Drop frames until specified bitrate is achieved.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 24 matching lines @@
 const int64_t kFrameLogIntervalMs = 60000;
 // We will never ask for a resolution lower than this.
 #if defined(WEBRTC_ANDROID)
 // TODO(kthelgason): Lower this limit when better testing
 // on MediaCodec and fallback implementations are in place.
 const int kMinPixelsPerFrame = 320 * 180;
 #else
 const int kMinPixelsPerFrame = 120 * 90;
 #endif
 
+// The maximum number of frames to drop at beginning of stream

[perkj_webrtc, 2017/01/23 08:54:22]

Isnt this comment a bit confusing? This will happen as soon as the resolution
change and the bitrate is lower than what we think is good. 

[kthelgason, 2017/01/23 12:39:06]

I'm not sure I understand what you mean. This should only happen in the
beginning when an encoder is configured. 

[perkj_webrtc, 2017/01/24 09:06:15]

but an encoder is reconfigured every time the resolution change. 
ie. after the cpu adapter or quality scaler kicks in.
Just to double check, Will the quality scaler request a scale up if the initial
bitrate is low so that ViEEncoder has requested a scale down.

+// to try and achieve desired bitrate.
+const int kMaxInitialFramedrop = 4;
+
 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
 // pipelining encoders better (multiple input frames before something comes
 // out). This should effectively turn off CPU adaptations for systems that
 // remotely cope with the load right now.
 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
   CpuOveruseOptions options;
   if (full_overuse_time) {
     options.low_encode_usage_threshold_percent = 150;
     options.high_encode_usage_threshold_percent = 200;
   }
   return options;
 }
 
+uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) {
+  if (kbps > 0) {
+    if (kbps < 300 /* qvga */) {
+      return 320 * 240;
+    } else if (kbps < 500 /* vga */) {
+      return 640 * 480;
+    }
+  }
+  return std::numeric_limits<uint32_t>::max();
+}
+
 }  //  namespace
 
 class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask {
  public:
   ConfigureEncoderTask(ViEEncoder* vie_encoder,
                        VideoEncoderConfig config,
                        size_t max_data_payload_length,
                        bool nack_enabled)
       : vie_encoder_(vie_encoder),
         config_(std::move(config)),
@@ skipping 169 matching lines @@
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
                        EncodedFrameObserver* encoder_timing)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
+      initial_rampup_(kMaxInitialFramedrop),
       source_proxy_(new VideoSourceProxy(this)),
       sink_(nullptr),
       settings_(settings),
       codec_type_(PayloadNameToCodecType(settings.payload_name)
                       .value_or(VideoCodecType::kVideoCodecUnknown)),
       video_sender_(Clock::GetRealTimeClock(), this, this),
       overuse_detector_(Clock::GetRealTimeClock(),
                         GetCpuOveruseOptions(settings.full_overuse_time),
                         this,
                         encoder_timing,
@@ skipping 175 matching lines @@
         encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate));
   }
 
   pending_encoder_reconfiguration_ = false;
 
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
 
   const auto scaling_settings = settings_.encoder->GetScalingSettings();
   if (scaling_enabled_ && scaling_settings.enabled) {
+    // Drop frames and scale down until desired quality is achieved.
+    initial_rampup_ = 0;
     if (scaling_settings.thresholds) {
       quality_scaler_.reset(
           new QualityScaler(this, *(scaling_settings.thresholds)));
     } else {
       quality_scaler_.reset(new QualityScaler(this, codec_type_));
     }
   } else {
     quality_scaler_.reset(nullptr);
     stats_proxy_->SetResolutionRestrictionStats(
         false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]);
@@ skipping 103 matching lines @@
                                           bitrate_observer_);
   }
   last_parameters_update_ms_.emplace(now_ms);
 
   if (EncoderPaused()) {
     TraceFrameDropStart();
     return;
   }
   TraceFrameDropEnd();
 
+  uint32_t expected_pixel_count =

[perkj_webrtc, 2017/01/23 08:54:22]

min_pixel_count? 

[kthelgason, 2017/01/23 12:39:06]

Shouldn't it rather be max_pixel_count? It's the maximum amount of pixels we
allow for this bitrate.

+      MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000);
+  if (initial_rampup_ < kMaxInitialFramedrop &&

[perkj_webrtc, 2017/01/23 08:54:22]

please name more appropriate. Maybe something like
frames_dropped_due_to_high_resolution_ ? 

[perkj_webrtc, 2017/01/23 08:54:22]

This will drop frames at anytime time  kMaxInitialFrameDrop times if the bitrate
estimates suddenly drops and a new lower resolution is requested. 
Is that ok with sprang and stefan? Does that look good? 

Will the quality scaler request a higher resolution later, even if the
resolution was lowered due to this? 
Can you add a test for this case as well please.

[perkj_webrtc, 2017/01/23 08:54:22]

Should kMaxInitilFrameDrop also change change name since frames are dropped at
anytime the bitrate drops below the threshold? 

[kthelgason, 2017/01/23 12:39:06]

I understand what you mean. That should not be how this works. If a frame ever
passes this condition |initial_rampup_| should be set to kMaxInitialFrameDrop so
that we never drop frames once one has been sent.

+      video_frame.size() > expected_pixel_count) {
+    LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
+    ScaleDown(kQuality);
+    ++initial_rampup_;
+    return;
+  }
+
   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                           "Encode");
 
   overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms);
 
   if (codec_type_ == webrtc::kVideoCodecVP8) {
     webrtc::CodecSpecificInfo codec_specific_info;
     codec_specific_info.codecType = webrtc::kVideoCodecVP8;
 
     codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = has_received_rpsi_;
@@ skipping 104 matching lines @@
   RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active.";
 
   LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps
                   << " packet loss " << static_cast<int>(fraction_lost)
                   << " rtt " << round_trip_time_ms;
 
   video_sender_.SetChannelParameters(bitrate_bps, fraction_lost,
                                      round_trip_time_ms, rate_allocator_.get(),
                                      bitrate_observer_);
 
   encoder_start_bitrate_bps_ =

[perkj_webrtc, 2017/01/23 08:54:22]

this is where encoder_start_bitrate_bps_ is set. it only exist since we might
reconfigure the encoder even though its paused. 
It might be better and make more sense to use last_observed_bitrate_bps_ for
this.

       bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_;
   bool video_is_suspended = bitrate_bps == 0;
   bool video_suspension_changed = video_is_suspended != EncoderPaused();
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
@@ skipping 58 matching lines @@
   --scale_counter_[reason];
   source_proxy_->RequestHigherResolutionThan(current_pixel_count);
   LOG(LS_INFO) << "Scaling up resolution.";
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
     LOG(LS_INFO) << "Scaled " << scale_counter_[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
 }  // namespace webrtc