Drop frames until specified bitrate is achieved.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 24 matching lines @@
 const int64_t kFrameLogIntervalMs = 60000;
 // We will never ask for a resolution lower than this.
 #if defined(WEBRTC_ANDROID)
 // TODO(kthelgason): Lower this limit when better testing
 // on MediaCodec and fallback implementations are in place.
 const int kMinPixelsPerFrame = 320 * 180;
 #else
 const int kMinPixelsPerFrame = 120 * 90;
 #endif
 
+// The maximum number of frames to drop at beginning of stream
+// to try and achieve desired bitrate.
+const int kMaxInitialFramedrop = 4;
+
 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
 // pipelining encoders better (multiple input frames before something comes
 // out). This should effectively turn off CPU adaptations for systems that
 // remotely cope with the load right now.
 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
   CpuOveruseOptions options;
   if (full_overuse_time) {
     options.low_encode_usage_threshold_percent = 150;
     options.high_encode_usage_threshold_percent = 200;
   }
   return options;
 }
 
+uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) {
+  if (kbps > 0) {
+    if (kbps < 300 /* qvga */) {
+      return 320 * 240;
+    } else if (kbps < 500 /* vga */) {
+      return 640 * 480;
+    }
+  }
+  return std::numeric_limits<uint32_t>::max();
+}
+
 }  //  namespace
 
 class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask {
  public:
   ConfigureEncoderTask(ViEEncoder* vie_encoder,
                        VideoEncoderConfig config,
                        size_t max_data_payload_length,
                        bool nack_enabled)
       : vie_encoder_(vie_encoder),
         config_(std::move(config)),
@@ skipping 169 matching lines @@
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
                        EncodedFrameObserver* encoder_timing)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
+      initial_rampup_(0),
       source_proxy_(new VideoSourceProxy(this)),
       sink_(nullptr),
       settings_(settings),
       codec_type_(PayloadNameToCodecType(settings.payload_name)
                       .value_or(VideoCodecType::kVideoCodecUnknown)),
       video_sender_(Clock::GetRealTimeClock(), this, this),
       overuse_detector_(Clock::GetRealTimeClock(),
                         GetCpuOveruseOptions(settings.full_overuse_time),
                         this,
                         encoder_timing,
@@ skipping 75 matching lines @@
   });
 }
 
 void ViEEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(source, degradation_preference);
   encoder_queue_.PostTask([this, degradation_preference] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     scaling_enabled_ = (degradation_preference !=

[perkj_webrtc, 2017/01/24 12:58:23]

Changing the source change scaling_enabled_ but does not reconfigure the encoder
unless the frame size also change.

[kthelgason, 2017/01/25 10:01:58]

Acknowledged.

          VideoSendStream::DegradationPreference::kMaintainResolution);
     stats_proxy_->SetResolutionRestrictionStats(
         scaling_enabled_, scale_counter_[kCpu] > 0, scale_counter_[kQuality]);
   });
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
   source_proxy_->SetWantsRotationApplied(rotation_applied);
   encoder_queue_.PostTask([this, sink] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
@@ skipping 79 matching lines @@
         encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate));
   }
 
   pending_encoder_reconfiguration_ = false;
 
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
 
   const auto scaling_settings = settings_.encoder->GetScalingSettings();
   if (scaling_enabled_ && scaling_settings.enabled) {
+    // Drop frames and scale down until desired quality is achieved.
     if (scaling_settings.thresholds) {
       quality_scaler_.reset(

[perkj_webrtc, 2017/01/24 12:58:22]

quality scaler is only created when the codec is reconfigured. it can be due to
frame size changes or that the the public method ViEEncoder::ConfigureEncoder is
called.

[kthelgason, 2017/01/25 10:01:58]

Acknowledged.

           new QualityScaler(this, *(scaling_settings.thresholds)));
     } else {
       quality_scaler_.reset(new QualityScaler(this, codec_type_));
     }
   } else {
+    initial_rampup_ = kMaxInitialFramedrop;

[perkj_webrtc, 2017/01/24 12:58:23]

This will not necessarily be called when scaling_enabled_ change. Ie, if the
source change.
You should maybe not change this here either then.

[kthelgason, 2017/01/25 10:01:58]

Acknowledged.

     quality_scaler_.reset(nullptr);
     stats_proxy_->SetResolutionRestrictionStats(
         false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]);
   }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   VideoFrame incoming_frame = video_frame;
 
@@ skipping 79 matching lines @@
     pending_encoder_reconfiguration_ = true;
     last_frame_info_ = rtc::Optional<VideoFrameInfo>(
         VideoFrameInfo(video_frame.width(), video_frame.height(),
                        video_frame.rotation(), video_frame.is_texture()));
     LOG(LS_INFO) << "Video frame parameters changed: dimensions="
                  << last_frame_info_->width << "x" << last_frame_info_->height
                  << ", rotation=" << last_frame_info_->rotation
                  << ", texture=" << last_frame_info_->is_texture;
   }
 
+  uint32_t max_pixel_count =
+      MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000);
+  if (initial_rampup_ < kMaxInitialFramedrop &&

[perkj_webrtc, 2017/01/24 12:58:22]

I guess you will also need to check if (scaling_enabled_ && quality_scaling_ )
to handle this, otherwise 
if !scaling_settings.enabled but scaling_enabled_ you will never request a
higher resolution again.  This gets a bit too complicated.

Any chance we can start over a bit and clean up the states?

[kthelgason, 2017/01/25 10:01:58]

Yes, this sucks.

+      video_frame.size() > max_pixel_count) {

[perkj_webrtc, 2017/01/24 12:58:23]

nit: prefer video_frame_size() >
MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000);

[kthelgason, 2017/01/25 10:01:58]

Done.

+    LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
+    ScaleDown(kQuality);
+    ++initial_rampup_;
+    return;
+  }
+  initial_rampup_ = kMaxInitialFramedrop;
+
+

[perkj_webrtc, 2017/01/24 12:58:23]

nit: remove extra empty line.

[kthelgason, 2017/01/25 10:01:58]

Done.

   int64_t now_ms = clock_->TimeInMilliseconds();
   if (pending_encoder_reconfiguration_) {
     ReconfigureEncoder();
   } else if (!last_parameters_update_ms_ ||
              now_ms - *last_parameters_update_ms_ >=
                  vcm::VCMProcessTimer::kDefaultProcessIntervalMs) {
     video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
                                           bitrate_observer_);
   }
   last_parameters_update_ms_.emplace(now_ms);
@@ skipping 136 matching lines @@
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
 void ViEEncoder::ScaleDown(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   if (!scaling_enabled_)

[perkj_webrtc, 2017/01/24 12:58:23]

so if the source change scaling_enabled_ can change but quality scaler may or
may not exist regardless.

[kthelgason, 2017/01/25 10:01:58]

This has now been changed such that the quality scaler is reinitialized both
when re =configuring the encoder and when changing the source.

     return;
   // Request lower resolution if the current resolution is lower than last time
   // we asked for the resolution to be lowered.
   int current_pixel_count =
       last_frame_info_ ? last_frame_info_->pixel_count() : 0;
   if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_)
     return;
   switch (reason) {
     case kQuality:
       stats_proxy_->OnQualityRestrictedResolutionChanged(
@@ skipping 43 matching lines @@
   --scale_counter_[reason];
   source_proxy_->RequestHigherResolutionThan(current_pixel_count);
   LOG(LS_INFO) << "Scaling up resolution.";
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
     LOG(LS_INFO) << "Scaled " << scale_counter_[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
 }  // namespace webrtc