Set overuse detector max frame interval based on target frame rate.

webrtc/video/vie_encoder.cc

Index: webrtc/video/vie_encoder.cc
diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc
index 203d1c405a8813d05b627cd464803bb9df39f645..d144b2035762266d2dc53e373fc0418d5de3dd87 100644
--- a/webrtc/video/vie_encoder.cc
+++ b/webrtc/video/vie_encoder.cc
@@ -44,24 +44,12 @@ const int64_t kFrameLogIntervalMs = 60000;
 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
 const int kMinPixelsPerFrame = 320 * 180;
 const int kMinFramerateFps = 2;
+const int kMaxFramerateFps = 120;
 
 // The maximum number of frames to drop at beginning of stream
 // to try and achieve desired bitrate.
 const int kMaxInitialFramedrop = 4;
 
-// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
-// pipelining encoders better (multiple input frames before something comes
-// out). This should effectively turn off CPU adaptations for systems that
-// remotely cope with the load right now.
-CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
-  CpuOveruseOptions options;
-  if (full_overuse_time) {
-    options.low_encode_usage_threshold_percent = 150;
-    options.high_encode_usage_threshold_percent = 200;
-  }
-  return options;
-}
-
 uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) {
   if (kbps > 0) {
     if (kbps < 300 /* qvga */) {
@@ -249,10 +237,11 @@ class ViEEncoder::VideoSourceProxy {
     return true;
   }
 
-  bool RequestFramerateLowerThan(int fps) {
+  int RequestFramerateLowerThan(int fps) {
     // Called on the encoder task queue.
     // The input video frame rate will be scaled down to 2/3, rounding down.
-    return RestrictFramerate((fps * 2) / 3);
+    int framerate_wanted = (fps * 2) / 3;
+    return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
   }
 
   bool RequestHigherResolutionThan(int pixel_count) {
@@ -289,14 +278,18 @@ class ViEEncoder::VideoSourceProxy {
     return true;
   }
 
-  bool RequestHigherFramerateThan(int fps) {
+  // Request upgrade in framerate. Returns the new requested frame, or -1 if
+  // no change requested. Note that maxint may be returned if limits due to
+  // adaptation requests are removed completely. In that case, consider
+  // |max_framerate_| to be the current limit (assuming the capturer complies).
+  int RequestHigherFramerateThan(int fps) {
     // Called on the encoder task queue.
     // The input frame rate will be scaled up to the last step, with rounding.
     int framerate_wanted = fps;
     if (fps != std::numeric_limits<int>::max())
       framerate_wanted = (fps * 3) / 2;
 
-    return IncreaseFramerate(framerate_wanted);
+    return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1;
   }
 
   bool RestrictFramerate(int fps) {
@@ -347,7 +340,8 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
-                       EncodedFrameObserver* encoder_timing)
+                       EncodedFrameObserver* encoder_timing,
+                       std::unique_ptr<OveruseFrameDetector> overuse_detector)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
       initial_rampup_(0),
@@ -357,13 +351,18 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
       codec_type_(PayloadNameToCodecType(settings.payload_name)
                       .value_or(VideoCodecType::kVideoCodecUnknown)),
       video_sender_(Clock::GetRealTimeClock(), this, this),
-      overuse_detector_(GetCpuOveruseOptions(settings.full_overuse_time),
-                        this,
-                        encoder_timing,
-                        stats_proxy),
+      overuse_detector_(
+          overuse_detector.get()
+              ? overuse_detector.release()
+              : new OveruseFrameDetector(
+                    GetCpuOveruseOptions(settings.full_overuse_time),
+                    this,
+                    encoder_timing,
+                    stats_proxy)),
       stats_proxy_(stats_proxy),
       pre_encode_callback_(pre_encode_callback),
       module_process_thread_(nullptr),
+      max_framerate_(-1),
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
@@ -384,7 +383,7 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
   RTC_DCHECK(stats_proxy);
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    overuse_detector_.StartCheckForOveruse();
+    overuse_detector_->StartCheckForOveruse();
     video_sender_.RegisterExternalEncoder(
         settings_.encoder, settings_.payload_type, settings_.internal_source);
   });
@@ -396,12 +395,25 @@ ViEEncoder::~ViEEncoder() {
       << "Must call ::Stop() before destruction.";
 }
 
+// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
+// pipelining encoders better (multiple input frames before something comes
+// out). This should effectively turn off CPU adaptations for systems that
+// remotely cope with the load right now.
+CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) {
+  CpuOveruseOptions options;
+  if (full_overuse_time) {
+    options.low_encode_usage_threshold_percent = 150;
+    options.high_encode_usage_threshold_percent = 200;
+  }
+  return options;
+}
+
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference());
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    overuse_detector_.StopCheckForOveruse();
+    overuse_detector_->StopCheckForOveruse();
     rate_allocator_.reset();
     bitrate_observer_ = nullptr;
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
@@ -452,6 +464,12 @@ void ViEEncoder::SetSource(
     bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_);
     initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop;
     ConfigureQualityScaler();
+    if (!IsFramerateScalingEnabled(degradation_preference) &&
+        max_framerate_ != -1) {
+      // If frame rate scaling is no longer allowed, remove any potential
+      // allowance for longer frame intervals.
+      overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
+    }
   });
 }
 
@@ -521,6 +539,8 @@ void ViEEncoder::ReconfigureEncoder() {
       std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
   codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
   codec.expect_encode_from_texture = last_frame_info_->is_texture;
+  max_framerate_ = codec.maxFramerate;
+  RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps);
 
   bool success = video_sender_.RegisterSendCodec(
                      &codec, number_of_cores_,
@@ -533,19 +553,35 @@ void ViEEncoder::ReconfigureEncoder() {
   video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
                                         bitrate_observer_);
 
-  int framerate = stats_proxy_->GetSendFrameRate();
-  if (framerate == 0)
-    framerate = codec.maxFramerate;
+  // Get the current actual framerate, as measured by the stats proxy. This is
+  // used to get the correct bitrate layer allocation.
+  int current_framerate = stats_proxy_->GetSendFrameRate();
+  if (current_framerate == 0)
+    current_framerate = codec.maxFramerate;
   stats_proxy_->OnEncoderReconfigured(
-      encoder_config_, rate_allocator_.get()
-                           ? rate_allocator_->GetPreferredBitrateBps(framerate)
-                           : codec.maxBitrate);
+      encoder_config_,
+      rate_allocator_.get()
+          ? rate_allocator_->GetPreferredBitrateBps(current_framerate)
+          : codec.maxBitrate);
 
   pending_encoder_reconfiguration_ = false;
 
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
 
+  // Get the current target framerate, ie the maximum framerate as specified by
+  // the current codec configuration, or any limit imposed by cpu adaption in
+  // maintain-resolution or balanced mode. This is used to make sure overuse
+  // detection doesn't needlessly trigger in low and/or variable framerate
+  // scenarios.
+  int target_framerate = max_framerate_;
+  if (last_adaptation_request_ &&
+      last_adaptation_request_->selected_framerate_fps_) {

[åsapersson, 2017/06/14 10:50:26]

would it be better to get sink_wants_.max_framerate_fps from source_proxy_?

[sprang_webrtc, 2017/06/14 13:41:28]

Sure, I'll expose a getter that can be used.

+    target_framerate = std::min(
+        target_framerate, *last_adaptation_request_->selected_framerate_fps_);
+  }
+  overuse_detector_->OnTargetFramerateUpdated(target_framerate);
+
   ConfigureQualityScaler();
 }
 
@@ -694,7 +730,7 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                           "Encode");
 
-  overuse_detector_.FrameCaptured(video_frame, time_when_posted_us);
+  overuse_detector_->FrameCaptured(video_frame, time_when_posted_us);
 
   video_sender_.AddVideoFrame(video_frame, nullptr);
 }
@@ -725,7 +761,7 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage(
   const int qp = encoded_image.qp_;
   encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    overuse_detector_.FrameSent(timestamp, time_sent_us);
+    overuse_detector_->FrameSent(timestamp, time_sent_us);
     if (quality_scaler_ && qp >= 0)
       quality_scaler_->ReportQP(qp);
   });
@@ -796,7 +832,7 @@ void ViEEncoder::AdaptDown(AdaptReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   AdaptationRequest adaptation_request = {
       last_frame_info_->pixel_count(),
-      stats_proxy_->GetStats().input_frame_rate,
+      stats_proxy_->GetStats().input_frame_rate, rtc::Optional<int>(),
       AdaptationRequest::Mode::kAdaptDown};
 
   bool downgrade_requested =
@@ -851,14 +887,18 @@ void ViEEncoder::AdaptDown(AdaptReason reason) {
       }
       GetAdaptCounter().IncrementResolution(reason, 1);
       break;
-    case VideoSendStream::DegradationPreference::kMaintainResolution:
+    case VideoSendStream::DegradationPreference::kMaintainResolution: {
       // Scale down framerate.
-      if (!source_proxy_->RequestFramerateLowerThan(
-              adaptation_request.framerate_fps_)) {
+      const int requested_framerate = source_proxy_->RequestFramerateLowerThan(
+          adaptation_request.framerate_fps_);
+      if (requested_framerate == -1)
         return;
-      }
+      adaptation_request.selected_framerate_fps_.emplace(requested_framerate);
+      overuse_detector_->OnTargetFramerateUpdated(
+          std::min(max_framerate_, requested_framerate));
       GetAdaptCounter().IncrementFramerate(reason, 1);
       break;
+    }
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
@@ -881,7 +921,7 @@ void ViEEncoder::AdaptUp(AdaptReason reason) {
 
   AdaptationRequest adaptation_request = {
       last_frame_info_->pixel_count(),
-      stats_proxy_->GetStats().input_frame_rate,
+      stats_proxy_->GetStats().input_frame_rate, rtc::Optional<int>(),
       AdaptationRequest::Mode::kAdaptUp};
 
   bool adapt_up_requested =
@@ -930,8 +970,16 @@ void ViEEncoder::AdaptUp(AdaptReason reason) {
         LOG(LS_INFO) << "Removing framerate down-scaling setting.";
         fps = std::numeric_limits<int>::max();
       }
-      if (!source_proxy_->RequestHigherFramerateThan(fps))
+
+      const int requested_framerate =
+          source_proxy_->RequestHigherFramerateThan(fps);
+      if (requested_framerate == -1) {
+        overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
         return;
+      }
+      adaptation_request.selected_framerate_fps_.emplace(requested_framerate);
+      overuse_detector_->OnTargetFramerateUpdated(
+          std::min(max_framerate_, requested_framerate));
       GetAdaptCounter().IncrementFramerate(reason, -1);
       break;
     }