Set overuse detector max frame interval based on target frame rate.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 31 matching lines @@
 // TODO(kthelgason): Lower this limit when better testing
 // on MediaCodec and fallback implementations are in place.
 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
 const int kMinPixelsPerFrame = 320 * 180;
 const int kMinFramerateFps = 2;
 
 // The maximum number of frames to drop at beginning of stream
 // to try and achieve desired bitrate.
 const int kMaxInitialFramedrop = 4;
 
-// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
-// pipelining encoders better (multiple input frames before something comes
-// out). This should effectively turn off CPU adaptations for systems that
-// remotely cope with the load right now.
-CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
-  CpuOveruseOptions options;
-  if (full_overuse_time) {
-    options.low_encode_usage_threshold_percent = 150;
-    options.high_encode_usage_threshold_percent = 200;
-  }
-  return options;
-}
-
 uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) {
   if (kbps > 0) {
     if (kbps < 300 /* qvga */) {
       return 320 * 240;
     } else if (kbps < 500 /* vga */) {
       return 640 * 480;
     }
   }
   return std::numeric_limits<uint32_t>::max();
 }
@@ skipping 167 matching lines @@
         pixels_wanted >= sink_wants_.max_pixel_count) {
       return false;
     }
     LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted;
     sink_wants_.max_pixel_count = pixels_wanted;
     sink_wants_.target_pixel_count = rtc::Optional<int>();
     source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
     return true;
   }
 
-  bool RequestFramerateLowerThan(int fps) {
+  int RequestFramerateLowerThan(int fps) {
     // Called on the encoder task queue.
     // The input video frame rate will be scaled down to 2/3, rounding down.
-    return RestrictFramerate((fps * 2) / 3);
+    int wanted_framerate = (fps * 2) / 3;

[åsapersson, 2017/06/13 12:15:09]

nit: maybe framerate_wanted for consistency with RequestHigherFramerateThan

[sprang_webrtc, 2017/06/14 08:39:16]

Done.

+    return RestrictFramerate(wanted_framerate) ? wanted_framerate : -1;
   }
 
   bool RequestHigherResolutionThan(int pixel_count) {
     // Called on the encoder task queue.
     rtc::CritScope lock(&crit_);
     if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
       // This can happen since |degradation_preference_| is set on libjingle's
       // worker thread but the adaptation is done on the encoder task queue.
       return false;
     }
@@ skipping 16 matching lines @@
       // not take a too large step up, we cap the requested pixel count to be at
       // most four time the current number of pixels.
       sink_wants_.target_pixel_count =
           rtc::Optional<int>((pixel_count * 5) / 3);
     }
     LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted;
     source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
     return true;
   }
 
-  bool RequestHigherFramerateThan(int fps) {
+  // Request upgrade in framerate. Returns the new requested frame, or -1 if
+  // no change requested. Note that maxint may be returned if limits due to
+  // adaptation requests are removed completely. In that case, consider
+  // |max_framerate_| to be the current limit (assuming the capturer complies).
+  int RequestHigherFramerateThan(int fps) {
     // Called on the encoder task queue.
     // The input frame rate will be scaled up to the last step, with rounding.
     int framerate_wanted = fps;
     if (fps != std::numeric_limits<int>::max())
       framerate_wanted = (fps * 3) / 2;
 
-    return IncreaseFramerate(framerate_wanted);
+    return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1;
   }
 
   bool RestrictFramerate(int fps) {
     // Called on the encoder task queue.
     rtc::CritScope lock(&crit_);
     if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
       return false;
 
     const int fps_wanted = std::max(kMinFramerateFps, fps);
     if (fps_wanted >= sink_wants_.max_framerate_fps)
@@ skipping 40 matching lines @@
                        EncodedFrameObserver* encoder_timing)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
       initial_rampup_(0),
       source_proxy_(new VideoSourceProxy(this)),
       sink_(nullptr),
       settings_(settings),
       codec_type_(PayloadNameToCodecType(settings.payload_name)
                       .value_or(VideoCodecType::kVideoCodecUnknown)),
       video_sender_(Clock::GetRealTimeClock(), this, this),
-      overuse_detector_(GetCpuOveruseOptions(settings.full_overuse_time),
-                        this,
-                        encoder_timing,
-                        stats_proxy),
+      overuse_detector_(new OveruseFrameDetector(
+          GetCpuOveruseOptions(settings.full_overuse_time),
+          this,
+          encoder_timing,
+          stats_proxy)),
       stats_proxy_(stats_proxy),
       pre_encode_callback_(pre_encode_callback),
       module_process_thread_(nullptr),
+      max_framerate_(-1),
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
       nack_enabled_(false),
       last_observed_bitrate_bps_(0),
       encoder_paused_and_dropped_frame_(false),
       clock_(Clock::GetRealTimeClock()),
       degradation_preference_(
           VideoSendStream::DegradationPreference::kDegradationDisabled),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       last_frame_log_ms_(clock_->TimeInMilliseconds()),
       captured_frame_count_(0),
       dropped_frame_count_(0),
       bitrate_observer_(nullptr),
       encoder_queue_("EncoderQueue") {
   RTC_DCHECK(stats_proxy);
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    overuse_detector_.StartCheckForOveruse();
+    overuse_detector_->StartCheckForOveruse();
     video_sender_.RegisterExternalEncoder(
         settings_.encoder, settings_.payload_type, settings_.internal_source);
   });
 }
 
 ViEEncoder::~ViEEncoder() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(shutdown_event_.Wait(0))
       << "Must call ::Stop() before destruction.";
 }
 
+// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
+// pipelining encoders better (multiple input frames before something comes
+// out). This should effectively turn off CPU adaptations for systems that
+// remotely cope with the load right now.
+CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) {
+  CpuOveruseOptions options;
+  if (full_overuse_time) {
+    options.low_encode_usage_threshold_percent = 150;
+    options.high_encode_usage_threshold_percent = 200;
+  }
+  return options;
+}
+
+void ViEEncoder::UpdateOveruseDetector(

[kthelgason, 2017/06/12 11:56:47]

Is this only used in the test? I'd prefer we not make all these changes just to
make that possible. Just make it injectable in the ctor instead.

[sprang_webrtc, 2017/06/14 08:39:17]

Done.

+    std::unique_ptr<OveruseFrameDetector> overuse_detector) {
+  rtc::Event event(false, false);
+  encoder_queue_.PostTask([this, &overuse_detector, &event] {
+    RTC_DCHECK_RUN_ON(&encoder_queue_);
+    overuse_detector_->StopCheckForOveruse();
+    overuse_detector_ = std::move(overuse_detector);
+    overuse_detector_->StartCheckForOveruse();
+    event.Set();
+  });
+  RTC_DCHECK(event.Wait(5000));
+}
+
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference());
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    overuse_detector_.StopCheckForOveruse();
+    overuse_detector_->StopCheckForOveruse();
     rate_allocator_.reset();
     bitrate_observer_ = nullptr;
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
     quality_scaler_ = nullptr;
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
@@ skipping 30 matching lines @@
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     if (degradation_preference_ != degradation_preference) {
       // Reset adaptation state, so that we're not tricked into thinking there's
       // an already pending request of the same type.
       last_adaptation_request_.reset();
     }
     degradation_preference_ = degradation_preference;
     bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_);
     initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop;
     ConfigureQualityScaler();
+    if (!IsFramerateScalingEnabled(degradation_preference) &&
+        max_framerate_ != -1) {
+      // If frame rate scaling is no longer allowed, remove any potential
+      // allowance for longer frame intervals.
+      overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
+    }
   });
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
   source_proxy_->SetWantsRotationApplied(rotation_applied);
   encoder_queue_.PostTask([this, sink] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     sink_ = sink;
   });
 }
@@ skipping 49 matching lines @@
   if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams,
                                          nack_enabled_, &codec,
                                          &rate_allocator_)) {
     LOG(LS_ERROR) << "Failed to create encoder configuration.";
   }
 
   codec.startBitrate =
       std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
   codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
   codec.expect_encode_from_texture = last_frame_info_->is_texture;
+  max_framerate_ = codec.maxFramerate;
 
   bool success = video_sender_.RegisterSendCodec(
                      &codec, number_of_cores_,
                      static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
   if (!success) {
     LOG(LS_ERROR) << "Failed to configure encoder.";
     rate_allocator_.reset();
   }
 
   video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
                                         bitrate_observer_);
 
-  int framerate = stats_proxy_->GetSendFrameRate();
-  if (framerate == 0)
-    framerate = codec.maxFramerate;
+  // Get the current actual framerate, as measured by the stats proxy. This is
+  // used to get the correct bitrate layer allocation.
+  int current_framerate = stats_proxy_->GetSendFrameRate();
+  if (current_framerate == 0)
+    current_framerate = codec.maxFramerate;
   stats_proxy_->OnEncoderReconfigured(
-      encoder_config_, rate_allocator_.get()
-                           ? rate_allocator_->GetPreferredBitrateBps(framerate)
-                           : codec.maxBitrate);
+      encoder_config_,
+      rate_allocator_.get()
+          ? rate_allocator_->GetPreferredBitrateBps(current_framerate)
+          : codec.maxBitrate);
 
   pending_encoder_reconfiguration_ = false;
 
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
 
+  // Get the current target framerate, ie the maximum framerate as specified by
+  // the current codec configuration, or any limit imposed by cpu adaption in
+  // maintain-resolution or balanced mode. This is used to make sure overuse
+  // detection doesn't needlessly trigger in low and/or variable framerate
+  // scenarios.
+  int target_framerate = max_framerate_;
+  if (last_adaptation_request_ &&
+      last_adaptation_request_->framerate_fps_ > 0) {

[åsapersson, 2017/06/13 12:15:09]

last_adaptation_request_->framerate_fps_ will not hold the requested framrate,
right? Holds the input fps before the request.

[sprang_webrtc, 2017/06/14 08:39:16]

Hm, you're right. Guess I'll have to add another field to the adaptation request
struct :/

+    target_framerate =
+        std::min(target_framerate, last_adaptation_request_->framerate_fps_);
+  }
+  overuse_detector_->OnTargetFramerateUpdated(target_framerate);
+
   ConfigureQualityScaler();
 }
 
 void ViEEncoder::ConfigureQualityScaler() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   const auto scaling_settings = settings_.encoder->GetScalingSettings();
   const bool quality_scaling_allowed =
       IsResolutionScalingEnabled(degradation_preference_) &&
       scaling_settings.enabled;
 
@@ skipping 128 matching lines @@
 
   if (EncoderPaused()) {
     TraceFrameDropStart();
     return;
   }
   TraceFrameDropEnd();
 
   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                           "Encode");
 
-  overuse_detector_.FrameCaptured(video_frame, time_when_posted_us);
+  overuse_detector_->FrameCaptured(video_frame, time_when_posted_us);
 
   video_sender_.AddVideoFrame(video_frame, nullptr);
 }
 
 void ViEEncoder::SendKeyFrame() {
   if (!encoder_queue_.IsCurrent()) {
     encoder_queue_.PostTask([this] { SendKeyFrame(); });
     return;
   }
   RTC_DCHECK_RUN_ON(&encoder_queue_);
@@ skipping 10 matching lines @@
   stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
 
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
 
   int64_t time_sent_us = rtc::TimeMicros();
   uint32_t timestamp = encoded_image._timeStamp;
   const int qp = encoded_image.qp_;
   encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    overuse_detector_.FrameSent(timestamp, time_sent_us);
+    overuse_detector_->FrameSent(timestamp, time_sent_us);
     if (quality_scaler_ && qp >= 0)
       quality_scaler_->ReportQP(qp);
   });
 
   return result;
 }
 
 void ViEEncoder::OnDroppedFrame() {
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
@@ skipping 105 matching lines @@
     case VideoSendStream::DegradationPreference::kBalanced:
       FALLTHROUGH();
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       // Scale down resolution.
       if (!source_proxy_->RequestResolutionLowerThan(
               adaptation_request.input_pixel_count_)) {
         return;
       }
       GetAdaptCounter().IncrementResolution(reason, 1);
       break;
-    case VideoSendStream::DegradationPreference::kMaintainResolution:
+    case VideoSendStream::DegradationPreference::kMaintainResolution: {
       // Scale down framerate.
-      if (!source_proxy_->RequestFramerateLowerThan(
-              adaptation_request.framerate_fps_)) {
+      int requested_framerate = source_proxy_->RequestFramerateLowerThan(

[kthelgason, 2017/06/12 11:56:48]

const

[sprang_webrtc, 2017/06/14 08:39:17]

Done.

+          adaptation_request.framerate_fps_);
+      if (requested_framerate == -1)
         return;
-      }
+      overuse_detector_->OnTargetFramerateUpdated(
+          std::min(max_framerate_, requested_framerate));
       GetAdaptCounter().IncrementFramerate(reason, 1);
       break;
+    }
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << GetConstAdaptCounter().ToString();
 }
@@ skipping 51 matching lines @@
       GetAdaptCounter().IncrementResolution(reason, -1);
       break;
     }
     case VideoSendStream::DegradationPreference::kMaintainResolution: {
       // Scale up framerate.
       int fps = adaptation_request.framerate_fps_;
       if (adapt_counter.FramerateCount() == 1) {
         LOG(LS_INFO) << "Removing framerate down-scaling setting.";
         fps = std::numeric_limits<int>::max();
       }
-      if (!source_proxy_->RequestHigherFramerateThan(fps))
+
+      int requested_framerate = source_proxy_->RequestHigherFramerateThan(fps);

[kthelgason, 2017/06/12 11:56:47]

const

[sprang_webrtc, 2017/06/14 08:39:16]

Done.

+      if (requested_framerate == -1) {
+        overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
         return;
+      }
+      overuse_detector_->OnTargetFramerateUpdated(
+          std::min(max_framerate_, requested_framerate));
       GetAdaptCounter().IncrementFramerate(reason, -1);
       break;
     }
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
@@ skipping 106 matching lines @@
 std::string ViEEncoder::AdaptCounter::ToString(
     const std::vector<int>& counters) const {
   std::stringstream ss;
   for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
     ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
   }
   return ss.str();
 }
 
 }  // namespace webrtc