Add framerate to VideoSinkWants and ability to signal on overuse

webrtc/video/vie_encoder.cc

Index: webrtc/video/vie_encoder.cc
diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc
index 7880c4a275fd5e4589722bd269f0db44a421dc33..075b3e36f2ccdf57daa7c53d9d399e50224abcf4 100644
--- a/webrtc/video/vie_encoder.cc
+++ b/webrtc/video/vie_encoder.cc
@@ -41,6 +41,7 @@ const int64_t kFrameLogIntervalMs = 60000;
 // on MediaCodec and fallback implementations are in place.
 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
 const int kMinPixelsPerFrame = 320 * 180;
+const int kMinFramerateFps = 2;
 
 // The maximum number of frames to drop at beginning of stream
 // to try and achieve desired bitrate.
@@ -160,10 +161,23 @@ class ViEEncoder::VideoSourceProxy {
     rtc::VideoSinkWants wants;
     {
       rtc::CritScope lock(&crit_);
+      wants = sink_wants_;
+      // If changing degradation preference, clear any constraints from the
+      // current sink wants that will no longer apply.
+      if (degradation_preference_ != degradation_preference) {
+        switch (degradation_preference) {
+          case DegradationPreference::kBalanced:
+            wants.max_framerate_fps_.reset();
+            break;
+          case DegradationPreference::kMaintainResolution:
+            wants.max_pixel_count.reset();
+            wants.target_pixel_count.reset();
+            break;
+        }
+      }
+      degradation_preference_ = degradation_preference;
       old_source = source_;
       source_ = source;
-      degradation_preference_ = degradation_preference;
-      wants = current_wants();
     }
 
     if (old_source != source && old_source != nullptr) {
@@ -180,10 +194,8 @@ class ViEEncoder::VideoSourceProxy {
   void SetWantsRotationApplied(bool rotation_applied) {
     rtc::CritScope lock(&crit_);
     sink_wants_.rotation_applied = rotation_applied;
-    disabled_scaling_sink_wants_.rotation_applied = rotation_applied;
-    if (source_) {
-      source_->AddOrUpdateSink(vie_encoder_, current_wants());
-    }
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
   void RequestResolutionLowerThan(int pixel_count) {
@@ -207,6 +219,24 @@ class ViEEncoder::VideoSourceProxy {
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
+  void RequestFramerateLowerThan(int framerate_fps) {
+    // Called on the encoder task queue.
+    rtc::CritScope lock(&crit_);
+    if (!IsFramerateScalingEnabledLocked()) {
+      // This can happen since |degradation_preference_| is set on
+      // libjingle's worker thread but the adaptation is done on the encoder
+      // task queue.
+      return;
+    }
+    // The input video frame rate will be scaled down to 2/3 of input fps,
+    // rounding down.
+    const int framerate_wanted =
+        std::max(kMinFramerateFps, (framerate_fps * 2) / 3);
+    sink_wants_.max_framerate_fps_.emplace(framerate_wanted);
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+  }
+
   void RequestHigherResolutionThan(int pixel_count) {
     rtc::CritScope lock(&crit_);
     if (!IsResolutionScalingEnabledLocked()) {
@@ -215,14 +245,44 @@ class ViEEncoder::VideoSourceProxy {
       // task queue.
       return;
     }
-    // On step down we request at most 3/5 the pixel count of the previous
-    // resolution, so in order to take "one step up" we request a resolution as
-    // close as possible to 5/3 of the current resolution. The actual pixel
-    // count selected depends on the capabilities of the source. In order to not
-    // take a too large step up, we cap the requested pixel count to be at most
-    // four time the current number of pixels.
-    sink_wants_.target_pixel_count = rtc::Optional<int>((pixel_count * 5) / 3);
-    sink_wants_.max_pixel_count = rtc::Optional<int>(pixel_count * 4);
+
+    if (pixel_count == std::numeric_limits<int>::max()) {
+      // Remove any restrains.
+      sink_wants_.target_pixel_count.reset();
+      sink_wants_.max_pixel_count.reset();
+    } else {
+      // On step down we request at most 3/5 the pixel count of the previous
+      // resolution, so in order to take "one step up" we request a resolution
+      // as close as possible to 5/3 of the current resolution. The actual pixel
+      // count selected depends on the capabilities of the source. In order to
+      // not take a too large step up, we cap the requested pixel count to be at
+      // most four time the current number of pixels.
+      sink_wants_.target_pixel_count =
+          rtc::Optional<int>((pixel_count * 5) / 3);
+      sink_wants_.max_pixel_count = rtc::Optional<int>(pixel_count * 4);
+    }
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+  }
+
+  void RequestHigherFramerateThan(int framerate_fps) {
+    // Called on the encoder task queue.
+    rtc::CritScope lock(&crit_);
+    if (!IsFramerateScalingEnabledLocked()) {
+      // This can happen since |degradation_preference_| is set on
+      // libjingle's worker thread but the adaptation is done on the encoder
+      // task queue.
+      return;
+    }
+    if (framerate_fps == std::numeric_limits<int>::max()) {
+      // Remove any restrains.
+      sink_wants_.max_framerate_fps_.reset();
+    } else {
+      // The input video frame rate will be scaled up to the last step, with
+      // rounding.
+      const int framerate_wanted = (framerate_fps * 3) / 2;
+      sink_wants_.max_framerate_fps_.emplace(framerate_wanted);
+    }
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
@@ -234,17 +294,16 @@ class ViEEncoder::VideoSourceProxy {
            DegradationPreference::kMaintainResolution;
   }
 
-  const rtc::VideoSinkWants& current_wants() const
+  bool IsFramerateScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
-    return IsResolutionScalingEnabledLocked() ? sink_wants_
-                                              : disabled_scaling_sink_wants_;
+    return degradation_preference_ ==
+           DegradationPreference::kMaintainResolution;
   }
 
   rtc::CriticalSection crit_;
   rtc::SequencedTaskChecker main_checker_;
   ViEEncoder* const vie_encoder_;
   rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
-  rtc::VideoSinkWants disabled_scaling_sink_wants_ GUARDED_BY(&crit_);
   DegradationPreference degradation_preference_ GUARDED_BY(&crit_);
   rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
 
@@ -283,7 +342,11 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      scale_counter_(kScaleReasonSize, 0),
+      scale_counters_{
+          {VideoSendStream::DegradationPreference::kMaintainResolution,
+           std::vector<int>(kScaleReasonSize, 0)},
+          {VideoSendStream::DegradationPreference::kBalanced,
+           std::vector<int>(kScaleReasonSize, 0)}},
       degradation_preference_(DegradationPreference::kMaintainResolution),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
@@ -474,9 +537,10 @@ void ViEEncoder::ConfigureQualityScaler() {
     quality_scaler_.reset(nullptr);
     initial_rampup_ = kMaxInitialFramedrop;
   }
+  const std::vector<int>& scale_counter = *GetScaleCounters();
   stats_proxy_->SetResolutionRestrictionStats(
-      degradation_preference_allows_scaling, scale_counter_[kCpu] > 0,
-      scale_counter_[kQuality]);
+      degradation_preference_allows_scaling, scale_counter[kCpu] > 0,
+      scale_counter[kQuality]);
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
@@ -739,79 +803,168 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps,
 
 void ViEEncoder::AdaptDown(AdaptReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ != DegradationPreference::kBalanced)
-    return;
-  RTC_DCHECK(static_cast<bool>(last_frame_info_));
-  int current_pixel_count = last_frame_info_->pixel_count();
-  if (last_adaptation_request_ &&
-      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown &&
-      current_pixel_count >= last_adaptation_request_->input_pixel_count_) {
-    // Don't request lower resolution if the current resolution is not lower
-    // than the last time we asked for the resolution to be lowered.
+  AdaptationRequest adaptation_request = {
+      last_frame_info_->pixel_count(),
+      stats_proxy_->GetStats().input_frame_rate,
+      AdaptationRequest::Mode::kAdaptDown};
+
+  if (degradation_preference_ == DegradationPreference::kMaintainResolution &&
+      adaptation_request.framerate_fps_ <= 0) {
+    // No input fps estimate available, can't determine how to scale down fps.
     return;
   }
-  last_adaptation_request_.emplace(AdaptationRequest{
-      current_pixel_count, AdaptationRequest::Mode::kAdaptDown});
+
+  bool downgrade_requested =
+      last_adaptation_request_ &&
+      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
+  int max_downgrades;
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      max_downgrades = kMaxCpuResolutionDowngrades;
+      if (downgrade_requested &&
+          adaptation_request.input_pixel_count_ >=
+              last_adaptation_request_->input_pixel_count_) {
+        // Don't request lower resolution if the current resolution is not
+        // lower than the last time we asked for the resolution to be lowered.
+        return;
+      }
+      break;
+    case DegradationPreference::kMaintainResolution:
+      max_downgrades = kMaxCpuFramerateDowngrades;
+      if (downgrade_requested &&
+          adaptation_request.framerate_fps_ < kMinFramerateFps) {
+        // Don't request lower framerate if we don't have a valid frame rate.
+        // Since framerate, unlike resolution, is a measure we have to
+        // estimate, and can fluctuate naturally over time, don't make the
+        // same kind of limitations as for resolution, but trust the overuse
+        // detector to not trigger too often.
+        return;
+      }
+      break;
+  }
+
+  last_adaptation_request_.emplace(adaptation_request);
+  std::vector<int>* scale_counter = GetScaleCounters();
 
   switch (reason) {
     case kQuality:
       stats_proxy_->OnQualityRestrictedResolutionChanged(
-          scale_counter_[reason] + 1);
+          (*scale_counter)[reason] + 1);
       break;
     case kCpu:
-      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+      if ((*scale_counter)[reason] >= max_downgrades)
         return;
       // Update stats accordingly.
       stats_proxy_->OnCpuRestrictedResolutionChanged(true);
       break;
   }
-  ++scale_counter_[reason];
-  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
-  LOG(LS_INFO) << "Scaling down resolution.";
+  ++(*scale_counter)[reason];
+
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      source_proxy_->RequestResolutionLowerThan(
+          adaptation_request.input_pixel_count_);
+      LOG(LS_INFO) << "Scaling down resolution.";
+      break;
+    case DegradationPreference::kMaintainResolution:
+      source_proxy_->RequestFramerateLowerThan(
+          adaptation_request.framerate_fps_);
+      LOG(LS_INFO) << "Scaling down framerate.";
+      break;
+  }
+
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
-    LOG(LS_INFO) << "Scaled " << scale_counter_[i]
+    LOG(LS_INFO) << "Scaled " << (*scale_counter)[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
 void ViEEncoder::AdaptUp(AdaptReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (scale_counter_[reason] == 0 ||
-      degradation_preference_ != DegradationPreference::kBalanced) {
-    return;
-  }
-  // Only scale if resolution is higher than last time we requested higher
-  // resolution.
-  RTC_DCHECK(static_cast<bool>(last_frame_info_));
-  int current_pixel_count = last_frame_info_->pixel_count();
-  if (last_adaptation_request_ &&
-      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp &&
-      current_pixel_count <= last_adaptation_request_->input_pixel_count_) {
-    // Don't request higher resolution if the current resolution is not higher
-    // than the last time we asked for the resolution to be higher.
+  int* scale_counter = &((*GetScaleCounters())[reason]);
+  if (*scale_counter == 0)
     return;
+  RTC_DCHECK_GT(*scale_counter, 0);
+  AdaptationRequest adaptation_request = {
+      last_frame_info_->pixel_count(),
+      stats_proxy_->GetStats().input_frame_rate,
+      AdaptationRequest::Mode::kAdaptUp};
+
+  bool adapt_up_requested =
+      last_adaptation_request_ &&
+      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      if (adapt_up_requested &&
+          adaptation_request.input_pixel_count_ <=
+              last_adaptation_request_->input_pixel_count_) {
+        // Don't request higher resolution if the current resolution is not
+        // higher than the last time we asked for the resolution to be higher.
+        return;
+      }
+      break;
+    case DegradationPreference::kMaintainResolution:
+      // TODO(sprang): Don't request higher framerate if we are already at
+      // max requested fps?
+      break;
   }
-  last_adaptation_request_.emplace(AdaptationRequest{
-      current_pixel_count, AdaptationRequest::Mode::kAdaptUp});
 
   switch (reason) {
     case kQuality:
-      stats_proxy_->OnQualityRestrictedResolutionChanged(
-          scale_counter_[reason] - 1);
+      stats_proxy_->OnQualityRestrictedResolutionChanged(*scale_counter - 1);
       break;
     case kCpu:
       // Update stats accordingly.
-      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
-                                                     1);
+      stats_proxy_->OnCpuRestrictedResolutionChanged(*scale_counter > 1);
       break;
   }
-  --scale_counter_[reason];
-  source_proxy_->RequestHigherResolutionThan(current_pixel_count);
-  LOG(LS_INFO) << "Scaling up resolution.";
+
+  // Decrease counter of how many times we have scaled down, for this
+  // degradation preference mode and reason.
+  --*scale_counter;
+
+  // Get a sum of how many times have scaled down, in total, for this
+  // degradation preference mode. If it is 0, remove any restraints.
+  const std::vector<int>& current_scale_counters = *GetScaleCounters();
+  int scale_sum = 0;
+  for (int i : current_scale_counters)
+    scale_sum += i;
+
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      if (scale_sum == 0) {
+        LOG(LS_INFO) << "Removing resolution down-scaling setting.";
+        source_proxy_->RequestHigherResolutionThan(
+            std::numeric_limits<int>::max());
+      } else {
+        source_proxy_->RequestHigherResolutionThan(
+            adaptation_request.input_pixel_count_);
+        LOG(LS_INFO) << "Scaling up resolution.";
+      }
+      break;
+    case DegradationPreference::kMaintainResolution:
+      if (scale_sum == 0) {
+        LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+        source_proxy_->RequestHigherFramerateThan(
+            std::numeric_limits<int>::max());
+      } else {
+        source_proxy_->RequestHigherFramerateThan(
+            adaptation_request.framerate_fps_);
+        LOG(LS_INFO) << "Scaling up framerate.";
+      }
+      break;
+  }
+
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
-    LOG(LS_INFO) << "Scaled " << scale_counter_[i]
+    LOG(LS_INFO) << "Scaled " << (*GetScaleCounters())[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
+std::vector<int>* ViEEncoder::GetScaleCounters() {
+  auto it = scale_counters_.find(degradation_preference_);
+  RTC_CHECK(it != scale_counters_.end());
+  return &it->second;
+}
+
 }  // namespace webrtc