Add framerate to VideoSinkWants and ability to signal on overuse

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 23 matching lines @@
 using DegradationPreference = VideoSendStream::DegradationPreference;
 
 // Time interval for logging frame counts.
 const int64_t kFrameLogIntervalMs = 60000;
 
 // We will never ask for a resolution lower than this.
 // TODO(kthelgason): Lower this limit when better testing
 // on MediaCodec and fallback implementations are in place.
 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
 const int kMinPixelsPerFrame = 320 * 180;
+const int kMinFramerateFps = 2;
 
 // The maximum number of frames to drop at beginning of stream
 // to try and achieve desired bitrate.
 const int kMaxInitialFramedrop = 4;
 
 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
 // pipelining encoders better (multiple input frames before something comes
 // out). This should effectively turn off CPU adaptations for systems that
 // remotely cope with the load right now.
 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
@@ skipping 88 matching lines @@
 };
 
 // VideoSourceProxy is responsible ensuring thread safety between calls to
 // ViEEncoder::SetSource that will happen on libjingle's worker thread when a
 // video capturer is connected to the encoder and the encoder task queue
 // (encoder_queue_) where the encoder reports its VideoSinkWants.
 class ViEEncoder::VideoSourceProxy {
  public:
   explicit VideoSourceProxy(ViEEncoder* vie_encoder)
       : vie_encoder_(vie_encoder),
-        degradation_preference_(DegradationPreference::kMaintainResolution),
+        degradation_preference_(DegradationPreference::kDegradationDisabled),
         source_(nullptr) {}
 
   void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
                  const DegradationPreference& degradation_preference) {
     // Called on libjingle's worker thread.
     RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_);
     rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
     rtc::VideoSinkWants wants;
     {
       rtc::CritScope lock(&crit_);
+      degradation_preference_ = degradation_preference;
       old_source = source_;
       source_ = source;
-      degradation_preference_ = degradation_preference;
-      wants = current_wants();
+      wants = GetActiveSinkWants();
     }
 
     if (old_source != source && old_source != nullptr) {
       old_source->RemoveSink(vie_encoder_);
     }
 
     if (!source) {
       return;
     }
 
     source->AddOrUpdateSink(vie_encoder_, wants);
   }
 
   void SetWantsRotationApplied(bool rotation_applied) {
     rtc::CritScope lock(&crit_);
     sink_wants_.rotation_applied = rotation_applied;
-    disabled_scaling_sink_wants_.rotation_applied = rotation_applied;
-    if (source_) {
-      source_->AddOrUpdateSink(vie_encoder_, current_wants());
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+  }
+
+  rtc::VideoSinkWants GetActiveSinkWants() EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
+    rtc::VideoSinkWants wants = sink_wants_;
+    // If clear any constraints from the current sink wants that don't apply to

[kthelgason, 2017/03/14 10:01:20]

I guess this `if` should not be here?

[sprang_webrtc, 2017/03/14 14:15:03]

Done.

+    // the used degradation_preference.
+    switch (degradation_preference_) {
+      case DegradationPreference::kBalanced:
+        FALLTHROUGH();
+      case DegradationPreference::kMaintainFramerate:
+        wants.max_framerate_fps.reset();
+        break;
+      case DegradationPreference::kMaintainResolution:
+        wants.max_pixel_count.reset();
+        wants.target_pixel_count.reset();
+        break;
+      case DegradationPreference::kDegradationDisabled:
+        wants.max_pixel_count.reset();
+        wants.target_pixel_count.reset();
+        wants.max_framerate_fps.reset();
     }
+    return wants;
   }
 
   void RequestResolutionLowerThan(int pixel_count) {
     // Called on the encoder task queue.
     rtc::CritScope lock(&crit_);
     if (!IsResolutionScalingEnabledLocked()) {
       // This can happen since |degradation_preference_| is set on
       // libjingle's worker thread but the adaptation is done on the encoder
       // task queue.
       return;
     }
     // The input video frame size will have a resolution with less than or
     // equal to |max_pixel_count| depending on how the source can scale the
     // input frame size.
     const int pixels_wanted = (pixel_count * 3) / 5;
     if (pixels_wanted < kMinPixelsPerFrame)
       return;
     sink_wants_.max_pixel_count = rtc::Optional<int>(pixels_wanted);
     sink_wants_.target_pixel_count = rtc::Optional<int>();
     if (source_)
-      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
+  }
+
+  void RequestFramerateLowerThan(int framerate_fps) {
+    // Called on the encoder task queue.
+    rtc::CritScope lock(&crit_);
+    if (!IsFramerateScalingEnabledLocked()) {
+      // This can happen since |degradation_preference_| is set on
+      // libjingle's worker thread but the adaptation is done on the encoder
+      // task queue.
+      return;
+    }
+    // The input video frame rate will be scaled down to 2/3 of input fps,
+    // rounding down.
+    const int framerate_wanted =
+        std::max(kMinFramerateFps, (framerate_fps * 2) / 3);
+    sink_wants_.max_framerate_fps.emplace(framerate_wanted);
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
   }
 
   void RequestHigherResolutionThan(int pixel_count) {
     rtc::CritScope lock(&crit_);
     if (!IsResolutionScalingEnabledLocked()) {
       // This can happen since |degradation_preference_| is set on
       // libjingle's worker thread but the adaptation is done on the encoder
       // task queue.
       return;
     }
-    // On step down we request at most 3/5 the pixel count of the previous
-    // resolution, so in order to take "one step up" we request a resolution as
-    // close as possible to 5/3 of the current resolution. The actual pixel
-    // count selected depends on the capabilities of the source. In order to not
-    // take a too large step up, we cap the requested pixel count to be at most
-    // four time the current number of pixels.
-    sink_wants_.target_pixel_count = rtc::Optional<int>((pixel_count * 5) / 3);
-    sink_wants_.max_pixel_count = rtc::Optional<int>(pixel_count * 4);
+
+    if (pixel_count == std::numeric_limits<int>::max()) {
+      // Remove any restrains.

[kthelgason, 2017/03/14 10:01:20]

nit: constraints.

[sprang_webrtc, 2017/03/14 14:15:03]

Done.

+      sink_wants_.target_pixel_count.reset();
+      sink_wants_.max_pixel_count.reset();
+    } else {
+      // On step down we request at most 3/5 the pixel count of the previous
+      // resolution, so in order to take "one step up" we request a resolution
+      // as close as possible to 5/3 of the current resolution. The actual pixel
+      // count selected depends on the capabilities of the source. In order to
+      // not take a too large step up, we cap the requested pixel count to be at
+      // most four time the current number of pixels.
+      sink_wants_.target_pixel_count =
+          rtc::Optional<int>((pixel_count * 5) / 3);
+      sink_wants_.max_pixel_count = rtc::Optional<int>(pixel_count * 4);
+    }
     if (source_)
-      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
+  }
+
+  void RequestHigherFramerateThan(int framerate_fps) {
+    // Called on the encoder task queue.
+    rtc::CritScope lock(&crit_);
+    if (!IsFramerateScalingEnabledLocked()) {
+      // This can happen since |degradation_preference_| is set on
+      // libjingle's worker thread but the adaptation is done on the encoder
+      // task queue.
+      return;
+    }
+    if (framerate_fps == std::numeric_limits<int>::max()) {
+      // Remove any restrains.
+      sink_wants_.max_framerate_fps.reset();
+    } else {
+      // The input video frame rate will be scaled up to the last step, with
+      // rounding.
+      const int framerate_wanted = (framerate_fps * 3) / 2;
+      sink_wants_.max_framerate_fps.emplace(framerate_wanted);
+    }
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
   }
 
  private:
   bool IsResolutionScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     return degradation_preference_ !=
            DegradationPreference::kMaintainResolution;
   }
 
-  const rtc::VideoSinkWants& current_wants() const
+  bool IsFramerateScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
-    return IsResolutionScalingEnabledLocked() ? sink_wants_
-                                              : disabled_scaling_sink_wants_;
+    return degradation_preference_ ==
+           DegradationPreference::kMaintainResolution;
   }
 
   rtc::CriticalSection crit_;
   rtc::SequencedTaskChecker main_checker_;
   ViEEncoder* const vie_encoder_;
   rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
-  rtc::VideoSinkWants disabled_scaling_sink_wants_ GUARDED_BY(&crit_);
   DegradationPreference degradation_preference_ GUARDED_BY(&crit_);
   rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
 
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
@@ skipping 18 matching lines @@
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
       nack_enabled_(false),
       last_observed_bitrate_bps_(0),
       encoder_paused_and_dropped_frame_(false),
       has_received_sli_(false),
       picture_id_sli_(0),
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      scale_counter_(kScaleReasonSize, 0),
-      degradation_preference_(DegradationPreference::kMaintainResolution),
+      degradation_preference_(DegradationPreference::kDegradationDisabled),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       last_frame_log_ms_(clock_->TimeInMilliseconds()),
       captured_frame_count_(0),
       dropped_frame_count_(0),
       bitrate_observer_(nullptr),
       encoder_queue_("EncoderQueue") {
   RTC_DCHECK(stats_proxy);
   encoder_queue_.PostTask([this] {
@@ skipping 50 matching lines @@
   });
 }
 
 void ViEEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(source, degradation_preference);
   encoder_queue_.PostTask([this, degradation_preference] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-
+    if (degradation_preference_ != degradation_preference) {
+      // Reset adaptation state, so that we're not tricked into thinking there's
+      // an already pending request of the same type.
+      last_adaptation_request_.reset();
+    }
     degradation_preference_ = degradation_preference;
     initial_rampup_ =
         degradation_preference_ != DegradationPreference::kMaintainResolution
             ? 0
             : kMaxInitialFramedrop;
     ConfigureQualityScaler();
   });
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
@@ skipping 98 matching lines @@
     if (scaling_settings.thresholds) {
       quality_scaler_.reset(
           new QualityScaler(this, *(scaling_settings.thresholds)));
     } else {
       quality_scaler_.reset(new QualityScaler(this, codec_type_));
     }
   } else {
     quality_scaler_.reset(nullptr);
     initial_rampup_ = kMaxInitialFramedrop;
   }
+  const std::vector<int>& scale_counter = *GetScaleCounters();
   stats_proxy_->SetResolutionRestrictionStats(
-      degradation_preference_allows_scaling, scale_counter_[kCpu] > 0,
-      scale_counter_[kQuality]);
+      degradation_preference_allows_scaling, scale_counter[kCpu] > 0,
+      scale_counter[kQuality]);
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   VideoFrame incoming_frame = video_frame;
 
   // Local time in webrtc time base.
   int64_t current_time_us = clock_->TimeInMicroseconds();
   int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec;
   // TODO(nisse): This always overrides the incoming timestamp. Don't
@@ skipping 242 matching lines @@
 
   if (video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
 void ViEEncoder::AdaptDown(AdaptReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ != DegradationPreference::kBalanced)
-    return;
-  RTC_DCHECK(static_cast<bool>(last_frame_info_));
-  int current_pixel_count = last_frame_info_->pixel_count();
-  if (last_adaptation_request_ &&
-      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown &&
-      current_pixel_count >= last_adaptation_request_->input_pixel_count_) {
-    // Don't request lower resolution if the current resolution is not lower
-    // than the last time we asked for the resolution to be lowered.
-    return;
+  AdaptationRequest adaptation_request = {
+      last_frame_info_->pixel_count(),
+      stats_proxy_->GetStats().input_frame_rate,
+      AdaptationRequest::Mode::kAdaptDown};
+  bool downgrade_requested =
+      last_adaptation_request_ &&
+      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
+
+  int max_downgrades = 0;
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      FALLTHROUGH();
+    case DegradationPreference::kMaintainFramerate:
+      max_downgrades = kMaxCpuResolutionDowngrades;
+      if (downgrade_requested &&
+          adaptation_request.input_pixel_count_ >=
+              last_adaptation_request_->input_pixel_count_) {
+        // Don't request lower resolution if the current resolution is not
+        // lower than the last time we asked for the resolution to be lowered.
+        return;
+      }
+      break;
+    case DegradationPreference::kMaintainResolution:
+      max_downgrades = kMaxCpuFramerateDowngrades;
+      if (adaptation_request.framerate_fps_ <= 0 ||
+          (downgrade_requested &&
+           adaptation_request.framerate_fps_ < kMinFramerateFps)) {
+        // If no input fps estimate available, can't determine how to scale down
+        // framerate. Otherwise, don't request lower framerate if we don't have
+        // a valid frame rate. Since framerate, unlike resolution, is a measure
+        // we have to estimate, and can fluctuate naturally over time, don't
+        // make the same kind of limitations as for resolution, but trust the
+        // overuse detector to not trigger too often.
+        return;
+      }
+      break;
+    case DegradationPreference::kDegradationDisabled:
+      return;
   }
-  last_adaptation_request_.emplace(AdaptationRequest{
-      current_pixel_count, AdaptationRequest::Mode::kAdaptDown});
+
+  last_adaptation_request_.emplace(adaptation_request);
+  std::vector<int>* scale_counter = GetScaleCounters();
 
   switch (reason) {
     case kQuality:
       stats_proxy_->OnQualityRestrictedResolutionChanged(
-          scale_counter_[reason] + 1);
+          (*scale_counter)[reason] + 1);
       break;
     case kCpu:
-      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+      if ((*scale_counter)[reason] >= max_downgrades)
         return;
       // Update stats accordingly.
       stats_proxy_->OnCpuRestrictedResolutionChanged(true);
       break;
   }
-  ++scale_counter_[reason];
-  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
-  LOG(LS_INFO) << "Scaling down resolution.";
+  ++(*scale_counter)[reason];
+
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      FALLTHROUGH();
+    case DegradationPreference::kMaintainFramerate:
+      source_proxy_->RequestResolutionLowerThan(
+          adaptation_request.input_pixel_count_);
+      LOG(LS_INFO) << "Scaling down resolution.";
+      break;
+    case DegradationPreference::kMaintainResolution:
+      source_proxy_->RequestFramerateLowerThan(
+          adaptation_request.framerate_fps_);
+      LOG(LS_INFO) << "Scaling down framerate.";
+      break;
+    case DegradationPreference::kDegradationDisabled:
+      RTC_NOTREACHED();
+  }
+
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
-    LOG(LS_INFO) << "Scaled " << scale_counter_[i]
+    LOG(LS_INFO) << "Scaled " << (*scale_counter)[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
 void ViEEncoder::AdaptUp(AdaptReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (scale_counter_[reason] == 0 ||
-      degradation_preference_ != DegradationPreference::kBalanced) {
+  int* scale_counter = &((*GetScaleCounters())[reason]);

[kthelgason, 2017/03/14 10:01:20]

I'd prefer this to just be an int and then the vector updated afterwards with
the new value. It's easier to reason about when it's clear when the counters are
being updated.

[sprang_webrtc, 2017/03/14 14:15:03]

Done.

+  if (*scale_counter == 0)
     return;
+  RTC_DCHECK_GT(*scale_counter, 0);
+  AdaptationRequest adaptation_request = {
+      last_frame_info_->pixel_count(),
+      stats_proxy_->GetStats().input_frame_rate,
+      AdaptationRequest::Mode::kAdaptUp};
+
+  bool adapt_up_requested =
+      last_adaptation_request_ &&
+      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      FALLTHROUGH();
+    case DegradationPreference::kMaintainFramerate:
+      if (adapt_up_requested &&
+          adaptation_request.input_pixel_count_ <=
+              last_adaptation_request_->input_pixel_count_) {
+        // Don't request higher resolution if the current resolution is not
+        // higher than the last time we asked for the resolution to be higher.
+        return;
+      }
+      break;
+    case DegradationPreference::kMaintainResolution:
+      // TODO(sprang): Don't request higher framerate if we are already at
+      // max requested fps?
+      break;
+    case DegradationPreference::kDegradationDisabled:
+      return;
   }
-  // Only scale if resolution is higher than last time we requested higher
-  // resolution.
-  RTC_DCHECK(static_cast<bool>(last_frame_info_));
-  int current_pixel_count = last_frame_info_->pixel_count();
-  if (last_adaptation_request_ &&
-      last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp &&
-      current_pixel_count <= last_adaptation_request_->input_pixel_count_) {
-    // Don't request higher resolution if the current resolution is not higher
-    // than the last time we asked for the resolution to be higher.
-    return;
-  }
-  last_adaptation_request_.emplace(AdaptationRequest{
-      current_pixel_count, AdaptationRequest::Mode::kAdaptUp});
 
   switch (reason) {
     case kQuality:
-      stats_proxy_->OnQualityRestrictedResolutionChanged(
-          scale_counter_[reason] - 1);
+      stats_proxy_->OnQualityRestrictedResolutionChanged(*scale_counter - 1);
       break;
     case kCpu:
       // Update stats accordingly.
-      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
-                                                     1);
+      stats_proxy_->OnCpuRestrictedResolutionChanged(*scale_counter > 1);
       break;
   }
-  --scale_counter_[reason];
-  source_proxy_->RequestHigherResolutionThan(current_pixel_count);
-  LOG(LS_INFO) << "Scaling up resolution.";
+
+  // Decrease counter of how many times we have scaled down, for this
+  // degradation preference mode and reason.
+  --*scale_counter;
+
+  // Get a sum of how many times have scaled down, in total, for this
+  // degradation preference mode. If it is 0, remove any restraints.
+  const std::vector<int>& current_scale_counters = *GetScaleCounters();
+  int scale_sum = 0;
+  for (int i : current_scale_counters)
+    scale_sum += i;
+
+  switch (degradation_preference_) {
+    case DegradationPreference::kBalanced:
+      FALLTHROUGH();
+    case DegradationPreference::kMaintainFramerate:
+      if (scale_sum == 0) {
+        LOG(LS_INFO) << "Removing resolution down-scaling setting.";
+        source_proxy_->RequestHigherResolutionThan(
+            std::numeric_limits<int>::max());
+      } else {
+        source_proxy_->RequestHigherResolutionThan(
+            adaptation_request.input_pixel_count_);
+        LOG(LS_INFO) << "Scaling up resolution.";
+      }
+      break;
+    case DegradationPreference::kMaintainResolution:
+      if (scale_sum == 0) {
+        LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+        source_proxy_->RequestHigherFramerateThan(
+            std::numeric_limits<int>::max());
+      } else {
+        source_proxy_->RequestHigherFramerateThan(
+            adaptation_request.framerate_fps_);
+        LOG(LS_INFO) << "Scaling up framerate.";
+      }
+      break;
+    case DegradationPreference::kDegradationDisabled:
+      RTC_NOTREACHED();
+  }
+
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
-    LOG(LS_INFO) << "Scaled " << scale_counter_[i]
+    LOG(LS_INFO) << "Scaled " << (*GetScaleCounters())[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
+std::vector<int>* ViEEncoder::GetScaleCounters() {

[kthelgason, 2017/03/14 10:01:20]

Can we have this return a `std::vector<int>&` instead? It seems like we are
having to do a lot of dereferencing in the code.

[sprang_webrtc, 2017/03/14 14:15:03]

Done.

+  auto it = scale_counters_.find(degradation_preference_);
+  if (it == scale_counters_.end()) {
+    scale_counters_[degradation_preference_].resize(kScaleReasonSize);
+    return GetScaleCounters();

[kthelgason, 2017/03/14 10:01:20]

Is there a reason for not just returning the newly inserted vector?

[sprang_webrtc, 2017/03/14 14:15:03]

Done.

+  }
+  return &it->second;
+}
+
 }  // namespace webrtc