Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.cc

Index: webrtc/video/vie_encoder.cc
diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc
index 7cc799a644058291dd577927b47af91e6cf5c85c..f4c26360f824b8260c42fdf122c730fa5de8f13c 100644
--- a/webrtc/video/vie_encoder.cc
+++ b/webrtc/video/vie_encoder.cc
@@ -30,6 +30,10 @@ namespace webrtc {
 namespace {
 // Time interval for logging frame counts.
 const int64_t kFrameLogIntervalMs = 60000;
+// Downscale resolution at most 2 times for CPU reasons.
+static const int kMaxCpuDowngrades = 2;
+// Downscale resolution at most 4 times for low-quality reasons.
+static const int kMaxQualityDowngrades = 4;
 
 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) {
   if (payload_name == "VP8")
@@ -337,6 +341,11 @@ class ViEEncoder::VideoSourceProxy {
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
+  bool IsResolutionScalingEnabled() const {
+    rtc::CritScope lock(&crit_);
+    return IsResolutionScalingEnabledLocked();
+  }
+
  private:
   bool IsResolutionScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
@@ -392,9 +401,6 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      degradation_preference_(
-          VideoSendStream::DegradationPreference::kBalanced),
-      cpu_restricted_counter_(0),
       last_frame_width_(0),
       last_frame_height_(0),
       last_captured_timestamp_(0),
@@ -426,6 +432,7 @@ void ViEEncoder::Stop() {
     overuse_detector_.StopCheckForOveruse();
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
+    quality_scaler_ = nullptr;
     shutdown_event_.Set();
   });
 
@@ -449,20 +456,15 @@ void ViEEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
+  const bool scaling_enabled =
+      degradation_preference !=
+      VideoSendStream::DegradationPreference::kMaintainResolution;
   source_proxy_->SetSource(source, degradation_preference);
-  encoder_queue_.PostTask([this, degradation_preference] {
+  encoder_queue_.PostTask([this, scaling_enabled] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    degradation_preference_ = degradation_preference;
-    // Set the stats for if we are currently CPU restricted. We are CPU
-    // restricted depending on degradation preference and
-    // if the overusedetector has currently detected overuse which is counted in
-    // |cpu_restricted_counter_|
-    // We do this on the encoder task queue to avoid a race with the stats set
-    // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected.
-    stats_proxy_->SetCpuRestrictedResolution(
-        degradation_preference_ !=
-            VideoSendStream::DegradationPreference::kMaintainResolution &&
-        cpu_restricted_counter_ != 0);
+    stats_proxy_->SetResolutionRestrictionStats(
+        scaling_enabled && scale_counter_[kQuality] > 0,
+        scaling_enabled && scale_counter_[kCpu] > 0);
   });
 }
 
@@ -535,10 +537,6 @@ void ViEEncoder::ReconfigureEncoder() {
   }
 
   rate_allocator_.reset(new SimulcastRateAllocator(codec));
-  if (stats_proxy_) {
-    stats_proxy_->OnEncoderReconfigured(encoder_config_,
-                                        rate_allocator_->GetPreferedBitrate());
-  }
 
   pending_encoder_reconfiguration_ = false;
   if (stats_proxy_) {
@@ -547,6 +545,17 @@ void ViEEncoder::ReconfigureEncoder() {
   }
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
+
+  const auto scaling_settings = settings_.encoder->GetQPThresholds();
+  if (scaling_settings.enabled && source_proxy_->IsResolutionScalingEnabled()) {
+    LOG(LS_INFO) << "Initializing quality scaler.";
+    if (scaling_settings.thresholds) {
+      quality_scaler_.reset(
+          new QualityScaler(this, *(scaling_settings.thresholds)));
+    } else {
+      quality_scaler_.reset(new QualityScaler(this, codec_type_));
+    }
+  }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
@@ -624,6 +633,7 @@ void ViEEncoder::TraceFrameDropEnd() {
 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                   int64_t time_when_posted_in_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
+
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
@@ -705,10 +715,11 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage(
 
   int64_t time_sent = clock_->TimeInMilliseconds();
   uint32_t timestamp = encoded_image._timeStamp;
-
-  encoder_queue_.PostTask([this, timestamp, time_sent] {
+  encoder_queue_.PostTask([this, timestamp, time_sent, encoded_image] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.FrameSent(timestamp, time_sent);
+    if (quality_scaler_)
+      quality_scaler_->ReportQP(encoded_image.qp_);
   });
 
   return result;
@@ -786,48 +797,59 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps,
   }
 }
 
-void ViEEncoder::OveruseDetected() {
+void ViEEncoder::ScaleDown(ScaleReason reason) {

[magjed_webrtc, 2016/11/10 13:07:00]

Wouldn't it be easier to send |current_pixel_count| as an argument? Wouldn't
that make Quality and CPU scaling more independent and avoid races?

[kthelgason, 2016/11/10 15:38:28]

The QualityScaler and CpuOveruseDetector don't know (and probably shouldn't
know) the current_pixel_count. 
Also, They aren't independent, as we maintain different counters, so even though
we have scaled down for quality reasons we might still let the cpu adapter scale
down further.
The stats also have to be updated depending on the scale reason.

[magjed_webrtc, 2016/11/10 19:29:07]

QualityScaler used to store the resolution in 'Resolution target_res_' and
OveruseDetector stores it in |int num_pixels_| so it should be possible for them
to know what pixel count they are on. My point is that those pixel counts might
be out of sync with the pixel count here (last_frame_width_/last_frame_height_)
since these are asynchronous events that take a few ms to propagate. What
happens if e.g. QP scaler decides to call ScaleDown one frame after CPU scaler
has called ScaleDown? It looks like we will scale down twice then, if the
|last_frame_| resolution was updated between. I don't think this is a problem in
practice, so I'm willing to leave this code as is.

   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ >= kMaxCpuDowngrades) {
-    return;
-  }
-  LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution.";
+
   // Request lower resolution if the current resolution is lower than last time
   // we asked for the resolution to be lowered.
-  // Update stats accordingly.
   int current_pixel_count = last_frame_height_ * last_frame_width_;
-  if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) {
-    max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
-    max_pixel_count_step_up_ = rtc::Optional<int>();
-    stats_proxy_->OnCpuRestrictedResolutionChanged(true);
-    ++cpu_restricted_counter_;
-    source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) {
+    return;
+  }
+  switch (reason) {
+    case kQuality:
+      if (scale_counter_[reason] >= kMaxQualityDowngrades)
+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(true);
+      break;
+    case kCpu:
+      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+        return;
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(true);
+      break;
   }
+  max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
+  max_pixel_count_step_up_ = rtc::Optional<int>();
+  ++scale_counter_[reason];
+  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
 }
 
-void ViEEncoder::NormalUsage() {
+void ViEEncoder::ScaleUp(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ == 0) {
+  if (scale_counter_[reason] == 0)
     return;
-  }
 
-  LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution.";
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  // Request higher resolution if we are CPU restricted and the the current
+  // Request higher resolution if we are cpu restricted and the the current
   // resolution is higher than last time we requested higher resolution.
-  // Update stats accordingly.
-  if (!max_pixel_count_step_up_ ||
-      current_pixel_count > *max_pixel_count_step_up_) {
-    max_pixel_count_ = rtc::Optional<int>();
-    max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
-    --cpu_restricted_counter_;
-    stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
-    source_proxy_->RequestHigherResolutionThan(current_pixel_count);
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count <= max_pixel_count_step_up_.value_or(0))
+    return;
+  switch (reason) {
+    case kQuality:
+      stats_proxy_->OnQualityRestrictedResolutionChanged(
+          scale_counter_[reason] > 1);
+      break;
+    case kCpu:
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
+                                                     1);
+      break;
   }
+  max_pixel_count_ = rtc::Optional<int>();
+  max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
+  --scale_counter_[reason];
+  source_proxy_->RequestHigherResolutionThan(current_pixel_count);
 }
 
 }  // namespace webrtc