Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.cc

Index: webrtc/video/vie_encoder.cc
diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc
index 7cc799a644058291dd577927b47af91e6cf5c85c..f4c26360f824b8260c42fdf122c730fa5de8f13c 100644
--- a/webrtc/video/vie_encoder.cc
+++ b/webrtc/video/vie_encoder.cc
@@ -30,6 +30,10 @@ namespace webrtc {
 namespace {
 // Time interval for logging frame counts.
 const int64_t kFrameLogIntervalMs = 60000;
+// Downscale resolution at most 2 times for CPU reasons.
+static const int kMaxCpuDowngrades = 2;
+// Downscale resolution at most 4 times for low-quality reasons.
+static const int kMaxQualityDowngrades = 4;
 
 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) {
   if (payload_name == "VP8")
@@ -337,6 +341,11 @@ class ViEEncoder::VideoSourceProxy {
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
+  bool IsResolutionScalingEnabled() const {
+    rtc::CritScope lock(&crit_);
+    return IsResolutionScalingEnabledLocked();
+  }
+
  private:
   bool IsResolutionScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
@@ -392,9 +401,6 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      degradation_preference_(
-          VideoSendStream::DegradationPreference::kBalanced),
-      cpu_restricted_counter_(0),
       last_frame_width_(0),
       last_frame_height_(0),
       last_captured_timestamp_(0),
@@ -426,6 +432,7 @@ void ViEEncoder::Stop() {
     overuse_detector_.StopCheckForOveruse();
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
+    quality_scaler_ = nullptr;
     shutdown_event_.Set();
   });
 
@@ -449,20 +456,15 @@ void ViEEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
+  const bool scaling_enabled =
+      degradation_preference !=
+      VideoSendStream::DegradationPreference::kMaintainResolution;
   source_proxy_->SetSource(source, degradation_preference);
-  encoder_queue_.PostTask([this, degradation_preference] {
+  encoder_queue_.PostTask([this, scaling_enabled] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    degradation_preference_ = degradation_preference;
-    // Set the stats for if we are currently CPU restricted. We are CPU
-    // restricted depending on degradation preference and
-    // if the overusedetector has currently detected overuse which is counted in
-    // |cpu_restricted_counter_|
-    // We do this on the encoder task queue to avoid a race with the stats set
-    // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected.
-    stats_proxy_->SetCpuRestrictedResolution(
-        degradation_preference_ !=
-            VideoSendStream::DegradationPreference::kMaintainResolution &&
-        cpu_restricted_counter_ != 0);
+    stats_proxy_->SetResolutionRestrictionStats(
+        scaling_enabled && scale_counter_[kQuality] > 0,
+        scaling_enabled && scale_counter_[kCpu] > 0);
   });
 }
 
@@ -535,10 +537,6 @@ void ViEEncoder::ReconfigureEncoder() {
   }
 
   rate_allocator_.reset(new SimulcastRateAllocator(codec));
-  if (stats_proxy_) {
-    stats_proxy_->OnEncoderReconfigured(encoder_config_,
-                                        rate_allocator_->GetPreferedBitrate());
-  }
 
   pending_encoder_reconfiguration_ = false;
   if (stats_proxy_) {
@@ -547,6 +545,17 @@ void ViEEncoder::ReconfigureEncoder() {
   }
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
+
+  const auto scaling_settings = settings_.encoder->GetQPThresholds();
+  if (scaling_settings.enabled && source_proxy_->IsResolutionScalingEnabled()) {

[perkj_webrtc, 2016/11/14 07:21:16]

should you delete the quality scaler if this is not true? 

ie- switching from camera to screen share? 

[kthelgason, 2016/11/14 14:36:44]

Good catch, thanks. It would not have actually scaled, as we check that scaling
is enabled for the current source before scaling, but it's a waste of CPU
cycles.

+    LOG(LS_INFO) << "Initializing quality scaler.";
+    if (scaling_settings.thresholds) {
+      quality_scaler_.reset(
+          new QualityScaler(this, *(scaling_settings.thresholds)));
+    } else {
+      quality_scaler_.reset(new QualityScaler(this, codec_type_));
+    }
+  }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
@@ -624,6 +633,7 @@ void ViEEncoder::TraceFrameDropEnd() {
 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                   int64_t time_when_posted_in_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
+
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
@@ -705,10 +715,11 @@ EncodedImageCallback::Result ViEEncoder::OnEncodedImage(
 
   int64_t time_sent = clock_->TimeInMilliseconds();
   uint32_t timestamp = encoded_image._timeStamp;
-
-  encoder_queue_.PostTask([this, timestamp, time_sent] {
+  encoder_queue_.PostTask([this, timestamp, time_sent, encoded_image] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.FrameSent(timestamp, time_sent);
+    if (quality_scaler_)

[perkj_webrtc, 2016/11/14 07:21:16]

quality_scaler is still set here if you switch from camera to screen share.

[kthelgason, 2016/11/14 14:36:45]

Acknowledged.

+      quality_scaler_->ReportQP(encoded_image.qp_);
   });
 
   return result;
@@ -786,48 +797,59 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps,
   }
 }
 
-void ViEEncoder::OveruseDetected() {
+void ViEEncoder::ScaleDown(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ >= kMaxCpuDowngrades) {
-    return;
-  }
-  LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution.";
+
   // Request lower resolution if the current resolution is lower than last time
   // we asked for the resolution to be lowered.
-  // Update stats accordingly.
   int current_pixel_count = last_frame_height_ * last_frame_width_;
-  if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) {
-    max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
-    max_pixel_count_step_up_ = rtc::Optional<int>();
-    stats_proxy_->OnCpuRestrictedResolutionChanged(true);
-    ++cpu_restricted_counter_;
-    source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) {
+    return;
+  }
+  switch (reason) {
+    case kQuality:
+      if (scale_counter_[reason] >= kMaxQualityDowngrades)
+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(true);
+      break;
+    case kCpu:
+      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+        return;
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(true);
+      break;
   }
+  max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
+  max_pixel_count_step_up_ = rtc::Optional<int>();
+  ++scale_counter_[reason];

[perkj_webrtc, 2016/11/14 07:21:16]

Please add a LOG(LS_INFO) with the info about why we go up/down in resolution
and include the value of both reasons.

[kthelgason, 2016/11/14 14:36:44]

Done.

+  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
 }
 
-void ViEEncoder::NormalUsage() {
+void ViEEncoder::ScaleUp(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ == 0) {
+  if (scale_counter_[reason] == 0)
     return;
-  }
 
-  LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution.";
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  // Request higher resolution if we are CPU restricted and the the current
+  // Request higher resolution if we are cpu restricted and the the current

[perkj_webrtc, 2016/11/14 07:21:16]

cpu ? 

[kthelgason, 2016/11/14 14:36:44]

Done.

   // resolution is higher than last time we requested higher resolution.
-  // Update stats accordingly.
-  if (!max_pixel_count_step_up_ ||
-      current_pixel_count > *max_pixel_count_step_up_) {
-    max_pixel_count_ = rtc::Optional<int>();
-    max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
-    --cpu_restricted_counter_;
-    stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
-    source_proxy_->RequestHigherResolutionThan(current_pixel_count);
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count <= max_pixel_count_step_up_.value_or(0))
+    return;
+  switch (reason) {
+    case kQuality:
+      stats_proxy_->OnQualityRestrictedResolutionChanged(
+          scale_counter_[reason] > 1);
+      break;
+    case kCpu:
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
+                                                     1);
+      break;
   }
+  max_pixel_count_ = rtc::Optional<int>();
+  max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
+  --scale_counter_[reason];
+  source_proxy_->RequestHigherResolutionThan(current_pixel_count);

[perkj_webrtc, 2016/11/14 07:21:16]

And a log here too please.

[kthelgason, 2016/11/14 14:36:44]

Done.

 }
 
 }  // namespace webrtc