Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 12 matching lines @@
 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
 #include "webrtc/video/overuse_frame_detector.h"
 #include "webrtc/video/send_statistics_proxy.h"
 #include "webrtc/video_frame.h"
 
 namespace webrtc {
 
 namespace {
 // Time interval for logging frame counts.
 const int64_t kFrameLogIntervalMs = 60000;
+// Downscale resolution at most 2 times for CPU reasons.
+static const int kMaxCpuDowngrades = 2;
+// Downscale resolution at most 4 times for low-quality reasons.
+static const int kMaxQualityDowngrades = 4;
 
 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) {
   if (payload_name == "VP8")
     return kVideoCodecVP8;
   if (payload_name == "VP9")
     return kVideoCodecVP9;
   if (payload_name == "H264")
     return kVideoCodecH264;
   return kVideoCodecGeneric;
 }
@@ skipping 287 matching lines @@
     }
     // The input video frame size will have a resolution with "one step up"
     // pixels than |max_pixel_count_step_up| where "one step up" depends on
     // how the source can scale the input frame size.
     sink_wants_.max_pixel_count = rtc::Optional<int>();
     sink_wants_.max_pixel_count_step_up = rtc::Optional<int>(pixel_count);
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
+  bool IsResolutionScalingEnabled() const {
+    rtc::CritScope lock(&crit_);
+    return IsResolutionScalingEnabledLocked();
+  }
+
  private:
   bool IsResolutionScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     return degradation_preference_ !=
            VideoSendStream::DegradationPreference::kMaintainResolution;
   }
 
   const rtc::VideoSinkWants& current_wants() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     return IsResolutionScalingEnabledLocked() ? sink_wants_
@@ skipping 35 matching lines @@
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
       last_observed_bitrate_bps_(0),
       encoder_paused_and_dropped_frame_(false),
       has_received_sli_(false),
       picture_id_sli_(0),
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      degradation_preference_(
-          VideoSendStream::DegradationPreference::kBalanced),
-      cpu_restricted_counter_(0),
       last_frame_width_(0),
       last_frame_height_(0),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       last_frame_log_ms_(clock_->TimeInMilliseconds()),
       captured_frame_count_(0),
       dropped_frame_count_(0),
       encoder_queue_("EncoderQueue") {
   encoder_queue_.PostTask([this] {
@@ skipping 11 matching lines @@
 }
 
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference());
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.StopCheckForOveruse();
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
+    quality_scaler_ = nullptr;
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(!module_process_thread_);
   module_process_thread_ = module_process_thread;
   module_process_thread_->RegisterModule(&video_sender_);
   module_process_thread_checker_.DetachFromThread();
 }
 
 void ViEEncoder::DeRegisterProcessThread() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   module_process_thread_->DeRegisterModule(&video_sender_);
 }
 
 void ViEEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
+  const bool scaling_enabled =
+      degradation_preference !=
+      VideoSendStream::DegradationPreference::kMaintainResolution;
   source_proxy_->SetSource(source, degradation_preference);
-  encoder_queue_.PostTask([this, degradation_preference] {
+  encoder_queue_.PostTask([this, scaling_enabled] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    degradation_preference_ = degradation_preference;
-    // Set the stats for if we are currently CPU restricted. We are CPU
-    // restricted depending on degradation preference and
-    // if the overusedetector has currently detected overuse which is counted in
-    // |cpu_restricted_counter_|
-    // We do this on the encoder task queue to avoid a race with the stats set
-    // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected.
-    stats_proxy_->SetCpuRestrictedResolution(
-        degradation_preference_ !=
-            VideoSendStream::DegradationPreference::kMaintainResolution &&
-        cpu_restricted_counter_ != 0);
+    stats_proxy_->SetResolutionRestrictionStats(
+        scaling_enabled && scale_counter_[kQuality] > 0,
+        scaling_enabled && scale_counter_[kCpu] > 0);
   });
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
   source_proxy_->SetWantsRotationApplied(rotation_applied);
   encoder_queue_.PostTask([this, sink] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     sink_ = sink;
   });
 }
@@ skipping 52 matching lines @@
 
   bool success = video_sender_.RegisterSendCodec(
                      &codec, number_of_cores_,
                      static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
   if (!success) {
     LOG(LS_ERROR) << "Failed to configure encoder.";
     RTC_DCHECK(success);
   }
 
   rate_allocator_.reset(new SimulcastRateAllocator(codec));
-  if (stats_proxy_) {
-    stats_proxy_->OnEncoderReconfigured(encoder_config_,
-                                        rate_allocator_->GetPreferedBitrate());
-  }
 
   pending_encoder_reconfiguration_ = false;
   if (stats_proxy_) {
     stats_proxy_->OnEncoderReconfigured(encoder_config_,
                                         rate_allocator_->GetPreferedBitrate());
   }
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
+
+  const auto scaling_settings = settings_.encoder->GetQPThresholds();
+  if (scaling_settings.enabled && source_proxy_->IsResolutionScalingEnabled()) {
+    LOG(LS_INFO) << "Initializing quality scaler.";
+    if (scaling_settings.thresholds) {
+      quality_scaler_.reset(
+          new QualityScaler(this, *(scaling_settings.thresholds)));
+    } else {
+      quality_scaler_.reset(new QualityScaler(this, codec_type_));
+    }
+  }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   VideoFrame incoming_frame = video_frame;
 
   // Local time in webrtc time base.
   int64_t current_time = clock_->TimeInMilliseconds();
   incoming_frame.set_render_time_ms(current_time);
 
@@ skipping 57 matching lines @@
   // End trace event on first frame after encoder resumes, if frame was dropped.
   if (encoder_paused_and_dropped_frame_) {
     TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
   }
   encoder_paused_and_dropped_frame_ = false;
 }
 
 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                   int64_t time_when_posted_in_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
+
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
   if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
       video_frame.height() != last_frame_info_->height ||
       video_frame.rotation() != last_frame_info_->rotation ||
       video_frame.is_texture() != last_frame_info_->is_texture) {
     pending_encoder_reconfiguration_ = true;
     last_frame_info_ = rtc::Optional<VideoFrameInfo>(
         VideoFrameInfo(video_frame.width(), video_frame.height(),
@@ skipping 61 matching lines @@
   // running in parallel on different threads.
   if (stats_proxy_) {
     stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
   }
 
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
 
   int64_t time_sent = clock_->TimeInMilliseconds();
   uint32_t timestamp = encoded_image._timeStamp;
-
-  encoder_queue_.PostTask([this, timestamp, time_sent] {
+  encoder_queue_.PostTask([this, timestamp, time_sent, encoded_image] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.FrameSent(timestamp, time_sent);
+    if (quality_scaler_)
+      quality_scaler_->ReportQP(encoded_image.qp_);
   });
 
   return result;
 }
 
 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) {
   RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
   if (stats_proxy_)
     stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate);
 }
@@ skipping 57 matching lines @@
       video_is_suspended != (last_observed_bitrate_bps_ == 0);
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
-void ViEEncoder::OveruseDetected() {
+void ViEEncoder::ScaleDown(ScaleReason reason) {

[magjed_webrtc, 2016/11/10 13:07:00]

Wouldn't it be easier to send |current_pixel_count| as an argument? Wouldn't
that make Quality and CPU scaling more independent and avoid races?

[kthelgason, 2016/11/10 15:38:28]

The QualityScaler and CpuOveruseDetector don't know (and probably shouldn't
know) the current_pixel_count. 
Also, They aren't independent, as we maintain different counters, so even though
we have scaled down for quality reasons we might still let the cpu adapter scale
down further.
The stats also have to be updated depending on the scale reason.

[magjed_webrtc, 2016/11/10 19:29:07]

QualityScaler used to store the resolution in 'Resolution target_res_' and
OveruseDetector stores it in |int num_pixels_| so it should be possible for them
to know what pixel count they are on. My point is that those pixel counts might
be out of sync with the pixel count here (last_frame_width_/last_frame_height_)
since these are asynchronous events that take a few ms to propagate. What
happens if e.g. QP scaler decides to call ScaleDown one frame after CPU scaler
has called ScaleDown? It looks like we will scale down twice then, if the
|last_frame_| resolution was updated between. I don't think this is a problem in
practice, so I'm willing to leave this code as is.

   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ >= kMaxCpuDowngrades) {
+
+  // Request lower resolution if the current resolution is lower than last time
+  // we asked for the resolution to be lowered.
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) {
     return;
   }
-  LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution.";
-  // Request lower resolution if the current resolution is lower than last time
-  // we asked for the resolution to be lowered.
-  // Update stats accordingly.
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) {
-    max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
-    max_pixel_count_step_up_ = rtc::Optional<int>();
-    stats_proxy_->OnCpuRestrictedResolutionChanged(true);
-    ++cpu_restricted_counter_;
-    source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  switch (reason) {
+    case kQuality:
+      if (scale_counter_[reason] >= kMaxQualityDowngrades)
+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(true);
+      break;
+    case kCpu:
+      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+        return;
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(true);
+      break;
   }
+  max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
+  max_pixel_count_step_up_ = rtc::Optional<int>();
+  ++scale_counter_[reason];
+  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
 }
 
-void ViEEncoder::NormalUsage() {
+void ViEEncoder::ScaleUp(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ == 0) {
+  if (scale_counter_[reason] == 0)
     return;
+
+  // Request higher resolution if we are cpu restricted and the the current
+  // resolution is higher than last time we requested higher resolution.
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count <= max_pixel_count_step_up_.value_or(0))
+    return;
+  switch (reason) {
+    case kQuality:
+      stats_proxy_->OnQualityRestrictedResolutionChanged(
+          scale_counter_[reason] > 1);
+      break;
+    case kCpu:
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
+                                                     1);
+      break;
   }
-
-  LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution.";
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  // Request higher resolution if we are CPU restricted and the the current
-  // resolution is higher than last time we requested higher resolution.
-  // Update stats accordingly.
-  if (!max_pixel_count_step_up_ ||
-      current_pixel_count > *max_pixel_count_step_up_) {
-    max_pixel_count_ = rtc::Optional<int>();
-    max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
-    --cpu_restricted_counter_;
-    stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
-    source_proxy_->RequestHigherResolutionThan(current_pixel_count);
-  }
+  max_pixel_count_ = rtc::Optional<int>();
+  max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
+  --scale_counter_[reason];
+  source_proxy_->RequestHigherResolutionThan(current_pixel_count);
 }
 
 }  // namespace webrtc