Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 235 matching lines @@
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
       last_observed_bitrate_bps_(0),
       encoder_paused_and_dropped_frame_(false),
       has_received_sli_(false),
       picture_id_sli_(0),
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      degradation_preference_(
-          VideoSendStream::DegradationPreference::kBalanced),
-      cpu_restricted_counter_(0),
       last_frame_width_(0),
       last_frame_height_(0),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       last_frame_log_ms_(clock_->TimeInMilliseconds()),
       captured_frame_count_(0),
       dropped_frame_count_(0),
       encoder_queue_("EncoderQueue") {
   encoder_queue_.PostTask([this] {
@@ skipping 12 matching lines @@
 
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference());
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.StopCheckForOveruse();
     rate_allocator_.reset();
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
+    quality_scaler_ = nullptr;
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(!module_process_thread_);
   module_process_thread_ = module_process_thread;
   module_process_thread_->RegisterModule(&video_sender_);
   module_process_thread_checker_.DetachFromThread();
 }
 
 void ViEEncoder::DeRegisterProcessThread() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   module_process_thread_->DeRegisterModule(&video_sender_);
 }
 
 void ViEEncoder::SetSource(
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(source, degradation_preference);
   encoder_queue_.PostTask([this, degradation_preference] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
-    degradation_preference_ = degradation_preference;
-    // Set the stats for if we are currently CPU restricted. We are CPU
-    // restricted depending on degradation preference and
-    // if the overusedetector has currently detected overuse which is counted in
-    // |cpu_restricted_counter_|
-    // We do this on the encoder task queue to avoid a race with the stats set
-    // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected.
-    stats_proxy_->SetCpuRestrictedResolution(
-        degradation_preference_ !=
-            VideoSendStream::DegradationPreference::kMaintainResolution &&
-        cpu_restricted_counter_ != 0);
+    scaling_enabled_ =

[perkj_webrtc, 2016/11/17 21:23:38]

Why not store degradation_preference ? 

[kthelgason, 2016/11/21 13:06:53]

Just because I thought this was enough. It also means that I don't always have
to make the comparison against that 70 character-long constant.

+        (degradation_preference !=
+        VideoSendStream::DegradationPreference::kMaintainResolution);
+    stats_proxy_->SetResolutionRestrictionStats(
+        scaling_enabled_ && scale_counter_[kQuality] > 0,
+        scaling_enabled_ && scale_counter_[kCpu] > 0);
   });
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
   source_proxy_->SetWantsRotationApplied(rotation_applied);
   encoder_queue_.PostTask([this, sink] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     sink_ = sink;
   });
 }
@@ skipping 67 matching lines @@
     if (framerate == 0)
       framerate = codec.maxFramerate;
     stats_proxy_->OnEncoderReconfigured(
         encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate));
   }
 
   pending_encoder_reconfiguration_ = false;
 
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
+
+  const auto scaling_settings = settings_.encoder->GetScalingSettings();
+  if (scaling_settings.enabled && scaling_enabled_) {
+    LOG(LS_INFO) << "Initializing quality scaler.";
+    if (scaling_settings.thresholds) {
+      quality_scaler_.reset(
+          new QualityScaler(this, *(scaling_settings.thresholds)));
+    } else {
+      quality_scaler_.reset(new QualityScaler(this, codec_type_));
+    }
+  } else {
+    quality_scaler_.reset(nullptr);
+  }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   VideoFrame incoming_frame = video_frame;
 
   // Local time in webrtc time base.
   int64_t current_time = clock_->TimeInMilliseconds();
   incoming_frame.set_render_time_ms(current_time);
 
@@ skipping 57 matching lines @@
   // End trace event on first frame after encoder resumes, if frame was dropped.
   if (encoder_paused_and_dropped_frame_) {
     TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
   }
   encoder_paused_and_dropped_frame_ = false;
 }
 
 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                   int64_t time_when_posted_in_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
+
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
   if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
       video_frame.height() != last_frame_info_->height ||
       video_frame.rotation() != last_frame_info_->rotation ||
       video_frame.is_texture() != last_frame_info_->is_texture) {
     pending_encoder_reconfiguration_ = true;
     last_frame_info_ = rtc::Optional<VideoFrameInfo>(
         VideoFrameInfo(video_frame.width(), video_frame.height(),
@@ skipping 57 matching lines @@
   // running in parallel on different threads.
   if (stats_proxy_) {
     stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
   }
 
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
 
   int64_t time_sent = clock_->TimeInMilliseconds();
   uint32_t timestamp = encoded_image._timeStamp;
-
-  encoder_queue_.PostTask([this, timestamp, time_sent] {
+  encoder_queue_.PostTask([this, timestamp, time_sent, encoded_image] {

[perkj_webrtc, 2016/11/17 21:23:38]

Sorry if I missed this before. But you can not post encoded_image. This will
create a copy of encoded_image but encoded_image contains pointers to data that
is not valid when the task is processed. So you just copy and post the value you
really need. ie, encoded_image.qp_.

[kthelgason, 2016/11/21 13:06:53]

Thanks, done.

     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.FrameSent(timestamp, time_sent);
+    if (quality_scaler_)
+      quality_scaler_->ReportQP(encoded_image.qp_);
   });
 
   return result;
 }
 
 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) {
   RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
   if (stats_proxy_)
     stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate);
 }
@@ skipping 56 matching lines @@
   bool video_suspension_changed = video_is_suspended != EncoderPaused();
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
-void ViEEncoder::OveruseDetected() {
+void ViEEncoder::ScaleDown(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ >= kMaxCpuDowngrades) {
+
+  // Request lower resolution if the current resolution is lower than last time
+  // we asked for the resolution to be lowered.
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) {

[stefan-webrtc, 2016/11/17 16:12:57]

This sounds incorrect to me. If current_pixel_count is greater than max we
should scale down until we're below max, no?

[perkj_webrtc, 2016/11/17 21:23:38]

No, when max_pixel_count is set it means we are ordering the capturer to reduce
the resolution to max_pixel_count_. 
So this happens when the capturer has not yet fulfilled the request but the
quality scaler or overusedetector reports overuse/bad quality again.  

[stefan-webrtc, 2016/11/21 13:02:57]

So max_pixel_count_ is the currently requested max resolution? Maybe the
variable name could be improved to avoid the ambiguity.

[kthelgason, 2016/11/21 13:06:53]

Acknowledged.

     return;
   }
-  LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution.";
-  // Request lower resolution if the current resolution is lower than last time
-  // we asked for the resolution to be lowered.
-  // Update stats accordingly.
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) {
-    max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
-    max_pixel_count_step_up_ = rtc::Optional<int>();
-    stats_proxy_->OnCpuRestrictedResolutionChanged(true);
-    ++cpu_restricted_counter_;
-    source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  switch (reason) {
+    case kQuality:
+      if (!scaling_enabled_)

[perkj_webrtc, 2016/11/17 21:23:38]

yes we should. Otherwise this will give you problem for cpu adaptation.

Also, if sprang is working on frame rate degradation I think you should keep
degradation_preferences instead of just a bool for resolution scaling.

[kthelgason, 2016/11/21 13:06:53]

If that helps his work he should do it in his CL. I don't want to make
assumptions here about how his implementation will work, as they will probably
turn out to be wrong.

+        return;

[stefan-webrtc, 2016/11/17 16:12:57]

Shouldn't we check this at the top and return early, or does scaling_enabled_
only apply to quality scaling for some reason?

[kthelgason, 2016/11/21 13:06:53]

For some reason I thought CPU adaptation should be exempt from this. Fixed.

+      if (scale_counter_[reason] >= kMaxQualityDowngrades)
+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(true);
+      break;
+    case kCpu:
+      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+        return;
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(true);
+      break;
+  }
+  max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
+  max_pixel_count_step_up_ = rtc::Optional<int>();
+  ++scale_counter_[reason];
+  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  LOG(LS_INFO) << "Scaling down resolution.";
+  for (size_t i = 0; i < kScaleReasonSize; ++i) {
+    LOG(LS_INFO) << "Scaled " << scale_counter_[i]
+                 << " times for reason: " << (i ? "quality" : "cpu");
   }
 }
 
-void ViEEncoder::NormalUsage() {
+void ViEEncoder::ScaleUp(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (degradation_preference_ ==
-          VideoSendStream::DegradationPreference::kMaintainResolution ||
-      cpu_restricted_counter_ == 0) {
+  if (scale_counter_[reason] == 0)
     return;
+
+  // Only scale if resolution is higher than last time
+  // we requested higher resolution.
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count <= max_pixel_count_step_up_.value_or(0))
+    return;
+  switch (reason) {
+    case kQuality:
+      if (!scaling_enabled_)

[perkj_webrtc, 2016/11/17 21:23:38]

dito.

[kthelgason, 2016/11/21 13:06:53]

Done.

+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(
+          scale_counter_[reason] > 1);
+      break;
+    case kCpu:
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
+                                                     1);
+      break;
   }
-
-  LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution.";
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  // Request higher resolution if we are CPU restricted and the the current
-  // resolution is higher than last time we requested higher resolution.
-  // Update stats accordingly.
-  if (!max_pixel_count_step_up_ ||
-      current_pixel_count > *max_pixel_count_step_up_) {
-    max_pixel_count_ = rtc::Optional<int>();
-    max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
-    --cpu_restricted_counter_;
-    stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
-    source_proxy_->RequestHigherResolutionThan(current_pixel_count);
+  max_pixel_count_ = rtc::Optional<int>();
+  max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
+  --scale_counter_[reason];
+  source_proxy_->RequestHigherResolutionThan(current_pixel_count);
+  LOG(LS_INFO) << "Scaling up resolution.";
+  for (size_t i = 0; i < kScaleReasonSize; ++i) {
+    LOG(LS_INFO) << "Scaled " << scale_counter_[i]
+                 << " times for reason: " << (i ? "quality" : "cpu");
   }
 }
 
 }  // namespace webrtc