Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 208 matching lines @@
         time_when_posted_ms_(time_when_posted_in_ms),
         log_stats_(log_stats) {
     frame_.ShallowCopy(frame);
     ++vie_encoder_->posted_frames_waiting_for_encode_;
   }
 
  private:
   bool Run() override {
     RTC_DCHECK_RUN_ON(&vie_encoder_->encoder_queue_);
     RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0);
-    vie_encoder_->stats_proxy_->OnIncomingFrame(
-        frame_.width(), frame_.height(),
-        vie_encoder_->cpu_restricted_counter_ != 0);
+    vie_encoder_->stats_proxy_->OnIncomingFrame(frame_.width(),
+                                                frame_.height());
     ++vie_encoder_->captured_frame_count_;
     if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) {
       vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_ms_);
     } else {
       // There is a newer frame in flight. Do not encode this frame.
       LOG(LS_VERBOSE)
           << "Incoming frame dropped due to that the encoder is blocked.";
       ++vie_encoder_->dropped_frame_count_;
     }
     if (log_stats_) {
@@ skipping 90 matching lines @@
     }
     // The input video frame size will have a resolution with "one step up"
     // pixels than |max_pixel_count_step_up| where "one step up" depends on
     // how the source can scale the input frame size.
     sink_wants_.max_pixel_count = rtc::Optional<int>();
     sink_wants_.max_pixel_count_step_up = rtc::Optional<int>(pixel_count);
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
+  bool CanScaleResolution() {
+    rtc::CritScope lock(&crit_);
+    return !resolution_scaling_disabled_;
+  }
+
  private:
   rtc::CriticalSection crit_;
   rtc::SequencedTaskChecker main_checker_;
   ViEEncoder* const vie_encoder_;
   rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
   rtc::VideoSinkWants disabled_scaling_sink_wants_ GUARDED_BY(&crit_);
   bool resolution_scaling_disabled_ GUARDED_BY(&crit_);
   rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
 
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
                        EncodedFrameObserver* encoder_timing)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
       source_proxy_(new VideoSourceProxy(this)),
       sink_(nullptr),
       settings_(settings),
       codec_type_(PayloadNameToCodecType(settings.payload_name)),
       video_sender_(Clock::GetRealTimeClock(), this, this),
-      disable_resolution_scaling_(false),
+      enable_quality_scaling_(false),
       overuse_detector_(Clock::GetRealTimeClock(),
                         GetCpuOveruseOptions(settings.full_overuse_time),
                         this,
                         encoder_timing,
                         stats_proxy),
+      quality_scaler_(new QualityScaler(this)),
       stats_proxy_(stats_proxy),
       pre_encode_callback_(pre_encode_callback),
       module_process_thread_(nullptr),
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
       last_observed_bitrate_bps_(0),
       encoder_paused_and_dropped_frame_(false),
       has_received_sli_(false),
       picture_id_sli_(0),
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
-      cpu_restricted_counter_(0),
       last_frame_width_(0),
       last_frame_height_(0),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       last_frame_log_ms_(clock_->TimeInMilliseconds()),
       captured_frame_count_(0),
       dropped_frame_count_(0),
       encoder_queue_("EncoderQueue") {
   encoder_queue_.PostTask([this, encoder_timing] {
@@ skipping 11 matching lines @@
 }
 
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr, false);
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
     overuse_detector_.StopCheckForOveruse();
+    if (enable_quality_scaling_)
+      quality_scaler_->Stop();
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(!module_process_thread_);
   module_process_thread_ = module_process_thread;
   module_process_thread_->RegisterModule(&video_sender_);
   module_process_thread_checker_.DetachFromThread();
 }
 
 void ViEEncoder::DeRegisterProcessThread() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   module_process_thread_->DeRegisterModule(&video_sender_);
 }
 
 void ViEEncoder::SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
                            bool disable_resolution_scaling) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(source, disable_resolution_scaling);
-  encoder_queue_.PostTask([this, disable_resolution_scaling] {
-    RTC_DCHECK_RUN_ON(&encoder_queue_);
-    if (disable_resolution_scaling_ != disable_resolution_scaling) {
-      stats_proxy_->SetCpuRestrictedResolution(!disable_resolution_scaling &&
-                                               cpu_restricted_counter_ != 0);
-    }
-    disable_resolution_scaling_ = disable_resolution_scaling;
-  });
+  stats_proxy_->SetResolutionRestrictionStats(
+      !disable_resolution_scaling && scale_counter_[kQuality] > 0,
+      !disable_resolution_scaling && scale_counter_[kCpu] > 0);
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
   source_proxy_->SetWantsRotationApplied(rotation_applied);
   encoder_queue_.PostTask([this, sink, rotation_applied] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     sink_ = sink;
   });
 }
 
@@ skipping 52 matching lines @@
 
   bool success = video_sender_.RegisterSendCodec(
                      &codec, number_of_cores_,
                      static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
   if (!success) {
     LOG(LS_ERROR) << "Failed to configure encoder.";
     RTC_DCHECK(success);
   }
 
   rate_allocator_.reset(new SimulcastRateAllocator(codec));
-  if (stats_proxy_) {
-    stats_proxy_->OnEncoderReconfigured(encoder_config_,
-                                        rate_allocator_->GetPreferedBitrate());
-  }
 
   pending_encoder_reconfiguration_ = false;
   if (stats_proxy_) {
     stats_proxy_->OnEncoderReconfigured(encoder_config_,
                                         rate_allocator_->GetPreferedBitrate());
   }
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
+
+  const auto scaling_settings = settings_.encoder->GetQPThresholds();
+  enable_quality_scaling_ = scaling_settings.enabled;
+  if (enable_quality_scaling_) {
+    LOG(LS_INFO) << "Initializing quality scaler.";
+    if (quality_scaler_)
+      quality_scaler_->Stop();
+    if (scaling_settings.thresholds) {
+      quality_scaler_->Init(*(scaling_settings.thresholds));
+    } else {
+      quality_scaler_->Init(codec_type_);
+    }
+  }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   VideoFrame incoming_frame = video_frame;
 
   // Local time in webrtc time base.
   int64_t current_time = clock_->TimeInMilliseconds();
   incoming_frame.set_render_time_ms(current_time);
 
@@ skipping 139 matching lines @@
   // running in parallel on different threads.
   if (stats_proxy_) {
     stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
   }
 
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
 
   int64_t time_sent = clock_->TimeInMilliseconds();
   uint32_t timestamp = encoded_image._timeStamp;
-  encoder_queue_.PostTask([this, timestamp, time_sent] {
+  encoder_queue_.PostTask([this, timestamp, time_sent, encoded_image] {

[perkj_webrtc, 2016/10/26 15:53:50]

This may contain invalid pointers. Copy only the value you need. encoded_image.

[kthelgason, 2016/10/26 19:02:50]

Sorry, I'm not following. Can you elaborate?

     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.FrameSent(timestamp, time_sent);
+    if (enable_quality_scaling_)
+      quality_scaler_->ReportQP(encoded_image.qp_);
   });
   return result;
 }
 
 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) {
   RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
   if (stats_proxy_)
     stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate);
 }
 
@@ skipping 56 matching lines @@
       video_is_suspended != (last_observed_bitrate_bps_ == 0);
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
-void ViEEncoder::OveruseDetected() {
+void ViEEncoder::ScaleDown(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-
-  if (disable_resolution_scaling_)
-    return;
-
-  if (cpu_restricted_counter_ >= kMaxCpuDowngrades) {
+  // Request lower resolution if the current resolution is lower than last time
+  // we asked for the resolution to be lowered.
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count >=
+      max_pixel_count_.value_or(current_pixel_count + 1)) {
     return;
   }
-  // Request lower resolution if the current resolution is lower than last time
-  // we asked for the resolution to be lowered.
-  // Update stats accordingly.
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
-  if (current_pixel_count <
-      max_pixel_count_.value_or(current_pixel_count + 1)) {
-    max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
-    max_pixel_count_step_up_ = rtc::Optional<int>();
-    stats_proxy_->OnCpuRestrictedResolutionChanged(true);
-    ++cpu_restricted_counter_;
-    source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  if (!source_proxy_->CanScaleResolution())
+    return;
+  max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
+  max_pixel_count_step_up_ = rtc::Optional<int>();
+
+  switch (reason) {
+    case kQuality:
+      if (!enable_quality_scaling_)
+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(true);
+      break;
+    case kCpu:
+      if (scale_counter_[reason] >= kMaxCpuDowngrades)
+        return;
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(true);
+      break;
   }
+  ++scale_counter_[reason];
+  source_proxy_->RequestResolutionLowerThan(current_pixel_count);
 }
 
-void ViEEncoder::NormalUsage() {
+void ViEEncoder::ScaleUp(ScaleReason reason) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (disable_resolution_scaling_)
-    return;
-
-  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  RTC_DCHECK_GE(scale_counter_[reason], 0);
   // Request higher resolution if we are cpu restricted and the the current
   // resolution is higher than last time we requested higher resolution.
-  // Update stats accordingly.
-  if (cpu_restricted_counter_ > 0 &&
-      current_pixel_count > max_pixel_count_step_up_.value_or(0)) {
-    max_pixel_count_ = rtc::Optional<int>();
-    max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
-    --cpu_restricted_counter_;
-    stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
-    source_proxy_->RequestHigherResolutionThan(current_pixel_count);
+  if (scale_counter_[reason] == 0)
+    return;
+  if (!source_proxy_->CanScaleResolution())
+    return;
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count <= max_pixel_count_step_up_.value_or(0))
+    return;
+  max_pixel_count_ = rtc::Optional<int>();
+  max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
+  switch (reason) {
+    case kQuality:
+      if (!enable_quality_scaling_)
+        return;
+      stats_proxy_->OnQualityRestrictedResolutionChanged(
+          scale_counter_[reason] > 1);
+      break;
+    case kCpu:
+      // Update stats accordingly.
+      stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
+                                                     1);
+      break;
   }
+  --scale_counter_[reason];
+  source_proxy_->RequestHigherResolutionThan(current_pixel_count);
 }
 
 }  // namespace webrtc