Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 219 matching lines @@
         frame_.width(), frame_.height(),
         vie_encoder_->cpu_restricted_counter_ != 0);
     ++vie_encoder_->captured_frame_count_;
     if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) {
       vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_ms_);
     } else {
       // There is a newer frame in flight. Do not encode this frame.
       LOG(LS_VERBOSE)
           << "Incoming frame dropped due to that the encoder is blocked.";
       ++vie_encoder_->dropped_frame_count_;
+      vie_encoder_->quality_scaler_.ReportDroppedFrame();

[perkj_webrtc, 2016/10/07 07:41:16]

note- this is not the same ReportDroppedFrame as previously. This are frames
dropped in the capture pipeline before beeing send to the encoder because the
encoder is not fast enough. 

ReportDroppedFrame was used when an encoder dropped a frame because either
bitrate was not enough, or probably also in the VideoSender when the encoder
used a too high bitrate.

[kthelgason, 2016/10/07 09:34:48]

Ok, that's good to know. At what point does ViEEncoder know that an encoder has
dropped a frame?

     }
     if (log_stats_) {
       LOG(LS_INFO) << "Number of frames: captured "
                    << vie_encoder_->captured_frame_count_
                    << ", dropped (due to encoder blocked) "
                    << vie_encoder_->dropped_frame_count_ << ", interval_ms "
                    << kFrameLogIntervalMs;
       vie_encoder_->captured_frame_count_ = 0;
       vie_encoder_->dropped_frame_count_ = 0;
     }
@@ skipping 157 matching lines @@
 }
 
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr, false);
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
     overuse_detector_.StopCheckForOveruse();
+    quality_scaler_.Stop();
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(!module_process_thread_);
   module_process_thread_ = module_process_thread;
@@ skipping 74 matching lines @@
           last_frame_info_->width, last_frame_info_->height, encoder_config_);
 
   VideoCodec codec = VideoEncoderConfigToVideoCodec(
       encoder_config_, streams, settings_.payload_name, settings_.payload_type);
 
   codec.startBitrate =
       std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
   codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
   codec.expect_encode_from_texture = last_frame_info_->is_texture;
 
+  const auto thresholds = settings_.encoder->GetQPThresholds();
+  if (thresholds)

[perkj_webrtc, 2016/10/07 07:41:16]

nit: {}

+    quality_scaler_.Init(this, thresholds->first, thresholds->second,
+                         streams[0].max_framerate);
+  else
+    quality_scaler_.Init(this, codec_type_, streams[0].max_framerate);

[perkj_webrtc, 2016/10/07 07:41:16]

{}

+
   bool success = video_sender_.RegisterSendCodec(
                      &codec, number_of_cores_,
                      static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
   if (!success) {
     LOG(LS_ERROR) << "Failed to configure encoder.";
     RTC_DCHECK(success);
   }
 
   rate_allocator_.reset(new SimulcastRateAllocator(codec));
   if (stats_proxy_) {
@@ skipping 160 matching lines @@
   // running in parallel on different threads.
   if (stats_proxy_) {
     stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
   }
 
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
 
   int64_t time_sent = clock_->TimeInMilliseconds();
   uint32_t timestamp = encoded_image._timeStamp;
-  encoder_queue_.PostTask([this, timestamp, time_sent] {
+  encoder_queue_.PostTask([this, timestamp, time_sent, encoded_image] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     overuse_detector_.FrameSent(timestamp, time_sent);
+    quality_scaler_.ReportQP(encoded_image.qp_);
   });
   return result;
 }
 
 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) {
   RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
   if (stats_proxy_)
     stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate);
 }
 
@@ skipping 56 matching lines @@
       video_is_suspended != (last_observed_bitrate_bps_ == 0);
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
-void ViEEncoder::OveruseDetected() {
+void ViEEncoder::ScaleDown() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
 
   if (disable_resolution_scaling_)
     return;
 
   if (cpu_restricted_counter_ >= kMaxCpuDowngrades) {

[perkj_webrtc, 2016/10/07 07:41:16]

This is a behavior change. Previously we did not adapt down more than
kMaxCpuDowngrades due to CPU adaptation. But with this change we will not adapt
down more than kMaxCpuDowngrades regardless if it is CPU adaptation or quality.

Also, what happens in ScaleUp if the CPU adapter requests scaling up but the
quality scaler does not want that due to that quality is too bad.

I think you need to keep two separate sets of callbacks to VieEncoder so that
you can treat them separately and take a combined decision. Ie, do not adapt up
until both CPU and Quality is ok with that.

     return;
   }
   // Request lower resolution if the current resolution is lower than last time
   // we asked for the resolution to be lowered.
   // Update stats accordingly.
   int current_pixel_count = last_frame_height_ * last_frame_width_;
   if (current_pixel_count <
       max_pixel_count_.value_or(current_pixel_count + 1)) {
     max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
     max_pixel_count_step_up_ = rtc::Optional<int>();
     stats_proxy_->OnCpuRestrictedResolutionChanged(true);
     ++cpu_restricted_counter_;
     source_proxy_->RequestResolutionLowerThan(current_pixel_count);
   }
 }
 
-void ViEEncoder::NormalUsage() {
+void ViEEncoder::ScaleUp() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   if (disable_resolution_scaling_)
     return;
 
   int current_pixel_count = last_frame_height_ * last_frame_width_;
   // Request higher resolution if we are cpu restricted and the the current
   // resolution is higher than last time we requested higher resolution.
   // Update stats accordingly.
   if (cpu_restricted_counter_ > 0 &&
       current_pixel_count > max_pixel_count_step_up_.value_or(0)) {
     max_pixel_count_ = rtc::Optional<int>();
     max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
     --cpu_restricted_counter_;
     stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
     source_proxy_->RequestHigherResolutionThan(current_pixel_count);
   }
 }
 
 }  // namespace webrtc