Let ViEEncoder express resolution requests as Sinkwants

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 208 matching lines @@
         time_when_posted_ms_(time_when_posted_in_ms),
         log_stats_(log_stats) {
     frame_.ShallowCopy(frame);
     ++vie_encoder_->posted_frames_waiting_for_encode_;
   }
 
  private:
   bool Run() override {
     RTC_DCHECK_RUN_ON(&vie_encoder_->encoder_queue_);
     RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0);
+    vie_encoder_->stats_proxy_->OnIncomingFrame(
+        frame_.width(), frame_.height(),
+        vie_encoder_->cpu_restricted_counter_ != 0);
     ++vie_encoder_->captured_frame_count_;
     if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) {
       vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_ms_);
     } else {
       // There is a newer frame in flight. Do not encode this frame.
       LOG(LS_VERBOSE)
           << "Incoming frame dropped due to that the encoder is blocked.";
       ++vie_encoder_->dropped_frame_count_;
     }
     if (log_stats_) {
       LOG(LS_INFO) << "Number of frames: captured "
                    << vie_encoder_->captured_frame_count_
                    << ", dropped (due to encoder blocked) "
                    << vie_encoder_->dropped_frame_count_ << ", interval_ms "
                    << kFrameLogIntervalMs;
       vie_encoder_->captured_frame_count_ = 0;
       vie_encoder_->dropped_frame_count_ = 0;
     }
     return true;
   }
   VideoFrame frame_;
   ViEEncoder* const vie_encoder_;
   const int64_t time_when_posted_ms_;
   const bool log_stats_;
 };
 
 // VideoSourceProxy is responsible ensuring thread safety between calls to
-// ViEEncoder::SetSource that will happen on libjingles worker thread when a
+// ViEEncoder::SetSource that will happen on libjingle's worker thread when a
 // video capturer is connected to the encoder and the encoder task queue
 // (encoder_queue_) where the encoder reports its VideoSinkWants.
 class ViEEncoder::VideoSourceProxy {
  public:
   explicit VideoSourceProxy(ViEEncoder* vie_encoder)
-      : vie_encoder_(vie_encoder), source_(nullptr) {}
+      : vie_encoder_(vie_encoder),
+        resolution_scaling_disabled_(false),
+        source_(nullptr) {}
 
-  void SetSource(rtc::VideoSourceInterface<VideoFrame>* source) {
+  void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
+                 bool disable_resolution_scaling) {
+    // Called on libjingle's worker thread.
     RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_);
     rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
+    rtc::VideoSinkWants wants;
     {
       rtc::CritScope lock(&crit_);
       old_source = source_;
       source_ = source;
+      resolution_scaling_disabled_ = disable_resolution_scaling;
+      wants = disable_resolution_scaling ? disabled_scaling_sink_wants_
+                                         : sink_wants_;
     }
 
     if (old_source != source && old_source != nullptr) {
       old_source->RemoveSink(vie_encoder_);
     }
 
     if (!source) {
       return;
     }
 
-    // TODO(perkj): Let VideoSourceProxy implement LoadObserver and truly send
-    // CPU load as sink wants.
-    rtc::VideoSinkWants wants;
     source->AddOrUpdateSink(vie_encoder_, wants);
   }
 
+  void SetWantsRotationApplied(bool rotation_applied) {
+    rtc::CritScope lock(&crit_);
+    sink_wants_.rotation_applied = rotation_applied;
+    disabled_scaling_sink_wants_.rotation_applied = rotation_applied;
+    if (source_) {
+      source_->AddOrUpdateSink(vie_encoder_, resolution_scaling_disabled_
+                                                 ? disabled_scaling_sink_wants_
+                                                 : sink_wants_);
+    }
+  }
+
+  void RequestResolutionLowerThan(int pixel_count) {
+    // Called on the encoder task queue.
+    rtc::CritScope lock(&crit_);
+    if (resolution_scaling_disabled_) {
+      // This can happen since |resolution_scaling_disabled_| is set on
+      // libjingle's worker thread but the adaptation is done on the encoder
+      // task
+      // queue.
+      return;
+    }
+    // The input video frame size will have a resolution with less than or
+    // equal to |max_pixel_count| depending on how the source can scale the
+    // input frame size.
+    sink_wants_.max_pixel_count = rtc::Optional<int>((pixel_count * 3) / 5);
+    sink_wants_.max_pixel_count_step_up = rtc::Optional<int>();
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+  }
+
+  void RequestHigherResolutionThan(int pixel_count) {
+    rtc::CritScope lock(&crit_);
+    if (resolution_scaling_disabled_) {
+      // This can happen since |resolution_scaling_disabled_| is set on
+      // libjingles worker thread but the adaptation is done on the encoder task
+      // queue.
+      return;
+    }
+    // The input video frame size will have a resolution with "one step up"
+    // pixels than |max_pixel_count_step_up| where "one step up" depends on
+    // how the source can scale the input frame size.
+    sink_wants_.max_pixel_count = rtc::Optional<int>();
+    sink_wants_.max_pixel_count_step_up = rtc::Optional<int>(pixel_count);
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+  }
+
  private:
   rtc::CriticalSection crit_;
   rtc::SequencedTaskChecker main_checker_;
-  ViEEncoder* vie_encoder_;
+  ViEEncoder* const vie_encoder_;
+  rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
+  rtc::VideoSinkWants disabled_scaling_sink_wants_ GUARDED_BY(&crit_);
+  bool resolution_scaling_disabled_ GUARDED_BY(&crit_);
   rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
 
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
-                       LoadObserver* overuse_callback,
                        EncodedFrameObserver* encoder_timing)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
       source_proxy_(new VideoSourceProxy(this)),
       sink_(nullptr),
       settings_(settings),
       codec_type_(PayloadNameToCodecType(settings.payload_name)),
       video_sender_(Clock::GetRealTimeClock(), this, this),
+      disable_resolution_scaling_(false),
       overuse_detector_(Clock::GetRealTimeClock(),
                         GetCpuOveruseOptions(settings.full_overuse_time),
                         this,
                         encoder_timing,
                         stats_proxy),
-      load_observer_(overuse_callback),
       stats_proxy_(stats_proxy),
       pre_encode_callback_(pre_encode_callback),
       module_process_thread_(nullptr),
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
       max_data_payload_length_(0),
       last_observed_bitrate_bps_(0),
       encoder_paused_and_dropped_frame_(false),
       has_received_sli_(false),
       picture_id_sli_(0),
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
+      cpu_restricted_counter_(0),
+      last_frame_width_(0),
+      last_frame_height_(0),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       last_frame_log_ms_(clock_->TimeInMilliseconds()),
       captured_frame_count_(0),
       dropped_frame_count_(0),
       encoder_queue_("EncoderQueue") {
   encoder_queue_.PostTask([this, encoder_timing] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     video_sender_.RegisterExternalEncoder(
         settings_.encoder, settings_.payload_type, settings_.internal_source);
     overuse_detector_.StartCheckForOveruse();
   });
 }
 
 ViEEncoder::~ViEEncoder() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(shutdown_event_.Wait(0))
       << "Must call ::Stop() before destruction.";
 }
 
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
-  source_proxy_->SetSource(nullptr);
+  source_proxy_->SetSource(nullptr, false);

[kthelgason, 2016/10/05 12:42:20]

Why hardcode false here?

[perkj_webrtc, 2016/10/18 07:48:21]

true to disable overuse detection maybe make more sence.

   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
     overuse_detector_.StopCheckForOveruse();
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(!module_process_thread_);
   module_process_thread_ = module_process_thread;
   module_process_thread_->RegisterModule(&video_sender_);
   module_process_thread_checker_.DetachFromThread();
 }
 
 void ViEEncoder::DeRegisterProcessThread() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   module_process_thread_->DeRegisterModule(&video_sender_);
 }
 
-void ViEEncoder::SetSource(rtc::VideoSourceInterface<VideoFrame>* source) {
+void ViEEncoder::SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
+                           bool disable_resolution_scaling) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
-  source_proxy_->SetSource(source);
+  source_proxy_->SetSource(source, disable_resolution_scaling);
+  encoder_queue_.PostTask([this, disable_resolution_scaling] {
+    RTC_DCHECK_RUN_ON(&encoder_queue_);
+    if (disable_resolution_scaling_ != disable_resolution_scaling) {
+      stats_proxy_->SetCpuRestrictedResolution(!disable_resolution_scaling &&
+                                               cpu_restricted_counter_ != 0);
+    }
+    disable_resolution_scaling_ = disable_resolution_scaling;
+  });
 }
 
-void ViEEncoder::SetSink(EncoderSink* sink) {
-  encoder_queue_.PostTask([this, sink] {
+void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
+  source_proxy_->SetWantsRotationApplied(rotation_applied);
+  encoder_queue_.PostTask([this, sink, rotation_applied] {

[kthelgason, 2016/10/06 16:46:12]

captured variable unused in lambda

[perkj_webrtc, 2016/10/18 07:48:21]

Done.

     RTC_DCHECK_RUN_ON(&encoder_queue_);
     sink_ = sink;
   });
 }
 
 void ViEEncoder::SetStartBitrate(int start_bitrate_bps) {
   encoder_queue_.PostTask([this, start_bitrate_bps] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     encoder_start_bitrate_bps_ = start_bitrate_bps;
   });
@@ skipping 63 matching lines @@
   if (stats_proxy_) {
     stats_proxy_->OnEncoderReconfigured(encoder_config_,
                                         rate_allocator_->GetPreferedBitrate());
   }
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
-  stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height());
-
   VideoFrame incoming_frame = video_frame;
 
   // Local time in webrtc time base.
   int64_t current_time = clock_->TimeInMilliseconds();
   incoming_frame.set_render_time_ms(current_time);
 
   // Capture time may come from clock with an offset and drift from clock_.
   int64_t capture_ntp_time_ms;
   if (video_frame.ntp_time_ms() != 0) {
     capture_ntp_time_ms = video_frame.ntp_time_ms();
@@ skipping 53 matching lines @@
   // End trace event on first frame after encoder resumes, if frame was dropped.
   if (encoder_paused_and_dropped_frame_) {
     TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
   }
   encoder_paused_and_dropped_frame_ = false;
 }
 
 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                   int64_t time_when_posted_in_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
+
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
   if (video_frame.width() != last_frame_info_->width ||
       video_frame.height() != last_frame_info_->height ||
       video_frame.rotation() != last_frame_info_->rotation ||
       video_frame.is_texture() != last_frame_info_->is_texture) {
     pending_encoder_reconfiguration_ = true;
     last_frame_info_ = rtc::Optional<VideoFrameInfo>(
         VideoFrameInfo(video_frame.width(), video_frame.height(),
                        video_frame.rotation(), video_frame.is_texture()));
     LOG(LS_INFO) << "Video frame parameters changed: dimensions="
                  << last_frame_info_->width << "x" << last_frame_info_->height
                  << ", rotation=" << last_frame_info_->rotation
                  << ", texture=" << last_frame_info_->is_texture;
   }
 
   if (pending_encoder_reconfiguration_) {
     ReconfigureEncoder();
   }
 
   if (EncoderPaused()) {
     TraceFrameDropStart();
     return;
   }
   TraceFrameDropEnd();
 
+  last_frame_height_ = video_frame.height();
+  last_frame_width_ = video_frame.width();
+
   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                           "Encode");
 
   overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms);
 
   if (codec_type_ == webrtc::kVideoCodecVP8) {
     webrtc::CodecSpecificInfo codec_specific_info;
     codec_specific_info.codecType = webrtc::kVideoCodecVP8;
 
       codec_specific_info.codecSpecific.VP8.hasReceivedRPSI =
@@ skipping 112 matching lines @@
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
 void ViEEncoder::OveruseDetected() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  // TODO(perkj): When ViEEncoder inherit rtc::VideoSink instead of
-  // VideoCaptureInput |load_observer_| should be removed and overuse be
-  // expressed as rtc::VideoSinkWants instead.
-  if (load_observer_)
-    load_observer_->OnLoadUpdate(LoadObserver::kOveruse);
+
+  if (disable_resolution_scaling_)

[kthelgason, 2016/10/05 12:42:20]

Rather than checking this here, I feel it would make sense to do this further up
the stack, and not instantiate the overuse_frame_detector_ unless this is false.
Then we don't waste cycles running that in the background, and it making a bunch
of fruitless callbacks back to this class.

+    return;
+
+  if (cpu_restricted_counter_ >= kMaxCpuDowngrades) {
+    return;
+  }
+  // Request lower resolution if the current resolution is lower than last time
+  // we asked for the resolution to be lowered.
+  // Update stats accordingly.
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  if (current_pixel_count <
+      max_pixel_count_.value_or(current_pixel_count + 1)) {
+    max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
+    max_pixel_count_step_up_ = rtc::Optional<int>();
+    stats_proxy_->OnCpuRestrictedResolutionChanged(true);
+    ++cpu_restricted_counter_;
+    source_proxy_->RequestResolutionLowerThan(current_pixel_count);
+  }
 }
 
 void ViEEncoder::NormalUsage() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
-  if (load_observer_)
-    load_observer_->OnLoadUpdate(LoadObserver::kUnderuse);
+  if (disable_resolution_scaling_)

[kthelgason, 2016/10/05 12:42:20]

Ditto

+    return;
+
+  int current_pixel_count = last_frame_height_ * last_frame_width_;
+  // Request higher resolution if we are cpu restricted and the the current
+  // resolution is higher than last time we requested higher resolution.
+  // Update stats accordingly.
+  if (cpu_restricted_counter_ > 0 &&
+      current_pixel_count > max_pixel_count_step_up_.value_or(0)) {
+    max_pixel_count_ = rtc::Optional<int>();
+    max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
+    --cpu_restricted_counter_;
+    stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0);
+    source_proxy_->RequestHigherResolutionThan(current_pixel_count);
+  }
 }
 
 }  // namespace webrtc