Change OverUseFrameDetector to use a task queue

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 181 matching lines @@
     options.low_encode_usage_threshold_percent = 150;
     options.high_encode_usage_threshold_percent = 200;
   }
   return options;
 }
 
 }  //  namespace
 
 class ViEEncoder::EncodeTask : public rtc::QueuedTask {
  public:
-  EncodeTask(const VideoFrame& frame, ViEEncoder* vie_encoder)
-      : vie_encoder_(vie_encoder) {
+  EncodeTask(const VideoFrame& frame,
+             ViEEncoder* vie_encoder,
+             int64_t time_when_posted_in_ms)
+      : vie_encoder_(vie_encoder), time_when_posted_(time_when_posted_in_ms) {
     frame_.ShallowCopy(frame);
     ++vie_encoder_->posted_frames_waiting_for_encode_;
   }
 
  private:
   bool Run() override {
     RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0);
     if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) {
-      vie_encoder_->EncodeVideoFrame(frame_);
+      vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_);
     } else {
       // There is a newer frame in flight. Do not encode this frame.
       LOG(LS_VERBOSE)
           << "Incoming frame dropped due to that the encoder is blocked.";
     }
     return true;
   }
   VideoFrame frame_;
-  ViEEncoder* vie_encoder_;
+  ViEEncoder* const vie_encoder_;
+  const int64_t time_when_posted_;

[åsapersson, 2016/08/24 08:59:05]

time_when_posted_ms_

[perkj_webrtc, 2016/09/01 10:03:30]

Done.

 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
                        LoadObserver* overuse_callback,
                        EncodedFrameObserver* encoder_timing)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
@@ skipping 17 matching lines @@
       picture_id_sli_(0),
       has_received_rpsi_(false),
       picture_id_rpsi_(0),
       clock_(Clock::GetRealTimeClock()),
       last_captured_timestamp_(0),
       delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
                              clock_->TimeInMilliseconds()),
       encoder_queue_("EncoderQueue") {
   vp_->EnableTemporalDecimation(false);
 
-  encoder_queue_.PostTask([this] {
+  encoder_queue_.PostTask([this, encoder_timing] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     video_sender_.RegisterExternalEncoder(
         settings_.encoder, settings_.payload_type, settings_.internal_source);
+    overuse_detector_.StartCheckForOveruse();
   });
 }
 
 ViEEncoder::~ViEEncoder() {
   RTC_DCHECK(shutdown_event_.Wait(0))
       << "Must call ::Stop() before destruction.";
 }
 
 void ViEEncoder::Stop() {
   if (!encoder_queue_.IsCurrent()) {
     encoder_queue_.PostTask([this] { Stop(); });
     shutdown_event_.Wait(rtc::Event::kForever);
     return;
   }
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false);
+  overuse_detector_.StopCheckForOveruse();
   shutdown_event_.Set();
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
   RTC_DCHECK(!module_process_thread_);
   module_process_thread_ = module_process_thread;
-  module_process_thread_->RegisterModule(&overuse_detector_);
   module_process_thread_->RegisterModule(&video_sender_);
   module_process_thread_checker_.DetachFromThread();
 }
 
 void ViEEncoder::DeRegisterProcessThread() {
-  module_process_thread_->DeRegisterModule(&overuse_detector_);
   module_process_thread_->DeRegisterModule(&video_sender_);
 }
 
 void ViEEncoder::SetSink(EncodedImageCallback* sink) {
   encoder_queue_.PostTask([this, sink] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     sink_ = sink;
   });
 }
 
@@ skipping 88 matching lines @@
   if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
     // We don't allow the same capture time for two frames, drop this one.
     LOG(LS_WARNING) << "Same/old NTP timestamp ("
                     << incoming_frame.ntp_time_ms()
                     << " <= " << last_captured_timestamp_
                     << ") for incoming frame. Dropping.";
     return;
   }
 
   last_captured_timestamp_ = incoming_frame.ntp_time_ms();
-  overuse_detector_.FrameCaptured(incoming_frame);
-  encoder_queue_.PostTask(
-      std::unique_ptr<rtc::QueuedTask>(new EncodeTask(incoming_frame, this)));
+  encoder_queue_.PostTask(std::unique_ptr<rtc::QueuedTask>(
+      new EncodeTask(incoming_frame, this, clock_->TimeInMilliseconds())));
 }
 
 bool ViEEncoder::EncoderPaused() const {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   // Pause video if paused by caller or as long as the network is down or the
   // pacer queue has grown too large in buffered mode.
   // If the pacer queue has grown too large or the network is down,
   // last_observed_bitrate_bps_ will be 0.
   return last_observed_bitrate_bps_ == 0;
 }
@@ skipping 10 matching lines @@
 
 void ViEEncoder::TraceFrameDropEnd() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   // End trace event on first frame after encoder resumes, if frame was dropped.
   if (encoder_paused_and_dropped_frame_) {
     TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
   }
   encoder_paused_and_dropped_frame_ = false;
 }
 
-void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) {
+void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
+                                  int64_t time_when_posted_in_ms) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
   if (EncoderPaused()) {
     TraceFrameDropStart();
     return;
   }
   TraceFrameDropEnd();
 
   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                           "Encode");
   const VideoFrame* frame_to_send = &video_frame;
   // TODO(wuchengli): support texture frames.
   if (!video_frame.video_frame_buffer()->native_handle()) {
     // Pass frame via preprocessor.
     frame_to_send = vp_->PreprocessFrame(video_frame);
     if (!frame_to_send) {
       // Drop this frame, or there was an error processing it.
       return;
     }
   }
 
+  overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms);
+
   if (encoder_config_.codecType == webrtc::kVideoCodecVP8) {
     webrtc::CodecSpecificInfo codec_specific_info;
     codec_specific_info.codecType = webrtc::kVideoCodecVP8;
 
       codec_specific_info.codecSpecific.VP8.hasReceivedRPSI =
           has_received_rpsi_;
       codec_specific_info.codecSpecific.VP8.hasReceivedSLI =
           has_received_sli_;
       codec_specific_info.codecSpecific.VP8.pictureIdRPSI =
           picture_id_rpsi_;
@@ skipping 24 matching lines @@
   // Encoded is called on whatever thread the real encoder implementation run
   // on. In the case of hardware encoders, there might be several encoders
   // running in parallel on different threads.
   if (stats_proxy_) {
     stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
   }
 
   EncodedImageCallback::Result result =
       sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
 
-  overuse_detector_.FrameSent(encoded_image._timeStamp);
+  int64_t time_sent = clock_->TimeInMilliseconds();
+  uint32_t timestamp = encoded_image._timeStamp;
+  encoder_queue_.PostTask([this, timestamp, time_sent] {
+    RTC_DCHECK_RUN_ON(&encoder_queue_);
+    overuse_detector_.FrameSent(timestamp, time_sent);
+  });
   return result;
 }
 
 void ViEEncoder::SendStatistics(uint32_t bit_rate,
                                 uint32_t frame_rate,
                                 const std::string& encoder_name) {
   RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
   if (stats_proxy_)
     stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name);
 }
@@ skipping 58 matching lines @@
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
                  << (video_is_suspended ? "suspended" : "not suspended");
     stats_proxy_->OnSuspendChange(video_is_suspended);
   }
 }
 
 void ViEEncoder::OveruseDetected() {
-  RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
+  RTC_DCHECK_RUN_ON(&encoder_queue_);
   // TODO(perkj): When ViEEncoder inherit rtc::VideoSink instead of
   // VideoCaptureInput |load_observer_| should be removed and overuse be
   // expressed as rtc::VideoSinkWants instead.
   if (load_observer_)
     load_observer_->OnLoadUpdate(LoadObserver::kOveruse);
 }
 
 void ViEEncoder::NormalUsage() {
-  RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
+  RTC_DCHECK_RUN_ON(&encoder_queue_);
   if (load_observer_)
     load_observer_->OnLoadUpdate(LoadObserver::kUnderuse);
 }
 
 }  // namespace webrtc