Add framerate to VideoSinkWants and ability to signal on overuse

webrtc/test/frame_generator_capturer.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 20 matching lines @@
   InsertFrameTask(
       webrtc::test::FrameGeneratorCapturer* frame_generator_capturer,
       uint32_t repeat_interval_ms)
       : frame_generator_capturer_(frame_generator_capturer),
         repeat_interval_ms_(repeat_interval_ms),
         intended_run_time_ms_(-1) {}
 
  private:
   bool Run() override {
     if (repeat_interval_ms_ > 0) {
-      int64_t delay_ms;
-      int64_t time_now_ms = rtc::TimeMillis();
-      if (intended_run_time_ms_ > 0) {
-        delay_ms = time_now_ms - intended_run_time_ms_;
+      // This is not a one-off frame. Check if the frame interval for this
+      // task queue is the same same as the current configures frame rate.
+      uint32_t current_interval_ms =
+          1000 / frame_generator_capturer_->GetCurrentConfiguredFramerate();
+      if (repeat_interval_ms_ != current_interval_ms) {
+        // Frame rate has changed since task was started, create a new instance.
+        rtc::TaskQueue::Current()->PostDelayedTask(
+            std::unique_ptr<rtc::QueuedTask>(new InsertFrameTask(
+                frame_generator_capturer_, current_interval_ms)),
+            current_interval_ms);
       } else {
-        delay_ms = 0;
-        intended_run_time_ms_ = time_now_ms;
-      }
-      intended_run_time_ms_ += repeat_interval_ms_;
-      if (delay_ms < repeat_interval_ms_) {
-        rtc::TaskQueue::Current()->PostDelayedTask(
-            std::unique_ptr<rtc::QueuedTask>(this),
-            repeat_interval_ms_ - delay_ms);
-      } else {
-        rtc::TaskQueue::Current()->PostDelayedTask(
-            std::unique_ptr<rtc::QueuedTask>(this), 0);
-        LOG(LS_ERROR)
-            << "Frame Generator Capturer can't keep up with requested fps";
+        // Schedule the next frame capture event to happen at approximately the
+        // correct absolute time point.
+        int64_t delay_ms;
+        int64_t time_now_ms = rtc::TimeMillis();
+        if (intended_run_time_ms_ > 0) {
+          delay_ms = time_now_ms - intended_run_time_ms_;
+        } else {
+          delay_ms = 0;
+          intended_run_time_ms_ = time_now_ms;
+        }
+        intended_run_time_ms_ += repeat_interval_ms_;
+        if (delay_ms < repeat_interval_ms_) {
+          rtc::TaskQueue::Current()->PostDelayedTask(
+              std::unique_ptr<rtc::QueuedTask>(this),
+              repeat_interval_ms_ - delay_ms);
+        } else {
+          rtc::TaskQueue::Current()->PostDelayedTask(
+              std::unique_ptr<rtc::QueuedTask>(this), 0);
+          LOG(LS_ERROR)
+              << "Frame Generator Capturer can't keep up with requested fps";
+        }
       }
     }
     frame_generator_capturer_->InsertFrame();
     // Task should be deleted only if it's not repeating.
     return repeat_interval_ms_ == 0;

[ilnik, 2017/03/20 10:02:52]

There is now a memory leak possibility. If a repeating task did not repost
itself because requested frame rate changed, it will not be removed. Either do
not create a new task above, or adjust a return value here, please.

[sprang_webrtc, 2017/03/20 10:18:45]

Good catch! Done.

   }
 
   webrtc::test::FrameGeneratorCapturer* const frame_generator_capturer_;
   const uint32_t repeat_interval_ms_;
   int64_t intended_run_time_ms_;
 };
 
 FrameGeneratorCapturer* FrameGeneratorCapturer::Create(int width,
                                                        int height,
                                                        int target_fps,
                                                        Clock* clock) {
-  FrameGeneratorCapturer* capturer = new FrameGeneratorCapturer(
-      clock, FrameGenerator::CreateSquareGenerator(width, height), target_fps);
-  if (!capturer->Init()) {
-    delete capturer;
-    return NULL;
-  }
+  std::unique_ptr<FrameGeneratorCapturer> capturer(new FrameGeneratorCapturer(
+      clock, FrameGenerator::CreateSquareGenerator(width, height), target_fps));
+  if (!capturer->Init())
+    return nullptr;
 
-  return capturer;
+  return capturer.release();
 }
 
 FrameGeneratorCapturer* FrameGeneratorCapturer::CreateFromYuvFile(
     const std::string& file_name,
     size_t width,
     size_t height,
     int target_fps,
     Clock* clock) {
-  FrameGeneratorCapturer* capturer = new FrameGeneratorCapturer(
-      clock, FrameGenerator::CreateFromYuvFile(
-                 std::vector<std::string>(1, file_name), width, height, 1),
-      target_fps);
-  if (!capturer->Init()) {
-    delete capturer;
-    return NULL;
-  }
+  std::unique_ptr<FrameGeneratorCapturer> capturer(new FrameGeneratorCapturer(
+      clock,
+      FrameGenerator::CreateFromYuvFile(std::vector<std::string>(1, file_name),
+                                        width, height, 1),
+      target_fps));
+  if (!capturer->Init())
+    return nullptr;
 
-  return capturer;
+  return capturer.release();
 }
 
 FrameGeneratorCapturer::FrameGeneratorCapturer(
     Clock* clock,
     std::unique_ptr<FrameGenerator> frame_generator,
     int target_fps)
     : clock_(clock),
       sending_(false),
       sink_(nullptr),
       sink_wants_observer_(nullptr),
@@ skipping 11 matching lines @@
 }
 
 void FrameGeneratorCapturer::SetFakeRotation(VideoRotation rotation) {
   rtc::CritScope cs(&lock_);
   fake_rotation_ = rotation;
 }
 
 bool FrameGeneratorCapturer::Init() {
   // This check is added because frame_generator_ might be file based and should
   // not crash because a file moved.
-  if (frame_generator_.get() == NULL)
+  if (frame_generator_.get() == nullptr)
     return false;
 
+  int framerate_fps = GetCurrentConfiguredFramerate();
   task_queue_.PostDelayedTask(
       std::unique_ptr<rtc::QueuedTask>(
-          new InsertFrameTask(this, 1000 / target_fps_)),
-      1000 / target_fps_);
+          new InsertFrameTask(this, 1000 / framerate_fps)),
+      1000 / framerate_fps);
 
   return true;
 }
 
 void FrameGeneratorCapturer::InsertFrame() {
-  {
-    rtc::CritScope cs(&lock_);
-    if (sending_) {
-      VideoFrame* frame = frame_generator_->NextFrame();
-      frame->set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
-      frame->set_rotation(fake_rotation_);
-      if (first_frame_capture_time_ == -1) {
-        first_frame_capture_time_ = frame->ntp_time_ms();
-      }
-      if (sink_)
-        sink_->OnFrame(*frame);
+  rtc::CritScope cs(&lock_);
+  if (sending_) {
+    VideoFrame* frame = frame_generator_->NextFrame();
+    frame->set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
+    frame->set_rotation(fake_rotation_);
+    if (first_frame_capture_time_ == -1) {
+      first_frame_capture_time_ = frame->ntp_time_ms();
+    }
+
+    if (sink_) {
+      rtc::Optional<VideoFrame> out_frame = AdaptFrame(*frame);
+      if (out_frame)
+        sink_->OnFrame(*out_frame);
     }
   }
 }
 
 void FrameGeneratorCapturer::Start() {
   rtc::CritScope cs(&lock_);
   sending_ = true;
 }
 
 void FrameGeneratorCapturer::Stop() {
@@ skipping 13 matching lines @@
 }
 
 void FrameGeneratorCapturer::AddOrUpdateSink(
     rtc::VideoSinkInterface<VideoFrame>* sink,
     const rtc::VideoSinkWants& wants) {
   rtc::CritScope cs(&lock_);
   RTC_CHECK(!sink_ || sink_ == sink);
   sink_ = sink;
   if (sink_wants_observer_)
     sink_wants_observer_->OnSinkWantsChanged(sink, wants);
+
+  // Handle framerate within this class, just pass on resolution for possible
+  // adaptation.
+  rtc::VideoSinkWants resolution_wants = wants;
+  resolution_wants.max_framerate_fps = std::numeric_limits<int>::max();
+  VideoCapturer::AddOrUpdateSink(sink, resolution_wants);
+
+  if (wants.max_framerate_fps < target_fps_)
+    wanted_fps_.emplace(wants.max_framerate_fps);

[ilnik, 2017/03/20 10:02:52]

May it happen that |wants.max_framerate_fps| may be less than |target_fps_|, but
later be larger than it? Issue here is that |wanted_fps_| will not be updated
and frame generator will continue to always generate less fps than it could.

[sprang_webrtc, 2017/03/20 10:18:45]

You're right. I botched the rebase here. Fixed.

 }
 
 void FrameGeneratorCapturer::RemoveSink(
     rtc::VideoSinkInterface<VideoFrame>* sink) {
   rtc::CritScope cs(&lock_);
   RTC_CHECK(sink_ == sink);
   sink_ = nullptr;
 }
 
 void FrameGeneratorCapturer::ForceFrame() {
   // One-time non-repeating task,
   // therefore repeat_interval_ms is 0 in InsertFrameTask()
   task_queue_.PostTask(
       std::unique_ptr<rtc::QueuedTask>(new InsertFrameTask(this, 0)));
 }
 
+int FrameGeneratorCapturer::GetCurrentConfiguredFramerate() {
+  rtc::CritScope cs(&lock_);
+  if (wanted_fps_ && *wanted_fps_ < target_fps_)
+    return *wanted_fps_;
+  return target_fps_;
+}
+
 }  // namespace test
 }  // namespace webrtc