Add framerate to VideoSinkWants and ability to signal on overuse

webrtc/video/vie_encoder_unittest.cc

 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <limits>
 #include <utility>
 
 #include "webrtc/api/video/i420_buffer.h"
+#include "webrtc/base/fakeclock.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/media/base/videoadapter.h"
 #include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h"
 #include "webrtc/system_wrappers/include/metrics_default.h"
 #include "webrtc/system_wrappers/include/sleep.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_encoder.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/gmock.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/video/send_statistics_proxy.h"
 #include "webrtc/video/vie_encoder.h"
 
 namespace {
 #if defined(WEBRTC_ANDROID)
 // TODO(kthelgason): Lower this limit when better testing
 // on MediaCodec and fallback implementations are in place.
 const int kMinPixelsPerFrame = 320 * 180;
 #else
 const int kMinPixelsPerFrame = 120 * 90;
 #endif
-}
+const int kMinFramerateFps = 2;
+const int64_t kFrameTimeoutMs = 100;
+}  // namespace
 
 namespace webrtc {
 
 using DegredationPreference = VideoSendStream::DegradationPreference;
 using ScaleReason = AdaptationObserverInterface::AdaptReason;
 using ::testing::_;
 using ::testing::Return;
 
 namespace {
 const size_t kMaxPayloadLength = 1440;
@@ skipping 79 matching lines @@
   bool adaption_enabled() {
     rtc::CritScope cs(&crit_);
     return adaptation_enabled_;
   }
 
   void IncomingCapturedFrame(const VideoFrame& video_frame) override {
     int cropped_width = 0;
     int cropped_height = 0;
     int out_width = 0;
     int out_height = 0;
-    if (adaption_enabled() &&
-        adapter_.AdaptFrameResolution(video_frame.width(), video_frame.height(),
-                                      video_frame.timestamp_us() * 1000,
-                                      &cropped_width, &cropped_height,
-                                      &out_width, &out_height)) {
-      VideoFrame adapted_frame(
-          new rtc::RefCountedObject<TestBuffer>(nullptr, out_width, out_height),
-          99, 99, kVideoRotation_0);
-      adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
-      test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
+    if (adaption_enabled()) {
+      if (adapter_.AdaptFrameResolution(
+              video_frame.width(), video_frame.height(),
+              video_frame.timestamp_us() * 1000, &cropped_width,
+              &cropped_height, &out_width, &out_height)) {
+        VideoFrame adapted_frame(new rtc::RefCountedObject<TestBuffer>(
+                                     nullptr, out_width, out_height),
+                                 99, 99, kVideoRotation_0);
+        adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
+        test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
+      }
     } else {
       test::FrameForwarder::IncomingCapturedFrame(video_frame);
     }
   }
 
   void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
                        const rtc::VideoSinkWants& wants) override {
     rtc::CritScope cs(&crit_);
-    adapter_.OnResolutionRequest(wants.target_pixel_count,
-                                 wants.max_pixel_count);
+    adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
+                                          wants.max_pixel_count,
+                                          wants.max_framerate_fps_);
     test::FrameForwarder::AddOrUpdateSink(sink, wants);
   }
 
   cricket::VideoAdapter adapter_;
   bool adaptation_enabled_ GUARDED_BY(crit_);
 };
 }  // namespace
 
 class ViEEncoderTest : public ::testing::Test {
  public:
@@ skipping 58 matching lines @@
                      99, 99, kVideoRotation_0);
     frame.set_ntp_time_ms(ntp_time_ms);
     return frame;
   }
 
   VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
     VideoFrame frame(
         new rtc::RefCountedObject<TestBuffer>(nullptr, width, height), 99, 99,
         kVideoRotation_0);
     frame.set_ntp_time_ms(ntp_time_ms);
+    frame.set_timestamp_us(ntp_time_ms * 1000);
     return frame;
   }
 
   class TestEncoder : public test::FakeEncoder {
    public:
     TestEncoder()
         : FakeEncoder(Clock::GetRealTimeClock()),
           continue_encode_event_(false, false) {}
 
     VideoCodec codec_config() {
@@ skipping 68 matching lines @@
       EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
       {
         rtc::CritScope lock(&crit_);
         timestamp = last_timestamp_;
       }
       test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
     }
 
     void WaitForEncodedFrame(uint32_t expected_width,
                              uint32_t expected_height) {
+      EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
+      CheckLastFrameSizeMathces(expected_width, expected_height);
+    }
+
+    void CheckLastFrameSizeMathces(uint32_t expected_width,
+                                   uint32_t expected_height) {
       uint32_t width = 0;
       uint32_t height = 0;
-      EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
       {
         rtc::CritScope lock(&crit_);
         width = last_width_;
         height = last_height_;
       }
       EXPECT_EQ(expected_height, height);
       EXPECT_EQ(expected_width, width);
     }
 
     void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(20)); }
 
+    bool WaitForFrame(int64_t timeout_ms) {
+      return encoded_frame_event_.Wait(timeout_ms);
+    }
+
     void SetExpectNoFrames() {
       rtc::CritScope lock(&crit_);
       expect_frames_ = false;
     }
 
     int number_of_reconfigurations() {
       rtc::CritScope lock(&crit_);
       return number_of_reconfigurations_;
     }
 
@@ skipping 276 matching lines @@
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
 
   int frame_width = 1280;
   int frame_height = 720;
 
   // Trigger CPU overuse kMaxCpuDowngrades times. Every time, ViEEncoder should
   // request lower resolution.
-  for (int i = 1; i <= ViEEncoder::kMaxCpuDowngrades; ++i) {
+  for (int i = 1; i <= ViEEncoder::kMaxCpuResolutionDowngrades; ++i) {
     video_source_.IncomingCapturedFrame(
         CreateFrame(i, frame_width, frame_height));
     sink_.WaitForEncodedFrame(i);
 
     vie_encoder_->TriggerCpuOveruse();
 
     EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
     EXPECT_LT(video_source_.sink_wants().max_pixel_count.value_or(
                   std::numeric_limits<int>::max()),
               frame_width * frame_height);
 
     frame_width /= 2;
     frame_height /= 2;
   }
 
   // Trigger CPU overuse one more time. This should not trigger a request for
   // lower resolution.
   rtc::VideoSinkWants current_wants = video_source_.sink_wants();
   video_source_.IncomingCapturedFrame(CreateFrame(
-      ViEEncoder::kMaxCpuDowngrades + 1, frame_width, frame_height));
-  sink_.WaitForEncodedFrame(ViEEncoder::kMaxCpuDowngrades + 1);
+      ViEEncoder::kMaxCpuResolutionDowngrades + 1, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(ViEEncoder::kMaxCpuResolutionDowngrades + 1);
   vie_encoder_->TriggerCpuOveruse();
   EXPECT_EQ(video_source_.sink_wants().target_pixel_count,
             current_wants.target_pixel_count);
   EXPECT_EQ(video_source_.sink_wants().max_pixel_count,
             current_wants.max_pixel_count);
 
   // Trigger CPU normal use.
   vie_encoder_->TriggerCpuNormalUsage();
   EXPECT_EQ(frame_width * frame_height * 5 / 3,
             video_source_.sink_wants().target_pixel_count.value_or(0));
@@ skipping 563 matching lines @@
   video_source_.IncomingCapturedFrame(
       CreateFrame(1, kFrameWidth, kFrameHeight));
   sink_.WaitForEncodedFrame(kFrameWidth, kFrameHeight);
 
   // Trigger CPU overuse, downscale by 3/4.
   vie_encoder_->TriggerCpuOveruse();
   video_source_.IncomingCapturedFrame(
       CreateFrame(2, kFrameWidth, kFrameHeight));
   sink_.WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
 
-  // Trigger CPU normal use, return to original resoluton;
+  // Trigger CPU normal use, return to original resolution;
   vie_encoder_->TriggerCpuNormalUsage();
   video_source_.IncomingCapturedFrame(
       CreateFrame(3, kFrameWidth, kFrameHeight));
   sink_.WaitForEncodedFrame(kFrameWidth, kFrameHeight);
 
   vie_encoder_->Stop();
 }
+
+TEST_F(ViEEncoderTest, AdaptsFrameOnOveruseWithMaintainResolution) {
+  const int kDefaultFramerateFps = 30;
+  const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerateFps;
+  const int kFrameWidth = 1280;
+  const int kFrameHeight = 720;
+  rtc::ScopedFakeClock fake_clock;
+
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+  vie_encoder_->SetSource(
+      &video_source_,
+      VideoSendStream::DegradationPreference::kMaintainResolution);
+  video_source_.set_adaptation_enabled(true);
+
+  fake_clock.SetTimeMicros(kFrameIntervalMs * 1000);
+  int64_t timestamp_ms = kFrameIntervalMs;
+
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+  sink_.WaitForEncodedFrame(timestamp_ms);
+
+  // Try to trigger overuse. No fps estimate available => no effect.
+  vie_encoder_->TriggerCpuOveruse();
+
+  // Insert frames for one second to get a stable estimate.
+  for (int i = 0; i < kDefaultFramerateFps; ++i) {
+    timestamp_ms += kFrameIntervalMs;
+    fake_clock.AdvanceTimeMicros(kFrameIntervalMs * 1000);
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+    sink_.WaitForEncodedFrame(timestamp_ms);
+  }
+
+  // Trigger CPU overuse, reduce framerate by 2/3.
+  vie_encoder_->TriggerCpuOveruse();
+  int num_frames_dropped = 0;
+  for (int i = 0; i < kDefaultFramerateFps; ++i) {
+    timestamp_ms += kFrameIntervalMs;
+    fake_clock.AdvanceTimeMicros(kFrameIntervalMs * 1000);
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+    if (!sink_.WaitForFrame(kFrameTimeoutMs)) {
+      ++num_frames_dropped;
+    } else {
+      sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
+    }
+  }
+
+  // TODO(sprang): Find where there's rounding errors or stuff causing the
+  // margin here to be a little larger than we'd like (input fps estimate is
+  // off) and the frame dropping is a little too aggressive.
+  const int kErrorMargin = 5;
+  EXPECT_NEAR(num_frames_dropped,
+              kDefaultFramerateFps - (kDefaultFramerateFps * 2 / 3),
+              kErrorMargin);
+
+  // Trigger CPU overuse, reduce framerate by 2/3 again.
+  vie_encoder_->TriggerCpuOveruse();
+  num_frames_dropped = 0;
+  for (int i = 0; i < kDefaultFramerateFps; ++i) {
+    timestamp_ms += kFrameIntervalMs;
+    fake_clock.AdvanceTimeMicros(kFrameIntervalMs * 1000);
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+    if (!sink_.WaitForFrame(kFrameTimeoutMs)) {
+      ++num_frames_dropped;
+    } else {
+      sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
+    }
+  }
+  EXPECT_NEAR(num_frames_dropped,
+              kDefaultFramerateFps - (kDefaultFramerateFps * 4 / 9),
+              kErrorMargin);
+
+  // Go back up one step.
+  vie_encoder_->TriggerCpuNormalUsage();
+  num_frames_dropped = 0;
+  for (int i = 0; i < kDefaultFramerateFps; ++i) {
+    timestamp_ms += kFrameIntervalMs;
+    fake_clock.AdvanceTimeMicros(kFrameIntervalMs * 1000);
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+    if (!sink_.WaitForFrame(kFrameTimeoutMs)) {
+      ++num_frames_dropped;
+    } else {
+      sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
+    }
+  }
+  EXPECT_NEAR(num_frames_dropped,
+              kDefaultFramerateFps - (kDefaultFramerateFps * 2 / 3),
+              kErrorMargin);
+
+  // Go back up to original mode.
+  vie_encoder_->TriggerCpuNormalUsage();
+  num_frames_dropped = 0;
+  for (int i = 0; i < kDefaultFramerateFps; ++i) {
+    timestamp_ms += kFrameIntervalMs;
+    fake_clock.AdvanceTimeMicros(kFrameIntervalMs * 1000);
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+    if (!sink_.WaitForFrame(kFrameTimeoutMs)) {
+      ++num_frames_dropped;
+    } else {
+      sink_.CheckLastFrameSizeMathces(kFrameWidth, kFrameHeight);
+    }
+  }
+  EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
+
+  vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, DoesntAdaptDownPastMinFramerate) {
+  const int kFramerateFps = 5;
+  const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
+  const int kMinFpsFrameInterval = rtc::kNumMillisecsPerSec / kMinFramerateFps;
+  const int kFrameWidth = 1280;
+  const int kFrameHeight = 720;
+
+  rtc::ScopedFakeClock fake_clock;
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+  vie_encoder_->SetSource(
+      &video_source_,
+      VideoSendStream::DegradationPreference::kMaintainResolution);
+  video_source_.set_adaptation_enabled(true);
+
+  fake_clock.SetTimeMicros(kFrameIntervalMs * 1000);
+  int64_t timestamp_ms = kFrameIntervalMs;
+
+  // Trigger overuse as much as we can.
+  for (int i = 0; i < ViEEncoder::kMaxCpuResolutionDowngrades; ++i) {
+    // Insert frames to get a new fps estimate...
+    for (int j = 0; j < kFramerateFps; ++j) {
+      video_source_.IncomingCapturedFrame(
+          CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+      timestamp_ms += kFrameIntervalMs;
+      fake_clock.AdvanceTimeMicros(kFrameIntervalMs * 1000);
+    }
+    // ...and then try to adapt again.
+    vie_encoder_->TriggerCpuOveruse();
+  }
+
+  // Drain any frame in the pipeline.
+  sink_.WaitForFrame(kDefaultTimeoutMs);
+
+  // Insert frames at min fps, all should go through.
+  for (int i = 0; i < 10; ++i) {
+    timestamp_ms += kMinFpsFrameInterval;
+    fake_clock.AdvanceTimeMicros(kMinFpsFrameInterval * 1000);
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
+    sink_.WaitForEncodedFrame(timestamp_ms);
+  }
+
+  vie_encoder_->Stop();
+}
 }  // namespace webrtc