Scale counter and resolution downgrade stats may be incorrect.

webrtc/video/vie_encoder_unittest.cc

 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 11 matching lines @@
 #include "webrtc/system_wrappers/include/sleep.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_encoder.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/gmock.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/video/send_statistics_proxy.h"
 #include "webrtc/video/vie_encoder.h"
 
 namespace {
-#if defined(WEBRTC_ANDROID)
 // TODO(kthelgason): Lower this limit when better testing
 // on MediaCodec and fallback implementations are in place.
 const int kMinPixelsPerFrame = 320 * 180;
-#else
-const int kMinPixelsPerFrame = 120 * 90;
-#endif
 const int kMinFramerateFps = 2;
 const int64_t kFrameTimeoutMs = 100;
 }  // namespace
 
 namespace webrtc {
 
 using DegredationPreference = VideoSendStream::DegradationPreference;
 using ScaleReason = AdaptationObserverInterface::AdaptReason;
 using ::testing::_;
 using ::testing::Return;
@@ skipping 670 matching lines @@
 }
 
 TEST_F(ViEEncoderTest, SinkWantsRotationApplied) {
   EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
   vie_encoder_->SetSink(&sink_, true /*rotation_applied*/);
   EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, SinkWantsFromOveruseDetector) {
+  const int kMaxDowngrades = ViEEncoder::kMaxCpuResolutionDowngrades;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   VerifyNoLimitation(video_source_.sink_wants());
 
   int frame_width = 1280;
   int frame_height = 720;
 
   // Trigger CPU overuse kMaxCpuDowngrades times. Every time, ViEEncoder should
   // request lower resolution.
-  for (int i = 1; i <= ViEEncoder::kMaxCpuResolutionDowngrades; ++i) {
+  for (int i = 1; i <= kMaxDowngrades; ++i) {
     video_source_.IncomingCapturedFrame(
         CreateFrame(i, frame_width, frame_height));
     sink_.WaitForEncodedFrame(i);
 
     vie_encoder_->TriggerCpuOveruse();
 
     EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
     EXPECT_LT(video_source_.sink_wants().max_pixel_count,
               frame_width * frame_height);
+    EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+    EXPECT_EQ(i, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
 
     frame_width /= 2;
     frame_height /= 2;
   }
 
   // Trigger CPU overuse one more time. This should not trigger a request for
   // lower resolution.
   rtc::VideoSinkWants current_wants = video_source_.sink_wants();
-  video_source_.IncomingCapturedFrame(CreateFrame(
-      ViEEncoder::kMaxCpuResolutionDowngrades + 1, frame_width, frame_height));
-  sink_.WaitForEncodedFrame(ViEEncoder::kMaxCpuResolutionDowngrades + 1);
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(kMaxDowngrades + 1, frame_width, frame_height));
+  sink_.WaitForEncodedFrame(kMaxDowngrades + 1);
   vie_encoder_->TriggerCpuOveruse();
   EXPECT_EQ(video_source_.sink_wants().target_pixel_count,
             current_wants.target_pixel_count);
   EXPECT_EQ(video_source_.sink_wants().max_pixel_count,
             current_wants.max_pixel_count);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(kMaxDowngrades,
+            stats_proxy_->GetStats().number_of_cpu_adapt_changes);
 
   // Trigger CPU normal use.
   vie_encoder_->TriggerCpuNormalUsage();
   EXPECT_EQ(frame_width * frame_height * 5 / 3,
             video_source_.sink_wants().target_pixel_count.value_or(0));
   EXPECT_EQ(frame_width * frame_height * 4,
             video_source_.sink_wants().max_pixel_count);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(kMaxDowngrades + 1,
+            stats_proxy_->GetStats().number_of_cpu_adapt_changes);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, SinkWantsStoredByDegradationPreference) {
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   VerifyNoLimitation(video_source_.sink_wants());
 
   const int kFrameWidth = 1280;
   const int kFrameHeight = 720;
@@ skipping 496 matching lines @@
 
   // Expect no scaling.
   EXPECT_EQ(std::numeric_limits<int>::max(),
             new_video_source.sink_wants().max_pixel_count);
 
   // Trigger scale up.
   vie_encoder_->TriggerQualityHigh();
   new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
   sink_.WaitForEncodedFrame(4);
 
-  // Expect nothing to change, still no scaling.
+  // Expect nothing to change, still no scaling

[kthelgason, 2017/04/19 08:23:04]

accident?

[åsapersson, 2017/04/20 13:44:47]

yes

   EXPECT_EQ(std::numeric_limits<int>::max(),
             new_video_source.sink_wants().max_pixel_count);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, SkipsSameAdaptDownRequest_MaintainFramerateMode) {
   const int kWidth = 1280;
   const int kHeight = 720;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
@@ skipping 103 matching lines @@
   // Trigger adapt up, expect no restriction.
   vie_encoder_->TriggerQualityHigh();
   VerifyNoLimitation(source.sink_wants());
   EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
   EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
   EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
 
   vie_encoder_->Stop();
 }
 
-TEST_F(ViEEncoderTest, DoesNotScaleBelowSetLimit) {
-  int frame_width = 1280;
-  int frame_height = 720;
+TEST_F(ViEEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
+  const int kWidth = 1280;
+  const int kHeight = 720;
+  const size_t kNumFrames = 10;
+
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
-  for (size_t i = 1; i <= 10; i++) {
-    video_source_.IncomingCapturedFrame(
-        CreateFrame(i, frame_width, frame_height));
+  // Enable adapter, expected input resolutions when downscaling:
+  // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (min resolution limit)
+  video_source_.set_adaptation_enabled(true);
+
+  EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  int downscales = 0;
+  for (size_t i = 1; i <= kNumFrames; i++) {
+    video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
     sink_.WaitForEncodedFrame(i);
+
     // Trigger scale down.
+    rtc::VideoSinkWants last_wants = video_source_.sink_wants();
     vie_encoder_->TriggerQualityLow();
     EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
+
+    if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
+      ++downscales;
+
+    EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+    EXPECT_EQ(downscales,
+              stats_proxy_->GetStats().number_of_quality_adapt_changes);
+    EXPECT_GT(downscales, 0);
   }
+  vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest,
+       AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
+  const int kWidth = 1280;
+  const int kHeight = 720;
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Enable kMaintainFramerate preference, no initial limitation.
+  AdaptingFrameForwarder source;
+  source.set_adaptation_enabled(true);
+  vie_encoder_->SetSource(
+      &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
+
+  source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(1);
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+  // Trigger adapt down, expect scaled down resolution.
+  vie_encoder_->TriggerCpuOveruse();
+  source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(2);
+  VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+  // Trigger adapt up, expect no restriction.
+  vie_encoder_->TriggerCpuNormalUsage();
+  source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(3);
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+  // Trigger adapt down, expect scaled down resolution.
+  vie_encoder_->TriggerCpuOveruse();
+  source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(4);
+  VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+  // Trigger adapt up, expect no restriction.
+  vie_encoder_->TriggerCpuNormalUsage();
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
 
   vie_encoder_->Stop();
 }
+
+TEST_F(ViEEncoderTest,
+       AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
+  const int kWidth = 1280;
+  const int kHeight = 720;
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Enable kMaintainFramerate preference, no initial limitation.
+  AdaptingFrameForwarder source;
+  source.set_adaptation_enabled(true);
+  vie_encoder_->SetSource(
+      &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
+
+  source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(1280, 720);
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger cpu adapt down, expect scaled down resolution.
+  vie_encoder_->TriggerCpuOveruse();
+  source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(960, 540);

[kthelgason, 2017/04/19 08:23:04]

I've tried to avoid making assumptions about the scale factors in these tests,
as they should be unrelated. If it's impossible to test the desired logic
without it I think it's ok, but it would be better if we could avoid it.

[åsapersson, 2017/04/20 13:44:47]

Done.

+  VerifyResolutionLimitationLessThan(source.sink_wants(), kWidth * kHeight);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger cpu adapt down, expect scaled down resolution.
+  vie_encoder_->TriggerCpuOveruse();
+  source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(640, 360);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger cpu adapt down, max cpu downgrades reached, expect no change.
+  vie_encoder_->TriggerCpuOveruse();
+  source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(640, 360);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger quality adapt down, expect scaled down resolution.
+  vie_encoder_->TriggerQualityLow();
+  source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(480, 270);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger cpu adapt up, expect upscaled resolution.
+  vie_encoder_->TriggerCpuNormalUsage();
+  source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(640, 360);
+  EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger cpu adapt up, expect upscaled resolution.
+  vie_encoder_->TriggerCpuNormalUsage();
+  source.IncomingCapturedFrame(CreateFrame(7, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(960, 540);
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger cpu adapt up, no cpu downgrades, expect no change.
+  vie_encoder_->TriggerCpuNormalUsage();
+  source.IncomingCapturedFrame(CreateFrame(8, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(960, 540);
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  // Trigger quality adapt up, expect no restriction.
+  vie_encoder_->TriggerQualityHigh();
+  source.IncomingCapturedFrame(CreateFrame(9, kWidth, kHeight));
+  sink_.WaitForEncodedFrame(1280, 720);
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+  EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
+
+  vie_encoder_->Stop();
+}
 
 TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) {
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   const int kWidth = 640;
   const int kHeight = 360;
 
   for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
     video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
     sink_.WaitForEncodedFrame(i);
@@ skipping 138 matching lines @@
                           VideoSendStream::DegradationPreference::kBalanced);
 
   video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
   // Frame should not be dropped, even if it's too large.
   sink_.WaitForEncodedFrame(1);
 
   vie_encoder_->Stop();
   fake_encoder_.SetQualityScaling(true);
 }
 
+TEST_F(ViEEncoderTest,
+       ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
+  const int kTooSmallWidth = 10;
+  const int kTooSmallHeight = 10;
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Enable kMaintainFramerate preference, no initial limitation.
+  test::FrameForwarder source;
+  vie_encoder_->SetSource(
+      &source, VideoSendStream::DegradationPreference::kMaintainFramerate);
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+
+  // Trigger adapt down, too small frame, expect no change.
+  source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
+  sink_.WaitForEncodedFrame(1);
+  vie_encoder_->TriggerCpuOveruse();
+  VerifyNoLimitation(source.sink_wants());
+  EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
+  EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
+
+  vie_encoder_->Stop();
+}
+
 TEST_F(ViEEncoderTest, FailingInitEncodeDoesntCauseCrash) {
   fake_encoder_.ForceInitEncodeFailure(true);
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   ResetEncoder("VP8", 2, 1, true);
   const int kFrameWidth = 1280;
   const int kFrameHeight = 720;
   video_source_.IncomingCapturedFrame(
       CreateFrame(1, kFrameWidth, kFrameHeight));
   sink_.ExpectDroppedFrame();
   vie_encoder_->Stop();
 }
 
 // TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
-TEST_F(ViEEncoderTest, AdaptsResolutionOnOveruse) {
+TEST_F(ViEEncoderTest, AdaptsResolutionOnOveruse_MaintainFramerateMode) {
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   const int kFrameWidth = 1280;
   const int kFrameHeight = 720;
   // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
   // requested by ViEEncoder::VideoSourceProxy::RequestResolutionLowerThan().
   video_source_.set_adaptation_enabled(true);
 
   video_source_.IncomingCapturedFrame(
       CreateFrame(1, kFrameWidth, kFrameHeight));
@@ skipping 161 matching lines @@
   for (int i = 0; i < 10; ++i) {
     timestamp_ms += kMinFpsFrameInterval;
     fake_clock.AdvanceTimeMicros(kMinFpsFrameInterval * 1000);
     video_source_.IncomingCapturedFrame(
         CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
     sink_.WaitForEncodedFrame(timestamp_ms);
   }
   vie_encoder_->Stop();
 }
 }  // namespace webrtc