Let ViEEncoder express resolution requests as Sinkwants

webrtc/media/engine/webrtcvideoengine2_unittest.cc

 /*
  *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 1986 matching lines @@
   AddSendStream();
 
   cricket::FakeVideoCapturer capturer;
   ASSERT_TRUE(channel_->SetVideoSend(last_ssrc_, true, nullptr, &capturer));
   ASSERT_EQ(cricket::CS_RUNNING,
             capturer.Start(capturer.GetSupportedFormats()->front()));
   ASSERT_TRUE(channel_->SetSend(true));
 
   ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size());
   FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front();
-  webrtc::LoadObserver* overuse_callback =
-      send_stream->GetConfig().overuse_callback;
-  ASSERT_TRUE(overuse_callback != NULL);
 
   EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420));
   EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1280, send_stream->GetLastWidth());
   EXPECT_EQ(720, send_stream->GetLastHeight());
 
   // Trigger overuse.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
+  rtc::VideoSinkWants wants;
+  wants.max_pixel_count = rtc::Optional<int>(
+      send_stream->GetLastWidth() * send_stream->GetLastHeight() - 1);
+  send_stream->InjectVideoSinkWants(wants);
   EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420));
   EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1280 * 3 / 4, send_stream->GetLastWidth());
   EXPECT_EQ(720 * 3 / 4, send_stream->GetLastHeight());
 
   // Trigger overuse again.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
+  wants.max_pixel_count = rtc::Optional<int>(
+      send_stream->GetLastWidth() * send_stream->GetLastHeight() - 1);
+  send_stream->InjectVideoSinkWants(wants);
   EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420));
   EXPECT_EQ(3, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1280 * 2 / 4, send_stream->GetLastWidth());
   EXPECT_EQ(720 * 2 / 4, send_stream->GetLastHeight());
 
-  // Trigger overuse again. This should not decrease the resolution since we
-  // should only adapt the resolution down max two steps.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
-  EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420));
-  EXPECT_EQ(4, send_stream->GetNumberOfSwappedFrames());
-  EXPECT_EQ(1280 * 2 / 4, send_stream->GetLastWidth());
-  EXPECT_EQ(720 * 2 / 4, send_stream->GetLastHeight());
-
   // Change input resolution.
   EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420));
-  EXPECT_EQ(5, send_stream->GetNumberOfSwappedFrames());
+  EXPECT_EQ(4, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1284 / 2, send_stream->GetLastWidth());
   EXPECT_EQ(724 / 2, send_stream->GetLastHeight());
 
   // Trigger underuse which should go back up in resolution.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse);
+  wants.max_pixel_count = rtc::Optional<int>();
+  wants.max_pixel_count_step_up = rtc::Optional<int>(
+      send_stream->GetLastWidth() * send_stream->GetLastHeight());
+  send_stream->InjectVideoSinkWants(wants);
   EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420));
-  EXPECT_EQ(6, send_stream->GetNumberOfSwappedFrames());
+  EXPECT_EQ(5, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1284 * 3 / 4, send_stream->GetLastWidth());
   EXPECT_EQ(724 * 3 / 4, send_stream->GetLastHeight());
 
   // Trigger underuse which should go back up in resolution.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse);
+  wants.max_pixel_count = rtc::Optional<int>();
+  wants.max_pixel_count_step_up = rtc::Optional<int>(
+      send_stream->GetLastWidth() * send_stream->GetLastHeight());
+  send_stream->InjectVideoSinkWants(wants);
   EXPECT_TRUE(capturer.CaptureCustomFrame(1284, 724, cricket::FOURCC_I420));
-  EXPECT_EQ(7, send_stream->GetNumberOfSwappedFrames());
+  EXPECT_EQ(6, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1284, send_stream->GetLastWidth());
   EXPECT_EQ(724, send_stream->GetLastHeight());
 
   EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, true, nullptr, nullptr));
 }
 
 TEST_F(WebRtcVideoChannel2Test, PreviousAdaptationDoesNotApplyToScreenshare) {
   cricket::VideoCodec codec = kVp8Codec720p;
   cricket::VideoSendParameters parameters;
   parameters.codecs.push_back(codec);
 
   MediaConfig media_config = MediaConfig();
   channel_.reset(
       engine_.CreateChannel(fake_call_.get(), media_config, VideoOptions()));
   ASSERT_TRUE(channel_->SetSendParameters(parameters));
 
   AddSendStream();
 
   cricket::FakeVideoCapturer capturer;
   ASSERT_EQ(cricket::CS_RUNNING,
             capturer.Start(capturer.GetSupportedFormats()->front()));
   ASSERT_TRUE(channel_->SetSend(true));
   cricket::VideoOptions camera_options;
+  camera_options.is_screencast = rtc::Optional<bool>(false);
   channel_->SetVideoSend(last_ssrc_, true /* enable */, &camera_options,
                          &capturer);
 
   ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size());
   FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front();
-  webrtc::LoadObserver* overuse_callback =
-      send_stream->GetConfig().overuse_callback;
-  ASSERT_TRUE(overuse_callback != NULL);
 
   EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420));
   EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1280, send_stream->GetLastWidth());
   EXPECT_EQ(720, send_stream->GetLastHeight());
 
   // Trigger overuse.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
+  rtc::VideoSinkWants wants;
+  wants.max_pixel_count = rtc::Optional<int>(
+      send_stream->GetLastWidth() * send_stream->GetLastHeight() - 1);
+  send_stream->InjectVideoSinkWants(wants);
   EXPECT_TRUE(capturer.CaptureCustomFrame(1280, 720, cricket::FOURCC_I420));
   EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames());
   EXPECT_EQ(1280 * 3 / 4, send_stream->GetLastWidth());
   EXPECT_EQ(720 * 3 / 4, send_stream->GetLastHeight());
 
   // Switch to screen share. Expect no CPU adaptation.
   cricket::FakeVideoCapturer screen_share(true);
   ASSERT_EQ(cricket::CS_RUNNING,
             screen_share.Start(screen_share.GetSupportedFormats()->front()));
   cricket::VideoOptions screenshare_options;
@@ skipping 35 matching lines @@
 
   cricket::FakeVideoCapturer capturer;
   VideoOptions options;
   options.is_screencast = rtc::Optional<bool>(is_screenshare);
   EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, true, &options, &capturer));
   EXPECT_EQ(cricket::CS_RUNNING,
             capturer.Start(capturer.GetSupportedFormats()->front()));
 
   EXPECT_TRUE(channel_->SetSend(true));
 
-  // Trigger overuse.
-  ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size());
   FakeVideoSendStream* send_stream = fake_call_->GetVideoSendStreams().front();
-  webrtc::LoadObserver* overuse_callback =
-      send_stream->GetConfig().overuse_callback;
 
-  if (!enable_overuse) {
-    ASSERT_TRUE(overuse_callback == NULL);
+  if (!enable_overuse || is_screenshare) {
+    EXPECT_TRUE(send_stream->resolution_scaling_disabled());
 
     EXPECT_TRUE(capturer.CaptureFrame());
     EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames());
 
     EXPECT_EQ(codec.width, send_stream->GetLastWidth());
     EXPECT_EQ(codec.height, send_stream->GetLastHeight());
 
     EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, true, nullptr, nullptr));
     return;
   }
 
-  ASSERT_TRUE(overuse_callback != NULL);
+  EXPECT_FALSE(send_stream->resolution_scaling_disabled());
+  // Trigger overuse.
+  ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size());
+
+  rtc::VideoSinkWants wants;
+  wants.max_pixel_count = rtc::Optional<int>(codec.width * codec.height - 1);
+  send_stream->InjectVideoSinkWants(wants);
+
   EXPECT_TRUE(capturer.CaptureFrame());
   EXPECT_EQ(1, send_stream->GetNumberOfSwappedFrames());
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
 
   EXPECT_TRUE(capturer.CaptureFrame());
   EXPECT_EQ(2, send_stream->GetNumberOfSwappedFrames());
 
-  if (is_screenshare) {
-    // Do not adapt screen share.
-    EXPECT_EQ(codec.width, send_stream->GetLastWidth());
-    EXPECT_EQ(codec.height, send_stream->GetLastHeight());
-  } else {
-    EXPECT_LT(send_stream->GetLastWidth(), codec.width);
-    EXPECT_LT(send_stream->GetLastHeight(), codec.height);
-  }
+  EXPECT_LT(send_stream->GetLastWidth(), codec.width);
+  EXPECT_LT(send_stream->GetLastHeight(), codec.height);
 
   // Trigger underuse which should go back to normal resolution.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse);
+  wants.max_pixel_count = rtc::Optional<int>();
+  wants.max_pixel_count_step_up = rtc::Optional<int>(
+      send_stream->GetLastWidth() * send_stream->GetLastHeight());
+  send_stream->InjectVideoSinkWants(wants);
+
   EXPECT_TRUE(capturer.CaptureFrame());
   EXPECT_EQ(3, send_stream->GetNumberOfSwappedFrames());
 
   EXPECT_EQ(codec.width, send_stream->GetLastWidth());
   EXPECT_EQ(codec.height, send_stream->GetLastHeight());
 
   EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, true, nullptr, nullptr));
 }
 
 TEST_F(WebRtcVideoChannel2Test, EstimatesNtpStartTimeCorrectly) {
@@ skipping 718 matching lines @@
   webrtc::VideoSendStream::Stats stats;
   stats.preferred_media_bitrate_bps = 5;
   stream->SetStats(stats);
 
   cricket::VideoMediaInfo info;
   ASSERT_TRUE(channel_->GetStats(&info));
   ASSERT_EQ(1u, info.senders.size());
   EXPECT_EQ(5, info.senders[0].preferred_bitrate);
 }
 
-TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationStats) {
-  AddSendStream(cricket::CreateSimStreamParams("cname", MAKE_VECTOR(kSsrcs3)));
+TEST_F(WebRtcVideoChannel2Test, GetStatsReportsCpuAdaptationStats) {
+  FakeVideoSendStream* stream = AddSendStream();
+  webrtc::VideoSendStream::Stats stats;
+  stats.number_of_cpu_adapt_changes = 2;
+  stats.cpu_limited_resolution = true;
+  stream->SetStats(stats);
 
-  // Capture format VGA.
-  cricket::FakeVideoCapturer video_capturer_vga;
-  const std::vector<cricket::VideoFormat>* formats =
-      video_capturer_vga.GetSupportedFormats();
-  cricket::VideoFormat capture_format_vga = (*formats)[1];
-  EXPECT_EQ(cricket::CS_RUNNING, video_capturer_vga.Start(capture_format_vga));
-  EXPECT_TRUE(
-      channel_->SetVideoSend(kSsrcs3[0], true, nullptr, &video_capturer_vga));
-  EXPECT_TRUE(video_capturer_vga.CaptureFrame());
-
-  cricket::VideoCodec send_codec(100, "VP8", 640, 480, 30);
-  cricket::VideoSendParameters parameters;
-  parameters.codecs.push_back(send_codec);
-  EXPECT_TRUE(channel_->SetSendParameters(parameters));
-  EXPECT_TRUE(channel_->SetSend(true));
-
-  // Verify that the CpuOveruseObserver is registered and trigger downgrade.
-
-  // Trigger overuse.
-  ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size());
-  webrtc::LoadObserver* overuse_callback =
-      fake_call_->GetVideoSendStreams().front()->GetConfig().overuse_callback;
-  ASSERT_TRUE(overuse_callback != NULL);
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
-
-  // Capture format VGA -> adapt (OnCpuResolutionRequest downgrade) -> VGA/2.
-  EXPECT_TRUE(video_capturer_vga.CaptureFrame());
-  cricket::VideoMediaInfo info;
-  EXPECT_TRUE(channel_->GetStats(&info));
-  ASSERT_EQ(1U, info.senders.size());
-  EXPECT_EQ(1, info.senders[0].adapt_changes);
-  EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU, info.senders[0].adapt_reason);
-
-  // Trigger upgrade and verify that we adapt back up to VGA.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse);
-  EXPECT_TRUE(video_capturer_vga.CaptureFrame());
-  info.Clear();
-  EXPECT_TRUE(channel_->GetStats(&info));
-  ASSERT_EQ(1U, info.senders.size());
-  EXPECT_EQ(2, info.senders[0].adapt_changes);
-  EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_NONE,
-            info.senders[0].adapt_reason);
-
-  // No capturer (no adapter). Adapt changes from old adapter should be kept.
-  EXPECT_TRUE(channel_->SetVideoSend(kSsrcs3[0], true, nullptr, nullptr));
-  info.Clear();
-  EXPECT_TRUE(channel_->GetStats(&info));
-  ASSERT_EQ(1U, info.senders.size());
-  EXPECT_EQ(2, info.senders[0].adapt_changes);
-  EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_NONE,
-            info.senders[0].adapt_reason);
-
-  // Set new capturer, capture format HD.
-  cricket::FakeVideoCapturer video_capturer_hd;
-  cricket::VideoFormat capture_format_hd = (*formats)[0];
-  EXPECT_EQ(cricket::CS_RUNNING, video_capturer_hd.Start(capture_format_hd));
-  EXPECT_TRUE(
-      channel_->SetVideoSend(kSsrcs3[0], true, nullptr, &video_capturer_hd));
-  EXPECT_TRUE(video_capturer_hd.CaptureFrame());
-
-  // Trigger overuse, HD -> adapt (OnCpuResolutionRequest downgrade) -> HD/2.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
-  EXPECT_TRUE(video_capturer_hd.CaptureFrame());
-  info.Clear();
-  EXPECT_TRUE(channel_->GetStats(&info));
-  ASSERT_EQ(1U, info.senders.size());
-  EXPECT_EQ(3, info.senders[0].adapt_changes);
-  EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU, info.senders[0].adapt_reason);
-
-  EXPECT_TRUE(channel_->SetVideoSend(kSsrcs3[0], true, nullptr, nullptr));
-}
-
-TEST_F(WebRtcVideoChannel2Test, GetStatsTracksAdaptationAndBandwidthStats) {
-  AddSendStream(cricket::CreateSimStreamParams("cname", MAKE_VECTOR(kSsrcs3)));
-
-  // Capture format VGA.
-  cricket::FakeVideoCapturer video_capturer_vga;
-  const std::vector<cricket::VideoFormat>* formats =
-      video_capturer_vga.GetSupportedFormats();
-  cricket::VideoFormat capture_format_vga = (*formats)[1];
-  EXPECT_EQ(cricket::CS_RUNNING, video_capturer_vga.Start(capture_format_vga));
-  EXPECT_TRUE(
-      channel_->SetVideoSend(kSsrcs3[0], true, nullptr, &video_capturer_vga));
-  EXPECT_TRUE(video_capturer_vga.CaptureFrame());
-
-  cricket::VideoCodec send_codec(100, "VP8", 640, 480, 30);
-  cricket::VideoSendParameters parameters;
-  parameters.codecs.push_back(send_codec);
-  EXPECT_TRUE(channel_->SetSendParameters(parameters));
-  EXPECT_TRUE(channel_->SetSend(true));
-
-  // Verify that the CpuOveruseObserver is registered and trigger downgrade.
-
-  // Trigger overuse -> adapt CPU.
-  ASSERT_EQ(1u, fake_call_->GetVideoSendStreams().size());
-  webrtc::LoadObserver* overuse_callback =
-      fake_call_->GetVideoSendStreams().front()->GetConfig().overuse_callback;
-  ASSERT_TRUE(overuse_callback != NULL);
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kOveruse);
-  EXPECT_TRUE(video_capturer_vga.CaptureFrame());
   cricket::VideoMediaInfo info;
   EXPECT_TRUE(channel_->GetStats(&info));
   ASSERT_EQ(1U, info.senders.size());
   EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU, info.senders[0].adapt_reason);
+  EXPECT_EQ(2, info.senders[0].adapt_changes);
+}
 
-  // Set bandwidth limitation stats for the stream -> adapt CPU + BW.
+TEST_F(WebRtcVideoChannel2Test, GetStatsReportsAdaptationAndBandwidthStats) {
+  FakeVideoSendStream* stream = AddSendStream();
   webrtc::VideoSendStream::Stats stats;
+  stats.number_of_cpu_adapt_changes = 2;
+  stats.cpu_limited_resolution = true;
   stats.bw_limited_resolution = true;
-  fake_call_->GetVideoSendStreams().front()->SetStats(stats);
-  info.Clear();
+  stream->SetStats(stats);
+
+  cricket::VideoMediaInfo info;
   EXPECT_TRUE(channel_->GetStats(&info));
   ASSERT_EQ(1U, info.senders.size());
   EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_CPU |
                 WebRtcVideoChannel2::ADAPTREASON_BANDWIDTH,
             info.senders[0].adapt_reason);
-
-  // Trigger upgrade -> adapt BW.
-  overuse_callback->OnLoadUpdate(webrtc::LoadObserver::kUnderuse);
-  EXPECT_TRUE(video_capturer_vga.CaptureFrame());
-  info.Clear();
-  EXPECT_TRUE(channel_->GetStats(&info));
-  ASSERT_EQ(1U, info.senders.size());
-  EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_BANDWIDTH,
-            info.senders[0].adapt_reason);
-
-  // Reset bandwidth limitation state -> adapt NONE.
-  stats.bw_limited_resolution = false;
-  fake_call_->GetVideoSendStreams().front()->SetStats(stats);
-  info.Clear();
-  EXPECT_TRUE(channel_->GetStats(&info));
-  ASSERT_EQ(1U, info.senders.size());
-  EXPECT_EQ(WebRtcVideoChannel2::ADAPTREASON_NONE,
-            info.senders[0].adapt_reason);
-
-  EXPECT_TRUE(channel_->SetVideoSend(kSsrcs3[0], true, nullptr, nullptr));
+  EXPECT_EQ(2, info.senders[0].adapt_changes);
 }
 
 TEST_F(WebRtcVideoChannel2Test,
        GetStatsTranslatesBandwidthLimitedResolutionCorrectly) {
   FakeVideoSendStream* stream = AddSendStream();
   webrtc::VideoSendStream::Stats stats;
   stats.bw_limited_resolution = true;
   stream->SetStats(stats);
 
   cricket::VideoMediaInfo info;
@@ skipping 779 matching lines @@
 }
 
 // Test that we normalize send codec format size in simulcast.
 TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWithOddSizeInSimulcast) {
   cricket::VideoCodec codec(kVp8Codec270p);
   codec.width += 1;
   codec.height += 1;
   VerifySimulcastSettings(codec, 2, 2);
 }
 }  // namespace cricket