Added large room scenario to full-stack tests. Added thumbnail streams functionality to call test/v…

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_quality_test.h"
@@ skipping 39 matching lines @@
 
 constexpr int kSendStatsPollingIntervalMs = 1000;
 constexpr int kPayloadTypeH264 = 122;
 constexpr int kPayloadTypeVP8 = 123;
 constexpr int kPayloadTypeVP9 = 124;
 constexpr size_t kMaxComparisons = 10;
 constexpr char kSyncGroup[] = "av_sync";
 constexpr int kOpusMinBitrateBps = 6000;
 constexpr int kOpusBitrateFbBps = 32000;
 constexpr int kFramesSentInQuickTest = 1;
+constexpr uint32_t kThumbnailSendSsrcStart = 0xE0000;
+constexpr uint32_t kThumbnailRtxSsrcStart = 0xF0000;
 
 struct VoiceEngineState {
   VoiceEngineState()
       : voice_engine(nullptr),
         base(nullptr),
         send_channel_id(-1),
         receive_channel_id(-1) {}
 
   webrtc::VoiceEngine* voice_engine;
   webrtc::VoEBase* base;
@@ skipping 953 matching lines @@
 
 VideoQualityTest::Params::Params()
     : call({false, Call::Config::BitrateConfig()}),
       video({false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false,
              false, "", ""}),
       audio({false, false}),
       screenshare({false, 10, 0}),
       analyzer({"", 0.0, 0.0, 0, "", ""}),
       pipe(),
       logs(false),
-      ss({std::vector<VideoStream>(), 0, 0, -1, std::vector<SpatialLayer>()}) {}
+      ss({std::vector<VideoStream>(), 0, 0, -1, std::vector<SpatialLayer>()}),
+      num_thumbnails(0) {}
 
 VideoQualityTest::Params::~Params() = default;
 
 void VideoQualityTest::TestBody() {}
 
 std::string VideoQualityTest::GenerateGraphTitle() const {
   std::stringstream ss;
   ss << params_.video.codec;
   ss << " (" << params_.video.target_bitrate_bps / 1000 << "kbps";
   ss << ", " << params_.video.fps << " FPS";
@@ skipping 47 matching lines @@
   RTC_CHECK_GE(params_.ss.num_spatial_layers, 1);
   RTC_CHECK_LE(params_.ss.selected_sl, params_.ss.num_spatial_layers);
   RTC_CHECK(params_.ss.spatial_layers.empty() ||
             params_.ss.spatial_layers.size() ==
                 static_cast<size_t>(params_.ss.num_spatial_layers));
   if (params_.video.codec == "VP8") {
     RTC_CHECK_EQ(params_.ss.num_spatial_layers, 1);
   } else if (params_.video.codec == "VP9") {
     RTC_CHECK_EQ(params_.ss.streams.size(), 1);
   }
+  RTC_CHECK_GE(params_.num_thumbnails, 0);
+  if (params_.num_thumbnails > 0) {
+    RTC_CHECK_EQ(params_.ss.num_spatial_layers, 1);
+    RTC_CHECK_EQ(params_.ss.streams.size(), 3);
+    RTC_CHECK_EQ(params_.video.num_temporal_layers, 3);
+    RTC_CHECK_EQ(params_.video.codec, "VP8");
+  }
 }
 
 // Static.
 std::vector<int> VideoQualityTest::ParseCSV(const std::string& str) {
   // Parse comma separated nonnegative integers, where some elements may be
   // empty. The empty values are replaced with -1.
   // E.g. "10,-20,,30,40" --> {10, 20, -1, 30,40}
   // E.g. ",,10,,20," --> {-1, -1, 10, -1, 20, -1}
   std::vector<int> result;
   if (str.empty())
@@ skipping 20 matching lines @@
 // Static.
 VideoStream VideoQualityTest::DefaultVideoStream(const Params& params) {
   VideoStream stream;
   stream.width = params.video.width;
   stream.height = params.video.height;
   stream.max_framerate = params.video.fps;
   stream.min_bitrate_bps = params.video.min_bitrate_bps;
   stream.target_bitrate_bps = params.video.target_bitrate_bps;
   stream.max_bitrate_bps = params.video.max_bitrate_bps;
   stream.max_qp = 52;
-  if (params.video.num_temporal_layers == 2)
+  if (params.video.num_temporal_layers == 2) {
     stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps);
+  } else if (params.video.num_temporal_layers == 3) {
+    stream.temporal_layer_thresholds_bps.push_back(stream.max_bitrate_bps / 4);
+    stream.temporal_layer_thresholds_bps.push_back(stream.target_bitrate_bps);
+  }
   return stream;
 }
 
+// Static.
+VideoStream VideoQualityTest::DefaultThumbnailStream() {
+  VideoStream stream;
+  stream.width = 320;
+  stream.height = 180;
+  stream.max_framerate = 7;
+  stream.min_bitrate_bps = 7500;
+  stream.target_bitrate_bps = 37500;
+  stream.max_bitrate_bps = 50000;
+  stream.max_qp = 52;
+  return stream;
+}
+
 // Static.
 void VideoQualityTest::FillScalabilitySettings(
     Params* params,
     const std::vector<std::string>& stream_descriptors,
     size_t selected_stream,
     int num_spatial_layers,
     int selected_sl,
     const std::vector<std::string>& sl_descriptors) {
   // Read VideoStream and SpatialLayer elements from a list of comma separated
   // lists. To use a default value for an element, use -1 or leave empty.
@@ skipping 154 matching lines @@
         video_send_config_.rtp.ulpfec.red_payload_type;
     video_receive_configs_[params_.ss.selected_stream]
         .rtp.ulpfec.ulpfec_payload_type =
         video_send_config_.rtp.ulpfec.ulpfec_payload_type;
     video_receive_configs_[params_.ss.selected_stream]
         .rtp.ulpfec.red_rtx_payload_type =
         video_send_config_.rtp.ulpfec.red_rtx_payload_type;
   }
 }
 
+void VideoQualityTest::SetupThumbnails(Transport* send_transport,
+                                       Transport* recv_transport) {
+  size_t num_video_streams = params_.ss.streams.size();
+  for (int i = 0; i < params_.num_thumbnails; ++i) {
+    thumbnail_encoders_.emplace_back(VP8Encoder::Create());
+
+    VideoSendStream::Config thumbnail_send_config(send_transport);
+    thumbnail_send_config.rtp.ssrcs.push_back(kThumbnailSendSsrcStart + i);
+    thumbnail_send_config.encoder_settings.encoder =
+        thumbnail_encoders_.back().get();
+    thumbnail_send_config.encoder_settings.payload_name = params_.video.codec;
+    thumbnail_send_config.encoder_settings.payload_type = kPayloadTypeVP8;
+    thumbnail_send_config.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
+    thumbnail_send_config.rtp.rtx.payload_type = kSendRtxPayloadType;
+    thumbnail_send_config.rtp.rtx.ssrcs.push_back(kThumbnailRtxSsrcStart + i);
+    thumbnail_send_config.rtp.extensions.clear();
+    if (params_.call.send_side_bwe) {
+      thumbnail_send_config.rtp.extensions.push_back(
+          RtpExtension(RtpExtension::kTransportSequenceNumberUri,
+                       test::kTransportSequenceNumberExtensionId));
+    } else {
+      thumbnail_send_config.rtp.extensions.push_back(RtpExtension(
+          RtpExtension::kAbsSendTimeUri, test::kAbsSendTimeExtensionId));
+    }
+
+    VideoEncoderConfig thumbnail_encoder_config;
+    thumbnail_encoder_config.min_transmit_bitrate_bps = 7500;
+    thumbnail_send_config.suspend_below_min_bitrate =
+        params_.video.suspend_below_min_bitrate;
+    thumbnail_encoder_config.number_of_streams = 1;
+    thumbnail_encoder_config.max_bitrate_bps = 50000;
+    thumbnail_encoder_config.video_stream_factory =
+        new rtc::RefCountedObject<VideoStreamFactory>(
+            std::vector<webrtc::VideoStream>{DefaultThumbnailStream()});
+    thumbnail_encoder_config.spatial_layers = params_.ss.spatial_layers;
+
+    thumbnail_encoder_configs_.push_back(thumbnail_encoder_config.Copy());
+    thumbnail_send_configs_.push_back(thumbnail_send_config.Copy());
+
+    CreateMatchingVideoReceiveConfigs(thumbnail_send_configs_[i],
+                                      recv_transport);
+    RTC_CHECK(video_receive_configs_.size() == num_video_streams + i + 1);
+    video_receive_configs_[num_video_streams + i].rtp.nack.rtp_history_ms =
+        kNackRtpHistoryMs;
+    video_receive_configs_[num_video_streams + i].rtp.rtx_ssrc =
+        kThumbnailRtxSsrcStart + i;
+    video_receive_configs_[num_video_streams + i]
+        .rtp.rtx_payload_types[kPayloadTypeVP8] = kSendRtxPayloadType;
+    video_receive_configs_[num_video_streams + i].rtp.transport_cc =
+        params_.call.send_side_bwe;
+    video_receive_configs_[num_video_streams + i].rtp.remb =
+        !params_.call.send_side_bwe;
+  }
+
+  for (size_t i = 0; i < thumbnail_send_configs_.size(); ++i) {
+    thumbnail_send_streams_.push_back(sender_call_->CreateVideoSendStream(
+        thumbnail_send_configs_[i].Copy(),
+        thumbnail_encoder_configs_[i].Copy()));
+  }
+}
+
 void VideoQualityTest::SetupScreenshareOrSVC() {
   if (params_.screenshare.enabled) {
     // Fill out codec settings.
     video_encoder_config_.content_type =
         VideoEncoderConfig::ContentType::kScreen;
     degradation_preference_ =
         VideoSendStream::DegradationPreference::kMaintainResolution;
     if (params_.video.codec == "VP8") {
       VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
       vp8_settings.denoisingOn = false;
@@ skipping 49 matching lines @@
     VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
     vp9_settings.numberOfTemporalLayers =
         static_cast<unsigned char>(params_.video.num_temporal_layers);
     vp9_settings.numberOfSpatialLayers =
         static_cast<unsigned char>(params_.ss.num_spatial_layers);
     video_encoder_config_.encoder_specific_settings = new rtc::RefCountedObject<
         VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
   }
 }
 
+void VideoQualityTest::SetupThumbnailCapturers(size_t num_thumbnail_streams) {
+  VideoStream thumbnail = DefaultThumbnailStream();
+  for (size_t i = 0; i < num_thumbnail_streams; ++i) {
+    thumbnail_capturers_.emplace_back(test::FrameGeneratorCapturer::Create(
+        static_cast<int>(thumbnail.width), static_cast<int>(thumbnail.height),
+        thumbnail.max_framerate, clock_));
+  }
+}
+
 void VideoQualityTest::CreateCapturer() {
   if (params_.screenshare.enabled) {
     test::FrameGeneratorCapturer* frame_generator_capturer =
         new test::FrameGeneratorCapturer(clock_, std::move(frame_generator_),
                                          params_.video.fps);
     EXPECT_TRUE(frame_generator_capturer->Init());
     video_capturer_.reset(frame_generator_capturer);
   } else {
     if (params_.video.clip_name.empty()) {
       video_capturer_.reset(test::VcmCapturer::Create(
@@ skipping 41 matching lines @@
 
   test::LayerFilteringTransport send_transport(
       params_.pipe, sender_call_.get(), kPayloadTypeVP8, kPayloadTypeVP9,
       params_.video.selected_tl, params_.ss.selected_sl);
   test::DirectTransport recv_transport(params_.pipe, receiver_call_.get());
 
   std::string graph_title = params_.analyzer.graph_title;
   if (graph_title.empty())
     graph_title = VideoQualityTest::GenerateGraphTitle();
 
-  // In the case of different resolutions, the functions calculating PSNR and
-  // SSIM return -1.0, instead of a positive value as usual. VideoAnalyzer
-  // aborts if the average psnr/ssim are below the given threshold, which is
-  // 0.0 by default. Setting the thresholds to -1.1 prevents the unnecessary
-  // abort.
   VideoStream& selected_stream = params_.ss.streams[params_.ss.selected_stream];
 
   bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest");
   VideoAnalyzer analyzer(
       &send_transport, params_.analyzer.test_label,
-
       params_.analyzer.avg_psnr_threshold, params_.analyzer.avg_ssim_threshold,
       is_quick_test_enabled
           ? kFramesSentInQuickTest
           : params_.analyzer.test_durations_secs * params_.video.fps,
       graph_data_output_file, graph_title,
       kVideoSendSsrcs[params_.ss.selected_stream],
       kSendRtxSsrcs[params_.ss.selected_stream],
       static_cast<uint32_t>(selected_stream.width),
       static_cast<uint32_t>(selected_stream.height), params.ss.selected_sl,
       params_.video.selected_tl, is_quick_test_enabled);
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupVideo(&analyzer, &recv_transport);
+  SetupThumbnails(&analyzer, &recv_transport);
   video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
   video_send_config_.pre_encode_callback = analyzer.pre_encode_proxy();
   RTC_DCHECK(!video_send_config_.post_encode_callback);
   video_send_config_.post_encode_callback = analyzer.encode_timing_proxy();
 
   SetupScreenshareOrSVC();
 
   CreateFlexfecStreams();
   CreateVideoStreams();
   analyzer.SetSendStream(video_send_stream_);
   if (video_receive_streams_.size() == 1)
     analyzer.SetReceiveStream(video_receive_streams_[0]);
 
   video_send_stream_->SetSource(analyzer.OutputInterface(),
                                 degradation_preference_);
 
+  SetupThumbnailCapturers(params_.num_thumbnails);
+  for (size_t i = 0; i < thumbnail_send_streams_.size(); ++i) {
+    thumbnail_send_streams_[i]->SetSource(thumbnail_capturers_[i].get(),
+                                          degradation_preference_);
+  }
+
   CreateCapturer();
+
   rtc::VideoSinkWants wants;
   video_capturer_->AddOrUpdateSink(analyzer.InputInterface(), wants);
 
   StartEncodedFrameLogs(video_send_stream_);
   StartEncodedFrameLogs(video_receive_streams_[0]);
   video_send_stream_->Start();
+  for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
+    thumbnail_send_stream->Start();
   for (VideoReceiveStream* receive_stream : video_receive_streams_)
     receive_stream->Start();
   for (FlexfecReceiveStream* receive_stream : flexfec_receive_streams_)
     receive_stream->Start();
+
   analyzer.StartMeasuringCpuProcessTime();
+
   video_capturer_->Start();
+  for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
+       thumbnail_capturers_) {
+    video_caputurer->Start();
+  }
 
   analyzer.Wait();
 
   send_transport.StopSending();
   recv_transport.StopSending();
 
+  for (std::unique_ptr<test::VideoCapturer>& video_caputurer :
+       thumbnail_capturers_)
+    video_caputurer->Stop();
   video_capturer_->Stop();
   for (FlexfecReceiveStream* receive_stream : flexfec_receive_streams_)
     receive_stream->Stop();
   for (VideoReceiveStream* receive_stream : video_receive_streams_)
     receive_stream->Stop();
+  for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
+    thumbnail_send_stream->Stop();
   video_send_stream_->Stop();
 
   DestroyStreams();
+  // Thumbnails are not managed by CallTest. We need to destroy them manually.
+  for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
+    sender_call_->DestroyVideoSendStream(thumbnail_send_stream);
+  thumbnail_send_streams_.clear();
 
   if (graph_data_output_file)
     fclose(graph_data_output_file);
 }
 
 void VideoQualityTest::SetupAudio(int send_channel_id,
                                   int receive_channel_id,
                                   Call* call,
                                   Transport* transport,
                                   AudioReceiveStream** audio_receive_stream) {
@@ skipping 184 matching lines @@
     std::ostringstream str;
     str << receive_logs_++;
     std::string path =
         params_.video.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
     stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
                                         10000000);
   }
 }
 
 }  // namespace webrtc