Add cropping to VIEEncoder to match simulcast streams resolution

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_quality_test.h"
@@ skipping 11 matching lines @@
 #include "webrtc/base/cpu_time.h"
 #include "webrtc/base/event.h"
 #include "webrtc/base/format_macros.h"
 #include "webrtc/base/memory_usage.h"
 #include "webrtc/base/optional.h"
 #include "webrtc/base/platform_file.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/logging/rtc_event_log/rtc_event_log.h"
+#include "webrtc/media/engine/webrtcvideoengine.h"
 #include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/system_wrappers/include/cpu_info.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
@@ skipping 98 matching lines @@
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames,
                 FILE* graph_data_output_file,
                 const std::string& graph_title,
                 uint32_t ssrc_to_analyze,
                 uint32_t rtx_ssrc_to_analyze,
                 size_t selected_stream,
                 int selected_sl,
                 int selected_tl,
-                bool is_quick_test_enabled)
+                bool is_quick_test_enabled,
+                Clock* clock)
       : transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         receive_stream_(nullptr),
-        captured_frame_forwarder_(this),
+        captured_frame_forwarder_(this, clock),
         test_label_(test_label),
         graph_data_output_file_(graph_data_output_file),
         graph_title_(graph_title),
         ssrc_to_analyze_(ssrc_to_analyze),
         rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
         selected_stream_(selected_stream),
         selected_sl_(selected_sl),
         selected_tl_(selected_tl),
         pre_encode_proxy_(this),
         encode_timing_proxy_(this),
@@ skipping 45 matching lines @@
 
   ~VideoAnalyzer() {
     for (rtc::PlatformThread* thread : comparison_thread_pool_) {
       thread->Stop();
       delete thread;
     }
   }
 
   virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
 
+  void SetSource(test::VideoCapturer* video_capturer, bool respect_sink_wants) {
+    if (respect_sink_wants)
+      captured_frame_forwarder_.SetSource(video_capturer);
+    rtc::VideoSinkWants wants;
+    video_capturer->AddOrUpdateSink(InputInterface(), wants);
+  }
+
   void SetSendStream(VideoSendStream* stream) {
     rtc::CritScope lock(&crit_);
     RTC_DCHECK(!send_stream_);
     send_stream_ = stream;
   }
 
   void SetReceiveStream(VideoReceiveStream* stream) {
     rtc::CritScope lock(&crit_);
     RTC_DCHECK(!receive_stream_);
     receive_stream_ = stream;
@@ skipping 543 matching lines @@
     if (!is_quick_test_enabled_) {
       EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
       EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
     }
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = -1.0;
     double ssim = -1.0;
-    if (comparison.reference) {
+    if (comparison.reference && !comparison.dropped) {
       psnr = I420PSNR(&*comparison.reference, &*comparison.render);
       ssim = I420SSIM(&*comparison.reference, &*comparison.render);
     }
 
     rtc::CritScope crit(&comparison_lock_);
     if (graph_data_output_file_) {
       samples_.push_back(Sample(
           comparison.dropped, comparison.input_time_ms, comparison.send_time_ms,
           comparison.recv_time_ms, comparison.render_time_ms,
           comparison.encoded_frame_size, psnr, ssim));
@@ skipping 98 matching lines @@
   }
 
   // Implements VideoSinkInterface to receive captured frames from a
   // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
   // as a source to VideoSendStream.
   // It forwards all input frames to the VideoAnalyzer for later comparison and
   // forwards the captured frames to the VideoSendStream.
   class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
                                  public rtc::VideoSourceInterface<VideoFrame> {
    public:
-    explicit CapturedFrameForwarder(VideoAnalyzer* analyzer)
-        : analyzer_(analyzer), send_stream_input_(nullptr) {}
+    explicit CapturedFrameForwarder(VideoAnalyzer* analyzer, Clock* clock)
+        : analyzer_(analyzer),
+          send_stream_input_(nullptr),
+          video_capturer_(nullptr),
+          clock_(clock) {}
+
+    void SetSource(test::VideoCapturer* video_capturer) {
+      video_capturer_ = video_capturer;
+    }
 
    private:
     void OnFrame(const VideoFrame& video_frame) override {
       VideoFrame copy = video_frame;
       // Frames from the capturer does not have a rtp timestamp.
       // Create one so it can be used for comparison.
       RTC_DCHECK_EQ(0, video_frame.timestamp());
-      if (copy.ntp_time_ms() == 0)
-        copy.set_ntp_time_ms(rtc::TimeMillis());
+      if (video_frame.ntp_time_ms() == 0)
+        copy.set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
       copy.set_timestamp(copy.ntp_time_ms() * 90);
       analyzer_->AddCapturedFrameForComparison(copy);
       rtc::CritScope lock(&crit_);
       if (send_stream_input_)
         send_stream_input_->OnFrame(copy);
     }
 
     // Called when |send_stream_.SetSource()| is called.
     void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
                          const rtc::VideoSinkWants& wants) override {
       rtc::CritScope lock(&crit_);
       RTC_DCHECK(!send_stream_input_ || send_stream_input_ == sink);
       send_stream_input_ = sink;
+      if (video_capturer_) {
+        video_capturer_->AddOrUpdateSink(this, wants);
+      }
     }
 
     // Called by |send_stream_| when |send_stream_.SetSource()| is called.
     void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override {
       rtc::CritScope lock(&crit_);
       RTC_DCHECK(sink == send_stream_input_);
       send_stream_input_ = nullptr;
     }
 
     VideoAnalyzer* const analyzer_;
     rtc::CriticalSection crit_;
     rtc::VideoSinkInterface<VideoFrame>* send_stream_input_ GUARDED_BY(crit_);
+    test::VideoCapturer* video_capturer_;
+    Clock* clock_;
   };
 
   void AddCapturedFrameForComparison(const VideoFrame& video_frame) {
     rtc::CritScope lock(&crit_);
     frames_.push_back(video_frame);
   }
 
   VideoSendStream* send_stream_;
   VideoReceiveStream* receive_stream_;
   CapturedFrameForwarder captured_frame_forwarder_;
@@ skipping 281 matching lines @@
       stream.temporal_layer_thresholds_bps.clear();
       stream.temporal_layer_thresholds_bps.insert(
           stream.temporal_layer_thresholds_bps.end(), v.begin() + 7, v.end());
     } else {
       // Automatic TL thresholds for more than two layers not supported.
       RTC_CHECK_LE(params->video.num_temporal_layers, 2);
     }
     params->ss.streams.push_back(stream);
   }
   params->ss.selected_stream = selected_stream;
+  params->ss.infer_streams = false;
 
   params->ss.num_spatial_layers = num_spatial_layers ? num_spatial_layers : 1;
   params->ss.selected_sl = selected_sl;
   RTC_CHECK(params->ss.spatial_layers.empty());
   for (auto descriptor : sl_descriptors) {
     if (descriptor.empty())
       continue;
     std::vector<int> v = VideoQualityTest::ParseCSV(descriptor);
     RTC_CHECK_GT(v[2], 0);
 
@@ skipping 61 matching lines @@
 
   video_send_config_.suspend_below_min_bitrate =
       params_.video.suspend_below_min_bitrate;
 
   video_encoder_config_.number_of_streams = params_.ss.streams.size();
   video_encoder_config_.max_bitrate_bps = 0;
   for (size_t i = 0; i < params_.ss.streams.size(); ++i) {
     video_encoder_config_.max_bitrate_bps +=
         params_.ss.streams[i].max_bitrate_bps;
   }
-  video_encoder_config_.video_stream_factory =
-      new rtc::RefCountedObject<VideoStreamFactory>(params_.ss.streams);
+  if (params_.ss.infer_streams) {
+    video_encoder_config_.video_stream_factory =
+        new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
+            params_.video.codec, params_.ss.streams[0].max_qp,
+            params_.video.fps, params_.screenshare.enabled, true);
+  } else {
+    video_encoder_config_.video_stream_factory =
+        new rtc::RefCountedObject<VideoStreamFactory>(params_.ss.streams);
+  }
 
   video_encoder_config_.spatial_layers = params_.ss.spatial_layers;
 
   CreateMatchingReceiveConfigs(recv_transport);
 
   for (size_t i = 0; i < num_video_streams; ++i) {
     video_receive_configs_[i].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
     video_receive_configs_[i].rtp.rtx_ssrc = kSendRtxSsrcs[i];
     video_receive_configs_[i].rtp.rtx_payload_types[payload_type] =
         kSendRtxPayloadType;
@@ skipping 82 matching lines @@
       thumbnail_send_config.rtp.extensions.push_back(RtpExtension(
           RtpExtension::kAbsSendTimeUri, test::kAbsSendTimeExtensionId));
     }
 
     VideoEncoderConfig thumbnail_encoder_config;
     thumbnail_encoder_config.min_transmit_bitrate_bps = 7500;
     thumbnail_send_config.suspend_below_min_bitrate =
         params_.video.suspend_below_min_bitrate;
     thumbnail_encoder_config.number_of_streams = 1;
     thumbnail_encoder_config.max_bitrate_bps = 50000;
-    thumbnail_encoder_config.video_stream_factory =
-        new rtc::RefCountedObject<VideoStreamFactory>(
-            std::vector<webrtc::VideoStream>{DefaultThumbnailStream()});
+    if (params_.ss.infer_streams) {
+      thumbnail_encoder_config.video_stream_factory =
+          new rtc::RefCountedObject<VideoStreamFactory>(params_.ss.streams);
+    } else {
+      thumbnail_encoder_config.video_stream_factory =
+          new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
+              params_.video.codec, params_.ss.streams[0].max_qp,
+              params_.video.fps, params_.screenshare.enabled, true);
+    }
     thumbnail_encoder_config.spatial_layers = params_.ss.spatial_layers;
 
     VideoReceiveStream::Config thumbnail_receive_config(send_transport);
     thumbnail_receive_config.rtp.remb = false;
     thumbnail_receive_config.rtp.transport_cc = true;
     thumbnail_receive_config.rtp.local_ssrc = kReceiverLocalVideoSsrc;
     for (const RtpExtension& extension : thumbnail_send_config.rtp.extensions)
       thumbnail_receive_config.rtp.extensions.push_back(extension);
     thumbnail_receive_config.renderer = &fake_renderer_;
 
@@ skipping 119 matching lines @@
 }
 
 void VideoQualityTest::CreateCapturer() {
   if (params_.screenshare.enabled) {
     test::FrameGeneratorCapturer* frame_generator_capturer =
         new test::FrameGeneratorCapturer(clock_, std::move(frame_generator_),
                                          params_.video.fps);
     EXPECT_TRUE(frame_generator_capturer->Init());
     video_capturer_.reset(frame_generator_capturer);
   } else {
-    if (params_.video.clip_name.empty()) {
+    if (params_.video.clip_name == "Generator") {
+      video_capturer_.reset(test::FrameGeneratorCapturer::Create(
+          static_cast<int>(params_.video.width),
+          static_cast<int>(params_.video.height), params_.video.fps, clock_));
+    } else if (params_.video.clip_name.empty()) {
       video_capturer_.reset(test::VcmCapturer::Create(
           params_.video.width, params_.video.height, params_.video.fps,
           params_.video.capture_device_index));
       if (!video_capturer_) {
         // Failed to get actual camera, use chroma generator as backup.
         video_capturer_.reset(test::FrameGeneratorCapturer::Create(
             static_cast<int>(params_.video.width),
             static_cast<int>(params_.video.height), params_.video.fps, clock_));
       }
     } else {
@@ skipping 45 matching lines @@
   VideoAnalyzer analyzer(
       &send_transport, params_.analyzer.test_label,
       params_.analyzer.avg_psnr_threshold, params_.analyzer.avg_ssim_threshold,
       is_quick_test_enabled
           ? kFramesSentInQuickTest
           : params_.analyzer.test_durations_secs * params_.video.fps,
       graph_data_output_file, graph_title,
       kVideoSendSsrcs[params_.ss.selected_stream],
       kSendRtxSsrcs[params_.ss.selected_stream],
       static_cast<size_t>(params_.ss.selected_stream), params.ss.selected_sl,
-      params_.video.selected_tl, is_quick_test_enabled);
+      params_.video.selected_tl, is_quick_test_enabled, clock_);
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupVideo(&analyzer, &recv_transport);
   SetupThumbnails(&analyzer, &recv_transport);
   video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
   video_send_config_.pre_encode_callback = analyzer.pre_encode_proxy();
   RTC_DCHECK(!video_send_config_.post_encode_callback);
   video_send_config_.post_encode_callback = analyzer.encode_timing_proxy();
@@ skipping 10 matching lines @@
                                 degradation_preference_);
 
   SetupThumbnailCapturers(params_.num_thumbnails);
   for (size_t i = 0; i < thumbnail_send_streams_.size(); ++i) {
     thumbnail_send_streams_[i]->SetSource(thumbnail_capturers_[i].get(),
                                           degradation_preference_);
   }
 
   CreateCapturer();
 
-  rtc::VideoSinkWants wants;
-  video_capturer_->AddOrUpdateSink(analyzer.InputInterface(), wants);
+  analyzer.SetSource(video_capturer_.get(), params_.ss.infer_streams);
 
   StartEncodedFrameLogs(video_send_stream_);
   StartEncodedFrameLogs(video_receive_streams_[0]);
   video_send_stream_->Start();
   for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_)
     thumbnail_send_stream->Start();
   for (VideoReceiveStream* receive_stream : video_receive_streams_)
     receive_stream->Start();
   for (FlexfecReceiveStream* receive_stream : flexfec_receive_streams_)
     receive_stream->Start();
@@ skipping 233 matching lines @@
   if (!params_.video.encoded_frame_base_path.empty()) {
     std::ostringstream str;
     str << receive_logs_++;
     std::string path =
         params_.video.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
     stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
                                         10000000);
   }
 }
 }  // namespace webrtc