Added WebRTC-QuickPerfTest field trial. If enabled only 1 frame will be sent.

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_quality_test.h"
@@ skipping 15 matching lines @@
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/logging/rtc_event_log/rtc_event_log.h"
 #include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/system_wrappers/include/cpu_info.h"
+#include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/test/layer_filtering_transport.h"
 #include "webrtc/test/run_loop.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/vcm_capturer.h"
 #include "webrtc/test/video_renderer.h"
 #include "webrtc/voice_engine/include/voe_base.h"
 
 namespace {
 
 constexpr int kSendStatsPollingIntervalMs = 1000;
 constexpr int kPayloadTypeH264 = 122;
 constexpr int kPayloadTypeVP8 = 123;
 constexpr int kPayloadTypeVP9 = 124;
 constexpr size_t kMaxComparisons = 10;
 constexpr char kSyncGroup[] = "av_sync";
 constexpr int kOpusMinBitrateBps = 6000;
 constexpr int kOpusBitrateFbBps = 32000;
+constexpr int kFramesSentInQuickTest = 1;
 
 struct VoiceEngineState {
   VoiceEngineState()
       : voice_engine(nullptr),
         base(nullptr),
         send_channel_id(-1),
         receive_channel_id(-1) {}
 
   webrtc::VoiceEngine* voice_engine;
   webrtc::VoEBase* base;
@@ skipping 64 matching lines @@
   VideoAnalyzer(test::LayerFilteringTransport* transport,
                 const std::string& test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames,
                 FILE* graph_data_output_file,
                 const std::string& graph_title,
                 uint32_t ssrc_to_analyze,
                 uint32_t rtx_ssrc_to_analyze,
                 uint32_t selected_width,
-                uint32_t selected_height)
+                uint32_t selected_height,
+                bool is_quick_test_enabled)
       : transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         receive_stream_(nullptr),
         captured_frame_forwarder_(this),
         test_label_(test_label),
         graph_data_output_file_(graph_data_output_file),
         graph_title_(graph_title),
         ssrc_to_analyze_(ssrc_to_analyze),
         rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
         selected_width_(selected_width),
         selected_height_(selected_height),
         pre_encode_proxy_(this),
         encode_timing_proxy_(this),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         dropped_frames_before_first_encode_(0),
         dropped_frames_before_rendering_(0),
         last_render_time_(0),
         rtp_timestamp_delta_(0),
         avg_psnr_threshold_(avg_psnr_threshold),
         avg_ssim_threshold_(avg_ssim_threshold),
+        is_quick_test_enabled_(is_quick_test_enabled),
         stats_polling_thread_(&PollStatsThread, this, "StatsPoller"),
         comparison_available_event_(false, false),
         done_(true, false) {
     // Create thread pool for CPU-expensive PSNR/SSIM calculations.
 
     // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
     // so that we don't accidentally starve "real" worker threads (codec etc).
     // Also, don't allocate more than kMaxComparisonThreads, even if there are
     // spare cores.
 
@@ skipping 502 matching lines @@
       PrintResult("decode_time", decode_time_ms_, " ms");
       PrintResult("decode_time_max", decode_time_max_ms_, " ms");
     }
 
     printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(),
            dropped_frames_);
     printf("RESULT dropped_frames_before_first_encode: %s = %d frames\n",
            test_label_.c_str(), dropped_frames_before_first_encode_);
     printf("RESULT dropped_frames_before_rendering: %s = %d frames\n",
            test_label_.c_str(), dropped_frames_before_rendering_);
-
-    EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
-    EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
+    //  Disable quality check for quick test, as quality checks may fail
+    //  because too few samples were collected.
+    if (!is_quick_test_enabled_) {
+      EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
+      EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
+    }
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = -1.0;
     double ssim = -1.0;
     if (comparison.reference) {
       psnr = I420PSNR(&*comparison.reference, &*comparison.render);
       ssim = I420SSIM(&*comparison.reference, &*comparison.render);
     }
@@ skipping 184 matching lines @@
   std::deque<VideoFrame> frames_ GUARDED_BY(crit_);
   rtc::Optional<VideoFrame> last_rendered_frame_ GUARDED_BY(crit_);
   rtc::TimestampWrapAroundHandler wrap_handler_ GUARDED_BY(crit_);
   std::map<int64_t, int64_t> send_times_ GUARDED_BY(crit_);
   std::map<int64_t, int64_t> recv_times_ GUARDED_BY(crit_);
   std::map<int64_t, size_t> encoded_frame_sizes_ GUARDED_BY(crit_);
   rtc::Optional<uint32_t> first_encoded_timestamp_ GUARDED_BY(crit_);
   rtc::Optional<uint32_t> first_sent_timestamp_ GUARDED_BY(crit_);
   const double avg_psnr_threshold_;
   const double avg_ssim_threshold_;
+  bool is_quick_test_enabled_;
 
   rtc::CriticalSection comparison_lock_;
   std::vector<rtc::PlatformThread*> comparison_thread_pool_;
   rtc::PlatformThread stats_polling_thread_;
   rtc::Event comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   rtc::Event done_;
 };
 
 VideoQualityTest::VideoQualityTest()
@@ skipping 429 matching lines @@
       selected_stream.height != params_.video.height ||
       (!params_.ss.spatial_layers.empty() &&
        params_.ss.spatial_layers[selected_sl].scaling_factor_num !=
            params_.ss.spatial_layers[selected_sl].scaling_factor_den);
   if (disable_quality_check) {
     fprintf(stderr,
             "Warning: Calculating PSNR and SSIM for downsized resolution "
             "not implemented yet! Skipping PSNR and SSIM calculations!\n");
   }
 
+  bool is_quick_test_enabled =
+      field_trial::FindFullName("WebRTC-QuickPerfTest") == "Enabled";
   VideoAnalyzer analyzer(
       &send_transport, params_.analyzer.test_label,
       disable_quality_check ? -1.1 : params_.analyzer.avg_psnr_threshold,
       disable_quality_check ? -1.1 : params_.analyzer.avg_ssim_threshold,
-      params_.analyzer.test_durations_secs * params_.video.fps,
+      is_quick_test_enabled
+          ? kFramesSentInQuickTest
+          : params_.analyzer.test_durations_secs * params_.video.fps,
       graph_data_output_file, graph_title,
       kVideoSendSsrcs[params_.ss.selected_stream],
       kSendRtxSsrcs[params_.ss.selected_stream],
       static_cast<uint32_t>(selected_stream.width),
-      static_cast<uint32_t>(selected_stream.height));
-
+      static_cast<uint32_t>(selected_stream.height), is_quick_test_enabled);
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupVideo(&analyzer, &recv_transport);
   video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
   video_send_config_.pre_encode_callback = analyzer.pre_encode_proxy();
   for (auto& config : video_receive_configs_)
     config.pre_decode_callback = &analyzer;
   RTC_DCHECK(!video_send_config_.post_encode_callback);
@@ skipping 234 matching lines @@
     std::ostringstream str;
     str << receive_logs_++;
     std::string path =
         params_.video.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
     stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
                                         10000000);
   }
 }
 
 }  // namespace webrtc