Make rate control and quality thresholds optional in VideoProcessorIntegrationTest.

webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
 
 #include <math.h>
 
 #include <limits>
 #include <memory>
 #include <string>
 #include <utility>
+#include <vector>
 
 #if defined(WEBRTC_ANDROID)
 #include "webrtc/modules/video_coding/codecs/test/android_test_initializer.h"
 #include "webrtc/sdk/android/src/jni/androidmediadecoder_jni.h"
 #include "webrtc/sdk/android/src/jni/androidmediaencoder_jni.h"
 #elif defined(WEBRTC_IOS)
 #include "webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h"
 #endif
 
 #include "webrtc/media/engine/internaldecoderfactory.h"
@@ skipping 28 matching lines @@
 const int kPercTargetvsActualMismatch = 20;
 const int kBaseKeyFrameInterval = 3000;
 
 // Parameters from VP8 wrapper, which control target size of key frames.
 const float kInitialBufferSize = 0.5f;
 const float kOptimalBufferSize = 0.6f;
 const float kScaleKeyFrameSize = 0.5f;
 
 // Thresholds for the quality metrics. Defaults are maximally minimal.
 struct QualityThresholds {
-  QualityThresholds() {}
   QualityThresholds(double min_avg_psnr,
                     double min_min_psnr,
                     double min_avg_ssim,
                     double min_min_ssim)
       : min_avg_psnr(min_avg_psnr),
         min_min_psnr(min_min_psnr),
         min_avg_ssim(min_avg_ssim),
         min_min_ssim(min_min_ssim) {}
-  double min_avg_psnr = std::numeric_limits<double>::min();
-  double min_min_psnr = std::numeric_limits<double>::min();
-  double min_avg_ssim = 0.0;
-  double min_min_ssim = 0.0;
+  double min_avg_psnr;
+  double min_min_psnr;
+  double min_avg_ssim;
+  double min_min_ssim;
 };
 
 // The sequence of bit rate and frame rate changes for the encoder, the frame
 // number where the changes are made, and the total number of frames for the
 // test.
 struct RateProfile {
   int target_bit_rate[kMaxNumRateUpdates];
   int input_frame_rate[kMaxNumRateUpdates];
   int frame_index_rate_update[kMaxNumRateUpdates + 1];
   int num_frames;
 };
 
 // Thresholds for the rate control metrics. The rate mismatch thresholds are
 // defined as percentages. |max_time_hit_target| is defined as number of frames,
 // after a rate update is made to the encoder, for the encoder to reach within
 // |kPercTargetvsActualMismatch| of new target rate. The thresholds are defined
 // for each rate update sequence.
 struct RateControlThresholds {
   int max_num_dropped_frames;
   int max_key_frame_size_mismatch;
   int max_delta_frame_size_mismatch;
   int max_encoding_rate_mismatch;
   int max_time_hit_target;
-  int num_spatial_resizes;  // Set to -1 to disable check.
-  int num_key_frames;       // Set to -1 to disable check.
+  int num_spatial_resizes;
+  int num_key_frames;
 };
 
 // Should video files be saved persistently to disk for post-run visualization?
 struct VisualizationParams {
   bool save_encoded_ivf;
   bool save_decoded_y4m;
 };
 
 // Integration test for video processor. Encodes+decodes a clip and
 // writes it to the output directory. After completion, quality metrics
@@ skipping 183 matching lines @@
     perc_encoding_rate_mismatch_ =
         100 * fabs(encoding_bitrate_total_ - bitrate_kbps_) / bitrate_kbps_;
     if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch &&
         !encoding_rate_within_target_) {
       num_frames_to_hit_target_ = num_frames_total_;
       encoding_rate_within_target_ = true;
     }
   }
 
   // Verify expected behavior of rate control and print out data.
-  void VerifyRateControlMetrics(int rate_update_index,
-                                const RateControlThresholds& rc_expected) {
+  void PrintAndMaybeVerifyRateControlMetrics(
+      int rate_update_index,
+      const std::vector<RateControlThresholds>* rc_thresholds) {
     int num_dropped_frames = processor_->NumberDroppedFrames();
     int num_resize_actions = processor_->NumberSpatialResizes();
     printf(
         "Rate update #%d:\n"
         " Target bitrate         : %d\n"
         " Encoded bitrate        : %f\n"
         " Frame rate             : %d\n",
         rate_update_index, bitrate_kbps_, encoding_bitrate_total_, framerate_);
     printf(
         " # processed frames     : %d\n"
         " # frames to convergence: %d\n"
         " # dropped frames       : %d\n"
         " # spatial resizes      : %d\n",
         num_frames_total_, num_frames_to_hit_target_, num_dropped_frames,
         num_resize_actions);
 
-    EXPECT_LE(perc_encoding_rate_mismatch_,
-              rc_expected.max_encoding_rate_mismatch);
+    const RateControlThresholds* rc_threshold = nullptr;
+    if (rc_thresholds) {
+      rc_threshold = &(*rc_thresholds)[rate_update_index];
+
+      EXPECT_LE(perc_encoding_rate_mismatch_,
+                rc_threshold->max_encoding_rate_mismatch);
+    }
     if (num_key_frames_ > 0) {
       int perc_key_frame_size_mismatch =
           100 * sum_key_frame_size_mismatch_ / num_key_frames_;
       printf(
           " # key frames           : %d\n"
           " Key frame rate mismatch: %d\n",
           num_key_frames_, perc_key_frame_size_mismatch);
-      EXPECT_LE(perc_key_frame_size_mismatch,
-                rc_expected.max_key_frame_size_mismatch);
+      if (rc_threshold) {
+        EXPECT_LE(perc_key_frame_size_mismatch,
+                  rc_threshold->max_key_frame_size_mismatch);
+      }
     }
 
     const int num_temporal_layers =
         NumberOfTemporalLayers(config_.codec_settings);
     for (int i = 0; i < num_temporal_layers; i++) {
       int perc_frame_size_mismatch =
           100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i];
       int perc_encoding_rate_mismatch =
           100 * fabs(encoding_bitrate_[i] - bitrate_layer_[i]) /
           bitrate_layer_[i];
       printf(
           " Temporal layer #%d:\n"
           "  Target layer bitrate                : %f\n"
           "  Layer frame rate                    : %f\n"
           "  Layer per frame bandwidth           : %f\n"
           "  Layer encoding bitrate              : %f\n"
           "  Layer percent frame size mismatch   : %d\n"
           "  Layer percent encoding rate mismatch: %d\n"
           "  # frames processed per layer        : %d\n",
           i, bitrate_layer_[i], framerate_layer_[i], per_frame_bandwidth_[i],
           encoding_bitrate_[i], perc_frame_size_mismatch,
           perc_encoding_rate_mismatch, num_frames_per_update_[i]);
-      EXPECT_LE(perc_frame_size_mismatch,
-                rc_expected.max_delta_frame_size_mismatch);
-      EXPECT_LE(perc_encoding_rate_mismatch,
-                rc_expected.max_encoding_rate_mismatch);
+      if (rc_threshold) {
+        EXPECT_LE(perc_frame_size_mismatch,
+                  rc_threshold->max_delta_frame_size_mismatch);
+        EXPECT_LE(perc_encoding_rate_mismatch,
+                  rc_threshold->max_encoding_rate_mismatch);
+      }
     }
     printf("\n");
 
-    EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target);
-    EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames);
-    if (rc_expected.num_spatial_resizes >= 0) {
-      EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions);
-    }
-    if (rc_expected.num_key_frames >= 0) {
-      EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_);
+    if (rc_threshold) {
+      EXPECT_LE(num_frames_to_hit_target_, rc_threshold->max_time_hit_target);
+      EXPECT_LE(num_dropped_frames, rc_threshold->max_num_dropped_frames);
+      EXPECT_EQ(rc_threshold->num_spatial_resizes, num_resize_actions);
+      EXPECT_EQ(rc_threshold->num_key_frames, num_key_frames_);
     }
   }
 
   static void VerifyQuality(const QualityMetricsResult& psnr_result,
                             const QualityMetricsResult& ssim_result,
                             const QualityThresholds& quality_thresholds) {
     EXPECT_GT(psnr_result.average, quality_thresholds.min_avg_psnr);
     EXPECT_GT(psnr_result.min, quality_thresholds.min_min_psnr);
     EXPECT_GT(ssim_result.average, quality_thresholds.min_avg_ssim);
     EXPECT_GT(ssim_result.min, quality_thresholds.min_min_ssim);
@@ skipping 68 matching lines @@
       }
       framerate_layer_[i] =
           framerate_ / static_cast<float>(1 << (num_temporal_layers - 1));
     }
     if (num_temporal_layers == 3) {
       framerate_layer_[2] = framerate_ / 2.0f;
     }
   }
 
   // Processes all frames in the clip and verifies the result.
-  // TODO(brandtr): Change the second last argument to be a
-  // const std::vector<RateControlThresholds>&, so we can ensure that the user
-  // does not expect us to do mid-clip rate updates when we are not able to,
-  // e.g., when we are operating in batch mode.
-  void ProcessFramesAndVerify(const QualityThresholds& quality_thresholds,
-                              const RateProfile& rate_profile,
-                              RateControlThresholds* rc_thresholds,
-                              const VisualizationParams* visualization_params) {
+  void ProcessFramesAndMaybeVerify(
+      const RateProfile& rate_profile,
+      const std::vector<RateControlThresholds>* rc_thresholds,
+      const QualityThresholds* quality_thresholds,
+      const VisualizationParams* visualization_params) {
     config_.codec_settings.startBitrate = rate_profile.target_bit_rate[0];
     SetUpObjects(visualization_params, rate_profile.target_bit_rate[0],
                  rate_profile.input_frame_rate[0]);
 
     // Set initial rates.
     bitrate_kbps_ = rate_profile.target_bit_rate[0];
     framerate_ = rate_profile.input_frame_rate[0];
     SetTemporalLayerRates();
     // Set the initial target size for key frame.
     target_size_key_frame_initial_ =
@@ skipping 39 matching lines @@
         ++num_frames_per_update_[tl_idx];
         ++num_frames_total_;
         UpdateRateControlMetrics(frame_number);
 
         ++frame_number;
 
         // If we hit another/next update, verify stats for current state and
         // update layers and codec with new rates.
         if (frame_number ==
             rate_profile.frame_index_rate_update[update_index + 1]) {
-          VerifyRateControlMetrics(update_index, rc_thresholds[update_index]);
+          PrintAndMaybeVerifyRateControlMetrics(update_index, rc_thresholds);
 
           // Update layer rates and the codec with new rates.
           ++update_index;
           bitrate_kbps_ = rate_profile.target_bit_rate[update_index];
           framerate_ = rate_profile.input_frame_rate[update_index];
           SetTemporalLayerRates();
           ResetRateControlMetrics(
               rate_profile.frame_index_rate_update[update_index + 1]);
           processor_->SetRates(bitrate_kbps_, framerate_);
         }
       }
       // TODO(brandtr): Refactor "frame number accounting" so we don't have to
       // call ProcessFrame one extra time here.
       processor_->ProcessFrame(frame_number);
     }
 
     // Verify rate control metrics for all frames (if in batch mode), or for all
     // frames since the last rate update (if not in batch mode).
-    VerifyRateControlMetrics(update_index, rc_thresholds[update_index]);
+    PrintAndMaybeVerifyRateControlMetrics(update_index, rc_thresholds);
     EXPECT_EQ(num_frames, frame_number);
     EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
 
     // Release encoder and decoder to make sure they have finished processing.
     processor_->Release();
     DestroyEncoderAndDecoder();
 
     // Close the analysis files before we use them for SSIM/PSNR calculations.
     analysis_frame_reader_->Close();
     analysis_frame_writer_->Close();
 
     // Close visualization files.
     if (encoded_frame_writer_) {
       EXPECT_TRUE(encoded_frame_writer_->Close());
     }
     if (decoded_frame_writer_) {
       decoded_frame_writer_->Close();
     }
 
     // TODO(marpan): Should compute these quality metrics per SetRates update.
     QualityMetricsResult psnr_result, ssim_result;
     EXPECT_EQ(0, I420MetricsFromFiles(config_.input_filename.c_str(),
                                       config_.output_filename.c_str(),
                                       config_.codec_settings.width,
                                       config_.codec_settings.height,
                                       &psnr_result, &ssim_result));
-    VerifyQuality(psnr_result, ssim_result, quality_thresholds);
+    if (quality_thresholds) {
+      VerifyQuality(psnr_result, ssim_result, *quality_thresholds);
+    }
     stats_.PrintSummary();
     printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",
            psnr_result.average, psnr_result.min, ssim_result.average,
            ssim_result.min);
     printf("\n");
 
     // Remove analysis file.
     if (remove(config_.output_filename.c_str()) < 0) {
       fprintf(stderr, "Failed to remove temporary file!\n");
     }
@@ skipping 69 matching lines @@
                              int rate_update_index,
                              int bitrate_kbps,
                              int framerate_fps,
                              int frame_index_rate_update) {
     rate_profile->target_bit_rate[rate_update_index] = bitrate_kbps;
     rate_profile->input_frame_rate[rate_update_index] = framerate_fps;
     rate_profile->frame_index_rate_update[rate_update_index] =
         frame_index_rate_update;
   }
 
-  static void SetRateControlThresholds(RateControlThresholds* rc_thresholds,
-                                       int update_index,
-                                       int max_num_dropped_frames,
-                                       int max_key_frame_size_mismatch,
-                                       int max_delta_frame_size_mismatch,
-                                       int max_encoding_rate_mismatch,
-                                       int max_time_hit_target,
-                                       int num_spatial_resizes,
-                                       int num_key_frames) {
-    rc_thresholds[update_index].max_num_dropped_frames = max_num_dropped_frames;
-    rc_thresholds[update_index].max_key_frame_size_mismatch =
-        max_key_frame_size_mismatch;
-    rc_thresholds[update_index].max_delta_frame_size_mismatch =
-        max_delta_frame_size_mismatch;
-    rc_thresholds[update_index].max_encoding_rate_mismatch =
-        max_encoding_rate_mismatch;
-    rc_thresholds[update_index].max_time_hit_target = max_time_hit_target;
-    rc_thresholds[update_index].num_spatial_resizes = num_spatial_resizes;
-    rc_thresholds[update_index].num_key_frames = num_key_frames;
+  static void AddRateControlThresholds(
+      int max_num_dropped_frames,
+      int max_key_frame_size_mismatch,
+      int max_delta_frame_size_mismatch,
+      int max_encoding_rate_mismatch,
+      int max_time_hit_target,
+      int num_spatial_resizes,
+      int num_key_frames,
+      std::vector<RateControlThresholds>* rc_thresholds) {
+    RTC_DCHECK(rc_thresholds);
+
+    rc_thresholds->emplace_back();
+    RateControlThresholds* rc_threshold = &rc_thresholds->back();
+    rc_threshold->max_num_dropped_frames = max_num_dropped_frames;
+    rc_threshold->max_key_frame_size_mismatch = max_key_frame_size_mismatch;
+    rc_threshold->max_delta_frame_size_mismatch = max_delta_frame_size_mismatch;
+    rc_threshold->max_encoding_rate_mismatch = max_encoding_rate_mismatch;
+    rc_threshold->max_time_hit_target = max_time_hit_target;
+    rc_threshold->num_spatial_resizes = num_spatial_resizes;
+    rc_threshold->num_key_frames = num_key_frames;
   }
 
   // Config.
   TestConfig config_;
 
   // Codecs.
   std::unique_ptr<cricket::WebRtcVideoEncoderFactory> encoder_factory_;
   VideoEncoder* encoder_;
   std::unique_ptr<cricket::WebRtcVideoDecoderFactory> decoder_factory_;
   VideoDecoder* decoder_;
@@ skipping 27 matching lines @@
   float target_size_key_frame_initial_;
   float target_size_key_frame_;
   float sum_key_frame_size_mismatch_;
   int num_key_frames_;
 };
 
 }  // namespace test
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_