Step #3: Add flag for correctness mode in VideoProcessor integration tests.

webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 68 matching lines @@
   bool error_concealment_on;
   bool denoising_on;
   bool frame_dropper_on;
   bool spatial_resize_on;
 
   float packet_loss_probability;  // [0.0, 1.0].
 
   std::string filename;
   bool verbose_logging;
 
+  // In correctness mode, EXPECTs are enabled for comparing the quality and
+  // rate control metrics to their respective thresholds. It can be useful to
+  // turn off correctness mode for HW codecs, since these in general are less
+  // reliable than SW codecs, and failing an EXPECT will fail the entire test.
+  bool correctness_mode;
+
   // In batch mode, the VideoProcessor is fed all the frames for processing
   // before any metrics are calculated. This is useful for pipelining HW codecs,
   // for which some calculated metrics otherwise would be incorrect. The
   // downside with batch mode is that mid-test rate allocation is not supported.
   bool batch_mode;
 };
 
 // Thresholds for the quality metrics.
 struct QualityThresholds {
   double min_avg_psnr;
@@ skipping 316 matching lines @@
         !encoding_rate_within_target_) {
       num_frames_to_hit_target_ = num_frames_total_;
       encoding_rate_within_target_ = true;
     }
   }
 
   // Verify expected behavior of rate control and print out data.
   void VerifyRateControlMetrics(int update_index,
                                 int num_dropped_frames,
                                 int num_resize_actions,
+                                bool correctness_mode,
                                 const RateControlThresholds& rc_expected) {
     printf(
         "For update #: %d,\n"
         " Target Bitrate: %d,\n"
         " Encoding bitrate: %f,\n"
         " Frame rate: %d \n",
         update_index, bit_rate_, encoding_bitrate_total_, frame_rate_);
     printf(
         " Number of frames to approach target rate: %d, \n"
         " Number of dropped frames: %d, \n"
         " Number of spatial resizes: %d, \n",
         num_frames_to_hit_target_, num_dropped_frames, num_resize_actions);
-    EXPECT_LE(perc_encoding_rate_mismatch_,
-              rc_expected.max_encoding_rate_mismatch);
+    if (correctness_mode) {
+      EXPECT_LE(perc_encoding_rate_mismatch_,
+                rc_expected.max_encoding_rate_mismatch);
+    }
     if (num_key_frames_ > 0) {
       int perc_key_frame_size_mismatch =
           100 * sum_key_frame_size_mismatch_ / num_key_frames_;
       printf(
           " Number of Key frames: %d \n"
           " Key frame rate mismatch: %d \n",
           num_key_frames_, perc_key_frame_size_mismatch);
-      EXPECT_LE(perc_key_frame_size_mismatch,
-                rc_expected.max_key_frame_size_mismatch);
+      if (correctness_mode) {
+        EXPECT_LE(perc_key_frame_size_mismatch,
+                  rc_expected.max_key_frame_size_mismatch);
+      }
     }
     printf("\n");
     printf("Rates statistics for Layer data \n");
     for (int i = 0; i < num_temporal_layers_; i++) {
       printf("Temporal layer #%d \n", i);
       int perc_frame_size_mismatch =
           100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i];
       int perc_encoding_rate_mismatch =
           100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) /
           bit_rate_layer_[i];
       printf(
           " Target Layer Bit rate: %f \n"
           " Layer frame rate: %f, \n"
           " Layer per frame bandwidth: %f, \n"
           " Layer Encoding bit rate: %f, \n"
           " Layer Percent frame size mismatch: %d,  \n"
           " Layer Percent encoding rate mismatch: %d, \n"
           " Number of frame processed per layer: %d \n",
           bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i],
           encoding_bitrate_[i], perc_frame_size_mismatch,
           perc_encoding_rate_mismatch, num_frames_per_update_[i]);
-      EXPECT_LE(perc_frame_size_mismatch,
-                rc_expected.max_delta_frame_size_mismatch);
-      EXPECT_LE(perc_encoding_rate_mismatch,
-                rc_expected.max_encoding_rate_mismatch);
+      if (correctness_mode) {
+        EXPECT_LE(perc_frame_size_mismatch,
+                  rc_expected.max_delta_frame_size_mismatch);
+        EXPECT_LE(perc_encoding_rate_mismatch,
+                  rc_expected.max_encoding_rate_mismatch);
+      }
     }
     printf("\n");
-    EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target);
-    EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames);
-    EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions);
-    EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_);
+    if (correctness_mode) {
+      EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target);
+      EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames);
+      EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions);
+      EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_);
+    }
   }
 
   void VerifyQuality(const test::QualityMetricsResult& psnr_result,
                      const test::QualityMetricsResult& ssim_result,
                      const QualityThresholds& quality_thresholds) {
     EXPECT_GT(psnr_result.average, quality_thresholds.min_avg_psnr);
     EXPECT_GT(psnr_result.min, quality_thresholds.min_min_psnr);
     EXPECT_GT(ssim_result.average, quality_thresholds.min_avg_ssim);
     EXPECT_GT(ssim_result.min, quality_thresholds.min_min_ssim);
   }
@@ skipping 96 matching lines @@
         ++num_frames_per_update_[LayerIndexForFrame(frame_number)];
         ++num_frames_total_;
         FrameType frame_type = processor_->EncodedFrameType(frame_number);
         size_t encoded_frame_size = processor_->EncodedFrameSize(frame_number);
         UpdateRateControlMetrics(frame_number, frame_type, encoded_frame_size);
       }
     } else {
       // In online mode, we calculate the metrics for a given frame right after
       // it has been sent for encoding.
 
+      if (!process.correctness_mode) {
+        // This is a check that we do not disable correctness mode by mistake
+        // on the bots. When doing manual experimentation and correctness mode
+        // is not desired, the check can be commented out.
+        RTC_NOTREACHED() << "This test is set up to run with both correctness "
+                            "mode and batch mode disabled. Are you sure you "
+                            "want to do that? In particular, this should never "
+                            "happen on the bots.";
+      }
       if (process.hw_codec) {
         LOG(LS_WARNING) << "HW codecs should mostly be run in batch mode, "
                            "since they may be pipelining.";
       }
 
       while (frame_number < num_frames) {
         EXPECT_TRUE(processor_->ProcessFrame(frame_number));
 
         ++num_frames_per_update_[LayerIndexForFrame(frame_number)];
         ++num_frames_total_;
         FrameType frame_type = processor_->EncodedFrameType(frame_number);
         size_t encoded_frame_size = processor_->EncodedFrameSize(frame_number);
         UpdateRateControlMetrics(frame_number, frame_type, encoded_frame_size);
 
         ++frame_number;
 
         // If we hit another/next update, verify stats for current state and
         // update layers and codec with new rates.
         if (frame_number ==
             rate_profile.frame_index_rate_update[update_index + 1]) {
           int num_dropped_frames = processor_->NumberDroppedFrames();
           int num_resize_actions = processor_->NumberSpatialResizes();
           VerifyRateControlMetrics(update_index, num_dropped_frames,
-                                   num_resize_actions,
+                                   num_resize_actions, process.correctness_mode,
                                    rc_thresholds[update_index]);
 
           // Update layer rates and the codec with new rates.
           ++update_index;
           bit_rate_ = rate_profile.target_bit_rate[update_index];
           frame_rate_ = rate_profile.input_frame_rate[update_index];
           SetLayerRates();
           ResetRateControlMetrics(
               rate_profile.frame_index_rate_update[update_index + 1]);
           processor_->SetRates(bit_rate_, frame_rate_);
         }
       }
       processor_->ProcessFrame(frame_number);
 
       // Release encoder and decoder to make sure they have finished processing.
       EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
       EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
     }
 
     // Verify rate control metrics for all frames (if in batch mode), or for all
     // frames since the last rate update (if not in batch mode).
     int num_dropped_frames = processor_->NumberDroppedFrames();
     int num_resize_actions = processor_->NumberSpatialResizes();
     VerifyRateControlMetrics(update_index, num_dropped_frames,
-                             num_resize_actions, rc_thresholds[update_index]);
+                             num_resize_actions, process.correctness_mode,
+                             rc_thresholds[update_index]);
     EXPECT_EQ(num_frames, frame_number);
     EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
 
     // Close the analysis files before we use them for SSIM/PSNR calculations.
     analysis_frame_reader_->Close();
     analysis_frame_writer_->Close();
 
     // Close visualization files.
     if (source_frame_writer_) {
       source_frame_writer_->Close();
     }
     if (encoded_frame_writer_) {
       EXPECT_TRUE(encoded_frame_writer_->Close());
     }
     if (decoded_frame_writer_) {
       decoded_frame_writer_->Close();
     }
 
     // TODO(marpan): Should compute these quality metrics per SetRates update.
     test::QualityMetricsResult psnr_result, ssim_result;
     EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(),
                                             config_.output_filename.c_str(),
                                             config_.codec_settings->width,
                                             config_.codec_settings->height,
                                             &psnr_result, &ssim_result));
     printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",
            psnr_result.average, psnr_result.min, ssim_result.average,
            ssim_result.min);
-    VerifyQuality(psnr_result, ssim_result, quality_thresholds);
+    if (process.correctness_mode) {
+      VerifyQuality(psnr_result, ssim_result, quality_thresholds);
+    }
     stats_.PrintSummary();
 
     // Remove analysis file.
     if (remove(config_.output_filename.c_str()) < 0) {
       fprintf(stderr, "Failed to remove temporary file!\n");
     }
   }
 
   static void SetCodecParams(CodecParams* process_settings,
                              VideoCodecType codec_type,
                              bool hw_codec,
                              bool use_single_core,
                              float packet_loss_probability,
                              int key_frame_interval,
                              int num_temporal_layers,
                              bool error_concealment_on,
                              bool denoising_on,
                              bool frame_dropper_on,
                              bool spatial_resize_on,
                              int width,
                              int height,
                              const std::string& filename,
                              bool verbose_logging,
+                             bool correctness_mode,
                              bool batch_mode) {
     process_settings->codec_type = codec_type;
     process_settings->hw_codec = hw_codec;
     process_settings->use_single_core = use_single_core;
     process_settings->packet_loss_probability = packet_loss_probability;
     process_settings->key_frame_interval = key_frame_interval;
     process_settings->num_temporal_layers = num_temporal_layers,
     process_settings->error_concealment_on = error_concealment_on;
     process_settings->denoising_on = denoising_on;
     process_settings->frame_dropper_on = frame_dropper_on;
     process_settings->spatial_resize_on = spatial_resize_on;
     process_settings->width = width;
     process_settings->height = height;
     process_settings->filename = filename;
     process_settings->verbose_logging = verbose_logging;
+    process_settings->correctness_mode = correctness_mode;
     process_settings->batch_mode = batch_mode;
   }
 
   static void SetCodecParams(CodecParams* process_settings,
                              VideoCodecType codec_type,
                              bool hw_codec,
                              bool use_single_core,
                              float packet_loss_probability,
                              int key_frame_interval,
                              int num_temporal_layers,
                              bool error_concealment_on,
                              bool denoising_on,
                              bool frame_dropper_on,
                              bool spatial_resize_on) {
     SetCodecParams(process_settings, codec_type, hw_codec, use_single_core,
                    packet_loss_probability, key_frame_interval,
                    num_temporal_layers, error_concealment_on, denoising_on,
                    frame_dropper_on, spatial_resize_on, kCifWidth, kCifHeight,
                    kFilenameForemanCif, false /* verbose_logging */,
-                   false /* batch_mode */);
+                   true /* correctness_mode */, false /* batch_mode */);
   }
 
   static void SetQualityThresholds(QualityThresholds* quality_thresholds,
                                    double min_avg_psnr,
                                    double min_min_psnr,
                                    double min_avg_ssim,
                                    double min_min_ssim) {
     quality_thresholds->min_avg_psnr = min_avg_psnr;
     quality_thresholds->min_min_psnr = min_min_psnr;
     quality_thresholds->min_avg_ssim = min_avg_ssim;
@@ skipping 79 matching lines @@
 
   // Codec and network settings.
   float packet_loss_probability_;
   int num_temporal_layers_;
 };
 
 }  // namespace test
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_