Reland of Add optional visualization file writers to VideoProcessor tests.

webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_
 
 #include <math.h>
 
 #include <memory>
 #include <string>
+#include <utility>
 
 #if defined(WEBRTC_ANDROID)
 #include "webrtc/modules/video_coding/codecs/test/android_test_initializer.h"
 #include "webrtc/sdk/android/src/jni/androidmediadecoder_jni.h"
 #include "webrtc/sdk/android/src/jni/androidmediaencoder_jni.h"
 #elif defined(WEBRTC_IOS)
 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_decoder.h"
 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.h"
 #endif
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/file.h"
 #include "webrtc/media/engine/webrtcvideodecoderfactory.h"
 #include "webrtc/media/engine/webrtcvideoencoderfactory.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h"
 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
-#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "webrtc/modules/video_coding/include/video_coding.h"
+#include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/testsupport/frame_reader.h"
 #include "webrtc/test/testsupport/frame_writer.h"
 #include "webrtc/test/testsupport/metrics/video_metrics.h"
 #include "webrtc/test/testsupport/packet_reader.h"
 #include "webrtc/typedefs.h"
 
 namespace webrtc {
 namespace test {
@@ skipping 53 matching lines @@
 struct RateControlMetrics {
   int max_num_dropped_frames;
   int max_key_frame_size_mismatch;
   int max_delta_frame_size_mismatch;
   int max_encoding_rate_mismatch;
   int max_time_hit_target;
   int num_spatial_resizes;
   int num_key_frames;
 };
 
+// Should video files be saved persistently to disk for post-run visualization?
+struct VisualizationParams {
+  bool save_source_y4m;
+  bool save_encoded_ivf;
+  bool save_decoded_y4m;
+};
+
 #if !defined(WEBRTC_IOS)
 const int kNumFramesShort = 100;
 #endif
 const int kNumFramesLong = 299;
 
 // Parameters from VP8 wrapper, which control target size of key frames.
 const float kInitialBufferSize = 0.5f;
 const float kOptimalBufferSize = 0.6f;
 const float kScaleKeyFrameSize = 0.5f;
 
@@ skipping 13 matching lines @@
     InitializeAndroidObjects();
 
     external_encoder_factory_.reset(
         new webrtc_jni::MediaCodecVideoEncoderFactory());
     external_decoder_factory_.reset(
         new webrtc_jni::MediaCodecVideoDecoderFactory());
 #endif
   }
   virtual ~VideoProcessorIntegrationTest() = default;
 
-  void SetUpCodecConfig(const CodecConfigPars& process) {
+  void SetUpCodecConfig(const CodecConfigPars& process,
+                        const VisualizationParams* visualization_params) {
     if (process.hw_codec) {
 #if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED)
 #if defined(WEBRTC_ANDROID)
       // In general, external codecs should be destroyed by the factories that
       // allocated them. For the particular case of the Android
       // MediaCodecVideo{En,De}coderFactory's, however, it turns out that it is
       // fine for the std::unique_ptr to destroy the owned codec directly.
       switch (process.codec_type) {
         case kVideoCodecH264:
           encoder_.reset(external_encoder_factory_->CreateVideoEncoder(
@@ skipping 11 matching lines @@
           encoder_.reset(external_encoder_factory_->CreateVideoEncoder(
               cricket::VideoCodec(cricket::kVp9CodecName)));
           decoder_.reset(
               external_decoder_factory_->CreateVideoDecoder(kVideoCodecVP9));
           break;
         default:
           RTC_NOTREACHED();
           break;
       }
 #elif defined(WEBRTC_IOS)
-      RTC_DCHECK_EQ(kVideoCodecH264, process.codec_type)
+      ASSERT_EQ(kVideoCodecH264, process.codec_type)
           << "iOS HW codecs only support H264.";
       encoder_.reset(new H264VideoToolboxEncoder(
           cricket::VideoCodec(cricket::kH264CodecName)));
       decoder_.reset(new H264VideoToolboxDecoder());
 #else
       RTC_NOTREACHED() << "Only support HW codecs on Android and iOS.";
 #endif
 #endif  // WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED
-      RTC_DCHECK(encoder_) << "HW encoder not successfully created.";
-      RTC_DCHECK(decoder_) << "HW decoder not successfully created.";
+      RTC_CHECK(encoder_) << "HW encoder not successfully created.";
+      RTC_CHECK(decoder_) << "HW decoder not successfully created.";
     } else {
       // SW codecs.
       switch (process.codec_type) {
         case kVideoCodecH264:
           encoder_.reset(H264Encoder::Create(
               cricket::VideoCodec(cricket::kH264CodecName)));
           decoder_.reset(H264Decoder::Create());
           break;
         case kVideoCodecVP8:
           encoder_.reset(VP8Encoder::Create());
@@ skipping 59 matching lines @@
         config_.codec_settings->VP9()->frameDroppingOn =
             process.frame_dropper_on;
         config_.codec_settings->VP9()->automaticResizeOn =
             process.spatial_resize_on;
         config_.codec_settings->VP9()->keyFrameInterval = kBaseKeyFrameInterval;
         break;
       default:
         RTC_NOTREACHED();
         break;
     }
-    frame_reader_.reset(new test::FrameReaderImpl(
+
+    // Create file objects for quality analysis.
+    analysis_frame_reader_.reset(new test::YuvFrameReaderImpl(
         config_.input_filename, config_.codec_settings->width,
         config_.codec_settings->height));
-    frame_writer_.reset(new test::FrameWriterImpl(
-        config_.output_filename, config_.frame_length_in_bytes));
-    RTC_CHECK(frame_reader_->Init());
-    RTC_CHECK(frame_writer_->Init());
+    analysis_frame_writer_.reset(new test::YuvFrameWriterImpl(
+        config_.output_filename, config_.codec_settings->width,
+        config_.codec_settings->height));
+    RTC_CHECK(analysis_frame_reader_->Init());
+    RTC_CHECK(analysis_frame_writer_->Init());
+
+    if (visualization_params) {
+      // clang-format off
+      const std::string output_filename_base =
+          test::OutputPath() + process.filename +
+          "_cd-" + CodecTypeToPayloadName(process.codec_type).value_or("") +
+          "_hw-" + std::to_string(process.hw_codec) +
+          "_fr-" + std::to_string(start_frame_rate_) +
+          "_br-" + std::to_string(static_cast<int>(start_bitrate_));
+      // clang-format on
+      if (visualization_params->save_source_y4m) {
+        source_frame_writer_.reset(new test::Y4mFrameWriterImpl(
+            output_filename_base + "_source.y4m", config_.codec_settings->width,
+            config_.codec_settings->height, start_frame_rate_));
+        RTC_CHECK(source_frame_writer_->Init());
+      }
+      if (visualization_params->save_encoded_ivf) {
+        rtc::File post_encode_file =
+            rtc::File::Create(output_filename_base + "_encoded.ivf");
+        encoded_frame_writer_ =
+            IvfFileWriter::Wrap(std::move(post_encode_file), 0);
+      }
+      if (visualization_params->save_decoded_y4m) {
+        decoded_frame_writer_.reset(new test::Y4mFrameWriterImpl(
+            output_filename_base + "_decoded.y4m",
+            config_.codec_settings->width, config_.codec_settings->height,
+            start_frame_rate_));
+        RTC_CHECK(decoded_frame_writer_->Init());
+      }
+    }
 
     packet_manipulator_.reset(new test::PacketManipulatorImpl(
         &packet_reader_, config_.networking_config, config_.verbose));
     processor_.reset(new test::VideoProcessorImpl(
-        encoder_.get(), decoder_.get(), frame_reader_.get(),
-        frame_writer_.get(), packet_manipulator_.get(), config_, &stats_));
+        encoder_.get(), decoder_.get(), analysis_frame_reader_.get(),
+        analysis_frame_writer_.get(), packet_manipulator_.get(), config_,
+        &stats_, source_frame_writer_.get(), encoded_frame_writer_.get(),
+        decoded_frame_writer_.get()));
     RTC_CHECK(processor_->Init());
   }
 
   // Reset quantities after each encoder update, update the target
   // per-frame bandwidth.
   void ResetRateControlMetrics(int num_frames) {
     for (int i = 0; i < num_temporal_layers_; i++) {
       num_frames_per_update_[i] = 0;
       sum_frame_size_mismatch_[i] = 0.0f;
       sum_encoded_frame_size_[i] = 0.0f;
@@ skipping 165 matching lines @@
     }
     if (num_temporal_layers_ == 3) {
       frame_rate_layer_[2] = frame_rate_ / 2.0f;
     }
   }
 
   // Processes all frames in the clip and verifies the result.
   void ProcessFramesAndVerify(QualityMetrics quality_metrics,
                               RateProfile rate_profile,
                               CodecConfigPars process,
-                              RateControlMetrics* rc_metrics) {
+                              RateControlMetrics* rc_metrics,
+                              const VisualizationParams* visualization_params) {
     // Codec/config settings.
     start_bitrate_ = rate_profile.target_bit_rate[0];
+    start_frame_rate_ = rate_profile.input_frame_rate[0];
     packet_loss_ = process.packet_loss;
     num_temporal_layers_ = process.num_temporal_layers;
-    SetUpCodecConfig(process);
+    SetUpCodecConfig(process, visualization_params);
     // Update the layers and the codec with the initial rates.
     bit_rate_ = rate_profile.target_bit_rate[0];
     frame_rate_ = rate_profile.input_frame_rate[0];
     SetLayerRates();
     // Set the initial target size for key frame.
     target_size_key_frame_initial_ =
         0.5 * kInitialBufferSize * bit_rate_layer_[0];
     processor_->SetRates(bit_rate_, frame_rate_);
 
     // Process each frame, up to |num_frames|.
@@ skipping 45 matching lines @@
                       rc_metrics[update_index].max_num_dropped_frames,
                       rc_metrics[update_index].num_spatial_resizes,
                       rc_metrics[update_index].num_key_frames);
     EXPECT_EQ(num_frames, frame_number);
     EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
 
     // Release encoder and decoder to make sure they have finished processing:
     EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
     EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
 
-    // Close the files before we start using them for SSIM/PSNR calculations.
-    frame_reader_->Close();
-    frame_writer_->Close();
+    // Close the analysis files before we use them for SSIM/PSNR calculations.
+    analysis_frame_reader_->Close();
+    analysis_frame_writer_->Close();
 
-    // TODO(marpan): should compute these quality metrics per SetRates update.
+    // Close visualization files.
+    if (source_frame_writer_) {
+      source_frame_writer_->Close();
+    }
+    if (encoded_frame_writer_) {
+      encoded_frame_writer_->Close();
+    }
+    if (decoded_frame_writer_) {
+      decoded_frame_writer_->Close();
+    }
+
+    // TODO(marpan): Should compute these quality metrics per SetRates update.
     test::QualityMetricsResult psnr_result, ssim_result;
     EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(),
                                             config_.output_filename.c_str(),
                                             config_.codec_settings->width,
                                             config_.codec_settings->height,
                                             &psnr_result, &ssim_result));
     printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",
            psnr_result.average, psnr_result.min, ssim_result.average,
            ssim_result.min);
     stats_.PrintSummary();
     EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr);
     EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr);
     EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim);
     EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim);
+
+    // Remove analysis file.
     if (remove(config_.output_filename.c_str()) < 0) {
       fprintf(stderr, "Failed to remove temporary file!\n");
     }
   }
 
-  static void SetRateProfilePars(RateProfile* rate_profile,
-                                 int update_index,
-                                 int bit_rate,
-                                 int frame_rate,
-                                 int frame_index_rate_update) {
-    rate_profile->target_bit_rate[update_index] = bit_rate;
-    rate_profile->input_frame_rate[update_index] = frame_rate;
-    rate_profile->frame_index_rate_update[update_index] =
-        frame_index_rate_update;
-  }
-
   static void SetCodecParameters(CodecConfigPars* process_settings,
                                  VideoCodecType codec_type,
                                  bool hw_codec,
                                  bool use_single_core,
                                  float packet_loss,
                                  int key_frame_interval,
                                  int num_temporal_layers,
                                  bool error_concealment_on,
                                  bool denoising_on,
                                  bool frame_dropper_on,
@@ skipping 40 matching lines @@
                                 double minimum_avg_psnr,
                                 double minimum_min_psnr,
                                 double minimum_avg_ssim,
                                 double minimum_min_ssim) {
     quality_metrics->minimum_avg_psnr = minimum_avg_psnr;
     quality_metrics->minimum_min_psnr = minimum_min_psnr;
     quality_metrics->minimum_avg_ssim = minimum_avg_ssim;
     quality_metrics->minimum_min_ssim = minimum_min_ssim;
   }
 
+  static void SetRateProfilePars(RateProfile* rate_profile,
+                                 int update_index,
+                                 int bit_rate,
+                                 int frame_rate,
+                                 int frame_index_rate_update) {
+    rate_profile->target_bit_rate[update_index] = bit_rate;
+    rate_profile->input_frame_rate[update_index] = frame_rate;
+    rate_profile->frame_index_rate_update[update_index] =
+        frame_index_rate_update;
+  }
+
   static void SetRateControlMetrics(RateControlMetrics* rc_metrics,
                                     int update_index,
                                     int max_num_dropped_frames,
                                     int max_key_frame_size_mismatch,
                                     int max_delta_frame_size_mismatch,
                                     int max_encoding_rate_mismatch,
                                     int max_time_hit_target,
                                     int num_spatial_resizes,
                                     int num_key_frames) {
     rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames;
     rc_metrics[update_index].max_key_frame_size_mismatch =
         max_key_frame_size_mismatch;
     rc_metrics[update_index].max_delta_frame_size_mismatch =
         max_delta_frame_size_mismatch;
     rc_metrics[update_index].max_encoding_rate_mismatch =
         max_encoding_rate_mismatch;
     rc_metrics[update_index].max_time_hit_target = max_time_hit_target;
     rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes;
     rc_metrics[update_index].num_key_frames = num_key_frames;
   }
 
+  // Codecs.
   std::unique_ptr<VideoEncoder> encoder_;
   std::unique_ptr<cricket::WebRtcVideoEncoderFactory> external_encoder_factory_;
   std::unique_ptr<VideoDecoder> decoder_;
   std::unique_ptr<cricket::WebRtcVideoDecoderFactory> external_decoder_factory_;
-  std::unique_ptr<test::FrameReader> frame_reader_;
-  std::unique_ptr<test::FrameWriter> frame_writer_;
+  VideoCodec codec_settings_;
+
+  // Helper objects.
+  std::unique_ptr<test::FrameReader> analysis_frame_reader_;
+  std::unique_ptr<test::FrameWriter> analysis_frame_writer_;
   test::PacketReader packet_reader_;
   std::unique_ptr<test::PacketManipulator> packet_manipulator_;
   test::Stats stats_;
   test::TestConfig config_;
-  VideoCodec codec_settings_;
   // Must be destroyed before |encoder_| and |decoder_|.
   std::unique_ptr<test::VideoProcessor> processor_;
-  TemporalLayersFactory tl_factory_;
+
+  // Visualization objects.
+  std::unique_ptr<test::FrameWriter> source_frame_writer_;
+  std::unique_ptr<IvfFileWriter> encoded_frame_writer_;
+  std::unique_ptr<test::FrameWriter> decoded_frame_writer_;
 
   // Quantities defined/updated for every encoder rate update.
   // Some quantities defined per temporal layer (at most 3 layers in this test).
   int num_frames_per_update_[3];
   float sum_frame_size_mismatch_[3];
   float sum_encoded_frame_size_[3];
   float encoding_bitrate_[3];
   float per_frame_bandwidth_[3];
   float bit_rate_layer_[3];
   float frame_rate_layer_[3];
   int num_frames_total_;
   float sum_encoded_frame_size_total_;
   float encoding_bitrate_total_;
   float perc_encoding_rate_mismatch_;
   int num_frames_to_hit_target_;
   bool encoding_rate_within_target_;
   int bit_rate_;
   int frame_rate_;
   int layer_;
   float target_size_key_frame_initial_;
   float target_size_key_frame_;
   float sum_key_frame_size_mismatch_;
   int num_key_frames_;
   float start_bitrate_;
+  int start_frame_rate_;
 
   // Codec and network settings.
   float packet_loss_;
   int num_temporal_layers_;
 };
 
 }  // namespace test
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H_