Report minimum PSNR in VideoQualityTest and save corresponding frame to file

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_quality_test.h"
@@ skipping 27 matching lines @@
 #include "webrtc/rtc_base/optional.h"
 #include "webrtc/rtc_base/platform_file.h"
 #include "webrtc/rtc_base/timeutils.h"
 #include "webrtc/system_wrappers/include/cpu_info.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/test/layer_filtering_transport.h"
 #include "webrtc/test/run_loop.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
+#include "webrtc/test/testsupport/frame_writer.h"
 #include "webrtc/test/vcm_capturer.h"
 #include "webrtc/test/video_renderer.h"
 #include "webrtc/voice_engine/include/voe_base.h"
 
 #include "webrtc/test/rtp_file_writer.h"
 
 namespace {
 
 constexpr int kSendStatsPollingIntervalMs = 1000;
 
@@ skipping 738 matching lines @@
     PrintResult("sender_time", sender_time_, " ms");
     PrintResult("receiver_time", receiver_time_, " ms");
     PrintResult("total_delay_incl_network", end_to_end_, " ms");
     PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
     PrintResult("encode_frame_rate", encode_frame_rate_, " fps");
     PrintResult("encode_time", encode_time_ms_, " ms");
     PrintResult("media_bitrate", media_bitrate_bps_, " bps");
     PrintResult("fec_bitrate", fec_bitrate_bps_, " bps");
     PrintResult("send_bandwidth", send_bandwidth_bps_, " bps");
 
+    if (worst_frame_) {
+      printf("RESULT min_psnr: %s = %lf dB\n", test_label_.c_str(),
+             worst_frame_->psnr);
+    }
+
     if (receive_stream_ != nullptr) {
       PrintResult("decode_time", decode_time_ms_, " ms");
     }
 
     printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(),
            dropped_frames_);
     printf("RESULT cpu_usage: %s = %lf %%\n", test_label_.c_str(),
            GetCpuUsagePercent());
 
 #if defined(WEBRTC_WIN)
       // On Linux and Mac in Resident Set some unused pages may be counted.
       // Therefore this metric will depend on order in which tests are run and
       // will be flaky.
     PrintResult("memory_usage", memory_usage_, " bytes");
 #endif
+    // TODO(ilnik): enable frame writing for android, once jpeg frame writer
+    // is implemented.
+
+#if !defined(WEBRTC_ANDROID)
+    if (worst_frame_) {
+      test::Y4mFrameWriterImpl frame_writer(test_label_ + ".y4m",

[stefan-webrtc, 2017/08/22 11:32:18]

Isn't it better to write this to some directory other than the root? I think
it's a bit polluting right now. Maybe webrtc/out/?

[ilnik, 2017/08/22 12:07:42]

Yes, this problem is acknowledged. As Erik stated, it's being fixed in
https://codereview.webrtc.org/2998143002/ : directory is now configurable from
the command-line (/out by default), special flag is required to enable the
saving in the first place, and files are much smaller.

+                                            worst_frame_->frame.width(),
+                                            worst_frame_->frame.height(), 1);
+      bool res = frame_writer.Init();
+      RTC_DCHECK(res);
+      size_t length =
+          CalcBufferSize(VideoType::kI420, worst_frame_->frame.width(),
+                         worst_frame_->frame.height());
+      rtc::Buffer extracted_buffer(length);
+      size_t extracted_length =
+          ExtractBuffer(worst_frame_->frame.video_frame_buffer()->ToI420(),
+                        length, extracted_buffer.data());
+      RTC_DCHECK_EQ(extracted_length, frame_writer.FrameLength());
+      res = frame_writer.WriteFrame(extracted_buffer.data());
+      RTC_DCHECK(res);
+      frame_writer.Close();
+    }
+#endif
 
     //  Disable quality check for quick test, as quality checks may fail
     //  because too few samples were collected.
     if (!is_quick_test_enabled_) {
       EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
       EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
     }
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = -1.0;
     double ssim = -1.0;
     if (comparison.reference && !comparison.dropped) {
       psnr = I420PSNR(&*comparison.reference, &*comparison.render);
       ssim = I420SSIM(&*comparison.reference, &*comparison.render);
     }
 
     rtc::CritScope crit(&comparison_lock_);
+
+    if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) {
+      worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render});
+    }
+
     if (graph_data_output_file_) {
       samples_.push_back(Sample(
           comparison.dropped, comparison.input_time_ms, comparison.send_time_ms,
           comparison.recv_time_ms, comparison.render_time_ms,
           comparison.encoded_frame_size, psnr, ssim));
     }
     if (psnr >= 0.0)
       psnr_.AddSample(psnr);
     if (ssim >= 0.0)
       ssim_.AddSample(ssim);
@@ skipping 183 matching lines @@
   test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_usage_percent_ GUARDED_BY(comparison_lock_);
   test::Statistics decode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics decode_time_max_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics media_bitrate_bps_ GUARDED_BY(comparison_lock_);
   test::Statistics fec_bitrate_bps_ GUARDED_BY(comparison_lock_);
   test::Statistics send_bandwidth_bps_ GUARDED_BY(comparison_lock_);
   test::Statistics memory_usage_ GUARDED_BY(comparison_lock_);
 
+  struct FrameWithPsnr {
+    double psnr;
+    VideoFrame frame;
+  };
+
+  // Rendered frame with worst PSNR is saved for further analysis.
+  rtc::Optional<FrameWithPsnr> worst_frame_ GUARDED_BY(comparison_lock_);
+
   size_t last_fec_bytes_;
 
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
   int dropped_frames_;
   int dropped_frames_before_first_encode_;
   int dropped_frames_before_rendering_;
   int64_t last_render_time_;
   uint32_t rtp_timestamp_delta_;
@@ skipping 1029 matching lines @@
   if (!params_.logging.encoded_frame_base_path.empty()) {
     std::ostringstream str;
     str << receive_logs_++;
     std::string path =
         params_.logging.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
     stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
                                         100000000);
   }
 }
 }  // namespace webrtc