Collecting encode_time_ms for each frame

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
@@ skipping 10 matching lines @@
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
 #include "webrtc/system_wrappers/interface/cpu_info.h"
 #include "webrtc/test/layer_filtering_transport.h"
 #include "webrtc/test/run_loop.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/video_renderer.h"
+#include "webrtc/video/video_capture_input.h"
 #include "webrtc/video/video_quality_test.h"
 
 namespace webrtc {
 
 static const int kTransportSeqExtensionId =
     VideoQualityTest::kAbsSendTimeExtensionId + 1;
 static const int kSendStatsPollingIntervalMs = 1000;
 static const int kPayloadTypeVP8 = 123;
 static const int kPayloadTypeVP9 = 124;
 
 class VideoAnalyzer : public PacketReceiver,
                       public Transport,
                       public VideoRenderer,
                       public VideoCaptureInput,
-                      public EncodedFrameObserver {
+                      public EncodedFrameObserver,
+                      public EncodingTimeObserver {
  public:
-  VideoAnalyzer(VideoCaptureInput* input,
-                Transport* transport,
+  VideoAnalyzer(Transport* transport,
                 const std::string& test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames,
                 FILE* graph_data_output_file)
-      : input_(input),
+      : input_(nullptr),

[pbos-webrtc, 2015/09/30 12:48:31]

When is this set/used?

[ivica, 2015/09/30 13:18:04]

Line 789 on the right side. The argument input was nullptr anyway, so I removed
it.

         transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         test_label_(test_label),
         graph_data_output_file_(graph_data_output_file),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         last_render_time_(0),
@@ skipping 51 matching lines @@
     parser->Parse(packet, length, &header);
     {
       rtc::CritScope lock(&crit_);
       recv_times_[header.timestamp - rtp_timestamp_delta_] =
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     }
 
     return receiver_->DeliverPacket(media_type, packet, length, packet_time);
   }
 
+  // EncodingTimeObserver.
+  void OnEncodedFrame(const VideoFrame& frame, int encode_time_ms) {

[pbos-webrtc, 2015/09/30 12:48:31]

override, and imo no need for the comment

[ivica, 2015/09/30 13:18:04]

VideoAnalyzer is full of random callbacks, it is really difficult to keep track
of which callback is from which interface. I'd like to keep it if that's fine
with you.

+    rtc::CritScope crit(&comparison_lock_);
+    samples_encode_time_ms_[frame.ntp_time_ms()] = encode_time_ms;
+  }
+
   void IncomingCapturedFrame(const VideoFrame& video_frame) override {
     VideoFrame copy = video_frame;
     copy.set_timestamp(copy.ntp_time_ms() * 90);
 
     {
       rtc::CritScope lock(&crit_);
       if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)
         first_send_frame_ = copy;
 
       frames_.push_back(copy);
@@ skipping 134 matching lines @@
     VideoFrame reference;
     VideoFrame render;
     bool dropped;
     int64_t send_time_ms;
     int64_t recv_time_ms;
     int64_t render_time_ms;
     size_t encoded_frame_size;
   };
 
   struct Sample {
-    Sample(double dropped,
-           double input_time_ms,
-           double send_time_ms,
-           double recv_time_ms,
-           double encoded_frame_size,
+    Sample(int dropped,
+           int64_t input_time_ms,
+           int64_t send_time_ms,
+           int64_t recv_time_ms,
+           int64_t render_time_ms,
+           size_t encoded_frame_size,
            double psnr,
-           double ssim,
-           double render_time_ms)
+           double ssim)
         : dropped(dropped),
           input_time_ms(input_time_ms),
           send_time_ms(send_time_ms),
           recv_time_ms(recv_time_ms),
+          render_time_ms(render_time_ms),
           encoded_frame_size(encoded_frame_size),
           psnr(psnr),
-          ssim(ssim),
-          render_time_ms(render_time_ms) {}
+          ssim(ssim) {}
 
-    double dropped;
-    double input_time_ms;
-    double send_time_ms;
-    double recv_time_ms;
-    double encoded_frame_size;
+    int dropped;
+    int64_t input_time_ms;
+    int64_t send_time_ms;
+    int64_t recv_time_ms;
+    int64_t render_time_ms;
+    size_t encoded_frame_size;
     double psnr;
     double ssim;
-    double render_time_ms;
   };
 
   void AddFrameComparison(const VideoFrame& reference,
                           const VideoFrame& render,
                           bool dropped,
                           int64_t render_time_ms)
       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
     int64_t send_time_ms = send_times_[reference.timestamp()];
     send_times_.erase(reference.timestamp());
     int64_t recv_time_ms = recv_times_[reference.timestamp()];
@@ skipping 141 matching lines @@
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = I420PSNR(&comparison.reference, &comparison.render);
     double ssim = I420SSIM(&comparison.reference, &comparison.render);
 
     int64_t input_time_ms = comparison.reference.ntp_time_ms();
 
     rtc::CritScope crit(&comparison_lock_);
     if (graph_data_output_file_) {
       samples_.push_back(
           Sample(comparison.dropped, input_time_ms, comparison.send_time_ms,
-                 comparison.recv_time_ms, comparison.encoded_frame_size, psnr,
-                 ssim, comparison.render_time_ms));
+                 comparison.recv_time_ms, comparison.render_time_ms,
+                 comparison.encoded_frame_size, psnr, ssim));
     }
     psnr_.AddSample(psnr);
     ssim_.AddSample(ssim);
 
     if (comparison.dropped) {
       ++dropped_frames_;
       return;
     }
     if (last_render_time_ != 0)
       rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
@@ skipping 25 matching lines @@
                 return A.input_time_ms < B.input_time_ms;
               });
 
     fprintf(out, "%s\n", test_label_.c_str());
     fprintf(out, "%" PRIuS "\n", samples_.size());
     fprintf(out,
             "dropped "
             "input_time_ms "
             "send_time_ms "
             "recv_time_ms "
+            "render_time_ms "
             "encoded_frame_size "
             "psnr "
             "ssim "
-            "render_time_ms\n");
-    for (const Sample& sample : samples_) {
-      fprintf(out, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped,
-              sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms,
+            "encode_time_ms\n");
+    int missing_encode_time_samples = 0;
+    for (size_t i = 0; i < samples_.size(); ++i) {
+      const Sample& sample = samples_[i];
+      auto it = samples_encode_time_ms_.find(sample.input_time_ms);
+      int encode_time_ms;
+      if (it != samples_encode_time_ms_.end()) {
+        encode_time_ms = it->second;
+      } else {
+        ++missing_encode_time_samples;
+        encode_time_ms = -1;
+      }
+      fprintf(out, "%d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRIuS
+                   " %lf %lf %d\n",
+              sample.dropped, sample.input_time_ms, sample.send_time_ms,
+              sample.recv_time_ms, sample.render_time_ms,
               sample.encoded_frame_size, sample.psnr, sample.ssim,
-              sample.render_time_ms);
+              encode_time_ms);
+    }
+    if (missing_encode_time_samples) {
+      fprintf(stderr,
+              "Warning: Missing encode_time_ms samples for %d frame(s).\n",
+              missing_encode_time_samples);
     }
   }
 
   const std::string test_label_;
   FILE* const graph_data_output_file_;
   std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);
+  std::map<int64_t, int> samples_encode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
   test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
   test::Statistics psnr_ GUARDED_BY(comparison_lock_);
   test::Statistics ssim_ GUARDED_BY(comparison_lock_);
   test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
   test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
   test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_time_ms GUARDED_BY(comparison_lock_);
   test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_);
@@ skipping 190 matching lines @@
     RTC_CHECK(graph_data_output_file != nullptr)
         << "Can't open the file "
         << params.analyzer.graph_data_output_filename << "!";
   }
 
   test::LayerFilteringTransport send_transport(
       params.pipe, kPayloadTypeVP8, kPayloadTypeVP9,
       static_cast<uint8_t>(params.common.tl_discard_threshold), 0);
   test::DirectTransport recv_transport(params.pipe);
   VideoAnalyzer analyzer(
-      nullptr, &send_transport, params.analyzer.test_label,
+      &send_transport, params.analyzer.test_label,
       params.analyzer.avg_psnr_threshold, params.analyzer.avg_ssim_threshold,
       params.analyzer.test_durations_secs * params.common.fps,
       graph_data_output_file);
 
   Call::Config call_config;
   call_config.bitrate_config = params.common.call_bitrate_config;
   CreateCalls(call_config, call_config);
 
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupFullStack(params, &analyzer, &recv_transport);
+  send_config_.encoding_time_observer = &analyzer;
   receive_configs_[0].renderer = &analyzer;
   for (auto& config : receive_configs_)
     config.pre_decode_callback = &analyzer;
 
   if (params.screenshare.enabled)
     SetupScreenshare(params);
 
   CreateCapturer(params, &analyzer);
 
   CreateStreams();
@@ skipping 68 matching lines @@
   send_stream_->Stop();
   receive_stream->Stop();
 
   call->DestroyVideoReceiveStream(receive_stream);
   call->DestroyVideoSendStream(send_stream_);
 
   transport.StopSending();
 }
 
 }  // namespace webrtc