Full stack graphs

webrtc/video/full_stack.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
 
 #include <deque>
 #include <map>
 
 #include "testing/gtest/include/gtest/gtest.h"
 
+#include "webrtc/base/format_macros.h"
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/base/thread_annotations.h"
 #include "webrtc/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/frame_callback.h"
 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
 #include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/system_wrappers/interface/cpu_info.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/event_wrapper.h"
 #include "webrtc/system_wrappers/interface/sleep.h"
-#include "webrtc/test/call_test.h"
-#include "webrtc/test/direct_transport.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_encoder.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/frame_generator_capturer.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/typedefs.h"
+#include "webrtc/video/full_stack.h"
 
 namespace webrtc {
 
 static const int kFullStackTestDurationSecs = 60;
 static const int kSendStatsPollingIntervalMs = 1000;
 
-struct FullStackTestParams {
-  const char* test_label;
-  struct {
-    const char* name;
-    size_t width, height;
-    int fps;
-  } clip;
-  bool screenshare;
-  int min_bitrate_bps;
-  int target_bitrate_bps;
-  int max_bitrate_bps;
-  double avg_psnr_threshold;
-  double avg_ssim_threshold;
-  int test_durations_secs;
-  std::string codec;
-  FakeNetworkPipe::Config link;
-};
-
-class FullStackTest : public test::CallTest {
- protected:
-  void RunTest(const FullStackTestParams& params);
-};
-
 class VideoAnalyzer : public PacketReceiver,
                       public newapi::Transport,
                       public VideoRenderer,
                       public VideoCaptureInput,
                       public EncodedFrameObserver {
  public:
   VideoAnalyzer(VideoCaptureInput* input,
                 Transport* transport,
                 const char* test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
-                int duration_frames)
+                int duration_frames,
+                const std::string& stats_output_filename)
       : input_(input),
         transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         test_label_(test_label),
+        stats_output_filename_(stats_output_filename),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         last_render_time_(0),
         rtp_timestamp_delta_(0),
         avg_psnr_threshold_(avg_psnr_threshold),
         avg_ssim_threshold_(avg_ssim_threshold),
         comparison_available_event_(EventWrapper::Create()),
         done_(EventWrapper::Create()) {
@@ skipping 71 matching lines @@
     RTPHeader header;
     parser->Parse(packet, length, &header);
 
     {
       rtc::CritScope lock(&crit_);
       if (rtp_timestamp_delta_ == 0) {
         rtp_timestamp_delta_ =
             header.timestamp - first_send_frame_.timestamp();
         first_send_frame_.Reset();
       }
-      send_times_[header.timestamp - rtp_timestamp_delta_] =
+      uint32_t timestamp = header.timestamp - rtp_timestamp_delta_;
+      send_times_[timestamp] =
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
+      encoded_frame_sizes_[timestamp] +=
+          length - (header.headerLength + header.paddingLength);

[sprang_webrtc, 2015/09/03 08:59:39]

Do we ever use simulcast in these tests? Then this could be racy?
For VP9 svc, this is the combined size of all spatial layers for this frame?

[ivica, 2015/09/03 09:30:34]

Simulcast is not supported here yet. That will be another CL.

[sprang_webrtc, 2015/09/03 11:53:22]

Acknowledged.

     }
 
     return transport_->SendRtp(packet, length);
   }
 
   bool SendRtcp(const uint8_t* packet, size_t length) override {
     return transport_->SendRtcp(packet, length);
   }
 
   void EncodedFrameCallback(const EncodedFrame& frame) override {
     rtc::CritScope lock(&comparison_lock_);
-    if (frames_recorded_ < frames_to_process_)
+    if (frames_recorded_ < frames_to_process_) {
       encoded_frame_size_.AddSample(frame.length_);
+    }

[sprang_webrtc, 2015/09/03 08:59:39]

Don't think you need these extra {}

[ivica, 2015/09/03 09:30:34]

Done.

   }
 
   void RenderFrame(const VideoFrame& video_frame,
                    int time_to_render_ms) override {
     int64_t render_time_ms =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
 
     rtc::CritScope lock(&crit_);
 
@@ skipping 47 matching lines @@
   }
 
   VideoCaptureInput* input_;
   Transport* transport_;
   PacketReceiver* receiver_;
   VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
     FrameComparison()
-        : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
+        : dropped(false),
+          send_time_ms(0),
+          recv_time_ms(0),
+          render_time_ms(0),
+          encoded_frame_size(0) {}
 
     FrameComparison(const VideoFrame& reference,
                     const VideoFrame& render,
                     bool dropped,
                     int64_t send_time_ms,
                     int64_t recv_time_ms,
-                    int64_t render_time_ms)
+                    int64_t render_time_ms,
+                    size_t encoded_frame_size)
         : reference(reference),
           render(render),
           dropped(dropped),
           send_time_ms(send_time_ms),
           recv_time_ms(recv_time_ms),
-          render_time_ms(render_time_ms) {}
+          render_time_ms(render_time_ms),
+          encoded_frame_size(encoded_frame_size) {}
 
     VideoFrame reference;
     VideoFrame render;
     bool dropped;
     int64_t send_time_ms;
     int64_t recv_time_ms;
     int64_t render_time_ms;
+    size_t encoded_frame_size;
+  };
+
+  struct Sample {
+    Sample(double dropped,
+           double input_time_ms,
+           double send_time_ms,
+           double recv_time_ms,
+           double encoded_frame_size,
+           double psnr,
+           double ssim,
+           double decoded_time_ms)
+        : dropped(dropped),
+          input_time_ms(input_time_ms),
+          send_time_ms(send_time_ms),
+          recv_time_ms(recv_time_ms),
+          encoded_frame_size(encoded_frame_size),
+          psnr(psnr),
+          ssim(ssim),
+          decoded_time_ms(decoded_time_ms) {}
+
+    double dropped;
+    double input_time_ms;
+    double send_time_ms;
+    double recv_time_ms;
+    double encoded_frame_size;
+    double psnr;
+    double ssim;
+    double decoded_time_ms;
   };
 
   void AddFrameComparison(const VideoFrame& reference,
                           const VideoFrame& render,
                           bool dropped,
                           int64_t render_time_ms)
       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
     int64_t send_time_ms = send_times_[reference.timestamp()];
     send_times_.erase(reference.timestamp());
     int64_t recv_time_ms = recv_times_[reference.timestamp()];
     recv_times_.erase(reference.timestamp());
 
+    size_t encoded_size = encoded_frame_sizes_[reference.timestamp()];
+    encoded_frame_sizes_.erase(reference.timestamp());
+
+    VideoFrame reference_copy;
+    VideoFrame render_copy;
+    reference_copy.CopyFrame(reference);
+    render_copy.CopyFrame(render);
+
     rtc::CritScope crit(&comparison_lock_);
-    comparisons_.push_back(FrameComparison(reference, render, dropped,
+    comparisons_.push_back(FrameComparison(reference_copy, render_copy, dropped,
                                            send_time_ms, recv_time_ms,
-                                           render_time_ms));
+                                           render_time_ms, encoded_size));
     comparison_available_event_->Set();
   }
 
   static bool PollStatsThread(void* obj) {
     return static_cast<VideoAnalyzer*>(obj)->PollStats();
   }
 
   bool PollStats() {
     switch (done_->Wait(kSendStatsPollingIntervalMs)) {
       case kEventSignaled:
@@ skipping 37 matching lines @@
         comparison_available_event_->Set();
         return false;
       }
       return true;  // Try again.
     }
 
     PerformFrameComparison(comparison);
 
     if (FrameProcessed()) {
       PrintResults();
+      if (!stats_output_filename_.empty())
+        PrintSamplesToFile();
       done_->Set();
       comparison_available_event_->Set();
       return false;
     }
 
     return true;
   }
 
   bool PopComparison(FrameComparison* comparison) {
     rtc::CritScope crit(&comparison_lock_);
@@ skipping 52 matching lines @@
 
     EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
     EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = I420PSNR(&comparison.reference, &comparison.render);
     double ssim = I420SSIM(&comparison.reference, &comparison.render);
 
+    int64_t input_time_ms = comparison.reference.ntp_time_ms();
+    Sample sample(comparison.dropped, input_time_ms, comparison.send_time_ms,
+                  comparison.recv_time_ms, comparison.encoded_frame_size, psnr,
+                  ssim,
+                  comparison.render_time_ms);  // decoded_time_ms

[sprang_webrtc, 2015/09/03 08:59:39]

Why use a temporary?

[ivica, 2015/09/03 09:30:34]

Done.

+
     rtc::CritScope crit(&comparison_lock_);
+    if (!stats_output_filename_.empty())
+      samples_.push_back(sample);
     psnr_.AddSample(psnr);
     ssim_.AddSample(ssim);
+
     if (comparison.dropped) {
       ++dropped_frames_;
       return;
     }
     if (last_render_time_ != 0)
       rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
     last_render_time_ = comparison.render_time_ms;
 
-    int64_t input_time_ms = comparison.reference.ntp_time_ms();
     sender_time_.AddSample(comparison.send_time_ms - input_time_ms);
     receiver_time_.AddSample(comparison.render_time_ms -
                              comparison.recv_time_ms);
     end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);
+    encoded_frame_size_.AddSample(comparison.encoded_frame_size);
   }
 
   void PrintResult(const char* result_type,
                    test::Statistics stats,
                    const char* unit) {
     printf("RESULT %s: %s = {%f, %f}%s\n",
            result_type,
            test_label_,
            stats.Mean(),
            stats.StandardDeviation(),
            unit);
   }
 
+  void PrintSamplesToFile(void) {
+    FILE* f = fopen(stats_output_filename_.c_str(), "w");
+    if (f == NULL) {
+      fprintf(stderr, "Couldn't open file %s\n",
+              stats_output_filename_.c_str());
+      abort();
+    }
+
+    rtc::CritScope crit(&comparison_lock_);
+    std::sort(samples_.begin(), samples_.end(),
+              [](const Sample& A, const Sample& B)
+                  -> bool { return A.input_time_ms < B.input_time_ms; });
+
+    fprintf(f, "%s\n", test_label_);
+    fprintf(f, "%" PRIuS "\n", samples_.size());
+    fprintf(f,
+            "dropped "
+            "input_time_ms "
+            "send_time_ms "
+            "recv_time_ms "
+            "encoded_frame_size "
+            "psnr "
+            "ssim "
+            "decoded_time_ms\n");
+    for (Sample& sample : samples_) {

[sprang_webrtc, 2015/09/03 08:59:39]

const Sample&

[ivica, 2015/09/03 09:30:34]

Done.

+      fprintf(f, "%lf %lf %lf %lf %lf %lf %lf %lf\n", sample.dropped,
+              sample.input_time_ms, sample.send_time_ms, sample.recv_time_ms,
+              sample.encoded_frame_size, sample.psnr, sample.ssim,
+              sample.decoded_time_ms);
+    }
+    fclose(f);
+  }
+
   const char* const test_label_;
+  std::string stats_output_filename_;
+  std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);
   test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
   test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
   test::Statistics psnr_ GUARDED_BY(comparison_lock_);
   test::Statistics ssim_ GUARDED_BY(comparison_lock_);
   test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
   test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
   test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_time_ms GUARDED_BY(comparison_lock_);
   test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_);
   test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_);
 
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
   int dropped_frames_;
   int64_t last_render_time_;
   uint32_t rtp_timestamp_delta_;
 
   rtc::CriticalSection crit_;
   std::deque<VideoFrame> frames_ GUARDED_BY(crit_);
+  std::deque<VideoSendStream::Stats> send_stats_ GUARDED_BY(crit_);
   VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
+  std::map<uint32_t, size_t> encoded_frame_sizes_ GUARDED_BY(crit_);
   VideoFrame first_send_frame_ GUARDED_BY(crit_);
   const double avg_psnr_threshold_;
   const double avg_ssim_threshold_;
 
   rtc::CriticalSection comparison_lock_;
   std::vector<ThreadWrapper*> comparison_thread_pool_;
   rtc::scoped_ptr<ThreadWrapper> stats_polling_thread_;
   const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   const rtc::scoped_ptr<EventWrapper> done_;
 };
 
 void FullStackTest::RunTest(const FullStackTestParams& params) {
+  // TODO(ivica): Add num_temporal_layers as a param.
+  unsigned char num_temporal_layers =
+      params.stats_output_filename.empty() ? 2 : 1;
+
   test::DirectTransport send_transport(params.link);
   test::DirectTransport recv_transport(params.link);
   VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label,
                          params.avg_psnr_threshold, params.avg_ssim_threshold,
-                         params.test_durations_secs * params.clip.fps);
+                         params.test_durations_secs * params.clip.fps,
+                         params.stats_output_filename);
 
   CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
 
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   CreateSendConfig(1);
 
   rtc::scoped_ptr<VideoEncoder> encoder;
@@ skipping 27 matching lines @@
 
   VideoCodecVP8 vp8_settings;
   VideoCodecVP9 vp9_settings;
   if (params.screenshare) {
     encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen;
     encoder_config_.min_transmit_bitrate_bps = 400 * 1000;
     if (params.codec == "VP8") {
       vp8_settings = VideoEncoder::GetDefaultVp8Settings();
       vp8_settings.denoisingOn = false;
       vp8_settings.frameDroppingOn = false;
-      vp8_settings.numberOfTemporalLayers = 2;
+      vp8_settings.numberOfTemporalLayers = num_temporal_layers;
       encoder_config_.encoder_specific_settings = &vp8_settings;
     } else if (params.codec == "VP9") {
       vp9_settings = VideoEncoder::GetDefaultVp9Settings();
       vp9_settings.denoisingOn = false;
       vp9_settings.frameDroppingOn = false;
-      vp9_settings.numberOfTemporalLayers = 2;
+      vp9_settings.numberOfTemporalLayers = num_temporal_layers;
       encoder_config_.encoder_specific_settings = &vp9_settings;
     }
 
     stream->temporal_layer_thresholds_bps.clear();
-    stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);
+    if (num_temporal_layers > 1) {
+      stream->temporal_layer_thresholds_bps.push_back(
+          stream->target_bitrate_bps);
+    }
   }
 
   CreateMatchingReceiveConfigs();
   receive_configs_[0].renderer = &analyzer;
   receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
       kSendRtxPayloadType;
 
   for (auto& config : receive_configs_)
@@ skipping 199 matching lines @@
       50000,
       200000,
       2000000,
       0.0,
       0.0,
       kFullStackTestDurationSecs,
       "VP9"};
   RunTest(screenshare_params);
 }
 }  // namespace webrtc