Full stack graphs

webrtc/video/full_stack.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
 
 #include <deque>
 #include <map>
 
 #include "testing/gtest/include/gtest/gtest.h"
 
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/base/thread_annotations.h"
 #include "webrtc/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/frame_callback.h"
 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
 #include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/system_wrappers/interface/cpu_info.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/event_wrapper.h"
 #include "webrtc/system_wrappers/interface/sleep.h"
-#include "webrtc/test/call_test.h"
-#include "webrtc/test/direct_transport.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_encoder.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/frame_generator_capturer.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/typedefs.h"
+#include "webrtc/video/full_stack.h"
 
 namespace webrtc {
 
 static const int kFullStackTestDurationSecs = 60;
 static const int kSendStatsPollingIntervalMs = 1000;
 
-struct FullStackTestParams {
-  const char* test_label;
-  struct {
-    const char* name;
-    size_t width, height;
-    int fps;
-  } clip;
-  bool screenshare;
-  int min_bitrate_bps;
-  int target_bitrate_bps;
-  int max_bitrate_bps;
-  double avg_psnr_threshold;
-  double avg_ssim_threshold;
-  int test_durations_secs;
-  std::string codec;
-  FakeNetworkPipe::Config link;
-};
-
-class FullStackTest : public test::CallTest {
- protected:
-  void RunTest(const FullStackTestParams& params);
-};
-
 class VideoAnalyzer : public PacketReceiver,
                       public newapi::Transport,
                       public VideoRenderer,
                       public VideoCaptureInput,
                       public EncodedFrameObserver {
  public:
   VideoAnalyzer(VideoCaptureInput* input,
                 Transport* transport,
                 const char* test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
-                int duration_frames)
+                int duration_frames,
+                const std::string& graph_data_filename)
       : input_(input),
         transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         test_label_(test_label),
+        save_samples_(!graph_data_filename.empty()),
+        graph_data_filename_(graph_data_filename),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         last_render_time_(0),
         rtp_timestamp_delta_(0),
         avg_psnr_threshold_(avg_psnr_threshold),
         avg_ssim_threshold_(avg_ssim_threshold),
         comparison_available_event_(EventWrapper::Create()),
         done_(EventWrapper::Create()) {
@@ skipping 48 matching lines @@
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     }
 
     return receiver_->DeliverPacket(media_type, packet, length);
   }
 
   void IncomingCapturedFrame(const VideoFrame& video_frame) override {
     VideoFrame copy = video_frame;
     copy.set_timestamp(copy.ntp_time_ms() * 90);
 
+    VideoSendStream::Stats stats = send_stream_->GetStats();
     {
       rtc::CritScope lock(&crit_);
       if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)
         first_send_frame_ = copy;
 
       frames_.push_back(copy);
+      send_stats_.push_back(stats);
     }
 
     input_->IncomingCapturedFrame(video_frame);
   }
 
   bool SendRtp(const uint8_t* packet, size_t length) override {
     rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
     RTPHeader header;
     parser->Parse(packet, length, &header);
 
     {
       rtc::CritScope lock(&crit_);
       if (rtp_timestamp_delta_ == 0) {
         rtp_timestamp_delta_ =
             header.timestamp - first_send_frame_.timestamp();
         first_send_frame_.Reset();
       }
-      send_times_[header.timestamp - rtp_timestamp_delta_] =
+      uint32_t timestamp = header.timestamp - rtp_timestamp_delta_;
+      send_times_[timestamp] =
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
+      encoded_frame_sizes_[timestamp] +=
+          length - (header.headerLength + header.paddingLength);

[sprang_webrtc, 2015/08/19 13:33:37]

Rebase error? This is handles by EncodedFrameCallback() now.

[ivica, 2015/08/20 13:25:59]

For graphs I need encoded frame size for each frame separately. Is there any
better way to gather that information?

     }
 
     return transport_->SendRtp(packet, length);
   }
 
   bool SendRtcp(const uint8_t* packet, size_t length) override {
     return transport_->SendRtcp(packet, length);
   }
 
   void EncodedFrameCallback(const EncodedFrame& frame) override {
     rtc::CritScope lock(&comparison_lock_);
     if (frames_recorded_ < frames_to_process_)
       encoded_frame_size_.AddSample(frame.length_);
   }
 
   void RenderFrame(const VideoFrame& video_frame,
                    int time_to_render_ms) override {
     int64_t render_time_ms =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
 
     rtc::CritScope lock(&crit_);
 
     while (frames_.front().timestamp() < send_timestamp) {
       AddFrameComparison(frames_.front(), last_rendered_frame_, true,
-                         render_time_ms);
+                         render_time_ms, send_stats_.front());
       frames_.pop_front();
+      send_stats_.pop_front();
     }
 
     VideoFrame reference_frame = frames_.front();
     frames_.pop_front();
     assert(!reference_frame.IsZeroSize());
     EXPECT_EQ(reference_frame.timestamp(), send_timestamp);
     assert(reference_frame.timestamp() == send_timestamp);
 
-    AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
+    AddFrameComparison(reference_frame, video_frame, false, render_time_ms,
+                       send_stats_.front());
+    send_stats_.pop_front();
 
     last_rendered_frame_ = video_frame;
   }
 
   bool IsTextureSupported() const override { return false; }
 
   void Wait() {
     // Frame comparisons can be very expensive. Wait for test to be done, but
     // at time-out check if frames_processed is going up. If so, give it more
     // time, otherwise fail. Hopefully this will reduce test flakiness.
@@ skipping 24 matching lines @@
   }
 
   VideoCaptureInput* input_;
   Transport* transport_;
   PacketReceiver* receiver_;
   VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
     FrameComparison()
-        : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
+        : dropped(false),
+          send_time_ms(0),
+          recv_time_ms(0),
+          render_time_ms(0),
+          encoded_frame_size(0) {}
 
     FrameComparison(const VideoFrame& reference,
                     const VideoFrame& render,
                     bool dropped,
                     int64_t send_time_ms,
                     int64_t recv_time_ms,
-                    int64_t render_time_ms)
+                    int64_t render_time_ms,
+                    size_t encoded_frame_size,
+                    const VideoSendStream::Stats& send_stats)
         : reference(reference),
           render(render),
           dropped(dropped),
           send_time_ms(send_time_ms),
           recv_time_ms(recv_time_ms),
-          render_time_ms(render_time_ms) {}
+          render_time_ms(render_time_ms),
+          encoded_frame_size(encoded_frame_size),
+          send_stats(send_stats) {}
 
     VideoFrame reference;
     VideoFrame render;
     bool dropped;
     int64_t send_time_ms;
     int64_t recv_time_ms;
     int64_t render_time_ms;
+    size_t encoded_frame_size;
+    VideoSendStream::Stats send_stats;
+  };
+
+  struct Sample {
+    Sample(double dropped,
+           double input_time_ms,
+           double send_time_ms,
+           double recv_time_ms,
+           double avg_encode_time_ms,
+           double encoded_frame_size,
+           double encode_frame_rate,
+           double encode_usage_percent,
+           double media_bitrate_bps,
+           double psnr,
+           double ssim,
+           double decoded_time_ms)
+        : dropped(dropped),
+          input_time_ms(input_time_ms),
+          send_time_ms(send_time_ms),
+          recv_time_ms(recv_time_ms),
+          avg_encode_time_ms(avg_encode_time_ms),
+          encoded_frame_size(encoded_frame_size),
+          encode_frame_rate(encode_frame_rate),
+          encode_usage_percent(encode_usage_percent),
+          media_bitrate_bps(media_bitrate_bps),
+          psnr(psnr),
+          ssim(ssim),
+          decoded_time_ms(decoded_time_ms) {}
+
+    double dropped;
+    double input_time_ms;
+    double send_time_ms;
+    double recv_time_ms;
+    double avg_encode_time_ms;
+    double encoded_frame_size;
+    double encode_frame_rate;
+    double encode_usage_percent;
+    double media_bitrate_bps;
+    double psnr;
+    double ssim;
+    double decoded_time_ms;
   };
 
   void AddFrameComparison(const VideoFrame& reference,
                           const VideoFrame& render,
                           bool dropped,
-                          int64_t render_time_ms)
+                          int64_t render_time_ms,
+                          const VideoSendStream::Stats& send_stats)
       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
     int64_t send_time_ms = send_times_[reference.timestamp()];
     send_times_.erase(reference.timestamp());
     int64_t recv_time_ms = recv_times_[reference.timestamp()];
     recv_times_.erase(reference.timestamp());
 
+    size_t encoded_size = encoded_frame_sizes_[reference.timestamp()];
+    encoded_frame_sizes_.erase(reference.timestamp());
+
+    VideoFrame reference_copy;
+    VideoFrame render_copy;
+    reference_copy.CopyFrame(reference);
+    render_copy.CopyFrame(render);
+
     rtc::CritScope crit(&comparison_lock_);
-    comparisons_.push_back(FrameComparison(reference, render, dropped,
-                                           send_time_ms, recv_time_ms,
-                                           render_time_ms));
+    comparisons_.push_back(FrameComparison(
+        reference_copy, render_copy, dropped, send_time_ms, recv_time_ms,
+        render_time_ms, encoded_size, send_stats));
     comparison_available_event_->Set();
   }
 
   static bool PollStatsThread(void* obj) {
     return static_cast<VideoAnalyzer*>(obj)->PollStats();
   }
 
   bool PollStats() {
     switch (done_->Wait(kSendStatsPollingIntervalMs)) {
       case kEventSignaled:
@@ skipping 37 matching lines @@
         comparison_available_event_->Set();
         return false;
       }
       return true;  // Try again.
     }
 
     PerformFrameComparison(comparison);
 
     if (FrameProcessed()) {
       PrintResults();
+      if (save_samples_) {
+        PrintSamplesToFile();
+      }

[sprang_webrtc, 2015/08/19 13:33:37]

Remove braces

[ivica, 2015/08/20 13:25:59]

Done.

       done_->Set();
       comparison_available_event_->Set();
       return false;
     }
 
     return true;
   }
 
   bool PopComparison(FrameComparison* comparison) {
     rtc::CritScope crit(&comparison_lock_);
@@ skipping 52 matching lines @@
 
     EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
     EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = I420PSNR(&comparison.reference, &comparison.render);
     double ssim = I420SSIM(&comparison.reference, &comparison.render);
 
+    int64_t input_time_ms = comparison.reference.ntp_time_ms();
+    Sample sample(
+        comparison.dropped, input_time_ms, comparison.send_time_ms,
+        comparison.recv_time_ms, comparison.send_stats.avg_encode_time_ms,
+        comparison.encoded_frame_size, comparison.send_stats.encode_frame_rate,
+        comparison.send_stats.encode_usage_percent,
+        comparison.send_stats.media_bitrate_bps, psnr, ssim,
+        comparison.render_time_ms);  // decoded_time_ms

[sprang_webrtc, 2015/08/19 13:33:37]

I don't think comparison.send_stats is available here. (Handled by PollStats()
since a later revision)
Either remove them, or if your really need them poll send stats here.

[ivica, 2015/08/20 13:25:59]

Removed. Can be added later if necessary.

+
     rtc::CritScope crit(&comparison_lock_);
+    if (save_samples_) {
+      samples_.push_back(sample);
+    }
     psnr_.AddSample(psnr);
     ssim_.AddSample(ssim);
+
     if (comparison.dropped) {
       ++dropped_frames_;
       return;
     }
     if (last_render_time_ != 0)
       rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
     last_render_time_ = comparison.render_time_ms;
 
-    int64_t input_time_ms = comparison.reference.ntp_time_ms();
     sender_time_.AddSample(comparison.send_time_ms - input_time_ms);
     receiver_time_.AddSample(comparison.render_time_ms -
                              comparison.recv_time_ms);
     end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);
+    encoded_frame_size_.AddSample(comparison.encoded_frame_size);
   }
 
   void PrintResult(const char* result_type,
                    test::Statistics stats,
                    const char* unit) {
     printf("RESULT %s: %s = {%f, %f}%s\n",
            result_type,
            test_label_,
            stats.Mean(),
            stats.StandardDeviation(),
            unit);
   }
 
+  void PrintSamplesToFile(void) {
+    FILE* f = fopen(graph_data_filename_.c_str(), "w");
+    if (f == NULL) {
+      fprintf(stderr, "Couldn't open file %s\n", graph_data_filename_.c_str());
+      abort();
+    }
+
+    rtc::CritScope crit(&comparison_lock_);
+    std::sort(samples_.begin(), samples_.end(),
+              [](const Sample& A, const Sample& B)
+                  -> bool { return A.input_time_ms < B.input_time_ms; });
+
+    fprintf(f, "%s\n", test_label_);
+    fprintf(f, "%d\n", (int)samples_.size());

[sprang_webrtc, 2015/08/19 13:33:37]

Use static_cast rather than c-style cast.
Or just use "%lu\n" ?

[ivica, 2015/08/20 13:25:59]

Using %" PRIuS " as described in style guide...

+#define PRINT_SAMPLES(field)           \
+  {                                    \
+    fprintf(f, "%s ", #field);         \
+    for (Sample & value : samples_) {  \
+      fprintf(f, "%lf ", value.field); \
+    }                                  \
+    fprintf(f, "\n");                  \
+  }

[sprang_webrtc, 2015/08/19 13:33:37]

Macros are unfortunately only allowed in some very rare circumstances.
Break PRINT_SAMPLES out as method instead.

[ivica, 2015/08/20 13:25:59]

Changed from one-field-one-row to one-field-one-column format and got rid of the
macro.

+    PRINT_SAMPLES(dropped);
+    PRINT_SAMPLES(input_time_ms);
+    PRINT_SAMPLES(send_time_ms);
+    PRINT_SAMPLES(recv_time_ms);
+    PRINT_SAMPLES(avg_encode_time_ms);
+    PRINT_SAMPLES(encoded_frame_size);
+    PRINT_SAMPLES(encode_frame_rate);
+    PRINT_SAMPLES(encode_usage_percent);
+    PRINT_SAMPLES(media_bitrate_bps);
+    PRINT_SAMPLES(psnr);
+    PRINT_SAMPLES(ssim);
+    PRINT_SAMPLES(decoded_time_ms);
+#undef PRINT_SAMPLES
+
+    // Add an end specifier, to be able to add comments to the end of file if
+    // necessary.
+    fprintf(f, "--END--\n");
+    fclose(f);
+  }
+
   const char* const test_label_;
+  bool save_samples_;

[sprang_webrtc, 2015/08/19 13:33:37]

Not sure this is necessary. Calling graph_data_filename_.empty() once per frame
shouldn't be too much overhead.

[ivica, 2015/08/20 13:25:59]

Removed.

+  std::string graph_data_filename_;
+  std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);
   test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
   test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
   test::Statistics psnr_ GUARDED_BY(comparison_lock_);
   test::Statistics ssim_ GUARDED_BY(comparison_lock_);
   test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
   test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
   test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_time_ms GUARDED_BY(comparison_lock_);
   test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_);
   test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_);
 
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
   int dropped_frames_;
   int64_t last_render_time_;
   uint32_t rtp_timestamp_delta_;
 
   rtc::CriticalSection crit_;
   std::deque<VideoFrame> frames_ GUARDED_BY(crit_);
+  std::deque<VideoSendStream::Stats> send_stats_ GUARDED_BY(crit_);
   VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
+  std::map<uint32_t, size_t> encoded_frame_sizes_ GUARDED_BY(crit_);
   VideoFrame first_send_frame_ GUARDED_BY(crit_);
   const double avg_psnr_threshold_;
   const double avg_ssim_threshold_;
 
   rtc::CriticalSection comparison_lock_;
   std::vector<ThreadWrapper*> comparison_thread_pool_;
   rtc::scoped_ptr<ThreadWrapper> stats_polling_thread_;
   const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   const rtc::scoped_ptr<EventWrapper> done_;
 };
 
 void FullStackTest::RunTest(const FullStackTestParams& params) {
+  // TODO(ivica): Add num_temporal_layers as a param.
+  unsigned char num_temporal_layers =
+      params.graph_data_filename.empty() ? 2 : 1;
+
   test::DirectTransport send_transport(params.link);
   test::DirectTransport recv_transport(params.link);
   VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label,
                          params.avg_psnr_threshold, params.avg_ssim_threshold,
-                         params.test_durations_secs * params.clip.fps);
+                         params.test_durations_secs * params.clip.fps,
+                         params.graph_data_filename);
 
   CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
 
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   CreateSendConfig(1);
 
   rtc::scoped_ptr<VideoEncoder> encoder;
@@ skipping 27 matching lines @@
 
   VideoCodecVP8 vp8_settings;
   VideoCodecVP9 vp9_settings;
   if (params.screenshare) {
     encoder_config_.content_type = VideoEncoderConfig::ContentType::kScreen;
     encoder_config_.min_transmit_bitrate_bps = 400 * 1000;
     if (params.codec == "VP8") {
       vp8_settings = VideoEncoder::GetDefaultVp8Settings();
       vp8_settings.denoisingOn = false;
       vp8_settings.frameDroppingOn = false;
-      vp8_settings.numberOfTemporalLayers = 2;
+      vp8_settings.numberOfTemporalLayers = num_temporal_layers;
       encoder_config_.encoder_specific_settings = &vp8_settings;
     } else if (params.codec == "VP9") {
       vp9_settings = VideoEncoder::GetDefaultVp9Settings();
       vp9_settings.denoisingOn = false;
       vp9_settings.frameDroppingOn = false;
-      vp9_settings.numberOfTemporalLayers = 2;
+      vp9_settings.numberOfTemporalLayers = num_temporal_layers;
       encoder_config_.encoder_specific_settings = &vp9_settings;
     }
 
     stream->temporal_layer_thresholds_bps.clear();
-    stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);
+    if (num_temporal_layers > 1) {
+      stream->temporal_layer_thresholds_bps.push_back(
+          stream->target_bitrate_bps);
+    }
   }
 
   CreateMatchingReceiveConfigs();
   receive_configs_[0].renderer = &analyzer;
   receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
       kSendRtxPayloadType;
 
   for (auto& config : receive_configs_)
@@ skipping 199 matching lines @@
       50000,
       200000,
       2000000,
       0.0,
       0.0,
       kFullStackTestDurationSecs,
       "VP9"};
   RunTest(screenshare_params);
 }
 }  // namespace webrtc