Extend full stack tests with more stats

webrtc/video/full_stack.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
 
 #include <deque>
 #include <map>
 
 #include "testing/gtest/include/gtest/gtest.h"
 
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/base/thread_annotations.h"
 #include "webrtc/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+#include "webrtc/frame_callback.h"
 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
 #include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/system_wrappers/interface/cpu_info.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/event_wrapper.h"
 #include "webrtc/system_wrappers/interface/sleep.h"
 #include "webrtc/test/call_test.h"
 #include "webrtc/test/direct_transport.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_encoder.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/frame_generator_capturer.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/typedefs.h"
 
 namespace webrtc {
 
 static const int kFullStackTestDurationSecs = 60;
+static const int kSendStatsPollingIntervalMs = 1000;
 
 struct FullStackTestParams {
   const char* test_label;
   struct {
     const char* name;
     size_t width, height;
     int fps;
   } clip;
   bool screenshare;
   int min_bitrate_bps;
   int target_bitrate_bps;
   int max_bitrate_bps;
   double avg_psnr_threshold;
   double avg_ssim_threshold;
   int test_durations_secs;
   FakeNetworkPipe::Config link;
 };
 
 class FullStackTest : public test::CallTest {
  protected:
   void RunTest(const FullStackTestParams& params);
 };
 
 class VideoAnalyzer : public PacketReceiver,
                       public newapi::Transport,
                       public VideoRenderer,
-                      public VideoCaptureInput {
+                      public VideoCaptureInput,
+                      public EncodedFrameObserver {
  public:
   VideoAnalyzer(VideoCaptureInput* input,
                 Transport* transport,
                 const char* test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames)
       : input_(input),
         transport_(transport),
         receiver_(nullptr),
+        send_stream_(nullptr),
         test_label_(test_label),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         last_render_time_(0),
         rtp_timestamp_delta_(0),
         avg_psnr_threshold_(avg_psnr_threshold),
         avg_ssim_threshold_(avg_ssim_threshold),
         comparison_available_event_(EventWrapper::Create()),
@@ skipping 16 matching lines @@
       num_cores -= kMinCoresLeft;
       num_cores = std::min(num_cores, kMaxComparisonThreads);
     }
 
     for (uint32_t i = 0; i < num_cores; ++i) {
       rtc::scoped_ptr<ThreadWrapper> thread =
           ThreadWrapper::CreateThread(&FrameComparisonThread, this, "Analyzer");
       EXPECT_TRUE(thread->Start());
       comparison_thread_pool_.push_back(thread.release());
     }
+
+    stats_polling_thread_ =
+        ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller");
+    EXPECT_TRUE(stats_polling_thread_->Start());
   }
 
   ~VideoAnalyzer() {
     for (ThreadWrapper* thread : comparison_thread_pool_) {
       EXPECT_TRUE(thread->Stop());
       delete thread;
     }
+    EXPECT_TRUE(stats_polling_thread_->Stop());
   }
 
   virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
 
   DeliveryStatus DeliverPacket(MediaType media_type, const uint8_t* packet,
                                size_t length) override {
     rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
     RTPHeader header;
     parser->Parse(packet, length, &header);
     {
@@ skipping 36 matching lines @@
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     }
 
     return transport_->SendRtp(packet, length);
   }
 
   bool SendRtcp(const uint8_t* packet, size_t length) override {
     return transport_->SendRtcp(packet, length);
   }
 
+  void EncodedFrameCallback(const EncodedFrame& frame) override {
+    rtc::CritScope lock(&comparison_lock_);
+    if (frames_recorded_ < frames_to_process_)
+      encoded_frame_size_.AddSample(frame.length_);
+  }
+
   void RenderFrame(const VideoFrame& video_frame,
                    int time_to_render_ms) override {
     int64_t render_time_ms =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
 
     rtc::CritScope lock(&crit_);
 
     while (frames_.front().timestamp() < send_timestamp) {
       AddFrameComparison(frames_.front(), last_rendered_frame_, true,
@@ skipping 29 matching lines @@
         frames_processed = frames_processed_;
       }
       if (last_frames_processed == -1) {
         last_frames_processed = frames_processed;
         continue;
       }
       ASSERT_GT(frames_processed, last_frames_processed)
           << "Analyzer stalled while waiting for test to finish.";
       last_frames_processed = frames_processed;
     }
+
+    done_->Set();  // Signal stats polling thread if that is still waiting.
   }
 
   VideoCaptureInput* input_;
   Transport* transport_;
   PacketReceiver* receiver_;
+  VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
     FrameComparison()
         : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
 
     FrameComparison(const VideoFrame& reference,
                     const VideoFrame& render,
                     bool dropped,
                     int64_t send_time_ms,
@@ skipping 18 matching lines @@
                           const VideoFrame& render,
                           bool dropped,
                           int64_t render_time_ms)
       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
     int64_t send_time_ms = send_times_[reference.timestamp()];
     send_times_.erase(reference.timestamp());
     int64_t recv_time_ms = recv_times_[reference.timestamp()];
     recv_times_.erase(reference.timestamp());
 
     rtc::CritScope crit(&comparison_lock_);
-    comparisons_.push_back(FrameComparison(reference,
-                                           render,
-                                           dropped,
-                                           send_time_ms,
-                                           recv_time_ms,
+    comparisons_.push_back(FrameComparison(reference, render, dropped,
+                                           send_time_ms, recv_time_ms,
                                            render_time_ms));
     comparison_available_event_->Set();
   }
 
+  static bool PollStatsThread(void* obj) {
+    return static_cast<VideoAnalyzer*>(obj)->PollStats();
+  }
+
+  bool PollStats() {
+    switch (done_->Wait(kSendStatsPollingIntervalMs)) {
+      case kEventSignaled:
+      case kEventError:
+        done_->Set();  // Make sure main thread is also signaled.
+        return false;
+      case kEventTimeout:
+        break;
+      default:

[pbos-webrtc, 2015/07/21 20:47:49]

Do you need default: here? (Is this an enum?)

[sprang_webrtc, 2015/07/22 12:12:10]

It's an enum. I'd like to get a compile-time error instead, but that doesn't
seem to happen :(

+        RTC_NOTREACHED();
+    }
+
+    VideoSendStream::Stats stats = send_stream_->GetStats();
+
+    rtc::CritScope crit(&comparison_lock_);
+    if (frames_recorded_ >= frames_to_process_)

[pbos-webrtc, 2015/07/21 20:47:49]

Does this mean that we're done? If so add a comment saying why listening for
done_ is not enough. If you're worried about adding one sample too many, then
don't bother I think. :)

[sprang_webrtc, 2015/07/22 12:12:10]

Done.

+      return false;
+    encode_frame_rate_.AddSample(stats.encode_frame_rate);
+    avg_encode_time_ms.AddSample(stats.avg_encode_time_ms);
+    encode_usage_percent.AddSample(stats.encode_usage_percent);
+    media_bitrate_bps.AddSample(stats.media_bitrate_bps);
+
+    return true;
+  }
+
   static bool FrameComparisonThread(void* obj) {
     return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
   }
 
   bool CompareFrames() {
     if (AllFramesRecorded())
       return false;
 
     VideoFrame reference;
     VideoFrame render;
@@ skipping 64 matching lines @@
   void PrintResults() {
     rtc::CritScope crit(&comparison_lock_);
     PrintResult("psnr", psnr_, " dB");
     PrintResult("ssim", ssim_, "");
     PrintResult("sender_time", sender_time_, " ms");
     printf("RESULT dropped_frames: %s = %d frames\n", test_label_,
            dropped_frames_);
     PrintResult("receiver_time", receiver_time_, " ms");
     PrintResult("total_delay_incl_network", end_to_end_, " ms");
     PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
+    PrintResult("encoded_frame_size", encoded_frame_size_, " bytes");
+    PrintResult("encode_frame_rate_", encode_frame_rate_, " fps");

[pbos-webrtc, 2015/07/21 20:47:49]

encode_frame_rate (drop the _ suffix)

[sprang_webrtc, 2015/07/22 12:12:10]

Done.

+    PrintResult("avg_encode_time", avg_encode_time_ms, " ms");

[pbos-webrtc, 2015/07/21 20:47:49]

Should this be called avg_ ? All of these are averages, even though this is an
average of averages.

[sprang_webrtc, 2015/07/22 12:12:10]

I'll drop the avg. Just copied the name from the stats struct.

+    PrintResult("encode_usage_percent", encode_usage_percent, " percent");
+    PrintResult("media_bitrate", media_bitrate_bps, " bps");
+
     EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
     EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = I420PSNR(&comparison.reference, &comparison.render);
     double ssim = I420SSIM(&comparison.reference, &comparison.render);
 
     rtc::CritScope crit(&comparison_lock_);
@@ skipping 19 matching lines @@
                    const char* unit) {
     printf("RESULT %s: %s = {%f, %f}%s\n",
            result_type,
            test_label_,
            stats.Mean(),
            stats.StandardDeviation(),
            unit);
   }
 
   const char* const test_label_;
-  test::Statistics sender_time_;
-  test::Statistics receiver_time_;
-  test::Statistics psnr_;
-  test::Statistics ssim_;
-  test::Statistics end_to_end_;
-  test::Statistics rendered_delta_;
+  test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
+  test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
+  test::Statistics psnr_ GUARDED_BY(comparison_lock_);
+  test::Statistics ssim_ GUARDED_BY(comparison_lock_);
+  test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
+  test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
+  test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
+  test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
+  test::Statistics avg_encode_time_ms GUARDED_BY(comparison_lock_);
+  test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_);
+  test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_);
+
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
   int dropped_frames_;
   int64_t last_render_time_;
   uint32_t rtp_timestamp_delta_;
 
   rtc::CriticalSection crit_;
   std::deque<VideoFrame> frames_ GUARDED_BY(crit_);
   VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
   VideoFrame first_send_frame_ GUARDED_BY(crit_);
   const double avg_psnr_threshold_;
   const double avg_ssim_threshold_;
 
   rtc::CriticalSection comparison_lock_;
   std::vector<ThreadWrapper*> comparison_thread_pool_;
+  rtc::scoped_ptr<ThreadWrapper> stats_polling_thread_;
   const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   const rtc::scoped_ptr<EventWrapper> done_;
 };
 
 void FullStackTest::RunTest(const FullStackTestParams& params) {
   test::DirectTransport send_transport(params.link);
   test::DirectTransport recv_transport(params.link);
   VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label,
                          params.avg_psnr_threshold, params.avg_ssim_threshold,
@@ skipping 37 matching lines @@
     stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);
   }
 
   CreateMatchingReceiveConfigs();
   receive_configs_[0].renderer = &analyzer;
   receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
       kSendRtxPayloadType;
 
+  for (auto& config : receive_configs_)
+    config.pre_decode_callback = &analyzer;
   CreateStreams();
   analyzer.input_ = send_stream_->Input();
+  analyzer.send_stream_ = send_stream_;
 
   if (params.screenshare) {
     std::vector<std::string> slides;
     slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
     slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
     slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
     slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv"));
 
     rtc::scoped_ptr<test::FrameGenerator> frame_generator(
         test::FrameGenerator::CreateFromYuvFile(
@@ skipping 157 matching lines @@
       true,
       50000,
       200000,
       2000000,
       0.0,
       0.0,
       kFullStackTestDurationSecs};
   RunTest(screenshare_params);
 }
 }  // namespace webrtc