Extend full stack tests with more stats

webrtc/video/full_stack.cc

Index: webrtc/video/full_stack.cc
diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc
index ad1b5c6356557e3e4152b1dbd0f3c8c0f31ede2c..85771f8615a79f47611b659f0bd2ad92d13c8f21 100644
--- a/webrtc/video/full_stack.cc
+++ b/webrtc/video/full_stack.cc
@@ -18,6 +18,7 @@
 #include "webrtc/base/thread_annotations.h"
 #include "webrtc/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+#include "webrtc/frame_callback.h"
 #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
 #include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/system_wrappers/interface/cpu_info.h"
@@ -37,6 +38,7 @@
 namespace webrtc {
 
 static const int kFullStackTestDurationSecs = 60;
+static const int kSendStatsPollingIntervalMs = 1000;
 
 struct FullStackTestParams {
   const char* test_label;
@@ -63,7 +65,8 @@ class FullStackTest : public test::CallTest {
 class VideoAnalyzer : public PacketReceiver,
                       public newapi::Transport,
                       public VideoRenderer,
-                      public VideoCaptureInput {
+                      public VideoCaptureInput,
+                      public EncodedFrameObserver {
  public:
   VideoAnalyzer(VideoCaptureInput* input,
                 Transport* transport,
@@ -74,6 +77,7 @@ class VideoAnalyzer : public PacketReceiver,
       : input_(input),
         transport_(transport),
         receiver_(nullptr),
+        send_stream_(nullptr),
         test_label_(test_label),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
@@ -110,6 +114,10 @@ class VideoAnalyzer : public PacketReceiver,
       EXPECT_TRUE(thread->Start());
       comparison_thread_pool_.push_back(thread.release());
     }
+
+    stats_polling_thread_ =
+        ThreadWrapper::CreateThread(&PollStatsThread, this, "StatsPoller");
+    EXPECT_TRUE(stats_polling_thread_->Start());
   }
 
   ~VideoAnalyzer() {
@@ -117,6 +125,7 @@ class VideoAnalyzer : public PacketReceiver,
       EXPECT_TRUE(thread->Stop());
       delete thread;
     }
+    EXPECT_TRUE(stats_polling_thread_->Stop());
   }
 
   virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
@@ -173,6 +182,12 @@ class VideoAnalyzer : public PacketReceiver,
     return transport_->SendRtcp(packet, length);
   }
 
+  void EncodedFrameCallback(const EncodedFrame& frame) override {
+    rtc::CritScope lock(&comparison_lock_);
+    if (frames_recorded_ < frames_to_process_)
+      encoded_frame_size_.AddSample(frame.length_);
+  }
+
   void RenderFrame(const VideoFrame& video_frame,
                    int time_to_render_ms) override {
     int64_t render_time_ms =
@@ -222,11 +237,14 @@ class VideoAnalyzer : public PacketReceiver,
           << "Analyzer stalled while waiting for test to finish.";
       last_frames_processed = frames_processed;
     }
+
+    done_->Set();  // Signal stats polling thread if that is still waiting.
   }
 
   VideoCaptureInput* input_;
   Transport* transport_;
   PacketReceiver* receiver_;
+  VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
@@ -265,15 +283,41 @@ class VideoAnalyzer : public PacketReceiver,
     recv_times_.erase(reference.timestamp());
 
     rtc::CritScope crit(&comparison_lock_);
-    comparisons_.push_back(FrameComparison(reference,
-                                           render,
-                                           dropped,
-                                           send_time_ms,
-                                           recv_time_ms,
+    comparisons_.push_back(FrameComparison(reference, render, dropped,
+                                           send_time_ms, recv_time_ms,
                                            render_time_ms));
     comparison_available_event_->Set();
   }
 
+  static bool PollStatsThread(void* obj) {
+    return static_cast<VideoAnalyzer*>(obj)->PollStats();
+  }
+
+  bool PollStats() {
+    switch (done_->Wait(kSendStatsPollingIntervalMs)) {
+      case kEventSignaled:
+      case kEventError:
+        done_->Set();  // Make sure main thread is also signaled.
+        return false;
+      case kEventTimeout:
+        break;
+      default:

[pbos-webrtc, 2015/07/21 20:47:49]

Do you need default: here? (Is this an enum?)

[sprang_webrtc, 2015/07/22 12:12:10]

It's an enum. I'd like to get a compile-time error instead, but that doesn't
seem to happen :(

+        RTC_NOTREACHED();
+    }
+
+    VideoSendStream::Stats stats = send_stream_->GetStats();
+
+    rtc::CritScope crit(&comparison_lock_);
+    if (frames_recorded_ >= frames_to_process_)

[pbos-webrtc, 2015/07/21 20:47:49]

Does this mean that we're done? If so add a comment saying why listening for
done_ is not enough. If you're worried about adding one sample too many, then
don't bother I think. :)

[sprang_webrtc, 2015/07/22 12:12:10]

Done.

+      return false;
+    encode_frame_rate_.AddSample(stats.encode_frame_rate);
+    avg_encode_time_ms.AddSample(stats.avg_encode_time_ms);
+    encode_usage_percent.AddSample(stats.encode_usage_percent);
+    media_bitrate_bps.AddSample(stats.media_bitrate_bps);
+
+    return true;
+  }
+
   static bool FrameComparisonThread(void* obj) {
     return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
   }
@@ -358,6 +402,12 @@ class VideoAnalyzer : public PacketReceiver,
     PrintResult("receiver_time", receiver_time_, " ms");
     PrintResult("total_delay_incl_network", end_to_end_, " ms");
     PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
+    PrintResult("encoded_frame_size", encoded_frame_size_, " bytes");
+    PrintResult("encode_frame_rate_", encode_frame_rate_, " fps");

[pbos-webrtc, 2015/07/21 20:47:49]

encode_frame_rate (drop the _ suffix)

[sprang_webrtc, 2015/07/22 12:12:10]

Done.

+    PrintResult("avg_encode_time", avg_encode_time_ms, " ms");

[pbos-webrtc, 2015/07/21 20:47:49]

Should this be called avg_ ? All of these are averages, even though this is an
average of averages.

[sprang_webrtc, 2015/07/22 12:12:10]

I'll drop the avg. Just copied the name from the stats struct.

+    PrintResult("encode_usage_percent", encode_usage_percent, " percent");
+    PrintResult("media_bitrate", media_bitrate_bps, " bps");
+
     EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
     EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
   }
@@ -397,12 +447,18 @@ class VideoAnalyzer : public PacketReceiver,
   }
 
   const char* const test_label_;
-  test::Statistics sender_time_;
-  test::Statistics receiver_time_;
-  test::Statistics psnr_;
-  test::Statistics ssim_;
-  test::Statistics end_to_end_;
-  test::Statistics rendered_delta_;
+  test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
+  test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
+  test::Statistics psnr_ GUARDED_BY(comparison_lock_);
+  test::Statistics ssim_ GUARDED_BY(comparison_lock_);
+  test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
+  test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
+  test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
+  test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
+  test::Statistics avg_encode_time_ms GUARDED_BY(comparison_lock_);
+  test::Statistics encode_usage_percent GUARDED_BY(comparison_lock_);
+  test::Statistics media_bitrate_bps GUARDED_BY(comparison_lock_);
+
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
@@ -421,6 +477,7 @@ class VideoAnalyzer : public PacketReceiver,
 
   rtc::CriticalSection comparison_lock_;
   std::vector<ThreadWrapper*> comparison_thread_pool_;
+  rtc::scoped_ptr<ThreadWrapper> stats_polling_thread_;
   const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   const rtc::scoped_ptr<EventWrapper> done_;
@@ -478,8 +535,11 @@ void FullStackTest::RunTest(const FullStackTestParams& params) {
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
       kSendRtxPayloadType;
 
+  for (auto& config : receive_configs_)
+    config.pre_decode_callback = &analyzer;
   CreateStreams();
   analyzer.input_ = send_stream_->Input();
+  analyzer.send_stream_ = send_stream_;
 
   if (params.screenshare) {
     std::vector<std::string> slides;