Extend full stack tests with more stats

webrtc/video/full_stack.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
@@ skipping 121 matching lines @@
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     }
 
     return receiver_->DeliverPacket(media_type, packet, length);
   }
 
   void IncomingCapturedFrame(const VideoFrame& video_frame) override {
     VideoFrame copy = video_frame;
     copy.set_timestamp(copy.ntp_time_ms() * 90);
 
+    VideoSendStream::Stats stats = send_stream_->GetStats();
     {
       rtc::CritScope lock(&crit_);
       if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)
         first_send_frame_ = copy;
 
       frames_.push_back(copy);
+      send_stats_.push_back(stats);
     }
 
     input_->IncomingCapturedFrame(video_frame);
   }
 
   bool SendRtp(const uint8_t* packet, size_t length) override {
     rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
     RTPHeader header;
     parser->Parse(packet, length, &header);
 
     {
       rtc::CritScope lock(&crit_);
       if (rtp_timestamp_delta_ == 0) {
         rtp_timestamp_delta_ =
             header.timestamp - first_send_frame_.timestamp();
         first_send_frame_.Reset();
       }
-      send_times_[header.timestamp - rtp_timestamp_delta_] =
+      uint32_t timestamp = header.timestamp - rtp_timestamp_delta_;
+      send_times_[timestamp] =
           Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
+      encoded_frame_sizes_[timestamp] +=

[pbos-webrtc, 2015/06/28 18:35:31]

Why do you need a += here if you remove the stats further down? This looks racy
if you want to catch all the combined simulcast layers.

[sprang_webrtc, 2015/07/06 11:19:00]

Is simulcast used in the full stack tests? Then how is send_times_ not racy?
The reason for the += is to capture the encoded size of a frame if it is
fragmented across several rtp packets. I'd rather have used
VideoSenderStream.post_encode_callback but EncodedFrameObserver takes an
EncodedFrame which doesn't give us anything to identify it by...
Any other ideas of how to solve this?
If it's even needed - I did this before adding the send stream stats, perhaps
the media send bitrate is enough..

[pbos-webrtc, 2015/07/06 11:36:00]

It won't be popped unless it's rendered, right? That requires all packets to be
delivered/given up. For one stream that's fine, we won't try to render it twice.
For simulcast we have 3 separate streams but one of them can be rendered before
the last one is even sent.

[sprang_webrtc, 2015/07/17 11:42:07]

True, what I'm saying is that send_times is racy as well then? Are we using
simulcast in these test?
Anyway, I removed this and instead have the pre_decode_callback directly add a
sample. This might not be entirely correct, but it'll have to be good enough.

+          length - (header.headerLength + header.paddingLength);
     }
 
     return transport_->SendRtp(packet, length);
   }
 
   bool SendRtcp(const uint8_t* packet, size_t length) override {
     return transport_->SendRtcp(packet, length);
   }
 
   void RenderFrame(const VideoFrame& video_frame,
                    int time_to_render_ms) override {
     int64_t render_time_ms =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
 
     rtc::CritScope lock(&crit_);
 
     while (frames_.front().timestamp() < send_timestamp) {
       AddFrameComparison(frames_.front(), last_rendered_frame_, true,
-                         render_time_ms);
+                         render_time_ms, send_stats_.front());
       frames_.pop_front();
+      send_stats_.pop_front();
     }
 
     VideoFrame reference_frame = frames_.front();
+    VideoSendStream::Stats send_stats = send_stats_.front();

[pbos-webrtc, 2015/06/28 18:35:32]

You can pop this one after AddFrameComparison so you don't need a temporary, up
to you though.

[sprang_webrtc, 2015/07/06 11:19:00]

Done.

     frames_.pop_front();
+    send_stats_.pop_front();
     assert(!reference_frame.IsZeroSize());
     EXPECT_EQ(reference_frame.timestamp(), send_timestamp);
     assert(reference_frame.timestamp() == send_timestamp);
 
-    AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
+    AddFrameComparison(reference_frame, video_frame, false, render_time_ms,
+                       send_stats);
 
     last_rendered_frame_ = video_frame;
   }
 
   bool IsTextureSupported() const override { return false; }
 
   void Wait() {
     // Frame comparisons can be very expensive. Wait for test to be done, but
     // at time-out check if frames_processed is going up. If so, give it more
     // time, otherwise fail. Hopefully this will reduce test flakiness.
@@ skipping 13 matching lines @@
       }
       ASSERT_GT(frames_processed, last_frames_processed)
           << "Analyzer stalled while waiting for test to finish.";
       last_frames_processed = frames_processed;
     }
   }
 
   VideoCaptureInput* input_;
   Transport* transport_;
   PacketReceiver* receiver_;
+  VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
     FrameComparison()
-        : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
+        : dropped(false),
+          send_time_ms(0),
+          recv_time_ms(0),
+          render_time_ms(0),
+          encoded_frame_size(0) {}
 
     FrameComparison(const VideoFrame& reference,
                     const VideoFrame& render,
                     bool dropped,
                     int64_t send_time_ms,
                     int64_t recv_time_ms,
-                    int64_t render_time_ms)
+                    int64_t render_time_ms,
+                    size_t encoded_frame_size,
+                    const VideoSendStream::Stats& send_stats)
         : reference(reference),
           render(render),
           dropped(dropped),
           send_time_ms(send_time_ms),
           recv_time_ms(recv_time_ms),
-          render_time_ms(render_time_ms) {}
+          render_time_ms(render_time_ms),
+          encoded_frame_size(encoded_frame_size),
+          send_stats(send_stats) {}
 
     VideoFrame reference;
     VideoFrame render;
     bool dropped;
     int64_t send_time_ms;
     int64_t recv_time_ms;
     int64_t render_time_ms;
+    size_t encoded_frame_size;
+    VideoSendStream::Stats send_stats;
   };
 
   void AddFrameComparison(const VideoFrame& reference,
                           const VideoFrame& render,
                           bool dropped,
-                          int64_t render_time_ms)
+                          int64_t render_time_ms,
+                          const VideoSendStream::Stats& send_stats)
       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
     int64_t send_time_ms = send_times_[reference.timestamp()];
     send_times_.erase(reference.timestamp());
     int64_t recv_time_ms = recv_times_[reference.timestamp()];
     recv_times_.erase(reference.timestamp());
+    size_t encoded_size = encoded_frame_sizes_[reference.timestamp()];
+    encoded_frame_sizes_.erase(reference.timestamp());
 
     rtc::CritScope crit(&comparison_lock_);
-    comparisons_.push_back(FrameComparison(reference,
-                                           render,
-                                           dropped,
-                                           send_time_ms,
-                                           recv_time_ms,
-                                           render_time_ms));
+    comparisons_.push_back(
+        FrameComparison(reference, render, dropped, send_time_ms, recv_time_ms,
+                        render_time_ms, encoded_size, send_stats));
     comparison_available_event_->Set();
   }
 
   static bool FrameComparisonThread(void* obj) {
     return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
   }
 
   bool CompareFrames() {
     if (AllFramesRecorded())
       return false;
@@ skipping 67 matching lines @@
   void PrintResults() {
     rtc::CritScope crit(&comparison_lock_);
     PrintResult("psnr", psnr_, " dB");
     PrintResult("ssim", ssim_, "");
     PrintResult("sender_time", sender_time_, " ms");
     printf("RESULT dropped_frames: %s = %d frames\n", test_label_,
            dropped_frames_);
     PrintResult("receiver_time", receiver_time_, " ms");
     PrintResult("total_delay_incl_network", end_to_end_, " ms");
     PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
+    PrintResult("encoded_frame_size", encoded_frame_size_, " bytes");
+    PrintResult("encode_frame_rate_", encode_frame_rate_, " fps");
+    PrintResult("avg_encode_time", avg_encode_time_ms, " ms");
+    PrintResult("encode_usage_percent", encode_usage_percent, " percent");
+    PrintResult("media_bitrate", media_bitrate_bps, " bps");
+
     EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
     EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
   }
 
   void PerformFrameComparison(const FrameComparison& comparison) {
     // Perform expensive psnr and ssim calculations while not holding lock.
     double psnr = I420PSNR(&comparison.reference, &comparison.render);
     double ssim = I420SSIM(&comparison.reference, &comparison.render);
 
     rtc::CritScope crit(&comparison_lock_);
     psnr_.AddSample(psnr);
     ssim_.AddSample(ssim);
     if (comparison.dropped) {
       ++dropped_frames_;
       return;
     }
     if (last_render_time_ != 0)
       rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
     last_render_time_ = comparison.render_time_ms;
 
     int64_t input_time_ms = comparison.reference.ntp_time_ms();
     sender_time_.AddSample(comparison.send_time_ms - input_time_ms);
     receiver_time_.AddSample(comparison.render_time_ms -
                              comparison.recv_time_ms);
     end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);
+    encoded_frame_size_.AddSample(comparison.encoded_frame_size);
+    encode_frame_rate_.AddSample(comparison.send_stats.encode_frame_rate);

[pbos-webrtc, 2015/06/28 18:35:31]

Does this send_stats even necessarily correspond to the same frame? Otherwise I
see no reason to add it here, can't we just add samples from when we call
GetStats in the first place?

[sprang_webrtc, 2015/07/06 11:19:00]

The reason I did this was that I wanted samples from the same time frame as the
the other stats. If we do .AddSample when we capture a frame we might be adding
data points for a while after the we actually stop adding frame comparisons.

[pbos-webrtc, 2015/07/06 11:36:00]

Ok, so this essentially replaces a separate thread doing GetStats() every k ms,
and should not be corresponding to frame samples? I'm wondering whether we
should do that, WDYT?

[sprang_webrtc, 2015/07/17 11:42:07]

Sure. I added a check for frames_captured_ instead to limit scope.

+    avg_encode_time_ms.AddSample(comparison.send_stats.avg_encode_time_ms);
+    encode_usage_percent.AddSample(comparison.send_stats.encode_usage_percent);
+    media_bitrate_bps.AddSample(comparison.send_stats.media_bitrate_bps);
   }
 
   void PrintResult(const char* result_type,
                    test::Statistics stats,
                    const char* unit) {
     printf("RESULT %s: %s = {%f, %f}%s\n",
            result_type,
            test_label_,
            stats.Mean(),
            stats.StandardDeviation(),
            unit);
   }
 
   const char* const test_label_;
   test::Statistics sender_time_;
   test::Statistics receiver_time_;
   test::Statistics psnr_;
   test::Statistics ssim_;
   test::Statistics end_to_end_;
   test::Statistics rendered_delta_;
+  test::Statistics encoded_frame_size_;
+  test::Statistics encode_frame_rate_;
+  test::Statistics avg_encode_time_ms;
+  test::Statistics encode_usage_percent;
+  test::Statistics media_bitrate_bps;
+
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
   int dropped_frames_;
   int64_t last_render_time_;
   uint32_t rtp_timestamp_delta_;
 
   rtc::CriticalSection crit_;
   std::deque<VideoFrame> frames_ GUARDED_BY(crit_);
+  std::deque<VideoSendStream::Stats> send_stats_ GUARDED_BY(crit_);
   VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
   std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
+  std::map<uint32_t, size_t> encoded_frame_sizes_ GUARDED_BY(crit_);
   VideoFrame first_send_frame_ GUARDED_BY(crit_);
   const double avg_psnr_threshold_;
   const double avg_ssim_threshold_;
 
   rtc::CriticalSection comparison_lock_;
   std::vector<ThreadWrapper*> comparison_thread_pool_;
   const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   const rtc::scoped_ptr<EventWrapper> done_;
 };
@@ skipping 45 matching lines @@
 
   CreateMatchingReceiveConfigs();
   receive_configs_[0].renderer = &analyzer;
   receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
   receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
       kSendRtxPayloadType;
 
   CreateStreams();
   analyzer.input_ = send_stream_->Input();
+  analyzer.send_stream_ = send_stream_;
 
   if (params.screenshare) {
     std::vector<std::string> slides;
     slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
     slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
     slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
     slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv"));
 
     rtc::scoped_ptr<test::FrameGenerator> frame_generator(
         test::FrameGenerator::CreateFromYuvFile(
@@ skipping 157 matching lines @@
       true,
       50000,
       200000,
       2000000,
       0.0,
       0.0,
       kFullStackTestDurationSecs};
   RunTest(screenshare_params);
 }
 }  // namespace webrtc