Fixed Full stack tests to correctly process selected TL and SL while

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_quality_test.h"
@@ skipping 10 matching lines @@
 #include "webrtc/base/event.h"
 #include "webrtc/base/format_macros.h"
 #include "webrtc/base/optional.h"
 #include "webrtc/base/platform_file.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/logging/rtc_event_log/rtc_event_log.h"
 #include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
+#include "webrtc/modules/rtp_rtcp/source/rtp_format.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/system_wrappers/include/cpu_info.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/test/layer_filtering_transport.h"
 #include "webrtc/test/run_loop.h"
 #include "webrtc/test/statistics.h"
@@ skipping 91 matching lines @@
                 const std::string& test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames,
                 FILE* graph_data_output_file,
                 const std::string& graph_title,
                 uint32_t ssrc_to_analyze,
                 uint32_t rtx_ssrc_to_analyze,
                 uint32_t selected_stream_width,
                 uint32_t selected_stream_height,
+                int selected_sl,
+                int selected_tl,
                 bool is_quick_test_enabled)
       : transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         receive_stream_(nullptr),
         captured_frame_forwarder_(this),
         test_label_(test_label),
         graph_data_output_file_(graph_data_output_file),
         graph_title_(graph_title),
         ssrc_to_analyze_(ssrc_to_analyze),
         rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze),
         selected_stream_width_(selected_stream_width),
         selected_stream_height_(selected_stream_height),
+        selected_sl_(selected_sl),
+        selected_tl_(selected_tl),
         pre_encode_proxy_(this),
         encode_timing_proxy_(this),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         dropped_frames_before_first_encode_(0),
         dropped_frames_before_rendering_(0),
         last_render_time_(0),
         rtp_timestamp_delta_(0),
+        total_media_bytes_(0),
+        first_sending_time_(0),
+        last_sending_time_(0),
         avg_psnr_threshold_(avg_psnr_threshold),
         avg_ssim_threshold_(avg_ssim_threshold),
         is_quick_test_enabled_(is_quick_test_enabled),
         stats_polling_thread_(&PollStatsThread, this, "StatsPoller"),
         comparison_available_event_(false, false),
         done_(true, false) {
     // Create thread pool for CPU-expensive PSNR/SSIM calculations.
 
     // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
     // so that we don't accidentally starve "real" worker threads (codec etc).
@@ skipping 123 matching lines @@
       }
 
       if (!IsFlexfec(header.payloadType) && header.ssrc == ssrc_to_analyze_) {
         // Ignore FlexFEC timestamps, to avoid collisions with media timestamps.
         // (FlexFEC and media are sent on different SSRCs, which have different
         // timestamps spaces.)
         // Also ignore packets from wrong SSRC and retransmits.
         int64_t timestamp =
             wrap_handler_.Unwrap(header.timestamp - rtp_timestamp_delta_);
         send_times_[timestamp] = current_time;
-        if (!transport_->DiscardedLastPacket() &&
-            header.ssrc == ssrc_to_analyze_) {
+
+        // Ignore incorrect temporal and spatial layers.
+        bool is_correct_layer = true;
+        if (header.payloadType == kPayloadTypeVP9 ||
+            header.payloadType == kPayloadTypeVP8) {
+          // Get VP8 and VP9 specific header to check layers indexes.
+          const uint8_t* payload = packet + header.headerLength;
+          const size_t payload_length = length - header.headerLength;
+          const size_t payload_data_length =
+              payload_length - header.paddingLength;
+          const bool is_vp8 = header.payloadType == kPayloadTypeVP8;
+          std::unique_ptr<RtpDepacketizer> depacketizer(
+              RtpDepacketizer::Create(is_vp8 ? kRtpVideoVp8 : kRtpVideoVp9));

[sprang_webrtc, 2017/02/22 16:56:07]

Can you extract this to a helper method?

[ilnik, 2017/02/23 08:58:29]

Done.

+          RtpDepacketizer::ParsedPayload parsed_payload;
+          bool result = depacketizer->Parse(&parsed_payload, payload,
+                                            payload_data_length);
+          RTC_DCHECK(result);
+          const int temporal_idx = static_cast<int>(
+              is_vp8 ? parsed_payload.type.Video.codecHeader.VP8.temporalIdx
+                     : parsed_payload.type.Video.codecHeader.VP9.temporal_idx);
+          const int spatial_idx = static_cast<int>(
+              is_vp8 ? kNoSpatialIdx
+                     : parsed_payload.type.Video.codecHeader.VP9.spatial_idx);
+          is_correct_layer =
+              (selected_tl_ < 0 || temporal_idx == kNoTemporalIdx ||
+               temporal_idx <= selected_tl_) &&
+              (selected_sl_ < 0 || spatial_idx == kNoSpatialIdx ||
+               spatial_idx <= selected_sl_);
+        }
+        if (is_correct_layer) {
           encoded_frame_sizes_[timestamp] +=
               length - (header.headerLength + header.paddingLength);
+          total_media_bytes_ +=
+              length - (header.headerLength + header.paddingLength);
         }
+        if (first_sending_time_ == 0)
+          first_sending_time_ = current_time;
+        last_sending_time_ = current_time;
       }
     }
     return result;
   }
 
   bool SendRtcp(const uint8_t* packet, size_t length) override {
     return transport_->SendRtcp(packet, length);
   }
 
-  void EncodedFrameCallback(const EncodedFrame& frame) override {
-    rtc::CritScope lock(&comparison_lock_);
-    if (frames_recorded_ < frames_to_process_)
-      encoded_frame_size_.AddSample(frame.length_);
-  }
+  void EncodedFrameCallback(const EncodedFrame& frame) override { }

[sprang_webrtc, 2017/02/22 16:56:07]

encoded_frame_size_ isn't being populate any more?

I think it'd still be interesting to have. Also note that this can be slightly
different than rtp media payload size, since it'd doesn't include and payload
specific headers.

[ilnik, 2017/02/23 08:58:29]

No, it is populated in sendRtp to encoded_frame_sizes_, from there it's inserted
to Comparison struct and finally, added to statistic in PerformFrameComparison
at line 785. It will be a bit different, but it's the only way to separate
different streams. Before my changes encoded_frame_size_ was populated in both
places.

[sprang_webrtc, 2017/02/23 10:46:54]

I see. 
Then I guess you can remove this method, remove the public EncodedFrameObserver
inheritance and the registering of this class as pre_decode_callback on the
video receive configs.

[ilnik, 2017/02/23 10:55:59]

Done.

 
   void OnFrame(const VideoFrame& video_frame) override {
     int64_t render_time_ms =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
 
     rtc::CritScope lock(&crit_);
     int64_t send_timestamp =
         wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_);
 
     while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) {
@@ skipping 332 matching lines @@
     PrintResult("sender_time", sender_time_, " ms");
     PrintResult("receiver_time", receiver_time_, " ms");
     PrintResult("total_delay_incl_network", end_to_end_, " ms");
     PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
     PrintResult("encoded_frame_size", encoded_frame_size_, " bytes");
     PrintResult("encode_frame_rate", encode_frame_rate_, " fps");
     PrintResult("encode_time", encode_time_ms_, " ms");
     PrintResult("encode_usage_percent", encode_usage_percent_, " percent");
     PrintResult("media_bitrate", media_bitrate_bps_, " bps");
 
+    printf("RESULT actual_bitrate: %s = %.6lf bps\n", test_label_.c_str(),
+           GetActualBitrate());
+
     if (receive_stream_ != nullptr) {
       PrintResult("decode_time", decode_time_ms_, " ms");
       PrintResult("decode_time_max", decode_time_max_ms_, " ms");
     }
 
     printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(),
            dropped_frames_);
     printf("RESULT dropped_frames_before_first_encode: %s = %d frames\n",
            test_label_.c_str(), dropped_frames_before_first_encode_);
     printf("RESULT dropped_frames_before_rendering: %s = %d frames\n",
@@ skipping 104 matching lines @@
               sample.encoded_frame_size, sample.psnr, sample.ssim,
               encode_time_ms);
     }
     if (missing_encode_time_samples) {
       fprintf(stderr,
               "Warning: Missing encode_time_ms samples for %d frame(s).\n",
               missing_encode_time_samples);
     }
   }
 
+  double GetActualBitrate() {

[sprang_webrtc, 2017/02/22 16:56:07]

Maybe GetAverageMediaPayloadBitrateBps() ?

[ilnik, 2017/02/23 08:58:29]

Done.

+    return static_cast<double>(total_media_bytes_) * 8 /
+           (last_sending_time_ - first_sending_time_) *
+           rtc::kNumMillisecsPerSec;

[sprang_webrtc, 2017/02/22 16:56:06]

Just to be sure, maybe add a dcheck verifying that last_sending_time_ and
first_sending_time_ are not equal. Though a single frame/single packet test
should be rare even with the quick mode.

[ilnik, 2017/02/23 08:58:29]

Actually, in quick mode there will be exactly one frame. Need to add fix for
that. Nice catch.

+  }
+
   // Implements VideoSinkInterface to receive captured frames from a
   // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
   // as a source to VideoSendStream.
   // It forwards all input frames to the VideoAnalyzer for later comparison and
   // forwards the captured frames to the VideoSendStream.
   class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
                                  public rtc::VideoSourceInterface<VideoFrame> {
    public:
     explicit CapturedFrameForwarder(VideoAnalyzer* analyzer)
         : analyzer_(analyzer), send_stream_input_(nullptr) {}
@@ skipping 39 matching lines @@
   VideoSendStream* send_stream_;
   VideoReceiveStream* receive_stream_;
   CapturedFrameForwarder captured_frame_forwarder_;
   const std::string test_label_;
   FILE* const graph_data_output_file_;
   const std::string graph_title_;
   const uint32_t ssrc_to_analyze_;
   const uint32_t rtx_ssrc_to_analyze_;
   const uint32_t selected_stream_width_;
   const uint32_t selected_stream_height_;
+  const int selected_sl_;
+  const int selected_tl_;
   PreEncodeProxy pre_encode_proxy_;
   OnEncodeTimingProxy encode_timing_proxy_;
   std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);
   std::map<int64_t, int> samples_encode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
   test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
   test::Statistics psnr_ GUARDED_BY(comparison_lock_);
   test::Statistics ssim_ GUARDED_BY(comparison_lock_);
   test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
   test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
   test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics encode_usage_percent_ GUARDED_BY(comparison_lock_);
   test::Statistics decode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics decode_time_max_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics media_bitrate_bps_ GUARDED_BY(comparison_lock_);
 
   const int frames_to_process_;
   int frames_recorded_;
   int frames_processed_;
   int dropped_frames_;
   int dropped_frames_before_first_encode_;
   int dropped_frames_before_rendering_;
   int64_t last_render_time_;
   uint32_t rtp_timestamp_delta_;
+  int64_t total_media_bytes_;
+
+  int64_t first_sending_time_;
+  int64_t last_sending_time_;
 
   rtc::CriticalSection crit_;
   std::deque<VideoFrame> frames_ GUARDED_BY(crit_);
   rtc::Optional<VideoFrame> last_rendered_frame_ GUARDED_BY(crit_);
   rtc::TimestampWrapAroundHandler wrap_handler_ GUARDED_BY(crit_);
   std::map<int64_t, int64_t> send_times_ GUARDED_BY(crit_);
   std::map<int64_t, int64_t> recv_times_ GUARDED_BY(crit_);
   std::map<int64_t, size_t> encoded_frame_sizes_ GUARDED_BY(crit_);
   rtc::Optional<uint32_t> first_encoded_timestamp_ GUARDED_BY(crit_);
   rtc::Optional<uint32_t> first_sent_timestamp_ GUARDED_BY(crit_);
@@ skipping 452 matching lines @@
       &send_transport, params_.analyzer.test_label,
 
       params_.analyzer.avg_psnr_threshold, params_.analyzer.avg_ssim_threshold,
       is_quick_test_enabled
           ? kFramesSentInQuickTest
           : params_.analyzer.test_durations_secs * params_.video.fps,
       graph_data_output_file, graph_title,
       kVideoSendSsrcs[params_.ss.selected_stream],
       kSendRtxSsrcs[params_.ss.selected_stream],
       static_cast<uint32_t>(selected_stream.width),
-      static_cast<uint32_t>(selected_stream.height), is_quick_test_enabled);
+      static_cast<uint32_t>(selected_stream.height), params.ss.selected_sl,
+      params_.video.selected_tl, is_quick_test_enabled);
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupVideo(&analyzer, &recv_transport);
   video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
   video_send_config_.pre_encode_callback = analyzer.pre_encode_proxy();
   for (auto& config : video_receive_configs_)
     config.pre_decode_callback = &analyzer;
   RTC_DCHECK(!video_send_config_.post_encode_callback);
@@ skipping 233 matching lines @@
     std::ostringstream str;
     str << receive_logs_++;
     std::string path =
         params_.video.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
     stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
                                         10000000);
   }
 }
 
 }  // namespace webrtc