Incorrect timestamps used by video analyzer with real camera

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
@@ skipping 25 matching lines @@
 
 namespace webrtc {
 
 static const int kSendStatsPollingIntervalMs = 1000;
 static const int kPayloadTypeVP8 = 123;
 static const int kPayloadTypeVP9 = 124;
 
 class VideoAnalyzer : public PacketReceiver,
                       public Transport,
                       public VideoRenderer,
-                      public VideoCaptureInput,
                       public EncodedFrameObserver,
                       public EncodingTimeObserver {
  public:
   VideoAnalyzer(test::LayerFilteringTransport* transport,
                 const std::string& test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames,
                 FILE* graph_data_output_file,
                 const std::string& graph_title,
                 uint32_t ssrc_to_analyze)
-      : input_(nullptr),
-        transport_(transport),
+      : transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         test_label_(test_label),
         graph_data_output_file_(graph_data_output_file),
         graph_title_(graph_title),
         ssrc_to_analyze_(ssrc_to_analyze),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
@@ skipping 55 matching lines @@
 
     return receiver_->DeliverPacket(media_type, packet, length, packet_time);
   }
 
   // EncodingTimeObserver.
   void OnReportEncodedTime(int64_t ntp_time_ms, int encode_time_ms) override {
     rtc::CritScope crit(&comparison_lock_);
     samples_encode_time_ms_[ntp_time_ms] = encode_time_ms;
   }
 
-  void IncomingCapturedFrame(const VideoFrame& video_frame) override {
-    VideoFrame copy = video_frame;
-    copy.set_timestamp(copy.ntp_time_ms() * 90);
-
-    {
-      rtc::CritScope lock(&crit_);
-      if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)
-        first_send_frame_ = copy;
-
-      frames_.push_back(copy);
-    }
-
-    input_->IncomingCapturedFrame(video_frame);
-  }
-
   bool SendRtp(const uint8_t* packet,
                size_t length,
                const PacketOptions& options) override {
     RtpUtility::RtpHeaderParser parser(packet, length);
     RTPHeader header;
     parser.Parse(header);
 
     int64_t current_time =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     bool result = transport_->SendRtp(packet, length, options);
@@ skipping 17 matching lines @@
   bool SendRtcp(const uint8_t* packet, size_t length) override {
     return transport_->SendRtcp(packet, length);
   }
 
   void EncodedFrameCallback(const EncodedFrame& frame) override {
     rtc::CritScope lock(&comparison_lock_);
     if (frames_recorded_ < frames_to_process_)
       encoded_frame_size_.AddSample(frame.length_);
   }
 
   void RenderFrame(const VideoFrame& video_frame,

[pbos-webrtc, 2015/12/19 11:02:34]

Put a comment on this function to document that this renderer is used both for
local rendering (on input) and rendering after frames have been decoded on the
remote side, otherwise the below block is confusing.

                    int time_to_render_ms) override {
+    rtc::CritScope lock(&crit_);
+    uint32_t last_timestamp =
+        frames_.empty() ? 0 : frames_.rbegin()->timestamp();
+    if (video_frame.timestamp() > last_timestamp) {
+      // This is must be an input frame, just add a copy to frames_ and return.
+      if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)
+        first_send_frame_ = video_frame;
+
+      frames_.push_back(video_frame);
+      return;
+    }
+
     int64_t render_time_ms =
         Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
     uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
 
-    rtc::CritScope lock(&crit_);
-
     while (frames_.front().timestamp() < send_timestamp) {
       AddFrameComparison(frames_.front(), last_rendered_frame_, true,
                          render_time_ms);
       frames_.pop_front();
     }
 
     VideoFrame reference_frame = frames_.front();
     frames_.pop_front();
     assert(!reference_frame.IsZeroSize());
     if (send_timestamp == reference_frame.timestamp() - 1) {
@@ skipping 46 matching lines @@
     if (iteration > 0)
       printf("- Farewell, sweet Concorde!\n");
 
     // Signal stats polling thread if that is still waiting and stop it now,
     // since it uses the send_stream_ reference that might be reclaimed after
     // returning from this method.
     done_.Set();
     stats_polling_thread_.Stop();
   }
 
-  VideoCaptureInput* input_;
   test::LayerFilteringTransport* const transport_;
   PacketReceiver* receiver_;
   VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
     FrameComparison()
         : dropped(false),
           send_time_ms(0),
           recv_time_ms(0),
@@ skipping 691 matching lines @@
       params_.analyzer.test_durations_secs * params_.common.fps,
       graph_data_output_file, graph_title,
       kSendSsrcs[params_.ss.selected_stream]);
 
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupCommon(&analyzer, &recv_transport);
   send_config_.encoding_time_observer = &analyzer;
+  send_config_.local_renderer = &analyzer;
   receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
   for (auto& config : receive_configs_)
     config.pre_decode_callback = &analyzer;
 
   if (params_.screenshare.enabled)
     SetupScreenshare();
 
   CreateStreams();
-  analyzer.input_ = send_stream_->Input();
   analyzer.send_stream_ = send_stream_;
 
-  CreateCapturer(&analyzer);
+  CreateCapturer(send_stream_->Input());
 
   send_stream_->Start();
   for (size_t i = 0; i < receive_streams_.size(); ++i)
     receive_streams_[i]->Start();
   capturer_->Start();
 
   analyzer.Wait();
 
   send_transport.StopSending();
   recv_transport.StopSending();
@@ skipping 66 matching lines @@
   send_stream_->Stop();
   receive_stream->Stop();
 
   call->DestroyVideoReceiveStream(receive_stream);
   call->DestroyVideoSendStream(send_stream_);
 
   transport.StopSending();
 }
 
 }  // namespace webrtc