Incorrect timestamps used by video analyzer with real camera

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
@@ skipping 81 matching lines @@
   voe->voice_engine = nullptr;
 }
 
 }  // namespace
 
 namespace webrtc {
 
 class VideoAnalyzer : public PacketReceiver,
                       public Transport,
                       public rtc::VideoSinkInterface<VideoFrame>,
-                      public VideoCaptureInput,
                       public EncodedFrameObserver {
  public:
   VideoAnalyzer(test::LayerFilteringTransport* transport,
                 const std::string& test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
                 int duration_frames,
                 FILE* graph_data_output_file,
                 const std::string& graph_title,
                 uint32_t ssrc_to_analyze)
-      : input_(nullptr),
-        transport_(transport),
+      : transport_(transport),
         receiver_(nullptr),
         send_stream_(nullptr),
         test_label_(test_label),
         graph_data_output_file_(graph_data_output_file),
         graph_title_(graph_title),
         ssrc_to_analyze_(ssrc_to_analyze),
         pre_encode_proxy_(this),
+        input_capture_proxy_(this),
         encode_timing_proxy_(this),
         frames_to_process_(duration_frames),
         frames_recorded_(0),
         frames_processed_(0),
         dropped_frames_(0),
         dropped_frames_before_first_encode_(0),
         dropped_frames_before_rendering_(0),
         last_render_time_(0),
         rtp_timestamp_delta_(0),
         avg_psnr_threshold_(avg_psnr_threshold),
@@ skipping 59 matching lines @@
     }
 
     return receiver_->DeliverPacket(media_type, packet, length, packet_time);
   }
 
   void MeasuredEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) {
     rtc::CritScope crit(&comparison_lock_);
     samples_encode_time_ms_[ntp_time_ms] = encode_time_ms;
   }
 
-  void IncomingCapturedFrame(const VideoFrame& video_frame) override {
-    VideoFrame copy = video_frame;
-    copy.set_timestamp(copy.ntp_time_ms() * 90);
-    {
-      rtc::CritScope lock(&crit_);
-      frames_.push_back(copy);
-    }
-
-    input_->IncomingCapturedFrame(video_frame);
+  void OnCapturedFrame(const VideoFrame& video_frame) {
+    rtc::CritScope lock(&crit_);
+    frames_.push_back(video_frame);

[stefan-webrtc, 2016/09/13 07:19:53]

Why did we have to use a copy before but not now?

[sprang_webrtc, 2016/09/13 08:35:26]

Because we now call this method after all the timestamps have all been properly
populated.
The old code relied upon ntp_time_ms being set, and that that was the timestamp
always used - which was the root cause of this bug.
Though I think this CL is now borked again, since things moved significantly
with the encoder task queue thingy :(

I'll ping you when this CL is relevant again.

   }
 
   void PreEncodeOnFrame(const VideoFrame& video_frame) {
     rtc::CritScope lock(&crit_);
     if (!first_send_timestamp_ && rtp_timestamp_delta_ == 0) {
       while (frames_.front().timestamp() != video_frame.timestamp()) {
         ++dropped_frames_before_first_encode_;
         frames_.pop_front();
         RTC_CHECK(!frames_.empty());
       }
@@ skipping 117 matching lines @@
 
     if (iteration > 0)
       printf("- Farewell, sweet Concorde!\n");
 
     stats_polling_thread_.Stop();
   }
 
   rtc::VideoSinkInterface<VideoFrame>* pre_encode_proxy() {
     return &pre_encode_proxy_;
   }
+  rtc::VideoSinkInterface<VideoFrame>* input_capture_proxy() {
+    return &input_capture_proxy_;
+  }
   EncodedFrameObserver* encode_timing_proxy() { return &encode_timing_proxy_; }
 
   VideoCaptureInput* input_;
   test::LayerFilteringTransport* const transport_;
   PacketReceiver* receiver_;
   VideoSendStream* send_stream_;
 
  private:
   struct FrameComparison {
     FrameComparison()
@@ skipping 63 matching lines @@
 
     void OnEncodeTiming(int64_t ntp_time_ms, int encode_time_ms) override {
       parent_->MeasuredEncodeTiming(ntp_time_ms, encode_time_ms);
     }
     void EncodedFrameCallback(const EncodedFrame& frame) override {}
 
    private:
     VideoAnalyzer* const parent_;
   };
 
-  // This class receives the send-side OnFrame callback and is provided to not
+  // This class receives the send-side OnFrame callback triggered on the encoder
+  // thread just before encoding the thread. It is provided as a proxy to not
   // conflict with the receiver-side renderer callback.
   class PreEncodeProxy : public rtc::VideoSinkInterface<VideoFrame> {
    public:
     explicit PreEncodeProxy(VideoAnalyzer* parent) : parent_(parent) {}
 
     void OnFrame(const VideoFrame& video_frame) override {
       parent_->PreEncodeOnFrame(video_frame);
     }
 
    private:
     VideoAnalyzer* const parent_;
   };
 
+  // This class receives the send-side OnFrame callback triggered when the
+  // capturer receives a new frame. It is provided as a proxy to not conflict
+  // with the receiver-side renderer callback.
+  class InputCaptureProxy : public rtc::VideoSinkInterface<VideoFrame> {
+   public:
+    explicit InputCaptureProxy(VideoAnalyzer* parent) : parent_(parent) {}
+
+    void OnFrame(const VideoFrame& video_frame) override {
+      parent_->OnCapturedFrame(video_frame);
+    }
+
+   private:
+    VideoAnalyzer* const parent_;
+  };
+
   void AddFrameComparison(const VideoFrame& reference,
                           const VideoFrame& render,
                           bool dropped,
                           int64_t render_time_ms)
       EXCLUSIVE_LOCKS_REQUIRED(crit_) {
     int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp());
     int64_t send_time_ms = send_times_[reference_timestamp];
     send_times_.erase(reference_timestamp);
     int64_t recv_time_ms = recv_times_[reference_timestamp];
     recv_times_.erase(reference_timestamp);
@@ skipping 241 matching lines @@
               "Warning: Missing encode_time_ms samples for %d frame(s).\n",
               missing_encode_time_samples);
     }
   }
 
   const std::string test_label_;
   FILE* const graph_data_output_file_;
   const std::string graph_title_;
   const uint32_t ssrc_to_analyze_;
   PreEncodeProxy pre_encode_proxy_;
+  InputCaptureProxy input_capture_proxy_;
   OnEncodeTimingProxy encode_timing_proxy_;
   std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);
   std::map<int64_t, int> samples_encode_time_ms_ GUARDED_BY(comparison_lock_);
   test::Statistics sender_time_ GUARDED_BY(comparison_lock_);
   test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);
   test::Statistics psnr_ GUARDED_BY(comparison_lock_);
   test::Statistics ssim_ GUARDED_BY(comparison_lock_);
   test::Statistics end_to_end_ GUARDED_BY(comparison_lock_);
   test::Statistics rendered_delta_ GUARDED_BY(comparison_lock_);
   test::Statistics encoded_frame_size_ GUARDED_BY(comparison_lock_);
@@ skipping 396 matching lines @@
       graph_data_output_file, graph_title,
       kVideoSendSsrcs[params_.ss.selected_stream]);
 
   analyzer.SetReceiver(receiver_call_->Receiver());
   send_transport.SetReceiver(&analyzer);
   recv_transport.SetReceiver(sender_call_->Receiver());
 
   SetupCommon(&analyzer, &recv_transport);
   video_receive_configs_[params_.ss.selected_stream].renderer = &analyzer;
   video_send_config_.pre_encode_callback = analyzer.pre_encode_proxy();
+  video_send_config_.local_renderer = analyzer.input_capture_proxy();
   for (auto& config : video_receive_configs_)
     config.pre_decode_callback = &analyzer;
   RTC_DCHECK(!video_send_config_.post_encode_callback);
   video_send_config_.post_encode_callback = analyzer.encode_timing_proxy();
 
   if (params_.screenshare.enabled)
     SetupScreenshare();
 
   CreateVideoStreams();
-  analyzer.input_ = video_send_stream_->Input();
   analyzer.send_stream_ = video_send_stream_;
 
-  CreateCapturer(&analyzer);
+  CreateCapturer(video_send_stream_->Input());
 
   video_send_stream_->Start();
   for (VideoReceiveStream* receive_stream : video_receive_streams_)
     receive_stream->Start();
   capturer_->Start();
 
   analyzer.Wait();
 
   send_transport.StopSending();
   recv_transport.StopSending();
@@ skipping 157 matching lines @@
      call->DestroyAudioSendStream(audio_send_stream_);
      call->DestroyAudioReceiveStream(audio_receive_stream);
   }
 
   transport.StopSending();
   if (params_.audio)
     DestroyVoiceEngine(&voe);
 }
 
 }  // namespace webrtc