Implement timestamp translation/filter in VideoCapturer.

webrtc/media/base/videocapturer.h

 /*
  *  Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 207 matching lines @@
   void RemoveSink(rtc::VideoSinkInterface<cricket::VideoFrame>* sink) override;
 
  protected:
   // OnSinkWantsChanged can be overridden to change the default behavior
   // when a sink changes its VideoSinkWants by calling AddOrUpdateSink.
   virtual void OnSinkWantsChanged(const rtc::VideoSinkWants& wants);
 
   // Reports the appropriate frame size after adaptation. Returns true
   // if a frame is wanted. Returns false if there are no interested
   // sinks, or if the VideoAdapter decides to drop the frame.
+
+  // This function also implements timestamp translation/filtering.
+  // |camera_time_ns| is the camera's timestamp for the captured
+  // frame; it is expected to have good accuracy, but it may use an
+  // arbitrary epoch and a small possibly free-running with a frequency
+  // slightly different from the system clock. |system_time_us| is the
+  // monotonic system time (in the same scale as rtc::TimeMicros) when
+  // the frame was captured; the application is expected to read the
+  // system time as soon as possible after frame capture, but it may
+  // suffer scheduling jitter or poor system clock resolution. The
+  // output |translated_camera_time_us| is a combined timestamp,
+  // taking advantage of the supposedly higher accuracy in the camera
+  // timestamp, but using the same epoch and frequency as system time.
   bool AdaptFrame(int width,
                   int height,
-                  int64_t capture_time_ns,
+                  int64_t camera_time_us,
+                  int64_t system_time_us,
                   int* out_width,
                   int* out_height,
                   int* crop_width,
                   int* crop_height,
                   int* crop_x,
-                  int* crop_y);
+                  int* crop_y,
+                  int64_t* translated_camera_time_us);
 
   // Callback attached to SignalFrameCaptured where SignalVideoFrames is called.
   void OnFrameCaptured(VideoCapturer* video_capturer,
                        const CapturedFrame* captured_frame);
 
   // Called when a frame has been captured and converted to a
   // VideoFrame. OnFrame can be called directly by an implementation
   // that does not use SignalFrameCaptured or OnFrameCaptured. The
   // orig_width and orig_height are used only to produce stats.
   void OnFrame(const VideoFrame& frame, int orig_width, int orig_height);
@@ skipping 11 matching lines @@
     id_ = id;
   }
 
   void SetCaptureFormat(const VideoFormat* format) {
     capture_format_.reset(format ? new VideoFormat(*format) : NULL);
   }
 
   void SetSupportedFormats(const std::vector<VideoFormat>& formats);
   VideoFrameFactory* frame_factory() { return frame_factory_.get(); }
 
+ protected:
+  // Access intended only for testing.

[sprang_webrtc, 2016/06/17 13:10:52]

Can't we use friend class to expose this instead, or do we really hate those?

[nisse-webrtc, 2016/06/17 13:43:01]

I have no strong opinion on that (and not much experience with the C++ "friend"
feature). But to me, a protected non-virtual inline method seems very cheap in
all respects. A friend declaration referencing some class in the testsuite (if
that's what you are suggesting?) seems kind-of ugly.

[sprang_webrtc, 2016/06/17 14:35:38]

Friend class isn't pretty, but exposing and commenting that it's only for test
is no fun either imo. But I'm not religious about it either, so do as you like

+  int64_t clip_bias_us() { return clip_bias_us_; }
+
  private:
   void Construct();
   // Get the distance between the desired format and the supported format.
   // Return the max distance if they mismatch. See the implementation for
   // details.
   int64_t GetFormatDistance(const VideoFormat& desired,
                             const VideoFormat& supported);
 
   // Convert captured frame to readable string for LOG messages.
   std::string ToString(const CapturedFrame* frame) const;
 
   // Updates filtered_supported_formats_ so that it contains the formats in
   // supported_formats_ that fulfill all applied restrictions.
   void UpdateFilteredSupportedFormats();
   // Returns true if format doesn't fulfill all applied restrictions.
   bool ShouldFilterFormat(const VideoFormat& format) const;
 
   void UpdateInputSize(int width, int height);
 
+  // Update the estimated offset between camera time and system monotonic time.
+  void UpdateOffset(int64_t camera_time_us, int64_t system_time_us);
+
+  int64_t ClipTimestamp(int64_t filtered_time_us, int64_t system_time_us);
+
   rtc::ThreadChecker thread_checker_;
   std::string id_;
   CaptureState capture_state_;
   std::unique_ptr<VideoFrameFactory> frame_factory_;
   std::unique_ptr<VideoFormat> capture_format_;
   std::vector<VideoFormat> supported_formats_;
   std::unique_ptr<VideoFormat> max_format_;
   std::vector<VideoFormat> filtered_supported_formats_;
 
   bool enable_camera_list_;
   int scaled_width_;  // Current output size from ComputeScale.
   int scaled_height_;
 
   rtc::VideoBroadcaster broadcaster_;
   bool enable_video_adapter_;
   VideoAdapter video_adapter_;
 
   rtc::CriticalSection frame_stats_crit_;
   // The captured frame size before potential adapation.
   bool input_size_valid_ GUARDED_BY(frame_stats_crit_) = false;
   int input_width_ GUARDED_BY(frame_stats_crit_);
   int input_height_ GUARDED_BY(frame_stats_crit_);
 
   // Whether capturer should apply rotation to the frame before signaling it.
   bool apply_rotation_;
 
+  // State for the timestamp translation.
+  int frames_seen_;
+  // Estimated offset between camera time and system monotonic time.
+  int64_t offset_us_;
+
+  // State for timestamp clipping, applied after the filter, to ensure
+  // that translated timestamps are monotonic and not in the future.
+  // Subtracted from the translated timestamps.
+  int64_t clip_bias_us_;
+  rtc::Optional<int64_t> prev_translated_time_us_;
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoCapturer);
 };
 
 }  // namespace cricket
 
 #endif  // WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_