Implement timestamp translation/filter in VideoCapturer.

webrtc/media/base/videocapturer.cc

Index: webrtc/media/base/videocapturer.cc
diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc
index 96a605585509220b12875ecf993fc5010fe5e1cc..5c0c1b3efc4ea9e9995a01ce9d14ddf1b79d00ce 100644
--- a/webrtc/media/base/videocapturer.cc
+++ b/webrtc/media/base/videocapturer.cc
@@ -59,7 +59,8 @@ bool CapturedFrame::GetDataSize(uint32_t* size) const {
 /////////////////////////////////////////////////////////////////////
 // Implementation of class VideoCapturer
 /////////////////////////////////////////////////////////////////////
-VideoCapturer::VideoCapturer() : apply_rotation_(false) {
+VideoCapturer::VideoCapturer()
+    : apply_rotation_(false), frames_seen_(0), offset_us_(0) {
   thread_checker_.DetachFromThread();
   Construct();
 }
@@ -214,16 +215,84 @@ void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) {
   }
 }
 
+void VideoCapturer::UpdateOffset(int64_t capture_time_us) {
+  // Estimate the offset between system monotonic time and the capture
+  // time from the camera. The camera is assumed to provide more
+  // accurate timestamps than we can do here. But the camera may use
+  // its own free-running clock with a large offset and a small drift
+  // compared to the system clock. So the model is basically
+  //
+  //   y_k = c_0 + c_1 x_k + v_k
+  //
+  // where x_k is the camera timestamp, believed to be accurate in its
+  // own scale. y_k is our reading of the system clock. v_k is the
+  // measurement noise, i.e., the delay from frame capture until we
+  // get here and read the clock.
+  //
+  // It's possible to do (weighted) least-squares estimation of both
+  // c_0 and c_1. Then we get the constants as c_1 = Cov(x,y) /
+  // Var(x), and c_0 = mean(y) - c_1 mean(x). Substituting this c_0,
+  // we can rearrange the model as
+  //
+  //   y_k = mean(y) + (x_k - mean(x)) + (c_1 - 1) (x_k - mean(x)) + v_k
+  //
+  // Now if we use a weighted average which gradually forgets old
+  // values, x_k - mean(x) is bounded, of the same order as the time
+  // constant (and close to constant for a steady frame rate). In
+  // addition, the frequency error |c_1 - 1| should be small. Cameras
+  // with a frequency error up to 3000 ppm (3 ms drift per second)
+  // have been observed, but frequency errors below 100 ppm could be
+  // expected of any cheap crystal.
+  //
+  // Bottom line is that we ignore the c_1 term, and use only the estimator
+  //
+  //    x_k + mean(y-x)
+  //
+  // where mean is plain averaging for initial samples, followed by
+  // exponential averaging.
+
+  // TODO(nisse): Don't read the clock here, instead let the caller
+  // pass in the current system time? Useful for testing, or if the
+  // application reads the system clock earlier.
+  int64_t diff_us = rtc::TimeMicros() - capture_time_us;
+
+  // We also try to detect if the camera timestamp actually is using
+  // the system monotonic clock (a common case, for cameras without
+  // builtin timestamping). In this case, we aim to keep the
+  // timestamps as if, i.e., set the offset to zero. In case the
+  // camera clock is drifting, and by chance the offset is crossing
+  // zero, this hack will only cause a small dent in the otherwise
+  // linear offset curve, temporarily forcing it closer to zero.
+  static const int64_t kDelayLimit = 50 * rtc::kNumMicrosecsPerMillisec;

[stefan-webrtc, 2016/05/27 00:49:10]

Maybe just kDelayLimitUs = 50000? Easier to read, IMO.

[nisse-webrtc, 2016/05/27 09:47:33]

Done.

+  if (diff_us > 0 && diff_us < kDelayLimit)
+    diff_us = 0;
+
+  // TODO(nisse): Do we need to detect jumps in the camera clock?
+  // E.g., if the camera is somehow reset mid-stream? We could check
+  // if abs(diff_us - offset_us) > 500ms or so, and in this case reset
+  // frames_seen_ to zero.
+  static const unsigned kWindowSize = 100;
+  if (frames_seen_ < kWindowSize)
+    frames_seen_++;

[stefan-webrtc, 2016/05/27 00:49:10]

++frames_seen_

[nisse-webrtc, 2016/05/27 09:47:33]

Done.

+
+  offset_us_ += diff_us / frames_seen_;

[qiangchen, 2016/05/26 20:01:35]

Can you double check the correctness of the math behind?

 
Here frames_seen_ is a variable, not a constant.
Suppose we have two arrays of observed value of x_k and y_k.

This would be
offset_us_ = (y_1-x_1) + (y_2 - x_2)/2 + (y_3 - x_3)/3 ...

Not average of (y_k - x_k).


based on your comment above I think the code should be
if (frames_seen_ < kWindowSize) {
  offset_us_ *= frames_seen_;
  ++ frames_seen_;
  offset_us_ /= frames_seen_;
  offset_us_ += diff_us /frames_seen_;
} else {
  offset_us_ = offset_us_ * rate + diff_us * (1-rate);
}

where |rate| is a pre-determined parameter in the exponential weighted average
formula.

[stefan-webrtc, 2016/05/27 00:49:10]

That's what I would have expected too.

[nisse-webrtc, 2016/05/27 09:47:32]

Fixed.

[qiangchen, 2016/05/27 15:43:50]

Acknowledged.

+}
+
 bool VideoCapturer::AdaptFrame(int width,
                                int height,
-                               // TODO(nisse): Switch to us unit.
+                               // TODO(nisse): Switch to us unit. In
+                               // progress in different cl.
                                int64_t capture_time_ns,
                                int* out_width,
                                int* out_height,
                                int* crop_width,
                                int* crop_height,
                                int* crop_x,
-                               int* crop_y) {
+                               int* crop_y,
+                               int64_t* time_us) {
+  int64_t capture_time_us = capture_time_ns / rtc::kNumNanosecsPerMicrosec;
+  UpdateOffset(capture_time_us);
+
   if (!broadcaster_.frame_wanted()) {
     return false;
   }
@@ -245,6 +314,7 @@ bool VideoCapturer::AdaptFrame(int width,
     *crop_x = 0;
     *crop_y = 0;
   }
+  *time_us = capture_time_us + offset_us_;
   return true;
 }
 
@@ -256,11 +326,12 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
   int crop_height;
   int crop_x;
   int crop_y;
+  int64_t time_us;
 
   if (!AdaptFrame(captured_frame->width, captured_frame->height,
                   captured_frame->time_stamp,
                   &out_width, &out_height,
-                  &crop_width, &crop_height, &crop_x, &crop_y)) {
+                  &crop_width, &crop_height, &crop_x, &crop_y, &time_us)) {
     return;
   }
 
@@ -269,6 +340,8 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
     return;
   }
 
+  frame_factory_->SetTimestampOffset(offset_us_);
+
   // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y
   // are ignored for now.
   std::unique_ptr<VideoFrame> adapted_frame(frame_factory_->CreateAliasedFrame(