OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
52 if (!size || data_size == CapturedFrame::kUnknownDataSize) { | 52 if (!size || data_size == CapturedFrame::kUnknownDataSize) { |
53 return false; | 53 return false; |
54 } | 54 } |
55 *size = data_size; | 55 *size = data_size; |
56 return true; | 56 return true; |
57 } | 57 } |
58 | 58 |
59 ///////////////////////////////////////////////////////////////////// | 59 ///////////////////////////////////////////////////////////////////// |
60 // Implementation of class VideoCapturer | 60 // Implementation of class VideoCapturer |
61 ///////////////////////////////////////////////////////////////////// | 61 ///////////////////////////////////////////////////////////////////// |
62 VideoCapturer::VideoCapturer() : apply_rotation_(false) { | 62 VideoCapturer::VideoCapturer() |
63 : apply_rotation_(false), frames_seen_(0), offset_us_(0) { | |
63 thread_checker_.DetachFromThread(); | 64 thread_checker_.DetachFromThread(); |
64 Construct(); | 65 Construct(); |
65 } | 66 } |
66 | 67 |
67 void VideoCapturer::Construct() { | 68 void VideoCapturer::Construct() { |
68 enable_camera_list_ = false; | 69 enable_camera_list_ = false; |
69 capture_state_ = CS_STOPPED; | 70 capture_state_ = CS_STOPPED; |
70 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); | 71 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); |
71 scaled_width_ = 0; | 72 scaled_width_ = 0; |
72 scaled_height_ = 0; | 73 scaled_height_ = 0; |
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
207 if (frame_factory_) { | 208 if (frame_factory_) { |
208 frame_factory_->SetApplyRotation(apply_rotation_); | 209 frame_factory_->SetApplyRotation(apply_rotation_); |
209 } | 210 } |
210 | 211 |
211 if (video_adapter()) { | 212 if (video_adapter()) { |
212 video_adapter()->OnResolutionRequest(wants.max_pixel_count, | 213 video_adapter()->OnResolutionRequest(wants.max_pixel_count, |
213 wants.max_pixel_count_step_up); | 214 wants.max_pixel_count_step_up); |
214 } | 215 } |
215 } | 216 } |
216 | 217 |
218 void VideoCapturer::UpdateOffset( | |
219 int64_t camera_time_us, int64_t system_time_us) { | |
220 // Estimate the offset between system monotonic time and the capture | |
221 // time from the camera. The camera is assumed to provide more | |
222 // accurate timestamps than we can do here. But the camera may use | |
223 // its own free-running clock with a large offset and a small drift | |
224 // compared to the system clock. So the model is basically | |
225 // | |
226 // y_k = c_0 + c_1 x_k + v_k | |
227 // | |
228 // where x_k is the camera timestamp, believed to be accurate in its | |
229 // own scale. y_k is our reading of the system clock. v_k is the | |
230 // measurement noise, i.e., the delay from frame capture until we | |
231 // get here and read the clock. | |
232 // | |
233 // It's possible to do (weighted) least-squares estimation of both | |
234 // c_0 and c_1. Then we get the constants as c_1 = Cov(x,y) / | |
235 // Var(x), and c_0 = mean(y) - c_1 mean(x). Substituting this c_0, | |
236 // we can rearrange the model as | |
237 // | |
238 // y_k = mean(y) + (x_k - mean(x)) + (c_1 - 1) (x_k - mean(x)) + v_k | |
239 // | |
240 // Now if we use a weighted average which gradually forgets old | |
241 // values, x_k - mean(x) is bounded, of the same order as the time | |
242 // constant (and close to constant for a steady frame rate). In | |
243 // addition, the frequency error |c_1 - 1| should be small. Cameras | |
244 // with a frequency error up to 3000 ppm (3 ms drift per second) | |
245 // have been observed, but frequency errors below 100 ppm could be | |
246 // expected of any cheap crystal. | |
247 // | |
248 // Bottom line is that we ignore the c_1 term, and use only the estimator | |
249 // | |
250 // x_k + mean(y-x) | |
251 // | |
252 // where mean is plain averaging for initial samples, followed by | |
253 // exponential averaging. | |
254 | |
255 // TODO(nisse): Don't read the clock here, instead let the caller | |
256 // pass in the current system time? Useful for testing, or if the | |
257 // application reads the system clock earlier. | |
258 int64_t diff_us = system_time_us - camera_time_us; | |
259 | |
260 // We also try to detect if the camera timestamp actually is using | |
261 // the system monotonic clock (a common case, for cameras without | |
262 // builtin timestamping). In this case, we aim to keep the | |
263 // timestamps as if, i.e., set the offset to zero. In case the | |
264 // camera clock is drifting, and by chance the offset is crossing | |
265 // zero, this hack will only cause a small dent in the otherwise | |
266 // linear offset curve, temporarily forcing it closer to zero. | |
267 static const int64_t kDelayLimitUs = 50000; | |
268 if (diff_us > 0 && diff_us < kDelayLimitUs) { | |
269 diff_us = 0; | |
270 } | |
stefan-webrtc
2016/06/08 11:28:42
Why do we have to do this?
nisse-webrtc
2016/06/09 10:02:58
The above? I'd like the filtering to add jitter in
stefan-webrtc
2016/06/09 11:59:09
But if it's using system time, shouldn't diff_us b
nisse-webrtc
2016/06/10 12:37:57
I think you're right, it's unlikely to make any no
| |
271 // TODO(nisse): Do we need to detect jumps in the camera clock? | |
272 // E.g., if the camera is somehow reset mid-stream? We could check | |
273 // if abs(diff_us - offset_us) > 500ms or so, and in this case reset | |
274 // frames_seen_ to zero. | |
stefan-webrtc
2016/06/08 11:28:42
I'd suggest using cusum change detection to detect
nisse-webrtc
2016/06/09 10:02:58
I think cusum is overkill (and when possible, I tr
stefan-webrtc
2016/06/09 11:59:09
I think the latter approach sounds pretty good the
nisse-webrtc
2016/06/10 12:37:57
I changed it to the second approach changed, i.e.,
| |
275 static const unsigned kWindowSize = 100; | |
276 if (frames_seen_ < kWindowSize) { | |
277 ++frames_seen_; | |
278 } | |
279 offset_us_ += (diff_us - offset_us_) / frames_seen_; | |
280 } | |
281 | |
217 bool VideoCapturer::AdaptFrame(int width, | 282 bool VideoCapturer::AdaptFrame(int width, |
218 int height, | 283 int height, |
219 // TODO(nisse): Switch to us unit. | 284 int64_t camera_time_us, |
220 int64_t capture_time_ns, | 285 int64_t system_time_us, |
221 int* out_width, | 286 int* out_width, |
222 int* out_height, | 287 int* out_height, |
223 int* crop_width, | 288 int* crop_width, |
224 int* crop_height, | 289 int* crop_height, |
225 int* crop_x, | 290 int* crop_x, |
226 int* crop_y) { | 291 int* crop_y, |
292 int64_t* time_us) { | |
stefan-webrtc
2016/06/08 11:28:42
Should this be called something else? system_captu
nisse-webrtc
2016/06/09 10:02:58
Maybe. In some of the callers, I use the name tran
stefan-webrtc
2016/06/09 11:59:09
Both work for me.
| |
293 if (time_us) | |
294 UpdateOffset(camera_time_us, system_time_us); | |
295 | |
227 if (!broadcaster_.frame_wanted()) { | 296 if (!broadcaster_.frame_wanted()) { |
228 return false; | 297 return false; |
229 } | 298 } |
230 | 299 |
231 if (enable_video_adapter_ && !IsScreencast()) { | 300 if (enable_video_adapter_ && !IsScreencast()) { |
232 if (!video_adapter_.AdaptFrameResolution( | 301 if (!video_adapter_.AdaptFrameResolution( |
233 width, height, capture_time_ns, | 302 width, height, camera_time_us * rtc::kNumNanosecsPerMicrosec, |
234 crop_width, crop_height, out_width, out_height)) { | 303 crop_width, crop_height, out_width, out_height)) { |
235 // VideoAdapter dropped the frame. | 304 // VideoAdapter dropped the frame. |
236 return false; | 305 return false; |
237 } | 306 } |
238 *crop_x = (width - *crop_width) / 2; | 307 *crop_x = (width - *crop_width) / 2; |
239 *crop_y = (height - *crop_height) / 2; | 308 *crop_y = (height - *crop_height) / 2; |
240 } else { | 309 } else { |
241 *out_width = width; | 310 *out_width = width; |
242 *out_height = height; | 311 *out_height = height; |
243 *crop_width = width; | 312 *crop_width = width; |
244 *crop_height = height; | 313 *crop_height = height; |
245 *crop_x = 0; | 314 *crop_x = 0; |
246 *crop_y = 0; | 315 *crop_y = 0; |
247 } | 316 } |
317 | |
318 if (time_us) | |
319 *time_us = camera_time_us + offset_us_; | |
320 | |
248 return true; | 321 return true; |
249 } | 322 } |
250 | 323 |
251 void VideoCapturer::OnFrameCaptured(VideoCapturer*, | 324 void VideoCapturer::OnFrameCaptured(VideoCapturer*, |
252 const CapturedFrame* captured_frame) { | 325 const CapturedFrame* captured_frame) { |
253 int out_width; | 326 int out_width; |
254 int out_height; | 327 int out_height; |
255 int crop_width; | 328 int crop_width; |
256 int crop_height; | 329 int crop_height; |
257 int crop_x; | 330 int crop_x; |
258 int crop_y; | 331 int crop_y; |
259 | 332 |
260 if (!AdaptFrame(captured_frame->width, captured_frame->height, | 333 if (!AdaptFrame(captured_frame->width, captured_frame->height, |
261 captured_frame->time_stamp, | 334 captured_frame->time_stamp / rtc::kNumNanosecsPerMicrosec, |
335 // TODO(nisse): Disable use of the timestamp | |
stefan-webrtc
2016/06/08 11:28:42
Should this be fixed now?
nisse-webrtc
2016/06/09 10:02:58
The comment is perhaps badly phrased, I'm trying t
stefan-webrtc
2016/06/09 11:59:09
Acknowledged.
| |
336 // translation when using the SignalCapturedFrame | |
337 // interface. Enabling it breaks the | |
338 // WebRtcVideoEngine2Test.PropagatesInputFrameTimestamp | |
339 // test. Probably not worth the effort to fix, | |
340 // instead, try to get rid of this method. | |
341 0, | |
262 &out_width, &out_height, | 342 &out_width, &out_height, |
263 &crop_width, &crop_height, &crop_x, &crop_y)) { | 343 &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) { |
264 return; | 344 return; |
265 } | 345 } |
266 | 346 |
267 if (!frame_factory_) { | 347 if (!frame_factory_) { |
268 LOG(LS_ERROR) << "No video frame factory."; | 348 LOG(LS_ERROR) << "No video frame factory."; |
269 return; | 349 return; |
270 } | 350 } |
271 | 351 |
272 // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y | 352 // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y |
273 // are ignored for now. | 353 // are ignored for now. |
(...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
433 void VideoCapturer::UpdateInputSize(int width, int height) { | 513 void VideoCapturer::UpdateInputSize(int width, int height) { |
434 // Update stats protected from fetches from different thread. | 514 // Update stats protected from fetches from different thread. |
435 rtc::CritScope cs(&frame_stats_crit_); | 515 rtc::CritScope cs(&frame_stats_crit_); |
436 | 516 |
437 input_size_valid_ = true; | 517 input_size_valid_ = true; |
438 input_width_ = width; | 518 input_width_ = width; |
439 input_height_ = height; | 519 input_height_ = height; |
440 } | 520 } |
441 | 521 |
442 } // namespace cricket | 522 } // namespace cricket |
OLD | NEW |