OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
52 if (!size || data_size == CapturedFrame::kUnknownDataSize) { | 52 if (!size || data_size == CapturedFrame::kUnknownDataSize) { |
53 return false; | 53 return false; |
54 } | 54 } |
55 *size = data_size; | 55 *size = data_size; |
56 return true; | 56 return true; |
57 } | 57 } |
58 | 58 |
59 ///////////////////////////////////////////////////////////////////// | 59 ///////////////////////////////////////////////////////////////////// |
60 // Implementation of class VideoCapturer | 60 // Implementation of class VideoCapturer |
61 ///////////////////////////////////////////////////////////////////// | 61 ///////////////////////////////////////////////////////////////////// |
62 VideoCapturer::VideoCapturer() : apply_rotation_(false) { | 62 VideoCapturer::VideoCapturer() |
63 : apply_rotation_(false), frames_seen_(0), offset_us_(0) { | |
63 thread_checker_.DetachFromThread(); | 64 thread_checker_.DetachFromThread(); |
64 Construct(); | 65 Construct(); |
65 } | 66 } |
66 | 67 |
67 void VideoCapturer::Construct() { | 68 void VideoCapturer::Construct() { |
68 enable_camera_list_ = false; | 69 enable_camera_list_ = false; |
69 capture_state_ = CS_STOPPED; | 70 capture_state_ = CS_STOPPED; |
70 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); | 71 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); |
71 scaled_width_ = 0; | 72 scaled_width_ = 0; |
72 scaled_height_ = 0; | 73 scaled_height_ = 0; |
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
207 if (frame_factory_) { | 208 if (frame_factory_) { |
208 frame_factory_->SetApplyRotation(apply_rotation_); | 209 frame_factory_->SetApplyRotation(apply_rotation_); |
209 } | 210 } |
210 | 211 |
211 if (video_adapter()) { | 212 if (video_adapter()) { |
212 video_adapter()->OnResolutionRequest(wants.max_pixel_count, | 213 video_adapter()->OnResolutionRequest(wants.max_pixel_count, |
213 wants.max_pixel_count_step_up); | 214 wants.max_pixel_count_step_up); |
214 } | 215 } |
215 } | 216 } |
216 | 217 |
218 void VideoCapturer::UpdateOffset(int64_t camera_time_us, | |
219 int64_t system_time_us) { | |
220 // Estimate the offset between system monotonic time and the capture | |
221 // time from the camera. The camera is assumed to provide more | |
222 // accurate timestamps than we get from the system time. But the | |
223 // camera may use its own free-running clock with a large offset and | |
224 // a small drift compared to the system clock. So the model is | |
225 // basically | |
226 // | |
227 // y_k = c_0 + c_1 * x_k + v_k | |
228 // | |
229 // where x_k is the camera timestamp, believed to be accurate in its | |
230 // own scale. y_k is our reading of the system clock. v_k is the | |
231 // measurement noise, i.e., the delay from frame capture until the | |
232 // system clock was read. | |
233 // | |
234 // It's possible to do (weighted) least-squares estimation of both | |
235 // c_0 and c_1. Then we get the constants as c_1 = Cov(x,y) / | |
236 // Var(x), and c_0 = mean(y) - c_1 * mean(x). Substituting this c_0, | |
237 // we can rearrange the model as | |
238 // | |
239 // y_k = mean(y) + (x_k - mean(x)) + (c_1 - 1) * (x_k - mean(x)) + v_k | |
240 // | |
241 // Now if we use a weighted average which gradually forgets old | |
242 // values, x_k - mean(x) is bounded, of the same order as the time | |
243 // constant (and close to constant for a steady frame rate). In | |
244 // addition, the frequency error |c_1 - 1| should be small. Cameras | |
245 // with a frequency error up to 3000 ppm (3 ms drift per second) | |
246 // have been observed, but frequency errors below 100 ppm could be | |
247 // expected of any cheap crystal. | |
248 // | |
249 // Bottom line is that we ignore the c_1 term, and use only the estimator | |
250 // | |
251 // x_k + mean(y-x) | |
252 // | |
253 // where mean is plain averaging for initial samples, followed by | |
254 // exponential averaging. | |
255 | |
256 // The input for averaging, y_k - x_k in the above notation. | |
257 int64_t diff_us = system_time_us - camera_time_us; | |
258 // The deviation from the current average. | |
259 int64_t error_us = diff_us - offset_us_; | |
260 | |
261 // If the current difference is far from the currently estimated | |
262 // offset, the filter is reset. This could happen, e.g., if the | |
263 // camera clock is reset, or cameras are plugged in and out, or if | |
264 // the application process is temporarily suspended. The limit of | |
265 // 300 ms should make this unlikely in normal operation, and at the | |
266 // same time, converging gradually rather than resetting the filter | |
267 // should be tolerable for jumps in camera time below this | |
268 // threshold. | |
269 static const int64_t kResetLimitUs = 300000; | |
270 if (std::abs(error_us) > kResetLimitUs) { | |
271 LOG(LS_INFO) << "Resetting timestamp translation after averaging " | |
272 << frames_seen_ << " frames. Old offset: " | |
273 << offset_us_ << ", new offset: " << diff_us; | |
274 frames_seen_ = 0; | |
275 prev_translated_time_us_ = rtc::Optional<int64_t>(); | |
276 } | |
277 | |
278 static const unsigned kWindowSize = 100; | |
279 if (frames_seen_ < kWindowSize) { | |
280 ++frames_seen_; | |
281 } | |
282 offset_us_ += error_us / frames_seen_; | |
283 } | |
sprang_webrtc
2016/06/17 13:10:52
Wow, quite extensive commenting there! Guess it wi
| |
284 | |
285 int64_t VideoCapturer::ClipTimestamp(int64_t time_us, int64_t system_time_us) { | |
286 // Make timestamps monotonic. | |
287 if (!prev_translated_time_us_) { | |
288 // Initialize. | |
289 clip_bias_us_ = 0; | |
290 } else if (time_us < *prev_translated_time_us_) { | |
291 time_us = *prev_translated_time_us_; | |
292 } | |
293 | |
294 // Clip to make sure we don't produce time stamps in the future. | |
295 time_us -= clip_bias_us_; | |
296 if (time_us > system_time_us) { | |
297 clip_bias_us_ += time_us - system_time_us; | |
298 time_us = system_time_us; | |
299 } | |
300 prev_translated_time_us_ = rtc::Optional<int64_t>(time_us); | |
301 return time_us; | |
302 } | |
303 | |
217 bool VideoCapturer::AdaptFrame(int width, | 304 bool VideoCapturer::AdaptFrame(int width, |
218 int height, | 305 int height, |
219 // TODO(nisse): Switch to us unit. | 306 int64_t camera_time_us, |
220 int64_t capture_time_ns, | 307 int64_t system_time_us, |
221 int* out_width, | 308 int* out_width, |
222 int* out_height, | 309 int* out_height, |
223 int* crop_width, | 310 int* crop_width, |
224 int* crop_height, | 311 int* crop_height, |
225 int* crop_x, | 312 int* crop_x, |
226 int* crop_y) { | 313 int* crop_y, |
314 int64_t* translated_camera_time_us) { | |
315 if (translated_camera_time_us) { | |
316 UpdateOffset(camera_time_us, system_time_us); | |
317 } | |
227 if (!broadcaster_.frame_wanted()) { | 318 if (!broadcaster_.frame_wanted()) { |
228 return false; | 319 return false; |
229 } | 320 } |
230 | 321 |
231 if (enable_video_adapter_ && !IsScreencast()) { | 322 if (enable_video_adapter_ && !IsScreencast()) { |
232 if (!video_adapter_.AdaptFrameResolution( | 323 if (!video_adapter_.AdaptFrameResolution( |
233 width, height, capture_time_ns, | 324 width, height, camera_time_us * rtc::kNumNanosecsPerMicrosec, |
234 crop_width, crop_height, out_width, out_height)) { | 325 crop_width, crop_height, out_width, out_height)) { |
235 // VideoAdapter dropped the frame. | 326 // VideoAdapter dropped the frame. |
236 return false; | 327 return false; |
237 } | 328 } |
238 *crop_x = (width - *crop_width) / 2; | 329 *crop_x = (width - *crop_width) / 2; |
239 *crop_y = (height - *crop_height) / 2; | 330 *crop_y = (height - *crop_height) / 2; |
240 } else { | 331 } else { |
241 *out_width = width; | 332 *out_width = width; |
242 *out_height = height; | 333 *out_height = height; |
243 *crop_width = width; | 334 *crop_width = width; |
244 *crop_height = height; | 335 *crop_height = height; |
245 *crop_x = 0; | 336 *crop_x = 0; |
246 *crop_y = 0; | 337 *crop_y = 0; |
247 } | 338 } |
339 | |
340 if (translated_camera_time_us) { | |
341 *translated_camera_time_us = | |
342 ClipTimestamp(camera_time_us + offset_us_, system_time_us); | |
343 } | |
248 return true; | 344 return true; |
249 } | 345 } |
250 | 346 |
251 void VideoCapturer::OnFrameCaptured(VideoCapturer*, | 347 void VideoCapturer::OnFrameCaptured(VideoCapturer*, |
252 const CapturedFrame* captured_frame) { | 348 const CapturedFrame* captured_frame) { |
253 int out_width; | 349 int out_width; |
254 int out_height; | 350 int out_height; |
255 int crop_width; | 351 int crop_width; |
256 int crop_height; | 352 int crop_height; |
257 int crop_x; | 353 int crop_x; |
258 int crop_y; | 354 int crop_y; |
259 | 355 |
356 // TODO(nisse): We don't do timestamp translation on this input | |
357 // path. It seems straight-forward to enable translation, but that | |
358 // breaks the WebRtcVideoEngine2Test.PropagatesInputFrameTimestamp | |
359 // test. Probably not worth the effort to fix, instead, try to | |
360 // delete or refactor all code using VideoFrameFactory and | |
361 // SignalCapturedFrame. | |
260 if (!AdaptFrame(captured_frame->width, captured_frame->height, | 362 if (!AdaptFrame(captured_frame->width, captured_frame->height, |
261 captured_frame->time_stamp, | 363 captured_frame->time_stamp / rtc::kNumNanosecsPerMicrosec, |
364 0, | |
262 &out_width, &out_height, | 365 &out_width, &out_height, |
263 &crop_width, &crop_height, &crop_x, &crop_y)) { | 366 &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) { |
264 return; | 367 return; |
265 } | 368 } |
266 | 369 |
267 if (!frame_factory_) { | 370 if (!frame_factory_) { |
268 LOG(LS_ERROR) << "No video frame factory."; | 371 LOG(LS_ERROR) << "No video frame factory."; |
269 return; | 372 return; |
270 } | 373 } |
271 | 374 |
272 // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y | 375 // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y |
273 // are ignored for now. | 376 // are ignored for now. |
(...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
433 void VideoCapturer::UpdateInputSize(int width, int height) { | 536 void VideoCapturer::UpdateInputSize(int width, int height) { |
434 // Update stats protected from fetches from different thread. | 537 // Update stats protected from fetches from different thread. |
435 rtc::CritScope cs(&frame_stats_crit_); | 538 rtc::CritScope cs(&frame_stats_crit_); |
436 | 539 |
437 input_size_valid_ = true; | 540 input_size_valid_ = true; |
438 input_width_ = width; | 541 input_width_ = width; |
439 input_height_ = height; | 542 input_height_ = height; |
440 } | 543 } |
441 | 544 |
442 } // namespace cricket | 545 } // namespace cricket |
OLD | NEW |