OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
52 if (!size || data_size == CapturedFrame::kUnknownDataSize) { | 52 if (!size || data_size == CapturedFrame::kUnknownDataSize) { |
53 return false; | 53 return false; |
54 } | 54 } |
55 *size = data_size; | 55 *size = data_size; |
56 return true; | 56 return true; |
57 } | 57 } |
58 | 58 |
59 ///////////////////////////////////////////////////////////////////// | 59 ///////////////////////////////////////////////////////////////////// |
60 // Implementation of class VideoCapturer | 60 // Implementation of class VideoCapturer |
61 ///////////////////////////////////////////////////////////////////// | 61 ///////////////////////////////////////////////////////////////////// |
62 VideoCapturer::VideoCapturer() : apply_rotation_(false) { | 62 VideoCapturer::VideoCapturer() |
63 : apply_rotation_(false), frames_seen_(0), offset_us_(0) { | |
63 thread_checker_.DetachFromThread(); | 64 thread_checker_.DetachFromThread(); |
64 Construct(); | 65 Construct(); |
65 } | 66 } |
66 | 67 |
67 void VideoCapturer::Construct() { | 68 void VideoCapturer::Construct() { |
68 enable_camera_list_ = false; | 69 enable_camera_list_ = false; |
69 capture_state_ = CS_STOPPED; | 70 capture_state_ = CS_STOPPED; |
70 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); | 71 SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured); |
71 scaled_width_ = 0; | 72 scaled_width_ = 0; |
72 scaled_height_ = 0; | 73 scaled_height_ = 0; |
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
207 if (frame_factory_) { | 208 if (frame_factory_) { |
208 frame_factory_->SetApplyRotation(apply_rotation_); | 209 frame_factory_->SetApplyRotation(apply_rotation_); |
209 } | 210 } |
210 | 211 |
211 if (video_adapter()) { | 212 if (video_adapter()) { |
212 video_adapter()->OnResolutionRequest(wants.max_pixel_count, | 213 video_adapter()->OnResolutionRequest(wants.max_pixel_count, |
213 wants.max_pixel_count_step_up); | 214 wants.max_pixel_count_step_up); |
214 } | 215 } |
215 } | 216 } |
216 | 217 |
218 void VideoCapturer::UpdateOffset( | |
219 int64_t camera_time_us, int64_t system_time_us) { | |
220 // Estimate the offset between system monotonic time and the capture | |
221 // time from the camera. The camera is assumed to provide more | |
222 // accurate timestamps than we get from the system time. But the | |
223 // camera may use its own free-running clock with a large offset and | |
224 // a small drift compared to the system clock. So the model is | |
225 // basically | |
226 // | |
227 // y_k = c_0 + c_1 x_k + v_k | |
228 // | |
229 // where x_k is the camera timestamp, believed to be accurate in its | |
230 // own scale. y_k is our reading of the system clock. v_k is the | |
231 // measurement noise, i.e., the delay from frame capture until the | |
232 // system clock was read. | |
233 // | |
234 // It's possible to do (weighted) least-squares estimation of both | |
235 // c_0 and c_1. Then we get the constants as c_1 = Cov(x,y) / | |
236 // Var(x), and c_0 = mean(y) - c_1 mean(x). Substituting this c_0, | |
237 // we can rearrange the model as | |
238 // | |
239 // y_k = mean(y) + (x_k - mean(x)) + (c_1 - 1) (x_k - mean(x)) + v_k | |
240 // | |
241 // Now if we use a weighted average which gradually forgets old | |
242 // values, x_k - mean(x) is bounded, of the same order as the time | |
243 // constant (and close to constant for a steady frame rate). In | |
244 // addition, the frequency error |c_1 - 1| should be small. Cameras | |
245 // with a frequency error up to 3000 ppm (3 ms drift per second) | |
246 // have been observed, but frequency errors below 100 ppm could be | |
247 // expected of any cheap crystal. | |
248 // | |
249 // Bottom line is that we ignore the c_1 term, and use only the estimator | |
250 // | |
251 // x_k + mean(y-x) | |
252 // | |
253 // where mean is plain averaging for initial samples, followed by | |
254 // exponential averaging. | |
255 | |
256 if (frames_seen_) { | |
257 // Reset the filter whenever there is a jump in camera timestamps. | |
258 // This happens both when capture is restarted, and if the camera | |
259 // clock for any reason is reset. | |
260 static const int64_t kResetLimitUs = 300000; | |
261 | |
262 if (camera_time_us < prev_camera_time_us_ | |
263 || camera_time_us > prev_camera_time_us_ + kResetLimitUs) { | |
stefan-webrtc
2016/06/10 13:13:31
Hm, what if we have only 1 fps input? I think we n
nisse-webrtc
2016/06/13 06:52:49
Then we'll ignore the camera timestamp and use onl
stefan-webrtc
2016/06/13 08:56:01
This is probably true, but it seems a bit ugly to
nisse-webrtc
2016/06/13 09:33:04
I've changed the reset logic back. Also added a lo
| |
264 frames_seen_ = 0; | |
265 } | |
266 } | |
267 prev_camera_time_us_ = camera_time_us; | |
268 | |
269 // The input for averaging, y_k - x_k in the above notation. | |
270 int64_t diff_us = system_time_us - camera_time_us; | |
271 | |
272 static const unsigned kWindowSize = 100; | |
273 if (frames_seen_ < kWindowSize) { | |
274 ++frames_seen_; | |
275 } | |
276 offset_us_ += (diff_us - offset_us_) / frames_seen_; | |
277 } | |
278 | |
217 bool VideoCapturer::AdaptFrame(int width, | 279 bool VideoCapturer::AdaptFrame(int width, |
218 int height, | 280 int height, |
219 // TODO(nisse): Switch to us unit. | 281 int64_t camera_time_us, |
220 int64_t capture_time_ns, | 282 int64_t system_time_us, |
221 int* out_width, | 283 int* out_width, |
222 int* out_height, | 284 int* out_height, |
223 int* crop_width, | 285 int* crop_width, |
224 int* crop_height, | 286 int* crop_height, |
225 int* crop_x, | 287 int* crop_x, |
226 int* crop_y) { | 288 int* crop_y, |
289 int64_t* translated_camera_time_us) { | |
290 if (translated_camera_time_us) | |
291 UpdateOffset(camera_time_us, system_time_us); | |
292 | |
227 if (!broadcaster_.frame_wanted()) { | 293 if (!broadcaster_.frame_wanted()) { |
228 return false; | 294 return false; |
229 } | 295 } |
230 | 296 |
231 if (enable_video_adapter_ && !IsScreencast()) { | 297 if (enable_video_adapter_ && !IsScreencast()) { |
232 if (!video_adapter_.AdaptFrameResolution( | 298 if (!video_adapter_.AdaptFrameResolution( |
233 width, height, capture_time_ns, | 299 width, height, camera_time_us * rtc::kNumNanosecsPerMicrosec, |
234 crop_width, crop_height, out_width, out_height)) { | 300 crop_width, crop_height, out_width, out_height)) { |
235 // VideoAdapter dropped the frame. | 301 // VideoAdapter dropped the frame. |
236 return false; | 302 return false; |
237 } | 303 } |
238 *crop_x = (width - *crop_width) / 2; | 304 *crop_x = (width - *crop_width) / 2; |
239 *crop_y = (height - *crop_height) / 2; | 305 *crop_y = (height - *crop_height) / 2; |
240 } else { | 306 } else { |
241 *out_width = width; | 307 *out_width = width; |
242 *out_height = height; | 308 *out_height = height; |
243 *crop_width = width; | 309 *crop_width = width; |
244 *crop_height = height; | 310 *crop_height = height; |
245 *crop_x = 0; | 311 *crop_x = 0; |
246 *crop_y = 0; | 312 *crop_y = 0; |
247 } | 313 } |
314 | |
315 if (translated_camera_time_us) | |
316 *translated_camera_time_us = camera_time_us + offset_us_; | |
317 | |
248 return true; | 318 return true; |
249 } | 319 } |
250 | 320 |
251 void VideoCapturer::OnFrameCaptured(VideoCapturer*, | 321 void VideoCapturer::OnFrameCaptured(VideoCapturer*, |
252 const CapturedFrame* captured_frame) { | 322 const CapturedFrame* captured_frame) { |
253 int out_width; | 323 int out_width; |
254 int out_height; | 324 int out_height; |
255 int crop_width; | 325 int crop_width; |
256 int crop_height; | 326 int crop_height; |
257 int crop_x; | 327 int crop_x; |
258 int crop_y; | 328 int crop_y; |
259 | 329 |
330 // TODO(nisse): We don't do timestamp translation on this input | |
331 // path. It seems straight-forward to enable translation, but that | |
332 // breaks the WebRtcVideoEngine2Test.PropagatesInputFrameTimestamp | |
333 // test. Probably not worth the effort to fix, instead, try to | |
334 // delete or refactor all code using VideoFrameFactory and | |
335 // SignalCapturedFrame. | |
260 if (!AdaptFrame(captured_frame->width, captured_frame->height, | 336 if (!AdaptFrame(captured_frame->width, captured_frame->height, |
261 captured_frame->time_stamp, | 337 captured_frame->time_stamp / rtc::kNumNanosecsPerMicrosec, |
338 0, | |
262 &out_width, &out_height, | 339 &out_width, &out_height, |
263 &crop_width, &crop_height, &crop_x, &crop_y)) { | 340 &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) { |
264 return; | 341 return; |
265 } | 342 } |
266 | 343 |
267 if (!frame_factory_) { | 344 if (!frame_factory_) { |
268 LOG(LS_ERROR) << "No video frame factory."; | 345 LOG(LS_ERROR) << "No video frame factory."; |
269 return; | 346 return; |
270 } | 347 } |
271 | 348 |
272 // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y | 349 // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y |
273 // are ignored for now. | 350 // are ignored for now. |
(...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
433 void VideoCapturer::UpdateInputSize(int width, int height) { | 510 void VideoCapturer::UpdateInputSize(int width, int height) { |
434 // Update stats protected from fetches from different thread. | 511 // Update stats protected from fetches from different thread. |
435 rtc::CritScope cs(&frame_stats_crit_); | 512 rtc::CritScope cs(&frame_stats_crit_); |
436 | 513 |
437 input_size_valid_ = true; | 514 input_size_valid_ = true; |
438 input_width_ = width; | 515 input_width_ = width; |
439 input_height_ = height; | 516 input_height_ = height; |
440 } | 517 } |
441 | 518 |
442 } // namespace cricket | 519 } // namespace cricket |
OLD | NEW |