Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(89)

Side by Side Diff: webrtc/modules/video_coding/generic_encoder.cc

Issue 2968153002: Refactor timing frame logic to work with encoders with internal sources (Closed)
Patch Set: Implement Sprang@ comments Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license 4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source 5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found 6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may 7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree. 8 * be found in the AUTHORS file in the root of the source tree.
9 */ 9 */
10 10
(...skipping 198 matching lines...) Expand 10 before | Expand all | Expand 10 after
209 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms[capture_time_ms] = 209 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms[capture_time_ms] =
210 rtc::TimeMillis(); 210 rtc::TimeMillis();
211 } 211 }
212 212
213 EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( 213 EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage(
214 const EncodedImage& encoded_image, 214 const EncodedImage& encoded_image,
215 const CodecSpecificInfo* codec_specific, 215 const CodecSpecificInfo* codec_specific,
216 const RTPFragmentationHeader* fragmentation_header) { 216 const RTPFragmentationHeader* fragmentation_header) {
217 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded", 217 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
218 "timestamp", encoded_image._timeStamp); 218 "timestamp", encoded_image._timeStamp);
219 bool is_timing_frame = false;
220 size_t outlier_frame_size = 0;
221 int64_t encode_start_ms = -1;
222 size_t simulcast_svc_idx = 0; 219 size_t simulcast_svc_idx = 0;
223 if (codec_specific->codecType == kVideoCodecVP9) { 220 if (codec_specific->codecType == kVideoCodecVP9) {
224 if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1) 221 if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1)
225 simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx; 222 simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx;
226 } else if (codec_specific->codecType == kVideoCodecVP8) { 223 } else if (codec_specific->codecType == kVideoCodecVP8) {
227 simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx; 224 simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx;
228 } else if (codec_specific->codecType == kVideoCodecGeneric) { 225 } else if (codec_specific->codecType == kVideoCodecGeneric) {
229 simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx; 226 simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx;
230 } else if (codec_specific->codecType == kVideoCodecH264) { 227 } else if (codec_specific->codecType == kVideoCodecH264) {
231 // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here. 228 // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here.
232 } 229 }
233 230
231 // Larger than current frame size, so it will not trigger timing frame as an
232 // outlier. If encoder didn't call OnFramerateChanged it will be used.
233 size_t outlier_frame_size = encoded_image._length + 1;
234 // Default best guess. If encoder didn't call OnEncodeStarted it will be used.
235 int64_t encode_start_ms = encoded_image.capture_time_ms_;
236 bool is_timing_frame = false;
234 { 237 {
235 rtc::CritScope crit(&timing_params_lock_); 238 rtc::CritScope crit(&timing_params_lock_);
236 // TODO(ilnik): Workaround for hardware encoders, which do not call 239
237 // |OnEncodeStarted| correctly. Once fixed, remove conditional check. 240 // Encoders with internal sources do not call OnEncodeStarted and
241 // OnFrameRateChanged. |timing_frames_info_| may be not filled here.
238 if (simulcast_svc_idx < timing_frames_info_.size()) { 242 if (simulcast_svc_idx < timing_frames_info_.size()) {
239 RTC_CHECK_LT(simulcast_svc_idx, timing_frames_info_.size());
240
241 auto encode_start_map = 243 auto encode_start_map =
242 &timing_frames_info_[simulcast_svc_idx].encode_start_time_ms; 244 &timing_frames_info_[simulcast_svc_idx].encode_start_time_ms;
243 auto it = encode_start_map->find(encoded_image.capture_time_ms_); 245 auto it = encode_start_map->find(encoded_image.capture_time_ms_);
244 if (it != encode_start_map->end()) { 246 if (it != encode_start_map->end()) {
245 encode_start_ms = it->second; 247 encode_start_ms = it->second;
246 // Assuming all encoders do not reorder frames within single stream, 248 // Assuming all encoders do not reorder frames within single stream,
247 // there may be some dropped frames with smaller timestamps. These 249 // there may be some dropped frames with smaller timestamps. These
248 // should be purged. 250 // should be purged.
249 encode_start_map->erase(encode_start_map->begin(), it); 251 encode_start_map->erase(encode_start_map->begin(), it);
250 encode_start_map->erase(it); 252 encode_start_map->erase(it);
251 } else { 253 } else {
252 // Some chromium remoting unittests use generic encoder incorrectly 254 // Encoder is with internal source: free our records of any frames just
253 // If timestamps do not match, purge them all. 255 // in case to free memory.
254 encode_start_map->erase(encode_start_map->begin(), 256 encode_start_map->clear();
255 encode_start_map->end());
256 } 257 }
257 258
258 int64_t timing_frame_delay_ms = 259 size_t target_bitrate =
259 encoded_image.capture_time_ms_ - last_timing_frame_time_ms_; 260 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec;
260 if (last_timing_frame_time_ms_ == -1 || 261 if (framerate_ > 0 && target_bitrate > 0) {
261 timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms || 262 // framerate and target bitrate were reported by encoder.
262 timing_frame_delay_ms == 0) { 263 size_t average_frame_size = target_bitrate / framerate_;
263 is_timing_frame = true;
264 last_timing_frame_time_ms_ = encoded_image.capture_time_ms_;
265 }
266 // TODO(ilnik): Once OnFramerateChanged is called correctly by hardware
267 // encoders, remove the conditional check below.
268 if (framerate_ > 0) {
269 RTC_CHECK_GT(framerate_, 0);
270 size_t average_frame_size =
271 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec
272 / framerate_;
273 outlier_frame_size = average_frame_size * 264 outlier_frame_size = average_frame_size *
274 timing_frames_thresholds_.outlier_ratio_percent / 265 timing_frames_thresholds_.outlier_ratio_percent /
275 100; 266 100;
276 } else {
277 outlier_frame_size = encoded_image._length + 1;
278 } 267 }
279 } else { 268 }
280 // We don't have any information prior to encode start, thus we can't 269
281 // reliably detect outliers. Set outlier size to anything larger than 270 // Check if it's time to send a timing frame.
282 // current frame size. 271 int64_t timing_frame_delay_ms =
283 outlier_frame_size = encoded_image._length + 1; 272 encoded_image.capture_time_ms_ - last_timing_frame_time_ms_;
273 // Trigger threshold if it's a first frame, too long passed since the last
274 // timing frame, or we already sent timing frame on a different simulcast
275 // stream with the same capture time.
276 if (last_timing_frame_time_ms_ == -1 ||
277 timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms ||
278 timing_frame_delay_ms == 0) {
279 is_timing_frame = true;
280 last_timing_frame_time_ms_ = encoded_image.capture_time_ms_;
281 }
282
283 // Outliers trigger timing frames, but do not affect scheduled timing
284 // frames.
285 if (encoded_image._length >= outlier_frame_size) {
286 is_timing_frame = true;
284 } 287 }
285 } 288 }
286 289
287 if (encoded_image._length >= outlier_frame_size) { 290 if (is_timing_frame) {
288 is_timing_frame = true;
289 }
290 if (encode_start_ms >= 0 && is_timing_frame) {
291 encoded_image.SetEncodeTime(encode_start_ms, rtc::TimeMillis()); 291 encoded_image.SetEncodeTime(encode_start_ms, rtc::TimeMillis());
292 } 292 }
293 293
294 Result result = post_encode_callback_->OnEncodedImage( 294 Result result = post_encode_callback_->OnEncodedImage(
295 encoded_image, codec_specific, fragmentation_header); 295 encoded_image, codec_specific, fragmentation_header);
296 if (result.error != Result::OK) 296 if (result.error != Result::OK)
297 return result; 297 return result;
298 298
299 if (media_opt_) { 299 if (media_opt_) {
300 media_opt_->UpdateWithEncodedData(encoded_image); 300 media_opt_->UpdateWithEncodedData(encoded_image);
301 if (internal_source_) { 301 if (internal_source_) {
302 // Signal to encoder to drop next frame. 302 // Signal to encoder to drop next frame.
303 result.drop_next_frame = media_opt_->DropFrame(); 303 result.drop_next_frame = media_opt_->DropFrame();
304 } 304 }
305 } 305 }
306 return result; 306 return result;
307 } 307 }
308 308
309 } // namespace webrtc 309 } // namespace webrtc
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698