Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(246)

Side by Side Diff: webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm

Issue 2398963003: Move usage of QualityScaler to ViEEncoder. (Closed)
Patch Set: Stats now properly reflect scaling status Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license 4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source 5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found 6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may 7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree. 8 * be found in the AUTHORS file in the root of the source tree.
9 * 9 *
10 */ 10 */
(...skipping 216 matching lines...) Expand 10 before | Expand all | Expand 10 after
227 227
228 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { 228 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
229 DestroyCompressionSession(); 229 DestroyCompressionSession();
230 } 230 }
231 231
232 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, 232 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
233 int number_of_cores, 233 int number_of_cores,
234 size_t max_payload_size) { 234 size_t max_payload_size) {
235 RTC_DCHECK(codec_settings); 235 RTC_DCHECK(codec_settings);
236 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); 236 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
237 { 237
238 rtc::CritScope lock(&quality_scaler_crit_); 238 width_ = codec_settings->width;
239 quality_scaler_.Init(internal::kLowH264QpThreshold, 239 height_ = codec_settings->height;
240 internal::kHighH264QpThreshold,
241 codec_settings->startBitrate, codec_settings->width,
242 codec_settings->height, codec_settings->maxFramerate);
243 QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
244 // TODO(tkchin): We may need to enforce width/height dimension restrictions
245 // to match what the encoder supports.
246 width_ = res.width;
247 height_ = res.height;
248 }
249 // We can only set average bitrate on the HW encoder. 240 // We can only set average bitrate on the HW encoder.
250 target_bitrate_bps_ = codec_settings->startBitrate; 241 target_bitrate_bps_ = codec_settings->startBitrate;
251 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); 242 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
252 243
253 // TODO(tkchin): Try setting payload size via 244 // TODO(tkchin): Try setting payload size via
254 // kVTCompressionPropertyKey_MaxH264SliceBytes. 245 // kVTCompressionPropertyKey_MaxH264SliceBytes.
255 246
256 return ResetCompressionSession(); 247 return ResetCompressionSession();
257 } 248 }
258 249
259 int H264VideoToolboxEncoder::Encode( 250 int H264VideoToolboxEncoder::Encode(
260 const VideoFrame& frame, 251 const VideoFrame& frame,
261 const CodecSpecificInfo* codec_specific_info, 252 const CodecSpecificInfo* codec_specific_info,
262 const std::vector<FrameType>* frame_types) { 253 const std::vector<FrameType>* frame_types) {
263 RTC_DCHECK(!frame.IsZeroSize()); 254 RTC_DCHECK(!frame.IsZeroSize());
264 if (!callback_ || !compression_session_) { 255 if (!callback_ || !compression_session_) {
265 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 256 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
266 } 257 }
267 #if defined(WEBRTC_IOS) 258 #if defined(WEBRTC_IOS)
268 if (!RTCIsUIApplicationActive()) { 259 if (!RTCIsUIApplicationActive()) {
269 // Ignore all encode requests when app isn't active. In this state, the 260 // Ignore all encode requests when app isn't active. In this state, the
270 // hardware encoder has been invalidated by the OS. 261 // hardware encoder has been invalidated by the OS.
271 return WEBRTC_VIDEO_CODEC_OK; 262 return WEBRTC_VIDEO_CODEC_OK;
272 } 263 }
273 #endif 264 #endif
274 bool is_keyframe_required = false; 265 bool is_keyframe_required = false;
275 266
276 quality_scaler_.OnEncodeFrame(frame.width(), frame.height());
magjed_webrtc 2016/10/26 14:28:36 ditto: Please add: RTC_DCHECK_EQ(frame.width(), wi
kthelgason 2016/10/26 19:02:49 Done.
277 const QualityScaler::Resolution scaled_res =
278 quality_scaler_.GetScaledResolution();
279
280 if (scaled_res.width != width_ || scaled_res.height != height_) {
281 width_ = scaled_res.width;
282 height_ = scaled_res.height;
283 int ret = ResetCompressionSession();
284 if (ret < 0)
285 return ret;
286 }
287
288 // Get a pixel buffer from the pool and copy frame data over. 267 // Get a pixel buffer from the pool and copy frame data over.
289 CVPixelBufferPoolRef pixel_buffer_pool = 268 CVPixelBufferPoolRef pixel_buffer_pool =
290 VTCompressionSessionGetPixelBufferPool(compression_session_); 269 VTCompressionSessionGetPixelBufferPool(compression_session_);
291 #if defined(WEBRTC_IOS) 270 #if defined(WEBRTC_IOS)
292 if (!pixel_buffer_pool) { 271 if (!pixel_buffer_pool) {
293 // Kind of a hack. On backgrounding, the compression session seems to get 272 // Kind of a hack. On backgrounding, the compression session seems to get
294 // invalidated, which causes this pool call to fail when the application 273 // invalidated, which causes this pool call to fail when the application
295 // is foregrounded and frames are being sent for encoding again. 274 // is foregrounded and frames are being sent for encoding again.
296 // Resetting the session when this happens fixes the issue. 275 // Resetting the session when this happens fixes the issue.
297 // In addition we request a keyframe so video can recover quickly. 276 // In addition we request a keyframe so video can recover quickly.
(...skipping 21 matching lines...) Expand all
319 } 298 }
320 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer( 299 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(
321 nullptr, pixel_buffer_pool, &pixel_buffer); 300 nullptr, pixel_buffer_pool, &pixel_buffer);
322 if (ret != kCVReturnSuccess) { 301 if (ret != kCVReturnSuccess) {
323 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; 302 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
324 // We probably want to drop frames here, since failure probably means 303 // We probably want to drop frames here, since failure probably means
325 // that the pool is empty. 304 // that the pool is empty.
326 return WEBRTC_VIDEO_CODEC_ERROR; 305 return WEBRTC_VIDEO_CODEC_ERROR;
327 } 306 }
328 RTC_DCHECK(pixel_buffer); 307 RTC_DCHECK(pixel_buffer);
329 // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer 308 if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer(),
330 // conversion once libyuv::MergeUVPlanes is available.
331 rtc::scoped_refptr<VideoFrameBuffer> scaled_i420_buffer =
332 quality_scaler_.GetScaledBuffer(frame.video_frame_buffer());
333 if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer,
334 pixel_buffer)) { 309 pixel_buffer)) {
335 LOG(LS_ERROR) << "Failed to copy frame data."; 310 LOG(LS_ERROR) << "Failed to copy frame data.";
336 CVBufferRelease(pixel_buffer); 311 CVBufferRelease(pixel_buffer);
337 return WEBRTC_VIDEO_CODEC_ERROR; 312 return WEBRTC_VIDEO_CODEC_ERROR;
338 } 313 }
339 } 314 }
340 315
341 // Check if we need a keyframe. 316 // Check if we need a keyframe.
342 if (!is_keyframe_required && frame_types) { 317 if (!is_keyframe_required && frame_types) {
343 for (auto frame_type : *frame_types) { 318 for (auto frame_type : *frame_types) {
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
379 } 354 }
380 return WEBRTC_VIDEO_CODEC_OK; 355 return WEBRTC_VIDEO_CODEC_OK;
381 } 356 }
382 357
383 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( 358 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
384 EncodedImageCallback* callback) { 359 EncodedImageCallback* callback) {
385 callback_ = callback; 360 callback_ = callback;
386 return WEBRTC_VIDEO_CODEC_OK; 361 return WEBRTC_VIDEO_CODEC_OK;
387 } 362 }
388 363
389 void H264VideoToolboxEncoder::OnDroppedFrame() {
390 rtc::CritScope lock(&quality_scaler_crit_);
391 quality_scaler_.ReportDroppedFrame();
392 }
393
394 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, 364 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
395 int64_t rtt) { 365 int64_t rtt) {
396 // Encoder doesn't know anything about packet loss or rtt so just return. 366 // Encoder doesn't know anything about packet loss or rtt so just return.
397 return WEBRTC_VIDEO_CODEC_OK; 367 return WEBRTC_VIDEO_CODEC_OK;
398 } 368 }
399 369
400 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, 370 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
401 uint32_t frame_rate) { 371 uint32_t frame_rate) {
402 target_bitrate_bps_ = 1000 * new_bitrate_kbit; 372 target_bitrate_bps_ = 1000 * new_bitrate_kbit;
403 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); 373 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
404 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); 374 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
405
406 rtc::CritScope lock(&quality_scaler_crit_);
407 quality_scaler_.ReportFramerate(frame_rate);
408
409 return WEBRTC_VIDEO_CODEC_OK; 375 return WEBRTC_VIDEO_CODEC_OK;
410 } 376 }
411 377
412 int H264VideoToolboxEncoder::Release() { 378 int H264VideoToolboxEncoder::Release() {
413 // Need to reset so that the session is invalidated and won't use the 379 // Need to reset so that the session is invalidated and won't use the
414 // callback anymore. Do not remove callback until the session is invalidated 380 // callback anymore. Do not remove callback until the session is invalidated
415 // since async encoder callbacks can occur until invalidation. 381 // since async encoder callbacks can occur until invalidation.
416 int ret = ResetCompressionSession(); 382 int ret = ResetCompressionSession();
417 callback_ = nullptr; 383 callback_ = nullptr;
418 return ret; 384 return ret;
(...skipping 152 matching lines...) Expand 10 before | Expand all | Expand 10 after
571 int32_t height, 537 int32_t height,
572 int64_t render_time_ms, 538 int64_t render_time_ms,
573 uint32_t timestamp, 539 uint32_t timestamp,
574 VideoRotation rotation) { 540 VideoRotation rotation) {
575 if (status != noErr) { 541 if (status != noErr) {
576 LOG(LS_ERROR) << "H264 encode failed."; 542 LOG(LS_ERROR) << "H264 encode failed.";
577 return; 543 return;
578 } 544 }
579 if (info_flags & kVTEncodeInfo_FrameDropped) { 545 if (info_flags & kVTEncodeInfo_FrameDropped) {
580 LOG(LS_INFO) << "H264 encode dropped frame."; 546 LOG(LS_INFO) << "H264 encode dropped frame.";
581 rtc::CritScope lock(&quality_scaler_crit_);
582 quality_scaler_.ReportDroppedFrame();
583 return; 547 return;
584 } 548 }
585 549
586 bool is_keyframe = false; 550 bool is_keyframe = false;
587 CFArrayRef attachments = 551 CFArrayRef attachments =
588 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); 552 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
589 if (attachments != nullptr && CFArrayGetCount(attachments)) { 553 if (attachments != nullptr && CFArrayGetCount(attachments)) {
590 CFDictionaryRef attachment = 554 CFDictionaryRef attachment =
591 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); 555 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
592 is_keyframe = 556 is_keyframe =
(...skipping 21 matching lines...) Expand all
614 frame._encodedWidth = width; 578 frame._encodedWidth = width;
615 frame._encodedHeight = height; 579 frame._encodedHeight = height;
616 frame._completeFrame = true; 580 frame._completeFrame = true;
617 frame._frameType = 581 frame._frameType =
618 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; 582 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
619 frame.capture_time_ms_ = render_time_ms; 583 frame.capture_time_ms_ = render_time_ms;
620 frame._timeStamp = timestamp; 584 frame._timeStamp = timestamp;
621 frame.rotation_ = rotation; 585 frame.rotation_ = rotation;
622 586
623 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); 587 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
624 int qp; 588 h264_bitstream_parser_.GetLastSliceQp(&frame.qp_);
625 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
626 rtc::CritScope lock(&quality_scaler_crit_);
627 quality_scaler_.ReportQP(qp);
628 }
629 589
630 int result = callback_->Encoded(frame, &codec_specific_info, header.get()); 590 int result = callback_->Encoded(frame, &codec_specific_info, header.get());
631 if (result != 0) { 591 if (result != 0) {
632 LOG(LS_ERROR) << "Encode callback failed: " << result; 592 LOG(LS_ERROR) << "Encode callback failed: " << result;
633 return; 593 return;
634 } 594 }
635 bitrate_adjuster_.Update(frame._size); 595 bitrate_adjuster_.Update(frame._size);
636 } 596 }
637 597
598 QualityScaler::Settings H264VideoToolboxEncoder::GetQPThresholds() const {
599 return QualityScaler::Settings(true, internal::kLowH264QpThreshold,
600 internal::kHighH264QpThreshold);
601 }
638 } // namespace webrtc 602 } // namespace webrtc
639 603
640 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) 604 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698