OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 * | 9 * |
10 */ | 10 */ |
(...skipping 342 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
353 | 353 |
354 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { | 354 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { |
355 DestroyCompressionSession(); | 355 DestroyCompressionSession(); |
356 } | 356 } |
357 | 357 |
358 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, | 358 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, |
359 int number_of_cores, | 359 int number_of_cores, |
360 size_t max_payload_size) { | 360 size_t max_payload_size) { |
361 RTC_DCHECK(codec_settings); | 361 RTC_DCHECK(codec_settings); |
362 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); | 362 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); |
363 { | 363 |
364 rtc::CritScope lock(&quality_scaler_crit_); | 364 width_ = codec_settings->width; |
365 quality_scaler_.Init(internal::kLowH264QpThreshold, | 365 height_ = codec_settings->height; |
366 internal::kHighH264QpThreshold, | |
367 codec_settings->startBitrate, codec_settings->width, | |
368 codec_settings->height, codec_settings->maxFramerate); | |
369 QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); | |
370 // TODO(tkchin): We may need to enforce width/height dimension restrictions | |
371 // to match what the encoder supports. | |
372 width_ = res.width; | |
373 height_ = res.height; | |
374 } | |
375 // We can only set average bitrate on the HW encoder. | 366 // We can only set average bitrate on the HW encoder. |
376 target_bitrate_bps_ = codec_settings->startBitrate; | 367 target_bitrate_bps_ = codec_settings->startBitrate; |
377 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); | 368 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); |
378 | 369 |
379 // TODO(tkchin): Try setting payload size via | 370 // TODO(tkchin): Try setting payload size via |
380 // kVTCompressionPropertyKey_MaxH264SliceBytes. | 371 // kVTCompressionPropertyKey_MaxH264SliceBytes. |
381 | 372 |
382 return ResetCompressionSession(); | 373 return ResetCompressionSession(); |
383 } | 374 } |
384 | 375 |
385 int H264VideoToolboxEncoder::Encode( | 376 int H264VideoToolboxEncoder::Encode( |
386 const VideoFrame& frame, | 377 const VideoFrame& frame, |
387 const CodecSpecificInfo* codec_specific_info, | 378 const CodecSpecificInfo* codec_specific_info, |
388 const std::vector<FrameType>* frame_types) { | 379 const std::vector<FrameType>* frame_types) { |
| 380 // |input_frame| size should always match codec settings. |
| 381 RTC_DCHECK_EQ(frame.width(), width_); |
| 382 RTC_DCHECK_EQ(frame.height(), height_); |
389 RTC_DCHECK(!frame.IsZeroSize()); | 383 RTC_DCHECK(!frame.IsZeroSize()); |
390 if (!callback_ || !compression_session_) { | 384 if (!callback_ || !compression_session_) { |
391 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 385 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
392 } | 386 } |
393 #if defined(WEBRTC_IOS) | 387 #if defined(WEBRTC_IOS) |
394 if (!RTCIsUIApplicationActive()) { | 388 if (!RTCIsUIApplicationActive()) { |
395 // Ignore all encode requests when app isn't active. In this state, the | 389 // Ignore all encode requests when app isn't active. In this state, the |
396 // hardware encoder has been invalidated by the OS. | 390 // hardware encoder has been invalidated by the OS. |
397 return WEBRTC_VIDEO_CODEC_OK; | 391 return WEBRTC_VIDEO_CODEC_OK; |
398 } | 392 } |
399 #endif | 393 #endif |
400 bool is_keyframe_required = false; | 394 bool is_keyframe_required = false; |
401 | 395 |
402 quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); | |
403 const QualityScaler::Resolution scaled_res = | |
404 quality_scaler_.GetScaledResolution(); | |
405 | |
406 if (scaled_res.width != width_ || scaled_res.height != height_) { | |
407 width_ = scaled_res.width; | |
408 height_ = scaled_res.height; | |
409 int ret = ResetCompressionSession(); | |
410 if (ret < 0) | |
411 return ret; | |
412 } | |
413 | |
414 // Get a pixel buffer from the pool and copy frame data over. | 396 // Get a pixel buffer from the pool and copy frame data over. |
415 CVPixelBufferPoolRef pixel_buffer_pool = | 397 CVPixelBufferPoolRef pixel_buffer_pool = |
416 VTCompressionSessionGetPixelBufferPool(compression_session_); | 398 VTCompressionSessionGetPixelBufferPool(compression_session_); |
417 #if defined(WEBRTC_IOS) | 399 #if defined(WEBRTC_IOS) |
418 if (!pixel_buffer_pool) { | 400 if (!pixel_buffer_pool) { |
419 // Kind of a hack. On backgrounding, the compression session seems to get | 401 // Kind of a hack. On backgrounding, the compression session seems to get |
420 // invalidated, which causes this pool call to fail when the application | 402 // invalidated, which causes this pool call to fail when the application |
421 // is foregrounded and frames are being sent for encoding again. | 403 // is foregrounded and frames are being sent for encoding again. |
422 // Resetting the session when this happens fixes the issue. | 404 // Resetting the session when this happens fixes the issue. |
423 // In addition we request a keyframe so video can recover quickly. | 405 // In addition we request a keyframe so video can recover quickly. |
(...skipping 26 matching lines...) Expand all Loading... |
450 if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_, | 432 if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_, |
451 pixel_buffer)) { | 433 pixel_buffer)) { |
452 return WEBRTC_VIDEO_CODEC_ERROR; | 434 return WEBRTC_VIDEO_CODEC_ERROR; |
453 } | 435 } |
454 } | 436 } |
455 } else { | 437 } else { |
456 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); | 438 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); |
457 if (!pixel_buffer) { | 439 if (!pixel_buffer) { |
458 return WEBRTC_VIDEO_CODEC_ERROR; | 440 return WEBRTC_VIDEO_CODEC_ERROR; |
459 } | 441 } |
460 // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer | 442 RTC_DCHECK(pixel_buffer); |
461 // conversion once libyuv::MergeUVPlanes is available. | 443 if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer(), |
462 rtc::scoped_refptr<VideoFrameBuffer> scaled_i420_buffer = | |
463 quality_scaler_.GetScaledBuffer(frame.video_frame_buffer()); | |
464 if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer, | |
465 pixel_buffer)) { | 444 pixel_buffer)) { |
466 LOG(LS_ERROR) << "Failed to copy frame data."; | 445 LOG(LS_ERROR) << "Failed to copy frame data."; |
467 CVBufferRelease(pixel_buffer); | 446 CVBufferRelease(pixel_buffer); |
468 return WEBRTC_VIDEO_CODEC_ERROR; | 447 return WEBRTC_VIDEO_CODEC_ERROR; |
469 } | 448 } |
470 } | 449 } |
471 | 450 |
472 // Check if we need a keyframe. | 451 // Check if we need a keyframe. |
473 if (!is_keyframe_required && frame_types) { | 452 if (!is_keyframe_required && frame_types) { |
474 for (auto frame_type : *frame_types) { | 453 for (auto frame_type : *frame_types) { |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
510 } | 489 } |
511 return WEBRTC_VIDEO_CODEC_OK; | 490 return WEBRTC_VIDEO_CODEC_OK; |
512 } | 491 } |
513 | 492 |
514 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( | 493 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( |
515 EncodedImageCallback* callback) { | 494 EncodedImageCallback* callback) { |
516 callback_ = callback; | 495 callback_ = callback; |
517 return WEBRTC_VIDEO_CODEC_OK; | 496 return WEBRTC_VIDEO_CODEC_OK; |
518 } | 497 } |
519 | 498 |
520 void H264VideoToolboxEncoder::OnDroppedFrame() { | |
521 rtc::CritScope lock(&quality_scaler_crit_); | |
522 quality_scaler_.ReportDroppedFrame(); | |
523 } | |
524 | |
525 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, | 499 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, |
526 int64_t rtt) { | 500 int64_t rtt) { |
527 // Encoder doesn't know anything about packet loss or rtt so just return. | 501 // Encoder doesn't know anything about packet loss or rtt so just return. |
528 return WEBRTC_VIDEO_CODEC_OK; | 502 return WEBRTC_VIDEO_CODEC_OK; |
529 } | 503 } |
530 | 504 |
531 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, | 505 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, |
532 uint32_t frame_rate) { | 506 uint32_t frame_rate) { |
533 target_bitrate_bps_ = 1000 * new_bitrate_kbit; | 507 target_bitrate_bps_ = 1000 * new_bitrate_kbit; |
534 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); | 508 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); |
535 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); | 509 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); |
536 | |
537 rtc::CritScope lock(&quality_scaler_crit_); | |
538 quality_scaler_.ReportFramerate(frame_rate); | |
539 | |
540 return WEBRTC_VIDEO_CODEC_OK; | 510 return WEBRTC_VIDEO_CODEC_OK; |
541 } | 511 } |
542 | 512 |
543 int H264VideoToolboxEncoder::Release() { | 513 int H264VideoToolboxEncoder::Release() { |
544 // Need to reset so that the session is invalidated and won't use the | 514 // Need to reset so that the session is invalidated and won't use the |
545 // callback anymore. Do not remove callback until the session is invalidated | 515 // callback anymore. Do not remove callback until the session is invalidated |
546 // since async encoder callbacks can occur until invalidation. | 516 // since async encoder callbacks can occur until invalidation. |
547 int ret = ResetCompressionSession(); | 517 int ret = ResetCompressionSession(); |
548 callback_ = nullptr; | 518 callback_ = nullptr; |
549 return ret; | 519 return ret; |
(...skipping 152 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
702 int32_t height, | 672 int32_t height, |
703 int64_t render_time_ms, | 673 int64_t render_time_ms, |
704 uint32_t timestamp, | 674 uint32_t timestamp, |
705 VideoRotation rotation) { | 675 VideoRotation rotation) { |
706 if (status != noErr) { | 676 if (status != noErr) { |
707 LOG(LS_ERROR) << "H264 encode failed."; | 677 LOG(LS_ERROR) << "H264 encode failed."; |
708 return; | 678 return; |
709 } | 679 } |
710 if (info_flags & kVTEncodeInfo_FrameDropped) { | 680 if (info_flags & kVTEncodeInfo_FrameDropped) { |
711 LOG(LS_INFO) << "H264 encode dropped frame."; | 681 LOG(LS_INFO) << "H264 encode dropped frame."; |
712 rtc::CritScope lock(&quality_scaler_crit_); | |
713 quality_scaler_.ReportDroppedFrame(); | |
714 return; | 682 return; |
715 } | 683 } |
716 | 684 |
717 bool is_keyframe = false; | 685 bool is_keyframe = false; |
718 CFArrayRef attachments = | 686 CFArrayRef attachments = |
719 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); | 687 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); |
720 if (attachments != nullptr && CFArrayGetCount(attachments)) { | 688 if (attachments != nullptr && CFArrayGetCount(attachments)) { |
721 CFDictionaryRef attachment = | 689 CFDictionaryRef attachment = |
722 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); | 690 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); |
723 is_keyframe = | 691 is_keyframe = |
(...skipping 21 matching lines...) Expand all Loading... |
745 frame._encodedWidth = width; | 713 frame._encodedWidth = width; |
746 frame._encodedHeight = height; | 714 frame._encodedHeight = height; |
747 frame._completeFrame = true; | 715 frame._completeFrame = true; |
748 frame._frameType = | 716 frame._frameType = |
749 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; | 717 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; |
750 frame.capture_time_ms_ = render_time_ms; | 718 frame.capture_time_ms_ = render_time_ms; |
751 frame._timeStamp = timestamp; | 719 frame._timeStamp = timestamp; |
752 frame.rotation_ = rotation; | 720 frame.rotation_ = rotation; |
753 | 721 |
754 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); | 722 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); |
755 int qp; | 723 h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); |
756 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { | |
757 rtc::CritScope lock(&quality_scaler_crit_); | |
758 quality_scaler_.ReportQP(qp); | |
759 frame.qp_ = qp; | |
760 } | |
761 | 724 |
762 EncodedImageCallback::Result result = | 725 EncodedImageCallback::Result res = |
763 callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); | 726 callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); |
764 if (result.error != EncodedImageCallback::Result::OK) { | 727 if (res.error != EncodedImageCallback::Result::OK) { |
765 LOG(LS_ERROR) << "Encode callback failed: " << result.error; | 728 LOG(LS_ERROR) << "Encode callback failed: " << res.error; |
766 return; | 729 return; |
767 } | 730 } |
768 bitrate_adjuster_.Update(frame._size); | 731 bitrate_adjuster_.Update(frame._size); |
769 } | 732 } |
770 | 733 |
| 734 VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() |
| 735 const { |
| 736 return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, |
| 737 internal::kHighH264QpThreshold); |
| 738 } |
771 } // namespace webrtc | 739 } // namespace webrtc |
OLD | NEW |