Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(442)

Unified Diff: webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm

Issue 2890513002: Revert of Split iOS sdk in to separate targets (Closed)
Patch Set: Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm
diff --git a/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm b/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm
new file mode 100644
index 0000000000000000000000000000000000000000..596a72777bf2d25b65e2bbcc572c1bddcac39b4e
--- /dev/null
+++ b/webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.mm
@@ -0,0 +1,746 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_encoder.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#if defined(WEBRTC_IOS)
+#import "WebRTC/UIDevice+RTCDevice.h"
+#include "RTCUIApplication.h"
+#endif
+#include "libyuv/convert_from.h"
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/h264/profile_level_id.h"
+#include "webrtc/sdk/objc/Framework/Classes/corevideo_frame_buffer.h"
+#include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_nalu.h"
+#include "webrtc/system_wrappers/include/clock.h"
+
+namespace internal {
+
+// The ratio between kVTCompressionPropertyKey_DataRateLimits and
+// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
+// than the average bit rate to avoid undershooting the target.
+const float kLimitToAverageBitRateFactor = 1.5f;
+// These thresholds deviate from the default h264 QP thresholds, as they
+// have been found to work better on devices that support VideoToolbox
+const int kLowH264QpThreshold = 28;
+const int kHighH264QpThreshold = 39;
+
+// Convenience function for creating a dictionary.
+inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
+ CFTypeRef* values,
+ size_t size) {
+ return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
+ &kCFTypeDictionaryKeyCallBacks,
+ &kCFTypeDictionaryValueCallBacks);
+}
+
+// Copies characters from a CFStringRef into a std::string.
+std::string CFStringToString(const CFStringRef cf_string) {
+ RTC_DCHECK(cf_string);
+ std::string std_string;
+ // Get the size needed for UTF8 plus terminating character.
+ size_t buffer_size =
+ CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string),
+ kCFStringEncodingUTF8) +
+ 1;
+ std::unique_ptr<char[]> buffer(new char[buffer_size]);
+ if (CFStringGetCString(cf_string, buffer.get(), buffer_size,
+ kCFStringEncodingUTF8)) {
+ // Copy over the characters.
+ std_string.assign(buffer.get());
+ }
+ return std_string;
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session,
+ CFStringRef key,
+ int32_t value) {
+ CFNumberRef cfNum =
+ CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
+ OSStatus status = VTSessionSetProperty(session, key, cfNum);
+ CFRelease(cfNum);
+ if (status != noErr) {
+ std::string key_string = CFStringToString(key);
+ LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+ << " to " << value << ": " << status;
+ }
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session,
+ CFStringRef key,
+ uint32_t value) {
+ int64_t value_64 = value;
+ CFNumberRef cfNum =
+ CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
+ OSStatus status = VTSessionSetProperty(session, key, cfNum);
+ CFRelease(cfNum);
+ if (status != noErr) {
+ std::string key_string = CFStringToString(key);
+ LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+ << " to " << value << ": " << status;
+ }
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
+ CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
+ OSStatus status = VTSessionSetProperty(session, key, cf_bool);
+ if (status != noErr) {
+ std::string key_string = CFStringToString(key);
+ LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+ << " to " << value << ": " << status;
+ }
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session,
+ CFStringRef key,
+ CFStringRef value) {
+ OSStatus status = VTSessionSetProperty(session, key, value);
+ if (status != noErr) {
+ std::string key_string = CFStringToString(key);
+ std::string val_string = CFStringToString(value);
+ LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+ << " to " << val_string << ": " << status;
+ }
+}
+
+// Struct that we pass to the encoder per frame to encode. We receive it again
+// in the encoder callback.
+struct FrameEncodeParams {
+ FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e,
+ const webrtc::CodecSpecificInfo* csi,
+ int32_t w,
+ int32_t h,
+ int64_t rtms,
+ uint32_t ts,
+ webrtc::VideoRotation r)
+ : encoder(e),
+ width(w),
+ height(h),
+ render_time_ms(rtms),
+ timestamp(ts),
+ rotation(r) {
+ if (csi) {
+ codec_specific_info = *csi;
+ } else {
+ codec_specific_info.codecType = webrtc::kVideoCodecH264;
+ }
+ }
+
+ webrtc::H264VideoToolboxEncoder* encoder;
+ webrtc::CodecSpecificInfo codec_specific_info;
+ int32_t width;
+ int32_t height;
+ int64_t render_time_ms;
+ uint32_t timestamp;
+ webrtc::VideoRotation rotation;
+};
+
+// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
+// encoder. This performs the copy and format conversion.
+// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
+bool CopyVideoFrameToPixelBuffer(
+ const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& frame,
+ CVPixelBufferRef pixel_buffer) {
+ RTC_DCHECK(pixel_buffer);
+ RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer),
+ kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
+ RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0),
+ static_cast<size_t>(frame->height()));
+ RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0),
+ static_cast<size_t>(frame->width()));
+
+ CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
+ if (cvRet != kCVReturnSuccess) {
+ LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
+ return false;
+ }
+ uint8_t* dst_y = reinterpret_cast<uint8_t*>(
+ CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
+ int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
+ uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
+ CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
+ int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
+ // Convert I420 to NV12.
+ int ret = libyuv::I420ToNV12(
+ frame->DataY(), frame->StrideY(),
+ frame->DataU(), frame->StrideU(),
+ frame->DataV(), frame->StrideV(),
+ dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ frame->width(), frame->height());
+ CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+ if (ret) {
+ LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
+ return false;
+ }
+ return true;
+}
+
+CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
+ if (!pixel_buffer_pool) {
+ LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
+ return nullptr;
+ }
+ CVPixelBufferRef pixel_buffer;
+ CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
+ &pixel_buffer);
+ if (ret != kCVReturnSuccess) {
+ LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
+ // We probably want to drop frames here, since failure probably means
+ // that the pool is empty.
+ return nullptr;
+ }
+ return pixel_buffer;
+}
+
+// This is the callback function that VideoToolbox calls when encode is
+// complete. From inspection this happens on its own queue.
+void VTCompressionOutputCallback(void* encoder,
+ void* params,
+ OSStatus status,
+ VTEncodeInfoFlags info_flags,
+ CMSampleBufferRef sample_buffer) {
+ std::unique_ptr<FrameEncodeParams> encode_params(
+ reinterpret_cast<FrameEncodeParams*>(params));
+ encode_params->encoder->OnEncodedFrame(
+ status, info_flags, sample_buffer, encode_params->codec_specific_info,
+ encode_params->width, encode_params->height,
+ encode_params->render_time_ms, encode_params->timestamp,
+ encode_params->rotation);
+}
+
+// Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no
+// specific VideoToolbox profile for the specified level, AutoLevel will be
+// returned. The user must initialize the encoder with a resolution and
+// framerate conforming to the selected H264 level regardless.
+CFStringRef ExtractProfile(const cricket::VideoCodec& codec) {
+ const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
+ webrtc::H264::ParseSdpProfileLevelId(codec.params);
+ RTC_DCHECK(profile_level_id);
+ switch (profile_level_id->profile) {
+ case webrtc::H264::kProfileConstrainedBaseline:
+ case webrtc::H264::kProfileBaseline:
+ switch (profile_level_id->level) {
+ case webrtc::H264::kLevel3:
+ return kVTProfileLevel_H264_Baseline_3_0;
+ case webrtc::H264::kLevel3_1:
+ return kVTProfileLevel_H264_Baseline_3_1;
+ case webrtc::H264::kLevel3_2:
+ return kVTProfileLevel_H264_Baseline_3_2;
+ case webrtc::H264::kLevel4:
+ return kVTProfileLevel_H264_Baseline_4_0;
+ case webrtc::H264::kLevel4_1:
+ return kVTProfileLevel_H264_Baseline_4_1;
+ case webrtc::H264::kLevel4_2:
+ return kVTProfileLevel_H264_Baseline_4_2;
+ case webrtc::H264::kLevel5:
+ return kVTProfileLevel_H264_Baseline_5_0;
+ case webrtc::H264::kLevel5_1:
+ return kVTProfileLevel_H264_Baseline_5_1;
+ case webrtc::H264::kLevel5_2:
+ return kVTProfileLevel_H264_Baseline_5_2;
+ case webrtc::H264::kLevel1:
+ case webrtc::H264::kLevel1_b:
+ case webrtc::H264::kLevel1_1:
+ case webrtc::H264::kLevel1_2:
+ case webrtc::H264::kLevel1_3:
+ case webrtc::H264::kLevel2:
+ case webrtc::H264::kLevel2_1:
+ case webrtc::H264::kLevel2_2:
+ return kVTProfileLevel_H264_Baseline_AutoLevel;
+ }
+
+ case webrtc::H264::kProfileMain:
+ switch (profile_level_id->level) {
+ case webrtc::H264::kLevel3:
+ return kVTProfileLevel_H264_Main_3_0;
+ case webrtc::H264::kLevel3_1:
+ return kVTProfileLevel_H264_Main_3_1;
+ case webrtc::H264::kLevel3_2:
+ return kVTProfileLevel_H264_Main_3_2;
+ case webrtc::H264::kLevel4:
+ return kVTProfileLevel_H264_Main_4_0;
+ case webrtc::H264::kLevel4_1:
+ return kVTProfileLevel_H264_Main_4_1;
+ case webrtc::H264::kLevel4_2:
+ return kVTProfileLevel_H264_Main_4_2;
+ case webrtc::H264::kLevel5:
+ return kVTProfileLevel_H264_Main_5_0;
+ case webrtc::H264::kLevel5_1:
+ return kVTProfileLevel_H264_Main_5_1;
+ case webrtc::H264::kLevel5_2:
+ return kVTProfileLevel_H264_Main_5_2;
+ case webrtc::H264::kLevel1:
+ case webrtc::H264::kLevel1_b:
+ case webrtc::H264::kLevel1_1:
+ case webrtc::H264::kLevel1_2:
+ case webrtc::H264::kLevel1_3:
+ case webrtc::H264::kLevel2:
+ case webrtc::H264::kLevel2_1:
+ case webrtc::H264::kLevel2_2:
+ return kVTProfileLevel_H264_Main_AutoLevel;
+ }
+
+ case webrtc::H264::kProfileConstrainedHigh:
+ case webrtc::H264::kProfileHigh:
+ switch (profile_level_id->level) {
+ case webrtc::H264::kLevel3:
+ return kVTProfileLevel_H264_High_3_0;
+ case webrtc::H264::kLevel3_1:
+ return kVTProfileLevel_H264_High_3_1;
+ case webrtc::H264::kLevel3_2:
+ return kVTProfileLevel_H264_High_3_2;
+ case webrtc::H264::kLevel4:
+ return kVTProfileLevel_H264_High_4_0;
+ case webrtc::H264::kLevel4_1:
+ return kVTProfileLevel_H264_High_4_1;
+ case webrtc::H264::kLevel4_2:
+ return kVTProfileLevel_H264_High_4_2;
+ case webrtc::H264::kLevel5:
+ return kVTProfileLevel_H264_High_5_0;
+ case webrtc::H264::kLevel5_1:
+ return kVTProfileLevel_H264_High_5_1;
+ case webrtc::H264::kLevel5_2:
+ return kVTProfileLevel_H264_High_5_2;
+ case webrtc::H264::kLevel1:
+ case webrtc::H264::kLevel1_b:
+ case webrtc::H264::kLevel1_1:
+ case webrtc::H264::kLevel1_2:
+ case webrtc::H264::kLevel1_3:
+ case webrtc::H264::kLevel2:
+ case webrtc::H264::kLevel2_1:
+ case webrtc::H264::kLevel2_2:
+ return kVTProfileLevel_H264_High_AutoLevel;
+ }
+ }
+}
+
+} // namespace internal
+
+namespace webrtc {
+
+// .5 is set as a mininum to prevent overcompensating for large temporary
+// overshoots. We don't want to degrade video quality too badly.
+// .95 is set to prevent oscillations. When a lower bitrate is set on the
+// encoder than previously set, its output seems to have a brief period of
+// drastically reduced bitrate, so we want to avoid that. In steady state
+// conditions, 0.95 seems to give us better overall bitrate over long periods
+// of time.
+H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec)
+ : callback_(nullptr),
+ compression_session_(nullptr),
+ bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95),
+ packetization_mode_(H264PacketizationMode::NonInterleaved),
+ profile_(internal::ExtractProfile(codec)) {
+ LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_);
+ RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName));
+}
+
+H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
+ DestroyCompressionSession();
+}
+
+int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
+ int number_of_cores,
+ size_t max_payload_size) {
+ RTC_DCHECK(codec_settings);
+ RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
+
+ width_ = codec_settings->width;
+ height_ = codec_settings->height;
+ mode_ = codec_settings->mode;
+ // We can only set average bitrate on the HW encoder.
+ target_bitrate_bps_ = codec_settings->startBitrate;
+ bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
+
+ // TODO(tkchin): Try setting payload size via
+ // kVTCompressionPropertyKey_MaxH264SliceBytes.
+
+ return ResetCompressionSession();
+}
+
+int H264VideoToolboxEncoder::Encode(
+ const VideoFrame& frame,
+ const CodecSpecificInfo* codec_specific_info,
+ const std::vector<FrameType>* frame_types) {
+ // |input_frame| size should always match codec settings.
+ RTC_DCHECK_EQ(frame.width(), width_);
+ RTC_DCHECK_EQ(frame.height(), height_);
+ if (!callback_ || !compression_session_) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+#if defined(WEBRTC_IOS)
+ if (!RTCIsUIApplicationActive()) {
+ // Ignore all encode requests when app isn't active. In this state, the
+ // hardware encoder has been invalidated by the OS.
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+#endif
+ bool is_keyframe_required = false;
+
+ // Get a pixel buffer from the pool and copy frame data over.
+ CVPixelBufferPoolRef pixel_buffer_pool =
+ VTCompressionSessionGetPixelBufferPool(compression_session_);
+#if defined(WEBRTC_IOS)
+ if (!pixel_buffer_pool) {
+ // Kind of a hack. On backgrounding, the compression session seems to get
+ // invalidated, which causes this pool call to fail when the application
+ // is foregrounded and frames are being sent for encoding again.
+ // Resetting the session when this happens fixes the issue.
+ // In addition we request a keyframe so video can recover quickly.
+ ResetCompressionSession();
+ pixel_buffer_pool =
+ VTCompressionSessionGetPixelBufferPool(compression_session_);
+ is_keyframe_required = true;
+ LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
+ }
+#endif
+
+ CVPixelBufferRef pixel_buffer = static_cast<CVPixelBufferRef>(
+ frame.video_frame_buffer()->native_handle());
+ if (pixel_buffer) {
+ // Native frame.
+ rtc::scoped_refptr<CoreVideoFrameBuffer> core_video_frame_buffer(
+ static_cast<CoreVideoFrameBuffer*>(frame.video_frame_buffer().get()));
+ if (!core_video_frame_buffer->RequiresCropping()) {
+ // This pixel buffer might have a higher resolution than what the
+ // compression session is configured to. The compression session can
+ // handle that and will output encoded frames in the configured
+ // resolution regardless of the input pixel buffer resolution.
+ CVBufferRetain(pixel_buffer);
+ } else {
+ // Cropping required, we need to crop and scale to a new pixel buffer.
+ pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
+ if (!pixel_buffer) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_,
+ pixel_buffer)) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ }
+ } else {
+ pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
+ if (!pixel_buffer) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ RTC_DCHECK(pixel_buffer);
+ if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer(),
+ pixel_buffer)) {
+ LOG(LS_ERROR) << "Failed to copy frame data.";
+ CVBufferRelease(pixel_buffer);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ }
+
+ // Check if we need a keyframe.
+ if (!is_keyframe_required && frame_types) {
+ for (auto frame_type : *frame_types) {
+ if (frame_type == kVideoFrameKey) {
+ is_keyframe_required = true;
+ break;
+ }
+ }
+ }
+
+ CMTime presentation_time_stamp =
+ CMTimeMake(frame.render_time_ms(), 1000);
+ CFDictionaryRef frame_properties = nullptr;
+ if (is_keyframe_required) {
+ CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
+ CFTypeRef values[] = {kCFBooleanTrue};
+ frame_properties = internal::CreateCFDictionary(keys, values, 1);
+ }
+ std::unique_ptr<internal::FrameEncodeParams> encode_params;
+ encode_params.reset(new internal::FrameEncodeParams(
+ this, codec_specific_info, width_, height_, frame.render_time_ms(),
+ frame.timestamp(), frame.rotation()));
+
+ encode_params->codec_specific_info.codecSpecific.H264.packetization_mode =
+ packetization_mode_;
+
+ // Update the bitrate if needed.
+ SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
+
+ OSStatus status = VTCompressionSessionEncodeFrame(
+ compression_session_, pixel_buffer, presentation_time_stamp,
+ kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
+ if (frame_properties) {
+ CFRelease(frame_properties);
+ }
+ if (pixel_buffer) {
+ CVBufferRelease(pixel_buffer);
+ }
+ if (status != noErr) {
+ LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) {
+ callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
+ int64_t rtt) {
+ // Encoder doesn't know anything about packet loss or rtt so just return.
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
+ uint32_t frame_rate) {
+ target_bitrate_bps_ = 1000 * new_bitrate_kbit;
+ bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
+ SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::Release() {
+ // Need to reset so that the session is invalidated and won't use the
+ // callback anymore. Do not remove callback until the session is invalidated
+ // since async encoder callbacks can occur until invalidation.
+ int ret = ResetCompressionSession();
+ callback_ = nullptr;
+ return ret;
+}
+
+int H264VideoToolboxEncoder::ResetCompressionSession() {
+ DestroyCompressionSession();
+
+ // Set source image buffer attributes. These attributes will be present on
+ // buffers retrieved from the encoder's pixel buffer pool.
+ const size_t attributes_size = 3;
+ CFTypeRef keys[attributes_size] = {
+#if defined(WEBRTC_IOS)
+ kCVPixelBufferOpenGLESCompatibilityKey,
+#elif defined(WEBRTC_MAC)
+ kCVPixelBufferOpenGLCompatibilityKey,
+#endif
+ kCVPixelBufferIOSurfacePropertiesKey,
+ kCVPixelBufferPixelFormatTypeKey
+ };
+ CFDictionaryRef io_surface_value =
+ internal::CreateCFDictionary(nullptr, nullptr, 0);
+ int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+ CFNumberRef pixel_format =
+ CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
+ CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
+ pixel_format};
+ CFDictionaryRef source_attributes =
+ internal::CreateCFDictionary(keys, values, attributes_size);
+ if (io_surface_value) {
+ CFRelease(io_surface_value);
+ io_surface_value = nullptr;
+ }
+ if (pixel_format) {
+ CFRelease(pixel_format);
+ pixel_format = nullptr;
+ }
+ OSStatus status = VTCompressionSessionCreate(
+ nullptr, // use default allocator
+ width_, height_, kCMVideoCodecType_H264,
+ nullptr, // use default encoder
+ source_attributes,
+ nullptr, // use default compressed data allocator
+ internal::VTCompressionOutputCallback, this, &compression_session_);
+ if (source_attributes) {
+ CFRelease(source_attributes);
+ source_attributes = nullptr;
+ }
+ if (status != noErr) {
+ LOG(LS_ERROR) << "Failed to create compression session: " << status;
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ ConfigureCompressionSession();
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void H264VideoToolboxEncoder::ConfigureCompressionSession() {
+ RTC_DCHECK(compression_session_);
+ internal::SetVTSessionProperty(compression_session_,
+ kVTCompressionPropertyKey_RealTime, true);
+ internal::SetVTSessionProperty(compression_session_,
+ kVTCompressionPropertyKey_ProfileLevel,
+ profile_);
+ internal::SetVTSessionProperty(compression_session_,
+ kVTCompressionPropertyKey_AllowFrameReordering,
+ false);
+ SetEncoderBitrateBps(target_bitrate_bps_);
+ // TODO(tkchin): Look at entropy mode and colorspace matrices.
+ // TODO(tkchin): Investigate to see if there's any way to make this work.
+ // May need it to interop with Android. Currently this call just fails.
+ // On inspecting encoder output on iOS8, this value is set to 6.
+ // internal::SetVTSessionProperty(compression_session_,
+ // kVTCompressionPropertyKey_MaxFrameDelayCount,
+ // 1);
+
+ // Set a relatively large value for keyframe emission (7200 frames or
+ // 4 minutes).
+ internal::SetVTSessionProperty(
+ compression_session_,
+ kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
+ internal::SetVTSessionProperty(
+ compression_session_,
+ kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
+}
+
+void H264VideoToolboxEncoder::DestroyCompressionSession() {
+ if (compression_session_) {
+ VTCompressionSessionInvalidate(compression_session_);
+ CFRelease(compression_session_);
+ compression_session_ = nullptr;
+ }
+}
+
+const char* H264VideoToolboxEncoder::ImplementationName() const {
+ return "VideoToolbox";
+}
+
+bool H264VideoToolboxEncoder::SupportsNativeHandle() const {
+ return true;
+}
+
+void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) {
+ if (encoder_bitrate_bps_ != bitrate_bps) {
+ SetEncoderBitrateBps(bitrate_bps);
+ }
+}
+
+void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) {
+ if (compression_session_) {
+ internal::SetVTSessionProperty(compression_session_,
+ kVTCompressionPropertyKey_AverageBitRate,
+ bitrate_bps);
+
+ // TODO(tkchin): Add a helper method to set array value.
+ int64_t data_limit_bytes_per_second_value = static_cast<int64_t>(
+ bitrate_bps * internal::kLimitToAverageBitRateFactor / 8);
+ CFNumberRef bytes_per_second =
+ CFNumberCreate(kCFAllocatorDefault,
+ kCFNumberSInt64Type,
+ &data_limit_bytes_per_second_value);
+ int64_t one_second_value = 1;
+ CFNumberRef one_second =
+ CFNumberCreate(kCFAllocatorDefault,
+ kCFNumberSInt64Type,
+ &one_second_value);
+ const void* nums[2] = { bytes_per_second, one_second };
+ CFArrayRef data_rate_limits =
+ CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
+ OSStatus status =
+ VTSessionSetProperty(compression_session_,
+ kVTCompressionPropertyKey_DataRateLimits,
+ data_rate_limits);
+ if (bytes_per_second) {
+ CFRelease(bytes_per_second);
+ }
+ if (one_second) {
+ CFRelease(one_second);
+ }
+ if (data_rate_limits) {
+ CFRelease(data_rate_limits);
+ }
+ if (status != noErr) {
+ LOG(LS_ERROR) << "Failed to set data rate limit";
+ }
+
+ encoder_bitrate_bps_ = bitrate_bps;
+ }
+}
+
+void H264VideoToolboxEncoder::OnEncodedFrame(
+ OSStatus status,
+ VTEncodeInfoFlags info_flags,
+ CMSampleBufferRef sample_buffer,
+ CodecSpecificInfo codec_specific_info,
+ int32_t width,
+ int32_t height,
+ int64_t render_time_ms,
+ uint32_t timestamp,
+ VideoRotation rotation) {
+ if (status != noErr) {
+ LOG(LS_ERROR) << "H264 encode failed.";
+ return;
+ }
+ if (info_flags & kVTEncodeInfo_FrameDropped) {
+ LOG(LS_INFO) << "H264 encode dropped frame.";
+ return;
+ }
+
+ bool is_keyframe = false;
+ CFArrayRef attachments =
+ CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
+ if (attachments != nullptr && CFArrayGetCount(attachments)) {
+ CFDictionaryRef attachment =
+ static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
+ is_keyframe =
+ !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
+ }
+
+ if (is_keyframe) {
+ LOG(LS_INFO) << "Generated keyframe";
+ }
+
+ // Convert the sample buffer into a buffer suitable for RTP packetization.
+ // TODO(tkchin): Allocate buffers through a pool.
+ std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer());
+ std::unique_ptr<webrtc::RTPFragmentationHeader> header;
+ {
+ webrtc::RTPFragmentationHeader* header_raw;
+ bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
+ buffer.get(), &header_raw);
+ header.reset(header_raw);
+ if (!result) {
+ return;
+ }
+ }
+ webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
+ frame._encodedWidth = width;
+ frame._encodedHeight = height;
+ frame._completeFrame = true;
+ frame._frameType =
+ is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
+ frame.capture_time_ms_ = render_time_ms;
+ frame._timeStamp = timestamp;
+ frame.rotation_ = rotation;
+
+ frame.content_type_ =
+ (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED;
+
+ h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
+ h264_bitstream_parser_.GetLastSliceQp(&frame.qp_);
+
+ EncodedImageCallback::Result res =
+ callback_->OnEncodedImage(frame, &codec_specific_info, header.get());
+ if (res.error != EncodedImageCallback::Result::OK) {
+ LOG(LS_ERROR) << "Encode callback failed: " << res.error;
+ return;
+ }
+ bitrate_adjuster_.Update(frame._length);
+}
+
+VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings()
+ const {
+ return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold,
+ internal::kHighH264QpThreshold);
+}
+} // namespace webrtc

Powered by Google App Engine
This is Rietveld 408576698