Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(26)

Unified Diff: webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm

Issue 2987413002: ObjC: Implement HW codecs in ObjC instead of C++ (Closed)
Patch Set: Move RTCH264PacketizationMode to RTCVideoCodec.h Created 3 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
similarity index 38%
rename from webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm
rename to webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
index 2bac68ef6de3185525711edb5faf41f5214eb662..4091886dd486eb3d811de000ef6427cf262d2ac1 100644
--- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm
+++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm
@@ -9,26 +9,45 @@
*
*/
-#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h"
+#import "WebRTC/RTCVideoCodecH264.h"
-#include <memory>
-#include <string>
+#import <VideoToolbox/VideoToolbox.h>
#include <vector>
#if defined(WEBRTC_IOS)
#import "Common/RTCUIApplicationStatusObserver.h"
#import "WebRTC/UIDevice+RTCDevice.h"
#endif
+#import "PeerConnection/RTCVideoCodec+Private.h"
+#import "WebRTC/RTCVideoCodec.h"
+#import "WebRTC/RTCVideoFrame.h"
#import "WebRTC/RTCVideoFrameBuffer.h"
+#import "helpers.h"
#include "libyuv/convert_from.h"
+#include "webrtc/common_video/h264/h264_bitstream_parser.h"
#include "webrtc/common_video/h264/profile_level_id.h"
-#include "webrtc/rtc_base/checks.h"
+#include "webrtc/common_video/include/bitrate_adjuster.h"
+#include "webrtc/modules/include/module_common_types.h"
+#include "webrtc/modules/video_coding/include/video_error_codes.h"
+#include "webrtc/rtc_base/buffer.h"
#include "webrtc/rtc_base/logging.h"
-#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h"
+#include "webrtc/rtc_base/timeutils.h"
#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
#include "webrtc/system_wrappers/include/clock.h"
-namespace internal {
+@interface RTCVideoEncoderH264 ()
+
+- (void)frameWasEncoded:(OSStatus)status
+ flags:(VTEncodeInfoFlags)infoFlags
+ sampleBuffer:(CMSampleBufferRef)sampleBuffer
+ codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
+ width:(int32_t)width
+ height:(int32_t)height
+ renderTimeMs:(int64_t)renderTimeMs
+ timestamp:(uint32_t)timestamp
+ rotation:(RTCVideoRotation)rotation;
+
+@end
// The ratio between kVTCompressionPropertyKey_DataRateLimits and
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
@@ -39,152 +58,66 @@ const float kLimitToAverageBitRateFactor = 1.5f;
const int kLowH264QpThreshold = 28;
const int kHighH264QpThreshold = 39;
-// Convenience function for creating a dictionary.
-inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
- CFTypeRef* values,
- size_t size) {
- return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
- &kCFTypeDictionaryKeyCallBacks,
- &kCFTypeDictionaryValueCallBacks);
-}
-
-// Copies characters from a CFStringRef into a std::string.
-std::string CFStringToString(const CFStringRef cf_string) {
- RTC_DCHECK(cf_string);
- std::string std_string;
- // Get the size needed for UTF8 plus terminating character.
- size_t buffer_size =
- CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string),
- kCFStringEncodingUTF8) +
- 1;
- std::unique_ptr<char[]> buffer(new char[buffer_size]);
- if (CFStringGetCString(cf_string, buffer.get(), buffer_size,
- kCFStringEncodingUTF8)) {
- // Copy over the characters.
- std_string.assign(buffer.get());
- }
- return std_string;
-}
-
-// Convenience function for setting a VT property.
-void SetVTSessionProperty(VTSessionRef session,
- CFStringRef key,
- int32_t value) {
- CFNumberRef cfNum =
- CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
- OSStatus status = VTSessionSetProperty(session, key, cfNum);
- CFRelease(cfNum);
- if (status != noErr) {
- std::string key_string = CFStringToString(key);
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
- << " to " << value << ": " << status;
- }
-}
-
-// Convenience function for setting a VT property.
-void SetVTSessionProperty(VTSessionRef session,
- CFStringRef key,
- uint32_t value) {
- int64_t value_64 = value;
- CFNumberRef cfNum =
- CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
- OSStatus status = VTSessionSetProperty(session, key, cfNum);
- CFRelease(cfNum);
- if (status != noErr) {
- std::string key_string = CFStringToString(key);
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
- << " to " << value << ": " << status;
- }
-}
-
-// Convenience function for setting a VT property.
-void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
- CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
- OSStatus status = VTSessionSetProperty(session, key, cf_bool);
- if (status != noErr) {
- std::string key_string = CFStringToString(key);
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
- << " to " << value << ": " << status;
- }
-}
-
-// Convenience function for setting a VT property.
-void SetVTSessionProperty(VTSessionRef session,
- CFStringRef key,
- CFStringRef value) {
- OSStatus status = VTSessionSetProperty(session, key, value);
- if (status != noErr) {
- std::string key_string = CFStringToString(key);
- std::string val_string = CFStringToString(value);
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
- << " to " << val_string << ": " << status;
- }
-}
-
// Struct that we pass to the encoder per frame to encode. We receive it again
// in the encoder callback.
-struct FrameEncodeParams {
- FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e,
- const webrtc::CodecSpecificInfo* csi,
- int32_t w,
- int32_t h,
- int64_t rtms,
- uint32_t ts,
- webrtc::VideoRotation r)
- : encoder(e),
- width(w),
- height(h),
- render_time_ms(rtms),
- timestamp(ts),
- rotation(r) {
+struct RTCFrameEncodeParams {
+ RTCFrameEncodeParams(RTCVideoEncoderH264 *e,
+ RTCCodecSpecificInfoH264 *csi,
+ int32_t w,
+ int32_t h,
+ int64_t rtms,
+ uint32_t ts,
+ RTCVideoRotation r)
+ : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) {
if (csi) {
- codec_specific_info = *csi;
+ codecSpecificInfo = csi;
} else {
- codec_specific_info.codecType = webrtc::kVideoCodecH264;
+ codecSpecificInfo = [[RTCCodecSpecificInfoH264 alloc] init];
}
}
- webrtc::H264VideoToolboxEncoder* encoder;
- webrtc::CodecSpecificInfo codec_specific_info;
+ RTCVideoEncoderH264 *encoder;
+ RTCCodecSpecificInfoH264 *codecSpecificInfo;
int32_t width;
int32_t height;
int64_t render_time_ms;
uint32_t timestamp;
- webrtc::VideoRotation rotation;
+ RTCVideoRotation rotation;
};
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
// encoder. This performs the copy and format conversion.
// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
-bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr<webrtc::I420BufferInterface>& frame,
- CVPixelBufferRef pixel_buffer) {
- RTC_DCHECK(pixel_buffer);
- RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer),
+bool CopyVideoFrameToPixelBuffer(id<RTCI420Buffer> frameBuffer, CVPixelBufferRef pixelBuffer) {
+ RTC_DCHECK(pixelBuffer);
+ RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer),
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
- RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0),
- static_cast<size_t>(frame->height()));
- RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0),
- static_cast<size_t>(frame->width()));
+ RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height);
+ RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width);
- CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
+ CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (cvRet != kCVReturnSuccess) {
LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
return false;
}
- uint8_t* dst_y = reinterpret_cast<uint8_t*>(
- CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
- int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
- uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
- CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
- int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
+ uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
+ int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
+ uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
+ int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
// Convert I420 to NV12.
- int ret = libyuv::I420ToNV12(
- frame->DataY(), frame->StrideY(),
- frame->DataU(), frame->StrideU(),
- frame->DataV(), frame->StrideV(),
- dst_y, dst_stride_y, dst_uv, dst_stride_uv,
- frame->width(), frame->height());
- CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+ int ret = libyuv::I420ToNV12(frameBuffer.dataY,
+ frameBuffer.strideY,
+ frameBuffer.dataU,
+ frameBuffer.strideU,
+ frameBuffer.dataV,
+ frameBuffer.strideV,
+ dstY,
+ dstStrideY,
+ dstUV,
+ dstStrideUV,
+ frameBuffer.width,
+ frameBuffer.height);
+ CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if (ret) {
LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
return false;
@@ -198,8 +131,7 @@ CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
return nullptr;
}
CVPixelBufferRef pixel_buffer;
- CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
- &pixel_buffer);
+ CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer);
if (ret != kCVReturnSuccess) {
LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
// We probably want to drop frames here, since failure probably means
@@ -211,25 +143,29 @@ CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
// This is the callback function that VideoToolbox calls when encode is
// complete. From inspection this happens on its own queue.
-void VTCompressionOutputCallback(void* encoder,
- void* params,
- OSStatus status,
- VTEncodeInfoFlags info_flags,
- CMSampleBufferRef sample_buffer) {
- std::unique_ptr<FrameEncodeParams> encode_params(
- reinterpret_cast<FrameEncodeParams*>(params));
- encode_params->encoder->OnEncodedFrame(
- status, info_flags, sample_buffer, encode_params->codec_specific_info,
- encode_params->width, encode_params->height,
- encode_params->render_time_ms, encode_params->timestamp,
- encode_params->rotation);
+void compressionOutputCallback(void *encoder,
+ void *params,
+ OSStatus status,
+ VTEncodeInfoFlags infoFlags,
+ CMSampleBufferRef sampleBuffer) {
+ std::unique_ptr<RTCFrameEncodeParams> encodeParams(
+ reinterpret_cast<RTCFrameEncodeParams *>(params));
+ [encodeParams->encoder frameWasEncoded:status
+ flags:infoFlags
+ sampleBuffer:sampleBuffer
+ codecSpecificInfo:encodeParams->codecSpecificInfo
+ width:encodeParams->width
+ height:encodeParams->height
+ renderTimeMs:encodeParams->render_time_ms
+ timestamp:encodeParams->timestamp
+ rotation:encodeParams->rotation];
}
// Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no
// specific VideoToolbox profile for the specified level, AutoLevel will be
// returned. The user must initialize the encoder with a resolution and
// framerate conforming to the selected H264 level regardless.
-CFStringRef ExtractProfile(const cricket::VideoCodec& codec) {
+CFStringRef ExtractProfile(const cricket::VideoCodec &codec) {
const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
webrtc::H264::ParseSdpProfileLevelId(codec.params);
RTC_DCHECK(profile_level_id);
@@ -331,9 +267,22 @@ CFStringRef ExtractProfile(const cricket::VideoCodec& codec) {
}
}
-} // namespace internal
-
-namespace webrtc {
+@implementation RTCVideoEncoderH264 {
+ RTCVideoCodecInfo *_codecInfo;
+ webrtc::BitrateAdjuster *_bitrateAdjuster;
+ uint32_t _targetBitrateBps;
+ uint32_t _encoderBitrateBps;
+ RTCH264PacketizationMode _packetizationMode;
+ CFStringRef _profile;
+ RTCVideoEncoderCallback _callback;
+ int32_t _width;
+ int32_t _height;
+ VTCompressionSessionRef _compressionSession;
+ RTCVideoCodecMode _mode;
+
+ webrtc::H264BitstreamParser _h264BitstreamParser;
+ std::vector<uint8_t> _nv12ScaleBuffer;
+}
// .5 is set as a mininum to prevent overcompensating for large temporary
// overshoots. We don't want to degrade video quality too badly.
@@ -342,47 +291,47 @@ namespace webrtc {
// drastically reduced bitrate, so we want to avoid that. In steady state
// conditions, 0.95 seems to give us better overall bitrate over long periods
// of time.
-H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec)
- : callback_(nullptr),
- compression_session_(nullptr),
- bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95),
- packetization_mode_(H264PacketizationMode::NonInterleaved),
- profile_(internal::ExtractProfile(codec)) {
- LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_);
- RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName));
+- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo {
+ if (self = [super init]) {
+ _codecInfo = codecInfo;
+ _bitrateAdjuster = new webrtc::BitrateAdjuster(webrtc::Clock::GetRealTimeClock(), .5, .95);
+ _packetizationMode = RTCH264PacketizationModeNonInterleaved;
+ _profile = ExtractProfile([codecInfo nativeVideoCodec]);
+ LOG(LS_INFO) << "Using profile " << CFStringToString(_profile);
+ RTC_CHECK([codecInfo.name isEqualToString:@"H264"]);
+ }
+ return self;
}
-H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
- DestroyCompressionSession();
+- (void)dealloc {
+ [self destroyCompressionSession];
}
-int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
- int number_of_cores,
- size_t max_payload_size) {
- RTC_DCHECK(codec_settings);
- RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
+- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings
+ numberOfCores:(int)numberOfCores {
+ RTC_DCHECK(settings);
+ RTC_DCHECK([settings.name isEqualToString:@"H264"]);
+
+ _width = settings.width;
+ _height = settings.height;
+ _mode = settings.mode;
- width_ = codec_settings->width;
- height_ = codec_settings->height;
- mode_ = codec_settings->mode;
// We can only set average bitrate on the HW encoder.
- target_bitrate_bps_ = codec_settings->startBitrate;
- bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
+ _targetBitrateBps = settings.startBitrate;
+ _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
// TODO(tkchin): Try setting payload size via
// kVTCompressionPropertyKey_MaxH264SliceBytes.
- return ResetCompressionSession();
+ return [self resetCompressionSession];
}
-int H264VideoToolboxEncoder::Encode(
- const VideoFrame& frame,
- const CodecSpecificInfo* codec_specific_info,
- const std::vector<FrameType>* frame_types) {
- // |input_frame| size should always match codec settings.
- RTC_DCHECK_EQ(frame.width(), width_);
- RTC_DCHECK_EQ(frame.height(), height_);
- if (!callback_ || !compression_session_) {
+- (NSInteger)encode:(RTCVideoFrame *)frame
+ codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
+ frameTypes:(NSArray<NSNumber *> *)frameTypes {
+ RTC_DCHECK_EQ(frame.width, _width);
+ RTC_DCHECK_EQ(frame.height, _height);
+ if (!_callback || !_compressionSession) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
#if defined(WEBRTC_IOS)
@@ -392,119 +341,116 @@ int H264VideoToolboxEncoder::Encode(
return WEBRTC_VIDEO_CODEC_OK;
}
#endif
- bool is_keyframe_required = false;
+ BOOL isKeyframeRequired = NO;
// Get a pixel buffer from the pool and copy frame data over.
- CVPixelBufferPoolRef pixel_buffer_pool =
- VTCompressionSessionGetPixelBufferPool(compression_session_);
+ CVPixelBufferPoolRef pixelBufferPool =
+ VTCompressionSessionGetPixelBufferPool(_compressionSession);
+
#if defined(WEBRTC_IOS)
- if (!pixel_buffer_pool) {
+ if (!pixelBufferPool) {
// Kind of a hack. On backgrounding, the compression session seems to get
// invalidated, which causes this pool call to fail when the application
// is foregrounded and frames are being sent for encoding again.
// Resetting the session when this happens fixes the issue.
// In addition we request a keyframe so video can recover quickly.
- ResetCompressionSession();
- pixel_buffer_pool =
- VTCompressionSessionGetPixelBufferPool(compression_session_);
- is_keyframe_required = true;
+ [self resetCompressionSession];
+ pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession);
+ isKeyframeRequired = YES;
LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
}
#endif
- CVPixelBufferRef pixel_buffer = nullptr;
- if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) {
- // Native frame.
- rtc::scoped_refptr<ObjCFrameBuffer> objc_frame_buffer(
- static_cast<ObjCFrameBuffer*>(frame.video_frame_buffer().get()));
- id<RTCVideoFrameBuffer> wrapped_frame_buffer =
- (id<RTCVideoFrameBuffer>)objc_frame_buffer->wrapped_frame_buffer();
-
- if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
- RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer;
- if (![rtc_pixel_buffer requiresCropping]) {
- // This pixel buffer might have a higher resolution than what the
- // compression session is configured to. The compression session can
- // handle that and will output encoded frames in the configured
- // resolution regardless of the input pixel buffer resolution.
- pixel_buffer = rtc_pixel_buffer.pixelBuffer;
- CVBufferRetain(pixel_buffer);
+ CVPixelBufferRef pixelBuffer = nullptr;
+ if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
+ // Native frame buffer
+ RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer;
+ if (![rtcPixelBuffer requiresCropping]) {
+ // This pixel buffer might have a higher resolution than what the
+ // compression session is configured to. The compression session can
+ // handle that and will output encoded frames in the configured
+ // resolution regardless of the input pixel buffer resolution.
+ pixelBuffer = rtcPixelBuffer.pixelBuffer;
+ CVBufferRetain(pixelBuffer);
+ } else {
+ // Cropping required, we need to crop and scale to a new pixel buffer.
+ pixelBuffer = CreatePixelBuffer(pixelBufferPool);
+ if (!pixelBuffer) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ int dstWidth = CVPixelBufferGetWidth(pixelBuffer);
+ int dstHeight = CVPixelBufferGetHeight(pixelBuffer);
+ if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) {
+ int size =
+ [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight];
+ _nv12ScaleBuffer.resize(size);
} else {
- // Cropping required, we need to crop and scale to a new pixel buffer.
- pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
- if (!pixel_buffer) {
- return WEBRTC_VIDEO_CODEC_ERROR;
- }
- int dst_width = CVPixelBufferGetWidth(pixel_buffer);
- int dst_height = CVPixelBufferGetHeight(pixel_buffer);
- if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) {
- int size =
- [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height];
- nv12_scale_buffer_.resize(size);
- } else {
- nv12_scale_buffer_.clear();
- }
- nv12_scale_buffer_.shrink_to_fit();
- if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer
- withTempBuffer:nv12_scale_buffer_.data()]) {
- return WEBRTC_VIDEO_CODEC_ERROR;
- }
+ _nv12ScaleBuffer.clear();
+ }
+ _nv12ScaleBuffer.shrink_to_fit();
+ if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer withTempBuffer:_nv12ScaleBuffer.data()]) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
}
}
}
- if (!pixel_buffer) {
- // We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame
- pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
- if (!pixel_buffer) {
+ if (!pixelBuffer) {
+ // We did not have a native frame buffer
+ pixelBuffer = CreatePixelBuffer(pixelBufferPool);
+ if (!pixelBuffer) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
- RTC_DCHECK(pixel_buffer);
- if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI420(),
- pixel_buffer)) {
+ RTC_DCHECK(pixelBuffer);
+ if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) {
LOG(LS_ERROR) << "Failed to copy frame data.";
- CVBufferRelease(pixel_buffer);
+ CVBufferRelease(pixelBuffer);
return WEBRTC_VIDEO_CODEC_ERROR;
}
}
// Check if we need a keyframe.
- if (!is_keyframe_required && frame_types) {
- for (auto frame_type : *frame_types) {
- if (frame_type == kVideoFrameKey) {
- is_keyframe_required = true;
+ if (!isKeyframeRequired && frameTypes) {
+ for (NSNumber *frameType in frameTypes) {
+ if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) {
+ isKeyframeRequired = YES;
break;
}
}
}
- CMTime presentation_time_stamp =
- CMTimeMake(frame.render_time_ms(), 1000);
- CFDictionaryRef frame_properties = nullptr;
- if (is_keyframe_required) {
+ CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000);
+ CFDictionaryRef frameProperties = nullptr;
+ if (isKeyframeRequired) {
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
CFTypeRef values[] = {kCFBooleanTrue};
- frame_properties = internal::CreateCFDictionary(keys, values, 1);
+ frameProperties = CreateCFTypeDictionary(keys, values, 1);
}
- std::unique_ptr<internal::FrameEncodeParams> encode_params;
- encode_params.reset(new internal::FrameEncodeParams(
- this, codec_specific_info, width_, height_, frame.render_time_ms(),
- frame.timestamp(), frame.rotation()));
- encode_params->codec_specific_info.codecSpecific.H264.packetization_mode =
- packetization_mode_;
+ std::unique_ptr<RTCFrameEncodeParams> encodeParams;
+ encodeParams.reset(new RTCFrameEncodeParams(self,
+ codecSpecificInfo,
+ _width,
+ _height,
+ frame.timeStampNs / rtc::kNumNanosecsPerMillisec,
+ frame.timeStamp,
+ frame.rotation));
+ encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode;
// Update the bitrate if needed.
- SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
+ [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
- OSStatus status = VTCompressionSessionEncodeFrame(
- compression_session_, pixel_buffer, presentation_time_stamp,
- kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
- if (frame_properties) {
- CFRelease(frame_properties);
+ OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession,
+ pixelBuffer,
+ presentationTimeStamp,
+ kCMTimeInvalid,
+ frameProperties,
+ encodeParams.release(),
+ nullptr);
+ if (frameProperties) {
+ CFRelease(frameProperties);
}
- if (pixel_buffer) {
- CVBufferRelease(pixel_buffer);
+ if (pixelBuffer) {
+ CVBufferRelease(pixelBuffer);
}
if (status != noErr) {
LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
@@ -513,42 +459,35 @@ int H264VideoToolboxEncoder::Encode(
return WEBRTC_VIDEO_CODEC_OK;
}
-int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
- EncodedImageCallback* callback) {
- callback_ = callback;
- return WEBRTC_VIDEO_CODEC_OK;
+- (void)setCallback:(RTCVideoEncoderCallback)callback {
+ _callback = callback;
}
-int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
- int64_t rtt) {
- // Encoder doesn't know anything about packet loss or rtt so just return.
+- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate {
+ _targetBitrateBps = 1000 * bitrateKbit;
+ _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps);
+ [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
return WEBRTC_VIDEO_CODEC_OK;
}
-int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
- uint32_t frame_rate) {
- target_bitrate_bps_ = 1000 * new_bitrate_kbit;
- bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
- SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
- return WEBRTC_VIDEO_CODEC_OK;
-}
+#pragma mark - Private
-int H264VideoToolboxEncoder::Release() {
+- (NSInteger)releaseEncoder {
// Need to destroy so that the session is invalidated and won't use the
// callback anymore. Do not remove callback until the session is invalidated
// since async encoder callbacks can occur until invalidation.
- DestroyCompressionSession();
- callback_ = nullptr;
+ [self destroyCompressionSession];
+ _callback = nullptr;
return WEBRTC_VIDEO_CODEC_OK;
}
-int H264VideoToolboxEncoder::ResetCompressionSession() {
- DestroyCompressionSession();
+- (int)resetCompressionSession {
+ [self destroyCompressionSession];
// Set source image buffer attributes. These attributes will be present on
// buffers retrieved from the encoder's pixel buffer pool.
- const size_t attributes_size = 3;
- CFTypeRef keys[attributes_size] = {
+ const size_t attributesSize = 3;
+ CFTypeRef keys[attributesSize] = {
#if defined(WEBRTC_IOS)
kCVPixelBufferOpenGLESCompatibilityKey,
#elif defined(WEBRTC_MAC)
@@ -557,53 +496,47 @@ int H264VideoToolboxEncoder::ResetCompressionSession() {
kCVPixelBufferIOSurfacePropertiesKey,
kCVPixelBufferPixelFormatTypeKey
};
- CFDictionaryRef io_surface_value =
- internal::CreateCFDictionary(nullptr, nullptr, 0);
+ CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0);
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
- CFNumberRef pixel_format =
- CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
- CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
- pixel_format};
- CFDictionaryRef source_attributes =
- internal::CreateCFDictionary(keys, values, attributes_size);
- if (io_surface_value) {
- CFRelease(io_surface_value);
- io_surface_value = nullptr;
- }
- if (pixel_format) {
- CFRelease(pixel_format);
- pixel_format = nullptr;
- }
- OSStatus status = VTCompressionSessionCreate(
- nullptr, // use default allocator
- width_, height_, kCMVideoCodecType_H264,
- nullptr, // use default encoder
- source_attributes,
- nullptr, // use default compressed data allocator
- internal::VTCompressionOutputCallback, this, &compression_session_);
- if (source_attributes) {
- CFRelease(source_attributes);
- source_attributes = nullptr;
+ CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
+ CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat};
+ CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize);
+ if (ioSurfaceValue) {
+ CFRelease(ioSurfaceValue);
+ ioSurfaceValue = nullptr;
+ }
+ if (pixelFormat) {
+ CFRelease(pixelFormat);
+ pixelFormat = nullptr;
+ }
+ OSStatus status = VTCompressionSessionCreate(nullptr, // use default allocator
+ _width,
+ _height,
+ kCMVideoCodecType_H264,
+ nullptr, // use default encoder
+ sourceAttributes,
+ nullptr, // use default compressed data allocator
+ compressionOutputCallback,
+ nullptr,
+ &_compressionSession);
+ if (sourceAttributes) {
+ CFRelease(sourceAttributes);
+ sourceAttributes = nullptr;
}
if (status != noErr) {
LOG(LS_ERROR) << "Failed to create compression session: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
- ConfigureCompressionSession();
+ [self configureCompressionSession];
return WEBRTC_VIDEO_CODEC_OK;
}
-void H264VideoToolboxEncoder::ConfigureCompressionSession() {
- RTC_DCHECK(compression_session_);
- internal::SetVTSessionProperty(compression_session_,
- kVTCompressionPropertyKey_RealTime, true);
- internal::SetVTSessionProperty(compression_session_,
- kVTCompressionPropertyKey_ProfileLevel,
- profile_);
- internal::SetVTSessionProperty(compression_session_,
- kVTCompressionPropertyKey_AllowFrameReordering,
- false);
- SetEncoderBitrateBps(target_bitrate_bps_);
+- (void)configureCompressionSession {
+ RTC_DCHECK(_compressionSession);
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true);
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, _profile);
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
+ [self setEncoderBitrateBps:_targetBitrateBps];
// TODO(tkchin): Look at entropy mode and colorspace matrices.
// TODO(tkchin): Investigate to see if there's any way to make this work.
// May need it to interop with Android. Currently this call just fails.
@@ -612,156 +545,136 @@ void H264VideoToolboxEncoder::ConfigureCompressionSession() {
// kVTCompressionPropertyKey_MaxFrameDelayCount,
// 1);
- // Set a relatively large value for keyframe emission (7200 frames or
- // 4 minutes).
- internal::SetVTSessionProperty(
- compression_session_,
- kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
- internal::SetVTSessionProperty(
- compression_session_,
- kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
+ // Set a relatively large value for keyframe emission (7200 frames or 4 minutes).
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
+ SetVTSessionProperty(
+ _compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
}
-void H264VideoToolboxEncoder::DestroyCompressionSession() {
- if (compression_session_) {
- VTCompressionSessionInvalidate(compression_session_);
- CFRelease(compression_session_);
- compression_session_ = nullptr;
+- (void)destroyCompressionSession {
+ if (_compressionSession) {
+ VTCompressionSessionInvalidate(_compressionSession);
+ CFRelease(_compressionSession);
+ _compressionSession = nullptr;
}
}
-const char* H264VideoToolboxEncoder::ImplementationName() const {
- return "VideoToolbox";
+- (NSString *)implementationName {
+ return @"VideoToolbox";
}
-bool H264VideoToolboxEncoder::SupportsNativeHandle() const {
- return true;
-}
-
-void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) {
- if (encoder_bitrate_bps_ != bitrate_bps) {
- SetEncoderBitrateBps(bitrate_bps);
+- (void)setBitrateBps:(uint32_t)bitrateBps {
+ if (_encoderBitrateBps != bitrateBps) {
+ [self setEncoderBitrateBps:bitrateBps];
}
}
-void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) {
- if (compression_session_) {
- internal::SetVTSessionProperty(compression_session_,
- kVTCompressionPropertyKey_AverageBitRate,
- bitrate_bps);
+- (void)setEncoderBitrateBps:(uint32_t)bitrateBps {
+ if (_compressionSession) {
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
// TODO(tkchin): Add a helper method to set array value.
- int64_t data_limit_bytes_per_second_value = static_cast<int64_t>(
- bitrate_bps * internal::kLimitToAverageBitRateFactor / 8);
- CFNumberRef bytes_per_second =
- CFNumberCreate(kCFAllocatorDefault,
- kCFNumberSInt64Type,
- &data_limit_bytes_per_second_value);
- int64_t one_second_value = 1;
- CFNumberRef one_second =
- CFNumberCreate(kCFAllocatorDefault,
- kCFNumberSInt64Type,
- &one_second_value);
- const void* nums[2] = { bytes_per_second, one_second };
- CFArrayRef data_rate_limits =
- CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
- OSStatus status =
- VTSessionSetProperty(compression_session_,
- kVTCompressionPropertyKey_DataRateLimits,
- data_rate_limits);
- if (bytes_per_second) {
- CFRelease(bytes_per_second);
+ int64_t dataLimitBytesPerSecondValue =
+ static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
+ CFNumberRef bytesPerSecond =
+ CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
+ int64_t oneSecondValue = 1;
+ CFNumberRef oneSecond =
+ CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue);
+ const void *nums[2] = {bytesPerSecond, oneSecond};
+ CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
+ OSStatus status = VTSessionSetProperty(
+ _compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits);
+ if (bytesPerSecond) {
+ CFRelease(bytesPerSecond);
}
- if (one_second) {
- CFRelease(one_second);
+ if (oneSecond) {
+ CFRelease(oneSecond);
}
- if (data_rate_limits) {
- CFRelease(data_rate_limits);
+ if (dataRateLimits) {
+ CFRelease(dataRateLimits);
}
if (status != noErr) {
LOG(LS_ERROR) << "Failed to set data rate limit";
}
- encoder_bitrate_bps_ = bitrate_bps;
+ _encoderBitrateBps = bitrateBps;
}
}
-void H264VideoToolboxEncoder::OnEncodedFrame(
- OSStatus status,
- VTEncodeInfoFlags info_flags,
- CMSampleBufferRef sample_buffer,
- CodecSpecificInfo codec_specific_info,
- int32_t width,
- int32_t height,
- int64_t render_time_ms,
- uint32_t timestamp,
- VideoRotation rotation) {
+- (void)frameWasEncoded:(OSStatus)status
+ flags:(VTEncodeInfoFlags)infoFlags
+ sampleBuffer:(CMSampleBufferRef)sampleBuffer
+ codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo
+ width:(int32_t)width
+ height:(int32_t)height
+ renderTimeMs:(int64_t)renderTimeMs
+ timestamp:(uint32_t)timestamp
+ rotation:(RTCVideoRotation)rotation {
if (status != noErr) {
LOG(LS_ERROR) << "H264 encode failed.";
return;
}
- if (info_flags & kVTEncodeInfo_FrameDropped) {
+ if (infoFlags & kVTEncodeInfo_FrameDropped) {
LOG(LS_INFO) << "H264 encode dropped frame.";
return;
}
- bool is_keyframe = false;
- CFArrayRef attachments =
- CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
+ BOOL isKeyframe = NO;
+ CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
if (attachments != nullptr && CFArrayGetCount(attachments)) {
CFDictionaryRef attachment =
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
- is_keyframe =
- !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
+ isKeyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
}
- if (is_keyframe) {
+ if (isKeyframe) {
LOG(LS_INFO) << "Generated keyframe";
}
// Convert the sample buffer into a buffer suitable for RTP packetization.
// TODO(tkchin): Allocate buffers through a pool.
std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer());
- std::unique_ptr<webrtc::RTPFragmentationHeader> header;
+ RTCRtpFragmentationHeader *header;
{
- webrtc::RTPFragmentationHeader* header_raw;
- bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
- buffer.get(), &header_raw);
- header.reset(header_raw);
+ webrtc::RTPFragmentationHeader *header_cpp;
+ bool result =
+ H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get(), &header_cpp);
+ header = [[RTCRtpFragmentationHeader alloc] initWithNativeFragmentationHeader:header_cpp];
if (!result) {
return;
}
}
- webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
- frame._encodedWidth = width;
- frame._encodedHeight = height;
- frame._completeFrame = true;
- frame._frameType =
- is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
- frame.capture_time_ms_ = render_time_ms;
- frame._timeStamp = timestamp;
- frame.rotation_ = rotation;
- frame.content_type_ =
- (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED;
- frame.timing_.is_timing_frame = false;
-
- h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
- h264_bitstream_parser_.GetLastSliceQp(&frame.qp_);
-
- EncodedImageCallback::Result res =
- callback_->OnEncodedImage(frame, &codec_specific_info, header.get());
- if (res.error != EncodedImageCallback::Result::OK) {
- LOG(LS_ERROR) << "Encode callback failed: " << res.error;
+
+ RTCEncodedImage *frame = [[RTCEncodedImage alloc] init];
+ frame.buffer = [NSData dataWithBytesNoCopy:buffer->data() length:buffer->size() freeWhenDone:NO];
+ frame.encodedWidth = width;
+ frame.encodedHeight = height;
+ frame.completeFrame = YES;
+ frame.frameType = isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta;
+ frame.captureTimeMs = renderTimeMs;
+ frame.timeStamp = timestamp;
+ frame.rotation = rotation;
+ frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare :
+ RTCVideoContentTypeUnspecified;
+ frame.isTimingFrame = NO;
+
+ int qp;
+ _h264BitstreamParser.ParseBitstream(buffer->data(), buffer->size());
+ _h264BitstreamParser.GetLastSliceQp(&qp);
+ frame.qp = @(qp);
+
+ BOOL res = _callback(frame, codecSpecificInfo, header);
+ if (!res) {
+ LOG(LS_ERROR) << "Encode callback failed";
return;
}
- bitrate_adjuster_.Update(frame._length);
+ _bitrateAdjuster->Update(frame.buffer.length);
}
-// TODO(magjed): This function is not used by RTCVideoEncoderH264, but this whole file will be
-// removed soon and inlined as ObjC.
-VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings()
- const {
- return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold,
- internal::kHighH264QpThreshold);
+- (RTCVideoEncoderQpThresholds *)scalingSettings {
+ return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold
+ high:kHighH264QpThreshold];
}
-} // namespace webrtc
+
+@end

Powered by Google App Engine
This is Rietveld 408576698