Index: webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm |
diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm |
similarity index 38% |
rename from webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm |
rename to webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm |
index 2bac68ef6de3185525711edb5faf41f5214eb662..1d710ec6d012c6aa35ef299a3ad8fd9b022daee6 100644 |
--- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm |
+++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm |
@@ -9,26 +9,45 @@ |
* |
*/ |
-#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" |
+#import "WebRTC/RTCVideoCodecH264.h" |
-#include <memory> |
-#include <string> |
+#import <VideoToolbox/VideoToolbox.h> |
#include <vector> |
#if defined(WEBRTC_IOS) |
#import "Common/RTCUIApplicationStatusObserver.h" |
#import "WebRTC/UIDevice+RTCDevice.h" |
#endif |
+#import "PeerConnection/RTCVideoCodec+Private.h" |
+#import "WebRTC/RTCVideoCodec.h" |
+#import "WebRTC/RTCVideoFrame.h" |
#import "WebRTC/RTCVideoFrameBuffer.h" |
+#import "helpers.h" |
#include "libyuv/convert_from.h" |
+#include "webrtc/common_video/h264/h264_bitstream_parser.h" |
#include "webrtc/common_video/h264/profile_level_id.h" |
-#include "webrtc/rtc_base/checks.h" |
+#include "webrtc/common_video/include/bitrate_adjuster.h" |
+#include "webrtc/modules/include/module_common_types.h" |
+#include "webrtc/modules/video_coding/include/video_error_codes.h" |
+#include "webrtc/rtc_base/buffer.h" |
#include "webrtc/rtc_base/logging.h" |
-#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" |
+#include "webrtc/rtc_base/timeutils.h" |
#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" |
#include "webrtc/system_wrappers/include/clock.h" |
-namespace internal { |
+@interface RTCVideoEncoderH264 () |
+ |
+- (void)frameWasEncoded:(OSStatus)status |
+ flags:(VTEncodeInfoFlags)infoFlags |
+ sampleBuffer:(CMSampleBufferRef)sampleBuffer |
+ codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo |
+ width:(int32_t)width |
+ height:(int32_t)height |
+ renderTimeMs:(int64_t)renderTimeMs |
+ timestamp:(uint32_t)timestamp |
+ rotation:(RTCVideoRotation)rotation; |
+ |
+@end |
// The ratio between kVTCompressionPropertyKey_DataRateLimits and |
// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher |
@@ -39,152 +58,66 @@ const float kLimitToAverageBitRateFactor = 1.5f; |
const int kLowH264QpThreshold = 28; |
const int kHighH264QpThreshold = 39; |
-// Convenience function for creating a dictionary. |
-inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, |
- CFTypeRef* values, |
- size_t size) { |
- return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, |
- &kCFTypeDictionaryKeyCallBacks, |
- &kCFTypeDictionaryValueCallBacks); |
-} |
- |
-// Copies characters from a CFStringRef into a std::string. |
-std::string CFStringToString(const CFStringRef cf_string) { |
- RTC_DCHECK(cf_string); |
- std::string std_string; |
- // Get the size needed for UTF8 plus terminating character. |
- size_t buffer_size = |
- CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), |
- kCFStringEncodingUTF8) + |
- 1; |
- std::unique_ptr<char[]> buffer(new char[buffer_size]); |
- if (CFStringGetCString(cf_string, buffer.get(), buffer_size, |
- kCFStringEncodingUTF8)) { |
- // Copy over the characters. |
- std_string.assign(buffer.get()); |
- } |
- return std_string; |
-} |
- |
-// Convenience function for setting a VT property. |
-void SetVTSessionProperty(VTSessionRef session, |
- CFStringRef key, |
- int32_t value) { |
- CFNumberRef cfNum = |
- CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); |
- OSStatus status = VTSessionSetProperty(session, key, cfNum); |
- CFRelease(cfNum); |
- if (status != noErr) { |
- std::string key_string = CFStringToString(key); |
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
- << " to " << value << ": " << status; |
- } |
-} |
- |
-// Convenience function for setting a VT property. |
-void SetVTSessionProperty(VTSessionRef session, |
- CFStringRef key, |
- uint32_t value) { |
- int64_t value_64 = value; |
- CFNumberRef cfNum = |
- CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); |
- OSStatus status = VTSessionSetProperty(session, key, cfNum); |
- CFRelease(cfNum); |
- if (status != noErr) { |
- std::string key_string = CFStringToString(key); |
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
- << " to " << value << ": " << status; |
- } |
-} |
- |
-// Convenience function for setting a VT property. |
-void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { |
- CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; |
- OSStatus status = VTSessionSetProperty(session, key, cf_bool); |
- if (status != noErr) { |
- std::string key_string = CFStringToString(key); |
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
- << " to " << value << ": " << status; |
- } |
-} |
- |
-// Convenience function for setting a VT property. |
-void SetVTSessionProperty(VTSessionRef session, |
- CFStringRef key, |
- CFStringRef value) { |
- OSStatus status = VTSessionSetProperty(session, key, value); |
- if (status != noErr) { |
- std::string key_string = CFStringToString(key); |
- std::string val_string = CFStringToString(value); |
- LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
- << " to " << val_string << ": " << status; |
- } |
-} |
- |
// Struct that we pass to the encoder per frame to encode. We receive it again |
// in the encoder callback. |
-struct FrameEncodeParams { |
- FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e, |
- const webrtc::CodecSpecificInfo* csi, |
- int32_t w, |
- int32_t h, |
- int64_t rtms, |
- uint32_t ts, |
- webrtc::VideoRotation r) |
- : encoder(e), |
- width(w), |
- height(h), |
- render_time_ms(rtms), |
- timestamp(ts), |
- rotation(r) { |
+struct RTCFrameEncodeParams { |
+ RTCFrameEncodeParams(RTCVideoEncoderH264 *e, |
+ RTCCodecSpecificInfoH264 *csi, |
+ int32_t w, |
+ int32_t h, |
+ int64_t rtms, |
+ uint32_t ts, |
+ RTCVideoRotation r) |
+ : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) { |
if (csi) { |
- codec_specific_info = *csi; |
+ codecSpecificInfo = csi; |
} else { |
- codec_specific_info.codecType = webrtc::kVideoCodecH264; |
+ codecSpecificInfo = [[RTCCodecSpecificInfoH264 alloc] init]; |
} |
} |
- webrtc::H264VideoToolboxEncoder* encoder; |
- webrtc::CodecSpecificInfo codec_specific_info; |
+ RTCVideoEncoderH264 *encoder; |
+ RTCCodecSpecificInfoH264 *codecSpecificInfo; |
int32_t width; |
int32_t height; |
int64_t render_time_ms; |
uint32_t timestamp; |
- webrtc::VideoRotation rotation; |
+ RTCVideoRotation rotation; |
}; |
// We receive I420Frames as input, but we need to feed CVPixelBuffers into the |
// encoder. This performs the copy and format conversion. |
// TODO(tkchin): See if encoder will accept i420 frames and compare performance. |
-bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr<webrtc::I420BufferInterface>& frame, |
- CVPixelBufferRef pixel_buffer) { |
- RTC_DCHECK(pixel_buffer); |
- RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer), |
+bool CopyVideoFrameToPixelBuffer(id<RTCI420Buffer> frameBuffer, CVPixelBufferRef pixelBuffer) { |
+ RTC_DCHECK(pixelBuffer); |
+ RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), |
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); |
- RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), |
- static_cast<size_t>(frame->height())); |
- RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), |
- static_cast<size_t>(frame->width())); |
+ RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height); |
+ RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width); |
- CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); |
+ CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0); |
if (cvRet != kCVReturnSuccess) { |
LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; |
return false; |
} |
- uint8_t* dst_y = reinterpret_cast<uint8_t*>( |
- CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); |
- int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); |
- uint8_t* dst_uv = reinterpret_cast<uint8_t*>( |
- CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); |
- int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); |
+ uint8_t *dstY = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); |
+ int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); |
+ uint8_t *dstUV = reinterpret_cast<uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); |
+ int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); |
// Convert I420 to NV12. |
- int ret = libyuv::I420ToNV12( |
- frame->DataY(), frame->StrideY(), |
- frame->DataU(), frame->StrideU(), |
- frame->DataV(), frame->StrideV(), |
- dst_y, dst_stride_y, dst_uv, dst_stride_uv, |
- frame->width(), frame->height()); |
- CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); |
+ int ret = libyuv::I420ToNV12(frameBuffer.dataY, |
+ frameBuffer.strideY, |
+ frameBuffer.dataU, |
+ frameBuffer.strideU, |
+ frameBuffer.dataV, |
+ frameBuffer.strideV, |
+ dstY, |
+ dstStrideY, |
+ dstUV, |
+ dstStrideUV, |
+ frameBuffer.width, |
+ frameBuffer.height); |
+ CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); |
if (ret) { |
LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; |
return false; |
@@ -198,8 +131,7 @@ CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { |
return nullptr; |
} |
CVPixelBufferRef pixel_buffer; |
- CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, |
- &pixel_buffer); |
+ CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer); |
if (ret != kCVReturnSuccess) { |
LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; |
// We probably want to drop frames here, since failure probably means |
@@ -211,25 +143,29 @@ CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { |
// This is the callback function that VideoToolbox calls when encode is |
// complete. From inspection this happens on its own queue. |
-void VTCompressionOutputCallback(void* encoder, |
- void* params, |
- OSStatus status, |
- VTEncodeInfoFlags info_flags, |
- CMSampleBufferRef sample_buffer) { |
- std::unique_ptr<FrameEncodeParams> encode_params( |
- reinterpret_cast<FrameEncodeParams*>(params)); |
- encode_params->encoder->OnEncodedFrame( |
- status, info_flags, sample_buffer, encode_params->codec_specific_info, |
- encode_params->width, encode_params->height, |
- encode_params->render_time_ms, encode_params->timestamp, |
- encode_params->rotation); |
+void compressionOutputCallback(void *encoder, |
+ void *params, |
+ OSStatus status, |
+ VTEncodeInfoFlags infoFlags, |
+ CMSampleBufferRef sampleBuffer) { |
+ std::unique_ptr<RTCFrameEncodeParams> encodeParams( |
+ reinterpret_cast<RTCFrameEncodeParams *>(params)); |
+ [encodeParams->encoder frameWasEncoded:status |
+ flags:infoFlags |
+ sampleBuffer:sampleBuffer |
+ codecSpecificInfo:encodeParams->codecSpecificInfo |
+ width:encodeParams->width |
+ height:encodeParams->height |
+ renderTimeMs:encodeParams->render_time_ms |
+ timestamp:encodeParams->timestamp |
+ rotation:encodeParams->rotation]; |
} |
// Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no |
// specific VideoToolbox profile for the specified level, AutoLevel will be |
// returned. The user must initialize the encoder with a resolution and |
// framerate conforming to the selected H264 level regardless. |
-CFStringRef ExtractProfile(const cricket::VideoCodec& codec) { |
+CFStringRef ExtractProfile(const cricket::VideoCodec &codec) { |
const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id = |
webrtc::H264::ParseSdpProfileLevelId(codec.params); |
RTC_DCHECK(profile_level_id); |
@@ -331,9 +267,22 @@ CFStringRef ExtractProfile(const cricket::VideoCodec& codec) { |
} |
} |
-} // namespace internal |
- |
-namespace webrtc { |
+@implementation RTCVideoEncoderH264 { |
+ RTCVideoCodecInfo *_codecInfo; |
+ webrtc::BitrateAdjuster *_bitrateAdjuster; |
+ uint32_t _targetBitrateBps; |
+ uint32_t _encoderBitrateBps; |
+ RTCH264PacketizationMode _packetizationMode; |
+ CFStringRef _profile; |
+ RTCVideoEncoderCallback _callback; |
+ int32_t _width; |
+ int32_t _height; |
+ VTCompressionSessionRef _compressionSession; |
+ RTCVideoCodecMode _mode; |
+ |
+ webrtc::H264BitstreamParser _h264BitstreamParser; |
+ std::vector<uint8_t> _nv12ScaleBuffer; |
+} |
// .5 is set as a mininum to prevent overcompensating for large temporary |
// overshoots. We don't want to degrade video quality too badly. |
@@ -342,47 +291,47 @@ namespace webrtc { |
// drastically reduced bitrate, so we want to avoid that. In steady state |
// conditions, 0.95 seems to give us better overall bitrate over long periods |
// of time. |
-H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec) |
- : callback_(nullptr), |
- compression_session_(nullptr), |
- bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95), |
- packetization_mode_(H264PacketizationMode::NonInterleaved), |
- profile_(internal::ExtractProfile(codec)) { |
- LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_); |
- RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName)); |
+- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo { |
+ if (self = [super init]) { |
+ _codecInfo = codecInfo; |
+ _bitrateAdjuster = new webrtc::BitrateAdjuster(webrtc::Clock::GetRealTimeClock(), .5, .95); |
+ _packetizationMode = NonInterleaved; |
+ _profile = ExtractProfile([codecInfo nativeVideoCodec]); |
+ LOG(LS_INFO) << "Using profile " << CFStringToString(_profile); |
+ RTC_CHECK([codecInfo.name isEqualToString:@"H264"]); |
+ } |
+ return self; |
} |
-H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { |
- DestroyCompressionSession(); |
+- (void)dealloc { |
+ [self destroyCompressionSession]; |
} |
-int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, |
- int number_of_cores, |
- size_t max_payload_size) { |
- RTC_DCHECK(codec_settings); |
- RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); |
+- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings |
+ numberOfCores:(int)numberOfCores { |
+ RTC_DCHECK(settings); |
+ RTC_DCHECK([settings.name isEqualToString:@"H264"]); |
+ |
+ _width = settings.width; |
+ _height = settings.height; |
+ _mode = settings.mode; |
- width_ = codec_settings->width; |
- height_ = codec_settings->height; |
- mode_ = codec_settings->mode; |
// We can only set average bitrate on the HW encoder. |
- target_bitrate_bps_ = codec_settings->startBitrate; |
- bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); |
+ _targetBitrateBps = settings.startBitrate; |
+ _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); |
// TODO(tkchin): Try setting payload size via |
// kVTCompressionPropertyKey_MaxH264SliceBytes. |
- return ResetCompressionSession(); |
+ return [self resetCompressionSession]; |
} |
-int H264VideoToolboxEncoder::Encode( |
- const VideoFrame& frame, |
- const CodecSpecificInfo* codec_specific_info, |
- const std::vector<FrameType>* frame_types) { |
- // |input_frame| size should always match codec settings. |
- RTC_DCHECK_EQ(frame.width(), width_); |
- RTC_DCHECK_EQ(frame.height(), height_); |
- if (!callback_ || !compression_session_) { |
+- (NSInteger)encode:(RTCVideoFrame *)frame |
+ codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo |
+ frameTypes:(NSArray<NSNumber *> *)frameTypes { |
+ RTC_DCHECK_EQ(frame.width, _width); |
+ RTC_DCHECK_EQ(frame.height, _height); |
+ if (!_callback || !_compressionSession) { |
return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
} |
#if defined(WEBRTC_IOS) |
@@ -392,119 +341,116 @@ int H264VideoToolboxEncoder::Encode( |
return WEBRTC_VIDEO_CODEC_OK; |
} |
#endif |
- bool is_keyframe_required = false; |
+ BOOL isKeyframeRequired = NO; |
// Get a pixel buffer from the pool and copy frame data over. |
- CVPixelBufferPoolRef pixel_buffer_pool = |
- VTCompressionSessionGetPixelBufferPool(compression_session_); |
+ CVPixelBufferPoolRef pixelBufferPool = |
+ VTCompressionSessionGetPixelBufferPool(_compressionSession); |
+ |
#if defined(WEBRTC_IOS) |
- if (!pixel_buffer_pool) { |
+ if (!pixelBufferPool) { |
// Kind of a hack. On backgrounding, the compression session seems to get |
// invalidated, which causes this pool call to fail when the application |
// is foregrounded and frames are being sent for encoding again. |
// Resetting the session when this happens fixes the issue. |
// In addition we request a keyframe so video can recover quickly. |
- ResetCompressionSession(); |
- pixel_buffer_pool = |
- VTCompressionSessionGetPixelBufferPool(compression_session_); |
- is_keyframe_required = true; |
+ [self resetCompressionSession]; |
+ pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession); |
+ isKeyframeRequired = YES; |
LOG(LS_INFO) << "Resetting compression session due to invalid pool."; |
} |
#endif |
- CVPixelBufferRef pixel_buffer = nullptr; |
- if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) { |
- // Native frame. |
- rtc::scoped_refptr<ObjCFrameBuffer> objc_frame_buffer( |
- static_cast<ObjCFrameBuffer*>(frame.video_frame_buffer().get())); |
- id<RTCVideoFrameBuffer> wrapped_frame_buffer = |
- (id<RTCVideoFrameBuffer>)objc_frame_buffer->wrapped_frame_buffer(); |
- |
- if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { |
- RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer; |
- if (![rtc_pixel_buffer requiresCropping]) { |
- // This pixel buffer might have a higher resolution than what the |
- // compression session is configured to. The compression session can |
- // handle that and will output encoded frames in the configured |
- // resolution regardless of the input pixel buffer resolution. |
- pixel_buffer = rtc_pixel_buffer.pixelBuffer; |
- CVBufferRetain(pixel_buffer); |
+ CVPixelBufferRef pixelBuffer = nullptr; |
+ if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { |
+ // Native frame buffer |
+ RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer; |
+ if (![rtcPixelBuffer requiresCropping]) { |
+ // This pixel buffer might have a higher resolution than what the |
+ // compression session is configured to. The compression session can |
+ // handle that and will output encoded frames in the configured |
+ // resolution regardless of the input pixel buffer resolution. |
+ pixelBuffer = rtcPixelBuffer.pixelBuffer; |
+ CVBufferRetain(pixelBuffer); |
+ } else { |
+ // Cropping required, we need to crop and scale to a new pixel buffer. |
+ pixelBuffer = CreatePixelBuffer(pixelBufferPool); |
+ if (!pixelBuffer) { |
+ return WEBRTC_VIDEO_CODEC_ERROR; |
+ } |
+ int dstWidth = CVPixelBufferGetWidth(pixelBuffer); |
+ int dstHeight = CVPixelBufferGetHeight(pixelBuffer); |
+ if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) { |
+ int size = |
+ [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight]; |
+ _nv12ScaleBuffer.resize(size); |
} else { |
- // Cropping required, we need to crop and scale to a new pixel buffer. |
- pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); |
- if (!pixel_buffer) { |
- return WEBRTC_VIDEO_CODEC_ERROR; |
- } |
- int dst_width = CVPixelBufferGetWidth(pixel_buffer); |
- int dst_height = CVPixelBufferGetHeight(pixel_buffer); |
- if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) { |
- int size = |
- [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height]; |
- nv12_scale_buffer_.resize(size); |
- } else { |
- nv12_scale_buffer_.clear(); |
- } |
- nv12_scale_buffer_.shrink_to_fit(); |
- if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer |
- withTempBuffer:nv12_scale_buffer_.data()]) { |
- return WEBRTC_VIDEO_CODEC_ERROR; |
- } |
+ _nv12ScaleBuffer.clear(); |
+ } |
+ _nv12ScaleBuffer.shrink_to_fit(); |
+ if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer withTempBuffer:_nv12ScaleBuffer.data()]) { |
+ return WEBRTC_VIDEO_CODEC_ERROR; |
} |
} |
} |
- if (!pixel_buffer) { |
- // We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame |
- pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); |
- if (!pixel_buffer) { |
+ if (!pixelBuffer) { |
+ // We did not have a native frame buffer |
+ pixelBuffer = CreatePixelBuffer(pixelBufferPool); |
+ if (!pixelBuffer) { |
return WEBRTC_VIDEO_CODEC_ERROR; |
} |
- RTC_DCHECK(pixel_buffer); |
- if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI420(), |
- pixel_buffer)) { |
+ RTC_DCHECK(pixelBuffer); |
+ if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) { |
LOG(LS_ERROR) << "Failed to copy frame data."; |
- CVBufferRelease(pixel_buffer); |
+ CVBufferRelease(pixelBuffer); |
return WEBRTC_VIDEO_CODEC_ERROR; |
} |
} |
// Check if we need a keyframe. |
- if (!is_keyframe_required && frame_types) { |
- for (auto frame_type : *frame_types) { |
- if (frame_type == kVideoFrameKey) { |
- is_keyframe_required = true; |
+ if (!isKeyframeRequired && frameTypes) { |
+ for (NSNumber *frameType in frameTypes) { |
+ if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) { |
+ isKeyframeRequired = YES; |
break; |
} |
} |
} |
- CMTime presentation_time_stamp = |
- CMTimeMake(frame.render_time_ms(), 1000); |
- CFDictionaryRef frame_properties = nullptr; |
- if (is_keyframe_required) { |
+ CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); |
+ CFDictionaryRef frameProperties = nullptr; |
+ if (isKeyframeRequired) { |
CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; |
CFTypeRef values[] = {kCFBooleanTrue}; |
- frame_properties = internal::CreateCFDictionary(keys, values, 1); |
+ frameProperties = CreateCFTypeDictionary(keys, values, 1); |
} |
- std::unique_ptr<internal::FrameEncodeParams> encode_params; |
- encode_params.reset(new internal::FrameEncodeParams( |
- this, codec_specific_info, width_, height_, frame.render_time_ms(), |
- frame.timestamp(), frame.rotation())); |
- encode_params->codec_specific_info.codecSpecific.H264.packetization_mode = |
- packetization_mode_; |
+ std::unique_ptr<RTCFrameEncodeParams> encodeParams; |
+ encodeParams.reset(new RTCFrameEncodeParams(self, |
+ codecSpecificInfo, |
+ _width, |
+ _height, |
+ frame.timeStampNs / rtc::kNumNanosecsPerMillisec, |
+ frame.timeStamp, |
+ frame.rotation)); |
+ encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode; |
// Update the bitrate if needed. |
- SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); |
+ [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; |
- OSStatus status = VTCompressionSessionEncodeFrame( |
- compression_session_, pixel_buffer, presentation_time_stamp, |
- kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); |
- if (frame_properties) { |
- CFRelease(frame_properties); |
+ OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession, |
+ pixelBuffer, |
+ presentationTimeStamp, |
+ kCMTimeInvalid, |
+ frameProperties, |
+ encodeParams.release(), |
+ nullptr); |
+ if (frameProperties) { |
+ CFRelease(frameProperties); |
} |
- if (pixel_buffer) { |
- CVBufferRelease(pixel_buffer); |
+ if (pixelBuffer) { |
+ CVBufferRelease(pixelBuffer); |
} |
if (status != noErr) { |
LOG(LS_ERROR) << "Failed to encode frame with code: " << status; |
@@ -513,42 +459,35 @@ int H264VideoToolboxEncoder::Encode( |
return WEBRTC_VIDEO_CODEC_OK; |
} |
-int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( |
- EncodedImageCallback* callback) { |
- callback_ = callback; |
- return WEBRTC_VIDEO_CODEC_OK; |
+- (void)setCallback:(RTCVideoEncoderCallback)callback { |
+ _callback = callback; |
} |
-int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, |
- int64_t rtt) { |
- // Encoder doesn't know anything about packet loss or rtt so just return. |
+- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { |
+ _targetBitrateBps = 1000 * bitrateKbit; |
+ _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); |
+ [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; |
return WEBRTC_VIDEO_CODEC_OK; |
} |
-int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, |
- uint32_t frame_rate) { |
- target_bitrate_bps_ = 1000 * new_bitrate_kbit; |
- bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); |
- SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); |
- return WEBRTC_VIDEO_CODEC_OK; |
-} |
+#pragma mark - Private |
-int H264VideoToolboxEncoder::Release() { |
+- (NSInteger)releaseEncoder { |
// Need to destroy so that the session is invalidated and won't use the |
// callback anymore. Do not remove callback until the session is invalidated |
// since async encoder callbacks can occur until invalidation. |
- DestroyCompressionSession(); |
- callback_ = nullptr; |
+ [self destroyCompressionSession]; |
+ _callback = nullptr; |
return WEBRTC_VIDEO_CODEC_OK; |
} |
-int H264VideoToolboxEncoder::ResetCompressionSession() { |
- DestroyCompressionSession(); |
+- (int)resetCompressionSession { |
+ [self destroyCompressionSession]; |
// Set source image buffer attributes. These attributes will be present on |
// buffers retrieved from the encoder's pixel buffer pool. |
- const size_t attributes_size = 3; |
- CFTypeRef keys[attributes_size] = { |
+ const size_t attributesSize = 3; |
+ CFTypeRef keys[attributesSize] = { |
#if defined(WEBRTC_IOS) |
kCVPixelBufferOpenGLESCompatibilityKey, |
#elif defined(WEBRTC_MAC) |
@@ -557,53 +496,47 @@ int H264VideoToolboxEncoder::ResetCompressionSession() { |
kCVPixelBufferIOSurfacePropertiesKey, |
kCVPixelBufferPixelFormatTypeKey |
}; |
- CFDictionaryRef io_surface_value = |
- internal::CreateCFDictionary(nullptr, nullptr, 0); |
+ CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); |
int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; |
- CFNumberRef pixel_format = |
- CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); |
- CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, |
- pixel_format}; |
- CFDictionaryRef source_attributes = |
- internal::CreateCFDictionary(keys, values, attributes_size); |
- if (io_surface_value) { |
- CFRelease(io_surface_value); |
- io_surface_value = nullptr; |
- } |
- if (pixel_format) { |
- CFRelease(pixel_format); |
- pixel_format = nullptr; |
- } |
- OSStatus status = VTCompressionSessionCreate( |
- nullptr, // use default allocator |
- width_, height_, kCMVideoCodecType_H264, |
- nullptr, // use default encoder |
- source_attributes, |
- nullptr, // use default compressed data allocator |
- internal::VTCompressionOutputCallback, this, &compression_session_); |
- if (source_attributes) { |
- CFRelease(source_attributes); |
- source_attributes = nullptr; |
+ CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); |
+ CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat}; |
+ CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize); |
+ if (ioSurfaceValue) { |
+ CFRelease(ioSurfaceValue); |
+ ioSurfaceValue = nullptr; |
+ } |
+ if (pixelFormat) { |
+ CFRelease(pixelFormat); |
+ pixelFormat = nullptr; |
+ } |
+ OSStatus status = VTCompressionSessionCreate(nullptr, // use default allocator |
+ _width, |
+ _height, |
+ kCMVideoCodecType_H264, |
+ nullptr, // use default encoder |
+ sourceAttributes, |
+ nullptr, // use default compressed data allocator |
+ compressionOutputCallback, |
+ nullptr, |
+ &_compressionSession); |
+ if (sourceAttributes) { |
+ CFRelease(sourceAttributes); |
+ sourceAttributes = nullptr; |
} |
if (status != noErr) { |
LOG(LS_ERROR) << "Failed to create compression session: " << status; |
return WEBRTC_VIDEO_CODEC_ERROR; |
} |
- ConfigureCompressionSession(); |
+ [self configureCompressionSession]; |
return WEBRTC_VIDEO_CODEC_OK; |
} |
-void H264VideoToolboxEncoder::ConfigureCompressionSession() { |
- RTC_DCHECK(compression_session_); |
- internal::SetVTSessionProperty(compression_session_, |
- kVTCompressionPropertyKey_RealTime, true); |
- internal::SetVTSessionProperty(compression_session_, |
- kVTCompressionPropertyKey_ProfileLevel, |
- profile_); |
- internal::SetVTSessionProperty(compression_session_, |
- kVTCompressionPropertyKey_AllowFrameReordering, |
- false); |
- SetEncoderBitrateBps(target_bitrate_bps_); |
+- (void)configureCompressionSession { |
+ RTC_DCHECK(_compressionSession); |
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true); |
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, _profile); |
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false); |
+ [self setEncoderBitrateBps:_targetBitrateBps]; |
// TODO(tkchin): Look at entropy mode and colorspace matrices. |
// TODO(tkchin): Investigate to see if there's any way to make this work. |
// May need it to interop with Android. Currently this call just fails. |
@@ -612,156 +545,136 @@ void H264VideoToolboxEncoder::ConfigureCompressionSession() { |
// kVTCompressionPropertyKey_MaxFrameDelayCount, |
// 1); |
- // Set a relatively large value for keyframe emission (7200 frames or |
- // 4 minutes). |
- internal::SetVTSessionProperty( |
- compression_session_, |
- kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); |
- internal::SetVTSessionProperty( |
- compression_session_, |
- kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); |
+ // Set a relatively large value for keyframe emission (7200 frames or 4 minutes). |
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); |
+ SetVTSessionProperty( |
+ _compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); |
} |
-void H264VideoToolboxEncoder::DestroyCompressionSession() { |
- if (compression_session_) { |
- VTCompressionSessionInvalidate(compression_session_); |
- CFRelease(compression_session_); |
- compression_session_ = nullptr; |
+- (void)destroyCompressionSession { |
+ if (_compressionSession) { |
+ VTCompressionSessionInvalidate(_compressionSession); |
+ CFRelease(_compressionSession); |
+ _compressionSession = nullptr; |
} |
} |
-const char* H264VideoToolboxEncoder::ImplementationName() const { |
- return "VideoToolbox"; |
+- (NSString *)implementationName { |
+ return @"VideoToolbox"; |
} |
-bool H264VideoToolboxEncoder::SupportsNativeHandle() const { |
- return true; |
-} |
- |
-void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) { |
- if (encoder_bitrate_bps_ != bitrate_bps) { |
- SetEncoderBitrateBps(bitrate_bps); |
+- (void)setBitrateBps:(uint32_t)bitrateBps { |
+ if (_encoderBitrateBps != bitrateBps) { |
+ [self setEncoderBitrateBps:bitrateBps]; |
} |
} |
-void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) { |
- if (compression_session_) { |
- internal::SetVTSessionProperty(compression_session_, |
- kVTCompressionPropertyKey_AverageBitRate, |
- bitrate_bps); |
+- (void)setEncoderBitrateBps:(uint32_t)bitrateBps { |
+ if (_compressionSession) { |
+ SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps); |
// TODO(tkchin): Add a helper method to set array value. |
- int64_t data_limit_bytes_per_second_value = static_cast<int64_t>( |
- bitrate_bps * internal::kLimitToAverageBitRateFactor / 8); |
- CFNumberRef bytes_per_second = |
- CFNumberCreate(kCFAllocatorDefault, |
- kCFNumberSInt64Type, |
- &data_limit_bytes_per_second_value); |
- int64_t one_second_value = 1; |
- CFNumberRef one_second = |
- CFNumberCreate(kCFAllocatorDefault, |
- kCFNumberSInt64Type, |
- &one_second_value); |
- const void* nums[2] = { bytes_per_second, one_second }; |
- CFArrayRef data_rate_limits = |
- CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); |
- OSStatus status = |
- VTSessionSetProperty(compression_session_, |
- kVTCompressionPropertyKey_DataRateLimits, |
- data_rate_limits); |
- if (bytes_per_second) { |
- CFRelease(bytes_per_second); |
+ int64_t dataLimitBytesPerSecondValue = |
+ static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8); |
+ CFNumberRef bytesPerSecond = |
+ CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue); |
+ int64_t oneSecondValue = 1; |
+ CFNumberRef oneSecond = |
+ CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue); |
+ const void *nums[2] = {bytesPerSecond, oneSecond}; |
+ CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); |
+ OSStatus status = VTSessionSetProperty( |
+ _compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits); |
+ if (bytesPerSecond) { |
+ CFRelease(bytesPerSecond); |
} |
- if (one_second) { |
- CFRelease(one_second); |
+ if (oneSecond) { |
+ CFRelease(oneSecond); |
} |
- if (data_rate_limits) { |
- CFRelease(data_rate_limits); |
+ if (dataRateLimits) { |
+ CFRelease(dataRateLimits); |
} |
if (status != noErr) { |
LOG(LS_ERROR) << "Failed to set data rate limit"; |
} |
- encoder_bitrate_bps_ = bitrate_bps; |
+ _encoderBitrateBps = bitrateBps; |
} |
} |
-void H264VideoToolboxEncoder::OnEncodedFrame( |
- OSStatus status, |
- VTEncodeInfoFlags info_flags, |
- CMSampleBufferRef sample_buffer, |
- CodecSpecificInfo codec_specific_info, |
- int32_t width, |
- int32_t height, |
- int64_t render_time_ms, |
- uint32_t timestamp, |
- VideoRotation rotation) { |
+- (void)frameWasEncoded:(OSStatus)status |
+ flags:(VTEncodeInfoFlags)infoFlags |
+ sampleBuffer:(CMSampleBufferRef)sampleBuffer |
+ codecSpecificInfo:(id<RTCCodecSpecificInfo>)codecSpecificInfo |
+ width:(int32_t)width |
+ height:(int32_t)height |
+ renderTimeMs:(int64_t)renderTimeMs |
+ timestamp:(uint32_t)timestamp |
+ rotation:(RTCVideoRotation)rotation { |
if (status != noErr) { |
LOG(LS_ERROR) << "H264 encode failed."; |
return; |
} |
- if (info_flags & kVTEncodeInfo_FrameDropped) { |
+ if (infoFlags & kVTEncodeInfo_FrameDropped) { |
LOG(LS_INFO) << "H264 encode dropped frame."; |
return; |
} |
- bool is_keyframe = false; |
- CFArrayRef attachments = |
- CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); |
+ BOOL isKeyframe = NO; |
+ CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); |
if (attachments != nullptr && CFArrayGetCount(attachments)) { |
CFDictionaryRef attachment = |
static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); |
- is_keyframe = |
- !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); |
+ isKeyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); |
} |
- if (is_keyframe) { |
+ if (isKeyframe) { |
LOG(LS_INFO) << "Generated keyframe"; |
} |
// Convert the sample buffer into a buffer suitable for RTP packetization. |
// TODO(tkchin): Allocate buffers through a pool. |
std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer()); |
- std::unique_ptr<webrtc::RTPFragmentationHeader> header; |
+ RTCRtpFragmentationHeader *header; |
{ |
- webrtc::RTPFragmentationHeader* header_raw; |
- bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe, |
- buffer.get(), &header_raw); |
- header.reset(header_raw); |
+ webrtc::RTPFragmentationHeader *header_cpp; |
+ bool result = |
+ H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get(), &header_cpp); |
+ header = [[RTCRtpFragmentationHeader alloc] initWithNativeFragmentationHeader:header_cpp]; |
if (!result) { |
return; |
} |
} |
- webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); |
- frame._encodedWidth = width; |
- frame._encodedHeight = height; |
- frame._completeFrame = true; |
- frame._frameType = |
- is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; |
- frame.capture_time_ms_ = render_time_ms; |
- frame._timeStamp = timestamp; |
- frame.rotation_ = rotation; |
- frame.content_type_ = |
- (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; |
- frame.timing_.is_timing_frame = false; |
- |
- h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); |
- h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); |
- |
- EncodedImageCallback::Result res = |
- callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); |
- if (res.error != EncodedImageCallback::Result::OK) { |
- LOG(LS_ERROR) << "Encode callback failed: " << res.error; |
+ |
+ RTCEncodedImage *frame = [[RTCEncodedImage alloc] init]; |
+ frame.buffer = [NSData dataWithBytesNoCopy:buffer->data() length:buffer->size() freeWhenDone:NO]; |
+ frame.encodedWidth = width; |
+ frame.encodedHeight = height; |
+ frame.completeFrame = YES; |
+ frame.frameType = isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta; |
+ frame.captureTimeMs = renderTimeMs; |
+ frame.timeStamp = timestamp; |
+ frame.rotation = rotation; |
+ frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare : |
+ RTCVideoContentTypeUnspecified; |
+ frame.isTimingFrame = NO; |
+ |
+ int qp; |
+ _h264BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); |
+ _h264BitstreamParser.GetLastSliceQp(&qp); |
+ frame.qp = @(qp); |
+ |
+ BOOL res = _callback(frame, codecSpecificInfo, header); |
+ if (!res) { |
+ LOG(LS_ERROR) << "Encode callback failed"; |
return; |
} |
- bitrate_adjuster_.Update(frame._length); |
+ _bitrateAdjuster->Update(frame.buffer.length); |
} |
-// TODO(magjed): This function is not used by RTCVideoEncoderH264, but this whole file will be |
-// removed soon and inlined as ObjC. |
-VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() |
- const { |
- return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, |
- internal::kHighH264QpThreshold); |
+- (RTCVideoEncoderQpThresholds *)scalingSettings { |
+ return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold |
+ high:kHighH264QpThreshold]; |
} |
-} // namespace webrtc |
+ |
+@end |