| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 * | |
| 10 */ | |
| 11 | |
| 12 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" | |
| 13 | |
| 14 #include <memory> | |
| 15 #include <string> | |
| 16 #include <vector> | |
| 17 | |
| 18 #if defined(WEBRTC_IOS) | |
| 19 #import "Common/RTCUIApplicationStatusObserver.h" | |
| 20 #import "WebRTC/UIDevice+RTCDevice.h" | |
| 21 #endif | |
| 22 #import "WebRTC/RTCVideoFrameBuffer.h" | |
| 23 #include "libyuv/convert_from.h" | |
| 24 #include "webrtc/common_video/h264/profile_level_id.h" | |
| 25 #include "webrtc/rtc_base/checks.h" | |
| 26 #include "webrtc/rtc_base/logging.h" | |
| 27 #include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" | |
| 28 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" | |
| 29 #include "webrtc/system_wrappers/include/clock.h" | |
| 30 | |
| 31 namespace internal { | |
| 32 | |
| 33 // The ratio between kVTCompressionPropertyKey_DataRateLimits and | |
| 34 // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher | |
| 35 // than the average bit rate to avoid undershooting the target. | |
| 36 const float kLimitToAverageBitRateFactor = 1.5f; | |
| 37 // These thresholds deviate from the default h264 QP thresholds, as they | |
| 38 // have been found to work better on devices that support VideoToolbox | |
| 39 const int kLowH264QpThreshold = 28; | |
| 40 const int kHighH264QpThreshold = 39; | |
| 41 | |
| 42 // Convenience function for creating a dictionary. | |
| 43 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
| 44 CFTypeRef* values, | |
| 45 size_t size) { | |
| 46 return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, | |
| 47 &kCFTypeDictionaryKeyCallBacks, | |
| 48 &kCFTypeDictionaryValueCallBacks); | |
| 49 } | |
| 50 | |
| 51 // Copies characters from a CFStringRef into a std::string. | |
| 52 std::string CFStringToString(const CFStringRef cf_string) { | |
| 53 RTC_DCHECK(cf_string); | |
| 54 std::string std_string; | |
| 55 // Get the size needed for UTF8 plus terminating character. | |
| 56 size_t buffer_size = | |
| 57 CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), | |
| 58 kCFStringEncodingUTF8) + | |
| 59 1; | |
| 60 std::unique_ptr<char[]> buffer(new char[buffer_size]); | |
| 61 if (CFStringGetCString(cf_string, buffer.get(), buffer_size, | |
| 62 kCFStringEncodingUTF8)) { | |
| 63 // Copy over the characters. | |
| 64 std_string.assign(buffer.get()); | |
| 65 } | |
| 66 return std_string; | |
| 67 } | |
| 68 | |
| 69 // Convenience function for setting a VT property. | |
| 70 void SetVTSessionProperty(VTSessionRef session, | |
| 71 CFStringRef key, | |
| 72 int32_t value) { | |
| 73 CFNumberRef cfNum = | |
| 74 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); | |
| 75 OSStatus status = VTSessionSetProperty(session, key, cfNum); | |
| 76 CFRelease(cfNum); | |
| 77 if (status != noErr) { | |
| 78 std::string key_string = CFStringToString(key); | |
| 79 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
| 80 << " to " << value << ": " << status; | |
| 81 } | |
| 82 } | |
| 83 | |
| 84 // Convenience function for setting a VT property. | |
| 85 void SetVTSessionProperty(VTSessionRef session, | |
| 86 CFStringRef key, | |
| 87 uint32_t value) { | |
| 88 int64_t value_64 = value; | |
| 89 CFNumberRef cfNum = | |
| 90 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); | |
| 91 OSStatus status = VTSessionSetProperty(session, key, cfNum); | |
| 92 CFRelease(cfNum); | |
| 93 if (status != noErr) { | |
| 94 std::string key_string = CFStringToString(key); | |
| 95 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
| 96 << " to " << value << ": " << status; | |
| 97 } | |
| 98 } | |
| 99 | |
| 100 // Convenience function for setting a VT property. | |
| 101 void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { | |
| 102 CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; | |
| 103 OSStatus status = VTSessionSetProperty(session, key, cf_bool); | |
| 104 if (status != noErr) { | |
| 105 std::string key_string = CFStringToString(key); | |
| 106 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
| 107 << " to " << value << ": " << status; | |
| 108 } | |
| 109 } | |
| 110 | |
| 111 // Convenience function for setting a VT property. | |
| 112 void SetVTSessionProperty(VTSessionRef session, | |
| 113 CFStringRef key, | |
| 114 CFStringRef value) { | |
| 115 OSStatus status = VTSessionSetProperty(session, key, value); | |
| 116 if (status != noErr) { | |
| 117 std::string key_string = CFStringToString(key); | |
| 118 std::string val_string = CFStringToString(value); | |
| 119 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
| 120 << " to " << val_string << ": " << status; | |
| 121 } | |
| 122 } | |
| 123 | |
| 124 // Struct that we pass to the encoder per frame to encode. We receive it again | |
| 125 // in the encoder callback. | |
| 126 struct FrameEncodeParams { | |
| 127 FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e, | |
| 128 const webrtc::CodecSpecificInfo* csi, | |
| 129 int32_t w, | |
| 130 int32_t h, | |
| 131 int64_t rtms, | |
| 132 uint32_t ts, | |
| 133 webrtc::VideoRotation r) | |
| 134 : encoder(e), | |
| 135 width(w), | |
| 136 height(h), | |
| 137 render_time_ms(rtms), | |
| 138 timestamp(ts), | |
| 139 rotation(r) { | |
| 140 if (csi) { | |
| 141 codec_specific_info = *csi; | |
| 142 } else { | |
| 143 codec_specific_info.codecType = webrtc::kVideoCodecH264; | |
| 144 } | |
| 145 } | |
| 146 | |
| 147 webrtc::H264VideoToolboxEncoder* encoder; | |
| 148 webrtc::CodecSpecificInfo codec_specific_info; | |
| 149 int32_t width; | |
| 150 int32_t height; | |
| 151 int64_t render_time_ms; | |
| 152 uint32_t timestamp; | |
| 153 webrtc::VideoRotation rotation; | |
| 154 }; | |
| 155 | |
| 156 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the | |
| 157 // encoder. This performs the copy and format conversion. | |
| 158 // TODO(tkchin): See if encoder will accept i420 frames and compare performance. | |
| 159 bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr<webrtc::I420BufferInte
rface>& frame, | |
| 160 CVPixelBufferRef pixel_buffer) { | |
| 161 RTC_DCHECK(pixel_buffer); | |
| 162 RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer), | |
| 163 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); | |
| 164 RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), | |
| 165 static_cast<size_t>(frame->height())); | |
| 166 RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), | |
| 167 static_cast<size_t>(frame->width())); | |
| 168 | |
| 169 CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); | |
| 170 if (cvRet != kCVReturnSuccess) { | |
| 171 LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; | |
| 172 return false; | |
| 173 } | |
| 174 uint8_t* dst_y = reinterpret_cast<uint8_t*>( | |
| 175 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); | |
| 176 int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); | |
| 177 uint8_t* dst_uv = reinterpret_cast<uint8_t*>( | |
| 178 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); | |
| 179 int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); | |
| 180 // Convert I420 to NV12. | |
| 181 int ret = libyuv::I420ToNV12( | |
| 182 frame->DataY(), frame->StrideY(), | |
| 183 frame->DataU(), frame->StrideU(), | |
| 184 frame->DataV(), frame->StrideV(), | |
| 185 dst_y, dst_stride_y, dst_uv, dst_stride_uv, | |
| 186 frame->width(), frame->height()); | |
| 187 CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); | |
| 188 if (ret) { | |
| 189 LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; | |
| 190 return false; | |
| 191 } | |
| 192 return true; | |
| 193 } | |
| 194 | |
| 195 CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { | |
| 196 if (!pixel_buffer_pool) { | |
| 197 LOG(LS_ERROR) << "Failed to get pixel buffer pool."; | |
| 198 return nullptr; | |
| 199 } | |
| 200 CVPixelBufferRef pixel_buffer; | |
| 201 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, | |
| 202 &pixel_buffer); | |
| 203 if (ret != kCVReturnSuccess) { | |
| 204 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; | |
| 205 // We probably want to drop frames here, since failure probably means | |
| 206 // that the pool is empty. | |
| 207 return nullptr; | |
| 208 } | |
| 209 return pixel_buffer; | |
| 210 } | |
| 211 | |
| 212 // This is the callback function that VideoToolbox calls when encode is | |
| 213 // complete. From inspection this happens on its own queue. | |
| 214 void VTCompressionOutputCallback(void* encoder, | |
| 215 void* params, | |
| 216 OSStatus status, | |
| 217 VTEncodeInfoFlags info_flags, | |
| 218 CMSampleBufferRef sample_buffer) { | |
| 219 std::unique_ptr<FrameEncodeParams> encode_params( | |
| 220 reinterpret_cast<FrameEncodeParams*>(params)); | |
| 221 encode_params->encoder->OnEncodedFrame( | |
| 222 status, info_flags, sample_buffer, encode_params->codec_specific_info, | |
| 223 encode_params->width, encode_params->height, | |
| 224 encode_params->render_time_ms, encode_params->timestamp, | |
| 225 encode_params->rotation); | |
| 226 } | |
| 227 | |
| 228 // Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no | |
| 229 // specific VideoToolbox profile for the specified level, AutoLevel will be | |
| 230 // returned. The user must initialize the encoder with a resolution and | |
| 231 // framerate conforming to the selected H264 level regardless. | |
| 232 CFStringRef ExtractProfile(const cricket::VideoCodec& codec) { | |
| 233 const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id = | |
| 234 webrtc::H264::ParseSdpProfileLevelId(codec.params); | |
| 235 RTC_DCHECK(profile_level_id); | |
| 236 switch (profile_level_id->profile) { | |
| 237 case webrtc::H264::kProfileConstrainedBaseline: | |
| 238 case webrtc::H264::kProfileBaseline: | |
| 239 switch (profile_level_id->level) { | |
| 240 case webrtc::H264::kLevel3: | |
| 241 return kVTProfileLevel_H264_Baseline_3_0; | |
| 242 case webrtc::H264::kLevel3_1: | |
| 243 return kVTProfileLevel_H264_Baseline_3_1; | |
| 244 case webrtc::H264::kLevel3_2: | |
| 245 return kVTProfileLevel_H264_Baseline_3_2; | |
| 246 case webrtc::H264::kLevel4: | |
| 247 return kVTProfileLevel_H264_Baseline_4_0; | |
| 248 case webrtc::H264::kLevel4_1: | |
| 249 return kVTProfileLevel_H264_Baseline_4_1; | |
| 250 case webrtc::H264::kLevel4_2: | |
| 251 return kVTProfileLevel_H264_Baseline_4_2; | |
| 252 case webrtc::H264::kLevel5: | |
| 253 return kVTProfileLevel_H264_Baseline_5_0; | |
| 254 case webrtc::H264::kLevel5_1: | |
| 255 return kVTProfileLevel_H264_Baseline_5_1; | |
| 256 case webrtc::H264::kLevel5_2: | |
| 257 return kVTProfileLevel_H264_Baseline_5_2; | |
| 258 case webrtc::H264::kLevel1: | |
| 259 case webrtc::H264::kLevel1_b: | |
| 260 case webrtc::H264::kLevel1_1: | |
| 261 case webrtc::H264::kLevel1_2: | |
| 262 case webrtc::H264::kLevel1_3: | |
| 263 case webrtc::H264::kLevel2: | |
| 264 case webrtc::H264::kLevel2_1: | |
| 265 case webrtc::H264::kLevel2_2: | |
| 266 return kVTProfileLevel_H264_Baseline_AutoLevel; | |
| 267 } | |
| 268 | |
| 269 case webrtc::H264::kProfileMain: | |
| 270 switch (profile_level_id->level) { | |
| 271 case webrtc::H264::kLevel3: | |
| 272 return kVTProfileLevel_H264_Main_3_0; | |
| 273 case webrtc::H264::kLevel3_1: | |
| 274 return kVTProfileLevel_H264_Main_3_1; | |
| 275 case webrtc::H264::kLevel3_2: | |
| 276 return kVTProfileLevel_H264_Main_3_2; | |
| 277 case webrtc::H264::kLevel4: | |
| 278 return kVTProfileLevel_H264_Main_4_0; | |
| 279 case webrtc::H264::kLevel4_1: | |
| 280 return kVTProfileLevel_H264_Main_4_1; | |
| 281 case webrtc::H264::kLevel4_2: | |
| 282 return kVTProfileLevel_H264_Main_4_2; | |
| 283 case webrtc::H264::kLevel5: | |
| 284 return kVTProfileLevel_H264_Main_5_0; | |
| 285 case webrtc::H264::kLevel5_1: | |
| 286 return kVTProfileLevel_H264_Main_5_1; | |
| 287 case webrtc::H264::kLevel5_2: | |
| 288 return kVTProfileLevel_H264_Main_5_2; | |
| 289 case webrtc::H264::kLevel1: | |
| 290 case webrtc::H264::kLevel1_b: | |
| 291 case webrtc::H264::kLevel1_1: | |
| 292 case webrtc::H264::kLevel1_2: | |
| 293 case webrtc::H264::kLevel1_3: | |
| 294 case webrtc::H264::kLevel2: | |
| 295 case webrtc::H264::kLevel2_1: | |
| 296 case webrtc::H264::kLevel2_2: | |
| 297 return kVTProfileLevel_H264_Main_AutoLevel; | |
| 298 } | |
| 299 | |
| 300 case webrtc::H264::kProfileConstrainedHigh: | |
| 301 case webrtc::H264::kProfileHigh: | |
| 302 switch (profile_level_id->level) { | |
| 303 case webrtc::H264::kLevel3: | |
| 304 return kVTProfileLevel_H264_High_3_0; | |
| 305 case webrtc::H264::kLevel3_1: | |
| 306 return kVTProfileLevel_H264_High_3_1; | |
| 307 case webrtc::H264::kLevel3_2: | |
| 308 return kVTProfileLevel_H264_High_3_2; | |
| 309 case webrtc::H264::kLevel4: | |
| 310 return kVTProfileLevel_H264_High_4_0; | |
| 311 case webrtc::H264::kLevel4_1: | |
| 312 return kVTProfileLevel_H264_High_4_1; | |
| 313 case webrtc::H264::kLevel4_2: | |
| 314 return kVTProfileLevel_H264_High_4_2; | |
| 315 case webrtc::H264::kLevel5: | |
| 316 return kVTProfileLevel_H264_High_5_0; | |
| 317 case webrtc::H264::kLevel5_1: | |
| 318 return kVTProfileLevel_H264_High_5_1; | |
| 319 case webrtc::H264::kLevel5_2: | |
| 320 return kVTProfileLevel_H264_High_5_2; | |
| 321 case webrtc::H264::kLevel1: | |
| 322 case webrtc::H264::kLevel1_b: | |
| 323 case webrtc::H264::kLevel1_1: | |
| 324 case webrtc::H264::kLevel1_2: | |
| 325 case webrtc::H264::kLevel1_3: | |
| 326 case webrtc::H264::kLevel2: | |
| 327 case webrtc::H264::kLevel2_1: | |
| 328 case webrtc::H264::kLevel2_2: | |
| 329 return kVTProfileLevel_H264_High_AutoLevel; | |
| 330 } | |
| 331 } | |
| 332 } | |
| 333 | |
| 334 } // namespace internal | |
| 335 | |
| 336 namespace webrtc { | |
| 337 | |
| 338 // .5 is set as a mininum to prevent overcompensating for large temporary | |
| 339 // overshoots. We don't want to degrade video quality too badly. | |
| 340 // .95 is set to prevent oscillations. When a lower bitrate is set on the | |
| 341 // encoder than previously set, its output seems to have a brief period of | |
| 342 // drastically reduced bitrate, so we want to avoid that. In steady state | |
| 343 // conditions, 0.95 seems to give us better overall bitrate over long periods | |
| 344 // of time. | |
| 345 H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& code
c) | |
| 346 : callback_(nullptr), | |
| 347 compression_session_(nullptr), | |
| 348 bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95), | |
| 349 packetization_mode_(H264PacketizationMode::NonInterleaved), | |
| 350 profile_(internal::ExtractProfile(codec)) { | |
| 351 LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_); | |
| 352 RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName)); | |
| 353 } | |
| 354 | |
| 355 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { | |
| 356 DestroyCompressionSession(); | |
| 357 } | |
| 358 | |
| 359 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, | |
| 360 int number_of_cores, | |
| 361 size_t max_payload_size) { | |
| 362 RTC_DCHECK(codec_settings); | |
| 363 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); | |
| 364 | |
| 365 width_ = codec_settings->width; | |
| 366 height_ = codec_settings->height; | |
| 367 mode_ = codec_settings->mode; | |
| 368 // We can only set average bitrate on the HW encoder. | |
| 369 target_bitrate_bps_ = codec_settings->startBitrate; | |
| 370 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); | |
| 371 | |
| 372 // TODO(tkchin): Try setting payload size via | |
| 373 // kVTCompressionPropertyKey_MaxH264SliceBytes. | |
| 374 | |
| 375 return ResetCompressionSession(); | |
| 376 } | |
| 377 | |
| 378 int H264VideoToolboxEncoder::Encode( | |
| 379 const VideoFrame& frame, | |
| 380 const CodecSpecificInfo* codec_specific_info, | |
| 381 const std::vector<FrameType>* frame_types) { | |
| 382 // |input_frame| size should always match codec settings. | |
| 383 RTC_DCHECK_EQ(frame.width(), width_); | |
| 384 RTC_DCHECK_EQ(frame.height(), height_); | |
| 385 if (!callback_ || !compression_session_) { | |
| 386 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | |
| 387 } | |
| 388 #if defined(WEBRTC_IOS) | |
| 389 if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { | |
| 390 // Ignore all encode requests when app isn't active. In this state, the | |
| 391 // hardware encoder has been invalidated by the OS. | |
| 392 return WEBRTC_VIDEO_CODEC_OK; | |
| 393 } | |
| 394 #endif | |
| 395 bool is_keyframe_required = false; | |
| 396 | |
| 397 // Get a pixel buffer from the pool and copy frame data over. | |
| 398 CVPixelBufferPoolRef pixel_buffer_pool = | |
| 399 VTCompressionSessionGetPixelBufferPool(compression_session_); | |
| 400 #if defined(WEBRTC_IOS) | |
| 401 if (!pixel_buffer_pool) { | |
| 402 // Kind of a hack. On backgrounding, the compression session seems to get | |
| 403 // invalidated, which causes this pool call to fail when the application | |
| 404 // is foregrounded and frames are being sent for encoding again. | |
| 405 // Resetting the session when this happens fixes the issue. | |
| 406 // In addition we request a keyframe so video can recover quickly. | |
| 407 ResetCompressionSession(); | |
| 408 pixel_buffer_pool = | |
| 409 VTCompressionSessionGetPixelBufferPool(compression_session_); | |
| 410 is_keyframe_required = true; | |
| 411 LOG(LS_INFO) << "Resetting compression session due to invalid pool."; | |
| 412 } | |
| 413 #endif | |
| 414 | |
| 415 CVPixelBufferRef pixel_buffer = nullptr; | |
| 416 if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) { | |
| 417 // Native frame. | |
| 418 rtc::scoped_refptr<ObjCFrameBuffer> objc_frame_buffer( | |
| 419 static_cast<ObjCFrameBuffer*>(frame.video_frame_buffer().get())); | |
| 420 id<RTCVideoFrameBuffer> wrapped_frame_buffer = | |
| 421 (id<RTCVideoFrameBuffer>)objc_frame_buffer->wrapped_frame_buffer(); | |
| 422 | |
| 423 if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { | |
| 424 RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buff
er; | |
| 425 if (![rtc_pixel_buffer requiresCropping]) { | |
| 426 // This pixel buffer might have a higher resolution than what the | |
| 427 // compression session is configured to. The compression session can | |
| 428 // handle that and will output encoded frames in the configured | |
| 429 // resolution regardless of the input pixel buffer resolution. | |
| 430 pixel_buffer = rtc_pixel_buffer.pixelBuffer; | |
| 431 CVBufferRetain(pixel_buffer); | |
| 432 } else { | |
| 433 // Cropping required, we need to crop and scale to a new pixel buffer. | |
| 434 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); | |
| 435 if (!pixel_buffer) { | |
| 436 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 437 } | |
| 438 int dst_width = CVPixelBufferGetWidth(pixel_buffer); | |
| 439 int dst_height = CVPixelBufferGetHeight(pixel_buffer); | |
| 440 if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height
]) { | |
| 441 int size = | |
| 442 [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width
height:dst_height]; | |
| 443 nv12_scale_buffer_.resize(size); | |
| 444 } else { | |
| 445 nv12_scale_buffer_.clear(); | |
| 446 } | |
| 447 nv12_scale_buffer_.shrink_to_fit(); | |
| 448 if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer | |
| 449 withTempBuffer:nv12_scale_buffer_.data()]) { | |
| 450 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 451 } | |
| 452 } | |
| 453 } | |
| 454 } | |
| 455 | |
| 456 if (!pixel_buffer) { | |
| 457 // We did not have a native frame, or the ObjCVideoFrame wrapped a non-nativ
e frame | |
| 458 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); | |
| 459 if (!pixel_buffer) { | |
| 460 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 461 } | |
| 462 RTC_DCHECK(pixel_buffer); | |
| 463 if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI42
0(), | |
| 464 pixel_buffer)) { | |
| 465 LOG(LS_ERROR) << "Failed to copy frame data."; | |
| 466 CVBufferRelease(pixel_buffer); | |
| 467 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 468 } | |
| 469 } | |
| 470 | |
| 471 // Check if we need a keyframe. | |
| 472 if (!is_keyframe_required && frame_types) { | |
| 473 for (auto frame_type : *frame_types) { | |
| 474 if (frame_type == kVideoFrameKey) { | |
| 475 is_keyframe_required = true; | |
| 476 break; | |
| 477 } | |
| 478 } | |
| 479 } | |
| 480 | |
| 481 CMTime presentation_time_stamp = | |
| 482 CMTimeMake(frame.render_time_ms(), 1000); | |
| 483 CFDictionaryRef frame_properties = nullptr; | |
| 484 if (is_keyframe_required) { | |
| 485 CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; | |
| 486 CFTypeRef values[] = {kCFBooleanTrue}; | |
| 487 frame_properties = internal::CreateCFDictionary(keys, values, 1); | |
| 488 } | |
| 489 std::unique_ptr<internal::FrameEncodeParams> encode_params; | |
| 490 encode_params.reset(new internal::FrameEncodeParams( | |
| 491 this, codec_specific_info, width_, height_, frame.render_time_ms(), | |
| 492 frame.timestamp(), frame.rotation())); | |
| 493 | |
| 494 encode_params->codec_specific_info.codecSpecific.H264.packetization_mode = | |
| 495 packetization_mode_; | |
| 496 | |
| 497 // Update the bitrate if needed. | |
| 498 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); | |
| 499 | |
| 500 OSStatus status = VTCompressionSessionEncodeFrame( | |
| 501 compression_session_, pixel_buffer, presentation_time_stamp, | |
| 502 kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); | |
| 503 if (frame_properties) { | |
| 504 CFRelease(frame_properties); | |
| 505 } | |
| 506 if (pixel_buffer) { | |
| 507 CVBufferRelease(pixel_buffer); | |
| 508 } | |
| 509 if (status != noErr) { | |
| 510 LOG(LS_ERROR) << "Failed to encode frame with code: " << status; | |
| 511 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 512 } | |
| 513 return WEBRTC_VIDEO_CODEC_OK; | |
| 514 } | |
| 515 | |
| 516 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( | |
| 517 EncodedImageCallback* callback) { | |
| 518 callback_ = callback; | |
| 519 return WEBRTC_VIDEO_CODEC_OK; | |
| 520 } | |
| 521 | |
| 522 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, | |
| 523 int64_t rtt) { | |
| 524 // Encoder doesn't know anything about packet loss or rtt so just return. | |
| 525 return WEBRTC_VIDEO_CODEC_OK; | |
| 526 } | |
| 527 | |
| 528 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, | |
| 529 uint32_t frame_rate) { | |
| 530 target_bitrate_bps_ = 1000 * new_bitrate_kbit; | |
| 531 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); | |
| 532 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); | |
| 533 return WEBRTC_VIDEO_CODEC_OK; | |
| 534 } | |
| 535 | |
| 536 int H264VideoToolboxEncoder::Release() { | |
| 537 // Need to destroy so that the session is invalidated and won't use the | |
| 538 // callback anymore. Do not remove callback until the session is invalidated | |
| 539 // since async encoder callbacks can occur until invalidation. | |
| 540 DestroyCompressionSession(); | |
| 541 callback_ = nullptr; | |
| 542 return WEBRTC_VIDEO_CODEC_OK; | |
| 543 } | |
| 544 | |
| 545 int H264VideoToolboxEncoder::ResetCompressionSession() { | |
| 546 DestroyCompressionSession(); | |
| 547 | |
| 548 // Set source image buffer attributes. These attributes will be present on | |
| 549 // buffers retrieved from the encoder's pixel buffer pool. | |
| 550 const size_t attributes_size = 3; | |
| 551 CFTypeRef keys[attributes_size] = { | |
| 552 #if defined(WEBRTC_IOS) | |
| 553 kCVPixelBufferOpenGLESCompatibilityKey, | |
| 554 #elif defined(WEBRTC_MAC) | |
| 555 kCVPixelBufferOpenGLCompatibilityKey, | |
| 556 #endif | |
| 557 kCVPixelBufferIOSurfacePropertiesKey, | |
| 558 kCVPixelBufferPixelFormatTypeKey | |
| 559 }; | |
| 560 CFDictionaryRef io_surface_value = | |
| 561 internal::CreateCFDictionary(nullptr, nullptr, 0); | |
| 562 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
| 563 CFNumberRef pixel_format = | |
| 564 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
| 565 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
| 566 pixel_format}; | |
| 567 CFDictionaryRef source_attributes = | |
| 568 internal::CreateCFDictionary(keys, values, attributes_size); | |
| 569 if (io_surface_value) { | |
| 570 CFRelease(io_surface_value); | |
| 571 io_surface_value = nullptr; | |
| 572 } | |
| 573 if (pixel_format) { | |
| 574 CFRelease(pixel_format); | |
| 575 pixel_format = nullptr; | |
| 576 } | |
| 577 OSStatus status = VTCompressionSessionCreate( | |
| 578 nullptr, // use default allocator | |
| 579 width_, height_, kCMVideoCodecType_H264, | |
| 580 nullptr, // use default encoder | |
| 581 source_attributes, | |
| 582 nullptr, // use default compressed data allocator | |
| 583 internal::VTCompressionOutputCallback, this, &compression_session_); | |
| 584 if (source_attributes) { | |
| 585 CFRelease(source_attributes); | |
| 586 source_attributes = nullptr; | |
| 587 } | |
| 588 if (status != noErr) { | |
| 589 LOG(LS_ERROR) << "Failed to create compression session: " << status; | |
| 590 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 591 } | |
| 592 ConfigureCompressionSession(); | |
| 593 return WEBRTC_VIDEO_CODEC_OK; | |
| 594 } | |
| 595 | |
| 596 void H264VideoToolboxEncoder::ConfigureCompressionSession() { | |
| 597 RTC_DCHECK(compression_session_); | |
| 598 internal::SetVTSessionProperty(compression_session_, | |
| 599 kVTCompressionPropertyKey_RealTime, true); | |
| 600 internal::SetVTSessionProperty(compression_session_, | |
| 601 kVTCompressionPropertyKey_ProfileLevel, | |
| 602 profile_); | |
| 603 internal::SetVTSessionProperty(compression_session_, | |
| 604 kVTCompressionPropertyKey_AllowFrameReordering, | |
| 605 false); | |
| 606 SetEncoderBitrateBps(target_bitrate_bps_); | |
| 607 // TODO(tkchin): Look at entropy mode and colorspace matrices. | |
| 608 // TODO(tkchin): Investigate to see if there's any way to make this work. | |
| 609 // May need it to interop with Android. Currently this call just fails. | |
| 610 // On inspecting encoder output on iOS8, this value is set to 6. | |
| 611 // internal::SetVTSessionProperty(compression_session_, | |
| 612 // kVTCompressionPropertyKey_MaxFrameDelayCount, | |
| 613 // 1); | |
| 614 | |
| 615 // Set a relatively large value for keyframe emission (7200 frames or | |
| 616 // 4 minutes). | |
| 617 internal::SetVTSessionProperty( | |
| 618 compression_session_, | |
| 619 kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); | |
| 620 internal::SetVTSessionProperty( | |
| 621 compression_session_, | |
| 622 kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); | |
| 623 } | |
| 624 | |
| 625 void H264VideoToolboxEncoder::DestroyCompressionSession() { | |
| 626 if (compression_session_) { | |
| 627 VTCompressionSessionInvalidate(compression_session_); | |
| 628 CFRelease(compression_session_); | |
| 629 compression_session_ = nullptr; | |
| 630 } | |
| 631 } | |
| 632 | |
| 633 const char* H264VideoToolboxEncoder::ImplementationName() const { | |
| 634 return "VideoToolbox"; | |
| 635 } | |
| 636 | |
| 637 bool H264VideoToolboxEncoder::SupportsNativeHandle() const { | |
| 638 return true; | |
| 639 } | |
| 640 | |
| 641 void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) { | |
| 642 if (encoder_bitrate_bps_ != bitrate_bps) { | |
| 643 SetEncoderBitrateBps(bitrate_bps); | |
| 644 } | |
| 645 } | |
| 646 | |
| 647 void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) { | |
| 648 if (compression_session_) { | |
| 649 internal::SetVTSessionProperty(compression_session_, | |
| 650 kVTCompressionPropertyKey_AverageBitRate, | |
| 651 bitrate_bps); | |
| 652 | |
| 653 // TODO(tkchin): Add a helper method to set array value. | |
| 654 int64_t data_limit_bytes_per_second_value = static_cast<int64_t>( | |
| 655 bitrate_bps * internal::kLimitToAverageBitRateFactor / 8); | |
| 656 CFNumberRef bytes_per_second = | |
| 657 CFNumberCreate(kCFAllocatorDefault, | |
| 658 kCFNumberSInt64Type, | |
| 659 &data_limit_bytes_per_second_value); | |
| 660 int64_t one_second_value = 1; | |
| 661 CFNumberRef one_second = | |
| 662 CFNumberCreate(kCFAllocatorDefault, | |
| 663 kCFNumberSInt64Type, | |
| 664 &one_second_value); | |
| 665 const void* nums[2] = { bytes_per_second, one_second }; | |
| 666 CFArrayRef data_rate_limits = | |
| 667 CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); | |
| 668 OSStatus status = | |
| 669 VTSessionSetProperty(compression_session_, | |
| 670 kVTCompressionPropertyKey_DataRateLimits, | |
| 671 data_rate_limits); | |
| 672 if (bytes_per_second) { | |
| 673 CFRelease(bytes_per_second); | |
| 674 } | |
| 675 if (one_second) { | |
| 676 CFRelease(one_second); | |
| 677 } | |
| 678 if (data_rate_limits) { | |
| 679 CFRelease(data_rate_limits); | |
| 680 } | |
| 681 if (status != noErr) { | |
| 682 LOG(LS_ERROR) << "Failed to set data rate limit"; | |
| 683 } | |
| 684 | |
| 685 encoder_bitrate_bps_ = bitrate_bps; | |
| 686 } | |
| 687 } | |
| 688 | |
| 689 void H264VideoToolboxEncoder::OnEncodedFrame( | |
| 690 OSStatus status, | |
| 691 VTEncodeInfoFlags info_flags, | |
| 692 CMSampleBufferRef sample_buffer, | |
| 693 CodecSpecificInfo codec_specific_info, | |
| 694 int32_t width, | |
| 695 int32_t height, | |
| 696 int64_t render_time_ms, | |
| 697 uint32_t timestamp, | |
| 698 VideoRotation rotation) { | |
| 699 if (status != noErr) { | |
| 700 LOG(LS_ERROR) << "H264 encode failed."; | |
| 701 return; | |
| 702 } | |
| 703 if (info_flags & kVTEncodeInfo_FrameDropped) { | |
| 704 LOG(LS_INFO) << "H264 encode dropped frame."; | |
| 705 return; | |
| 706 } | |
| 707 | |
| 708 bool is_keyframe = false; | |
| 709 CFArrayRef attachments = | |
| 710 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); | |
| 711 if (attachments != nullptr && CFArrayGetCount(attachments)) { | |
| 712 CFDictionaryRef attachment = | |
| 713 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); | |
| 714 is_keyframe = | |
| 715 !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); | |
| 716 } | |
| 717 | |
| 718 if (is_keyframe) { | |
| 719 LOG(LS_INFO) << "Generated keyframe"; | |
| 720 } | |
| 721 | |
| 722 // Convert the sample buffer into a buffer suitable for RTP packetization. | |
| 723 // TODO(tkchin): Allocate buffers through a pool. | |
| 724 std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer()); | |
| 725 std::unique_ptr<webrtc::RTPFragmentationHeader> header; | |
| 726 { | |
| 727 webrtc::RTPFragmentationHeader* header_raw; | |
| 728 bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe, | |
| 729 buffer.get(), &header_raw); | |
| 730 header.reset(header_raw); | |
| 731 if (!result) { | |
| 732 return; | |
| 733 } | |
| 734 } | |
| 735 webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); | |
| 736 frame._encodedWidth = width; | |
| 737 frame._encodedHeight = height; | |
| 738 frame._completeFrame = true; | |
| 739 frame._frameType = | |
| 740 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; | |
| 741 frame.capture_time_ms_ = render_time_ms; | |
| 742 frame._timeStamp = timestamp; | |
| 743 frame.rotation_ = rotation; | |
| 744 frame.content_type_ = | |
| 745 (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentTy
pe::UNSPECIFIED; | |
| 746 frame.timing_.is_timing_frame = false; | |
| 747 | |
| 748 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); | |
| 749 h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); | |
| 750 | |
| 751 EncodedImageCallback::Result res = | |
| 752 callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); | |
| 753 if (res.error != EncodedImageCallback::Result::OK) { | |
| 754 LOG(LS_ERROR) << "Encode callback failed: " << res.error; | |
| 755 return; | |
| 756 } | |
| 757 bitrate_adjuster_.Update(frame._length); | |
| 758 } | |
| 759 | |
| 760 // TODO(magjed): This function is not used by RTCVideoEncoderH264, but this whol
e file will be | |
| 761 // removed soon and inlined as ObjC. | |
| 762 VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() | |
| 763 const { | |
| 764 return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, | |
| 765 internal::kHighH264QpThreshold); | |
| 766 } | |
| 767 } // namespace webrtc | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2987413002:
ObjC: Implement HW codecs in ObjC instead of C++ (Closed)
