OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 * |
| 10 */ |
| 11 |
| 12 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h" |
| 13 |
| 14 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) |
| 15 |
| 16 #include <string> |
| 17 #include <vector> |
| 18 |
| 19 #include "libyuv/convert_from.h" |
| 20 #include "webrtc/base/checks.h" |
| 21 #include "webrtc/base/logging.h" |
| 22 #include "webrtc/base/scoped_ptr.h" |
| 23 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" |
| 24 |
| 25 namespace internal { |
| 26 |
| 27 // Convenience function for creating a dictionary. |
| 28 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, |
| 29 CFTypeRef* values, |
| 30 size_t size) { |
| 31 return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, |
| 32 &kCFTypeDictionaryKeyCallBacks, |
| 33 &kCFTypeDictionaryValueCallBacks); |
| 34 } |
| 35 |
| 36 // Copies characters from a CFStringRef into a std::string. |
| 37 std::string CFStringToString(const CFStringRef cf_string) { |
| 38 DCHECK(cf_string); |
| 39 std::string std_string; |
| 40 // Get the size needed for UTF8 plus terminating character. |
| 41 size_t buffer_size = |
| 42 CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), |
| 43 kCFStringEncodingUTF8) + |
| 44 1; |
| 45 rtc::scoped_ptr<char[]> buffer(new char[buffer_size]); |
| 46 if (CFStringGetCString(cf_string, buffer.get(), buffer_size, |
| 47 kCFStringEncodingUTF8)) { |
| 48 // Copy over the characters. |
| 49 std_string.assign(buffer.get()); |
| 50 } |
| 51 return std_string; |
| 52 } |
| 53 |
| 54 // Convenience function for setting a VT property. |
| 55 void SetVTSessionProperty(VTSessionRef session, |
| 56 CFStringRef key, |
| 57 int32_t value) { |
| 58 CFNumberRef cfNum = |
| 59 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); |
| 60 OSStatus status = VTSessionSetProperty(session, key, cfNum); |
| 61 CFRelease(cfNum); |
| 62 if (status != noErr) { |
| 63 std::string key_string = CFStringToString(key); |
| 64 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
| 65 << " to " << value << ": " << status; |
| 66 } |
| 67 } |
| 68 |
| 69 // Convenience function for setting a VT property. |
| 70 void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { |
| 71 CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; |
| 72 OSStatus status = VTSessionSetProperty(session, key, cf_bool); |
| 73 if (status != noErr) { |
| 74 std::string key_string = CFStringToString(key); |
| 75 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
| 76 << " to " << value << ": " << status; |
| 77 } |
| 78 } |
| 79 |
| 80 // Convenience function for setting a VT property. |
| 81 void SetVTSessionProperty(VTSessionRef session, |
| 82 CFStringRef key, |
| 83 CFStringRef value) { |
| 84 OSStatus status = VTSessionSetProperty(session, key, value); |
| 85 if (status != noErr) { |
| 86 std::string key_string = CFStringToString(key); |
| 87 std::string val_string = CFStringToString(value); |
| 88 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string |
| 89 << " to " << val_string << ": " << status; |
| 90 } |
| 91 } |
| 92 |
| 93 // Struct that we pass to the encoder per frame to encode. We receive it again |
| 94 // in the encoder callback. |
| 95 struct FrameEncodeParams { |
| 96 FrameEncodeParams(webrtc::EncodedImageCallback* cb, |
| 97 const webrtc::CodecSpecificInfo* csi, |
| 98 int32_t w, |
| 99 int32_t h, |
| 100 int64_t rtms, |
| 101 uint32_t ts) |
| 102 : callback(cb), |
| 103 width(w), |
| 104 height(h), |
| 105 render_time_ms(rtms), |
| 106 timestamp(ts) { |
| 107 if (csi) { |
| 108 codec_specific_info = *csi; |
| 109 } else { |
| 110 codec_specific_info.codecType = webrtc::kVideoCodecH264; |
| 111 } |
| 112 } |
| 113 webrtc::EncodedImageCallback* callback; |
| 114 webrtc::CodecSpecificInfo codec_specific_info; |
| 115 int32_t width; |
| 116 int32_t height; |
| 117 int64_t render_time_ms; |
| 118 uint32_t timestamp; |
| 119 }; |
| 120 |
| 121 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the |
| 122 // encoder. This performs the copy and format conversion. |
| 123 // TODO(tkchin): See if encoder will accept i420 frames and compare performance. |
| 124 bool CopyVideoFrameToPixelBuffer(const webrtc::VideoFrame& frame, |
| 125 CVPixelBufferRef pixel_buffer) { |
| 126 DCHECK(pixel_buffer); |
| 127 DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == |
| 128 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); |
| 129 DCHECK(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0) == |
| 130 static_cast<size_t>(frame.height())); |
| 131 DCHECK(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0) == |
| 132 static_cast<size_t>(frame.width())); |
| 133 |
| 134 CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); |
| 135 if (cvRet != kCVReturnSuccess) { |
| 136 LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; |
| 137 return false; |
| 138 } |
| 139 uint8* dst_y = reinterpret_cast<uint8*>( |
| 140 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); |
| 141 int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); |
| 142 uint8* dst_uv = reinterpret_cast<uint8*>( |
| 143 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); |
| 144 int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); |
| 145 // Convert I420 to NV12. |
| 146 int ret = libyuv::I420ToNV12( |
| 147 frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), |
| 148 frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), |
| 149 frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), |
| 150 dst_y, dst_stride_y, dst_uv, dst_stride_uv, |
| 151 frame.width(), frame.height()); |
| 152 CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); |
| 153 if (ret) { |
| 154 LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; |
| 155 return false; |
| 156 } |
| 157 return true; |
| 158 } |
| 159 |
| 160 // This is the callback function that VideoToolbox calls when encode is |
| 161 // complete. |
| 162 void VTCompressionOutputCallback(void* encoder, |
| 163 void* params, |
| 164 OSStatus status, |
| 165 VTEncodeInfoFlags info_flags, |
| 166 CMSampleBufferRef sample_buffer) { |
| 167 rtc::scoped_ptr<FrameEncodeParams> encode_params( |
| 168 reinterpret_cast<FrameEncodeParams*>(params)); |
| 169 if (status != noErr) { |
| 170 LOG(LS_ERROR) << "H264 encoding failed."; |
| 171 return; |
| 172 } |
| 173 if (info_flags & kVTEncodeInfo_FrameDropped) { |
| 174 LOG(LS_INFO) << "H264 encode dropped frame."; |
| 175 } |
| 176 |
| 177 bool is_keyframe = false; |
| 178 CFArrayRef attachments = |
| 179 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); |
| 180 if (attachments != nullptr && CFArrayGetCount(attachments)) { |
| 181 CFDictionaryRef attachment = |
| 182 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); |
| 183 is_keyframe = |
| 184 !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); |
| 185 } |
| 186 |
| 187 // Convert the sample buffer into a buffer suitable for RTP packetization. |
| 188 // TODO(tkchin): Allocate buffers through a pool. |
| 189 rtc::scoped_ptr<rtc::Buffer> buffer(new rtc::Buffer()); |
| 190 rtc::scoped_ptr<webrtc::RTPFragmentationHeader> header; |
| 191 if (!H264CMSampleBufferToAnnexBBuffer(sample_buffer, |
| 192 is_keyframe, |
| 193 buffer.get(), |
| 194 header.accept())) { |
| 195 return; |
| 196 } |
| 197 webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); |
| 198 frame._encodedWidth = encode_params->width; |
| 199 frame._encodedHeight = encode_params->height; |
| 200 frame._completeFrame = true; |
| 201 frame._frameType = is_keyframe ? webrtc::kKeyFrame : webrtc::kDeltaFrame; |
| 202 frame.capture_time_ms_ = encode_params->render_time_ms; |
| 203 frame._timeStamp = encode_params->timestamp; |
| 204 |
| 205 int result = encode_params->callback->Encoded( |
| 206 frame, &(encode_params->codec_specific_info), header.get()); |
| 207 if (result != 0) { |
| 208 LOG(LS_ERROR) << "Encoded callback failed: " << result; |
| 209 } |
| 210 } |
| 211 |
| 212 } // namespace internal |
| 213 |
| 214 namespace webrtc { |
| 215 |
| 216 H264VideoToolboxEncoder::H264VideoToolboxEncoder() |
| 217 : callback_(nullptr), compression_session_(nullptr) { |
| 218 } |
| 219 |
| 220 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { |
| 221 DestroyCompressionSession(); |
| 222 } |
| 223 |
| 224 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, |
| 225 int number_of_cores, |
| 226 size_t max_payload_size) { |
| 227 DCHECK(codec_settings); |
| 228 DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); |
| 229 // TODO(tkchin): We may need to enforce width/height dimension restrictions |
| 230 // to match what the encoder supports. |
| 231 width_ = codec_settings->width; |
| 232 height_ = codec_settings->height; |
| 233 // We can only set average bitrate on the HW encoder. |
| 234 bitrate_ = codec_settings->startBitrate * 1000; |
| 235 |
| 236 // TODO(tkchin): Try setting payload size via |
| 237 // kVTCompressionPropertyKey_MaxH264SliceBytes. |
| 238 |
| 239 return ResetCompressionSession(); |
| 240 } |
| 241 |
| 242 int H264VideoToolboxEncoder::Encode( |
| 243 const VideoFrame& input_image, |
| 244 const CodecSpecificInfo* codec_specific_info, |
| 245 const std::vector<VideoFrameType>* frame_types) { |
| 246 if (input_image.IsZeroSize()) { |
| 247 // It's possible to get zero sizes as a signal to produce keyframes (this |
| 248 // happens for internal sources). But this shouldn't happen in |
| 249 // webrtcvideoengine2. |
| 250 RTC_NOTREACHED(); |
| 251 return WEBRTC_VIDEO_CODEC_OK; |
| 252 } |
| 253 if (!callback_ || !compression_session_) { |
| 254 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 255 } |
| 256 |
| 257 // Get a pixel buffer from the pool and copy frame data over. |
| 258 CVPixelBufferPoolRef pixel_buffer_pool = |
| 259 VTCompressionSessionGetPixelBufferPool(compression_session_); |
| 260 CVPixelBufferRef pixel_buffer = nullptr; |
| 261 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, |
| 262 &pixel_buffer); |
| 263 if (ret != kCVReturnSuccess) { |
| 264 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; |
| 265 // We probably want to drop frames here, since failure probably means |
| 266 // that the pool is empty. |
| 267 return WEBRTC_VIDEO_CODEC_ERROR; |
| 268 } |
| 269 DCHECK(pixel_buffer); |
| 270 if (!internal::CopyVideoFrameToPixelBuffer(input_image, pixel_buffer)) { |
| 271 LOG(LS_ERROR) << "Failed to copy frame data."; |
| 272 CVBufferRelease(pixel_buffer); |
| 273 return WEBRTC_VIDEO_CODEC_ERROR; |
| 274 } |
| 275 |
| 276 // Check if we need a keyframe. |
| 277 bool is_keyframe_required = false; |
| 278 if (frame_types) { |
| 279 for (auto frame_type : *frame_types) { |
| 280 if (frame_type == kKeyFrame) { |
| 281 is_keyframe_required = true; |
| 282 break; |
| 283 } |
| 284 } |
| 285 } |
| 286 |
| 287 CMTime presentation_time_stamp = |
| 288 CMTimeMake(input_image.render_time_ms(), 1000); |
| 289 CFDictionaryRef frame_properties = nullptr; |
| 290 if (is_keyframe_required) { |
| 291 CFTypeRef keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame }; |
| 292 CFTypeRef values[] = { kCFBooleanTrue }; |
| 293 frame_properties = internal::CreateCFDictionary(keys, values, 1); |
| 294 } |
| 295 rtc::scoped_ptr<internal::FrameEncodeParams> encode_params; |
| 296 encode_params.reset(new internal::FrameEncodeParams( |
| 297 callback_, codec_specific_info, width_, height_, |
| 298 input_image.render_time_ms(), input_image.timestamp())); |
| 299 VTCompressionSessionEncodeFrame( |
| 300 compression_session_, pixel_buffer, presentation_time_stamp, |
| 301 kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); |
| 302 if (frame_properties) { |
| 303 CFRelease(frame_properties); |
| 304 } |
| 305 if (pixel_buffer) { |
| 306 CVBufferRelease(pixel_buffer); |
| 307 } |
| 308 return WEBRTC_VIDEO_CODEC_OK; |
| 309 } |
| 310 |
| 311 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( |
| 312 EncodedImageCallback* callback) { |
| 313 callback_ = callback; |
| 314 return WEBRTC_VIDEO_CODEC_OK; |
| 315 } |
| 316 |
| 317 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, |
| 318 int64_t rtt) { |
| 319 // Encoder doesn't know anything about packet loss or rtt so just return. |
| 320 return WEBRTC_VIDEO_CODEC_OK; |
| 321 } |
| 322 |
| 323 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, |
| 324 uint32_t frame_rate) { |
| 325 bitrate_ = new_bitrate_kbit * 1000; |
| 326 if (compression_session_) { |
| 327 internal::SetVTSessionProperty(compression_session_, |
| 328 kVTCompressionPropertyKey_AverageBitRate, |
| 329 bitrate_); |
| 330 } |
| 331 return WEBRTC_VIDEO_CODEC_OK; |
| 332 } |
| 333 |
| 334 int H264VideoToolboxEncoder::Release() { |
| 335 callback_ = nullptr; |
| 336 // Need to reset to that the session is invalidated and won't use the |
| 337 // callback anymore. |
| 338 return ResetCompressionSession(); |
| 339 } |
| 340 |
| 341 int H264VideoToolboxEncoder::ResetCompressionSession() { |
| 342 DestroyCompressionSession(); |
| 343 |
| 344 // Set source image buffer attributes. These attributes will be present on |
| 345 // buffers retrieved from the encoder's pixel buffer pool. |
| 346 const size_t attributes_size = 3; |
| 347 CFTypeRef keys[attributes_size] = { |
| 348 #if defined(WEBRTC_IOS) |
| 349 kCVPixelBufferOpenGLESCompatibilityKey, |
| 350 #elif defined(WEBRTC_MAC) |
| 351 kCVPixelBufferOpenGLCompatibilityKey, |
| 352 #endif |
| 353 kCVPixelBufferIOSurfacePropertiesKey, |
| 354 kCVPixelBufferPixelFormatTypeKey |
| 355 }; |
| 356 CFDictionaryRef io_surface_value = |
| 357 internal::CreateCFDictionary(nullptr, nullptr, 0); |
| 358 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; |
| 359 CFNumberRef pixel_format = |
| 360 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); |
| 361 CFTypeRef values[attributes_size] = { |
| 362 kCFBooleanTrue, |
| 363 io_surface_value, |
| 364 pixel_format |
| 365 }; |
| 366 CFDictionaryRef source_attributes = |
| 367 internal::CreateCFDictionary(keys, values, attributes_size); |
| 368 if (io_surface_value) { |
| 369 CFRelease(io_surface_value); |
| 370 io_surface_value = nullptr; |
| 371 } |
| 372 if (pixel_format) { |
| 373 CFRelease(pixel_format); |
| 374 pixel_format = nullptr; |
| 375 } |
| 376 OSStatus status = VTCompressionSessionCreate( |
| 377 nullptr, // use default allocator |
| 378 width_, |
| 379 height_, |
| 380 kCMVideoCodecType_H264, |
| 381 nullptr, // use default encoder |
| 382 source_attributes, |
| 383 nullptr, // use default compressed data allocator |
| 384 internal::VTCompressionOutputCallback, |
| 385 this, |
| 386 &compression_session_); |
| 387 if (source_attributes) { |
| 388 CFRelease(source_attributes); |
| 389 source_attributes = nullptr; |
| 390 } |
| 391 if (status != noErr) { |
| 392 LOG(LS_ERROR) << "Failed to create compression session: " << status; |
| 393 return WEBRTC_VIDEO_CODEC_ERROR; |
| 394 } |
| 395 ConfigureCompressionSession(); |
| 396 return WEBRTC_VIDEO_CODEC_OK; |
| 397 } |
| 398 |
| 399 void H264VideoToolboxEncoder::ConfigureCompressionSession() { |
| 400 DCHECK(compression_session_); |
| 401 internal::SetVTSessionProperty(compression_session_, |
| 402 kVTCompressionPropertyKey_RealTime, true); |
| 403 internal::SetVTSessionProperty(compression_session_, |
| 404 kVTCompressionPropertyKey_ProfileLevel, |
| 405 kVTProfileLevel_H264_Baseline_AutoLevel); |
| 406 internal::SetVTSessionProperty( |
| 407 compression_session_, kVTCompressionPropertyKey_AverageBitRate, bitrate_); |
| 408 internal::SetVTSessionProperty(compression_session_, |
| 409 kVTCompressionPropertyKey_AllowFrameReordering, |
| 410 false); |
| 411 // TODO(tkchin): Look at entropy mode and colorspace matrices. |
| 412 // TODO(tkchin): Investigate to see if there's any way to make this work. |
| 413 // May need it to interop with Android. Currently this call just fails. |
| 414 // On inspecting encoder output on iOS8, this value is set to 6. |
| 415 // internal::SetVTSessionProperty(compression_session_, |
| 416 // kVTCompressionPropertyKey_MaxFrameDelayCount, |
| 417 // 1); |
| 418 // TODO(tkchin): See if enforcing keyframe frequency is beneficial in any |
| 419 // way. |
| 420 // internal::SetVTSessionProperty( |
| 421 // compression_session_, |
| 422 // kVTCompressionPropertyKey_MaxKeyFrameInterval, 240); |
| 423 // internal::SetVTSessionProperty( |
| 424 // compression_session_, |
| 425 // kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); |
| 426 } |
| 427 |
| 428 void H264VideoToolboxEncoder::DestroyCompressionSession() { |
| 429 if (compression_session_) { |
| 430 VTCompressionSessionInvalidate(compression_session_); |
| 431 CFRelease(compression_session_); |
| 432 compression_session_ = nullptr; |
| 433 } |
| 434 } |
| 435 |
| 436 } // namespace webrtc |
| 437 |
| 438 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) |
OLD | NEW |