iOS HW H264 support.

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc

Index: webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc
diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc
new file mode 100644
index 0000000000000000000000000000000000000000..3dfd6cf43865d5fc12b9130d6a63fafc66fc18a7
--- /dev/null
+++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc
@@ -0,0 +1,438 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h"
+
+#if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
+
+#include <string>
+#include <vector>
+
+#include "libyuv/convert_from.h"
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
+
+namespace internal {
+
+// Convenience function for creating a dictionary.
+inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
+                                          CFTypeRef* values,
+                                          size_t size) {
+  return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
+                            &kCFTypeDictionaryKeyCallBacks,
+                            &kCFTypeDictionaryValueCallBacks);
+}
+
+// Copies characters from a CFStringRef into a std::string.
+std::string CFStringToString(const CFStringRef cf_string) {
+  DCHECK(cf_string);
+  std::string std_string;
+  // Get the size needed for UTF8 plus terminating character.
+  size_t buffer_size =
+      CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string),
+                                        kCFStringEncodingUTF8) +
+      1;
+  rtc::scoped_ptr<char[]> buffer(new char[buffer_size]);
+  if (CFStringGetCString(cf_string, buffer.get(), buffer_size,
+                         kCFStringEncodingUTF8)) {
+    // Copy over the characters.
+    std_string.assign(buffer.get());
+  }
+  return std_string;
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session,
+                          CFStringRef key,
+                          int32_t value) {
+  CFNumberRef cfNum =
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
+  OSStatus status = VTSessionSetProperty(session, key, cfNum);
+  CFRelease(cfNum);
+  if (status != noErr) {
+    std::string key_string = CFStringToString(key);
+    LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+                  << " to " << value << ": " << status;
+  }
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
+  CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
+  OSStatus status = VTSessionSetProperty(session, key, cf_bool);
+  if (status != noErr) {
+    std::string key_string = CFStringToString(key);
+    LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+                  << " to " << value << ": " << status;
+  }
+}
+
+// Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session,
+                          CFStringRef key,
+                          CFStringRef value) {
+  OSStatus status = VTSessionSetProperty(session, key, value);
+  if (status != noErr) {
+    std::string key_string = CFStringToString(key);
+    std::string val_string = CFStringToString(value);
+    LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+                  << " to " << val_string << ": " << status;
+  }
+}
+
+// Struct that we pass to the encoder per frame to encode. We receive it again
+// in the encoder callback.
+struct FrameEncodeParams {
+  FrameEncodeParams(webrtc::EncodedImageCallback* cb,
+                    const webrtc::CodecSpecificInfo* csi,
+                    int32_t w,
+                    int32_t h,
+                    int64_t rtms,
+                    uint32_t ts)
+      : callback(cb),
+        width(w),
+        height(h),
+        render_time_ms(rtms),
+        timestamp(ts) {
+    if (csi) {
+      codec_specific_info = *csi;
+    } else {
+      codec_specific_info.codecType = webrtc::kVideoCodecH264;
+    }
+  }
+  webrtc::EncodedImageCallback* callback;
+  webrtc::CodecSpecificInfo codec_specific_info;
+  int32_t width;
+  int32_t height;
+  int64_t render_time_ms;
+  uint32_t timestamp;
+};
+
+// We receive I420Frames as input, but we need to feed CVPixelBuffers into the
+// encoder. This performs the copy and format conversion.
+// TODO(tkchin): See if encoder will accept i420 frames and compare performance.
+bool CopyVideoFrameToPixelBuffer(const webrtc::VideoFrame& frame,
+                                 CVPixelBufferRef pixel_buffer) {
+  DCHECK(pixel_buffer);
+  DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) ==
+         kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
+  DCHECK(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0) ==
+         static_cast<size_t>(frame.height()));
+  DCHECK(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0) ==
+         static_cast<size_t>(frame.width()));
+
+  CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
+  if (cvRet != kCVReturnSuccess) {
+    LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
+    return false;
+  }
+  uint8* dst_y = reinterpret_cast<uint8*>(
+      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
+  int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
+  uint8* dst_uv = reinterpret_cast<uint8*>(
+      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
+  int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
+  // Convert I420 to NV12.
+  int ret = libyuv::I420ToNV12(
+      frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane),
+      frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane),
+      frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane),
+      dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+      frame.width(), frame.height());
+  CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+  if (ret) {
+    LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
+    return false;
+  }
+  return true;
+}
+
+// This is the callback function that VideoToolbox calls when encode is
+// complete.
+void VTCompressionOutputCallback(void* encoder,
+                                 void* params,
+                                 OSStatus status,
+                                 VTEncodeInfoFlags info_flags,
+                                 CMSampleBufferRef sample_buffer) {
+  rtc::scoped_ptr<FrameEncodeParams> encode_params(
+      reinterpret_cast<FrameEncodeParams*>(params));
+  if (status != noErr) {
+    LOG(LS_ERROR) << "H264 encoding failed.";
+    return;
+  }
+  if (info_flags & kVTEncodeInfo_FrameDropped) {
+    LOG(LS_INFO) << "H264 encode dropped frame.";
+  }
+
+  bool is_keyframe = false;
+  CFArrayRef attachments =
+      CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
+  if (attachments != nullptr && CFArrayGetCount(attachments)) {
+    CFDictionaryRef attachment =
+        static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
+    is_keyframe =
+        !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
+  }
+
+  // Convert the sample buffer into a buffer suitable for RTP packetization.
+  // TODO(tkchin): Allocate buffers through a pool.
+  rtc::scoped_ptr<rtc::Buffer> buffer(new rtc::Buffer());
+  rtc::scoped_ptr<webrtc::RTPFragmentationHeader> header;
+  if (!H264CMSampleBufferToAnnexBBuffer(sample_buffer,
+                                        is_keyframe,
+                                        buffer.get(),
+                                        header.accept())) {
+    return;
+  }
+  webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
+  frame._encodedWidth = encode_params->width;
+  frame._encodedHeight = encode_params->height;
+  frame._completeFrame = true;
+  frame._frameType = is_keyframe ? webrtc::kKeyFrame : webrtc::kDeltaFrame;
+  frame.capture_time_ms_ = encode_params->render_time_ms;
+  frame._timeStamp = encode_params->timestamp;
+
+  int result = encode_params->callback->Encoded(
+      frame, &(encode_params->codec_specific_info), header.get());
+  if (result != 0) {
+    LOG(LS_ERROR) << "Encoded callback failed: " << result;
+  }
+}
+
+}  // namespace internal
+
+namespace webrtc {
+
+H264VideoToolboxEncoder::H264VideoToolboxEncoder()
+    : callback_(nullptr), compression_session_(nullptr) {
+}
+
+H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
+  DestroyCompressionSession();
+}
+
+int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
+                                        int number_of_cores,
+                                        size_t max_payload_size) {
+  DCHECK(codec_settings);
+  DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
+  // TODO(tkchin): We may need to enforce width/height dimension restrictions
+  // to match what the encoder supports.
+  width_ = codec_settings->width;
+  height_ = codec_settings->height;
+  // We can only set average bitrate on the HW encoder.
+  bitrate_ = codec_settings->startBitrate * 1000;
+
+  // TODO(tkchin): Try setting payload size via
+  // kVTCompressionPropertyKey_MaxH264SliceBytes.
+
+  return ResetCompressionSession();
+}
+
+int H264VideoToolboxEncoder::Encode(
+    const VideoFrame& input_image,
+    const CodecSpecificInfo* codec_specific_info,
+    const std::vector<VideoFrameType>* frame_types) {
+  if (input_image.IsZeroSize()) {
+    // It's possible to get zero sizes as a signal to produce keyframes (this
+    // happens for internal sources). But this shouldn't happen in
+    // webrtcvideoengine2.
+    RTC_NOTREACHED();
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+  if (!callback_ || !compression_session_) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+
+  // Get a pixel buffer from the pool and copy frame data over.
+  CVPixelBufferPoolRef pixel_buffer_pool =
+      VTCompressionSessionGetPixelBufferPool(compression_session_);
+  CVPixelBufferRef pixel_buffer = nullptr;
+  CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
+                                                    &pixel_buffer);
+  if (ret != kCVReturnSuccess) {
+    LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
+    // We probably want to drop frames here, since failure probably means
+    // that the pool is empty.
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  DCHECK(pixel_buffer);
+  if (!internal::CopyVideoFrameToPixelBuffer(input_image, pixel_buffer)) {
+    LOG(LS_ERROR) << "Failed to copy frame data.";
+    CVBufferRelease(pixel_buffer);
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  // Check if we need a keyframe.
+  bool is_keyframe_required = false;
+  if (frame_types) {
+    for (auto frame_type : *frame_types) {
+      if (frame_type == kKeyFrame) {
+        is_keyframe_required = true;
+        break;
+      }
+    }
+  }
+
+  CMTime presentation_time_stamp =
+      CMTimeMake(input_image.render_time_ms(), 1000);
+  CFDictionaryRef frame_properties = nullptr;
+  if (is_keyframe_required) {
+    CFTypeRef keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
+    CFTypeRef values[] = { kCFBooleanTrue };
+    frame_properties = internal::CreateCFDictionary(keys, values, 1);
+  }
+  rtc::scoped_ptr<internal::FrameEncodeParams> encode_params;
+  encode_params.reset(new internal::FrameEncodeParams(
+      callback_, codec_specific_info, width_, height_,
+      input_image.render_time_ms(), input_image.timestamp()));
+  VTCompressionSessionEncodeFrame(
+      compression_session_, pixel_buffer, presentation_time_stamp,
+      kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
+  if (frame_properties) {
+    CFRelease(frame_properties);
+  }
+  if (pixel_buffer) {
+    CVBufferRelease(pixel_buffer);
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
+    EncodedImageCallback* callback) {
+  callback_ = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
+                                                  int64_t rtt) {
+  // Encoder doesn't know anything about packet loss or rtt so just return.
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
+                                      uint32_t frame_rate) {
+  bitrate_ = new_bitrate_kbit * 1000;
+  if (compression_session_) {
+    internal::SetVTSessionProperty(compression_session_,
+                                   kVTCompressionPropertyKey_AverageBitRate,
+                                   bitrate_);
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int H264VideoToolboxEncoder::Release() {
+  callback_ = nullptr;
+  // Need to reset to that the session is invalidated and won't use the
+  // callback anymore.
+  return ResetCompressionSession();
+}
+
+int H264VideoToolboxEncoder::ResetCompressionSession() {
+  DestroyCompressionSession();
+
+  // Set source image buffer attributes. These attributes will be present on
+  // buffers retrieved from the encoder's pixel buffer pool.
+  const size_t attributes_size = 3;
+  CFTypeRef keys[attributes_size] = {
+#if defined(WEBRTC_IOS)
+    kCVPixelBufferOpenGLESCompatibilityKey,
+#elif defined(WEBRTC_MAC)
+    kCVPixelBufferOpenGLCompatibilityKey,
+#endif
+    kCVPixelBufferIOSurfacePropertiesKey,
+    kCVPixelBufferPixelFormatTypeKey
+  };
+  CFDictionaryRef io_surface_value =
+      internal::CreateCFDictionary(nullptr, nullptr, 0);
+  int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+  CFNumberRef pixel_format =
+      CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
+  CFTypeRef values[attributes_size] = {
+    kCFBooleanTrue,
+    io_surface_value,
+    pixel_format
+  };
+  CFDictionaryRef source_attributes =
+      internal::CreateCFDictionary(keys, values, attributes_size);
+  if (io_surface_value) {
+    CFRelease(io_surface_value);
+    io_surface_value = nullptr;
+  }
+  if (pixel_format) {
+    CFRelease(pixel_format);
+    pixel_format = nullptr;
+  }
+  OSStatus status = VTCompressionSessionCreate(
+      nullptr,  // use default allocator
+      width_,
+      height_,
+      kCMVideoCodecType_H264,
+      nullptr,  // use default encoder
+      source_attributes,
+      nullptr,  // use default compressed data allocator
+      internal::VTCompressionOutputCallback,
+      this,
+      &compression_session_);
+  if (source_attributes) {
+    CFRelease(source_attributes);
+    source_attributes = nullptr;
+  }
+  if (status != noErr) {
+    LOG(LS_ERROR) << "Failed to create compression session: " << status;
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  ConfigureCompressionSession();
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void H264VideoToolboxEncoder::ConfigureCompressionSession() {
+  DCHECK(compression_session_);
+  internal::SetVTSessionProperty(compression_session_,
+                                 kVTCompressionPropertyKey_RealTime, true);
+  internal::SetVTSessionProperty(compression_session_,
+                                 kVTCompressionPropertyKey_ProfileLevel,
+                                 kVTProfileLevel_H264_Baseline_AutoLevel);
+  internal::SetVTSessionProperty(
+      compression_session_, kVTCompressionPropertyKey_AverageBitRate, bitrate_);
+  internal::SetVTSessionProperty(compression_session_,
+                                 kVTCompressionPropertyKey_AllowFrameReordering,
+                                 false);
+  // TODO(tkchin): Look at entropy mode and colorspace matrices.
+  // TODO(tkchin): Investigate to see if there's any way to make this work.
+  // May need it to interop with Android. Currently this call just fails.
+  // On inspecting encoder output on iOS8, this value is set to 6.
+  // internal::SetVTSessionProperty(compression_session_,
+  //     kVTCompressionPropertyKey_MaxFrameDelayCount,
+  //     1);
+  // TODO(tkchin): See if enforcing keyframe frequency is beneficial in any
+  // way.
+  // internal::SetVTSessionProperty(
+  //     compression_session_,
+  //     kVTCompressionPropertyKey_MaxKeyFrameInterval, 240);
+  // internal::SetVTSessionProperty(
+  //     compression_session_,
+  //     kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);

[stefan-webrtc, 2015/06/26 07:19:41]

Do you have any idea what the default is? Wondering if we should set it to 3000
as we do for VP8 and VP9.

[tkchin_webrtc, 2015/06/26 20:28:09]

No, I wanted to do all that work as part of a separate CL that tunes the
performance per the TODO.

+}
+
+void H264VideoToolboxEncoder::DestroyCompressionSession() {
+  if (compression_session_) {
+    VTCompressionSessionInvalidate(compression_session_);
+    CFRelease(compression_session_);
+    compression_session_ = nullptr;
+  }
+}
+
+}  // namespace webrtc
+
+#endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)