change videotoolbox format from NV12 to I420

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc

Index: webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc
diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc
index 6d9a4c14c709c5ac61200431b1d4d47107f2dcf1..cd0ef48709a023dba2fd8bb7eb38f50526d07c66 100644
--- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc
+++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc
@@ -48,31 +48,55 @@ struct FrameDecodeParams {
 rtc::scoped_refptr<webrtc::VideoFrameBuffer> VideoFrameBufferForPixelBuffer(
     CVPixelBufferRef pixel_buffer) {
   RTC_DCHECK(pixel_buffer);
-  RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) ==
-             kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
   size_t width = CVPixelBufferGetWidthOfPlane(pixel_buffer, 0);
   size_t height = CVPixelBufferGetHeightOfPlane(pixel_buffer, 0);
   // TODO(tkchin): Use a frame buffer pool.
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
       new rtc::RefCountedObject<webrtc::I420Buffer>(width, height);
-  CVPixelBufferLockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
-  const uint8_t* src_y = reinterpret_cast<const uint8_t*>(
-      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
-  int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
-  const uint8_t* src_uv = reinterpret_cast<const uint8_t*>(
-      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
-  int src_uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
-  int ret = libyuv::NV12ToI420(
-      src_y, src_y_stride, src_uv, src_uv_stride,
-      buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane),
-      buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane),
-      buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane),
-      width, height);
-  CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
-  if (ret) {
-    LOG(LS_ERROR) << "Error converting NV12 to I420: " << ret;
-    return nullptr;
-  }
+
+    OSType format = CVPixelBufferGetPixelFormatType(pixel_buffer);

[pbos-webrtc, 2015/12/10 12:05:17]

This is not style compliant (indentation etc), please run git cl format.

+    if (format == kCVPixelFormatType_420YpCbCr8Planar ||
+        format == kCVPixelFormatType_420YpCbCr8PlanarFullRange) {
+        int stride;
+        CVPixelBufferLockBaseAddress(pixel_buffer, 0);
+        size_t height = CVPixelBufferGetHeight(pixel_buffer);
+        const int half_height = (height + 1) / 2;
+        for (size_t planeIndex = 0; planeIndex < CVPixelBufferGetPlaneCount(pixel_buffer);
+             planeIndex++) {
+            void *baseAddress = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, planeIndex);
+            stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, planeIndex);
+            memcpy(buffer->MutableData((webrtc::PlaneType)planeIndex), baseAddress,
+                   (planeIndex == webrtc::kYPlane) ? stride * height :
+                   stride * half_height);
+        }
+        CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+
+    }else if (format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
+              format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange){
+        CVPixelBufferLockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
+        const uint8_t* src_y = reinterpret_cast<const uint8_t*>(
+                                                                CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
+        int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
+        const uint8_t* src_uv = reinterpret_cast<const uint8_t*>(
+                                                                 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
+        int src_uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
+        int ret = libyuv::NV12ToI420(
+                                     src_y, src_y_stride, src_uv, src_uv_stride,
+                                     buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane),
+                                     buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane),
+                                     buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane),
+                                     width, height);
+        CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
+        if (ret) {
+            LOG(LS_ERROR) << "Error converting NV12 to I420: " << ret;
+            return nullptr;
+        }
+    }else{
+        LOG(LS_ERROR) << "Unsupported pixel buffer format: " << format;
+        return nullptr;
+    }
+
+
   return buffer;
 }
 
@@ -203,9 +227,9 @@ int H264VideoToolboxDecoder::ResetDecompressionSession() {
   };
   CFDictionaryRef io_surface_value =
       internal::CreateCFDictionary(nullptr, nullptr, 0);
-  int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+    OSType i420type = kCVPixelFormatType_420YpCbCr8Planar;
   CFNumberRef pixel_format =
-      CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
+      CFNumberCreate(nullptr, kCFNumberSInt32Type, &i420type);
   CFTypeRef values[attributes_size] = {
     kCFBooleanTrue,
     io_surface_value,