AVFoundationVideoCapturer: Output native frames instead of I420 frames

webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "avfoundationvideocapturer.h"
 
 #import <AVFoundation/AVFoundation.h>
 #import <Foundation/Foundation.h>
 #if TARGET_OS_IPHONE
 #import <UIKit/UIKit.h>
 #endif
 
 #import "RTCDispatcher+Private.h"
 #import "WebRTC/RTCLogging.h"
 
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/thread.h"
+#include "webrtc/common_video/include/corevideo_frame_buffer.h"
 
 // TODO(tkchin): support other formats.
 static NSString *const kDefaultPreset = AVCaptureSessionPreset640x480;
 static cricket::VideoFormat const kDefaultFormat =
     cricket::VideoFormat(640,
                          480,
                          cricket::VideoFormat::FpsToInterval(30),
                          cricket::FOURCC_NV12);
 
 // This class used to capture frames using AVFoundation APIs on iOS. It is meant
@@ skipping 600 matching lines @@
         static_cast<rtc::TypedMessageData<AVFoundationFrame>*>(msg->pdata);
       const AVFoundationFrame& frame = data->data();
       OnFrameMessage(frame.image_buffer, frame.capture_time);
       delete data;
       break;
     }
   }
 }
 
 void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer,
-                                               int64_t capture_time) {
+                                               int64_t capture_time_ns) {
   RTC_DCHECK(_startThread->IsCurrent());
 
-  // Base address must be unlocked to access frame data.
-  CVOptionFlags lock_flags = kCVPixelBufferLock_ReadOnly;
-  CVReturn ret = CVPixelBufferLockBaseAddress(image_buffer, lock_flags);
-  if (ret != kCVReturnSuccess) {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
+      new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer);
+
+  const int captured_width = buffer->width();
+  const int captured_height = buffer->height();
+
+  int adapted_width;
+  int adapted_height;
+  int crop_width;
+  int crop_height;
+  int crop_x;
+  int crop_y;
+  int64_t translated_camera_time_us;
+
+  if (!AdaptFrame(captured_width, captured_height,

[tkchin_webrtc, 2016/07/11 18:49:01]

superclass no longer does adaptation? Just curious for my education.

[magjed_webrtc, 2016/07/12 14:42:18]

Right, this function now bypasses all adapation in the cricket::VideoCapturer
superclass.

+                  capture_time_ns / rtc::kNumNanosecsPerMicrosec,
+                  rtc::TimeMicros(), &adapted_width, &adapted_height,
+                  &crop_width, &crop_height, &crop_x, &crop_y,
+                  &translated_camera_time_us)) {
     return;
   }
 
-  static size_t const kYPlaneIndex = 0;
-  static size_t const kUVPlaneIndex = 1;
-  uint8_t* y_plane_address =
-      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(image_buffer,
-                                                               kYPlaneIndex));
-  size_t y_plane_height =
-      CVPixelBufferGetHeightOfPlane(image_buffer, kYPlaneIndex);
-  size_t y_plane_width =
-      CVPixelBufferGetWidthOfPlane(image_buffer, kYPlaneIndex);
-  size_t y_plane_bytes_per_row =
-      CVPixelBufferGetBytesPerRowOfPlane(image_buffer, kYPlaneIndex);
-  size_t uv_plane_height =
-      CVPixelBufferGetHeightOfPlane(image_buffer, kUVPlaneIndex);
-  size_t uv_plane_bytes_per_row =
-      CVPixelBufferGetBytesPerRowOfPlane(image_buffer, kUVPlaneIndex);
-  size_t frame_size = y_plane_bytes_per_row * y_plane_height +
-      uv_plane_bytes_per_row * uv_plane_height;
+  if (adapted_width != captured_width || crop_width != captured_width ||
+      adapted_height != captured_height || crop_height != captured_height) {
+    // TODO(magjed): Avoid converting to I420.
+    rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer(
+        _buffer_pool.CreateBuffer(adapted_width, adapted_height));
+    scaled_buffer->CropAndScaleFrom(buffer->NativeToI420Buffer(), crop_x,
+                                    crop_y, crop_width, crop_height);
+    buffer = scaled_buffer;
+  }
 
-  // Sanity check assumption that planar bytes are contiguous.
-  uint8_t* uv_plane_address =
-      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(image_buffer,
-                                                               kUVPlaneIndex));
-  RTC_DCHECK(uv_plane_address ==
-             y_plane_address + y_plane_height * y_plane_bytes_per_row);
+  OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0,

[tkchin_webrtc, 2016/07/11 18:49:01]

Is this the new way to do things? (As opposed to SignalFrameCaptured) 

iirc there was some mention that we can avoid thread dispatch now, do we still
need to thread hop from the frame output queue to the capturer thread?

[magjed_webrtc, 2016/07/12 14:42:18]

Yes, the new OnFrame() method was added by nisse@ recently, and provides a way
to skip the adaptation logic in the super class. It also provides a way to input
reference counted objects, which is not possible with CapturedFrame.

The downside is that we have to do scaling ourselves, but that is typically
needed anyway if we want to scale native formats efficiently like OpenGL
textures or CVPixelBuffers.

On Android, we will probably implement webrtc::VideoTrackSourceInterface
directly, which will remove the dependency to cricket::VideoCapturer completely.
The reason for this is that we barely use any logic in cricket::VideoCapturer
and it just gets in the way. Maybe this is something we should consider for iOS
as well in the future.
No, it's not needed anymore. I created a CL for removing the thread dispatch:
https://codereview.webrtc.org/2137503003/. My only worry is the discussion in
https://bugs.chromium.org/p/webrtc/issues/detail?id=5679, especially:
"Clients that need to minimize the chances of frames being dropped should
specify a queue on which a sufficiently small amount of processing is being done
outside of receiving sample buffers."

+                                    translated_camera_time_us),
+          captured_width, captured_height);
 
-  // Stuff data into a cricket::CapturedFrame.
-  cricket::CapturedFrame frame;
-  frame.width = y_plane_width;
-  frame.height = y_plane_height;
-  frame.pixel_width = 1;
-  frame.pixel_height = 1;
-  frame.fourcc = static_cast<uint32_t>(cricket::FOURCC_NV12);
-  frame.time_stamp = capture_time;
-  frame.data = y_plane_address;
-  frame.data_size = frame_size;
-
-  // This will call a superclass method that will perform the frame conversion
-  // to I420.
-  SignalFrameCaptured(this, &frame);
-
-  CVPixelBufferUnlockBaseAddress(image_buffer, lock_flags);
   CVBufferRelease(image_buffer);
 }
 
 }  // namespace webrtc