AVFoundationVideoCapturer: Output native frames instead of I420 frames

webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm

Index: webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
diff --git a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
index 778d70b897cd9777b2cd8aa13ced4ae6f9d0c47a..f4b3efcd4bd298f4e2b3e71d226b7b7ddc59c85b 100644
--- a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
+++ b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
@@ -22,6 +22,7 @@
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/thread.h"
+#include "webrtc/common_video/include/corevideo_frame_buffer.h"
 
 // TODO(tkchin): support other formats.
 static NSString *const kDefaultPreset = AVCaptureSessionPreset640x480;
@@ -642,57 +643,45 @@ void AVFoundationVideoCapturer::OnMessage(rtc::Message *msg) {
 }
 
 void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer,
-                                               int64_t capture_time) {
+                                               int64_t capture_time_ns) {
   RTC_DCHECK(_startThread->IsCurrent());
 
-  // Base address must be unlocked to access frame data.
-  CVOptionFlags lock_flags = kCVPixelBufferLock_ReadOnly;
-  CVReturn ret = CVPixelBufferLockBaseAddress(image_buffer, lock_flags);
-  if (ret != kCVReturnSuccess) {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
+      new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer);
+
+  const int captured_width = buffer->width();
+  const int captured_height = buffer->height();
+
+  int adapted_width;
+  int adapted_height;
+  int crop_width;
+  int crop_height;
+  int crop_x;
+  int crop_y;
+  int64_t translated_camera_time_us;
+
+  if (!AdaptFrame(captured_width, captured_height,

[tkchin_webrtc, 2016/07/11 18:49:01]

superclass no longer does adaptation? Just curious for my education.

[magjed_webrtc, 2016/07/12 14:42:18]

Right, this function now bypasses all adapation in the cricket::VideoCapturer
superclass.

+                  capture_time_ns / rtc::kNumNanosecsPerMicrosec,
+                  rtc::TimeMicros(), &adapted_width, &adapted_height,
+                  &crop_width, &crop_height, &crop_x, &crop_y,
+                  &translated_camera_time_us)) {
     return;
   }
 
-  static size_t const kYPlaneIndex = 0;
-  static size_t const kUVPlaneIndex = 1;
-  uint8_t* y_plane_address =
-      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(image_buffer,
-                                                               kYPlaneIndex));
-  size_t y_plane_height =
-      CVPixelBufferGetHeightOfPlane(image_buffer, kYPlaneIndex);
-  size_t y_plane_width =
-      CVPixelBufferGetWidthOfPlane(image_buffer, kYPlaneIndex);
-  size_t y_plane_bytes_per_row =
-      CVPixelBufferGetBytesPerRowOfPlane(image_buffer, kYPlaneIndex);
-  size_t uv_plane_height =
-      CVPixelBufferGetHeightOfPlane(image_buffer, kUVPlaneIndex);
-  size_t uv_plane_bytes_per_row =
-      CVPixelBufferGetBytesPerRowOfPlane(image_buffer, kUVPlaneIndex);
-  size_t frame_size = y_plane_bytes_per_row * y_plane_height +
-      uv_plane_bytes_per_row * uv_plane_height;
-
-  // Sanity check assumption that planar bytes are contiguous.
-  uint8_t* uv_plane_address =
-      static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(image_buffer,
-                                                               kUVPlaneIndex));
-  RTC_DCHECK(uv_plane_address ==
-             y_plane_address + y_plane_height * y_plane_bytes_per_row);
-
-  // Stuff data into a cricket::CapturedFrame.
-  cricket::CapturedFrame frame;
-  frame.width = y_plane_width;
-  frame.height = y_plane_height;
-  frame.pixel_width = 1;
-  frame.pixel_height = 1;
-  frame.fourcc = static_cast<uint32_t>(cricket::FOURCC_NV12);
-  frame.time_stamp = capture_time;
-  frame.data = y_plane_address;
-  frame.data_size = frame_size;
-
-  // This will call a superclass method that will perform the frame conversion
-  // to I420.
-  SignalFrameCaptured(this, &frame);
-
-  CVPixelBufferUnlockBaseAddress(image_buffer, lock_flags);
+  if (adapted_width != captured_width || crop_width != captured_width ||
+      adapted_height != captured_height || crop_height != captured_height) {
+    // TODO(magjed): Avoid converting to I420.
+    rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer(
+        _buffer_pool.CreateBuffer(adapted_width, adapted_height));
+    scaled_buffer->CropAndScaleFrom(buffer->NativeToI420Buffer(), crop_x,
+                                    crop_y, crop_width, crop_height);
+    buffer = scaled_buffer;
+  }
+
+  OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0,

[tkchin_webrtc, 2016/07/11 18:49:01]

Is this the new way to do things? (As opposed to SignalFrameCaptured) 

iirc there was some mention that we can avoid thread dispatch now, do we still
need to thread hop from the frame output queue to the capturer thread?

[magjed_webrtc, 2016/07/12 14:42:18]

Yes, the new OnFrame() method was added by nisse@ recently, and provides a way
to skip the adaptation logic in the super class. It also provides a way to input
reference counted objects, which is not possible with CapturedFrame.

The downside is that we have to do scaling ourselves, but that is typically
needed anyway if we want to scale native formats efficiently like OpenGL
textures or CVPixelBuffers.

On Android, we will probably implement webrtc::VideoTrackSourceInterface
directly, which will remove the dependency to cricket::VideoCapturer completely.
The reason for this is that we barely use any logic in cricket::VideoCapturer
and it just gets in the way. Maybe this is something we should consider for iOS
as well in the future.
No, it's not needed anymore. I created a CL for removing the thread dispatch:
https://codereview.webrtc.org/2137503003/. My only worry is the discussion in
https://bugs.chromium.org/p/webrtc/issues/detail?id=5679, especially:
"Clients that need to minimize the chances of frames being dropped should
specify a queue on which a sufficiently small amount of processing is being done
outside of receiving sample buffers."

+                                    translated_camera_time_us),
+          captured_width, captured_height);
+
   CVBufferRelease(image_buffer);
 }