Implement CVO for iOS capturer

webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm

Index: webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
diff --git a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
index ddeedb5ab6ffe19aedcc48627b571bd59880cc7d..a965e16f63ec63640ab64c18f0fd5fd1df67ff9c 100644
--- a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
+++ b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm
@@ -22,10 +22,13 @@
 #import "WebRTC/UIDevice+RTCDevice.h"
 #endif
 
+#include "libyuv/rotate.h"
+
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/common_video/include/corevideo_frame_buffer.h"
+#include "webrtc/common_video/rotation.h"
 
 struct AVCaptureSessionPresetResolution {
   NSString *sessionPreset;
@@ -98,7 +101,7 @@ static NSString *GetSessionPresetForVideoFormat(
   AVCaptureVideoDataOutput *_videoDataOutput;
   // The cricket::VideoCapturer that owns this class. Should never be NULL.
   webrtc::AVFoundationVideoCapturer *_capturer;
-  BOOL _orientationHasChanged;
+  webrtc::VideoRotation _rotation;
   BOOL _hasRetriedOnFatalError;
   BOOL _isRunning;
   BOOL _hasStarted;
@@ -228,7 +231,7 @@ static NSString *GetSessionPresetForVideoFormat(
   self.hasStarted = YES;
   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                                block:^{
-    _orientationHasChanged = NO;
+    _rotation = webrtc::kVideoRotation_0;

[tkchin_webrtc, 2016/08/23 17:33:19]

Can you add comment that this sets 0 rotation is orientation is
unknown/faceup/facedown?
What happens to local preview if say device was in landscape orientation, then
set faceup/facedown on table?

[tkchin_webrtc, 2016/08/23 17:37:25]

To clarify - what happens to an AVCapturePreviewLayer attached to the
AVCaptureSession? 

[magjed_webrtc, 2016/08/24 11:08:36]

Nothing happens when the device is set faceup/facedown with the new code. This
is the behavior I observe for the old code as well. The only difference I can
observe from the previous code overall is the brief flash when the device is
rotated.

I don't understand the purpose of the previous |_orientationHasChanged|. Was it
only used to initialize a specific value for iPhone the first time
updateOrientation is called and the orientation happens to be
unknown/faceup/facedown? Do I need to preserve that behavior?

     [self updateOrientation];
 #if TARGET_OS_IPHONE
     [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
@@ -263,7 +266,6 @@ static NSString *GetSessionPresetForVideoFormat(
 - (void)deviceOrientationDidChange:(NSNotification *)notification {
   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                                block:^{
-    _orientationHasChanged = YES;
     [self updateOrientation];
   }];
 }
@@ -278,7 +280,7 @@ static NSString *GetSessionPresetForVideoFormat(
   if (!self.hasStarted) {
     return;
   }
-  _capturer->CaptureSampleBuffer(sampleBuffer);
+  _capturer->CaptureSampleBuffer(sampleBuffer, _rotation);
 }
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
@@ -510,36 +512,26 @@ static NSString *GetSessionPresetForVideoFormat(
 
 // Called from capture session queue.
 - (void)updateOrientation {
-  AVCaptureConnection *connection =
-      [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
-  if (!connection.supportsVideoOrientation) {
-    // TODO(tkchin): set rotation bit on frames.
-    return;
-  }
 #if TARGET_OS_IPHONE
-  AVCaptureVideoOrientation orientation = AVCaptureVideoOrientationPortrait;
   switch ([UIDevice currentDevice].orientation) {
     case UIDeviceOrientationPortrait:
-      orientation = AVCaptureVideoOrientationPortrait;
+      _rotation = webrtc::kVideoRotation_90;
       break;
     case UIDeviceOrientationPortraitUpsideDown:
-      orientation = AVCaptureVideoOrientationPortraitUpsideDown;
+      _rotation = webrtc::kVideoRotation_270;
       break;
     case UIDeviceOrientationLandscapeLeft:
-      orientation = AVCaptureVideoOrientationLandscapeRight;
+      _rotation = webrtc::kVideoRotation_180;
       break;
     case UIDeviceOrientationLandscapeRight:
-      orientation = AVCaptureVideoOrientationLandscapeLeft;
+      _rotation = webrtc::kVideoRotation_0;
       break;
     case UIDeviceOrientationFaceUp:
     case UIDeviceOrientationFaceDown:
     case UIDeviceOrientationUnknown:
-      if (!_orientationHasChanged) {
-        connection.videoOrientation = orientation;
-      }
-      return;
+      // Ignore.
+      break;
   }
-  connection.videoOrientation = orientation;
 #endif
 }
 
@@ -578,9 +570,12 @@ enum AVFoundationVideoCapturerMessageType : uint32_t {
 };
 
 struct AVFoundationFrame {
-  AVFoundationFrame(CVImageBufferRef buffer, int64_t time)
-    : image_buffer(buffer), capture_time(time) {}
+  AVFoundationFrame(CVImageBufferRef buffer,
+                    webrtc::VideoRotation rotation,
+                    int64_t time)
+      : image_buffer(buffer), rotation(rotation), capture_time(time) {}
   CVImageBufferRef image_buffer;
+  webrtc::VideoRotation rotation;
   int64_t capture_time;
 };
 
@@ -688,7 +683,7 @@ bool AVFoundationVideoCapturer::GetUseBackCamera() const {
 }
 
 void AVFoundationVideoCapturer::CaptureSampleBuffer(
-    CMSampleBufferRef sampleBuffer) {
+    CMSampleBufferRef sampleBuffer, webrtc::VideoRotation rotation) {
   if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 ||
       !CMSampleBufferIsValid(sampleBuffer) ||
       !CMSampleBufferDataIsReady(sampleBuffer)) {
@@ -703,7 +698,7 @@ void AVFoundationVideoCapturer::CaptureSampleBuffer(
   // Retain the buffer and post it to the webrtc thread. It will be released
   // after it has successfully been signaled.
   CVBufferRetain(image_buffer);
-  AVFoundationFrame frame(image_buffer, rtc::TimeNanos());
+  AVFoundationFrame frame(image_buffer, rotation, rtc::TimeNanos());
   _startThread->Post(RTC_FROM_HERE, this, kMessageTypeFrame,
                      new rtc::TypedMessageData<AVFoundationFrame>(frame));
 }
@@ -714,7 +709,7 @@ void AVFoundationVideoCapturer::OnMessage(rtc::Message *msg) {
       rtc::TypedMessageData<AVFoundationFrame>* data =
         static_cast<rtc::TypedMessageData<AVFoundationFrame>*>(msg->pdata);
       const AVFoundationFrame& frame = data->data();
-      OnFrameMessage(frame.image_buffer, frame.capture_time);
+      OnFrameMessage(frame.image_buffer, frame.rotation, frame.capture_time);
       delete data;
       break;
     }
@@ -722,6 +717,7 @@ void AVFoundationVideoCapturer::OnMessage(rtc::Message *msg) {
 }
 
 void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer,
+                                               webrtc::VideoRotation rotation,
                                                int64_t capture_time_ns) {
   RTC_DCHECK(_startThread->IsCurrent());
 
@@ -749,16 +745,33 @@ void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer,
   }
 
   if (adapted_width != captured_width || crop_width != captured_width ||
-      adapted_height != captured_height || crop_height != captured_height) {
+      adapted_height != captured_height || crop_height != captured_height ||
+      (apply_rotation() && rotation != webrtc::kVideoRotation_0)) {
     // TODO(magjed): Avoid converting to I420.
     rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer(
         _buffer_pool.CreateBuffer(adapted_width, adapted_height));
     scaled_buffer->CropAndScaleFrom(buffer->NativeToI420Buffer(), crop_x,
                                     crop_y, crop_width, crop_height);
-    buffer = scaled_buffer;
+    if (!apply_rotation() || rotation == webrtc::kVideoRotation_0) {
+      buffer = scaled_buffer;
+    } else {
+      // Applying rotation is only supported for legacy reasons and performance
+      // is not critical here.
+      buffer = (rotation == webrtc::kVideoRotation_180)
+          ? I420Buffer::Create(adapted_width, adapted_height)
+          : I420Buffer::Create(adapted_height, adapted_width);
+      libyuv::I420Rotate(scaled_buffer->DataY(), scaled_buffer->StrideY(),
+                         scaled_buffer->DataU(), scaled_buffer->StrideU(),
+                         scaled_buffer->DataV(), scaled_buffer->StrideV(),
+                         buffer->MutableDataY(), buffer->StrideY(),
+                         buffer->MutableDataU(), buffer->StrideU(),
+                         buffer->MutableDataV(), buffer->StrideV(),
+                         crop_width, crop_height,
+                         static_cast<libyuv::RotationMode>(rotation));
+    }
   }
 
-  OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0,
+  OnFrame(cricket::WebRtcVideoFrame(buffer, rotation,
                                     translated_camera_time_us, 0),
           captured_width, captured_height);