Implement CVO for iOS capturer

webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "avfoundationvideocapturer.h"
 
 #import <AVFoundation/AVFoundation.h>
 #import <Foundation/Foundation.h>
 #if TARGET_OS_IPHONE
 #import <UIKit/UIKit.h>
 #endif
 
 #import "RTCDispatcher+Private.h"
 #import "WebRTC/RTCLogging.h"
 #if TARGET_OS_IPHONE
 #import "WebRTC/UIDevice+RTCDevice.h"
 #endif
 
+#include "libyuv/rotate.h"
+
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/common_video/include/corevideo_frame_buffer.h"
+#include "webrtc/common_video/rotation.h"
 
 struct AVCaptureSessionPresetResolution {
   NSString *sessionPreset;
   int width;
   int height;
 };
 
 #if TARGET_OS_IPHONE
 static const AVCaptureSessionPresetResolution kAvailablePresets[] = {
   { AVCaptureSessionPreset352x288, 352, 288},
@@ skipping 52 matching lines @@
 
 @end
 
 @implementation RTCAVFoundationVideoCapturerInternal {
   // Keep pointers to inputs for convenience.
   AVCaptureDeviceInput *_frontCameraInput;
   AVCaptureDeviceInput *_backCameraInput;
   AVCaptureVideoDataOutput *_videoDataOutput;
   // The cricket::VideoCapturer that owns this class. Should never be NULL.
   webrtc::AVFoundationVideoCapturer *_capturer;
-  BOOL _orientationHasChanged;
+  webrtc::VideoRotation _rotation;
   BOOL _hasRetriedOnFatalError;
   BOOL _isRunning;
   BOOL _hasStarted;
   rtc::CriticalSection _crit;
 }
 
 @synthesize captureSession = _captureSession;
 @synthesize frameQueue = _frameQueue;
 @synthesize useBackCamera = _useBackCamera;
 @synthesize hasStarted = _hasStarted;
@@ skipping 109 matching lines @@
 }
 
 // Called from WebRTC thread.
 - (void)start {
   if (self.hasStarted) {
     return;
   }
   self.hasStarted = YES;
   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                                block:^{
-    _orientationHasChanged = NO;
+    _rotation = webrtc::kVideoRotation_0;

[tkchin_webrtc, 2016/08/23 17:33:19]

Can you add comment that this sets 0 rotation is orientation is
unknown/faceup/facedown?
What happens to local preview if say device was in landscape orientation, then
set faceup/facedown on table?

[tkchin_webrtc, 2016/08/23 17:37:25]

To clarify - what happens to an AVCapturePreviewLayer attached to the
AVCaptureSession? 

[magjed_webrtc, 2016/08/24 11:08:36]

Nothing happens when the device is set faceup/facedown with the new code. This
is the behavior I observe for the old code as well. The only difference I can
observe from the previous code overall is the brief flash when the device is
rotated.

I don't understand the purpose of the previous |_orientationHasChanged|. Was it
only used to initialize a specific value for iPhone the first time
updateOrientation is called and the orientation happens to be
unknown/faceup/facedown? Do I need to preserve that behavior?

     [self updateOrientation];
 #if TARGET_OS_IPHONE
     [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
 #endif
     AVCaptureSession *captureSession = self.captureSession;
     [captureSession startRunning];
   }];
 }
 
 // Called from same thread as start.
@@ skipping 14 matching lines @@
 #endif
   }];
 }
 
 #pragma mark iOS notifications
 
 #if TARGET_OS_IPHONE
 - (void)deviceOrientationDidChange:(NSNotification *)notification {
   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                                block:^{
-    _orientationHasChanged = YES;
     [self updateOrientation];
   }];
 }
 #endif
 
 #pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
     didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
            fromConnection:(AVCaptureConnection *)connection {
   NSParameterAssert(captureOutput == _videoDataOutput);
   if (!self.hasStarted) {
     return;
   }
-  _capturer->CaptureSampleBuffer(sampleBuffer);
+  _capturer->CaptureSampleBuffer(sampleBuffer, _rotation);
 }
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
     didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
          fromConnection:(AVCaptureConnection *)connection {
   RTCLogError(@"Dropped sample buffer.");
 }
 
 #pragma mark - AVCaptureSession notifications
 
@@ skipping 211 matching lines @@
   if (![device lockForConfiguration:&error]) {
     RTCLogError(@"Failed to lock device for configuration. Error: %@", error.localizedDescription);
     return;
   }
   device.activeVideoMinFrameDuration = minFrameDuration;
   [device unlockForConfiguration];
 }
 
 // Called from capture session queue.
 - (void)updateOrientation {
-  AVCaptureConnection *connection =
-      [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
-  if (!connection.supportsVideoOrientation) {
-    // TODO(tkchin): set rotation bit on frames.
-    return;
-  }
 #if TARGET_OS_IPHONE
-  AVCaptureVideoOrientation orientation = AVCaptureVideoOrientationPortrait;
   switch ([UIDevice currentDevice].orientation) {
     case UIDeviceOrientationPortrait:
-      orientation = AVCaptureVideoOrientationPortrait;
+      _rotation = webrtc::kVideoRotation_90;
       break;
     case UIDeviceOrientationPortraitUpsideDown:
-      orientation = AVCaptureVideoOrientationPortraitUpsideDown;
+      _rotation = webrtc::kVideoRotation_270;
       break;
     case UIDeviceOrientationLandscapeLeft:
-      orientation = AVCaptureVideoOrientationLandscapeRight;
+      _rotation = webrtc::kVideoRotation_180;
       break;
     case UIDeviceOrientationLandscapeRight:
-      orientation = AVCaptureVideoOrientationLandscapeLeft;
+      _rotation = webrtc::kVideoRotation_0;
       break;
     case UIDeviceOrientationFaceUp:
     case UIDeviceOrientationFaceDown:
     case UIDeviceOrientationUnknown:
-      if (!_orientationHasChanged) {
-        connection.videoOrientation = orientation;
-      }
-      return;
+      // Ignore.
+      break;
   }
-  connection.videoOrientation = orientation;
 #endif
 }
 
 // Update the current session input to match what's stored in _useBackCamera.
 - (void)updateSessionInputForUseBackCamera:(BOOL)useBackCamera {
   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                                block:^{
     [_captureSession beginConfiguration];
     AVCaptureDeviceInput *oldInput = _backCameraInput;
     AVCaptureDeviceInput *newInput = _frontCameraInput;
@@ skipping 18 matching lines @@
 
 @end
 
 namespace webrtc {
 
 enum AVFoundationVideoCapturerMessageType : uint32_t {
   kMessageTypeFrame,
 };
 
 struct AVFoundationFrame {
-  AVFoundationFrame(CVImageBufferRef buffer, int64_t time)
-    : image_buffer(buffer), capture_time(time) {}
+  AVFoundationFrame(CVImageBufferRef buffer,
+                    webrtc::VideoRotation rotation,
+                    int64_t time)
+      : image_buffer(buffer), rotation(rotation), capture_time(time) {}
   CVImageBufferRef image_buffer;
+  webrtc::VideoRotation rotation;
   int64_t capture_time;
 };
 
 AVFoundationVideoCapturer::AVFoundationVideoCapturer()
     : _capturer(nil), _startThread(nullptr) {
   // Set our supported formats. This matches kAvailablePresets.
   _capturer =
       [[RTCAVFoundationVideoCapturerInternal alloc] initWithCapturer:this];
 
   std::vector<cricket::VideoFormat> supported_formats;
@@ skipping 87 matching lines @@
 
 void AVFoundationVideoCapturer::SetUseBackCamera(bool useBackCamera) {
   _capturer.useBackCamera = useBackCamera;
 }
 
 bool AVFoundationVideoCapturer::GetUseBackCamera() const {
   return _capturer.useBackCamera;
 }
 
 void AVFoundationVideoCapturer::CaptureSampleBuffer(
-    CMSampleBufferRef sampleBuffer) {
+    CMSampleBufferRef sampleBuffer, webrtc::VideoRotation rotation) {
   if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 ||
       !CMSampleBufferIsValid(sampleBuffer) ||
       !CMSampleBufferDataIsReady(sampleBuffer)) {
     return;
   }
 
   CVImageBufferRef image_buffer = CMSampleBufferGetImageBuffer(sampleBuffer);
   if (image_buffer == NULL) {
     return;
   }
 
   // Retain the buffer and post it to the webrtc thread. It will be released
   // after it has successfully been signaled.
   CVBufferRetain(image_buffer);
-  AVFoundationFrame frame(image_buffer, rtc::TimeNanos());
+  AVFoundationFrame frame(image_buffer, rotation, rtc::TimeNanos());
   _startThread->Post(RTC_FROM_HERE, this, kMessageTypeFrame,
                      new rtc::TypedMessageData<AVFoundationFrame>(frame));
 }
 
 void AVFoundationVideoCapturer::OnMessage(rtc::Message *msg) {
   switch (msg->message_id) {
     case kMessageTypeFrame: {
       rtc::TypedMessageData<AVFoundationFrame>* data =
         static_cast<rtc::TypedMessageData<AVFoundationFrame>*>(msg->pdata);
       const AVFoundationFrame& frame = data->data();
-      OnFrameMessage(frame.image_buffer, frame.capture_time);
+      OnFrameMessage(frame.image_buffer, frame.rotation, frame.capture_time);
       delete data;
       break;
     }
   }
 }
 
 void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer,
+                                               webrtc::VideoRotation rotation,
                                                int64_t capture_time_ns) {
   RTC_DCHECK(_startThread->IsCurrent());
 
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
       new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer);
 
   const int captured_width = buffer->width();
   const int captured_height = buffer->height();
 
   int adapted_width;
   int adapted_height;
   int crop_width;
   int crop_height;
   int crop_x;
   int crop_y;
   int64_t translated_camera_time_us;
 
   if (!AdaptFrame(captured_width, captured_height,
                   capture_time_ns / rtc::kNumNanosecsPerMicrosec,
                   rtc::TimeMicros(), &adapted_width, &adapted_height,
                   &crop_width, &crop_height, &crop_x, &crop_y,
                   &translated_camera_time_us)) {
     CVBufferRelease(image_buffer);
     return;
   }
 
   if (adapted_width != captured_width || crop_width != captured_width ||
-      adapted_height != captured_height || crop_height != captured_height) {
+      adapted_height != captured_height || crop_height != captured_height ||
+      (apply_rotation() && rotation != webrtc::kVideoRotation_0)) {
     // TODO(magjed): Avoid converting to I420.
     rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer(
         _buffer_pool.CreateBuffer(adapted_width, adapted_height));
     scaled_buffer->CropAndScaleFrom(buffer->NativeToI420Buffer(), crop_x,
                                     crop_y, crop_width, crop_height);
-    buffer = scaled_buffer;
+    if (!apply_rotation() || rotation == webrtc::kVideoRotation_0) {
+      buffer = scaled_buffer;
+    } else {
+      // Applying rotation is only supported for legacy reasons and performance
+      // is not critical here.
+      buffer = (rotation == webrtc::kVideoRotation_180)
+          ? I420Buffer::Create(adapted_width, adapted_height)
+          : I420Buffer::Create(adapted_height, adapted_width);
+      libyuv::I420Rotate(scaled_buffer->DataY(), scaled_buffer->StrideY(),
+                         scaled_buffer->DataU(), scaled_buffer->StrideU(),
+                         scaled_buffer->DataV(), scaled_buffer->StrideV(),
+                         buffer->MutableDataY(), buffer->StrideY(),
+                         buffer->MutableDataU(), buffer->StrideU(),
+                         buffer->MutableDataV(), buffer->StrideV(),
+                         crop_width, crop_height,
+                         static_cast<libyuv::RotationMode>(rotation));
+    }
   }
 
-  OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0,
+  OnFrame(cricket::WebRtcVideoFrame(buffer, rotation,
                                     translated_camera_time_us, 0),
           captured_width, captured_height);
 
   CVBufferRelease(image_buffer);
 }
 
 }  // namespace webrtc