[iOS] Fix incorrectly oriented frames when rapidly switching between cameras.

webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCCameraVideoCapturer.m

 /*
  *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #import <Foundation/Foundation.h>
 
 #import "WebRTC/RTCCameraVideoCapturer.h"
 #import "WebRTC/RTCLogging.h"
 #import "WebRTC/RTCVideoFrameBuffer.h"
 
 #if TARGET_OS_IPHONE
 #import "WebRTC/UIDevice+RTCDevice.h"
 #endif
 
 #import "RTCDispatcher+Private.h"
+#import "RTCImageHelper.h"
 
 const int64_t kNanosecondsPerSecond = 1000000000;
 
 static inline BOOL IsMediaSubTypeSupported(FourCharCode mediaSubType) {
   return (mediaSubType == kCVPixelFormatType_420YpCbCr8PlanarFullRange ||
           mediaSubType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
 }
 
 @interface RTCCameraVideoCapturer ()<AVCaptureVideoDataOutputSampleBufferDelegate>
 @property(nonatomic, readonly) dispatch_queue_t frameQueue;
+@property(assign) BOOL switchingCameras;

[magjed_webrtc, 2017/07/24 10:12:50]

It looks like we are accessing this variable from multiple threads without
synchronization.

 @end
 
 @implementation RTCCameraVideoCapturer {
   AVCaptureVideoDataOutput *_videoDataOutput;
   AVCaptureSession *_captureSession;
   AVCaptureDevice *_currentDevice;
-  RTCVideoRotation _rotation;
   BOOL _hasRetriedOnFatalError;
   BOOL _isRunning;
   // Will the session be running once all asynchronous operations have been completed?
   BOOL _willBeRunning;
+  BOOL _switchingCameras;
+#if TARGET_OS_IPHONE
+  UIDeviceOrientation _orientation;
+#endif
 }
 
 @synthesize frameQueue = _frameQueue;
 @synthesize captureSession = _captureSession;
+@synthesize switchingCameras = _switchingCameras;
 
 - (instancetype)initWithDelegate:(__weak id<RTCVideoCapturerDelegate>)delegate {
   if (self = [super initWithDelegate:delegate]) {
     // Create the capture session and all relevant inputs and outputs. We need
     // to do this in init because the application may want the capture session
     // before we start the capturer for e.g. AVCapturePreviewLayer. All objects
     // created here are retained until dealloc and never recreated.
     if (![self setupCaptureSession]) {
       return nil;
     }
     NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
+    self.switchingCameras = NO;
 #if TARGET_OS_IPHONE
+    _orientation = UIDeviceOrientationPortrait;
     [center addObserver:self
                selector:@selector(deviceOrientationDidChange:)
                    name:UIDeviceOrientationDidChangeNotification
                  object:nil];
     [center addObserver:self
                selector:@selector(handleCaptureSessionInterruption:)
                    name:AVCaptureSessionWasInterruptedNotification
                  object:_captureSession];
     [center addObserver:self
                selector:@selector(handleCaptureSessionInterruptionEnded:)
@@ skipping 41 matching lines @@
       [eligibleDeviceFormats addObject:format];
     }
   }
 
   return eligibleDeviceFormats;
 }
 
 - (void)startCaptureWithDevice:(AVCaptureDevice *)device
                         format:(AVCaptureDeviceFormat *)format
                            fps:(NSInteger)fps {
-  _willBeRunning = true;
+  _willBeRunning = YES;
   [RTCDispatcher
       dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                     block:^{
                       RTCLogInfo("startCaptureWithDevice %@ @ %zd fps", format, fps);
 
 #if TARGET_OS_IPHONE
                       [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
 #endif
 
                       _currentDevice = device;
 
                       NSError *error = nil;
                       if (![_currentDevice lockForConfiguration:&error]) {
                         RTCLogError(
                             @"Failed to lock device %@. Error: %@", _currentDevice, error.userInfo);
                         return;
                       }
 
+                      self.switchingCameras = YES;
                       [self reconfigureCaptureSessionInput];
                       [self updateOrientation];
                       [_captureSession startRunning];
                       [self updateDeviceCaptureFormat:format fps:fps];
                       [_currentDevice unlockForConfiguration];
-                      _isRunning = true;
+                      self.switchingCameras = NO;
+                      _isRunning = YES;
                     }];
 }
 
 - (void)stopCapture {
-  _willBeRunning = false;
+  _willBeRunning = NO;
   [RTCDispatcher
       dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                     block:^{
                       RTCLogInfo("Stop");
                       _currentDevice = nil;
                       for (AVCaptureDeviceInput *oldInput in [_captureSession.inputs copy]) {
                         [_captureSession removeInput:oldInput];
                       }
                       [_captureSession stopRunning];
 
 #if TARGET_OS_IPHONE
                       [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
 #endif
-                      _isRunning = false;
+                      _isRunning = NO;
                     }];
 }
 
 #pragma mark iOS notifications
 
 #if TARGET_OS_IPHONE
 - (void)deviceOrientationDidChange:(NSNotification *)notification {
   [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                                block:^{
                                  [self updateOrientation];
@@ skipping 11 matching lines @@
   if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
       !CMSampleBufferDataIsReady(sampleBuffer)) {
     return;
   }
 
   CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
   if (pixelBuffer == nil) {
     return;
   }
 
+#if TARGET_OS_IPHONE
+  // Default to portrait orientation on iPhone.
+  RTCVideoRotation rotation = RTCVideoRotation_90;
+  // Check here, which camera this frame is from, to avoid any race conditions.
+  AVCaptureDeviceInput *deviceInput =
+      (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
+  BOOL usingFrontCamera = deviceInput.device.position == AVCaptureDevicePositionFront;
+  // We gate checking the image's EXIF only if we're switching cameras as we don't need to parse
+  // the image's attachments and dictionaries for every video image.
+  if (self.switchingCameras) {
+    // Check the image's EXIF for the actual camera the image came as the image could have been
+    // delayed as we set alwaysDiscardsLateVideoFrames to NO.
+    usingFrontCamera = [RTCImageHelper isFrontCameraFromSampleBuffer:sampleBuffer];

[magjed_webrtc, 2017/07/24 10:12:50]

Is this method guaranteed to work on all devices? How expensive is it to call
it? If it's reliable and cheap to call it, can we always use it to simplify the
code?

[jtt_webrtc, 2017/07/24 16:40:36]

I've tested it with an iPhone SE and iPhone 7 and 7+. 

Elapsed time to call is ~0.004ms, so I'll remove the switchingCameras flag.

+  }
+  switch (_orientation) {
+    case UIDeviceOrientationPortrait:
+      rotation = RTCVideoRotation_90;
+      break;
+    case UIDeviceOrientationPortraitUpsideDown:
+      rotation = RTCVideoRotation_270;
+      break;
+    case UIDeviceOrientationLandscapeLeft:
+      rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
+      break;
+    case UIDeviceOrientationLandscapeRight:
+      rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
+      break;
+    case UIDeviceOrientationFaceUp:
+    case UIDeviceOrientationFaceDown:
+    case UIDeviceOrientationUnknown:
+      // Ignore.
+      break;
+  }
+#else
+  // No rotation on Mac.
+  RTCVideoRotation rotation = RTCVideoRotation_0;
+#endif
+
   RTCCVPixelBuffer *rtcPixelBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer];
   int64_t timeStampNs = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) *
-                        kNanosecondsPerSecond;
+      kNanosecondsPerSecond;
   RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithBuffer:rtcPixelBuffer
-                                                           rotation:_rotation
+                                                           rotation:rotation
                                                         timeStampNs:timeStampNs];
   [self.delegate capturer:self didCaptureVideoFrame:videoFrame];
 }
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
     didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
          fromConnection:(AVCaptureConnection *)connection {
   RTCLogError(@"Dropped sample buffer.");
 }
 
@@ skipping 182 matching lines @@
   } else {
     RTCLogError(@"Cannot add camera as an input to the session.");
   }
   [_captureSession commitConfiguration];
 }
 
 - (void)updateOrientation {
   NSAssert([RTCDispatcher isOnQueueForType:RTCDispatcherTypeCaptureSession],
            @"updateOrientation must be called on the capture queue.");
 #if TARGET_OS_IPHONE
-  BOOL usingFrontCamera = _currentDevice.position == AVCaptureDevicePositionFront;
-  switch ([UIDevice currentDevice].orientation) {
-    case UIDeviceOrientationPortrait:
-      _rotation = RTCVideoRotation_90;
-      break;
-    case UIDeviceOrientationPortraitUpsideDown:
-      _rotation = RTCVideoRotation_270;
-      break;
-    case UIDeviceOrientationLandscapeLeft:
-      _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
-      break;
-    case UIDeviceOrientationLandscapeRight:
-      _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
-      break;
-    case UIDeviceOrientationFaceUp:
-    case UIDeviceOrientationFaceDown:
-    case UIDeviceOrientationUnknown:
-      // Ignore.
-      break;
-  }
+  _orientation = [UIDevice currentDevice].orientation;
 #endif
 }
 
 @end