[iOS] Fix incorrectly oriented frames when rapidly switching between cameras.

webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm

 /*
  *  Copyright 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 10 matching lines @@
 
 #include "avfoundationformatmapper.h"
 
 @implementation RTCAVFoundationVideoCapturerInternal {
   // Keep pointers to inputs for convenience.
   AVCaptureDeviceInput *_frontCameraInput;
   AVCaptureDeviceInput *_backCameraInput;
   AVCaptureVideoDataOutput *_videoDataOutput;
   // The cricket::VideoCapturer that owns this class. Should never be NULL.
   webrtc::AVFoundationVideoCapturer *_capturer;
-  webrtc::VideoRotation _rotation;
   BOOL _hasRetriedOnFatalError;
   BOOL _isRunning;
   BOOL _hasStarted;
   rtc::CriticalSection _crit;
+  BOOL _switchingCameras;
+#if TARGET_OS_IPHONE
+  UIDeviceOrientation _orientation;
+#endif
 }
 
 @synthesize captureSession = _captureSession;
 @synthesize frameQueue = _frameQueue;
 @synthesize useBackCamera = _useBackCamera;
 
 @synthesize isRunning = _isRunning;
 @synthesize hasStarted = _hasStarted;
 
 // This is called from the thread that creates the video source, which is likely
 // the main thread.
 - (instancetype)initWithCapturer:(webrtc::AVFoundationVideoCapturer *)capturer {
   RTC_DCHECK(capturer);
   if (self = [super init]) {
     _capturer = capturer;
     // Create the capture session and all relevant inputs and outputs. We need
     // to do this in init because the application may want the capture session
     // before we start the capturer for e.g. AVCapturePreviewLayer. All objects
     // created here are retained until dealloc and never recreated.
     if (![self setupCaptureSession]) {
       return nil;
     }
     NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
+    _switchingCameras = NO;
 #if TARGET_OS_IPHONE
+    _orientation = UIDeviceOrientationPortrait;
     [center addObserver:self
                selector:@selector(deviceOrientationDidChange:)
                    name:UIDeviceOrientationDidChangeNotification
                  object:nil];
     [center addObserver:self
                selector:@selector(handleCaptureSessionInterruption:)
                    name:AVCaptureSessionWasInterruptedNotification
                  object:_captureSession];
     [center addObserver:self
                selector:@selector(handleCaptureSessionInterruptionEnded:)
@@ skipping 80 matching lines @@
     _useBackCamera = useBackCamera;
     [self updateSessionInputForUseBackCamera:useBackCamera];
   }
 }
 
 // Called from WebRTC thread.
 - (void)start {
   if (self.hasStarted) {
     return;
   }
-  self.hasStarted = YES;
   [RTCDispatcher
       dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                     block:^{
-#if TARGET_OS_IPHONE
-                        // Default to portrait orientation on iPhone. This will be reset in
-                        // updateOrientation unless orientation is unknown/faceup/facedown.
-                        _rotation = webrtc::kVideoRotation_90;
-#else
-                        // No rotation on Mac.
-                        _rotation = webrtc::kVideoRotation_0;
-#endif
                         [self updateOrientation];
 #if TARGET_OS_IPHONE
                         [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
 #endif
                         AVCaptureSession *captureSession = self.captureSession;
                         [captureSession startRunning];
+                        self.hasStarted = YES;

[kthelgason, 2017/07/20 08:07:54]

This change seems unrelated.

[jtt_webrtc, 2017/07/20 16:26:28]

We don't really want to mark the session as started until the capture session
has started running.

                     }];
 }
 
 // Called from same thread as start.
 - (void)stop {
   if (!self.hasStarted) {
     return;
   }
   self.hasStarted = NO;
   // Due to this async block, it's possible that the ObjC object outlives the
@@ skipping 23 matching lines @@
 
 #pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
     didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
            fromConnection:(AVCaptureConnection *)connection {
   NSParameterAssert(captureOutput == _videoDataOutput);
   if (!self.hasStarted) {
     return;
   }
-  _capturer->CaptureSampleBuffer(sampleBuffer, _rotation);
+
+#if TARGET_OS_IPHONE
+  // Default to portrait orientation on iPhone.
+  webrtc::VideoRotation rotation = webrtc::kVideoRotation_90;
+  AVCaptureDeviceInput *deviceInput =
+      (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
+  BOOL usingFrontCamera = deviceInput.device.position == AVCaptureDevicePositionFront;
+  if (_switchingCameras) {
+    // Check the image's EXIF for the actual camera the image came from only when switching
+    // cameras as the image may still be from the old camera.
+    CFDictionaryRef attachments = CMCopyDictionaryOfAttachments(
+        kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate);
+    if (attachments) {
+      int size = CFDictionaryGetCount(attachments);
+      if (size > 0) {
+        CFDictionaryRef cfExifDictVal = NULL;
+        if (CFDictionaryGetValueIfPresent(
+                attachments, (const void *)CFSTR("{Exif}"), (const void **)&cfExifDictVal)) {
+          CFStringRef cfLensModelStrVal;
+          if (CFDictionaryGetValueIfPresent(cfExifDictVal,
+                                            (const void *)CFSTR("LensModel"),
+                                            (const void **)&cfLensModelStrVal)) {
+            if ([(__bridge NSString *)cfLensModelStrVal containsString:@"front"]) {
+              usingFrontCamera = YES;
+            } else if ([(__bridge NSString *)cfLensModelStrVal containsString:@"back"]) {
+              usingFrontCamera = NO;
+            }
+          }
+        }
+      }
+      CFRelease(attachments);
+    }
+  }

[kthelgason, 2017/07/20 08:07:54]

Can we put this logic into it's own class instead of duplicating it?

[jtt_webrtc, 2017/07/20 16:26:28]

I moved the EXIF check to it's own class RTCImageHelper.

+  switch (_orientation) {
+    case UIDeviceOrientationPortrait:
+      rotation = webrtc::kVideoRotation_90;
+      break;
+    case UIDeviceOrientationPortraitUpsideDown:
+      rotation = webrtc::kVideoRotation_270;
+      break;
+    case UIDeviceOrientationLandscapeLeft:
+      rotation = usingFrontCamera ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
+      break;
+    case UIDeviceOrientationLandscapeRight:
+      rotation = usingFrontCamera ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
+      break;
+    case UIDeviceOrientationFaceUp:
+    case UIDeviceOrientationFaceDown:
+    case UIDeviceOrientationUnknown:
+      // Ignore.
+      break;
+  }
+#else
+  // No rotation on Mac.
+  webrtc::VideoRotation rotation = webrtc::kVideoRotation_0;
+#endif
+
+  _capturer->CaptureSampleBuffer(sampleBuffer, rotation);
 }
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
     didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
          fromConnection:(AVCaptureConnection *)connection {
   RTCLogError(@"Dropped sample buffer.");
 }
 
 #pragma mark - AVCaptureSession notifications
 
@@ skipping 209 matching lines @@
       return nil;
     }
     _backCameraInput = backCameraInput;
   }
   return _backCameraInput;
 }
 
 // Called from capture session queue.
 - (void)updateOrientation {
 #if TARGET_OS_IPHONE
-  switch ([UIDevice currentDevice].orientation) {
-    case UIDeviceOrientationPortrait:
-      _rotation = webrtc::kVideoRotation_90;
-      break;
-    case UIDeviceOrientationPortraitUpsideDown:
-      _rotation = webrtc::kVideoRotation_270;
-      break;
-    case UIDeviceOrientationLandscapeLeft:
-      _rotation =
-          _capturer->GetUseBackCamera() ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
-      break;
-    case UIDeviceOrientationLandscapeRight:
-      _rotation =
-          _capturer->GetUseBackCamera() ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
-      break;
-    case UIDeviceOrientationFaceUp:
-    case UIDeviceOrientationFaceDown:
-    case UIDeviceOrientationUnknown:
-      // Ignore.
-      break;
-  }
+  _orientation = [UIDevice currentDevice].orientation;
 #endif
 }
 
 // Update the current session input to match what's stored in _useBackCamera.
 - (void)updateSessionInputForUseBackCamera:(BOOL)useBackCamera {
-  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
-                               block:^{
-                                   [_captureSession beginConfiguration];
-                                   AVCaptureDeviceInput *oldInput = _backCameraInput;
-                                   AVCaptureDeviceInput *newInput = _frontCameraInput;
-                                   if (useBackCamera) {
-                                     oldInput = _frontCameraInput;
-                                     newInput = _backCameraInput;
-                                   }
-                                   if (oldInput) {
-                                     // Ok to remove this even if it's not attached. Will be no-op.
-                                     [_captureSession removeInput:oldInput];
-                                   }
-                                   if (newInput) {
-                                     [_captureSession addInput:newInput];
-                                   }
-                                   [self updateOrientation];
-                                   AVCaptureDevice *newDevice = newInput.device;
-                                   const cricket::VideoFormat *format =
-                                     _capturer->GetCaptureFormat();
-                                   webrtc::SetFormatForCaptureDevice(
-                                       newDevice, _captureSession, *format);
-                                   [_captureSession commitConfiguration];
-                               }];
+  [RTCDispatcher
+      dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                    block:^{
+                      _switchingCameras = YES;
+                      [_captureSession beginConfiguration];
+                      AVCaptureDeviceInput *oldInput = _backCameraInput;
+                      AVCaptureDeviceInput *newInput = _frontCameraInput;
+                      if (useBackCamera) {
+                        oldInput = _frontCameraInput;
+                        newInput = _backCameraInput;
+                      }
+                      if (oldInput) {
+                        // Ok to remove this even if it's not attached. Will be no-op.
+                        [_captureSession removeInput:oldInput];
+                      }
+                      if (newInput) {
+                        [_captureSession addInput:newInput];
+                      }
+                      [self updateOrientation];
+                      AVCaptureDevice *newDevice = newInput.device;
+                      const cricket::VideoFormat *format = _capturer->GetCaptureFormat();
+                      webrtc::SetFormatForCaptureDevice(newDevice, _captureSession, *format);
+                      [_captureSession commitConfiguration];
+                      _switchingCameras = NO;
+                    }];
 }
 
 @end