[iOS] Fix incorrectly oriented frames when rapidly switching between cameras.

webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm

Index: webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
diff --git a/webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm b/webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
index 166a609824941775e629936bb6e4c03905d20242..04c2b52e6f7da703fe6ddf1d14a60b0b295475b9 100644
--- a/webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
+++ b/webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
@@ -28,11 +28,14 @@
   AVCaptureVideoDataOutput *_videoDataOutput;
   // The cricket::VideoCapturer that owns this class. Should never be NULL.
   webrtc::AVFoundationVideoCapturer *_capturer;
-  webrtc::VideoRotation _rotation;
   BOOL _hasRetriedOnFatalError;
   BOOL _isRunning;
   BOOL _hasStarted;
   rtc::CriticalSection _crit;
+  BOOL _switchingCameras;
+#if TARGET_OS_IPHONE
+  UIDeviceOrientation _orientation;
+#endif
 }
 
 @synthesize captureSession = _captureSession;
@@ -56,7 +59,9 @@
       return nil;
     }
     NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
+    _switchingCameras = NO;
 #if TARGET_OS_IPHONE
+    _orientation = UIDeviceOrientationPortrait;
     [center addObserver:self
                selector:@selector(deviceOrientationDidChange:)
                    name:UIDeviceOrientationDidChangeNotification
@@ -157,24 +162,16 @@
   if (self.hasStarted) {
     return;
   }
-  self.hasStarted = YES;
   [RTCDispatcher
       dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
                     block:^{
-#if TARGET_OS_IPHONE
-                        // Default to portrait orientation on iPhone. This will be reset in
-                        // updateOrientation unless orientation is unknown/faceup/facedown.
-                        _rotation = webrtc::kVideoRotation_90;
-#else
-                        // No rotation on Mac.
-                        _rotation = webrtc::kVideoRotation_0;
-#endif
                         [self updateOrientation];
 #if TARGET_OS_IPHONE
                         [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
 #endif
                         AVCaptureSession *captureSession = self.captureSession;
                         [captureSession startRunning];
+                        self.hasStarted = YES;

[kthelgason, 2017/07/20 08:07:54]

This change seems unrelated.

[jtt_webrtc, 2017/07/20 16:26:28]

We don't really want to mark the session as started until the capture session
has started running.

                     }];
 }
 
@@ -218,7 +215,64 @@
   if (!self.hasStarted) {
     return;
   }
-  _capturer->CaptureSampleBuffer(sampleBuffer, _rotation);
+
+#if TARGET_OS_IPHONE
+  // Default to portrait orientation on iPhone.
+  webrtc::VideoRotation rotation = webrtc::kVideoRotation_90;
+  AVCaptureDeviceInput *deviceInput =
+      (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
+  BOOL usingFrontCamera = deviceInput.device.position == AVCaptureDevicePositionFront;
+  if (_switchingCameras) {
+    // Check the image's EXIF for the actual camera the image came from only when switching
+    // cameras as the image may still be from the old camera.
+    CFDictionaryRef attachments = CMCopyDictionaryOfAttachments(
+        kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate);
+    if (attachments) {
+      int size = CFDictionaryGetCount(attachments);
+      if (size > 0) {
+        CFDictionaryRef cfExifDictVal = NULL;
+        if (CFDictionaryGetValueIfPresent(
+                attachments, (const void *)CFSTR("{Exif}"), (const void **)&cfExifDictVal)) {
+          CFStringRef cfLensModelStrVal;
+          if (CFDictionaryGetValueIfPresent(cfExifDictVal,
+                                            (const void *)CFSTR("LensModel"),
+                                            (const void **)&cfLensModelStrVal)) {
+            if ([(__bridge NSString *)cfLensModelStrVal containsString:@"front"]) {
+              usingFrontCamera = YES;
+            } else if ([(__bridge NSString *)cfLensModelStrVal containsString:@"back"]) {
+              usingFrontCamera = NO;
+            }
+          }
+        }
+      }
+      CFRelease(attachments);
+    }
+  }

[kthelgason, 2017/07/20 08:07:54]

Can we put this logic into it's own class instead of duplicating it?

[jtt_webrtc, 2017/07/20 16:26:28]

I moved the EXIF check to it's own class RTCImageHelper.

+  switch (_orientation) {
+    case UIDeviceOrientationPortrait:
+      rotation = webrtc::kVideoRotation_90;
+      break;
+    case UIDeviceOrientationPortraitUpsideDown:
+      rotation = webrtc::kVideoRotation_270;
+      break;
+    case UIDeviceOrientationLandscapeLeft:
+      rotation = usingFrontCamera ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
+      break;
+    case UIDeviceOrientationLandscapeRight:
+      rotation = usingFrontCamera ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
+      break;
+    case UIDeviceOrientationFaceUp:
+    case UIDeviceOrientationFaceDown:
+    case UIDeviceOrientationUnknown:
+      // Ignore.
+      break;
+  }
+#else
+  // No rotation on Mac.
+  webrtc::VideoRotation rotation = webrtc::kVideoRotation_0;
+#endif
+
+  _capturer->CaptureSampleBuffer(sampleBuffer, rotation);
 }
 
 - (void)captureOutput:(AVCaptureOutput *)captureOutput
@@ -448,56 +502,37 @@
 // Called from capture session queue.
 - (void)updateOrientation {
 #if TARGET_OS_IPHONE
-  switch ([UIDevice currentDevice].orientation) {
-    case UIDeviceOrientationPortrait:
-      _rotation = webrtc::kVideoRotation_90;
-      break;
-    case UIDeviceOrientationPortraitUpsideDown:
-      _rotation = webrtc::kVideoRotation_270;
-      break;
-    case UIDeviceOrientationLandscapeLeft:
-      _rotation =
-          _capturer->GetUseBackCamera() ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
-      break;
-    case UIDeviceOrientationLandscapeRight:
-      _rotation =
-          _capturer->GetUseBackCamera() ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
-      break;
-    case UIDeviceOrientationFaceUp:
-    case UIDeviceOrientationFaceDown:
-    case UIDeviceOrientationUnknown:
-      // Ignore.
-      break;
-  }
+  _orientation = [UIDevice currentDevice].orientation;
 #endif
 }
 
 // Update the current session input to match what's stored in _useBackCamera.
 - (void)updateSessionInputForUseBackCamera:(BOOL)useBackCamera {
-  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
-                               block:^{
-                                   [_captureSession beginConfiguration];
-                                   AVCaptureDeviceInput *oldInput = _backCameraInput;
-                                   AVCaptureDeviceInput *newInput = _frontCameraInput;
-                                   if (useBackCamera) {
-                                     oldInput = _frontCameraInput;
-                                     newInput = _backCameraInput;
-                                   }
-                                   if (oldInput) {
-                                     // Ok to remove this even if it's not attached. Will be no-op.
-                                     [_captureSession removeInput:oldInput];
-                                   }
-                                   if (newInput) {
-                                     [_captureSession addInput:newInput];
-                                   }
-                                   [self updateOrientation];
-                                   AVCaptureDevice *newDevice = newInput.device;
-                                   const cricket::VideoFormat *format =
-                                     _capturer->GetCaptureFormat();
-                                   webrtc::SetFormatForCaptureDevice(
-                                       newDevice, _captureSession, *format);
-                                   [_captureSession commitConfiguration];
-                               }];
+  [RTCDispatcher
+      dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                    block:^{
+                      _switchingCameras = YES;
+                      [_captureSession beginConfiguration];
+                      AVCaptureDeviceInput *oldInput = _backCameraInput;
+                      AVCaptureDeviceInput *newInput = _frontCameraInput;
+                      if (useBackCamera) {
+                        oldInput = _frontCameraInput;
+                        newInput = _backCameraInput;
+                      }
+                      if (oldInput) {
+                        // Ok to remove this even if it's not attached. Will be no-op.
+                        [_captureSession removeInput:oldInput];
+                      }
+                      if (newInput) {
+                        [_captureSession addInput:newInput];
+                      }
+                      [self updateOrientation];
+                      AVCaptureDevice *newDevice = newInput.device;
+                      const cricket::VideoFormat *format = _capturer->GetCaptureFormat();
+                      webrtc::SetFormatForCaptureDevice(newDevice, _captureSession, *format);
+                      [_captureSession commitConfiguration];
+                      _switchingCameras = NO;
+                    }];
 }
 
 @end