New RTCCameraVideoCapturer.

webrtc/sdk/objc/Framework/Classes/RTCCameraVideoCapturer.mm

+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "WebRTC/RTCCameraVideoCapturer.h"

[daniela-webrtc, 2017/03/26 18:08:58]

Systems import first. Then empty line, then local frameworks imports, empty line
then local files

[sakal, 2017/03/29 11:45:44]

Done.

+
+#import <Foundation/Foundation.h>
+
+#if TARGET_OS_IPHONE
+#import "WebRTC/UIDevice+RTCDevice.h"
+#endif
+
+#import "RTCDispatcher+Private.h"
+#import "WebRTC/RTCLogging.h"
+
+const int64_t kNanosecondsPerSecond = 1000000000;
+
+@interface RTCCameraVideoCapturer ()
+@property(nonatomic, readonly) dispatch_queue_t frameQueue;
+@end
+
+@implementation RTCCameraVideoCapturer {
+  AVCaptureVideoDataOutput *_videoDataOutput;
+  AVCaptureSession *_captureSession;
+  AVCaptureDevice *_currentDevice;
+  RTCVideoRotation _rotation;
+  BOOL _hasRetriedOnFatalError;
+}
+
+@synthesize frameQueue = _frameQueue;
+@synthesize captureSession = _captureSession;
+
+- (instancetype)initWithDelegate:(__weak id<RTCVideoCapturerDelegate>)delegate {
+  if (self = [super initWithDelegate:delegate]) {
+    // Create the capture session and all relevant inputs and outputs. We need
+    // to do this in init because the application may want the capture session
+    // before we start the capturer for e.g. AVCapturePreviewLayer. All objects
+    // created here are retained until dealloc and never recreated.
+    if (![self setupCaptureSession]) {
+      return nil;
+    }
+    NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
+#if TARGET_OS_IPHONE
+    [center addObserver:self
+               selector:@selector(deviceOrientationDidChange:)
+                   name:UIDeviceOrientationDidChangeNotification
+                 object:nil];
+    [center addObserver:self
+               selector:@selector(handleCaptureSessionInterruption:)
+                   name:AVCaptureSessionWasInterruptedNotification
+                 object:_captureSession];
+    [center addObserver:self
+               selector:@selector(handleCaptureSessionInterruptionEnded:)
+                   name:AVCaptureSessionInterruptionEndedNotification
+                 object:_captureSession];
+    [center addObserver:self
+               selector:@selector(handleApplicationDidBecomeActive:)
+                   name:UIApplicationDidBecomeActiveNotification
+                 object:[UIApplication sharedApplication]];
+#endif
+    [center addObserver:self
+               selector:@selector(handleCaptureSessionRuntimeError:)
+                   name:AVCaptureSessionRuntimeErrorNotification
+                 object:_captureSession];
+    [center addObserver:self
+               selector:@selector(handleCaptureSessionDidStartRunning:)
+                   name:AVCaptureSessionDidStartRunningNotification
+                 object:_captureSession];
+    [center addObserver:self
+               selector:@selector(handleCaptureSessionDidStopRunning:)
+                   name:AVCaptureSessionDidStopRunningNotification
+                 object:_captureSession];
+  }
+  return self;
+}
+
++ (NSArray<AVCaptureDevice *> *)captureDevices {
+  return [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+}
+
+static inline BOOL IsMediaSubTypeSupported(FourCharCode mediaSubType) {

[daniela-webrtc, 2017/03/26 18:08:58]

Usually static function go at the top of the file together with the static
properties.

[sakal, 2017/03/29 11:45:44]

Done.

+  return (mediaSubType == kCVPixelFormatType_420YpCbCr8PlanarFullRange ||
+          mediaSubType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
+}
+
++ (NSArray<AVCaptureDeviceFormat *> *)supportedFormatsForDevice:(AVCaptureDevice *)device {
+  NSMutableArray<AVCaptureDeviceFormat *> *eligibleDeviceFormats = [NSMutableArray array];
+
+  for (AVCaptureDeviceFormat *format in device.formats) {
+    // Filter out subTypes that we currently don't support in the stack
+    FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
+    if (IsMediaSubTypeSupported(mediaSubType)) {
+      [eligibleDeviceFormats addObject:format];
+    }
+  }
+
+  return eligibleDeviceFormats;
+}
+
+- (void)dealloc {

[daniela-webrtc, 2017/03/26 18:08:58]

Idea: How about we make sure that the session is not capturing when we are being
dealloc-ated. That way we force devs to call `-stop` and give this class
opportunity to clean up and also we would prevent simple mistakes (like this
object not living long enough) to be prevented.

[daniela-webrtc, 2017/03/26 18:08:58]

We should stop the capture session somewhere as well. The dealloc seems like a
good place to do it.

[sakal, 2017/03/29 11:45:44]

Capture session is now stopped in stopCapture.

+  RTCLogInfo("dealloc");

[daniela-webrtc, 2017/03/26 18:08:57]

I don't think this logging is useful.

[sakal, 2017/03/29 11:45:44]

Done.

+  [[NSNotificationCenter defaultCenter] removeObserver:self];
+}

[daniela-webrtc, 2017/03/26 18:08:57]

[super dealloc];
at the end of the method.

[sakal, 2017/03/29 11:45:44]

Done.

+
+- (dispatch_queue_t)frameQueue {
+  if (!_frameQueue) {
+    _frameQueue =
+        dispatch_queue_create("org.webrtc.avfoundationvideocapturer.video", DISPATCH_QUEUE_SERIAL);
+    dispatch_set_target_queue(_frameQueue,
+                              dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
+  }
+  return _frameQueue;
+}
+
+- (void)startCaptureWithDevice:(AVCaptureDevice *)device

[daniela-webrtc, 2017/03/26 18:08:57]

In implementation file, keep public methods together and follow the order in
which the methods were defined in the header file.

[sakal, 2017/03/29 11:45:44]

Done.

+                        format:(AVCaptureDeviceFormat *)format
+                           fps:(int)fps {
+  [RTCDispatcher
+      dispatchAsyncOnType:RTCDispatcherTypeCaptureSession

[daniela-webrtc, 2017/03/26 18:08:57]

Suggestion: extract some of the functionality from this block in methods. It
will make testing easier and also increase readability.

[sakal, 2017/03/29 11:45:44]

Done.

+                    block:^{
+                      RTCLogInfo("startCaptureWithDevice %@ @ %d fps", format, fps);
+                      NSError *error = nil;
+
+                      AVCaptureDeviceInput *input =
+                          [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
+                      if (!input) {
+                        RTCLogError(@"Failed to create front camera input: %@",
+                                    error.localizedDescription);
+                        // TODO(magjed): Error callback?

[magjed_webrtc, 2017/03/27 13:11:48]

Remove TODOs if you are not going to add the error callback in this CL.

[sakal, 2017/03/29 11:45:44]

Done.

+                        return;
+                      }
+
+                      [_captureSession beginConfiguration];
+                      _currentDevice = device;
+                      for (AVCaptureDeviceInput *oldInput in _captureSession.inputs) {
+                        [_captureSession removeInput:oldInput];
+                      }
+                      if ([_captureSession canAddInput:input]) {
+                        [_captureSession addInput:input];
+                      } else {
+                        RTCLogError(@"Cannot add camera as an input to the session.");
+                        // TODO(sakal): Error callback?
+                        return;
+                      }
+                      [self updateOrientation];
+                      if ([device lockForConfiguration:&error]) {
+                        @try {
+                          device.activeFormat = format;
+                          device.activeVideoMinFrameDuration = CMTimeMake(1, fps);
+                        } @catch (NSException *exception) {
+                          RTCLogError(@"Failed to set active format!\n User info:%@",
+                                      exception.userInfo);
+                          // TODO(sakal): Error callback?
+                          return;
+                        }
+                        [device unlockForConfiguration];
+                      } else {
+                        RTCLogError(@"Failed to lock device %@. Error: %@", device, error.userInfo);
+                        // TODO(sakal): Error callback?
+                        return;
+                      }
+                      [_captureSession commitConfiguration];
+                    }];
+}
+
+- (void)stop {

[daniela-webrtc, 2017/03/26 18:08:58]

I don't see a reason with this type of workload in `stop` to perform it
async-ly. As a matter of fact the dispatch here might be an overhead.

[magjed_webrtc, 2017/03/27 13:11:48]

Maybe it's necessary for thread safety?

We need to be careful here that it's ok to stop async with respect to e.g.
dealloc.

+  [RTCDispatcher
+      dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                    block:^{
+                      RTCLogInfo("Stop");
+                      [_captureSession beginConfiguration];

[daniela-webrtc, 2017/03/26 18:08:57]

No need to wrap the input removal in `beginConfiguration` and
`commitConfiguration` according to the documentation

[sakal, 2017/03/29 11:45:44]

Done.

+                      for (AVCaptureDeviceInput *oldInput in _captureSession.inputs) {
+                        [_captureSession removeInput:oldInput];
+                      }
+                      [_captureSession commitConfiguration];
+#if TARGET_OS_IPHONE
+                      [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
+#endif
+                    }];
+}
+
+#pragma mark iOS notifications
+
+#if TARGET_OS_IPHONE
+- (void)deviceOrientationDidChange:(NSNotification *)notification {
+  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                               block:^{
+                                 [self updateOrientation];
+                               }];
+}
+#endif
+
+#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+    didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+           fromConnection:(AVCaptureConnection *)connection {
+  NSParameterAssert(captureOutput == _videoDataOutput);
+
+  if (CMSampleBufferGetNumSamples(sampleBuffer) != 1 || !CMSampleBufferIsValid(sampleBuffer) ||
+      !CMSampleBufferDataIsReady(sampleBuffer)) {
+    return;
+  }
+
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (pixelBuffer == nullptr) {
+    return;
+  }
+
+  int64_t timeStampNs = CACurrentMediaTime() * kNanosecondsPerSecond;
+  RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] initWithPixelBuffer:pixelBuffer
+                                                                rotation:_rotation
+                                                             timeStampNs:timeStampNs];
+  [self.delegate capturer:self didCaptureVideoFrame:videoFrame];
+}
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput
+    didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
+         fromConnection:(AVCaptureConnection *)connection {
+  RTCLogError(@"Dropped sample buffer.");
+}
+
+#pragma mark - AVCaptureSession notifications
+
+- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
+  NSString *reasonString = nil;
+#if defined(__IPHONE_9_0) && defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && \
+    __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_9_0
+  if ([UIDevice isIOS9OrLater]) {
+    NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey];
+    if (reason) {
+      switch (reason.intValue) {
+        case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground:
+          reasonString = @"VideoDeviceNotAvailableInBackground";
+          break;
+        case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient:
+          reasonString = @"AudioDeviceInUseByAnotherClient";
+          break;
+        case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient:
+          reasonString = @"VideoDeviceInUseByAnotherClient";
+          break;
+        case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps:
+          reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps";
+          break;
+      }
+    }
+  }
+#endif
+  RTCLog(@"Capture session interrupted: %@", reasonString);
+  // TODO(tkchin): Handle this case.
+}
+
+- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
+  RTCLog(@"Capture session interruption ended.");
+  // TODO(tkchin): Handle this case.
+}
+
+- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
+  NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
+  RTCLogError(@"Capture session runtime error: %@", error);
+
+  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                               block:^{
+#if TARGET_OS_IPHONE
+                                 if (error.code == AVErrorMediaServicesWereReset) {
+                                   [self handleNonFatalError];
+                                 } else {
+                                   [self handleFatalError];
+                                 }
+#else
+                                [self handleFatalError];
+#endif
+                               }];
+}
+
+- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
+  RTCLog(@"Capture session started.");
+
+  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                               block:^{
+                                 // If we successfully restarted after an unknown error,
+                                 // allow future retries on fatal errors.
+                                 _hasRetriedOnFatalError = NO;
+                               }];
+}
+
+- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
+  RTCLog(@"Capture session stopped.");
+}
+
+- (void)handleFatalError {
+  [RTCDispatcher
+      dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                    block:^{
+                      if (!_hasRetriedOnFatalError) {
+                        RTCLogWarning(@"Attempting to recover from fatal capture error.");
+                        [self handleNonFatalError];
+                        _hasRetriedOnFatalError = YES;
+                      } else {
+                        RTCLogError(@"Previous fatal error recovery failed.");
+                      }
+                    }];
+}
+
+- (void)handleNonFatalError {
+  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                               block:^{
+                                 RTCLog(@"Restarting capture session after error.");
+                                 [_captureSession startRunning];
+                               }];
+}
+
+#if TARGET_OS_IPHONE
+
+#pragma mark - UIApplication notifications
+
+- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
+  [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                               block:^{
+                                 if (!_captureSession.isRunning) {
+                                   RTCLog(@"Restarting capture session on active.");
+                                   [_captureSession startRunning];

[daniela-webrtc, 2017/03/26 18:08:57]

Not sure if this is still proper way of handling this use case as the
captureSession runs throughout the life of this object. Have you tested this
code path what happens?

[sakal, 2017/03/29 11:45:44]

This is what we used to do and seems to work fine for me. I added a check so we
don't restart the session if stopCapture has been called.

+                                 }
+                               }];
+}
+
+#endif  // TARGET_OS_IPHONE
+
+#pragma mark - Private

[daniela-webrtc, 2017/03/26 18:08:58]

This pragma mark should be way above, there are plenty of other private methods
above.

[sakal, 2017/03/29 11:45:44]

Done.

+
+- (BOOL)setupCaptureSession {
+  _captureSession = [[AVCaptureSession alloc] init];

[daniela-webrtc, 2017/03/26 18:08:58]

Add asserts (for instance if the _captureSession already exists etc) here to
make sure `setupCaptureSession` is called only once

+#if defined(WEBRTC_IOS)
+  _captureSession.usesApplicationAudioSession = NO;
+#endif
+  // Add the output.
+  AVCaptureVideoDataOutput *videoDataOutput = [self videoDataOutput];
+  if (![_captureSession canAddOutput:videoDataOutput]) {
+    RTCLogError(@"Video data output unsupported.");
+    return NO;
+  }
+  [_captureSession addOutput:videoDataOutput];
+
+  [RTCDispatcher
+      dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
+                    block:^{
+                      [self updateOrientation];

[daniela-webrtc, 2017/03/26 18:08:58]

No need to call this here yet. We'll update rotation once the capturing for
particular device starts

[sakal, 2017/03/29 11:45:44]

Done.

+#if TARGET_OS_IPHONE
+                      [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];

[daniela-webrtc, 2017/03/26 18:08:58]

Let's remove this from the dispatch block and from the setup as well.
No need to respond to orientation updates if no capture device is running.
Better add this in `startCaptureForDevice...` and then
`end​Generating​Device​Orientation​Notifications` in `stop...`

[sakal, 2017/03/29 11:45:44]

Done.

+#endif
+                      [_captureSession startRunning];
+                    }];
+  return YES;
+}
+
+- (AVCaptureVideoDataOutput *)videoDataOutput {
+  if (!_videoDataOutput) {
+    // Make the capturer output NV12. Ideally we want I420 but that's not
+    // currently supported on iPhone / iPad.
+    AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
+    videoDataOutput.videoSettings = @{
+      (NSString *)
+      // TODO(denicija): Remove this color conversion and use the original capture format directly.
+      kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
+    };
+    videoDataOutput.alwaysDiscardsLateVideoFrames = NO;
+    [videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
+    _videoDataOutput = videoDataOutput;
+  }
+  return _videoDataOutput;
+}
+
+// Called from capture session queue.
+- (void)updateOrientation {
+#if TARGET_OS_IPHONE
+  bool usingFrontCamera = _currentDevice.position == AVCaptureDevicePositionFront;

[daniela-webrtc, 2017/03/26 18:08:57]

Preferably use BOOL in ObjC code.

[sakal, 2017/03/29 11:45:44]

Done.

+  switch ([UIDevice currentDevice].orientation) {
+    case UIDeviceOrientationPortrait:
+      _rotation = RTCVideoRotation_90;
+      break;
+    case UIDeviceOrientationPortraitUpsideDown:
+      _rotation = RTCVideoRotation_270;
+      break;
+    case UIDeviceOrientationLandscapeLeft:
+      _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0;
+      break;
+    case UIDeviceOrientationLandscapeRight:
+      _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180;
+      break;
+    case UIDeviceOrientationFaceUp:
+    case UIDeviceOrientationFaceDown:
+    case UIDeviceOrientationUnknown:
+      // Ignore.
+      break;
+  }
+#endif
+}
+
+@end