Add support for more resolutions on iOS/macOS

webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "avfoundationvideocapturer.h"
 
 #import <AVFoundation/AVFoundation.h>
 #import <Foundation/Foundation.h>
 #if TARGET_OS_IPHONE
 #import <UIKit/UIKit.h>
 #endif
 
 #import "RTCDispatcher+Private.h"
 #import "WebRTC/RTCLogging.h"
 #if TARGET_OS_IPHONE
 #import "WebRTC/UIDevice+RTCDevice.h"
 #endif
 
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/common_video/include/corevideo_frame_buffer.h"
 
-// TODO(tkchin): support other formats.
-static NSString *const kDefaultPreset = AVCaptureSessionPreset640x480;
-static NSString *const kIPhone4SPreset = AVCaptureSessionPreset352x288;
-static cricket::VideoFormat const kDefaultFormat =
-    cricket::VideoFormat(640,
-                         480,
-                         cricket::VideoFormat::FpsToInterval(30),
-                         cricket::FOURCC_NV12);
-// iPhone4S is too slow to handle 30fps.
-static cricket::VideoFormat const kIPhone4SFormat =
-    cricket::VideoFormat(352,
-                         288,
-                         cricket::VideoFormat::FpsToInterval(15),
-                         cricket::FOURCC_NV12);
+struct AVCaptureSessionPresetResolution {
+  NSString *sessionPreset;
+  int width;
+  int height;
+};
+
+#if TARGET_OS_IPHONE
+static const AVCaptureSessionPresetResolution kAvailablePresets[] = {
+  { AVCaptureSessionPreset352x288, 352, 288},
+  { AVCaptureSessionPreset640x480, 640, 480},
+  { AVCaptureSessionPreset1280x720, 1280, 720},
+  { AVCaptureSessionPreset1920x1080, 1920, 1080},
+};
+#else // macOS
+static const AVCaptureSessionPresetResolution kAvailablePresets[] = {
+  { AVCaptureSessionPreset320x240, 320, 240},
+  { AVCaptureSessionPreset352x288, 352, 288},
+  { AVCaptureSessionPreset640x480, 640, 480},
+  { AVCaptureSessionPreset960x540, 960, 540},
+  { AVCaptureSessionPreset1280x720, 1280, 720},
+};
+#endif
+
+// Mapping from cricket::VideoFormat to AVCaptureSession presets.
+static NSString *GetSessionPresetForVideoFormat(
+  const cricket::VideoFormat& format) {
+  for (const auto preset : kAvailablePresets) {
+    // Check both orientations
+    if ((format.width == preset.width && format.height == preset.height) ||
+        (format.width == preset.height && format.height == preset.width)) {
+      return preset.sessionPreset;
+    }
+  }
+  // If no matching preset is found, use a default one.
+  return AVCaptureSessionPreset640x480;
+}
 
 // This class used to capture frames using AVFoundation APIs on iOS. It is meant
 // to be owned by an instance of AVFoundationVideoCapturer. The reason for this
 // because other webrtc objects own cricket::VideoCapturer, which is not
 // ref counted. To prevent bad behavior we do not expose this class directly.
 @interface RTCAVFoundationVideoCapturerInternal : NSObject
     <AVCaptureVideoDataOutputSampleBufferDelegate>
 
 @property(nonatomic, readonly) AVCaptureSession *captureSession;
 @property(nonatomic, readonly) dispatch_queue_t frameQueue;
@@ skipping 300 matching lines @@
 #pragma mark - Private
 
 - (BOOL)setupCaptureSession {
   AVCaptureSession *captureSession = [[AVCaptureSession alloc] init];
 #if defined(__IPHONE_7_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_7_0
   NSString *version = [[UIDevice currentDevice] systemVersion];
   if ([version integerValue] >= 7) {
     captureSession.usesApplicationAudioSession = NO;
   }
 #endif
-  NSString *preset = kDefaultPreset;
-#if TARGET_OS_IPHONE
-  if ([UIDevice deviceType] == RTCDeviceTypeIPhone4S) {
-    preset = kIPhone4SPreset;
-  }
-#endif
-  if (![captureSession canSetSessionPreset:preset]) {
-    RTCLogError(@"Session preset unsupported.");
-    return NO;
-  }
-  captureSession.sessionPreset = preset;
 
   // Add the output.
   AVCaptureVideoDataOutput *videoDataOutput = [self videoDataOutput];
   if (![captureSession canAddOutput:videoDataOutput]) {
     RTCLogError(@"Video data output unsupported.");
     return NO;
   }
   [captureSession addOutput:videoDataOutput];
 
   // Get the front and back cameras. If there isn't a front camera
   // give up.
   AVCaptureDeviceInput *frontCameraInput = [self frontCameraInput];
   AVCaptureDeviceInput *backCameraInput = [self backCameraInput];
   if (!frontCameraInput) {
     RTCLogError(@"No front camera for capture session.");
     return NO;
   }
 
   // Add the inputs.
   if (![captureSession canAddInput:frontCameraInput] ||
       (backCameraInput && ![captureSession canAddInput:backCameraInput])) {
     RTCLogError(@"Session does not support capture inputs.");
     return NO;
   }
   AVCaptureDeviceInput *input = self.useBackCamera ?
       backCameraInput : frontCameraInput;
   [captureSession addInput:input];
-#if TARGET_OS_IPHONE
-  if ([UIDevice deviceType] == RTCDeviceTypeIPhone4S) {
-    [self setMinFrameDuration:CMTimeMake(1, 15) forDevice:input.device];
-  }
-#endif
+
   _captureSession = captureSession;
   return YES;
 }
 
 - (AVCaptureVideoDataOutput *)videoDataOutput {
   if (!_videoDataOutput) {
     // Make the capturer output NV12. Ideally we want I420 but that's not
     // currently supported on iPhone / iPad.
     AVCaptureVideoDataOutput *videoDataOutput =
         [[AVCaptureVideoDataOutput alloc] init];
@@ skipping 62 matching lines @@
     if (!backCameraInput) {
       RTCLogError(@"Failed to create front camera input: %@",
                   error.localizedDescription);
       return nil;
     }
     _backCameraInput = backCameraInput;
   }
   return _backCameraInput;
 }
 
-- (void)setMinFrameDuration:(CMTime)minFrameDuration
-                  forDevice:(AVCaptureDevice *)device {
+- (void)setMinFrameDuration:(CMTime)minFrameDuration {
   NSError *error = nil;
-  if (![device lockForConfiguration:&error]) {
-    RTCLogError(@"Failed to lock device for configuration. Error: %@", error.localizedDescription);
-    return;
+  for (AVCaptureDevice *device in
+       [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) {

[tkchin_webrtc, 2016/08/18 15:17:24]

Seems odd to do this for every capture device. Could be laptop camera + other
things. I guess it's unlikely that user has a lot of video devices though so it
should be fine as long as this doesn't end up blocking the thread.

+    if (![device lockForConfiguration:&error]) {
+        RTCLogError(@"Failed to lock device for configuration. Error: %@",
+                    error.localizedDescription);
+        return;

[magjed_webrtc, 2016/08/18 09:27:37]

nit: continue might be more appropriate than return here.

+    }
+    device.activeVideoMinFrameDuration = minFrameDuration;
+    [device unlockForConfiguration];
   }
-  device.activeVideoMinFrameDuration = minFrameDuration;
-  [device unlockForConfiguration];
 }
 
 // Called from capture session queue.
 - (void)updateOrientation {
   AVCaptureConnection *connection =
       [_videoDataOutput connectionWithMediaType:AVMediaTypeVideo];
   if (!connection.supportsVideoOrientation) {
     // TODO(tkchin): set rotation bit on frames.
     return;
   }
@@ skipping 37 matching lines @@
     }
     if (oldInput) {
       // Ok to remove this even if it's not attached. Will be no-op.
       [_captureSession removeInput:oldInput];
     }
     if (newInput) {
       [_captureSession addInput:newInput];
     }
     [self updateOrientation];
     [_captureSession commitConfiguration];
-#if TARGET_OS_IPHONE
-    if ([UIDevice deviceType] == RTCDeviceTypeIPhone4S) {
-      [self setMinFrameDuration:CMTimeMake(1, 15) forDevice:newInput.device];
-    }
-#endif
   }];
 }
 
 @end
 
 namespace webrtc {
 
 enum AVFoundationVideoCapturerMessageType : uint32_t {
   kMessageTypeFrame,
 };
 
 struct AVFoundationFrame {
   AVFoundationFrame(CVImageBufferRef buffer, int64_t time)
     : image_buffer(buffer), capture_time(time) {}
   CVImageBufferRef image_buffer;
   int64_t capture_time;
 };
 
 AVFoundationVideoCapturer::AVFoundationVideoCapturer()
     : _capturer(nil), _startThread(nullptr) {
-  // Set our supported formats. This matches preset.
+  // Set our supported formats. This matches kAvailablePresets.
+  _capturer =
+      [[RTCAVFoundationVideoCapturerInternal alloc] initWithCapturer:this];
+
   std::vector<cricket::VideoFormat> supported_formats;
+  int framerate = 30;
+
 #if TARGET_OS_IPHONE
   if ([UIDevice deviceType] == RTCDeviceTypeIPhone4S) {
-    supported_formats.push_back(cricket::VideoFormat(kIPhone4SFormat));
     set_enable_video_adapter(false);
-  } else {
-    supported_formats.push_back(cricket::VideoFormat(kDefaultFormat));
+    framerate = 15;
   }
-#else
-  supported_formats.push_back(cricket::VideoFormat(kDefaultFormat));
 #endif
+
+  for (const auto preset : kAvailablePresets) {
+    if ([_capturer.captureSession canSetSessionPreset:preset.sessionPreset]) {
+      const auto format = cricket::VideoFormat(
+        preset.width,
+        preset.height,
+        cricket::VideoFormat::FpsToInterval(framerate),
+        cricket::FOURCC_NV12);
+      supported_formats.push_back(format);
+    }
+  }
+
   SetSupportedFormats(supported_formats);
-  _capturer =
-      [[RTCAVFoundationVideoCapturerInternal alloc] initWithCapturer:this];
 }
 
 AVFoundationVideoCapturer::~AVFoundationVideoCapturer() {
   _capturer = nil;
 }
 
 cricket::CaptureState AVFoundationVideoCapturer::Start(
     const cricket::VideoFormat& format) {
   if (!_capturer) {
     LOG(LS_ERROR) << "Failed to create AVFoundation capturer.";
     return cricket::CaptureState::CS_FAILED;
   }
   if (_capturer.isRunning) {
     LOG(LS_ERROR) << "The capturer is already running.";
     return cricket::CaptureState::CS_FAILED;
   }
-  if (format != kDefaultFormat && format != kIPhone4SFormat) {
-    LOG(LS_ERROR) << "Unsupported format provided.";
+
+  NSString *desiredPreset = GetSessionPresetForVideoFormat(format);
+  RTC_DCHECK(desiredPreset);
+
+  [_capturer.captureSession beginConfiguration];
+  if (![_capturer.captureSession canSetSessionPreset:desiredPreset]) {
+    LOG(LS_ERROR) << "Unsupported video format.";
+    [_capturer.captureSession commitConfiguration];
     return cricket::CaptureState::CS_FAILED;
   }
+  _capturer.captureSession.sessionPreset = desiredPreset;
+  [_capturer.captureSession commitConfiguration];
 
   // Keep track of which thread capture started on. This is the thread that
   // frames need to be sent to.
   RTC_DCHECK(!_startThread);
   _startThread = rtc::Thread::Current();
 
   SetCaptureFormat(&format);
   // This isn't super accurate because it takes a while for the AVCaptureSession
   // to spin up, and this call returns async.
   // TODO(tkchin): make this better.
   [_capturer start];
   SetCaptureState(cricket::CaptureState::CS_RUNNING);
 
+  // Adjust the framerate for all capture devices.
+  const auto fps = cricket::VideoFormat::IntervalToFps(format.interval);

[magjed_webrtc, 2016/08/18 09:27:37]

nit: use int instead of auto here. We typically only use auto when the type is
long and cluttered or clear from the context.
https://google.github.io/styleguide/cppguide.html#auto

+  [_capturer setMinFrameDuration:CMTimeMake(1, fps)];
+
   return cricket::CaptureState::CS_STARTING;
 }
 
 void AVFoundationVideoCapturer::Stop() {
   [_capturer stop];
   SetCaptureFormat(NULL);
   _startThread = nullptr;
 }
 
 bool AVFoundationVideoCapturer::IsRunning() {
@@ skipping 88 matching lines @@
   }
 
   OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0,
                                     translated_camera_time_us, 0),
           captured_width, captured_height);
 
   CVBufferRelease(image_buffer);
 }
 
 }  // namespace webrtc