iOS: Optimize video scaling and cropping

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h"
 
 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
 
 #include <memory>
 #include <string>
 #include <vector>
 
 #if defined(WEBRTC_IOS)
 #import "WebRTC/UIDevice+RTCDevice.h"
 #include "RTCUIApplication.h"
 #endif
 #include "libyuv/convert_from.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/common_video/include/corevideo_frame_buffer.h"
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
 #include "webrtc/system_wrappers/include/clock.h"
 
 namespace internal {
 
 // The ratio between kVTCompressionPropertyKey_DataRateLimits and
 // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
 // than the average bit rate to avoid undershooting the target.
 const float kLimitToAverageBitRateFactor = 1.5f;
 // These thresholds deviate from the default h264 QP thresholds, as they
@@ skipping 148 matching lines @@
       dst_y, dst_stride_y, dst_uv, dst_stride_uv,
       frame->width(), frame->height());
   CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
   if (ret) {
     LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
     return false;
   }
   return true;
 }
 
+CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
+  if (!pixel_buffer_pool) {
+    LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
+    return nil;

[tkchin_webrtc, 2016/10/19 00:48:32]

nullptr

[magjed_webrtc, 2016/10/19 13:19:34]

Done.

+  }
+  CVPixelBufferRef pixel_buffer;
+  CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
+                                                    &pixel_buffer);
+  if (ret != kCVReturnSuccess) {
+    LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
+    // We probably want to drop frames here, since failure probably means
+    // that the pool is empty.
+    return nil;

[tkchin_webrtc, 2016/10/19 00:48:32]

CVPixelBufferRef isn't an object, so return nullptr instead.

[magjed_webrtc, 2016/10/19 13:19:34]

Done.

+  }
+  return pixel_buffer;
+}
+
 // This is the callback function that VideoToolbox calls when encode is
 // complete. From inspection this happens on its own queue.
 void VTCompressionOutputCallback(void* encoder,
                                  void* params,
                                  OSStatus status,
                                  VTEncodeInfoFlags info_flags,
                                  CMSampleBufferRef sample_buffer) {
   std::unique_ptr<FrameEncodeParams> encode_params(
       reinterpret_cast<FrameEncodeParams*>(params));
   encode_params->encoder->OnEncodedFrame(
@@ skipping 94 matching lines @@
     pixel_buffer_pool =
         VTCompressionSessionGetPixelBufferPool(compression_session_);
     is_keyframe_required = true;
     LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
   }
 #endif
 
   CVPixelBufferRef pixel_buffer = static_cast<CVPixelBufferRef>(
       frame.video_frame_buffer()->native_handle());
   if (pixel_buffer) {
-    // This pixel buffer might have a higher resolution than what the
-    // compression session is configured to. The compression session can handle
-    // that and will output encoded frames in the configured resolution
-    // regardless of the input pixel buffer resolution.
-    CVBufferRetain(pixel_buffer);
-    pixel_buffer_pool = nullptr;
+    // Native frame.
+    rtc::scoped_refptr<CoreVideoFrameBuffer> core_video_frame_buffer(
+        static_cast<CoreVideoFrameBuffer*>(frame.video_frame_buffer().get()));
+    if (!core_video_frame_buffer->RequiresCropping()) {
+      // This pixel buffer might have a higher resolution than what the
+      // compression session is configured to. The compression session can
+      // handle that and will output encoded frames in the configured
+      // resolution regardless of the input pixel buffer resolution.
+      CVBufferRetain(pixel_buffer);
+    } else {
+      // Cropping required, we need to crop and scale to a new pixel buffer.
+      pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
+      if (!pixel_buffer) {
+        return WEBRTC_VIDEO_CODEC_ERROR;
+      }
+      if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_,
+                                                   pixel_buffer)) {
+        return WEBRTC_VIDEO_CODEC_ERROR;
+      }
+    }
   } else {
-    if (!pixel_buffer_pool) {
-      LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
+    pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
+    if (!pixel_buffer) {
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
-    CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(
-        nullptr, pixel_buffer_pool, &pixel_buffer);
-    if (ret != kCVReturnSuccess) {
-      LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
-      // We probably want to drop frames here, since failure probably means
-      // that the pool is empty.
-      return WEBRTC_VIDEO_CODEC_ERROR;
-    }
-    RTC_DCHECK(pixel_buffer);
     // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer
     // conversion once libyuv::MergeUVPlanes is available.
     rtc::scoped_refptr<VideoFrameBuffer> scaled_i420_buffer =
         quality_scaler_.GetScaledBuffer(frame.video_frame_buffer());
     if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer,
                                                pixel_buffer)) {
       LOG(LS_ERROR) << "Failed to copy frame data.";
       CVBufferRelease(pixel_buffer);
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
@@ skipping 292 matching lines @@
   if (result != 0) {
     LOG(LS_ERROR) << "Encode callback failed: " << result;
     return;
   }
   bitrate_adjuster_.Update(frame._size);
 }
 
 }  // namespace webrtc
 
 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)