iOS: Optimize video scaling and cropping

webrtc/common_video/corevideo_frame_buffer.cc

Index: webrtc/common_video/corevideo_frame_buffer.cc
diff --git a/webrtc/common_video/corevideo_frame_buffer.cc b/webrtc/common_video/corevideo_frame_buffer.cc
index 3245bf5e498b17337fa6bdf5983bfecc5ebdd6c5..3c2bae26a60eaa0d3973e8650e200ab0a5246d0c 100644
--- a/webrtc/common_video/corevideo_frame_buffer.cc
+++ b/webrtc/common_video/corevideo_frame_buffer.cc
@@ -13,14 +13,35 @@
 #include "libyuv/convert.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 
 namespace webrtc {
 
+CoreVideoFrameBuffer::CoreVideoFrameBuffer(CVPixelBufferRef pixel_buffer,
+                                           int adapted_width,
+                                           int adapted_height,
+                                           int crop_width,
+                                           int crop_height,
+                                           int crop_x,
+                                           int crop_y)
+    : NativeHandleBuffer(pixel_buffer, adapted_width, adapted_height),
+      pixel_buffer_(pixel_buffer),
+      buffer_width_(CVPixelBufferGetWidth(pixel_buffer)),
+      buffer_height_(CVPixelBufferGetHeight(pixel_buffer)),
+      crop_width_(crop_width), crop_height_(crop_height),
+      // Can only crop at even pixels.
+      crop_x_(crop_x & ~1), crop_y_(crop_y & ~1) {
+  CVBufferRetain(pixel_buffer_);
+}
+
 CoreVideoFrameBuffer::CoreVideoFrameBuffer(CVPixelBufferRef pixel_buffer)
     : NativeHandleBuffer(pixel_buffer,
                          CVPixelBufferGetWidth(pixel_buffer),
                          CVPixelBufferGetHeight(pixel_buffer)),
-      pixel_buffer_(pixel_buffer) {
+      pixel_buffer_(pixel_buffer),
+      buffer_width_(width_), buffer_height_(height_),
+      crop_width_(width_), crop_height_(height_),
+      crop_x_(0), crop_y_(0) {
   CVBufferRetain(pixel_buffer_);
 }
 
@@ -30,32 +51,98 @@ CoreVideoFrameBuffer::~CoreVideoFrameBuffer() {
 
 rtc::scoped_refptr<VideoFrameBuffer>
 CoreVideoFrameBuffer::NativeToI420Buffer() {
-  RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer_) ==
-             kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
-  size_t width = CVPixelBufferGetWidthOfPlane(pixel_buffer_, 0);
-  size_t height = CVPixelBufferGetHeightOfPlane(pixel_buffer_, 0);
-  // TODO(tkchin): Use a frame buffer pool.
-  rtc::scoped_refptr<webrtc::I420Buffer> buffer =
-      new rtc::RefCountedObject<webrtc::I420Buffer>(width, height);
+  const OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer_);
+  RTC_DCHECK(pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
+             pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
+
   CVPixelBufferLockBaseAddress(pixel_buffer_, kCVPixelBufferLock_ReadOnly);
   const uint8_t* src_y = static_cast<const uint8_t*>(
       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer_, 0));
-  int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer_, 0);
+  const int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer_, 0);
   const uint8_t* src_uv = static_cast<const uint8_t*>(
       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer_, 1));
-  int src_uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer_, 1);
-  int ret = libyuv::NV12ToI420(
-      src_y, src_y_stride, src_uv, src_uv_stride,
+  const int src_uv_stride =
+      CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer_, 1);
+
+  // Crop just by modifying pointers.
+  src_y += src_y_stride * crop_y_ + crop_x_;
+  src_uv += src_uv_stride * (crop_y_ / 2) + crop_x_;
+
+  // TODO(magjed): Use a frame buffer pool.
+  NV12ToI420Scaler nv12_to_i420_scaler;
+  rtc::scoped_refptr<I420Buffer> buffer =
+      new rtc::RefCountedObject<I420Buffer>(width_, height_);
+  nv12_to_i420_scaler.NV12ToI420Scale(
+      src_y, src_y_stride,
+      src_uv, src_uv_stride,
+      crop_width_, crop_height_,
       buffer->MutableDataY(), buffer->StrideY(),
       buffer->MutableDataU(), buffer->StrideU(),
       buffer->MutableDataV(), buffer->StrideV(),
-      width, height);
+      buffer->width(), buffer->height());
+
   CVPixelBufferUnlockBaseAddress(pixel_buffer_, kCVPixelBufferLock_ReadOnly);
-  if (ret) {
-    LOG(LS_ERROR) << "Error converting NV12 to I420: " << ret;
-    return nullptr;
-  }
+
   return buffer;
 }
 
+bool CoreVideoFrameBuffer::RequiresCropping() const {
+  return crop_width_ != buffer_width_ || crop_height_ != buffer_height_;
+}
+
+bool CoreVideoFrameBuffer::CropAndScaleTo(
+    std::vector<uint8_t>* tmp_buffer,

[tkchin_webrtc, 2016/10/19 00:48:32]

is buffer passed in so that it can be re-used across multiple crop/scale ops?

[magjed_webrtc, 2016/10/19 13:19:33]

Yes exactly.

+    CVPixelBufferRef output_pixel_buffer) const {
+  // Prepare output pointers.
+  RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(output_pixel_buffer),
+                kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
+  CVReturn cvRet = CVPixelBufferLockBaseAddress(output_pixel_buffer, 0);

[tkchin_webrtc, 2016/10/19 00:48:32]

nit: cv_ret

[magjed_webrtc, 2016/10/19 13:19:34]

Done.

+  if (cvRet != kCVReturnSuccess) {
+    LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
+    return false;
+  }
+  const int dst_width = CVPixelBufferGetWidth(output_pixel_buffer);
+  const int dst_height = CVPixelBufferGetHeight(output_pixel_buffer);
+  uint8_t* dst_y = reinterpret_cast<uint8_t*>(
+      CVPixelBufferGetBaseAddressOfPlane(output_pixel_buffer, 0));
+  const int dst_y_stride =
+      CVPixelBufferGetBytesPerRowOfPlane(output_pixel_buffer, 0);
+  uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
+      CVPixelBufferGetBaseAddressOfPlane(output_pixel_buffer, 1));
+  const int dst_uv_stride =
+      CVPixelBufferGetBytesPerRowOfPlane(output_pixel_buffer, 1);
+
+  // Prepare source pointers.
+  const OSType src_pixel_format =
+      CVPixelBufferGetPixelFormatType(pixel_buffer_);
+  RTC_DCHECK(
+      src_pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ||
+      src_pixel_format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
+  CVPixelBufferLockBaseAddress(pixel_buffer_, kCVPixelBufferLock_ReadOnly);
+  const uint8_t* src_y = static_cast<const uint8_t*>(
+      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer_, 0));
+  const int src_y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer_, 0);
+  const uint8_t* src_uv = static_cast<const uint8_t*>(
+      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer_, 1));
+  const int src_uv_stride =
+      CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer_, 1);
+
+  // Crop just by modifying pointers.
+  src_y += src_y_stride * crop_y_ + crop_x_;
+  src_uv += src_uv_stride * (crop_y_ / 2) + crop_x_;
+
+  NV12Scale(tmp_buffer,
+            src_y, src_y_stride,
+            src_uv, src_uv_stride,
+            crop_width_, crop_height_,
+            dst_y, dst_y_stride,
+            dst_uv, dst_uv_stride,
+            dst_width, dst_height);
+
+  CVPixelBufferUnlockBaseAddress(pixel_buffer_, kCVPixelBufferLock_ReadOnly);

[tkchin_webrtc, 2016/10/19 00:48:32]

not locked as readonly

[magjed_webrtc, 2016/10/19 13:19:34]

Yes it is?

[tkchin_webrtc, 2016/10/19 22:50:17]

ah, misread. You are correct.

+  CVPixelBufferUnlockBaseAddress(output_pixel_buffer, 0);
+
+  return true;
+}
+
 }  // namespace webrtc