iOS H264VideoToolBoxEncoder: Stop scaling native CVPixelBuffers

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
@@ skipping 137 matching lines @@
   webrtc::VideoRotation rotation;
 };
 
 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the
 // encoder. This performs the copy and format conversion.
 // TODO(tkchin): See if encoder will accept i420 frames and compare performance.
 bool CopyVideoFrameToPixelBuffer(
     const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& frame,
     CVPixelBufferRef pixel_buffer) {
   RTC_DCHECK(pixel_buffer);
-  RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) ==
-             kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
-  RTC_DCHECK(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0) ==
-             static_cast<size_t>(frame->height()));
-  RTC_DCHECK(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0) ==
-             static_cast<size_t>(frame->width()));
+  RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer),
+                kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
+  RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0),
+                static_cast<size_t>(frame->height()));
+  RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0),
+                static_cast<size_t>(frame->width()));
 
   CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
   if (cvRet != kCVReturnSuccess) {
     LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
     return false;
   }
   uint8_t* dst_y = reinterpret_cast<uint8_t*>(
       CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
   int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
   uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
@@ skipping 37 matching lines @@
 // .5 is set as a mininum to prevent overcompensating for large temporary
 // overshoots. We don't want to degrade video quality too badly.
 // .95 is set to prevent oscillations. When a lower bitrate is set on the
 // encoder than previously set, its output seems to have a brief period of
 // drastically reduced bitrate, so we want to avoid that. In steady state
 // conditions, 0.95 seems to give us better overall bitrate over long periods
 // of time.
 H264VideoToolboxEncoder::H264VideoToolboxEncoder()
     : callback_(nullptr),
       compression_session_(nullptr),
-      bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95),
-      enable_scaling_(true) {
-#if defined(WEBRTC_IOS)
-  if ([UIDevice deviceType] == RTCDeviceTypeIPhone4S) {
-    enable_scaling_ = false;
-  }
-#endif
+      bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95) {
 }
 
 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
   DestroyCompressionSession();
 }
 
 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
                                         int number_of_cores,
                                         size_t max_payload_size) {
   RTC_DCHECK(codec_settings);
   RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
-  width_ = codec_settings->width;
-  height_ = codec_settings->height;
   {
     rtc::CritScope lock(&quality_scaler_crit_);
     quality_scaler_.Init(QualityScaler::kLowH264QpThreshold,
                          QualityScaler::kBadH264QpThreshold,
                          codec_settings->startBitrate, codec_settings->width,
                          codec_settings->height, codec_settings->maxFramerate);
     QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
     // TODO(tkchin): We may need to enforce width/height dimension restrictions
     // to match what the encoder supports.
-    if (enable_scaling_) {
-      width_ = res.width;
-      height_ = res.height;
-    }
+    width_ = res.width;
+    height_ = res.height;
   }
   // We can only set average bitrate on the HW encoder.
   target_bitrate_bps_ = codec_settings->startBitrate;
   bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
 
   // TODO(tkchin): Try setting payload size via
   // kVTCompressionPropertyKey_MaxH264SliceBytes.
 
   return ResetCompressionSession();
 }
 
-rtc::scoped_refptr<VideoFrameBuffer>
-H264VideoToolboxEncoder::GetScaledBufferOnEncode(
-    const rtc::scoped_refptr<VideoFrameBuffer>& frame) {
-  rtc::CritScope lock(&quality_scaler_crit_);
-  quality_scaler_.OnEncodeFrame(frame->width(), frame->height());
-  if (!frame->native_handle())
-    return quality_scaler_.GetScaledBuffer(frame);
-
-  // Handle native (CVImageRef) scaling.
-  const QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
-  if (!enable_scaling_ ||
-      (res.width == frame->width() && res.height == frame->height()))
-    return frame;
-  // TODO(magjed): Implement efficient CVImageRef -> CVImageRef scaling instead
-  // of doing it via I420.
-  return quality_scaler_.GetScaledBuffer(frame->NativeToI420Buffer());
-}
-
 int H264VideoToolboxEncoder::Encode(
     const VideoFrame& frame,
     const CodecSpecificInfo* codec_specific_info,
     const std::vector<FrameType>* frame_types) {
   RTC_DCHECK(!frame.IsZeroSize());
   if (!callback_ || !compression_session_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 #if defined(WEBRTC_IOS)
   if (!RTCIsUIApplicationActive()) {
     // Ignore all encode requests when app isn't active. In this state, the
     // hardware encoder has been invalidated by the OS.
     return WEBRTC_VIDEO_CODEC_OK;
   }
 #endif
   bool is_keyframe_required = false;
-  rtc::scoped_refptr<VideoFrameBuffer> input_image(
-      GetScaledBufferOnEncode(frame.video_frame_buffer()));
 
-  if (input_image->width() != width_ || input_image->height() != height_) {
-    width_ = input_image->width();
-    height_ = input_image->height();
+  quality_scaler_.OnEncodeFrame(frame.width(), frame.height());
+  const QualityScaler::Resolution scaled_res =
+      quality_scaler_.GetScaledResolution();
+
+  if (scaled_res.width != width_ || scaled_res.height != height_) {
+    width_ = scaled_res.width;
+    height_ = scaled_res.height;
     int ret = ResetCompressionSession();
     if (ret < 0)
       return ret;
   }
 
   // Get a pixel buffer from the pool and copy frame data over.
   CVPixelBufferPoolRef pixel_buffer_pool =
       VTCompressionSessionGetPixelBufferPool(compression_session_);
 #if defined(WEBRTC_IOS)
   if (!pixel_buffer_pool) {
     // Kind of a hack. On backgrounding, the compression session seems to get
     // invalidated, which causes this pool call to fail when the application
     // is foregrounded and frames are being sent for encoding again.
     // Resetting the session when this happens fixes the issue.
     // In addition we request a keyframe so video can recover quickly.
     ResetCompressionSession();
     pixel_buffer_pool =
         VTCompressionSessionGetPixelBufferPool(compression_session_);
     is_keyframe_required = true;
     LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
   }
 #endif
 
-  CVPixelBufferRef pixel_buffer =
-      static_cast<CVPixelBufferRef>(input_image->native_handle());
+  CVPixelBufferRef pixel_buffer = static_cast<CVPixelBufferRef>(
+      frame.video_frame_buffer()->native_handle());
   if (pixel_buffer) {
+    // This pixel buffer might have a higher resolution than what the
+    // compression session is configured to. The compression session can handle
+    // that and will output encoded frames in the configured resolution
+    // regardless of the input pixel buffer resolution.
     CVBufferRetain(pixel_buffer);
     pixel_buffer_pool = nullptr;
   } else {
     if (!pixel_buffer_pool) {
       LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(
         nullptr, pixel_buffer_pool, &pixel_buffer);
     if (ret != kCVReturnSuccess) {
       LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
       // We probably want to drop frames here, since failure probably means
       // that the pool is empty.
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     RTC_DCHECK(pixel_buffer);
-    if (!internal::CopyVideoFrameToPixelBuffer(input_image, pixel_buffer)) {
+    // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer
+    // conversion once libyuv::MergeUVPlanes is available.
+    rtc::scoped_refptr<VideoFrameBuffer> scaled_i420_buffer =
+        quality_scaler_.GetScaledBuffer(frame.video_frame_buffer());
+    if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer,
+                                               pixel_buffer)) {
       LOG(LS_ERROR) << "Failed to copy frame data.";
       CVBufferRelease(pixel_buffer);
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
 
   // Check if we need a keyframe.
   if (!is_keyframe_required && frame_types) {
     for (auto frame_type : *frame_types) {
       if (frame_type == kVideoFrameKey) {
@@ skipping 286 matching lines @@
   if (result != 0) {
     LOG(LS_ERROR) << "Encode callback failed: " << result;
     return;
   }
   bitrate_adjuster_.Update(frame._size);
 }
 
 }  // namespace webrtc
 
 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)