Integrate QualityScaler into VideoToolboxEncoder.

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
@@ skipping 207 matching lines @@
 
 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
   DestroyCompressionSession();
 }
 
 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
                                         int number_of_cores,
                                         size_t max_payload_size) {
   RTC_DCHECK(codec_settings);
   RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
-  // TODO(tkchin): We may need to enforce width/height dimension restrictions
-  // to match what the encoder supports.
-  width_ = codec_settings->width;
-  height_ = codec_settings->height;
+  {
+    rtc::CritScope lock(&quality_scaler_crit_);
+    quality_scaler_.Init(QualityScaler::kLowH264QpThreshold,
+                         QualityScaler::kBadH264QpThreshold,
+                         codec_settings->startBitrate, codec_settings->width,
+                         codec_settings->height, codec_settings->maxFramerate);
+    QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
+    // TODO(tkchin): We may need to enforce width/height dimension restrictions
+    // to match what the encoder supports.
+    width_ = res.width;
+    height_ = res.height;
+  }
   // We can only set average bitrate on the HW encoder.
   target_bitrate_bps_ = codec_settings->startBitrate;
   bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
 
   // TODO(tkchin): Try setting payload size via
   // kVTCompressionPropertyKey_MaxH264SliceBytes.
 
   return ResetCompressionSession();
 }
 
+const VideoFrame& H264VideoToolboxEncoder::GetScaledFrameOnEncode(
+    const VideoFrame& frame) {
+  rtc::CritScope lock(&quality_scaler_crit_);
+  quality_scaler_.OnEncodeFrame(frame);
+  return quality_scaler_.GetScaledFrame(frame);
+}
+
 int H264VideoToolboxEncoder::Encode(
-    const VideoFrame& input_image,
+    const VideoFrame& frame,
     const CodecSpecificInfo* codec_specific_info,
     const std::vector<FrameType>* frame_types) {
-  if (input_image.IsZeroSize()) {
-    // It's possible to get zero sizes as a signal to produce keyframes (this
-    // happens for internal sources). But this shouldn't happen in
-    // webrtcvideoengine2.
-    RTC_NOTREACHED();
-    return WEBRTC_VIDEO_CODEC_OK;
-  }
+  RTC_DCHECK(frame.IsZeroSize());

[tkchin_webrtc, 2016/05/23 21:26:56]

why are frames expected to be zero size?

[pbos-webrtc, 2016/05/23 21:36:25]

Why indeed? :| Inverted it.

   if (!callback_ || !compression_session_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 #if defined(WEBRTC_IOS)
   if (!RTCIsUIApplicationActive()) {
     // Ignore all encode requests when app isn't active. In this state, the
     // hardware encoder has been invalidated by the OS.
     return WEBRTC_VIDEO_CODEC_OK;
   }
 #endif
   bool is_keyframe_required = false;
+  const VideoFrame& input_image = GetScaledFrameOnEncode(frame);
+
+  if (input_image.width() != width_ || input_image.height() != height_) {
+    width_ = input_image.width();
+    height_ = input_image.height();
+    int ret = ResetCompressionSession();
+    if (ret < 0)
+      return ret;
+  }
+
   // Get a pixel buffer from the pool and copy frame data over.
   CVPixelBufferPoolRef pixel_buffer_pool =
       VTCompressionSessionGetPixelBufferPool(compression_session_);
 #if defined(WEBRTC_IOS)
   if (!pixel_buffer_pool) {
     // Kind of a hack. On backgrounding, the compression session seems to get
     // invalidated, which causes this pool call to fail when the application
     // is foregrounded and frames are being sent for encoding again.
     // Resetting the session when this happens fixes the issue.
     // In addition we request a keyframe so video can recover quickly.
@@ skipping 64 matching lines @@
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+void H264VideoToolboxEncoder::OnDroppedFrame() {
+  rtc::CritScope lock(&quality_scaler_crit_);
+  quality_scaler_.ReportDroppedFrame();
+}
+
 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
                                                   int64_t rtt) {
   // Encoder doesn't know anything about packet loss or rtt so just return.
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
                                       uint32_t frame_rate) {
   target_bitrate_bps_ = 1000 * new_bitrate_kbit;
   bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
   SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
 
+  rtc::CritScope lock(&quality_scaler_crit_);
+  quality_scaler_.ReportFramerate(frame_rate);
+
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::Release() {
   // Need to reset so that the session is invalidated and won't use the
   // callback anymore. Do not remove callback until the session is invalidated
   // since async encoder callbacks can occur until invalidation.
   int ret = ResetCompressionSession();
   callback_ = nullptr;
   return ret;
@@ skipping 114 matching lines @@
     int32_t height,
     int64_t render_time_ms,
     uint32_t timestamp,
     VideoRotation rotation) {
   if (status != noErr) {
     LOG(LS_ERROR) << "H264 encode failed.";
     return;
   }
   if (info_flags & kVTEncodeInfo_FrameDropped) {
     LOG(LS_INFO) << "H264 encode dropped frame.";
+    rtc::CritScope lock(&quality_scaler_crit_);
+    quality_scaler_.ReportDroppedFrame();
+    return;
   }
 
   bool is_keyframe = false;
   CFArrayRef attachments =
       CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
   if (attachments != nullptr && CFArrayGetCount(attachments)) {
     CFDictionaryRef attachment =
         static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
     is_keyframe =
         !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
@@ skipping 15 matching lines @@
   webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
   frame._encodedWidth = width;
   frame._encodedHeight = height;
   frame._completeFrame = true;
   frame._frameType =
       is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
   frame.capture_time_ms_ = render_time_ms;
   frame._timeStamp = timestamp;
   frame.rotation_ = rotation;
 
+  h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
+  int qp;
+  if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
+    rtc::CritScope lock(&quality_scaler_crit_);
+    quality_scaler_.ReportQP(qp);
+  }
+
   int result = callback_->Encoded(frame, &codec_specific_info, header.get());
   if (result != 0) {
     LOG(LS_ERROR) << "Encode callback failed: " << result;
     return;
   }
   bitrate_adjuster_.Update(frame._size);
 }
 
 }  // namespace webrtc
 
 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)