Bitrate controller for VideoToolbox encoder.

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h"
 
 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
 
 #include <string>
 #include <vector>
 
 #include "libyuv/convert_from.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
+#include "webrtc/system_wrappers/include/clock.h"
 
 namespace internal {
 
 // Convenience function for creating a dictionary.
 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
                                           CFTypeRef* values,
                                           size_t size) {
   return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
                             &kCFTypeDictionaryKeyCallBacks,
                             &kCFTypeDictionaryValueCallBacks);
@@ skipping 26 matching lines @@
   OSStatus status = VTSessionSetProperty(session, key, cfNum);
   CFRelease(cfNum);
   if (status != noErr) {
     std::string key_string = CFStringToString(key);
     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
                   << " to " << value << ": " << status;
   }
 }
 
 // Convenience function for setting a VT property.
+void SetVTSessionProperty(VTSessionRef session,
+                          CFStringRef key,
+                          uint32_t value) {
+  int64_t value_64 = value;
+  CFNumberRef cfNum =
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
+  OSStatus status = VTSessionSetProperty(session, key, cfNum);
+  CFRelease(cfNum);
+  if (status != noErr) {
+    std::string key_string = CFStringToString(key);
+    LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
+                  << " to " << value << ": " << status;
+  }
+}
+
+// Convenience function for setting a VT property.
 void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
   CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
   OSStatus status = VTSessionSetProperty(session, key, cf_bool);
   if (status != noErr) {
     std::string key_string = CFStringToString(key);
     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
                   << " to " << value << ": " << status;
   }
 }
 
 // Convenience function for setting a VT property.
 void SetVTSessionProperty(VTSessionRef session,
                           CFStringRef key,
                           CFStringRef value) {
   OSStatus status = VTSessionSetProperty(session, key, value);
   if (status != noErr) {
     std::string key_string = CFStringToString(key);
     std::string val_string = CFStringToString(value);
     LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
                   << " to " << val_string << ": " << status;
   }
 }
 
 // Struct that we pass to the encoder per frame to encode. We receive it again
 // in the encoder callback.
 struct FrameEncodeParams {
-  FrameEncodeParams(webrtc::EncodedImageCallback* cb,
+  FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e,
                     const webrtc::CodecSpecificInfo* csi,
                     int32_t w,
                     int32_t h,
                     int64_t rtms,
                     uint32_t ts)
-      : callback(cb), width(w), height(h), render_time_ms(rtms), timestamp(ts) {
+      : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts) {
     if (csi) {
       codec_specific_info = *csi;
     } else {
       codec_specific_info.codecType = webrtc::kVideoCodecH264;
     }
   }
-  webrtc::EncodedImageCallback* callback;
+
+  webrtc::H264VideoToolboxEncoder* encoder;
   webrtc::CodecSpecificInfo codec_specific_info;
   int32_t width;
   int32_t height;
   int64_t render_time_ms;
   uint32_t timestamp;
 };
 
 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the
 // encoder. This performs the copy and format conversion.
 // TODO(tkchin): See if encoder will accept i420 frames and compare performance.
@@ skipping 26 matching lines @@
       dst_stride_y, dst_uv, dst_stride_uv, frame.width(), frame.height());
   CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
   if (ret) {
     LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
     return false;
   }
   return true;
 }
 
 // This is the callback function that VideoToolbox calls when encode is
-// complete.
+// complete. From inspection this happens on its own queue.
 void VTCompressionOutputCallback(void* encoder,
                                  void* params,
                                  OSStatus status,
                                  VTEncodeInfoFlags info_flags,
                                  CMSampleBufferRef sample_buffer) {
   rtc::scoped_ptr<FrameEncodeParams> encode_params(
       reinterpret_cast<FrameEncodeParams*>(params));
-  if (status != noErr) {
-    LOG(LS_ERROR) << "H264 encoding failed.";
-    return;
-  }
-  if (info_flags & kVTEncodeInfo_FrameDropped) {
-    LOG(LS_INFO) << "H264 encode dropped frame.";
-  }
-
-  bool is_keyframe = false;
-  CFArrayRef attachments =
-      CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
-  if (attachments != nullptr && CFArrayGetCount(attachments)) {
-    CFDictionaryRef attachment =
-        static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
-    is_keyframe =
-        !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
-  }
-
-  // Convert the sample buffer into a buffer suitable for RTP packetization.
-  // TODO(tkchin): Allocate buffers through a pool.
-  rtc::scoped_ptr<rtc::Buffer> buffer(new rtc::Buffer());
-  rtc::scoped_ptr<webrtc::RTPFragmentationHeader> header;
-  if (!H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
-                                        buffer.get(), header.accept())) {
-    return;
-  }
-  webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
-  frame._encodedWidth = encode_params->width;
-  frame._encodedHeight = encode_params->height;
-  frame._completeFrame = true;
-  frame._frameType =
-      is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
-  frame.capture_time_ms_ = encode_params->render_time_ms;
-  frame._timeStamp = encode_params->timestamp;
-
-  int result = encode_params->callback->Encoded(
-      frame, &(encode_params->codec_specific_info), header.get());
-  if (result != 0) {
-    LOG(LS_ERROR) << "Encoded callback failed: " << result;
-  }
+  encode_params->encoder->OnEncodedFrame(
+      status, info_flags, sample_buffer, encode_params->codec_specific_info,
+      encode_params->width, encode_params->height,
+      encode_params->render_time_ms, encode_params->timestamp);
 }
 
 }  // namespace internal
 
 namespace webrtc {
 
 H264VideoToolboxEncoder::H264VideoToolboxEncoder()
-    : callback_(nullptr), compression_session_(nullptr) {}
+    : callback_(nullptr),
+      compression_session_(nullptr),
+      bitrate_adjuster_(Clock::GetRealTimeClock()) {}
 
 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
   DestroyCompressionSession();
 }
 
 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
                                         int number_of_cores,
                                         size_t max_payload_size) {
   RTC_DCHECK(codec_settings);
   RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
   // TODO(tkchin): We may need to enforce width/height dimension restrictions
   // to match what the encoder supports.
   width_ = codec_settings->width;
   height_ = codec_settings->height;
   // We can only set average bitrate on the HW encoder.
-  bitrate_ = codec_settings->startBitrate * 1000;
+  target_bitrate_bps_ = codec_settings->startBitrate;
+  bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
+  // Prevent overcompensating for large temporary overshoots. We don't want to
+  // degrade video quality too badly.
+  bitrate_adjuster_.SetMinAdjustedBitratePct(.5);
+  // Prevent oscillations. When a lower bitrate is set on the encoder than
+  // previously set, its output seems to have a brief period of drastically
+  // reduced bitrate, so we want to avoid that. In steady state conditions,
+  // 0.95 seems to give us better overall bitrate over long periods of time.
+  bitrate_adjuster_.SetMaxAdjustedBitratePct(.95);
 
   // TODO(tkchin): Try setting payload size via
   // kVTCompressionPropertyKey_MaxH264SliceBytes.
 
   return ResetCompressionSession();
 }
 
 int H264VideoToolboxEncoder::Encode(
     const VideoFrame& input_image,
     const CodecSpecificInfo* codec_specific_info,
@@ skipping 42 matching lines @@
   CMTime presentation_time_stamp =
       CMTimeMake(input_image.render_time_ms(), 1000);
   CFDictionaryRef frame_properties = nullptr;
   if (is_keyframe_required) {
     CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
     CFTypeRef values[] = {kCFBooleanTrue};
     frame_properties = internal::CreateCFDictionary(keys, values, 1);
   }
   rtc::scoped_ptr<internal::FrameEncodeParams> encode_params;
   encode_params.reset(new internal::FrameEncodeParams(
-      callback_, codec_specific_info, width_, height_,
-      input_image.render_time_ms(), input_image.timestamp()));
+      this, codec_specific_info, width_, height_, input_image.render_time_ms(),
+      input_image.timestamp()));
+
+  // Update the bitrate if needed.
+  SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
+
   VTCompressionSessionEncodeFrame(
       compression_session_, pixel_buffer, presentation_time_stamp,
       kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
   if (frame_properties) {
     CFRelease(frame_properties);
   }
   if (pixel_buffer) {
     CVBufferRelease(pixel_buffer);
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
                                                   int64_t rtt) {
   // Encoder doesn't know anything about packet loss or rtt so just return.
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
                                       uint32_t frame_rate) {
-  bitrate_ = new_bitrate_kbit * 1000;
-  if (compression_session_) {
-    internal::SetVTSessionProperty(compression_session_,
-                                   kVTCompressionPropertyKey_AverageBitRate,
-                                   bitrate_);
-  }
+  target_bitrate_bps_ = 1000 * new_bitrate_kbit;
+  bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
+  SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
+
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::Release() {
+  // Need to reset so that the session is invalidated and won't use the
+  // callback anymore. Do not remove callback until the session is invalidated
+  // since async encoder callbacks can occur until invalidation.
+  int ret = ResetCompressionSession();
   callback_ = nullptr;
-  // Need to reset to that the session is invalidated and won't use the
-  // callback anymore.
-  return ResetCompressionSession();
+  return ret;
 }
 
 int H264VideoToolboxEncoder::ResetCompressionSession() {
   DestroyCompressionSession();
 
   // Set source image buffer attributes. These attributes will be present on
   // buffers retrieved from the encoder's pixel buffer pool.
   const size_t attributes_size = 3;
   CFTypeRef keys[attributes_size] = {
 #if defined(WEBRTC_IOS)
@@ skipping 40 matching lines @@
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 void H264VideoToolboxEncoder::ConfigureCompressionSession() {
   RTC_DCHECK(compression_session_);
   internal::SetVTSessionProperty(compression_session_,
                                  kVTCompressionPropertyKey_RealTime, true);
   internal::SetVTSessionProperty(compression_session_,
                                  kVTCompressionPropertyKey_ProfileLevel,
                                  kVTProfileLevel_H264_Baseline_AutoLevel);
-  internal::SetVTSessionProperty(
-      compression_session_, kVTCompressionPropertyKey_AverageBitRate, bitrate_);
   internal::SetVTSessionProperty(compression_session_,
                                  kVTCompressionPropertyKey_AllowFrameReordering,
                                  false);
+  SetEncoderBitrateBps(target_bitrate_bps_);
   // TODO(tkchin): Look at entropy mode and colorspace matrices.
   // TODO(tkchin): Investigate to see if there's any way to make this work.
   // May need it to interop with Android. Currently this call just fails.
   // On inspecting encoder output on iOS8, this value is set to 6.
   // internal::SetVTSessionProperty(compression_session_,
   //     kVTCompressionPropertyKey_MaxFrameDelayCount,
   //     1);
   // TODO(tkchin): See if enforcing keyframe frequency is beneficial in any
   // way.
   // internal::SetVTSessionProperty(
   //     compression_session_,
   //     kVTCompressionPropertyKey_MaxKeyFrameInterval, 240);
   // internal::SetVTSessionProperty(
   //     compression_session_,
   //     kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
 }
 
 void H264VideoToolboxEncoder::DestroyCompressionSession() {
   if (compression_session_) {
     VTCompressionSessionInvalidate(compression_session_);
     CFRelease(compression_session_);
     compression_session_ = nullptr;
   }
 }
 
 const char* H264VideoToolboxEncoder::ImplementationName() const {
   return "VideoToolbox";
 }
 
+void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) {
+  if (encoder_bitrate_bps_ != bitrate_bps) {
+    SetEncoderBitrateBps(bitrate_bps);
+  }
+}
+
+void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) {
+  if (compression_session_) {
+    internal::SetVTSessionProperty(compression_session_,
+                                   kVTCompressionPropertyKey_AverageBitRate,
+                                   bitrate_bps);
+    encoder_bitrate_bps_ = bitrate_bps;
+  }
+}
+
+void H264VideoToolboxEncoder::OnEncodedFrame(
+    OSStatus status,
+    VTEncodeInfoFlags info_flags,
+    CMSampleBufferRef sample_buffer,
+    CodecSpecificInfo codec_specific_info,
+    int32_t width,
+    int32_t height,
+    int64_t render_time_ms,
+    uint32_t timestamp) {
+  if (status != noErr) {
+    LOG(LS_ERROR) << "H264 encode failed.";
+    return;
+  }
+  if (info_flags & kVTEncodeInfo_FrameDropped) {
+    LOG(LS_INFO) << "H264 encode dropped frame.";
+  }
+
+  bool is_keyframe = false;
+  CFArrayRef attachments =
+      CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
+  if (attachments != nullptr && CFArrayGetCount(attachments)) {
+    CFDictionaryRef attachment =
+        static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
+    is_keyframe =
+        !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
+  }
+
+  // Convert the sample buffer into a buffer suitable for RTP packetization.
+  // TODO(tkchin): Allocate buffers through a pool.
+  rtc::scoped_ptr<rtc::Buffer> buffer(new rtc::Buffer());
+  rtc::scoped_ptr<webrtc::RTPFragmentationHeader> header;
+  if (!H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
+                                        buffer.get(), header.accept())) {
+    return;
+  }
+  webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
+  frame._encodedWidth = width;
+  frame._encodedHeight = height;
+  frame._completeFrame = true;
+  frame._frameType =
+      is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
+  frame.capture_time_ms_ = render_time_ms;
+  frame._timeStamp = timestamp;
+
+  int result = callback_->Encoded(frame, &codec_specific_info, header.get());
+  if (result != 0) {
+    LOG(LS_ERROR) << "Encode callback failed: " << result;
+    return;
+  }
+  bitrate_adjuster_.Update(frame._size);
+}
+
 }  // namespace webrtc
 
 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)