H.264 video codec support using OpenH264/FFmpeg

webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc

+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h"
+
+// OpenH264
+#include "codec_api.h"
+#include "codec_app_def.h"
+#include "codec_def.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+
+namespace webrtc {
+
+namespace {
+const bool kOpenH264EncoderDetailedLogging = false;
+}  // namespace
+
+static VideoFrameType EVideoFrameType_to_VideoFrameType(
+    EVideoFrameType type) {
+  switch (type) {
+    case videoFrameTypeInvalid:
+      return kSkipFrame;
+    case videoFrameTypeSkip:
+      return kDeltaFrame;
+    case videoFrameTypeIDR:
+      return kKeyFrame;
+    case videoFrameTypeI:
+    case videoFrameTypeP:
+    case videoFrameTypeIPMixed:
+      return kDeltaFrame;
+    default:
+      LOG(LS_WARNING) << "Unknown EVideoFrameType: " << type;
+      return kDeltaFrame;
+  }
+}
+// Helper method used by H264EncoderImpl::Encode.
+// Copies the encoded bytes from |info| to |encoded_image| and updates the
+// fragmentation information of |frag_header|. The |encoded_image->_buffer| may
+// be deleted and reallocated if a bigger buffer is required.
+//   After OpenH264 encoding, the encoded bytes are stored in |info| spread out
+// over a number of layers and "NAL units". Each NAL unit is a fragment starting
+// with the four-byte start code {0,0,0,1}. All of this data (including the
+// start codes) is copied to the |encoded_image->_buffer| and the |frag_header|
+// is updated to point to each fragment, with offsets and lengths set as to
+// exclude the start codes.
+static void RtpFragmentize(EncodedImage* encoded_image,
+                           rtc::scoped_ptr<uint8_t>* encoded_image_buffer,
+                           const VideoFrame& frame,
+                           SFrameBSInfo* info,
+                           RTPFragmentationHeader* frag_header) {
+  // Calculate minimum buffer size required to hold encoded data.
+  size_t required_size = 0;
+  size_t fragments_count = 0;
+  for (int iLayer = 0; iLayer < info->iLayerNum; ++iLayer) {
+    const SLayerBSInfo& layerInfo = info->sLayerInfo[iLayer];
+    for (int iNal = 0; iNal < layerInfo.iNalCount; ++iNal) {
+      required_size += layerInfo.pNalLengthInByte[iNal];
+      ++fragments_count;
+    }
+  }
+  if (encoded_image->_size < required_size) {
+    // Increase buffer size. Allocate enough to hold an unencoded image, this
+    // should be more than enough to hold any encoded data of future frames of
+    // the same size (avoiding possible future reallocation due to variations in
+    // required size).
+    encoded_image->_size = CalcBufferSize(
+        VideoType::kI420, frame.width(), frame.height());
+    if (encoded_image->_size < required_size) {
+      // Encoded data > unencoded data, wtf? Allocate required bytes.
+      LOG(LS_WARNING) << "Encoding produced more bytes than the original image "
+                      << "data! Original bytes: " << encoded_image->_size
+                      << ", encoded bytes: " << required_size << ".";
+      encoded_image->_size = required_size;
+    }
+    encoded_image->_buffer = new uint8_t[encoded_image->_size];
+    encoded_image_buffer->reset(encoded_image->_buffer);
+  }
+
+  // Iterate layers and NAL units, note each NAL unit as a fragment and copy
+  // the data to |encoded_image->_buffer|.
+  frag_header->VerifyAndAllocateFragmentationHeader(fragments_count);
+  size_t frag_i = 0;
+  encoded_image->_length = 0;
+  for (int iLayer = 0; iLayer < info->iLayerNum; ++iLayer) {
+    const SLayerBSInfo& layerInfo = info->sLayerInfo[iLayer];
+    // Iterate NAL units making up this layer, noting fragments.
+    size_t iLayerLen = 0;
+    for (int iNal = 0; iNal < layerInfo.iNalCount; ++iNal, ++frag_i) {
+      // Expecting start code constant {0,0,0,1}.
+      DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+0], static_cast<unsigned char>(0));
+      DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+1], static_cast<unsigned char>(0));
+      DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+2], static_cast<unsigned char>(0));
+      DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+3], static_cast<unsigned char>(1));
+      // Fragment: +4/-4 is for excluding the start code.

[stefan-webrtc, 2015/10/01 08:19:30]

Instead of this comment you can name the constant 4. Or even better, create a
constant array:
const uint8_t kStartCode[4] = {0, 0, 0, 1};

and use it above and use sizeof(kStartCode) below.

[hbos, 2015/10/01 12:19:45]

Done.

+      frag_header->fragmentationOffset[frag_i] =
+          encoded_image->_length + iLayerLen + 4;
+      frag_header->fragmentationLength[frag_i] =
+          layerInfo.pNalLengthInByte[iNal] - 4;
+      iLayerLen += layerInfo.pNalLengthInByte[iNal];
+    }
+    // Copy the entire layer's data (including start codes).
+    memcpy(encoded_image->_buffer + encoded_image->_length,
+           layerInfo.pBsBuf,
+           iLayerLen * sizeof(unsigned char));
+    encoded_image->_length += iLayerLen;
+  }
+}
+
+H264EncoderImpl::H264EncoderImpl()
+    : openh264_encoder_(nullptr),
+      encoded_image_callback_(nullptr) {
+}
+
+H264EncoderImpl::~H264EncoderImpl() {
+  Release();
+}
+
+int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
+                                    int32_t /*number_of_cores*/,
+                                    size_t /*max_payload_size*/) {
+  if (!codec_settings ||
+      codec_settings->codecType != VideoCodecType::kVideoCodecH264) {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+  if (codec_settings->maxFramerate == 0)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  if (codec_settings->width < 1 || codec_settings->height < 1)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+
+  int release_ret = Release();
+  if (release_ret != WEBRTC_VIDEO_CODEC_OK)
+    return release_ret;
+  DCHECK(!openh264_encoder_);
+
+  // Create encoder.
+  if (WelsCreateSVCEncoder(&openh264_encoder_) != 0) {
+    // Failed to create encoder.
+    LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
+    DCHECK(!openh264_encoder_);
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  DCHECK(openh264_encoder_);
+  if (kOpenH264EncoderDetailedLogging) {
+    int trace_level = WELS_LOG_DETAIL;
+    openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL,
+                                 &trace_level);
+  }
+  // else WELS_LOG_DEFAULT is used by default.
+
+  codec_settings_ = *codec_settings;
+  if (codec_settings_.targetBitrate == 0)
+    codec_settings_.targetBitrate = codec_settings_.startBitrate;
+
+  // Initialization parameters.
+  //   There are two ways to initialize. There is SEncParamBase (cleared with
+  // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
+  // which is a superset of SEncParamBase (cleared with GetDefaultParams) used
+  // in InitializeExt.
+  SEncParamExt init_params;
+  openh264_encoder_->GetDefaultParams(&init_params);
+  if (codec_settings_.mode == kRealtimeVideo) {
+    init_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
+  } else if (codec_settings_.mode == kScreensharing) {
+    init_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
+  } else {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+  init_params.iPicWidth = codec_settings_.width;
+  init_params.iPicHeight = codec_settings_.height;
+  // |iTargetBitrate| is in bit/s, |targetBitrate| is in kbit/s.
+  init_params.iTargetBitrate = codec_settings_.targetBitrate * 1000;
+  // Rate Control mode
+  // TODO(hbos): Switch to RC_TIMESTAMP_MODE? VideoProcessorImpl::ProcessFrame
+  // need to be updated to use proper time stamps or else it will not work and
+  // VideoProcessorIntegrationTest.Process0PercentPacketLossH264 will fail.
+  init_params.iRCMode = RC_BITRATE_MODE;
+  init_params.fMaxFrameRate = static_cast<float>(codec_settings_.maxFramerate);
+
+  // The following parameters are extension parameters (they're in SEncParamExt,
+  // not in SEncParamBase).
+  init_params.bEnableFrameSkip =
+      codec_settings_.codecSpecific.H264.frameDroppingOn;
+  // |uiIntraPeriod|    - multiple of GOP size
+  // |keyFrameInterval| - ? number of frames? multiple of GOP size?

[stefan-webrtc, 2015/10/01 08:19:30]

keyFrameInterval is in frames. I'm not sure what multiple of GOP size means when
you have an IPPP structure, where the GOP can be very large... Maybe you can try
a few settings and see how it behaves by logging when a key frame is generated?

[hbos, 2015/10/01 12:19:45]

I did some logging and for me setting uiIntraPeriod to x meant every x-th frame
was a key frame (= videoFrameTypeIDR). If I made the images change drastically
(like putting my hand in front of the camera) I could get more key frames
in-between. All other frames was videoFrameTypeP.

So it works for now at least with the default video_loopback settings/frames,
but I don't know what affects the GOP size or if there is a way to check what it
is...

+  init_params.uiIntraPeriod =
+      codec_settings_.codecSpecific.H264.keyFrameInterval;
+  init_params.uiMaxNalSize = 0;
+  // Threading model: use auto.
+  //  0: auto (dynamic imp. internal encoder)
+  //  1: single thread (default value)
+  // >1: number of threads
+  init_params.iMultipleThreadIdc = 0;
+  // The base spatial layer 0 is the only one we use.
+  init_params.sSpatialLayers[0].iVideoWidth        = init_params.iPicWidth;
+  init_params.sSpatialLayers[0].iVideoHeight       = init_params.iPicHeight;
+  init_params.sSpatialLayers[0].fFrameRate         = init_params.fMaxFrameRate;
+  init_params.sSpatialLayers[0].iSpatialBitrate    = init_params.iTargetBitrate;
+  init_params.sSpatialLayers[0].iMaxSpatialBitrate = init_params.iMaxBitrate;
+  // Slice num according to number of threads.
+  init_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE;
+
+  // Initialize.
+  if (openh264_encoder_->InitializeExt(&init_params) != 0) {
+    LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
+    Release();
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  int video_format = EVideoFormatType::videoFormatI420;
+  openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT,
+                               &video_format);
+
+  // Initialize encoded image. Default buffer size: size of unencoded data.
+  encoded_image_._size = CalcBufferSize(
+      VideoType::kI420, codec_settings_.width, codec_settings_.height);
+  encoded_image_._buffer = new uint8_t[encoded_image_._size];
+  encoded_image_buffer_.reset(encoded_image_._buffer);
+  encoded_image_._completeFrame = true;
+  encoded_image_._encodedWidth = 0;
+  encoded_image_._encodedHeight = 0;
+  encoded_image_._length = 0;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::Release() {
+  if (openh264_encoder_) {
+    int uninit_ret = openh264_encoder_->Uninitialize();
+    if (uninit_ret != 0) {
+      LOG(LS_WARNING) << "OpenH264 encoder's Uninitialize() returned "
+                      << "unsuccessful: " << uninit_ret;
+    }
+    WelsDestroySVCEncoder(openh264_encoder_);
+    openh264_encoder_ = nullptr;
+  }
+  if (encoded_image_._buffer != nullptr) {
+    encoded_image_._buffer = nullptr;
+    encoded_image_buffer_.reset(nullptr);

[stefan-webrtc, 2015/10/01 08:19:30]

Don't think you have to pass nullptr here.

[hbos, 2015/10/01 12:19:45]

Done.

+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
+    EncodedImageCallback* callback) {
+  encoded_image_callback_ = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) {
+  if (bitrate <= 0 || framerate <= 0) {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+  codec_settings_.targetBitrate = bitrate;
+  codec_settings_.maxFramerate = framerate;
+
+  SBitrateInfo target_bitrate;
+  memset(&target_bitrate, 0, sizeof(SBitrateInfo));
+  target_bitrate.iLayer = SPATIAL_LAYER_ALL,
+  target_bitrate.iBitrate = codec_settings_.targetBitrate * 1000;
+  openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE,
+                               &target_bitrate);
+  float max_framerate = static_cast<float>(codec_settings_.maxFramerate);
+  openh264_encoder_->SetOption(ENCODER_OPTION_FRAME_RATE,
+                               &max_framerate);
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::Encode(
+    const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info,
+    const std::vector<VideoFrameType>* frame_types) {
+  if (!IsInitialized())
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  if (frame.IsZeroSize())
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  if (!encoded_image_callback_) {
+    LOG(LS_WARNING) << "InitEncode() has been called, but a callback function "
+                    << "has not been set with RegisterEncodeCompleteCallback()";
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  if (frame.width()  != codec_settings_.width ||
+      frame.height() != codec_settings_.height) {
+    LOG(LS_WARNING) << "Encoder initialized for " << codec_settings_.width
+                    << "x" << codec_settings_.height << " but trying to encode "
+                    << frame.width() << "x" << frame.height() << " frame.";
+    return WEBRTC_VIDEO_CODEC_ERR_SIZE;
+  }
+
+  bool force_key_frame = false;
+  if (frame_types != nullptr) {
+    // Skip frame?
+    if (std::find(frame_types->begin(), frame_types->end(),

[stefan-webrtc, 2015/10/01 08:19:30]

Since we only encode a single stream you might as well DCHECK(frame_types.size()
== 1)
and then use frame_types[0].

[hbos, 2015/10/01 12:19:45]

Done. Did not realize |frame_types| one type per stream.

+                  kSkipFrame) != frame_types->end()) {
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+    // Force key frame?
+    force_key_frame = std::find(frame_types->begin(), frame_types->end(),
+                                kKeyFrame) != frame_types->end();
+  }
+  if (force_key_frame) {
+    // Only need to call ForceIntraFrame when true. API doc says
+    // ForceIntraFrame(false) does nothing but really if you call it for every
+    // frame it introduces massive delays and lag in the video stream.

[stefan-webrtc, 2015/10/01 08:19:30]

What the... :O

[hbos, 2015/10/01 12:19:45]

Yes :) and then it crashes after a few seconds.

+    openh264_encoder_->ForceIntraFrame(true);
+  }
+
+  // EncodeFrame input.
+  SSourcePicture picture;
+  memset(&picture, 0, sizeof(SSourcePicture));
+  picture.iPicWidth = frame.width();
+  picture.iPicHeight = frame.height();
+  picture.iColorFormat = EVideoFormatType::videoFormatI420;
+  // 90 kHz -> milliseconds (1 kHz)
+  picture.uiTimeStamp = frame.timestamp() / 90;

[stefan-webrtc, 2015/10/01 08:19:30]

You have to make you handle a wraparound in frame.timestamp() since it's 90 kHz
and uint32_t.

An option might be to use ntp_time_ms() instead?

[hbos, 2015/10/01 12:19:45]

Done (using ntp_time_ms instead).

+  picture.iStride[0] = frame.stride(kYPlane);
+  picture.iStride[1] = frame.stride(kUPlane);
+  picture.iStride[2] = frame.stride(kVPlane);
+  picture.pData[0] = const_cast<uint8_t*>(frame.buffer(kYPlane));
+  picture.pData[1] = const_cast<uint8_t*>(frame.buffer(kUPlane));
+  picture.pData[2] = const_cast<uint8_t*>(frame.buffer(kVPlane));
+
+  // EncodeFrame output.
+  SFrameBSInfo info;
+  memset(&info, 0, sizeof(SFrameBSInfo));
+
+  // Encode!
+  int enc_ret = openh264_encoder_->EncodeFrame(&picture, &info);
+  if (enc_ret != 0) {
+    LOG(LS_ERROR) << "OpenH264 frame encoding failed, EncodeFrame returned "
+                  << enc_ret << ".";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  encoded_image_._encodedWidth = frame.width();
+  encoded_image_._encodedHeight = frame.height();
+  encoded_image_._timeStamp = frame.timestamp();
+  encoded_image_.ntp_time_ms_ = frame.ntp_time_ms();
+  encoded_image_.capture_time_ms_ = frame.render_time_ms();
+  encoded_image_._frameType = EVideoFrameType_to_VideoFrameType(
+      info.eFrameType);
+
+  // Split encoded image up into fragments. This also updates |encoded_image_|.
+  RTPFragmentationHeader frag_header;
+  RtpFragmentize(&encoded_image_, &encoded_image_buffer_,
+                 frame, &info, &frag_header);
+
+  // Encoder can skip frames to save bandwidth in which case
+  // |encoded_image_._length| == 0.
+  if (encoded_image_._length > 0) {
+    // Deliver encoded image.
+    encoded_image_callback_->Encoded(encoded_image_, codec_specific_info,
+                                     &frag_header);
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+bool H264EncoderImpl::IsInitialized() {
+  return openh264_encoder_ != nullptr;
+}
+
+int32_t H264EncoderImpl::SetChannelParameters(
+    uint32_t packet_loss, int64_t rtt) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::CodecConfigParameters(uint8_t* buffer, int32_t size) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void H264EncoderImpl::OnDroppedFrame() {
+}
+
+}  // namespace webrtc