H.264 video codec support using OpenH264/FFmpeg

webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc

+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h"
+
+#include <iostream>
+
+// OpenH264
+#include "codec_api.h"
+#include "codec_app_def.h"
+#include "codec_def.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+
+namespace webrtc {
+
+namespace {
+const bool OPENH264_ENCODER_LOGGING = false;
+}  // anonymous namespace
+
+static VideoFrameType EVideoFrameType_to_VideoFrameType(
+    const EVideoFrameType& type) {
+  switch (type) {
+    case videoFrameTypeInvalid:
+    case videoFrameTypeSkip:
+      return kSkipFrame;
+    case videoFrameTypeIDR:
+    case videoFrameTypeI:
+    case videoFrameTypeP:
+    case videoFrameTypeIPMixed:
+      return kDeltaFrame;
+    default:
+      LOG(LS_WARNING) << "Unknown EVideoFrameType: " << type;
+      return kDeltaFrame;
+  }
+}
+
+// Copies the encoded bytes from |info| to |encoded_image| and updates the
+// fragmentation information of |frag_header|.
+//   After OpenH264 encoding, the encoded bytes are stored in |info| spread out
+// over a number of layers and "NAL units". Each NAL unit is a fragment starting
+// with the four-byte NAL header {0,0,0,1}. To save bytes, the NAL headers are
+// excluded when copying to |encoded_image->_buffer|. However these headers must
+// be included for an H264Decoder to be able to decode the data.
+//   When fragments are sent over a network, the receiving end must re-insert
+// the NAL headers before each fragment.
+//   For image data encoded and decoded locally, RTPDefragmentize can be used
+// to convert (EncodedImage, RTPFragmentationHeader) with NAL headers excluded
+// to a decodable EncodedImage buffer with NAL headers included.
+static void RTPFragmentize(EncodedImage* encoded_image,
+                           const VideoFrame& frame,
+                           SFrameBSInfo* info,
+                           RTPFragmentationHeader* frag_header) {
+  encoded_image->_length = 0;
+  std::vector<int> frags;
+
+  int aLen = 0, iLayerLen = 0;
+  for (int iLayer = 0; iLayer < info->iLayerNum; ++iLayer) {
+    iLayerLen = 0;
+    const SLayerBSInfo& layerInfo = info->sLayerInfo[iLayer];
+    // Copy the layer data to |encoded_image_._buffer|, excluding the 4-byte
+    // NAL headers.
+    for (int iNal = 0; iNal < layerInfo.iNalCount; ++iNal) {
+      memcpy(encoded_image->_buffer + encoded_image->_length,
+             layerInfo.pBsBuf + iLayerLen + 4,
+             (layerInfo.pNalLengthInByte[iNal] - 4) * sizeof(unsigned char));
+      encoded_image->_length += (layerInfo.pNalLengthInByte[iNal] - 4);
+      frags.push_back(layerInfo.pNalLengthInByte[iNal] - 4);
+
+      iLayerLen += layerInfo.pNalLengthInByte[iNal];
+    }
+    aLen += iLayerLen;
+  }
+
+  frag_header->VerifyAndAllocateFragmentationHeader(frags.size());
+  for (size_t i = 0, off = 0; i < frags.size(); ++i) {
+    frag_header->fragmentationOffset[i] = off;
+    frag_header->fragmentationLength[i] = frags[i];
+    off += frags[i];
+  }
+}
+
+void H264EncoderImpl::RTPDefragmentize(
+    const EncodedImage& encoded_image,
+    const RTPFragmentationHeader* frag_header,
+    uint8_t* enc_buffer_with_nal, size_t enc_buffer_with_nal_length) {
+  DCHECK_GE(enc_buffer_with_nal_length,
+            RTPDefragmentizeBufferLengthWithNAL(encoded_image, frag_header));
+  const unsigned char nal_header[] = { 0, 0, 0, 1 };
+  for (size_t i = 0; i < frag_header->fragmentationVectorSize; ++i) {
+    DCHECK_LE(frag_header->fragmentationOffset[i] +
+              frag_header->fragmentationLength[i],
+              encoded_image._length);
+
+    // Insert a NAL header.
+    memcpy(enc_buffer_with_nal, nal_header, 4);
+    // Copy fragment data.
+    memcpy(enc_buffer_with_nal + 4,
+           encoded_image._buffer + frag_header->fragmentationOffset[i],
+           frag_header->fragmentationLength[i]);
+
+    enc_buffer_with_nal += (4 + frag_header->fragmentationLength[i]);
+  }
+}
+
+size_t H264EncoderImpl::RTPDefragmentizeBufferLengthWithNAL(
+    const EncodedImage& encoded_image,
+    const webrtc::RTPFragmentationHeader* frag_header) {
+  return encoded_image._length +
+      4 * static_cast<size_t>(frag_header->fragmentationVectorSize);
+}
+
+H264EncoderImpl::H264EncoderImpl()
+    : openh264_encoder_(nullptr),
+      encoded_image_callback_(nullptr) {
+}
+
+H264EncoderImpl::~H264EncoderImpl() {
+  Release();
+}
+
+int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
+                                    int32_t /*number_of_cores*/,
+                                    size_t /*max_payload_size*/) {
+  if (!codec_settings ||
+      codec_settings->codecType != VideoCodecType::kVideoCodecH264) {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+  if (codec_settings->maxFramerate == 0)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  if (codec_settings->width < 1 || codec_settings->height < 1)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+
+  int release_ret = Release();
+  if (release_ret != WEBRTC_VIDEO_CODEC_OK)
+    return release_ret;
+  DCHECK(!openh264_encoder_);
+
+  // Create encoder.
+  if (WelsCreateSVCEncoder(&openh264_encoder_) != 0) {
+    // Failed to create encoder.
+    LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
+    DCHECK(!openh264_encoder_);
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  DCHECK(openh264_encoder_);
+  if (&codec_settings_ != codec_settings)
+    codec_settings_ = *codec_settings;
+
+  if (codec_settings_.targetBitrate == 0)
+    codec_settings_.targetBitrate = codec_settings_.startBitrate;
+
+  // Note: H264 codec specifics are ignored:
+  // - codec_settings->codecSpecific.H264.frameDroppingOn
+  // - codec_settings->codecSpecific.H264.keyFrameInterval
+
+  // Initialization parameters.
+  //   There are two ways to initialize. There is SEncParamBase (cleared with
+  // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
+  // which is a superset of SEncParamBase (cleared with GetDefaultParams) used
+  // in InitializeExt. We use SEncParamBase/Initialize.
+  SEncParamBase init_params;
+  memset(&init_params, 0, sizeof(SEncParamBase));
+  if (codec_settings_.mode == kRealtimeVideo) {
+    init_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
+  } else if (codec_settings_.mode == kScreensharing) {
+    init_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
+  } else {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+  init_params.iPicWidth = codec_settings_.width;
+  init_params.iPicHeight = codec_settings_.height;
+  // iTargetBitrate is in bit/s, targetBitrate is in kbit/s.
+  init_params.iTargetBitrate = codec_settings_.targetBitrate * 1000;
+  // Rate Control mode
+  init_params.iRCMode = RC_QUALITY_MODE;
+  init_params.fMaxFrameRate = static_cast<float>(codec_settings_.maxFramerate);
+
+  // Initialize.
+  if (openh264_encoder_->Initialize(&init_params) != 0) {
+    // Failed to initialize.
+    LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
+    Release();
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  // Initialize encoded image.
+  // (Safe?) assumption: encoded data is always less than unencoded data.
+  // (Unsafe?) assumption: don't need to realloc buffer if frame size differs.
+  encoded_image_._size = CalcBufferSize(
+      VideoType::kI420, codec_settings_.width, codec_settings_.height);
+  encoded_image_._buffer = new uint8_t[encoded_image_._size];
+  encoded_image_._completeFrame = true;
+  encoded_image_._encodedWidth = 0;
+  encoded_image_._encodedHeight = 0;
+  encoded_image_._length = 0;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::Release() {
+  if (openh264_encoder_) {
+    int uninit_ret = openh264_encoder_->Uninitialize();
+    if (uninit_ret != 0) {
+      LOG(LS_WARNING) << "OpenH264 encoder's Uninitialize() returned "
+                      << "unsuccessful: " << uninit_ret;
+    }
+    WelsDestroySVCEncoder(openh264_encoder_);
+    openh264_encoder_ = nullptr;
+  }
+  if (encoded_image_._buffer != nullptr) {
+    delete[] encoded_image_._buffer;
+    encoded_image_._buffer = nullptr;
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
+    EncodedImageCallback* callback) {
+  encoded_image_callback_ = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) {
+  if (bitrate <= 0 || framerate <= 0) {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+  codec_settings_.targetBitrate = bitrate;
+  codec_settings_.maxFramerate = framerate;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::Encode(
+    const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info,
+    const std::vector<VideoFrameType>* frame_types) {
+  if (!IsInitialized())
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  if (frame.IsZeroSize())
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  if (!encoded_image_callback_) {
+    LOG(LS_WARNING) << "InitEncode() has been called, but a callback function "
+                    << "has not been set with RegisterEncodeCompleteCallback()";
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+
+  // Make |codec_settings_|'s size reflect the latest frame's size.
+  // (Not necessary for encoding to work.)
+  if (codec_settings_.width != frame.width() ||
+      codec_settings_.height != frame.height()) {
+    codec_settings_.width = frame.width();
+    codec_settings_.height = frame.height();
+  }
+
+  // Set encoder options.
+  int video_format = EVideoFormatType::videoFormatI420;
+  openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT,
+                               &video_format);
+  SBitrateInfo target_bitrate;
+  memset(&target_bitrate, 0, sizeof(SBitrateInfo));
+  target_bitrate.iLayer = SPATIAL_LAYER_ALL,
+  target_bitrate.iBitrate = codec_settings_.targetBitrate * 1000;
+  openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE,
+                               &target_bitrate);
+  float max_framerate = static_cast<float>(codec_settings_.maxFramerate);
+  openh264_encoder_->SetOption(ENCODER_OPTION_FRAME_RATE,
+                               &max_framerate);
+
+  if (OPENH264_ENCODER_LOGGING) {
+    int trace_level = WELS_LOG_DETAIL;
+    openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL,
+                                 &trace_level);
+  }
+
+  // EncodeFrame input.
+  SSourcePicture picture;
+  memset(&picture, 0, sizeof(SSourcePicture));
+  picture.iPicWidth = frame.width();
+  picture.iPicHeight = frame.height();
+  picture.iColorFormat = video_format;
+  picture.uiTimeStamp = frame.timestamp();
+  picture.iStride[0] = frame.stride(kYPlane);
+  picture.iStride[1] = frame.stride(kUPlane);
+  picture.iStride[2] = frame.stride(kVPlane);
+  picture.pData[0] = const_cast<uint8_t*>(frame.buffer(kYPlane));
+  picture.pData[1] = const_cast<uint8_t*>(frame.buffer(kUPlane));
+  picture.pData[2] = const_cast<uint8_t*>(frame.buffer(kVPlane));
+
+  // EncodeFrame output.
+  SFrameBSInfo info;
+  memset(&info, 0, sizeof(SFrameBSInfo));
+
+  // Encode!
+  if (openh264_encoder_->EncodeFrame(&picture, &info) != 0) {
+    LOG(LS_ERROR) << "OpenH264 frame encoding failed (EncodeFrame)";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  encoded_image_._encodedWidth = frame.width();
+  encoded_image_._encodedHeight = frame.height();
+  encoded_image_._timeStamp = frame.timestamp();
+  encoded_image_.capture_time_ms_ = frame.render_time_ms();
+  encoded_image_._frameType = EVideoFrameType_to_VideoFrameType(
+      info.eFrameType);
+
+  // Split encoded image up into fragments. This also updates |encoded_image_|.
+  RTPFragmentationHeader frag_header;
+  RTPFragmentize(&encoded_image_, frame, &info, &frag_header);
+
+  // Encoder can skip frames to save bandwidth in which case
+  // |encoded_image_._length| == 0.
+  if (encoded_image_._length > 0) {
+    // Deliver encoded image.
+    encoded_image_callback_->Encoded(encoded_image_, codec_specific_info,
+                                     &frag_header);
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+bool H264EncoderImpl::IsInitialized() {
+  return openh264_encoder_ != nullptr;
+}
+
+int32_t H264EncoderImpl::SetChannelParameters(
+    uint32_t packet_loss, int64_t rtt) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264EncoderImpl::CodecConfigParameters(uint8_t* buffer, int32_t size) {
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void H264EncoderImpl::OnDroppedFrame() {
+}
+
+}  // namespace webrtc