H.264 video codec support using OpenH264/FFmpeg

webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc

Index: webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc
diff --git a/webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc
new file mode 100644
index 0000000000000000000000000000000000000000..a337ba8740bf728d0f6d5100b6dcea44c91bd04f
--- /dev/null
+++ b/webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc
@@ -0,0 +1,185 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ *
+ */
+
+#include "webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.h"
+
+#include <bitset>
+
+// OpenH264
+#include "codec_api.h"
+#include "codec_app_def.h"
+#include "codec_def.h"
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
+
+namespace webrtc {
+
+namespace {
+const bool kOpenH264DecoderDetailedLogging = false;
+}  // namespace
+
+H264DecoderImpl::H264DecoderImpl()
+    : openh264_decoder_(nullptr),
+      decoded_image_(),
+      decoded_image_callback_(nullptr) {
+}
+
+H264DecoderImpl::~H264DecoderImpl() {
+  Release();
+}
+
+int32_t H264DecoderImpl::InitDecode(const VideoCodec* codec_settings,
+                                    int32_t /*number_of_cores*/) {
+  if (codec_settings &&
+      codec_settings->codecType != VideoCodecType::kVideoCodecH264) {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;

[palmer, 2015/11/25 02:20:50]

Ultimately it may be a good idea to re-declare these constants as an enum, and
to return that enum type from this function/other similar functions. And then to
use switch/case in call sites.

[hbos, 2015/11/27 14:43:42]

Acknowledged. (Won't do that as part of this work though, it's used in a million
places.)

+  }
+
+  int release_ret = Release();

[palmer, 2015/11/25 02:20:50]

Nit: Are these codes intended to be ints, or int32s? It probably doesn't matter,
but it's better to be consistent.

[hbos, 2015/11/27 14:43:42]

Done.

+  if (release_ret != WEBRTC_VIDEO_CODEC_OK)
+    return release_ret;
+  RTC_DCHECK(!openh264_decoder_);
+
+  // Create decoder.
+  if (WelsCreateDecoder(&openh264_decoder_) != 0) {
+    LOG(LS_ERROR) << "Failed to create OpenH264 decoder";
+    RTC_DCHECK(!openh264_decoder_);
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  RTC_DCHECK(openh264_decoder_);
+  if (kOpenH264DecoderDetailedLogging) {
+    int trace_level = WELS_LOG_DETAIL;
+    openh264_decoder_->SetOption(DECODER_OPTION_TRACE_LEVEL,
+                                 &trace_level);
+  }
+  // else WELS_LOG_DEFAULT is used by default.
+
+  // Initialization parameters.
+  SDecodingParam init_params;
+  memset(&init_params, 0, sizeof(SDecodingParam));

[palmer, 2015/11/25 02:20:50]

Is this necessary? The SDecodingParam default constructor should enforce this,
right?

[hbos, 2015/11/27 14:43:43]

It's necessary. It's a struct with a default constructor that doesn't initialize
its primitive variables and memset is what the OpenH264 tutorial code does :)
(Open source third party code maintained by Cisco, they do things differently
than we'd like)

+  init_params.eOutputColorFormat = EVideoFormatType::videoFormatI420;
+  init_params.uiCpuLoad = 0;
+  init_params.uiTargetDqLayer = 0xFF;
+  init_params.eEcActiveIdc = ERROR_CON_IDC::ERROR_CON_DISABLE;
+  init_params.bParseOnly = false;
+  init_params.sVideoProperty.eVideoBsType =

[palmer, 2015/11/25 02:20:50]

Nit: Yeah, it seems like all this could be done with a constructor call?

[hbos, 2015/11/27 14:43:43]

I'm afraid not (unless we start contributing to the OpenH264 project and add a
sensible constructor)

+      VIDEO_BITSTREAM_TYPE::VIDEO_BITSTREAM_DEFAULT;
+
+  // Initialize.
+  if (openh264_decoder_->Initialize(&init_params) != 0) {
+    LOG(LS_ERROR) << "Failed to initialize OpenH264 decoder";
+    Release();
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264DecoderImpl::Release() {
+  if (openh264_decoder_) {
+    int64_t uninit_ret = openh264_decoder_->Uninitialize();
+    if (uninit_ret != 0) {
+      LOG(LS_WARNING) << "OpenH264 decoder's Uninitialize() returned "
+                      << "unsuccessful: " << uninit_ret;
+    }
+    WelsDestroyDecoder(openh264_decoder_);
+    openh264_decoder_ = nullptr;
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264DecoderImpl::Reset() {
+  if (!IsInitialized())
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  InitDecode(nullptr, 1);
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264DecoderImpl::RegisterDecodeCompleteCallback(
+    DecodedImageCallback* callback) {
+  decoded_image_callback_ = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
+                                bool /*missing_frames*/,
+                                const RTPFragmentationHeader* /*fragmentation*/,
+                                const CodecSpecificInfo* codec_specific_info,
+                                int64_t /*render_time_ms*/) {
+  if (!IsInitialized()) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  if (!decoded_image_callback_) {
+    LOG(LS_WARNING) << "InitDecode() has been called, but a callback function "
+                    << "has not been set with RegisterDecodeCompleteCallback()";
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  if (!input_image._buffer || !input_image._length)

[palmer, 2015/11/25 02:20:50]

BUG?: It looks like _length is declared as a size_t, but you pass it to
DecodeFrameNoDelay which treats it as a signed int. This is a change in
signedness and possibly in width, so it's a potentially unsafe cast.

I haven't yet seen the implementation of DecodeFrameNoDelay, so I don't yet know
what it's going to do with its arguments, but it seems like retaining type
information and checking for more than just !input_image._length would be a very
good idea. Deserializing data from the internet is always dangerous...

[hbos, 2015/11/27 14:43:42]

Hmh I don't know if it's possible to receive crazy big frames but I added a
check to ensure it can be converted safely to int.

+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  if (codec_specific_info &&
+      codec_specific_info->codecType != VideoCodecType::kVideoCodecH264) {
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+  }
+
+  // DecodeFrameNoDelay output.
+  uint8_t* data[3] = { 0, 0, 0 };
+  SBufferInfo buffer_info;
+  memset(&buffer_info, 0, sizeof(SBufferInfo));

[palmer, 2015/11/25 02:20:50]

Same default constructor issue here, too. (Unless I'm misunderstanding something
and there is a reason you have to use this old style?)

[hbos, 2015/11/27 14:43:42]

See other comment.

+
+  // Decode!
+  DECODING_STATE decode_ret = openh264_decoder_->DecodeFrameNoDelay(
+      input_image._buffer, input_image._length, data, &buffer_info);
+  if (decode_ret != 0) {
+    LOG(LS_ERROR) << "H264 decode failed, returned error bitmask: "
+                  << std::bitset<8>(decode_ret).to_string() << " = "
+                  << decode_ret;
+    if (decode_ret & dsFramePending)
+      LOG(LS_ERROR) << "  dsFramePending";
+    if (decode_ret & dsRefLost)
+      LOG(LS_ERROR) << "  dsRefLost";
+    if (decode_ret & dsBitstreamError)
+      LOG(LS_ERROR) << "  dsBitstreamError";
+    if (decode_ret & dsDepLayerLost)
+      LOG(LS_ERROR) << "  dsDepLayerLost";
+    if (decode_ret & dsNoParamSets)
+      LOG(LS_ERROR) << "  dsNoParamSets";
+    if (decode_ret & dsDataErrorConcealed)
+      LOG(LS_ERROR) << "  dsDataErrorConcealed";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  RTC_DCHECK_EQ(decode_ret, 0);
+
+  // Frame data ready?
+  if (buffer_info.iBufferStatus == 1) {
+    // Copy decoded data into |decoded_image_|. Must copy because the internal
+    // VideoFrameBuffer is reference counted.
+    decoded_image_.CreateFrame(data[0], data[1], data[2],
+                               buffer_info.UsrData.sSystemBuffer.iWidth,
+                               buffer_info.UsrData.sSystemBuffer.iHeight,
+                               buffer_info.UsrData.sSystemBuffer.iStride[0],
+                               buffer_info.UsrData.sSystemBuffer.iStride[1],
+                               buffer_info.UsrData.sSystemBuffer.iStride[1]);
+    decoded_image_.set_timestamp(input_image._timeStamp);
+    decoded_image_.set_ntp_time_ms(input_image.ntp_time_ms_);

[stefan-webrtc, 2015/10/01 13:13:52]

I think you have to call set_render_time_ms() too.

[hbos, 2015/11/27 14:43:42]

Done. Looks like every encoder uses a different timestamp :P

+
+    // Deliver decoded image.
+    decoded_image_callback_->Decoded(decoded_image_);
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+bool H264DecoderImpl::IsInitialized() {
+  return openh264_decoder_ != nullptr;
+}
+
+}  // namespace webrtc