H264: Improve FFmpeg decoder performance by using I420BufferPool.

webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
@@ skipping 59 matching lines @@
       RTC_NOTREACHED() << "av_lockmgr_register failed.";
       return;
     }
     av_register_all();
     ffmpeg_initialized = true;
   }
 }
 
 #endif  // !defined(WEBRTC_CHROMIUM_BUILD)
 
-// Called by FFmpeg when it is done with a frame buffer, see AVGetBuffer2.
-void AVFreeBuffer2(void* opaque, uint8_t* data) {
-  VideoFrame* video_frame = static_cast<VideoFrame*>(opaque);
-  delete video_frame;
-}
+}  // namespace
 
 // Called by FFmpeg when it needs a frame buffer to store decoded frames in.
 // The VideoFrames returned by FFmpeg at |Decode| originate from here. They are
 // reference counted and freed by FFmpeg using |AVFreeBuffer2|.
 // TODO(hbos): Use a frame pool for better performance instead of create/free.
 // Could be owned by decoder, |static_cast<H264DecoderImpl*>(context->opaque)|.

[stefan-webrtc, 2016/02/01 14:02:09]

Should this comment be removed now?

[hbos, 2016/02/02 16:13:03]

Done.

 // Consider verifying that the buffer was allocated by us to avoid unsafe type
 // cast. See https://bugs.chromium.org/p/webrtc/issues/detail?id=5428.
-int AVGetBuffer2(AVCodecContext* context, AVFrame* av_frame, int flags) {
-  RTC_CHECK_EQ(context->pix_fmt, kPixelFormat);  // Same as in InitDecode.
+int H264DecoderImpl::AVGetBuffer2(
+    AVCodecContext* context, AVFrame* av_frame, int flags) {
+  H264DecoderImpl* decoder =
+      reinterpret_cast<H264DecoderImpl*>(context->opaque);

[hbos, 2016/01/29 10:16:56]

To reinterpret_cast or static_cast, that is the question...

[hbos, 2016/02/02 16:13:03]

static_cast, that is the answer.

+
+  // DCHECK values set in |InitDecode|.
+  RTC_DCHECK(decoder);
+  RTC_DCHECK_EQ(context->pix_fmt, kPixelFormat);
   // Necessary capability to be allowed to provide our own buffers.
-  RTC_CHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
+  RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
+
   // |av_frame->width| and |av_frame->height| are set by FFmpeg. These are the
   // actual image's dimensions and may be different from |context->width| and
   // |context->coded_width| due to reordering.
   int width = av_frame->width;
   int height = av_frame->height;
   // See |lowres|, if used the decoder scales the image by 1/2^(lowres). This
   // has implications on which resolutions are valid, but we don't use it.
   RTC_CHECK_EQ(context->lowres, 0);
   // Adjust the |width| and |height| to values acceptable by the decoder.
   // Without this, FFmpeg may overflow the buffer. If modified, |width| and/or
   // |height| are larger than the actual image and the image has to be cropped
   // (top-left corner) after decoding to avoid visible borders to the right and
   // bottom of the actual image.
   avcodec_align_dimensions(context, &width, &height);
 
   RTC_CHECK_GE(width, 0);
   RTC_CHECK_GE(height, 0);
   int ret = av_image_check_size(static_cast<unsigned int>(width),
                                 static_cast<unsigned int>(height), 0, nullptr);
   if (ret < 0) {
     LOG(LS_ERROR) << "Invalid picture size " << width << "x" << height;
     return ret;
   }
 
   // The video frame is stored in |video_frame|. |av_frame| is FFmpeg's version
   // of a video frame and will be set up to reference |video_frame|'s buffers.
   VideoFrame* video_frame = new VideoFrame();
-  int stride_y = width;
-  int stride_uv = (width + 1) / 2;
-  RTC_CHECK_EQ(0, video_frame->CreateEmptyFrame(
-      width, height, stride_y, stride_uv, stride_uv));
-  int total_size = video_frame->allocated_size(kYPlane) +
-                   video_frame->allocated_size(kUPlane) +
-                   video_frame->allocated_size(kVPlane);
-  RTC_DCHECK_EQ(total_size, stride_y * height +
-                (stride_uv + stride_uv) * ((height + 1) / 2));
-
   // FFmpeg expects the initial allocation to be zero-initialized according to
-  // http://crbug.com/390941.
-  // Using a single |av_frame->buf| - YUV is required to be a continuous blob of
-  // memory. We can zero-initialize with one memset operation for all planes.
+  // http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
+  video_frame->set_video_frame_buffer(
+      decoder->pool_.CreateBuffer(width, height));
+  // DCHECK that we have a continuous buffer as is required.
   RTC_DCHECK_EQ(video_frame->buffer(kUPlane),
       video_frame->buffer(kYPlane) + video_frame->allocated_size(kYPlane));
   RTC_DCHECK_EQ(video_frame->buffer(kVPlane),
       video_frame->buffer(kUPlane) + video_frame->allocated_size(kUPlane));
-  memset(video_frame->buffer(kYPlane), 0, total_size);

[stefan-webrtc, 2016/02/01 14:02:09]

The memset you did in the pool doesn't memset the total_size, but only the
DataSize(), which appears to be the part of the buffer that is actually being
used, and not what was allocated?

[hbos, 2016/02/02 16:13:03]

That memset and this memset are the same. See previous comment, the extra bytes
due to memory alignment doesn't look like they need to be memset.

+  int total_size = video_frame->allocated_size(kYPlane) +
+                   video_frame->allocated_size(kUPlane) +
+                   video_frame->allocated_size(kVPlane);
 
   av_frame->format = context->pix_fmt;
   av_frame->reordered_opaque = context->reordered_opaque;
 
   // Set |av_frame| members as required by FFmpeg.
   av_frame->data[kYPlaneIndex] = video_frame->buffer(kYPlane);
   av_frame->linesize[kYPlaneIndex] = video_frame->stride(kYPlane);
   av_frame->data[kUPlaneIndex] = video_frame->buffer(kUPlane);
   av_frame->linesize[kUPlaneIndex] = video_frame->stride(kUPlane);
   av_frame->data[kVPlaneIndex] = video_frame->buffer(kVPlane);
   av_frame->linesize[kVPlaneIndex] = video_frame->stride(kVPlane);
   RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
 
   av_frame->buf[0] = av_buffer_create(av_frame->data[kYPlaneIndex],
                                       total_size,
                                       AVFreeBuffer2,
                                       static_cast<void*>(video_frame),
                                       0);
   RTC_CHECK(av_frame->buf[0]);
   return 0;
 }
 
-}  // namespace
+// Called by FFmpeg when it is done with a frame buffer, see AVGetBuffer2.
+void H264DecoderImpl::AVFreeBuffer2(void* opaque, uint8_t* data) {
+  VideoFrame* video_frame = static_cast<VideoFrame*>(opaque);
+  delete video_frame;

[stefan-webrtc, 2016/02/01 14:02:09]

Seems odd that we delete frames when we are using a pool?

[hbos, 2016/02/02 16:13:03]

Added a comment explaining why.

+}
 
-H264DecoderImpl::H264DecoderImpl()
-    : decoded_image_callback_(nullptr) {
+H264DecoderImpl::H264DecoderImpl() : pool_(true),
+                                     decoded_image_callback_(nullptr) {
 }
 
 H264DecoderImpl::~H264DecoderImpl() {
   Release();
 }
 
 int32_t H264DecoderImpl::InitDecode(const VideoCodec* codec_settings,
                                     int32_t number_of_cores) {
   if (codec_settings &&
       codec_settings->codecType != kVideoCodecH264) {
@@ skipping 22 matching lines @@
   av_context_->codec_type = AVMEDIA_TYPE_VIDEO;
   av_context_->codec_id = AV_CODEC_ID_H264;
   if (codec_settings) {
     av_context_->coded_width = codec_settings->width;
     av_context_->coded_height = codec_settings->height;
   }
   av_context_->pix_fmt = kPixelFormat;
   av_context_->extradata = nullptr;
   av_context_->extradata_size = 0;
 
+  // If this is ever increased, look at |av_context_->thread_safe_callbacks| and
+  // make it possible to disable the thread checker in the frame buffer pool.
   av_context_->thread_count = 1;
   av_context_->thread_type = FF_THREAD_SLICE;
 
-  // FFmpeg will get video buffers from our AVGetBuffer2, memory managed by us.
+  // Function used by FFmpeg to get buffers to store decoded frames in.
   av_context_->get_buffer2 = AVGetBuffer2;
-  // get_buffer2 is called with the context, there |opaque| can be used to get a
-  // pointer |this|.
+  // |get_buffer2| is called with the context, there |opaque| can be used to get
+  // a pointer |this|.
   av_context_->opaque = this;
   // Use ref counted frames (av_frame_unref).
   av_context_->refcounted_frames = 1;  // true
 
   AVCodec* codec = avcodec_find_decoder(av_context_->codec_id);
   if (!codec) {
     // This is an indication that FFmpeg has not been initialized or it has not
     // been compiled/initialized with the correct set of codecs.
     LOG(LS_ERROR) << "FFmpeg H.264 decoder not found.";
     Release();
@@ skipping 125 matching lines @@
     return ret;
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 bool H264DecoderImpl::IsInitialized() const {
   return av_context_ != nullptr;
 }
 
 }  // namespace webrtc