OLD | NEW |
---|---|
(Empty) | |
1 /* | |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 * | |
10 */ | |
11 | |
12 #include "webrtc/modules/video_coding/codecs/h264/h264_decoder_impl.h" | |
13 | |
14 #include <algorithm> | |
15 | |
16 extern "C" { | |
17 #include "third_party/ffmpeg/libavcodec/avcodec.h" | |
18 #include "third_party/ffmpeg/libavformat/avformat.h" | |
19 #include "third_party/ffmpeg/libavutil/imgutils.h" | |
20 } // extern "C" | |
21 | |
22 #include "webrtc/base/checks.h" | |
23 #include "webrtc/base/criticalsection.h" | |
24 #include "webrtc/base/logging.h" | |
25 | |
26 namespace webrtc { | |
27 | |
28 namespace { | |
29 | |
30 static bool ffmpeg_initialized = false; | |
31 static const AVPixelFormat PIXEL_FORMAT = AV_PIX_FMT_YUV420P; | |
32 | |
33 // Called by FFmpeg to do mutex operations if init using InitializeFFmpeg. | |
34 static int LockManagerOperation(void** lock, AVLockOp op) | |
35 EXCLUSIVE_LOCK_FUNCTION() UNLOCK_FUNCTION() { | |
36 switch (op) { | |
37 case AV_LOCK_CREATE: | |
38 *lock = new rtc::CriticalSection(); | |
39 return 0; | |
40 case AV_LOCK_OBTAIN: | |
41 static_cast<rtc::CriticalSection*>(*lock)->Enter(); | |
42 return 0; | |
43 case AV_LOCK_RELEASE: | |
44 static_cast<rtc::CriticalSection*>(*lock)->Leave(); | |
45 return 0; | |
46 case AV_LOCK_DESTROY: | |
47 delete static_cast<rtc::CriticalSection*>(*lock); | |
48 *lock = nullptr; | |
49 return 0; | |
50 } | |
51 return 1; | |
52 } | |
53 | |
54 // TODO(hbos): Assumed to be called on a single thread. Should DCHECK that | |
55 // InitializeFFmpeg is only called on one thread or make it thread safe. | |
56 static bool InitializeFFmpeg() { | |
57 if (!ffmpeg_initialized) { | |
58 if (av_lockmgr_register(LockManagerOperation) < 0) { | |
59 LOG(LS_ERROR) << "av_lockmgr_register failed."; | |
60 return false; | |
61 } | |
62 av_register_all(); | |
63 ffmpeg_initialized = true; | |
64 } | |
65 return true; | |
66 } | |
67 | |
68 // Called by FFmpeg when it is done with a frame buffer, see AVGetBuffer2. | |
69 static void AVFreeBuffer2(void* opaque, uint8_t* data) { | |
70 VideoFrame* video_frame = static_cast<VideoFrame*>(opaque); | |
71 delete video_frame; | |
72 } | |
73 | |
74 // Called by FFmpeg when it needs a frame buffer to store decoded frames in. | |
75 // The VideoFrames returned by FFmpeg at Decode originate from here. They are | |
76 // reference counted and freed by FFmpeg using AVFreeBuffer2. | |
noahric
2016/01/04 20:49:30
Is it also true that the AVFrame is basically unus
hbos
2016/01/07 19:41:14
It is used by ffmpeg, we are required to set certa
| |
77 // TODO(hbos): Use a frame pool for better performance instead of create/free. | |
78 // Could be owned by decoder, static_cast<H264DecoderImpl*>(context->opaque). | |
79 static int AVGetBuffer2(AVCodecContext* context, AVFrame* frame, int flags) { | |
noahric
2016/01/04 20:49:30
consider naming this out_frame or av_frame. Normal
hbos
2016/01/07 19:41:14
Done.
| |
80 RTC_CHECK_EQ(context->pix_fmt, PIXEL_FORMAT); // Same as in InitDecode. | |
81 // width/height and coded_width/coded_height can be different due to cropping | |
82 // or |lowres|. | |
83 int width = std::max(context->width, context->coded_width); | |
84 int height = std::max(context->height, context->coded_height); | |
85 // See |lowres|, if used the decoder scales the image by 1/2^(lowres). This | |
86 // has implications on which resolutions are valid, but we don't use it. | |
87 RTC_CHECK_EQ(context->lowres, 0); | |
88 | |
89 RTC_CHECK_GE(width, 0); | |
90 RTC_CHECK_GE(height, 0); | |
91 int ret = av_image_check_size(width, height, 0, nullptr); | |
92 if (ret < 0) { | |
93 LOG(LS_ERROR) << "Invalid picture size " << width << "x" << height; | |
94 return ret; | |
95 } | |
96 | |
97 VideoFrame* video_frame = new VideoFrame(); | |
98 int stride_y = width; | |
99 int stride_uv = (width + 1) / 2; | |
100 RTC_CHECK_EQ(0, video_frame->CreateEmptyFrame( | |
101 width, height, stride_y, stride_uv, stride_uv)); | |
102 size_t total_size = video_frame->allocated_size(kYPlane) + | |
103 video_frame->allocated_size(kUPlane) + | |
104 video_frame->allocated_size(kVPlane); | |
105 RTC_DCHECK_EQ(total_size, static_cast<size_t>(stride_y * height + | |
106 (stride_uv + stride_uv) * ((height + 1) / 2))); | |
107 // FFmpeg note: "Each data plane must be aligned to the maximum required by | |
108 // the target CPU." See get_buffer2. | |
109 // TODO(hbos): Memory alignment on a per-plane basis. CreateEmptyFrame only | |
110 // guarantees that the buffer of all planes is memory aligned, not each | |
111 // individual plane. | |
noahric
2016/01/04 20:49:30
Sounds like an issue for i420 frame, then. Is it w
hbos
2016/01/07 19:41:14
I read some FFmpeg documentation a second time and
hbos
2016/01/12 13:56:26
Removing the TODO based on new interpretation of t
| |
112 | |
113 // FFmpeg expects the initial allocation to be zero-initialized according to | |
114 // http://crbug.com/390941. | |
115 // Expect YUV to be a continuous blob of memory so that we can zero-initialize | |
noahric
2016/01/04 20:49:30
Is it worth it to just do three memsets? It's prob
hbos
2016/01/07 19:41:14
Probably not. But as you pointed out, it looks lik
| |
116 // with a single memset operation instead of three. | |
117 RTC_DCHECK_EQ(video_frame->buffer(kUPlane), | |
118 video_frame->buffer(kYPlane) + video_frame->allocated_size(kYPlane)); | |
119 RTC_DCHECK_EQ(video_frame->buffer(kVPlane), | |
120 video_frame->buffer(kUPlane) + video_frame->allocated_size(kUPlane)); | |
121 memset(video_frame->buffer(kYPlane), 0, total_size); | |
122 | |
123 frame->width = width; | |
124 frame->height = height; | |
125 frame->format = context->pix_fmt; | |
126 frame->reordered_opaque = context->reordered_opaque; | |
127 | |
128 frame->data[kYPlane] = video_frame->buffer(kYPlane); | |
noahric
2016/01/04 20:49:30
If the absolute values of these matter (it looks l
hbos
2016/01/07 19:41:14
Done.
| |
129 frame->linesize[kYPlane] = video_frame->stride(kYPlane); | |
130 frame->data[kUPlane] = video_frame->buffer(kUPlane); | |
131 frame->linesize[kUPlane] = video_frame->stride(kUPlane); | |
132 frame->data[kVPlane] = video_frame->buffer(kVPlane); | |
133 frame->linesize[kVPlane] = video_frame->stride(kVPlane); | |
134 RTC_DCHECK_EQ(frame->extended_data, frame->data); | |
135 | |
136 frame->buf[0] = av_buffer_create(frame->data[0], | |
noahric
2016/01/04 20:49:30
It looks like this is also requiring that the data
hbos
2016/01/07 19:41:14
Done.
| |
137 total_size, | |
138 AVFreeBuffer2, | |
139 static_cast<void*>(video_frame), | |
140 0); | |
141 RTC_CHECK(frame->buf[0]); | |
142 return 0; | |
143 } | |
144 | |
145 } // namespace | |
146 | |
147 H264DecoderImpl::H264DecoderImpl() | |
148 : decoded_image_callback_(nullptr) { | |
149 } | |
150 | |
151 H264DecoderImpl::~H264DecoderImpl() { | |
152 Release(); | |
153 } | |
154 | |
155 int32_t H264DecoderImpl::InitDecode(const VideoCodec* codec_settings, | |
156 int32_t /*number_of_cores*/) { | |
157 if (codec_settings && | |
158 codec_settings->codecType != kVideoCodecH264) { | |
159 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | |
160 } | |
161 | |
162 // In Chromium FFmpeg will be initialized outside of WebRTC and we should not | |
163 // attempt to do so ourselves or it will be initialized twice. | |
164 #if !defined(WEBRTC_CHROMIUM_BUILD) | |
noahric
2016/01/04 20:49:30
Consider putting this behind a different flag, if
hbos
2016/01/07 19:41:14
Good point. Added a TODO, I'll address this in a f
| |
165 // Make sure FFmpeg has been initialized. | |
166 InitializeFFmpeg(); | |
167 #endif | |
168 | |
169 // Release necessary in case of re-initializing. | |
170 int32_t ret = Release(); | |
171 if (ret != WEBRTC_VIDEO_CODEC_OK) | |
172 return ret; | |
173 RTC_DCHECK(!av_context_); | |
174 | |
175 // Initialize AVCodecContext. | |
176 av_context_.reset(avcodec_alloc_context3(nullptr)); | |
177 | |
178 av_context_->codec_type = AVMEDIA_TYPE_VIDEO; | |
179 av_context_->codec_id = AV_CODEC_ID_H264; | |
180 // This is meant to be able to decode OpenH264 streams, which should be | |
181 // baseline profile. | |
noahric
2016/01/04 20:49:30
I assume it's also meant to be able to decode stre
hbos
2016/01/07 19:41:14
You're right! I looked this up, it should not be s
| |
182 av_context_->profile = FF_PROFILE_H264_BASELINE; | |
183 if (codec_settings) { | |
184 av_context_->coded_width = codec_settings->width; | |
185 av_context_->coded_height = codec_settings->height; | |
186 } | |
187 av_context_->pix_fmt = PIXEL_FORMAT; | |
188 av_context_->extradata = nullptr; | |
189 av_context_->extradata_size = 0; | |
190 | |
191 av_context_->thread_count = 4; | |
noahric
2016/01/04 20:49:30
Should this be gleaned from something else? Number
hbos
2016/01/07 19:41:14
Done. Copied from vp8_impl.cc.
| |
192 av_context_->thread_type = FF_THREAD_SLICE; | |
193 | |
194 // FFmpeg will get video buffers from our AVGetBuffer2, memory managed by us. | |
195 av_context_->get_buffer2 = AVGetBuffer2; | |
196 // get_buffer2 is called with the context, there |opaque| can be used to get a | |
197 // pointer |this|. | |
198 av_context_->opaque = this; | |
199 // Use ref counted frames (av_frame_unref). | |
200 av_context_->refcounted_frames = 1; // true | |
201 | |
202 AVCodec* codec = avcodec_find_decoder(av_context_->codec_id); | |
203 if (!codec) { | |
204 // This is an indication that FFmpeg has not been initialized or it has not | |
205 // been compiled/initialized with the correct set of codecs. | |
206 LOG(LS_ERROR) << "FFmpeg H.264 decoder not found."; | |
207 Release(); | |
208 return WEBRTC_VIDEO_CODEC_ERROR; | |
209 } | |
210 int res = avcodec_open2(av_context_.get(), codec, nullptr); | |
211 if (res < 0) { | |
212 LOG(LS_ERROR) << "avcodec_open2 error: " << res; | |
213 Release(); | |
214 return WEBRTC_VIDEO_CODEC_ERROR; | |
215 } | |
216 | |
217 av_frame_.reset(av_frame_alloc()); | |
218 return WEBRTC_VIDEO_CODEC_OK; | |
219 } | |
220 | |
221 int32_t H264DecoderImpl::Release() { | |
222 av_context_.reset(); | |
223 av_frame_.reset(); | |
224 return WEBRTC_VIDEO_CODEC_OK; | |
225 } | |
226 | |
227 int32_t H264DecoderImpl::Reset() { | |
228 if (!IsInitialized()) | |
229 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | |
230 InitDecode(nullptr, 1); | |
231 return WEBRTC_VIDEO_CODEC_OK; | |
232 } | |
233 | |
234 int32_t H264DecoderImpl::RegisterDecodeCompleteCallback( | |
235 DecodedImageCallback* callback) { | |
236 decoded_image_callback_ = callback; | |
237 return WEBRTC_VIDEO_CODEC_OK; | |
238 } | |
239 | |
240 int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, | |
241 bool /*missing_frames*/, | |
242 const RTPFragmentationHeader* /*fragmentation*/, | |
243 const CodecSpecificInfo* codec_specific_info, | |
244 int64_t /*render_time_ms*/) { | |
245 if (!IsInitialized()) | |
246 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | |
247 if (!decoded_image_callback_) { | |
248 LOG(LS_WARNING) << "InitDecode() has been called, but a callback function " | |
249 "has not been set with RegisterDecodeCompleteCallback()"; | |
250 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | |
251 } | |
252 if (!input_image._buffer || !input_image._length) | |
253 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | |
254 if (codec_specific_info && | |
255 codec_specific_info->codecType != kVideoCodecH264) { | |
256 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | |
257 } | |
258 | |
259 AVPacket packet; | |
260 av_init_packet(&packet); | |
261 // TODO(hbos): "The input buffer must be AV_INPUT_BUFFER_PADDING_SIZE larger | |
262 // than the actual read bytes because some optimized bitstream readers read 32 | |
263 // or 64 bits at once and could read over the end." See avcodec_decode_video2. | |
264 // - Is this an issue? Do we have to make sure EncodedImage is allocated with | |
265 // additional bytes or do we have to do an otherwise unnecessary copy? Might | |
266 // only be a problem with non-mul-32 frame widths? | |
noahric
2016/01/04 20:49:30
I'd buy the second part of this sentence, and it's
hbos
2016/01/07 19:41:14
(I'll address this tomorrow, getting late)
hbos
2016/01/11 16:21:53
Thanks for the info, yeah should be easy. Created
| |
267 // ("If the first 23 bits of the additional bytes are not 0, then damaged MPEG | |
268 // bitstreams could cause overread and segfault.") | |
noahric
2016/01/04 20:49:30
I think this could happen anyways today (VCMFrameB
hbos
2016/01/11 16:21:53
Done. (Same bug)
| |
269 packet.data = input_image._buffer; | |
270 packet.size = input_image._length; | |
271 av_context_->reordered_opaque = input_image.ntp_time_ms_ * 1000; // ms -> μs | |
noahric
2016/01/04 20:49:30
Can you add a comment here? avcodec.h says this is
hbos
2016/01/07 19:41:14
Lies! In some versions of ffmpeg it is deprecated,
| |
272 | |
273 int frame_decoded = 0; | |
274 int result = avcodec_decode_video2(av_context_.get(), | |
275 av_frame_.get(), | |
276 &frame_decoded, | |
277 &packet); | |
278 if (result < 0) { | |
279 LOG(LS_ERROR) << "avcodec_decode_video2 error: " << result; | |
280 return WEBRTC_VIDEO_CODEC_ERROR; | |
281 } | |
282 // |result| is number of bytes used, which should be all of them. | |
283 if (result != packet.size) { | |
284 LOG(LS_ERROR) << "avcodec_decode_video2 consumed " << result << " bytes " | |
285 "when " << packet.size << " bytes were expected."; | |
286 return WEBRTC_VIDEO_CODEC_ERROR; | |
287 } | |
288 | |
289 if (!frame_decoded) { | |
290 LOG(LS_WARNING) << "avcodec_decode_video2 successful but no frame was " | |
291 "decoded."; | |
292 return WEBRTC_VIDEO_CODEC_OK; | |
293 } | |
294 | |
295 // Obtain the |video_frame| containing the decoded image. | |
296 VideoFrame* video_frame = static_cast<VideoFrame*>( | |
297 av_buffer_get_opaque(av_frame_->buf[0])); | |
298 RTC_DCHECK(video_frame); | |
299 RTC_CHECK_EQ(av_frame_->data[kYPlane], video_frame->buffer(kYPlane)); | |
300 RTC_CHECK_EQ(av_frame_->data[kUPlane], video_frame->buffer(kUPlane)); | |
301 RTC_CHECK_EQ(av_frame_->data[kVPlane], video_frame->buffer(kVPlane)); | |
302 video_frame->set_timestamp(input_image._timeStamp); | |
303 | |
304 // Return decoded frame. | |
305 int32_t ret = decoded_image_callback_->Decoded(*video_frame); | |
306 // Stop referencing it, possibly freeing |video_frame|. | |
307 av_frame_unref(av_frame_.get()); | |
308 video_frame = nullptr; | |
309 | |
310 if (ret) { | |
311 LOG(LS_WARNING) << "DecodedImageCallback::Decoded returned " << ret; | |
312 return ret; | |
313 } | |
314 return WEBRTC_VIDEO_CODEC_OK; | |
315 } | |
316 | |
317 bool H264DecoderImpl::IsInitialized() const { | |
318 return av_context_; | |
noahric
2016/01/04 20:49:30
av_context_ != nullptr
(implicit cast to bool wil
hbos
2016/01/07 19:41:14
Done.
| |
319 } | |
320 | |
321 } // namespace webrtc | |
OLD | NEW |