OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 * | |
10 */ | |
11 | |
12 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.h" | |
13 | |
14 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) | |
15 | |
16 #include <memory> | |
17 | |
18 #if defined(WEBRTC_IOS) | |
19 #include "RTCUIApplication.h" | |
20 #endif | |
21 #include "libyuv/convert.h" | |
22 #include "webrtc/base/checks.h" | |
23 #include "webrtc/base/logging.h" | |
24 #include "webrtc/common_video/include/corevideo_frame_buffer.h" | |
25 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" | |
26 #include "webrtc/video_frame.h" | |
27 | |
28 namespace internal { | |
29 | |
30 static const int64_t kMsPerSec = 1000; | |
31 | |
32 // Convenience function for creating a dictionary. | |
33 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
34 CFTypeRef* values, | |
35 size_t size) { | |
36 return CFDictionaryCreate(nullptr, keys, values, size, | |
37 &kCFTypeDictionaryKeyCallBacks, | |
38 &kCFTypeDictionaryValueCallBacks); | |
39 } | |
40 | |
41 // Struct that we pass to the decoder per frame to decode. We receive it again | |
42 // in the decoder callback. | |
43 struct FrameDecodeParams { | |
44 FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts) | |
45 : callback(cb), timestamp(ts) {} | |
46 webrtc::DecodedImageCallback* callback; | |
47 int64_t timestamp; | |
48 }; | |
49 | |
50 // This is the callback function that VideoToolbox calls when decode is | |
51 // complete. | |
52 void VTDecompressionOutputCallback(void* decoder, | |
53 void* params, | |
54 OSStatus status, | |
55 VTDecodeInfoFlags info_flags, | |
56 CVImageBufferRef image_buffer, | |
57 CMTime timestamp, | |
58 CMTime duration) { | |
59 std::unique_ptr<FrameDecodeParams> decode_params( | |
60 reinterpret_cast<FrameDecodeParams*>(params)); | |
61 if (status != noErr) { | |
62 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; | |
63 return; | |
64 } | |
65 // TODO(tkchin): Handle CVO properly. | |
66 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = | |
67 new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer); | |
68 webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, | |
69 CMTimeGetSeconds(timestamp) * kMsPerSec, | |
70 webrtc::kVideoRotation_0); | |
71 decode_params->callback->Decoded(decoded_frame); | |
72 } | |
73 | |
74 } // namespace internal | |
75 | |
76 namespace webrtc { | |
77 | |
78 H264VideoToolboxDecoder::H264VideoToolboxDecoder() | |
79 : callback_(nullptr), | |
80 video_format_(nullptr), | |
81 decompression_session_(nullptr) {} | |
82 | |
83 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { | |
84 DestroyDecompressionSession(); | |
85 SetVideoFormat(nullptr); | |
86 } | |
87 | |
88 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, | |
89 int number_of_cores) { | |
90 return WEBRTC_VIDEO_CODEC_OK; | |
91 } | |
92 | |
93 int H264VideoToolboxDecoder::Decode( | |
94 const EncodedImage& input_image, | |
95 bool missing_frames, | |
96 const RTPFragmentationHeader* fragmentation, | |
97 const CodecSpecificInfo* codec_specific_info, | |
98 int64_t render_time_ms) { | |
99 RTC_DCHECK(input_image._buffer); | |
100 | |
101 #if defined(WEBRTC_IOS) | |
102 if (!RTCIsUIApplicationActive()) { | |
103 // Ignore all decode requests when app isn't active. In this state, the | |
104 // hardware decoder has been invalidated by the OS. | |
105 // Reset video format so that we won't process frames until the next | |
106 // keyframe. | |
107 SetVideoFormat(nullptr); | |
108 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; | |
109 } | |
110 #endif | |
111 CMVideoFormatDescriptionRef input_format = nullptr; | |
112 if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, | |
113 input_image._length)) { | |
114 input_format = CreateVideoFormatDescription(input_image._buffer, | |
115 input_image._length); | |
116 if (input_format) { | |
117 // Check if the video format has changed, and reinitialize decoder if | |
118 // needed. | |
119 if (!CMFormatDescriptionEqual(input_format, video_format_)) { | |
120 SetVideoFormat(input_format); | |
121 ResetDecompressionSession(); | |
122 } | |
123 CFRelease(input_format); | |
124 } | |
125 } | |
126 if (!video_format_) { | |
127 // We received a frame but we don't have format information so we can't | |
128 // decode it. | |
129 // This can happen after backgrounding. We need to wait for the next | |
130 // sps/pps before we can resume so we request a keyframe by returning an | |
131 // error. | |
132 LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; | |
133 return WEBRTC_VIDEO_CODEC_ERROR; | |
134 } | |
135 CMSampleBufferRef sample_buffer = nullptr; | |
136 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, | |
137 input_image._length, video_format_, | |
138 &sample_buffer)) { | |
139 return WEBRTC_VIDEO_CODEC_ERROR; | |
140 } | |
141 RTC_DCHECK(sample_buffer); | |
142 VTDecodeFrameFlags decode_flags = | |
143 kVTDecodeFrame_EnableAsynchronousDecompression; | |
144 std::unique_ptr<internal::FrameDecodeParams> frame_decode_params; | |
145 frame_decode_params.reset( | |
146 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
147 OSStatus status = VTDecompressionSessionDecodeFrame( | |
148 decompression_session_, sample_buffer, decode_flags, | |
149 frame_decode_params.release(), nullptr); | |
150 #if defined(WEBRTC_IOS) | |
151 // Re-initialize the decoder if we have an invalid session while the app is | |
152 // active and retry the decode request. | |
153 if (status == kVTInvalidSessionErr && | |
154 ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { | |
155 frame_decode_params.reset( | |
156 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
157 status = VTDecompressionSessionDecodeFrame( | |
158 decompression_session_, sample_buffer, decode_flags, | |
159 frame_decode_params.release(), nullptr); | |
160 } | |
161 #endif | |
162 CFRelease(sample_buffer); | |
163 if (status != noErr) { | |
164 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; | |
165 return WEBRTC_VIDEO_CODEC_ERROR; | |
166 } | |
167 return WEBRTC_VIDEO_CODEC_OK; | |
168 } | |
169 | |
170 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( | |
171 DecodedImageCallback* callback) { | |
172 RTC_DCHECK(!callback_); | |
173 callback_ = callback; | |
174 return WEBRTC_VIDEO_CODEC_OK; | |
175 } | |
176 | |
177 int H264VideoToolboxDecoder::Release() { | |
178 // Need to invalidate the session so that callbacks no longer occur and it | |
179 // is safe to null out the callback. | |
180 DestroyDecompressionSession(); | |
181 SetVideoFormat(nullptr); | |
182 callback_ = nullptr; | |
183 return WEBRTC_VIDEO_CODEC_OK; | |
184 } | |
185 | |
186 int H264VideoToolboxDecoder::ResetDecompressionSession() { | |
187 DestroyDecompressionSession(); | |
188 | |
189 // Need to wait for the first SPS to initialize decoder. | |
190 if (!video_format_) { | |
191 return WEBRTC_VIDEO_CODEC_OK; | |
192 } | |
193 | |
194 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder | |
195 // create pixel buffers with GPU backed memory. The intent here is to pass | |
196 // the pixel buffers directly so we avoid a texture upload later during | |
197 // rendering. This currently is moot because we are converting back to an | |
198 // I420 frame after decode, but eventually we will be able to plumb | |
199 // CVPixelBuffers directly to the renderer. | |
200 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that | |
201 // we can pass CVPixelBuffers as native handles in decoder output. | |
202 static size_t const attributes_size = 3; | |
203 CFTypeRef keys[attributes_size] = { | |
204 #if defined(WEBRTC_IOS) | |
205 kCVPixelBufferOpenGLESCompatibilityKey, | |
206 #elif defined(WEBRTC_MAC) | |
207 kCVPixelBufferOpenGLCompatibilityKey, | |
208 #endif | |
209 kCVPixelBufferIOSurfacePropertiesKey, | |
210 kCVPixelBufferPixelFormatTypeKey | |
211 }; | |
212 CFDictionaryRef io_surface_value = | |
213 internal::CreateCFDictionary(nullptr, nullptr, 0); | |
214 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
215 CFNumberRef pixel_format = | |
216 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
217 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
218 pixel_format}; | |
219 CFDictionaryRef attributes = | |
220 internal::CreateCFDictionary(keys, values, attributes_size); | |
221 if (io_surface_value) { | |
222 CFRelease(io_surface_value); | |
223 io_surface_value = nullptr; | |
224 } | |
225 if (pixel_format) { | |
226 CFRelease(pixel_format); | |
227 pixel_format = nullptr; | |
228 } | |
229 VTDecompressionOutputCallbackRecord record = { | |
230 internal::VTDecompressionOutputCallback, this, | |
231 }; | |
232 OSStatus status = | |
233 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, | |
234 &record, &decompression_session_); | |
235 CFRelease(attributes); | |
236 if (status != noErr) { | |
237 DestroyDecompressionSession(); | |
238 return WEBRTC_VIDEO_CODEC_ERROR; | |
239 } | |
240 ConfigureDecompressionSession(); | |
241 | |
242 return WEBRTC_VIDEO_CODEC_OK; | |
243 } | |
244 | |
245 void H264VideoToolboxDecoder::ConfigureDecompressionSession() { | |
246 RTC_DCHECK(decompression_session_); | |
247 #if defined(WEBRTC_IOS) | |
248 VTSessionSetProperty(decompression_session_, | |
249 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); | |
250 #endif | |
251 } | |
252 | |
253 void H264VideoToolboxDecoder::DestroyDecompressionSession() { | |
254 if (decompression_session_) { | |
255 VTDecompressionSessionInvalidate(decompression_session_); | |
256 CFRelease(decompression_session_); | |
257 decompression_session_ = nullptr; | |
258 } | |
259 } | |
260 | |
261 void H264VideoToolboxDecoder::SetVideoFormat( | |
262 CMVideoFormatDescriptionRef video_format) { | |
263 if (video_format_ == video_format) { | |
264 return; | |
265 } | |
266 if (video_format_) { | |
267 CFRelease(video_format_); | |
268 } | |
269 video_format_ = video_format; | |
270 if (video_format_) { | |
271 CFRetain(video_format_); | |
272 } | |
273 } | |
274 | |
275 const char* H264VideoToolboxDecoder::ImplementationName() const { | |
276 return "VideoToolbox"; | |
277 } | |
278 | |
279 } // namespace webrtc | |
280 | |
281 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) | |
OLD | NEW |