OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 * | |
10 */ | |
11 | |
12 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_decoder.h" | |
13 | |
14 #include <memory> | |
15 | |
16 #if defined(WEBRTC_IOS) | |
17 #include "RTCUIApplication.h" | |
18 #endif | |
19 #include "libyuv/convert.h" | |
20 #include "webrtc/base/checks.h" | |
21 #include "webrtc/base/logging.h" | |
22 #include "webrtc/common_video/include/corevideo_frame_buffer.h" | |
23 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_nalu.h" | |
24 #include "webrtc/video_frame.h" | |
25 | |
26 namespace internal { | |
27 | |
28 static const int64_t kMsPerSec = 1000; | |
29 | |
30 // Convenience function for creating a dictionary. | |
31 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
32 CFTypeRef* values, | |
33 size_t size) { | |
34 return CFDictionaryCreate(nullptr, keys, values, size, | |
35 &kCFTypeDictionaryKeyCallBacks, | |
36 &kCFTypeDictionaryValueCallBacks); | |
37 } | |
38 | |
39 // Struct that we pass to the decoder per frame to decode. We receive it again | |
40 // in the decoder callback. | |
41 struct FrameDecodeParams { | |
42 FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts) | |
43 : callback(cb), timestamp(ts) {} | |
44 webrtc::DecodedImageCallback* callback; | |
45 int64_t timestamp; | |
46 }; | |
47 | |
48 // This is the callback function that VideoToolbox calls when decode is | |
49 // complete. | |
50 void VTDecompressionOutputCallback(void* decoder, | |
51 void* params, | |
52 OSStatus status, | |
53 VTDecodeInfoFlags info_flags, | |
54 CVImageBufferRef image_buffer, | |
55 CMTime timestamp, | |
56 CMTime duration) { | |
57 std::unique_ptr<FrameDecodeParams> decode_params( | |
58 reinterpret_cast<FrameDecodeParams*>(params)); | |
59 if (status != noErr) { | |
60 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; | |
61 return; | |
62 } | |
63 // TODO(tkchin): Handle CVO properly. | |
64 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = | |
65 new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer); | |
66 webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, | |
67 CMTimeGetSeconds(timestamp) * kMsPerSec, | |
68 webrtc::kVideoRotation_0); | |
69 decode_params->callback->Decoded(decoded_frame); | |
70 } | |
71 | |
72 } // namespace internal | |
73 | |
74 namespace webrtc { | |
75 | |
76 H264VideoToolboxDecoder::H264VideoToolboxDecoder() | |
77 : callback_(nullptr), | |
78 video_format_(nullptr), | |
79 decompression_session_(nullptr) {} | |
80 | |
81 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { | |
82 DestroyDecompressionSession(); | |
83 SetVideoFormat(nullptr); | |
84 } | |
85 | |
86 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, | |
87 int number_of_cores) { | |
88 return WEBRTC_VIDEO_CODEC_OK; | |
89 } | |
90 | |
91 int H264VideoToolboxDecoder::Decode( | |
92 const EncodedImage& input_image, | |
93 bool missing_frames, | |
94 const RTPFragmentationHeader* fragmentation, | |
95 const CodecSpecificInfo* codec_specific_info, | |
96 int64_t render_time_ms) { | |
97 RTC_DCHECK(input_image._buffer); | |
98 | |
99 #if defined(WEBRTC_IOS) | |
100 if (!RTCIsUIApplicationActive()) { | |
101 // Ignore all decode requests when app isn't active. In this state, the | |
102 // hardware decoder has been invalidated by the OS. | |
103 // Reset video format so that we won't process frames until the next | |
104 // keyframe. | |
105 SetVideoFormat(nullptr); | |
106 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; | |
107 } | |
108 #endif | |
109 CMVideoFormatDescriptionRef input_format = nullptr; | |
110 if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, | |
111 input_image._length)) { | |
112 input_format = CreateVideoFormatDescription(input_image._buffer, | |
113 input_image._length); | |
114 if (input_format) { | |
115 // Check if the video format has changed, and reinitialize decoder if | |
116 // needed. | |
117 if (!CMFormatDescriptionEqual(input_format, video_format_)) { | |
118 SetVideoFormat(input_format); | |
119 ResetDecompressionSession(); | |
120 } | |
121 CFRelease(input_format); | |
122 } | |
123 } | |
124 if (!video_format_) { | |
125 // We received a frame but we don't have format information so we can't | |
126 // decode it. | |
127 // This can happen after backgrounding. We need to wait for the next | |
128 // sps/pps before we can resume so we request a keyframe by returning an | |
129 // error. | |
130 LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; | |
131 return WEBRTC_VIDEO_CODEC_ERROR; | |
132 } | |
133 CMSampleBufferRef sample_buffer = nullptr; | |
134 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, | |
135 input_image._length, video_format_, | |
136 &sample_buffer)) { | |
137 return WEBRTC_VIDEO_CODEC_ERROR; | |
138 } | |
139 RTC_DCHECK(sample_buffer); | |
140 VTDecodeFrameFlags decode_flags = | |
141 kVTDecodeFrame_EnableAsynchronousDecompression; | |
142 std::unique_ptr<internal::FrameDecodeParams> frame_decode_params; | |
143 frame_decode_params.reset( | |
144 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
145 OSStatus status = VTDecompressionSessionDecodeFrame( | |
146 decompression_session_, sample_buffer, decode_flags, | |
147 frame_decode_params.release(), nullptr); | |
148 #if defined(WEBRTC_IOS) | |
149 // Re-initialize the decoder if we have an invalid session while the app is | |
150 // active and retry the decode request. | |
151 if (status == kVTInvalidSessionErr && | |
152 ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { | |
153 frame_decode_params.reset( | |
154 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
155 status = VTDecompressionSessionDecodeFrame( | |
156 decompression_session_, sample_buffer, decode_flags, | |
157 frame_decode_params.release(), nullptr); | |
158 } | |
159 #endif | |
160 CFRelease(sample_buffer); | |
161 if (status != noErr) { | |
162 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; | |
163 return WEBRTC_VIDEO_CODEC_ERROR; | |
164 } | |
165 return WEBRTC_VIDEO_CODEC_OK; | |
166 } | |
167 | |
168 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( | |
169 DecodedImageCallback* callback) { | |
170 RTC_DCHECK(!callback_); | |
171 callback_ = callback; | |
172 return WEBRTC_VIDEO_CODEC_OK; | |
173 } | |
174 | |
175 int H264VideoToolboxDecoder::Release() { | |
176 // Need to invalidate the session so that callbacks no longer occur and it | |
177 // is safe to null out the callback. | |
178 DestroyDecompressionSession(); | |
179 SetVideoFormat(nullptr); | |
180 callback_ = nullptr; | |
181 return WEBRTC_VIDEO_CODEC_OK; | |
182 } | |
183 | |
184 int H264VideoToolboxDecoder::ResetDecompressionSession() { | |
185 DestroyDecompressionSession(); | |
186 | |
187 // Need to wait for the first SPS to initialize decoder. | |
188 if (!video_format_) { | |
189 return WEBRTC_VIDEO_CODEC_OK; | |
190 } | |
191 | |
192 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder | |
193 // create pixel buffers with GPU backed memory. The intent here is to pass | |
194 // the pixel buffers directly so we avoid a texture upload later during | |
195 // rendering. This currently is moot because we are converting back to an | |
196 // I420 frame after decode, but eventually we will be able to plumb | |
197 // CVPixelBuffers directly to the renderer. | |
198 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that | |
199 // we can pass CVPixelBuffers as native handles in decoder output. | |
200 static size_t const attributes_size = 3; | |
201 CFTypeRef keys[attributes_size] = { | |
202 #if defined(WEBRTC_IOS) | |
203 kCVPixelBufferOpenGLESCompatibilityKey, | |
204 #elif defined(WEBRTC_MAC) | |
205 kCVPixelBufferOpenGLCompatibilityKey, | |
206 #endif | |
207 kCVPixelBufferIOSurfacePropertiesKey, | |
208 kCVPixelBufferPixelFormatTypeKey | |
209 }; | |
210 CFDictionaryRef io_surface_value = | |
211 internal::CreateCFDictionary(nullptr, nullptr, 0); | |
212 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
213 CFNumberRef pixel_format = | |
214 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
215 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
216 pixel_format}; | |
217 CFDictionaryRef attributes = | |
218 internal::CreateCFDictionary(keys, values, attributes_size); | |
219 if (io_surface_value) { | |
220 CFRelease(io_surface_value); | |
221 io_surface_value = nullptr; | |
222 } | |
223 if (pixel_format) { | |
224 CFRelease(pixel_format); | |
225 pixel_format = nullptr; | |
226 } | |
227 VTDecompressionOutputCallbackRecord record = { | |
228 internal::VTDecompressionOutputCallback, this, | |
229 }; | |
230 OSStatus status = | |
231 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, | |
232 &record, &decompression_session_); | |
233 CFRelease(attributes); | |
234 if (status != noErr) { | |
235 DestroyDecompressionSession(); | |
236 return WEBRTC_VIDEO_CODEC_ERROR; | |
237 } | |
238 ConfigureDecompressionSession(); | |
239 | |
240 return WEBRTC_VIDEO_CODEC_OK; | |
241 } | |
242 | |
243 void H264VideoToolboxDecoder::ConfigureDecompressionSession() { | |
244 RTC_DCHECK(decompression_session_); | |
245 #if defined(WEBRTC_IOS) | |
246 VTSessionSetProperty(decompression_session_, | |
247 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); | |
248 #endif | |
249 } | |
250 | |
251 void H264VideoToolboxDecoder::DestroyDecompressionSession() { | |
252 if (decompression_session_) { | |
253 VTDecompressionSessionInvalidate(decompression_session_); | |
254 CFRelease(decompression_session_); | |
255 decompression_session_ = nullptr; | |
256 } | |
257 } | |
258 | |
259 void H264VideoToolboxDecoder::SetVideoFormat( | |
260 CMVideoFormatDescriptionRef video_format) { | |
261 if (video_format_ == video_format) { | |
262 return; | |
263 } | |
264 if (video_format_) { | |
265 CFRelease(video_format_); | |
266 } | |
267 video_format_ = video_format; | |
268 if (video_format_) { | |
269 CFRetain(video_format_); | |
270 } | |
271 } | |
272 | |
273 const char* H264VideoToolboxDecoder::ImplementationName() const { | |
274 return "VideoToolbox"; | |
275 } | |
276 | |
277 } // namespace webrtc | |
OLD | NEW |