| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 * | |
| 10 */ | |
| 11 | |
| 12 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.h" | |
| 13 | |
| 14 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) | |
| 15 | |
| 16 #include <memory> | |
| 17 | |
| 18 #if defined(WEBRTC_IOS) | |
| 19 #include "RTCUIApplication.h" | |
| 20 #endif | |
| 21 #include "libyuv/convert.h" | |
| 22 #include "webrtc/base/checks.h" | |
| 23 #include "webrtc/base/logging.h" | |
| 24 #include "webrtc/common_video/include/corevideo_frame_buffer.h" | |
| 25 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" | |
| 26 #include "webrtc/video_frame.h" | |
| 27 | |
| 28 namespace internal { | |
| 29 | |
| 30 static const int64_t kMsPerSec = 1000; | |
| 31 | |
| 32 // Convenience function for creating a dictionary. | |
| 33 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
| 34 CFTypeRef* values, | |
| 35 size_t size) { | |
| 36 return CFDictionaryCreate(nullptr, keys, values, size, | |
| 37 &kCFTypeDictionaryKeyCallBacks, | |
| 38 &kCFTypeDictionaryValueCallBacks); | |
| 39 } | |
| 40 | |
| 41 // Struct that we pass to the decoder per frame to decode. We receive it again | |
| 42 // in the decoder callback. | |
| 43 struct FrameDecodeParams { | |
| 44 FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts) | |
| 45 : callback(cb), timestamp(ts) {} | |
| 46 webrtc::DecodedImageCallback* callback; | |
| 47 int64_t timestamp; | |
| 48 }; | |
| 49 | |
| 50 // This is the callback function that VideoToolbox calls when decode is | |
| 51 // complete. | |
| 52 void VTDecompressionOutputCallback(void* decoder, | |
| 53 void* params, | |
| 54 OSStatus status, | |
| 55 VTDecodeInfoFlags info_flags, | |
| 56 CVImageBufferRef image_buffer, | |
| 57 CMTime timestamp, | |
| 58 CMTime duration) { | |
| 59 std::unique_ptr<FrameDecodeParams> decode_params( | |
| 60 reinterpret_cast<FrameDecodeParams*>(params)); | |
| 61 if (status != noErr) { | |
| 62 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; | |
| 63 return; | |
| 64 } | |
| 65 // TODO(tkchin): Handle CVO properly. | |
| 66 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = | |
| 67 new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer); | |
| 68 webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, | |
| 69 CMTimeGetSeconds(timestamp) * kMsPerSec, | |
| 70 webrtc::kVideoRotation_0); | |
| 71 decode_params->callback->Decoded(decoded_frame); | |
| 72 } | |
| 73 | |
| 74 } // namespace internal | |
| 75 | |
| 76 namespace webrtc { | |
| 77 | |
| 78 H264VideoToolboxDecoder::H264VideoToolboxDecoder() | |
| 79 : callback_(nullptr), | |
| 80 video_format_(nullptr), | |
| 81 decompression_session_(nullptr) {} | |
| 82 | |
| 83 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { | |
| 84 DestroyDecompressionSession(); | |
| 85 SetVideoFormat(nullptr); | |
| 86 } | |
| 87 | |
| 88 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, | |
| 89 int number_of_cores) { | |
| 90 return WEBRTC_VIDEO_CODEC_OK; | |
| 91 } | |
| 92 | |
| 93 int H264VideoToolboxDecoder::Decode( | |
| 94 const EncodedImage& input_image, | |
| 95 bool missing_frames, | |
| 96 const RTPFragmentationHeader* fragmentation, | |
| 97 const CodecSpecificInfo* codec_specific_info, | |
| 98 int64_t render_time_ms) { | |
| 99 RTC_DCHECK(input_image._buffer); | |
| 100 | |
| 101 #if defined(WEBRTC_IOS) | |
| 102 if (!RTCIsUIApplicationActive()) { | |
| 103 // Ignore all decode requests when app isn't active. In this state, the | |
| 104 // hardware decoder has been invalidated by the OS. | |
| 105 // Reset video format so that we won't process frames until the next | |
| 106 // keyframe. | |
| 107 SetVideoFormat(nullptr); | |
| 108 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; | |
| 109 } | |
| 110 #endif | |
| 111 CMVideoFormatDescriptionRef input_format = nullptr; | |
| 112 if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, | |
| 113 input_image._length)) { | |
| 114 input_format = CreateVideoFormatDescription(input_image._buffer, | |
| 115 input_image._length); | |
| 116 if (input_format) { | |
| 117 // Check if the video format has changed, and reinitialize decoder if | |
| 118 // needed. | |
| 119 if (!CMFormatDescriptionEqual(input_format, video_format_)) { | |
| 120 SetVideoFormat(input_format); | |
| 121 ResetDecompressionSession(); | |
| 122 } | |
| 123 CFRelease(input_format); | |
| 124 } | |
| 125 } | |
| 126 if (!video_format_) { | |
| 127 // We received a frame but we don't have format information so we can't | |
| 128 // decode it. | |
| 129 // This can happen after backgrounding. We need to wait for the next | |
| 130 // sps/pps before we can resume so we request a keyframe by returning an | |
| 131 // error. | |
| 132 LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; | |
| 133 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 134 } | |
| 135 CMSampleBufferRef sample_buffer = nullptr; | |
| 136 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, | |
| 137 input_image._length, video_format_, | |
| 138 &sample_buffer)) { | |
| 139 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 140 } | |
| 141 RTC_DCHECK(sample_buffer); | |
| 142 VTDecodeFrameFlags decode_flags = | |
| 143 kVTDecodeFrame_EnableAsynchronousDecompression; | |
| 144 std::unique_ptr<internal::FrameDecodeParams> frame_decode_params; | |
| 145 frame_decode_params.reset( | |
| 146 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 147 OSStatus status = VTDecompressionSessionDecodeFrame( | |
| 148 decompression_session_, sample_buffer, decode_flags, | |
| 149 frame_decode_params.release(), nullptr); | |
| 150 #if defined(WEBRTC_IOS) | |
| 151 // Re-initialize the decoder if we have an invalid session while the app is | |
| 152 // active and retry the decode request. | |
| 153 if (status == kVTInvalidSessionErr && | |
| 154 ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { | |
| 155 frame_decode_params.reset( | |
| 156 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 157 status = VTDecompressionSessionDecodeFrame( | |
| 158 decompression_session_, sample_buffer, decode_flags, | |
| 159 frame_decode_params.release(), nullptr); | |
| 160 } | |
| 161 #endif | |
| 162 CFRelease(sample_buffer); | |
| 163 if (status != noErr) { | |
| 164 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; | |
| 165 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 166 } | |
| 167 return WEBRTC_VIDEO_CODEC_OK; | |
| 168 } | |
| 169 | |
| 170 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( | |
| 171 DecodedImageCallback* callback) { | |
| 172 RTC_DCHECK(!callback_); | |
| 173 callback_ = callback; | |
| 174 return WEBRTC_VIDEO_CODEC_OK; | |
| 175 } | |
| 176 | |
| 177 int H264VideoToolboxDecoder::Release() { | |
| 178 // Need to invalidate the session so that callbacks no longer occur and it | |
| 179 // is safe to null out the callback. | |
| 180 DestroyDecompressionSession(); | |
| 181 SetVideoFormat(nullptr); | |
| 182 callback_ = nullptr; | |
| 183 return WEBRTC_VIDEO_CODEC_OK; | |
| 184 } | |
| 185 | |
| 186 int H264VideoToolboxDecoder::ResetDecompressionSession() { | |
| 187 DestroyDecompressionSession(); | |
| 188 | |
| 189 // Need to wait for the first SPS to initialize decoder. | |
| 190 if (!video_format_) { | |
| 191 return WEBRTC_VIDEO_CODEC_OK; | |
| 192 } | |
| 193 | |
| 194 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder | |
| 195 // create pixel buffers with GPU backed memory. The intent here is to pass | |
| 196 // the pixel buffers directly so we avoid a texture upload later during | |
| 197 // rendering. This currently is moot because we are converting back to an | |
| 198 // I420 frame after decode, but eventually we will be able to plumb | |
| 199 // CVPixelBuffers directly to the renderer. | |
| 200 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that | |
| 201 // we can pass CVPixelBuffers as native handles in decoder output. | |
| 202 static size_t const attributes_size = 3; | |
| 203 CFTypeRef keys[attributes_size] = { | |
| 204 #if defined(WEBRTC_IOS) | |
| 205 kCVPixelBufferOpenGLESCompatibilityKey, | |
| 206 #elif defined(WEBRTC_MAC) | |
| 207 kCVPixelBufferOpenGLCompatibilityKey, | |
| 208 #endif | |
| 209 kCVPixelBufferIOSurfacePropertiesKey, | |
| 210 kCVPixelBufferPixelFormatTypeKey | |
| 211 }; | |
| 212 CFDictionaryRef io_surface_value = | |
| 213 internal::CreateCFDictionary(nullptr, nullptr, 0); | |
| 214 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
| 215 CFNumberRef pixel_format = | |
| 216 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
| 217 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
| 218 pixel_format}; | |
| 219 CFDictionaryRef attributes = | |
| 220 internal::CreateCFDictionary(keys, values, attributes_size); | |
| 221 if (io_surface_value) { | |
| 222 CFRelease(io_surface_value); | |
| 223 io_surface_value = nullptr; | |
| 224 } | |
| 225 if (pixel_format) { | |
| 226 CFRelease(pixel_format); | |
| 227 pixel_format = nullptr; | |
| 228 } | |
| 229 VTDecompressionOutputCallbackRecord record = { | |
| 230 internal::VTDecompressionOutputCallback, this, | |
| 231 }; | |
| 232 OSStatus status = | |
| 233 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, | |
| 234 &record, &decompression_session_); | |
| 235 CFRelease(attributes); | |
| 236 if (status != noErr) { | |
| 237 DestroyDecompressionSession(); | |
| 238 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 239 } | |
| 240 ConfigureDecompressionSession(); | |
| 241 | |
| 242 return WEBRTC_VIDEO_CODEC_OK; | |
| 243 } | |
| 244 | |
| 245 void H264VideoToolboxDecoder::ConfigureDecompressionSession() { | |
| 246 RTC_DCHECK(decompression_session_); | |
| 247 #if defined(WEBRTC_IOS) | |
| 248 VTSessionSetProperty(decompression_session_, | |
| 249 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); | |
| 250 #endif | |
| 251 } | |
| 252 | |
| 253 void H264VideoToolboxDecoder::DestroyDecompressionSession() { | |
| 254 if (decompression_session_) { | |
| 255 VTDecompressionSessionInvalidate(decompression_session_); | |
| 256 CFRelease(decompression_session_); | |
| 257 decompression_session_ = nullptr; | |
| 258 } | |
| 259 } | |
| 260 | |
| 261 void H264VideoToolboxDecoder::SetVideoFormat( | |
| 262 CMVideoFormatDescriptionRef video_format) { | |
| 263 if (video_format_ == video_format) { | |
| 264 return; | |
| 265 } | |
| 266 if (video_format_) { | |
| 267 CFRelease(video_format_); | |
| 268 } | |
| 269 video_format_ = video_format; | |
| 270 if (video_format_) { | |
| 271 CFRetain(video_format_); | |
| 272 } | |
| 273 } | |
| 274 | |
| 275 const char* H264VideoToolboxDecoder::ImplementationName() const { | |
| 276 return "VideoToolbox"; | |
| 277 } | |
| 278 | |
| 279 } // namespace webrtc | |
| 280 | |
| 281 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2487723004:
Reland of Add a webrtc{en,de}coderfactory implementation for VideoToolbox (Closed)
