| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 * | |
| 10 */ | |
| 11 | |
| 12 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_decoder.h" | |
| 13 | |
| 14 #include <memory> | |
| 15 | |
| 16 #if defined(WEBRTC_IOS) | |
| 17 #include "RTCUIApplication.h" | |
| 18 #endif | |
| 19 #include "libyuv/convert.h" | |
| 20 #include "webrtc/api/video/video_frame.h" | |
| 21 #include "webrtc/base/checks.h" | |
| 22 #include "webrtc/base/logging.h" | |
| 23 #include "webrtc/common_video/include/video_frame.h" | |
| 24 #include "webrtc/sdk/objc/Framework/Classes/corevideo_frame_buffer.h" | |
| 25 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_nalu.h" | |
| 26 | |
| 27 namespace webrtc { | |
| 28 namespace { | |
| 29 | |
| 30 static const int64_t kMsPerSec = 1000; | |
| 31 | |
| 32 // Convenience function for creating a dictionary. | |
| 33 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
| 34 CFTypeRef* values, | |
| 35 size_t size) { | |
| 36 return CFDictionaryCreate(nullptr, keys, values, size, | |
| 37 &kCFTypeDictionaryKeyCallBacks, | |
| 38 &kCFTypeDictionaryValueCallBacks); | |
| 39 } | |
| 40 | |
| 41 // Struct that we pass to the decoder per frame to decode. We receive it again | |
| 42 // in the decoder callback. | |
| 43 struct FrameDecodeParams { | |
| 44 FrameDecodeParams(DecodedImageCallback* cb, int64_t ts) | |
| 45 : callback(cb), timestamp(ts) {} | |
| 46 DecodedImageCallback* callback; | |
| 47 int64_t timestamp; | |
| 48 }; | |
| 49 | |
| 50 // This is the callback function that VideoToolbox calls when decode is | |
| 51 // complete. | |
| 52 void VTDecompressionOutputCallback(void* decoder, | |
| 53 void* params, | |
| 54 OSStatus status, | |
| 55 VTDecodeInfoFlags info_flags, | |
| 56 CVImageBufferRef image_buffer, | |
| 57 CMTime timestamp, | |
| 58 CMTime duration) { | |
| 59 std::unique_ptr<FrameDecodeParams> decode_params( | |
| 60 reinterpret_cast<FrameDecodeParams*>(params)); | |
| 61 if (status != noErr) { | |
| 62 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; | |
| 63 return; | |
| 64 } | |
| 65 // TODO(tkchin): Handle CVO properly. | |
| 66 rtc::scoped_refptr<VideoFrameBuffer> buffer = | |
| 67 new rtc::RefCountedObject<CoreVideoFrameBuffer>(image_buffer); | |
| 68 VideoFrame decoded_frame(buffer, decode_params->timestamp, | |
| 69 CMTimeGetSeconds(timestamp) * kMsPerSec, | |
| 70 kVideoRotation_0); | |
| 71 decode_params->callback->Decoded(decoded_frame); | |
| 72 } | |
| 73 | |
| 74 } // namespace | |
| 75 | |
| 76 H264VideoToolboxDecoder::H264VideoToolboxDecoder() | |
| 77 : callback_(nullptr), | |
| 78 video_format_(nullptr), | |
| 79 decompression_session_(nullptr) {} | |
| 80 | |
| 81 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { | |
| 82 DestroyDecompressionSession(); | |
| 83 SetVideoFormat(nullptr); | |
| 84 } | |
| 85 | |
| 86 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, | |
| 87 int number_of_cores) { | |
| 88 return WEBRTC_VIDEO_CODEC_OK; | |
| 89 } | |
| 90 | |
| 91 int H264VideoToolboxDecoder::Decode( | |
| 92 const EncodedImage& input_image, | |
| 93 bool missing_frames, | |
| 94 const RTPFragmentationHeader* fragmentation, | |
| 95 const CodecSpecificInfo* codec_specific_info, | |
| 96 int64_t render_time_ms) { | |
| 97 RTC_DCHECK(input_image._buffer); | |
| 98 | |
| 99 #if defined(WEBRTC_IOS) | |
| 100 if (!RTCIsUIApplicationActive()) { | |
| 101 // Ignore all decode requests when app isn't active. In this state, the | |
| 102 // hardware decoder has been invalidated by the OS. | |
| 103 // Reset video format so that we won't process frames until the next | |
| 104 // keyframe. | |
| 105 SetVideoFormat(nullptr); | |
| 106 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; | |
| 107 } | |
| 108 #endif | |
| 109 CMVideoFormatDescriptionRef input_format = nullptr; | |
| 110 if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, | |
| 111 input_image._length)) { | |
| 112 input_format = CreateVideoFormatDescription(input_image._buffer, | |
| 113 input_image._length); | |
| 114 if (input_format) { | |
| 115 // Check if the video format has changed, and reinitialize decoder if | |
| 116 // needed. | |
| 117 if (!CMFormatDescriptionEqual(input_format, video_format_)) { | |
| 118 SetVideoFormat(input_format); | |
| 119 ResetDecompressionSession(); | |
| 120 } | |
| 121 CFRelease(input_format); | |
| 122 } | |
| 123 } | |
| 124 if (!video_format_) { | |
| 125 // We received a frame but we don't have format information so we can't | |
| 126 // decode it. | |
| 127 // This can happen after backgrounding. We need to wait for the next | |
| 128 // sps/pps before we can resume so we request a keyframe by returning an | |
| 129 // error. | |
| 130 LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; | |
| 131 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 132 } | |
| 133 CMSampleBufferRef sample_buffer = nullptr; | |
| 134 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, | |
| 135 input_image._length, video_format_, | |
| 136 &sample_buffer)) { | |
| 137 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 138 } | |
| 139 RTC_DCHECK(sample_buffer); | |
| 140 VTDecodeFrameFlags decode_flags = | |
| 141 kVTDecodeFrame_EnableAsynchronousDecompression; | |
| 142 std::unique_ptr<FrameDecodeParams> frame_decode_params; | |
| 143 frame_decode_params.reset( | |
| 144 new FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 145 OSStatus status = VTDecompressionSessionDecodeFrame( | |
| 146 decompression_session_, sample_buffer, decode_flags, | |
| 147 frame_decode_params.release(), nullptr); | |
| 148 #if defined(WEBRTC_IOS) | |
| 149 // Re-initialize the decoder if we have an invalid session while the app is | |
| 150 // active and retry the decode request. | |
| 151 if (status == kVTInvalidSessionErr && | |
| 152 ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { | |
| 153 frame_decode_params.reset( | |
| 154 new FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 155 status = VTDecompressionSessionDecodeFrame( | |
| 156 decompression_session_, sample_buffer, decode_flags, | |
| 157 frame_decode_params.release(), nullptr); | |
| 158 } | |
| 159 #endif | |
| 160 CFRelease(sample_buffer); | |
| 161 if (status != noErr) { | |
| 162 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; | |
| 163 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 164 } | |
| 165 return WEBRTC_VIDEO_CODEC_OK; | |
| 166 } | |
| 167 | |
| 168 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( | |
| 169 DecodedImageCallback* callback) { | |
| 170 RTC_DCHECK(!callback_); | |
| 171 callback_ = callback; | |
| 172 return WEBRTC_VIDEO_CODEC_OK; | |
| 173 } | |
| 174 | |
| 175 int H264VideoToolboxDecoder::Release() { | |
| 176 // Need to invalidate the session so that callbacks no longer occur and it | |
| 177 // is safe to null out the callback. | |
| 178 DestroyDecompressionSession(); | |
| 179 SetVideoFormat(nullptr); | |
| 180 callback_ = nullptr; | |
| 181 return WEBRTC_VIDEO_CODEC_OK; | |
| 182 } | |
| 183 | |
| 184 int H264VideoToolboxDecoder::ResetDecompressionSession() { | |
| 185 DestroyDecompressionSession(); | |
| 186 | |
| 187 // Need to wait for the first SPS to initialize decoder. | |
| 188 if (!video_format_) { | |
| 189 return WEBRTC_VIDEO_CODEC_OK; | |
| 190 } | |
| 191 | |
| 192 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder | |
| 193 // create pixel buffers with GPU backed memory. The intent here is to pass | |
| 194 // the pixel buffers directly so we avoid a texture upload later during | |
| 195 // rendering. This currently is moot because we are converting back to an | |
| 196 // I420 frame after decode, but eventually we will be able to plumb | |
| 197 // CVPixelBuffers directly to the renderer. | |
| 198 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that | |
| 199 // we can pass CVPixelBuffers as native handles in decoder output. | |
| 200 static size_t const attributes_size = 3; | |
| 201 CFTypeRef keys[attributes_size] = { | |
| 202 #if defined(WEBRTC_IOS) | |
| 203 kCVPixelBufferOpenGLESCompatibilityKey, | |
| 204 #elif defined(WEBRTC_MAC) | |
| 205 kCVPixelBufferOpenGLCompatibilityKey, | |
| 206 #endif | |
| 207 kCVPixelBufferIOSurfacePropertiesKey, | |
| 208 kCVPixelBufferPixelFormatTypeKey | |
| 209 }; | |
| 210 CFDictionaryRef io_surface_value = CreateCFDictionary(nullptr, nullptr, 0); | |
| 211 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
| 212 CFNumberRef pixel_format = | |
| 213 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
| 214 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
| 215 pixel_format}; | |
| 216 CFDictionaryRef attributes = | |
| 217 CreateCFDictionary(keys, values, attributes_size); | |
| 218 if (io_surface_value) { | |
| 219 CFRelease(io_surface_value); | |
| 220 io_surface_value = nullptr; | |
| 221 } | |
| 222 if (pixel_format) { | |
| 223 CFRelease(pixel_format); | |
| 224 pixel_format = nullptr; | |
| 225 } | |
| 226 VTDecompressionOutputCallbackRecord record = { | |
| 227 VTDecompressionOutputCallback, this, | |
| 228 }; | |
| 229 OSStatus status = | |
| 230 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, | |
| 231 &record, &decompression_session_); | |
| 232 CFRelease(attributes); | |
| 233 if (status != noErr) { | |
| 234 DestroyDecompressionSession(); | |
| 235 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 236 } | |
| 237 ConfigureDecompressionSession(); | |
| 238 | |
| 239 return WEBRTC_VIDEO_CODEC_OK; | |
| 240 } | |
| 241 | |
| 242 void H264VideoToolboxDecoder::ConfigureDecompressionSession() { | |
| 243 RTC_DCHECK(decompression_session_); | |
| 244 #if defined(WEBRTC_IOS) | |
| 245 VTSessionSetProperty(decompression_session_, | |
| 246 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); | |
| 247 #endif | |
| 248 } | |
| 249 | |
| 250 void H264VideoToolboxDecoder::DestroyDecompressionSession() { | |
| 251 if (decompression_session_) { | |
| 252 VTDecompressionSessionInvalidate(decompression_session_); | |
| 253 CFRelease(decompression_session_); | |
| 254 decompression_session_ = nullptr; | |
| 255 } | |
| 256 } | |
| 257 | |
| 258 void H264VideoToolboxDecoder::SetVideoFormat( | |
| 259 CMVideoFormatDescriptionRef video_format) { | |
| 260 if (video_format_ == video_format) { | |
| 261 return; | |
| 262 } | |
| 263 if (video_format_) { | |
| 264 CFRelease(video_format_); | |
| 265 } | |
| 266 video_format_ = video_format; | |
| 267 if (video_format_) { | |
| 268 CFRetain(video_format_); | |
| 269 } | |
| 270 } | |
| 271 | |
| 272 const char* H264VideoToolboxDecoder::ImplementationName() const { | |
| 273 return "VideoToolbox"; | |
| 274 } | |
| 275 | |
| 276 } // namespace webrtc | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2862543002:
Split iOS sdk in to separate targets (Closed)
