| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 * | |
| 10 */ | |
| 11 | |
| 12 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_decoder.h" | |
| 13 | |
| 14 #include <memory> | |
| 15 | |
| 16 #if defined(WEBRTC_IOS) | |
| 17 #include "RTCUIApplication.h" | |
| 18 #endif | |
| 19 #include "libyuv/convert.h" | |
| 20 #include "webrtc/base/checks.h" | |
| 21 #include "webrtc/base/logging.h" | |
| 22 #include "webrtc/common_video/include/corevideo_frame_buffer.h" | |
| 23 #include "webrtc/sdk/objc/Framework/Classes/h264_video_toolbox_nalu.h" | |
| 24 #include "webrtc/video_frame.h" | |
| 25 | |
| 26 namespace internal { | |
| 27 | |
| 28 static const int64_t kMsPerSec = 1000; | |
| 29 | |
| 30 // Convenience function for creating a dictionary. | |
| 31 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
| 32 CFTypeRef* values, | |
| 33 size_t size) { | |
| 34 return CFDictionaryCreate(nullptr, keys, values, size, | |
| 35 &kCFTypeDictionaryKeyCallBacks, | |
| 36 &kCFTypeDictionaryValueCallBacks); | |
| 37 } | |
| 38 | |
| 39 // Struct that we pass to the decoder per frame to decode. We receive it again | |
| 40 // in the decoder callback. | |
| 41 struct FrameDecodeParams { | |
| 42 FrameDecodeParams(webrtc::DecodedImageCallback* cb, int64_t ts) | |
| 43 : callback(cb), timestamp(ts) {} | |
| 44 webrtc::DecodedImageCallback* callback; | |
| 45 int64_t timestamp; | |
| 46 }; | |
| 47 | |
| 48 // This is the callback function that VideoToolbox calls when decode is | |
| 49 // complete. | |
| 50 void VTDecompressionOutputCallback(void* decoder, | |
| 51 void* params, | |
| 52 OSStatus status, | |
| 53 VTDecodeInfoFlags info_flags, | |
| 54 CVImageBufferRef image_buffer, | |
| 55 CMTime timestamp, | |
| 56 CMTime duration) { | |
| 57 std::unique_ptr<FrameDecodeParams> decode_params( | |
| 58 reinterpret_cast<FrameDecodeParams*>(params)); | |
| 59 if (status != noErr) { | |
| 60 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; | |
| 61 return; | |
| 62 } | |
| 63 // TODO(tkchin): Handle CVO properly. | |
| 64 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = | |
| 65 new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(image_buffer); | |
| 66 webrtc::VideoFrame decoded_frame(buffer, decode_params->timestamp, | |
| 67 CMTimeGetSeconds(timestamp) * kMsPerSec, | |
| 68 webrtc::kVideoRotation_0); | |
| 69 decode_params->callback->Decoded(decoded_frame); | |
| 70 } | |
| 71 | |
| 72 } // namespace internal | |
| 73 | |
| 74 namespace webrtc { | |
| 75 | |
| 76 H264VideoToolboxDecoder::H264VideoToolboxDecoder() | |
| 77 : callback_(nullptr), | |
| 78 video_format_(nullptr), | |
| 79 decompression_session_(nullptr) {} | |
| 80 | |
| 81 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { | |
| 82 DestroyDecompressionSession(); | |
| 83 SetVideoFormat(nullptr); | |
| 84 } | |
| 85 | |
| 86 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, | |
| 87 int number_of_cores) { | |
| 88 return WEBRTC_VIDEO_CODEC_OK; | |
| 89 } | |
| 90 | |
| 91 int H264VideoToolboxDecoder::Decode( | |
| 92 const EncodedImage& input_image, | |
| 93 bool missing_frames, | |
| 94 const RTPFragmentationHeader* fragmentation, | |
| 95 const CodecSpecificInfo* codec_specific_info, | |
| 96 int64_t render_time_ms) { | |
| 97 RTC_DCHECK(input_image._buffer); | |
| 98 | |
| 99 #if defined(WEBRTC_IOS) | |
| 100 if (!RTCIsUIApplicationActive()) { | |
| 101 // Ignore all decode requests when app isn't active. In this state, the | |
| 102 // hardware decoder has been invalidated by the OS. | |
| 103 // Reset video format so that we won't process frames until the next | |
| 104 // keyframe. | |
| 105 SetVideoFormat(nullptr); | |
| 106 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; | |
| 107 } | |
| 108 #endif | |
| 109 CMVideoFormatDescriptionRef input_format = nullptr; | |
| 110 if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, | |
| 111 input_image._length)) { | |
| 112 input_format = CreateVideoFormatDescription(input_image._buffer, | |
| 113 input_image._length); | |
| 114 if (input_format) { | |
| 115 // Check if the video format has changed, and reinitialize decoder if | |
| 116 // needed. | |
| 117 if (!CMFormatDescriptionEqual(input_format, video_format_)) { | |
| 118 SetVideoFormat(input_format); | |
| 119 ResetDecompressionSession(); | |
| 120 } | |
| 121 CFRelease(input_format); | |
| 122 } | |
| 123 } | |
| 124 if (!video_format_) { | |
| 125 // We received a frame but we don't have format information so we can't | |
| 126 // decode it. | |
| 127 // This can happen after backgrounding. We need to wait for the next | |
| 128 // sps/pps before we can resume so we request a keyframe by returning an | |
| 129 // error. | |
| 130 LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; | |
| 131 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 132 } | |
| 133 CMSampleBufferRef sample_buffer = nullptr; | |
| 134 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, | |
| 135 input_image._length, video_format_, | |
| 136 &sample_buffer)) { | |
| 137 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 138 } | |
| 139 RTC_DCHECK(sample_buffer); | |
| 140 VTDecodeFrameFlags decode_flags = | |
| 141 kVTDecodeFrame_EnableAsynchronousDecompression; | |
| 142 std::unique_ptr<internal::FrameDecodeParams> frame_decode_params; | |
| 143 frame_decode_params.reset( | |
| 144 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 145 OSStatus status = VTDecompressionSessionDecodeFrame( | |
| 146 decompression_session_, sample_buffer, decode_flags, | |
| 147 frame_decode_params.release(), nullptr); | |
| 148 #if defined(WEBRTC_IOS) | |
| 149 // Re-initialize the decoder if we have an invalid session while the app is | |
| 150 // active and retry the decode request. | |
| 151 if (status == kVTInvalidSessionErr && | |
| 152 ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { | |
| 153 frame_decode_params.reset( | |
| 154 new internal::FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 155 status = VTDecompressionSessionDecodeFrame( | |
| 156 decompression_session_, sample_buffer, decode_flags, | |
| 157 frame_decode_params.release(), nullptr); | |
| 158 } | |
| 159 #endif | |
| 160 CFRelease(sample_buffer); | |
| 161 if (status != noErr) { | |
| 162 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; | |
| 163 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 164 } | |
| 165 return WEBRTC_VIDEO_CODEC_OK; | |
| 166 } | |
| 167 | |
| 168 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( | |
| 169 DecodedImageCallback* callback) { | |
| 170 RTC_DCHECK(!callback_); | |
| 171 callback_ = callback; | |
| 172 return WEBRTC_VIDEO_CODEC_OK; | |
| 173 } | |
| 174 | |
| 175 int H264VideoToolboxDecoder::Release() { | |
| 176 // Need to invalidate the session so that callbacks no longer occur and it | |
| 177 // is safe to null out the callback. | |
| 178 DestroyDecompressionSession(); | |
| 179 SetVideoFormat(nullptr); | |
| 180 callback_ = nullptr; | |
| 181 return WEBRTC_VIDEO_CODEC_OK; | |
| 182 } | |
| 183 | |
| 184 int H264VideoToolboxDecoder::ResetDecompressionSession() { | |
| 185 DestroyDecompressionSession(); | |
| 186 | |
| 187 // Need to wait for the first SPS to initialize decoder. | |
| 188 if (!video_format_) { | |
| 189 return WEBRTC_VIDEO_CODEC_OK; | |
| 190 } | |
| 191 | |
| 192 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder | |
| 193 // create pixel buffers with GPU backed memory. The intent here is to pass | |
| 194 // the pixel buffers directly so we avoid a texture upload later during | |
| 195 // rendering. This currently is moot because we are converting back to an | |
| 196 // I420 frame after decode, but eventually we will be able to plumb | |
| 197 // CVPixelBuffers directly to the renderer. | |
| 198 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that | |
| 199 // we can pass CVPixelBuffers as native handles in decoder output. | |
| 200 static size_t const attributes_size = 3; | |
| 201 CFTypeRef keys[attributes_size] = { | |
| 202 #if defined(WEBRTC_IOS) | |
| 203 kCVPixelBufferOpenGLESCompatibilityKey, | |
| 204 #elif defined(WEBRTC_MAC) | |
| 205 kCVPixelBufferOpenGLCompatibilityKey, | |
| 206 #endif | |
| 207 kCVPixelBufferIOSurfacePropertiesKey, | |
| 208 kCVPixelBufferPixelFormatTypeKey | |
| 209 }; | |
| 210 CFDictionaryRef io_surface_value = | |
| 211 internal::CreateCFDictionary(nullptr, nullptr, 0); | |
| 212 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
| 213 CFNumberRef pixel_format = | |
| 214 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
| 215 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
| 216 pixel_format}; | |
| 217 CFDictionaryRef attributes = | |
| 218 internal::CreateCFDictionary(keys, values, attributes_size); | |
| 219 if (io_surface_value) { | |
| 220 CFRelease(io_surface_value); | |
| 221 io_surface_value = nullptr; | |
| 222 } | |
| 223 if (pixel_format) { | |
| 224 CFRelease(pixel_format); | |
| 225 pixel_format = nullptr; | |
| 226 } | |
| 227 VTDecompressionOutputCallbackRecord record = { | |
| 228 internal::VTDecompressionOutputCallback, this, | |
| 229 }; | |
| 230 OSStatus status = | |
| 231 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, | |
| 232 &record, &decompression_session_); | |
| 233 CFRelease(attributes); | |
| 234 if (status != noErr) { | |
| 235 DestroyDecompressionSession(); | |
| 236 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 237 } | |
| 238 ConfigureDecompressionSession(); | |
| 239 | |
| 240 return WEBRTC_VIDEO_CODEC_OK; | |
| 241 } | |
| 242 | |
| 243 void H264VideoToolboxDecoder::ConfigureDecompressionSession() { | |
| 244 RTC_DCHECK(decompression_session_); | |
| 245 #if defined(WEBRTC_IOS) | |
| 246 VTSessionSetProperty(decompression_session_, | |
| 247 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); | |
| 248 #endif | |
| 249 } | |
| 250 | |
| 251 void H264VideoToolboxDecoder::DestroyDecompressionSession() { | |
| 252 if (decompression_session_) { | |
| 253 VTDecompressionSessionInvalidate(decompression_session_); | |
| 254 CFRelease(decompression_session_); | |
| 255 decompression_session_ = nullptr; | |
| 256 } | |
| 257 } | |
| 258 | |
| 259 void H264VideoToolboxDecoder::SetVideoFormat( | |
| 260 CMVideoFormatDescriptionRef video_format) { | |
| 261 if (video_format_ == video_format) { | |
| 262 return; | |
| 263 } | |
| 264 if (video_format_) { | |
| 265 CFRelease(video_format_); | |
| 266 } | |
| 267 video_format_ = video_format; | |
| 268 if (video_format_) { | |
| 269 CFRetain(video_format_); | |
| 270 } | |
| 271 } | |
| 272 | |
| 273 const char* H264VideoToolboxDecoder::ImplementationName() const { | |
| 274 return "VideoToolbox"; | |
| 275 } | |
| 276 | |
| 277 } // namespace webrtc | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2483273002:
Revert of Add a webrtc{en,de}coderfactory implementation for VideoToolbox (Closed)
