| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 * | |
| 10 */ | |
| 11 | |
| 12 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h" | |
| 13 | |
| 14 #include <memory> | |
| 15 | |
| 16 #include "libyuv/convert.h" | |
| 17 #include "webrtc/api/video/video_frame.h" | |
| 18 #include "webrtc/common_video/include/video_frame.h" | |
| 19 #include "webrtc/rtc_base/checks.h" | |
| 20 #include "webrtc/rtc_base/logging.h" | |
| 21 #include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" | |
| 22 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" | |
| 23 | |
| 24 #import "WebRTC/RTCVideoFrameBuffer.h" | |
| 25 | |
| 26 #if defined(WEBRTC_IOS) | |
| 27 #import "Common/RTCUIApplicationStatusObserver.h" | |
| 28 #endif | |
| 29 | |
| 30 namespace webrtc { | |
| 31 namespace { | |
| 32 | |
| 33 static const int64_t kMsPerSec = 1000; | |
| 34 | |
| 35 // Convenience function for creating a dictionary. | |
| 36 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
| 37 CFTypeRef* values, | |
| 38 size_t size) { | |
| 39 return CFDictionaryCreate(nullptr, keys, values, size, | |
| 40 &kCFTypeDictionaryKeyCallBacks, | |
| 41 &kCFTypeDictionaryValueCallBacks); | |
| 42 } | |
| 43 | |
| 44 // Struct that we pass to the decoder per frame to decode. We receive it again | |
| 45 // in the decoder callback. | |
| 46 struct FrameDecodeParams { | |
| 47 FrameDecodeParams(DecodedImageCallback* cb, int64_t ts) | |
| 48 : callback(cb), timestamp(ts) {} | |
| 49 DecodedImageCallback* callback; | |
| 50 int64_t timestamp; | |
| 51 }; | |
| 52 | |
| 53 // This is the callback function that VideoToolbox calls when decode is | |
| 54 // complete. | |
| 55 void VTDecompressionOutputCallback(void* decoder, | |
| 56 void* params, | |
| 57 OSStatus status, | |
| 58 VTDecodeInfoFlags info_flags, | |
| 59 CVImageBufferRef image_buffer, | |
| 60 CMTime timestamp, | |
| 61 CMTime duration) { | |
| 62 std::unique_ptr<FrameDecodeParams> decode_params( | |
| 63 reinterpret_cast<FrameDecodeParams*>(params)); | |
| 64 if (status != noErr) { | |
| 65 LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; | |
| 66 return; | |
| 67 } | |
| 68 // TODO(tkchin): Handle CVO properly. | |
| 69 rtc::scoped_refptr<VideoFrameBuffer> buffer = new rtc::RefCountedObject<ObjCFr
ameBuffer>( | |
| 70 [[RTCCVPixelBuffer alloc] initWithPixelBuffer:image_buffer]); | |
| 71 VideoFrame decoded_frame(buffer, decode_params->timestamp, | |
| 72 CMTimeGetSeconds(timestamp) * kMsPerSec, | |
| 73 kVideoRotation_0); | |
| 74 decode_params->callback->Decoded(decoded_frame); | |
| 75 } | |
| 76 | |
| 77 } // namespace | |
| 78 | |
| 79 H264VideoToolboxDecoder::H264VideoToolboxDecoder() | |
| 80 : callback_(nullptr), video_format_(nullptr), decompression_session_(nullptr
) {} | |
| 81 | |
| 82 H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { | |
| 83 DestroyDecompressionSession(); | |
| 84 SetVideoFormat(nullptr); | |
| 85 } | |
| 86 | |
| 87 int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, | |
| 88 int number_of_cores) { | |
| 89 return WEBRTC_VIDEO_CODEC_OK; | |
| 90 } | |
| 91 | |
| 92 int H264VideoToolboxDecoder::Decode( | |
| 93 const EncodedImage& input_image, | |
| 94 bool missing_frames, | |
| 95 const RTPFragmentationHeader* fragmentation, | |
| 96 const CodecSpecificInfo* codec_specific_info, | |
| 97 int64_t render_time_ms) { | |
| 98 RTC_DCHECK(input_image._buffer); | |
| 99 | |
| 100 #if defined(WEBRTC_IOS) | |
| 101 if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { | |
| 102 // Ignore all decode requests when app isn't active. In this state, the | |
| 103 // hardware decoder has been invalidated by the OS. | |
| 104 // Reset video format so that we won't process frames until the next | |
| 105 // keyframe. | |
| 106 SetVideoFormat(nullptr); | |
| 107 return WEBRTC_VIDEO_CODEC_NO_OUTPUT; | |
| 108 } | |
| 109 #endif | |
| 110 CMVideoFormatDescriptionRef input_format = nullptr; | |
| 111 if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, | |
| 112 input_image._length)) { | |
| 113 input_format = CreateVideoFormatDescription(input_image._buffer, | |
| 114 input_image._length); | |
| 115 if (input_format) { | |
| 116 // Check if the video format has changed, and reinitialize decoder if | |
| 117 // needed. | |
| 118 if (!CMFormatDescriptionEqual(input_format, video_format_)) { | |
| 119 SetVideoFormat(input_format); | |
| 120 ResetDecompressionSession(); | |
| 121 } | |
| 122 CFRelease(input_format); | |
| 123 } | |
| 124 } | |
| 125 if (!video_format_) { | |
| 126 // We received a frame but we don't have format information so we can't | |
| 127 // decode it. | |
| 128 // This can happen after backgrounding. We need to wait for the next | |
| 129 // sps/pps before we can resume so we request a keyframe by returning an | |
| 130 // error. | |
| 131 LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; | |
| 132 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 133 } | |
| 134 CMSampleBufferRef sample_buffer = nullptr; | |
| 135 if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, | |
| 136 input_image._length, video_format_, | |
| 137 &sample_buffer)) { | |
| 138 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 139 } | |
| 140 RTC_DCHECK(sample_buffer); | |
| 141 VTDecodeFrameFlags decode_flags = | |
| 142 kVTDecodeFrame_EnableAsynchronousDecompression; | |
| 143 std::unique_ptr<FrameDecodeParams> frame_decode_params; | |
| 144 frame_decode_params.reset( | |
| 145 new FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 146 OSStatus status = VTDecompressionSessionDecodeFrame( | |
| 147 decompression_session_, sample_buffer, decode_flags, | |
| 148 frame_decode_params.release(), nullptr); | |
| 149 #if defined(WEBRTC_IOS) | |
| 150 // Re-initialize the decoder if we have an invalid session while the app is | |
| 151 // active and retry the decode request. | |
| 152 if (status == kVTInvalidSessionErr && | |
| 153 ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { | |
| 154 frame_decode_params.reset( | |
| 155 new FrameDecodeParams(callback_, input_image._timeStamp)); | |
| 156 status = VTDecompressionSessionDecodeFrame( | |
| 157 decompression_session_, sample_buffer, decode_flags, | |
| 158 frame_decode_params.release(), nullptr); | |
| 159 } | |
| 160 #endif | |
| 161 CFRelease(sample_buffer); | |
| 162 if (status != noErr) { | |
| 163 LOG(LS_ERROR) << "Failed to decode frame with code: " << status; | |
| 164 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 165 } | |
| 166 return WEBRTC_VIDEO_CODEC_OK; | |
| 167 } | |
| 168 | |
| 169 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( | |
| 170 DecodedImageCallback* callback) { | |
| 171 RTC_DCHECK(!callback_); | |
| 172 callback_ = callback; | |
| 173 return WEBRTC_VIDEO_CODEC_OK; | |
| 174 } | |
| 175 | |
| 176 int H264VideoToolboxDecoder::Release() { | |
| 177 // Need to invalidate the session so that callbacks no longer occur and it | |
| 178 // is safe to null out the callback. | |
| 179 DestroyDecompressionSession(); | |
| 180 SetVideoFormat(nullptr); | |
| 181 callback_ = nullptr; | |
| 182 return WEBRTC_VIDEO_CODEC_OK; | |
| 183 } | |
| 184 | |
| 185 int H264VideoToolboxDecoder::ResetDecompressionSession() { | |
| 186 DestroyDecompressionSession(); | |
| 187 | |
| 188 // Need to wait for the first SPS to initialize decoder. | |
| 189 if (!video_format_) { | |
| 190 return WEBRTC_VIDEO_CODEC_OK; | |
| 191 } | |
| 192 | |
| 193 // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder | |
| 194 // create pixel buffers with GPU backed memory. The intent here is to pass | |
| 195 // the pixel buffers directly so we avoid a texture upload later during | |
| 196 // rendering. This currently is moot because we are converting back to an | |
| 197 // I420 frame after decode, but eventually we will be able to plumb | |
| 198 // CVPixelBuffers directly to the renderer. | |
| 199 // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that | |
| 200 // we can pass CVPixelBuffers as native handles in decoder output. | |
| 201 static size_t const attributes_size = 3; | |
| 202 CFTypeRef keys[attributes_size] = { | |
| 203 #if defined(WEBRTC_IOS) | |
| 204 kCVPixelBufferOpenGLESCompatibilityKey, | |
| 205 #elif defined(WEBRTC_MAC) | |
| 206 kCVPixelBufferOpenGLCompatibilityKey, | |
| 207 #endif | |
| 208 kCVPixelBufferIOSurfacePropertiesKey, | |
| 209 kCVPixelBufferPixelFormatTypeKey | |
| 210 }; | |
| 211 CFDictionaryRef io_surface_value = CreateCFDictionary(nullptr, nullptr, 0); | |
| 212 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
| 213 CFNumberRef pixel_format = | |
| 214 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
| 215 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
| 216 pixel_format}; | |
| 217 CFDictionaryRef attributes = | |
| 218 CreateCFDictionary(keys, values, attributes_size); | |
| 219 if (io_surface_value) { | |
| 220 CFRelease(io_surface_value); | |
| 221 io_surface_value = nullptr; | |
| 222 } | |
| 223 if (pixel_format) { | |
| 224 CFRelease(pixel_format); | |
| 225 pixel_format = nullptr; | |
| 226 } | |
| 227 VTDecompressionOutputCallbackRecord record = { | |
| 228 VTDecompressionOutputCallback, this, | |
| 229 }; | |
| 230 OSStatus status = | |
| 231 VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, | |
| 232 &record, &decompression_session_); | |
| 233 CFRelease(attributes); | |
| 234 if (status != noErr) { | |
| 235 DestroyDecompressionSession(); | |
| 236 return WEBRTC_VIDEO_CODEC_ERROR; | |
| 237 } | |
| 238 ConfigureDecompressionSession(); | |
| 239 | |
| 240 return WEBRTC_VIDEO_CODEC_OK; | |
| 241 } | |
| 242 | |
| 243 void H264VideoToolboxDecoder::ConfigureDecompressionSession() { | |
| 244 RTC_DCHECK(decompression_session_); | |
| 245 #if defined(WEBRTC_IOS) | |
| 246 VTSessionSetProperty(decompression_session_, | |
| 247 kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); | |
| 248 #endif | |
| 249 } | |
| 250 | |
| 251 void H264VideoToolboxDecoder::DestroyDecompressionSession() { | |
| 252 if (decompression_session_) { | |
| 253 VTDecompressionSessionInvalidate(decompression_session_); | |
| 254 CFRelease(decompression_session_); | |
| 255 decompression_session_ = nullptr; | |
| 256 } | |
| 257 } | |
| 258 | |
| 259 void H264VideoToolboxDecoder::SetVideoFormat( | |
| 260 CMVideoFormatDescriptionRef video_format) { | |
| 261 if (video_format_ == video_format) { | |
| 262 return; | |
| 263 } | |
| 264 if (video_format_) { | |
| 265 CFRelease(video_format_); | |
| 266 } | |
| 267 video_format_ = video_format; | |
| 268 if (video_format_) { | |
| 269 CFRetain(video_format_); | |
| 270 } | |
| 271 } | |
| 272 | |
| 273 const char* H264VideoToolboxDecoder::ImplementationName() const { | |
| 274 return "VideoToolbox"; | |
| 275 } | |
| 276 | |
| 277 } // namespace webrtc | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2987413002:
ObjC: Implement HW codecs in ObjC instead of C++ (Closed)
