OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 * | |
10 */ | |
11 | |
12 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h" | |
13 | |
14 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) | |
15 | |
16 #include <memory> | |
17 #include <string> | |
18 #include <vector> | |
19 | |
20 #if defined(WEBRTC_IOS) | |
21 #import "WebRTC/UIDevice+RTCDevice.h" | |
22 #include "RTCUIApplication.h" | |
23 #endif | |
24 #include "libyuv/convert_from.h" | |
25 #include "webrtc/base/checks.h" | |
26 #include "webrtc/base/logging.h" | |
27 #include "webrtc/common_video/include/corevideo_frame_buffer.h" | |
28 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" | |
29 #include "webrtc/system_wrappers/include/clock.h" | |
30 | |
31 namespace internal { | |
32 | |
33 // The ratio between kVTCompressionPropertyKey_DataRateLimits and | |
34 // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher | |
35 // than the average bit rate to avoid undershooting the target. | |
36 const float kLimitToAverageBitRateFactor = 1.5f; | |
37 // These thresholds deviate from the default h264 QP thresholds, as they | |
38 // have been found to work better on devices that support VideoToolbox | |
39 const int kLowH264QpThreshold = 28; | |
40 const int kHighH264QpThreshold = 39; | |
41 | |
42 // Convenience function for creating a dictionary. | |
43 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, | |
44 CFTypeRef* values, | |
45 size_t size) { | |
46 return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, | |
47 &kCFTypeDictionaryKeyCallBacks, | |
48 &kCFTypeDictionaryValueCallBacks); | |
49 } | |
50 | |
51 // Copies characters from a CFStringRef into a std::string. | |
52 std::string CFStringToString(const CFStringRef cf_string) { | |
53 RTC_DCHECK(cf_string); | |
54 std::string std_string; | |
55 // Get the size needed for UTF8 plus terminating character. | |
56 size_t buffer_size = | |
57 CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), | |
58 kCFStringEncodingUTF8) + | |
59 1; | |
60 std::unique_ptr<char[]> buffer(new char[buffer_size]); | |
61 if (CFStringGetCString(cf_string, buffer.get(), buffer_size, | |
62 kCFStringEncodingUTF8)) { | |
63 // Copy over the characters. | |
64 std_string.assign(buffer.get()); | |
65 } | |
66 return std_string; | |
67 } | |
68 | |
69 // Convenience function for setting a VT property. | |
70 void SetVTSessionProperty(VTSessionRef session, | |
71 CFStringRef key, | |
72 int32_t value) { | |
73 CFNumberRef cfNum = | |
74 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); | |
75 OSStatus status = VTSessionSetProperty(session, key, cfNum); | |
76 CFRelease(cfNum); | |
77 if (status != noErr) { | |
78 std::string key_string = CFStringToString(key); | |
79 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
80 << " to " << value << ": " << status; | |
81 } | |
82 } | |
83 | |
84 // Convenience function for setting a VT property. | |
85 void SetVTSessionProperty(VTSessionRef session, | |
86 CFStringRef key, | |
87 uint32_t value) { | |
88 int64_t value_64 = value; | |
89 CFNumberRef cfNum = | |
90 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); | |
91 OSStatus status = VTSessionSetProperty(session, key, cfNum); | |
92 CFRelease(cfNum); | |
93 if (status != noErr) { | |
94 std::string key_string = CFStringToString(key); | |
95 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
96 << " to " << value << ": " << status; | |
97 } | |
98 } | |
99 | |
100 // Convenience function for setting a VT property. | |
101 void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { | |
102 CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; | |
103 OSStatus status = VTSessionSetProperty(session, key, cf_bool); | |
104 if (status != noErr) { | |
105 std::string key_string = CFStringToString(key); | |
106 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
107 << " to " << value << ": " << status; | |
108 } | |
109 } | |
110 | |
111 // Convenience function for setting a VT property. | |
112 void SetVTSessionProperty(VTSessionRef session, | |
113 CFStringRef key, | |
114 CFStringRef value) { | |
115 OSStatus status = VTSessionSetProperty(session, key, value); | |
116 if (status != noErr) { | |
117 std::string key_string = CFStringToString(key); | |
118 std::string val_string = CFStringToString(value); | |
119 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string | |
120 << " to " << val_string << ": " << status; | |
121 } | |
122 } | |
123 | |
124 // Struct that we pass to the encoder per frame to encode. We receive it again | |
125 // in the encoder callback. | |
126 struct FrameEncodeParams { | |
127 FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e, | |
128 const webrtc::CodecSpecificInfo* csi, | |
129 int32_t w, | |
130 int32_t h, | |
131 int64_t rtms, | |
132 uint32_t ts, | |
133 webrtc::VideoRotation r) | |
134 : encoder(e), | |
135 width(w), | |
136 height(h), | |
137 render_time_ms(rtms), | |
138 timestamp(ts), | |
139 rotation(r) { | |
140 if (csi) { | |
141 codec_specific_info = *csi; | |
142 } else { | |
143 codec_specific_info.codecType = webrtc::kVideoCodecH264; | |
144 } | |
145 } | |
146 | |
147 webrtc::H264VideoToolboxEncoder* encoder; | |
148 webrtc::CodecSpecificInfo codec_specific_info; | |
149 int32_t width; | |
150 int32_t height; | |
151 int64_t render_time_ms; | |
152 uint32_t timestamp; | |
153 webrtc::VideoRotation rotation; | |
154 }; | |
155 | |
156 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the | |
157 // encoder. This performs the copy and format conversion. | |
158 // TODO(tkchin): See if encoder will accept i420 frames and compare performance. | |
159 bool CopyVideoFrameToPixelBuffer( | |
160 const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& frame, | |
161 CVPixelBufferRef pixel_buffer) { | |
162 RTC_DCHECK(pixel_buffer); | |
163 RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer), | |
164 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); | |
165 RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), | |
166 static_cast<size_t>(frame->height())); | |
167 RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), | |
168 static_cast<size_t>(frame->width())); | |
169 | |
170 CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); | |
171 if (cvRet != kCVReturnSuccess) { | |
172 LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; | |
173 return false; | |
174 } | |
175 uint8_t* dst_y = reinterpret_cast<uint8_t*>( | |
176 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); | |
177 int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); | |
178 uint8_t* dst_uv = reinterpret_cast<uint8_t*>( | |
179 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); | |
180 int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); | |
181 // Convert I420 to NV12. | |
182 int ret = libyuv::I420ToNV12( | |
183 frame->DataY(), frame->StrideY(), | |
184 frame->DataU(), frame->StrideU(), | |
185 frame->DataV(), frame->StrideV(), | |
186 dst_y, dst_stride_y, dst_uv, dst_stride_uv, | |
187 frame->width(), frame->height()); | |
188 CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); | |
189 if (ret) { | |
190 LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; | |
191 return false; | |
192 } | |
193 return true; | |
194 } | |
195 | |
196 CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { | |
197 if (!pixel_buffer_pool) { | |
198 LOG(LS_ERROR) << "Failed to get pixel buffer pool."; | |
199 return nullptr; | |
200 } | |
201 CVPixelBufferRef pixel_buffer; | |
202 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, | |
203 &pixel_buffer); | |
204 if (ret != kCVReturnSuccess) { | |
205 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; | |
206 // We probably want to drop frames here, since failure probably means | |
207 // that the pool is empty. | |
208 return nullptr; | |
209 } | |
210 return pixel_buffer; | |
211 } | |
212 | |
213 // This is the callback function that VideoToolbox calls when encode is | |
214 // complete. From inspection this happens on its own queue. | |
215 void VTCompressionOutputCallback(void* encoder, | |
216 void* params, | |
217 OSStatus status, | |
218 VTEncodeInfoFlags info_flags, | |
219 CMSampleBufferRef sample_buffer) { | |
220 std::unique_ptr<FrameEncodeParams> encode_params( | |
221 reinterpret_cast<FrameEncodeParams*>(params)); | |
222 encode_params->encoder->OnEncodedFrame( | |
223 status, info_flags, sample_buffer, encode_params->codec_specific_info, | |
224 encode_params->width, encode_params->height, | |
225 encode_params->render_time_ms, encode_params->timestamp, | |
226 encode_params->rotation); | |
227 } | |
228 | |
229 } // namespace internal | |
230 | |
231 namespace webrtc { | |
232 | |
233 // .5 is set as a mininum to prevent overcompensating for large temporary | |
234 // overshoots. We don't want to degrade video quality too badly. | |
235 // .95 is set to prevent oscillations. When a lower bitrate is set on the | |
236 // encoder than previously set, its output seems to have a brief period of | |
237 // drastically reduced bitrate, so we want to avoid that. In steady state | |
238 // conditions, 0.95 seems to give us better overall bitrate over long periods | |
239 // of time. | |
240 H264VideoToolboxEncoder::H264VideoToolboxEncoder() | |
241 : callback_(nullptr), | |
242 compression_session_(nullptr), | |
243 bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95) { | |
244 } | |
245 | |
246 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { | |
247 DestroyCompressionSession(); | |
248 } | |
249 | |
250 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, | |
251 int number_of_cores, | |
252 size_t max_payload_size) { | |
253 RTC_DCHECK(codec_settings); | |
254 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); | |
255 { | |
256 rtc::CritScope lock(&quality_scaler_crit_); | |
257 quality_scaler_.Init(internal::kLowH264QpThreshold, | |
258 internal::kHighH264QpThreshold, | |
259 codec_settings->startBitrate, codec_settings->width, | |
260 codec_settings->height, codec_settings->maxFramerate); | |
261 QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); | |
262 // TODO(tkchin): We may need to enforce width/height dimension restrictions | |
263 // to match what the encoder supports. | |
264 width_ = res.width; | |
265 height_ = res.height; | |
266 } | |
267 // We can only set average bitrate on the HW encoder. | |
268 target_bitrate_bps_ = codec_settings->startBitrate; | |
269 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); | |
270 | |
271 // TODO(tkchin): Try setting payload size via | |
272 // kVTCompressionPropertyKey_MaxH264SliceBytes. | |
273 | |
274 return ResetCompressionSession(); | |
275 } | |
276 | |
277 int H264VideoToolboxEncoder::Encode( | |
278 const VideoFrame& frame, | |
279 const CodecSpecificInfo* codec_specific_info, | |
280 const std::vector<FrameType>* frame_types) { | |
281 RTC_DCHECK(!frame.IsZeroSize()); | |
282 if (!callback_ || !compression_session_) { | |
283 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | |
284 } | |
285 #if defined(WEBRTC_IOS) | |
286 if (!RTCIsUIApplicationActive()) { | |
287 // Ignore all encode requests when app isn't active. In this state, the | |
288 // hardware encoder has been invalidated by the OS. | |
289 return WEBRTC_VIDEO_CODEC_OK; | |
290 } | |
291 #endif | |
292 bool is_keyframe_required = false; | |
293 | |
294 quality_scaler_.OnEncodeFrame(frame.width(), frame.height()); | |
295 const QualityScaler::Resolution scaled_res = | |
296 quality_scaler_.GetScaledResolution(); | |
297 | |
298 if (scaled_res.width != width_ || scaled_res.height != height_) { | |
299 width_ = scaled_res.width; | |
300 height_ = scaled_res.height; | |
301 int ret = ResetCompressionSession(); | |
302 if (ret < 0) | |
303 return ret; | |
304 } | |
305 | |
306 // Get a pixel buffer from the pool and copy frame data over. | |
307 CVPixelBufferPoolRef pixel_buffer_pool = | |
308 VTCompressionSessionGetPixelBufferPool(compression_session_); | |
309 #if defined(WEBRTC_IOS) | |
310 if (!pixel_buffer_pool) { | |
311 // Kind of a hack. On backgrounding, the compression session seems to get | |
312 // invalidated, which causes this pool call to fail when the application | |
313 // is foregrounded and frames are being sent for encoding again. | |
314 // Resetting the session when this happens fixes the issue. | |
315 // In addition we request a keyframe so video can recover quickly. | |
316 ResetCompressionSession(); | |
317 pixel_buffer_pool = | |
318 VTCompressionSessionGetPixelBufferPool(compression_session_); | |
319 is_keyframe_required = true; | |
320 LOG(LS_INFO) << "Resetting compression session due to invalid pool."; | |
321 } | |
322 #endif | |
323 | |
324 CVPixelBufferRef pixel_buffer = static_cast<CVPixelBufferRef>( | |
325 frame.video_frame_buffer()->native_handle()); | |
326 if (pixel_buffer) { | |
327 // Native frame. | |
328 rtc::scoped_refptr<CoreVideoFrameBuffer> core_video_frame_buffer( | |
329 static_cast<CoreVideoFrameBuffer*>(frame.video_frame_buffer().get())); | |
330 if (!core_video_frame_buffer->RequiresCropping()) { | |
331 // This pixel buffer might have a higher resolution than what the | |
332 // compression session is configured to. The compression session can | |
333 // handle that and will output encoded frames in the configured | |
334 // resolution regardless of the input pixel buffer resolution. | |
335 CVBufferRetain(pixel_buffer); | |
336 } else { | |
337 // Cropping required, we need to crop and scale to a new pixel buffer. | |
338 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); | |
339 if (!pixel_buffer) { | |
340 return WEBRTC_VIDEO_CODEC_ERROR; | |
341 } | |
342 if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_, | |
343 pixel_buffer)) { | |
344 return WEBRTC_VIDEO_CODEC_ERROR; | |
345 } | |
346 } | |
347 } else { | |
348 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); | |
349 if (!pixel_buffer) { | |
350 return WEBRTC_VIDEO_CODEC_ERROR; | |
351 } | |
352 // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer | |
353 // conversion once libyuv::MergeUVPlanes is available. | |
354 rtc::scoped_refptr<VideoFrameBuffer> scaled_i420_buffer = | |
355 quality_scaler_.GetScaledBuffer(frame.video_frame_buffer()); | |
356 if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer, | |
357 pixel_buffer)) { | |
358 LOG(LS_ERROR) << "Failed to copy frame data."; | |
359 CVBufferRelease(pixel_buffer); | |
360 return WEBRTC_VIDEO_CODEC_ERROR; | |
361 } | |
362 } | |
363 | |
364 // Check if we need a keyframe. | |
365 if (!is_keyframe_required && frame_types) { | |
366 for (auto frame_type : *frame_types) { | |
367 if (frame_type == kVideoFrameKey) { | |
368 is_keyframe_required = true; | |
369 break; | |
370 } | |
371 } | |
372 } | |
373 | |
374 CMTime presentation_time_stamp = | |
375 CMTimeMake(frame.render_time_ms(), 1000); | |
376 CFDictionaryRef frame_properties = nullptr; | |
377 if (is_keyframe_required) { | |
378 CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; | |
379 CFTypeRef values[] = {kCFBooleanTrue}; | |
380 frame_properties = internal::CreateCFDictionary(keys, values, 1); | |
381 } | |
382 std::unique_ptr<internal::FrameEncodeParams> encode_params; | |
383 encode_params.reset(new internal::FrameEncodeParams( | |
384 this, codec_specific_info, width_, height_, frame.render_time_ms(), | |
385 frame.timestamp(), frame.rotation())); | |
386 | |
387 // Update the bitrate if needed. | |
388 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); | |
389 | |
390 OSStatus status = VTCompressionSessionEncodeFrame( | |
391 compression_session_, pixel_buffer, presentation_time_stamp, | |
392 kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); | |
393 if (frame_properties) { | |
394 CFRelease(frame_properties); | |
395 } | |
396 if (pixel_buffer) { | |
397 CVBufferRelease(pixel_buffer); | |
398 } | |
399 if (status != noErr) { | |
400 LOG(LS_ERROR) << "Failed to encode frame with code: " << status; | |
401 return WEBRTC_VIDEO_CODEC_ERROR; | |
402 } | |
403 return WEBRTC_VIDEO_CODEC_OK; | |
404 } | |
405 | |
406 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( | |
407 EncodedImageCallback* callback) { | |
408 callback_ = callback; | |
409 return WEBRTC_VIDEO_CODEC_OK; | |
410 } | |
411 | |
412 void H264VideoToolboxEncoder::OnDroppedFrame() { | |
413 rtc::CritScope lock(&quality_scaler_crit_); | |
414 quality_scaler_.ReportDroppedFrame(); | |
415 } | |
416 | |
417 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, | |
418 int64_t rtt) { | |
419 // Encoder doesn't know anything about packet loss or rtt so just return. | |
420 return WEBRTC_VIDEO_CODEC_OK; | |
421 } | |
422 | |
423 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, | |
424 uint32_t frame_rate) { | |
425 target_bitrate_bps_ = 1000 * new_bitrate_kbit; | |
426 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); | |
427 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); | |
428 | |
429 rtc::CritScope lock(&quality_scaler_crit_); | |
430 quality_scaler_.ReportFramerate(frame_rate); | |
431 | |
432 return WEBRTC_VIDEO_CODEC_OK; | |
433 } | |
434 | |
435 int H264VideoToolboxEncoder::Release() { | |
436 // Need to reset so that the session is invalidated and won't use the | |
437 // callback anymore. Do not remove callback until the session is invalidated | |
438 // since async encoder callbacks can occur until invalidation. | |
439 int ret = ResetCompressionSession(); | |
440 callback_ = nullptr; | |
441 return ret; | |
442 } | |
443 | |
444 int H264VideoToolboxEncoder::ResetCompressionSession() { | |
445 DestroyCompressionSession(); | |
446 | |
447 // Set source image buffer attributes. These attributes will be present on | |
448 // buffers retrieved from the encoder's pixel buffer pool. | |
449 const size_t attributes_size = 3; | |
450 CFTypeRef keys[attributes_size] = { | |
451 #if defined(WEBRTC_IOS) | |
452 kCVPixelBufferOpenGLESCompatibilityKey, | |
453 #elif defined(WEBRTC_MAC) | |
454 kCVPixelBufferOpenGLCompatibilityKey, | |
455 #endif | |
456 kCVPixelBufferIOSurfacePropertiesKey, | |
457 kCVPixelBufferPixelFormatTypeKey | |
458 }; | |
459 CFDictionaryRef io_surface_value = | |
460 internal::CreateCFDictionary(nullptr, nullptr, 0); | |
461 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; | |
462 CFNumberRef pixel_format = | |
463 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); | |
464 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, | |
465 pixel_format}; | |
466 CFDictionaryRef source_attributes = | |
467 internal::CreateCFDictionary(keys, values, attributes_size); | |
468 if (io_surface_value) { | |
469 CFRelease(io_surface_value); | |
470 io_surface_value = nullptr; | |
471 } | |
472 if (pixel_format) { | |
473 CFRelease(pixel_format); | |
474 pixel_format = nullptr; | |
475 } | |
476 OSStatus status = VTCompressionSessionCreate( | |
477 nullptr, // use default allocator | |
478 width_, height_, kCMVideoCodecType_H264, | |
479 nullptr, // use default encoder | |
480 source_attributes, | |
481 nullptr, // use default compressed data allocator | |
482 internal::VTCompressionOutputCallback, this, &compression_session_); | |
483 if (source_attributes) { | |
484 CFRelease(source_attributes); | |
485 source_attributes = nullptr; | |
486 } | |
487 if (status != noErr) { | |
488 LOG(LS_ERROR) << "Failed to create compression session: " << status; | |
489 return WEBRTC_VIDEO_CODEC_ERROR; | |
490 } | |
491 ConfigureCompressionSession(); | |
492 return WEBRTC_VIDEO_CODEC_OK; | |
493 } | |
494 | |
495 void H264VideoToolboxEncoder::ConfigureCompressionSession() { | |
496 RTC_DCHECK(compression_session_); | |
497 internal::SetVTSessionProperty(compression_session_, | |
498 kVTCompressionPropertyKey_RealTime, true); | |
499 internal::SetVTSessionProperty(compression_session_, | |
500 kVTCompressionPropertyKey_ProfileLevel, | |
501 kVTProfileLevel_H264_Baseline_AutoLevel); | |
502 internal::SetVTSessionProperty(compression_session_, | |
503 kVTCompressionPropertyKey_AllowFrameReordering, | |
504 false); | |
505 SetEncoderBitrateBps(target_bitrate_bps_); | |
506 // TODO(tkchin): Look at entropy mode and colorspace matrices. | |
507 // TODO(tkchin): Investigate to see if there's any way to make this work. | |
508 // May need it to interop with Android. Currently this call just fails. | |
509 // On inspecting encoder output on iOS8, this value is set to 6. | |
510 // internal::SetVTSessionProperty(compression_session_, | |
511 // kVTCompressionPropertyKey_MaxFrameDelayCount, | |
512 // 1); | |
513 | |
514 // Set a relatively large value for keyframe emission (7200 frames or | |
515 // 4 minutes). | |
516 internal::SetVTSessionProperty( | |
517 compression_session_, | |
518 kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); | |
519 internal::SetVTSessionProperty( | |
520 compression_session_, | |
521 kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); | |
522 } | |
523 | |
524 void H264VideoToolboxEncoder::DestroyCompressionSession() { | |
525 if (compression_session_) { | |
526 VTCompressionSessionInvalidate(compression_session_); | |
527 CFRelease(compression_session_); | |
528 compression_session_ = nullptr; | |
529 } | |
530 } | |
531 | |
532 const char* H264VideoToolboxEncoder::ImplementationName() const { | |
533 return "VideoToolbox"; | |
534 } | |
535 | |
536 bool H264VideoToolboxEncoder::SupportsNativeHandle() const { | |
537 return true; | |
538 } | |
539 | |
540 void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) { | |
541 if (encoder_bitrate_bps_ != bitrate_bps) { | |
542 SetEncoderBitrateBps(bitrate_bps); | |
543 } | |
544 } | |
545 | |
546 void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) { | |
547 if (compression_session_) { | |
548 internal::SetVTSessionProperty(compression_session_, | |
549 kVTCompressionPropertyKey_AverageBitRate, | |
550 bitrate_bps); | |
551 | |
552 // TODO(tkchin): Add a helper method to set array value. | |
553 int64_t data_limit_bytes_per_second_value = static_cast<int64_t>( | |
554 bitrate_bps * internal::kLimitToAverageBitRateFactor / 8); | |
555 CFNumberRef bytes_per_second = | |
556 CFNumberCreate(kCFAllocatorDefault, | |
557 kCFNumberSInt64Type, | |
558 &data_limit_bytes_per_second_value); | |
559 int64_t one_second_value = 1; | |
560 CFNumberRef one_second = | |
561 CFNumberCreate(kCFAllocatorDefault, | |
562 kCFNumberSInt64Type, | |
563 &one_second_value); | |
564 const void* nums[2] = { bytes_per_second, one_second }; | |
565 CFArrayRef data_rate_limits = | |
566 CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); | |
567 OSStatus status = | |
568 VTSessionSetProperty(compression_session_, | |
569 kVTCompressionPropertyKey_DataRateLimits, | |
570 data_rate_limits); | |
571 if (bytes_per_second) { | |
572 CFRelease(bytes_per_second); | |
573 } | |
574 if (one_second) { | |
575 CFRelease(one_second); | |
576 } | |
577 if (data_rate_limits) { | |
578 CFRelease(data_rate_limits); | |
579 } | |
580 if (status != noErr) { | |
581 LOG(LS_ERROR) << "Failed to set data rate limit"; | |
582 } | |
583 | |
584 encoder_bitrate_bps_ = bitrate_bps; | |
585 } | |
586 } | |
587 | |
588 void H264VideoToolboxEncoder::OnEncodedFrame( | |
589 OSStatus status, | |
590 VTEncodeInfoFlags info_flags, | |
591 CMSampleBufferRef sample_buffer, | |
592 CodecSpecificInfo codec_specific_info, | |
593 int32_t width, | |
594 int32_t height, | |
595 int64_t render_time_ms, | |
596 uint32_t timestamp, | |
597 VideoRotation rotation) { | |
598 if (status != noErr) { | |
599 LOG(LS_ERROR) << "H264 encode failed."; | |
600 return; | |
601 } | |
602 if (info_flags & kVTEncodeInfo_FrameDropped) { | |
603 LOG(LS_INFO) << "H264 encode dropped frame."; | |
604 rtc::CritScope lock(&quality_scaler_crit_); | |
605 quality_scaler_.ReportDroppedFrame(); | |
606 return; | |
607 } | |
608 | |
609 bool is_keyframe = false; | |
610 CFArrayRef attachments = | |
611 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); | |
612 if (attachments != nullptr && CFArrayGetCount(attachments)) { | |
613 CFDictionaryRef attachment = | |
614 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); | |
615 is_keyframe = | |
616 !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); | |
617 } | |
618 | |
619 if (is_keyframe) { | |
620 LOG(LS_INFO) << "Generated keyframe"; | |
621 } | |
622 | |
623 // Convert the sample buffer into a buffer suitable for RTP packetization. | |
624 // TODO(tkchin): Allocate buffers through a pool. | |
625 std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer()); | |
626 std::unique_ptr<webrtc::RTPFragmentationHeader> header; | |
627 { | |
628 webrtc::RTPFragmentationHeader* header_raw; | |
629 bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe, | |
630 buffer.get(), &header_raw); | |
631 header.reset(header_raw); | |
632 if (!result) { | |
633 return; | |
634 } | |
635 } | |
636 webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); | |
637 frame._encodedWidth = width; | |
638 frame._encodedHeight = height; | |
639 frame._completeFrame = true; | |
640 frame._frameType = | |
641 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; | |
642 frame.capture_time_ms_ = render_time_ms; | |
643 frame._timeStamp = timestamp; | |
644 frame.rotation_ = rotation; | |
645 | |
646 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); | |
647 int qp; | |
648 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { | |
649 rtc::CritScope lock(&quality_scaler_crit_); | |
650 quality_scaler_.ReportQP(qp); | |
651 } | |
652 | |
653 EncodedImageCallback::Result result = | |
654 callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); | |
655 if (result.error != EncodedImageCallback::Result::OK) { | |
656 LOG(LS_ERROR) << "Encode callback failed: " << result.error; | |
657 return; | |
658 } | |
659 bitrate_adjuster_.Update(frame._size); | |
660 } | |
661 | |
662 } // namespace webrtc | |
663 | |
664 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) | |
OLD | NEW |