Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(426)

Side by Side Diff: webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm

Issue 2890513002: Revert of Split iOS sdk in to separate targets (Closed)
Patch Set: Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
12 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h"
13
14 #include <memory>
15 #include <string>
16 #include <vector>
17
18 #if defined(WEBRTC_IOS)
19 #import "WebRTC/UIDevice+RTCDevice.h"
20 #include "Common/RTCUIApplication.h"
21 #endif
22 #include "libyuv/convert_from.h"
23 #include "webrtc/base/checks.h"
24 #include "webrtc/base/logging.h"
25 #include "webrtc/common_video/h264/profile_level_id.h"
26 #include "webrtc/sdk/objc/Framework/Classes/Video/corevideo_frame_buffer.h"
27 #include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
28 #include "webrtc/system_wrappers/include/clock.h"
29
30 namespace internal {
31
32 // The ratio between kVTCompressionPropertyKey_DataRateLimits and
33 // kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher
34 // than the average bit rate to avoid undershooting the target.
35 const float kLimitToAverageBitRateFactor = 1.5f;
36 // These thresholds deviate from the default h264 QP thresholds, as they
37 // have been found to work better on devices that support VideoToolbox
38 const int kLowH264QpThreshold = 28;
39 const int kHighH264QpThreshold = 39;
40
41 // Convenience function for creating a dictionary.
42 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
43 CFTypeRef* values,
44 size_t size) {
45 return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
46 &kCFTypeDictionaryKeyCallBacks,
47 &kCFTypeDictionaryValueCallBacks);
48 }
49
50 // Copies characters from a CFStringRef into a std::string.
51 std::string CFStringToString(const CFStringRef cf_string) {
52 RTC_DCHECK(cf_string);
53 std::string std_string;
54 // Get the size needed for UTF8 plus terminating character.
55 size_t buffer_size =
56 CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string),
57 kCFStringEncodingUTF8) +
58 1;
59 std::unique_ptr<char[]> buffer(new char[buffer_size]);
60 if (CFStringGetCString(cf_string, buffer.get(), buffer_size,
61 kCFStringEncodingUTF8)) {
62 // Copy over the characters.
63 std_string.assign(buffer.get());
64 }
65 return std_string;
66 }
67
68 // Convenience function for setting a VT property.
69 void SetVTSessionProperty(VTSessionRef session,
70 CFStringRef key,
71 int32_t value) {
72 CFNumberRef cfNum =
73 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value);
74 OSStatus status = VTSessionSetProperty(session, key, cfNum);
75 CFRelease(cfNum);
76 if (status != noErr) {
77 std::string key_string = CFStringToString(key);
78 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
79 << " to " << value << ": " << status;
80 }
81 }
82
83 // Convenience function for setting a VT property.
84 void SetVTSessionProperty(VTSessionRef session,
85 CFStringRef key,
86 uint32_t value) {
87 int64_t value_64 = value;
88 CFNumberRef cfNum =
89 CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64);
90 OSStatus status = VTSessionSetProperty(session, key, cfNum);
91 CFRelease(cfNum);
92 if (status != noErr) {
93 std::string key_string = CFStringToString(key);
94 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
95 << " to " << value << ": " << status;
96 }
97 }
98
99 // Convenience function for setting a VT property.
100 void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
101 CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse;
102 OSStatus status = VTSessionSetProperty(session, key, cf_bool);
103 if (status != noErr) {
104 std::string key_string = CFStringToString(key);
105 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
106 << " to " << value << ": " << status;
107 }
108 }
109
110 // Convenience function for setting a VT property.
111 void SetVTSessionProperty(VTSessionRef session,
112 CFStringRef key,
113 CFStringRef value) {
114 OSStatus status = VTSessionSetProperty(session, key, value);
115 if (status != noErr) {
116 std::string key_string = CFStringToString(key);
117 std::string val_string = CFStringToString(value);
118 LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string
119 << " to " << val_string << ": " << status;
120 }
121 }
122
123 // Struct that we pass to the encoder per frame to encode. We receive it again
124 // in the encoder callback.
125 struct FrameEncodeParams {
126 FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e,
127 const webrtc::CodecSpecificInfo* csi,
128 int32_t w,
129 int32_t h,
130 int64_t rtms,
131 uint32_t ts,
132 webrtc::VideoRotation r)
133 : encoder(e),
134 width(w),
135 height(h),
136 render_time_ms(rtms),
137 timestamp(ts),
138 rotation(r) {
139 if (csi) {
140 codec_specific_info = *csi;
141 } else {
142 codec_specific_info.codecType = webrtc::kVideoCodecH264;
143 }
144 }
145
146 webrtc::H264VideoToolboxEncoder* encoder;
147 webrtc::CodecSpecificInfo codec_specific_info;
148 int32_t width;
149 int32_t height;
150 int64_t render_time_ms;
151 uint32_t timestamp;
152 webrtc::VideoRotation rotation;
153 };
154
155 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the
156 // encoder. This performs the copy and format conversion.
157 // TODO(tkchin): See if encoder will accept i420 frames and compare performance.
158 bool CopyVideoFrameToPixelBuffer(
159 const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& frame,
160 CVPixelBufferRef pixel_buffer) {
161 RTC_DCHECK(pixel_buffer);
162 RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer),
163 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange);
164 RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0),
165 static_cast<size_t>(frame->height()));
166 RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0),
167 static_cast<size_t>(frame->width()));
168
169 CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
170 if (cvRet != kCVReturnSuccess) {
171 LOG(LS_ERROR) << "Failed to lock base address: " << cvRet;
172 return false;
173 }
174 uint8_t* dst_y = reinterpret_cast<uint8_t*>(
175 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
176 int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
177 uint8_t* dst_uv = reinterpret_cast<uint8_t*>(
178 CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
179 int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
180 // Convert I420 to NV12.
181 int ret = libyuv::I420ToNV12(
182 frame->DataY(), frame->StrideY(),
183 frame->DataU(), frame->StrideU(),
184 frame->DataV(), frame->StrideV(),
185 dst_y, dst_stride_y, dst_uv, dst_stride_uv,
186 frame->width(), frame->height());
187 CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
188 if (ret) {
189 LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret;
190 return false;
191 }
192 return true;
193 }
194
195 CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) {
196 if (!pixel_buffer_pool) {
197 LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
198 return nullptr;
199 }
200 CVPixelBufferRef pixel_buffer;
201 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
202 &pixel_buffer);
203 if (ret != kCVReturnSuccess) {
204 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
205 // We probably want to drop frames here, since failure probably means
206 // that the pool is empty.
207 return nullptr;
208 }
209 return pixel_buffer;
210 }
211
212 // This is the callback function that VideoToolbox calls when encode is
213 // complete. From inspection this happens on its own queue.
214 void VTCompressionOutputCallback(void* encoder,
215 void* params,
216 OSStatus status,
217 VTEncodeInfoFlags info_flags,
218 CMSampleBufferRef sample_buffer) {
219 std::unique_ptr<FrameEncodeParams> encode_params(
220 reinterpret_cast<FrameEncodeParams*>(params));
221 encode_params->encoder->OnEncodedFrame(
222 status, info_flags, sample_buffer, encode_params->codec_specific_info,
223 encode_params->width, encode_params->height,
224 encode_params->render_time_ms, encode_params->timestamp,
225 encode_params->rotation);
226 }
227
228 // Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no
229 // specific VideoToolbox profile for the specified level, AutoLevel will be
230 // returned. The user must initialize the encoder with a resolution and
231 // framerate conforming to the selected H264 level regardless.
232 CFStringRef ExtractProfile(const cricket::VideoCodec& codec) {
233 const rtc::Optional<webrtc::H264::ProfileLevelId> profile_level_id =
234 webrtc::H264::ParseSdpProfileLevelId(codec.params);
235 RTC_DCHECK(profile_level_id);
236 switch (profile_level_id->profile) {
237 case webrtc::H264::kProfileConstrainedBaseline:
238 case webrtc::H264::kProfileBaseline:
239 switch (profile_level_id->level) {
240 case webrtc::H264::kLevel3:
241 return kVTProfileLevel_H264_Baseline_3_0;
242 case webrtc::H264::kLevel3_1:
243 return kVTProfileLevel_H264_Baseline_3_1;
244 case webrtc::H264::kLevel3_2:
245 return kVTProfileLevel_H264_Baseline_3_2;
246 case webrtc::H264::kLevel4:
247 return kVTProfileLevel_H264_Baseline_4_0;
248 case webrtc::H264::kLevel4_1:
249 return kVTProfileLevel_H264_Baseline_4_1;
250 case webrtc::H264::kLevel4_2:
251 return kVTProfileLevel_H264_Baseline_4_2;
252 case webrtc::H264::kLevel5:
253 return kVTProfileLevel_H264_Baseline_5_0;
254 case webrtc::H264::kLevel5_1:
255 return kVTProfileLevel_H264_Baseline_5_1;
256 case webrtc::H264::kLevel5_2:
257 return kVTProfileLevel_H264_Baseline_5_2;
258 case webrtc::H264::kLevel1:
259 case webrtc::H264::kLevel1_b:
260 case webrtc::H264::kLevel1_1:
261 case webrtc::H264::kLevel1_2:
262 case webrtc::H264::kLevel1_3:
263 case webrtc::H264::kLevel2:
264 case webrtc::H264::kLevel2_1:
265 case webrtc::H264::kLevel2_2:
266 return kVTProfileLevel_H264_Baseline_AutoLevel;
267 }
268
269 case webrtc::H264::kProfileMain:
270 switch (profile_level_id->level) {
271 case webrtc::H264::kLevel3:
272 return kVTProfileLevel_H264_Main_3_0;
273 case webrtc::H264::kLevel3_1:
274 return kVTProfileLevel_H264_Main_3_1;
275 case webrtc::H264::kLevel3_2:
276 return kVTProfileLevel_H264_Main_3_2;
277 case webrtc::H264::kLevel4:
278 return kVTProfileLevel_H264_Main_4_0;
279 case webrtc::H264::kLevel4_1:
280 return kVTProfileLevel_H264_Main_4_1;
281 case webrtc::H264::kLevel4_2:
282 return kVTProfileLevel_H264_Main_4_2;
283 case webrtc::H264::kLevel5:
284 return kVTProfileLevel_H264_Main_5_0;
285 case webrtc::H264::kLevel5_1:
286 return kVTProfileLevel_H264_Main_5_1;
287 case webrtc::H264::kLevel5_2:
288 return kVTProfileLevel_H264_Main_5_2;
289 case webrtc::H264::kLevel1:
290 case webrtc::H264::kLevel1_b:
291 case webrtc::H264::kLevel1_1:
292 case webrtc::H264::kLevel1_2:
293 case webrtc::H264::kLevel1_3:
294 case webrtc::H264::kLevel2:
295 case webrtc::H264::kLevel2_1:
296 case webrtc::H264::kLevel2_2:
297 return kVTProfileLevel_H264_Main_AutoLevel;
298 }
299
300 case webrtc::H264::kProfileConstrainedHigh:
301 case webrtc::H264::kProfileHigh:
302 switch (profile_level_id->level) {
303 case webrtc::H264::kLevel3:
304 return kVTProfileLevel_H264_High_3_0;
305 case webrtc::H264::kLevel3_1:
306 return kVTProfileLevel_H264_High_3_1;
307 case webrtc::H264::kLevel3_2:
308 return kVTProfileLevel_H264_High_3_2;
309 case webrtc::H264::kLevel4:
310 return kVTProfileLevel_H264_High_4_0;
311 case webrtc::H264::kLevel4_1:
312 return kVTProfileLevel_H264_High_4_1;
313 case webrtc::H264::kLevel4_2:
314 return kVTProfileLevel_H264_High_4_2;
315 case webrtc::H264::kLevel5:
316 return kVTProfileLevel_H264_High_5_0;
317 case webrtc::H264::kLevel5_1:
318 return kVTProfileLevel_H264_High_5_1;
319 case webrtc::H264::kLevel5_2:
320 return kVTProfileLevel_H264_High_5_2;
321 case webrtc::H264::kLevel1:
322 case webrtc::H264::kLevel1_b:
323 case webrtc::H264::kLevel1_1:
324 case webrtc::H264::kLevel1_2:
325 case webrtc::H264::kLevel1_3:
326 case webrtc::H264::kLevel2:
327 case webrtc::H264::kLevel2_1:
328 case webrtc::H264::kLevel2_2:
329 return kVTProfileLevel_H264_High_AutoLevel;
330 }
331 }
332 }
333
334 } // namespace internal
335
336 namespace webrtc {
337
338 // .5 is set as a mininum to prevent overcompensating for large temporary
339 // overshoots. We don't want to degrade video quality too badly.
340 // .95 is set to prevent oscillations. When a lower bitrate is set on the
341 // encoder than previously set, its output seems to have a brief period of
342 // drastically reduced bitrate, so we want to avoid that. In steady state
343 // conditions, 0.95 seems to give us better overall bitrate over long periods
344 // of time.
345 H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& code c)
346 : callback_(nullptr),
347 compression_session_(nullptr),
348 bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95),
349 packetization_mode_(H264PacketizationMode::NonInterleaved),
350 profile_(internal::ExtractProfile(codec)) {
351 LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_);
352 RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName));
353 }
354
355 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
356 DestroyCompressionSession();
357 }
358
359 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
360 int number_of_cores,
361 size_t max_payload_size) {
362 RTC_DCHECK(codec_settings);
363 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
364
365 width_ = codec_settings->width;
366 height_ = codec_settings->height;
367 mode_ = codec_settings->mode;
368 // We can only set average bitrate on the HW encoder.
369 target_bitrate_bps_ = codec_settings->startBitrate;
370 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
371
372 // TODO(tkchin): Try setting payload size via
373 // kVTCompressionPropertyKey_MaxH264SliceBytes.
374
375 return ResetCompressionSession();
376 }
377
378 int H264VideoToolboxEncoder::Encode(
379 const VideoFrame& frame,
380 const CodecSpecificInfo* codec_specific_info,
381 const std::vector<FrameType>* frame_types) {
382 // |input_frame| size should always match codec settings.
383 RTC_DCHECK_EQ(frame.width(), width_);
384 RTC_DCHECK_EQ(frame.height(), height_);
385 if (!callback_ || !compression_session_) {
386 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
387 }
388 #if defined(WEBRTC_IOS)
389 if (!RTCIsUIApplicationActive()) {
390 // Ignore all encode requests when app isn't active. In this state, the
391 // hardware encoder has been invalidated by the OS.
392 return WEBRTC_VIDEO_CODEC_OK;
393 }
394 #endif
395 bool is_keyframe_required = false;
396
397 // Get a pixel buffer from the pool and copy frame data over.
398 CVPixelBufferPoolRef pixel_buffer_pool =
399 VTCompressionSessionGetPixelBufferPool(compression_session_);
400 #if defined(WEBRTC_IOS)
401 if (!pixel_buffer_pool) {
402 // Kind of a hack. On backgrounding, the compression session seems to get
403 // invalidated, which causes this pool call to fail when the application
404 // is foregrounded and frames are being sent for encoding again.
405 // Resetting the session when this happens fixes the issue.
406 // In addition we request a keyframe so video can recover quickly.
407 ResetCompressionSession();
408 pixel_buffer_pool =
409 VTCompressionSessionGetPixelBufferPool(compression_session_);
410 is_keyframe_required = true;
411 LOG(LS_INFO) << "Resetting compression session due to invalid pool.";
412 }
413 #endif
414
415 CVPixelBufferRef pixel_buffer = static_cast<CVPixelBufferRef>(
416 frame.video_frame_buffer()->native_handle());
417 if (pixel_buffer) {
418 // Native frame.
419 rtc::scoped_refptr<CoreVideoFrameBuffer> core_video_frame_buffer(
420 static_cast<CoreVideoFrameBuffer*>(frame.video_frame_buffer().get()));
421 if (!core_video_frame_buffer->RequiresCropping()) {
422 // This pixel buffer might have a higher resolution than what the
423 // compression session is configured to. The compression session can
424 // handle that and will output encoded frames in the configured
425 // resolution regardless of the input pixel buffer resolution.
426 CVBufferRetain(pixel_buffer);
427 } else {
428 // Cropping required, we need to crop and scale to a new pixel buffer.
429 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
430 if (!pixel_buffer) {
431 return WEBRTC_VIDEO_CODEC_ERROR;
432 }
433 if (!core_video_frame_buffer->CropAndScaleTo(&nv12_scale_buffer_,
434 pixel_buffer)) {
435 return WEBRTC_VIDEO_CODEC_ERROR;
436 }
437 }
438 } else {
439 pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool);
440 if (!pixel_buffer) {
441 return WEBRTC_VIDEO_CODEC_ERROR;
442 }
443 RTC_DCHECK(pixel_buffer);
444 if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer(),
445 pixel_buffer)) {
446 LOG(LS_ERROR) << "Failed to copy frame data.";
447 CVBufferRelease(pixel_buffer);
448 return WEBRTC_VIDEO_CODEC_ERROR;
449 }
450 }
451
452 // Check if we need a keyframe.
453 if (!is_keyframe_required && frame_types) {
454 for (auto frame_type : *frame_types) {
455 if (frame_type == kVideoFrameKey) {
456 is_keyframe_required = true;
457 break;
458 }
459 }
460 }
461
462 CMTime presentation_time_stamp =
463 CMTimeMake(frame.render_time_ms(), 1000);
464 CFDictionaryRef frame_properties = nullptr;
465 if (is_keyframe_required) {
466 CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame};
467 CFTypeRef values[] = {kCFBooleanTrue};
468 frame_properties = internal::CreateCFDictionary(keys, values, 1);
469 }
470 std::unique_ptr<internal::FrameEncodeParams> encode_params;
471 encode_params.reset(new internal::FrameEncodeParams(
472 this, codec_specific_info, width_, height_, frame.render_time_ms(),
473 frame.timestamp(), frame.rotation()));
474
475 encode_params->codec_specific_info.codecSpecific.H264.packetization_mode =
476 packetization_mode_;
477
478 // Update the bitrate if needed.
479 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
480
481 OSStatus status = VTCompressionSessionEncodeFrame(
482 compression_session_, pixel_buffer, presentation_time_stamp,
483 kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
484 if (frame_properties) {
485 CFRelease(frame_properties);
486 }
487 if (pixel_buffer) {
488 CVBufferRelease(pixel_buffer);
489 }
490 if (status != noErr) {
491 LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
492 return WEBRTC_VIDEO_CODEC_ERROR;
493 }
494 return WEBRTC_VIDEO_CODEC_OK;
495 }
496
497 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
498 EncodedImageCallback* callback) {
499 callback_ = callback;
500 return WEBRTC_VIDEO_CODEC_OK;
501 }
502
503 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
504 int64_t rtt) {
505 // Encoder doesn't know anything about packet loss or rtt so just return.
506 return WEBRTC_VIDEO_CODEC_OK;
507 }
508
509 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
510 uint32_t frame_rate) {
511 target_bitrate_bps_ = 1000 * new_bitrate_kbit;
512 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
513 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
514 return WEBRTC_VIDEO_CODEC_OK;
515 }
516
517 int H264VideoToolboxEncoder::Release() {
518 // Need to reset so that the session is invalidated and won't use the
519 // callback anymore. Do not remove callback until the session is invalidated
520 // since async encoder callbacks can occur until invalidation.
521 int ret = ResetCompressionSession();
522 callback_ = nullptr;
523 return ret;
524 }
525
526 int H264VideoToolboxEncoder::ResetCompressionSession() {
527 DestroyCompressionSession();
528
529 // Set source image buffer attributes. These attributes will be present on
530 // buffers retrieved from the encoder's pixel buffer pool.
531 const size_t attributes_size = 3;
532 CFTypeRef keys[attributes_size] = {
533 #if defined(WEBRTC_IOS)
534 kCVPixelBufferOpenGLESCompatibilityKey,
535 #elif defined(WEBRTC_MAC)
536 kCVPixelBufferOpenGLCompatibilityKey,
537 #endif
538 kCVPixelBufferIOSurfacePropertiesKey,
539 kCVPixelBufferPixelFormatTypeKey
540 };
541 CFDictionaryRef io_surface_value =
542 internal::CreateCFDictionary(nullptr, nullptr, 0);
543 int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
544 CFNumberRef pixel_format =
545 CFNumberCreate(nullptr, kCFNumberLongType, &nv12type);
546 CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value,
547 pixel_format};
548 CFDictionaryRef source_attributes =
549 internal::CreateCFDictionary(keys, values, attributes_size);
550 if (io_surface_value) {
551 CFRelease(io_surface_value);
552 io_surface_value = nullptr;
553 }
554 if (pixel_format) {
555 CFRelease(pixel_format);
556 pixel_format = nullptr;
557 }
558 OSStatus status = VTCompressionSessionCreate(
559 nullptr, // use default allocator
560 width_, height_, kCMVideoCodecType_H264,
561 nullptr, // use default encoder
562 source_attributes,
563 nullptr, // use default compressed data allocator
564 internal::VTCompressionOutputCallback, this, &compression_session_);
565 if (source_attributes) {
566 CFRelease(source_attributes);
567 source_attributes = nullptr;
568 }
569 if (status != noErr) {
570 LOG(LS_ERROR) << "Failed to create compression session: " << status;
571 return WEBRTC_VIDEO_CODEC_ERROR;
572 }
573 ConfigureCompressionSession();
574 return WEBRTC_VIDEO_CODEC_OK;
575 }
576
577 void H264VideoToolboxEncoder::ConfigureCompressionSession() {
578 RTC_DCHECK(compression_session_);
579 internal::SetVTSessionProperty(compression_session_,
580 kVTCompressionPropertyKey_RealTime, true);
581 internal::SetVTSessionProperty(compression_session_,
582 kVTCompressionPropertyKey_ProfileLevel,
583 profile_);
584 internal::SetVTSessionProperty(compression_session_,
585 kVTCompressionPropertyKey_AllowFrameReordering,
586 false);
587 SetEncoderBitrateBps(target_bitrate_bps_);
588 // TODO(tkchin): Look at entropy mode and colorspace matrices.
589 // TODO(tkchin): Investigate to see if there's any way to make this work.
590 // May need it to interop with Android. Currently this call just fails.
591 // On inspecting encoder output on iOS8, this value is set to 6.
592 // internal::SetVTSessionProperty(compression_session_,
593 // kVTCompressionPropertyKey_MaxFrameDelayCount,
594 // 1);
595
596 // Set a relatively large value for keyframe emission (7200 frames or
597 // 4 minutes).
598 internal::SetVTSessionProperty(
599 compression_session_,
600 kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200);
601 internal::SetVTSessionProperty(
602 compression_session_,
603 kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240);
604 }
605
606 void H264VideoToolboxEncoder::DestroyCompressionSession() {
607 if (compression_session_) {
608 VTCompressionSessionInvalidate(compression_session_);
609 CFRelease(compression_session_);
610 compression_session_ = nullptr;
611 }
612 }
613
614 const char* H264VideoToolboxEncoder::ImplementationName() const {
615 return "VideoToolbox";
616 }
617
618 bool H264VideoToolboxEncoder::SupportsNativeHandle() const {
619 return true;
620 }
621
622 void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) {
623 if (encoder_bitrate_bps_ != bitrate_bps) {
624 SetEncoderBitrateBps(bitrate_bps);
625 }
626 }
627
628 void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) {
629 if (compression_session_) {
630 internal::SetVTSessionProperty(compression_session_,
631 kVTCompressionPropertyKey_AverageBitRate,
632 bitrate_bps);
633
634 // TODO(tkchin): Add a helper method to set array value.
635 int64_t data_limit_bytes_per_second_value = static_cast<int64_t>(
636 bitrate_bps * internal::kLimitToAverageBitRateFactor / 8);
637 CFNumberRef bytes_per_second =
638 CFNumberCreate(kCFAllocatorDefault,
639 kCFNumberSInt64Type,
640 &data_limit_bytes_per_second_value);
641 int64_t one_second_value = 1;
642 CFNumberRef one_second =
643 CFNumberCreate(kCFAllocatorDefault,
644 kCFNumberSInt64Type,
645 &one_second_value);
646 const void* nums[2] = { bytes_per_second, one_second };
647 CFArrayRef data_rate_limits =
648 CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks);
649 OSStatus status =
650 VTSessionSetProperty(compression_session_,
651 kVTCompressionPropertyKey_DataRateLimits,
652 data_rate_limits);
653 if (bytes_per_second) {
654 CFRelease(bytes_per_second);
655 }
656 if (one_second) {
657 CFRelease(one_second);
658 }
659 if (data_rate_limits) {
660 CFRelease(data_rate_limits);
661 }
662 if (status != noErr) {
663 LOG(LS_ERROR) << "Failed to set data rate limit";
664 }
665
666 encoder_bitrate_bps_ = bitrate_bps;
667 }
668 }
669
670 void H264VideoToolboxEncoder::OnEncodedFrame(
671 OSStatus status,
672 VTEncodeInfoFlags info_flags,
673 CMSampleBufferRef sample_buffer,
674 CodecSpecificInfo codec_specific_info,
675 int32_t width,
676 int32_t height,
677 int64_t render_time_ms,
678 uint32_t timestamp,
679 VideoRotation rotation) {
680 if (status != noErr) {
681 LOG(LS_ERROR) << "H264 encode failed.";
682 return;
683 }
684 if (info_flags & kVTEncodeInfo_FrameDropped) {
685 LOG(LS_INFO) << "H264 encode dropped frame.";
686 return;
687 }
688
689 bool is_keyframe = false;
690 CFArrayRef attachments =
691 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
692 if (attachments != nullptr && CFArrayGetCount(attachments)) {
693 CFDictionaryRef attachment =
694 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
695 is_keyframe =
696 !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
697 }
698
699 if (is_keyframe) {
700 LOG(LS_INFO) << "Generated keyframe";
701 }
702
703 // Convert the sample buffer into a buffer suitable for RTP packetization.
704 // TODO(tkchin): Allocate buffers through a pool.
705 std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer());
706 std::unique_ptr<webrtc::RTPFragmentationHeader> header;
707 {
708 webrtc::RTPFragmentationHeader* header_raw;
709 bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe,
710 buffer.get(), &header_raw);
711 header.reset(header_raw);
712 if (!result) {
713 return;
714 }
715 }
716 webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
717 frame._encodedWidth = width;
718 frame._encodedHeight = height;
719 frame._completeFrame = true;
720 frame._frameType =
721 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
722 frame.capture_time_ms_ = render_time_ms;
723 frame._timeStamp = timestamp;
724 frame.rotation_ = rotation;
725
726 frame.content_type_ =
727 (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentTy pe::UNSPECIFIED;
728
729 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
730 h264_bitstream_parser_.GetLastSliceQp(&frame.qp_);
731
732 EncodedImageCallback::Result res =
733 callback_->OnEncodedImage(frame, &codec_specific_info, header.get());
734 if (res.error != EncodedImageCallback::Result::OK) {
735 LOG(LS_ERROR) << "Encode callback failed: " << res.error;
736 return;
737 }
738 bitrate_adjuster_.Update(frame._length);
739 }
740
741 VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings()
742 const {
743 return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold,
744 internal::kHighH264QpThreshold);
745 }
746 } // namespace webrtc
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698