Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(10)

Side by Side Diff: webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.mm

Issue 2398963003: Move usage of QualityScaler to ViEEncoder. (Closed)
Patch Set: Code review comments Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license 4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source 5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found 6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may 7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree. 8 * be found in the AUTHORS file in the root of the source tree.
9 * 9 *
10 */ 10 */
(...skipping 216 matching lines...) Expand 10 before | Expand all | Expand 10 after
227 227
228 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { 228 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
229 DestroyCompressionSession(); 229 DestroyCompressionSession();
230 } 230 }
231 231
232 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, 232 int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings,
233 int number_of_cores, 233 int number_of_cores,
234 size_t max_payload_size) { 234 size_t max_payload_size) {
235 RTC_DCHECK(codec_settings); 235 RTC_DCHECK(codec_settings);
236 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); 236 RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264);
237 {
238 rtc::CritScope lock(&quality_scaler_crit_);
239 quality_scaler_.Init(internal::kLowH264QpThreshold,
240 internal::kHighH264QpThreshold,
241 codec_settings->startBitrate, codec_settings->width,
242 codec_settings->height, codec_settings->maxFramerate);
243 QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
244 // TODO(tkchin): We may need to enforce width/height dimension restrictions
245 // to match what the encoder supports.
246 width_ = res.width;
247 height_ = res.height;
248 }
249 // We can only set average bitrate on the HW encoder. 237 // We can only set average bitrate on the HW encoder.
250 target_bitrate_bps_ = codec_settings->startBitrate; 238 target_bitrate_bps_ = codec_settings->startBitrate;
251 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); 239 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
252 240
253 // TODO(tkchin): Try setting payload size via 241 // TODO(tkchin): Try setting payload size via
254 // kVTCompressionPropertyKey_MaxH264SliceBytes. 242 // kVTCompressionPropertyKey_MaxH264SliceBytes.
255 243
256 return ResetCompressionSession(); 244 return ResetCompressionSession();
257 } 245 }
258 246
259 int H264VideoToolboxEncoder::Encode( 247 int H264VideoToolboxEncoder::Encode(
260 const VideoFrame& frame, 248 const VideoFrame& frame,
261 const CodecSpecificInfo* codec_specific_info, 249 const CodecSpecificInfo* codec_specific_info,
262 const std::vector<FrameType>* frame_types) { 250 const std::vector<FrameType>* frame_types) {
263 RTC_DCHECK(!frame.IsZeroSize()); 251 RTC_DCHECK(!frame.IsZeroSize());
264 if (!callback_ || !compression_session_) { 252 if (!callback_ || !compression_session_) {
265 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 253 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
266 } 254 }
267 #if defined(WEBRTC_IOS) 255 #if defined(WEBRTC_IOS)
268 if (!RTCIsUIApplicationActive()) { 256 if (!RTCIsUIApplicationActive()) {
269 // Ignore all encode requests when app isn't active. In this state, the 257 // Ignore all encode requests when app isn't active. In this state, the
270 // hardware encoder has been invalidated by the OS. 258 // hardware encoder has been invalidated by the OS.
271 return WEBRTC_VIDEO_CODEC_OK; 259 return WEBRTC_VIDEO_CODEC_OK;
272 } 260 }
273 #endif 261 #endif
274 bool is_keyframe_required = false; 262 bool is_keyframe_required = false;
275 263
276 quality_scaler_.OnEncodeFrame(frame.width(), frame.height());
magjed_webrtc 2016/10/07 12:02:54 ditto: I don't think this class can handle dynamic
kthelgason 2016/10/07 13:54:48 It looks like the case that frame.width() != width
277 const QualityScaler::Resolution scaled_res =
278 quality_scaler_.GetScaledResolution();
279
280 if (scaled_res.width != width_ || scaled_res.height != height_) {
281 width_ = scaled_res.width;
282 height_ = scaled_res.height;
283 int ret = ResetCompressionSession();
284 if (ret < 0)
285 return ret;
286 }
287
288 // Get a pixel buffer from the pool and copy frame data over. 264 // Get a pixel buffer from the pool and copy frame data over.
289 CVPixelBufferPoolRef pixel_buffer_pool = 265 CVPixelBufferPoolRef pixel_buffer_pool =
290 VTCompressionSessionGetPixelBufferPool(compression_session_); 266 VTCompressionSessionGetPixelBufferPool(compression_session_);
291 #if defined(WEBRTC_IOS) 267 #if defined(WEBRTC_IOS)
292 if (!pixel_buffer_pool) { 268 if (!pixel_buffer_pool) {
293 // Kind of a hack. On backgrounding, the compression session seems to get 269 // Kind of a hack. On backgrounding, the compression session seems to get
294 // invalidated, which causes this pool call to fail when the application 270 // invalidated, which causes this pool call to fail when the application
295 // is foregrounded and frames are being sent for encoding again. 271 // is foregrounded and frames are being sent for encoding again.
296 // Resetting the session when this happens fixes the issue. 272 // Resetting the session when this happens fixes the issue.
297 // In addition we request a keyframe so video can recover quickly. 273 // In addition we request a keyframe so video can recover quickly.
(...skipping 21 matching lines...) Expand all
319 } 295 }
320 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer( 296 CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(
321 nullptr, pixel_buffer_pool, &pixel_buffer); 297 nullptr, pixel_buffer_pool, &pixel_buffer);
322 if (ret != kCVReturnSuccess) { 298 if (ret != kCVReturnSuccess) {
323 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; 299 LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
324 // We probably want to drop frames here, since failure probably means 300 // We probably want to drop frames here, since failure probably means
325 // that the pool is empty. 301 // that the pool is empty.
326 return WEBRTC_VIDEO_CODEC_ERROR; 302 return WEBRTC_VIDEO_CODEC_ERROR;
327 } 303 }
328 RTC_DCHECK(pixel_buffer); 304 RTC_DCHECK(pixel_buffer);
329 // TODO(magjed): Optimize by merging scaling and NV12 pixel buffer
330 // conversion once libyuv::MergeUVPlanes is available.
331 rtc::scoped_refptr<VideoFrameBuffer> scaled_i420_buffer =
332 quality_scaler_.GetScaledBuffer(frame.video_frame_buffer());
333 if (!internal::CopyVideoFrameToPixelBuffer(scaled_i420_buffer,
334 pixel_buffer)) {
335 LOG(LS_ERROR) << "Failed to copy frame data.";
336 CVBufferRelease(pixel_buffer);
337 return WEBRTC_VIDEO_CODEC_ERROR;
338 }
339 } 305 }
340 306
341 // Check if we need a keyframe. 307 // Check if we need a keyframe.
342 if (!is_keyframe_required && frame_types) { 308 if (!is_keyframe_required && frame_types) {
343 for (auto frame_type : *frame_types) { 309 for (auto frame_type : *frame_types) {
344 if (frame_type == kVideoFrameKey) { 310 if (frame_type == kVideoFrameKey) {
345 is_keyframe_required = true; 311 is_keyframe_required = true;
346 break; 312 break;
347 } 313 }
348 } 314 }
(...skipping 30 matching lines...) Expand all
379 } 345 }
380 return WEBRTC_VIDEO_CODEC_OK; 346 return WEBRTC_VIDEO_CODEC_OK;
381 } 347 }
382 348
383 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( 349 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
384 EncodedImageCallback* callback) { 350 EncodedImageCallback* callback) {
385 callback_ = callback; 351 callback_ = callback;
386 return WEBRTC_VIDEO_CODEC_OK; 352 return WEBRTC_VIDEO_CODEC_OK;
387 } 353 }
388 354
389 void H264VideoToolboxEncoder::OnDroppedFrame() {
390 rtc::CritScope lock(&quality_scaler_crit_);
391 quality_scaler_.ReportDroppedFrame();
392 }
393
394 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, 355 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
395 int64_t rtt) { 356 int64_t rtt) {
396 // Encoder doesn't know anything about packet loss or rtt so just return. 357 // Encoder doesn't know anything about packet loss or rtt so just return.
397 return WEBRTC_VIDEO_CODEC_OK; 358 return WEBRTC_VIDEO_CODEC_OK;
398 } 359 }
399 360
400 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, 361 int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit,
401 uint32_t frame_rate) { 362 uint32_t frame_rate) {
402 target_bitrate_bps_ = 1000 * new_bitrate_kbit; 363 target_bitrate_bps_ = 1000 * new_bitrate_kbit;
403 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); 364 bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_);
404 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); 365 SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
405
406 rtc::CritScope lock(&quality_scaler_crit_);
407 quality_scaler_.ReportFramerate(frame_rate);
408
409 return WEBRTC_VIDEO_CODEC_OK; 366 return WEBRTC_VIDEO_CODEC_OK;
410 } 367 }
411 368
412 int H264VideoToolboxEncoder::Release() { 369 int H264VideoToolboxEncoder::Release() {
413 // Need to reset so that the session is invalidated and won't use the 370 // Need to reset so that the session is invalidated and won't use the
414 // callback anymore. Do not remove callback until the session is invalidated 371 // callback anymore. Do not remove callback until the session is invalidated
415 // since async encoder callbacks can occur until invalidation. 372 // since async encoder callbacks can occur until invalidation.
416 int ret = ResetCompressionSession(); 373 int ret = ResetCompressionSession();
417 callback_ = nullptr; 374 callback_ = nullptr;
418 return ret; 375 return ret;
(...skipping 152 matching lines...) Expand 10 before | Expand all | Expand 10 after
571 int32_t height, 528 int32_t height,
572 int64_t render_time_ms, 529 int64_t render_time_ms,
573 uint32_t timestamp, 530 uint32_t timestamp,
574 VideoRotation rotation) { 531 VideoRotation rotation) {
575 if (status != noErr) { 532 if (status != noErr) {
576 LOG(LS_ERROR) << "H264 encode failed."; 533 LOG(LS_ERROR) << "H264 encode failed.";
577 return; 534 return;
578 } 535 }
579 if (info_flags & kVTEncodeInfo_FrameDropped) { 536 if (info_flags & kVTEncodeInfo_FrameDropped) {
580 LOG(LS_INFO) << "H264 encode dropped frame."; 537 LOG(LS_INFO) << "H264 encode dropped frame.";
581 rtc::CritScope lock(&quality_scaler_crit_);
582 quality_scaler_.ReportDroppedFrame();
583 return; 538 return;
584 } 539 }
585 540
586 bool is_keyframe = false; 541 bool is_keyframe = false;
587 CFArrayRef attachments = 542 CFArrayRef attachments =
588 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); 543 CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0);
589 if (attachments != nullptr && CFArrayGetCount(attachments)) { 544 if (attachments != nullptr && CFArrayGetCount(attachments)) {
590 CFDictionaryRef attachment = 545 CFDictionaryRef attachment =
591 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)); 546 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
592 is_keyframe = 547 is_keyframe =
(...skipping 20 matching lines...) Expand all
613 webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); 568 webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size());
614 frame._encodedWidth = width; 569 frame._encodedWidth = width;
615 frame._encodedHeight = height; 570 frame._encodedHeight = height;
616 frame._completeFrame = true; 571 frame._completeFrame = true;
617 frame._frameType = 572 frame._frameType =
618 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; 573 is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
619 frame.capture_time_ms_ = render_time_ms; 574 frame.capture_time_ms_ = render_time_ms;
620 frame._timeStamp = timestamp; 575 frame._timeStamp = timestamp;
621 frame.rotation_ = rotation; 576 frame.rotation_ = rotation;
622 577
623 h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size());
624 int qp;
625 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
626 rtc::CritScope lock(&quality_scaler_crit_);
627 quality_scaler_.ReportQP(qp);
magjed_webrtc 2016/10/07 12:02:54 ditto: You need to set qp_ in |frame| in order for
628 }
629
630 int result = callback_->Encoded(frame, &codec_specific_info, header.get()); 578 int result = callback_->Encoded(frame, &codec_specific_info, header.get());
631 if (result != 0) { 579 if (result != 0) {
632 LOG(LS_ERROR) << "Encode callback failed: " << result; 580 LOG(LS_ERROR) << "Encode callback failed: " << result;
633 return; 581 return;
634 } 582 }
635 bitrate_adjuster_.Update(frame._size); 583 bitrate_adjuster_.Update(frame._size);
636 } 584 }
637 585
586 rtc::Optional<VideoEncoder::QPThresholds>
587 H264VideoToolboxEncoder::GetQPThresholds() {
588 return rtc::Optional<VideoEncoder::QPThresholds>(
589 VideoEncoder::QPThresholds(internal::kLowH264QpThreshold,
590 internal::kHighH264QpThreshold));
591 }
638 } // namespace webrtc 592 } // namespace webrtc
639 593
640 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) 594 #endif // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698