OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 235 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
246 pending_encoder_reconfiguration_(false), | 246 pending_encoder_reconfiguration_(false), |
247 encoder_start_bitrate_bps_(0), | 247 encoder_start_bitrate_bps_(0), |
248 max_data_payload_length_(0), | 248 max_data_payload_length_(0), |
249 last_observed_bitrate_bps_(0), | 249 last_observed_bitrate_bps_(0), |
250 encoder_paused_and_dropped_frame_(false), | 250 encoder_paused_and_dropped_frame_(false), |
251 has_received_sli_(false), | 251 has_received_sli_(false), |
252 picture_id_sli_(0), | 252 picture_id_sli_(0), |
253 has_received_rpsi_(false), | 253 has_received_rpsi_(false), |
254 picture_id_rpsi_(0), | 254 picture_id_rpsi_(0), |
255 clock_(Clock::GetRealTimeClock()), | 255 clock_(Clock::GetRealTimeClock()), |
256 degradation_preference_( | |
257 VideoSendStream::DegradationPreference::kBalanced), | |
258 cpu_restricted_counter_(0), | |
259 last_frame_width_(0), | 256 last_frame_width_(0), |
260 last_frame_height_(0), | 257 last_frame_height_(0), |
261 last_captured_timestamp_(0), | 258 last_captured_timestamp_(0), |
262 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 259 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - |
263 clock_->TimeInMilliseconds()), | 260 clock_->TimeInMilliseconds()), |
264 last_frame_log_ms_(clock_->TimeInMilliseconds()), | 261 last_frame_log_ms_(clock_->TimeInMilliseconds()), |
265 captured_frame_count_(0), | 262 captured_frame_count_(0), |
266 dropped_frame_count_(0), | 263 dropped_frame_count_(0), |
267 encoder_queue_("EncoderQueue") { | 264 encoder_queue_("EncoderQueue") { |
268 encoder_queue_.PostTask([this] { | 265 encoder_queue_.PostTask([this] { |
(...skipping 12 matching lines...) Expand all Loading... | |
281 | 278 |
282 void ViEEncoder::Stop() { | 279 void ViEEncoder::Stop() { |
283 RTC_DCHECK_RUN_ON(&thread_checker_); | 280 RTC_DCHECK_RUN_ON(&thread_checker_); |
284 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); | 281 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); |
285 encoder_queue_.PostTask([this] { | 282 encoder_queue_.PostTask([this] { |
286 RTC_DCHECK_RUN_ON(&encoder_queue_); | 283 RTC_DCHECK_RUN_ON(&encoder_queue_); |
287 overuse_detector_.StopCheckForOveruse(); | 284 overuse_detector_.StopCheckForOveruse(); |
288 rate_allocator_.reset(); | 285 rate_allocator_.reset(); |
289 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, | 286 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, |
290 false); | 287 false); |
288 quality_scaler_ = nullptr; | |
291 shutdown_event_.Set(); | 289 shutdown_event_.Set(); |
292 }); | 290 }); |
293 | 291 |
294 shutdown_event_.Wait(rtc::Event::kForever); | 292 shutdown_event_.Wait(rtc::Event::kForever); |
295 } | 293 } |
296 | 294 |
297 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | 295 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { |
298 RTC_DCHECK_RUN_ON(&thread_checker_); | 296 RTC_DCHECK_RUN_ON(&thread_checker_); |
299 RTC_DCHECK(!module_process_thread_); | 297 RTC_DCHECK(!module_process_thread_); |
300 module_process_thread_ = module_process_thread; | 298 module_process_thread_ = module_process_thread; |
301 module_process_thread_->RegisterModule(&video_sender_); | 299 module_process_thread_->RegisterModule(&video_sender_); |
302 module_process_thread_checker_.DetachFromThread(); | 300 module_process_thread_checker_.DetachFromThread(); |
303 } | 301 } |
304 | 302 |
305 void ViEEncoder::DeRegisterProcessThread() { | 303 void ViEEncoder::DeRegisterProcessThread() { |
306 RTC_DCHECK_RUN_ON(&thread_checker_); | 304 RTC_DCHECK_RUN_ON(&thread_checker_); |
307 module_process_thread_->DeRegisterModule(&video_sender_); | 305 module_process_thread_->DeRegisterModule(&video_sender_); |
308 } | 306 } |
309 | 307 |
310 void ViEEncoder::SetSource( | 308 void ViEEncoder::SetSource( |
311 rtc::VideoSourceInterface<VideoFrame>* source, | 309 rtc::VideoSourceInterface<VideoFrame>* source, |
312 const VideoSendStream::DegradationPreference& degradation_preference) { | 310 const VideoSendStream::DegradationPreference& degradation_preference) { |
313 RTC_DCHECK_RUN_ON(&thread_checker_); | 311 RTC_DCHECK_RUN_ON(&thread_checker_); |
314 source_proxy_->SetSource(source, degradation_preference); | 312 source_proxy_->SetSource(source, degradation_preference); |
315 encoder_queue_.PostTask([this, degradation_preference] { | 313 encoder_queue_.PostTask([this, degradation_preference] { |
316 RTC_DCHECK_RUN_ON(&encoder_queue_); | 314 RTC_DCHECK_RUN_ON(&encoder_queue_); |
317 degradation_preference_ = degradation_preference; | 315 scaling_enabled_ = |
318 // Set the stats for if we are currently CPU restricted. We are CPU | 316 (degradation_preference != |
319 // restricted depending on degradation preference and | 317 VideoSendStream::DegradationPreference::kMaintainResolution); |
320 // if the overusedetector has currently detected overuse which is counted in | 318 stats_proxy_->SetResolutionRestrictionStats( |
321 // |cpu_restricted_counter_| | 319 scaling_enabled_ && scale_counter_[kQuality] > 0, |
322 // We do this on the encoder task queue to avoid a race with the stats set | 320 scaling_enabled_ && scale_counter_[kCpu] > 0); |
323 // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected. | |
324 stats_proxy_->SetCpuRestrictedResolution( | |
325 degradation_preference_ != | |
326 VideoSendStream::DegradationPreference::kMaintainResolution && | |
327 cpu_restricted_counter_ != 0); | |
328 }); | 321 }); |
329 } | 322 } |
330 | 323 |
331 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { | 324 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { |
332 source_proxy_->SetWantsRotationApplied(rotation_applied); | 325 source_proxy_->SetWantsRotationApplied(rotation_applied); |
333 encoder_queue_.PostTask([this, sink] { | 326 encoder_queue_.PostTask([this, sink] { |
334 RTC_DCHECK_RUN_ON(&encoder_queue_); | 327 RTC_DCHECK_RUN_ON(&encoder_queue_); |
335 sink_ = sink; | 328 sink_ = sink; |
336 }); | 329 }); |
337 } | 330 } |
(...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
405 if (framerate == 0) | 398 if (framerate == 0) |
406 framerate = codec.maxFramerate; | 399 framerate = codec.maxFramerate; |
407 stats_proxy_->OnEncoderReconfigured( | 400 stats_proxy_->OnEncoderReconfigured( |
408 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); | 401 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); |
409 } | 402 } |
410 | 403 |
411 pending_encoder_reconfiguration_ = false; | 404 pending_encoder_reconfiguration_ = false; |
412 | 405 |
413 sink_->OnEncoderConfigurationChanged( | 406 sink_->OnEncoderConfigurationChanged( |
414 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | 407 std::move(streams), encoder_config_.min_transmit_bitrate_bps); |
408 | |
409 const auto scaling_settings = settings_.encoder->GetScalingSettings(); | |
410 if (scaling_settings.enabled && scaling_enabled_) { | |
411 LOG(LS_INFO) << "Initializing quality scaler."; | |
perkj_webrtc
2016/11/22 19:04:16
nit: I think you should remove this log. It will b
kthelgason
2016/11/23 12:05:54
Done.
| |
412 if (scaling_settings.thresholds) { | |
413 quality_scaler_.reset( | |
414 new QualityScaler(this, *(scaling_settings.thresholds))); | |
415 } else { | |
416 quality_scaler_.reset(new QualityScaler(this, codec_type_)); | |
417 } | |
418 } else { | |
419 quality_scaler_.reset(nullptr); | |
420 } | |
415 } | 421 } |
416 | 422 |
417 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | 423 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { |
418 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | 424 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
419 VideoFrame incoming_frame = video_frame; | 425 VideoFrame incoming_frame = video_frame; |
420 | 426 |
421 // Local time in webrtc time base. | 427 // Local time in webrtc time base. |
422 int64_t current_time = clock_->TimeInMilliseconds(); | 428 int64_t current_time = clock_->TimeInMilliseconds(); |
423 incoming_frame.set_render_time_ms(current_time); | 429 incoming_frame.set_render_time_ms(current_time); |
424 | 430 |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
482 // End trace event on first frame after encoder resumes, if frame was dropped. | 488 // End trace event on first frame after encoder resumes, if frame was dropped. |
483 if (encoder_paused_and_dropped_frame_) { | 489 if (encoder_paused_and_dropped_frame_) { |
484 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | 490 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
485 } | 491 } |
486 encoder_paused_and_dropped_frame_ = false; | 492 encoder_paused_and_dropped_frame_ = false; |
487 } | 493 } |
488 | 494 |
489 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, | 495 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, |
490 int64_t time_when_posted_in_ms) { | 496 int64_t time_when_posted_in_ms) { |
491 RTC_DCHECK_RUN_ON(&encoder_queue_); | 497 RTC_DCHECK_RUN_ON(&encoder_queue_); |
498 | |
492 if (pre_encode_callback_) | 499 if (pre_encode_callback_) |
493 pre_encode_callback_->OnFrame(video_frame); | 500 pre_encode_callback_->OnFrame(video_frame); |
494 | 501 |
495 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || | 502 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || |
496 video_frame.height() != last_frame_info_->height || | 503 video_frame.height() != last_frame_info_->height || |
497 video_frame.rotation() != last_frame_info_->rotation || | 504 video_frame.rotation() != last_frame_info_->rotation || |
498 video_frame.is_texture() != last_frame_info_->is_texture) { | 505 video_frame.is_texture() != last_frame_info_->is_texture) { |
499 pending_encoder_reconfiguration_ = true; | 506 pending_encoder_reconfiguration_ = true; |
500 last_frame_info_ = rtc::Optional<VideoFrameInfo>( | 507 last_frame_info_ = rtc::Optional<VideoFrameInfo>( |
501 VideoFrameInfo(video_frame.width(), video_frame.height(), | 508 VideoFrameInfo(video_frame.width(), video_frame.height(), |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
559 // running in parallel on different threads. | 566 // running in parallel on different threads. |
560 if (stats_proxy_) { | 567 if (stats_proxy_) { |
561 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 568 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
562 } | 569 } |
563 | 570 |
564 EncodedImageCallback::Result result = | 571 EncodedImageCallback::Result result = |
565 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | 572 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); |
566 | 573 |
567 int64_t time_sent = clock_->TimeInMilliseconds(); | 574 int64_t time_sent = clock_->TimeInMilliseconds(); |
568 uint32_t timestamp = encoded_image._timeStamp; | 575 uint32_t timestamp = encoded_image._timeStamp; |
569 | 576 const int qp = encoded_image.qp_; |
570 encoder_queue_.PostTask([this, timestamp, time_sent] { | 577 encoder_queue_.PostTask([this, timestamp, time_sent, qp] { |
571 RTC_DCHECK_RUN_ON(&encoder_queue_); | 578 RTC_DCHECK_RUN_ON(&encoder_queue_); |
572 overuse_detector_.FrameSent(timestamp, time_sent); | 579 overuse_detector_.FrameSent(timestamp, time_sent); |
580 if (quality_scaler_) | |
581 quality_scaler_->ReportQP(qp); | |
573 }); | 582 }); |
574 | 583 |
575 return result; | 584 return result; |
576 } | 585 } |
577 | 586 |
578 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { | 587 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { |
579 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | 588 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); |
580 if (stats_proxy_) | 589 if (stats_proxy_) |
581 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate); | 590 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate); |
582 } | 591 } |
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
639 bool video_suspension_changed = video_is_suspended != EncoderPaused(); | 648 bool video_suspension_changed = video_is_suspended != EncoderPaused(); |
640 last_observed_bitrate_bps_ = bitrate_bps; | 649 last_observed_bitrate_bps_ = bitrate_bps; |
641 | 650 |
642 if (stats_proxy_ && video_suspension_changed) { | 651 if (stats_proxy_ && video_suspension_changed) { |
643 LOG(LS_INFO) << "Video suspend state changed to: " | 652 LOG(LS_INFO) << "Video suspend state changed to: " |
644 << (video_is_suspended ? "suspended" : "not suspended"); | 653 << (video_is_suspended ? "suspended" : "not suspended"); |
645 stats_proxy_->OnSuspendChange(video_is_suspended); | 654 stats_proxy_->OnSuspendChange(video_is_suspended); |
646 } | 655 } |
647 } | 656 } |
648 | 657 |
649 void ViEEncoder::OveruseDetected() { | 658 void ViEEncoder::ScaleDown(ScaleReason reason) { |
650 RTC_DCHECK_RUN_ON(&encoder_queue_); | 659 RTC_DCHECK_RUN_ON(&encoder_queue_); |
651 if (degradation_preference_ == | 660 if (!scaling_enabled_) |
652 VideoSendStream::DegradationPreference::kMaintainResolution || | |
653 cpu_restricted_counter_ >= kMaxCpuDowngrades) { | |
654 return; | 661 return; |
655 } | |
656 LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution."; | |
657 // Request lower resolution if the current resolution is lower than last time | 662 // Request lower resolution if the current resolution is lower than last time |
658 // we asked for the resolution to be lowered. | 663 // we asked for the resolution to be lowered. |
659 // Update stats accordingly. | |
660 int current_pixel_count = last_frame_height_ * last_frame_width_; | 664 int current_pixel_count = last_frame_height_ * last_frame_width_; |
661 if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) { | 665 if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) |
662 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); | 666 return; |
663 max_pixel_count_step_up_ = rtc::Optional<int>(); | 667 switch (reason) { |
664 stats_proxy_->OnCpuRestrictedResolutionChanged(true); | 668 case kQuality: |
665 ++cpu_restricted_counter_; | 669 if (scale_counter_[reason] >= kMaxQualityDowngrades) |
666 source_proxy_->RequestResolutionLowerThan(current_pixel_count); | 670 return; |
671 stats_proxy_->OnQualityRestrictedResolutionChanged(true); | |
672 break; | |
673 case kCpu: | |
674 if (scale_counter_[reason] >= kMaxCpuDowngrades) | |
675 return; | |
676 // Update stats accordingly. | |
677 stats_proxy_->OnCpuRestrictedResolutionChanged(true); | |
678 break; | |
679 } | |
680 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); | |
681 max_pixel_count_step_up_ = rtc::Optional<int>(); | |
682 ++scale_counter_[reason]; | |
683 source_proxy_->RequestResolutionLowerThan(current_pixel_count); | |
684 LOG(LS_INFO) << "Scaling down resolution."; | |
685 for (size_t i = 0; i < kScaleReasonSize; ++i) { | |
686 LOG(LS_INFO) << "Scaled " << scale_counter_[i] | |
687 << " times for reason: " << (i ? "quality" : "cpu"); | |
667 } | 688 } |
668 } | 689 } |
669 | 690 |
670 void ViEEncoder::NormalUsage() { | 691 void ViEEncoder::ScaleUp(ScaleReason reason) { |
671 RTC_DCHECK_RUN_ON(&encoder_queue_); | 692 RTC_DCHECK_RUN_ON(&encoder_queue_); |
672 if (degradation_preference_ == | 693 if (scale_counter_[reason] == 0 || !scaling_enabled_) |
673 VideoSendStream::DegradationPreference::kMaintainResolution || | |
674 cpu_restricted_counter_ == 0) { | |
675 return; | 694 return; |
695 // Only scale if resolution is higher than last time | |
696 // we requested higher resolution. | |
697 int current_pixel_count = last_frame_height_ * last_frame_width_; | |
698 if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) | |
699 return; | |
700 switch (reason) { | |
701 case kQuality: | |
702 stats_proxy_->OnQualityRestrictedResolutionChanged( | |
703 scale_counter_[reason] > 1); | |
704 break; | |
705 case kCpu: | |
706 // Update stats accordingly. | |
707 stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > | |
708 1); | |
709 break; | |
676 } | 710 } |
677 | 711 max_pixel_count_ = rtc::Optional<int>(); |
678 LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution."; | 712 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); |
679 int current_pixel_count = last_frame_height_ * last_frame_width_; | 713 --scale_counter_[reason]; |
680 // Request higher resolution if we are CPU restricted and the the current | 714 source_proxy_->RequestHigherResolutionThan(current_pixel_count); |
681 // resolution is higher than last time we requested higher resolution. | 715 LOG(LS_INFO) << "Scaling up resolution."; |
682 // Update stats accordingly. | 716 for (size_t i = 0; i < kScaleReasonSize; ++i) { |
683 if (!max_pixel_count_step_up_ || | 717 LOG(LS_INFO) << "Scaled " << scale_counter_[i] |
684 current_pixel_count > *max_pixel_count_step_up_) { | 718 << " times for reason: " << (i ? "quality" : "cpu"); |
685 max_pixel_count_ = rtc::Optional<int>(); | |
686 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); | |
687 --cpu_restricted_counter_; | |
688 stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0); | |
689 source_proxy_->RequestHigherResolutionThan(current_pixel_count); | |
690 } | 719 } |
691 } | 720 } |
692 | 721 |
693 } // namespace webrtc | 722 } // namespace webrtc |
OLD | NEW |