OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 239 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
250 encoder_start_bitrate_bps_(0), | 250 encoder_start_bitrate_bps_(0), |
251 max_data_payload_length_(0), | 251 max_data_payload_length_(0), |
252 nack_enabled_(false), | 252 nack_enabled_(false), |
253 last_observed_bitrate_bps_(0), | 253 last_observed_bitrate_bps_(0), |
254 encoder_paused_and_dropped_frame_(false), | 254 encoder_paused_and_dropped_frame_(false), |
255 has_received_sli_(false), | 255 has_received_sli_(false), |
256 picture_id_sli_(0), | 256 picture_id_sli_(0), |
257 has_received_rpsi_(false), | 257 has_received_rpsi_(false), |
258 picture_id_rpsi_(0), | 258 picture_id_rpsi_(0), |
259 clock_(Clock::GetRealTimeClock()), | 259 clock_(Clock::GetRealTimeClock()), |
260 degradation_preference_( | |
261 VideoSendStream::DegradationPreference::kBalanced), | |
262 cpu_restricted_counter_(0), | |
263 last_frame_width_(0), | 260 last_frame_width_(0), |
264 last_frame_height_(0), | 261 last_frame_height_(0), |
265 last_captured_timestamp_(0), | 262 last_captured_timestamp_(0), |
266 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 263 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - |
267 clock_->TimeInMilliseconds()), | 264 clock_->TimeInMilliseconds()), |
268 last_frame_log_ms_(clock_->TimeInMilliseconds()), | 265 last_frame_log_ms_(clock_->TimeInMilliseconds()), |
269 captured_frame_count_(0), | 266 captured_frame_count_(0), |
270 dropped_frame_count_(0), | 267 dropped_frame_count_(0), |
271 encoder_queue_("EncoderQueue") { | 268 encoder_queue_("EncoderQueue") { |
272 encoder_queue_.PostTask([this] { | 269 encoder_queue_.PostTask([this] { |
(...skipping 12 matching lines...) Expand all Loading... |
285 | 282 |
286 void ViEEncoder::Stop() { | 283 void ViEEncoder::Stop() { |
287 RTC_DCHECK_RUN_ON(&thread_checker_); | 284 RTC_DCHECK_RUN_ON(&thread_checker_); |
288 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); | 285 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); |
289 encoder_queue_.PostTask([this] { | 286 encoder_queue_.PostTask([this] { |
290 RTC_DCHECK_RUN_ON(&encoder_queue_); | 287 RTC_DCHECK_RUN_ON(&encoder_queue_); |
291 overuse_detector_.StopCheckForOveruse(); | 288 overuse_detector_.StopCheckForOveruse(); |
292 rate_allocator_.reset(); | 289 rate_allocator_.reset(); |
293 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, | 290 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, |
294 false); | 291 false); |
| 292 quality_scaler_ = nullptr; |
295 shutdown_event_.Set(); | 293 shutdown_event_.Set(); |
296 }); | 294 }); |
297 | 295 |
298 shutdown_event_.Wait(rtc::Event::kForever); | 296 shutdown_event_.Wait(rtc::Event::kForever); |
299 } | 297 } |
300 | 298 |
301 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | 299 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { |
302 RTC_DCHECK_RUN_ON(&thread_checker_); | 300 RTC_DCHECK_RUN_ON(&thread_checker_); |
303 RTC_DCHECK(!module_process_thread_); | 301 RTC_DCHECK(!module_process_thread_); |
304 module_process_thread_ = module_process_thread; | 302 module_process_thread_ = module_process_thread; |
305 module_process_thread_->RegisterModule(&video_sender_); | 303 module_process_thread_->RegisterModule(&video_sender_); |
306 module_process_thread_checker_.DetachFromThread(); | 304 module_process_thread_checker_.DetachFromThread(); |
307 } | 305 } |
308 | 306 |
309 void ViEEncoder::DeRegisterProcessThread() { | 307 void ViEEncoder::DeRegisterProcessThread() { |
310 RTC_DCHECK_RUN_ON(&thread_checker_); | 308 RTC_DCHECK_RUN_ON(&thread_checker_); |
311 module_process_thread_->DeRegisterModule(&video_sender_); | 309 module_process_thread_->DeRegisterModule(&video_sender_); |
312 } | 310 } |
313 | 311 |
314 void ViEEncoder::SetSource( | 312 void ViEEncoder::SetSource( |
315 rtc::VideoSourceInterface<VideoFrame>* source, | 313 rtc::VideoSourceInterface<VideoFrame>* source, |
316 const VideoSendStream::DegradationPreference& degradation_preference) { | 314 const VideoSendStream::DegradationPreference& degradation_preference) { |
317 RTC_DCHECK_RUN_ON(&thread_checker_); | 315 RTC_DCHECK_RUN_ON(&thread_checker_); |
318 source_proxy_->SetSource(source, degradation_preference); | 316 source_proxy_->SetSource(source, degradation_preference); |
319 encoder_queue_.PostTask([this, degradation_preference] { | 317 encoder_queue_.PostTask([this, degradation_preference] { |
320 RTC_DCHECK_RUN_ON(&encoder_queue_); | 318 RTC_DCHECK_RUN_ON(&encoder_queue_); |
321 degradation_preference_ = degradation_preference; | 319 scaling_enabled_ = |
322 // Set the stats for if we are currently CPU restricted. We are CPU | 320 (degradation_preference != |
323 // restricted depending on degradation preference and | 321 VideoSendStream::DegradationPreference::kMaintainResolution); |
324 // if the overusedetector has currently detected overuse which is counted in | 322 stats_proxy_->SetResolutionRestrictionStats( |
325 // |cpu_restricted_counter_| | 323 scaling_enabled_ && scale_counter_[kQuality] > 0, |
326 // We do this on the encoder task queue to avoid a race with the stats set | 324 scaling_enabled_ && scale_counter_[kCpu] > 0); |
327 // in ViEEncoder::NormalUsage and ViEEncoder::OveruseDetected. | |
328 stats_proxy_->SetCpuRestrictedResolution( | |
329 degradation_preference_ != | |
330 VideoSendStream::DegradationPreference::kMaintainResolution && | |
331 cpu_restricted_counter_ != 0); | |
332 }); | 325 }); |
333 } | 326 } |
334 | 327 |
335 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { | 328 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { |
336 source_proxy_->SetWantsRotationApplied(rotation_applied); | 329 source_proxy_->SetWantsRotationApplied(rotation_applied); |
337 encoder_queue_.PostTask([this, sink] { | 330 encoder_queue_.PostTask([this, sink] { |
338 RTC_DCHECK_RUN_ON(&encoder_queue_); | 331 RTC_DCHECK_RUN_ON(&encoder_queue_); |
339 sink_ = sink; | 332 sink_ = sink; |
340 }); | 333 }); |
341 } | 334 } |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
413 if (framerate == 0) | 406 if (framerate == 0) |
414 framerate = codec.maxFramerate; | 407 framerate = codec.maxFramerate; |
415 stats_proxy_->OnEncoderReconfigured( | 408 stats_proxy_->OnEncoderReconfigured( |
416 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); | 409 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); |
417 } | 410 } |
418 | 411 |
419 pending_encoder_reconfiguration_ = false; | 412 pending_encoder_reconfiguration_ = false; |
420 | 413 |
421 sink_->OnEncoderConfigurationChanged( | 414 sink_->OnEncoderConfigurationChanged( |
422 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | 415 std::move(streams), encoder_config_.min_transmit_bitrate_bps); |
| 416 |
| 417 const auto scaling_settings = settings_.encoder->GetScalingSettings(); |
| 418 if (scaling_settings.enabled && scaling_enabled_) { |
| 419 if (scaling_settings.thresholds) { |
| 420 quality_scaler_.reset( |
| 421 new QualityScaler(this, *(scaling_settings.thresholds))); |
| 422 } else { |
| 423 quality_scaler_.reset(new QualityScaler(this, codec_type_)); |
| 424 } |
| 425 } else { |
| 426 quality_scaler_.reset(nullptr); |
| 427 } |
423 } | 428 } |
424 | 429 |
425 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | 430 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { |
426 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | 431 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
427 VideoFrame incoming_frame = video_frame; | 432 VideoFrame incoming_frame = video_frame; |
428 | 433 |
429 // Local time in webrtc time base. | 434 // Local time in webrtc time base. |
430 int64_t current_time = clock_->TimeInMilliseconds(); | 435 int64_t current_time = clock_->TimeInMilliseconds(); |
431 incoming_frame.set_render_time_ms(current_time); | 436 incoming_frame.set_render_time_ms(current_time); |
432 | 437 |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
490 // End trace event on first frame after encoder resumes, if frame was dropped. | 495 // End trace event on first frame after encoder resumes, if frame was dropped. |
491 if (encoder_paused_and_dropped_frame_) { | 496 if (encoder_paused_and_dropped_frame_) { |
492 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | 497 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
493 } | 498 } |
494 encoder_paused_and_dropped_frame_ = false; | 499 encoder_paused_and_dropped_frame_ = false; |
495 } | 500 } |
496 | 501 |
497 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, | 502 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, |
498 int64_t time_when_posted_in_ms) { | 503 int64_t time_when_posted_in_ms) { |
499 RTC_DCHECK_RUN_ON(&encoder_queue_); | 504 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 505 |
500 if (pre_encode_callback_) | 506 if (pre_encode_callback_) |
501 pre_encode_callback_->OnFrame(video_frame); | 507 pre_encode_callback_->OnFrame(video_frame); |
502 | 508 |
503 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || | 509 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || |
504 video_frame.height() != last_frame_info_->height || | 510 video_frame.height() != last_frame_info_->height || |
505 video_frame.rotation() != last_frame_info_->rotation || | 511 video_frame.rotation() != last_frame_info_->rotation || |
506 video_frame.is_texture() != last_frame_info_->is_texture) { | 512 video_frame.is_texture() != last_frame_info_->is_texture) { |
507 pending_encoder_reconfiguration_ = true; | 513 pending_encoder_reconfiguration_ = true; |
508 last_frame_info_ = rtc::Optional<VideoFrameInfo>( | 514 last_frame_info_ = rtc::Optional<VideoFrameInfo>( |
509 VideoFrameInfo(video_frame.width(), video_frame.height(), | 515 VideoFrameInfo(video_frame.width(), video_frame.height(), |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
567 // running in parallel on different threads. | 573 // running in parallel on different threads. |
568 if (stats_proxy_) { | 574 if (stats_proxy_) { |
569 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 575 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
570 } | 576 } |
571 | 577 |
572 EncodedImageCallback::Result result = | 578 EncodedImageCallback::Result result = |
573 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | 579 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); |
574 | 580 |
575 int64_t time_sent = clock_->TimeInMilliseconds(); | 581 int64_t time_sent = clock_->TimeInMilliseconds(); |
576 uint32_t timestamp = encoded_image._timeStamp; | 582 uint32_t timestamp = encoded_image._timeStamp; |
577 | 583 const int qp = encoded_image.qp_; |
578 encoder_queue_.PostTask([this, timestamp, time_sent] { | 584 encoder_queue_.PostTask([this, timestamp, time_sent, qp] { |
579 RTC_DCHECK_RUN_ON(&encoder_queue_); | 585 RTC_DCHECK_RUN_ON(&encoder_queue_); |
580 overuse_detector_.FrameSent(timestamp, time_sent); | 586 overuse_detector_.FrameSent(timestamp, time_sent); |
| 587 if (quality_scaler_) |
| 588 quality_scaler_->ReportQP(qp); |
581 }); | 589 }); |
582 | 590 |
583 return result; | 591 return result; |
584 } | 592 } |
585 | 593 |
| 594 void ViEEncoder::OnDroppedFrame() { |
| 595 encoder_queue_.PostTask([this] { |
| 596 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 597 if (quality_scaler_) |
| 598 quality_scaler_->ReportDroppedFrame(); |
| 599 }); |
| 600 } |
| 601 |
586 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { | 602 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { |
587 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | 603 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); |
588 if (stats_proxy_) | 604 if (stats_proxy_) |
589 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate); | 605 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate); |
590 } | 606 } |
591 | 607 |
592 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { | 608 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
593 if (!encoder_queue_.IsCurrent()) { | 609 if (!encoder_queue_.IsCurrent()) { |
594 encoder_queue_.PostTask([this, picture_id] { OnReceivedSLI(picture_id); }); | 610 encoder_queue_.PostTask([this, picture_id] { OnReceivedSLI(picture_id); }); |
595 return; | 611 return; |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
647 bool video_suspension_changed = video_is_suspended != EncoderPaused(); | 663 bool video_suspension_changed = video_is_suspended != EncoderPaused(); |
648 last_observed_bitrate_bps_ = bitrate_bps; | 664 last_observed_bitrate_bps_ = bitrate_bps; |
649 | 665 |
650 if (stats_proxy_ && video_suspension_changed) { | 666 if (stats_proxy_ && video_suspension_changed) { |
651 LOG(LS_INFO) << "Video suspend state changed to: " | 667 LOG(LS_INFO) << "Video suspend state changed to: " |
652 << (video_is_suspended ? "suspended" : "not suspended"); | 668 << (video_is_suspended ? "suspended" : "not suspended"); |
653 stats_proxy_->OnSuspendChange(video_is_suspended); | 669 stats_proxy_->OnSuspendChange(video_is_suspended); |
654 } | 670 } |
655 } | 671 } |
656 | 672 |
657 void ViEEncoder::OveruseDetected() { | 673 void ViEEncoder::ScaleDown(ScaleReason reason) { |
658 RTC_DCHECK_RUN_ON(&encoder_queue_); | 674 RTC_DCHECK_RUN_ON(&encoder_queue_); |
659 if (degradation_preference_ == | 675 if (!scaling_enabled_) |
660 VideoSendStream::DegradationPreference::kMaintainResolution || | |
661 cpu_restricted_counter_ >= kMaxCpuDowngrades) { | |
662 return; | 676 return; |
663 } | |
664 LOG(LS_INFO) << "CPU overuse detected. Requesting lower resolution."; | |
665 // Request lower resolution if the current resolution is lower than last time | 677 // Request lower resolution if the current resolution is lower than last time |
666 // we asked for the resolution to be lowered. | 678 // we asked for the resolution to be lowered. |
667 // Update stats accordingly. | |
668 int current_pixel_count = last_frame_height_ * last_frame_width_; | 679 int current_pixel_count = last_frame_height_ * last_frame_width_; |
669 if (!max_pixel_count_ || current_pixel_count < *max_pixel_count_) { | 680 if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) |
670 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); | 681 return; |
671 max_pixel_count_step_up_ = rtc::Optional<int>(); | 682 switch (reason) { |
672 stats_proxy_->OnCpuRestrictedResolutionChanged(true); | 683 case kQuality: |
673 ++cpu_restricted_counter_; | 684 if (scale_counter_[reason] >= kMaxQualityDowngrades) |
674 source_proxy_->RequestResolutionLowerThan(current_pixel_count); | 685 return; |
| 686 stats_proxy_->OnQualityRestrictedResolutionChanged(true); |
| 687 break; |
| 688 case kCpu: |
| 689 if (scale_counter_[reason] >= kMaxCpuDowngrades) |
| 690 return; |
| 691 // Update stats accordingly. |
| 692 stats_proxy_->OnCpuRestrictedResolutionChanged(true); |
| 693 break; |
| 694 } |
| 695 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); |
| 696 max_pixel_count_step_up_ = rtc::Optional<int>(); |
| 697 ++scale_counter_[reason]; |
| 698 source_proxy_->RequestResolutionLowerThan(current_pixel_count); |
| 699 LOG(LS_INFO) << "Scaling down resolution."; |
| 700 for (size_t i = 0; i < kScaleReasonSize; ++i) { |
| 701 LOG(LS_INFO) << "Scaled " << scale_counter_[i] |
| 702 << " times for reason: " << (i ? "quality" : "cpu"); |
675 } | 703 } |
676 } | 704 } |
677 | 705 |
678 void ViEEncoder::NormalUsage() { | 706 void ViEEncoder::ScaleUp(ScaleReason reason) { |
679 RTC_DCHECK_RUN_ON(&encoder_queue_); | 707 RTC_DCHECK_RUN_ON(&encoder_queue_); |
680 if (degradation_preference_ == | 708 if (scale_counter_[reason] == 0 || !scaling_enabled_) |
681 VideoSendStream::DegradationPreference::kMaintainResolution || | |
682 cpu_restricted_counter_ == 0) { | |
683 return; | 709 return; |
| 710 // Only scale if resolution is higher than last time |
| 711 // we requested higher resolution. |
| 712 int current_pixel_count = last_frame_height_ * last_frame_width_; |
| 713 if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) |
| 714 return; |
| 715 switch (reason) { |
| 716 case kQuality: |
| 717 stats_proxy_->OnQualityRestrictedResolutionChanged( |
| 718 scale_counter_[reason] > 1); |
| 719 break; |
| 720 case kCpu: |
| 721 // Update stats accordingly. |
| 722 stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > |
| 723 1); |
| 724 break; |
684 } | 725 } |
685 | 726 max_pixel_count_ = rtc::Optional<int>(); |
686 LOG(LS_INFO) << "CPU underuse detected. Requesting higher resolution."; | 727 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); |
687 int current_pixel_count = last_frame_height_ * last_frame_width_; | 728 --scale_counter_[reason]; |
688 // Request higher resolution if we are CPU restricted and the the current | 729 source_proxy_->RequestHigherResolutionThan(current_pixel_count); |
689 // resolution is higher than last time we requested higher resolution. | 730 LOG(LS_INFO) << "Scaling up resolution."; |
690 // Update stats accordingly. | 731 for (size_t i = 0; i < kScaleReasonSize; ++i) { |
691 if (!max_pixel_count_step_up_ || | 732 LOG(LS_INFO) << "Scaled " << scale_counter_[i] |
692 current_pixel_count > *max_pixel_count_step_up_) { | 733 << " times for reason: " << (i ? "quality" : "cpu"); |
693 max_pixel_count_ = rtc::Optional<int>(); | |
694 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); | |
695 --cpu_restricted_counter_; | |
696 stats_proxy_->OnCpuRestrictedResolutionChanged(cpu_restricted_counter_ > 0); | |
697 source_proxy_->RequestHigherResolutionThan(current_pixel_count); | |
698 } | 734 } |
699 } | 735 } |
700 | 736 |
701 } // namespace webrtc | 737 } // namespace webrtc |
OLD | NEW |