| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
| 12 | 12 |
| 13 #include <algorithm> | 13 #include <algorithm> |
| 14 #include <limits> | 14 #include <limits> |
| 15 #include <utility> | 15 #include <utility> |
| 16 | 16 |
| 17 #include "webrtc/modules/video_coding/include/video_codec_initializer.h" | 17 #include "webrtc/modules/video_coding/include/video_codec_initializer.h" |
| 18 #include "webrtc/base/arraysize.h" |
| 18 #include "webrtc/base/checks.h" | 19 #include "webrtc/base/checks.h" |
| 19 #include "webrtc/base/logging.h" | 20 #include "webrtc/base/logging.h" |
| 20 #include "webrtc/base/trace_event.h" | 21 #include "webrtc/base/trace_event.h" |
| 21 #include "webrtc/base/timeutils.h" | 22 #include "webrtc/base/timeutils.h" |
| 22 #include "webrtc/common_video/include/video_bitrate_allocator.h" | 23 #include "webrtc/common_video/include/video_bitrate_allocator.h" |
| 23 #include "webrtc/modules/pacing/paced_sender.h" | 24 #include "webrtc/modules/pacing/paced_sender.h" |
| 24 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" | 25 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" |
| 25 #include "webrtc/modules/video_coding/include/video_coding.h" | 26 #include "webrtc/modules/video_coding/include/video_coding.h" |
| 26 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 27 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| 27 #include "webrtc/video/overuse_frame_detector.h" | 28 #include "webrtc/video/overuse_frame_detector.h" |
| (...skipping 227 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 255 encoder_start_bitrate_bps_(0), | 256 encoder_start_bitrate_bps_(0), |
| 256 max_data_payload_length_(0), | 257 max_data_payload_length_(0), |
| 257 nack_enabled_(false), | 258 nack_enabled_(false), |
| 258 last_observed_bitrate_bps_(0), | 259 last_observed_bitrate_bps_(0), |
| 259 encoder_paused_and_dropped_frame_(false), | 260 encoder_paused_and_dropped_frame_(false), |
| 260 has_received_sli_(false), | 261 has_received_sli_(false), |
| 261 picture_id_sli_(0), | 262 picture_id_sli_(0), |
| 262 has_received_rpsi_(false), | 263 has_received_rpsi_(false), |
| 263 picture_id_rpsi_(0), | 264 picture_id_rpsi_(0), |
| 264 clock_(Clock::GetRealTimeClock()), | 265 clock_(Clock::GetRealTimeClock()), |
| 266 scale_counter_(kScaleReasonSize, 0), |
| 265 last_frame_width_(0), | 267 last_frame_width_(0), |
| 266 last_frame_height_(0), | 268 last_frame_height_(0), |
| 267 last_captured_timestamp_(0), | 269 last_captured_timestamp_(0), |
| 268 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 270 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - |
| 269 clock_->TimeInMilliseconds()), | 271 clock_->TimeInMilliseconds()), |
| 270 last_frame_log_ms_(clock_->TimeInMilliseconds()), | 272 last_frame_log_ms_(clock_->TimeInMilliseconds()), |
| 271 captured_frame_count_(0), | 273 captured_frame_count_(0), |
| 272 dropped_frame_count_(0), | 274 dropped_frame_count_(0), |
| 273 bitrate_observer_(nullptr), | 275 bitrate_observer_(nullptr), |
| 274 encoder_queue_("EncoderQueue") { | 276 encoder_queue_("EncoderQueue") { |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 326 }); | 328 }); |
| 327 } | 329 } |
| 328 | 330 |
| 329 void ViEEncoder::SetSource( | 331 void ViEEncoder::SetSource( |
| 330 rtc::VideoSourceInterface<VideoFrame>* source, | 332 rtc::VideoSourceInterface<VideoFrame>* source, |
| 331 const VideoSendStream::DegradationPreference& degradation_preference) { | 333 const VideoSendStream::DegradationPreference& degradation_preference) { |
| 332 RTC_DCHECK_RUN_ON(&thread_checker_); | 334 RTC_DCHECK_RUN_ON(&thread_checker_); |
| 333 source_proxy_->SetSource(source, degradation_preference); | 335 source_proxy_->SetSource(source, degradation_preference); |
| 334 encoder_queue_.PostTask([this, degradation_preference] { | 336 encoder_queue_.PostTask([this, degradation_preference] { |
| 335 RTC_DCHECK_RUN_ON(&encoder_queue_); | 337 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 336 scaling_enabled_ = | 338 scaling_enabled_ = (degradation_preference != |
| 337 (degradation_preference != | |
| 338 VideoSendStream::DegradationPreference::kMaintainResolution); | 339 VideoSendStream::DegradationPreference::kMaintainResolution); |
| 339 stats_proxy_->SetResolutionRestrictionStats( | 340 stats_proxy_->SetResolutionRestrictionStats( |
| 340 scaling_enabled_ && scale_counter_[kQuality] > 0, | 341 scaling_enabled_, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); |
| 341 scaling_enabled_ && scale_counter_[kCpu] > 0); | |
| 342 }); | 342 }); |
| 343 } | 343 } |
| 344 | 344 |
| 345 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { | 345 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { |
| 346 source_proxy_->SetWantsRotationApplied(rotation_applied); | 346 source_proxy_->SetWantsRotationApplied(rotation_applied); |
| 347 encoder_queue_.PostTask([this, sink] { | 347 encoder_queue_.PostTask([this, sink] { |
| 348 RTC_DCHECK_RUN_ON(&encoder_queue_); | 348 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 349 sink_ = sink; | 349 sink_ = sink; |
| 350 }); | 350 }); |
| 351 } | 351 } |
| (...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 427 stats_proxy_->OnEncoderReconfigured( | 427 stats_proxy_->OnEncoderReconfigured( |
| 428 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); | 428 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); |
| 429 } | 429 } |
| 430 | 430 |
| 431 pending_encoder_reconfiguration_ = false; | 431 pending_encoder_reconfiguration_ = false; |
| 432 | 432 |
| 433 sink_->OnEncoderConfigurationChanged( | 433 sink_->OnEncoderConfigurationChanged( |
| 434 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | 434 std::move(streams), encoder_config_.min_transmit_bitrate_bps); |
| 435 | 435 |
| 436 const auto scaling_settings = settings_.encoder->GetScalingSettings(); | 436 const auto scaling_settings = settings_.encoder->GetScalingSettings(); |
| 437 if (scaling_settings.enabled && scaling_enabled_) { | 437 scaling_enabled_ &= scaling_settings.enabled; |
| 438 if (scaling_enabled_) { |
| 438 if (scaling_settings.thresholds) { | 439 if (scaling_settings.thresholds) { |
| 439 quality_scaler_.reset( | 440 quality_scaler_.reset( |
| 440 new QualityScaler(this, *(scaling_settings.thresholds))); | 441 new QualityScaler(this, *(scaling_settings.thresholds))); |
| 441 } else { | 442 } else { |
| 442 quality_scaler_.reset(new QualityScaler(this, codec_type_)); | 443 quality_scaler_.reset(new QualityScaler(this, codec_type_)); |
| 443 } | 444 } |
| 444 } else { | 445 } else { |
| 445 quality_scaler_.reset(nullptr); | 446 quality_scaler_.reset(nullptr); |
| 447 stats_proxy_->SetResolutionRestrictionStats( |
| 448 false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); |
| 446 } | 449 } |
| 447 } | 450 } |
| 448 | 451 |
| 449 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | 452 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { |
| 450 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | 453 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
| 451 VideoFrame incoming_frame = video_frame; | 454 VideoFrame incoming_frame = video_frame; |
| 452 | 455 |
| 453 // Local time in webrtc time base. | 456 // Local time in webrtc time base. |
| 454 int64_t current_time = clock_->TimeInMilliseconds(); | 457 int64_t current_time = clock_->TimeInMilliseconds(); |
| 455 incoming_frame.set_render_time_ms(current_time); | 458 incoming_frame.set_render_time_ms(current_time); |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 583 video_sender_.IntraFrameRequest(0); | 586 video_sender_.IntraFrameRequest(0); |
| 584 } | 587 } |
| 585 | 588 |
| 586 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( | 589 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( |
| 587 const EncodedImage& encoded_image, | 590 const EncodedImage& encoded_image, |
| 588 const CodecSpecificInfo* codec_specific_info, | 591 const CodecSpecificInfo* codec_specific_info, |
| 589 const RTPFragmentationHeader* fragmentation) { | 592 const RTPFragmentationHeader* fragmentation) { |
| 590 // Encoded is called on whatever thread the real encoder implementation run | 593 // Encoded is called on whatever thread the real encoder implementation run |
| 591 // on. In the case of hardware encoders, there might be several encoders | 594 // on. In the case of hardware encoders, there might be several encoders |
| 592 // running in parallel on different threads. | 595 // running in parallel on different threads. |
| 593 if (stats_proxy_) { | 596 if (stats_proxy_) |
| 594 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 597 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
| 595 } | |
| 596 | 598 |
| 597 EncodedImageCallback::Result result = | 599 EncodedImageCallback::Result result = |
| 598 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | 600 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); |
| 599 | 601 |
| 600 int64_t time_sent = clock_->TimeInMilliseconds(); | 602 int64_t time_sent = clock_->TimeInMilliseconds(); |
| 601 uint32_t timestamp = encoded_image._timeStamp; | 603 uint32_t timestamp = encoded_image._timeStamp; |
| 602 const int qp = encoded_image.qp_; | 604 const int qp = encoded_image.qp_; |
| 603 encoder_queue_.PostTask([this, timestamp, time_sent, qp] { | 605 encoder_queue_.PostTask([this, timestamp, time_sent, qp] { |
| 604 RTC_DCHECK_RUN_ON(&encoder_queue_); | 606 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 605 overuse_detector_.FrameSent(timestamp, time_sent); | 607 overuse_detector_.FrameSent(timestamp, time_sent); |
| (...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 694 RTC_DCHECK_RUN_ON(&encoder_queue_); | 696 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 695 if (!scaling_enabled_) | 697 if (!scaling_enabled_) |
| 696 return; | 698 return; |
| 697 // Request lower resolution if the current resolution is lower than last time | 699 // Request lower resolution if the current resolution is lower than last time |
| 698 // we asked for the resolution to be lowered. | 700 // we asked for the resolution to be lowered. |
| 699 int current_pixel_count = last_frame_height_ * last_frame_width_; | 701 int current_pixel_count = last_frame_height_ * last_frame_width_; |
| 700 if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) | 702 if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) |
| 701 return; | 703 return; |
| 702 switch (reason) { | 704 switch (reason) { |
| 703 case kQuality: | 705 case kQuality: |
| 704 stats_proxy_->OnQualityRestrictedResolutionChanged(true); | 706 stats_proxy_->OnQualityRestrictedResolutionChanged( |
| 707 scale_counter_[reason] + 1); |
| 705 break; | 708 break; |
| 706 case kCpu: | 709 case kCpu: |
| 707 if (scale_counter_[reason] >= kMaxCpuDowngrades) | 710 if (scale_counter_[reason] >= kMaxCpuDowngrades) |
| 708 return; | 711 return; |
| 709 // Update stats accordingly. | 712 // Update stats accordingly. |
| 710 stats_proxy_->OnCpuRestrictedResolutionChanged(true); | 713 stats_proxy_->OnCpuRestrictedResolutionChanged(true); |
| 711 break; | 714 break; |
| 712 } | 715 } |
| 713 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); | 716 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); |
| 714 max_pixel_count_step_up_ = rtc::Optional<int>(); | 717 max_pixel_count_step_up_ = rtc::Optional<int>(); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 726 if (scale_counter_[reason] == 0 || !scaling_enabled_) | 729 if (scale_counter_[reason] == 0 || !scaling_enabled_) |
| 727 return; | 730 return; |
| 728 // Only scale if resolution is higher than last time | 731 // Only scale if resolution is higher than last time |
| 729 // we requested higher resolution. | 732 // we requested higher resolution. |
| 730 int current_pixel_count = last_frame_height_ * last_frame_width_; | 733 int current_pixel_count = last_frame_height_ * last_frame_width_; |
| 731 if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) | 734 if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) |
| 732 return; | 735 return; |
| 733 switch (reason) { | 736 switch (reason) { |
| 734 case kQuality: | 737 case kQuality: |
| 735 stats_proxy_->OnQualityRestrictedResolutionChanged( | 738 stats_proxy_->OnQualityRestrictedResolutionChanged( |
| 736 scale_counter_[reason] > 1); | 739 scale_counter_[reason] - 1); |
| 737 break; | 740 break; |
| 738 case kCpu: | 741 case kCpu: |
| 739 // Update stats accordingly. | 742 // Update stats accordingly. |
| 740 stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > | 743 stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > |
| 741 1); | 744 1); |
| 742 break; | 745 break; |
| 743 } | 746 } |
| 744 max_pixel_count_ = rtc::Optional<int>(); | 747 max_pixel_count_ = rtc::Optional<int>(); |
| 745 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); | 748 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); |
| 746 --scale_counter_[reason]; | 749 --scale_counter_[reason]; |
| 747 source_proxy_->RequestHigherResolutionThan(current_pixel_count); | 750 source_proxy_->RequestHigherResolutionThan(current_pixel_count); |
| 748 LOG(LS_INFO) << "Scaling up resolution."; | 751 LOG(LS_INFO) << "Scaling up resolution."; |
| 749 for (size_t i = 0; i < kScaleReasonSize; ++i) { | 752 for (size_t i = 0; i < kScaleReasonSize; ++i) { |
| 750 LOG(LS_INFO) << "Scaled " << scale_counter_[i] | 753 LOG(LS_INFO) << "Scaled " << scale_counter_[i] |
| 751 << " times for reason: " << (i ? "quality" : "cpu"); | 754 << " times for reason: " << (i ? "quality" : "cpu"); |
| 752 } | 755 } |
| 753 } | 756 } |
| 754 | 757 |
| 755 } // namespace webrtc | 758 } // namespace webrtc |
| OLD | NEW |