| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
| 12 | 12 |
| 13 #include <algorithm> | 13 #include <algorithm> |
| 14 #include <limits> | 14 #include <limits> |
| 15 #include <utility> | 15 #include <utility> |
| 16 | 16 |
| 17 #include "webrtc/modules/video_coding/include/video_codec_initializer.h" | 17 #include "webrtc/modules/video_coding/include/video_codec_initializer.h" |
| 18 #include "webrtc/base/arraysize.h" | |
| 19 #include "webrtc/base/checks.h" | 18 #include "webrtc/base/checks.h" |
| 20 #include "webrtc/base/logging.h" | 19 #include "webrtc/base/logging.h" |
| 21 #include "webrtc/base/trace_event.h" | 20 #include "webrtc/base/trace_event.h" |
| 22 #include "webrtc/base/timeutils.h" | 21 #include "webrtc/base/timeutils.h" |
| 23 #include "webrtc/common_video/include/video_bitrate_allocator.h" | 22 #include "webrtc/common_video/include/video_bitrate_allocator.h" |
| 24 #include "webrtc/modules/pacing/paced_sender.h" | 23 #include "webrtc/modules/pacing/paced_sender.h" |
| 25 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" | 24 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" |
| 26 #include "webrtc/modules/video_coding/include/video_coding.h" | 25 #include "webrtc/modules/video_coding/include/video_coding.h" |
| 27 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 26 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| 28 #include "webrtc/video/overuse_frame_detector.h" | 27 #include "webrtc/video/overuse_frame_detector.h" |
| (...skipping 227 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 256 encoder_start_bitrate_bps_(0), | 255 encoder_start_bitrate_bps_(0), |
| 257 max_data_payload_length_(0), | 256 max_data_payload_length_(0), |
| 258 nack_enabled_(false), | 257 nack_enabled_(false), |
| 259 last_observed_bitrate_bps_(0), | 258 last_observed_bitrate_bps_(0), |
| 260 encoder_paused_and_dropped_frame_(false), | 259 encoder_paused_and_dropped_frame_(false), |
| 261 has_received_sli_(false), | 260 has_received_sli_(false), |
| 262 picture_id_sli_(0), | 261 picture_id_sli_(0), |
| 263 has_received_rpsi_(false), | 262 has_received_rpsi_(false), |
| 264 picture_id_rpsi_(0), | 263 picture_id_rpsi_(0), |
| 265 clock_(Clock::GetRealTimeClock()), | 264 clock_(Clock::GetRealTimeClock()), |
| 266 scale_counter_(kScaleReasonSize, 0), | |
| 267 last_frame_width_(0), | 265 last_frame_width_(0), |
| 268 last_frame_height_(0), | 266 last_frame_height_(0), |
| 269 last_captured_timestamp_(0), | 267 last_captured_timestamp_(0), |
| 270 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 268 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - |
| 271 clock_->TimeInMilliseconds()), | 269 clock_->TimeInMilliseconds()), |
| 272 last_frame_log_ms_(clock_->TimeInMilliseconds()), | 270 last_frame_log_ms_(clock_->TimeInMilliseconds()), |
| 273 captured_frame_count_(0), | 271 captured_frame_count_(0), |
| 274 dropped_frame_count_(0), | 272 dropped_frame_count_(0), |
| 275 bitrate_observer_(nullptr), | 273 bitrate_observer_(nullptr), |
| 276 encoder_queue_("EncoderQueue") { | 274 encoder_queue_("EncoderQueue") { |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 328 }); | 326 }); |
| 329 } | 327 } |
| 330 | 328 |
| 331 void ViEEncoder::SetSource( | 329 void ViEEncoder::SetSource( |
| 332 rtc::VideoSourceInterface<VideoFrame>* source, | 330 rtc::VideoSourceInterface<VideoFrame>* source, |
| 333 const VideoSendStream::DegradationPreference& degradation_preference) { | 331 const VideoSendStream::DegradationPreference& degradation_preference) { |
| 334 RTC_DCHECK_RUN_ON(&thread_checker_); | 332 RTC_DCHECK_RUN_ON(&thread_checker_); |
| 335 source_proxy_->SetSource(source, degradation_preference); | 333 source_proxy_->SetSource(source, degradation_preference); |
| 336 encoder_queue_.PostTask([this, degradation_preference] { | 334 encoder_queue_.PostTask([this, degradation_preference] { |
| 337 RTC_DCHECK_RUN_ON(&encoder_queue_); | 335 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 338 scaling_enabled_ = (degradation_preference != | 336 scaling_enabled_ = |
| 337 (degradation_preference != |
| 339 VideoSendStream::DegradationPreference::kMaintainResolution); | 338 VideoSendStream::DegradationPreference::kMaintainResolution); |
| 340 stats_proxy_->SetResolutionRestrictionStats( | 339 stats_proxy_->SetResolutionRestrictionStats( |
| 341 scaling_enabled_, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); | 340 scaling_enabled_ && scale_counter_[kQuality] > 0, |
| 341 scaling_enabled_ && scale_counter_[kCpu] > 0); |
| 342 }); | 342 }); |
| 343 } | 343 } |
| 344 | 344 |
| 345 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { | 345 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { |
| 346 source_proxy_->SetWantsRotationApplied(rotation_applied); | 346 source_proxy_->SetWantsRotationApplied(rotation_applied); |
| 347 encoder_queue_.PostTask([this, sink] { | 347 encoder_queue_.PostTask([this, sink] { |
| 348 RTC_DCHECK_RUN_ON(&encoder_queue_); | 348 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 349 sink_ = sink; | 349 sink_ = sink; |
| 350 }); | 350 }); |
| 351 } | 351 } |
| (...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 427 stats_proxy_->OnEncoderReconfigured( | 427 stats_proxy_->OnEncoderReconfigured( |
| 428 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); | 428 encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); |
| 429 } | 429 } |
| 430 | 430 |
| 431 pending_encoder_reconfiguration_ = false; | 431 pending_encoder_reconfiguration_ = false; |
| 432 | 432 |
| 433 sink_->OnEncoderConfigurationChanged( | 433 sink_->OnEncoderConfigurationChanged( |
| 434 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | 434 std::move(streams), encoder_config_.min_transmit_bitrate_bps); |
| 435 | 435 |
| 436 const auto scaling_settings = settings_.encoder->GetScalingSettings(); | 436 const auto scaling_settings = settings_.encoder->GetScalingSettings(); |
| 437 scaling_enabled_ &= scaling_settings.enabled; | 437 if (scaling_settings.enabled && scaling_enabled_) { |
| 438 if (scaling_enabled_) { | |
| 439 if (scaling_settings.thresholds) { | 438 if (scaling_settings.thresholds) { |
| 440 quality_scaler_.reset( | 439 quality_scaler_.reset( |
| 441 new QualityScaler(this, *(scaling_settings.thresholds))); | 440 new QualityScaler(this, *(scaling_settings.thresholds))); |
| 442 } else { | 441 } else { |
| 443 quality_scaler_.reset(new QualityScaler(this, codec_type_)); | 442 quality_scaler_.reset(new QualityScaler(this, codec_type_)); |
| 444 } | 443 } |
| 445 } else { | 444 } else { |
| 446 quality_scaler_.reset(nullptr); | 445 quality_scaler_.reset(nullptr); |
| 447 stats_proxy_->SetResolutionRestrictionStats( | |
| 448 false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]); | |
| 449 } | 446 } |
| 450 } | 447 } |
| 451 | 448 |
| 452 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | 449 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { |
| 453 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | 450 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
| 454 VideoFrame incoming_frame = video_frame; | 451 VideoFrame incoming_frame = video_frame; |
| 455 | 452 |
| 456 // Local time in webrtc time base. | 453 // Local time in webrtc time base. |
| 457 int64_t current_time = clock_->TimeInMilliseconds(); | 454 int64_t current_time = clock_->TimeInMilliseconds(); |
| 458 incoming_frame.set_render_time_ms(current_time); | 455 incoming_frame.set_render_time_ms(current_time); |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 586 video_sender_.IntraFrameRequest(0); | 583 video_sender_.IntraFrameRequest(0); |
| 587 } | 584 } |
| 588 | 585 |
| 589 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( | 586 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( |
| 590 const EncodedImage& encoded_image, | 587 const EncodedImage& encoded_image, |
| 591 const CodecSpecificInfo* codec_specific_info, | 588 const CodecSpecificInfo* codec_specific_info, |
| 592 const RTPFragmentationHeader* fragmentation) { | 589 const RTPFragmentationHeader* fragmentation) { |
| 593 // Encoded is called on whatever thread the real encoder implementation run | 590 // Encoded is called on whatever thread the real encoder implementation run |
| 594 // on. In the case of hardware encoders, there might be several encoders | 591 // on. In the case of hardware encoders, there might be several encoders |
| 595 // running in parallel on different threads. | 592 // running in parallel on different threads. |
| 596 if (stats_proxy_) | 593 if (stats_proxy_) { |
| 597 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 594 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
| 595 } |
| 598 | 596 |
| 599 EncodedImageCallback::Result result = | 597 EncodedImageCallback::Result result = |
| 600 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | 598 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); |
| 601 | 599 |
| 602 int64_t time_sent = clock_->TimeInMilliseconds(); | 600 int64_t time_sent = clock_->TimeInMilliseconds(); |
| 603 uint32_t timestamp = encoded_image._timeStamp; | 601 uint32_t timestamp = encoded_image._timeStamp; |
| 604 const int qp = encoded_image.qp_; | 602 const int qp = encoded_image.qp_; |
| 605 encoder_queue_.PostTask([this, timestamp, time_sent, qp] { | 603 encoder_queue_.PostTask([this, timestamp, time_sent, qp] { |
| 606 RTC_DCHECK_RUN_ON(&encoder_queue_); | 604 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 607 overuse_detector_.FrameSent(timestamp, time_sent); | 605 overuse_detector_.FrameSent(timestamp, time_sent); |
| (...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 696 RTC_DCHECK_RUN_ON(&encoder_queue_); | 694 RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 697 if (!scaling_enabled_) | 695 if (!scaling_enabled_) |
| 698 return; | 696 return; |
| 699 // Request lower resolution if the current resolution is lower than last time | 697 // Request lower resolution if the current resolution is lower than last time |
| 700 // we asked for the resolution to be lowered. | 698 // we asked for the resolution to be lowered. |
| 701 int current_pixel_count = last_frame_height_ * last_frame_width_; | 699 int current_pixel_count = last_frame_height_ * last_frame_width_; |
| 702 if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) | 700 if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_) |
| 703 return; | 701 return; |
| 704 switch (reason) { | 702 switch (reason) { |
| 705 case kQuality: | 703 case kQuality: |
| 706 stats_proxy_->OnQualityRestrictedResolutionChanged( | 704 stats_proxy_->OnQualityRestrictedResolutionChanged(true); |
| 707 scale_counter_[reason] + 1); | |
| 708 break; | 705 break; |
| 709 case kCpu: | 706 case kCpu: |
| 710 if (scale_counter_[reason] >= kMaxCpuDowngrades) | 707 if (scale_counter_[reason] >= kMaxCpuDowngrades) |
| 711 return; | 708 return; |
| 712 // Update stats accordingly. | 709 // Update stats accordingly. |
| 713 stats_proxy_->OnCpuRestrictedResolutionChanged(true); | 710 stats_proxy_->OnCpuRestrictedResolutionChanged(true); |
| 714 break; | 711 break; |
| 715 } | 712 } |
| 716 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); | 713 max_pixel_count_ = rtc::Optional<int>(current_pixel_count); |
| 717 max_pixel_count_step_up_ = rtc::Optional<int>(); | 714 max_pixel_count_step_up_ = rtc::Optional<int>(); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 729 if (scale_counter_[reason] == 0 || !scaling_enabled_) | 726 if (scale_counter_[reason] == 0 || !scaling_enabled_) |
| 730 return; | 727 return; |
| 731 // Only scale if resolution is higher than last time | 728 // Only scale if resolution is higher than last time |
| 732 // we requested higher resolution. | 729 // we requested higher resolution. |
| 733 int current_pixel_count = last_frame_height_ * last_frame_width_; | 730 int current_pixel_count = last_frame_height_ * last_frame_width_; |
| 734 if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) | 731 if (current_pixel_count <= max_pixel_count_step_up_.value_or(0)) |
| 735 return; | 732 return; |
| 736 switch (reason) { | 733 switch (reason) { |
| 737 case kQuality: | 734 case kQuality: |
| 738 stats_proxy_->OnQualityRestrictedResolutionChanged( | 735 stats_proxy_->OnQualityRestrictedResolutionChanged( |
| 739 scale_counter_[reason] - 1); | 736 scale_counter_[reason] > 1); |
| 740 break; | 737 break; |
| 741 case kCpu: | 738 case kCpu: |
| 742 // Update stats accordingly. | 739 // Update stats accordingly. |
| 743 stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > | 740 stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] > |
| 744 1); | 741 1); |
| 745 break; | 742 break; |
| 746 } | 743 } |
| 747 max_pixel_count_ = rtc::Optional<int>(); | 744 max_pixel_count_ = rtc::Optional<int>(); |
| 748 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); | 745 max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count); |
| 749 --scale_counter_[reason]; | 746 --scale_counter_[reason]; |
| 750 source_proxy_->RequestHigherResolutionThan(current_pixel_count); | 747 source_proxy_->RequestHigherResolutionThan(current_pixel_count); |
| 751 LOG(LS_INFO) << "Scaling up resolution."; | 748 LOG(LS_INFO) << "Scaling up resolution."; |
| 752 for (size_t i = 0; i < kScaleReasonSize; ++i) { | 749 for (size_t i = 0; i < kScaleReasonSize; ++i) { |
| 753 LOG(LS_INFO) << "Scaled " << scale_counter_[i] | 750 LOG(LS_INFO) << "Scaled " << scale_counter_[i] |
| 754 << " times for reason: " << (i ? "quality" : "cpu"); | 751 << " times for reason: " << (i ? "quality" : "cpu"); |
| 755 } | 752 } |
| 756 } | 753 } |
| 757 | 754 |
| 758 } // namespace webrtc | 755 } // namespace webrtc |
| OLD | NEW |