| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 */ | |
| 10 | |
| 11 #include "webrtc/video/vie_encoder.h" | |
| 12 | |
| 13 #include <algorithm> | |
| 14 #include <limits> | |
| 15 #include <numeric> | |
| 16 #include <utility> | |
| 17 | |
| 18 #include "webrtc/api/video/i420_buffer.h" | |
| 19 #include "webrtc/common_video/include/video_bitrate_allocator.h" | |
| 20 #include "webrtc/common_video/include/video_frame.h" | |
| 21 #include "webrtc/modules/pacing/paced_sender.h" | |
| 22 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" | |
| 23 #include "webrtc/modules/video_coding/include/video_codec_initializer.h" | |
| 24 #include "webrtc/modules/video_coding/include/video_coding.h" | |
| 25 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | |
| 26 #include "webrtc/rtc_base/arraysize.h" | |
| 27 #include "webrtc/rtc_base/checks.h" | |
| 28 #include "webrtc/rtc_base/location.h" | |
| 29 #include "webrtc/rtc_base/logging.h" | |
| 30 #include "webrtc/rtc_base/timeutils.h" | |
| 31 #include "webrtc/rtc_base/trace_event.h" | |
| 32 #include "webrtc/video/overuse_frame_detector.h" | |
| 33 #include "webrtc/video/send_statistics_proxy.h" | |
| 34 | |
| 35 namespace webrtc { | |
| 36 | |
| 37 namespace { | |
| 38 | |
| 39 // Time interval for logging frame counts. | |
| 40 const int64_t kFrameLogIntervalMs = 60000; | |
| 41 | |
| 42 // We will never ask for a resolution lower than this. | |
| 43 // TODO(kthelgason): Lower this limit when better testing | |
| 44 // on MediaCodec and fallback implementations are in place. | |
| 45 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206 | |
| 46 const int kMinPixelsPerFrame = 320 * 180; | |
| 47 const int kMinFramerateFps = 2; | |
| 48 const int kMaxFramerateFps = 120; | |
| 49 | |
| 50 // The maximum number of frames to drop at beginning of stream | |
| 51 // to try and achieve desired bitrate. | |
| 52 const int kMaxInitialFramedrop = 4; | |
| 53 | |
| 54 uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { | |
| 55 if (kbps > 0) { | |
| 56 if (kbps < 300 /* qvga */) { | |
| 57 return 320 * 240; | |
| 58 } else if (kbps < 500 /* vga */) { | |
| 59 return 640 * 480; | |
| 60 } | |
| 61 } | |
| 62 return std::numeric_limits<uint32_t>::max(); | |
| 63 } | |
| 64 | |
| 65 // Initial limits for kBalanced degradation preference. | |
| 66 int MinFps(int pixels) { | |
| 67 if (pixels <= 320 * 240) { | |
| 68 return 7; | |
| 69 } else if (pixels <= 480 * 270) { | |
| 70 return 10; | |
| 71 } else if (pixels <= 640 * 480) { | |
| 72 return 15; | |
| 73 } else { | |
| 74 return std::numeric_limits<int>::max(); | |
| 75 } | |
| 76 } | |
| 77 | |
| 78 int MaxFps(int pixels) { | |
| 79 if (pixels <= 320 * 240) { | |
| 80 return 10; | |
| 81 } else if (pixels <= 480 * 270) { | |
| 82 return 15; | |
| 83 } else { | |
| 84 return std::numeric_limits<int>::max(); | |
| 85 } | |
| 86 } | |
| 87 | |
| 88 bool IsResolutionScalingEnabled( | |
| 89 VideoSendStream::DegradationPreference degradation_preference) { | |
| 90 return degradation_preference == | |
| 91 VideoSendStream::DegradationPreference::kMaintainFramerate || | |
| 92 degradation_preference == | |
| 93 VideoSendStream::DegradationPreference::kBalanced; | |
| 94 } | |
| 95 | |
| 96 bool IsFramerateScalingEnabled( | |
| 97 VideoSendStream::DegradationPreference degradation_preference) { | |
| 98 return degradation_preference == | |
| 99 VideoSendStream::DegradationPreference::kMaintainResolution || | |
| 100 degradation_preference == | |
| 101 VideoSendStream::DegradationPreference::kBalanced; | |
| 102 } | |
| 103 | |
| 104 } // namespace | |
| 105 | |
| 106 class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask { | |
| 107 public: | |
| 108 ConfigureEncoderTask(ViEEncoder* vie_encoder, | |
| 109 VideoEncoderConfig config, | |
| 110 size_t max_data_payload_length, | |
| 111 bool nack_enabled) | |
| 112 : vie_encoder_(vie_encoder), | |
| 113 config_(std::move(config)), | |
| 114 max_data_payload_length_(max_data_payload_length), | |
| 115 nack_enabled_(nack_enabled) {} | |
| 116 | |
| 117 private: | |
| 118 bool Run() override { | |
| 119 vie_encoder_->ConfigureEncoderOnTaskQueue( | |
| 120 std::move(config_), max_data_payload_length_, nack_enabled_); | |
| 121 return true; | |
| 122 } | |
| 123 | |
| 124 ViEEncoder* const vie_encoder_; | |
| 125 VideoEncoderConfig config_; | |
| 126 size_t max_data_payload_length_; | |
| 127 bool nack_enabled_; | |
| 128 }; | |
| 129 | |
| 130 class ViEEncoder::EncodeTask : public rtc::QueuedTask { | |
| 131 public: | |
| 132 EncodeTask(const VideoFrame& frame, | |
| 133 ViEEncoder* vie_encoder, | |
| 134 int64_t time_when_posted_us, | |
| 135 bool log_stats) | |
| 136 : frame_(frame), | |
| 137 vie_encoder_(vie_encoder), | |
| 138 time_when_posted_us_(time_when_posted_us), | |
| 139 log_stats_(log_stats) { | |
| 140 ++vie_encoder_->posted_frames_waiting_for_encode_; | |
| 141 } | |
| 142 | |
| 143 private: | |
| 144 bool Run() override { | |
| 145 RTC_DCHECK_RUN_ON(&vie_encoder_->encoder_queue_); | |
| 146 RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0); | |
| 147 vie_encoder_->stats_proxy_->OnIncomingFrame(frame_.width(), | |
| 148 frame_.height()); | |
| 149 ++vie_encoder_->captured_frame_count_; | |
| 150 if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) { | |
| 151 vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_us_); | |
| 152 } else { | |
| 153 // There is a newer frame in flight. Do not encode this frame. | |
| 154 LOG(LS_VERBOSE) | |
| 155 << "Incoming frame dropped due to that the encoder is blocked."; | |
| 156 ++vie_encoder_->dropped_frame_count_; | |
| 157 } | |
| 158 if (log_stats_) { | |
| 159 LOG(LS_INFO) << "Number of frames: captured " | |
| 160 << vie_encoder_->captured_frame_count_ | |
| 161 << ", dropped (due to encoder blocked) " | |
| 162 << vie_encoder_->dropped_frame_count_ << ", interval_ms " | |
| 163 << kFrameLogIntervalMs; | |
| 164 vie_encoder_->captured_frame_count_ = 0; | |
| 165 vie_encoder_->dropped_frame_count_ = 0; | |
| 166 } | |
| 167 return true; | |
| 168 } | |
| 169 VideoFrame frame_; | |
| 170 ViEEncoder* const vie_encoder_; | |
| 171 const int64_t time_when_posted_us_; | |
| 172 const bool log_stats_; | |
| 173 }; | |
| 174 | |
| 175 // VideoSourceProxy is responsible ensuring thread safety between calls to | |
| 176 // ViEEncoder::SetSource that will happen on libjingle's worker thread when a | |
| 177 // video capturer is connected to the encoder and the encoder task queue | |
| 178 // (encoder_queue_) where the encoder reports its VideoSinkWants. | |
| 179 class ViEEncoder::VideoSourceProxy { | |
| 180 public: | |
| 181 explicit VideoSourceProxy(ViEEncoder* vie_encoder) | |
| 182 : vie_encoder_(vie_encoder), | |
| 183 degradation_preference_( | |
| 184 VideoSendStream::DegradationPreference::kDegradationDisabled), | |
| 185 source_(nullptr) {} | |
| 186 | |
| 187 void SetSource( | |
| 188 rtc::VideoSourceInterface<VideoFrame>* source, | |
| 189 const VideoSendStream::DegradationPreference& degradation_preference) { | |
| 190 // Called on libjingle's worker thread. | |
| 191 RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_); | |
| 192 rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr; | |
| 193 rtc::VideoSinkWants wants; | |
| 194 { | |
| 195 rtc::CritScope lock(&crit_); | |
| 196 degradation_preference_ = degradation_preference; | |
| 197 old_source = source_; | |
| 198 source_ = source; | |
| 199 wants = GetActiveSinkWantsInternal(); | |
| 200 } | |
| 201 | |
| 202 if (old_source != source && old_source != nullptr) { | |
| 203 old_source->RemoveSink(vie_encoder_); | |
| 204 } | |
| 205 | |
| 206 if (!source) { | |
| 207 return; | |
| 208 } | |
| 209 | |
| 210 source->AddOrUpdateSink(vie_encoder_, wants); | |
| 211 } | |
| 212 | |
| 213 void SetWantsRotationApplied(bool rotation_applied) { | |
| 214 rtc::CritScope lock(&crit_); | |
| 215 sink_wants_.rotation_applied = rotation_applied; | |
| 216 if (source_) | |
| 217 source_->AddOrUpdateSink(vie_encoder_, sink_wants_); | |
| 218 } | |
| 219 | |
| 220 rtc::VideoSinkWants GetActiveSinkWants() { | |
| 221 rtc::CritScope lock(&crit_); | |
| 222 return GetActiveSinkWantsInternal(); | |
| 223 } | |
| 224 | |
| 225 void ResetPixelFpsCount() { | |
| 226 rtc::CritScope lock(&crit_); | |
| 227 sink_wants_.max_pixel_count = std::numeric_limits<int>::max(); | |
| 228 sink_wants_.target_pixel_count.reset(); | |
| 229 sink_wants_.max_framerate_fps = std::numeric_limits<int>::max(); | |
| 230 if (source_) | |
| 231 source_->AddOrUpdateSink(vie_encoder_, sink_wants_); | |
| 232 } | |
| 233 | |
| 234 bool RequestResolutionLowerThan(int pixel_count) { | |
| 235 // Called on the encoder task queue. | |
| 236 rtc::CritScope lock(&crit_); | |
| 237 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { | |
| 238 // This can happen since |degradation_preference_| is set on libjingle's | |
| 239 // worker thread but the adaptation is done on the encoder task queue. | |
| 240 return false; | |
| 241 } | |
| 242 // The input video frame size will have a resolution less than or equal to | |
| 243 // |max_pixel_count| depending on how the source can scale the frame size. | |
| 244 const int pixels_wanted = (pixel_count * 3) / 5; | |
| 245 if (pixels_wanted < kMinPixelsPerFrame || | |
| 246 pixels_wanted >= sink_wants_.max_pixel_count) { | |
| 247 return false; | |
| 248 } | |
| 249 LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; | |
| 250 sink_wants_.max_pixel_count = pixels_wanted; | |
| 251 sink_wants_.target_pixel_count = rtc::Optional<int>(); | |
| 252 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
| 253 return true; | |
| 254 } | |
| 255 | |
| 256 int RequestFramerateLowerThan(int fps) { | |
| 257 // Called on the encoder task queue. | |
| 258 // The input video frame rate will be scaled down to 2/3, rounding down. | |
| 259 int framerate_wanted = (fps * 2) / 3; | |
| 260 return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; | |
| 261 } | |
| 262 | |
| 263 bool RequestHigherResolutionThan(int pixel_count) { | |
| 264 // Called on the encoder task queue. | |
| 265 rtc::CritScope lock(&crit_); | |
| 266 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { | |
| 267 // This can happen since |degradation_preference_| is set on libjingle's | |
| 268 // worker thread but the adaptation is done on the encoder task queue. | |
| 269 return false; | |
| 270 } | |
| 271 int max_pixels_wanted = pixel_count; | |
| 272 if (max_pixels_wanted != std::numeric_limits<int>::max()) | |
| 273 max_pixels_wanted = pixel_count * 4; | |
| 274 | |
| 275 if (max_pixels_wanted <= sink_wants_.max_pixel_count) | |
| 276 return false; | |
| 277 | |
| 278 sink_wants_.max_pixel_count = max_pixels_wanted; | |
| 279 if (max_pixels_wanted == std::numeric_limits<int>::max()) { | |
| 280 // Remove any constraints. | |
| 281 sink_wants_.target_pixel_count.reset(); | |
| 282 } else { | |
| 283 // On step down we request at most 3/5 the pixel count of the previous | |
| 284 // resolution, so in order to take "one step up" we request a resolution | |
| 285 // as close as possible to 5/3 of the current resolution. The actual pixel | |
| 286 // count selected depends on the capabilities of the source. In order to | |
| 287 // not take a too large step up, we cap the requested pixel count to be at | |
| 288 // most four time the current number of pixels. | |
| 289 sink_wants_.target_pixel_count = | |
| 290 rtc::Optional<int>((pixel_count * 5) / 3); | |
| 291 } | |
| 292 LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; | |
| 293 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
| 294 return true; | |
| 295 } | |
| 296 | |
| 297 // Request upgrade in framerate. Returns the new requested frame, or -1 if | |
| 298 // no change requested. Note that maxint may be returned if limits due to | |
| 299 // adaptation requests are removed completely. In that case, consider | |
| 300 // |max_framerate_| to be the current limit (assuming the capturer complies). | |
| 301 int RequestHigherFramerateThan(int fps) { | |
| 302 // Called on the encoder task queue. | |
| 303 // The input frame rate will be scaled up to the last step, with rounding. | |
| 304 int framerate_wanted = fps; | |
| 305 if (fps != std::numeric_limits<int>::max()) | |
| 306 framerate_wanted = (fps * 3) / 2; | |
| 307 | |
| 308 return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; | |
| 309 } | |
| 310 | |
| 311 bool RestrictFramerate(int fps) { | |
| 312 // Called on the encoder task queue. | |
| 313 rtc::CritScope lock(&crit_); | |
| 314 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) | |
| 315 return false; | |
| 316 | |
| 317 const int fps_wanted = std::max(kMinFramerateFps, fps); | |
| 318 if (fps_wanted >= sink_wants_.max_framerate_fps) | |
| 319 return false; | |
| 320 | |
| 321 LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; | |
| 322 sink_wants_.max_framerate_fps = fps_wanted; | |
| 323 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
| 324 return true; | |
| 325 } | |
| 326 | |
| 327 bool IncreaseFramerate(int fps) { | |
| 328 // Called on the encoder task queue. | |
| 329 rtc::CritScope lock(&crit_); | |
| 330 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) | |
| 331 return false; | |
| 332 | |
| 333 const int fps_wanted = std::max(kMinFramerateFps, fps); | |
| 334 if (fps_wanted <= sink_wants_.max_framerate_fps) | |
| 335 return false; | |
| 336 | |
| 337 LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; | |
| 338 sink_wants_.max_framerate_fps = fps_wanted; | |
| 339 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
| 340 return true; | |
| 341 } | |
| 342 | |
| 343 private: | |
| 344 rtc::VideoSinkWants GetActiveSinkWantsInternal() | |
| 345 EXCLUSIVE_LOCKS_REQUIRED(&crit_) { | |
| 346 rtc::VideoSinkWants wants = sink_wants_; | |
| 347 // Clear any constraints from the current sink wants that don't apply to | |
| 348 // the used degradation_preference. | |
| 349 switch (degradation_preference_) { | |
| 350 case VideoSendStream::DegradationPreference::kBalanced: | |
| 351 break; | |
| 352 case VideoSendStream::DegradationPreference::kMaintainFramerate: | |
| 353 wants.max_framerate_fps = std::numeric_limits<int>::max(); | |
| 354 break; | |
| 355 case VideoSendStream::DegradationPreference::kMaintainResolution: | |
| 356 wants.max_pixel_count = std::numeric_limits<int>::max(); | |
| 357 wants.target_pixel_count.reset(); | |
| 358 break; | |
| 359 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
| 360 wants.max_pixel_count = std::numeric_limits<int>::max(); | |
| 361 wants.target_pixel_count.reset(); | |
| 362 wants.max_framerate_fps = std::numeric_limits<int>::max(); | |
| 363 } | |
| 364 return wants; | |
| 365 } | |
| 366 | |
| 367 rtc::CriticalSection crit_; | |
| 368 rtc::SequencedTaskChecker main_checker_; | |
| 369 ViEEncoder* const vie_encoder_; | |
| 370 rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_); | |
| 371 VideoSendStream::DegradationPreference degradation_preference_ | |
| 372 GUARDED_BY(&crit_); | |
| 373 rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_); | |
| 374 | |
| 375 RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); | |
| 376 }; | |
| 377 | |
| 378 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | |
| 379 SendStatisticsProxy* stats_proxy, | |
| 380 const VideoSendStream::Config::EncoderSettings& settings, | |
| 381 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | |
| 382 EncodedFrameObserver* encoder_timing, | |
| 383 std::unique_ptr<OveruseFrameDetector> overuse_detector) | |
| 384 : shutdown_event_(true /* manual_reset */, false), | |
| 385 number_of_cores_(number_of_cores), | |
| 386 initial_rampup_(0), | |
| 387 source_proxy_(new VideoSourceProxy(this)), | |
| 388 sink_(nullptr), | |
| 389 settings_(settings), | |
| 390 codec_type_(PayloadNameToCodecType(settings.payload_name) | |
| 391 .value_or(VideoCodecType::kVideoCodecUnknown)), | |
| 392 video_sender_(Clock::GetRealTimeClock(), this, this), | |
| 393 overuse_detector_( | |
| 394 overuse_detector.get() | |
| 395 ? overuse_detector.release() | |
| 396 : new OveruseFrameDetector( | |
| 397 GetCpuOveruseOptions(settings.full_overuse_time), | |
| 398 this, | |
| 399 encoder_timing, | |
| 400 stats_proxy)), | |
| 401 stats_proxy_(stats_proxy), | |
| 402 pre_encode_callback_(pre_encode_callback), | |
| 403 module_process_thread_(nullptr), | |
| 404 max_framerate_(-1), | |
| 405 pending_encoder_reconfiguration_(false), | |
| 406 encoder_start_bitrate_bps_(0), | |
| 407 max_data_payload_length_(0), | |
| 408 nack_enabled_(false), | |
| 409 last_observed_bitrate_bps_(0), | |
| 410 encoder_paused_and_dropped_frame_(false), | |
| 411 clock_(Clock::GetRealTimeClock()), | |
| 412 degradation_preference_( | |
| 413 VideoSendStream::DegradationPreference::kDegradationDisabled), | |
| 414 last_captured_timestamp_(0), | |
| 415 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | |
| 416 clock_->TimeInMilliseconds()), | |
| 417 last_frame_log_ms_(clock_->TimeInMilliseconds()), | |
| 418 captured_frame_count_(0), | |
| 419 dropped_frame_count_(0), | |
| 420 bitrate_observer_(nullptr), | |
| 421 encoder_queue_("EncoderQueue") { | |
| 422 RTC_DCHECK(stats_proxy); | |
| 423 encoder_queue_.PostTask([this] { | |
| 424 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 425 overuse_detector_->StartCheckForOveruse(); | |
| 426 video_sender_.RegisterExternalEncoder( | |
| 427 settings_.encoder, settings_.payload_type, settings_.internal_source); | |
| 428 }); | |
| 429 } | |
| 430 | |
| 431 ViEEncoder::~ViEEncoder() { | |
| 432 RTC_DCHECK_RUN_ON(&thread_checker_); | |
| 433 RTC_DCHECK(shutdown_event_.Wait(0)) | |
| 434 << "Must call ::Stop() before destruction."; | |
| 435 } | |
| 436 | |
| 437 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle | |
| 438 // pipelining encoders better (multiple input frames before something comes | |
| 439 // out). This should effectively turn off CPU adaptations for systems that | |
| 440 // remotely cope with the load right now. | |
| 441 CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) { | |
| 442 CpuOveruseOptions options; | |
| 443 if (full_overuse_time) { | |
| 444 options.low_encode_usage_threshold_percent = 150; | |
| 445 options.high_encode_usage_threshold_percent = 200; | |
| 446 } | |
| 447 return options; | |
| 448 } | |
| 449 | |
| 450 void ViEEncoder::Stop() { | |
| 451 RTC_DCHECK_RUN_ON(&thread_checker_); | |
| 452 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); | |
| 453 encoder_queue_.PostTask([this] { | |
| 454 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 455 overuse_detector_->StopCheckForOveruse(); | |
| 456 rate_allocator_.reset(); | |
| 457 bitrate_observer_ = nullptr; | |
| 458 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, | |
| 459 false); | |
| 460 quality_scaler_ = nullptr; | |
| 461 shutdown_event_.Set(); | |
| 462 }); | |
| 463 | |
| 464 shutdown_event_.Wait(rtc::Event::kForever); | |
| 465 } | |
| 466 | |
| 467 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | |
| 468 RTC_DCHECK_RUN_ON(&thread_checker_); | |
| 469 RTC_DCHECK(!module_process_thread_); | |
| 470 module_process_thread_ = module_process_thread; | |
| 471 module_process_thread_->RegisterModule(&video_sender_, RTC_FROM_HERE); | |
| 472 module_process_thread_checker_.DetachFromThread(); | |
| 473 } | |
| 474 | |
| 475 void ViEEncoder::DeRegisterProcessThread() { | |
| 476 RTC_DCHECK_RUN_ON(&thread_checker_); | |
| 477 module_process_thread_->DeRegisterModule(&video_sender_); | |
| 478 } | |
| 479 | |
| 480 void ViEEncoder::SetBitrateObserver( | |
| 481 VideoBitrateAllocationObserver* bitrate_observer) { | |
| 482 RTC_DCHECK_RUN_ON(&thread_checker_); | |
| 483 encoder_queue_.PostTask([this, bitrate_observer] { | |
| 484 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 485 RTC_DCHECK(!bitrate_observer_); | |
| 486 bitrate_observer_ = bitrate_observer; | |
| 487 }); | |
| 488 } | |
| 489 | |
| 490 void ViEEncoder::SetSource( | |
| 491 rtc::VideoSourceInterface<VideoFrame>* source, | |
| 492 const VideoSendStream::DegradationPreference& degradation_preference) { | |
| 493 RTC_DCHECK_RUN_ON(&thread_checker_); | |
| 494 source_proxy_->SetSource(source, degradation_preference); | |
| 495 encoder_queue_.PostTask([this, degradation_preference] { | |
| 496 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 497 if (degradation_preference_ != degradation_preference) { | |
| 498 // Reset adaptation state, so that we're not tricked into thinking there's | |
| 499 // an already pending request of the same type. | |
| 500 last_adaptation_request_.reset(); | |
| 501 if (degradation_preference == | |
| 502 VideoSendStream::DegradationPreference::kBalanced || | |
| 503 degradation_preference_ == | |
| 504 VideoSendStream::DegradationPreference::kBalanced) { | |
| 505 // TODO(asapersson): Consider removing |adapt_counters_| map and use one | |
| 506 // AdaptCounter for all modes. | |
| 507 source_proxy_->ResetPixelFpsCount(); | |
| 508 adapt_counters_.clear(); | |
| 509 } | |
| 510 } | |
| 511 degradation_preference_ = degradation_preference; | |
| 512 bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); | |
| 513 initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; | |
| 514 ConfigureQualityScaler(); | |
| 515 if (!IsFramerateScalingEnabled(degradation_preference) && | |
| 516 max_framerate_ != -1) { | |
| 517 // If frame rate scaling is no longer allowed, remove any potential | |
| 518 // allowance for longer frame intervals. | |
| 519 overuse_detector_->OnTargetFramerateUpdated(max_framerate_); | |
| 520 } | |
| 521 }); | |
| 522 } | |
| 523 | |
| 524 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { | |
| 525 source_proxy_->SetWantsRotationApplied(rotation_applied); | |
| 526 encoder_queue_.PostTask([this, sink] { | |
| 527 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 528 sink_ = sink; | |
| 529 }); | |
| 530 } | |
| 531 | |
| 532 void ViEEncoder::SetStartBitrate(int start_bitrate_bps) { | |
| 533 encoder_queue_.PostTask([this, start_bitrate_bps] { | |
| 534 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 535 encoder_start_bitrate_bps_ = start_bitrate_bps; | |
| 536 }); | |
| 537 } | |
| 538 | |
| 539 void ViEEncoder::ConfigureEncoder(VideoEncoderConfig config, | |
| 540 size_t max_data_payload_length, | |
| 541 bool nack_enabled) { | |
| 542 encoder_queue_.PostTask( | |
| 543 std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask( | |
| 544 this, std::move(config), max_data_payload_length, nack_enabled))); | |
| 545 } | |
| 546 | |
| 547 void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config, | |
| 548 size_t max_data_payload_length, | |
| 549 bool nack_enabled) { | |
| 550 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 551 RTC_DCHECK(sink_); | |
| 552 LOG(LS_INFO) << "ConfigureEncoder requested."; | |
| 553 | |
| 554 max_data_payload_length_ = max_data_payload_length; | |
| 555 nack_enabled_ = nack_enabled; | |
| 556 encoder_config_ = std::move(config); | |
| 557 pending_encoder_reconfiguration_ = true; | |
| 558 | |
| 559 // Reconfigure the encoder now if the encoder has an internal source or | |
| 560 // if the frame resolution is known. Otherwise, the reconfiguration is | |
| 561 // deferred until the next frame to minimize the number of reconfigurations. | |
| 562 // The codec configuration depends on incoming video frame size. | |
| 563 if (last_frame_info_) { | |
| 564 ReconfigureEncoder(); | |
| 565 } else if (settings_.internal_source) { | |
| 566 last_frame_info_ = | |
| 567 rtc::Optional<VideoFrameInfo>(VideoFrameInfo(176, 144, false)); | |
| 568 ReconfigureEncoder(); | |
| 569 } | |
| 570 } | |
| 571 | |
| 572 void ViEEncoder::ReconfigureEncoder() { | |
| 573 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 574 RTC_DCHECK(pending_encoder_reconfiguration_); | |
| 575 std::vector<VideoStream> streams = | |
| 576 encoder_config_.video_stream_factory->CreateEncoderStreams( | |
| 577 last_frame_info_->width, last_frame_info_->height, encoder_config_); | |
| 578 | |
| 579 // TODO(ilnik): If configured resolution is significantly less than provided, | |
| 580 // e.g. because there are not enough SSRCs for all simulcast streams, | |
| 581 // signal new resolutions via SinkWants to video source. | |
| 582 | |
| 583 // Stream dimensions may be not equal to given because of a simulcast | |
| 584 // restrictions. | |
| 585 int highest_stream_width = static_cast<int>(streams.back().width); | |
| 586 int highest_stream_height = static_cast<int>(streams.back().height); | |
| 587 // Dimension may be reduced to be, e.g. divisible by 4. | |
| 588 RTC_CHECK_GE(last_frame_info_->width, highest_stream_width); | |
| 589 RTC_CHECK_GE(last_frame_info_->height, highest_stream_height); | |
| 590 crop_width_ = last_frame_info_->width - highest_stream_width; | |
| 591 crop_height_ = last_frame_info_->height - highest_stream_height; | |
| 592 | |
| 593 VideoCodec codec; | |
| 594 if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams, | |
| 595 nack_enabled_, &codec, | |
| 596 &rate_allocator_)) { | |
| 597 LOG(LS_ERROR) << "Failed to create encoder configuration."; | |
| 598 } | |
| 599 | |
| 600 codec.startBitrate = | |
| 601 std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); | |
| 602 codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); | |
| 603 codec.expect_encode_from_texture = last_frame_info_->is_texture; | |
| 604 max_framerate_ = codec.maxFramerate; | |
| 605 RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps); | |
| 606 | |
| 607 bool success = video_sender_.RegisterSendCodec( | |
| 608 &codec, number_of_cores_, | |
| 609 static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK; | |
| 610 if (!success) { | |
| 611 LOG(LS_ERROR) << "Failed to configure encoder."; | |
| 612 rate_allocator_.reset(); | |
| 613 } | |
| 614 | |
| 615 video_sender_.UpdateChannelParemeters(rate_allocator_.get(), | |
| 616 bitrate_observer_); | |
| 617 | |
| 618 // Get the current actual framerate, as measured by the stats proxy. This is | |
| 619 // used to get the correct bitrate layer allocation. | |
| 620 int current_framerate = stats_proxy_->GetSendFrameRate(); | |
| 621 if (current_framerate == 0) | |
| 622 current_framerate = codec.maxFramerate; | |
| 623 stats_proxy_->OnEncoderReconfigured( | |
| 624 encoder_config_, | |
| 625 rate_allocator_.get() | |
| 626 ? rate_allocator_->GetPreferredBitrateBps(current_framerate) | |
| 627 : codec.maxBitrate); | |
| 628 | |
| 629 pending_encoder_reconfiguration_ = false; | |
| 630 | |
| 631 sink_->OnEncoderConfigurationChanged( | |
| 632 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | |
| 633 | |
| 634 // Get the current target framerate, ie the maximum framerate as specified by | |
| 635 // the current codec configuration, or any limit imposed by cpu adaption in | |
| 636 // maintain-resolution or balanced mode. This is used to make sure overuse | |
| 637 // detection doesn't needlessly trigger in low and/or variable framerate | |
| 638 // scenarios. | |
| 639 int target_framerate = std::min( | |
| 640 max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); | |
| 641 overuse_detector_->OnTargetFramerateUpdated(target_framerate); | |
| 642 | |
| 643 ConfigureQualityScaler(); | |
| 644 } | |
| 645 | |
| 646 void ViEEncoder::ConfigureQualityScaler() { | |
| 647 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 648 const auto scaling_settings = settings_.encoder->GetScalingSettings(); | |
| 649 const bool quality_scaling_allowed = | |
| 650 IsResolutionScalingEnabled(degradation_preference_) && | |
| 651 scaling_settings.enabled; | |
| 652 | |
| 653 if (quality_scaling_allowed) { | |
| 654 if (quality_scaler_.get() == nullptr) { | |
| 655 // Quality scaler has not already been configured. | |
| 656 // Drop frames and scale down until desired quality is achieved. | |
| 657 if (scaling_settings.thresholds) { | |
| 658 quality_scaler_.reset( | |
| 659 new QualityScaler(this, *(scaling_settings.thresholds))); | |
| 660 } else { | |
| 661 quality_scaler_.reset(new QualityScaler(this, codec_type_)); | |
| 662 } | |
| 663 } | |
| 664 } else { | |
| 665 quality_scaler_.reset(nullptr); | |
| 666 initial_rampup_ = kMaxInitialFramedrop; | |
| 667 } | |
| 668 | |
| 669 stats_proxy_->SetAdaptationStats(GetActiveCounts(kCpu), | |
| 670 GetActiveCounts(kQuality)); | |
| 671 } | |
| 672 | |
| 673 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | |
| 674 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | |
| 675 VideoFrame incoming_frame = video_frame; | |
| 676 | |
| 677 // Local time in webrtc time base. | |
| 678 int64_t current_time_us = clock_->TimeInMicroseconds(); | |
| 679 int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec; | |
| 680 // In some cases, e.g., when the frame from decoder is fed to encoder, | |
| 681 // the timestamp may be set to the future. As the encoding pipeline assumes | |
| 682 // capture time to be less than present time, we should reset the capture | |
| 683 // timestamps here. Otherwise there may be issues with RTP send stream. | |
| 684 if (incoming_frame.timestamp_us() > current_time_us) | |
| 685 incoming_frame.set_timestamp_us(current_time_us); | |
| 686 | |
| 687 // Capture time may come from clock with an offset and drift from clock_. | |
| 688 int64_t capture_ntp_time_ms; | |
| 689 if (video_frame.ntp_time_ms() > 0) { | |
| 690 capture_ntp_time_ms = video_frame.ntp_time_ms(); | |
| 691 } else if (video_frame.render_time_ms() != 0) { | |
| 692 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; | |
| 693 } else { | |
| 694 capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; | |
| 695 } | |
| 696 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); | |
| 697 | |
| 698 // Convert NTP time, in ms, to RTP timestamp. | |
| 699 const int kMsToRtpTimestamp = 90; | |
| 700 incoming_frame.set_timestamp( | |
| 701 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms())); | |
| 702 | |
| 703 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { | |
| 704 // We don't allow the same capture time for two frames, drop this one. | |
| 705 LOG(LS_WARNING) << "Same/old NTP timestamp (" | |
| 706 << incoming_frame.ntp_time_ms() | |
| 707 << " <= " << last_captured_timestamp_ | |
| 708 << ") for incoming frame. Dropping."; | |
| 709 return; | |
| 710 } | |
| 711 | |
| 712 bool log_stats = false; | |
| 713 if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) { | |
| 714 last_frame_log_ms_ = current_time_ms; | |
| 715 log_stats = true; | |
| 716 } | |
| 717 | |
| 718 last_captured_timestamp_ = incoming_frame.ntp_time_ms(); | |
| 719 encoder_queue_.PostTask(std::unique_ptr<rtc::QueuedTask>(new EncodeTask( | |
| 720 incoming_frame, this, rtc::TimeMicros(), log_stats))); | |
| 721 } | |
| 722 | |
| 723 bool ViEEncoder::EncoderPaused() const { | |
| 724 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 725 // Pause video if paused by caller or as long as the network is down or the | |
| 726 // pacer queue has grown too large in buffered mode. | |
| 727 // If the pacer queue has grown too large or the network is down, | |
| 728 // last_observed_bitrate_bps_ will be 0. | |
| 729 return last_observed_bitrate_bps_ == 0; | |
| 730 } | |
| 731 | |
| 732 void ViEEncoder::TraceFrameDropStart() { | |
| 733 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 734 // Start trace event only on the first frame after encoder is paused. | |
| 735 if (!encoder_paused_and_dropped_frame_) { | |
| 736 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); | |
| 737 } | |
| 738 encoder_paused_and_dropped_frame_ = true; | |
| 739 } | |
| 740 | |
| 741 void ViEEncoder::TraceFrameDropEnd() { | |
| 742 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 743 // End trace event on first frame after encoder resumes, if frame was dropped. | |
| 744 if (encoder_paused_and_dropped_frame_) { | |
| 745 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | |
| 746 } | |
| 747 encoder_paused_and_dropped_frame_ = false; | |
| 748 } | |
| 749 | |
| 750 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, | |
| 751 int64_t time_when_posted_us) { | |
| 752 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 753 | |
| 754 if (pre_encode_callback_) | |
| 755 pre_encode_callback_->OnFrame(video_frame); | |
| 756 | |
| 757 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || | |
| 758 video_frame.height() != last_frame_info_->height || | |
| 759 video_frame.is_texture() != last_frame_info_->is_texture) { | |
| 760 pending_encoder_reconfiguration_ = true; | |
| 761 last_frame_info_ = rtc::Optional<VideoFrameInfo>(VideoFrameInfo( | |
| 762 video_frame.width(), video_frame.height(), video_frame.is_texture())); | |
| 763 LOG(LS_INFO) << "Video frame parameters changed: dimensions=" | |
| 764 << last_frame_info_->width << "x" << last_frame_info_->height | |
| 765 << ", texture=" << last_frame_info_->is_texture << "."; | |
| 766 } | |
| 767 | |
| 768 if (initial_rampup_ < kMaxInitialFramedrop && | |
| 769 video_frame.size() > | |
| 770 MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) { | |
| 771 LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; | |
| 772 AdaptDown(kQuality); | |
| 773 ++initial_rampup_; | |
| 774 return; | |
| 775 } | |
| 776 initial_rampup_ = kMaxInitialFramedrop; | |
| 777 | |
| 778 int64_t now_ms = clock_->TimeInMilliseconds(); | |
| 779 if (pending_encoder_reconfiguration_) { | |
| 780 ReconfigureEncoder(); | |
| 781 last_parameters_update_ms_.emplace(now_ms); | |
| 782 } else if (!last_parameters_update_ms_ || | |
| 783 now_ms - *last_parameters_update_ms_ >= | |
| 784 vcm::VCMProcessTimer::kDefaultProcessIntervalMs) { | |
| 785 video_sender_.UpdateChannelParemeters(rate_allocator_.get(), | |
| 786 bitrate_observer_); | |
| 787 last_parameters_update_ms_.emplace(now_ms); | |
| 788 } | |
| 789 | |
| 790 if (EncoderPaused()) { | |
| 791 TraceFrameDropStart(); | |
| 792 return; | |
| 793 } | |
| 794 TraceFrameDropEnd(); | |
| 795 | |
| 796 VideoFrame out_frame(video_frame); | |
| 797 // Crop frame if needed. | |
| 798 if (crop_width_ > 0 || crop_height_ > 0) { | |
| 799 int cropped_width = video_frame.width() - crop_width_; | |
| 800 int cropped_height = video_frame.height() - crop_height_; | |
| 801 rtc::scoped_refptr<I420Buffer> cropped_buffer = | |
| 802 I420Buffer::Create(cropped_width, cropped_height); | |
| 803 // TODO(ilnik): Remove scaling if cropping is too big, as it should never | |
| 804 // happen after SinkWants signaled correctly from ReconfigureEncoder. | |
| 805 if (crop_width_ < 4 && crop_height_ < 4) { | |
| 806 cropped_buffer->CropAndScaleFrom( | |
| 807 *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2, | |
| 808 crop_height_ / 2, cropped_width, cropped_height); | |
| 809 } else { | |
| 810 cropped_buffer->ScaleFrom( | |
| 811 *video_frame.video_frame_buffer()->ToI420().get()); | |
| 812 } | |
| 813 out_frame = | |
| 814 VideoFrame(cropped_buffer, video_frame.timestamp(), | |
| 815 video_frame.render_time_ms(), video_frame.rotation()); | |
| 816 out_frame.set_ntp_time_ms(video_frame.ntp_time_ms()); | |
| 817 } | |
| 818 | |
| 819 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), | |
| 820 "Encode"); | |
| 821 | |
| 822 overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); | |
| 823 | |
| 824 video_sender_.AddVideoFrame(out_frame, nullptr); | |
| 825 } | |
| 826 | |
| 827 void ViEEncoder::SendKeyFrame() { | |
| 828 if (!encoder_queue_.IsCurrent()) { | |
| 829 encoder_queue_.PostTask([this] { SendKeyFrame(); }); | |
| 830 return; | |
| 831 } | |
| 832 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 833 video_sender_.IntraFrameRequest(0); | |
| 834 } | |
| 835 | |
| 836 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( | |
| 837 const EncodedImage& encoded_image, | |
| 838 const CodecSpecificInfo* codec_specific_info, | |
| 839 const RTPFragmentationHeader* fragmentation) { | |
| 840 // Encoded is called on whatever thread the real encoder implementation run | |
| 841 // on. In the case of hardware encoders, there might be several encoders | |
| 842 // running in parallel on different threads. | |
| 843 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | |
| 844 | |
| 845 EncodedImageCallback::Result result = | |
| 846 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | |
| 847 | |
| 848 int64_t time_sent_us = rtc::TimeMicros(); | |
| 849 uint32_t timestamp = encoded_image._timeStamp; | |
| 850 const int qp = encoded_image.qp_; | |
| 851 encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { | |
| 852 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 853 overuse_detector_->FrameSent(timestamp, time_sent_us); | |
| 854 if (quality_scaler_ && qp >= 0) | |
| 855 quality_scaler_->ReportQP(qp); | |
| 856 }); | |
| 857 | |
| 858 return result; | |
| 859 } | |
| 860 | |
| 861 void ViEEncoder::OnDroppedFrame() { | |
| 862 encoder_queue_.PostTask([this] { | |
| 863 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 864 if (quality_scaler_) | |
| 865 quality_scaler_->ReportDroppedFrame(); | |
| 866 }); | |
| 867 } | |
| 868 | |
| 869 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { | |
| 870 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | |
| 871 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate); | |
| 872 } | |
| 873 | |
| 874 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { | |
| 875 if (!encoder_queue_.IsCurrent()) { | |
| 876 encoder_queue_.PostTask( | |
| 877 [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); }); | |
| 878 return; | |
| 879 } | |
| 880 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 881 // Key frame request from remote side, signal to VCM. | |
| 882 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | |
| 883 video_sender_.IntraFrameRequest(stream_index); | |
| 884 } | |
| 885 | |
| 886 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | |
| 887 uint8_t fraction_lost, | |
| 888 int64_t round_trip_time_ms) { | |
| 889 if (!encoder_queue_.IsCurrent()) { | |
| 890 encoder_queue_.PostTask( | |
| 891 [this, bitrate_bps, fraction_lost, round_trip_time_ms] { | |
| 892 OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms); | |
| 893 }); | |
| 894 return; | |
| 895 } | |
| 896 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 897 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; | |
| 898 | |
| 899 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps | |
| 900 << " packet loss " << static_cast<int>(fraction_lost) | |
| 901 << " rtt " << round_trip_time_ms; | |
| 902 | |
| 903 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | |
| 904 round_trip_time_ms, rate_allocator_.get(), | |
| 905 bitrate_observer_); | |
| 906 | |
| 907 encoder_start_bitrate_bps_ = | |
| 908 bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; | |
| 909 bool video_is_suspended = bitrate_bps == 0; | |
| 910 bool video_suspension_changed = video_is_suspended != EncoderPaused(); | |
| 911 last_observed_bitrate_bps_ = bitrate_bps; | |
| 912 | |
| 913 if (video_suspension_changed) { | |
| 914 LOG(LS_INFO) << "Video suspend state changed to: " | |
| 915 << (video_is_suspended ? "suspended" : "not suspended"); | |
| 916 stats_proxy_->OnSuspendChange(video_is_suspended); | |
| 917 } | |
| 918 } | |
| 919 | |
| 920 void ViEEncoder::AdaptDown(AdaptReason reason) { | |
| 921 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 922 AdaptationRequest adaptation_request = { | |
| 923 last_frame_info_->pixel_count(), | |
| 924 stats_proxy_->GetStats().input_frame_rate, | |
| 925 AdaptationRequest::Mode::kAdaptDown}; | |
| 926 | |
| 927 bool downgrade_requested = | |
| 928 last_adaptation_request_ && | |
| 929 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; | |
| 930 | |
| 931 switch (degradation_preference_) { | |
| 932 case VideoSendStream::DegradationPreference::kBalanced: | |
| 933 break; | |
| 934 case VideoSendStream::DegradationPreference::kMaintainFramerate: | |
| 935 if (downgrade_requested && | |
| 936 adaptation_request.input_pixel_count_ >= | |
| 937 last_adaptation_request_->input_pixel_count_) { | |
| 938 // Don't request lower resolution if the current resolution is not | |
| 939 // lower than the last time we asked for the resolution to be lowered. | |
| 940 return; | |
| 941 } | |
| 942 break; | |
| 943 case VideoSendStream::DegradationPreference::kMaintainResolution: | |
| 944 if (adaptation_request.framerate_fps_ <= 0 || | |
| 945 (downgrade_requested && | |
| 946 adaptation_request.framerate_fps_ < kMinFramerateFps)) { | |
| 947 // If no input fps estimate available, can't determine how to scale down | |
| 948 // framerate. Otherwise, don't request lower framerate if we don't have | |
| 949 // a valid frame rate. Since framerate, unlike resolution, is a measure | |
| 950 // we have to estimate, and can fluctuate naturally over time, don't | |
| 951 // make the same kind of limitations as for resolution, but trust the | |
| 952 // overuse detector to not trigger too often. | |
| 953 return; | |
| 954 } | |
| 955 break; | |
| 956 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
| 957 return; | |
| 958 } | |
| 959 | |
| 960 if (reason == kCpu) { | |
| 961 if (GetConstAdaptCounter().ResolutionCount(kCpu) >= | |
| 962 kMaxCpuResolutionDowngrades || | |
| 963 GetConstAdaptCounter().FramerateCount(kCpu) >= | |
| 964 kMaxCpuFramerateDowngrades) { | |
| 965 return; | |
| 966 } | |
| 967 } | |
| 968 | |
| 969 switch (degradation_preference_) { | |
| 970 case VideoSendStream::DegradationPreference::kBalanced: { | |
| 971 // Try scale down framerate, if lower. | |
| 972 int fps = MinFps(last_frame_info_->pixel_count()); | |
| 973 if (source_proxy_->RestrictFramerate(fps)) { | |
| 974 GetAdaptCounter().IncrementFramerate(reason); | |
| 975 break; | |
| 976 } | |
| 977 // Scale down resolution. | |
| 978 FALLTHROUGH(); | |
| 979 } | |
| 980 case VideoSendStream::DegradationPreference::kMaintainFramerate: | |
| 981 // Scale down resolution. | |
| 982 if (!source_proxy_->RequestResolutionLowerThan( | |
| 983 adaptation_request.input_pixel_count_)) { | |
| 984 return; | |
| 985 } | |
| 986 GetAdaptCounter().IncrementResolution(reason); | |
| 987 break; | |
| 988 case VideoSendStream::DegradationPreference::kMaintainResolution: { | |
| 989 // Scale down framerate. | |
| 990 const int requested_framerate = source_proxy_->RequestFramerateLowerThan( | |
| 991 adaptation_request.framerate_fps_); | |
| 992 if (requested_framerate == -1) | |
| 993 return; | |
| 994 RTC_DCHECK_NE(max_framerate_, -1); | |
| 995 overuse_detector_->OnTargetFramerateUpdated( | |
| 996 std::min(max_framerate_, requested_framerate)); | |
| 997 GetAdaptCounter().IncrementFramerate(reason); | |
| 998 break; | |
| 999 } | |
| 1000 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
| 1001 RTC_NOTREACHED(); | |
| 1002 } | |
| 1003 | |
| 1004 last_adaptation_request_.emplace(adaptation_request); | |
| 1005 | |
| 1006 UpdateAdaptationStats(reason); | |
| 1007 | |
| 1008 LOG(LS_INFO) << GetConstAdaptCounter().ToString(); | |
| 1009 } | |
| 1010 | |
| 1011 void ViEEncoder::AdaptUp(AdaptReason reason) { | |
| 1012 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 1013 | |
| 1014 const AdaptCounter& adapt_counter = GetConstAdaptCounter(); | |
| 1015 int num_downgrades = adapt_counter.TotalCount(reason); | |
| 1016 if (num_downgrades == 0) | |
| 1017 return; | |
| 1018 RTC_DCHECK_GT(num_downgrades, 0); | |
| 1019 | |
| 1020 AdaptationRequest adaptation_request = { | |
| 1021 last_frame_info_->pixel_count(), | |
| 1022 stats_proxy_->GetStats().input_frame_rate, | |
| 1023 AdaptationRequest::Mode::kAdaptUp}; | |
| 1024 | |
| 1025 bool adapt_up_requested = | |
| 1026 last_adaptation_request_ && | |
| 1027 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; | |
| 1028 | |
| 1029 if (degradation_preference_ == | |
| 1030 VideoSendStream::DegradationPreference::kMaintainFramerate) { | |
| 1031 if (adapt_up_requested && | |
| 1032 adaptation_request.input_pixel_count_ <= | |
| 1033 last_adaptation_request_->input_pixel_count_) { | |
| 1034 // Don't request higher resolution if the current resolution is not | |
| 1035 // higher than the last time we asked for the resolution to be higher. | |
| 1036 return; | |
| 1037 } | |
| 1038 } | |
| 1039 | |
| 1040 switch (degradation_preference_) { | |
| 1041 case VideoSendStream::DegradationPreference::kBalanced: { | |
| 1042 // Try scale up framerate, if higher. | |
| 1043 int fps = MaxFps(last_frame_info_->pixel_count()); | |
| 1044 if (source_proxy_->IncreaseFramerate(fps)) { | |
| 1045 GetAdaptCounter().DecrementFramerate(reason, fps); | |
| 1046 // Reset framerate in case of fewer fps steps down than up. | |
| 1047 if (adapt_counter.FramerateCount() == 0 && | |
| 1048 fps != std::numeric_limits<int>::max()) { | |
| 1049 LOG(LS_INFO) << "Removing framerate down-scaling setting."; | |
| 1050 source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max()); | |
| 1051 } | |
| 1052 break; | |
| 1053 } | |
| 1054 // Scale up resolution. | |
| 1055 FALLTHROUGH(); | |
| 1056 } | |
| 1057 case VideoSendStream::DegradationPreference::kMaintainFramerate: { | |
| 1058 // Scale up resolution. | |
| 1059 int pixel_count = adaptation_request.input_pixel_count_; | |
| 1060 if (adapt_counter.ResolutionCount() == 1) { | |
| 1061 LOG(LS_INFO) << "Removing resolution down-scaling setting."; | |
| 1062 pixel_count = std::numeric_limits<int>::max(); | |
| 1063 } | |
| 1064 if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) | |
| 1065 return; | |
| 1066 GetAdaptCounter().DecrementResolution(reason); | |
| 1067 break; | |
| 1068 } | |
| 1069 case VideoSendStream::DegradationPreference::kMaintainResolution: { | |
| 1070 // Scale up framerate. | |
| 1071 int fps = adaptation_request.framerate_fps_; | |
| 1072 if (adapt_counter.FramerateCount() == 1) { | |
| 1073 LOG(LS_INFO) << "Removing framerate down-scaling setting."; | |
| 1074 fps = std::numeric_limits<int>::max(); | |
| 1075 } | |
| 1076 | |
| 1077 const int requested_framerate = | |
| 1078 source_proxy_->RequestHigherFramerateThan(fps); | |
| 1079 if (requested_framerate == -1) { | |
| 1080 overuse_detector_->OnTargetFramerateUpdated(max_framerate_); | |
| 1081 return; | |
| 1082 } | |
| 1083 overuse_detector_->OnTargetFramerateUpdated( | |
| 1084 std::min(max_framerate_, requested_framerate)); | |
| 1085 GetAdaptCounter().DecrementFramerate(reason); | |
| 1086 break; | |
| 1087 } | |
| 1088 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
| 1089 return; | |
| 1090 } | |
| 1091 | |
| 1092 last_adaptation_request_.emplace(adaptation_request); | |
| 1093 | |
| 1094 UpdateAdaptationStats(reason); | |
| 1095 | |
| 1096 LOG(LS_INFO) << adapt_counter.ToString(); | |
| 1097 } | |
| 1098 | |
| 1099 void ViEEncoder::UpdateAdaptationStats(AdaptReason reason) { | |
| 1100 switch (reason) { | |
| 1101 case kCpu: | |
| 1102 stats_proxy_->OnCpuAdaptationChanged(GetActiveCounts(kCpu), | |
| 1103 GetActiveCounts(kQuality)); | |
| 1104 break; | |
| 1105 case kQuality: | |
| 1106 stats_proxy_->OnQualityAdaptationChanged(GetActiveCounts(kCpu), | |
| 1107 GetActiveCounts(kQuality)); | |
| 1108 break; | |
| 1109 } | |
| 1110 } | |
| 1111 | |
| 1112 ViEEncoder::AdaptCounts ViEEncoder::GetActiveCounts(AdaptReason reason) { | |
| 1113 ViEEncoder::AdaptCounts counts = GetConstAdaptCounter().Counts(reason); | |
| 1114 switch (reason) { | |
| 1115 case kCpu: | |
| 1116 if (!IsFramerateScalingEnabled(degradation_preference_)) | |
| 1117 counts.fps = -1; | |
| 1118 if (!IsResolutionScalingEnabled(degradation_preference_)) | |
| 1119 counts.resolution = -1; | |
| 1120 break; | |
| 1121 case kQuality: | |
| 1122 if (!IsFramerateScalingEnabled(degradation_preference_) || | |
| 1123 !quality_scaler_) { | |
| 1124 counts.fps = -1; | |
| 1125 } | |
| 1126 if (!IsResolutionScalingEnabled(degradation_preference_) || | |
| 1127 !quality_scaler_) { | |
| 1128 counts.resolution = -1; | |
| 1129 } | |
| 1130 break; | |
| 1131 } | |
| 1132 return counts; | |
| 1133 } | |
| 1134 | |
| 1135 ViEEncoder::AdaptCounter& ViEEncoder::GetAdaptCounter() { | |
| 1136 return adapt_counters_[degradation_preference_]; | |
| 1137 } | |
| 1138 | |
| 1139 const ViEEncoder::AdaptCounter& ViEEncoder::GetConstAdaptCounter() { | |
| 1140 return adapt_counters_[degradation_preference_]; | |
| 1141 } | |
| 1142 | |
| 1143 // Class holding adaptation information. | |
| 1144 ViEEncoder::AdaptCounter::AdaptCounter() { | |
| 1145 fps_counters_.resize(kScaleReasonSize); | |
| 1146 resolution_counters_.resize(kScaleReasonSize); | |
| 1147 static_assert(kScaleReasonSize == 2, "Update MoveCount."); | |
| 1148 } | |
| 1149 | |
| 1150 ViEEncoder::AdaptCounter::~AdaptCounter() {} | |
| 1151 | |
| 1152 std::string ViEEncoder::AdaptCounter::ToString() const { | |
| 1153 std::stringstream ss; | |
| 1154 ss << "Downgrade counts: fps: {" << ToString(fps_counters_); | |
| 1155 ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; | |
| 1156 return ss.str(); | |
| 1157 } | |
| 1158 | |
| 1159 ViEEncoder::AdaptCounts ViEEncoder::AdaptCounter::Counts(int reason) const { | |
| 1160 AdaptCounts counts; | |
| 1161 counts.fps = fps_counters_[reason]; | |
| 1162 counts.resolution = resolution_counters_[reason]; | |
| 1163 return counts; | |
| 1164 } | |
| 1165 | |
| 1166 void ViEEncoder::AdaptCounter::IncrementFramerate(int reason) { | |
| 1167 ++(fps_counters_[reason]); | |
| 1168 } | |
| 1169 | |
| 1170 void ViEEncoder::AdaptCounter::IncrementResolution(int reason) { | |
| 1171 ++(resolution_counters_[reason]); | |
| 1172 } | |
| 1173 | |
| 1174 void ViEEncoder::AdaptCounter::DecrementFramerate(int reason) { | |
| 1175 if (fps_counters_[reason] == 0) { | |
| 1176 // Balanced mode: Adapt up is in a different order, switch reason. | |
| 1177 // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). | |
| 1178 // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} | |
| 1179 // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} | |
| 1180 // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} | |
| 1181 // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0} | |
| 1182 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; | |
| 1183 RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; | |
| 1184 MoveCount(&resolution_counters_, reason); | |
| 1185 MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); | |
| 1186 } | |
| 1187 --(fps_counters_[reason]); | |
| 1188 RTC_DCHECK_GE(fps_counters_[reason], 0); | |
| 1189 } | |
| 1190 | |
| 1191 void ViEEncoder::AdaptCounter::DecrementResolution(int reason) { | |
| 1192 if (resolution_counters_[reason] == 0) { | |
| 1193 // Balanced mode: Adapt up is in a different order, switch reason. | |
| 1194 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; | |
| 1195 RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; | |
| 1196 MoveCount(&fps_counters_, reason); | |
| 1197 MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); | |
| 1198 } | |
| 1199 --(resolution_counters_[reason]); | |
| 1200 RTC_DCHECK_GE(resolution_counters_[reason], 0); | |
| 1201 } | |
| 1202 | |
| 1203 void ViEEncoder::AdaptCounter::DecrementFramerate(int reason, int cur_fps) { | |
| 1204 DecrementFramerate(reason); | |
| 1205 // Reset if at max fps (i.e. in case of fewer steps up than down). | |
| 1206 if (cur_fps == std::numeric_limits<int>::max()) | |
| 1207 std::fill(fps_counters_.begin(), fps_counters_.end(), 0); | |
| 1208 } | |
| 1209 | |
| 1210 int ViEEncoder::AdaptCounter::FramerateCount() const { | |
| 1211 return Count(fps_counters_); | |
| 1212 } | |
| 1213 | |
| 1214 int ViEEncoder::AdaptCounter::ResolutionCount() const { | |
| 1215 return Count(resolution_counters_); | |
| 1216 } | |
| 1217 | |
| 1218 int ViEEncoder::AdaptCounter::FramerateCount(int reason) const { | |
| 1219 return fps_counters_[reason]; | |
| 1220 } | |
| 1221 | |
| 1222 int ViEEncoder::AdaptCounter::ResolutionCount(int reason) const { | |
| 1223 return resolution_counters_[reason]; | |
| 1224 } | |
| 1225 | |
| 1226 int ViEEncoder::AdaptCounter::TotalCount(int reason) const { | |
| 1227 return FramerateCount(reason) + ResolutionCount(reason); | |
| 1228 } | |
| 1229 | |
| 1230 int ViEEncoder::AdaptCounter::Count(const std::vector<int>& counters) const { | |
| 1231 return std::accumulate(counters.begin(), counters.end(), 0); | |
| 1232 } | |
| 1233 | |
| 1234 void ViEEncoder::AdaptCounter::MoveCount(std::vector<int>* counters, | |
| 1235 int from_reason) { | |
| 1236 int to_reason = (from_reason + 1) % kScaleReasonSize; | |
| 1237 ++((*counters)[to_reason]); | |
| 1238 --((*counters)[from_reason]); | |
| 1239 } | |
| 1240 | |
| 1241 std::string ViEEncoder::AdaptCounter::ToString( | |
| 1242 const std::vector<int>& counters) const { | |
| 1243 std::stringstream ss; | |
| 1244 for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { | |
| 1245 ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; | |
| 1246 } | |
| 1247 return ss.str(); | |
| 1248 } | |
| 1249 | |
| 1250 } // namespace webrtc | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2995433002:
Rename ViEEncoder to VideoStreamEncoder. (Closed)
