OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 */ | |
10 | |
11 #include "webrtc/video/vie_encoder.h" | |
12 | |
13 #include <algorithm> | |
14 #include <limits> | |
15 #include <numeric> | |
16 #include <utility> | |
17 | |
18 #include "webrtc/api/video/i420_buffer.h" | |
19 #include "webrtc/common_video/include/video_bitrate_allocator.h" | |
20 #include "webrtc/common_video/include/video_frame.h" | |
21 #include "webrtc/modules/pacing/paced_sender.h" | |
22 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" | |
23 #include "webrtc/modules/video_coding/include/video_codec_initializer.h" | |
24 #include "webrtc/modules/video_coding/include/video_coding.h" | |
25 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | |
26 #include "webrtc/rtc_base/arraysize.h" | |
27 #include "webrtc/rtc_base/checks.h" | |
28 #include "webrtc/rtc_base/location.h" | |
29 #include "webrtc/rtc_base/logging.h" | |
30 #include "webrtc/rtc_base/timeutils.h" | |
31 #include "webrtc/rtc_base/trace_event.h" | |
32 #include "webrtc/video/overuse_frame_detector.h" | |
33 #include "webrtc/video/send_statistics_proxy.h" | |
34 | |
35 namespace webrtc { | |
36 | |
37 namespace { | |
38 | |
39 // Time interval for logging frame counts. | |
40 const int64_t kFrameLogIntervalMs = 60000; | |
41 | |
42 // We will never ask for a resolution lower than this. | |
43 // TODO(kthelgason): Lower this limit when better testing | |
44 // on MediaCodec and fallback implementations are in place. | |
45 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206 | |
46 const int kMinPixelsPerFrame = 320 * 180; | |
47 const int kMinFramerateFps = 2; | |
48 const int kMaxFramerateFps = 120; | |
49 | |
50 // The maximum number of frames to drop at beginning of stream | |
51 // to try and achieve desired bitrate. | |
52 const int kMaxInitialFramedrop = 4; | |
53 | |
54 uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { | |
55 if (kbps > 0) { | |
56 if (kbps < 300 /* qvga */) { | |
57 return 320 * 240; | |
58 } else if (kbps < 500 /* vga */) { | |
59 return 640 * 480; | |
60 } | |
61 } | |
62 return std::numeric_limits<uint32_t>::max(); | |
63 } | |
64 | |
65 // Initial limits for kBalanced degradation preference. | |
66 int MinFps(int pixels) { | |
67 if (pixels <= 320 * 240) { | |
68 return 7; | |
69 } else if (pixels <= 480 * 270) { | |
70 return 10; | |
71 } else if (pixels <= 640 * 480) { | |
72 return 15; | |
73 } else { | |
74 return std::numeric_limits<int>::max(); | |
75 } | |
76 } | |
77 | |
78 int MaxFps(int pixels) { | |
79 if (pixels <= 320 * 240) { | |
80 return 10; | |
81 } else if (pixels <= 480 * 270) { | |
82 return 15; | |
83 } else { | |
84 return std::numeric_limits<int>::max(); | |
85 } | |
86 } | |
87 | |
88 bool IsResolutionScalingEnabled( | |
89 VideoSendStream::DegradationPreference degradation_preference) { | |
90 return degradation_preference == | |
91 VideoSendStream::DegradationPreference::kMaintainFramerate || | |
92 degradation_preference == | |
93 VideoSendStream::DegradationPreference::kBalanced; | |
94 } | |
95 | |
96 bool IsFramerateScalingEnabled( | |
97 VideoSendStream::DegradationPreference degradation_preference) { | |
98 return degradation_preference == | |
99 VideoSendStream::DegradationPreference::kMaintainResolution || | |
100 degradation_preference == | |
101 VideoSendStream::DegradationPreference::kBalanced; | |
102 } | |
103 | |
104 } // namespace | |
105 | |
106 class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask { | |
107 public: | |
108 ConfigureEncoderTask(ViEEncoder* vie_encoder, | |
109 VideoEncoderConfig config, | |
110 size_t max_data_payload_length, | |
111 bool nack_enabled) | |
112 : vie_encoder_(vie_encoder), | |
113 config_(std::move(config)), | |
114 max_data_payload_length_(max_data_payload_length), | |
115 nack_enabled_(nack_enabled) {} | |
116 | |
117 private: | |
118 bool Run() override { | |
119 vie_encoder_->ConfigureEncoderOnTaskQueue( | |
120 std::move(config_), max_data_payload_length_, nack_enabled_); | |
121 return true; | |
122 } | |
123 | |
124 ViEEncoder* const vie_encoder_; | |
125 VideoEncoderConfig config_; | |
126 size_t max_data_payload_length_; | |
127 bool nack_enabled_; | |
128 }; | |
129 | |
130 class ViEEncoder::EncodeTask : public rtc::QueuedTask { | |
131 public: | |
132 EncodeTask(const VideoFrame& frame, | |
133 ViEEncoder* vie_encoder, | |
134 int64_t time_when_posted_us, | |
135 bool log_stats) | |
136 : frame_(frame), | |
137 vie_encoder_(vie_encoder), | |
138 time_when_posted_us_(time_when_posted_us), | |
139 log_stats_(log_stats) { | |
140 ++vie_encoder_->posted_frames_waiting_for_encode_; | |
141 } | |
142 | |
143 private: | |
144 bool Run() override { | |
145 RTC_DCHECK_RUN_ON(&vie_encoder_->encoder_queue_); | |
146 RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0); | |
147 vie_encoder_->stats_proxy_->OnIncomingFrame(frame_.width(), | |
148 frame_.height()); | |
149 ++vie_encoder_->captured_frame_count_; | |
150 if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) { | |
151 vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_us_); | |
152 } else { | |
153 // There is a newer frame in flight. Do not encode this frame. | |
154 LOG(LS_VERBOSE) | |
155 << "Incoming frame dropped due to that the encoder is blocked."; | |
156 ++vie_encoder_->dropped_frame_count_; | |
157 } | |
158 if (log_stats_) { | |
159 LOG(LS_INFO) << "Number of frames: captured " | |
160 << vie_encoder_->captured_frame_count_ | |
161 << ", dropped (due to encoder blocked) " | |
162 << vie_encoder_->dropped_frame_count_ << ", interval_ms " | |
163 << kFrameLogIntervalMs; | |
164 vie_encoder_->captured_frame_count_ = 0; | |
165 vie_encoder_->dropped_frame_count_ = 0; | |
166 } | |
167 return true; | |
168 } | |
169 VideoFrame frame_; | |
170 ViEEncoder* const vie_encoder_; | |
171 const int64_t time_when_posted_us_; | |
172 const bool log_stats_; | |
173 }; | |
174 | |
175 // VideoSourceProxy is responsible ensuring thread safety between calls to | |
176 // ViEEncoder::SetSource that will happen on libjingle's worker thread when a | |
177 // video capturer is connected to the encoder and the encoder task queue | |
178 // (encoder_queue_) where the encoder reports its VideoSinkWants. | |
179 class ViEEncoder::VideoSourceProxy { | |
180 public: | |
181 explicit VideoSourceProxy(ViEEncoder* vie_encoder) | |
182 : vie_encoder_(vie_encoder), | |
183 degradation_preference_( | |
184 VideoSendStream::DegradationPreference::kDegradationDisabled), | |
185 source_(nullptr) {} | |
186 | |
187 void SetSource( | |
188 rtc::VideoSourceInterface<VideoFrame>* source, | |
189 const VideoSendStream::DegradationPreference& degradation_preference) { | |
190 // Called on libjingle's worker thread. | |
191 RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_); | |
192 rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr; | |
193 rtc::VideoSinkWants wants; | |
194 { | |
195 rtc::CritScope lock(&crit_); | |
196 degradation_preference_ = degradation_preference; | |
197 old_source = source_; | |
198 source_ = source; | |
199 wants = GetActiveSinkWantsInternal(); | |
200 } | |
201 | |
202 if (old_source != source && old_source != nullptr) { | |
203 old_source->RemoveSink(vie_encoder_); | |
204 } | |
205 | |
206 if (!source) { | |
207 return; | |
208 } | |
209 | |
210 source->AddOrUpdateSink(vie_encoder_, wants); | |
211 } | |
212 | |
213 void SetWantsRotationApplied(bool rotation_applied) { | |
214 rtc::CritScope lock(&crit_); | |
215 sink_wants_.rotation_applied = rotation_applied; | |
216 if (source_) | |
217 source_->AddOrUpdateSink(vie_encoder_, sink_wants_); | |
218 } | |
219 | |
220 rtc::VideoSinkWants GetActiveSinkWants() { | |
221 rtc::CritScope lock(&crit_); | |
222 return GetActiveSinkWantsInternal(); | |
223 } | |
224 | |
225 void ResetPixelFpsCount() { | |
226 rtc::CritScope lock(&crit_); | |
227 sink_wants_.max_pixel_count = std::numeric_limits<int>::max(); | |
228 sink_wants_.target_pixel_count.reset(); | |
229 sink_wants_.max_framerate_fps = std::numeric_limits<int>::max(); | |
230 if (source_) | |
231 source_->AddOrUpdateSink(vie_encoder_, sink_wants_); | |
232 } | |
233 | |
234 bool RequestResolutionLowerThan(int pixel_count) { | |
235 // Called on the encoder task queue. | |
236 rtc::CritScope lock(&crit_); | |
237 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { | |
238 // This can happen since |degradation_preference_| is set on libjingle's | |
239 // worker thread but the adaptation is done on the encoder task queue. | |
240 return false; | |
241 } | |
242 // The input video frame size will have a resolution less than or equal to | |
243 // |max_pixel_count| depending on how the source can scale the frame size. | |
244 const int pixels_wanted = (pixel_count * 3) / 5; | |
245 if (pixels_wanted < kMinPixelsPerFrame || | |
246 pixels_wanted >= sink_wants_.max_pixel_count) { | |
247 return false; | |
248 } | |
249 LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; | |
250 sink_wants_.max_pixel_count = pixels_wanted; | |
251 sink_wants_.target_pixel_count = rtc::Optional<int>(); | |
252 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
253 return true; | |
254 } | |
255 | |
256 int RequestFramerateLowerThan(int fps) { | |
257 // Called on the encoder task queue. | |
258 // The input video frame rate will be scaled down to 2/3, rounding down. | |
259 int framerate_wanted = (fps * 2) / 3; | |
260 return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; | |
261 } | |
262 | |
263 bool RequestHigherResolutionThan(int pixel_count) { | |
264 // Called on the encoder task queue. | |
265 rtc::CritScope lock(&crit_); | |
266 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { | |
267 // This can happen since |degradation_preference_| is set on libjingle's | |
268 // worker thread but the adaptation is done on the encoder task queue. | |
269 return false; | |
270 } | |
271 int max_pixels_wanted = pixel_count; | |
272 if (max_pixels_wanted != std::numeric_limits<int>::max()) | |
273 max_pixels_wanted = pixel_count * 4; | |
274 | |
275 if (max_pixels_wanted <= sink_wants_.max_pixel_count) | |
276 return false; | |
277 | |
278 sink_wants_.max_pixel_count = max_pixels_wanted; | |
279 if (max_pixels_wanted == std::numeric_limits<int>::max()) { | |
280 // Remove any constraints. | |
281 sink_wants_.target_pixel_count.reset(); | |
282 } else { | |
283 // On step down we request at most 3/5 the pixel count of the previous | |
284 // resolution, so in order to take "one step up" we request a resolution | |
285 // as close as possible to 5/3 of the current resolution. The actual pixel | |
286 // count selected depends on the capabilities of the source. In order to | |
287 // not take a too large step up, we cap the requested pixel count to be at | |
288 // most four time the current number of pixels. | |
289 sink_wants_.target_pixel_count = | |
290 rtc::Optional<int>((pixel_count * 5) / 3); | |
291 } | |
292 LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; | |
293 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
294 return true; | |
295 } | |
296 | |
297 // Request upgrade in framerate. Returns the new requested frame, or -1 if | |
298 // no change requested. Note that maxint may be returned if limits due to | |
299 // adaptation requests are removed completely. In that case, consider | |
300 // |max_framerate_| to be the current limit (assuming the capturer complies). | |
301 int RequestHigherFramerateThan(int fps) { | |
302 // Called on the encoder task queue. | |
303 // The input frame rate will be scaled up to the last step, with rounding. | |
304 int framerate_wanted = fps; | |
305 if (fps != std::numeric_limits<int>::max()) | |
306 framerate_wanted = (fps * 3) / 2; | |
307 | |
308 return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; | |
309 } | |
310 | |
311 bool RestrictFramerate(int fps) { | |
312 // Called on the encoder task queue. | |
313 rtc::CritScope lock(&crit_); | |
314 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) | |
315 return false; | |
316 | |
317 const int fps_wanted = std::max(kMinFramerateFps, fps); | |
318 if (fps_wanted >= sink_wants_.max_framerate_fps) | |
319 return false; | |
320 | |
321 LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; | |
322 sink_wants_.max_framerate_fps = fps_wanted; | |
323 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
324 return true; | |
325 } | |
326 | |
327 bool IncreaseFramerate(int fps) { | |
328 // Called on the encoder task queue. | |
329 rtc::CritScope lock(&crit_); | |
330 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) | |
331 return false; | |
332 | |
333 const int fps_wanted = std::max(kMinFramerateFps, fps); | |
334 if (fps_wanted <= sink_wants_.max_framerate_fps) | |
335 return false; | |
336 | |
337 LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; | |
338 sink_wants_.max_framerate_fps = fps_wanted; | |
339 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal()); | |
340 return true; | |
341 } | |
342 | |
343 private: | |
344 rtc::VideoSinkWants GetActiveSinkWantsInternal() | |
345 EXCLUSIVE_LOCKS_REQUIRED(&crit_) { | |
346 rtc::VideoSinkWants wants = sink_wants_; | |
347 // Clear any constraints from the current sink wants that don't apply to | |
348 // the used degradation_preference. | |
349 switch (degradation_preference_) { | |
350 case VideoSendStream::DegradationPreference::kBalanced: | |
351 break; | |
352 case VideoSendStream::DegradationPreference::kMaintainFramerate: | |
353 wants.max_framerate_fps = std::numeric_limits<int>::max(); | |
354 break; | |
355 case VideoSendStream::DegradationPreference::kMaintainResolution: | |
356 wants.max_pixel_count = std::numeric_limits<int>::max(); | |
357 wants.target_pixel_count.reset(); | |
358 break; | |
359 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
360 wants.max_pixel_count = std::numeric_limits<int>::max(); | |
361 wants.target_pixel_count.reset(); | |
362 wants.max_framerate_fps = std::numeric_limits<int>::max(); | |
363 } | |
364 return wants; | |
365 } | |
366 | |
367 rtc::CriticalSection crit_; | |
368 rtc::SequencedTaskChecker main_checker_; | |
369 ViEEncoder* const vie_encoder_; | |
370 rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_); | |
371 VideoSendStream::DegradationPreference degradation_preference_ | |
372 GUARDED_BY(&crit_); | |
373 rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_); | |
374 | |
375 RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); | |
376 }; | |
377 | |
378 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | |
379 SendStatisticsProxy* stats_proxy, | |
380 const VideoSendStream::Config::EncoderSettings& settings, | |
381 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | |
382 EncodedFrameObserver* encoder_timing, | |
383 std::unique_ptr<OveruseFrameDetector> overuse_detector) | |
384 : shutdown_event_(true /* manual_reset */, false), | |
385 number_of_cores_(number_of_cores), | |
386 initial_rampup_(0), | |
387 source_proxy_(new VideoSourceProxy(this)), | |
388 sink_(nullptr), | |
389 settings_(settings), | |
390 codec_type_(PayloadNameToCodecType(settings.payload_name) | |
391 .value_or(VideoCodecType::kVideoCodecUnknown)), | |
392 video_sender_(Clock::GetRealTimeClock(), this, this), | |
393 overuse_detector_( | |
394 overuse_detector.get() | |
395 ? overuse_detector.release() | |
396 : new OveruseFrameDetector( | |
397 GetCpuOveruseOptions(settings.full_overuse_time), | |
398 this, | |
399 encoder_timing, | |
400 stats_proxy)), | |
401 stats_proxy_(stats_proxy), | |
402 pre_encode_callback_(pre_encode_callback), | |
403 module_process_thread_(nullptr), | |
404 max_framerate_(-1), | |
405 pending_encoder_reconfiguration_(false), | |
406 encoder_start_bitrate_bps_(0), | |
407 max_data_payload_length_(0), | |
408 nack_enabled_(false), | |
409 last_observed_bitrate_bps_(0), | |
410 encoder_paused_and_dropped_frame_(false), | |
411 clock_(Clock::GetRealTimeClock()), | |
412 degradation_preference_( | |
413 VideoSendStream::DegradationPreference::kDegradationDisabled), | |
414 last_captured_timestamp_(0), | |
415 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | |
416 clock_->TimeInMilliseconds()), | |
417 last_frame_log_ms_(clock_->TimeInMilliseconds()), | |
418 captured_frame_count_(0), | |
419 dropped_frame_count_(0), | |
420 bitrate_observer_(nullptr), | |
421 encoder_queue_("EncoderQueue") { | |
422 RTC_DCHECK(stats_proxy); | |
423 encoder_queue_.PostTask([this] { | |
424 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
425 overuse_detector_->StartCheckForOveruse(); | |
426 video_sender_.RegisterExternalEncoder( | |
427 settings_.encoder, settings_.payload_type, settings_.internal_source); | |
428 }); | |
429 } | |
430 | |
431 ViEEncoder::~ViEEncoder() { | |
432 RTC_DCHECK_RUN_ON(&thread_checker_); | |
433 RTC_DCHECK(shutdown_event_.Wait(0)) | |
434 << "Must call ::Stop() before destruction."; | |
435 } | |
436 | |
437 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle | |
438 // pipelining encoders better (multiple input frames before something comes | |
439 // out). This should effectively turn off CPU adaptations for systems that | |
440 // remotely cope with the load right now. | |
441 CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) { | |
442 CpuOveruseOptions options; | |
443 if (full_overuse_time) { | |
444 options.low_encode_usage_threshold_percent = 150; | |
445 options.high_encode_usage_threshold_percent = 200; | |
446 } | |
447 return options; | |
448 } | |
449 | |
450 void ViEEncoder::Stop() { | |
451 RTC_DCHECK_RUN_ON(&thread_checker_); | |
452 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); | |
453 encoder_queue_.PostTask([this] { | |
454 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
455 overuse_detector_->StopCheckForOveruse(); | |
456 rate_allocator_.reset(); | |
457 bitrate_observer_ = nullptr; | |
458 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, | |
459 false); | |
460 quality_scaler_ = nullptr; | |
461 shutdown_event_.Set(); | |
462 }); | |
463 | |
464 shutdown_event_.Wait(rtc::Event::kForever); | |
465 } | |
466 | |
467 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | |
468 RTC_DCHECK_RUN_ON(&thread_checker_); | |
469 RTC_DCHECK(!module_process_thread_); | |
470 module_process_thread_ = module_process_thread; | |
471 module_process_thread_->RegisterModule(&video_sender_, RTC_FROM_HERE); | |
472 module_process_thread_checker_.DetachFromThread(); | |
473 } | |
474 | |
475 void ViEEncoder::DeRegisterProcessThread() { | |
476 RTC_DCHECK_RUN_ON(&thread_checker_); | |
477 module_process_thread_->DeRegisterModule(&video_sender_); | |
478 } | |
479 | |
480 void ViEEncoder::SetBitrateObserver( | |
481 VideoBitrateAllocationObserver* bitrate_observer) { | |
482 RTC_DCHECK_RUN_ON(&thread_checker_); | |
483 encoder_queue_.PostTask([this, bitrate_observer] { | |
484 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
485 RTC_DCHECK(!bitrate_observer_); | |
486 bitrate_observer_ = bitrate_observer; | |
487 }); | |
488 } | |
489 | |
490 void ViEEncoder::SetSource( | |
491 rtc::VideoSourceInterface<VideoFrame>* source, | |
492 const VideoSendStream::DegradationPreference& degradation_preference) { | |
493 RTC_DCHECK_RUN_ON(&thread_checker_); | |
494 source_proxy_->SetSource(source, degradation_preference); | |
495 encoder_queue_.PostTask([this, degradation_preference] { | |
496 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
497 if (degradation_preference_ != degradation_preference) { | |
498 // Reset adaptation state, so that we're not tricked into thinking there's | |
499 // an already pending request of the same type. | |
500 last_adaptation_request_.reset(); | |
501 if (degradation_preference == | |
502 VideoSendStream::DegradationPreference::kBalanced || | |
503 degradation_preference_ == | |
504 VideoSendStream::DegradationPreference::kBalanced) { | |
505 // TODO(asapersson): Consider removing |adapt_counters_| map and use one | |
506 // AdaptCounter for all modes. | |
507 source_proxy_->ResetPixelFpsCount(); | |
508 adapt_counters_.clear(); | |
509 } | |
510 } | |
511 degradation_preference_ = degradation_preference; | |
512 bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); | |
513 initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; | |
514 ConfigureQualityScaler(); | |
515 if (!IsFramerateScalingEnabled(degradation_preference) && | |
516 max_framerate_ != -1) { | |
517 // If frame rate scaling is no longer allowed, remove any potential | |
518 // allowance for longer frame intervals. | |
519 overuse_detector_->OnTargetFramerateUpdated(max_framerate_); | |
520 } | |
521 }); | |
522 } | |
523 | |
524 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { | |
525 source_proxy_->SetWantsRotationApplied(rotation_applied); | |
526 encoder_queue_.PostTask([this, sink] { | |
527 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
528 sink_ = sink; | |
529 }); | |
530 } | |
531 | |
532 void ViEEncoder::SetStartBitrate(int start_bitrate_bps) { | |
533 encoder_queue_.PostTask([this, start_bitrate_bps] { | |
534 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
535 encoder_start_bitrate_bps_ = start_bitrate_bps; | |
536 }); | |
537 } | |
538 | |
539 void ViEEncoder::ConfigureEncoder(VideoEncoderConfig config, | |
540 size_t max_data_payload_length, | |
541 bool nack_enabled) { | |
542 encoder_queue_.PostTask( | |
543 std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask( | |
544 this, std::move(config), max_data_payload_length, nack_enabled))); | |
545 } | |
546 | |
547 void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config, | |
548 size_t max_data_payload_length, | |
549 bool nack_enabled) { | |
550 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
551 RTC_DCHECK(sink_); | |
552 LOG(LS_INFO) << "ConfigureEncoder requested."; | |
553 | |
554 max_data_payload_length_ = max_data_payload_length; | |
555 nack_enabled_ = nack_enabled; | |
556 encoder_config_ = std::move(config); | |
557 pending_encoder_reconfiguration_ = true; | |
558 | |
559 // Reconfigure the encoder now if the encoder has an internal source or | |
560 // if the frame resolution is known. Otherwise, the reconfiguration is | |
561 // deferred until the next frame to minimize the number of reconfigurations. | |
562 // The codec configuration depends on incoming video frame size. | |
563 if (last_frame_info_) { | |
564 ReconfigureEncoder(); | |
565 } else if (settings_.internal_source) { | |
566 last_frame_info_ = | |
567 rtc::Optional<VideoFrameInfo>(VideoFrameInfo(176, 144, false)); | |
568 ReconfigureEncoder(); | |
569 } | |
570 } | |
571 | |
572 void ViEEncoder::ReconfigureEncoder() { | |
573 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
574 RTC_DCHECK(pending_encoder_reconfiguration_); | |
575 std::vector<VideoStream> streams = | |
576 encoder_config_.video_stream_factory->CreateEncoderStreams( | |
577 last_frame_info_->width, last_frame_info_->height, encoder_config_); | |
578 | |
579 // TODO(ilnik): If configured resolution is significantly less than provided, | |
580 // e.g. because there are not enough SSRCs for all simulcast streams, | |
581 // signal new resolutions via SinkWants to video source. | |
582 | |
583 // Stream dimensions may be not equal to given because of a simulcast | |
584 // restrictions. | |
585 int highest_stream_width = static_cast<int>(streams.back().width); | |
586 int highest_stream_height = static_cast<int>(streams.back().height); | |
587 // Dimension may be reduced to be, e.g. divisible by 4. | |
588 RTC_CHECK_GE(last_frame_info_->width, highest_stream_width); | |
589 RTC_CHECK_GE(last_frame_info_->height, highest_stream_height); | |
590 crop_width_ = last_frame_info_->width - highest_stream_width; | |
591 crop_height_ = last_frame_info_->height - highest_stream_height; | |
592 | |
593 VideoCodec codec; | |
594 if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams, | |
595 nack_enabled_, &codec, | |
596 &rate_allocator_)) { | |
597 LOG(LS_ERROR) << "Failed to create encoder configuration."; | |
598 } | |
599 | |
600 codec.startBitrate = | |
601 std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); | |
602 codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); | |
603 codec.expect_encode_from_texture = last_frame_info_->is_texture; | |
604 max_framerate_ = codec.maxFramerate; | |
605 RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps); | |
606 | |
607 bool success = video_sender_.RegisterSendCodec( | |
608 &codec, number_of_cores_, | |
609 static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK; | |
610 if (!success) { | |
611 LOG(LS_ERROR) << "Failed to configure encoder."; | |
612 rate_allocator_.reset(); | |
613 } | |
614 | |
615 video_sender_.UpdateChannelParemeters(rate_allocator_.get(), | |
616 bitrate_observer_); | |
617 | |
618 // Get the current actual framerate, as measured by the stats proxy. This is | |
619 // used to get the correct bitrate layer allocation. | |
620 int current_framerate = stats_proxy_->GetSendFrameRate(); | |
621 if (current_framerate == 0) | |
622 current_framerate = codec.maxFramerate; | |
623 stats_proxy_->OnEncoderReconfigured( | |
624 encoder_config_, | |
625 rate_allocator_.get() | |
626 ? rate_allocator_->GetPreferredBitrateBps(current_framerate) | |
627 : codec.maxBitrate); | |
628 | |
629 pending_encoder_reconfiguration_ = false; | |
630 | |
631 sink_->OnEncoderConfigurationChanged( | |
632 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | |
633 | |
634 // Get the current target framerate, ie the maximum framerate as specified by | |
635 // the current codec configuration, or any limit imposed by cpu adaption in | |
636 // maintain-resolution or balanced mode. This is used to make sure overuse | |
637 // detection doesn't needlessly trigger in low and/or variable framerate | |
638 // scenarios. | |
639 int target_framerate = std::min( | |
640 max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); | |
641 overuse_detector_->OnTargetFramerateUpdated(target_framerate); | |
642 | |
643 ConfigureQualityScaler(); | |
644 } | |
645 | |
646 void ViEEncoder::ConfigureQualityScaler() { | |
647 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
648 const auto scaling_settings = settings_.encoder->GetScalingSettings(); | |
649 const bool quality_scaling_allowed = | |
650 IsResolutionScalingEnabled(degradation_preference_) && | |
651 scaling_settings.enabled; | |
652 | |
653 if (quality_scaling_allowed) { | |
654 if (quality_scaler_.get() == nullptr) { | |
655 // Quality scaler has not already been configured. | |
656 // Drop frames and scale down until desired quality is achieved. | |
657 if (scaling_settings.thresholds) { | |
658 quality_scaler_.reset( | |
659 new QualityScaler(this, *(scaling_settings.thresholds))); | |
660 } else { | |
661 quality_scaler_.reset(new QualityScaler(this, codec_type_)); | |
662 } | |
663 } | |
664 } else { | |
665 quality_scaler_.reset(nullptr); | |
666 initial_rampup_ = kMaxInitialFramedrop; | |
667 } | |
668 | |
669 stats_proxy_->SetAdaptationStats(GetActiveCounts(kCpu), | |
670 GetActiveCounts(kQuality)); | |
671 } | |
672 | |
673 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | |
674 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | |
675 VideoFrame incoming_frame = video_frame; | |
676 | |
677 // Local time in webrtc time base. | |
678 int64_t current_time_us = clock_->TimeInMicroseconds(); | |
679 int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec; | |
680 // In some cases, e.g., when the frame from decoder is fed to encoder, | |
681 // the timestamp may be set to the future. As the encoding pipeline assumes | |
682 // capture time to be less than present time, we should reset the capture | |
683 // timestamps here. Otherwise there may be issues with RTP send stream. | |
684 if (incoming_frame.timestamp_us() > current_time_us) | |
685 incoming_frame.set_timestamp_us(current_time_us); | |
686 | |
687 // Capture time may come from clock with an offset and drift from clock_. | |
688 int64_t capture_ntp_time_ms; | |
689 if (video_frame.ntp_time_ms() > 0) { | |
690 capture_ntp_time_ms = video_frame.ntp_time_ms(); | |
691 } else if (video_frame.render_time_ms() != 0) { | |
692 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; | |
693 } else { | |
694 capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; | |
695 } | |
696 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); | |
697 | |
698 // Convert NTP time, in ms, to RTP timestamp. | |
699 const int kMsToRtpTimestamp = 90; | |
700 incoming_frame.set_timestamp( | |
701 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms())); | |
702 | |
703 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { | |
704 // We don't allow the same capture time for two frames, drop this one. | |
705 LOG(LS_WARNING) << "Same/old NTP timestamp (" | |
706 << incoming_frame.ntp_time_ms() | |
707 << " <= " << last_captured_timestamp_ | |
708 << ") for incoming frame. Dropping."; | |
709 return; | |
710 } | |
711 | |
712 bool log_stats = false; | |
713 if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) { | |
714 last_frame_log_ms_ = current_time_ms; | |
715 log_stats = true; | |
716 } | |
717 | |
718 last_captured_timestamp_ = incoming_frame.ntp_time_ms(); | |
719 encoder_queue_.PostTask(std::unique_ptr<rtc::QueuedTask>(new EncodeTask( | |
720 incoming_frame, this, rtc::TimeMicros(), log_stats))); | |
721 } | |
722 | |
723 bool ViEEncoder::EncoderPaused() const { | |
724 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
725 // Pause video if paused by caller or as long as the network is down or the | |
726 // pacer queue has grown too large in buffered mode. | |
727 // If the pacer queue has grown too large or the network is down, | |
728 // last_observed_bitrate_bps_ will be 0. | |
729 return last_observed_bitrate_bps_ == 0; | |
730 } | |
731 | |
732 void ViEEncoder::TraceFrameDropStart() { | |
733 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
734 // Start trace event only on the first frame after encoder is paused. | |
735 if (!encoder_paused_and_dropped_frame_) { | |
736 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); | |
737 } | |
738 encoder_paused_and_dropped_frame_ = true; | |
739 } | |
740 | |
741 void ViEEncoder::TraceFrameDropEnd() { | |
742 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
743 // End trace event on first frame after encoder resumes, if frame was dropped. | |
744 if (encoder_paused_and_dropped_frame_) { | |
745 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | |
746 } | |
747 encoder_paused_and_dropped_frame_ = false; | |
748 } | |
749 | |
750 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, | |
751 int64_t time_when_posted_us) { | |
752 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
753 | |
754 if (pre_encode_callback_) | |
755 pre_encode_callback_->OnFrame(video_frame); | |
756 | |
757 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || | |
758 video_frame.height() != last_frame_info_->height || | |
759 video_frame.is_texture() != last_frame_info_->is_texture) { | |
760 pending_encoder_reconfiguration_ = true; | |
761 last_frame_info_ = rtc::Optional<VideoFrameInfo>(VideoFrameInfo( | |
762 video_frame.width(), video_frame.height(), video_frame.is_texture())); | |
763 LOG(LS_INFO) << "Video frame parameters changed: dimensions=" | |
764 << last_frame_info_->width << "x" << last_frame_info_->height | |
765 << ", texture=" << last_frame_info_->is_texture << "."; | |
766 } | |
767 | |
768 if (initial_rampup_ < kMaxInitialFramedrop && | |
769 video_frame.size() > | |
770 MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) { | |
771 LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; | |
772 AdaptDown(kQuality); | |
773 ++initial_rampup_; | |
774 return; | |
775 } | |
776 initial_rampup_ = kMaxInitialFramedrop; | |
777 | |
778 int64_t now_ms = clock_->TimeInMilliseconds(); | |
779 if (pending_encoder_reconfiguration_) { | |
780 ReconfigureEncoder(); | |
781 last_parameters_update_ms_.emplace(now_ms); | |
782 } else if (!last_parameters_update_ms_ || | |
783 now_ms - *last_parameters_update_ms_ >= | |
784 vcm::VCMProcessTimer::kDefaultProcessIntervalMs) { | |
785 video_sender_.UpdateChannelParemeters(rate_allocator_.get(), | |
786 bitrate_observer_); | |
787 last_parameters_update_ms_.emplace(now_ms); | |
788 } | |
789 | |
790 if (EncoderPaused()) { | |
791 TraceFrameDropStart(); | |
792 return; | |
793 } | |
794 TraceFrameDropEnd(); | |
795 | |
796 VideoFrame out_frame(video_frame); | |
797 // Crop frame if needed. | |
798 if (crop_width_ > 0 || crop_height_ > 0) { | |
799 int cropped_width = video_frame.width() - crop_width_; | |
800 int cropped_height = video_frame.height() - crop_height_; | |
801 rtc::scoped_refptr<I420Buffer> cropped_buffer = | |
802 I420Buffer::Create(cropped_width, cropped_height); | |
803 // TODO(ilnik): Remove scaling if cropping is too big, as it should never | |
804 // happen after SinkWants signaled correctly from ReconfigureEncoder. | |
805 if (crop_width_ < 4 && crop_height_ < 4) { | |
806 cropped_buffer->CropAndScaleFrom( | |
807 *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2, | |
808 crop_height_ / 2, cropped_width, cropped_height); | |
809 } else { | |
810 cropped_buffer->ScaleFrom( | |
811 *video_frame.video_frame_buffer()->ToI420().get()); | |
812 } | |
813 out_frame = | |
814 VideoFrame(cropped_buffer, video_frame.timestamp(), | |
815 video_frame.render_time_ms(), video_frame.rotation()); | |
816 out_frame.set_ntp_time_ms(video_frame.ntp_time_ms()); | |
817 } | |
818 | |
819 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), | |
820 "Encode"); | |
821 | |
822 overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); | |
823 | |
824 video_sender_.AddVideoFrame(out_frame, nullptr); | |
825 } | |
826 | |
827 void ViEEncoder::SendKeyFrame() { | |
828 if (!encoder_queue_.IsCurrent()) { | |
829 encoder_queue_.PostTask([this] { SendKeyFrame(); }); | |
830 return; | |
831 } | |
832 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
833 video_sender_.IntraFrameRequest(0); | |
834 } | |
835 | |
836 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( | |
837 const EncodedImage& encoded_image, | |
838 const CodecSpecificInfo* codec_specific_info, | |
839 const RTPFragmentationHeader* fragmentation) { | |
840 // Encoded is called on whatever thread the real encoder implementation run | |
841 // on. In the case of hardware encoders, there might be several encoders | |
842 // running in parallel on different threads. | |
843 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | |
844 | |
845 EncodedImageCallback::Result result = | |
846 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | |
847 | |
848 int64_t time_sent_us = rtc::TimeMicros(); | |
849 uint32_t timestamp = encoded_image._timeStamp; | |
850 const int qp = encoded_image.qp_; | |
851 encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { | |
852 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
853 overuse_detector_->FrameSent(timestamp, time_sent_us); | |
854 if (quality_scaler_ && qp >= 0) | |
855 quality_scaler_->ReportQP(qp); | |
856 }); | |
857 | |
858 return result; | |
859 } | |
860 | |
861 void ViEEncoder::OnDroppedFrame() { | |
862 encoder_queue_.PostTask([this] { | |
863 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
864 if (quality_scaler_) | |
865 quality_scaler_->ReportDroppedFrame(); | |
866 }); | |
867 } | |
868 | |
869 void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) { | |
870 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | |
871 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate); | |
872 } | |
873 | |
874 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { | |
875 if (!encoder_queue_.IsCurrent()) { | |
876 encoder_queue_.PostTask( | |
877 [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); }); | |
878 return; | |
879 } | |
880 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
881 // Key frame request from remote side, signal to VCM. | |
882 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | |
883 video_sender_.IntraFrameRequest(stream_index); | |
884 } | |
885 | |
886 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | |
887 uint8_t fraction_lost, | |
888 int64_t round_trip_time_ms) { | |
889 if (!encoder_queue_.IsCurrent()) { | |
890 encoder_queue_.PostTask( | |
891 [this, bitrate_bps, fraction_lost, round_trip_time_ms] { | |
892 OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms); | |
893 }); | |
894 return; | |
895 } | |
896 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
897 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; | |
898 | |
899 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps | |
900 << " packet loss " << static_cast<int>(fraction_lost) | |
901 << " rtt " << round_trip_time_ms; | |
902 | |
903 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | |
904 round_trip_time_ms, rate_allocator_.get(), | |
905 bitrate_observer_); | |
906 | |
907 encoder_start_bitrate_bps_ = | |
908 bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; | |
909 bool video_is_suspended = bitrate_bps == 0; | |
910 bool video_suspension_changed = video_is_suspended != EncoderPaused(); | |
911 last_observed_bitrate_bps_ = bitrate_bps; | |
912 | |
913 if (video_suspension_changed) { | |
914 LOG(LS_INFO) << "Video suspend state changed to: " | |
915 << (video_is_suspended ? "suspended" : "not suspended"); | |
916 stats_proxy_->OnSuspendChange(video_is_suspended); | |
917 } | |
918 } | |
919 | |
920 void ViEEncoder::AdaptDown(AdaptReason reason) { | |
921 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
922 AdaptationRequest adaptation_request = { | |
923 last_frame_info_->pixel_count(), | |
924 stats_proxy_->GetStats().input_frame_rate, | |
925 AdaptationRequest::Mode::kAdaptDown}; | |
926 | |
927 bool downgrade_requested = | |
928 last_adaptation_request_ && | |
929 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; | |
930 | |
931 switch (degradation_preference_) { | |
932 case VideoSendStream::DegradationPreference::kBalanced: | |
933 break; | |
934 case VideoSendStream::DegradationPreference::kMaintainFramerate: | |
935 if (downgrade_requested && | |
936 adaptation_request.input_pixel_count_ >= | |
937 last_adaptation_request_->input_pixel_count_) { | |
938 // Don't request lower resolution if the current resolution is not | |
939 // lower than the last time we asked for the resolution to be lowered. | |
940 return; | |
941 } | |
942 break; | |
943 case VideoSendStream::DegradationPreference::kMaintainResolution: | |
944 if (adaptation_request.framerate_fps_ <= 0 || | |
945 (downgrade_requested && | |
946 adaptation_request.framerate_fps_ < kMinFramerateFps)) { | |
947 // If no input fps estimate available, can't determine how to scale down | |
948 // framerate. Otherwise, don't request lower framerate if we don't have | |
949 // a valid frame rate. Since framerate, unlike resolution, is a measure | |
950 // we have to estimate, and can fluctuate naturally over time, don't | |
951 // make the same kind of limitations as for resolution, but trust the | |
952 // overuse detector to not trigger too often. | |
953 return; | |
954 } | |
955 break; | |
956 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
957 return; | |
958 } | |
959 | |
960 if (reason == kCpu) { | |
961 if (GetConstAdaptCounter().ResolutionCount(kCpu) >= | |
962 kMaxCpuResolutionDowngrades || | |
963 GetConstAdaptCounter().FramerateCount(kCpu) >= | |
964 kMaxCpuFramerateDowngrades) { | |
965 return; | |
966 } | |
967 } | |
968 | |
969 switch (degradation_preference_) { | |
970 case VideoSendStream::DegradationPreference::kBalanced: { | |
971 // Try scale down framerate, if lower. | |
972 int fps = MinFps(last_frame_info_->pixel_count()); | |
973 if (source_proxy_->RestrictFramerate(fps)) { | |
974 GetAdaptCounter().IncrementFramerate(reason); | |
975 break; | |
976 } | |
977 // Scale down resolution. | |
978 FALLTHROUGH(); | |
979 } | |
980 case VideoSendStream::DegradationPreference::kMaintainFramerate: | |
981 // Scale down resolution. | |
982 if (!source_proxy_->RequestResolutionLowerThan( | |
983 adaptation_request.input_pixel_count_)) { | |
984 return; | |
985 } | |
986 GetAdaptCounter().IncrementResolution(reason); | |
987 break; | |
988 case VideoSendStream::DegradationPreference::kMaintainResolution: { | |
989 // Scale down framerate. | |
990 const int requested_framerate = source_proxy_->RequestFramerateLowerThan( | |
991 adaptation_request.framerate_fps_); | |
992 if (requested_framerate == -1) | |
993 return; | |
994 RTC_DCHECK_NE(max_framerate_, -1); | |
995 overuse_detector_->OnTargetFramerateUpdated( | |
996 std::min(max_framerate_, requested_framerate)); | |
997 GetAdaptCounter().IncrementFramerate(reason); | |
998 break; | |
999 } | |
1000 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
1001 RTC_NOTREACHED(); | |
1002 } | |
1003 | |
1004 last_adaptation_request_.emplace(adaptation_request); | |
1005 | |
1006 UpdateAdaptationStats(reason); | |
1007 | |
1008 LOG(LS_INFO) << GetConstAdaptCounter().ToString(); | |
1009 } | |
1010 | |
1011 void ViEEncoder::AdaptUp(AdaptReason reason) { | |
1012 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
1013 | |
1014 const AdaptCounter& adapt_counter = GetConstAdaptCounter(); | |
1015 int num_downgrades = adapt_counter.TotalCount(reason); | |
1016 if (num_downgrades == 0) | |
1017 return; | |
1018 RTC_DCHECK_GT(num_downgrades, 0); | |
1019 | |
1020 AdaptationRequest adaptation_request = { | |
1021 last_frame_info_->pixel_count(), | |
1022 stats_proxy_->GetStats().input_frame_rate, | |
1023 AdaptationRequest::Mode::kAdaptUp}; | |
1024 | |
1025 bool adapt_up_requested = | |
1026 last_adaptation_request_ && | |
1027 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; | |
1028 | |
1029 if (degradation_preference_ == | |
1030 VideoSendStream::DegradationPreference::kMaintainFramerate) { | |
1031 if (adapt_up_requested && | |
1032 adaptation_request.input_pixel_count_ <= | |
1033 last_adaptation_request_->input_pixel_count_) { | |
1034 // Don't request higher resolution if the current resolution is not | |
1035 // higher than the last time we asked for the resolution to be higher. | |
1036 return; | |
1037 } | |
1038 } | |
1039 | |
1040 switch (degradation_preference_) { | |
1041 case VideoSendStream::DegradationPreference::kBalanced: { | |
1042 // Try scale up framerate, if higher. | |
1043 int fps = MaxFps(last_frame_info_->pixel_count()); | |
1044 if (source_proxy_->IncreaseFramerate(fps)) { | |
1045 GetAdaptCounter().DecrementFramerate(reason, fps); | |
1046 // Reset framerate in case of fewer fps steps down than up. | |
1047 if (adapt_counter.FramerateCount() == 0 && | |
1048 fps != std::numeric_limits<int>::max()) { | |
1049 LOG(LS_INFO) << "Removing framerate down-scaling setting."; | |
1050 source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max()); | |
1051 } | |
1052 break; | |
1053 } | |
1054 // Scale up resolution. | |
1055 FALLTHROUGH(); | |
1056 } | |
1057 case VideoSendStream::DegradationPreference::kMaintainFramerate: { | |
1058 // Scale up resolution. | |
1059 int pixel_count = adaptation_request.input_pixel_count_; | |
1060 if (adapt_counter.ResolutionCount() == 1) { | |
1061 LOG(LS_INFO) << "Removing resolution down-scaling setting."; | |
1062 pixel_count = std::numeric_limits<int>::max(); | |
1063 } | |
1064 if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) | |
1065 return; | |
1066 GetAdaptCounter().DecrementResolution(reason); | |
1067 break; | |
1068 } | |
1069 case VideoSendStream::DegradationPreference::kMaintainResolution: { | |
1070 // Scale up framerate. | |
1071 int fps = adaptation_request.framerate_fps_; | |
1072 if (adapt_counter.FramerateCount() == 1) { | |
1073 LOG(LS_INFO) << "Removing framerate down-scaling setting."; | |
1074 fps = std::numeric_limits<int>::max(); | |
1075 } | |
1076 | |
1077 const int requested_framerate = | |
1078 source_proxy_->RequestHigherFramerateThan(fps); | |
1079 if (requested_framerate == -1) { | |
1080 overuse_detector_->OnTargetFramerateUpdated(max_framerate_); | |
1081 return; | |
1082 } | |
1083 overuse_detector_->OnTargetFramerateUpdated( | |
1084 std::min(max_framerate_, requested_framerate)); | |
1085 GetAdaptCounter().DecrementFramerate(reason); | |
1086 break; | |
1087 } | |
1088 case VideoSendStream::DegradationPreference::kDegradationDisabled: | |
1089 return; | |
1090 } | |
1091 | |
1092 last_adaptation_request_.emplace(adaptation_request); | |
1093 | |
1094 UpdateAdaptationStats(reason); | |
1095 | |
1096 LOG(LS_INFO) << adapt_counter.ToString(); | |
1097 } | |
1098 | |
1099 void ViEEncoder::UpdateAdaptationStats(AdaptReason reason) { | |
1100 switch (reason) { | |
1101 case kCpu: | |
1102 stats_proxy_->OnCpuAdaptationChanged(GetActiveCounts(kCpu), | |
1103 GetActiveCounts(kQuality)); | |
1104 break; | |
1105 case kQuality: | |
1106 stats_proxy_->OnQualityAdaptationChanged(GetActiveCounts(kCpu), | |
1107 GetActiveCounts(kQuality)); | |
1108 break; | |
1109 } | |
1110 } | |
1111 | |
1112 ViEEncoder::AdaptCounts ViEEncoder::GetActiveCounts(AdaptReason reason) { | |
1113 ViEEncoder::AdaptCounts counts = GetConstAdaptCounter().Counts(reason); | |
1114 switch (reason) { | |
1115 case kCpu: | |
1116 if (!IsFramerateScalingEnabled(degradation_preference_)) | |
1117 counts.fps = -1; | |
1118 if (!IsResolutionScalingEnabled(degradation_preference_)) | |
1119 counts.resolution = -1; | |
1120 break; | |
1121 case kQuality: | |
1122 if (!IsFramerateScalingEnabled(degradation_preference_) || | |
1123 !quality_scaler_) { | |
1124 counts.fps = -1; | |
1125 } | |
1126 if (!IsResolutionScalingEnabled(degradation_preference_) || | |
1127 !quality_scaler_) { | |
1128 counts.resolution = -1; | |
1129 } | |
1130 break; | |
1131 } | |
1132 return counts; | |
1133 } | |
1134 | |
1135 ViEEncoder::AdaptCounter& ViEEncoder::GetAdaptCounter() { | |
1136 return adapt_counters_[degradation_preference_]; | |
1137 } | |
1138 | |
1139 const ViEEncoder::AdaptCounter& ViEEncoder::GetConstAdaptCounter() { | |
1140 return adapt_counters_[degradation_preference_]; | |
1141 } | |
1142 | |
1143 // Class holding adaptation information. | |
1144 ViEEncoder::AdaptCounter::AdaptCounter() { | |
1145 fps_counters_.resize(kScaleReasonSize); | |
1146 resolution_counters_.resize(kScaleReasonSize); | |
1147 static_assert(kScaleReasonSize == 2, "Update MoveCount."); | |
1148 } | |
1149 | |
1150 ViEEncoder::AdaptCounter::~AdaptCounter() {} | |
1151 | |
1152 std::string ViEEncoder::AdaptCounter::ToString() const { | |
1153 std::stringstream ss; | |
1154 ss << "Downgrade counts: fps: {" << ToString(fps_counters_); | |
1155 ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; | |
1156 return ss.str(); | |
1157 } | |
1158 | |
1159 ViEEncoder::AdaptCounts ViEEncoder::AdaptCounter::Counts(int reason) const { | |
1160 AdaptCounts counts; | |
1161 counts.fps = fps_counters_[reason]; | |
1162 counts.resolution = resolution_counters_[reason]; | |
1163 return counts; | |
1164 } | |
1165 | |
1166 void ViEEncoder::AdaptCounter::IncrementFramerate(int reason) { | |
1167 ++(fps_counters_[reason]); | |
1168 } | |
1169 | |
1170 void ViEEncoder::AdaptCounter::IncrementResolution(int reason) { | |
1171 ++(resolution_counters_[reason]); | |
1172 } | |
1173 | |
1174 void ViEEncoder::AdaptCounter::DecrementFramerate(int reason) { | |
1175 if (fps_counters_[reason] == 0) { | |
1176 // Balanced mode: Adapt up is in a different order, switch reason. | |
1177 // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). | |
1178 // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} | |
1179 // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} | |
1180 // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} | |
1181 // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0} | |
1182 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; | |
1183 RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; | |
1184 MoveCount(&resolution_counters_, reason); | |
1185 MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); | |
1186 } | |
1187 --(fps_counters_[reason]); | |
1188 RTC_DCHECK_GE(fps_counters_[reason], 0); | |
1189 } | |
1190 | |
1191 void ViEEncoder::AdaptCounter::DecrementResolution(int reason) { | |
1192 if (resolution_counters_[reason] == 0) { | |
1193 // Balanced mode: Adapt up is in a different order, switch reason. | |
1194 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; | |
1195 RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; | |
1196 MoveCount(&fps_counters_, reason); | |
1197 MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); | |
1198 } | |
1199 --(resolution_counters_[reason]); | |
1200 RTC_DCHECK_GE(resolution_counters_[reason], 0); | |
1201 } | |
1202 | |
1203 void ViEEncoder::AdaptCounter::DecrementFramerate(int reason, int cur_fps) { | |
1204 DecrementFramerate(reason); | |
1205 // Reset if at max fps (i.e. in case of fewer steps up than down). | |
1206 if (cur_fps == std::numeric_limits<int>::max()) | |
1207 std::fill(fps_counters_.begin(), fps_counters_.end(), 0); | |
1208 } | |
1209 | |
1210 int ViEEncoder::AdaptCounter::FramerateCount() const { | |
1211 return Count(fps_counters_); | |
1212 } | |
1213 | |
1214 int ViEEncoder::AdaptCounter::ResolutionCount() const { | |
1215 return Count(resolution_counters_); | |
1216 } | |
1217 | |
1218 int ViEEncoder::AdaptCounter::FramerateCount(int reason) const { | |
1219 return fps_counters_[reason]; | |
1220 } | |
1221 | |
1222 int ViEEncoder::AdaptCounter::ResolutionCount(int reason) const { | |
1223 return resolution_counters_[reason]; | |
1224 } | |
1225 | |
1226 int ViEEncoder::AdaptCounter::TotalCount(int reason) const { | |
1227 return FramerateCount(reason) + ResolutionCount(reason); | |
1228 } | |
1229 | |
1230 int ViEEncoder::AdaptCounter::Count(const std::vector<int>& counters) const { | |
1231 return std::accumulate(counters.begin(), counters.end(), 0); | |
1232 } | |
1233 | |
1234 void ViEEncoder::AdaptCounter::MoveCount(std::vector<int>* counters, | |
1235 int from_reason) { | |
1236 int to_reason = (from_reason + 1) % kScaleReasonSize; | |
1237 ++((*counters)[to_reason]); | |
1238 --((*counters)[from_reason]); | |
1239 } | |
1240 | |
1241 std::string ViEEncoder::AdaptCounter::ToString( | |
1242 const std::vector<int>& counters) const { | |
1243 std::stringstream ss; | |
1244 for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { | |
1245 ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; | |
1246 } | |
1247 return ss.str(); | |
1248 } | |
1249 | |
1250 } // namespace webrtc | |
OLD | NEW |