Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(205)

Side by Side Diff: webrtc/media/base/videoadapter.cc

Issue 1836043004: Cleanup the VideoAdapter (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master
Patch Set: Addressed nits Created 4 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « webrtc/media/base/videoadapter.h ('k') | webrtc/media/base/videoadapter_unittest.cc » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license 4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source 5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found 6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may 7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree. 8 * be found in the AUTHORS file in the root of the source tree.
9 */ 9 */
10 10
11 #include "webrtc/media/base/videoadapter.h" 11 #include "webrtc/media/base/videoadapter.h"
12 12
13 #include <limits.h> // For INT_MAX
14 #include <algorithm> 13 #include <algorithm>
14 #include <limits>
15 15
16 #include "webrtc/base/logging.h" 16 #include "webrtc/base/logging.h"
17 #include "webrtc/base/timeutils.h"
18 #include "webrtc/media/base/mediaconstants.h" 17 #include "webrtc/media/base/mediaconstants.h"
19 #include "webrtc/media/base/videocommon.h" 18 #include "webrtc/media/base/videocommon.h"
20 #include "webrtc/media/base/videoframe.h"
21 19
22 namespace cricket { 20 namespace {
23 21
24 // TODO(fbarchard): Make downgrades settable 22 // Scale factors optimized for in libYUV that we accept.
25 static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU. 23 // Must be sorted in decreasing scale factors for FindScaleLargerThan to work.
26 // The number of cpu samples to require before adapting. This value depends on 24 const float kScaleFactors[] = {
27 // the cpu monitor sampling frequency being 2000ms. 25 1.f / 1.f, // Full size.
28 static const int kCpuLoadMinSamples = 3; 26 3.f / 4.f, // 3/4 scale.
29 // The amount of weight to give to each new cpu load sample. The lower the 27 1.f / 2.f, // 1/2 scale.
30 // value, the slower we'll adapt to changing cpu conditions. 28 3.f / 8.f, // 3/8 scale.
31 static const float kCpuLoadWeightCoefficient = 0.4f; 29 1.f / 4.f, // 1/4 scale.
32 // The seed value for the cpu load moving average. 30 3.f / 16.f, // 3/16 scale.
33 static const float kCpuLoadInitialAverage = 0.5f;
34
35 // Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90)
36 static const float kScaleFactors[] = {
37 1.f / 1.f, // Full size.
38 3.f / 4.f, // 3/4 scale.
39 1.f / 2.f, // 1/2 scale.
40 3.f / 8.f, // 3/8 scale.
41 1.f / 4.f, // 1/4 scale.
42 3.f / 16.f, // 3/16 scale.
43 1.f / 8.f, // 1/8 scale.
44 0.f // End of table.
45 }; 31 };
46 32
47 // TODO(fbarchard): Use this table (optionally) for CPU and GD as well. 33 float FindScaleLessThanOrEqual(int width,
48 static const float kViewScaleFactors[] = { 34 int height,
49 1.f / 1.f, // Full size. 35 int target_num_pixels,
50 3.f / 4.f, // 3/4 scale. 36 int* resulting_number_of_pixels) {
51 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p. 37 float best_distance = std::numeric_limits<float>::max();
52 1.f / 2.f, // 1/2 scale. 38 float best_scale = 0.0f; // Default to 0 if nothing matches.
53 3.f / 8.f, // 3/8 scale. 39 float pixels = width * height;
54 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p. 40 float best_number_of_pixels = 0.0f;
55 1.f / 4.f, // 1/4 scale. 41 for (const auto& scale : kScaleFactors) {
56 3.f / 16.f, // 3/16 scale.
57 1.f / 8.f, // 1/8 scale.
58 0.f // End of table.
59 };
60
61 const float* VideoAdapter::GetViewScaleFactors() const {
62 return scale_third_ ? kViewScaleFactors : kScaleFactors;
63 }
64
65 // For resolutions that would scale down a little instead of up a little,
66 // bias toward scaling up a little. This will tend to choose 3/4 scale instead
67 // of 2/3 scale, when the 2/3 is not an exact match.
68 static const float kUpBias = -0.9f;
69 // Find the scale factor that, when applied to width and height, is closest
70 // to num_pixels.
71 float VideoAdapter::FindScale(const float* scale_factors,
72 const float upbias,
73 int width, int height,
74 int target_num_pixels) {
75 const float kMinNumPixels = 160 * 90;
76 if (!target_num_pixels) {
77 return 0.f;
78 }
79 float best_distance = static_cast<float>(INT_MAX);
80 float best_scale = 1.f; // Default to unscaled if nothing matches.
81 float pixels = static_cast<float>(width * height);
82 for (int i = 0; ; ++i) {
83 float scale = scale_factors[i];
84 float test_num_pixels = pixels * scale * scale; 42 float test_num_pixels = pixels * scale * scale;
85 // Do not consider scale factors that produce too small images.
86 // Scale factor of 0 at end of table will also exit here.
87 if (test_num_pixels < kMinNumPixels) {
88 break;
89 }
90 float diff = target_num_pixels - test_num_pixels; 43 float diff = target_num_pixels - test_num_pixels;
91 // If resolution is higher than desired, bias the difference based on
92 // preference for slightly larger for nearest, or avoid completely if
93 // looking for lower resolutions only.
94 if (diff < 0) { 44 if (diff < 0) {
95 diff = diff * kUpBias; 45 continue;
96 } 46 }
97 if (diff < best_distance) { 47 if (diff < best_distance) {
98 best_distance = diff; 48 best_distance = diff;
99 best_scale = scale; 49 best_scale = scale;
50 best_number_of_pixels = test_num_pixels;
100 if (best_distance == 0) { // Found exact match. 51 if (best_distance == 0) { // Found exact match.
101 break; 52 break;
102 } 53 }
103 } 54 }
104 } 55 }
56 if (resulting_number_of_pixels) {
57 *resulting_number_of_pixels = static_cast<int>(best_number_of_pixels + .5f);
58 }
105 return best_scale; 59 return best_scale;
106 } 60 }
107 61
108 // Find the closest scale factor. 62 float FindScaleLargerThan(int width,
109 float VideoAdapter::FindClosestScale(int width, int height, 63 int height,
110 int target_num_pixels) { 64 int target_num_pixels,
111 return FindScale(kScaleFactors, kUpBias, 65 int* resulting_number_of_pixels) {
112 width, height, target_num_pixels); 66 float best_distance = std::numeric_limits<float>::max();
67 float best_scale = 1.f; // Default to unscaled if nothing matches.
68 float pixels = width * height;
69 float best_number_of_pixels = pixels; // Default to input number of pixels.
70 for (const auto& scale : kScaleFactors) {
71 float test_num_pixels = pixels * scale * scale;
72 float diff = test_num_pixels - target_num_pixels;
73 if (diff <= 0) {
74 break;
75 }
76 if (diff < best_distance) {
77 best_distance = diff;
78 best_scale = scale;
79 best_number_of_pixels = test_num_pixels;
80 }
81 }
82
83 *resulting_number_of_pixels = static_cast<int>(best_number_of_pixels + .5f);
84 return best_scale;
113 } 85 }
114 86
115 // Find the closest view scale factor. 87 } // namespace
116 float VideoAdapter::FindClosestViewScale(int width, int height,
117 int target_num_pixels) {
118 return FindScale(GetViewScaleFactors(), kUpBias,
119 width, height, target_num_pixels);
120 }
121 88
122 // Finds the scale factor that, when applied to width and height, produces 89 namespace cricket {
123 // fewer than num_pixels.
124 static const float kUpAvoidBias = -1000000000.f;
125 float VideoAdapter::FindLowerScale(int width, int height,
126 int target_num_pixels) {
127 return FindScale(GetViewScaleFactors(), kUpAvoidBias,
128 width, height, target_num_pixels);
129 }
130 90
131 // There are several frame sizes used by Adapter. This explains them
132 // input_format - set once by server to frame size expected from the camera.
133 // The input frame size is also updated in AdaptFrameResolution.
134 // output_format - size that output would like to be. Includes framerate.
135 // The output frame size is also updated in AdaptFrameResolution.
136 // output_num_pixels - size that output should be constrained to. Used to
137 // compute output_format from in_frame.
138 // in_frame - actual camera captured frame size, which is typically the same
139 // as input_format. This can also be rotated or cropped for aspect ratio.
140 // out_frame - actual frame output by adapter. Should be a direct scale of
141 // in_frame maintaining rotation and aspect ratio.
142 // OnOutputFormatRequest - server requests you send this resolution based on
143 // view requests.
144 // OnEncoderResolutionRequest - encoder requests you send this resolution based
145 // on bandwidth
146 // OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
147 // cpu load.
148
149 ///////////////////////////////////////////////////////////////////////
150 // Implementation of VideoAdapter
151 VideoAdapter::VideoAdapter() 91 VideoAdapter::VideoAdapter()
152 : output_num_pixels_(INT_MAX), 92 : output_num_pixels_(std::numeric_limits<int>::max()),
153 scale_third_(false),
154 frames_in_(0), 93 frames_in_(0),
155 frames_out_(0), 94 frames_out_(0),
156 frames_scaled_(0), 95 frames_scaled_(0),
157 adaption_changes_(0), 96 adaption_changes_(0),
158 previous_width_(0), 97 previous_width_(0),
159 previous_height_(0), 98 previous_height_(0),
160 interval_next_frame_(0) { 99 interval_next_frame_(0),
161 } 100 format_request_max_pixel_count_(std::numeric_limits<int>::max()),
101 resolution_request_max_pixel_count_(std::numeric_limits<int>::max()) {}
162 102
163 VideoAdapter::~VideoAdapter() { 103 VideoAdapter::~VideoAdapter() {}
104
105 void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) {
106 // TODO(perkj): Consider measuring input frame rate instead.
107 // Frame rate typically varies depending on lighting.
108 rtc::CritScope cs(&critical_section_);
109 input_format_.interval = interval;
164 } 110 }
165 111
166 void VideoAdapter::SetInputFormat(const VideoFormat& format) { 112 void VideoAdapter::SetInputFormat(const VideoFormat& format) {
167 rtc::CritScope cs(&critical_section_); 113 bool is_resolution_change = (input_format().width != format.width ||
114 input_format().height != format.height);
168 int64_t old_input_interval = input_format_.interval; 115 int64_t old_input_interval = input_format_.interval;
169 input_format_ = format; 116 input_format_ = format;
170 output_format_.interval = 117 output_format_.interval =
171 std::max(output_format_.interval, input_format_.interval); 118 std::max(output_format_.interval, input_format_.interval);
172 if (old_input_interval != input_format_.interval) { 119 if (old_input_interval != input_format_.interval) {
173 LOG(LS_INFO) << "VAdapt input interval changed from " 120 LOG(LS_INFO) << "VAdapt input interval changed from "
174 << old_input_interval << " to " << input_format_.interval; 121 << old_input_interval << " to " << input_format_.interval;
175 } 122 }
176 }
177
178 void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) {
179 int previous_width = input_format().width;
180 int previous_height = input_format().height;
181 bool is_resolution_change = previous_width > 0 && format.width > 0 &&
182 (previous_width != format.width ||
183 previous_height != format.height);
184 VideoAdapter::SetInputFormat(format);
185 if (is_resolution_change) { 123 if (is_resolution_change) {
186 int width, height;
187 // Trigger the adaptation logic again, to potentially reset the adaptation 124 // Trigger the adaptation logic again, to potentially reset the adaptation
188 // state for things like view requests that may not longer be capping 125 // state for things like view requests that may not longer be capping
189 // output (or may now cap output). 126 // output (or may now cap output).
190 AdaptToMinimumFormat(&width, &height); 127 Adapt(std::min(format_request_max_pixel_count_,
191 LOG(LS_INFO) << "VAdapt Input Resolution Change: " 128 resolution_request_max_pixel_count_),
192 << "Previous input resolution: " 129 0);
193 << previous_width << "x" << previous_height
194 << " New input resolution: "
195 << format.width << "x" << format.height
196 << " New output resolution: "
197 << width << "x" << height;
198 } 130 }
199 } 131 }
200 132
201 void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) { 133 const VideoFormat& VideoAdapter::input_format() const {
202 LOG(LS_INFO) << "CPU smoothing is now "
203 << (enable ? "enabled" : "disabled");
204 cpu_smoothing_ = enable;
205 }
206
207 void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
208 rtc::CritScope cs(&critical_section_);
209 int64_t old_output_interval = output_format_.interval;
210 output_format_ = format;
211 output_num_pixels_ = output_format_.width * output_format_.height;
212 output_format_.interval =
213 std::max(output_format_.interval, input_format_.interval);
214 if (old_output_interval != output_format_.interval) {
215 LOG(LS_INFO) << "VAdapt output interval changed from "
216 << old_output_interval << " to " << output_format_.interval;
217 }
218 }
219
220 const VideoFormat& VideoAdapter::input_format() {
221 rtc::CritScope cs(&critical_section_); 134 rtc::CritScope cs(&critical_section_);
222 return input_format_; 135 return input_format_;
223 } 136 }
224 137
225 bool VideoAdapter::drops_all_frames() const {
226 return output_num_pixels_ == 0;
227 }
228
229 const VideoFormat& VideoAdapter::output_format() {
230 rtc::CritScope cs(&critical_section_);
231 return output_format_;
232 }
233
234 // Constrain output resolution to this many pixels overall
235 void VideoAdapter::SetOutputNumPixels(int num_pixels) {
236 output_num_pixels_ = num_pixels;
237 }
238
239 int VideoAdapter::GetOutputNumPixels() const {
240 return output_num_pixels_;
241 }
242
243 VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { 138 VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) {
244 rtc::CritScope cs(&critical_section_); 139 rtc::CritScope cs(&critical_section_);
245 ++frames_in_; 140 ++frames_in_;
246 141
247 SetInputFormat(VideoFormat( 142 SetInputFormat(VideoFormat(
248 in_width, in_height, input_format_.interval, input_format_.fourcc)); 143 in_width, in_height, input_format_.interval, input_format_.fourcc));
249 144
250 // Drop the input frame if necessary. 145 // Drop the input frame if necessary.
251 bool should_drop = false; 146 bool should_drop = false;
252 if (!output_num_pixels_) { 147 if (!output_num_pixels_) {
253 // Drop all frames as the output format is 0x0. 148 // Drop all frames as the output format is 0x0.
254 should_drop = true; 149 should_drop = true;
255 } else { 150 } else {
256 // Drop some frames based on input fps and output fps. 151 // Drop some frames based on input fps and output fps.
257 // Normally output fps is less than input fps. 152 // Normally output fps is less than input fps.
258 // TODO(fbarchard): Consider adjusting interval to reflect the adjusted
259 // interval between frames after dropping some frames.
260 interval_next_frame_ += input_format_.interval; 153 interval_next_frame_ += input_format_.interval;
261 if (output_format_.interval > 0) { 154 if (output_format_.interval > 0) {
262 if (interval_next_frame_ >= output_format_.interval) { 155 if (interval_next_frame_ >= output_format_.interval) {
263 interval_next_frame_ %= output_format_.interval; 156 interval_next_frame_ %= output_format_.interval;
264 } else { 157 } else {
265 should_drop = true; 158 should_drop = true;
266 } 159 }
267 } 160 }
268 } 161 }
269 if (should_drop) { 162 if (should_drop) {
270 // Show VAdapt log every 90 frames dropped. (3 seconds) 163 // Show VAdapt log every 90 frames dropped. (3 seconds)
271 if ((frames_in_ - frames_out_) % 90 == 0) { 164 if ((frames_in_ - frames_out_) % 90 == 0) {
272 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed 165 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
273 // in default calls. 166 // in default calls.
274 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_ 167 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
275 << " / out " << frames_out_ 168 << " / out " << frames_out_
276 << " / in " << frames_in_ 169 << " / in " << frames_in_
277 << " Changes: " << adaption_changes_ 170 << " Changes: " << adaption_changes_
278 << " Input: " << in_width 171 << " Input: " << in_width
279 << "x" << in_height 172 << "x" << in_height
280 << " i" << input_format_.interval 173 << " i" << input_format_.interval
281 << " Output: i" << output_format_.interval; 174 << " Output: i" << output_format_.interval;
282 } 175 }
283 176
284 return VideoFormat(); // Drop frame. 177 return VideoFormat(); // Drop frame.
285 } 178 }
286 179
287 const float scale = VideoAdapter::FindClosestViewScale( 180 const float scale = FindScaleLessThanOrEqual(in_width, in_height,
288 in_width, in_height, output_num_pixels_); 181 output_num_pixels_, nullptr);
289 const size_t output_width = static_cast<size_t>(in_width * scale + .5f); 182 const int output_width = static_cast<int>(in_width * scale + .5f);
290 const size_t output_height = static_cast<size_t>(in_height * scale + .5f); 183 const int output_height = static_cast<int>(in_height * scale + .5f);
291 184
292 ++frames_out_; 185 ++frames_out_;
293 if (scale != 1) 186 if (scale != 1)
294 ++frames_scaled_; 187 ++frames_scaled_;
295 // Show VAdapt log every 90 frames output. (3 seconds)
296 // TODO(fbarchard): Consider GetLogSeverity() to change interval to less
297 // for LS_VERBOSE and more for LS_INFO.
298 bool show = (frames_out_) % 90 == 0;
299 188
300 // TODO(fbarchard): LOG the previous output resolution and track input
301 // resolution changes as well. Consider dropping the statistics into their
302 // own class which could be queried publically.
303 bool changed = false;
304 if (previous_width_ && (previous_width_ != output_width || 189 if (previous_width_ && (previous_width_ != output_width ||
305 previous_height_ != output_height)) { 190 previous_height_ != output_height)) {
306 show = true;
307 ++adaption_changes_; 191 ++adaption_changes_;
308 changed = true; 192 LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out "
309 } 193 << frames_out_ << " / in " << frames_in_
310 if (show) { 194 << " Changes: " << adaption_changes_ << " Input: " << in_width
311 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed 195 << "x" << in_height << " i" << input_format_.interval
312 // in default calls. 196 << " Scale: " << scale << " Output: " << output_width << "x"
313 LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_ 197 << output_height << " i" << output_format_.interval;
314 << " / out " << frames_out_
315 << " / in " << frames_in_
316 << " Changes: " << adaption_changes_
317 << " Input: " << in_width
318 << "x" << in_height
319 << " i" << input_format_.interval
320 << " Scale: " << scale
321 << " Output: " << output_width
322 << "x" << output_height
323 << " i" << output_format_.interval
324 << " Changed: " << (changed ? "true" : "false");
325 } 198 }
326 199
327 output_format_.width = output_width; 200 output_format_.width = output_width;
328 output_format_.height = output_height; 201 output_format_.height = output_height;
329 previous_width_ = output_width; 202 previous_width_ = output_width;
330 previous_height_ = output_height; 203 previous_height_ = output_height;
331 204
332 return output_format_; 205 return output_format_;
333 } 206 }
334 207
335 void VideoAdapter::set_scale_third(bool enable) { 208 void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
336 LOG(LS_INFO) << "Video Adapter third scaling is now " 209 rtc::CritScope cs(&critical_section_);
337 << (enable ? "enabled" : "disabled"); 210 format_request_max_pixel_count_ = format.width * format.height;
338 scale_third_ = enable; 211 output_format_.interval = format.interval;
212 Adapt(std::min(format_request_max_pixel_count_,
213 resolution_request_max_pixel_count_),
214 0);
339 } 215 }
340 216
341 /////////////////////////////////////////////////////////////////////// 217 void VideoAdapter::OnResolutionRequest(
342 // Implementation of CoordinatedVideoAdapter 218 rtc::Optional<int> max_pixel_count,
343 CoordinatedVideoAdapter::CoordinatedVideoAdapter() 219 rtc::Optional<int> max_pixel_count_step_up) {
344 : cpu_adaptation_(true), 220 rtc::CritScope cs(&critical_section_);
345 cpu_smoothing_(false), 221 resolution_request_max_pixel_count_ =
346 gd_adaptation_(true), 222 max_pixel_count.value_or(std::numeric_limits<int>::max());
347 view_adaptation_(true), 223 Adapt(std::min(format_request_max_pixel_count_,
348 view_switch_(false), 224 resolution_request_max_pixel_count_),
349 cpu_downgrade_count_(0), 225 max_pixel_count_step_up.value_or(0));
350 cpu_load_min_samples_(kCpuLoadMinSamples),
351 cpu_load_num_samples_(0),
352 high_system_threshold_(kHighSystemCpuThreshold),
353 low_system_threshold_(kLowSystemCpuThreshold),
354 process_threshold_(kProcessCpuThreshold),
355 view_desired_num_pixels_(INT_MAX),
356 view_desired_interval_(0),
357 encoder_desired_num_pixels_(INT_MAX),
358 cpu_desired_num_pixels_(INT_MAX),
359 adapt_reason_(ADAPTREASON_NONE),
360 system_load_average_(kCpuLoadInitialAverage) {
361 } 226 }
362 227
363 // Helper function to UPGRADE or DOWNGRADE a number of pixels 228 bool VideoAdapter::Adapt(int max_num_pixels, int max_pixel_count_step_up) {
364 void CoordinatedVideoAdapter::StepPixelCount( 229 float scale_lower =
365 CoordinatedVideoAdapter::AdaptRequest request, 230 FindScaleLessThanOrEqual(input_format_.width, input_format_.height,
366 int* num_pixels) { 231 max_num_pixels, &max_num_pixels);
367 switch (request) { 232 float scale_upper =
368 case CoordinatedVideoAdapter::DOWNGRADE: 233 max_pixel_count_step_up > 0
369 *num_pixels /= 2; 234 ? FindScaleLargerThan(input_format_.width, input_format_.height,
370 break; 235 max_pixel_count_step_up,
236 &max_pixel_count_step_up)
237 : 1.f;
371 238
372 case CoordinatedVideoAdapter::UPGRADE: 239 bool use_max_pixel_count_step_up =
373 *num_pixels *= 2; 240 max_pixel_count_step_up > 0 && max_num_pixels > max_pixel_count_step_up;
374 break;
375 241
376 default: // No change in pixel count 242 int old_num_pixels = output_num_pixels_;
377 break; 243 output_num_pixels_ =
378 } 244 use_max_pixel_count_step_up ? max_pixel_count_step_up : max_num_pixels;
379 return; 245 // Log the new size.
380 } 246 float scale = use_max_pixel_count_step_up ? scale_upper : scale_lower;
247 int new_width = static_cast<int>(input_format_.width * scale + .5f);
248 int new_height = static_cast<int>(input_format_.height * scale + .5f);
381 249
382 // Find the adaptation request of the cpu based on the load. Return UPGRADE if 250 bool changed = output_num_pixels_ != old_num_pixels;
383 // the load is low, DOWNGRADE if the load is high, and KEEP otherwise. 251 LOG(LS_INFO) << "OnResolutionRequest: "
384 CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest( 252 << " Max pixels: " << max_num_pixels
385 int current_cpus, int max_cpus, 253 << " Max pixels step up: " << max_pixel_count_step_up
386 float process_load, float system_load) { 254 << " Output Pixels: " << output_num_pixels_
387 // Downgrade if system is high and plugin is at least more than midrange. 255 << " Input: " << input_format_.width << "x"
388 if (system_load >= high_system_threshold_ * max_cpus && 256 << input_format_.height << " Scale: " << scale
389 process_load >= process_threshold_ * current_cpus) { 257 << " Resolution: " << new_width << "x" << new_height
390 return CoordinatedVideoAdapter::DOWNGRADE; 258 << " Changed: " << (changed ? "true" : "false");
391 // Upgrade if system is low.
392 } else if (system_load < low_system_threshold_ * max_cpus) {
393 return CoordinatedVideoAdapter::UPGRADE;
394 }
395 return CoordinatedVideoAdapter::KEEP;
396 }
397
398 // A remote view request for a new resolution.
399 void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
400 rtc::CritScope cs(&request_critical_section_);
401 if (!view_adaptation_) {
402 return;
403 }
404 // Set output for initial aspect ratio in mediachannel unittests.
405 int old_num_pixels = GetOutputNumPixels();
406 SetOutputFormat(format);
407 SetOutputNumPixels(old_num_pixels);
408 view_desired_num_pixels_ = format.width * format.height;
409 view_desired_interval_ = format.interval;
410 int new_width, new_height;
411 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
412 LOG(LS_INFO) << "VAdapt View Request: "
413 << format.width << "x" << format.height
414 << " Pixels: " << view_desired_num_pixels_
415 << " Changed: " << (changed ? "true" : "false")
416 << " To: " << new_width << "x" << new_height;
417 }
418
419 void CoordinatedVideoAdapter::set_cpu_load_min_samples(
420 int cpu_load_min_samples) {
421 if (cpu_load_min_samples_ != cpu_load_min_samples) {
422 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: "
423 << cpu_load_min_samples_ << " to "
424 << cpu_load_min_samples;
425 cpu_load_min_samples_ = cpu_load_min_samples;
426 }
427 }
428
429 void CoordinatedVideoAdapter::set_high_system_threshold(
430 float high_system_threshold) {
431 ASSERT(high_system_threshold <= 1.0f);
432 ASSERT(high_system_threshold >= 0.0f);
433 if (high_system_threshold_ != high_system_threshold) {
434 LOG(LS_INFO) << "VAdapt Change High System Threshold from: "
435 << high_system_threshold_ << " to " << high_system_threshold;
436 high_system_threshold_ = high_system_threshold;
437 }
438 }
439
440 void CoordinatedVideoAdapter::set_low_system_threshold(
441 float low_system_threshold) {
442 ASSERT(low_system_threshold <= 1.0f);
443 ASSERT(low_system_threshold >= 0.0f);
444 if (low_system_threshold_ != low_system_threshold) {
445 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: "
446 << low_system_threshold_ << " to " << low_system_threshold;
447 low_system_threshold_ = low_system_threshold;
448 }
449 }
450
451 void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) {
452 ASSERT(process_threshold <= 1.0f);
453 ASSERT(process_threshold >= 0.0f);
454 if (process_threshold_ != process_threshold) {
455 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: "
456 << process_threshold_ << " to " << process_threshold;
457 process_threshold_ = process_threshold;
458 }
459 }
460
461 // A Bandwidth GD request for new resolution
462 void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
463 int width, int height, AdaptRequest request) {
464 rtc::CritScope cs(&request_critical_section_);
465 if (!gd_adaptation_) {
466 return;
467 }
468 int old_encoder_desired_num_pixels = encoder_desired_num_pixels_;
469 if (KEEP != request) {
470 int new_encoder_desired_num_pixels = width * height;
471 int old_num_pixels = GetOutputNumPixels();
472 if (new_encoder_desired_num_pixels != old_num_pixels) {
473 LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored";
474 } else {
475 // Update the encoder desired format based on the request.
476 encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
477 StepPixelCount(request, &encoder_desired_num_pixels_);
478 }
479 }
480 int new_width, new_height;
481 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
482
483 // Ignore up or keep if no change.
484 if (DOWNGRADE != request && view_switch_ && !changed) {
485 encoder_desired_num_pixels_ = old_encoder_desired_num_pixels;
486 LOG(LS_VERBOSE) << "VAdapt ignoring GD request.";
487 }
488
489 LOG(LS_INFO) << "VAdapt GD Request: "
490 << (DOWNGRADE == request ? "down" :
491 (UPGRADE == request ? "up" : "keep"))
492 << " From: " << width << "x" << height
493 << " Pixels: " << encoder_desired_num_pixels_
494 << " Changed: " << (changed ? "true" : "false")
495 << " To: " << new_width << "x" << new_height;
496 }
497
498 void CoordinatedVideoAdapter::OnCpuResolutionRequest(
499 rtc::Optional<int> max_pixel_count,
500 rtc::Optional<int> max_pixel_count_step_up) {
501 rtc::CritScope cs(&request_critical_section_);
502 // TODO(perkj): We should support taking larger steps up and down and
503 // actually look at the values set in max_pixel_count and
504 // max_pixel_count_step_up.
505 if (max_pixel_count && *max_pixel_count < GetOutputNumPixels()) {
506 OnCpuResolutionRequest(DOWNGRADE);
507 } else if (max_pixel_count_step_up &&
508 *max_pixel_count_step_up >= GetOutputNumPixels()) {
509 OnCpuResolutionRequest(UPGRADE);
510 }
511 }
512
513 // A Bandwidth GD request for new resolution
514 void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) {
515 rtc::CritScope cs(&request_critical_section_);
516 if (!cpu_adaptation_) {
517 return;
518 }
519
520 // Update how many times we have downgraded due to the cpu load.
521 switch (request) {
522 case DOWNGRADE:
523 // Ignore downgrades if we have downgraded the maximum times.
524 if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
525 ++cpu_downgrade_count_;
526 } else {
527 LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
528 "because maximum downgrades reached";
529 SignalCpuAdaptationUnable();
530 }
531 break;
532 case UPGRADE:
533 if (cpu_downgrade_count_ > 0) {
534 bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
535 if (is_min) {
536 --cpu_downgrade_count_;
537 } else {
538 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
539 "because cpu is not limiting resolution";
540 }
541 } else {
542 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
543 "because minimum downgrades reached";
544 }
545 break;
546 case KEEP:
547 default:
548 break;
549 }
550 if (KEEP != request) {
551 // TODO(fbarchard): compute stepping up/down from OutputNumPixels but
552 // clamp to inputpixels / 4 (2 steps)
553 cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX :
554 static_cast<int>(input_format().width * input_format().height >>
555 cpu_downgrade_count_);
556 }
557 int new_width, new_height;
558 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
559 LOG(LS_INFO) << "VAdapt CPU Request: "
560 << (DOWNGRADE == request ? "down" :
561 (UPGRADE == request ? "up" : "keep"))
562 << " Steps: " << cpu_downgrade_count_
563 << " Changed: " << (changed ? "true" : "false")
564 << " To: " << new_width << "x" << new_height;
565 }
566
567 // A CPU request for new resolution
568 // TODO(fbarchard): Move outside adapter.
569 void CoordinatedVideoAdapter::OnCpuLoadUpdated(
570 int current_cpus, int max_cpus, float process_load, float system_load) {
571 rtc::CritScope cs(&request_critical_section_);
572 if (!cpu_adaptation_) {
573 return;
574 }
575 // Update the moving average of system load. Even if we aren't smoothing,
576 // we'll still calculate this information, in case smoothing is later enabled.
577 system_load_average_ = kCpuLoadWeightCoefficient * system_load +
578 (1.0f - kCpuLoadWeightCoefficient) * system_load_average_;
579 ++cpu_load_num_samples_;
580 if (cpu_smoothing_) {
581 system_load = system_load_average_;
582 }
583 AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
584 process_load, system_load);
585 // Make sure we're not adapting too quickly.
586 if (request != KEEP) {
587 if (cpu_load_num_samples_ < cpu_load_min_samples_) {
588 LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until "
589 << (cpu_load_min_samples_ - cpu_load_num_samples_)
590 << " more samples";
591 request = KEEP;
592 }
593 }
594
595 OnCpuResolutionRequest(request);
596 }
597
598 // Called by cpu adapter on up requests.
599 bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
600 // Find closest scale factor that matches input resolution to min_num_pixels
601 // and set that for output resolution. This is not needed for VideoAdapter,
602 // but provides feedback to unittests and users on expected resolution.
603 // Actual resolution is based on input frame.
604 VideoFormat new_output = output_format();
605 VideoFormat input = input_format();
606 if (input_format().IsSize0x0()) {
607 input = new_output;
608 }
609 float scale = 1.0f;
610 if (!input.IsSize0x0()) {
611 scale = FindClosestScale(input.width,
612 input.height,
613 pixels);
614 }
615 new_output.width = static_cast<int>(input.width * scale + .5f);
616 new_output.height = static_cast<int>(input.height * scale + .5f);
617 int new_pixels = new_output.width * new_output.height;
618 int num_pixels = GetOutputNumPixels();
619 return new_pixels <= num_pixels;
620 }
621
622 // Called by all coordinators when there is a change.
623 bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width,
624 int* new_height) {
625 VideoFormat new_output = output_format();
626 VideoFormat input = input_format();
627 if (input_format().IsSize0x0()) {
628 input = new_output;
629 }
630 int old_num_pixels = GetOutputNumPixels();
631 int min_num_pixels = INT_MAX;
632 adapt_reason_ = ADAPTREASON_NONE;
633
634 // Reduce resolution based on encoder bandwidth (GD).
635 if (encoder_desired_num_pixels_ &&
636 (encoder_desired_num_pixels_ < min_num_pixels)) {
637 adapt_reason_ |= ADAPTREASON_BANDWIDTH;
638 min_num_pixels = encoder_desired_num_pixels_;
639 }
640 // Reduce resolution based on CPU.
641 if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
642 (cpu_desired_num_pixels_ <= min_num_pixels)) {
643 if (cpu_desired_num_pixels_ < min_num_pixels) {
644 adapt_reason_ = ADAPTREASON_CPU;
645 } else {
646 adapt_reason_ |= ADAPTREASON_CPU;
647 }
648 min_num_pixels = cpu_desired_num_pixels_;
649 }
650 // Round resolution for GD or CPU to allow 1/2 to map to 9/16.
651 if (!input.IsSize0x0() && min_num_pixels != INT_MAX) {
652 float scale = FindClosestScale(input.width, input.height, min_num_pixels);
653 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
654 static_cast<int>(input.height * scale + .5f);
655 }
656 // Reduce resolution based on View Request.
657 if (view_desired_num_pixels_ <= min_num_pixels) {
658 if (view_desired_num_pixels_ < min_num_pixels) {
659 adapt_reason_ = ADAPTREASON_VIEW;
660 } else {
661 adapt_reason_ |= ADAPTREASON_VIEW;
662 }
663 min_num_pixels = view_desired_num_pixels_;
664 }
665 // Snap to a scale factor.
666 float scale = 1.0f;
667 if (!input.IsSize0x0()) {
668 scale = FindLowerScale(input.width, input.height, min_num_pixels);
669 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
670 static_cast<int>(input.height * scale + .5f);
671 }
672 if (scale == 1.0f) {
673 adapt_reason_ = ADAPTREASON_NONE;
674 }
675 *new_width = new_output.width = static_cast<int>(input.width * scale + .5f);
676 *new_height = new_output.height = static_cast<int>(input.height * scale +
677 .5f);
678 SetOutputNumPixels(min_num_pixels);
679
680 new_output.interval = view_desired_interval_;
681 SetOutputFormat(new_output);
682 int new_num_pixels = GetOutputNumPixels();
683 bool changed = new_num_pixels != old_num_pixels;
684
685 static const char* kReasons[8] = {
686 "None",
687 "CPU",
688 "BANDWIDTH",
689 "CPU+BANDWIDTH",
690 "VIEW",
691 "CPU+VIEW",
692 "BANDWIDTH+VIEW",
693 "CPU+BANDWIDTH+VIEW",
694 };
695
696 LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
697 << " GD: " << encoder_desired_num_pixels_
698 << " CPU: " << cpu_desired_num_pixels_
699 << " Pixels: " << min_num_pixels
700 << " Input: " << input.width
701 << "x" << input.height
702 << " Scale: " << scale
703 << " Resolution: " << new_output.width
704 << "x" << new_output.height
705 << " Changed: " << (changed ? "true" : "false")
706 << " Reason: " << kReasons[adapt_reason_];
707
708 if (changed) {
709 // When any adaptation occurs, historic CPU load levels are no longer
710 // accurate. Clear out our state so we can re-learn at the new normal.
711 cpu_load_num_samples_ = 0;
712 system_load_average_ = kCpuLoadInitialAverage;
713 }
714 259
715 return changed; 260 return changed;
716 } 261 }
717 262
718 } // namespace cricket 263 } // namespace cricket
OLDNEW
« no previous file with comments | « webrtc/media/base/videoadapter.h ('k') | webrtc/media/base/videoadapter_unittest.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698