OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/media/base/videoadapter.h" | 11 #include "webrtc/media/base/videoadapter.h" |
12 | 12 |
13 #include <limits.h> // For INT_MAX | |
14 #include <algorithm> | 13 #include <algorithm> |
14 #include <limits> | |
15 | 15 |
16 #include "webrtc/base/logging.h" | 16 #include "webrtc/base/logging.h" |
17 #include "webrtc/base/timeutils.h" | |
18 #include "webrtc/media/base/mediaconstants.h" | 17 #include "webrtc/media/base/mediaconstants.h" |
19 #include "webrtc/media/base/videocommon.h" | 18 #include "webrtc/media/base/videocommon.h" |
20 #include "webrtc/media/base/videoframe.h" | |
21 | 19 |
22 namespace cricket { | 20 namespace { |
23 | 21 |
24 // TODO(fbarchard): Make downgrades settable | 22 // Scale factors optimized for in libYUV that we accept. |
25 static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU. | 23 // Must be sorted in decreasing scale factors for FindScaleLargerThan to work. |
26 // The number of cpu samples to require before adapting. This value depends on | 24 const float kScaleFactors[] = { |
27 // the cpu monitor sampling frequency being 2000ms. | 25 1.f / 1.f, // Full size. |
28 static const int kCpuLoadMinSamples = 3; | 26 3.f / 4.f, // 3/4 scale. |
29 // The amount of weight to give to each new cpu load sample. The lower the | 27 1.f / 2.f, // 1/2 scale. |
30 // value, the slower we'll adapt to changing cpu conditions. | 28 3.f / 8.f, // 3/8 scale. |
31 static const float kCpuLoadWeightCoefficient = 0.4f; | 29 1.f / 4.f, // 1/4 scale. |
32 // The seed value for the cpu load moving average. | 30 3.f / 16.f, // 3/16 scale. |
33 static const float kCpuLoadInitialAverage = 0.5f; | |
34 | |
35 // Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90) | |
36 static const float kScaleFactors[] = { | |
37 1.f / 1.f, // Full size. | |
38 3.f / 4.f, // 3/4 scale. | |
39 1.f / 2.f, // 1/2 scale. | |
40 3.f / 8.f, // 3/8 scale. | |
41 1.f / 4.f, // 1/4 scale. | |
42 3.f / 16.f, // 3/16 scale. | |
43 1.f / 8.f, // 1/8 scale. | |
44 0.f // End of table. | |
45 }; | 31 }; |
46 | 32 |
47 // TODO(fbarchard): Use this table (optionally) for CPU and GD as well. | 33 float FindScaleLessThanOrEqual(int width, |
48 static const float kViewScaleFactors[] = { | 34 int height, |
49 1.f / 1.f, // Full size. | 35 int target_num_pixels, |
50 3.f / 4.f, // 3/4 scale. | 36 int* resulting_number_of_pixels) { |
51 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p. | 37 float best_distance = std::numeric_limits<float>::max(); |
52 1.f / 2.f, // 1/2 scale. | 38 float best_scale = 0.0f; // Default to 0 if nothing matches. |
53 3.f / 8.f, // 3/8 scale. | 39 float pixels = width * height; |
54 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p. | 40 float best_number_of_pixels = 0.0f; |
55 1.f / 4.f, // 1/4 scale. | 41 for (const auto& scale : kScaleFactors) { |
56 3.f / 16.f, // 3/16 scale. | |
57 1.f / 8.f, // 1/8 scale. | |
58 0.f // End of table. | |
59 }; | |
60 | |
61 const float* VideoAdapter::GetViewScaleFactors() const { | |
62 return scale_third_ ? kViewScaleFactors : kScaleFactors; | |
63 } | |
64 | |
65 // For resolutions that would scale down a little instead of up a little, | |
66 // bias toward scaling up a little. This will tend to choose 3/4 scale instead | |
67 // of 2/3 scale, when the 2/3 is not an exact match. | |
68 static const float kUpBias = -0.9f; | |
69 // Find the scale factor that, when applied to width and height, is closest | |
70 // to num_pixels. | |
71 float VideoAdapter::FindScale(const float* scale_factors, | |
72 const float upbias, | |
73 int width, int height, | |
74 int target_num_pixels) { | |
75 const float kMinNumPixels = 160 * 90; | |
76 if (!target_num_pixels) { | |
77 return 0.f; | |
78 } | |
79 float best_distance = static_cast<float>(INT_MAX); | |
80 float best_scale = 1.f; // Default to unscaled if nothing matches. | |
81 float pixels = static_cast<float>(width * height); | |
82 for (int i = 0; ; ++i) { | |
83 float scale = scale_factors[i]; | |
84 float test_num_pixels = pixels * scale * scale; | 42 float test_num_pixels = pixels * scale * scale; |
85 // Do not consider scale factors that produce too small images. | |
86 // Scale factor of 0 at end of table will also exit here. | |
87 if (test_num_pixels < kMinNumPixels) { | |
88 break; | |
89 } | |
90 float diff = target_num_pixels - test_num_pixels; | 43 float diff = target_num_pixels - test_num_pixels; |
91 // If resolution is higher than desired, bias the difference based on | |
92 // preference for slightly larger for nearest, or avoid completely if | |
93 // looking for lower resolutions only. | |
94 if (diff < 0) { | 44 if (diff < 0) { |
95 diff = diff * kUpBias; | 45 continue; |
96 } | 46 } |
97 if (diff < best_distance) { | 47 if (diff < best_distance) { |
98 best_distance = diff; | 48 best_distance = diff; |
99 best_scale = scale; | 49 best_scale = scale; |
50 best_number_of_pixels = test_num_pixels; | |
100 if (best_distance == 0) { // Found exact match. | 51 if (best_distance == 0) { // Found exact match. |
101 break; | 52 break; |
102 } | 53 } |
103 } | 54 } |
104 } | 55 } |
56 if (resulting_number_of_pixels) { | |
57 *resulting_number_of_pixels = static_cast<int>(best_number_of_pixels + .5f); | |
58 } | |
105 return best_scale; | 59 return best_scale; |
106 } | 60 } |
107 | 61 |
108 // Find the closest scale factor. | 62 float FindScaleLargerThan(int width, |
109 float VideoAdapter::FindClosestScale(int width, int height, | 63 int height, |
110 int target_num_pixels) { | 64 int target_num_pixels, |
111 return FindScale(kScaleFactors, kUpBias, | 65 int* resulting_number_of_pixels) { |
112 width, height, target_num_pixels); | 66 float best_distance = std::numeric_limits<float>::max(); |
67 float best_scale = 1.f; // Default to unscaled if nothing matches. | |
68 float pixels = width * height; | |
69 float best_number_of_pixels = pixels; // Default to input number of pixels. | |
70 for (const auto& scale : kScaleFactors) { | |
71 float test_num_pixels = pixels * scale * scale; | |
72 float diff = test_num_pixels - target_num_pixels; | |
73 if (diff <= 0) { | |
74 break; | |
75 } | |
76 if (diff < best_distance) { | |
77 best_distance = diff; | |
78 best_scale = scale; | |
79 best_number_of_pixels = test_num_pixels; | |
80 } | |
81 } | |
82 | |
83 *resulting_number_of_pixels = static_cast<int>(best_number_of_pixels + .5f); | |
84 return best_scale; | |
113 } | 85 } |
114 | 86 |
115 // Find the closest view scale factor. | 87 } // namespace |
116 float VideoAdapter::FindClosestViewScale(int width, int height, | |
117 int target_num_pixels) { | |
118 return FindScale(GetViewScaleFactors(), kUpBias, | |
119 width, height, target_num_pixels); | |
120 } | |
121 | 88 |
122 // Finds the scale factor that, when applied to width and height, produces | 89 namespace cricket { |
123 // fewer than num_pixels. | |
124 static const float kUpAvoidBias = -1000000000.f; | |
125 float VideoAdapter::FindLowerScale(int width, int height, | |
126 int target_num_pixels) { | |
127 return FindScale(GetViewScaleFactors(), kUpAvoidBias, | |
128 width, height, target_num_pixels); | |
129 } | |
130 | 90 |
131 // There are several frame sizes used by Adapter. This explains them | |
132 // input_format - set once by server to frame size expected from the camera. | |
133 // The input frame size is also updated in AdaptFrameResolution. | |
134 // output_format - size that output would like to be. Includes framerate. | |
135 // The output frame size is also updated in AdaptFrameResolution. | |
136 // output_num_pixels - size that output should be constrained to. Used to | |
137 // compute output_format from in_frame. | |
138 // in_frame - actual camera captured frame size, which is typically the same | |
139 // as input_format. This can also be rotated or cropped for aspect ratio. | |
140 // out_frame - actual frame output by adapter. Should be a direct scale of | |
141 // in_frame maintaining rotation and aspect ratio. | |
142 // OnOutputFormatRequest - server requests you send this resolution based on | |
143 // view requests. | |
144 // OnEncoderResolutionRequest - encoder requests you send this resolution based | |
145 // on bandwidth | |
146 // OnCpuLoadUpdated - cpu monitor requests you send this resolution based on | |
147 // cpu load. | |
148 | |
149 /////////////////////////////////////////////////////////////////////// | |
150 // Implementation of VideoAdapter | |
151 VideoAdapter::VideoAdapter() | 91 VideoAdapter::VideoAdapter() |
152 : output_num_pixels_(INT_MAX), | 92 : output_num_pixels_(std::numeric_limits<int>::max()), |
153 scale_third_(false), | |
154 frames_in_(0), | 93 frames_in_(0), |
155 frames_out_(0), | 94 frames_out_(0), |
156 frames_scaled_(0), | 95 frames_scaled_(0), |
157 adaption_changes_(0), | 96 adaption_changes_(0), |
158 previous_width_(0), | 97 previous_width_(0), |
159 previous_height_(0), | 98 previous_height_(0), |
160 interval_next_frame_(0) { | 99 interval_next_frame_(0), |
161 } | 100 format_request_max_pixel_count_(std::numeric_limits<int>::max()), |
101 resolution_request_max_pixel_count_(std::numeric_limits<int>::max()) {} | |
162 | 102 |
163 VideoAdapter::~VideoAdapter() { | 103 VideoAdapter::~VideoAdapter() {} |
104 | |
105 void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) { | |
106 // TODO(perkj): Consider measuring input frame rate instead. | |
107 // Frame rate typically varies depending on lighting. | |
108 rtc::CritScope cs(&critical_section_); | |
109 input_format_.interval = interval; | |
164 } | 110 } |
165 | 111 |
166 void VideoAdapter::SetInputFormat(const VideoFormat& format) { | 112 void VideoAdapter::SetInputFormat(const VideoFormat& format) { |
167 rtc::CritScope cs(&critical_section_); | 113 bool is_resolution_change = (input_format().width != format.width || |
114 input_format().height != format.height); | |
168 int64_t old_input_interval = input_format_.interval; | 115 int64_t old_input_interval = input_format_.interval; |
169 input_format_ = format; | 116 input_format_ = format; |
170 output_format_.interval = | 117 output_format_.interval = |
171 std::max(output_format_.interval, input_format_.interval); | 118 std::max(output_format_.interval, input_format_.interval); |
172 if (old_input_interval != input_format_.interval) { | 119 if (old_input_interval != input_format_.interval) { |
173 LOG(LS_INFO) << "VAdapt input interval changed from " | 120 LOG(LS_INFO) << "VAdapt input interval changed from " |
174 << old_input_interval << " to " << input_format_.interval; | 121 << old_input_interval << " to " << input_format_.interval; |
175 } | 122 } |
176 } | |
177 | |
178 void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) { | |
179 int previous_width = input_format().width; | |
180 int previous_height = input_format().height; | |
181 bool is_resolution_change = previous_width > 0 && format.width > 0 && | |
182 (previous_width != format.width || | |
183 previous_height != format.height); | |
184 VideoAdapter::SetInputFormat(format); | |
185 if (is_resolution_change) { | 123 if (is_resolution_change) { |
186 int width, height; | |
187 // Trigger the adaptation logic again, to potentially reset the adaptation | 124 // Trigger the adaptation logic again, to potentially reset the adaptation |
188 // state for things like view requests that may not longer be capping | 125 // state for things like view requests that may not longer be capping |
189 // output (or may now cap output). | 126 // output (or may now cap output). |
190 AdaptToMinimumFormat(&width, &height); | 127 Adapt(std::min(format_request_max_pixel_count_, |
191 LOG(LS_INFO) << "VAdapt Input Resolution Change: " | 128 resolution_request_max_pixel_count_), |
192 << "Previous input resolution: " | 129 0); |
193 << previous_width << "x" << previous_height | |
194 << " New input resolution: " | |
195 << format.width << "x" << format.height | |
196 << " New output resolution: " | |
197 << width << "x" << height; | |
198 } | 130 } |
199 } | 131 } |
200 | 132 |
201 void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) { | 133 const VideoFormat& VideoAdapter::input_format() const { |
202 LOG(LS_INFO) << "CPU smoothing is now " | |
203 << (enable ? "enabled" : "disabled"); | |
204 cpu_smoothing_ = enable; | |
205 } | |
206 | |
207 void VideoAdapter::SetOutputFormat(const VideoFormat& format) { | |
208 rtc::CritScope cs(&critical_section_); | |
209 int64_t old_output_interval = output_format_.interval; | |
210 output_format_ = format; | |
211 output_num_pixels_ = output_format_.width * output_format_.height; | |
212 output_format_.interval = | |
213 std::max(output_format_.interval, input_format_.interval); | |
214 if (old_output_interval != output_format_.interval) { | |
215 LOG(LS_INFO) << "VAdapt output interval changed from " | |
216 << old_output_interval << " to " << output_format_.interval; | |
217 } | |
218 } | |
219 | |
220 const VideoFormat& VideoAdapter::input_format() { | |
221 rtc::CritScope cs(&critical_section_); | 134 rtc::CritScope cs(&critical_section_); |
222 return input_format_; | 135 return input_format_; |
223 } | 136 } |
224 | 137 |
225 bool VideoAdapter::drops_all_frames() const { | |
226 return output_num_pixels_ == 0; | |
227 } | |
228 | |
229 const VideoFormat& VideoAdapter::output_format() { | |
230 rtc::CritScope cs(&critical_section_); | |
231 return output_format_; | |
232 } | |
233 | |
234 // Constrain output resolution to this many pixels overall | |
235 void VideoAdapter::SetOutputNumPixels(int num_pixels) { | |
236 output_num_pixels_ = num_pixels; | |
237 } | |
238 | |
239 int VideoAdapter::GetOutputNumPixels() const { | |
240 return output_num_pixels_; | |
241 } | |
242 | |
243 VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { | 138 VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) { |
244 rtc::CritScope cs(&critical_section_); | 139 rtc::CritScope cs(&critical_section_); |
245 ++frames_in_; | 140 ++frames_in_; |
246 | 141 |
247 SetInputFormat(VideoFormat( | 142 SetInputFormat(VideoFormat( |
248 in_width, in_height, input_format_.interval, input_format_.fourcc)); | 143 in_width, in_height, input_format_.interval, input_format_.fourcc)); |
249 | 144 |
250 // Drop the input frame if necessary. | 145 // Drop the input frame if necessary. |
251 bool should_drop = false; | 146 bool should_drop = false; |
252 if (!output_num_pixels_) { | 147 if (!output_num_pixels_) { |
253 // Drop all frames as the output format is 0x0. | 148 // Drop all frames as the output format is 0x0. |
254 should_drop = true; | 149 should_drop = true; |
255 } else { | 150 } else { |
256 // Drop some frames based on input fps and output fps. | 151 // Drop some frames based on input fps and output fps. |
257 // Normally output fps is less than input fps. | 152 // Normally output fps is less than input fps. |
258 // TODO(fbarchard): Consider adjusting interval to reflect the adjusted | |
259 // interval between frames after dropping some frames. | |
260 interval_next_frame_ += input_format_.interval; | 153 interval_next_frame_ += input_format_.interval; |
261 if (output_format_.interval > 0) { | 154 if (output_format_.interval > 0) { |
262 if (interval_next_frame_ >= output_format_.interval) { | 155 if (interval_next_frame_ >= output_format_.interval) { |
263 interval_next_frame_ %= output_format_.interval; | 156 interval_next_frame_ %= output_format_.interval; |
264 } else { | 157 } else { |
265 should_drop = true; | 158 should_drop = true; |
266 } | 159 } |
267 } | 160 } |
268 } | 161 } |
269 if (should_drop) { | 162 if (should_drop) { |
270 // Show VAdapt log every 90 frames dropped. (3 seconds) | 163 // Show VAdapt log every 90 frames dropped. (3 seconds) |
271 if ((frames_in_ - frames_out_) % 90 == 0) { | 164 if ((frames_in_ - frames_out_) % 90 == 0) { |
272 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed | 165 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed |
273 // in default calls. | 166 // in default calls. |
274 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_ | 167 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_ |
275 << " / out " << frames_out_ | 168 << " / out " << frames_out_ |
276 << " / in " << frames_in_ | 169 << " / in " << frames_in_ |
277 << " Changes: " << adaption_changes_ | 170 << " Changes: " << adaption_changes_ |
278 << " Input: " << in_width | 171 << " Input: " << in_width |
279 << "x" << in_height | 172 << "x" << in_height |
280 << " i" << input_format_.interval | 173 << " i" << input_format_.interval |
281 << " Output: i" << output_format_.interval; | 174 << " Output: i" << output_format_.interval; |
282 } | 175 } |
283 | 176 |
284 return VideoFormat(); // Drop frame. | 177 return VideoFormat(); // Drop frame. |
285 } | 178 } |
286 | 179 |
287 const float scale = VideoAdapter::FindClosestViewScale( | 180 const float scale = FindScaleLessThanOrEqual(in_width, in_height, |
288 in_width, in_height, output_num_pixels_); | 181 output_num_pixels_, nullptr); |
289 const size_t output_width = static_cast<size_t>(in_width * scale + .5f); | 182 const int output_width = static_cast<int>(in_width * scale + .5f); |
290 const size_t output_height = static_cast<size_t>(in_height * scale + .5f); | 183 const int output_height = static_cast<int>(in_height * scale + .5f); |
291 | 184 |
292 ++frames_out_; | 185 ++frames_out_; |
293 if (scale != 1) | 186 if (scale != 1) |
294 ++frames_scaled_; | 187 ++frames_scaled_; |
295 // Show VAdapt log every 90 frames output. (3 seconds) | |
296 // TODO(fbarchard): Consider GetLogSeverity() to change interval to less | |
297 // for LS_VERBOSE and more for LS_INFO. | |
298 bool show = (frames_out_) % 90 == 0; | |
299 | 188 |
300 // TODO(fbarchard): LOG the previous output resolution and track input | |
301 // resolution changes as well. Consider dropping the statistics into their | |
302 // own class which could be queried publically. | |
303 bool changed = false; | |
304 if (previous_width_ && (previous_width_ != output_width || | 189 if (previous_width_ && (previous_width_ != output_width || |
305 previous_height_ != output_height)) { | 190 previous_height_ != output_height)) { |
306 show = true; | |
307 ++adaption_changes_; | 191 ++adaption_changes_; |
308 changed = true; | 192 LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_ << " / out " |
pbos-webrtc
2016/04/05 13:17:53
I think this should log "Frame size changed: " ins
perkj_webrtc
2016/04/05 13:24:15
Done.
| |
309 } | 193 << frames_out_ << " / in " << frames_in_ |
310 if (show) { | 194 << " Changes: " << adaption_changes_ << " Input: " << in_width |
311 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed | 195 << "x" << in_height << " i" << input_format_.interval |
312 // in default calls. | 196 << " Scale: " << scale << " Output: " << output_width << "x" |
313 LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_ | 197 << output_height << " i" << output_format_.interval; |
314 << " / out " << frames_out_ | |
315 << " / in " << frames_in_ | |
316 << " Changes: " << adaption_changes_ | |
317 << " Input: " << in_width | |
318 << "x" << in_height | |
319 << " i" << input_format_.interval | |
320 << " Scale: " << scale | |
321 << " Output: " << output_width | |
322 << "x" << output_height | |
323 << " i" << output_format_.interval | |
324 << " Changed: " << (changed ? "true" : "false"); | |
325 } | 198 } |
326 | 199 |
327 output_format_.width = output_width; | 200 output_format_.width = output_width; |
328 output_format_.height = output_height; | 201 output_format_.height = output_height; |
329 previous_width_ = output_width; | 202 previous_width_ = output_width; |
330 previous_height_ = output_height; | 203 previous_height_ = output_height; |
331 | 204 |
332 return output_format_; | 205 return output_format_; |
333 } | 206 } |
334 | 207 |
335 void VideoAdapter::set_scale_third(bool enable) { | 208 void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { |
336 LOG(LS_INFO) << "Video Adapter third scaling is now " | 209 rtc::CritScope cs(&critical_section_); |
337 << (enable ? "enabled" : "disabled"); | 210 format_request_max_pixel_count_ = format.width * format.height; |
338 scale_third_ = enable; | 211 output_format_.interval = format.interval; |
212 Adapt(std::min(format_request_max_pixel_count_, | |
213 resolution_request_max_pixel_count_), | |
214 0); | |
339 } | 215 } |
340 | 216 |
341 /////////////////////////////////////////////////////////////////////// | 217 void VideoAdapter::OnResolutionRequest( |
342 // Implementation of CoordinatedVideoAdapter | 218 rtc::Optional<int> max_pixel_count, |
343 CoordinatedVideoAdapter::CoordinatedVideoAdapter() | 219 rtc::Optional<int> max_pixel_count_step_up) { |
344 : cpu_adaptation_(true), | 220 rtc::CritScope cs(&critical_section_); |
345 cpu_smoothing_(false), | 221 resolution_request_max_pixel_count_ = |
346 gd_adaptation_(true), | 222 max_pixel_count.value_or(std::numeric_limits<int>::max()); |
347 view_adaptation_(true), | 223 Adapt(std::min(format_request_max_pixel_count_, |
348 view_switch_(false), | 224 resolution_request_max_pixel_count_), |
349 cpu_downgrade_count_(0), | 225 max_pixel_count_step_up.value_or(0)); |
350 cpu_load_min_samples_(kCpuLoadMinSamples), | |
351 cpu_load_num_samples_(0), | |
352 high_system_threshold_(kHighSystemCpuThreshold), | |
353 low_system_threshold_(kLowSystemCpuThreshold), | |
354 process_threshold_(kProcessCpuThreshold), | |
355 view_desired_num_pixels_(INT_MAX), | |
356 view_desired_interval_(0), | |
357 encoder_desired_num_pixels_(INT_MAX), | |
358 cpu_desired_num_pixels_(INT_MAX), | |
359 adapt_reason_(ADAPTREASON_NONE), | |
360 system_load_average_(kCpuLoadInitialAverage) { | |
361 } | 226 } |
362 | 227 |
363 // Helper function to UPGRADE or DOWNGRADE a number of pixels | 228 bool VideoAdapter::Adapt(int max_num_pixels, int max_pixel_count_step_up) { |
364 void CoordinatedVideoAdapter::StepPixelCount( | 229 float scale_lower = |
365 CoordinatedVideoAdapter::AdaptRequest request, | 230 FindScaleLessThanOrEqual(input_format_.width, input_format_.height, |
366 int* num_pixels) { | 231 max_num_pixels, &max_num_pixels); |
367 switch (request) { | 232 float scale_upper = |
368 case CoordinatedVideoAdapter::DOWNGRADE: | 233 max_pixel_count_step_up > 0 |
369 *num_pixels /= 2; | 234 ? FindScaleLargerThan(input_format_.width, input_format_.height, |
370 break; | 235 max_pixel_count_step_up, |
236 &max_pixel_count_step_up) | |
237 : 1.f; | |
371 | 238 |
372 case CoordinatedVideoAdapter::UPGRADE: | 239 bool use_max_pixel_count_step_up = |
373 *num_pixels *= 2; | 240 max_pixel_count_step_up > 0 && max_num_pixels > max_pixel_count_step_up; |
374 break; | |
375 | 241 |
376 default: // No change in pixel count | 242 int old_num_pixels = output_num_pixels_; |
377 break; | 243 output_num_pixels_ = |
378 } | 244 use_max_pixel_count_step_up ? max_pixel_count_step_up : max_num_pixels; |
379 return; | 245 // Log the new size. |
380 } | 246 float scale = use_max_pixel_count_step_up ? scale_upper : scale_lower; |
247 int new_width = static_cast<int>(input_format_.width * scale + .5f); | |
248 int new_height = static_cast<int>(input_format_.height * scale + .5f); | |
381 | 249 |
382 // Find the adaptation request of the cpu based on the load. Return UPGRADE if | 250 bool changed = output_num_pixels_ != old_num_pixels; |
383 // the load is low, DOWNGRADE if the load is high, and KEEP otherwise. | 251 LOG(LS_INFO) << "Adapt: " |
pbos-webrtc
2016/04/05 13:17:53
Change Adapt: To OnResolutionRequest:
I think tha
perkj_webrtc
2016/04/05 13:24:15
Done.
| |
384 CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest( | 252 << " Max pixels: " << max_num_pixels |
385 int current_cpus, int max_cpus, | 253 << " Max pixels step up: " << max_pixel_count_step_up |
386 float process_load, float system_load) { | 254 << " Output Pixels: " << output_num_pixels_ |
387 // Downgrade if system is high and plugin is at least more than midrange. | 255 << " Input: " << input_format_.width << "x" |
388 if (system_load >= high_system_threshold_ * max_cpus && | 256 << input_format_.height << " Scale: " << scale |
389 process_load >= process_threshold_ * current_cpus) { | 257 << " Resolution: " << new_width << "x" << new_height |
390 return CoordinatedVideoAdapter::DOWNGRADE; | 258 << " Changed: " << (changed ? "true" : "false"); |
391 // Upgrade if system is low. | |
392 } else if (system_load < low_system_threshold_ * max_cpus) { | |
393 return CoordinatedVideoAdapter::UPGRADE; | |
394 } | |
395 return CoordinatedVideoAdapter::KEEP; | |
396 } | |
397 | |
398 // A remote view request for a new resolution. | |
399 void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { | |
400 rtc::CritScope cs(&request_critical_section_); | |
401 if (!view_adaptation_) { | |
402 return; | |
403 } | |
404 // Set output for initial aspect ratio in mediachannel unittests. | |
405 int old_num_pixels = GetOutputNumPixels(); | |
406 SetOutputFormat(format); | |
407 SetOutputNumPixels(old_num_pixels); | |
408 view_desired_num_pixels_ = format.width * format.height; | |
409 view_desired_interval_ = format.interval; | |
410 int new_width, new_height; | |
411 bool changed = AdaptToMinimumFormat(&new_width, &new_height); | |
412 LOG(LS_INFO) << "VAdapt View Request: " | |
413 << format.width << "x" << format.height | |
414 << " Pixels: " << view_desired_num_pixels_ | |
415 << " Changed: " << (changed ? "true" : "false") | |
416 << " To: " << new_width << "x" << new_height; | |
417 } | |
418 | |
419 void CoordinatedVideoAdapter::set_cpu_load_min_samples( | |
420 int cpu_load_min_samples) { | |
421 if (cpu_load_min_samples_ != cpu_load_min_samples) { | |
422 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: " | |
423 << cpu_load_min_samples_ << " to " | |
424 << cpu_load_min_samples; | |
425 cpu_load_min_samples_ = cpu_load_min_samples; | |
426 } | |
427 } | |
428 | |
429 void CoordinatedVideoAdapter::set_high_system_threshold( | |
430 float high_system_threshold) { | |
431 ASSERT(high_system_threshold <= 1.0f); | |
432 ASSERT(high_system_threshold >= 0.0f); | |
433 if (high_system_threshold_ != high_system_threshold) { | |
434 LOG(LS_INFO) << "VAdapt Change High System Threshold from: " | |
435 << high_system_threshold_ << " to " << high_system_threshold; | |
436 high_system_threshold_ = high_system_threshold; | |
437 } | |
438 } | |
439 | |
440 void CoordinatedVideoAdapter::set_low_system_threshold( | |
441 float low_system_threshold) { | |
442 ASSERT(low_system_threshold <= 1.0f); | |
443 ASSERT(low_system_threshold >= 0.0f); | |
444 if (low_system_threshold_ != low_system_threshold) { | |
445 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: " | |
446 << low_system_threshold_ << " to " << low_system_threshold; | |
447 low_system_threshold_ = low_system_threshold; | |
448 } | |
449 } | |
450 | |
451 void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) { | |
452 ASSERT(process_threshold <= 1.0f); | |
453 ASSERT(process_threshold >= 0.0f); | |
454 if (process_threshold_ != process_threshold) { | |
455 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: " | |
456 << process_threshold_ << " to " << process_threshold; | |
457 process_threshold_ = process_threshold; | |
458 } | |
459 } | |
460 | |
461 // A Bandwidth GD request for new resolution | |
462 void CoordinatedVideoAdapter::OnEncoderResolutionRequest( | |
463 int width, int height, AdaptRequest request) { | |
464 rtc::CritScope cs(&request_critical_section_); | |
465 if (!gd_adaptation_) { | |
466 return; | |
467 } | |
468 int old_encoder_desired_num_pixels = encoder_desired_num_pixels_; | |
469 if (KEEP != request) { | |
470 int new_encoder_desired_num_pixels = width * height; | |
471 int old_num_pixels = GetOutputNumPixels(); | |
472 if (new_encoder_desired_num_pixels != old_num_pixels) { | |
473 LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored"; | |
474 } else { | |
475 // Update the encoder desired format based on the request. | |
476 encoder_desired_num_pixels_ = new_encoder_desired_num_pixels; | |
477 StepPixelCount(request, &encoder_desired_num_pixels_); | |
478 } | |
479 } | |
480 int new_width, new_height; | |
481 bool changed = AdaptToMinimumFormat(&new_width, &new_height); | |
482 | |
483 // Ignore up or keep if no change. | |
484 if (DOWNGRADE != request && view_switch_ && !changed) { | |
485 encoder_desired_num_pixels_ = old_encoder_desired_num_pixels; | |
486 LOG(LS_VERBOSE) << "VAdapt ignoring GD request."; | |
487 } | |
488 | |
489 LOG(LS_INFO) << "VAdapt GD Request: " | |
490 << (DOWNGRADE == request ? "down" : | |
491 (UPGRADE == request ? "up" : "keep")) | |
492 << " From: " << width << "x" << height | |
493 << " Pixels: " << encoder_desired_num_pixels_ | |
494 << " Changed: " << (changed ? "true" : "false") | |
495 << " To: " << new_width << "x" << new_height; | |
496 } | |
497 | |
498 void CoordinatedVideoAdapter::OnCpuResolutionRequest( | |
499 rtc::Optional<int> max_pixel_count, | |
500 rtc::Optional<int> max_pixel_count_step_up) { | |
501 rtc::CritScope cs(&request_critical_section_); | |
502 // TODO(perkj): We should support taking larger steps up and down and | |
503 // actually look at the values set in max_pixel_count and | |
504 // max_pixel_count_step_up. | |
505 if (max_pixel_count && *max_pixel_count < GetOutputNumPixels()) { | |
506 OnCpuResolutionRequest(DOWNGRADE); | |
507 } else if (max_pixel_count_step_up && | |
508 *max_pixel_count_step_up >= GetOutputNumPixels()) { | |
509 OnCpuResolutionRequest(UPGRADE); | |
510 } | |
511 } | |
512 | |
513 // A Bandwidth GD request for new resolution | |
514 void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) { | |
515 rtc::CritScope cs(&request_critical_section_); | |
516 if (!cpu_adaptation_) { | |
517 return; | |
518 } | |
519 | |
520 // Update how many times we have downgraded due to the cpu load. | |
521 switch (request) { | |
522 case DOWNGRADE: | |
523 // Ignore downgrades if we have downgraded the maximum times. | |
524 if (cpu_downgrade_count_ < kMaxCpuDowngrades) { | |
525 ++cpu_downgrade_count_; | |
526 } else { | |
527 LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade " | |
528 "because maximum downgrades reached"; | |
529 SignalCpuAdaptationUnable(); | |
530 } | |
531 break; | |
532 case UPGRADE: | |
533 if (cpu_downgrade_count_ > 0) { | |
534 bool is_min = IsMinimumFormat(cpu_desired_num_pixels_); | |
535 if (is_min) { | |
536 --cpu_downgrade_count_; | |
537 } else { | |
538 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " | |
539 "because cpu is not limiting resolution"; | |
540 } | |
541 } else { | |
542 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade " | |
543 "because minimum downgrades reached"; | |
544 } | |
545 break; | |
546 case KEEP: | |
547 default: | |
548 break; | |
549 } | |
550 if (KEEP != request) { | |
551 // TODO(fbarchard): compute stepping up/down from OutputNumPixels but | |
552 // clamp to inputpixels / 4 (2 steps) | |
553 cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX : | |
554 static_cast<int>(input_format().width * input_format().height >> | |
555 cpu_downgrade_count_); | |
556 } | |
557 int new_width, new_height; | |
558 bool changed = AdaptToMinimumFormat(&new_width, &new_height); | |
559 LOG(LS_INFO) << "VAdapt CPU Request: " | |
560 << (DOWNGRADE == request ? "down" : | |
561 (UPGRADE == request ? "up" : "keep")) | |
562 << " Steps: " << cpu_downgrade_count_ | |
563 << " Changed: " << (changed ? "true" : "false") | |
564 << " To: " << new_width << "x" << new_height; | |
565 } | |
566 | |
567 // A CPU request for new resolution | |
568 // TODO(fbarchard): Move outside adapter. | |
569 void CoordinatedVideoAdapter::OnCpuLoadUpdated( | |
570 int current_cpus, int max_cpus, float process_load, float system_load) { | |
571 rtc::CritScope cs(&request_critical_section_); | |
572 if (!cpu_adaptation_) { | |
573 return; | |
574 } | |
575 // Update the moving average of system load. Even if we aren't smoothing, | |
576 // we'll still calculate this information, in case smoothing is later enabled. | |
577 system_load_average_ = kCpuLoadWeightCoefficient * system_load + | |
578 (1.0f - kCpuLoadWeightCoefficient) * system_load_average_; | |
579 ++cpu_load_num_samples_; | |
580 if (cpu_smoothing_) { | |
581 system_load = system_load_average_; | |
582 } | |
583 AdaptRequest request = FindCpuRequest(current_cpus, max_cpus, | |
584 process_load, system_load); | |
585 // Make sure we're not adapting too quickly. | |
586 if (request != KEEP) { | |
587 if (cpu_load_num_samples_ < cpu_load_min_samples_) { | |
588 LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until " | |
589 << (cpu_load_min_samples_ - cpu_load_num_samples_) | |
590 << " more samples"; | |
591 request = KEEP; | |
592 } | |
593 } | |
594 | |
595 OnCpuResolutionRequest(request); | |
596 } | |
597 | |
598 // Called by cpu adapter on up requests. | |
599 bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) { | |
600 // Find closest scale factor that matches input resolution to min_num_pixels | |
601 // and set that for output resolution. This is not needed for VideoAdapter, | |
602 // but provides feedback to unittests and users on expected resolution. | |
603 // Actual resolution is based on input frame. | |
604 VideoFormat new_output = output_format(); | |
605 VideoFormat input = input_format(); | |
606 if (input_format().IsSize0x0()) { | |
607 input = new_output; | |
608 } | |
609 float scale = 1.0f; | |
610 if (!input.IsSize0x0()) { | |
611 scale = FindClosestScale(input.width, | |
612 input.height, | |
613 pixels); | |
614 } | |
615 new_output.width = static_cast<int>(input.width * scale + .5f); | |
616 new_output.height = static_cast<int>(input.height * scale + .5f); | |
617 int new_pixels = new_output.width * new_output.height; | |
618 int num_pixels = GetOutputNumPixels(); | |
619 return new_pixels <= num_pixels; | |
620 } | |
621 | |
622 // Called by all coordinators when there is a change. | |
623 bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width, | |
624 int* new_height) { | |
625 VideoFormat new_output = output_format(); | |
626 VideoFormat input = input_format(); | |
627 if (input_format().IsSize0x0()) { | |
628 input = new_output; | |
629 } | |
630 int old_num_pixels = GetOutputNumPixels(); | |
631 int min_num_pixels = INT_MAX; | |
632 adapt_reason_ = ADAPTREASON_NONE; | |
633 | |
634 // Reduce resolution based on encoder bandwidth (GD). | |
635 if (encoder_desired_num_pixels_ && | |
636 (encoder_desired_num_pixels_ < min_num_pixels)) { | |
637 adapt_reason_ |= ADAPTREASON_BANDWIDTH; | |
638 min_num_pixels = encoder_desired_num_pixels_; | |
639 } | |
640 // Reduce resolution based on CPU. | |
641 if (cpu_adaptation_ && cpu_desired_num_pixels_ && | |
642 (cpu_desired_num_pixels_ <= min_num_pixels)) { | |
643 if (cpu_desired_num_pixels_ < min_num_pixels) { | |
644 adapt_reason_ = ADAPTREASON_CPU; | |
645 } else { | |
646 adapt_reason_ |= ADAPTREASON_CPU; | |
647 } | |
648 min_num_pixels = cpu_desired_num_pixels_; | |
649 } | |
650 // Round resolution for GD or CPU to allow 1/2 to map to 9/16. | |
651 if (!input.IsSize0x0() && min_num_pixels != INT_MAX) { | |
652 float scale = FindClosestScale(input.width, input.height, min_num_pixels); | |
653 min_num_pixels = static_cast<int>(input.width * scale + .5f) * | |
654 static_cast<int>(input.height * scale + .5f); | |
655 } | |
656 // Reduce resolution based on View Request. | |
657 if (view_desired_num_pixels_ <= min_num_pixels) { | |
658 if (view_desired_num_pixels_ < min_num_pixels) { | |
659 adapt_reason_ = ADAPTREASON_VIEW; | |
660 } else { | |
661 adapt_reason_ |= ADAPTREASON_VIEW; | |
662 } | |
663 min_num_pixels = view_desired_num_pixels_; | |
664 } | |
665 // Snap to a scale factor. | |
666 float scale = 1.0f; | |
667 if (!input.IsSize0x0()) { | |
668 scale = FindLowerScale(input.width, input.height, min_num_pixels); | |
669 min_num_pixels = static_cast<int>(input.width * scale + .5f) * | |
670 static_cast<int>(input.height * scale + .5f); | |
671 } | |
672 if (scale == 1.0f) { | |
673 adapt_reason_ = ADAPTREASON_NONE; | |
674 } | |
675 *new_width = new_output.width = static_cast<int>(input.width * scale + .5f); | |
676 *new_height = new_output.height = static_cast<int>(input.height * scale + | |
677 .5f); | |
678 SetOutputNumPixels(min_num_pixels); | |
679 | |
680 new_output.interval = view_desired_interval_; | |
681 SetOutputFormat(new_output); | |
682 int new_num_pixels = GetOutputNumPixels(); | |
683 bool changed = new_num_pixels != old_num_pixels; | |
684 | |
685 static const char* kReasons[8] = { | |
686 "None", | |
687 "CPU", | |
688 "BANDWIDTH", | |
689 "CPU+BANDWIDTH", | |
690 "VIEW", | |
691 "CPU+VIEW", | |
692 "BANDWIDTH+VIEW", | |
693 "CPU+BANDWIDTH+VIEW", | |
694 }; | |
695 | |
696 LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_ | |
697 << " GD: " << encoder_desired_num_pixels_ | |
698 << " CPU: " << cpu_desired_num_pixels_ | |
699 << " Pixels: " << min_num_pixels | |
700 << " Input: " << input.width | |
701 << "x" << input.height | |
702 << " Scale: " << scale | |
703 << " Resolution: " << new_output.width | |
704 << "x" << new_output.height | |
705 << " Changed: " << (changed ? "true" : "false") | |
706 << " Reason: " << kReasons[adapt_reason_]; | |
707 | |
708 if (changed) { | |
709 // When any adaptation occurs, historic CPU load levels are no longer | |
710 // accurate. Clear out our state so we can re-learn at the new normal. | |
711 cpu_load_num_samples_ = 0; | |
712 system_load_average_ = kCpuLoadInitialAverage; | |
713 } | |
714 | 259 |
715 return changed; | 260 return changed; |
716 } | 261 } |
717 | 262 |
718 } // namespace cricket | 263 } // namespace cricket |
OLD | NEW |