OLD | NEW |
---|---|
(Empty) | |
1 /* | |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 */ | |
10 | |
11 #include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" | |
12 | |
13 #include <algorithm> | |
14 | |
15 namespace webrtc { | |
16 namespace testing { | |
17 namespace bwe { | |
18 | |
19 namespace { | |
20 | |
21 template <typename T> | |
22 T Sum(const std::vector<T>& input) { | |
23 T total = 0; | |
24 for (T val : input) { | |
25 total += val; | |
26 } | |
27 return total; | |
28 } | |
29 | |
30 template <typename T> | |
31 double Average(const std::vector<T>& array, size_t size) { | |
32 return static_cast<double>(Sum(array)) / size; | |
33 } | |
34 | |
35 template <typename T> | |
36 std::vector<T> Abs(const std::vector<T>& input) { | |
37 std::vector<T> output(input); | |
38 for (T val : output) { | |
39 val = std::abs(val); | |
40 } | |
41 return output; | |
42 } | |
43 | |
44 template <typename T> | |
45 std::vector<double> Pow(const std::vector<T>& input, double p) { | |
46 std::vector<double> output; | |
47 for (T val : input) { | |
48 output.push_back(pow(static_cast<double>(val), p)); | |
49 } | |
50 return output; | |
51 } | |
52 | |
53 template <typename T> | |
54 double StandardDeviation(const std::vector<T>& array, size_t size) { | |
55 double mean = Average(array, size); | |
56 std::vector<double> square_values = Pow(array, 2.0); | |
57 double var = Average(square_values, size) - mean * mean; | |
58 return sqrt(var); | |
59 } | |
60 | |
61 // Holder mean, Manhattan distance for p=1, EuclidianNorm/sqrt(n) for p=2. | |
62 template <typename T> | |
63 double NormLp(const std::vector<T>& array, size_t size, double p) { | |
64 std::vector<T> abs_values = Abs(array); | |
65 std::vector<double> pow_values = Pow(abs_values, p); | |
66 return pow(Sum(pow_values) / size, 1.0 / p); | |
67 } | |
68 | |
69 template <typename T> | |
70 std::vector<T> PositiveFilter(const std::vector<T>& input) { | |
71 std::vector<T> output(input); | |
72 for (T val : output) { | |
73 val = val > 0 ? val : 0; | |
74 } | |
75 return output; | |
76 } | |
77 | |
78 template <typename T> | |
79 std::vector<T> NegativeFilter(const std::vector<T>& input) { | |
80 std::vector<T> output(input); | |
81 for (T val : output) { | |
82 val = val < 0 ? -val : 0; | |
83 } | |
84 return output; | |
85 } | |
86 } // namespace | |
87 | |
88 LinkShare::LinkShare(ChokeFilter* choke_filter) | |
89 : choke_filter_(choke_filter), running_flows_(choke_filter->flow_ids()) { | |
90 } | |
91 | |
92 void LinkShare::PauseFlow(int flow_id) { | |
93 running_flows_.erase(flow_id); | |
94 } | |
95 | |
96 void LinkShare::ResumeFlow(int flow_id) { | |
97 running_flows_.insert(flow_id); | |
98 } | |
99 | |
100 uint32_t LinkShare::TotalAvailableKbps() { | |
101 return choke_filter_->capacity_kbps(); | |
102 } | |
103 | |
104 uint32_t LinkShare::AvailablePerFlowKbps(int flow_id) { | |
105 uint32_t available_capacity_per_flow_kbps = 0; | |
106 if (running_flows_.find(flow_id) != running_flows_.end()) { | |
107 available_capacity_per_flow_kbps = | |
108 TotalAvailableKbps() / static_cast<uint32_t>(running_flows_.size()); | |
109 } | |
110 return available_capacity_per_flow_kbps; | |
111 } | |
112 | |
113 MetricRecorder::MetricRecorder(const std::string algorithm_name, | |
114 int flow_id, | |
115 PacketSender* packet_sender, | |
116 LinkShare* link_share) | |
117 : algorithm_name_(algorithm_name), | |
118 flow_id_(flow_id), | |
119 packet_sender_(packet_sender), | |
120 link_share_(link_share), | |
121 now_ms_(0), | |
122 delays_ms_(), | |
123 throughput_bytes_(), | |
124 weighted_estimate_error_(), | |
125 last_unweighted_estimate_error_(0), | |
126 optimal_throughput_bits_(0), | |
127 last_available_bitrate_per_flow_kbps_(0), | |
128 start_computing_metrics_ms_(0), | |
129 started_computing_metrics_(false) { | |
130 } | |
131 | |
132 void MetricRecorder::SetPlotInformation( | |
133 const std::vector<std::string>& prefixs) { | |
stefan-webrtc
2015/07/06 08:24:51
prefixes
magalhaesc
2015/07/06 09:28:06
Done.
| |
134 assert(prefixs.size() == kNumMetrics); | |
135 for (size_t i = 0; i < kNumMetrics; ++i) { | |
136 plot_information_[i].prefix = prefixs[i]; | |
137 } | |
138 plot_information_[kThroughput].plot_interval_ms = 100; | |
139 plot_information_[kDelay].plot_interval_ms = 100; | |
140 plot_information_[kLoss].plot_interval_ms = 500; | |
141 plot_information_[kObjective].plot_interval_ms = 1000; | |
142 plot_information_[kTotalAvailable].plot_interval_ms = 1000; | |
143 plot_information_[kAvailablePerFlow].plot_interval_ms = 1000; | |
144 | |
145 for (int i = kThroughput; i < kNumMetrics; ++i) { | |
146 plot_information_[i].last_plot_ms = 0; | |
147 if (i == kObjective || i == kAvailablePerFlow) { | |
148 plot_information_[i].plot = false; | |
149 } else { | |
150 plot_information_[i].plot = true; | |
151 } | |
152 } | |
153 } | |
154 | |
155 void MetricRecorder::PlotAllDynamics() { | |
156 for (int i = kThroughput; i < kNumMetrics; ++i) { | |
157 if (plot_information_[i].plot && | |
158 now_ms_ - plot_information_[i].last_plot_ms > | |
159 plot_information_[i].plot_interval_ms) { | |
160 PlotDynamics(i); | |
161 } | |
162 } | |
163 } | |
164 | |
165 void MetricRecorder::PlotDynamics(int metric) { | |
166 if (metric == kTotalAvailable) { | |
167 BWE_TEST_LOGGING_PLOT_WITH_NAME( | |
168 0, plot_information_[kTotalAvailable].prefix, now_ms_, | |
169 GetTotalAvailableKbps(), "Available"); | |
170 } else if (metric == kAvailablePerFlow) { | |
171 BWE_TEST_LOGGING_PLOT_WITH_NAME( | |
172 0, plot_information_[kAvailablePerFlow].prefix, now_ms_, | |
173 GetAvailablePerFlowKbps(), "Available_per_flow"); | |
174 } else { | |
175 PlotLine(metric, plot_information_[metric].prefix, | |
176 plot_information_[metric].time_ms, | |
177 plot_information_[metric].value); | |
178 } | |
179 plot_information_[metric].last_plot_ms = now_ms_; | |
180 } | |
181 | |
182 void MetricRecorder::PlotLine(int windows_id, | |
183 const std::string& prefix, | |
184 int64_t x, | |
185 int64_t y) { | |
186 BWE_TEST_LOGGING_PLOT_WITH_NAME(windows_id, prefix, x, y, algorithm_name_); | |
187 } | |
188 | |
189 void MetricRecorder::UpdateTime(int64_t time_ms) { | |
190 now_ms_ = std::max(now_ms_, time_ms); | |
191 } | |
192 | |
193 void MetricRecorder::UpdateThroughput(int64_t bitrate_kbps, | |
194 size_t payload_size) { | |
195 PushThroughputBytes(payload_size, now_ms_); | |
196 plot_information_[kThroughput].Update(now_ms_, bitrate_kbps); | |
197 } | |
198 | |
199 void MetricRecorder::UpdateDelay(int64_t delay_ms) { | |
200 PushDelayMs(delay_ms, now_ms_); | |
201 plot_information_[kDelay].Update(now_ms_, delay_ms); | |
202 } | |
203 | |
204 void MetricRecorder::UpdateLoss(float loss_ratio) { | |
205 plot_information_[kLoss].Update(now_ms_, loss_ratio); | |
206 } | |
207 | |
208 void MetricRecorder::UpdateObjective() { | |
209 plot_information_[kObjective].Update(now_ms_, ObjectiveFunction()); | |
210 } | |
211 | |
212 uint32_t MetricRecorder::GetTotalAvailableKbps() { | |
213 return link_share_->TotalAvailableKbps(); | |
214 } | |
215 | |
216 uint32_t MetricRecorder::GetAvailablePerFlowKbps() { | |
217 return link_share_->AvailablePerFlowKbps(flow_id_); | |
218 } | |
219 | |
220 uint32_t MetricRecorder::GetSendingEstimateKbps() { | |
221 return packet_sender_->TargetBitrateKbps(); | |
222 } | |
223 | |
224 void MetricRecorder::PushDelayMs(int64_t delay_ms, int64_t arrival_time_ms) { | |
225 if (ShouldRecord(arrival_time_ms)) { | |
226 delays_ms_.push_back(delay_ms); | |
227 } | |
228 } | |
229 | |
230 void MetricRecorder::PushThroughputBytes(size_t payload_size, | |
231 int64_t arrival_time_ms) { | |
232 if (ShouldRecord(arrival_time_ms)) { | |
233 throughput_bytes_.push_back(payload_size); | |
234 | |
235 int64_t current_available_per_flow_kbps = | |
236 static_cast<int64_t>(GetAvailablePerFlowKbps()); | |
237 | |
238 int64_t current_bitrate_diff_kbps = | |
239 static_cast<int64_t>(GetSendingEstimateKbps()) - | |
240 current_available_per_flow_kbps; | |
241 | |
242 // time_ms was still not updated here. | |
stefan-webrtc
2015/07/06 08:24:51
I don't understand this comment. time_ms isn't upd
magalhaesc
2015/07/06 09:28:06
The comment will be moved to UpdateThroughput meth
| |
243 weighted_estimate_error_.push_back( | |
244 ((current_bitrate_diff_kbps + last_unweighted_estimate_error_) * | |
245 (arrival_time_ms - plot_information_[kThroughput].time_ms)) / | |
246 2); | |
247 | |
248 optimal_throughput_bits_ += | |
249 ((current_available_per_flow_kbps + | |
250 last_available_bitrate_per_flow_kbps_) * | |
251 (arrival_time_ms - plot_information_[kThroughput].time_ms)) / | |
252 2; | |
253 | |
254 last_available_bitrate_per_flow_kbps_ = current_available_per_flow_kbps; | |
255 } | |
256 } | |
257 | |
258 bool MetricRecorder::ShouldRecord(int64_t arrival_time_ms) { | |
259 if (arrival_time_ms >= start_computing_metrics_ms_) { | |
260 if (!started_computing_metrics_) { | |
261 start_computing_metrics_ms_ = arrival_time_ms; | |
262 now_ms_ = arrival_time_ms; | |
263 started_computing_metrics_ = true; | |
264 } | |
265 return true; | |
266 } else { | |
267 return false; | |
268 } | |
269 } | |
270 | |
271 // The weighted_estimate_error_ was weighted based on time windows. | |
272 // This function scales back the result before plotting. | |
273 double MetricRecorder::Renormalize(double x) { | |
274 size_t num_packets_received = delays_ms_.size(); | |
275 return (x * num_packets_received) / now_ms_; | |
276 } | |
277 | |
278 inline double U(int64_t x, double alpha) { | |
279 if (alpha == 1.0) { | |
280 return log(static_cast<double>(x)); | |
281 } | |
282 return pow(static_cast<double>(x), 1.0 - alpha) / (1.0 - alpha); | |
283 } | |
284 | |
285 inline double U(size_t x, double alpha) { | |
286 return U(static_cast<int64_t>(x), alpha); | |
287 } | |
288 | |
289 // TODO(magalhaesc): Update ObjectiveFunction. | |
290 double MetricRecorder::ObjectiveFunction() { | |
291 const double kDelta = 0.15; // Delay penalty factor. | |
292 const double kAlpha = 1.0; | |
293 const double kBeta = 1.0; | |
294 | |
295 double throughput_metric = U(Sum(throughput_bytes_), kAlpha); | |
296 double delay_penalty = kDelta * U(Sum(delays_ms_), kBeta); | |
297 | |
298 return throughput_metric - delay_penalty; | |
299 } | |
300 | |
301 void MetricRecorder::PlotThroughputHistogram(const std::string& title, | |
302 const std::string& bwe_name, | |
303 int num_flows, | |
304 int64_t extra_offset_ms, | |
305 const std::string optimum_id) { | |
306 size_t num_packets_received = delays_ms_.size(); | |
307 | |
308 int64_t duration_ms = now_ms_ - start_computing_metrics_ms_ - extra_offset_ms; | |
309 | |
310 double average_bitrate_kbps = | |
311 static_cast<double>(8 * Sum(throughput_bytes_) / duration_ms); | |
312 | |
313 double optimal_bitrate_per_flow_kbps = | |
314 static_cast<double>(optimal_throughput_bits_ / duration_ms); | |
315 | |
316 std::vector<int64_t> positive = PositiveFilter(weighted_estimate_error_); | |
317 std::vector<int64_t> negative = NegativeFilter(weighted_estimate_error_); | |
318 | |
319 double p_error = Renormalize(NormLp(positive, num_packets_received, 1.0)); | |
320 double n_error = Renormalize(NormLp(negative, num_packets_received, 1.0)); | |
321 | |
322 // Prevent the error to be too close to zero (plotting issue). | |
323 double extra_error = average_bitrate_kbps / 500; | |
324 | |
325 std::string optimum_title = | |
326 optimum_id.empty() ? "optimal_bitrate" : "optimal_bitrates#" + optimum_id; | |
327 | |
328 BWE_TEST_LOGGING_LABEL(4, title, "average_bitrate_(kbps)", num_flows); | |
329 BWE_TEST_LOGGING_LIMITERRORBAR( | |
330 4, bwe_name, average_bitrate_kbps, | |
331 average_bitrate_kbps - n_error - extra_error, | |
332 average_bitrate_kbps + p_error + extra_error, "estimate_error", | |
333 optimal_bitrate_per_flow_kbps, optimum_title, flow_id_); | |
334 | |
335 RTC_UNUSED(p_error); | |
336 RTC_UNUSED(n_error); | |
337 RTC_UNUSED(extra_error); | |
338 RTC_UNUSED(optimal_bitrate_per_flow_kbps); | |
339 } | |
340 | |
341 void MetricRecorder::PlotThroughputHistogram(const std::string& title, | |
342 const std::string& bwe_name, | |
343 int num_flows, | |
344 int64_t extra_offset_ms) { | |
345 PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, ""); | |
346 } | |
347 | |
348 void MetricRecorder::PlotDelayHistogram(const std::string& title, | |
349 const std::string& bwe_name, | |
350 int num_flows) { | |
351 size_t num_packets_received = delays_ms_.size(); | |
352 double average_delay_ms = Average(delays_ms_, num_packets_received); | |
353 | |
354 // Prevent the error to be too close to zero (plotting issue). | |
355 double extra_error = average_delay_ms / 500; | |
356 | |
357 double tenth_sigma_ms = | |
358 StandardDeviation(delays_ms_, num_packets_received) / 10.0 + extra_error; | |
359 | |
360 BWE_TEST_LOGGING_LABEL(5, title, "average_delay_(ms)", num_flows) | |
361 BWE_TEST_LOGGING_ERRORBAR( | |
362 5, bwe_name, average_delay_ms, average_delay_ms - tenth_sigma_ms, | |
363 average_delay_ms + tenth_sigma_ms, "sigma/10", flow_id_); | |
364 | |
365 RTC_UNUSED(tenth_sigma_ms); | |
366 } | |
367 | |
368 void MetricRecorder::PlotLossHistogram(const std::string& title, | |
369 const std::string& bwe_name, | |
370 int num_flows, | |
371 float global_loss_ratio) { | |
372 BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows) | |
373 BWE_TEST_LOGGING_BAR(6, bwe_name, 100.0f * global_loss_ratio, flow_id_); | |
374 } | |
375 | |
376 void MetricRecorder::PlotObjectiveHistogram(const std::string& title, | |
377 const std::string& bwe_name, | |
378 int num_flows) { | |
379 BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows) | |
380 BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), flow_id_); | |
381 } | |
382 | |
383 } // namespace bwe | |
384 } // namespace testing | |
385 } // namespace webrtc | |
OLD | NEW |