OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ |
| 10 |
| 11 #include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h" |
| 12 |
| 13 #include <algorithm> |
| 14 |
| 15 namespace webrtc { |
| 16 namespace testing { |
| 17 namespace bwe { |
| 18 |
| 19 namespace { |
| 20 |
| 21 template <typename T> |
| 22 T Sum(const std::vector<T>& input) { |
| 23 T total = 0; |
| 24 for (T val : input) { |
| 25 total += val; |
| 26 } |
| 27 return total; |
| 28 } |
| 29 |
| 30 template <typename T> |
| 31 double Average(const std::vector<T>& array, size_t size) { |
| 32 return static_cast<double>(Sum(array)) / size; |
| 33 } |
| 34 |
| 35 template <typename T> |
| 36 std::vector<T> Abs(const std::vector<T>& input) { |
| 37 std::vector<T> output(input); |
| 38 for (T val : output) { |
| 39 val = std::abs(val); |
| 40 } |
| 41 return output; |
| 42 } |
| 43 |
| 44 template <typename T> |
| 45 std::vector<double> Pow(const std::vector<T>& input, double p) { |
| 46 std::vector<double> output; |
| 47 for (T val : input) { |
| 48 output.push_back(pow(static_cast<double>(val), p)); |
| 49 } |
| 50 return output; |
| 51 } |
| 52 |
| 53 template <typename T> |
| 54 double StandardDeviation(const std::vector<T>& array, size_t size) { |
| 55 double mean = Average(array, size); |
| 56 std::vector<double> square_values = Pow(array, 2.0); |
| 57 double var = Average(square_values, size) - mean * mean; |
| 58 return sqrt(var); |
| 59 } |
| 60 |
| 61 // Holder mean, Manhattan distance for p=1, EuclidianNorm/sqrt(n) for p=2. |
| 62 template <typename T> |
| 63 double NormLp(const std::vector<T>& array, size_t size, double p) { |
| 64 std::vector<T> abs_values = Abs(array); |
| 65 std::vector<double> pow_values = Pow(abs_values, p); |
| 66 return pow(Sum(pow_values) / size, 1.0 / p); |
| 67 } |
| 68 |
| 69 template <typename T> |
| 70 std::vector<T> PositiveFilter(const std::vector<T>& input) { |
| 71 std::vector<T> output(input); |
| 72 for (T val : output) { |
| 73 val = val > 0 ? val : 0; |
| 74 } |
| 75 return output; |
| 76 } |
| 77 |
| 78 template <typename T> |
| 79 std::vector<T> NegativeFilter(const std::vector<T>& input) { |
| 80 std::vector<T> output(input); |
| 81 for (T val : output) { |
| 82 val = val < 0 ? -val : 0; |
| 83 } |
| 84 return output; |
| 85 } |
| 86 } // namespace |
| 87 |
| 88 LinkShare::LinkShare(ChokeFilter* choke_filter) |
| 89 : choke_filter_(choke_filter), running_flows_(choke_filter->flow_ids()) { |
| 90 } |
| 91 |
| 92 void LinkShare::PauseFlow(int flow_id) { |
| 93 running_flows_.erase(flow_id); |
| 94 } |
| 95 |
| 96 void LinkShare::ResumeFlow(int flow_id) { |
| 97 running_flows_.insert(flow_id); |
| 98 } |
| 99 |
| 100 uint32_t LinkShare::TotalAvailableKbps() { |
| 101 return choke_filter_->capacity_kbps(); |
| 102 } |
| 103 |
| 104 uint32_t LinkShare::AvailablePerFlowKbps(int flow_id) { |
| 105 uint32_t available_capacity_per_flow_kbps = 0; |
| 106 if (running_flows_.find(flow_id) != running_flows_.end()) { |
| 107 available_capacity_per_flow_kbps = |
| 108 TotalAvailableKbps() / static_cast<uint32_t>(running_flows_.size()); |
| 109 } |
| 110 return available_capacity_per_flow_kbps; |
| 111 } |
| 112 |
| 113 MetricRecorder::MetricRecorder(const std::string algorithm_name, |
| 114 int flow_id, |
| 115 PacketSender* packet_sender, |
| 116 LinkShare* link_share) |
| 117 : algorithm_name_(algorithm_name), |
| 118 flow_id_(flow_id), |
| 119 packet_sender_(packet_sender), |
| 120 link_share_(link_share), |
| 121 now_ms_(0), |
| 122 delays_ms_(), |
| 123 throughput_bytes_(), |
| 124 weighted_estimate_error_(), |
| 125 last_unweighted_estimate_error_(0), |
| 126 optimal_throughput_bits_(0), |
| 127 last_available_bitrate_per_flow_kbps_(0), |
| 128 start_computing_metrics_ms_(0), |
| 129 started_computing_metrics_(false) { |
| 130 } |
| 131 |
| 132 void MetricRecorder::SetPlotInformation( |
| 133 const std::vector<std::string>& prefixs) { |
| 134 assert(prefixs.size() == kNumMetrics); |
| 135 for (size_t i = 0; i < kNumMetrics; ++i) { |
| 136 plot_information_[i].prefix = prefixs[i]; |
| 137 } |
| 138 plot_information_[kThroughput].plot_interval_ms = 100; |
| 139 plot_information_[kDelay].plot_interval_ms = 100; |
| 140 plot_information_[kLoss].plot_interval_ms = 500; |
| 141 plot_information_[kObjective].plot_interval_ms = 1000; |
| 142 plot_information_[kTotalAvailable].plot_interval_ms = 1000; |
| 143 plot_information_[kAvailablePerFlow].plot_interval_ms = 1000; |
| 144 |
| 145 for (int i = kThroughput; i < kNumMetrics; ++i) { |
| 146 plot_information_[i].last_plot_ms = 0; |
| 147 if (i == kObjective || i == kAvailablePerFlow) { |
| 148 plot_information_[i].plot = false; |
| 149 } else { |
| 150 plot_information_[i].plot = true; |
| 151 } |
| 152 } |
| 153 } |
| 154 |
| 155 void MetricRecorder::PlotAllDynamics() { |
| 156 for (int i = kThroughput; i < kNumMetrics; ++i) { |
| 157 if (plot_information_[i].plot && |
| 158 now_ms_ - plot_information_[i].last_plot_ms > |
| 159 plot_information_[i].plot_interval_ms) { |
| 160 PlotDynamics(i); |
| 161 } |
| 162 } |
| 163 } |
| 164 |
| 165 void MetricRecorder::PlotDynamics(int metric) { |
| 166 if (metric == kTotalAvailable) { |
| 167 BWE_TEST_LOGGING_PLOT_WITH_NAME(0, |
| 168 plot_information_[kTotalAvailable].prefix, |
| 169 plot_information_[kTotalAvailable].time_ms, |
| 170 GetTotalAvailableKbps(), "Available"); |
| 171 } else if (metric == kAvailablePerFlow) { |
| 172 BWE_TEST_LOGGING_PLOT_WITH_NAME( |
| 173 0, plot_information_[kAvailablePerFlow].prefix, |
| 174 plot_information_[kAvailablePerFlow].time_ms, GetAvailablePerFlowKbps(), |
| 175 "Available_per_flow"); |
| 176 } else { |
| 177 PlotLine(metric, plot_information_[metric].prefix, |
| 178 plot_information_[metric].time_ms, |
| 179 plot_information_[metric].value); |
| 180 } |
| 181 plot_information_[metric].last_plot_ms = now_ms_; |
| 182 } |
| 183 |
| 184 void MetricRecorder::PlotLine(int windows_id, |
| 185 const std::string& prefix, |
| 186 int64_t x, |
| 187 int64_t y) { |
| 188 BWE_TEST_LOGGING_PLOT_WITH_NAME(windows_id, prefix, x, y, algorithm_name_); |
| 189 } |
| 190 |
| 191 void MetricRecorder::UpdateTime(int64_t time_ms) { |
| 192 now_ms_ = std::max(now_ms_, time_ms); |
| 193 } |
| 194 |
| 195 void MetricRecorder::UpdateThroughput(int64_t bitrate_kbps, |
| 196 size_t payload_size) { |
| 197 PushThroughputBytes(payload_size, now_ms_); |
| 198 plot_information_[kThroughput].Update(now_ms_, bitrate_kbps); |
| 199 } |
| 200 |
| 201 void MetricRecorder::UpdateDelay(int64_t delay_ms) { |
| 202 PushDelayMs(delay_ms, now_ms_); |
| 203 plot_information_[kDelay].Update(now_ms_, delay_ms); |
| 204 } |
| 205 |
| 206 void MetricRecorder::UpdateLoss(float loss_ratio) { |
| 207 plot_information_[kLoss].Update(now_ms_, loss_ratio); |
| 208 } |
| 209 |
| 210 void MetricRecorder::UpdateObjective() { |
| 211 plot_information_[kObjective].Update(now_ms_, ObjectiveFunction()); |
| 212 } |
| 213 |
| 214 uint32_t MetricRecorder::GetTotalAvailableKbps() { |
| 215 return link_share_->TotalAvailableKbps(); |
| 216 } |
| 217 |
| 218 uint32_t MetricRecorder::GetAvailablePerFlowKbps() { |
| 219 return link_share_->AvailablePerFlowKbps(flow_id_); |
| 220 } |
| 221 |
| 222 uint32_t MetricRecorder::GetSendingEstimateKbps() { |
| 223 return packet_sender_->TargetBitrateKbps(); |
| 224 } |
| 225 |
| 226 void MetricRecorder::PushDelayMs(int64_t delay_ms, int64_t arrival_time_ms) { |
| 227 if (ShouldRecord(arrival_time_ms)) { |
| 228 delays_ms_.push_back(delay_ms); |
| 229 } |
| 230 } |
| 231 |
| 232 void MetricRecorder::PushThroughputBytes(size_t payload_size, |
| 233 int64_t arrival_time_ms) { |
| 234 if (ShouldRecord(arrival_time_ms)) { |
| 235 throughput_bytes_.push_back(payload_size); |
| 236 |
| 237 int64_t current_available_per_flow_kbps = |
| 238 static_cast<int64_t>(GetAvailablePerFlowKbps()); |
| 239 |
| 240 int64_t current_bitrate_diff_kbps = |
| 241 static_cast<int64_t>(GetSendingEstimateKbps()) - |
| 242 current_available_per_flow_kbps; |
| 243 |
| 244 // time_ms was still not updated here. |
| 245 weighted_estimate_error_.push_back( |
| 246 ((current_bitrate_diff_kbps + last_unweighted_estimate_error_) * |
| 247 (arrival_time_ms - plot_information_[kThroughput].time_ms)) / |
| 248 2); |
| 249 |
| 250 optimal_throughput_bits_ += |
| 251 ((current_available_per_flow_kbps + |
| 252 last_available_bitrate_per_flow_kbps_) * |
| 253 (arrival_time_ms - plot_information_[kThroughput].time_ms)) / |
| 254 2; |
| 255 |
| 256 last_available_bitrate_per_flow_kbps_ = current_available_per_flow_kbps; |
| 257 } |
| 258 } |
| 259 |
| 260 bool MetricRecorder::ShouldRecord(int64_t arrival_time_ms) { |
| 261 if (arrival_time_ms >= start_computing_metrics_ms_) { |
| 262 if (!started_computing_metrics_) { |
| 263 start_computing_metrics_ms_ = arrival_time_ms; |
| 264 now_ms_ = arrival_time_ms; |
| 265 started_computing_metrics_ = true; |
| 266 } |
| 267 return true; |
| 268 } else { |
| 269 return false; |
| 270 } |
| 271 } |
| 272 |
| 273 // The weighted_estimate_error_ was weighted based on time windows. |
| 274 // This function scales back the result before plotting. |
| 275 double MetricRecorder::Renormalize(double x) { |
| 276 size_t num_packets_received = delays_ms_.size(); |
| 277 return (x * num_packets_received) / now_ms_; |
| 278 } |
| 279 |
| 280 inline double U(int64_t x, double alpha) { |
| 281 if (alpha == 1.0) { |
| 282 return log(static_cast<double>(x)); |
| 283 } |
| 284 return pow(static_cast<double>(x), 1.0 - alpha) / (1.0 - alpha); |
| 285 } |
| 286 |
| 287 inline double U(size_t x, double alpha) { |
| 288 return U(static_cast<int64_t>(x), alpha); |
| 289 } |
| 290 |
| 291 // TODO(magalhaesc): Update ObjectiveFunction. |
| 292 double MetricRecorder::ObjectiveFunction() { |
| 293 const double kDelta = 0.15; // Delay penalty factor. |
| 294 const double kAlpha = 1.0; |
| 295 const double kBeta = 1.0; |
| 296 |
| 297 double throughput_metric = U(Sum(throughput_bytes_), kAlpha); |
| 298 double delay_penalty = kDelta * U(Sum(delays_ms_), kBeta); |
| 299 |
| 300 return throughput_metric - delay_penalty; |
| 301 } |
| 302 |
| 303 void MetricRecorder::PlotThroughputHistogram(const std::string& title, |
| 304 const std::string& bwe_name, |
| 305 int num_flows, |
| 306 int64_t extra_offset_ms, |
| 307 const std::string optimum_id) { |
| 308 size_t num_packets_received = delays_ms_.size(); |
| 309 |
| 310 int64_t duration_ms = now_ms_ - start_computing_metrics_ms_ - extra_offset_ms; |
| 311 |
| 312 double average_bitrate_kbps = |
| 313 static_cast<double>(8 * Sum(throughput_bytes_) / duration_ms); |
| 314 |
| 315 double optimal_bitrate_per_flow_kbps = |
| 316 static_cast<double>(optimal_throughput_bits_ / duration_ms); |
| 317 |
| 318 std::vector<int64_t> positive = PositiveFilter(weighted_estimate_error_); |
| 319 std::vector<int64_t> negative = NegativeFilter(weighted_estimate_error_); |
| 320 |
| 321 double p_error = Renormalize(NormLp(positive, num_packets_received, 1.0)); |
| 322 double n_error = Renormalize(NormLp(negative, num_packets_received, 1.0)); |
| 323 |
| 324 // Prevent the error to be too close to zero (plotting issue). |
| 325 double extra_error = average_bitrate_kbps / 500; |
| 326 |
| 327 std::string optimum_title = |
| 328 optimum_id.empty() ? "optimal_bitrate" : "optimal_bitrates#" + optimum_id; |
| 329 |
| 330 BWE_TEST_LOGGING_LABEL(4, title, "average_bitrate_(kbps)", num_flows); |
| 331 BWE_TEST_LOGGING_LIMITERRORBAR( |
| 332 4, bwe_name, average_bitrate_kbps, |
| 333 average_bitrate_kbps - n_error - extra_error, |
| 334 average_bitrate_kbps + p_error + extra_error, "estimate_error", |
| 335 optimal_bitrate_per_flow_kbps, optimum_title, flow_id_); |
| 336 |
| 337 RTC_UNUSED(p_error); |
| 338 RTC_UNUSED(n_error); |
| 339 RTC_UNUSED(extra_error); |
| 340 RTC_UNUSED(optimal_bitrate_per_flow_kbps); |
| 341 } |
| 342 |
| 343 void MetricRecorder::PlotThroughputHistogram(const std::string& title, |
| 344 const std::string& bwe_name, |
| 345 int num_flows, |
| 346 int64_t extra_offset_ms) { |
| 347 PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, ""); |
| 348 } |
| 349 |
| 350 void MetricRecorder::PlotDelayHistogram(const std::string& title, |
| 351 const std::string& bwe_name, |
| 352 int num_flows) { |
| 353 size_t num_packets_received = delays_ms_.size(); |
| 354 double average_delay_ms = Average(delays_ms_, num_packets_received); |
| 355 |
| 356 // Prevent the error to be too close to zero (plotting issue). |
| 357 double extra_error = average_delay_ms / 500; |
| 358 |
| 359 double tenth_sigma_ms = |
| 360 StandardDeviation(delays_ms_, num_packets_received) / 10.0 + extra_error; |
| 361 |
| 362 BWE_TEST_LOGGING_LABEL(5, title, "average_delay_(ms)", num_flows) |
| 363 BWE_TEST_LOGGING_ERRORBAR( |
| 364 5, bwe_name, average_delay_ms, average_delay_ms - tenth_sigma_ms, |
| 365 average_delay_ms + tenth_sigma_ms, "sigma/10", flow_id_); |
| 366 |
| 367 RTC_UNUSED(tenth_sigma_ms); |
| 368 } |
| 369 |
| 370 void MetricRecorder::PlotLossHistogram(const std::string& title, |
| 371 const std::string& bwe_name, |
| 372 int num_flows, |
| 373 float global_loss_ratio) { |
| 374 BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows) |
| 375 BWE_TEST_LOGGING_BAR(6, bwe_name, 100.0f * global_loss_ratio, flow_id_); |
| 376 } |
| 377 |
| 378 void MetricRecorder::PlotObjectiveHistogram(const std::string& title, |
| 379 const std::string& bwe_name, |
| 380 int num_flows) { |
| 381 BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows) |
| 382 BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), flow_id_); |
| 383 } |
| 384 |
| 385 } // namespace bwe |
| 386 } // namespace testing |
| 387 } // namespace webrtc |
OLD | NEW |