Index: webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc |
diff --git a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc |
index c13f14437a5239be926acfeb7d6b7a772d35f239..90b61f2f89457f7a3fb2c355b13a3caf7f49d6b2 100644 |
--- a/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc |
+++ b/webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc |
@@ -31,24 +31,39 @@ PacketReceiver::PacketReceiver(PacketProcessorListener* listener, |
bool plot_delay, |
bool plot_bwe) |
: PacketProcessor(listener, flow_id, kReceiver), |
+ now_ms_(0), |
delay_log_prefix_(), |
metric_log_prefix_(), |
packet_loss_log_prefix_(), |
+ available_capacity_log_prefix_(), |
+ throughput_log_prefix_(), |
last_delay_plot_ms_(0), |
+ last_throughput_plot_ms_(0), |
last_metric_plot_ms_(0), |
last_packet_loss_plot_ms_(0), |
+ last_total_capacity_plot_ms_(0), |
+ last_per_flow_capacity_plot_ms_(0), |
plot_delay_(plot_delay), |
- // TODO(magalhaesc) Add separated plot_objective_function and |
- // plot_packet_loss parameters to the constructor. |
- plot_objective_function_(plot_delay), |
- plot_packet_loss_(plot_delay), |
+ // TODO(magalhaesc): Add following bool parameters to the constructor. |
+ plot_throughput_(true), |
+ plot_objective_function_(false), |
+ plot_packet_loss_(true), |
+ plot_total_available_capacity_(true), |
+ plot_available_capacity_per_flow_(false), |
bwe_receiver_(CreateBweReceiver(bwe_type, flow_id, plot_bwe)), |
- total_delay_ms_(0), |
- total_throughput_(0), |
- number_packets_(0) { |
+ delays_ms_(), |
+ throughput_bytes_(), |
+ weighted_estimate_error_(), |
+ last_unweighted_estimate_error_(0), |
+ optimal_throughput_bits_(0), |
+ last_total_available_bitrate_kbps_(0), |
+ last_available_bitrate_per_flow_kbps_(0), |
+ start_computing_metrics_ms_(0), |
+ started_computing_metrics_(false), |
+ alg_name_(bwe_names[bwe_type]) { |
// Setup the prefix ststd::rings used when logging. |
std::stringstream ss1; |
- ss1 << "Delay_" << flow_id << "#2"; |
+ ss1 << "Delay_ms_" << flow_id << "#2"; |
delay_log_prefix_ = ss1.str(); |
std::stringstream ss2; |
@@ -58,6 +73,15 @@ PacketReceiver::PacketReceiver(PacketProcessorListener* listener, |
std::stringstream ss3; |
ss3 << "Packet_Loss_" << flow_id << "#2"; |
packet_loss_log_prefix_ = ss3.str(); |
+ |
+ // Plot Available capacity together with throughputs. |
+ std::stringstream ss4; |
+ ss4 << "Throughput_kbps" << flow_id << "#1"; |
+ available_capacity_log_prefix_ = ss4.str(); |
+ |
+ std::stringstream ss5; |
+ ss5 << "Throughput_kbps_" << flow_id << "#2"; |
+ throughput_log_prefix_ = ss5.str(); |
} |
PacketReceiver::~PacketReceiver() { |
@@ -81,12 +105,48 @@ void PacketReceiver::RunFor(int64_t time_ms, Packets* in_out) { |
int64_t send_time_ms = (media_packet->creation_time_us() + 500) / 1000; |
delay_stats_.Push(arrival_time_ms - send_time_ms); |
PlotDelay(arrival_time_ms, send_time_ms); |
+ PlotThroughput(arrival_time_ms); |
+ PlotTotalAvailableCapacity(arrival_time_ms); |
+ PlotAvailableCapacityPerFlow(arrival_time_ms); |
PlotObjectiveFunction(arrival_time_ms); |
PlotPacketLoss(arrival_time_ms); |
- total_delay_ms_ += arrival_time_ms - send_time_ms; |
- total_throughput_ += media_packet->payload_size(); |
- ++number_packets_; |
+ int64_t current_capacity_per_flow_kbps = |
+ static_cast<int64_t>(media_packet->get_capacity_per_flow_kbps()); |
+ |
+ if (arrival_time_ms >= start_computing_metrics_ms_) { |
+ if (!started_computing_metrics_) { |
+ start_computing_metrics_ms_ = arrival_time_ms; |
+ now_ms_ = arrival_time_ms; |
+ started_computing_metrics_ = true; |
+ } |
+ |
+ delays_ms_.push_back(arrival_time_ms - send_time_ms); |
+ throughput_bytes_.push_back(media_packet->payload_size()); |
+ |
+ int64_t current_bitrate_diff_kbps = |
+ static_cast<int64_t>(media_packet->get_sending_estimate_kbps()) - |
+ current_capacity_per_flow_kbps; |
+ |
+ // now_ms_ was still not updated here. |
+ weighted_estimate_error_.push_back( |
+ ((current_bitrate_diff_kbps + last_unweighted_estimate_error_) * |
+ (arrival_time_ms - now_ms_)) / |
+ 2); |
+ |
+ optimal_throughput_bits_ += ((current_capacity_per_flow_kbps + |
+ last_available_bitrate_per_flow_kbps_) * |
+ (arrival_time_ms - now_ms_)) / |
+ 2; |
+ |
+ last_unweighted_estimate_error_ = current_bitrate_diff_kbps; |
+ } |
+ |
+ last_available_bitrate_per_flow_kbps_ = current_capacity_per_flow_kbps; |
+ last_total_available_bitrate_kbps_ = |
+ media_packet->get_total_capacity_kbps(); |
+ |
+ now_ms_ = std::max(now_ms_, arrival_time_ms); |
stefan-webrtc
2015/06/25 14:44:05
Break this plotting stuff out to a method.
magalhaesc
2015/07/01 12:48:40
Done, moved to MetricRecorder
|
bwe_receiver_->ReceivePacket(arrival_time_ms, *media_packet); |
FeedbackPacket* fb = bwe_receiver_->GetFeedback(arrival_time_ms); |
@@ -107,16 +167,142 @@ void PacketReceiver::PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms) { |
if (!plot_delay_) |
return; |
if (arrival_time_ms - last_delay_plot_ms_ > kDelayPlotIntervalMs) { |
- BWE_TEST_LOGGING_PLOT(0, delay_log_prefix_, arrival_time_ms, |
- arrival_time_ms - send_time_ms); |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME(0, delay_log_prefix_, arrival_time_ms, |
+ arrival_time_ms - send_time_ms, alg_name_); |
last_delay_plot_ms_ = arrival_time_ms; |
} |
} |
+void PacketReceiver::PlotThroughput(int64_t arrival_time_ms) { |
+ static const int kThroughputPlotIntervalMs = 1000; |
+ if (!plot_throughput_) |
+ return; |
+ if (arrival_time_ms - last_throughput_plot_ms_ > kThroughputPlotIntervalMs) { |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME(0, throughput_log_prefix_, arrival_time_ms, |
+ bwe_receiver_->RecentKbps(), alg_name_); |
+ last_throughput_plot_ms_ = arrival_time_ms; |
+ } |
+} |
+ |
+void PacketReceiver::PlotTotalAvailableCapacity(int64_t arrival_time_ms) { |
+ static const int kAvailableCapacityPerFlowMs = 1000; |
+ if (!plot_total_available_capacity_) |
+ return; |
+ if (arrival_time_ms - last_total_capacity_plot_ms_ > |
+ kAvailableCapacityPerFlowMs) { |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME( |
+ 0, available_capacity_log_prefix_, arrival_time_ms, |
+ last_total_available_bitrate_kbps_, "Available"); |
+ last_total_capacity_plot_ms_ = arrival_time_ms; |
+ } |
+} |
+ |
+void PacketReceiver::PlotAvailableCapacityPerFlow(int64_t arrival_time_ms) { |
+ static const int kAvailableCapacityPerFlowMs = 1000; |
+ if (!plot_available_capacity_per_flow_) |
+ return; |
+ if (arrival_time_ms - last_per_flow_capacity_plot_ms_ > |
+ kAvailableCapacityPerFlowMs) { |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME( |
+ 0, available_capacity_log_prefix_, arrival_time_ms, |
+ last_available_bitrate_per_flow_kbps_, "Available_per_flow"); |
+ last_per_flow_capacity_plot_ms_ = arrival_time_ms; |
+ } |
+} |
+ |
+template <typename T> |
stefan-webrtc
2015/06/25 14:44:05
All of these methods should go in the beginning an
magalhaesc
2015/07/01 12:48:40
Done, moved to MetricReceiver
|
+inline T Sum(std::vector<T>& input) { |
stefan-webrtc
2015/06/25 14:44:05
No need for inline.
magalhaesc
2015/07/01 12:48:40
Done.
|
+ T total = 0; |
+ for (auto it = input.begin(); it != input.end(); ++it) { |
+ total += *it; |
+ } |
+ return total; |
+} |
+ |
+template <typename T> |
+inline double Average(std::vector<T>& array, size_t size) { |
+ return static_cast<double>(Sum(array)) / size; |
+} |
+ |
+template <typename T> |
+inline std::vector<T> Abs(std::vector<T>& input) { |
+ std::vector<T> output; |
+ for (auto it = input.begin(); it != input.end(); ++it) { |
+ output.push_back(std::abs(*it)); |
+ } |
+ return output; |
+} |
+ |
+template <typename T> |
+inline std::vector<double> Pow(std::vector<T>& input, double p) { |
+ std::vector<double> output; |
+ for (auto it = input.begin(); it != input.end(); ++it) { |
+ output.push_back(pow(static_cast<double>(*it), p)); |
+ } |
+ return output; |
+} |
+ |
+template <typename T> |
+inline double StandardDeviation(std::vector<T>& array, size_t size) { |
+ double mean = Average(array, size); |
+ std::vector<double> square_values = Pow(array, 2.0); |
+ double var = Average(square_values, size) - mean * mean; |
+ return sqrt(var); |
+} |
+ |
+// Holder mean, Manhattan distance for p=1, EuclidianNorm/sqrt(n) for p=2. |
+template <typename T> |
+inline double NormLp(std::vector<T>& array, size_t size, double p) { |
+ std::vector<T> abs_values = Abs(array); |
+ std::vector<double> pow_values = Pow(abs_values, p); |
+ return pow(Sum(pow_values) / size, 1.0 / p); |
+} |
+ |
+template <typename T> |
+inline std::vector<T> PositiveFilter(std::vector<T>& input) { |
+ std::vector<T> output(input); |
+ for (auto it = output.begin(); it != output.end(); ++it) { |
+ (*it) = (*it) > 0 ? (*it) : 0; |
+ } |
+ return output; |
+} |
+ |
+template <typename T> |
+inline std::vector<T> NegativeFilter(std::vector<T>& input) { |
+ std::vector<T> output(input); |
+ for (auto it = output.begin(); it != output.end(); ++it) { |
+ (*it) = (*it) < 0 ? -(*it) : 0; |
+ } |
+ return output; |
+} |
+ |
+// The weighted_estimate_error_ was weighted based on time windows. |
+// This function scales back the result before plotting. |
+double PacketReceiver::Renormalize(double x) { |
+ size_t num_packets_received = delays_ms_.size(); |
+ return (x * num_packets_received) / now_ms_; |
+} |
+ |
+inline double U(int64_t x, double alpha) { |
+ if (alpha == 1.0) { |
+ return log(static_cast<double>(x)); |
+ } |
+ return pow(static_cast<double>(x), 1.0 - alpha) / (1.0 - alpha); |
+} |
+ |
+inline double U(size_t x, double alpha) { |
+ return U(static_cast<int64_t>(x), alpha); |
+} |
+ |
+// TODO(magalhaesc): Update ObjectiveFunction. |
double PacketReceiver::ObjectiveFunction() { |
stefan-webrtc
2015/06/25 14:44:05
Not sure we ended up using this? Maybe better to r
magalhaesc
2015/07/01 12:48:40
They are now in the MetricRecorder file, so separa
|
- const double kDelta = 1.0; // Delay penalty factor. |
- double throughput_metric = log(static_cast<double>(total_throughput_)); |
- double delay_penalty = kDelta * log(static_cast<double>(total_delay_ms_)); |
+ const double kDelta = 0.15; // Delay penalty factor. |
+ const double kAlpha = 1.0; |
+ const double kBeta = 1.0; |
+ |
+ double throughput_metric = U(Sum(throughput_bytes_), kAlpha); |
+ double delay_penalty = kDelta * U(Sum(delays_ms_), kBeta); |
+ |
return throughput_metric - delay_penalty; |
} |
@@ -138,12 +324,98 @@ void PacketReceiver::PlotPacketLoss(int64_t arrival_time_ms) { |
return; |
} |
if (arrival_time_ms - last_packet_loss_plot_ms_ > kPacketLossPlotIntervalMs) { |
- BWE_TEST_LOGGING_PLOT(2, packet_loss_log_prefix_, arrival_time_ms, |
- bwe_receiver_->RecentPacketLossRatio()); |
+ BWE_TEST_LOGGING_PLOT_WITH_NAME( |
+ 2, "Recent_" + packet_loss_log_prefix_, arrival_time_ms, |
+ bwe_receiver_->RecentPacketLossRatio(), alg_name_); |
last_packet_loss_plot_ms_ = arrival_time_ms; |
} |
} |
+void PacketReceiver::PlotThroughputHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows, |
+ int64_t extra_offset_ms, |
+ const std::string optimum_id) { |
+ size_t num_packets_received = delays_ms_.size(); |
+ |
+ int64_t duration_ms = now_ms_ - start_computing_metrics_ms_ - extra_offset_ms; |
+ |
+ double average_bitrate_kbps = |
+ static_cast<double>(8 * Sum(throughput_bytes_) / duration_ms); |
+ |
+ double optimal_bitrate_per_flow_kbps = |
+ static_cast<double>(optimal_throughput_bits_ / duration_ms); |
+ |
+ std::vector<int64_t> positive = PositiveFilter(weighted_estimate_error_); |
+ std::vector<int64_t> negative = NegativeFilter(weighted_estimate_error_); |
+ |
+ double p_error = Renormalize(NormLp(positive, num_packets_received, 1.0)); |
+ double n_error = Renormalize(NormLp(negative, num_packets_received, 1.0)); |
+ |
+ // Prevent the error to be too close to zero (plotting issue). |
+ double extra_error = average_bitrate_kbps / 500; |
+ |
+ std::string optimum_title = |
+ optimum_id.empty() ? "optimal_bitrate" : "optimal_bitrates#" + optimum_id; |
+ |
+ BWE_TEST_LOGGING_LABEL(4, title, "average_bitrate_(kbps)", num_flows); |
+ BWE_TEST_LOGGING_LIMITERRORBAR( |
+ 4, bwe_name, average_bitrate_kbps, |
+ average_bitrate_kbps - n_error - extra_error, |
+ average_bitrate_kbps + p_error + extra_error, "estimate_error", |
+ optimal_bitrate_per_flow_kbps, optimum_title, *flow_ids().begin()); |
+ |
+ // Silencing unused variable compiling error. |
+ (void)p_error; |
+ (void)n_error; |
+ (void)extra_error; |
+ (void)optimal_bitrate_per_flow_kbps; |
stefan-webrtc
2015/06/25 14:44:05
RTC_UNUSED
magalhaesc
2015/07/01 12:48:40
Done.
|
+} |
+ |
+void PacketReceiver::PlotThroughputHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows, |
+ int64_t extra_offset_ms) { |
+ PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, ""); |
+} |
+ |
+void PacketReceiver::PlotDelayHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows) { |
+ size_t num_packets_received = delays_ms_.size(); |
+ double average_delay_ms = Average(delays_ms_, num_packets_received); |
+ |
+ // Prevent the error to be too close to zero (plotting issue). |
+ double extra_error = average_delay_ms / 500; |
+ |
+ double tenth_sigma_ms = |
+ StandardDeviation(delays_ms_, num_packets_received) / 10.0 + extra_error; |
+ |
+ BWE_TEST_LOGGING_LABEL(5, title, "average_delay_(ms)", num_flows) |
+ BWE_TEST_LOGGING_ERRORBAR( |
+ 5, bwe_name, average_delay_ms, average_delay_ms - tenth_sigma_ms, |
+ average_delay_ms + tenth_sigma_ms, "sigma/10", *flow_ids().begin()); |
+ |
+ // Silencing unused variable compiling error. |
+ (void)tenth_sigma_ms; |
stefan-webrtc
2015/06/25 14:44:05
RTC_UNUSED
magalhaesc
2015/07/01 12:48:40
Done.
|
+} |
+ |
+void PacketReceiver::PlotLossHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows) { |
+ BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows) |
+ BWE_TEST_LOGGING_BAR(6, bwe_name, |
+ 100.0f * bwe_receiver_->GlobalReceiverPacketLossRatio(), |
+ *flow_ids().begin()); |
+} |
+ |
+void PacketReceiver::PlotObjectiveHistogram(const std::string& title, |
+ const std::string& bwe_name, |
+ int num_flows) { |
+ BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows) |
+ BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), *flow_ids().begin()); |
+} |
+ |
Stats<double> PacketReceiver::GetDelayStats() const { |
return delay_stats_; |
} |