More Simulation Framework features

webrtc/modules/remote_bitrate_estimator/test/packet_receiver.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
 namespace webrtc {
 namespace testing {
 namespace bwe {
 
 PacketReceiver::PacketReceiver(PacketProcessorListener* listener,
                                int flow_id,
                                BandwidthEstimatorType bwe_type,
                                bool plot_delay,
                                bool plot_bwe)
     : PacketProcessor(listener, flow_id, kReceiver),
+      now_ms_(0),
       delay_log_prefix_(),
       metric_log_prefix_(),
       packet_loss_log_prefix_(),
+      available_capacity_log_prefix_(),
+      throughput_log_prefix_(),
       last_delay_plot_ms_(0),
+      last_throughput_plot_ms_(0),
       last_metric_plot_ms_(0),
       last_packet_loss_plot_ms_(0),
+      last_total_capacity_plot_ms_(0),
+      last_per_flow_capacity_plot_ms_(0),
       plot_delay_(plot_delay),
-      // TODO(magalhaesc) Add separated plot_objective_function and
-      // plot_packet_loss parameters to the constructor.
-      plot_objective_function_(plot_delay),
-      plot_packet_loss_(plot_delay),
+      // TODO(magalhaesc): Add following bool parameters to the constructor.
+      plot_throughput_(true),
+      plot_objective_function_(false),
+      plot_packet_loss_(true),
+      plot_total_available_capacity_(true),
+      plot_available_capacity_per_flow_(false),
       bwe_receiver_(CreateBweReceiver(bwe_type, flow_id, plot_bwe)),
-      total_delay_ms_(0),
-      total_throughput_(0),
-      number_packets_(0) {
+      delays_ms_(),
+      throughput_bytes_(),
+      weighted_estimate_error_(),
+      last_unweighted_estimate_error_(0),
+      optimal_throughput_bits_(0),
+      last_total_available_bitrate_kbps_(0),
+      last_available_bitrate_per_flow_kbps_(0),
+      start_computing_metrics_ms_(0),
+      started_computing_metrics_(false),
+      alg_name_(bwe_names[bwe_type]) {
   // Setup the prefix ststd::rings used when logging.
   std::stringstream ss1;
-  ss1 << "Delay_" << flow_id << "#2";
+  ss1 << "Delay_ms_" << flow_id << "#2";
   delay_log_prefix_ = ss1.str();
 
   std::stringstream ss2;
   ss2 << "Objective_function_" << flow_id << "#2";
   metric_log_prefix_ = ss2.str();
 
   std::stringstream ss3;
   ss3 << "Packet_Loss_" << flow_id << "#2";
   packet_loss_log_prefix_ = ss3.str();
+
+  // Plot Available capacity together with throughputs.
+  std::stringstream ss4;
+  ss4 << "Throughput_kbps" << flow_id << "#1";
+  available_capacity_log_prefix_ = ss4.str();
+
+  std::stringstream ss5;
+  ss5 << "Throughput_kbps_" << flow_id << "#2";
+  throughput_log_prefix_ = ss5.str();
 }
 
 PacketReceiver::~PacketReceiver() {
 }
 
 void PacketReceiver::RunFor(int64_t time_ms, Packets* in_out) {
   Packets feedback;
   for (auto it = in_out->begin(); it != in_out->end();) {
     // PacketReceivers are only associated with a single stream, and therefore
     // should only process a single flow id.
     // TODO(holmer): Break this out into a Demuxer which implements both
     // PacketProcessorListener and PacketProcessor.
     BWE_TEST_LOGGING_CONTEXT("Receiver");
     if ((*it)->GetPacketType() == Packet::kMedia &&
         (*it)->flow_id() == *flow_ids().begin()) {
       BWE_TEST_LOGGING_CONTEXT(*flow_ids().begin());
       const MediaPacket* media_packet = static_cast<const MediaPacket*>(*it);
       // We're treating the send time (from previous filter) as the arrival
       // time once packet reaches the estimator.
       int64_t arrival_time_ms = (media_packet->send_time_us() + 500) / 1000;
       int64_t send_time_ms = (media_packet->creation_time_us() + 500) / 1000;
       delay_stats_.Push(arrival_time_ms - send_time_ms);
       PlotDelay(arrival_time_ms, send_time_ms);
+      PlotThroughput(arrival_time_ms);
+      PlotTotalAvailableCapacity(arrival_time_ms);
+      PlotAvailableCapacityPerFlow(arrival_time_ms);
       PlotObjectiveFunction(arrival_time_ms);
       PlotPacketLoss(arrival_time_ms);
 
-      total_delay_ms_ += arrival_time_ms - send_time_ms;
-      total_throughput_ += media_packet->payload_size();
-      ++number_packets_;
+      int64_t current_capacity_per_flow_kbps =
+          static_cast<int64_t>(media_packet->get_capacity_per_flow_kbps());
+
+      if (arrival_time_ms >= start_computing_metrics_ms_) {
+        if (!started_computing_metrics_) {
+          start_computing_metrics_ms_ = arrival_time_ms;
+          now_ms_ = arrival_time_ms;
+          started_computing_metrics_ = true;
+        }
+
+        delays_ms_.push_back(arrival_time_ms - send_time_ms);
+        throughput_bytes_.push_back(media_packet->payload_size());
+
+        int64_t current_bitrate_diff_kbps =
+            static_cast<int64_t>(media_packet->get_sending_estimate_kbps()) -
+            current_capacity_per_flow_kbps;
+
+        // now_ms_ was still not updated here.
+        weighted_estimate_error_.push_back(
+            ((current_bitrate_diff_kbps + last_unweighted_estimate_error_) *
+             (arrival_time_ms - now_ms_)) /
+            2);
+
+        optimal_throughput_bits_ += ((current_capacity_per_flow_kbps +
+                                      last_available_bitrate_per_flow_kbps_) *
+                                     (arrival_time_ms - now_ms_)) /
+                                    2;
+
+        last_unweighted_estimate_error_ = current_bitrate_diff_kbps;
+      }
+
+      last_available_bitrate_per_flow_kbps_ = current_capacity_per_flow_kbps;
+      last_total_available_bitrate_kbps_ =
+          media_packet->get_total_capacity_kbps();
+
+      now_ms_ = std::max(now_ms_, arrival_time_ms);

[stefan-webrtc, 2015/06/25 14:44:05]

Break this plotting stuff out to a method.

[magalhaesc, 2015/07/01 12:48:40]

Done, moved to MetricRecorder

 
       bwe_receiver_->ReceivePacket(arrival_time_ms, *media_packet);
       FeedbackPacket* fb = bwe_receiver_->GetFeedback(arrival_time_ms);
       if (fb)
         feedback.push_back(fb);
       delete media_packet;
       it = in_out->erase(it);
     } else {
       ++it;
     }
   }
   // Insert feedback packets to be sent back to the sender.
   in_out->merge(feedback, DereferencingComparator<Packet>);
 }
 
 void PacketReceiver::PlotDelay(int64_t arrival_time_ms, int64_t send_time_ms) {
   static const int kDelayPlotIntervalMs = 100;
   if (!plot_delay_)
     return;
   if (arrival_time_ms - last_delay_plot_ms_ > kDelayPlotIntervalMs) {
-    BWE_TEST_LOGGING_PLOT(0, delay_log_prefix_, arrival_time_ms,
-                          arrival_time_ms - send_time_ms);
+    BWE_TEST_LOGGING_PLOT_WITH_NAME(0, delay_log_prefix_, arrival_time_ms,
+                                    arrival_time_ms - send_time_ms, alg_name_);
     last_delay_plot_ms_ = arrival_time_ms;
   }
 }
 
+void PacketReceiver::PlotThroughput(int64_t arrival_time_ms) {
+  static const int kThroughputPlotIntervalMs = 1000;
+  if (!plot_throughput_)
+    return;
+  if (arrival_time_ms - last_throughput_plot_ms_ > kThroughputPlotIntervalMs) {
+    BWE_TEST_LOGGING_PLOT_WITH_NAME(0, throughput_log_prefix_, arrival_time_ms,
+                                    bwe_receiver_->RecentKbps(), alg_name_);
+    last_throughput_plot_ms_ = arrival_time_ms;
+  }
+}
+
+void PacketReceiver::PlotTotalAvailableCapacity(int64_t arrival_time_ms) {
+  static const int kAvailableCapacityPerFlowMs = 1000;
+  if (!plot_total_available_capacity_)
+    return;
+  if (arrival_time_ms - last_total_capacity_plot_ms_ >
+      kAvailableCapacityPerFlowMs) {
+    BWE_TEST_LOGGING_PLOT_WITH_NAME(
+        0, available_capacity_log_prefix_, arrival_time_ms,
+        last_total_available_bitrate_kbps_, "Available");
+    last_total_capacity_plot_ms_ = arrival_time_ms;
+  }
+}
+
+void PacketReceiver::PlotAvailableCapacityPerFlow(int64_t arrival_time_ms) {
+  static const int kAvailableCapacityPerFlowMs = 1000;
+  if (!plot_available_capacity_per_flow_)
+    return;
+  if (arrival_time_ms - last_per_flow_capacity_plot_ms_ >
+      kAvailableCapacityPerFlowMs) {
+    BWE_TEST_LOGGING_PLOT_WITH_NAME(
+        0, available_capacity_log_prefix_, arrival_time_ms,
+        last_available_bitrate_per_flow_kbps_, "Available_per_flow");
+    last_per_flow_capacity_plot_ms_ = arrival_time_ms;
+  }
+}
+
+template <typename T>

[stefan-webrtc, 2015/06/25 14:44:05]

All of these methods should go in the beginning and be in under namespace
vector_operation {} I think. 

[magalhaesc, 2015/07/01 12:48:40]

Done, moved to MetricReceiver

+inline T Sum(std::vector<T>& input) {

[stefan-webrtc, 2015/06/25 14:44:05]

No need for inline.

[magalhaesc, 2015/07/01 12:48:40]

Done.

+  T total = 0;
+  for (auto it = input.begin(); it != input.end(); ++it) {
+    total += *it;
+  }
+  return total;
+}
+
+template <typename T>
+inline double Average(std::vector<T>& array, size_t size) {
+  return static_cast<double>(Sum(array)) / size;
+}
+
+template <typename T>
+inline std::vector<T> Abs(std::vector<T>& input) {
+  std::vector<T> output;
+  for (auto it = input.begin(); it != input.end(); ++it) {
+    output.push_back(std::abs(*it));
+  }
+  return output;
+}
+
+template <typename T>
+inline std::vector<double> Pow(std::vector<T>& input, double p) {
+  std::vector<double> output;
+  for (auto it = input.begin(); it != input.end(); ++it) {
+    output.push_back(pow(static_cast<double>(*it), p));
+  }
+  return output;
+}
+
+template <typename T>
+inline double StandardDeviation(std::vector<T>& array, size_t size) {
+  double mean = Average(array, size);
+  std::vector<double> square_values = Pow(array, 2.0);
+  double var = Average(square_values, size) - mean * mean;
+  return sqrt(var);
+}
+
+// Holder mean, Manhattan distance for p=1, EuclidianNorm/sqrt(n) for p=2.
+template <typename T>
+inline double NormLp(std::vector<T>& array, size_t size, double p) {
+  std::vector<T> abs_values = Abs(array);
+  std::vector<double> pow_values = Pow(abs_values, p);
+  return pow(Sum(pow_values) / size, 1.0 / p);
+}
+
+template <typename T>
+inline std::vector<T> PositiveFilter(std::vector<T>& input) {
+  std::vector<T> output(input);
+  for (auto it = output.begin(); it != output.end(); ++it) {
+    (*it) = (*it) > 0 ? (*it) : 0;
+  }
+  return output;
+}
+
+template <typename T>
+inline std::vector<T> NegativeFilter(std::vector<T>& input) {
+  std::vector<T> output(input);
+  for (auto it = output.begin(); it != output.end(); ++it) {
+    (*it) = (*it) < 0 ? -(*it) : 0;
+  }
+  return output;
+}
+
+// The weighted_estimate_error_ was weighted based on time windows.
+// This function scales back the result before plotting.
+double PacketReceiver::Renormalize(double x) {
+  size_t num_packets_received = delays_ms_.size();
+  return (x * num_packets_received) / now_ms_;
+}
+
+inline double U(int64_t x, double alpha) {
+  if (alpha == 1.0) {
+    return log(static_cast<double>(x));
+  }
+  return pow(static_cast<double>(x), 1.0 - alpha) / (1.0 - alpha);
+}
+
+inline double U(size_t x, double alpha) {
+  return U(static_cast<int64_t>(x), alpha);
+}
+
+// TODO(magalhaesc): Update ObjectiveFunction.
 double PacketReceiver::ObjectiveFunction() {

[stefan-webrtc, 2015/06/25 14:44:05]

Not sure we ended up using this? Maybe better to remove?

[magalhaesc, 2015/07/01 12:48:40]

They are now in the MetricRecorder file, so separated from the framework. 
We can remove it or keep it in case we think it could be useful.
WDYT?

-  const double kDelta = 1.0;  // Delay penalty factor.
-  double throughput_metric = log(static_cast<double>(total_throughput_));
-  double delay_penalty = kDelta * log(static_cast<double>(total_delay_ms_));
+  const double kDelta = 0.15;  // Delay penalty factor.
+  const double kAlpha = 1.0;
+  const double kBeta = 1.0;
+
+  double throughput_metric = U(Sum(throughput_bytes_), kAlpha);
+  double delay_penalty = kDelta * U(Sum(delays_ms_), kBeta);
+
   return throughput_metric - delay_penalty;
 }
 
 void PacketReceiver::PlotObjectiveFunction(int64_t arrival_time_ms) {
   static const int kMetricPlotIntervalMs = 1000;
   if (!plot_objective_function_) {
     return;
   }
   if (arrival_time_ms - last_metric_plot_ms_ > kMetricPlotIntervalMs) {
     BWE_TEST_LOGGING_PLOT(1, metric_log_prefix_, arrival_time_ms,
                           ObjectiveFunction());
     last_metric_plot_ms_ = arrival_time_ms;
   }
 }
 
 void PacketReceiver::PlotPacketLoss(int64_t arrival_time_ms) {
   static const int kPacketLossPlotIntervalMs = 500;
   if (!plot_packet_loss_) {
     return;
   }
   if (arrival_time_ms - last_packet_loss_plot_ms_ > kPacketLossPlotIntervalMs) {
-    BWE_TEST_LOGGING_PLOT(2, packet_loss_log_prefix_, arrival_time_ms,
-                          bwe_receiver_->RecentPacketLossRatio());
+    BWE_TEST_LOGGING_PLOT_WITH_NAME(
+        2, "Recent_" + packet_loss_log_prefix_, arrival_time_ms,
+        bwe_receiver_->RecentPacketLossRatio(), alg_name_);
     last_packet_loss_plot_ms_ = arrival_time_ms;
   }
 }
 
+void PacketReceiver::PlotThroughputHistogram(const std::string& title,
+                                             const std::string& bwe_name,
+                                             int num_flows,
+                                             int64_t extra_offset_ms,
+                                             const std::string optimum_id) {
+  size_t num_packets_received = delays_ms_.size();
+
+  int64_t duration_ms = now_ms_ - start_computing_metrics_ms_ - extra_offset_ms;
+
+  double average_bitrate_kbps =
+      static_cast<double>(8 * Sum(throughput_bytes_) / duration_ms);
+
+  double optimal_bitrate_per_flow_kbps =
+      static_cast<double>(optimal_throughput_bits_ / duration_ms);
+
+  std::vector<int64_t> positive = PositiveFilter(weighted_estimate_error_);
+  std::vector<int64_t> negative = NegativeFilter(weighted_estimate_error_);
+
+  double p_error = Renormalize(NormLp(positive, num_packets_received, 1.0));
+  double n_error = Renormalize(NormLp(negative, num_packets_received, 1.0));
+
+  // Prevent the error to be too close to zero (plotting issue).
+  double extra_error = average_bitrate_kbps / 500;
+
+  std::string optimum_title =
+      optimum_id.empty() ? "optimal_bitrate" : "optimal_bitrates#" + optimum_id;
+
+  BWE_TEST_LOGGING_LABEL(4, title, "average_bitrate_(kbps)", num_flows);
+  BWE_TEST_LOGGING_LIMITERRORBAR(
+      4, bwe_name, average_bitrate_kbps,
+      average_bitrate_kbps - n_error - extra_error,
+      average_bitrate_kbps + p_error + extra_error, "estimate_error",
+      optimal_bitrate_per_flow_kbps, optimum_title, *flow_ids().begin());
+
+  // Silencing unused variable compiling error.
+  (void)p_error;
+  (void)n_error;
+  (void)extra_error;
+  (void)optimal_bitrate_per_flow_kbps;

[stefan-webrtc, 2015/06/25 14:44:05]

RTC_UNUSED

[magalhaesc, 2015/07/01 12:48:40]

Done.

+}
+
+void PacketReceiver::PlotThroughputHistogram(const std::string& title,
+                                             const std::string& bwe_name,
+                                             int num_flows,
+                                             int64_t extra_offset_ms) {
+  PlotThroughputHistogram(title, bwe_name, num_flows, extra_offset_ms, "");
+}
+
+void PacketReceiver::PlotDelayHistogram(const std::string& title,
+                                        const std::string& bwe_name,
+                                        int num_flows) {
+  size_t num_packets_received = delays_ms_.size();
+  double average_delay_ms = Average(delays_ms_, num_packets_received);
+
+  // Prevent the error to be too close to zero (plotting issue).
+  double extra_error = average_delay_ms / 500;
+
+  double tenth_sigma_ms =
+      StandardDeviation(delays_ms_, num_packets_received) / 10.0 + extra_error;
+
+  BWE_TEST_LOGGING_LABEL(5, title, "average_delay_(ms)", num_flows)
+  BWE_TEST_LOGGING_ERRORBAR(
+      5, bwe_name, average_delay_ms, average_delay_ms - tenth_sigma_ms,
+      average_delay_ms + tenth_sigma_ms, "sigma/10", *flow_ids().begin());
+
+  // Silencing unused variable compiling error.
+  (void)tenth_sigma_ms;

[stefan-webrtc, 2015/06/25 14:44:05]

RTC_UNUSED

[magalhaesc, 2015/07/01 12:48:40]

Done.

+}
+
+void PacketReceiver::PlotLossHistogram(const std::string& title,
+                                       const std::string& bwe_name,
+                                       int num_flows) {
+  BWE_TEST_LOGGING_LABEL(6, title, "packet_loss_ratio_(%)", num_flows)
+  BWE_TEST_LOGGING_BAR(6, bwe_name,
+                       100.0f * bwe_receiver_->GlobalReceiverPacketLossRatio(),
+                       *flow_ids().begin());
+}
+
+void PacketReceiver::PlotObjectiveHistogram(const std::string& title,
+                                            const std::string& bwe_name,
+                                            int num_flows) {
+  BWE_TEST_LOGGING_LABEL(7, title, "objective_function", num_flows)
+  BWE_TEST_LOGGING_BAR(7, bwe_name, ObjectiveFunction(), *flow_ids().begin());
+}
+
 Stats<double> PacketReceiver::GetDelayStats() const {
   return delay_stats_;
 }
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc