Evaluation tests

webrtc/modules/remote_bitrate_estimator/test/bwe_test.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test.h"
 
+#include <sstream>
+
 #include "webrtc/base/common.h"
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/modules/interface/module_common_types.h"
 #include "webrtc/modules/remote_bitrate_estimator/test/bwe_test_framework.h"
+#include "webrtc/modules/remote_bitrate_estimator/test/metric_recorder.h"
 #include "webrtc/modules/remote_bitrate_estimator/test/packet_receiver.h"
 #include "webrtc/modules/remote_bitrate_estimator/test/packet_sender.h"
 #include "webrtc/system_wrappers/interface/clock.h"
 #include "webrtc/test/testsupport/perf_test.h"
 
 using std::string;
 using std::vector;
 
 namespace webrtc {
 namespace testing {
@@ skipping 55 matching lines @@
   for (; it != queue_.end(); ++it) {
     if ((*it)->send_time_us() > end_of_batch_time_us) {
       break;
     }
   }
   Packets to_transfer;
   to_transfer.splice(to_transfer.begin(), queue_, queue_.begin(), it);
   batch->merge(to_transfer, DereferencingComparator<Packet>);
 }
 
-BweTest::BweTest()
-    : run_time_ms_(0), time_now_ms_(-1), simulation_interval_ms_(-1) {
+// Plot link capacity by default.
+BweTest::BweTest() : BweTest(true) {
+}
+
+BweTest::BweTest(bool plot_capacity)
+    : run_time_ms_(0),
+      time_now_ms_(-1),
+      simulation_interval_ms_(-1),
+      plot_total_available_capacity_(plot_capacity) {
   links_.push_back(&uplink_);
   links_.push_back(&downlink_);
 }
 
 BweTest::~BweTest() {
   for (Packet* packet : packets_)
     delete packet;
 }
 
 void BweTest::SetUp() {
@@ skipping 123 matching lines @@
     fairness_index /= flow_throughput_kbps.size() * squared_bitrate_sum;
   }
   webrtc::test::PrintResult("BwePerformance", GetTestName(), "Fairness",
                             fairness_index * 100, "%", false);
 }
 
 void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
                               size_t num_media_flows,
                               size_t num_tcp_flows,
                               int64_t run_time_seconds,
-                              int capacity_kbps,
-                              int max_delay_ms) {
+                              uint32_t capacity_kbps,
+                              int64_t max_delay_ms,
+                              int64_t rtt_ms,
+                              int64_t max_jitter_ms,
+                              const int64_t* offsets_ms) {
+  RunFairnessTest(bwe_type, num_media_flows, num_tcp_flows, run_time_seconds,
+                  capacity_kbps, max_delay_ms, rtt_ms, max_jitter_ms,
+                  offsets_ms, "Fairness_test", bwe_names[bwe_type]);
+}
+
+void BweTest::RunFairnessTest(BandwidthEstimatorType bwe_type,
+                              size_t num_media_flows,
+                              size_t num_tcp_flows,
+                              int64_t run_time_seconds,
+                              uint32_t capacity_kbps,
+                              int64_t max_delay_ms,
+                              int64_t rtt_ms,
+                              int64_t max_jitter_ms,
+                              const int64_t* offsets_ms,
+                              const std::string& title,
+                              const std::string& flow_name) {
   std::set<int> all_flow_ids;
   std::set<int> media_flow_ids;
   std::set<int> tcp_flow_ids;
   int next_flow_id = 0;
   for (size_t i = 0; i < num_media_flows; ++i) {
     media_flow_ids.insert(next_flow_id);
     all_flow_ids.insert(next_flow_id);
     ++next_flow_id;
   }
   for (size_t i = 0; i < num_tcp_flows; ++i) {
     tcp_flow_ids.insert(next_flow_id);
     all_flow_ids.insert(next_flow_id);
     ++next_flow_id;
   }
 
   std::vector<VideoSource*> sources;
   std::vector<PacketSender*> senders;
+  std::vector<MetricRecorder*> metric_recorders;
 
-  size_t i = 1;
+  int64_t max_offset_ms = 0;
+
   for (int media_flow : media_flow_ids) {
-    // Streams started 20 seconds apart to give them different advantage when
-    // competing for the bandwidth.
-    const int64_t kFlowStartOffsetMs = i++ * (rand() % 10000);
     sources.push_back(new AdaptiveVideoSource(media_flow, 30, 300, 0,
-                                              kFlowStartOffsetMs));
+                                              offsets_ms[media_flow]));
     senders.push_back(new PacedVideoSender(&uplink_, sources.back(), bwe_type));
+    max_offset_ms = std::max(max_offset_ms, offsets_ms[media_flow]);
   }
 
-  const int64_t kTcpStartOffsetMs = 5000;
-  for (int tcp_flow : tcp_flow_ids)
-    senders.push_back(new TcpSender(&uplink_, tcp_flow, kTcpStartOffsetMs));
+  for (int tcp_flow : tcp_flow_ids) {
+    senders.push_back(new TcpSender(&uplink_, tcp_flow, offsets_ms[tcp_flow]));
+    max_offset_ms = std::max(max_offset_ms, offsets_ms[tcp_flow]);
+  }
 
   ChokeFilter choke(&uplink_, all_flow_ids);
   choke.set_capacity_kbps(capacity_kbps);
   choke.set_max_delay_ms(max_delay_ms);
+  LinkShare link_share(&choke);
 
+  int64_t one_way_delay_ms = rtt_ms / 2;
   DelayFilter delay_uplink(&uplink_, all_flow_ids);
-  delay_uplink.SetOneWayDelayMs(25);
+  delay_uplink.SetOneWayDelayMs(one_way_delay_ms);
+
+  JitterFilter jitter(&uplink_, all_flow_ids);
+  jitter.SetMaxJitter(max_jitter_ms);
 
   std::vector<RateCounterFilter*> rate_counters;
   for (int flow : all_flow_ids) {
     rate_counters.push_back(
         new RateCounterFilter(&uplink_, flow, "receiver_input"));
   }
 
   RateCounterFilter total_utilization(&uplink_, all_flow_ids,
                                       "total_utilization");
 
   std::vector<PacketReceiver*> receivers;
-  i = 0;
+  // Delays is being plotted only for the first flow.
+  // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
   for (int media_flow : media_flow_ids) {
-    receivers.push_back(
-        new PacketReceiver(&uplink_, media_flow, bwe_type, i++ == 0, false));
+    metric_recorders.push_back(
+        new MetricRecorder(bwe_names[bwe_type], static_cast<int>(media_flow),
+                           senders[media_flow], &link_share));
+    receivers.push_back(new PacketReceiver(&uplink_, media_flow, bwe_type,
+                                           media_flow == 0, false,
+                                           metric_recorders[media_flow]));
+    metric_recorders[media_flow]->set_plot_available_capacity(
+        media_flow == 0 && plot_total_available_capacity_);
+    metric_recorders[media_flow]->set_start_computing_metrics_ms(max_offset_ms);
   }
+  // Delays is not being plotted only for TCP flows. To plot all of them,
+  // replace first "false" occurence with "true" on new PacketReceiver().
   for (int tcp_flow : tcp_flow_ids) {
-    receivers.push_back(
-        new PacketReceiver(&uplink_, tcp_flow, kTcpEstimator, false, false));
+    metric_recorders.push_back(
+        new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(tcp_flow),
+                           senders[tcp_flow], &link_share));
+    receivers.push_back(new PacketReceiver(&uplink_, tcp_flow, kTcpEstimator,
+                                           false, false,
+                                           metric_recorders[tcp_flow]));
+    metric_recorders[tcp_flow]->set_plot_available_capacity(
+        tcp_flow == 0 && plot_total_available_capacity_);
   }
 
   DelayFilter delay_downlink(&downlink_, all_flow_ids);
-  delay_downlink.SetOneWayDelayMs(25);
+  delay_downlink.SetOneWayDelayMs(one_way_delay_ms);
 
   RunFor(run_time_seconds * 1000);
 
   std::map<int, Stats<double>> flow_throughput_kbps;
   for (RateCounterFilter* rate_counter : rate_counters) {
     int flow_id = *rate_counter->flow_ids().begin();
     flow_throughput_kbps[flow_id] = rate_counter->GetBitrateStats();
   }
 
   std::map<int, Stats<double>> flow_delay_ms;
   for (PacketReceiver* receiver : receivers) {
     int flow_id = *receiver->flow_ids().begin();
     flow_delay_ms[flow_id] = receiver->GetDelayStats();
   }
 
   PrintResults(capacity_kbps, total_utilization.GetBitrateStats(),
                flow_delay_ms, flow_throughput_kbps);
 
+  for (int i : all_flow_ids) {
+    metric_recorders[i]->PlotThroughputHistogram(
+        title, flow_name, static_cast<int>(num_media_flows), 0);
+
+    metric_recorders[i]->PlotLossHistogram(title, flow_name,
+                                           static_cast<int>(num_media_flows),
+                                           receivers[i]->GlobalPacketLoss());
+  }
+
+  // Pointless to show delay histogram for TCP flow.
+  for (int i : media_flow_ids) {
+    metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type],
+                                            static_cast<int>(num_media_flows),
+                                            one_way_delay_ms);
+    BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], one_way_delay_ms, i);
+  }
+
   for (VideoSource* source : sources)
     delete source;
   for (PacketSender* sender : senders)
     delete sender;
   for (RateCounterFilter* rate_counter : rate_counters)
     delete rate_counter;
   for (PacketReceiver* receiver : receivers)
     delete receiver;
-}
+  for (MetricRecorder* recorder : metric_recorders)
+    delete recorder;
+}
+
+void BweTest::RunChoke(BandwidthEstimatorType bwe_type,
+                       std::vector<int> capacities_kbps) {
+  int flow_id = bwe_type;
+  AdaptiveVideoSource source(flow_id, 30, 300, 0, 0);
+  VideoSender sender(&uplink_, &source, bwe_type);
+  ChokeFilter choke(&uplink_, flow_id);
+  LinkShare link_share(&choke);
+  MetricRecorder metric_recorder(bwe_names[bwe_type], flow_id, &sender,
+                                 &link_share);
+  PacketReceiver receiver(&uplink_, flow_id, bwe_type, true, false,
+                          &metric_recorder);
+  metric_recorder.set_plot_available_capacity(plot_total_available_capacity_);
+
+  choke.set_max_delay_ms(500);
+  const int64_t kRunTimeMs = 60 * 1000;
+
+  std::stringstream title("Choke");
+  char delimiter = '_';
+
+  for (auto it = capacities_kbps.begin(); it != capacities_kbps.end(); ++it) {
+    choke.set_capacity_kbps(*it);
+    RunFor(kRunTimeMs);
+    title << delimiter << (*it);
+    delimiter = '-';
+  }
+
+  title << "_kbps,_" << (kRunTimeMs / 1000) << "s_each";
+  metric_recorder.PlotThroughputHistogram(title.str(), bwe_names[bwe_type], 1,
+                                          0);
+  metric_recorder.PlotDelayHistogram(title.str(), bwe_names[bwe_type], 1, 0);
+  // receiver.PlotLossHistogram(title, bwe_names[bwe_type], 1);
+  // receiver.PlotObjectiveHistogram(title, bwe_names[bwe_type], 1);
+}
+
+// 5.1. Single Video and Audio media traffic, forward direction.
+void BweTest::RunVariableCapacity1SingleFlow(BandwidthEstimatorType bwe_type) {
+  int flow_id = bwe_type;  // Arbitrary value.

[stefan-webrtc, 2015/07/23 11:33:33]

Hm, looks weird. Why not just 0?

[magalhaesc, 2015/07/23 12:11:53]

Done.

+  AdaptiveVideoSource source(flow_id, 30, 300, 0, 0);
+  PacedVideoSender sender(&uplink_, &source, bwe_type);
+
+  DefaultEvaluationFilter up_filter(&uplink_, flow_id);
+  DelayFilter down_filter(&downlink_, flow_id);
+
+  LinkShare link_share(&(up_filter.choke));
+  MetricRecorder metric_recorder(bwe_names[bwe_type], flow_id, &sender,
+                                 &link_share);
+
+  PacketReceiver receiver(&uplink_, flow_id, bwe_type, true, true,
+                          &metric_recorder);
+
+  metric_recorder.set_plot_available_capacity(plot_total_available_capacity_);
+
+  down_filter.SetOneWayDelayMs(kOneWayDelayMs);
+
+  // Test also with one way propagation delay = 100ms.
+  // up_filter.delay.SetOneWayDelayMs(100);
+  // down_filter.SetOneWayDelayMs(100);
+
+  up_filter.choke.set_capacity_kbps(1000);
+  RunFor(40 * 1000);  // 0-40s.
+  up_filter.choke.set_capacity_kbps(2500);
+  RunFor(20 * 1000);  // 40-60s.
+  up_filter.choke.set_capacity_kbps(600);
+  RunFor(20 * 1000);  // 60-80s.
+  up_filter.choke.set_capacity_kbps(1000);
+  RunFor(20 * 1000);  // 80-100s.
+
+  std::string title("5.1_Variable_capacity_single_flow");
+  metric_recorder.PlotThroughputHistogram(title, bwe_names[bwe_type], 1, 0);
+  // metric_recorder.PlotDelayHistogram(title, bwe_names[bwe_type], 1,
+  //                                    kOneWayDelayMs);
+  // metric_recorder.PlotLossHistogram(title, bwe_names[bwe_type], 1,
+  //                                   receiver.GlobalPacketLoss());
+  // BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs,
+  // flow_id);

[stefan-webrtc, 2015/07/23 11:33:33]

Delete or uncomment.

[magalhaesc, 2015/07/23 12:11:53]

Done.

+}
+
+// 5.2. Two forward direction competing flows, variable capacity.
+void BweTest::RunVariableCapacity2MultipleFlows(BandwidthEstimatorType bwe_type,
+                                                size_t num_flows) {
+  std::vector<VideoSource*> sources;
+  std::vector<PacketSender*> senders;
+  std::vector<MetricRecorder*> metric_recorders;
+  std::vector<PacketReceiver*> receivers;
+
+  const int64_t kStartingApartMs = 0;  // Flows initialized simultaneously.
+
+  for (size_t i = 0; i < num_flows; ++i) {
+    sources.push_back(new AdaptiveVideoSource(static_cast<int>(i), 30, 300, 0,
+                                              i * kStartingApartMs));
+    senders.push_back(new VideoSender(&uplink_, sources[i], bwe_type));
+  }
+
+  FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(num_flows - 1));
+
+  DefaultEvaluationFilter up_filter(&uplink_, flow_ids);
+  LinkShare link_share(&(up_filter.choke));
+  DelayFilter down_filter(&downlink_, flow_ids);

[stefan-webrtc, 2015/07/23 11:33:33]

I think it would make sense to move the down_filter to after the receivers are
instantiated to make it more clear that they are on the downlink (after
receivers).

[magalhaesc, 2015/07/23 12:11:53]

Right, done.

+
+  RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization");
+
+  // Delays is being plotted only for the first flow.
+  // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
+  for (size_t i = 0; i < num_flows; ++i) {
+    metric_recorders.push_back(new MetricRecorder(
+        bwe_names[bwe_type], static_cast<int>(i), senders[i], &link_share));
+
+    receivers.push_back(new PacketReceiver(&uplink_, static_cast<int>(i),
+                                           bwe_type, i == 0, false,
+                                           metric_recorders[i]));
+    metric_recorders[i]->set_plot_available_capacity(
+        i == 0 && plot_total_available_capacity_);
+  }
+
+  down_filter.SetOneWayDelayMs(kOneWayDelayMs);
+  // Test also with one way propagation delay = 100ms.
+  // up_filter.delay.SetOneWayDelayMs(100);
+  // down_filter.SetOneWayDelayMs(100);
+
+  up_filter.choke.set_capacity_kbps(4000);
+  RunFor(25 * 1000);  // 0-25s.
+  up_filter.choke.set_capacity_kbps(2000);
+  RunFor(25 * 1000);  // 25-50s.
+  up_filter.choke.set_capacity_kbps(3500);
+  RunFor(25 * 1000);  // 50-75s.
+  up_filter.choke.set_capacity_kbps(1000);
+  RunFor(25 * 1000);  // 75-100s.
+  up_filter.choke.set_capacity_kbps(2000);
+  RunFor(25 * 1000);  // 100-125s.
+
+  std::string title("5.2_Variable_capacity_two_flows");
+  for (size_t i = 0; i < num_flows; ++i) {
+    metric_recorders[i]->PlotThroughputHistogram(title, bwe_names[bwe_type],
+                                                 num_flows, 0);
+    metric_recorders[i]->PlotDelayHistogram(title, bwe_names[bwe_type],
+                                            num_flows, kOneWayDelayMs);
+    metric_recorders[i]->PlotLossHistogram(title, bwe_names[bwe_type],
+                                           num_flows,
+                                           receivers[i]->GlobalPacketLoss());
+    BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
+  }
+
+  for (VideoSource* source : sources)
+    delete source;
+  for (PacketSender* sender : senders)
+    delete sender;
+  for (MetricRecorder* recorder : metric_recorders)
+    delete recorder;
+  for (PacketReceiver* receiver : receivers)
+    delete receiver;
+}
+
+// 5.3. Bi-directional RMCAT flows.
+void BweTest::RunBidirectionalFlow(BandwidthEstimatorType bwe_type) {
+  enum direction { kForward = 0, kBackward };
+  const size_t kNumFlows = 2;
+  rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows];
+  rtc::scoped_ptr<VideoSender> senders[kNumFlows];
+  rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows];
+  rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
+
+  sources[kForward].reset(new AdaptiveVideoSource(kForward, 30, 300, 0, 0));
+  senders[kForward].reset(
+      new VideoSender(&uplink_, sources[kForward].get(), bwe_type));
+
+  sources[kBackward].reset(new AdaptiveVideoSource(kBackward, 30, 300, 0, 0));
+  senders[kBackward].reset(
+      new VideoSender(&downlink_, sources[kBackward].get(), bwe_type));
+
+  DefaultEvaluationFilter up_filter(&uplink_, kForward);
+  DefaultEvaluationFilter down_filter(&downlink_, kBackward);
+  LinkShare up_link_share(&(up_filter.choke));
+  LinkShare down_link_share(&(down_filter.choke));
+
+  metric_recorders[kForward].reset(new MetricRecorder(
+      bwe_names[bwe_type], kForward, senders[kForward].get(), &up_link_share));
+  receivers[kForward].reset(
+      new PacketReceiver(&uplink_, kForward, bwe_type, true, false,
+                         metric_recorders[kForward].get()));
+
+  metric_recorders[kForward].get()->set_plot_available_capacity(
+      plot_total_available_capacity_);
+
+  metric_recorders[kBackward].reset(
+      new MetricRecorder(bwe_names[bwe_type], kBackward,
+                         senders[kBackward].get(), &down_link_share));
+  receivers[kBackward].reset(
+      new PacketReceiver(&downlink_, kBackward, bwe_type, true, false,
+                         metric_recorders[kBackward].get()));
+
+  metric_recorders[kBackward].get()->set_plot_available_capacity(
+      plot_total_available_capacity_);
+
+  // Test also with one way propagation delay = 100ms.
+  // up_filter.delay.SetOneWayDelayMs(100);
+  // down_filter.delay.SetOneWayDelayMs(100);
+
+  up_filter.choke.set_capacity_kbps(2000);
+  down_filter.choke.set_capacity_kbps(2000);
+  RunFor(20 * 1000);  // 0-20s.
+
+  up_filter.choke.set_capacity_kbps(1000);
+  RunFor(15 * 1000);  // 20-35s.
+
+  down_filter.choke.set_capacity_kbps(800);
+  RunFor(5 * 1000);  // 35-40s.
+
+  up_filter.choke.set_capacity_kbps(500);
+  RunFor(20 * 1000);  // 40-60s.
+
+  up_filter.choke.set_capacity_kbps(2000);
+  RunFor(10 * 1000);  // 60-70s.
+
+  down_filter.choke.set_capacity_kbps(2000);
+  RunFor(30 * 1000);  // 70-100s.
+
+  std::string title("5.3_Bidirectional_flows");
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    metric_recorders[i].get()->PlotThroughputHistogram(
+        title, bwe_names[bwe_type], kNumFlows, 0);
+    metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
+                                                  kNumFlows, kOneWayDelayMs);
+    metric_recorders[i].get()->PlotLossHistogram(
+        title, bwe_names[bwe_type], kNumFlows,
+        receivers[i].get()->GlobalPacketLoss());
+    BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
+  }
+}
+
+// 5.4. Three forward direction competing flows, constant capacity.
+void BweTest::RunSelfFairness(BandwidthEstimatorType bwe_type) {
+  const int kNumRmcatFlows = 3;
+  const int kNumTcpFlows = 0;
+  const int64_t kRunTimeS = 120;
+  const int kLinkCapacity = 3500;
+
+  int64_t max_delay_ms = kMaxQueueingDelayMs;
+  int64_t rtt_ms = 2 * kOneWayDelayMs;
+
+  const int64_t kStartingApartMs = 20 * 1000;
+  int64_t offsets_ms[kNumRmcatFlows];
+  for (int i = 0; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = kStartingApartMs * i;
+  }
+
+  // Test also with one way propagation delay = 100ms.
+  // rtt_ms = 2 * 100;
+  // Test also with bottleneck queue size = 20ms and 1000ms.
+  // max_delay_ms = 20;
+  // max_delay_ms = 1000;
+
+  std::string title("5.4_Self_fairness_test");
+
+  // Test also with one way propagation delay = 100ms.
+  RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
+                  kLinkCapacity, max_delay_ms, rtt_ms, kMaxJitterMs, offsets_ms,
+                  title, bwe_names[bwe_type]);
+}
+
+// 5.5. Five competing RMCAT flows under different RTTs.
+void BweTest::RunRoundTripTimeFairness(BandwidthEstimatorType bwe_type) {
+  const int kAllFlowIds[] = {0, 1, 2, 3, 4};  // Five RMCAT flows.
+  const int64_t kAllOneWayDelayMs[] = {10, 25, 50, 100, 150};
+  const size_t kNumFlows = ARRAY_SIZE(kAllFlowIds);
+  rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows];
+  rtc::scoped_ptr<VideoSender> senders[kNumFlows];
+  rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows];
+
+  // Flows initialized 10 seconds apart.
+  const int64_t kStartingApartMs = 10 * 1000;
+
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
+                                             i * kStartingApartMs));
+    senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type));
+  }
+
+  ChokeFilter choke_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
+  JitterFilter jitter_filter(&uplink_, CreateFlowIds(kAllFlowIds, kNumFlows));
+  rtc::scoped_ptr<DelayFilter> up_delay_filters[kNumFlows];
+  rtc::scoped_ptr<DelayFilter> down_delay_filters[kNumFlows];
+
+  LinkShare link_share(&choke_filter);
+
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    up_delay_filters[i].reset(new DelayFilter(&uplink_, kAllFlowIds[i]));
+    down_delay_filters[i].reset(new DelayFilter(&downlink_, kAllFlowIds[i]));
+  }
+
+  RateCounterFilter total_utilization(
+      &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization");
+
+  // Delays is being plotted only for the first flow.
+  // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
+  rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    metric_recorders[i].reset(
+        new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i),
+                           senders[i].get(), &link_share));
+
+    receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type,
+                                          i == 0, false,
+                                          metric_recorders[i].get()));
+    metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs *
+                                                              (kNumFlows - 1));
+    metric_recorders[i].get()->set_plot_available_capacity(
+        i == 0 && plot_total_available_capacity_);
+  }
+
+  jitter_filter.SetMaxJitter(kMaxJitterMs);
+  choke_filter.set_max_delay_ms(kMaxQueueingDelayMs);
+
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    up_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]);
+    down_delay_filters[i]->SetOneWayDelayMs(kAllOneWayDelayMs[i]);
+  }
+
+  choke_filter.set_capacity_kbps(3500);
+
+  RunFor(300 * 1000);  // 0-300s.
+
+  std::string title("5.5_Round_Trip_Time_Fairness");
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    metric_recorders[i].get()->PlotThroughputHistogram(
+        title, bwe_names[bwe_type], kNumFlows, 0);
+    metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
+                                                  kNumFlows, kOneWayDelayMs);
+    metric_recorders[i].get()->PlotLossHistogram(
+        title, bwe_names[bwe_type], kNumFlows,
+        receivers[i].get()->GlobalPacketLoss());
+    BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kAllOneWayDelayMs[i],
+                                 i);
+  }
+}
+
+// 5.6. RMCAT Flow competing with a long TCP Flow.
+void BweTest::RunLongTcpFairness(BandwidthEstimatorType bwe_type) {
+  const size_t kNumRmcatFlows = 1;
+  const size_t kNumTcpFlows = 1;
+  const int64_t kRunTimeS = 120;
+  const int kCapacityKbps = 2000;
+  // Tcp starts at t = 0, media flow at t = 5s.
+  const int64_t kOffSetsMs[] = {5000, 0};
+
+  int64_t max_delay_ms = kMaxQueueingDelayMs;
+  int64_t rtt_ms = 2 * kOneWayDelayMs;
+
+  // Test also with one way propagation delay = 100ms.
+  // rtt_ms = 2 * 100;
+  // Test also with bottleneck queue size = 20ms and 1000ms.
+  // max_delay_ms = 20;
+  // max_delay_ms = 1000;
+
+  std::string title("5.6_Long_TCP_Fairness");
+  std::string flow_name(bwe_names[bwe_type] + 'x' + bwe_names[kTcpEstimator]);
+
+  RunFairnessTest(bwe_type, kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
+                  kCapacityKbps, max_delay_ms, rtt_ms, kMaxJitterMs, kOffSetsMs,
+                  title, flow_name);
+}
+
+// 5.7. RMCAT Flows competing with multiple short TCP Flows.
+void BweTest::RunMultipleShortTcpFairness(
+    BandwidthEstimatorType bwe_type,
+    std::vector<int> tcp_file_sizes_bytes,
+    std::vector<int64_t> tcp_starting_times_ms) {
+  // Two RMCAT flows and ten TCP flows.
+  const int kAllRmcatFlowIds[] = {0, 1};
+  const int kAllTcpFlowIds[] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+
+  assert(tcp_starting_times_ms.size() == tcp_file_sizes_bytes.size() &&
+         tcp_starting_times_ms.size() == ARRAY_SIZE(kAllTcpFlowIds));
+
+  const size_t kNumRmcatFlows = ARRAY_SIZE(kAllRmcatFlowIds);
+  const size_t kNumTotalFlows = kNumRmcatFlows + ARRAY_SIZE(kAllTcpFlowIds);
+
+  rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumRmcatFlows];
+  rtc::scoped_ptr<PacketSender> senders[kNumTotalFlows];
+  rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumTotalFlows];
+  rtc::scoped_ptr<PacketReceiver> receivers[kNumTotalFlows];
+
+  // RMCAT Flows are initialized simultaneosly at t=5 seconds.
+  const int64_t kRmcatStartingTimeMs = 5 * 1000;
+  for (size_t id : kAllRmcatFlowIds) {
+    sources[id].reset(new AdaptiveVideoSource(static_cast<int>(id), 30, 300, 0,
+                                              kRmcatStartingTimeMs));
+    senders[id].reset(new VideoSender(&uplink_, sources[id].get(), bwe_type));
+  }
+
+  for (size_t id : kAllTcpFlowIds) {
+    senders[id].reset(new TcpSender(&uplink_, static_cast<int>(id),
+                                    tcp_starting_times_ms[id - kNumRmcatFlows],
+                                    tcp_file_sizes_bytes[id - kNumRmcatFlows]));
+  }
+
+  FlowIds flow_ids = CreateFlowIdRange(0, static_cast<int>(kNumTotalFlows - 1));
+  DefaultEvaluationFilter up_filter(&uplink_, flow_ids);
+  DelayFilter down_filter(&downlink_, flow_ids);
+
+  LinkShare link_share(&(up_filter.choke));
+
+  RateCounterFilter total_utilization(&uplink_, flow_ids, "Total_utilization");
+
+  // Delays is being plotted only for the first flow.
+  // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
+  for (size_t id : kAllRmcatFlowIds) {
+    metric_recorders[id].reset(
+        new MetricRecorder(bwe_names[bwe_type], static_cast<int>(id),
+                           senders[id].get(), &link_share));
+    receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id),
+                                           bwe_type, id == 0, false,
+                                           metric_recorders[id].get()));
+    metric_recorders[id].get()->set_start_computing_metrics_ms(
+        kRmcatStartingTimeMs);
+    metric_recorders[id].get()->set_plot_available_capacity(
+        id == 0 && plot_total_available_capacity_);
+  }
+
+  // Delays is not being plotted only for TCP flows. To plot all of them,
+  // replace first "false" occurence with "true" on new PacketReceiver().
+  for (size_t id : kAllTcpFlowIds) {
+    metric_recorders[id].reset(
+        new MetricRecorder(bwe_names[kTcpEstimator], static_cast<int>(id),
+                           senders[id].get(), &link_share));
+    receivers[id].reset(new PacketReceiver(&uplink_, static_cast<int>(id),
+                                           kTcpEstimator, false, false,
+                                           metric_recorders[id].get()));
+    metric_recorders[id].get()->set_plot_available_capacity(
+        id == 0 && plot_total_available_capacity_);
+  }
+
+  down_filter.SetOneWayDelayMs(kOneWayDelayMs);
+
+  // Test also with one way propagation delay = 100ms.
+  // up_filter.delay.SetOneWayDelayMs(100);
+  // down_filter.SetOneWayDelayms(100);
+
+  // Test also with bottleneck queue size = 20ms and 1000ms.
+  // up_filter.choke.set_max_delay_ms(20);
+  // up_filter.choke.set_max_delay_ms(1000);
+
+  // Test also with no Jitter:
+  // up_filter.jitter.SetMaxJitter(0);
+
+  up_filter.choke.set_capacity_kbps(2000);
+
+  RunFor(300 * 1000);  // 0-300s.
+
+  std::string title("5.7_Multiple_short_TCP_flows");
+  for (size_t id : kAllRmcatFlowIds) {
+    metric_recorders[id].get()->PlotThroughputHistogram(
+        title, bwe_names[bwe_type], kNumRmcatFlows, 0);
+    metric_recorders[id].get()->PlotDelayHistogram(
+        title, bwe_names[bwe_type], kNumRmcatFlows, kOneWayDelayMs);
+    metric_recorders[id].get()->PlotLossHistogram(
+        title, bwe_names[bwe_type], kNumRmcatFlows,
+        receivers[id].get()->GlobalPacketLoss());
+    BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, id);
+  }
+}
+
+// 5.8. Three forward direction competing flows, constant capacity.
+// During the test, one of the flows is paused and later resumed.
+void BweTest::RunPauseResumeFlows(BandwidthEstimatorType bwe_type) {
+  const int kAllFlowIds[] = {0, 1, 2};  // Three RMCAT flows.
+  const size_t kNumFlows = ARRAY_SIZE(kAllFlowIds);
+
+  rtc::scoped_ptr<AdaptiveVideoSource> sources[kNumFlows];
+  rtc::scoped_ptr<VideoSender> senders[kNumFlows];
+  rtc::scoped_ptr<MetricRecorder> metric_recorders[kNumFlows];
+  rtc::scoped_ptr<PacketReceiver> receivers[kNumFlows];
+
+  // Flows initialized simultaneously.
+  const int64_t kStartingApartMs = 0;
+
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    sources[i].reset(new AdaptiveVideoSource(kAllFlowIds[i], 30, 300, 0,
+                                             i * kStartingApartMs));
+    senders[i].reset(new VideoSender(&uplink_, sources[i].get(), bwe_type));
+  }
+
+  DefaultEvaluationFilter filter(&uplink_,
+                                 CreateFlowIds(kAllFlowIds, kNumFlows));
+
+  LinkShare link_share(&(filter.choke));
+
+  RateCounterFilter total_utilization(
+      &uplink_, CreateFlowIds(kAllFlowIds, kNumFlows), "Total_utilization");
+
+  // Delays is being plotted only for the first flow.
+  // To plot all of them, replace "i == 0" with "true" on new PacketReceiver().
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    metric_recorders[i].reset(
+        new MetricRecorder(bwe_names[bwe_type], static_cast<int>(i),
+                           senders[i].get(), &link_share));
+    receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], bwe_type,
+                                          i == 0, false,
+                                          metric_recorders[i].get()));
+    metric_recorders[i].get()->set_start_computing_metrics_ms(kStartingApartMs *
+                                                              (kNumFlows - 1));
+    metric_recorders[i].get()->set_plot_available_capacity(
+        i == 0 && plot_total_available_capacity_);
+  }
+
+  // Test also with one way propagation delay = 100ms.
+  // filter.delay.SetOneWayDelayMs(100);
+  filter.choke.set_capacity_kbps(3500);
+
+  RunFor(40 * 1000);  // 0-40s.
+
+  senders[0].get()->Pause();
+  metric_recorders[0].get()->PauseFlow();
+  RunFor(20 * 1000);  // 40-60s.
+
+  senders[0].get()->Resume();
+  metric_recorders[0].get()->ResumeFlow(20 * 1000);
+  RunFor(60 * 1000);  // 60-120s.
+
+  int64_t paused[] = {20 * 1000, 0, 0};
+
+  // First flow is being paused, hence having a different optimum.
+  const std::string optima_lines[] = {"1", "2", "2"};
+
+  std::string title("5.8_Pause_and_resume_media_flow");
+  for (size_t i = 0; i < kNumFlows; ++i) {
+    metric_recorders[i].get()->PlotThroughputHistogram(
+        title, bwe_names[bwe_type], kNumFlows, paused[i], optima_lines[i]);
+    metric_recorders[i].get()->PlotDelayHistogram(title, bwe_names[bwe_type],
+                                                  kNumFlows, kOneWayDelayMs);
+    metric_recorders[i].get()->PlotLossHistogram(
+        title, bwe_names[bwe_type], kNumFlows,
+        receivers[i].get()->GlobalPacketLoss());
+    BWE_TEST_LOGGING_BASELINEBAR(5, bwe_names[bwe_type], kOneWayDelayMs, i);
+  }
+}
+
+// Following functions are used for randomizing TCP file size and
+// starting time, used on 5.7 RunMultipleShortTcpFairness.
+// They are pseudo-random generators, creating always the same
+// value sequence for a given Random seed.
+
+std::vector<int> BweTest::GetFileSizesBytes(int num_files) {
+  // File size chosen from uniform distribution between [100,1000] kB.
+  const int kMinKbytes = 100;
+  const int kMaxKbytes = 1000;
+
+  Random random(0x12345678);
+  std::vector<int> tcp_file_sizes_bytes;
+
+  while (num_files-- > 0) {
+    tcp_file_sizes_bytes.push_back(random.Rand(kMinKbytes, kMaxKbytes) * 1000);
+  }
+
+  return tcp_file_sizes_bytes;
+}
+
+std::vector<int64_t> BweTest::GetStartingTimesMs(int num_files) {
+  // OFF state behaves as an exp. distribution with mean = 10 seconds.
+  const float kMeanMs = 10000.0f;
+  Random random(0x12345678);
+
+  std::vector<int64_t> tcp_starting_times_ms;
+
+  // Two TCP Flows are initialized simultaneosly at t=0 seconds.
+  for (int i = 0; i < 2; ++i, --num_files) {
+    tcp_starting_times_ms.push_back(0);
+  }
+
+  // Other TCP Flows are initialized in an OFF state.
+  while (num_files-- > 0) {
+    tcp_starting_times_ms.push_back(
+        static_cast<int64_t>(random.Exponential(1.0f / kMeanMs)));
+  }
+
+  return tcp_starting_times_ms;
+}
+
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc