Evaluation tests

webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc

Index: webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc
diff --git a/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc b/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc
index 2c4cf753b74ce32f28506519990b980648374f2d..41c9b16da2984a84332ec7e750388ea1c1899ea3 100644
--- a/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc
+++ b/webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc
@@ -270,35 +270,230 @@ TEST_P(BweSimulation, SelfFairnessTest) {
 
 TEST_P(BweSimulation, PacedSelfFairness50msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 50);
+  Random random(0x12345678);

[stefan-webrtc, 2015/07/21 12:12:32]

I'd prefer if these (PacedSelfFairness*) by default had a random seed.

[magalhaesc, 2015/07/21 14:46:53]

Done.

[stefan-webrtc, 2015/07/22 09:30:29]

I think you misunderstood. I wanted you to not have a fixed constant for the
seed passed to Random's constructor. Instead do:
Random random(Clock::GetRealTimeClock()->TimeInMicroseconds());

Or possibly use rand() instead?

[magalhaesc, 2015/07/22 11:22:54]

Yes, I had misunderstood. Since we're using the method Random::Rand(low_bound,
up_bound), I'll initialize it with a random seed.

+  const int64_t kAverageOffsetMs = 20 * 1000;
+  const int kNumRmcatFlows = 4;
+  int64_t offsets_ms[kNumRmcatFlows];
+  offsets_ms[0] = static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  for (int i = 1; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = offsets_ms[i - 1] +
+                    static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  }
+  RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 50, 50, 0,
+                  offsets_ms);
 }
 
 TEST_P(BweSimulation, PacedSelfFairness500msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 500);
+  Random random(0x12345678);
+  const int64_t kAverageOffsetMs = 20 * 1000;
+  const int kNumRmcatFlows = 4;
+  int64_t offsets_ms[kNumRmcatFlows];
+  offsets_ms[0] = static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  for (int i = 1; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = offsets_ms[i - 1] +
+                    static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  }
+  RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 500, 50, 0,
+                  offsets_ms);
 }
 
 TEST_P(BweSimulation, PacedSelfFairness1000msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000);
+  Random random(0x12345678);
+  const int64_t kAverageOffsetMs = 20 * 1000;
+  const int kNumRmcatFlows = 4;
+  int64_t offsets_ms[kNumRmcatFlows];
+  offsets_ms[0] = static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  for (int i = 1; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = offsets_ms[i - 1] +
+                    static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  }
+  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000, 50, 0, offsets_ms);
 }
 
 TEST_P(BweSimulation, TcpFairness50msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50);
+  Random random(0x12345678);

[stefan-webrtc, 2015/07/21 12:12:31]

Same for TcpFairness*Test

[magalhaesc, 2015/07/21 14:46:53]

Done.

+  const int64_t kAverageOffsetMs = 20 * 1000;
+  int64_t offset_ms =
+      static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50, 50, 0, &offset_ms);
 }
 
 TEST_P(BweSimulation, TcpFairness500msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500);
+  Random random(0x12345678);
+  const int64_t kAverageOffsetMs = 20 * 1000;
+  int64_t offset_ms =
+      static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500, 50, 0, &offset_ms);
 }
 
 TEST_P(BweSimulation, TcpFairness1000msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000);
+  Random random(0x12345678);
+  const int kAverageOffsetMs = 20 * 1000;
+  int64_t offset_ms =
+      static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000, 50, 0, &offset_ms);
+}
+
+// Following test cases begin with "Evaluation" as a referrence to the

[stefan-webrtc, 2015/07/21 12:12:31]

The following...

[magalhaesc, 2015/07/21 14:46:53]

Done.

+// Internet draft https://tools.ietf.org/html/draft-ietf-rmcat-eval-test-01.
+
+TEST_P(BweSimulation, Evaluation1) {
+  this->RunVariableCapacitySingleFlow(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->.

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+TEST_P(BweSimulation, Evaluation2) {
+  this->RunVariableCapacityTwoFlows(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->.

Can we generalize RunVariableCapacitySingleFlows and RunVariableCapacityTwoFlows
to RunVariableCapacity(GetParam(), N), or do they test different variations in
capacity? In that case maybe they should be named to reflect that instead in
that case?

[magalhaesc, 2015/07/21 14:46:53]

They test different variations in capacity. Names updated.

[stefan-webrtc, 2015/07/22 09:30:29]

Thanks! Maybe also make it possible to pass in the number of flows here, since
it now says MultipleFlows?

[magalhaesc, 2015/07/22 11:22:54]

Yes, it will be better. Done.

+}
+
+TEST_P(BweSimulation, Evaluation3) {
+  this->RunBidirectionalFlow(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->.

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+// 5.4. Three forward direction competing flows, constant capacity.
+TEST_P(BweSimulation, Evaluation4) {
+  const int kNumRmcatFlows = 3;
+  const int kNumTcpFlows = 0;
+  const int64_t kRunTimeS = 120;
+  const int kLinkCapacity = 3500;
+
+  const int64_t kStartingApartMs = 20 * 1000;
+  int64_t offsets_ms[kNumRmcatFlows];
+  for (int i = 0; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = kStartingApartMs * i;
+  }
+
+  // Test also with one way propagation delay = 100ms.
+  RunFairnessTest(GetParam(), kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
+                  kLinkCapacity, kMaxQueueingDelayMs, 2 * kOneWayDelayMs,
+                  kMaxJitterMs, offsets_ms);
+}
+
+TEST_P(BweSimulation, Evaluation5) {
+  this->RunRoundTripTimeFairness(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+TEST_P(BweSimulation, Evaluation6) {
+  this->RunLongTcpFairness(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->

Btw, can this be implemented with RunFairnessTest(GetParam(), 1, 1...) instead?

[magalhaesc, 2015/07/21 14:46:53]

Yes, with a few modifications

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+// Different calls to the Evaluation7 will create the same FileSizes
+// and StartingTimes as long as the seeds remain unchanged. This is essential
+// when calling it with multiple estimators for comparison purposes.
+TEST_P(BweSimulation, Evaluation7) {
+  const int kNumTcpFiles = 10;
+  this->RunMultipleShortTcpFairness(GetParam(),

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->

[magalhaesc, 2015/07/21 14:46:53]

Done.

+                                    BweTest::GetFileSizesBytes(kNumTcpFiles),
+                                    BweTest::GetStartingTimesMs(kNumTcpFiles));
+}
+
+TEST_P(BweSimulation, Evaluation8) {
+  this->RunPauseResumeFlows(GetParam());

[stefan-webrtc, 2015/07/21 12:12:32]

Remove this->

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+// Following test cases begin with "GccComparison" run the
+// evaluation test cases for both GCC and other calling RMCAT.
+
+TEST_P(BweSimulation, GccComparison1) {
+  this->RunVariableCapacitySingleFlow(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this-> here and all below.

[magalhaesc, 2015/07/21 14:46:53]

Done.

+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);

[stefan-webrtc, 2015/07/21 12:12:31]

Maybe make this part of a second constructor of BweTest?

[magalhaesc, 2015/07/21 14:46:53]

Done.

+  gcc_test.RunVariableCapacitySingleFlow(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison2) {
+  this->RunVariableCapacityTwoFlows(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunVariableCapacityTwoFlows(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison3) {
+  this->RunBidirectionalFlow(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunBidirectionalFlow(kFullSendSideEstimator);
+}
+
+// 5.4. Three forward direction competing flows, constant capacity.
+TEST_P(BweSimulation, GccComparison4) {
+  const int kNumRmcatFlows = 3;
+  const int kNumTcpFlows = 0;
+  const int64_t kRunTimeS = 120;
+  const int kLinkCapacity = 3500;
+
+  const int64_t kStartingApartMs = 20 * 1000;
+  int64_t offsets_ms[kNumRmcatFlows];
+  for (int i = 0; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = kStartingApartMs * i;
+  }
+
+  // Test also with one way propagation delay = 100ms.
+  this->RunFairnessTest(GetParam(), kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
+                        kLinkCapacity, kMaxQueueingDelayMs, 2 * kOneWayDelayMs,
+                        kMaxJitterMs, offsets_ms);
+
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunFairnessTest(kFullSendSideEstimator, kNumRmcatFlows, kNumTcpFlows,
+                           kRunTimeS, kLinkCapacity, kMaxQueueingDelayMs,
+                           2 * kOneWayDelayMs, kMaxJitterMs, offsets_ms);
+}
+
+TEST_P(BweSimulation, GccComparison5) {
+  this->RunRoundTripTimeFairness(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunRoundTripTimeFairness(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison6) {
+  this->RunLongTcpFairness(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunLongTcpFairness(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison7) {
+  const int kNumTcpFiles = 10;
+
+  std::vector<int> tcp_file_sizes_bytes =
+      BweTest::GetFileSizesBytes(kNumTcpFiles);
+  std::vector<int64_t> tcp_starting_times_ms =
+      BweTest::GetStartingTimesMs(kNumTcpFiles);
+
+  this->RunMultipleShortTcpFairness(GetParam(), tcp_file_sizes_bytes,
+                                    tcp_starting_times_ms);
+
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunMultipleShortTcpFairness(
+      kFullSendSideEstimator, tcp_file_sizes_bytes, tcp_starting_times_ms);
+}
+
+TEST_P(BweSimulation, GccComparison8) {
+  this->RunPauseResumeFlows(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunPauseResumeFlows(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparisonChoke) {
+  int array[] = {1000, 500, 1000};
+  std::vector<int> capacities_kbps(array, array + 3);
+  this->RunChoke(GetParam(), capacities_kbps);
+
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunChoke(kFullSendSideEstimator, capacities_kbps);
 }
 
 #endif  // BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc
+