Evaluation tests

webrtc/modules/remote_bitrate_estimator/bwe_simulations.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 252 matching lines @@
   for (size_t i = 0; i < kNumFlows; ++i) {
     receivers[i].reset(new PacketReceiver(&uplink_, kAllFlowIds[i], GetParam(),
                                           i == 0, false));
   }
 
   RunFor(30 * 60 * 1000);
 }
 
 TEST_P(BweSimulation, PacedSelfFairness50msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 50);
+  Random random(0x12345678);

[stefan-webrtc, 2015/07/21 12:12:32]

I'd prefer if these (PacedSelfFairness*) by default had a random seed.

[magalhaesc, 2015/07/21 14:46:53]

Done.

[stefan-webrtc, 2015/07/22 09:30:29]

I think you misunderstood. I wanted you to not have a fixed constant for the
seed passed to Random's constructor. Instead do:
Random random(Clock::GetRealTimeClock()->TimeInMicroseconds());

Or possibly use rand() instead?

[magalhaesc, 2015/07/22 11:22:54]

Yes, I had misunderstood. Since we're using the method Random::Rand(low_bound,
up_bound), I'll initialize it with a random seed.

+  const int64_t kAverageOffsetMs = 20 * 1000;
+  const int kNumRmcatFlows = 4;
+  int64_t offsets_ms[kNumRmcatFlows];
+  offsets_ms[0] = static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  for (int i = 1; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = offsets_ms[i - 1] +
+                    static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  }
+  RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 50, 50, 0,
+                  offsets_ms);
 }
 
 TEST_P(BweSimulation, PacedSelfFairness500msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 500);
+  Random random(0x12345678);
+  const int64_t kAverageOffsetMs = 20 * 1000;
+  const int kNumRmcatFlows = 4;
+  int64_t offsets_ms[kNumRmcatFlows];
+  offsets_ms[0] = static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  for (int i = 1; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = offsets_ms[i - 1] +
+                    static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  }
+  RunFairnessTest(GetParam(), kNumRmcatFlows, 0, 1000, 3000, 500, 50, 0,
+                  offsets_ms);
 }
 
 TEST_P(BweSimulation, PacedSelfFairness1000msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000);
+  Random random(0x12345678);
+  const int64_t kAverageOffsetMs = 20 * 1000;
+  const int kNumRmcatFlows = 4;
+  int64_t offsets_ms[kNumRmcatFlows];
+  offsets_ms[0] = static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  for (int i = 1; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = offsets_ms[i - 1] +
+                    static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  }
+  RunFairnessTest(GetParam(), 4, 0, 1000, 3000, 1000, 50, 0, offsets_ms);
 }
 
 TEST_P(BweSimulation, TcpFairness50msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50);
+  Random random(0x12345678);

[stefan-webrtc, 2015/07/21 12:12:31]

Same for TcpFairness*Test

[magalhaesc, 2015/07/21 14:46:53]

Done.

+  const int64_t kAverageOffsetMs = 20 * 1000;
+  int64_t offset_ms =
+      static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 50, 50, 0, &offset_ms);
 }
 
 TEST_P(BweSimulation, TcpFairness500msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500);
+  Random random(0x12345678);
+  const int64_t kAverageOffsetMs = 20 * 1000;
+  int64_t offset_ms =
+      static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 500, 50, 0, &offset_ms);
 }
 
 TEST_P(BweSimulation, TcpFairness1000msTest) {
   srand(Clock::GetRealTimeClock()->TimeInMicroseconds());
-  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000);
+  Random random(0x12345678);
+  const int kAverageOffsetMs = 20 * 1000;
+  int64_t offset_ms =
+      static_cast<int64_t>(random.Rand(0, 2 * kAverageOffsetMs));
+  RunFairnessTest(GetParam(), 1, 1, 1000, 2000, 1000, 50, 0, &offset_ms);
+}
+
+// Following test cases begin with "Evaluation" as a referrence to the

[stefan-webrtc, 2015/07/21 12:12:31]

The following...

[magalhaesc, 2015/07/21 14:46:53]

Done.

+// Internet draft https://tools.ietf.org/html/draft-ietf-rmcat-eval-test-01.
+
+TEST_P(BweSimulation, Evaluation1) {
+  this->RunVariableCapacitySingleFlow(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->.

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+TEST_P(BweSimulation, Evaluation2) {
+  this->RunVariableCapacityTwoFlows(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->.

Can we generalize RunVariableCapacitySingleFlows and RunVariableCapacityTwoFlows
to RunVariableCapacity(GetParam(), N), or do they test different variations in
capacity? In that case maybe they should be named to reflect that instead in
that case?

[magalhaesc, 2015/07/21 14:46:53]

They test different variations in capacity. Names updated.

[stefan-webrtc, 2015/07/22 09:30:29]

Thanks! Maybe also make it possible to pass in the number of flows here, since
it now says MultipleFlows?

[magalhaesc, 2015/07/22 11:22:54]

Yes, it will be better. Done.

+}
+
+TEST_P(BweSimulation, Evaluation3) {
+  this->RunBidirectionalFlow(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->.

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+// 5.4. Three forward direction competing flows, constant capacity.
+TEST_P(BweSimulation, Evaluation4) {
+  const int kNumRmcatFlows = 3;
+  const int kNumTcpFlows = 0;
+  const int64_t kRunTimeS = 120;
+  const int kLinkCapacity = 3500;
+
+  const int64_t kStartingApartMs = 20 * 1000;
+  int64_t offsets_ms[kNumRmcatFlows];
+  for (int i = 0; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = kStartingApartMs * i;
+  }
+
+  // Test also with one way propagation delay = 100ms.
+  RunFairnessTest(GetParam(), kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
+                  kLinkCapacity, kMaxQueueingDelayMs, 2 * kOneWayDelayMs,
+                  kMaxJitterMs, offsets_ms);
+}
+
+TEST_P(BweSimulation, Evaluation5) {
+  this->RunRoundTripTimeFairness(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+TEST_P(BweSimulation, Evaluation6) {
+  this->RunLongTcpFairness(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->

Btw, can this be implemented with RunFairnessTest(GetParam(), 1, 1...) instead?

[magalhaesc, 2015/07/21 14:46:53]

Yes, with a few modifications

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+// Different calls to the Evaluation7 will create the same FileSizes
+// and StartingTimes as long as the seeds remain unchanged. This is essential
+// when calling it with multiple estimators for comparison purposes.
+TEST_P(BweSimulation, Evaluation7) {
+  const int kNumTcpFiles = 10;
+  this->RunMultipleShortTcpFairness(GetParam(),

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this->

[magalhaesc, 2015/07/21 14:46:53]

Done.

+                                    BweTest::GetFileSizesBytes(kNumTcpFiles),
+                                    BweTest::GetStartingTimesMs(kNumTcpFiles));
+}
+
+TEST_P(BweSimulation, Evaluation8) {
+  this->RunPauseResumeFlows(GetParam());

[stefan-webrtc, 2015/07/21 12:12:32]

Remove this->

[magalhaesc, 2015/07/21 14:46:53]

Done.

+}
+
+// Following test cases begin with "GccComparison" run the
+// evaluation test cases for both GCC and other calling RMCAT.
+
+TEST_P(BweSimulation, GccComparison1) {
+  this->RunVariableCapacitySingleFlow(GetParam());

[stefan-webrtc, 2015/07/21 12:12:31]

Remove this-> here and all below.

[magalhaesc, 2015/07/21 14:46:53]

Done.

+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);

[stefan-webrtc, 2015/07/21 12:12:31]

Maybe make this part of a second constructor of BweTest?

[magalhaesc, 2015/07/21 14:46:53]

Done.

+  gcc_test.RunVariableCapacitySingleFlow(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison2) {
+  this->RunVariableCapacityTwoFlows(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunVariableCapacityTwoFlows(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison3) {
+  this->RunBidirectionalFlow(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunBidirectionalFlow(kFullSendSideEstimator);
+}
+
+// 5.4. Three forward direction competing flows, constant capacity.
+TEST_P(BweSimulation, GccComparison4) {
+  const int kNumRmcatFlows = 3;
+  const int kNumTcpFlows = 0;
+  const int64_t kRunTimeS = 120;
+  const int kLinkCapacity = 3500;
+
+  const int64_t kStartingApartMs = 20 * 1000;
+  int64_t offsets_ms[kNumRmcatFlows];
+  for (int i = 0; i < kNumRmcatFlows; ++i) {
+    offsets_ms[i] = kStartingApartMs * i;
+  }
+
+  // Test also with one way propagation delay = 100ms.
+  this->RunFairnessTest(GetParam(), kNumRmcatFlows, kNumTcpFlows, kRunTimeS,
+                        kLinkCapacity, kMaxQueueingDelayMs, 2 * kOneWayDelayMs,
+                        kMaxJitterMs, offsets_ms);
+
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunFairnessTest(kFullSendSideEstimator, kNumRmcatFlows, kNumTcpFlows,
+                           kRunTimeS, kLinkCapacity, kMaxQueueingDelayMs,
+                           2 * kOneWayDelayMs, kMaxJitterMs, offsets_ms);
+}
+
+TEST_P(BweSimulation, GccComparison5) {
+  this->RunRoundTripTimeFairness(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunRoundTripTimeFairness(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison6) {
+  this->RunLongTcpFairness(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunLongTcpFairness(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparison7) {
+  const int kNumTcpFiles = 10;
+
+  std::vector<int> tcp_file_sizes_bytes =
+      BweTest::GetFileSizesBytes(kNumTcpFiles);
+  std::vector<int64_t> tcp_starting_times_ms =
+      BweTest::GetStartingTimesMs(kNumTcpFiles);
+
+  this->RunMultipleShortTcpFairness(GetParam(), tcp_file_sizes_bytes,
+                                    tcp_starting_times_ms);
+
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunMultipleShortTcpFairness(
+      kFullSendSideEstimator, tcp_file_sizes_bytes, tcp_starting_times_ms);
+}
+
+TEST_P(BweSimulation, GccComparison8) {
+  this->RunPauseResumeFlows(GetParam());
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunPauseResumeFlows(kFullSendSideEstimator);
+}
+
+TEST_P(BweSimulation, GccComparisonChoke) {
+  int array[] = {1000, 500, 1000};
+  std::vector<int> capacities_kbps(array, array + 3);
+  this->RunChoke(GetParam(), capacities_kbps);
+
+  BweTest gcc_test;
+  gcc_test.set_plot_available_capacity(false);
+  gcc_test.RunChoke(kFullSendSideEstimator, capacities_kbps);
 }
 
 #endif  // BWE_TEST_LOGGING_COMPILE_TIME_ENABLE
 }  // namespace bwe
 }  // namespace testing
 }  // namespace webrtc
+