Reland of "Added large room scenario to full-stack tests

webrtc/video/full_stack_tests.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
 
 #include "webrtc/test/gtest.h"
 #include "webrtc/video/video_quality_test.h"
 
 namespace webrtc {
 
+#if defined(WEBRTC_ANDROID) || defined(WEBRTC_WIN)
+// Fails on Android and win

[kjellander_webrtc, 2017/03/07 09:39:27]

Please track fixing this in a bug that you add a reference to here. Not tracking
this in a bug will likely lead to this staying disabled forever for these
platforms.

If Android is too weak, could you consider reducing the size down from 1280x720
to VGA (or small) instead of disabling the test entirely? Or will 50 thumbnails
always be too many for Android?

For Windows, I assume the HD version with 50 thumbnails must be possible to fix,
right? 

[ilnik, 2017/03/07 09:54:07]

50 thumbnails will be too large even without main HD feed. It consumes 6 full
cores on my workstation and around 4GB of memory. But it would be nice to have
such huge test. Why it fails on Win I don't know yet. maybe buildbot doesn't
have enough computing power or memory.

Added a bug for it.

+#define MAYBE_FullStackTest DISABLED_FullStackTest
+#else
+#define MAYBE_FullStackTest FullStackTest

[kjellander_webrtc, 2017/03/07 09:39:27]

If you're only looking for disabling a single test case, you should add the
MAYBE_ pattern to the test case name instead:

#if defined(WEBRTC_ANDROID) || defined(WEBRTC_WIN)
#define MAYBE_LargeRoomVP8_50thumb DISABLED_LargeRoomVP8_50thumb
#else
#define MAYBE_LargeRoomVP8_50thumb LargeRoomVP8_50thumb

And move it to line 600.

[ilnik, 2017/03/07 09:54:07]

Done.

+#endif
+
 static const int kFullStackTestDurationSecs = 45;
 
 class FullStackTest : public VideoQualityTest {
  public:
   void RunTest(const VideoQualityTest::Params &params) {
     RunWithAnalyzer(params);
   }
 };
 
 // VideoQualityTest::Params params = {
@@ skipping 330 matching lines @@
   screenshare.analyzer = {"screenshare_slides_vp9_2sl", 0.0, 0.0,
                           kFullStackTestDurationSecs};
   screenshare.logs = false;
   screenshare.ss = {std::vector<VideoStream>(), 0, 2, 1};
   RunTest(screenshare);
 }
 
 TEST_F(FullStackTest, VP9SVC_3SL_High) {
   VideoQualityTest::Params simulcast;
   simulcast.call.send_side_bwe = true;
-  simulcast.video = {true,   1280,    720,     50,
+  simulcast.video = {true,   1280,    720,     30,
                      800000, 2500000, 2500000, false,
                      "VP9",  1,       0,       400000,
                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
   simulcast.analyzer = {"vp9svc_3sl_high", 0.0, 0.0,
                         kFullStackTestDurationSecs};
   simulcast.logs = false;
   simulcast.ss = {std::vector<VideoStream>(), 0, 3, 2};
   RunTest(simulcast);
 }
 
 TEST_F(FullStackTest, VP9SVC_3SL_Medium) {
   VideoQualityTest::Params simulcast;
   simulcast.call.send_side_bwe = true;
-  simulcast.video = {true,   1280,    720,     50,
+  simulcast.video = {true,   1280,    720,     30,
                      800000, 2500000, 2500000, false,
                      "VP9",  1,       0,       400000,
                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
   simulcast.analyzer = {"vp9svc_3sl_medium", 0.0, 0.0,
                         kFullStackTestDurationSecs};
   simulcast.logs = false;
   simulcast.ss = {std::vector<VideoStream>(), 0, 3, 1};
   RunTest(simulcast);
 }
 
 TEST_F(FullStackTest, VP9SVC_3SL_Low) {
   VideoQualityTest::Params simulcast;
   simulcast.call.send_side_bwe = true;
-  simulcast.video = {true,   1280,    720,     50,
+  simulcast.video = {true,   1280,    720,     30,
                      800000, 2500000, 2500000, false,
                      "VP9",  1,       0,       400000,
                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
   simulcast.analyzer = {"vp9svc_3sl_low", 0.0, 0.0, kFullStackTestDurationSecs};
   simulcast.logs = false;
   simulcast.ss = {std::vector<VideoStream>(), 0, 3, 0};
   RunTest(simulcast);
 }
 #endif  // !defined(RTC_DISABLE_VP9)
 
 TEST_F(FullStackTest, SimulcastVP8_3SL_High) {
   VideoQualityTest::Params simulcast;
   simulcast.call.send_side_bwe = true;
-  simulcast.video = {true,   1280,    720,     50,
+  simulcast.video = {true,   1280,    720,     30,
                      800000, 2500000, 2500000, false,
                      "VP8",  1,       0,       400000,
                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
   simulcast.analyzer = {"simulcast_vp8_3sl_high", 0.0, 0.0,
                         kFullStackTestDurationSecs};
   simulcast.pipe.loss_percent = 0;
   simulcast.pipe.queue_delay_ms = 100;
   VideoQualityTest::Params video_params_high;
   video_params_high.video = {
-      true,   1280,    720,     50,
+      true,   1280,    720,     30,
       800000, 2500000, 2500000, false,
       "VP8",  1,       0,       400000,
       false,  false,   "",      "ConferenceMotion_1280_720_50"};
   VideoQualityTest::Params video_params_medium;
   video_params_medium.video = {
-      true,   640,    360,    50,
+      true,   640,    360,    30,
       150000, 500000, 700000, false,
       "VP8",  1,      0,      400000,
       false,  false,  "",     "ConferenceMotion_1280_720_50"};
   VideoQualityTest::Params video_params_low;
   video_params_low.video = {
-      true,  320,    180,    50,
+      true,  320,    180,    30,
       30000, 150000, 200000, false,
       "VP8", 1,      0,      400000,
       false, false,  "",     "ConferenceMotion_1280_720_50"};
 
   std::vector<VideoStream> streams = {DefaultVideoStream(video_params_low),
                                       DefaultVideoStream(video_params_medium),
                                       DefaultVideoStream(video_params_high)};
   simulcast.ss = {streams, 2, 1, 0};
   RunTest(simulcast);
 }
 
 TEST_F(FullStackTest, SimulcastVP8_3SL_Medium) {
   VideoQualityTest::Params simulcast;
   simulcast.call.send_side_bwe = true;
-  simulcast.video = {true,   1280,    720,     50,
+  simulcast.video = {true,   1280,    720,     30,
                      800000, 2500000, 2500000, false,
                      "VP8",  1,       0,       400000,
                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
   simulcast.analyzer = {"simulcast_vp8_3sl_medium", 0.0, 0.0,
                         kFullStackTestDurationSecs};
   simulcast.pipe.loss_percent = 0;
   simulcast.pipe.queue_delay_ms = 100;
   VideoQualityTest::Params video_params_high;
   video_params_high.video = {
-      true,   1280,    720,     50,
+      true,   1280,    720,     30,
       800000, 2500000, 2500000, false,
       "VP8",  1,       0,       400000,
       false,  false,   "",      "ConferenceMotion_1280_720_50"};
   VideoQualityTest::Params video_params_medium;
   video_params_medium.video = {
-      true,   640,    360,    50,
+      true,   640,    360,    30,
       150000, 500000, 700000, false,
       "VP8",  1,      0,      400000,
       false,  false,  "",     "ConferenceMotion_1280_720_50"};
   VideoQualityTest::Params video_params_low;
   video_params_low.video = {
-      true,  320,    180,    50,
+      true,  320,    180,    30,
       30000, 150000, 200000, false,
       "VP8", 1,      0,      400000,
       false, false,  "",     "ConferenceMotion_1280_720_50"};
 
   std::vector<VideoStream> streams = {DefaultVideoStream(video_params_low),
                                       DefaultVideoStream(video_params_medium),
                                       DefaultVideoStream(video_params_high)};
   simulcast.ss = {streams, 1, 1, 0};
   RunTest(simulcast);
 }
 
 TEST_F(FullStackTest, SimulcastVP8_3SL_Low) {
   VideoQualityTest::Params simulcast;
   simulcast.call.send_side_bwe = true;
-  simulcast.video = {true,   1280,    720,     50,
+  simulcast.video = {true,   1280,    720,     30,
                      800000, 2500000, 2500000, false,
                      "VP8",  1,       0,       400000,
                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
   simulcast.analyzer = {"simulcast_vp8_3sl_low", 0.0, 0.0,
                         kFullStackTestDurationSecs};
   simulcast.pipe.loss_percent = 0;
   simulcast.pipe.queue_delay_ms = 100;
   VideoQualityTest::Params video_params_high;
   video_params_high.video = {
-      true,   1280,    720,     50,
+      true,   1280,    720,     30,
       800000, 2500000, 2500000, false,
       "VP8",  1,       0,       400000,
       false,  false,   "",      "ConferenceMotion_1280_720_50"};
   VideoQualityTest::Params video_params_medium;
   video_params_medium.video = {
-      true,   640,    360,    50,
+      true,   640,    360,    30,
       150000, 500000, 700000, false,
       "VP8",  1,      0,      400000,
       false,  false,  "",     "ConferenceMotion_1280_720_50"};
   VideoQualityTest::Params video_params_low;
   video_params_low.video = {
-      true,  320,    180,    50,
+      true,  320,    180,    30,
       30000, 150000, 200000, false,
       "VP8", 1,      0,      400000,
       false, false,  "",     "ConferenceMotion_1280_720_50"};
 
   std::vector<VideoStream> streams = {DefaultVideoStream(video_params_low),
                                       DefaultVideoStream(video_params_medium),
                                       DefaultVideoStream(video_params_high)};
   simulcast.ss = {streams, 0, 1, 0};
   RunTest(simulcast);
 }
 
+TEST_F(FullStackTest, LargeRoomVP8_5thumb) {
+  VideoQualityTest::Params large_room;
+  large_room.call.send_side_bwe = true;
+  large_room.video = {true,   1280,    720,     30,
+                      800000, 2500000, 2500000, false,
+                      "VP8",  3,       2,       400000,
+                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
+  large_room.analyzer = {"largeroom_5thumb", 0.0, 0.0,
+                         kFullStackTestDurationSecs};
+  large_room.pipe.loss_percent = 0;
+  large_room.pipe.queue_delay_ms = 100;
+  VideoQualityTest::Params video_params_high;
+  video_params_high.video = {
+      true,   1280,    720,     30,
+      800000, 2500000, 2500000, false,
+      "VP8",  3,       2,       400000,
+      false,  false,   "",      "ConferenceMotion_1280_720_50"};
+  VideoQualityTest::Params video_params_medium;
+  video_params_medium.video = {
+      true,   640,    360,    30,
+      150000, 500000, 700000, false,
+      "VP8",  3,      2,      400000,
+      false,  false,  "",     "ConferenceMotion_1280_720_50"};
+  VideoQualityTest::Params video_params_low;
+  video_params_low.video = {
+      true,  320,    180,    30,
+      30000, 150000, 200000, false,
+      "VP8", 3,      2,      400000,
+      false, false,  "",     "ConferenceMotion_1280_720_50"};
+
+  std::vector<VideoStream> streams = {DefaultVideoStream(video_params_low),
+                                      DefaultVideoStream(video_params_medium),
+                                      DefaultVideoStream(video_params_high)};
+  large_room.num_thumbnails = 5;
+  large_room.ss = {streams, 2, 1, 0};
+  RunTest(large_room);
+}
+
+TEST_F(FullStackTest, LargeRoomVP8_15thumb) {
+  VideoQualityTest::Params large_room;
+  large_room.call.send_side_bwe = true;
+  large_room.video = {true,   1280,    720,     30,
+                      800000, 2500000, 2500000, false,
+                      "VP8",  3,       2,       400000,
+                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
+  large_room.analyzer = {"largeroom_15thumb", 0.0, 0.0,
+                         kFullStackTestDurationSecs};
+  large_room.pipe.loss_percent = 0;
+  large_room.pipe.queue_delay_ms = 100;
+  VideoQualityTest::Params video_params_high;
+  video_params_high.video = {
+      true,   1280,    720,     30,
+      800000, 2500000, 2500000, false,
+      "VP8",  3,       2,       400000,
+      false,  false,   "",      "ConferenceMotion_1280_720_50"};
+  VideoQualityTest::Params video_params_medium;
+  video_params_medium.video = {
+      true,   640,    360,    30,
+      150000, 500000, 700000, false,
+      "VP8",  3,      2,      400000,
+      false,  false,  "",     "ConferenceMotion_1280_720_50"};
+  VideoQualityTest::Params video_params_low;
+  video_params_low.video = {
+      true,  320,    180,    30,
+      30000, 150000, 200000, false,
+      "VP8", 3,      2,      400000,
+      false, false,  "",     "ConferenceMotion_1280_720_50"};
+
+  std::vector<VideoStream> streams = {DefaultVideoStream(video_params_low),
+                                      DefaultVideoStream(video_params_medium),
+                                      DefaultVideoStream(video_params_high)};
+  large_room.num_thumbnails = 15;
+  large_room.ss = {streams, 2, 1, 0};
+  RunTest(large_room);
+}
+
+TEST_F(MAYBE_FullStackTest, LargeRoomVP8_50thumb) {
+  VideoQualityTest::Params large_room;
+  large_room.call.send_side_bwe = true;
+  large_room.video = {true,   1280,    720,     30,
+                      800000, 2500000, 2500000, false,
+                      "VP8",  3,       2,       400000,
+                      false,  false,   "",      "ConferenceMotion_1280_720_50"};
+  large_room.analyzer = {"largeroom_50thumb", 0.0, 0.0,
+                         kFullStackTestDurationSecs};
+  large_room.pipe.loss_percent = 0;
+  large_room.pipe.queue_delay_ms = 100;
+  VideoQualityTest::Params video_params_high;
+  video_params_high.video = {
+      true,   1280,    720,     30,
+      800000, 2500000, 2500000, false,
+      "VP8",  3,       2,       400000,
+      false,  false,   "",      "ConferenceMotion_1280_720_50"};
+  VideoQualityTest::Params video_params_medium;
+  video_params_medium.video = {
+      true,   640,    360,    30,
+      150000, 500000, 700000, false,
+      "VP8",  3,      2,      400000,
+      false,  false,  "",     "ConferenceMotion_1280_720_50"};
+  VideoQualityTest::Params video_params_low;
+  video_params_low.video = {
+      true,  320,    180,    30,
+      30000, 150000, 200000, false,
+      "VP8", 3,      2,      400000,
+      false, false,  "",     "ConferenceMotion_1280_720_50"};
+
+  std::vector<VideoStream> streams = {DefaultVideoStream(video_params_low),
+                                      DefaultVideoStream(video_params_medium),
+                                      DefaultVideoStream(video_params_high)};
+  large_room.num_thumbnails = 50;
+  large_room.ss = {streams, 2, 1, 0};
+  RunTest(large_room);
+}
+
+
 }  // namespace webrtc