OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 #include <stdio.h> | 10 #include <stdio.h> |
11 | 11 |
12 #include "webrtc/test/gtest.h" | 12 #include "webrtc/test/gtest.h" |
13 #include "webrtc/video/video_quality_test.h" | 13 #include "webrtc/video/video_quality_test.h" |
14 | 14 |
15 namespace webrtc { | 15 namespace webrtc { |
16 | 16 |
17 static const int kFullStackTestDurationSecs = 45; | 17 static const int kFullStackTestDurationSecs = 45; |
18 | 18 |
19 class FullStackTest : public VideoQualityTest { | 19 class FullStackTest : public VideoQualityTest { |
20 public: | 20 public: |
21 void RunTest(const VideoQualityTest::Params ¶ms) { | 21 void RunTest(const VideoQualityTest::Params ¶ms) { |
22 RunWithAnalyzer(params); | 22 RunWithAnalyzer(params); |
23 } | 23 } |
24 | |
25 void ForemanCifWithoutPacketLoss(const std::string& video_codec) { | |
26 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. | |
27 VideoQualityTest::Params foreman_cif; | |
28 foreman_cif.video = {true, 352, 288, 30, 700000, 700000, 700000, false, | |
29 video_codec, 1, 0, 0, false, false, "", "foreman_cif"}; | |
30 foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_" + video_codec, | |
31 0.0, 0.0, kFullStackTestDurationSecs}; | |
32 RunTest(foreman_cif); | |
33 } | |
34 | |
35 void ForemanCifPlr5(const std::string& video_codec, | |
36 bool use_ulpfec, | |
37 bool use_flexfec) { | |
38 VideoQualityTest::Params foreman_cif; | |
39 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, | |
40 video_codec, 1, 0, 0, use_ulpfec, use_flexfec, "", | |
41 "foreman_cif"}; | |
42 std::string fec_description; | |
43 if (use_ulpfec) | |
44 fec_description += "_ulpfec"; | |
45 if (use_flexfec) | |
46 fec_description += "_flexfec"; | |
47 foreman_cif.analyzer = { | |
48 "foreman_cif_delay_50_0_plr_5_" + video_codec + fec_description, 0.0, | |
49 0.0, kFullStackTestDurationSecs}; | |
50 foreman_cif.pipe.loss_percent = 5; | |
51 foreman_cif.pipe.queue_delay_ms = 50; | |
52 RunTest(foreman_cif); | |
53 } | |
54 }; | 24 }; |
55 | 25 |
56 // VideoQualityTest::Params params = { | 26 // VideoQualityTest::Params params = { |
57 // { ... }, // Common. | 27 // { ... }, // Common. |
58 // { ... }, // Video-specific settings. | 28 // { ... }, // Video-specific settings. |
59 // { ... }, // Screenshare-specific settings. | 29 // { ... }, // Screenshare-specific settings. |
60 // { ... }, // Analyzer settings. | 30 // { ... }, // Analyzer settings. |
61 // pipe, // FakeNetworkPipe::Config | 31 // pipe, // FakeNetworkPipe::Config |
62 // { ... }, // Spatial scalability. | 32 // { ... }, // Spatial scalability. |
63 // logs // bool | 33 // logs // bool |
64 // }; | 34 // }; |
65 | 35 |
66 #if !defined(RTC_DISABLE_VP9) | 36 #if !defined(RTC_DISABLE_VP9) |
67 TEST_F(FullStackTest, ForemanCifWithoutPacketLossVp9) { | 37 TEST_F(FullStackTest, ForemanCifWithoutPacketLossVp9) { |
68 ForemanCifWithoutPacketLoss("VP9"); | 38 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. |
| 39 VideoQualityTest::Params foreman_cif; |
| 40 foreman_cif.call.send_side_bwe = true; |
| 41 foreman_cif.video = {true, 352, 288, 30, 700000, 700000, 700000, false, |
| 42 "VP9", 1, 0, 0, false, false, "", "foreman_cif"}; |
| 43 foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_VP9", 0.0, 0.0, |
| 44 kFullStackTestDurationSecs}; |
| 45 RunTest(foreman_cif); |
69 } | 46 } |
70 | 47 |
71 TEST_F(FullStackTest, ForemanCifPlr5Vp9) { | 48 TEST_F(FullStackTest, ForemanCifPlr5Vp9) { |
72 const bool kUlpfec = false; | 49 VideoQualityTest::Params foreman_cif; |
73 const bool kFlexfec = false; | 50 foreman_cif.call.send_side_bwe = true; |
74 ForemanCifPlr5("VP9", kUlpfec, kFlexfec); | 51 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, |
| 52 "VP9", 1, 0, 0, false, false, "", "foreman_cif"}; |
| 53 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5_VP9", 0.0, 0.0, |
| 54 kFullStackTestDurationSecs}; |
| 55 foreman_cif.pipe.loss_percent = 5; |
| 56 foreman_cif.pipe.queue_delay_ms = 50; |
| 57 RunTest(foreman_cif); |
75 } | 58 } |
76 #endif // !defined(RTC_DISABLE_VP9) | 59 #endif // !defined(RTC_DISABLE_VP9) |
77 | 60 |
78 TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { | 61 TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { |
79 VideoQualityTest::Params paris_qcif; | 62 VideoQualityTest::Params paris_qcif; |
80 paris_qcif.call.send_side_bwe = true; | 63 paris_qcif.call.send_side_bwe = true; |
81 paris_qcif.video = {true, 176, 144, 30, 300000, 300000, 300000, false, | 64 paris_qcif.video = {true, 176, 144, 30, 300000, 300000, 300000, false, |
82 "VP8", 1, 0, 0, false, false, "", "paris_qcif"}; | 65 "VP8", 1, 0, 0, false, false, "", "paris_qcif"}; |
83 paris_qcif.analyzer = {"net_delay_0_0_plr_0", 36.0, 0.96, | 66 paris_qcif.analyzer = {"net_delay_0_0_plr_0", 36.0, 0.96, |
84 kFullStackTestDurationSecs}; | 67 kFullStackTestDurationSecs}; |
(...skipping 16 matching lines...) Expand all Loading... |
101 foreman_cif.call.send_side_bwe = true; | 84 foreman_cif.call.send_side_bwe = true; |
102 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", | 85 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", |
103 1, 0, 0, false, false, "", "foreman_cif"}; | 86 1, 0, 0, false, false, "", "foreman_cif"}; |
104 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5", 0.0, 0.0, | 87 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5", 0.0, 0.0, |
105 kFullStackTestDurationSecs}; | 88 kFullStackTestDurationSecs}; |
106 foreman_cif.pipe.loss_percent = 5; | 89 foreman_cif.pipe.loss_percent = 5; |
107 foreman_cif.pipe.queue_delay_ms = 50; | 90 foreman_cif.pipe.queue_delay_ms = 50; |
108 RunTest(foreman_cif); | 91 RunTest(foreman_cif); |
109 } | 92 } |
110 | 93 |
| 94 TEST_F(FullStackTest, ForemanCifPlr5Ulpfec) { |
| 95 VideoQualityTest::Params foreman_cif; |
| 96 foreman_cif.call.send_side_bwe = true; |
| 97 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", |
| 98 1, 0, 0, true, false, "", "foreman_cif"}; |
| 99 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5_ulpfec", 0.0, 0.0, |
| 100 kFullStackTestDurationSecs}; |
| 101 foreman_cif.pipe.loss_percent = 5; |
| 102 foreman_cif.pipe.queue_delay_ms = 50; |
| 103 RunTest(foreman_cif); |
| 104 } |
| 105 |
| 106 TEST_F(FullStackTest, ForemanCifPlr5Flexfec) { |
| 107 VideoQualityTest::Params foreman_cif; |
| 108 foreman_cif.call.send_side_bwe = true; |
| 109 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", |
| 110 1, 0, 0, false, true, "", "foreman_cif"}; |
| 111 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5_flexfec", 0.0, 0.0, |
| 112 kFullStackTestDurationSecs}; |
| 113 foreman_cif.pipe.loss_percent = 5; |
| 114 foreman_cif.pipe.queue_delay_ms = 50; |
| 115 RunTest(foreman_cif); |
| 116 } |
| 117 |
111 #if defined(WEBRTC_USE_H264) | 118 #if defined(WEBRTC_USE_H264) |
112 TEST_F(FullStackTest, ForemanCifWithoutPacketlossH264) { | 119 TEST_F(FullStackTest, ForemanCifWithoutPacketlossH264) { |
113 ForemanCifWithoutPacketLoss("H264"); | 120 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. |
| 121 VideoQualityTest::Params foreman_cif; |
| 122 foreman_cif.call.send_side_bwe = true; |
| 123 foreman_cif.video = {true, 352, 288, 30, 700000, 700000, 700000, false, |
| 124 "H264", 1, 0, 0, false, false, "", "foreman_cif"}; |
| 125 foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_H264", 0.0, 0.0, |
| 126 kFullStackTestDurationSecs}; |
| 127 RunTest(foreman_cif); |
114 } | 128 } |
115 | 129 |
116 TEST_F(FullStackTest, ForemanCifPlr5H264) { | 130 TEST_F(FullStackTest, ForemanCifPlr5H264) { |
117 const bool kUlpfec = false; | 131 VideoQualityTest::Params foreman_cif; |
118 const bool kFlexfec = false; | 132 foreman_cif.call.send_side_bwe = true; |
119 ForemanCifPlr5("H264", kUlpfec, kFlexfec); | 133 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, |
| 134 "H264", 1, 0, 0, false, false, "", "foreman_cif"}; |
| 135 std::string fec_description; |
| 136 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5_H264", 0.0, 0.0, |
| 137 kFullStackTestDurationSecs}; |
| 138 foreman_cif.pipe.loss_percent = 5; |
| 139 foreman_cif.pipe.queue_delay_ms = 50; |
| 140 RunTest(foreman_cif); |
120 } | 141 } |
121 | 142 |
122 // Verify that this is worth the bot time, before enabling. | 143 // Verify that this is worth the bot time, before enabling. |
123 TEST_F(FullStackTest, ForemanCifPlr5H264Flexfec) { | 144 TEST_F(FullStackTest, ForemanCifPlr5H264Flexfec) { |
124 const bool kUlpfec = false; | 145 VideoQualityTest::Params foreman_cif; |
125 const bool kFlexfec = true; | 146 foreman_cif.call.send_side_bwe = true; |
126 ForemanCifPlr5("H264", kUlpfec, kFlexfec); | 147 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, |
| 148 "H264", 1, 0, 0, false, true, "", "foreman_cif"}; |
| 149 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5_H264_flexfec", 0.0, 0.0, |
| 150 kFullStackTestDurationSecs}; |
| 151 foreman_cif.pipe.loss_percent = 5; |
| 152 foreman_cif.pipe.queue_delay_ms = 50; |
| 153 RunTest(foreman_cif); |
127 } | 154 } |
128 | 155 |
129 // Ulpfec with H264 is an unsupported combination, so this test is only useful | 156 // Ulpfec with H264 is an unsupported combination, so this test is only useful |
130 // for debugging. It is therefore disabled by default. | 157 // for debugging. It is therefore disabled by default. |
131 TEST_F(FullStackTest, DISABLED_ForemanCifPlr5H264Ulpfec) { | 158 TEST_F(FullStackTest, DISABLED_ForemanCifPlr5H264Ulpfec) { |
132 const bool kUlpfec = true; | 159 VideoQualityTest::Params foreman_cif; |
133 const bool kFlexfec = false; | 160 foreman_cif.call.send_side_bwe = true; |
134 ForemanCifPlr5("H264", kUlpfec, kFlexfec); | 161 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, |
| 162 "H264", 1, 0, 0, true, false, "", "foreman_cif"}; |
| 163 foreman_cif.analyzer = {"foreman_cif_delay_50_0_plr_5_H264_ulpfec", 0.0, 0.0, |
| 164 kFullStackTestDurationSecs}; |
| 165 foreman_cif.pipe.loss_percent = 5; |
| 166 foreman_cif.pipe.queue_delay_ms = 50; |
| 167 RunTest(foreman_cif); |
135 } | 168 } |
136 #endif // defined(WEBRTC_USE_H264) | 169 #endif // defined(WEBRTC_USE_H264) |
137 | 170 |
138 TEST_F(FullStackTest, ForemanCif500kbps) { | 171 TEST_F(FullStackTest, ForemanCif500kbps) { |
139 VideoQualityTest::Params foreman_cif; | 172 VideoQualityTest::Params foreman_cif; |
140 foreman_cif.call.send_side_bwe = true; | 173 foreman_cif.call.send_side_bwe = true; |
141 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", | 174 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", |
142 1, 0, 0, false, false, "", "foreman_cif"}; | 175 1, 0, 0, false, false, "", "foreman_cif"}; |
143 foreman_cif.analyzer = {"foreman_cif_500kbps", 0.0, 0.0, | 176 foreman_cif.analyzer = {"foreman_cif_500kbps", 0.0, 0.0, |
144 kFullStackTestDurationSecs}; | 177 kFullStackTestDurationSecs}; |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
182 foreman_cif.analyzer = {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, | 215 foreman_cif.analyzer = {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, |
183 kFullStackTestDurationSecs}; | 216 kFullStackTestDurationSecs}; |
184 foreman_cif.pipe.queue_length_packets = 32; | 217 foreman_cif.pipe.queue_length_packets = 32; |
185 foreman_cif.pipe.queue_delay_ms = 100; | 218 foreman_cif.pipe.queue_delay_ms = 100; |
186 foreman_cif.pipe.link_capacity_kbps = 500; | 219 foreman_cif.pipe.link_capacity_kbps = 500; |
187 RunTest(foreman_cif); | 220 RunTest(foreman_cif); |
188 } | 221 } |
189 | 222 |
190 TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) { | 223 TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) { |
191 VideoQualityTest::Params foreman_cif; | 224 VideoQualityTest::Params foreman_cif; |
| 225 foreman_cif.call.send_side_bwe = false; |
192 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", | 226 foreman_cif.video = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", |
193 1, 0, 0, false, false, "", "foreman_cif"}; | 227 1, 0, 0, false, false, "", "foreman_cif"}; |
194 foreman_cif.analyzer = {"foreman_cif_500kbps_100ms_32pkts_queue", 0.0, 0.0, | 228 foreman_cif.analyzer = {"foreman_cif_500kbps_100ms_32pkts_queue_recv_bwe", |
195 kFullStackTestDurationSecs}; | 229 0.0, 0.0, kFullStackTestDurationSecs}; |
196 foreman_cif.pipe.queue_length_packets = 32; | 230 foreman_cif.pipe.queue_length_packets = 32; |
197 foreman_cif.pipe.queue_delay_ms = 100; | 231 foreman_cif.pipe.queue_delay_ms = 100; |
198 foreman_cif.pipe.link_capacity_kbps = 500; | 232 foreman_cif.pipe.link_capacity_kbps = 500; |
199 RunTest(foreman_cif); | 233 RunTest(foreman_cif); |
200 } | 234 } |
201 | 235 |
202 TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { | 236 TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { |
203 VideoQualityTest::Params foreman_cif; | 237 VideoQualityTest::Params foreman_cif; |
204 foreman_cif.call.send_side_bwe = true; | 238 foreman_cif.call.send_side_bwe = true; |
205 foreman_cif.video = {true, 352, 288, 30, 30000, 2000000, 2000000, false, | 239 foreman_cif.video = {true, 352, 288, 30, 30000, 2000000, 2000000, false, |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
283 screenshare.video = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, | 317 screenshare.video = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, |
284 "VP9", 1, 0, 400000, false, false, "", ""}; | 318 "VP9", 1, 0, 400000, false, false, "", ""}; |
285 screenshare.screenshare = {true, 10}; | 319 screenshare.screenshare = {true, 10}; |
286 screenshare.analyzer = {"screenshare_slides_vp9_2sl", 0.0, 0.0, | 320 screenshare.analyzer = {"screenshare_slides_vp9_2sl", 0.0, 0.0, |
287 kFullStackTestDurationSecs}; | 321 kFullStackTestDurationSecs}; |
288 screenshare.logs = false; | 322 screenshare.logs = false; |
289 screenshare.ss = {std::vector<VideoStream>(), 0, 2, 1}; | 323 screenshare.ss = {std::vector<VideoStream>(), 0, 2, 1}; |
290 RunTest(screenshare); | 324 RunTest(screenshare); |
291 } | 325 } |
292 #endif // !defined(RTC_DISABLE_VP9) | 326 #endif // !defined(RTC_DISABLE_VP9) |
| 327 |
293 } // namespace webrtc | 328 } // namespace webrtc |
OLD | NEW |