OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H
_ | 11 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H
_ |
12 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H
_ | 12 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTEST_H
_ |
13 | 13 |
14 #include <math.h> | |
15 | |
16 #include <limits> | |
17 #include <memory> | 14 #include <memory> |
18 #include <string> | 15 #include <string> |
19 #include <utility> | |
20 #include <vector> | 16 #include <vector> |
21 | 17 |
22 #if defined(WEBRTC_ANDROID) | 18 #include "webrtc/common_types.h" |
23 #include "webrtc/modules/video_coding/codecs/test/android_test_initializer.h" | |
24 #include "webrtc/sdk/android/src/jni/androidmediadecoder_jni.h" | |
25 #include "webrtc/sdk/android/src/jni/androidmediaencoder_jni.h" | |
26 #elif defined(WEBRTC_IOS) | |
27 #include "webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h" | |
28 #endif | |
29 | |
30 #include "webrtc/media/engine/internaldecoderfactory.h" | |
31 #include "webrtc/media/engine/internalencoderfactory.h" | |
32 #include "webrtc/media/engine/webrtcvideodecoderfactory.h" | 19 #include "webrtc/media/engine/webrtcvideodecoderfactory.h" |
33 #include "webrtc/media/engine/webrtcvideoencoderfactory.h" | 20 #include "webrtc/media/engine/webrtcvideoencoderfactory.h" |
34 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" | 21 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" |
| 22 #include "webrtc/modules/video_coding/codecs/test/stats.h" |
35 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" | 23 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" |
36 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" | |
37 #include "webrtc/modules/video_coding/include/video_codec_interface.h" | |
38 #include "webrtc/modules/video_coding/include/video_coding.h" | |
39 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h" | 24 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h" |
40 #include "webrtc/rtc_base/checks.h" | |
41 #include "webrtc/rtc_base/event.h" | |
42 #include "webrtc/rtc_base/file.h" | |
43 #include "webrtc/rtc_base/logging.h" | |
44 #include "webrtc/rtc_base/ptr_util.h" | |
45 #include "webrtc/system_wrappers/include/sleep.h" | |
46 #include "webrtc/test/gtest.h" | 25 #include "webrtc/test/gtest.h" |
47 #include "webrtc/test/testsupport/fileutils.h" | |
48 #include "webrtc/test/testsupport/frame_reader.h" | 26 #include "webrtc/test/testsupport/frame_reader.h" |
49 #include "webrtc/test/testsupport/frame_writer.h" | 27 #include "webrtc/test/testsupport/frame_writer.h" |
50 #include "webrtc/test/testsupport/metrics/video_metrics.h" | |
51 #include "webrtc/test/testsupport/packet_reader.h" | 28 #include "webrtc/test/testsupport/packet_reader.h" |
52 #include "webrtc/test/video_codec_settings.h" | |
53 #include "webrtc/typedefs.h" | |
54 | 29 |
55 namespace webrtc { | 30 namespace webrtc { |
56 namespace test { | 31 namespace test { |
57 | 32 |
58 const int kMaxNumRateUpdates = 3; | |
59 const int kMaxNumTemporalLayers = 3; | |
60 | |
61 const int kPercTargetvsActualMismatch = 20; | |
62 const int kBaseKeyFrameInterval = 3000; | |
63 | |
64 // Parameters from VP8 wrapper, which control target size of key frames. | |
65 const float kInitialBufferSize = 0.5f; | |
66 const float kOptimalBufferSize = 0.6f; | |
67 const float kScaleKeyFrameSize = 0.5f; | |
68 | |
69 // Thresholds for the quality metrics. Defaults are maximally minimal. | |
70 struct QualityThresholds { | |
71 QualityThresholds(double min_avg_psnr, | |
72 double min_min_psnr, | |
73 double min_avg_ssim, | |
74 double min_min_ssim) | |
75 : min_avg_psnr(min_avg_psnr), | |
76 min_min_psnr(min_min_psnr), | |
77 min_avg_ssim(min_avg_ssim), | |
78 min_min_ssim(min_min_ssim) {} | |
79 double min_avg_psnr; | |
80 double min_min_psnr; | |
81 double min_avg_ssim; | |
82 double min_min_ssim; | |
83 }; | |
84 | |
85 // The sequence of bit rate and frame rate changes for the encoder, the frame | 33 // The sequence of bit rate and frame rate changes for the encoder, the frame |
86 // number where the changes are made, and the total number of frames for the | 34 // number where the changes are made, and the total number of frames for the |
87 // test. | 35 // test. |
88 struct RateProfile { | 36 struct RateProfile { |
| 37 static const int kMaxNumRateUpdates = 3; |
| 38 |
89 int target_bit_rate[kMaxNumRateUpdates]; | 39 int target_bit_rate[kMaxNumRateUpdates]; |
90 int input_frame_rate[kMaxNumRateUpdates]; | 40 int input_frame_rate[kMaxNumRateUpdates]; |
91 int frame_index_rate_update[kMaxNumRateUpdates + 1]; | 41 int frame_index_rate_update[kMaxNumRateUpdates + 1]; |
92 int num_frames; | 42 int num_frames; |
93 }; | 43 }; |
94 | 44 |
95 // Thresholds for the rate control metrics. The rate mismatch thresholds are | 45 // Thresholds for the rate control metrics. The rate mismatch thresholds are |
96 // defined as percentages. |max_time_hit_target| is defined as number of frames, | 46 // defined as percentages. |max_time_hit_target| is defined as number of frames, |
97 // after a rate update is made to the encoder, for the encoder to reach within | 47 // after a rate update is made to the encoder, for the encoder to reach within |
98 // |kPercTargetvsActualMismatch| of new target rate. The thresholds are defined | 48 // |kPercTargetvsActualMismatch| of new target rate. The thresholds are defined |
99 // for each rate update sequence. | 49 // for each rate update sequence. |
100 struct RateControlThresholds { | 50 struct RateControlThresholds { |
101 int max_num_dropped_frames; | 51 int max_num_dropped_frames; |
102 int max_key_frame_size_mismatch; | 52 int max_key_frame_size_mismatch; |
103 int max_delta_frame_size_mismatch; | 53 int max_delta_frame_size_mismatch; |
104 int max_encoding_rate_mismatch; | 54 int max_encoding_rate_mismatch; |
105 int max_time_hit_target; | 55 int max_time_hit_target; |
106 int num_spatial_resizes; | 56 int num_spatial_resizes; |
107 int num_key_frames; | 57 int num_key_frames; |
108 }; | 58 }; |
109 | 59 |
| 60 // Thresholds for the quality metrics. |
| 61 struct QualityThresholds { |
| 62 QualityThresholds(double min_avg_psnr, |
| 63 double min_min_psnr, |
| 64 double min_avg_ssim, |
| 65 double min_min_ssim) |
| 66 : min_avg_psnr(min_avg_psnr), |
| 67 min_min_psnr(min_min_psnr), |
| 68 min_avg_ssim(min_avg_ssim), |
| 69 min_min_ssim(min_min_ssim) {} |
| 70 double min_avg_psnr; |
| 71 double min_min_psnr; |
| 72 double min_avg_ssim; |
| 73 double min_min_ssim; |
| 74 }; |
| 75 |
110 // Should video files be saved persistently to disk for post-run visualization? | 76 // Should video files be saved persistently to disk for post-run visualization? |
111 struct VisualizationParams { | 77 struct VisualizationParams { |
112 bool save_encoded_ivf; | 78 bool save_encoded_ivf; |
113 bool save_decoded_y4m; | 79 bool save_decoded_y4m; |
114 }; | 80 }; |
115 | 81 |
116 // Integration test for video processor. Encodes+decodes a clip and | 82 // Integration test for video processor. Encodes+decodes a clip and |
117 // writes it to the output directory. After completion, quality metrics | 83 // writes it to the output directory. After completion, quality metrics |
118 // (PSNR and SSIM) and rate control metrics are computed and compared to given | 84 // (PSNR and SSIM) and rate control metrics are computed and compared to given |
119 // thresholds, to verify that the quality and encoder response is acceptable. | 85 // thresholds, to verify that the quality and encoder response is acceptable. |
120 // The rate control tests allow us to verify the behavior for changing bit rate, | 86 // The rate control tests allow us to verify the behavior for changing bit rate, |
121 // changing frame rate, frame dropping/spatial resize, and temporal layers. | 87 // changing frame rate, frame dropping/spatial resize, and temporal layers. |
122 // The thresholds for the rate control metrics are set to be fairly | 88 // The thresholds for the rate control metrics are set to be fairly |
123 // conservative, so failure should only happen when some significant regression | 89 // conservative, so failure should only happen when some significant regression |
124 // or breakdown occurs. | 90 // or breakdown occurs. |
125 class VideoProcessorIntegrationTest : public testing::Test { | 91 class VideoProcessorIntegrationTest : public testing::Test { |
126 protected: | 92 protected: |
127 VideoProcessorIntegrationTest() { | 93 VideoProcessorIntegrationTest(); |
128 #if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED) && \ | 94 ~VideoProcessorIntegrationTest() override; |
129 defined(WEBRTC_ANDROID) | |
130 InitializeAndroidObjects(); | |
131 #endif | |
132 } | |
133 ~VideoProcessorIntegrationTest() override = default; | |
134 | |
135 void CreateEncoderAndDecoder() { | |
136 if (config_.hw_codec) { | |
137 #if defined(WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED) | |
138 #if defined(WEBRTC_ANDROID) | |
139 encoder_factory_.reset(new jni::MediaCodecVideoEncoderFactory()); | |
140 decoder_factory_.reset(new jni::MediaCodecVideoDecoderFactory()); | |
141 #elif defined(WEBRTC_IOS) | |
142 EXPECT_EQ(kVideoCodecH264, config_.codec_settings.codecType) | |
143 << "iOS HW codecs only support H264."; | |
144 encoder_factory_ = CreateObjCEncoderFactory(); | |
145 decoder_factory_ = CreateObjCDecoderFactory(); | |
146 #else | |
147 RTC_NOTREACHED() << "Only support HW codecs on Android and iOS."; | |
148 #endif | |
149 #endif // WEBRTC_VIDEOPROCESSOR_INTEGRATIONTEST_HW_CODECS_ENABLED | |
150 } else { | |
151 // SW codecs. | |
152 encoder_factory_.reset(new cricket::InternalEncoderFactory()); | |
153 decoder_factory_.reset(new cricket::InternalDecoderFactory()); | |
154 } | |
155 | |
156 switch (config_.codec_settings.codecType) { | |
157 case kVideoCodecVP8: | |
158 encoder_ = encoder_factory_->CreateVideoEncoder( | |
159 cricket::VideoCodec(cricket::kVp8CodecName)); | |
160 decoder_ = decoder_factory_->CreateVideoDecoder(kVideoCodecVP8); | |
161 break; | |
162 case kVideoCodecVP9: | |
163 encoder_ = encoder_factory_->CreateVideoEncoder( | |
164 cricket::VideoCodec(cricket::kVp9CodecName)); | |
165 decoder_ = decoder_factory_->CreateVideoDecoder(kVideoCodecVP9); | |
166 break; | |
167 case kVideoCodecH264: | |
168 // TODO(brandtr): Generalize so that we support multiple profiles here. | |
169 encoder_ = encoder_factory_->CreateVideoEncoder( | |
170 cricket::VideoCodec(cricket::kH264CodecName)); | |
171 decoder_ = decoder_factory_->CreateVideoDecoder(kVideoCodecH264); | |
172 break; | |
173 default: | |
174 RTC_NOTREACHED(); | |
175 break; | |
176 } | |
177 | |
178 EXPECT_TRUE(encoder_) << "Encoder not successfully created."; | |
179 EXPECT_TRUE(decoder_) << "Decoder not successfully created."; | |
180 } | |
181 | |
182 void DestroyEncoderAndDecoder() { | |
183 encoder_factory_->DestroyVideoEncoder(encoder_); | |
184 decoder_factory_->DestroyVideoDecoder(decoder_); | |
185 } | |
186 | |
187 void SetUpAndInitObjects(rtc::TaskQueue* task_queue, | |
188 const int initial_bitrate_kbps, | |
189 const int initial_framerate_fps, | |
190 const VisualizationParams* visualization_params) { | |
191 CreateEncoderAndDecoder(); | |
192 | |
193 // Create file objects for quality analysis. | |
194 analysis_frame_reader_.reset(new YuvFrameReaderImpl( | |
195 config_.input_filename, config_.codec_settings.width, | |
196 config_.codec_settings.height)); | |
197 analysis_frame_writer_.reset(new YuvFrameWriterImpl( | |
198 config_.output_filename, config_.codec_settings.width, | |
199 config_.codec_settings.height)); | |
200 EXPECT_TRUE(analysis_frame_reader_->Init()); | |
201 EXPECT_TRUE(analysis_frame_writer_->Init()); | |
202 | |
203 if (visualization_params) { | |
204 const std::string codec_name = | |
205 CodecTypeToPayloadString(config_.codec_settings.codecType); | |
206 const std::string implementation_type = config_.hw_codec ? "hw" : "sw"; | |
207 // clang-format off | |
208 const std::string output_filename_base = | |
209 OutputPath() + config_.filename + "-" + | |
210 codec_name + "-" + implementation_type + "-" + | |
211 std::to_string(initial_bitrate_kbps); | |
212 // clang-format on | |
213 if (visualization_params->save_encoded_ivf) { | |
214 rtc::File post_encode_file = | |
215 rtc::File::Create(output_filename_base + ".ivf"); | |
216 encoded_frame_writer_ = | |
217 IvfFileWriter::Wrap(std::move(post_encode_file), 0); | |
218 } | |
219 if (visualization_params->save_decoded_y4m) { | |
220 decoded_frame_writer_.reset(new Y4mFrameWriterImpl( | |
221 output_filename_base + ".y4m", config_.codec_settings.width, | |
222 config_.codec_settings.height, initial_framerate_fps)); | |
223 EXPECT_TRUE(decoded_frame_writer_->Init()); | |
224 } | |
225 } | |
226 | |
227 packet_manipulator_.reset(new PacketManipulatorImpl( | |
228 &packet_reader_, config_.networking_config, config_.verbose)); | |
229 | |
230 config_.codec_settings.minBitrate = 0; | |
231 config_.codec_settings.startBitrate = initial_bitrate_kbps; | |
232 config_.codec_settings.maxFramerate = initial_framerate_fps; | |
233 | |
234 rtc::Event sync_event(false, false); | |
235 task_queue->PostTask([this, &sync_event]() { | |
236 processor_ = rtc::MakeUnique<VideoProcessor>( | |
237 encoder_, decoder_, analysis_frame_reader_.get(), | |
238 analysis_frame_writer_.get(), packet_manipulator_.get(), config_, | |
239 &stats_, encoded_frame_writer_.get(), decoded_frame_writer_.get()); | |
240 processor_->Init(); | |
241 sync_event.Set(); | |
242 }); | |
243 sync_event.Wait(rtc::Event::kForever); | |
244 } | |
245 | |
246 void ReleaseAndCloseObjects(rtc::TaskQueue* task_queue) { | |
247 rtc::Event sync_event(false, false); | |
248 task_queue->PostTask([this, &sync_event]() { | |
249 processor_->Release(); | |
250 sync_event.Set(); | |
251 }); | |
252 sync_event.Wait(rtc::Event::kForever); | |
253 | |
254 // The VideoProcessor must be ::Release()'d before we destroy the codecs. | |
255 DestroyEncoderAndDecoder(); | |
256 | |
257 // Close the analysis files before we use them for SSIM/PSNR calculations. | |
258 analysis_frame_reader_->Close(); | |
259 analysis_frame_writer_->Close(); | |
260 | |
261 // Close visualization files. | |
262 if (encoded_frame_writer_) { | |
263 EXPECT_TRUE(encoded_frame_writer_->Close()); | |
264 } | |
265 if (decoded_frame_writer_) { | |
266 decoded_frame_writer_->Close(); | |
267 } | |
268 } | |
269 | |
270 // For every encoded frame, update the rate control metrics. | |
271 void UpdateRateControlMetrics(int frame_number) { | |
272 RTC_CHECK_GE(frame_number, 0); | |
273 | |
274 const int tl_idx = TemporalLayerIndexForFrame(frame_number); | |
275 ++num_frames_per_update_[tl_idx]; | |
276 ++num_frames_total_; | |
277 | |
278 FrameType frame_type = stats_.stats_[frame_number].frame_type; | |
279 float encoded_size_kbits = | |
280 stats_.stats_[frame_number].encoded_frame_length_in_bytes * 8.0f / | |
281 1000.0f; | |
282 | |
283 // Update layer data. | |
284 // Update rate mismatch relative to per-frame bandwidth for delta frames. | |
285 if (frame_type == kVideoFrameDelta) { | |
286 // TODO(marpan): Should we count dropped (zero size) frames in mismatch? | |
287 sum_frame_size_mismatch_[tl_idx] += | |
288 fabs(encoded_size_kbits - per_frame_bandwidth_[tl_idx]) / | |
289 per_frame_bandwidth_[tl_idx]; | |
290 } else { | |
291 float target_size = (frame_number == 0) ? target_size_key_frame_initial_ | |
292 : target_size_key_frame_; | |
293 sum_key_frame_size_mismatch_ += | |
294 fabs(encoded_size_kbits - target_size) / target_size; | |
295 num_key_frames_ += 1; | |
296 } | |
297 sum_encoded_frame_size_[tl_idx] += encoded_size_kbits; | |
298 // Encoding bit rate per temporal layer: from the start of the update/run | |
299 // to the current frame. | |
300 encoding_bitrate_[tl_idx] = sum_encoded_frame_size_[tl_idx] * | |
301 framerate_layer_[tl_idx] / | |
302 num_frames_per_update_[tl_idx]; | |
303 // Total encoding rate: from the start of the update/run to current frame. | |
304 sum_encoded_frame_size_total_ += encoded_size_kbits; | |
305 encoding_bitrate_total_ = | |
306 sum_encoded_frame_size_total_ * framerate_ / num_frames_total_; | |
307 perc_encoding_rate_mismatch_ = | |
308 100 * fabs(encoding_bitrate_total_ - bitrate_kbps_) / bitrate_kbps_; | |
309 if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && | |
310 !encoding_rate_within_target_) { | |
311 num_frames_to_hit_target_ = num_frames_total_; | |
312 encoding_rate_within_target_ = true; | |
313 } | |
314 } | |
315 | |
316 // Verify expected behavior of rate control and print out data. | |
317 void PrintAndMaybeVerifyRateControlMetrics( | |
318 int rate_update_index, | |
319 const std::vector<RateControlThresholds>* rc_thresholds, | |
320 const std::vector<int>& num_dropped_frames, | |
321 const std::vector<int>& num_resize_actions) { | |
322 printf( | |
323 "Rate update #%d:\n" | |
324 " Target bitrate : %d\n" | |
325 " Encoded bitrate : %f\n" | |
326 " Frame rate : %d\n", | |
327 rate_update_index, bitrate_kbps_, encoding_bitrate_total_, framerate_); | |
328 printf( | |
329 " # processed frames : %d\n" | |
330 " # frames to convergence: %d\n" | |
331 " # dropped frames : %d\n" | |
332 " # spatial resizes : %d\n", | |
333 num_frames_total_, num_frames_to_hit_target_, | |
334 num_dropped_frames[rate_update_index], | |
335 num_resize_actions[rate_update_index]); | |
336 | |
337 const RateControlThresholds* rc_threshold = nullptr; | |
338 if (rc_thresholds) { | |
339 rc_threshold = &(*rc_thresholds)[rate_update_index]; | |
340 | |
341 EXPECT_LE(perc_encoding_rate_mismatch_, | |
342 rc_threshold->max_encoding_rate_mismatch); | |
343 } | |
344 if (num_key_frames_ > 0) { | |
345 int perc_key_frame_size_mismatch = | |
346 100 * sum_key_frame_size_mismatch_ / num_key_frames_; | |
347 printf( | |
348 " # key frames : %d\n" | |
349 " Key frame rate mismatch: %d\n", | |
350 num_key_frames_, perc_key_frame_size_mismatch); | |
351 if (rc_threshold) { | |
352 EXPECT_LE(perc_key_frame_size_mismatch, | |
353 rc_threshold->max_key_frame_size_mismatch); | |
354 } | |
355 } | |
356 | |
357 const int num_temporal_layers = | |
358 NumberOfTemporalLayers(config_.codec_settings); | |
359 for (int i = 0; i < num_temporal_layers; i++) { | |
360 int perc_frame_size_mismatch = | |
361 100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i]; | |
362 int perc_encoding_rate_mismatch = | |
363 100 * fabs(encoding_bitrate_[i] - bitrate_layer_[i]) / | |
364 bitrate_layer_[i]; | |
365 printf( | |
366 " Temporal layer #%d:\n" | |
367 " Target layer bitrate : %f\n" | |
368 " Layer frame rate : %f\n" | |
369 " Layer per frame bandwidth : %f\n" | |
370 " Layer encoding bitrate : %f\n" | |
371 " Layer percent frame size mismatch : %d\n" | |
372 " Layer percent encoding rate mismatch: %d\n" | |
373 " # frames processed per layer : %d\n", | |
374 i, bitrate_layer_[i], framerate_layer_[i], per_frame_bandwidth_[i], | |
375 encoding_bitrate_[i], perc_frame_size_mismatch, | |
376 perc_encoding_rate_mismatch, num_frames_per_update_[i]); | |
377 if (rc_threshold) { | |
378 EXPECT_LE(perc_frame_size_mismatch, | |
379 rc_threshold->max_delta_frame_size_mismatch); | |
380 EXPECT_LE(perc_encoding_rate_mismatch, | |
381 rc_threshold->max_encoding_rate_mismatch); | |
382 } | |
383 } | |
384 printf("\n"); | |
385 | |
386 if (rc_threshold) { | |
387 EXPECT_LE(num_frames_to_hit_target_, rc_threshold->max_time_hit_target); | |
388 EXPECT_LE(num_dropped_frames[rate_update_index], | |
389 rc_threshold->max_num_dropped_frames); | |
390 EXPECT_EQ(rc_threshold->num_spatial_resizes, | |
391 num_resize_actions[rate_update_index]); | |
392 EXPECT_EQ(rc_threshold->num_key_frames, num_key_frames_); | |
393 } | |
394 } | |
395 | |
396 static void VerifyQuality(const QualityMetricsResult& psnr_result, | |
397 const QualityMetricsResult& ssim_result, | |
398 const QualityThresholds& quality_thresholds) { | |
399 EXPECT_GT(psnr_result.average, quality_thresholds.min_avg_psnr); | |
400 EXPECT_GT(psnr_result.min, quality_thresholds.min_min_psnr); | |
401 EXPECT_GT(ssim_result.average, quality_thresholds.min_avg_ssim); | |
402 EXPECT_GT(ssim_result.min, quality_thresholds.min_min_ssim); | |
403 } | |
404 | |
405 static int NumberOfTemporalLayers(const VideoCodec& codec_settings) { | |
406 if (codec_settings.codecType == kVideoCodecVP8) { | |
407 return codec_settings.VP8().numberOfTemporalLayers; | |
408 } else if (codec_settings.codecType == kVideoCodecVP9) { | |
409 return codec_settings.VP9().numberOfTemporalLayers; | |
410 } else { | |
411 return 1; | |
412 } | |
413 } | |
414 | |
415 // Temporal layer index corresponding to frame number, for up to 3 layers. | |
416 int TemporalLayerIndexForFrame(int frame_number) { | |
417 const int num_temporal_layers = | |
418 NumberOfTemporalLayers(config_.codec_settings); | |
419 int tl_idx = -1; | |
420 switch (num_temporal_layers) { | |
421 case 1: | |
422 tl_idx = 0; | |
423 break; | |
424 case 2: | |
425 // temporal layer 0: 0 2 4 ... | |
426 // temporal layer 1: 1 3 | |
427 tl_idx = (frame_number % 2 == 0) ? 0 : 1; | |
428 break; | |
429 case 3: | |
430 // temporal layer 0: 0 4 8 ... | |
431 // temporal layer 1: 2 6 | |
432 // temporal layer 2: 1 3 5 7 | |
433 if (frame_number % 4 == 0) { | |
434 tl_idx = 0; | |
435 } else if ((frame_number + 2) % 4 == 0) { | |
436 tl_idx = 1; | |
437 } else if ((frame_number + 1) % 2 == 0) { | |
438 tl_idx = 2; | |
439 } | |
440 break; | |
441 default: | |
442 RTC_NOTREACHED(); | |
443 break; | |
444 } | |
445 return tl_idx; | |
446 } | |
447 | |
448 // Reset quantities before each encoder rate update. | |
449 void ResetRateControlMetrics(int rate_update_index, | |
450 const RateProfile& rate_profile) { | |
451 // Set new rates. | |
452 bitrate_kbps_ = rate_profile.target_bit_rate[rate_update_index]; | |
453 framerate_ = rate_profile.input_frame_rate[rate_update_index]; | |
454 const int num_temporal_layers = | |
455 NumberOfTemporalLayers(config_.codec_settings); | |
456 RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers); | |
457 for (int i = 0; i < num_temporal_layers; i++) { | |
458 float bit_rate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i]; | |
459 if (i > 0) { | |
460 float bit_rate_delta_ratio = | |
461 kVp8LayerRateAlloction[num_temporal_layers - 1][i] - | |
462 kVp8LayerRateAlloction[num_temporal_layers - 1][i - 1]; | |
463 bitrate_layer_[i] = bitrate_kbps_ * bit_rate_delta_ratio; | |
464 } else { | |
465 bitrate_layer_[i] = bitrate_kbps_ * bit_rate_ratio; | |
466 } | |
467 framerate_layer_[i] = | |
468 framerate_ / static_cast<float>(1 << (num_temporal_layers - 1)); | |
469 } | |
470 if (num_temporal_layers == 3) { | |
471 framerate_layer_[2] = framerate_ / 2.0f; | |
472 } | |
473 if (rate_update_index == 0) { | |
474 target_size_key_frame_initial_ = | |
475 0.5 * kInitialBufferSize * bitrate_layer_[0]; | |
476 } | |
477 | |
478 // Reset rate control metrics. | |
479 for (int i = 0; i < num_temporal_layers; i++) { | |
480 num_frames_per_update_[i] = 0; | |
481 sum_frame_size_mismatch_[i] = 0.0f; | |
482 sum_encoded_frame_size_[i] = 0.0f; | |
483 encoding_bitrate_[i] = 0.0f; | |
484 // Update layer per-frame-bandwidth. | |
485 per_frame_bandwidth_[i] = static_cast<float>(bitrate_layer_[i]) / | |
486 static_cast<float>(framerate_layer_[i]); | |
487 } | |
488 // Set maximum size of key frames, following setting in the VP8 wrapper. | |
489 float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * framerate_; | |
490 // We don't know exact target size of the key frames (except for first one), | |
491 // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is | |
492 // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average | |
493 // as reference for mismatch. Note key frames always correspond to base | |
494 // layer frame in this test. | |
495 target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; | |
496 num_frames_total_ = 0; | |
497 sum_encoded_frame_size_total_ = 0.0f; | |
498 encoding_bitrate_total_ = 0.0f; | |
499 perc_encoding_rate_mismatch_ = 0.0f; | |
500 num_frames_to_hit_target_ = | |
501 rate_profile.frame_index_rate_update[rate_update_index + 1]; | |
502 encoding_rate_within_target_ = false; | |
503 sum_key_frame_size_mismatch_ = 0.0; | |
504 num_key_frames_ = 0; | |
505 } | |
506 | |
507 // Processes all frames in the clip and verifies the result. | |
508 void ProcessFramesAndMaybeVerify( | |
509 const RateProfile& rate_profile, | |
510 const std::vector<RateControlThresholds>* rc_thresholds, | |
511 const QualityThresholds* quality_thresholds, | |
512 const VisualizationParams* visualization_params) { | |
513 // The Android HW codec needs to be run on a task queue, so we simply always | |
514 // run the test on a task queue. | |
515 rtc::TaskQueue task_queue("VidProc TQ"); | |
516 rtc::Event sync_event(false, false); | |
517 | |
518 SetUpAndInitObjects(&task_queue, rate_profile.target_bit_rate[0], | |
519 rate_profile.input_frame_rate[0], visualization_params); | |
520 | |
521 // Set initial rates. | |
522 int rate_update_index = 0; | |
523 task_queue.PostTask([this, &rate_profile, rate_update_index] { | |
524 processor_->SetRates(rate_profile.target_bit_rate[rate_update_index], | |
525 rate_profile.input_frame_rate[rate_update_index]); | |
526 }); | |
527 | |
528 // Process all frames. | |
529 int frame_number = 0; | |
530 const int num_frames = rate_profile.num_frames; | |
531 RTC_DCHECK_GE(num_frames, 1); | |
532 while (frame_number < num_frames) { | |
533 // In order to not overwhelm the OpenMAX buffers in the Android | |
534 // MediaCodec API, we roughly pace the frames here. The downside | |
535 // of this is that the encode run will be done in real-time. | |
536 // TODO(brandtr): Investigate if this is needed on iOS. | |
537 if (config_.hw_codec) { | |
538 SleepMs(rtc::kNumMillisecsPerSec / | |
539 rate_profile.input_frame_rate[rate_update_index]); | |
540 } | |
541 | |
542 task_queue.PostTask( | |
543 [this, frame_number] { processor_->ProcessFrame(frame_number); }); | |
544 ++frame_number; | |
545 | |
546 if (frame_number == | |
547 rate_profile.frame_index_rate_update[rate_update_index + 1]) { | |
548 ++rate_update_index; | |
549 | |
550 task_queue.PostTask([this, &rate_profile, rate_update_index] { | |
551 processor_->SetRates( | |
552 rate_profile.target_bit_rate[rate_update_index], | |
553 rate_profile.input_frame_rate[rate_update_index]); | |
554 }); | |
555 } | |
556 } | |
557 | |
558 // Give the VideoProcessor pipeline some time to process the last frame, | |
559 // and then release the codecs. | |
560 if (config_.hw_codec) { | |
561 SleepMs(1 * rtc::kNumMillisecsPerSec); | |
562 } | |
563 ReleaseAndCloseObjects(&task_queue); | |
564 | |
565 // Calculate and print rate control statistics. | |
566 rate_update_index = 0; | |
567 frame_number = 0; | |
568 ResetRateControlMetrics(rate_update_index, rate_profile); | |
569 std::vector<int> num_dropped_frames; | |
570 std::vector<int> num_resize_actions; | |
571 sync_event.Reset(); | |
572 task_queue.PostTask( | |
573 [this, &num_dropped_frames, &num_resize_actions, &sync_event]() { | |
574 num_dropped_frames = processor_->NumberDroppedFramesPerRateUpdate(); | |
575 num_resize_actions = processor_->NumberSpatialResizesPerRateUpdate(); | |
576 sync_event.Set(); | |
577 }); | |
578 sync_event.Wait(rtc::Event::kForever); | |
579 while (frame_number < num_frames) { | |
580 UpdateRateControlMetrics(frame_number); | |
581 | |
582 ++frame_number; | |
583 | |
584 if (frame_number == | |
585 rate_profile.frame_index_rate_update[rate_update_index + 1]) { | |
586 PrintAndMaybeVerifyRateControlMetrics(rate_update_index, rc_thresholds, | |
587 num_dropped_frames, | |
588 num_resize_actions); | |
589 ++rate_update_index; | |
590 ResetRateControlMetrics(rate_update_index, rate_profile); | |
591 } | |
592 } | |
593 PrintAndMaybeVerifyRateControlMetrics(rate_update_index, rc_thresholds, | |
594 num_dropped_frames, | |
595 num_resize_actions); | |
596 | |
597 // Calculate and print other statistics. | |
598 EXPECT_EQ(num_frames, static_cast<int>(stats_.stats_.size())); | |
599 stats_.PrintSummary(); | |
600 | |
601 // Calculate and print image quality statistics. | |
602 // TODO(marpan): Should compute these quality metrics per SetRates update. | |
603 QualityMetricsResult psnr_result, ssim_result; | |
604 EXPECT_EQ(0, I420MetricsFromFiles(config_.input_filename.c_str(), | |
605 config_.output_filename.c_str(), | |
606 config_.codec_settings.width, | |
607 config_.codec_settings.height, | |
608 &psnr_result, &ssim_result)); | |
609 if (quality_thresholds) { | |
610 VerifyQuality(psnr_result, ssim_result, *quality_thresholds); | |
611 } | |
612 printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n", | |
613 psnr_result.average, psnr_result.min, ssim_result.average, | |
614 ssim_result.min); | |
615 printf("\n"); | |
616 | |
617 // Remove analysis file. | |
618 if (remove(config_.output_filename.c_str()) < 0) { | |
619 fprintf(stderr, "Failed to remove temporary file!\n"); | |
620 } | |
621 } | |
622 | 95 |
623 static void SetTestConfig(TestConfig* config, | 96 static void SetTestConfig(TestConfig* config, |
624 bool hw_codec, | 97 bool hw_codec, |
625 bool use_single_core, | 98 bool use_single_core, |
626 float packet_loss_probability, | 99 float packet_loss_probability, |
627 std::string filename, | 100 std::string filename, |
628 bool verbose_logging) { | 101 bool verbose_logging); |
629 config->filename = filename; | |
630 config->input_filename = ResourcePath(filename, "yuv"); | |
631 // Generate an output filename in a safe way. | |
632 config->output_filename = | |
633 TempFilename(OutputPath(), "videoprocessor_integrationtest"); | |
634 config->networking_config.packet_loss_probability = packet_loss_probability; | |
635 config->use_single_core = use_single_core; | |
636 config->verbose = verbose_logging; | |
637 config->hw_codec = hw_codec; | |
638 } | |
639 | 102 |
640 static void SetCodecSettings(TestConfig* config, | 103 static void SetCodecSettings(TestConfig* config, |
641 VideoCodecType codec_type, | 104 VideoCodecType codec_type, |
642 int num_temporal_layers, | 105 int num_temporal_layers, |
643 bool error_concealment_on, | 106 bool error_concealment_on, |
644 bool denoising_on, | 107 bool denoising_on, |
645 bool frame_dropper_on, | 108 bool frame_dropper_on, |
646 bool spatial_resize_on, | 109 bool spatial_resize_on, |
647 bool resilience_on, | 110 bool resilience_on, |
648 int width, | 111 int width, |
649 int height) { | 112 int height); |
650 webrtc::test::CodecSettings(codec_type, &config->codec_settings); | |
651 config->codec_settings.width = width; | |
652 config->codec_settings.height = height; | |
653 switch (config->codec_settings.codecType) { | |
654 case kVideoCodecVP8: | |
655 config->codec_settings.VP8()->resilience = | |
656 resilience_on ? kResilientStream : kResilienceOff; | |
657 config->codec_settings.VP8()->numberOfTemporalLayers = | |
658 num_temporal_layers; | |
659 config->codec_settings.VP8()->denoisingOn = denoising_on; | |
660 config->codec_settings.VP8()->errorConcealmentOn = error_concealment_on; | |
661 config->codec_settings.VP8()->automaticResizeOn = spatial_resize_on; | |
662 config->codec_settings.VP8()->frameDroppingOn = frame_dropper_on; | |
663 config->codec_settings.VP8()->keyFrameInterval = kBaseKeyFrameInterval; | |
664 break; | |
665 case kVideoCodecVP9: | |
666 config->codec_settings.VP9()->resilienceOn = resilience_on; | |
667 config->codec_settings.VP9()->numberOfTemporalLayers = | |
668 num_temporal_layers; | |
669 config->codec_settings.VP9()->denoisingOn = denoising_on; | |
670 config->codec_settings.VP9()->frameDroppingOn = frame_dropper_on; | |
671 config->codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval; | |
672 config->codec_settings.VP9()->automaticResizeOn = spatial_resize_on; | |
673 break; | |
674 case kVideoCodecH264: | |
675 config->codec_settings.H264()->frameDroppingOn = frame_dropper_on; | |
676 config->codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval; | |
677 break; | |
678 default: | |
679 RTC_NOTREACHED(); | |
680 break; | |
681 } | |
682 | |
683 config->frame_length_in_bytes = | |
684 CalcBufferSize(VideoType::kI420, width, height); | |
685 } | |
686 | 113 |
687 static void SetRateProfile(RateProfile* rate_profile, | 114 static void SetRateProfile(RateProfile* rate_profile, |
688 int rate_update_index, | 115 int rate_update_index, |
689 int bitrate_kbps, | 116 int bitrate_kbps, |
690 int framerate_fps, | 117 int framerate_fps, |
691 int frame_index_rate_update) { | 118 int frame_index_rate_update); |
692 rate_profile->target_bit_rate[rate_update_index] = bitrate_kbps; | |
693 rate_profile->input_frame_rate[rate_update_index] = framerate_fps; | |
694 rate_profile->frame_index_rate_update[rate_update_index] = | |
695 frame_index_rate_update; | |
696 } | |
697 | 119 |
698 static void AddRateControlThresholds( | 120 static void AddRateControlThresholds( |
699 int max_num_dropped_frames, | 121 int max_num_dropped_frames, |
700 int max_key_frame_size_mismatch, | 122 int max_key_frame_size_mismatch, |
701 int max_delta_frame_size_mismatch, | 123 int max_delta_frame_size_mismatch, |
702 int max_encoding_rate_mismatch, | 124 int max_encoding_rate_mismatch, |
703 int max_time_hit_target, | 125 int max_time_hit_target, |
704 int num_spatial_resizes, | 126 int num_spatial_resizes, |
705 int num_key_frames, | 127 int num_key_frames, |
706 std::vector<RateControlThresholds>* rc_thresholds) { | 128 std::vector<RateControlThresholds>* rc_thresholds); |
707 RTC_DCHECK(rc_thresholds); | |
708 | 129 |
709 rc_thresholds->emplace_back(); | 130 void ProcessFramesAndMaybeVerify( |
710 RateControlThresholds* rc_threshold = &rc_thresholds->back(); | 131 const RateProfile& rate_profile, |
711 rc_threshold->max_num_dropped_frames = max_num_dropped_frames; | 132 const std::vector<RateControlThresholds>* rc_thresholds, |
712 rc_threshold->max_key_frame_size_mismatch = max_key_frame_size_mismatch; | 133 const QualityThresholds* quality_thresholds, |
713 rc_threshold->max_delta_frame_size_mismatch = max_delta_frame_size_mismatch; | 134 const VisualizationParams* visualization_params); |
714 rc_threshold->max_encoding_rate_mismatch = max_encoding_rate_mismatch; | |
715 rc_threshold->max_time_hit_target = max_time_hit_target; | |
716 rc_threshold->num_spatial_resizes = num_spatial_resizes; | |
717 rc_threshold->num_key_frames = num_key_frames; | |
718 } | |
719 | 135 |
720 // Config. | 136 // Config. |
721 TestConfig config_; | 137 TestConfig config_; |
722 | 138 |
| 139 private: |
| 140 static const int kMaxNumTemporalLayers = 3; |
| 141 |
| 142 void CreateEncoderAndDecoder(); |
| 143 void DestroyEncoderAndDecoder(); |
| 144 void SetUpAndInitObjects(rtc::TaskQueue* task_queue, |
| 145 const int initial_bitrate_kbps, |
| 146 const int initial_framerate_fps, |
| 147 const VisualizationParams* visualization_params); |
| 148 void ReleaseAndCloseObjects(rtc::TaskQueue* task_queue); |
| 149 void UpdateRateControlMetrics(int frame_number); |
| 150 void PrintAndMaybeVerifyRateControlMetrics( |
| 151 int rate_update_index, |
| 152 const std::vector<RateControlThresholds>* rc_thresholds, |
| 153 const std::vector<int>& num_dropped_frames, |
| 154 const std::vector<int>& num_resize_actions); |
| 155 int TemporalLayerIndexForFrame(int frame_number) const; |
| 156 void ResetRateControlMetrics(int rate_update_index, |
| 157 const RateProfile& rate_profile); |
| 158 |
723 // Codecs. | 159 // Codecs. |
724 std::unique_ptr<cricket::WebRtcVideoEncoderFactory> encoder_factory_; | 160 std::unique_ptr<cricket::WebRtcVideoEncoderFactory> encoder_factory_; |
725 VideoEncoder* encoder_; | 161 VideoEncoder* encoder_; |
726 std::unique_ptr<cricket::WebRtcVideoDecoderFactory> decoder_factory_; | 162 std::unique_ptr<cricket::WebRtcVideoDecoderFactory> decoder_factory_; |
727 VideoDecoder* decoder_; | 163 VideoDecoder* decoder_; |
728 | 164 |
729 // Helper objects. | 165 // Helper objects. |
730 std::unique_ptr<FrameReader> analysis_frame_reader_; | 166 std::unique_ptr<FrameReader> analysis_frame_reader_; |
731 std::unique_ptr<FrameWriter> analysis_frame_writer_; | 167 std::unique_ptr<FrameWriter> analysis_frame_writer_; |
732 std::unique_ptr<IvfFileWriter> encoded_frame_writer_; | 168 std::unique_ptr<IvfFileWriter> encoded_frame_writer_; |
(...skipping 22 matching lines...) Expand all Loading... |
755 float target_size_key_frame_initial_; | 191 float target_size_key_frame_initial_; |
756 float target_size_key_frame_; | 192 float target_size_key_frame_; |
757 float sum_key_frame_size_mismatch_; | 193 float sum_key_frame_size_mismatch_; |
758 int num_key_frames_; | 194 int num_key_frames_; |
759 }; | 195 }; |
760 | 196 |
761 } // namespace test | 197 } // namespace test |
762 } // namespace webrtc | 198 } // namespace webrtc |
763 | 199 |
764 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTES
T_H_ | 200 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_INTEGRATIONTES
T_H_ |
OLD | NEW |