| OLD | NEW |
|---|
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include <math.h> | 11 #include "webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest
.h" |
| 12 | |
| 13 #include <memory> | |
| 14 | |
| 15 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h" | |
| 16 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" | |
| 17 #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" | |
| 18 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" | |
| 19 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" | |
| 20 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" | |
| 21 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" | |
| 22 #include "webrtc/modules/video_coding/include/video_codec_interface.h" | |
| 23 #include "webrtc/modules/video_coding/include/video_coding.h" | |
| 24 #include "webrtc/test/gtest.h" | |
| 25 #include "webrtc/test/testsupport/fileutils.h" | |
| 26 #include "webrtc/test/testsupport/frame_reader.h" | |
| 27 #include "webrtc/test/testsupport/frame_writer.h" | |
| 28 #include "webrtc/test/testsupport/metrics/video_metrics.h" | |
| 29 #include "webrtc/test/testsupport/packet_reader.h" | |
| 30 #include "webrtc/typedefs.h" | |
| 31 | 12 |
| 32 namespace webrtc { | 13 namespace webrtc { |
| 33 namespace { | 14 namespace test { |
| 34 // Maximum number of rate updates (i.e., calls to encoder to change bitrate | |
| 35 // and/or frame rate) for the current tests. | |
| 36 const int kMaxNumRateUpdates = 3; | |
| 37 | |
| 38 const int kPercTargetvsActualMismatch = 20; | |
| 39 const int kBaseKeyFrameInterval = 3000; | |
| 40 | |
| 41 // Codec and network settings. | |
| 42 struct CodecConfigPars { | |
| 43 VideoCodecType codec_type; | |
| 44 float packet_loss; | |
| 45 int num_temporal_layers; | |
| 46 int key_frame_interval; | |
| 47 bool error_concealment_on; | |
| 48 bool denoising_on; | |
| 49 bool frame_dropper_on; | |
| 50 bool spatial_resize_on; | |
| 51 }; | |
| 52 | |
| 53 // Quality metrics. | |
| 54 struct QualityMetrics { | |
| 55 double minimum_avg_psnr; | |
| 56 double minimum_min_psnr; | |
| 57 double minimum_avg_ssim; | |
| 58 double minimum_min_ssim; | |
| 59 }; | |
| 60 | |
| 61 // The sequence of bitrate and frame rate changes for the encoder, the frame | |
| 62 // number where the changes are made, and the total number of frames for the | |
| 63 // test. | |
| 64 struct RateProfile { | |
| 65 int target_bit_rate[kMaxNumRateUpdates]; | |
| 66 int input_frame_rate[kMaxNumRateUpdates]; | |
| 67 int frame_index_rate_update[kMaxNumRateUpdates + 1]; | |
| 68 int num_frames; | |
| 69 }; | |
| 70 | |
| 71 // Metrics for the rate control. The rate mismatch metrics are defined as | |
| 72 // percentages.|max_time_hit_target| is defined as number of frames, after a | |
| 73 // rate update is made to the encoder, for the encoder to reach within | |
| 74 // |kPercTargetvsActualMismatch| of new target rate. The metrics are defined for | |
| 75 // each rate update sequence. | |
| 76 struct RateControlMetrics { | |
| 77 int max_num_dropped_frames; | |
| 78 int max_key_frame_size_mismatch; | |
| 79 int max_delta_frame_size_mismatch; | |
| 80 int max_encoding_rate_mismatch; | |
| 81 int max_time_hit_target; | |
| 82 int num_spatial_resizes; | |
| 83 int num_key_frames; | |
| 84 }; | |
| 85 | |
| 86 // Sequence used is foreman (CIF): may be better to use VGA for resize test. | |
| 87 const int kCIFWidth = 352; | |
| 88 const int kCIFHeight = 288; | |
| 89 #if !defined(WEBRTC_IOS) | |
| 90 const int kNbrFramesShort = 100; // Some tests are run for shorter sequence. | |
| 91 #endif | |
| 92 const int kNbrFramesLong = 299; | |
| 93 | |
| 94 // Parameters from VP8 wrapper, which control target size of key frames. | |
| 95 const float kInitialBufferSize = 0.5f; | |
| 96 const float kOptimalBufferSize = 0.6f; | |
| 97 const float kScaleKeyFrameSize = 0.5f; | |
| 98 | |
| 99 void SetRateProfilePars(RateProfile* rate_profile, | |
| 100 int update_index, | |
| 101 int bit_rate, | |
| 102 int frame_rate, | |
| 103 int frame_index_rate_update) { | |
| 104 rate_profile->target_bit_rate[update_index] = bit_rate; | |
| 105 rate_profile->input_frame_rate[update_index] = frame_rate; | |
| 106 rate_profile->frame_index_rate_update[update_index] = frame_index_rate_update; | |
| 107 } | |
| 108 | |
| 109 void SetCodecParameters(CodecConfigPars* process_settings, | |
| 110 VideoCodecType codec_type, | |
| 111 float packet_loss, | |
| 112 int key_frame_interval, | |
| 113 int num_temporal_layers, | |
| 114 bool error_concealment_on, | |
| 115 bool denoising_on, | |
| 116 bool frame_dropper_on, | |
| 117 bool spatial_resize_on) { | |
| 118 process_settings->codec_type = codec_type; | |
| 119 process_settings->packet_loss = packet_loss; | |
| 120 process_settings->key_frame_interval = key_frame_interval; | |
| 121 process_settings->num_temporal_layers = num_temporal_layers, | |
| 122 process_settings->error_concealment_on = error_concealment_on; | |
| 123 process_settings->denoising_on = denoising_on; | |
| 124 process_settings->frame_dropper_on = frame_dropper_on; | |
| 125 process_settings->spatial_resize_on = spatial_resize_on; | |
| 126 } | |
| 127 | |
| 128 void SetQualityMetrics(QualityMetrics* quality_metrics, | |
| 129 double minimum_avg_psnr, | |
| 130 double minimum_min_psnr, | |
| 131 double minimum_avg_ssim, | |
| 132 double minimum_min_ssim) { | |
| 133 quality_metrics->minimum_avg_psnr = minimum_avg_psnr; | |
| 134 quality_metrics->minimum_min_psnr = minimum_min_psnr; | |
| 135 quality_metrics->minimum_avg_ssim = minimum_avg_ssim; | |
| 136 quality_metrics->minimum_min_ssim = minimum_min_ssim; | |
| 137 } | |
| 138 | |
| 139 void SetRateControlMetrics(RateControlMetrics* rc_metrics, | |
| 140 int update_index, | |
| 141 int max_num_dropped_frames, | |
| 142 int max_key_frame_size_mismatch, | |
| 143 int max_delta_frame_size_mismatch, | |
| 144 int max_encoding_rate_mismatch, | |
| 145 int max_time_hit_target, | |
| 146 int num_spatial_resizes, | |
| 147 int num_key_frames) { | |
| 148 rc_metrics[update_index].max_num_dropped_frames = max_num_dropped_frames; | |
| 149 rc_metrics[update_index].max_key_frame_size_mismatch = | |
| 150 max_key_frame_size_mismatch; | |
| 151 rc_metrics[update_index].max_delta_frame_size_mismatch = | |
| 152 max_delta_frame_size_mismatch; | |
| 153 rc_metrics[update_index].max_encoding_rate_mismatch = | |
| 154 max_encoding_rate_mismatch; | |
| 155 rc_metrics[update_index].max_time_hit_target = max_time_hit_target; | |
| 156 rc_metrics[update_index].num_spatial_resizes = num_spatial_resizes; | |
| 157 rc_metrics[update_index].num_key_frames = num_key_frames; | |
| 158 } | |
| 159 } // namespace | |
| 160 | |
| 161 // Integration test for video processor. Encodes+decodes a clip and | |
| 162 // writes it to the output directory. After completion, quality metrics | |
| 163 // (PSNR and SSIM) and rate control metrics are computed to verify that the | |
| 164 // quality and encoder response is acceptable. The rate control tests allow us | |
| 165 // to verify the behavior for changing bitrate, changing frame rate, frame | |
| 166 // dropping/spatial resize, and temporal layers. The limits for the rate | |
| 167 // control metrics are set to be fairly conservative, so failure should only | |
| 168 // happen when some significant regression or breakdown occurs. | |
| 169 class VideoProcessorIntegrationTest : public testing::Test { | |
| 170 protected: | |
| 171 std::unique_ptr<VideoEncoder> encoder_; | |
| 172 std::unique_ptr<VideoDecoder> decoder_; | |
| 173 std::unique_ptr<test::FrameReader> frame_reader_; | |
| 174 std::unique_ptr<test::FrameWriter> frame_writer_; | |
| 175 test::PacketReader packet_reader_; | |
| 176 std::unique_ptr<test::PacketManipulator> packet_manipulator_; | |
| 177 test::Stats stats_; | |
| 178 test::TestConfig config_; | |
| 179 VideoCodec codec_settings_; | |
| 180 std::unique_ptr<test::VideoProcessor> processor_; | |
| 181 TemporalLayersFactory tl_factory_; | |
| 182 | |
| 183 // Quantities defined/updated for every encoder rate update. | |
| 184 // Some quantities defined per temporal layer (at most 3 layers in this test). | |
| 185 int num_frames_per_update_[3]; | |
| 186 float sum_frame_size_mismatch_[3]; | |
| 187 float sum_encoded_frame_size_[3]; | |
| 188 float encoding_bitrate_[3]; | |
| 189 float per_frame_bandwidth_[3]; | |
| 190 float bit_rate_layer_[3]; | |
| 191 float frame_rate_layer_[3]; | |
| 192 int num_frames_total_; | |
| 193 float sum_encoded_frame_size_total_; | |
| 194 float encoding_bitrate_total_; | |
| 195 float perc_encoding_rate_mismatch_; | |
| 196 int num_frames_to_hit_target_; | |
| 197 bool encoding_rate_within_target_; | |
| 198 int bit_rate_; | |
| 199 int frame_rate_; | |
| 200 int layer_; | |
| 201 float target_size_key_frame_initial_; | |
| 202 float target_size_key_frame_; | |
| 203 float sum_key_frame_size_mismatch_; | |
| 204 int num_key_frames_; | |
| 205 float start_bitrate_; | |
| 206 | |
| 207 // Codec and network settings. | |
| 208 VideoCodecType codec_type_; | |
| 209 float packet_loss_; | |
| 210 int num_temporal_layers_; | |
| 211 int key_frame_interval_; | |
| 212 bool error_concealment_on_; | |
| 213 bool denoising_on_; | |
| 214 bool frame_dropper_on_; | |
| 215 bool spatial_resize_on_; | |
| 216 | |
| 217 VideoProcessorIntegrationTest() {} | |
| 218 virtual ~VideoProcessorIntegrationTest() {} | |
| 219 | |
| 220 void SetUpCodecConfig() { | |
| 221 if (codec_type_ == kVideoCodecH264) { | |
| 222 encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264"))); | |
| 223 decoder_.reset(H264Decoder::Create()); | |
| 224 VideoCodingModule::Codec(kVideoCodecH264, &codec_settings_); | |
| 225 } else if (codec_type_ == kVideoCodecVP8) { | |
| 226 encoder_.reset(VP8Encoder::Create()); | |
| 227 decoder_.reset(VP8Decoder::Create()); | |
| 228 VideoCodingModule::Codec(kVideoCodecVP8, &codec_settings_); | |
| 229 } else if (codec_type_ == kVideoCodecVP9) { | |
| 230 encoder_.reset(VP9Encoder::Create()); | |
| 231 decoder_.reset(VP9Decoder::Create()); | |
| 232 VideoCodingModule::Codec(kVideoCodecVP9, &codec_settings_); | |
| 233 } | |
| 234 | |
| 235 // CIF is currently used for all tests below. | |
| 236 // Setup the TestConfig struct for processing of a clip in CIF resolution. | |
| 237 config_.input_filename = webrtc::test::ResourcePath("foreman_cif", "yuv"); | |
| 238 | |
| 239 // Generate an output filename in a safe way. | |
| 240 config_.output_filename = webrtc::test::TempFilename( | |
| 241 webrtc::test::OutputPath(), "videoprocessor_integrationtest"); | |
| 242 config_.frame_length_in_bytes = | |
| 243 CalcBufferSize(kI420, kCIFWidth, kCIFHeight); | |
| 244 config_.verbose = false; | |
| 245 // Only allow encoder/decoder to use single core, for predictability. | |
| 246 config_.use_single_core = true; | |
| 247 // Key frame interval and packet loss are set for each test. | |
| 248 config_.keyframe_interval = key_frame_interval_; | |
| 249 config_.networking_config.packet_loss_probability = packet_loss_; | |
| 250 | |
| 251 // Configure codec settings. | |
| 252 config_.codec_settings = &codec_settings_; | |
| 253 config_.codec_settings->startBitrate = start_bitrate_; | |
| 254 config_.codec_settings->width = kCIFWidth; | |
| 255 config_.codec_settings->height = kCIFHeight; | |
| 256 | |
| 257 // These features may be set depending on the test. | |
| 258 switch (config_.codec_settings->codecType) { | |
| 259 case kVideoCodecH264: | |
| 260 config_.codec_settings->H264()->frameDroppingOn = frame_dropper_on_; | |
| 261 config_.codec_settings->H264()->keyFrameInterval = | |
| 262 kBaseKeyFrameInterval; | |
| 263 break; | |
| 264 case kVideoCodecVP8: | |
| 265 config_.codec_settings->VP8()->errorConcealmentOn = | |
| 266 error_concealment_on_; | |
| 267 config_.codec_settings->VP8()->denoisingOn = denoising_on_; | |
| 268 config_.codec_settings->VP8()->numberOfTemporalLayers = | |
| 269 num_temporal_layers_; | |
| 270 config_.codec_settings->VP8()->frameDroppingOn = frame_dropper_on_; | |
| 271 config_.codec_settings->VP8()->automaticResizeOn = spatial_resize_on_; | |
| 272 config_.codec_settings->VP8()->keyFrameInterval = kBaseKeyFrameInterval; | |
| 273 break; | |
| 274 case kVideoCodecVP9: | |
| 275 config_.codec_settings->VP9()->denoisingOn = denoising_on_; | |
| 276 config_.codec_settings->VP9()->numberOfTemporalLayers = | |
| 277 num_temporal_layers_; | |
| 278 config_.codec_settings->VP9()->frameDroppingOn = frame_dropper_on_; | |
| 279 config_.codec_settings->VP9()->automaticResizeOn = spatial_resize_on_; | |
| 280 config_.codec_settings->VP9()->keyFrameInterval = kBaseKeyFrameInterval; | |
| 281 break; | |
| 282 default: | |
| 283 assert(false); | |
| 284 break; | |
| 285 } | |
| 286 frame_reader_.reset(new test::FrameReaderImpl( | |
| 287 config_.input_filename, config_.codec_settings->width, | |
| 288 config_.codec_settings->height)); | |
| 289 frame_writer_.reset(new test::FrameWriterImpl( | |
| 290 config_.output_filename, config_.frame_length_in_bytes)); | |
| 291 ASSERT_TRUE(frame_reader_->Init()); | |
| 292 ASSERT_TRUE(frame_writer_->Init()); | |
| 293 | |
| 294 packet_manipulator_.reset(new test::PacketManipulatorImpl( | |
| 295 &packet_reader_, config_.networking_config, config_.verbose)); | |
| 296 processor_.reset(new test::VideoProcessorImpl( | |
| 297 encoder_.get(), decoder_.get(), frame_reader_.get(), | |
| 298 frame_writer_.get(), packet_manipulator_.get(), config_, &stats_)); | |
| 299 ASSERT_TRUE(processor_->Init()); | |
| 300 } | |
| 301 | |
| 302 // Reset quantities after each encoder update, update the target | |
| 303 // per-frame bandwidth. | |
| 304 void ResetRateControlMetrics(int num_frames) { | |
| 305 for (int i = 0; i < num_temporal_layers_; i++) { | |
| 306 num_frames_per_update_[i] = 0; | |
| 307 sum_frame_size_mismatch_[i] = 0.0f; | |
| 308 sum_encoded_frame_size_[i] = 0.0f; | |
| 309 encoding_bitrate_[i] = 0.0f; | |
| 310 // Update layer per-frame-bandwidth. | |
| 311 per_frame_bandwidth_[i] = static_cast<float>(bit_rate_layer_[i]) / | |
| 312 static_cast<float>(frame_rate_layer_[i]); | |
| 313 } | |
| 314 // Set maximum size of key frames, following setting in the VP8 wrapper. | |
| 315 float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * frame_rate_; | |
| 316 // We don't know exact target size of the key frames (except for first one), | |
| 317 // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is | |
| 318 // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average | |
| 319 // as reference for mismatch. Note key frames always correspond to base | |
| 320 // layer frame in this test. | |
| 321 target_size_key_frame_ = 0.5 * (3 + max_key_size) * per_frame_bandwidth_[0]; | |
| 322 num_frames_total_ = 0; | |
| 323 sum_encoded_frame_size_total_ = 0.0f; | |
| 324 encoding_bitrate_total_ = 0.0f; | |
| 325 perc_encoding_rate_mismatch_ = 0.0f; | |
| 326 num_frames_to_hit_target_ = num_frames; | |
| 327 encoding_rate_within_target_ = false; | |
| 328 sum_key_frame_size_mismatch_ = 0.0; | |
| 329 num_key_frames_ = 0; | |
| 330 } | |
| 331 | |
| 332 // For every encoded frame, update the rate control metrics. | |
| 333 void UpdateRateControlMetrics(int frame_num, FrameType frame_type) { | |
| 334 float encoded_size_kbits = processor_->EncodedFrameSize() * 8.0f / 1000.0f; | |
| 335 // Update layer data. | |
| 336 // Update rate mismatch relative to per-frame bandwidth for delta frames. | |
| 337 if (frame_type == kVideoFrameDelta) { | |
| 338 // TODO(marpan): Should we count dropped (zero size) frames in mismatch? | |
| 339 sum_frame_size_mismatch_[layer_] += | |
| 340 fabs(encoded_size_kbits - per_frame_bandwidth_[layer_]) / | |
| 341 per_frame_bandwidth_[layer_]; | |
| 342 } else { | |
| 343 float target_size = (frame_num == 1) ? target_size_key_frame_initial_ | |
| 344 : target_size_key_frame_; | |
| 345 sum_key_frame_size_mismatch_ += | |
| 346 fabs(encoded_size_kbits - target_size) / target_size; | |
| 347 num_key_frames_ += 1; | |
| 348 } | |
| 349 sum_encoded_frame_size_[layer_] += encoded_size_kbits; | |
| 350 // Encoding bitrate per layer: from the start of the update/run to the | |
| 351 // current frame. | |
| 352 encoding_bitrate_[layer_] = sum_encoded_frame_size_[layer_] * | |
| 353 frame_rate_layer_[layer_] / | |
| 354 num_frames_per_update_[layer_]; | |
| 355 // Total encoding rate: from the start of the update/run to current frame. | |
| 356 sum_encoded_frame_size_total_ += encoded_size_kbits; | |
| 357 encoding_bitrate_total_ = | |
| 358 sum_encoded_frame_size_total_ * frame_rate_ / num_frames_total_; | |
| 359 perc_encoding_rate_mismatch_ = | |
| 360 100 * fabs(encoding_bitrate_total_ - bit_rate_) / bit_rate_; | |
| 361 if (perc_encoding_rate_mismatch_ < kPercTargetvsActualMismatch && | |
| 362 !encoding_rate_within_target_) { | |
| 363 num_frames_to_hit_target_ = num_frames_total_; | |
| 364 encoding_rate_within_target_ = true; | |
| 365 } | |
| 366 } | |
| 367 | |
| 368 // Verify expected behavior of rate control and print out data. | |
| 369 void VerifyRateControl(int update_index, | |
| 370 int max_key_frame_size_mismatch, | |
| 371 int max_delta_frame_size_mismatch, | |
| 372 int max_encoding_rate_mismatch, | |
| 373 int max_time_hit_target, | |
| 374 int max_num_dropped_frames, | |
| 375 int num_spatial_resizes, | |
| 376 int num_key_frames) { | |
| 377 int num_dropped_frames = processor_->NumberDroppedFrames(); | |
| 378 int num_resize_actions = processor_->NumberSpatialResizes(); | |
| 379 printf( | |
| 380 "For update #: %d,\n " | |
| 381 " Target Bitrate: %d,\n" | |
| 382 " Encoding bitrate: %f,\n" | |
| 383 " Frame rate: %d \n", | |
| 384 update_index, bit_rate_, encoding_bitrate_total_, frame_rate_); | |
| 385 printf( | |
| 386 " Number of frames to approach target rate: %d, \n" | |
| 387 " Number of dropped frames: %d, \n" | |
| 388 " Number of spatial resizes: %d, \n", | |
| 389 num_frames_to_hit_target_, num_dropped_frames, num_resize_actions); | |
| 390 EXPECT_LE(perc_encoding_rate_mismatch_, max_encoding_rate_mismatch); | |
| 391 if (num_key_frames_ > 0) { | |
| 392 int perc_key_frame_size_mismatch = | |
| 393 100 * sum_key_frame_size_mismatch_ / num_key_frames_; | |
| 394 printf( | |
| 395 " Number of Key frames: %d \n" | |
| 396 " Key frame rate mismatch: %d \n", | |
| 397 num_key_frames_, perc_key_frame_size_mismatch); | |
| 398 EXPECT_LE(perc_key_frame_size_mismatch, max_key_frame_size_mismatch); | |
| 399 } | |
| 400 printf("\n"); | |
| 401 printf("Rates statistics for Layer data \n"); | |
| 402 for (int i = 0; i < num_temporal_layers_; i++) { | |
| 403 printf("Temporal layer #%d \n", i); | |
| 404 int perc_frame_size_mismatch = | |
| 405 100 * sum_frame_size_mismatch_[i] / num_frames_per_update_[i]; | |
| 406 int perc_encoding_rate_mismatch = | |
| 407 100 * fabs(encoding_bitrate_[i] - bit_rate_layer_[i]) / | |
| 408 bit_rate_layer_[i]; | |
| 409 printf( | |
| 410 " Target Layer Bit rate: %f \n" | |
| 411 " Layer frame rate: %f, \n" | |
| 412 " Layer per frame bandwidth: %f, \n" | |
| 413 " Layer Encoding bit rate: %f, \n" | |
| 414 " Layer Percent frame size mismatch: %d, \n" | |
| 415 " Layer Percent encoding rate mismatch: %d, \n" | |
| 416 " Number of frame processed per layer: %d \n", | |
| 417 bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i], | |
| 418 encoding_bitrate_[i], perc_frame_size_mismatch, | |
| 419 perc_encoding_rate_mismatch, num_frames_per_update_[i]); | |
| 420 EXPECT_LE(perc_frame_size_mismatch, max_delta_frame_size_mismatch); | |
| 421 EXPECT_LE(perc_encoding_rate_mismatch, max_encoding_rate_mismatch); | |
| 422 } | |
| 423 printf("\n"); | |
| 424 EXPECT_LE(num_frames_to_hit_target_, max_time_hit_target); | |
| 425 EXPECT_LE(num_dropped_frames, max_num_dropped_frames); | |
| 426 EXPECT_EQ(num_resize_actions, num_spatial_resizes); | |
| 427 EXPECT_EQ(num_key_frames_, num_key_frames); | |
| 428 } | |
| 429 | |
| 430 // Layer index corresponding to frame number, for up to 3 layers. | |
| 431 void LayerIndexForFrame(int frame_number) { | |
| 432 if (num_temporal_layers_ == 1) { | |
| 433 layer_ = 0; | |
| 434 } else if (num_temporal_layers_ == 2) { | |
| 435 // layer 0: 0 2 4 ... | |
| 436 // layer 1: 1 3 | |
| 437 if (frame_number % 2 == 0) { | |
| 438 layer_ = 0; | |
| 439 } else { | |
| 440 layer_ = 1; | |
| 441 } | |
| 442 } else if (num_temporal_layers_ == 3) { | |
| 443 // layer 0: 0 4 8 ... | |
| 444 // layer 1: 2 6 | |
| 445 // layer 2: 1 3 5 7 | |
| 446 if (frame_number % 4 == 0) { | |
| 447 layer_ = 0; | |
| 448 } else if ((frame_number + 2) % 4 == 0) { | |
| 449 layer_ = 1; | |
| 450 } else if ((frame_number + 1) % 2 == 0) { | |
| 451 layer_ = 2; | |
| 452 } | |
| 453 } else { | |
| 454 assert(false); // Only up to 3 layers. | |
| 455 } | |
| 456 } | |
| 457 | |
| 458 // Set the bitrate and frame rate per layer, for up to 3 layers. | |
| 459 void SetLayerRates() { | |
| 460 assert(num_temporal_layers_ <= 3); | |
| 461 for (int i = 0; i < num_temporal_layers_; i++) { | |
| 462 float bit_rate_ratio = | |
| 463 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i]; | |
| 464 if (i > 0) { | |
| 465 float bit_rate_delta_ratio = | |
| 466 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i] - | |
| 467 kVp8LayerRateAlloction[num_temporal_layers_ - 1][i - 1]; | |
| 468 bit_rate_layer_[i] = bit_rate_ * bit_rate_delta_ratio; | |
| 469 } else { | |
| 470 bit_rate_layer_[i] = bit_rate_ * bit_rate_ratio; | |
| 471 } | |
| 472 frame_rate_layer_[i] = | |
| 473 frame_rate_ / static_cast<float>(1 << (num_temporal_layers_ - 1)); | |
| 474 } | |
| 475 if (num_temporal_layers_ == 3) { | |
| 476 frame_rate_layer_[2] = frame_rate_ / 2.0f; | |
| 477 } | |
| 478 } | |
| 479 | |
| 480 // Processes all frames in the clip and verifies the result. | |
| 481 void ProcessFramesAndVerify(QualityMetrics quality_metrics, | |
| 482 RateProfile rate_profile, | |
| 483 CodecConfigPars process, | |
| 484 RateControlMetrics* rc_metrics) { | |
| 485 // Codec/config settings. | |
| 486 codec_type_ = process.codec_type; | |
| 487 start_bitrate_ = rate_profile.target_bit_rate[0]; | |
| 488 packet_loss_ = process.packet_loss; | |
| 489 key_frame_interval_ = process.key_frame_interval; | |
| 490 num_temporal_layers_ = process.num_temporal_layers; | |
| 491 error_concealment_on_ = process.error_concealment_on; | |
| 492 denoising_on_ = process.denoising_on; | |
| 493 frame_dropper_on_ = process.frame_dropper_on; | |
| 494 spatial_resize_on_ = process.spatial_resize_on; | |
| 495 SetUpCodecConfig(); | |
| 496 // Update the layers and the codec with the initial rates. | |
| 497 bit_rate_ = rate_profile.target_bit_rate[0]; | |
| 498 frame_rate_ = rate_profile.input_frame_rate[0]; | |
| 499 SetLayerRates(); | |
| 500 // Set the initial target size for key frame. | |
| 501 target_size_key_frame_initial_ = | |
| 502 0.5 * kInitialBufferSize * bit_rate_layer_[0]; | |
| 503 processor_->SetRates(bit_rate_, frame_rate_); | |
| 504 // Process each frame, up to |num_frames|. | |
| 505 int num_frames = rate_profile.num_frames; | |
| 506 int update_index = 0; | |
| 507 ResetRateControlMetrics( | |
| 508 rate_profile.frame_index_rate_update[update_index + 1]); | |
| 509 int frame_number = 0; | |
| 510 FrameType frame_type = kVideoFrameDelta; | |
| 511 while (processor_->ProcessFrame(frame_number) && | |
| 512 frame_number < num_frames) { | |
| 513 // Get the layer index for the frame |frame_number|. | |
| 514 LayerIndexForFrame(frame_number); | |
| 515 // Get the frame_type. | |
| 516 frame_type = processor_->EncodedFrameType(); | |
| 517 // Counter for whole sequence run. | |
| 518 ++frame_number; | |
| 519 // Counters for each rate update. | |
| 520 ++num_frames_per_update_[layer_]; | |
| 521 ++num_frames_total_; | |
| 522 UpdateRateControlMetrics(frame_number, frame_type); | |
| 523 // If we hit another/next update, verify stats for current state and | |
| 524 // update layers and codec with new rates. | |
| 525 if (frame_number == | |
| 526 rate_profile.frame_index_rate_update[update_index + 1]) { | |
| 527 VerifyRateControl( | |
| 528 update_index, rc_metrics[update_index].max_key_frame_size_mismatch, | |
| 529 rc_metrics[update_index].max_delta_frame_size_mismatch, | |
| 530 rc_metrics[update_index].max_encoding_rate_mismatch, | |
| 531 rc_metrics[update_index].max_time_hit_target, | |
| 532 rc_metrics[update_index].max_num_dropped_frames, | |
| 533 rc_metrics[update_index].num_spatial_resizes, | |
| 534 rc_metrics[update_index].num_key_frames); | |
| 535 // Update layer rates and the codec with new rates. | |
| 536 ++update_index; | |
| 537 bit_rate_ = rate_profile.target_bit_rate[update_index]; | |
| 538 frame_rate_ = rate_profile.input_frame_rate[update_index]; | |
| 539 SetLayerRates(); | |
| 540 ResetRateControlMetrics( | |
| 541 rate_profile.frame_index_rate_update[update_index + 1]); | |
| 542 processor_->SetRates(bit_rate_, frame_rate_); | |
| 543 } | |
| 544 } | |
| 545 VerifyRateControl(update_index, | |
| 546 rc_metrics[update_index].max_key_frame_size_mismatch, | |
| 547 rc_metrics[update_index].max_delta_frame_size_mismatch, | |
| 548 rc_metrics[update_index].max_encoding_rate_mismatch, | |
| 549 rc_metrics[update_index].max_time_hit_target, | |
| 550 rc_metrics[update_index].max_num_dropped_frames, | |
| 551 rc_metrics[update_index].num_spatial_resizes, | |
| 552 rc_metrics[update_index].num_key_frames); | |
| 553 EXPECT_EQ(num_frames, frame_number); | |
| 554 EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size())); | |
| 555 | |
| 556 // Release encoder and decoder to make sure they have finished processing: | |
| 557 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); | |
| 558 EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); | |
| 559 // Close the files before we start using them for SSIM/PSNR calculations. | |
| 560 frame_reader_->Close(); | |
| 561 frame_writer_->Close(); | |
| 562 | |
| 563 // TODO(marpan): should compute these quality metrics per SetRates update. | |
| 564 test::QualityMetricsResult psnr_result, ssim_result; | |
| 565 EXPECT_EQ(0, test::I420MetricsFromFiles(config_.input_filename.c_str(), | |
| 566 config_.output_filename.c_str(), | |
| 567 config_.codec_settings->width, | |
| 568 config_.codec_settings->height, | |
| 569 &psnr_result, &ssim_result)); | |
| 570 printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n", | |
| 571 psnr_result.average, psnr_result.min, ssim_result.average, | |
| 572 ssim_result.min); | |
| 573 stats_.PrintSummary(); | |
| 574 EXPECT_GT(psnr_result.average, quality_metrics.minimum_avg_psnr); | |
| 575 EXPECT_GT(psnr_result.min, quality_metrics.minimum_min_psnr); | |
| 576 EXPECT_GT(ssim_result.average, quality_metrics.minimum_avg_ssim); | |
| 577 EXPECT_GT(ssim_result.min, quality_metrics.minimum_min_ssim); | |
| 578 if (remove(config_.output_filename.c_str()) < 0) { | |
| 579 fprintf(stderr, "Failed to remove temporary file!\n"); | |
| 580 } | |
| 581 } | |
| 582 }; | |
| 583 | 15 |
| 584 #if defined(WEBRTC_VIDEOPROCESSOR_H264_TESTS) | 16 #if defined(WEBRTC_VIDEOPROCESSOR_H264_TESTS) |
| 585 | 17 |
| 586 // H264: Run with no packet loss and fixed bitrate. Quality should be very high. | 18 // H264: Run with no packet loss and fixed bitrate. Quality should be very high. |
| 587 // Note(hbos): The PacketManipulatorImpl code used to simulate packet loss in | 19 // Note(hbos): The PacketManipulatorImpl code used to simulate packet loss in |
| 588 // these unittests appears to drop "packets" in a way that is not compatible | 20 // these unittests appears to drop "packets" in a way that is not compatible |
| 589 // with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have | 21 // with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have |
| 590 // not been added. | 22 // not been added. |
| 591 TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) { | 23 TEST_F(VideoProcessorIntegrationTest, Process0PercentPacketLossH264) { |
| 592 // Bitrate and frame rate profile. | 24 // Bitrate and frame rate profile. |
| (...skipping 361 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 954 // Metrics for expected quality. | 386 // Metrics for expected quality. |
| 955 QualityMetrics quality_metrics; | 387 QualityMetrics quality_metrics; |
| 956 SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80); | 388 SetQualityMetrics(&quality_metrics, 32.5, 30.0, 0.85, 0.80); |
| 957 // Metrics for rate control. | 389 // Metrics for rate control. |
| 958 RateControlMetrics rc_metrics[2]; | 390 RateControlMetrics rc_metrics[2]; |
| 959 SetRateControlMetrics(rc_metrics, 0, 0, 20, 30, 10, 10, 0, 1); | 391 SetRateControlMetrics(rc_metrics, 0, 0, 20, 30, 10, 10, 0, 1); |
| 960 SetRateControlMetrics(rc_metrics, 1, 0, 0, 30, 15, 10, 0, 0); | 392 SetRateControlMetrics(rc_metrics, 1, 0, 0, 30, 15, 10, 0, 0); |
| 961 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, | 393 ProcessFramesAndVerify(quality_metrics, rate_profile, process_settings, |
| 962 rc_metrics); | 394 rc_metrics); |
| 963 } | 395 } |
| 396 } // namespace test |
| 964 } // namespace webrtc | 397 } // namespace webrtc |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2643853002:
Add script for plotting statistics from webrtc integration test logs. (Closed)
