webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h - Issue 2710913004: Step #3: Add flag for correctness mode in VideoProcessor integration tests.

Unified Diff: webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

Issue 2710913004: Step #3: Add flag for correctness mode in VideoProcessor integration tests. (Closed)

Patch Set: Nit. Created 3 years, 9 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

« webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc ('K') | « webrtc/modules/video_coding/codecs/test/plot_videoprocessor_integrationtest.cc ('k') | webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Hide Comments ('s')

Index: webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

index 55bc4efae4c45c15dc638c18982ce404fdc84062..d8a6c523e63c7245b1fedbc81629bb254e8651a8 100644

--- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

+++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

@@ -86,6 +86,12 @@ struct CodecParams {

std::string filename;

bool verbose_logging;

+ // In correctness mode, EXPECTs are enabled for comparing the quality and

+ // rate control metrics to their respective thresholds. It can be useful to

+ // turn off correctness mode for HW codecs, since these in general are less

+ // reliable than SW codecs, and failing an EXPECT will fail the entire test.

+ bool correctness_mode;

// In batch mode, the VideoProcessor is fed all the frames for processing

// before any metrics are calculated. This is useful for pipelining HW codecs,

// for which some calculated metrics otherwise would be incorrect. The

@@ -422,6 +428,7 @@ class VideoProcessorIntegrationTest : public testing::Test {

void VerifyRateControlMetrics(int update_index,

int num_dropped_frames,

int num_resize_actions,

+ bool correctness_mode,

const RateControlThresholds& rc_expected) {

printf(

"For update #: %d,\n"

@@ -434,8 +441,10 @@ class VideoProcessorIntegrationTest : public testing::Test {

" Number of dropped frames: %d, \n"

" Number of spatial resizes: %d, \n",

num_frames_to_hit_target_, num_dropped_frames, num_resize_actions);

- EXPECT_LE(perc_encoding_rate_mismatch_,

- rc_expected.max_encoding_rate_mismatch);

+ if (correctness_mode) {

+ EXPECT_LE(perc_encoding_rate_mismatch_,

+ rc_expected.max_encoding_rate_mismatch);

+ }

if (num_key_frames_ > 0) {

int perc_key_frame_size_mismatch =

100 * sum_key_frame_size_mismatch_ / num_key_frames_;

@@ -443,8 +452,10 @@ class VideoProcessorIntegrationTest : public testing::Test {

" Number of Key frames: %d \n"

" Key frame rate mismatch: %d \n",

num_key_frames_, perc_key_frame_size_mismatch);

- EXPECT_LE(perc_key_frame_size_mismatch,

- rc_expected.max_key_frame_size_mismatch);

+ if (correctness_mode) {

+ EXPECT_LE(perc_key_frame_size_mismatch,

+ rc_expected.max_key_frame_size_mismatch);

+ }

}

printf("\n");

printf("Rates statistics for Layer data \n");

@@ -466,16 +477,20 @@ class VideoProcessorIntegrationTest : public testing::Test {

bit_rate_layer_[i], frame_rate_layer_[i], per_frame_bandwidth_[i],

encoding_bitrate_[i], perc_frame_size_mismatch,

perc_encoding_rate_mismatch, num_frames_per_update_[i]);

- EXPECT_LE(perc_frame_size_mismatch,

- rc_expected.max_delta_frame_size_mismatch);

- EXPECT_LE(perc_encoding_rate_mismatch,

- rc_expected.max_encoding_rate_mismatch);

+ if (correctness_mode) {

+ EXPECT_LE(perc_frame_size_mismatch,

+ rc_expected.max_delta_frame_size_mismatch);

+ EXPECT_LE(perc_encoding_rate_mismatch,

+ rc_expected.max_encoding_rate_mismatch);

+ }

}

printf("\n");

- EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target);

- EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames);

- EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions);

- EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_);

+ if (correctness_mode) {

+ EXPECT_LE(num_frames_to_hit_target_, rc_expected.max_time_hit_target);

+ EXPECT_LE(num_dropped_frames, rc_expected.max_num_dropped_frames);

+ EXPECT_EQ(rc_expected.num_spatial_resizes, num_resize_actions);

+ EXPECT_EQ(rc_expected.num_key_frames, num_key_frames_);

+ }

}

void VerifyQuality(const test::QualityMetricsResult& psnr_result,

@@ -592,6 +607,15 @@ class VideoProcessorIntegrationTest : public testing::Test {

// In online mode, we calculate the metrics for a given frame right after

// it has been sent for encoding.

+ if (!process.correctness_mode) {

+ // This is a check that we do not disable correctness mode by mistake

+ // on the bots. When doing manual experimentation and correctness mode

+ // is not desired, the check can be commented out.

+ RTC_NOTREACHED() << "This test is set up to run with both correctness "

+ "mode and batch mode disabled. Are you sure you "

+ "want to do that? In particular, this should never "

+ "happen on the bots.";

+ }

if (process.hw_codec) {

LOG(LS_WARNING) << "HW codecs should mostly be run in batch mode, "

"since they may be pipelining.";

@@ -615,7 +639,7 @@ class VideoProcessorIntegrationTest : public testing::Test {

int num_dropped_frames = processor_->NumberDroppedFrames();

int num_resize_actions = processor_->NumberSpatialResizes();

VerifyRateControlMetrics(update_index, num_dropped_frames,

- num_resize_actions,

+ num_resize_actions, process.correctness_mode,

rc_thresholds[update_index]);

// Update layer rates and the codec with new rates.

@@ -640,7 +664,8 @@ class VideoProcessorIntegrationTest : public testing::Test {

int num_dropped_frames = processor_->NumberDroppedFrames();

int num_resize_actions = processor_->NumberSpatialResizes();

VerifyRateControlMetrics(update_index, num_dropped_frames,

- num_resize_actions, rc_thresholds[update_index]);

+ num_resize_actions, process.correctness_mode,

+ rc_thresholds[update_index]);

EXPECT_EQ(num_frames, frame_number);

EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));

@@ -669,7 +694,9 @@ class VideoProcessorIntegrationTest : public testing::Test {

printf("PSNR avg: %f, min: %f\nSSIM avg: %f, min: %f\n",

psnr_result.average, psnr_result.min, ssim_result.average,

ssim_result.min);

- VerifyQuality(psnr_result, ssim_result, quality_thresholds);

+ if (process.correctness_mode) {

+ VerifyQuality(psnr_result, ssim_result, quality_thresholds);

+ }

stats_.PrintSummary();

// Remove analysis file.

@@ -693,6 +720,7 @@ class VideoProcessorIntegrationTest : public testing::Test {

int height,

const std::string& filename,

bool verbose_logging,

+ bool correctness_mode,

bool batch_mode) {

process_settings->codec_type = codec_type;

process_settings->hw_codec = hw_codec;

@@ -708,6 +736,7 @@ class VideoProcessorIntegrationTest : public testing::Test {

process_settings->height = height;

process_settings->filename = filename;

process_settings->verbose_logging = verbose_logging;

+ process_settings->correctness_mode = correctness_mode;

process_settings->batch_mode = batch_mode;

}

@@ -727,7 +756,7 @@ class VideoProcessorIntegrationTest : public testing::Test {

num_temporal_layers, error_concealment_on, denoising_on,

frame_dropper_on, spatial_resize_on, kCifWidth, kCifHeight,

kFilenameForemanCif, false /* verbose_logging */,

- false /* batch_mode */);

+ true /* correctness_mode */, false /* batch_mode */);

}

static void SetQualityThresholds(QualityThresholds* quality_thresholds,