Step #4: Run VideoProcessor integration test batch mode on task queue.

webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h

Index: webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
index a3776186e1169152c1e2b960803438f694a448af..64e220b7b1d46ca70481f13bc97bf29d7885f72d 100644
--- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
+++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h
@@ -28,8 +28,10 @@
 #endif
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/event.h"
 #include "webrtc/base/file.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/base/task_queue.h"
 #include "webrtc/media/engine/webrtcvideodecoderfactory.h"
 #include "webrtc/media/engine/webrtcvideoencoderfactory.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
@@ -41,6 +43,7 @@
 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "webrtc/modules/video_coding/include/video_coding.h"
 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
+#include "webrtc/system_wrappers/include/sleep.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/testsupport/frame_reader.h"
@@ -66,6 +69,8 @@ const int kCifWidth = 352;
 const int kCifHeight = 288;
 const char kFilenameForemanCif[] = "foreman_cif";
 
+const int kNumMillisecondsPerSecond = 1000;

[sprang_webrtc, 2017/03/20 19:36:17]

use rtc::kNumMillisPerSecond instead?

[brandtr, 2017/03/21 09:40:54]

Done.

+
 // Codec and network settings.
 struct CodecParams {
   VideoCodecType codec_type;
@@ -236,6 +241,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
   }
 
   void SetUpCodecConfig(const CodecParams& process,
+                        const RateProfile& rate_profile,
                         const VisualizationParams* visualization_params) {
     CreateEncoderAndDecoder(process.hw_codec, process.codec_type);
 
@@ -254,12 +260,13 @@ class VideoProcessorIntegrationTest : public testing::Test {
     // Key frame interval and packet loss are set for each test.
     config_.keyframe_interval = process.key_frame_interval;
     config_.networking_config.packet_loss_probability =
-        packet_loss_probability_;
+        process.packet_loss_probability;
 
     // Configure codec settings.
     VideoCodingModule::Codec(process.codec_type, &codec_settings_);
     config_.codec_settings = &codec_settings_;
-    config_.codec_settings->startBitrate = start_bitrate_;
+    float start_bitrate = rate_profile.target_bit_rate[0];
+    config_.codec_settings->startBitrate = start_bitrate;
     config_.codec_settings->width = process.width;
     config_.codec_settings->height = process.height;
 
@@ -309,18 +316,19 @@ class VideoProcessorIntegrationTest : public testing::Test {
     RTC_CHECK(analysis_frame_writer_->Init());
 
     if (visualization_params) {
+      int start_frame_rate = rate_profile.input_frame_rate[0];
       // clang-format off
       const std::string output_filename_base =
           test::OutputPath() + process.filename +
           "_cd-" + CodecTypeToPayloadName(process.codec_type).value_or("") +
           "_hw-" + std::to_string(process.hw_codec) +
-          "_fr-" + std::to_string(start_frame_rate_) +
-          "_br-" + std::to_string(static_cast<int>(start_bitrate_));
+          "_fr-" + std::to_string(start_frame_rate) +
+          "_br-" + std::to_string(static_cast<int>(start_bitrate));
       // clang-format on
       if (visualization_params->save_source_y4m) {
         source_frame_writer_.reset(new test::Y4mFrameWriterImpl(
             output_filename_base + "_source.y4m", config_.codec_settings->width,
-            config_.codec_settings->height, start_frame_rate_));
+            config_.codec_settings->height, start_frame_rate));
         RTC_CHECK(source_frame_writer_->Init());
       }
       if (visualization_params->save_encoded_ivf) {
@@ -333,7 +341,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
         decoded_frame_writer_.reset(new test::Y4mFrameWriterImpl(
             output_filename_base + "_decoded.y4m",
             config_.codec_settings->width, config_.codec_settings->height,
-            start_frame_rate_));
+            start_frame_rate));
         RTC_CHECK(decoded_frame_writer_->Init());
       }
     }
@@ -345,7 +353,6 @@ class VideoProcessorIntegrationTest : public testing::Test {
         analysis_frame_writer_.get(), packet_manipulator_.get(), config_,
         &stats_, source_frame_writer_.get(), encoded_frame_writer_.get(),
         decoded_frame_writer_.get()));
-    RTC_CHECK(processor_->Init());
   }
 
   // Reset quantities after each encoder update, update the target
@@ -379,12 +386,12 @@ class VideoProcessorIntegrationTest : public testing::Test {
   }
 
   // For every encoded frame, update the rate control metrics.
-  void UpdateRateControlMetrics(int frame_number) {
+  void UpdateRateControlMetrics(int frame_number,
+                                FrameType frame_type,
+                                size_t encoded_frame_size) {
     RTC_CHECK_GE(frame_number, 0);
     int tl_idx = TemporalLayerIndexForFrame(frame_number);
-    FrameType frame_type = processor_->EncodedFrameType(frame_number);
-    float encoded_size_kbits =
-        processor_->EncodedFrameSize(frame_number) * 8.0f / 1000.0f;
+    float encoded_size_kbits = encoded_frame_size * 8.0f / 1000.0f;
 
     // Update layer data.
     // Update rate mismatch relative to per-frame bandwidth for delta frames.
@@ -419,11 +426,36 @@ class VideoProcessorIntegrationTest : public testing::Test {
     }
   }
 
+  void UpdateRateControlMetrics(int frame_number, rtc::TaskQueue* task_queue) {
+    FrameType frame_type;
+    size_t encoded_frame_size;
+
+    if (task_queue) {
+      rtc::Event sync_event(false, false);
+      task_queue->PostTask([this, &frame_type, &encoded_frame_size,
+                            frame_number, &sync_event]() {
+        frame_type = processor_->EncodedFrameType(frame_number);
+        encoded_frame_size = processor_->EncodedFrameSize(frame_number);
+        sync_event.Set();
+      });
+      sync_event.Wait(rtc::Event::kForever);

[sprang_webrtc, 2017/03/20 19:36:17]

nit: ASSERT_TRUE() maybe?

[brandtr, 2017/03/21 09:40:54]

Done.

+    } else {
+      frame_type = processor_->EncodedFrameType(frame_number);
+      encoded_frame_size = processor_->EncodedFrameSize(frame_number);
+    }
+
+    UpdateRateControlMetrics(frame_number, frame_type, encoded_frame_size);
+  }
+
+  void UpdateRateControlMetrics(int frame_number) {
+    UpdateRateControlMetrics(frame_number, nullptr);
+  }
+
   // Verify expected behavior of rate control and print out data.
   void VerifyRateControlMetrics(int update_index,
-                                const RateControlThresholds& rc_expected) {
-    int num_dropped_frames = processor_->NumberDroppedFrames();
-    int num_resize_actions = processor_->NumberSpatialResizes();
+                                const RateControlThresholds& rc_expected,
+                                int num_dropped_frames,
+                                int num_resize_actions) {
     printf(
         "For update #: %d,\n"
         " Target Bitrate: %d,\n"
@@ -483,6 +515,35 @@ class VideoProcessorIntegrationTest : public testing::Test {
     }
   }
 
+  void VerifyRateControlMetrics(int update_index,
+                                const RateControlThresholds& rc_expected,
+                                rtc::TaskQueue* task_queue) {
+    int num_dropped_frames;
+    int num_resize_actions;
+
+    if (task_queue) {
+      rtc::Event sync_event(false, false);
+      task_queue->PostTask(
+          [this, &num_dropped_frames, &num_resize_actions, &sync_event]() {
+            num_dropped_frames = processor_->NumberDroppedFrames();
+            num_resize_actions = processor_->NumberSpatialResizes();
+            sync_event.Set();
+          });
+      sync_event.Wait(rtc::Event::kForever);
+    } else {
+      num_dropped_frames = processor_->NumberDroppedFrames();
+      num_resize_actions = processor_->NumberSpatialResizes();
+    }
+
+    VerifyRateControlMetrics(update_index, rc_expected, num_dropped_frames,
+                             num_resize_actions);
+  }
+
+  void VerifyRateControlMetrics(int update_index,
+                                const RateControlThresholds& rc_expected) {
+    VerifyRateControlMetrics(update_index, rc_expected, nullptr);
+  }
+
   void VerifyQuality(const test::QualityMetricsResult& psnr_result,
                      const test::QualityMetricsResult& ssim_result,
                      const QualityThresholds& quality_thresholds) {
@@ -556,11 +617,9 @@ class VideoProcessorIntegrationTest : public testing::Test {
                               RateControlThresholds* rc_thresholds,
                               const VisualizationParams* visualization_params) {
     // Codec/config settings.
-    start_bitrate_ = rate_profile.target_bit_rate[0];
-    start_frame_rate_ = rate_profile.input_frame_rate[0];
-    packet_loss_probability_ = process.packet_loss_probability;
     num_temporal_layers_ = process.num_temporal_layers;
-    SetUpCodecConfig(process, visualization_params);
+    SetUpCodecConfig(process, rate_profile, visualization_params);
+
     // Update the temporal layers and the codec with the initial rates.
     bit_rate_ = rate_profile.target_bit_rate[0];
     frame_rate_ = rate_profile.input_frame_rate[0];
@@ -568,7 +627,6 @@ class VideoProcessorIntegrationTest : public testing::Test {
     // Set the initial target size for key frame.
     target_size_key_frame_initial_ =
         0.5 * kInitialBufferSize * bit_rate_layer_[0];
-    processor_->SetRates(bit_rate_, frame_rate_);
 
     // Process each frame, up to |num_frames|.
     int frame_number = 0;
@@ -581,17 +639,57 @@ class VideoProcessorIntegrationTest : public testing::Test {
       // In batch mode, we calculate the metrics for all frames after all frames
       // have been sent for encoding.
 
+      // AndroidMediaEncoder must be called on a task queue, so during the batch
+      // run we will call |processor_| on a task queue.
+      std::unique_ptr<rtc::TaskQueue> task_queue(
+          new rtc::TaskQueue("Integration test task queue."));

[sprang_webrtc, 2017/03/20 19:36:17]

might want to choose a shorter name. think some oses has as 12 character limit
or something on the thread names...

[brandtr, 2017/03/21 09:40:54]

Done.

+
+      // Initialize |processor_|, which initializes the encoder and decoder.
+      rtc::Event sync_event(false, false);
+      task_queue->PostTask([this, &sync_event]() {
+        RTC_CHECK(processor_->Init());
+        processor_->SetRates(bit_rate_, frame_rate_);
+        sync_event.Set();
+      });
+      sync_event.Wait(rtc::Event::kForever);
+
+      // Post all frames (in order) to encoder.
+      int start_frame_rate = rate_profile.input_frame_rate[0];
       // TODO(brandtr): Refactor "frame number accounting" so we don't have to
       // call ProcessFrame num_frames+1 times here.
       for (frame_number = 0; frame_number <= num_frames; ++frame_number) {
-        EXPECT_TRUE(processor_->ProcessFrame(frame_number));
+        task_queue->PostTask([this, frame_number]() {
+          EXPECT_TRUE(processor_->ProcessFrame(frame_number));
+        });
+
+        // HW codecs can get stressed out if frames are arriving to quickly,
+        // so we do some rough pacing here.

[sprang_webrtc, 2017/03/20 19:36:17]

How do they get "stressed out"? Any other form of backpressure we can add to
avoid sleeping?

[brandtr, 2017/03/21 09:40:54]

Expanded the comment.

Another approach would be to post delayed tasks with a linearly increasing delay
calculated something like "delay_ms = (1000/frame_rate) * frame_number". (This
assumes that the posting of the tasks is almost instantaneous.) This would leave
the pacing of the encode operations to the task queue but I would have to post a
delayed "synchronization task" at the end, which seems a bit odd. I don't fully
understand what happens when normal and delayed tasks are mixed on the same task
queue and that's why I went with the sleep operation instead.

+        if (process.hw_codec) {
+          SleepMs(kNumMillisecondsPerSecond / start_frame_rate);
+        }
       }
 
+      // Give the task queue some time to finish processing the posted frames,
+      // then shut down processing and synchronize with main thread.
+      SleepMs(1 * kNumMillisecondsPerSecond);

[sprang_webrtc, 2017/03/20 19:36:17]

drop "1 *"

[brandtr, 2017/03/21 09:40:54]

Done.

+      sync_event.Reset();
+      task_queue->PostTask([this, &sync_event]() {
+        EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+        EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
+        processor_->DeregisterCallbacks();
+        sync_event.Set();
+      });
+      sync_event.Wait(rtc::Event::kForever);
+
+      // Calculate and print rate control metrics.
       for (frame_number = 0; frame_number < num_frames; ++frame_number) {
         ++num_frames_per_update_[TemporalLayerIndexForFrame(frame_number)];
         ++num_frames_total_;
-        UpdateRateControlMetrics(frame_number);
+        UpdateRateControlMetrics(frame_number, task_queue.get());
       }
+
+      VerifyRateControlMetrics(update_index, rc_thresholds[0],
+                               task_queue.get());
     } else {
       // In online mode, we calculate the metrics for a given frame right after
       // it has been sent for encoding.
@@ -601,6 +699,10 @@ class VideoProcessorIntegrationTest : public testing::Test {
                            "since they may be pipelining.";
       }
 
+      // Initialize |processor_| on main thread.
+      RTC_CHECK(processor_->Init());
+      processor_->SetRates(bit_rate_, frame_rate_);
+
       while (frame_number < num_frames) {
         EXPECT_TRUE(processor_->ProcessFrame(frame_number));
 
@@ -629,18 +731,18 @@ class VideoProcessorIntegrationTest : public testing::Test {
       // TODO(brandtr): Refactor "frame number accounting" so we don't have to
       // call ProcessFrame one extra time here.
       EXPECT_TRUE(processor_->ProcessFrame(frame_number));
-    }
 
-    // Verify rate control metrics for all frames (if in batch mode), or for all
-    // frames since the last rate update (if not in batch mode).
-    VerifyRateControlMetrics(update_index, rc_thresholds[update_index]);
+      // Release codecs to make sure they have finished processing.
+      EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+      EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
+      processor_->DeregisterCallbacks();
+
+      // Verify for final rate update.
+      VerifyRateControlMetrics(update_index, rc_thresholds[update_index]);
+    }
     EXPECT_EQ(num_frames, frame_number);
     EXPECT_EQ(num_frames + 1, static_cast<int>(stats_.stats_.size()));
 
-    // Release encoder and decoder to make sure they have finished processing.
-    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
-    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
-
     // Close the analysis files before we use them for SSIM/PSNR calculations.
     analysis_frame_reader_->Close();
     analysis_frame_writer_->Close();
@@ -812,11 +914,8 @@ class VideoProcessorIntegrationTest : public testing::Test {
   float target_size_key_frame_;
   float sum_key_frame_size_mismatch_;
   int num_key_frames_;
-  float start_bitrate_;
-  int start_frame_rate_;
 
   // Codec and network settings.
-  float packet_loss_probability_;
   int num_temporal_layers_;
 };