Turn off error resilience for vp8 for no temporal layers if nack is enabled.

webrtc/video/vie_encoder_unittest.cc

 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <limits>
 #include <utility>
 
 #include "webrtc/base/logging.h"
 #include "webrtc/system_wrappers/include/metrics_default.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_encoder.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/video/send_statistics_proxy.h"
 #include "webrtc/video/vie_encoder.h"
 
 namespace webrtc {
 
 namespace {
+const size_t kMaxPayloadLength = 1440;
+const int kTargetBitrateBps = 100000;
+
 class TestBuffer : public webrtc::I420Buffer {
  public:
   TestBuffer(rtc::Event* event, int width, int height)
       : I420Buffer(width, height), event_(event) {}
 
  private:
   friend class rtc::RefCountedObject<TestBuffer>;
   ~TestBuffer() override {
     if (event_)
       event_->Set();
@@ skipping 24 matching lines @@
   void TriggerCpuNormalUsage() {
     rtc::Event event(false, false);
     encoder_queue()->PostTask([this, &event] {
       NormalUsage();
       event.Set();
     });
     event.Wait(rtc::Event::kForever);
   }
 };
 
+class VideoStreamFactory
+    : public VideoEncoderConfig::VideoStreamFactoryInterface {
+ public:
+  explicit VideoStreamFactory(size_t num_temporal_layers)
+      : num_temporal_layers_(num_temporal_layers) {
+    EXPECT_GT(num_temporal_layers, 0u);
+  }
+
+ private:
+  std::vector<VideoStream> CreateEncoderStreams(
+      int width,
+      int height,
+      const VideoEncoderConfig& encoder_config) override {
+    std::vector<VideoStream> streams =
+        test::CreateVideoStreams(width, height, encoder_config);
+    for (VideoStream& stream : streams) {
+      stream.temporal_layer_thresholds_bps.resize(num_temporal_layers_ - 1);
+    }
+    return streams;
+  }
+  const size_t num_temporal_layers_;
+};
+
 }  // namespace
 
 class ViEEncoderTest : public ::testing::Test {
  public:
   static const int kDefaultTimeoutMs = 30 * 1000;
 
   ViEEncoderTest()
       : video_send_config_(VideoSendStream::Config(nullptr)),
         codec_width_(320),
         codec_height_(240),
         fake_encoder_(),
         stats_proxy_(new SendStatisticsProxy(
             Clock::GetRealTimeClock(),
             video_send_config_,
             webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
         sink_(&fake_encoder_) {}
 
   void SetUp() override {
     metrics::Reset();
     video_send_config_ = VideoSendStream::Config(nullptr);
     video_send_config_.encoder_settings.encoder = &fake_encoder_;
     video_send_config_.encoder_settings.payload_name = "FAKE";
     video_send_config_.encoder_settings.payload_type = 125;
 
     VideoEncoderConfig video_encoder_config;
     test::FillEncoderConfiguration(1, &video_encoder_config);
     video_encoder_config_ = video_encoder_config.Copy();
+    ConfigureEncoder(std::move(video_encoder_config), true /* nack_enabled */);
+  }
+
+  void ConfigureEncoder(VideoEncoderConfig video_encoder_config,
+                        bool nack_enabled) {
+    if (vie_encoder_)
+      vie_encoder_->Stop();
     vie_encoder_.reset(new ViEEncoderUnderTest(
         stats_proxy_.get(), video_send_config_.encoder_settings));
     vie_encoder_->SetSink(&sink_, false /* rotation_applied */);
     vie_encoder_->SetSource(&video_source_,
                             VideoSendStream::DegradationPreference::kBalanced);
     vie_encoder_->SetStartBitrate(10000);
-    vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), 1440);
+    vie_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+                                   kMaxPayloadLength, nack_enabled);
+  }
+
+  void ResetEncoder(const std::string& payload_name,
+                    size_t num_streams,
+                    size_t num_temporal_layers,
+                    bool nack_enabled) {
+    video_send_config_.encoder_settings.payload_name = payload_name;
+
+    VideoEncoderConfig video_encoder_config;
+    video_encoder_config.number_of_streams = num_streams;
+    video_encoder_config.max_bitrate_bps = 1000000;
+    video_encoder_config.video_stream_factory =
+        new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers);
+    ConfigureEncoder(std::move(video_encoder_config), nack_enabled);
   }
 
   VideoFrame CreateFrame(int64_t ntp_ts, rtc::Event* destruction_event) const {
     VideoFrame frame(new rtc::RefCountedObject<TestBuffer>(
                          destruction_event, codec_width_, codec_height_),
                      99, 99, kVideoRotation_0);
     frame.set_ntp_time_ms(ntp_ts);
     return frame;
   }
 
@@ skipping 127 matching lines @@
   int codec_width_;
   int codec_height_;
   TestEncoder fake_encoder_;
   std::unique_ptr<SendStatisticsProxy> stats_proxy_;
   TestSink sink_;
   test::FrameForwarder video_source_;
   std::unique_ptr<ViEEncoderUnderTest> vie_encoder_;
 };
 
 TEST_F(ViEEncoderTest, EncodeOneFrame) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   rtc::Event frame_destroyed_event(false, false);
   video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
   sink_.WaitForEncodedFrame(1);
   EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
   // Dropped since no target bitrate has been set.
   rtc::Event frame_destroyed_event(false, false);
   video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
   EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
 
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
   sink_.WaitForEncodedFrame(2);
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, DropsFramesWhenRateSetToZero) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
   sink_.WaitForEncodedFrame(1);
 
   vie_encoder_->OnBitrateUpdated(0, 0, 0);
   // Dropped since bitrate is zero.
   video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
 
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
   sink_.WaitForEncodedFrame(3);
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
   sink_.WaitForEncodedFrame(1);
 
   // This frame will be dropped since it has the same ntp timestamp.
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
 
   video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
   sink_.WaitForEncodedFrame(2);
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, DropsFrameAfterStop) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
   sink_.WaitForEncodedFrame(1);
 
   vie_encoder_->Stop();
   sink_.SetExpectNoFrames();
   rtc::Event frame_destroyed_event(false, false);
   video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
   EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
 }
 
 TEST_F(ViEEncoderTest, DropsPendingFramesOnSlowEncode) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   fake_encoder_.BlockNextEncode();
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
   sink_.WaitForEncodedFrame(1);
   // Here, the encoder thread will be blocked in the TestEncoder waiting for a
   // call to ContinueEncode.
   video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
   video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
   fake_encoder_.ContinueEncode();
   sink_.WaitForEncodedFrame(3);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
   EXPECT_EQ(0, sink_.number_of_reconfigurations());
 
   // Capture a frame and wait for it to synchronize with the encoder thread.
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
   sink_.WaitForEncodedFrame(1);
   // The encoder will have been configured once when the first frame is
   // received.
   EXPECT_EQ(1, sink_.number_of_reconfigurations());
 
   VideoEncoderConfig video_encoder_config;
   test::FillEncoderConfiguration(1, &video_encoder_config);
   video_encoder_config.min_transmit_bitrate_bps = 9999;
-  vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), 1440);
+  vie_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+                                 kMaxPayloadLength, true /* nack_enabled */);
 
   // Capture a frame and wait for it to synchronize with the encoder thread.
   video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
   sink_.WaitForEncodedFrame(2);
   EXPECT_EQ(2, sink_.number_of_reconfigurations());
   EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, FrameResolutionChangeReconfigureEncoder) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   // Capture a frame and wait for it to synchronize with the encoder thread.
   video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
   sink_.WaitForEncodedFrame(1);
   // The encoder will have been configured once.
   EXPECT_EQ(1, sink_.number_of_reconfigurations());
   EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
   EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
 
   codec_width_ *= 2;
   codec_height_ *= 2;
   // Capture a frame with a higher resolution and wait for it to synchronize
   // with the encoder thread.
   video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
   sink_.WaitForEncodedFrame(2);
   EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
   EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
   EXPECT_EQ(2, sink_.number_of_reconfigurations());
 
   vie_encoder_->Stop();
 }
 
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOffFor1S1TLWithNackEnabled) {
+  const bool kNackEnabled = true;
+  const size_t kNumStreams = 1;
+  const size_t kNumTl = 1;
+  ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Capture a frame and wait for it to synchronize with the encoder thread.
+  video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+  sink_.WaitForEncodedFrame(1);
+  // The encoder have been configured once when the first frame is received.
+  EXPECT_EQ(1, sink_.number_of_reconfigurations());
+  EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+  EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+  EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+  // Resilience is off for no temporal layers with nack on.
+  EXPECT_EQ(kResilienceOff, fake_encoder_.codec_config().VP8()->resilience);
+  vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOffFor2S1TlWithNackEnabled) {
+  const bool kNackEnabled = true;
+  const size_t kNumStreams = 2;
+  const size_t kNumTl = 1;
+  ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Capture a frame and wait for it to synchronize with the encoder thread.
+  video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+  sink_.WaitForEncodedFrame(1);
+  // The encoder have been configured once when the first frame is received.
+  EXPECT_EQ(1, sink_.number_of_reconfigurations());
+  EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+  EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+  EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+  // Resilience is off for no temporal layers and >1 streams with nack on.
+  EXPECT_EQ(kResilienceOff, fake_encoder_.codec_config().VP8()->resilience);
+  vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOnFor1S1TLWithNackDisabled) {
+  const bool kNackEnabled = false;
+  const size_t kNumStreams = 1;
+  const size_t kNumTl = 1;
+  ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Capture a frame and wait for it to synchronize with the encoder thread.
+  video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+  sink_.WaitForEncodedFrame(1);
+  // The encoder have been configured once when the first frame is received.
+  EXPECT_EQ(1, sink_.number_of_reconfigurations());
+  EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+  EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+  EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+  // Resilience is on for no temporal layers with nack off.
+  EXPECT_EQ(kResilientStream, fake_encoder_.codec_config().VP8()->resilience);
+  vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOnFor1S2TlWithNackEnabled) {
+  const bool kNackEnabled = true;
+  const size_t kNumStreams = 1;
+  const size_t kNumTl = 2;
+  ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+  vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+  // Capture a frame and wait for it to synchronize with the encoder thread.
+  video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+  sink_.WaitForEncodedFrame(1);
+  // The encoder have been configured once when the first frame is received.
+  EXPECT_EQ(1, sink_.number_of_reconfigurations());
+  EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+  EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+  EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+  // Resilience is on for temporal layers.
+  EXPECT_EQ(kResilientStream, fake_encoder_.codec_config().VP8()->resilience);
+  vie_encoder_->Stop();
+}
+
 TEST_F(ViEEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
   EXPECT_TRUE(video_source_.has_sinks());
   test::FrameForwarder new_video_source;
   vie_encoder_->SetSource(&new_video_source,
                           VideoSendStream::DegradationPreference::kBalanced);
   EXPECT_FALSE(video_source_.has_sinks());
   EXPECT_TRUE(new_video_source.has_sinks());
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, SinkWantsRotationApplied) {
   EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
   vie_encoder_->SetSink(&sink_, true /*rotation_applied*/);
   EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, SinkWantsFromOveruseDetector) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count_step_up);
 
   int frame_width = 1280;
   int frame_height = 720;
 
   // Trigger CPU overuse kMaxCpuDowngrades times. Every time, ViEEncoder should
   // request lower resolution.
@@ skipping 29 matching lines @@
   vie_encoder_->TriggerCpuNormalUsage();
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
   EXPECT_EQ(video_source_.sink_wants().max_pixel_count_step_up.value_or(0),
             frame_width * frame_height);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest,
        ResolutionSinkWantsResetOnSetSourceWithDisabledResolutionScaling) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
   EXPECT_FALSE(video_source_.sink_wants().max_pixel_count_step_up);
 
   int frame_width = 1280;
   int frame_height = 720;
 
   // Trigger CPU overuse.
   vie_encoder_->TriggerCpuOveruse();
@@ skipping 26 matching lines @@
                           VideoSendStream::DegradationPreference::kBalanced);
   EXPECT_LT(new_video_source.sink_wants().max_pixel_count.value_or(
                 std::numeric_limits<int>::max()),
             frame_width * frame_height);
   EXPECT_FALSE(new_video_source.sink_wants().max_pixel_count_step_up);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, StatsTracksAdaptationStats) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   int frame_width = 1280;
   int frame_height = 720;
 
   video_source_.IncomingCapturedFrame(
       CreateFrame(1, frame_width, frame_height));
   sink_.WaitForEncodedFrame(1);
   VideoSendStream::Stats stats = stats_proxy_->GetStats();
   EXPECT_FALSE(stats.cpu_limited_resolution);
@@ skipping 16 matching lines @@
   sink_.WaitForEncodedFrame(3);
 
   stats = stats_proxy_->GetStats();
   EXPECT_FALSE(stats.cpu_limited_resolution);
   EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, StatsTracksAdaptationStatsWhenSwitchingSource) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   // Trigger CPU overuse.
   vie_encoder_->TriggerCpuOveruse();
   int frame_width = 1280;
   int frame_height = 720;
 
   video_source_.IncomingCapturedFrame(
       CreateFrame(1, frame_width, frame_height));
   sink_.WaitForEncodedFrame(1);
@@ skipping 41 matching lines @@
       CreateFrame(5, frame_width, frame_height));
   sink_.WaitForEncodedFrame(5);
   stats = stats_proxy_->GetStats();
   EXPECT_FALSE(stats.cpu_limited_resolution);
   EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
   sink_.WaitForEncodedFrame(1);
 
   VideoSendStream::Stats stats = stats_proxy_->GetStats();
   EXPECT_EQ(video_encoder_config_.max_bitrate_bps,
             stats.preferred_media_bitrate_bps);
 
   vie_encoder_->Stop();
 }
 
 TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) {
-  const int kTargetBitrateBps = 100000;
   vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
 
   int frame_width = 640;
   int frame_height = 360;
 
   for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
     video_source_.IncomingCapturedFrame(
         CreateFrame(i, frame_width, frame_height));
     sink_.WaitForEncodedFrame(i);
   }
@@ skipping 10 matching lines @@
   vie_encoder_->Stop();
 
   stats_proxy_.reset();
   EXPECT_EQ(1,
             metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
   EXPECT_EQ(
       1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
 }
 
 }  // namespace webrtc