Add stats for forced software encoder fallback for VP8.

webrtc/video/send_statistics_proxy_unittest.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/video/send_statistics_proxy.h"
 
 #include <map>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "webrtc/system_wrappers/include/metrics.h"
 #include "webrtc/system_wrappers/include/metrics_default.h"
+#include "webrtc/test/field_trial.h"
 #include "webrtc/test/gtest.h"
 
 namespace webrtc {
 namespace {
 const uint32_t kFirstSsrc = 17;
 const uint32_t kSecondSsrc = 42;
 const uint32_t kFirstRtxSsrc = 18;
 const uint32_t kSecondRtxSsrc = 43;
 const uint32_t kFlexFecSsrc = 55;
 const int kFpsPeriodicIntervalMs = 2000;
 const int kWidth = 640;
 const int kHeight = 480;
 const int kQpIdx0 = 21;
 const int kQpIdx1 = 39;
+const int kMinFirstFallbackIntervalMs = 1500;
 const CodecSpecificInfo kDefaultCodecInfo = []() {
   CodecSpecificInfo codec_info;
   codec_info.codecType = kVideoCodecVP8;
   codec_info.codecSpecific.VP8.simulcastIdx = 0;
   return codec_info;
 }();
 }  // namespace
 
 class SendStatisticsProxyTest : public ::testing::Test {
  public:
-  SendStatisticsProxyTest()
-      : fake_clock_(1234), config_(GetTestConfig()), avg_delay_ms_(0),
+  SendStatisticsProxyTest() : SendStatisticsProxyTest("") {}
+  explicit SendStatisticsProxyTest(const std::string& field_trials)
+      : override_field_trials_(field_trials),
+        fake_clock_(1234),
+        config_(GetTestConfig()),
+        avg_delay_ms_(0),
         max_delay_ms_(0) {}
   virtual ~SendStatisticsProxyTest() {}
 
  protected:
   virtual void SetUp() {
     metrics::Reset();
     statistics_proxy_.reset(new SendStatisticsProxy(
         &fake_clock_, GetTestConfig(),
         VideoEncoderConfig::ContentType::kRealtimeVideo));
     expected_ = VideoSendStream::Stats();
@@ skipping 77 matching lines @@
       EXPECT_EQ(a.rtp_stats.fec.packets, b.rtp_stats.fec.packets);
 
       EXPECT_EQ(a.rtcp_stats.fraction_lost, b.rtcp_stats.fraction_lost);
       EXPECT_EQ(a.rtcp_stats.packets_lost, b.rtcp_stats.packets_lost);
       EXPECT_EQ(a.rtcp_stats.extended_highest_sequence_number,
                 b.rtcp_stats.extended_highest_sequence_number);
       EXPECT_EQ(a.rtcp_stats.jitter, b.rtcp_stats.jitter);
     }
   }
 
+  test::ScopedFieldTrials override_field_trials_;
   SimulatedClock fake_clock_;
   std::unique_ptr<SendStatisticsProxy> statistics_proxy_;
   VideoSendStream::Config config_;
   int avg_delay_ms_;
   int max_delay_ms_;
   VideoSendStream::Stats expected_;
   typedef std::map<uint32_t, VideoSendStream::StreamStats>::const_iterator
       StreamIterator;
 };
 
@@ skipping 1639 matching lines @@
     // Advance one interval and update counters.
     fake_clock_.AdvanceTimeMilliseconds(kPeriodIntervalMs);
     proxy->DataCountersUpdated(counters, kFirstSsrc);
   }
 
   // FEC not enabled.
   statistics_proxy_.reset();
   EXPECT_EQ(0, metrics::NumSamples("WebRTC.Video.FecBitrateSentInKbps"));
 }
 
+TEST_F(SendStatisticsProxyTest, GetStatsReportsEncoderImplementationName) {
+  const char* kName = "encoderName";
+  EncodedImage encoded_image;
+  CodecSpecificInfo codec_info;
+  codec_info.codec_name = kName;
+  statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
+  EXPECT_STREQ(
+      kName, statistics_proxy_->GetStats().encoder_implementation_name.c_str());
+}
+

[brandtr, 2017/09/14 12:30:12]

Add unit test checking that FallbackTimeInPercent.Vp8 is not outputed when the
field trial is not set.

[åsapersson, 2017/09/14 14:41:44]

Done.

+class ForcedFallbackEnabled : public SendStatisticsProxyTest {
+ public:
+  ForcedFallbackEnabled()
+      : SendStatisticsProxyTest(
+            "WebRTC-VP8-Forced-Fallback-Encoder/Enabled-1,2," +
+            std::to_string(kMinFirstFallbackIntervalMs) + ",4/") {
+    codec_info_.codecType = kVideoCodecVP8;
+    codec_info_.codecSpecific.VP8.simulcastIdx = 0;
+    codec_info_.codecSpecific.VP8.temporalIdx = 0;
+    codec_info_.codec_name = "fake_codec";
+  }
+
+ protected:
+  void InsertEncodedFrames(int num_frames, int interval_ms) {
+    // First frame is not updating stats, insert initial frame.
+    if (statistics_proxy_->GetStats().frames_encoded == 0) {
+      statistics_proxy_->OnSendEncodedImage(encoded_image_, &codec_info_);
+    }
+    for (int i = 0; i < num_frames; ++i) {
+      statistics_proxy_->OnSendEncodedImage(encoded_image_, &codec_info_);
+      fake_clock_.AdvanceTimeMilliseconds(interval_ms);
+    }
+    statistics_proxy_->OnSendEncodedImage(encoded_image_, &codec_info_);

[brandtr, 2017/09/14 12:30:12]

Add comment.

[åsapersson, 2017/09/14 14:41:44]

Done.

+  }
+
+  EncodedImage encoded_image_;
+  CodecSpecificInfo codec_info_;
+  const std::string kPrefix = "WebRTC.Video.Encoder.ForcedSw";
+  const int kFrameIntervalMs = 1000;
+  const int kMinFrames = 20;  // Min run time 20 sec.
+};
+
+TEST_F(ForcedFallbackEnabled, StatsNotUpdatedIfMinRunTimeHasNotPassed) {
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs - 1);
+  statistics_proxy_.reset();
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+}
+
+TEST_F(ForcedFallbackEnabled, StatsUpdated) {
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
+  statistics_proxy_.reset();
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(1, metrics::NumEvents(kPrefix + "FallbackTimeInPercent.Vp8", 0));
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+  EXPECT_EQ(1, metrics::NumEvents(kPrefix + "FallbackChangesPerMinute.Vp8", 0));
+}
+
+TEST_F(ForcedFallbackEnabled, StatsNotUpdatedIfNotVp8) {
+  codec_info_.codecType = kVideoCodecVP9;
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
+  statistics_proxy_.reset();
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+}
+
+TEST_F(ForcedFallbackEnabled, StatsNotUpdatedForTemporalLayers) {
+  codec_info_.codecSpecific.VP8.temporalIdx = 1;
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
+  statistics_proxy_.reset();
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+}
+
+TEST_F(ForcedFallbackEnabled, StatsNotUpdatedForSimulcast) {
+  codec_info_.codecSpecific.VP8.simulcastIdx = 1;
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
+  statistics_proxy_.reset();
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+}
+
+TEST_F(ForcedFallbackEnabled, OneFallbackEvent) {
+  // One change. Video: 20000 ms, fallback: 5000 ms (25%).
+  InsertEncodedFrames(15, 1000);
+  codec_info_.codec_name = "libvpx";
+  InsertEncodedFrames(5, 1000);
+
+  statistics_proxy_.reset();
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(1, metrics::NumEvents(kPrefix + "FallbackTimeInPercent.Vp8", 25));
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+  EXPECT_EQ(1, metrics::NumEvents(kPrefix + "FallbackChangesPerMinute.Vp8", 3));
+}
+
+TEST_F(ForcedFallbackEnabled, ThreeFallbackEvents) {
+  codec_info_.codecSpecific.VP8.temporalIdx = kNoTemporalIdx;

[brandtr, 2017/09/14 12:30:12]

Delete this line?

[åsapersson, 2017/09/14 14:41:44]

Added comment..

+  const int kMaxFrameDiffMs = 2000;
+
+  // Three changes. Video: 60000 ms, fallback: 15000 ms (25%).
+  InsertEncodedFrames(10, 1000);
+  codec_info_.codec_name = "libvpx";
+  InsertEncodedFrames(15, 500);
+  codec_info_.codec_name = "notlibvpx";
+  InsertEncodedFrames(20, 1000);
+  InsertEncodedFrames(3, kMaxFrameDiffMs);  // Should not be included.
+  InsertEncodedFrames(10, 1000);
+  codec_info_.codec_name = "notlibvpx2";
+  InsertEncodedFrames(10, 500);
+  codec_info_.codec_name = "libvpx";
+  InsertEncodedFrames(15, 500);
+
+  statistics_proxy_.reset();
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(1, metrics::NumEvents(kPrefix + "FallbackTimeInPercent.Vp8", 25));
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+  EXPECT_EQ(1, metrics::NumEvents(kPrefix + "FallbackChangesPerMinute.Vp8", 3));
+}
+
+TEST_F(ForcedFallbackEnabled, NoFallbackIfMinIntervalHasNotPassed) {
+  InsertEncodedFrames(1, kMinFirstFallbackIntervalMs - 1);
+  codec_info_.codec_name = "libvpx";
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
+
+  statistics_proxy_.reset();
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(0, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+}
+
+TEST_F(ForcedFallbackEnabled, FallbackIfMinIntervalPassed) {
+  InsertEncodedFrames(1, kMinFirstFallbackIntervalMs);
+  codec_info_.codec_name = "libvpx";
+  InsertEncodedFrames(kMinFrames, kFrameIntervalMs);
+
+  statistics_proxy_.reset();
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackTimeInPercent.Vp8"));
+  EXPECT_EQ(1, metrics::NumSamples(kPrefix + "FallbackChangesPerMinute.Vp8"));
+}
+
 }  // namespace webrtc