Now run EndToEndTest with the WebRTC-NewVideoJitterBuffer experiment.

webrtc/video/end_to_end_tests.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <algorithm>
@@ skipping 24 matching lines @@
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
 #include "webrtc/system_wrappers/include/metrics.h"
 #include "webrtc/system_wrappers/include/metrics_default.h"
 #include "webrtc/system_wrappers/include/sleep.h"
 #include "webrtc/test/call_test.h"
 #include "webrtc/test/direct_transport.h"
 #include "webrtc/test/encoder_settings.h"
 #include "webrtc/test/fake_decoder.h"
 #include "webrtc/test/fake_encoder.h"
+#include "webrtc/test/field_trial.h"
 #include "webrtc/test/frame_generator.h"
 #include "webrtc/test/frame_generator_capturer.h"
 #include "webrtc/test/gtest.h"
+#include "webrtc/test/gmock.h"
 #include "webrtc/test/null_transport.h"
 #include "webrtc/test/rtcp_packet_parser.h"
 #include "webrtc/test/rtp_rtcp_observer.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/testsupport/perf_test.h"
 #include "webrtc/video/transport_adapter.h"
 #include "webrtc/video_encoder.h"
 
 namespace webrtc {
 
 static const int kSilenceTimeoutMs = 2000;
 
-class EndToEndTest : public test::CallTest {
+class EndToEndTest : public test::CallTest,
+                     public ::testing::WithParamInterface<std::string> {
  public:
-  EndToEndTest() {}
+  EndToEndTest() : scoped_field_trial_(GetParam()) {}
 
   virtual ~EndToEndTest() {
     EXPECT_EQ(nullptr, video_send_stream_);
     EXPECT_TRUE(video_receive_streams_.empty());
   }
 
  protected:
   class UnusedTransport : public Transport {
    private:
     bool SendRtp(const uint8_t* packet,
@@ skipping 48 matching lines @@
   void TestSendsSetSsrcs(size_t num_ssrcs, bool send_single_ssrc_first);
   void TestRtpStatePreservation(bool use_rtx, bool provoke_rtcpsr_before_rtp);
   void VerifyHistogramStats(bool use_rtx, bool use_red, bool screenshare);
   void VerifyNewVideoSendStreamsRespectNetworkState(
       MediaType network_to_bring_down,
       VideoEncoder* encoder,
       Transport* transport);
   void VerifyNewVideoReceiveStreamsRespectNetworkState(
       MediaType network_to_bring_down,
       Transport* transport);
+  test::ScopedFieldTrials scoped_field_trial_;
 };
 
-TEST_F(EndToEndTest, ReceiverCanBeStartedTwice) {
+INSTANTIATE_TEST_CASE_P(
+    TestWithNewVideoJitterBuffer,
+    EndToEndTest,
+    ::testing::Values("WebRTC-NewVideoJitterBuffer/Enabled/",
+                      "WebRTC-NewVideoJitterBuffer/Disabled/"));
+
+TEST_P(EndToEndTest, ReceiverCanBeStartedTwice) {
   CreateCalls(Call::Config(&event_log_), Call::Config(&event_log_));
 
   test::NullTransport transport;
   CreateSendConfig(1, 0, 0, &transport);
   CreateMatchingReceiveConfigs(&transport);
 
   CreateVideoStreams();
 
   video_receive_streams_[0]->Start();
   video_receive_streams_[0]->Start();
 
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, ReceiverCanBeStoppedTwice) {
+TEST_P(EndToEndTest, ReceiverCanBeStoppedTwice) {
   CreateCalls(Call::Config(&event_log_), Call::Config(&event_log_));
 
   test::NullTransport transport;
   CreateSendConfig(1, 0, 0, &transport);
   CreateMatchingReceiveConfigs(&transport);
 
   CreateVideoStreams();
 
   video_receive_streams_[0]->Stop();
   video_receive_streams_[0]->Stop();
 
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, ReceiverCanBeStoppedAndRestarted) {
+TEST_P(EndToEndTest, ReceiverCanBeStoppedAndRestarted) {
   CreateCalls(Call::Config(&event_log_), Call::Config(&event_log_));
 
   test::NullTransport transport;
   CreateSendConfig(1, 0, 0, &transport);
   CreateMatchingReceiveConfigs(&transport);
 
   CreateVideoStreams();
 
   video_receive_streams_[0]->Stop();
   video_receive_streams_[0]->Start();
   video_receive_streams_[0]->Stop();
 
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, RendersSingleDelayedFrame) {
+TEST_P(EndToEndTest, RendersSingleDelayedFrame) {
   static const int kWidth = 320;
   static const int kHeight = 240;
   // This constant is chosen to be higher than the timeout in the video_render
   // module. This makes sure that frames aren't dropped if there are no other
   // frames in the queue.
   static const int kDelayRenderCallbackMs = 1000;
 
   class Renderer : public rtc::VideoSinkInterface<VideoFrame> {
    public:
     Renderer() : event_(false, false) {}
@@ skipping 52 matching lines @@
       << "Timed out while waiting for the frame to render.";
 
   Stop();
 
   sender_transport.StopSending();
   receiver_transport.StopSending();
 
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, TransmitsFirstFrame) {
+TEST_P(EndToEndTest, TransmitsFirstFrame) {
   class Renderer : public rtc::VideoSinkInterface<VideoFrame> {
    public:
     Renderer() : event_(false, false) {}
 
     void OnFrame(const VideoFrame& video_frame) override { event_.Set(); }
 
     bool Wait() { return event_.Wait(kDefaultTimeoutMs); }
 
     rtc::Event event_;
   } renderer;
@@ skipping 82 matching lines @@
 
  private:
   int no_frames_to_wait_for_;
   VideoRotation expected_rotation_;
   std::string payload_name_;
   std::unique_ptr<webrtc::VideoEncoder> encoder_;
   std::unique_ptr<webrtc::VideoDecoder> decoder_;
   int frame_counter_;
 };
 
-TEST_F(EndToEndTest, SendsAndReceivesVP8Rotation90) {
+TEST_P(EndToEndTest, SendsAndReceivesVP8) {
+  CodecObserver test(5, kVideoRotation_0, "VP8", VP8Encoder::Create(),
+                     VP8Decoder::Create());
+  RunBaseTest(&test);
+}
+
+TEST_P(EndToEndTest, SendsAndReceivesVP8Rotation90) {
   CodecObserver test(5, kVideoRotation_90, "VP8", VP8Encoder::Create(),
                      VP8Decoder::Create());
   RunBaseTest(&test);
 }
 
 #if !defined(RTC_DISABLE_VP9)
-TEST_F(EndToEndTest, SendsAndReceivesVP9) {
+TEST_P(EndToEndTest, SendsAndReceivesVP9) {
   CodecObserver test(500, kVideoRotation_0, "VP9", VP9Encoder::Create(),
                      VP9Decoder::Create());
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, SendsAndReceivesVP9VideoRotation90) {
+TEST_P(EndToEndTest, SendsAndReceivesVP9VideoRotation90) {
   CodecObserver test(5, kVideoRotation_90, "VP9", VP9Encoder::Create(),
                      VP9Decoder::Create());
   RunBaseTest(&test);
 }
 #endif  // !defined(RTC_DISABLE_VP9)
 
 #if defined(WEBRTC_USE_H264)
-TEST_F(EndToEndTest, SendsAndReceivesH264) {
+TEST_P(EndToEndTest, SendsAndReceivesH264) {
   CodecObserver test(500, kVideoRotation_0, "H264", H264Encoder::Create(),
                      H264Decoder::Create());
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, SendsAndReceivesH264VideoRotation90) {
+TEST_P(EndToEndTest, SendsAndReceivesH264VideoRotation90) {
   CodecObserver test(5, kVideoRotation_90, "H264", H264Encoder::Create(),
                      H264Decoder::Create());
   RunBaseTest(&test);
 }
 #endif  // defined(WEBRTC_USE_H264)
 
-TEST_F(EndToEndTest, ReceiverUsesLocalSsrc) {
+TEST_P(EndToEndTest, ReceiverUsesLocalSsrc) {
   class SyncRtcpObserver : public test::EndToEndTest {
    public:
     SyncRtcpObserver() : EndToEndTest(kDefaultTimeoutMs) {}
 
     Action OnReceiveRtcp(const uint8_t* packet, size_t length) override {
       test::RtcpPacketParser parser;
       EXPECT_TRUE(parser.Parse(packet, length));
       EXPECT_EQ(kReceiverLocalVideoSsrc, parser.sender_ssrc());
       observation_complete_.Set();
 
       return SEND_PACKET;
     }
 
     void PerformTest() override {
       EXPECT_TRUE(Wait())
           << "Timed out while waiting for a receiver RTCP packet to be sent.";
     }
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ReceivesAndRetransmitsNack) {
+TEST_P(EndToEndTest, ReceivesAndRetransmitsNack) {
   static const int kNumberOfNacksToObserve = 2;
   static const int kLossBurstSize = 2;
   static const int kPacketsBetweenLossBursts = 9;
   class NackObserver : public test::EndToEndTest {
    public:
     NackObserver()
         : EndToEndTest(kLongTimeoutMs),
           sent_rtp_packets_(0),
           packets_left_to_drop_(0),
           nacks_left_(kNumberOfNacksToObserve) {}
@@ skipping 61 matching lines @@
     std::set<uint16_t> dropped_packets_;
     std::set<uint16_t> retransmitted_packets_;
     uint64_t sent_rtp_packets_;
     int packets_left_to_drop_;
     int nacks_left_ GUARDED_BY(&crit_);
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ReceivesNackAndRetransmitsAudio) {
+TEST_P(EndToEndTest, ReceivesNackAndRetransmitsAudio) {
   class NackObserver : public test::EndToEndTest {
    public:
     NackObserver()
         : EndToEndTest(kLongTimeoutMs),
           local_ssrc_(0),
           remote_ssrc_(0),
           receive_transport_(nullptr) {}
 
    private:
     size_t GetNumVideoStreams() const override { return 0; }
@@ skipping 50 matching lines @@
 
     uint32_t local_ssrc_;
     uint32_t remote_ssrc_;
     Transport* receive_transport_;
     rtc::Optional<uint16_t> sequence_number_to_retransmit_;
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, CanReceiveUlpfec) {
+TEST_P(EndToEndTest, CanReceiveUlpfec) {
   class UlpfecRenderObserver : public test::EndToEndTest,
                                public rtc::VideoSinkInterface<VideoFrame> {
    public:
     UlpfecRenderObserver()
         : EndToEndTest(kDefaultTimeoutMs), state_(kFirstPacket) {}
 
    private:
     Action OnSendRtp(const uint8_t* packet, size_t length) override {
       rtc::CritScope lock(&crit_);
       RTPHeader header;
@@ skipping 72 matching lines @@
       (*receive_configs)[0].renderer = this;
     }
 
     void PerformTest() override {
       EXPECT_TRUE(Wait())
           << "Timed out waiting for dropped frames to be rendered.";
     }
 
     rtc::CriticalSection crit_;
     std::set<uint32_t> protected_sequence_numbers_ GUARDED_BY(crit_);
-    std::set<uint32_t> protected_timestamps_ GUARDED_BY(crit_);
+    std::multiset<uint32_t> protected_timestamps_ GUARDED_BY(crit_);
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, CanReceiveFlexfec) {
+TEST_P(EndToEndTest, CanReceiveFlexfec) {
   class FlexfecRenderObserver : public test::EndToEndTest,
                                 public rtc::VideoSinkInterface<VideoFrame> {
    public:
     FlexfecRenderObserver()
         : EndToEndTest(kDefaultTimeoutMs), state_(kFirstPacket) {}
 
     size_t GetNumFlexfecStreams() const override { return 1; }
 
    private:
     Action OnSendRtp(const uint8_t* packet, size_t length) override {
@@ skipping 63 matching lines @@
       (*receive_configs)[0].renderer = this;
     }
 
     void PerformTest() override {
       EXPECT_TRUE(Wait())
           << "Timed out waiting for dropped frames to be rendered.";
     }
 
     rtc::CriticalSection crit_;
     std::set<uint32_t> protected_sequence_numbers_ GUARDED_BY(crit_);
-    std::set<uint32_t> protected_timestamps_ GUARDED_BY(crit_);
+    std::multiset<uint32_t> protected_timestamps_ GUARDED_BY(crit_);

[brandtr, 2016/11/21 16:46:45]

Thanks for fixing this. Could you add a short comment why this multiset is
needed. (I.e., multi-packet frames where several sequence numbers belong to the
same timestamp.)

[philipel, 2016/11/22 14:56:23]

Done.

   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ReceivedUlpfecPacketsNotNacked) {
+TEST_P(EndToEndTest, ReceivedUlpfecPacketsNotNacked) {
   class UlpfecNackObserver : public test::EndToEndTest {
    public:
     UlpfecNackObserver()
         : EndToEndTest(kDefaultTimeoutMs),
           state_(kFirstPacket),
           ulpfec_sequence_number_(0),
           has_last_sequence_number_(false),
           last_sequence_number_(0),
           encoder_(VP8Encoder::Create()),
           decoder_(VP8Decoder::Create()) {}
@@ skipping 277 matching lines @@
     std::unique_ptr<VideoEncoder> encoder_;
     const std::string payload_name_;
     int marker_bits_observed_;
     uint32_t retransmitted_timestamp_ GUARDED_BY(&crit_);
     std::vector<uint32_t> rendered_timestamps_ GUARDED_BY(&crit_);
   } test(enable_rtx, enable_red);
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, DecodesRetransmittedFrame) {
+TEST_P(EndToEndTest, DecodesRetransmittedFrame) {
   DecodesRetransmittedFrame(false, false);
 }
 
-TEST_F(EndToEndTest, DecodesRetransmittedFrameOverRtx) {
+TEST_P(EndToEndTest, DecodesRetransmittedFrameOverRtx) {
   DecodesRetransmittedFrame(true, false);
 }
 
-TEST_F(EndToEndTest, DecodesRetransmittedFrameByRed) {
+TEST_P(EndToEndTest, DecodesRetransmittedFrameByRed) {
   DecodesRetransmittedFrame(false, true);
 }
 
-TEST_F(EndToEndTest, DecodesRetransmittedFrameByRedOverRtx) {
+TEST_P(EndToEndTest, DecodesRetransmittedFrameByRedOverRtx) {
   DecodesRetransmittedFrame(true, true);
 }
 
 void EndToEndTest::ReceivesPliAndRecovers(int rtp_history_ms) {
   static const int kPacketsToDrop = 1;
 
   class PliObserver : public test::EndToEndTest,
                       public rtc::VideoSinkInterface<VideoFrame> {
    public:
     explicit PliObserver(int rtp_history_ms)
@@ skipping 63 matching lines @@
     int rtp_history_ms_;
     bool nack_enabled_;
     uint32_t highest_dropped_timestamp_ GUARDED_BY(&crit_);
     int frames_to_drop_ GUARDED_BY(&crit_);
     bool received_pli_ GUARDED_BY(&crit_);
   } test(rtp_history_ms);
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ReceivesPliAndRecoversWithNack) {
+TEST_P(EndToEndTest, ReceivesPliAndRecoversWithNack) {
   ReceivesPliAndRecovers(1000);
 }
 
-TEST_F(EndToEndTest, ReceivesPliAndRecoversWithoutNack) {
+TEST_P(EndToEndTest, ReceivesPliAndRecoversWithoutNack) {
+  // This test makes no sense for the new video jitter buffer.
+  if (GetParam() == "WebRTC-NewVideoJitterBuffer/Enabled/")

[sprang_webrtc, 2016/11/21 16:49:14]

Can you use constant for this name instead?

+    return;
   ReceivesPliAndRecovers(0);
 }
 
-TEST_F(EndToEndTest, UnknownRtpPacketGivesUnknownSsrcReturnCode) {
+TEST_P(EndToEndTest, UnknownRtpPacketGivesUnknownSsrcReturnCode) {
   class PacketInputObserver : public PacketReceiver {
    public:
     explicit PacketInputObserver(PacketReceiver* receiver)
         : receiver_(receiver), delivered_packet_(false, false) {}
 
     bool Wait() { return delivered_packet_.Wait(kDefaultTimeoutMs); }
 
    private:
     DeliveryStatus DeliverPacket(MediaType media_type,
                                  const uint8_t* packet,
@@ skipping 111 matching lines @@
     }
 
     RtcpMode rtcp_mode_;
     int sent_rtp_;
     int sent_rtcp_;
   } test(rtcp_mode);
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, UsesRtcpCompoundMode) {
+TEST_P(EndToEndTest, UsesRtcpCompoundMode) {
   RespectsRtcpMode(RtcpMode::kCompound);
 }
 
-TEST_F(EndToEndTest, UsesRtcpReducedSizeMode) {
+TEST_P(EndToEndTest, UsesRtcpReducedSizeMode) {
   RespectsRtcpMode(RtcpMode::kReducedSize);
 }
 
 // Test sets up a Call multiple senders with different resolutions and SSRCs.
 // Another is set up to receive all three of these with different renderers.
 class MultiStreamTest {
  public:
   static const size_t kNumStreams = 3;
   struct CodecSettings {
     uint32_t ssrc;
@@ skipping 103 matching lines @@
   virtual test::DirectTransport* CreateSendTransport(Call* sender_call) {
     return new test::DirectTransport(sender_call);
   }
   virtual test::DirectTransport* CreateReceiveTransport(Call* receiver_call) {
     return new test::DirectTransport(receiver_call);
   }
 };
 
 // Each renderer verifies that it receives the expected resolution, and as soon
 // as every renderer has received a frame, the test finishes.
-TEST_F(EndToEndTest, SendsAndReceivesMultipleStreams) {
+TEST_P(EndToEndTest, SendsAndReceivesMultipleStreams) {
   class VideoOutputObserver : public rtc::VideoSinkInterface<VideoFrame> {
    public:
     VideoOutputObserver(const MultiStreamTest::CodecSettings& settings,
                         uint32_t ssrc,
                         test::FrameGeneratorCapturer** frame_generator)
         : settings_(settings),
           ssrc_(ssrc),
           frame_generator_(frame_generator),
           done_(false, false) {}
 
@@ skipping 44 matching lines @@
       receive_config->renderer = observers_[stream_index].get();
     }
 
    private:
     std::unique_ptr<VideoOutputObserver> observers_[kNumStreams];
   } tester;
 
   tester.RunTest();
 }
 
-TEST_F(EndToEndTest, AssignsTransportSequenceNumbers) {
+TEST_P(EndToEndTest, AssignsTransportSequenceNumbers) {
   static const int kExtensionId = 5;
 
   class RtpExtensionHeaderObserver : public test::DirectTransport {
    public:
     RtpExtensionHeaderObserver(Call* sender_call,
                                const uint32_t& first_media_ssrc,
                                const std::map<uint32_t, uint32_t>& ssrc_map)
         : DirectTransport(sender_call),
           done_(false, false),
           parser_(RtpHeaderParser::Create()),
@@ skipping 246 matching lines @@
   }
 
  private:
   static const int kExtensionId = 5;
   const bool feedback_enabled_;
   const size_t num_video_streams_;
   const size_t num_audio_streams_;
   Call* receiver_call_;
 };
 
-TEST_F(EndToEndTest, VideoReceivesTransportFeedback) {
+TEST_P(EndToEndTest, VideoReceivesTransportFeedback) {
   TransportFeedbackTester test(true, 1, 0);
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, VideoTransportFeedbackNotConfigured) {
+TEST_P(EndToEndTest, VideoTransportFeedbackNotConfigured) {
   TransportFeedbackTester test(false, 1, 0);
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, AudioReceivesTransportFeedback) {
+TEST_P(EndToEndTest, AudioReceivesTransportFeedback) {
   TransportFeedbackTester test(true, 0, 1);
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, AudioTransportFeedbackNotConfigured) {
+TEST_P(EndToEndTest, AudioTransportFeedbackNotConfigured) {
   TransportFeedbackTester test(false, 0, 1);
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, AudioVideoReceivesTransportFeedback) {
+TEST_P(EndToEndTest, AudioVideoReceivesTransportFeedback) {
   TransportFeedbackTester test(true, 1, 1);
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ObserversEncodedFrames) {
+TEST_P(EndToEndTest, ObserversEncodedFrames) {
   class EncodedFrameTestObserver : public EncodedFrameObserver {
    public:
     EncodedFrameTestObserver()
         : length_(0), frame_type_(kEmptyFrame), called_(false, false) {}
     virtual ~EncodedFrameTestObserver() {}
 
     virtual void EncodedFrameCallback(const EncodedFrame& encoded_frame) {
       frame_type_ = encoded_frame.frame_type_;
       length_ = encoded_frame.length_;
       buffer_.reset(new uint8_t[length_]);
@@ skipping 54 matching lines @@
   post_encode_observer.ExpectEqualFrames(pre_decode_observer);
 
   Stop();
 
   sender_transport.StopSending();
   receiver_transport.StopSending();
 
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, ReceiveStreamSendsRemb) {
+TEST_P(EndToEndTest, ReceiveStreamSendsRemb) {
   class RembObserver : public test::EndToEndTest {
    public:
     RembObserver() : EndToEndTest(kDefaultTimeoutMs) {}
 
     Action OnReceiveRtcp(const uint8_t* packet, size_t length) override {
       test::RtcpPacketParser parser;
       EXPECT_TRUE(parser.Parse(packet, length));
 
       if (parser.remb()->num_packets() > 0) {
         EXPECT_EQ(kReceiverLocalVideoSsrc, parser.remb()->sender_ssrc());
         EXPECT_LT(0U, parser.remb()->bitrate_bps());
         EXPECT_EQ(1U, parser.remb()->ssrcs().size());
         EXPECT_EQ(kVideoSendSsrcs[0], parser.remb()->ssrcs()[0]);
         observation_complete_.Set();
       }
 
       return SEND_PACKET;
     }
     void PerformTest() override {
       EXPECT_TRUE(Wait()) << "Timed out while waiting for a "
                              "receiver RTCP REMB packet to be "
                              "sent.";
     }
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, VerifyBandwidthStats) {
+TEST_P(EndToEndTest, VerifyBandwidthStats) {
   class RtcpObserver : public test::EndToEndTest {
    public:
     RtcpObserver()
         : EndToEndTest(kDefaultTimeoutMs),
           sender_call_(nullptr),
           receiver_call_(nullptr),
           has_seen_pacer_delay_(false) {}
 
     Action OnSendRtp(const uint8_t* packet, size_t length) override {
       Call::Stats sender_stats = sender_call_->GetStats();
@@ skipping 25 matching lines @@
 
   RunBaseTest(&test);
 }
 
 
 // Verifies that it's possible to limit the send BWE by sending a REMB.
 // This is verified by allowing the send BWE to ramp-up to >1000 kbps,
 // then have the test generate a REMB of 500 kbps and verify that the send BWE
 // is reduced to exactly 500 kbps. Then a REMB of 1000 kbps is generated and the
 // test verifies that the send BWE ramps back up to exactly 1000 kbps.
-TEST_F(EndToEndTest, RembWithSendSideBwe) {
+TEST_P(EndToEndTest, RembWithSendSideBwe) {
   class BweObserver : public test::EndToEndTest {
    public:
     BweObserver()
         : EndToEndTest(kDefaultTimeoutMs),
           sender_call_(nullptr),
           clock_(Clock::GetRealTimeClock()),
           sender_ssrc_(0),
           remb_bitrate_bps_(1000000),
           receive_transport_(nullptr),
           event_(false, false),
@@ skipping 112 matching lines @@
     test::PacketTransport* receive_transport_;
     rtc::Event event_;
     rtc::PlatformThread poller_thread_;
     TestState state_;
     RateLimiter retransmission_rate_limiter_;
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, VerifyNackStats) {
+TEST_P(EndToEndTest, VerifyNackStats) {
   static const int kPacketNumberToDrop = 200;
   class NackObserver : public test::EndToEndTest {
    public:
     NackObserver()
         : EndToEndTest(kLongTimeoutMs),
           sent_rtp_packets_(0),
           dropped_rtp_packet_(0),
           dropped_rtp_packet_requested_(false),
           send_stream_(nullptr),
           start_runtime_ms_(-1) {}
@@ skipping 311 matching lines @@
 
   int num_red_samples = use_red ? 1 : 0;
   EXPECT_EQ(num_red_samples,
             metrics::NumSamples("WebRTC.Video.FecBitrateSentInKbps"));
   EXPECT_EQ(num_red_samples,
             metrics::NumSamples("WebRTC.Video.FecBitrateReceivedInKbps"));
   EXPECT_EQ(num_red_samples,
             metrics::NumSamples("WebRTC.Video.ReceivedFecPacketsInPercent"));
 }
 
-TEST_F(EndToEndTest, VerifyHistogramStatsWithRtx) {
+TEST_P(EndToEndTest, VerifyHistogramStatsWithRtx) {
   const bool kEnabledRtx = true;
   const bool kEnabledRed = false;
   const bool kScreenshare = false;
   VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare);
 }
 
-TEST_F(EndToEndTest, VerifyHistogramStatsWithRed) {
+TEST_P(EndToEndTest, VerifyHistogramStatsWithRed) {
   const bool kEnabledRtx = false;
   const bool kEnabledRed = true;
   const bool kScreenshare = false;
   VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare);
 }
 
-TEST_F(EndToEndTest, VerifyHistogramStatsWithScreenshare) {
+TEST_P(EndToEndTest, VerifyHistogramStatsWithScreenshare) {
   const bool kEnabledRtx = false;
   const bool kEnabledRed = false;
   const bool kScreenshare = true;
   VerifyHistogramStats(kEnabledRtx, kEnabledRed, kScreenshare);
 }
 
 void EndToEndTest::TestXrReceiverReferenceTimeReport(bool enable_rrtr) {
   static const int kNumRtcpReportPacketsToObserve = 5;
   class RtcpXrObserver : public test::EndToEndTest {
    public:
@@ skipping 187 matching lines @@
     size_t ssrcs_to_observe_;
     bool expect_single_ssrc_;
 
     VideoSendStream* send_stream_;
     VideoEncoderConfig video_encoder_config_all_streams_;
   } test(kVideoSendSsrcs, num_ssrcs, send_single_ssrc_first);
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ReportsSetEncoderRates) {
+TEST_P(EndToEndTest, ReportsSetEncoderRates) {
   class EncoderRateStatsTest : public test::EndToEndTest,
                                public test::FakeEncoder {
    public:
     EncoderRateStatsTest()
         : EndToEndTest(kDefaultTimeoutMs),
           FakeEncoder(Clock::GetRealTimeClock()),
           send_stream_(nullptr),
           bitrate_kbps_(0) {}
 
     void OnVideoStreamsCreated(
@@ skipping 59 matching lines @@
 
    private:
     rtc::CriticalSection crit_;
     VideoSendStream* send_stream_;
     uint32_t bitrate_kbps_ GUARDED_BY(crit_);
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, GetStats) {
+TEST_P(EndToEndTest, GetStats) {
   static const int kStartBitrateBps = 3000000;
   static const int kExpectedRenderDelayMs = 20;
 
   class ReceiveStreamRenderer : public rtc::VideoSinkInterface<VideoFrame> {
    public:
     ReceiveStreamRenderer() {}
 
    private:
     void OnFrame(const VideoFrame& video_frame) override {}
   };
@@ skipping 320 matching lines @@
     std::map<std::string, bool> send_stats_filled_;
 
     std::vector<uint32_t> expected_receive_ssrcs_;
     std::set<uint32_t> expected_send_ssrcs_;
     std::string expected_cname_;
 
     rtc::Event check_stats_event_;
     ReceiveStreamRenderer receive_stream_renderer_;
   } test;
 
+  // TODO(philipel): Implement statistics for the new video jitter buffer.
+  if (GetParam() == "WebRTC-NewVideoJitterBuffer/Enabled/")

[philipel, 2016/11/21 15:15:21]

The reason I haven't fixed this test yet is because I'm not 100% sure what all
the metrics means, so I think we might have to discuss how we want the metrics
to be measured.

[sprang_webrtc, 2016/11/21 16:49:14]

This sounds important to get right, so that we can have confidence in the
rollout.

+    return;
+
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, ReceiverReferenceTimeReportEnabled) {
+TEST_P(EndToEndTest, ReceiverReferenceTimeReportEnabled) {
   TestXrReceiverReferenceTimeReport(true);
 }
 
-TEST_F(EndToEndTest, ReceiverReferenceTimeReportDisabled) {
+TEST_P(EndToEndTest, ReceiverReferenceTimeReportDisabled) {
   TestXrReceiverReferenceTimeReport(false);
 }
 
-TEST_F(EndToEndTest, TestReceivedRtpPacketStats) {
+TEST_P(EndToEndTest, TestReceivedRtpPacketStats) {
   static const size_t kNumRtpPacketsToSend = 5;
   class ReceivedRtpStatsObserver : public test::EndToEndTest {
    public:
     ReceivedRtpStatsObserver()
         : EndToEndTest(kDefaultTimeoutMs),
           receive_stream_(nullptr),
           sent_rtp_(0) {}
 
    private:
     void OnVideoStreamsCreated(
@@ skipping 19 matching lines @@
           << "Timed out while verifying number of received RTP packets.";
     }
 
     VideoReceiveStream* receive_stream_;
     uint32_t sent_rtp_;
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, SendsSetSsrc) { TestSendsSetSsrcs(1, false); }
+TEST_P(EndToEndTest, SendsSetSsrc) {
+  TestSendsSetSsrcs(1, false);
+}
 
-TEST_F(EndToEndTest, SendsSetSimulcastSsrcs) {
+TEST_P(EndToEndTest, SendsSetSimulcastSsrcs) {
   TestSendsSetSsrcs(kNumSsrcs, false);
 }
 
-TEST_F(EndToEndTest, CanSwitchToUseAllSsrcs) {
+TEST_P(EndToEndTest, CanSwitchToUseAllSsrcs) {
   TestSendsSetSsrcs(kNumSsrcs, true);
 }
 
-TEST_F(EndToEndTest, DISABLED_RedundantPayloadsTransmittedOnAllSsrcs) {
+TEST_P(EndToEndTest, DISABLED_RedundantPayloadsTransmittedOnAllSsrcs) {
   class ObserveRedundantPayloads: public test::EndToEndTest {
    public:
     ObserveRedundantPayloads()
         : EndToEndTest(kDefaultTimeoutMs), ssrcs_to_observe_(kNumSsrcs) {
           for (size_t i = 0; i < kNumSsrcs; ++i) {
             registered_rtx_ssrc_[kSendRtxSsrcs[i]] = true;
           }
         }
 
    private:
@@ skipping 313 matching lines @@
         << "Timed out waiting for all SSRCs to send packets.";
   }
 
   send_transport.StopSending();
   receive_transport.StopSending();
 
   Stop();
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, RestartingSendStreamPreservesRtpState) {
+TEST_P(EndToEndTest, RestartingSendStreamPreservesRtpState) {
   TestRtpStatePreservation(false, false);
 }
 
-TEST_F(EndToEndTest, RestartingSendStreamPreservesRtpStatesWithRtx) {
+TEST_P(EndToEndTest, RestartingSendStreamPreservesRtpStatesWithRtx) {
   TestRtpStatePreservation(true, false);
 }
 
-TEST_F(EndToEndTest, RestartingSendStreamKeepsRtpAndRtcpTimestampsSynced) {
+TEST_P(EndToEndTest, RestartingSendStreamKeepsRtpAndRtcpTimestampsSynced) {
   TestRtpStatePreservation(true, true);
 }
 
-TEST_F(EndToEndTest, RespectsNetworkState) {
+TEST_P(EndToEndTest, RespectsNetworkState) {
   // TODO(pbos): Remove accepted downtime packets etc. when signaling network
   // down blocks until no more packets will be sent.
 
   // Pacer will send from its packet list and then send required padding before
   // checking paused_ again. This should be enough for one round of pacing,
   // otherwise increase.
   static const int kNumAcceptedDowntimeRtp = 5;
   // A single RTCP may be in the pipeline.
   static const int kNumAcceptedDowntimeRtcp = 1;
   class NetworkStateTest : public test::EndToEndTest, public test::FakeEncoder {
@@ skipping 173 matching lines @@
     NetworkState sender_state_ GUARDED_BY(test_crit_);
     int sender_rtp_ GUARDED_BY(test_crit_);
     int sender_rtcp_ GUARDED_BY(test_crit_);
     int receiver_rtcp_ GUARDED_BY(test_crit_);
     int down_frames_ GUARDED_BY(test_crit_);
   } test;
 
   RunBaseTest(&test);
 }
 
-TEST_F(EndToEndTest, CallReportsRttForSender) {
+TEST_P(EndToEndTest, CallReportsRttForSender) {
   static const int kSendDelayMs = 30;
   static const int kReceiveDelayMs = 70;
   CreateCalls(Call::Config(&event_log_), Call::Config(&event_log_));
 
   FakeNetworkPipe::Config config;
   config.queue_delay_ms = kSendDelayMs;
   test::DirectTransport sender_transport(config, sender_call_.get());
   config.queue_delay_ms = kReceiveDelayMs;
   test::DirectTransport receiver_transport(config, receiver_call_.get());
   sender_transport.SetReceiver(receiver_call_->Receiver());
@@ skipping 65 matching lines @@
 
   Start();
   SleepMs(kSilenceTimeoutMs);
   Stop();
 
   sender_transport.StopSending();
 
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, NewVideoSendStreamsRespectVideoNetworkDown) {
+TEST_P(EndToEndTest, NewVideoSendStreamsRespectVideoNetworkDown) {
   class UnusedEncoder : public test::FakeEncoder {
    public:
     UnusedEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
 
     int32_t InitEncode(const VideoCodec* config,
                        int32_t number_of_cores,
                        size_t max_payload_size) override {
       EXPECT_GT(config->startBitrate, 0u);
       return 0;
     }
     int32_t Encode(const VideoFrame& input_image,
                    const CodecSpecificInfo* codec_specific_info,
                    const std::vector<FrameType>* frame_types) override {
       ADD_FAILURE() << "Unexpected frame encode.";
       return test::FakeEncoder::Encode(input_image, codec_specific_info,
                                        frame_types);
     }
   };
 
   UnusedEncoder unused_encoder;
   UnusedTransport unused_transport;
   VerifyNewVideoSendStreamsRespectNetworkState(
       MediaType::VIDEO, &unused_encoder, &unused_transport);
 }
 
-TEST_F(EndToEndTest, NewVideoSendStreamsIgnoreAudioNetworkDown) {
+TEST_P(EndToEndTest, NewVideoSendStreamsIgnoreAudioNetworkDown) {
   class RequiredEncoder : public test::FakeEncoder {
    public:
     RequiredEncoder()
         : FakeEncoder(Clock::GetRealTimeClock()), encoded_frame_(false) {}
     ~RequiredEncoder() {
       if (!encoded_frame_) {
         ADD_FAILURE() << "Didn't encode an expected frame";
       }
     }
     int32_t Encode(const VideoFrame& input_image,
                    const CodecSpecificInfo* codec_specific_info,
                    const std::vector<FrameType>* frame_types) override {
       encoded_frame_ = true;
       return test::FakeEncoder::Encode(input_image, codec_specific_info,
                                        frame_types);
     }
 
    private:
     bool encoded_frame_;
   };
 
   RequiredTransport required_transport(true /*rtp*/, false /*rtcp*/);
   RequiredEncoder required_encoder;
   VerifyNewVideoSendStreamsRespectNetworkState(
       MediaType::AUDIO, &required_encoder, &required_transport);
 }
 
-TEST_F(EndToEndTest, NewVideoReceiveStreamsRespectVideoNetworkDown) {
+TEST_P(EndToEndTest, NewVideoReceiveStreamsRespectVideoNetworkDown) {
   UnusedTransport transport;
   VerifyNewVideoReceiveStreamsRespectNetworkState(MediaType::VIDEO, &transport);
 }
 
-TEST_F(EndToEndTest, NewVideoReceiveStreamsIgnoreAudioNetworkDown) {
+TEST_P(EndToEndTest, NewVideoReceiveStreamsIgnoreAudioNetworkDown) {
   RequiredTransport transport(false /*rtp*/, true /*rtcp*/);
   VerifyNewVideoReceiveStreamsRespectNetworkState(MediaType::AUDIO, &transport);
 }
 
 void VerifyEmptyNackConfig(const NackConfig& config) {
   EXPECT_EQ(0, config.rtp_history_ms)
       << "Enabling NACK requires rtcp-fb: nack negotiation.";
 }
 
 void VerifyEmptyUlpfecConfig(const UlpfecConfig& config) {
   EXPECT_EQ(-1, config.ulpfec_payload_type)
       << "Enabling ULPFEC requires rtpmap: ulpfec negotiation.";
   EXPECT_EQ(-1, config.red_payload_type)
       << "Enabling ULPFEC requires rtpmap: red negotiation.";
   EXPECT_EQ(-1, config.red_rtx_payload_type)
       << "Enabling RTX in ULPFEC requires rtpmap: rtx negotiation.";
 }
 
 void VerifyEmptyFlexfecConfig(const FlexfecConfig& config) {
   EXPECT_EQ(-1, config.flexfec_payload_type)
       << "Enabling FlexFEC requires rtpmap: flexfec negotiation.";
   EXPECT_TRUE(config.protected_media_ssrcs.empty())
       << "Enabling FlexFEC requires ssrc-group: FEC-FR negotiation.";
 }
 
-TEST_F(EndToEndTest, VerifyDefaultSendConfigParameters) {
+TEST_P(EndToEndTest, VerifyDefaultSendConfigParameters) {
   VideoSendStream::Config default_send_config(nullptr);
   EXPECT_EQ(0, default_send_config.rtp.nack.rtp_history_ms)
       << "Enabling NACK require rtcp-fb: nack negotiation.";
   EXPECT_TRUE(default_send_config.rtp.rtx.ssrcs.empty())
       << "Enabling RTX requires rtpmap: rtx negotiation.";
   EXPECT_TRUE(default_send_config.rtp.extensions.empty())
       << "Enabling RTP extensions require negotiation.";
 
   VerifyEmptyNackConfig(default_send_config.rtp.nack);
   VerifyEmptyUlpfecConfig(default_send_config.rtp.ulpfec);
   VerifyEmptyFlexfecConfig(default_send_config.rtp.flexfec);
 }
 
-TEST_F(EndToEndTest, VerifyDefaultVideoReceiveConfigParameters) {
+TEST_P(EndToEndTest, VerifyDefaultVideoReceiveConfigParameters) {
   VideoReceiveStream::Config default_receive_config(nullptr);
   EXPECT_EQ(RtcpMode::kCompound, default_receive_config.rtp.rtcp_mode)
       << "Reduced-size RTCP require rtcp-rsize to be negotiated.";
   EXPECT_FALSE(default_receive_config.rtp.remb)
       << "REMB require rtcp-fb: goog-remb to be negotiated.";
   EXPECT_FALSE(
       default_receive_config.rtp.rtcp_xr.receiver_reference_time_report)
       << "RTCP XR settings require rtcp-xr to be negotiated.";
   EXPECT_TRUE(default_receive_config.rtp.rtx.empty())
       << "Enabling RTX requires rtpmap: rtx negotiation.";
   EXPECT_TRUE(default_receive_config.rtp.extensions.empty())
       << "Enabling RTP extensions require negotiation.";
 
   VerifyEmptyNackConfig(default_receive_config.rtp.nack);
   VerifyEmptyUlpfecConfig(default_receive_config.rtp.ulpfec);
 }
 
-TEST_F(EndToEndTest, VerifyDefaultFlexfecReceiveConfigParameters) {
+TEST_P(EndToEndTest, VerifyDefaultFlexfecReceiveConfigParameters) {
   FlexfecReceiveStream::Config default_receive_config;
   VerifyEmptyFlexfecConfig(default_receive_config);
 }
 
-TEST_F(EndToEndTest, TransportSeqNumOnAudioAndVideo) {
+TEST_P(EndToEndTest, TransportSeqNumOnAudioAndVideo) {
   static const int kExtensionId = 8;
   class TransportSequenceNumberTest : public test::EndToEndTest {
    public:
     TransportSequenceNumberTest()
         : EndToEndTest(kDefaultTimeoutMs),
           video_observed_(false),
           audio_observed_(false) {
       parser_->RegisterRtpHeaderExtension(kRtpExtensionTransportSequenceNumber,
                                           kExtensionId);
     }
@@ skipping 90 matching lines @@
       video_receive_streams_[0]->EnableEncodedFrameRecording(
           OpenFile(AddFile()), 0);
     } else {
       video_receive_streams_[0]->DisableEncodedFrameRecording();
     }
   }
 
   std::vector<std::string> paths_;
 };
 
-TEST_F(EndToEndLogTest, LogsEncodedFramesWhenRequested) {
+TEST_P(EndToEndLogTest, LogsEncodedFramesWhenRequested) {
   static const int kNumFramesToRecord = 10;
   class LogEncodingObserver : public test::EndToEndTest,
                               public EncodedFrameObserver {
    public:
     explicit LogEncodingObserver(EndToEndLogTest* fixture)
         : EndToEndTest(kDefaultTimeoutMs),
           fixture_(fixture),
           recorded_frames_(0) {}
 
     void PerformTest() override {
@@ skipping 42 matching lines @@
     std::unique_ptr<VideoEncoder> encoder_;
     std::unique_ptr<VideoDecoder> decoder_;
     rtc::CriticalSection crit_;
     int recorded_frames_ GUARDED_BY(crit_);
   } test(this);
 
   RunBaseTest(&test);
 }
 
 }  // namespace webrtc