NetEq: Implement muted output

webrtc/modules/audio_coding/neteq/neteq_unittest.cc

 /*
  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 101 matching lines @@
   digest->Update(&size, sizeof(size));
 
   if (file)
     ASSERT_EQ(static_cast<size_t>(size),
               fwrite(message.data(), sizeof(char), size, file));
   digest->Update(message.data(), sizeof(char) * size);
 }
 
 #endif  // WEBRTC_NETEQ_UNITTEST_BITEXACT
 
+void LoadDecoders(webrtc::NetEq* neteq) {
+  // Load PCMu.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderPCMu,
+                                          "pcmu", 0));
+  // Load PCMa.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderPCMa,
+                                          "pcma", 8));
+#ifdef WEBRTC_CODEC_ILBC
+  // Load iLBC.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderILBC,
+                                          "ilbc", 102));
+#endif
+#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
+  // Load iSAC.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderISAC,
+                                          "isac", 103));
+#endif
+#ifdef WEBRTC_CODEC_ISAC
+  // Load iSAC SWB.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderISACswb,
+                                          "isac-swb", 104));
+#endif
+#ifdef WEBRTC_CODEC_OPUS
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderOpus,
+                                          "opus", 111));
+#endif
+  // Load PCM16B nb.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderPCM16B,
+                                          "pcm16-nb", 93));
+  // Load PCM16B wb.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(
+                   webrtc::NetEqDecoder::kDecoderPCM16Bwb, "pcm16-wb", 94));
+  // Load PCM16B swb32.
+  ASSERT_EQ(
+      0, neteq->RegisterPayloadType(
+             webrtc::NetEqDecoder::kDecoderPCM16Bswb32kHz, "pcm16-swb32", 95));
+  // Load CNG 8 kHz.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderCNGnb,
+                                          "cng-nb", 13));
+  // Load CNG 16 kHz.
+  ASSERT_EQ(0, neteq->RegisterPayloadType(webrtc::NetEqDecoder::kDecoderCNGwb,
+                                          "cng-wb", 98));
+}
 }  // namespace
 
 namespace webrtc {
 
 class ResultSink {
  public:
   explicit ResultSink(const std::string& output_file);
   ~ResultSink();
 
   template<typename T, size_t n> void AddResult(
@@ skipping 74 matching lines @@
   static const size_t kBlockSize8kHz = kTimeStepMs * 8;
   static const size_t kBlockSize16kHz = kTimeStepMs * 16;
   static const size_t kBlockSize32kHz = kTimeStepMs * 32;
   static const size_t kBlockSize48kHz = kTimeStepMs * 48;
   static const int kInitSampleRateHz = 8000;
 
   NetEqDecodingTest();
   virtual void SetUp();
   virtual void TearDown();
   void SelectDecoders(NetEqDecoder* used_codec);
-  void LoadDecoders();
   void OpenInputFile(const std::string &rtp_file);
   void Process();
 
   void DecodeAndCompare(const std::string& rtp_file,
                         const std::string& output_checksum,
                         const std::string& network_stats_checksum,
                         const std::string& rtcp_stats_checksum,
                         bool gen_ref);
 
   static void PopulateRtpInfo(int frame_index,
@@ skipping 44 matching lines @@
       algorithmic_delay_ms_(0) {
   config_.sample_rate_hz = kInitSampleRateHz;
 }
 
 void NetEqDecodingTest::SetUp() {
   neteq_ = NetEq::Create(config_);
   NetEqNetworkStatistics stat;
   ASSERT_EQ(0, neteq_->NetworkStatistics(&stat));
   algorithmic_delay_ms_ = stat.current_buffer_size_ms;
   ASSERT_TRUE(neteq_);
-  LoadDecoders();
+  LoadDecoders(neteq_);
 }
 
 void NetEqDecodingTest::TearDown() {
   delete neteq_;
 }
 
-void NetEqDecodingTest::LoadDecoders() {
-  // Load PCMu.
-  ASSERT_EQ(0,
-            neteq_->RegisterPayloadType(NetEqDecoder::kDecoderPCMu, "pcmu", 0));
-  // Load PCMa.
-  ASSERT_EQ(0,
-            neteq_->RegisterPayloadType(NetEqDecoder::kDecoderPCMa, "pcma", 8));
-#ifdef WEBRTC_CODEC_ILBC
-  // Load iLBC.
-  ASSERT_EQ(
-      0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderILBC, "ilbc", 102));
-#endif
-#if defined(WEBRTC_CODEC_ISAC) || defined(WEBRTC_CODEC_ISACFX)
-  // Load iSAC.
-  ASSERT_EQ(
-      0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderISAC, "isac", 103));
-#endif
-#ifdef WEBRTC_CODEC_ISAC
-  // Load iSAC SWB.
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderISACswb,
-                                           "isac-swb", 104));
-#endif
-#ifdef WEBRTC_CODEC_OPUS
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderOpus,
-                                           "opus", 111));
-#endif
-  // Load PCM16B nb.
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderPCM16B,
-                                           "pcm16-nb", 93));
-  // Load PCM16B wb.
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderPCM16Bwb,
-                                           "pcm16-wb", 94));
-  // Load PCM16B swb32.
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderPCM16Bswb32kHz,
-                                           "pcm16-swb32", 95));
-  // Load CNG 8 kHz.
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderCNGnb,
-                                           "cng-nb", 13));
-  // Load CNG 16 kHz.
-  ASSERT_EQ(0, neteq_->RegisterPayloadType(NetEqDecoder::kDecoderCNGwb,
-                                           "cng-wb", 98));
-}
-
 void NetEqDecodingTest::OpenInputFile(const std::string &rtp_file) {
   rtp_source_.reset(test::RtpFileSource::Create(rtp_file));
 }
 
 void NetEqDecodingTest::Process() {
   // Check if time to receive.
   while (packet_ && sim_clock_ >= packet_->time_ms()) {
     if (packet_->payload_length_bytes() > 0) {
       WebRtcRTPHeader rtp_header;
       packet_->ConvertHeader(&rtp_header);
 #ifndef WEBRTC_CODEC_ISAC
       // Ignore payload type 104 (iSAC-swb) if ISAC is not supported.
       if (rtp_header.header.payloadType != 104)
 #endif
       ASSERT_EQ(0, neteq_->InsertPacket(
                        rtp_header,
                        rtc::ArrayView<const uint8_t>(
                            packet_->payload(), packet_->payload_length_bytes()),
                        static_cast<uint32_t>(packet_->time_ms() *
                                              (output_sample_rate_ / 1000))));
     }
     // Get next packet.
     packet_.reset(rtp_source_->NextPacket());
   }
 
   // Get audio from NetEq.
-  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+  bool muted_output;
+  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
+  ASSERT_FALSE(muted_output);
   ASSERT_TRUE((out_frame_.samples_per_channel_ == kBlockSize8kHz) ||
               (out_frame_.samples_per_channel_ == kBlockSize16kHz) ||
               (out_frame_.samples_per_channel_ == kBlockSize32kHz) ||
               (out_frame_.samples_per_channel_ == kBlockSize48kHz));
   output_sample_rate_ = out_frame_.sample_rate_hz_;
   EXPECT_EQ(output_sample_rate_, neteq_->last_output_sample_rate_hz());
 
   // Increase time.
   sim_clock_ += kTimeStepMs;
 }
@@ skipping 170 matching lines @@
     WebRtcRTPHeader rtp_info;
     rtp_info.header.sequenceNumber = i;
     rtp_info.header.timestamp = i * kSamples;
     rtp_info.header.ssrc = 0x1234;  // Just an arbitrary SSRC.
     rtp_info.header.payloadType = 94;  // PCM16b WB codec.
     rtp_info.header.markerBit = 0;
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
   }
   // Pull out all data.
   for (size_t i = 0; i < num_frames; ++i) {
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    bool muted_output;
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
 
   NetEqNetworkStatistics stats;
   EXPECT_EQ(0, neteq_->NetworkStatistics(&stats));
   // Since all frames are dumped into NetEQ at once, but pulled out with 10 ms
   // spacing (per definition), we expect the delay to increase with 10 ms for
   // each packet. Thus, we are calculating the statistics for a series from 10
   // to 300, in steps of 10 ms.
   EXPECT_EQ(155, stats.mean_waiting_time_ms);
@@ skipping 20 matching lines @@
     int num_packets = (frame_index % 10 == 0 ? 2 : 1);
     for (int n = 0; n < num_packets; ++n) {
       uint8_t payload[kPayloadBytes] = {0};
       WebRtcRTPHeader rtp_info;
       PopulateRtpInfo(frame_index, frame_index * kSamples, &rtp_info);
       ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
       ++frame_index;
     }
 
     // Pull out data once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    bool muted_output;
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
 
   NetEqNetworkStatistics network_stats;
   ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats));
   EXPECT_EQ(-103196, network_stats.clockdrift_ppm);
 }
 
 TEST_F(NetEqDecodingTest, TestAverageInterArrivalTimePositive) {
   const int kNumFrames = 5000;  // Needed for convergence.
   int frame_index = 0;
   const size_t kSamples = 10 * 16;
   const size_t kPayloadBytes = kSamples * 2;
   for (int i = 0; i < kNumFrames; ++i) {
     // Insert one packet each time, except every 10th time where we don't insert
     // any packet. This will create a positive clock-drift of approx. 11%.
     int num_packets = (i % 10 == 9 ? 0 : 1);
     for (int n = 0; n < num_packets; ++n) {
       uint8_t payload[kPayloadBytes] = {0};
       WebRtcRTPHeader rtp_info;
       PopulateRtpInfo(frame_index, frame_index * kSamples, &rtp_info);
       ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
       ++frame_index;
     }
 
     // Pull out data once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    bool muted_output;
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
 
   NetEqNetworkStatistics network_stats;
   ASSERT_EQ(0, neteq_->NetworkStatistics(&network_stats));
   EXPECT_EQ(110946, network_stats.clockdrift_ppm);
 }
 
 void NetEqDecodingTest::LongCngWithClockDrift(double drift_factor,
                                               double network_freeze_ms,
                                               bool pull_audio_during_freeze,
                                               int delay_tolerance_ms,
                                               int max_time_to_speech_ms) {
   uint16_t seq_no = 0;
   uint32_t timestamp = 0;
   const int kFrameSizeMs = 30;
   const size_t kSamples = kFrameSizeMs * 16;
   const size_t kPayloadBytes = kSamples * 2;
   double next_input_time_ms = 0.0;
   double t_ms;
+  bool muted_output;
 
   // Insert speech for 5 seconds.
   const int kSpeechDurationMs = 5000;
   for (t_ms = 0; t_ms < kSpeechDurationMs; t_ms += 10) {
     // Each turn in this for loop is 10 ms.
     while (next_input_time_ms <= t_ms) {
       // Insert one 30 ms speech frame.
       uint8_t payload[kPayloadBytes] = {0};
       WebRtcRTPHeader rtp_info;
       PopulateRtpInfo(seq_no, timestamp, &rtp_info);
       ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
       ++seq_no;
       timestamp += kSamples;
       next_input_time_ms += static_cast<double>(kFrameSizeMs) * drift_factor;
     }
     // Pull out data once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
 
   EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_);
   rtc::Optional<uint32_t> playout_timestamp = PlayoutTimestamp();
   ASSERT_TRUE(playout_timestamp);
   int32_t delay_before = timestamp - *playout_timestamp;
 
   // Insert CNG for 1 minute (= 60000 ms).
   const int kCngPeriodMs = 100;
   const int kCngPeriodSamples = kCngPeriodMs * 16;  // Period in 16 kHz samples.
   const int kCngDurationMs = 60000;
   for (; t_ms < kSpeechDurationMs + kCngDurationMs; t_ms += 10) {
     // Each turn in this for loop is 10 ms.
     while (next_input_time_ms <= t_ms) {
       // Insert one CNG frame each 100 ms.
       uint8_t payload[kPayloadBytes];
       size_t payload_len;
       WebRtcRTPHeader rtp_info;
       PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
       ASSERT_EQ(0, neteq_->InsertPacket(
                        rtp_info,
                        rtc::ArrayView<const uint8_t>(payload, payload_len), 0));
       ++seq_no;
       timestamp += kCngPeriodSamples;
       next_input_time_ms += static_cast<double>(kCngPeriodMs) * drift_factor;
     }
     // Pull out data once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
 
   EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
 
   if (network_freeze_ms > 0) {
     // First keep pulling audio for |network_freeze_ms| without inserting
     // any data, then insert CNG data corresponding to |network_freeze_ms|
     // without pulling any output audio.
     const double loop_end_time = t_ms + network_freeze_ms;
     for (; t_ms < loop_end_time; t_ms += 10) {
       // Pull out data once.
-      ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+      ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
       ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
       EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
     }
     bool pull_once = pull_audio_during_freeze;
     // If |pull_once| is true, GetAudio will be called once half-way through
     // the network recovery period.
     double pull_time_ms = (t_ms + next_input_time_ms) / 2;
     while (next_input_time_ms <= t_ms) {
       if (pull_once && next_input_time_ms >= pull_time_ms) {
         pull_once = false;
         // Pull out data once.
-        ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+        ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
         ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
         EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
         t_ms += 10;
       }
       // Insert one CNG frame each 100 ms.
       uint8_t payload[kPayloadBytes];
       size_t payload_len;
       WebRtcRTPHeader rtp_info;
       PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
       ASSERT_EQ(0, neteq_->InsertPacket(
@@ skipping 13 matching lines @@
       // Insert one 30 ms speech frame.
       uint8_t payload[kPayloadBytes] = {0};
       WebRtcRTPHeader rtp_info;
       PopulateRtpInfo(seq_no, timestamp, &rtp_info);
       ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
       ++seq_no;
       timestamp += kSamples;
       next_input_time_ms += kFrameSizeMs * drift_factor;
     }
     // Pull out data once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
     // Increase clock.
     t_ms += 10;
   }
 
   // Check that the speech starts again within reasonable time.
   double time_until_speech_returns_ms = t_ms - speech_restart_time_ms;
   EXPECT_LT(time_until_speech_returns_ms, max_time_to_speech_ms);
   playout_timestamp = PlayoutTimestamp();
   ASSERT_TRUE(playout_timestamp);
@@ skipping 107 matching lines @@
   uint8_t payload[kPayloadBytes] = {0};
   WebRtcRTPHeader rtp_info;
   PopulateRtpInfo(0, 0, &rtp_info);
   rtp_info.header.payloadType = 103;  // iSAC, but the payload is invalid.
   EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
   // Set all of |out_data_| to 1, and verify that it was set to 0 by the call
   // to GetAudio.
   for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; ++i) {
     out_frame_.data_[i] = 1;
   }
-  EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&out_frame_));
+  bool muted_output;
+  EXPECT_EQ(NetEq::kFail, neteq_->GetAudio(&out_frame_, &muted_output));
+  ASSERT_FALSE(muted_output);

[minyue-webrtc, 2016/05/11 11:29:54]

does this need to be assert? + all similar places

[hlundin-webrtc, 2016/05/12 07:44:40]

I choose to use ASSERT in all places where the test may read the samples from
the output frame after the GetAudio call. Since the samples are not set by NetEq
when muted_output is true, I'd rather abort the test than continue with
undefined data.

[minyue-webrtc, 2016/05/12 10:56:06]

True, and with this regards, many EXPECT can become ASSERT in this test also.
But of course, not a duty in this CL

   // Verify that there is a decoder error to check.
   EXPECT_EQ(NetEq::kDecoderErrorCode, neteq_->LastError());
 
   enum NetEqDecoderError {
     ISAC_LENGTH_MISMATCH = 6730,
     ISAC_RANGE_ERROR_DECODE_FRAME_LENGTH = 6640
   };
 #if defined(WEBRTC_CODEC_ISAC)
   EXPECT_EQ(ISAC_LENGTH_MISMATCH, neteq_->LastDecoderError());
 #elif defined(WEBRTC_CODEC_ISACFX)
@@ skipping 16 matching lines @@
     EXPECT_EQ(1, out_frame_.data_[i]);
   }
 }
 
 TEST_F(NetEqDecodingTest, GetAudioBeforeInsertPacket) {
   // Set all of |out_data_| to 1, and verify that it was set to 0 by the call
   // to GetAudio.
   for (size_t i = 0; i < AudioFrame::kMaxDataSizeSamples; ++i) {
     out_frame_.data_[i] = 1;
   }
-  EXPECT_EQ(0, neteq_->GetAudio(&out_frame_));
+  bool muted_output;
+  EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
+  ASSERT_FALSE(muted_output);
   // Verify that the first block of samples is set to 0.
   static const int kExpectedOutputLength =
       kInitSampleRateHz / 100;  // 10 ms at initial sample rate.
   for (int i = 0; i < kExpectedOutputLength; ++i) {
     std::ostringstream ss;
     ss << "i = " << i;
     SCOPED_TRACE(ss.str());  // Print out the parameter values on failure.
     EXPECT_EQ(0, out_frame_.data_[i]);
   }
   // Verify that the sample rate did not change from the initial configuration.
@@ skipping 31 matching lines @@
         10 * sampling_rate_hz,  // Max 10 seconds loop length.
         expected_samples_per_channel));
 
     // Payload of 10 ms of PCM16 32 kHz.
     uint8_t payload[kBlockSize32kHz * sizeof(int16_t)];
     WebRtcRTPHeader rtp_info;
     PopulateRtpInfo(0, 0, &rtp_info);
     rtp_info.header.payloadType = payload_type;
 
     uint32_t receive_timestamp = 0;
+    bool muted_output;
     for (int n = 0; n < 10; ++n) {  // Insert few packets and get audio.
       auto block = input.GetNextBlock();
       ASSERT_EQ(expected_samples_per_channel, block.size());
       size_t enc_len_bytes =
           WebRtcPcm16b_Encode(block.data(), block.size(), payload);
       ASSERT_EQ(enc_len_bytes, expected_samples_per_channel * 2);
 
       ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, rtc::ArrayView<const uint8_t>(
                                                       payload, enc_len_bytes),
                                         receive_timestamp));
       output.Reset();
-      ASSERT_EQ(0, neteq_->GetAudio(&output));
+      ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
       ASSERT_EQ(1u, output.num_channels_);
       ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_);
       ASSERT_EQ(AudioFrame::kNormalSpeech, output.speech_type_);
 
       // Next packet.
       rtp_info.header.timestamp += expected_samples_per_channel;
       rtp_info.header.sequenceNumber++;
       receive_timestamp += expected_samples_per_channel;
     }
 
     output.Reset();
 
     // Get audio without inserting packets, expecting PLC and PLC-to-CNG. Pull
     // one frame without checking speech-type. This is the first frame pulled
     // without inserting any packet, and might not be labeled as PLC.
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
     ASSERT_EQ(1u, output.num_channels_);
     ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_);
 
     // To be able to test the fading of background noise we need at lease to
     // pull 611 frames.
     const int kFadingThreshold = 611;
 
     // Test several CNG-to-PLC packet for the expected behavior. The number 20
     // is arbitrary, but sufficiently large to test enough number of frames.
     const int kNumPlcToCngTestFrames = 20;
     bool plc_to_cng = false;
     for (int n = 0; n < kFadingThreshold + kNumPlcToCngTestFrames; ++n) {
       output.Reset();
       memset(output.data_, 1, sizeof(output.data_));  // Set to non-zero.
-      ASSERT_EQ(0, neteq_->GetAudio(&output));
+      ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
+      ASSERT_FALSE(muted_output);
       ASSERT_EQ(1u, output.num_channels_);
       ASSERT_EQ(expected_samples_per_channel, output.samples_per_channel_);
       if (output.speech_type_ == AudioFrame::kPLCCNG) {
         plc_to_cng = true;
         double sum_squared = 0;
         for (size_t k = 0;
              k < output.num_channels_ * output.samples_per_channel_; ++k)
           sum_squared += output.data_[k] * output.data_[k];
         TestCondition(sum_squared, n > kFadingThreshold);
       } else {
@@ skipping 153 matching lines @@
   const size_t kPayloadBytes = kBlockSize16kHz * sizeof(int16_t);
   uint8_t payload[kPayloadBytes];
   AudioFrame output;
   int algorithmic_frame_delay = algorithmic_delay_ms_ / 10 + 1;
   for (size_t n = 0; n < kPayloadBytes; ++n) {
     payload[n] = (rand() & 0xF0) + 1;  // Non-zero random sequence.
   }
   // Insert some packets which decode to noise. We are not interested in
   // actual decoded values.
   uint32_t receive_timestamp = 0;
+  bool muted_output;
   for (int n = 0; n < 100; ++n) {
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, receive_timestamp));
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
     ASSERT_EQ(1u, output.num_channels_);
 
     rtp_info.header.sequenceNumber++;
     rtp_info.header.timestamp += kBlockSize16kHz;
     receive_timestamp += kBlockSize16kHz;
   }
   const int kNumSyncPackets = 10;
 
   // Make sure sufficient number of sync packets are inserted that we can
   // conduct a test.
   ASSERT_GT(kNumSyncPackets, algorithmic_frame_delay);
   // Insert sync-packets, the decoded sequence should be all-zero.
   for (int n = 0; n < kNumSyncPackets; ++n) {
     ASSERT_EQ(0, neteq_->InsertSyncPacket(rtp_info, receive_timestamp));
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
+    ASSERT_FALSE(muted_output);
     ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
     ASSERT_EQ(1u, output.num_channels_);
     if (n > algorithmic_frame_delay) {
       EXPECT_TRUE(IsAllZero(
           output.data_, output.samples_per_channel_ * output.num_channels_));
     }
     rtp_info.header.sequenceNumber++;
     rtp_info.header.timestamp += kBlockSize16kHz;
     receive_timestamp += kBlockSize16kHz;
   }
 
   // We insert regular packets, if sync packet are not correctly buffered then
   // network statistics would show some packet loss.
   for (int n = 0; n <= algorithmic_frame_delay + 10; ++n) {
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, receive_timestamp));
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
+    ASSERT_FALSE(muted_output);
     if (n >= algorithmic_frame_delay + 1) {
       // Expect that this frame contain samples from regular RTP.
       EXPECT_TRUE(IsAllNonZero(
           output.data_, output.samples_per_channel_ * output.num_channels_));
     }
     rtp_info.header.sequenceNumber++;
     rtp_info.header.timestamp += kBlockSize16kHz;
     receive_timestamp += kBlockSize16kHz;
   }
   NetEqNetworkStatistics network_stats;
@@ skipping 15 matching lines @@
   const size_t kPayloadBytes = kBlockSize16kHz * sizeof(int16_t);
   uint8_t payload[kPayloadBytes];
   AudioFrame output;
   for (size_t n = 0; n < kPayloadBytes; ++n) {
     payload[n] = (rand() & 0xF0) + 1;  // Non-zero random sequence.
   }
   // Insert some packets which decode to noise. We are not interested in
   // actual decoded values.
   uint32_t receive_timestamp = 0;
   int algorithmic_frame_delay = algorithmic_delay_ms_ / 10 + 1;
+  bool muted_output;
   for (int n = 0; n < algorithmic_frame_delay; ++n) {
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, receive_timestamp));
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
     ASSERT_EQ(1u, output.num_channels_);
     rtp_info.header.sequenceNumber++;
     rtp_info.header.timestamp += kBlockSize16kHz;
     receive_timestamp += kBlockSize16kHz;
   }
   const int kNumSyncPackets = 10;
 
   WebRtcRTPHeader first_sync_packet_rtp_info;
   memcpy(&first_sync_packet_rtp_info, &rtp_info, sizeof(rtp_info));
@@ skipping 16 matching lines @@
   // Insert.
   for (int n = 0; n < kNumSyncPackets; ++n) {
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, receive_timestamp));
     rtp_info.header.sequenceNumber++;
     rtp_info.header.timestamp += kBlockSize16kHz;
     receive_timestamp += kBlockSize16kHz;
   }
 
   // Decode.
   for (int n = 0; n < kNumSyncPackets; ++n) {
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
+    ASSERT_FALSE(muted_output);
     ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
     ASSERT_EQ(1u, output.num_channels_);
     EXPECT_TRUE(IsAllNonZero(
         output.data_, output.samples_per_channel_ * output.num_channels_));
   }
 }
 
 void NetEqDecodingTest::WrapTest(uint16_t start_seq_no,
                                  uint32_t start_timestamp,
                                  const std::set<uint16_t>& drop_seq_numbers,
@@ skipping 46 matching lines @@
       ++seq_no;
       timestamp += kSamples;
       receive_timestamp += kSamples;
       next_input_time_ms += static_cast<double>(kFrameSizeMs);
 
       seq_no_wrapped |= seq_no < last_seq_no;
       timestamp_wrapped |= timestamp < last_timestamp;
     }
     // Pull out data once.
     AudioFrame output;
-    ASSERT_EQ(0, neteq_->GetAudio(&output));
+    bool muted_output;
+    ASSERT_EQ(0, neteq_->GetAudio(&output, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, output.samples_per_channel_);
     ASSERT_EQ(1u, output.num_channels_);
 
     // Expect delay (in samples) to be less than 2 packets.
     rtc::Optional<uint32_t> playout_timestamp = PlayoutTimestamp();
     ASSERT_TRUE(playout_timestamp);
     EXPECT_LE(timestamp - *playout_timestamp,
               static_cast<uint32_t>(kSamples * 2));
   }
   // Make sure we have actually tested wrap-around.
@@ skipping 35 matching lines @@
   const int kSampleRateKhz = 16;
   const int kSamples = kFrameSizeMs * kSampleRateKhz;
   const size_t kPayloadBytes = kSamples * 2;
 
   const int algorithmic_delay_samples = std::max(
       algorithmic_delay_ms_ * kSampleRateKhz, 5 * kSampleRateKhz / 8);
   // Insert three speech packets. Three are needed to get the frame length
   // correct.
   uint8_t payload[kPayloadBytes] = {0};
   WebRtcRTPHeader rtp_info;
+  bool muted_output;
   for (int i = 0; i < 3; ++i) {
     PopulateRtpInfo(seq_no, timestamp, &rtp_info);
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
     ++seq_no;
     timestamp += kSamples;
 
     // Pull audio once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
   // Verify speech output.
   EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_);
 
   // Insert same CNG packet twice.
   const int kCngPeriodMs = 100;
   const int kCngPeriodSamples = kCngPeriodMs * kSampleRateKhz;
   size_t payload_len;
   PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
   // This is the first time this CNG packet is inserted.
   ASSERT_EQ(
       0, neteq_->InsertPacket(
              rtp_info, rtc::ArrayView<const uint8_t>(payload, payload_len), 0));
 
   // Pull audio once and make sure CNG is played.
-  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
   ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
   EXPECT_FALSE(PlayoutTimestamp());  // Returns empty value during CNG.
   EXPECT_EQ(timestamp - algorithmic_delay_samples,
             out_frame_.timestamp_ + out_frame_.samples_per_channel_);
 
   // Insert the same CNG packet again. Note that at this point it is old, since
   // we have already decoded the first copy of it.
   ASSERT_EQ(
       0, neteq_->InsertPacket(
              rtp_info, rtc::ArrayView<const uint8_t>(payload, payload_len), 0));
 
   // Pull audio until we have played |kCngPeriodMs| of CNG. Start at 10 ms since
   // we have already pulled out CNG once.
   for (int cng_time_ms = 10; cng_time_ms < kCngPeriodMs; cng_time_ms += 10) {
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
     EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
     EXPECT_FALSE(PlayoutTimestamp());  // Returns empty value during CNG.
     EXPECT_EQ(timestamp - algorithmic_delay_samples,
               out_frame_.timestamp_ + out_frame_.samples_per_channel_);
   }
 
   // Insert speech again.
   ++seq_no;
   timestamp += kCngPeriodSamples;
   PopulateRtpInfo(seq_no, timestamp, &rtp_info);
   ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
 
   // Pull audio once and verify that the output is speech again.
-  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
   ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_);
   rtc::Optional<uint32_t> playout_timestamp = PlayoutTimestamp();
   ASSERT_TRUE(playout_timestamp);
   EXPECT_EQ(timestamp + kSamples - algorithmic_delay_samples,
             *playout_timestamp);
 }
 
 rtc::Optional<uint32_t> NetEqDecodingTest::PlayoutTimestamp() {
   return neteq_->GetPlayoutTimestamp();
@@ skipping 17 matching lines @@
 
   PopulateCng(seq_no, timestamp, &rtp_info, payload, &payload_len);
   ASSERT_EQ(
       NetEq::kOK,
       neteq_->InsertPacket(
           rtp_info, rtc::ArrayView<const uint8_t>(payload, payload_len), 0));
   ++seq_no;
   timestamp += kCngPeriodSamples;
 
   // Pull audio once and make sure CNG is played.
-  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+  bool muted_output;
+  ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
   ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   EXPECT_EQ(AudioFrame::kCNG, out_frame_.speech_type_);
 
   // Insert some speech packets.
   for (int i = 0; i < 3; ++i) {
     PopulateRtpInfo(seq_no, timestamp, &rtp_info);
     ASSERT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
     ++seq_no;
     timestamp += kSamples;
 
     // Pull audio once.
-    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_));
+    ASSERT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
     ASSERT_EQ(kBlockSize16kHz, out_frame_.samples_per_channel_);
   }
   // Verify speech output.
   EXPECT_EQ(AudioFrame::kNormalSpeech, out_frame_.speech_type_);
 }
+
+class NetEqDecodingTestWithMutedState : public NetEqDecodingTest {
+ public:
+  NetEqDecodingTestWithMutedState() : NetEqDecodingTest() {
+    config_.enable_muted_state = true;
+  }
+};
+
+// Verifies that NetEq goes in and out of muted state as expected.
+TEST_F(NetEqDecodingTestWithMutedState, MutedState) {
+  const size_t kSamples = 10 * 16;
+  const size_t kPayloadBytes = kSamples * 2;
+  // Insert one speech packet.
+  uint8_t payload[kPayloadBytes] = {0};
+  WebRtcRTPHeader rtp_info;
+  PopulateRtpInfo(0, 0, &rtp_info);
+  EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
+
+  // Pull out data once.
+  bool muted_output;
+  EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
+  EXPECT_FALSE(muted_output);
+
+  // Pull data until expand starts.
+  int counter = 0;
+  while (out_frame_.speech_type_ != AudioFrame::kPLC) {
+    EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
+    EXPECT_FALSE(muted_output);
+    ASSERT_LT(counter++, 1000) << "Test timed out";
+  }
+
+  // Pull data until faded out.
+  while (!muted_output) {
+    EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
+    ASSERT_LT(counter++, 1000) << "Test timed out";
+  }
+
+  // Verify that output audio is not written during muted mode. Other parameters

[minyue-webrtc, 2016/05/11 11:29:54]

is this important?

[hlundin-webrtc, 2016/05/12 07:44:40]

Yes. One of the points with muted state is that we do not even spend cycles on
writing zero-samples. However, the other parameters in AudioFrame must still be
populated.

[minyue-webrtc, 2016/05/12 10:56:07]

But this is not sufficient in checking the efficiency of the implementation.
On the other hand, maintaining audio frame untouched when muted_output is not a
promise from the API perspective.

[hlundin-webrtc, 2016/05/12 12:06:20]

I updated the comment in neteq.h regarding this. See the diff between ps1 and
ps2. The comment says that the data_ (samples) may be left untouched when
muted==true.

+  // should be correct, though.
+  AudioFrame new_frame;
+  for (auto& d : new_frame.data_) {

[minyue-webrtc, 2016/05/11 11:29:54]

and how about memset and memcmp

[hlundin-webrtc, 2016/05/12 07:44:40]

I cannot set a non-zero value to elements wider than a byte using memset. And
for memcmp, I'd need a reference array, which I don't have. I thought this was
the most compact way to write it.

+    d = 17;
+  }
+  EXPECT_EQ(0, neteq_->GetAudio(&new_frame, &muted_output));
+  EXPECT_TRUE(muted_output);
+  for (auto d : new_frame.data_) {
+    EXPECT_EQ(17, d);
+  }
+  EXPECT_EQ(out_frame_.timestamp_ + out_frame_.samples_per_channel_,
+            new_frame.timestamp_);
+  EXPECT_EQ(out_frame_.samples_per_channel_, new_frame.samples_per_channel_);
+  EXPECT_EQ(out_frame_.sample_rate_hz_, new_frame.sample_rate_hz_);
+  EXPECT_EQ(out_frame_.num_channels_, new_frame.num_channels_);
+  EXPECT_EQ(out_frame_.speech_type_, new_frame.speech_type_);
+  EXPECT_EQ(out_frame_.vad_activity_, new_frame.vad_activity_);
+
+  // Insert new data. Timestamp is corrected for the time elapsed since the last
+  // packet.
+  PopulateRtpInfo(0, 16 * 10 * counter, &rtp_info);

[minyue-webrtc, 2016/05/11 11:29:54]

what if the timestamp is not  16 * 10 * counter

[hlundin-webrtc, 2016/05/12 07:44:40]

I changed to kSamples to be consistent within the test.

+  EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
+
+  while (out_frame_.speech_type_ != AudioFrame::kNormalSpeech) {
+    EXPECT_EQ(0, neteq_->GetAudio(&out_frame_, &muted_output));
+    ASSERT_LT(counter++, 1000) << "Test timed out";
+  }
+  EXPECT_FALSE(muted_output);
+}
+
+class NetEqDecodingTestTwoInstances : public NetEqDecodingTest {
+ public:
+  NetEqDecodingTestTwoInstances() : NetEqDecodingTest() {}
+
+  void SetUp() override {
+    NetEqDecodingTest::SetUp();
+    config2_ = config_;
+  }
+
+  void CreateSecondInstance() {
+    neteq2_.reset(NetEq::Create(config2_));
+    ASSERT_TRUE(neteq2_);
+    LoadDecoders(neteq2_.get());
+  }
+
+ protected:
+  std::unique_ptr<NetEq> neteq2_;
+  NetEq::Config config2_;
+};
+
+namespace {
+::testing::AssertionResult AudioFramesEqualExceptData(const AudioFrame& a,
+                                                      const AudioFrame& b) {
+  if (a.timestamp_ != b.timestamp_)
+    return ::testing::AssertionFailure() << "timestamp_ diff (" << a.timestamp_
+                                         << " != " << b.timestamp_ << ")";
+  if (a.sample_rate_hz_ != b.sample_rate_hz_)
+    return ::testing::AssertionFailure() << "sample_rate_hz_ diff ("
+                                         << a.sample_rate_hz_
+                                         << " != " << b.sample_rate_hz_ << ")";
+  if (a.samples_per_channel_ != b.samples_per_channel_)
+    return ::testing::AssertionFailure()
+           << "samples_per_channel_ diff (" << a.samples_per_channel_
+           << " != " << b.samples_per_channel_ << ")";
+  if (a.num_channels_ != b.num_channels_)
+    return ::testing::AssertionFailure() << "num_channels_ diff ("
+                                         << a.num_channels_
+                                         << " != " << b.num_channels_ << ")";
+  if (a.speech_type_ != b.speech_type_)
+    return ::testing::AssertionFailure() << "speech_type_ diff ("
+                                         << a.speech_type_
+                                         << " != " << b.speech_type_ << ")";
+  if (a.vad_activity_ != b.vad_activity_)
+    return ::testing::AssertionFailure() << "vad_activity_ diff ("
+                                         << a.vad_activity_
+                                         << " != " << b.vad_activity_ << ")";
+  return ::testing::AssertionSuccess();
+}
+
+::testing::AssertionResult AudioFramesEqual(const AudioFrame& a,
+                                            const AudioFrame& b) {
+  ::testing::AssertionResult res = AudioFramesEqualExceptData(a, b);
+  if (!res)
+    return res;
+  if (memcmp(a.data_, b.data_, a.samples_per_channel_ * a.num_channels_ *
+                                   sizeof(a.data_[0])) != 0) {

[minyue-webrtc, 2016/05/11 11:29:54]

weird wrapping

[hlundin-webrtc, 2016/05/12 07:44:40]

Better?

[minyue-webrtc, 2016/05/12 10:56:07]

yes

+    return ::testing::AssertionFailure() << "data_ diff";
+  }
+  return ::testing::AssertionSuccess();
+}
+
+}  // namespace
+
+TEST_F(NetEqDecodingTestTwoInstances, CompareMutedStateOnOff) {
+  ASSERT_FALSE(config_.enable_muted_state);
+  config2_.enable_muted_state = true;
+  CreateSecondInstance();
+
+  // Insert one speech packet into both NetEqs.
+  const size_t kSamples = 10 * 16;
+  const size_t kPayloadBytes = kSamples * 2;
+  uint8_t payload[kPayloadBytes] = {0};
+  WebRtcRTPHeader rtp_info;
+  PopulateRtpInfo(0, 0, &rtp_info);
+  EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
+  EXPECT_EQ(0, neteq2_->InsertPacket(rtp_info, payload, 0));
+
+  AudioFrame out_frame1, out_frame2;
+  bool muted_output;
+  for (int i = 0; i < 1000; ++i) {
+    std::ostringstream ss;
+    ss << "i = " << i;
+    SCOPED_TRACE(ss.str());  // Print out the loop iterator on failure.
+    EXPECT_EQ(0, neteq_->GetAudio(&out_frame1, &muted_output));
+    EXPECT_FALSE(muted_output);
+    EXPECT_EQ(0, neteq2_->GetAudio(&out_frame2, &muted_output));
+    if (muted_output) {
+      EXPECT_TRUE(AudioFramesEqualExceptData(out_frame1, out_frame2));
+    } else {
+      EXPECT_TRUE(AudioFramesEqual(out_frame1, out_frame2));
+    }
+  }
+  EXPECT_TRUE(muted_output);
+
+  // Insert new data. Timestamp is corrected for the time elapsed since the last
+  // packet.
+  PopulateRtpInfo(0, 16 * 10 * 1000, &rtp_info);
+  EXPECT_EQ(0, neteq_->InsertPacket(rtp_info, payload, 0));
+  EXPECT_EQ(0, neteq2_->InsertPacket(rtp_info, payload, 0));
+
+  int counter = 0;
+  while (out_frame1.speech_type_ != AudioFrame::kNormalSpeech) {
+    ASSERT_LT(counter++, 1000) << "Test timed out";
+    std::ostringstream ss;
+    ss << "counter = " << counter;
+    SCOPED_TRACE(ss.str());  // Print out the loop iterator on failure.
+    EXPECT_EQ(0, neteq_->GetAudio(&out_frame1, &muted_output));
+    EXPECT_FALSE(muted_output);
+    EXPECT_EQ(0, neteq2_->GetAudio(&out_frame2, &muted_output));
+    if (muted_output) {
+      EXPECT_TRUE(AudioFramesEqualExceptData(out_frame1, out_frame2));
+    } else {
+      EXPECT_TRUE(AudioFramesEqual(out_frame1, out_frame2));
+    }
+  }
+  EXPECT_FALSE(muted_output);
+}
+
 }  // namespace webrtc