WIP: Adding a centralized NetEq Clock

webrtc/modules/audio_coding/neteq/neteq_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 24 matching lines @@
 #include "webrtc/modules/audio_coding/neteq/expand.h"
 #include "webrtc/modules/audio_coding/neteq/merge.h"
 #include "webrtc/modules/audio_coding/neteq/nack.h"
 #include "webrtc/modules/audio_coding/neteq/normal.h"
 #include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
 #include "webrtc/modules/audio_coding/neteq/packet.h"
 #include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
 #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
 #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
 #include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
+#include "webrtc/modules/audio_coding/neteq/tick_timer.h"
 #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
 #include "webrtc/modules/include/module_common_types.h"
 
 // Modify the code to obtain backwards bit-exactness. Once bit-exactness is no
 // longer required, this #define should be removed (and the code that it
 // enables).
 #define LEGACY_BITEXACT
 
 namespace webrtc {
 
 NetEqImpl::NetEqImpl(const NetEq::Config& config,
+                     std::unique_ptr<TickTimer> tick_timer,
                      BufferLevelFilter* buffer_level_filter,
                      DecoderDatabase* decoder_database,
                      DelayManager* delay_manager,
                      DelayPeakDetector* delay_peak_detector,
                      DtmfBuffer* dtmf_buffer,
                      DtmfToneGenerator* dtmf_tone_generator,
                      PacketBuffer* packet_buffer,
                      PayloadSplitter* payload_splitter,
                      TimestampScaler* timestamp_scaler,
                      AccelerateFactory* accelerate_factory,
                      ExpandFactory* expand_factory,
                      PreemptiveExpandFactory* preemptive_expand_factory,
                      bool create_components)
-    : buffer_level_filter_(buffer_level_filter),
+    : tick_timer_(std::move(tick_timer)),
+      buffer_level_filter_(buffer_level_filter),
       decoder_database_(decoder_database),
       delay_manager_(delay_manager),
       delay_peak_detector_(delay_peak_detector),
       dtmf_buffer_(dtmf_buffer),
       dtmf_tone_generator_(dtmf_tone_generator),
       packet_buffer_(packet_buffer),
       payload_splitter_(payload_splitter),
       timestamp_scaler_(timestamp_scaler),
       vad_(new PostDecodeVad()),
       expand_factory_(expand_factory),
@@ skipping 445 matching lines @@
     // is not defined outside of this block.
     Packet* packet = new Packet;
     packet->header.markerBit = false;
     packet->header.payloadType = rtp_header.header.payloadType;
     packet->header.sequenceNumber = rtp_header.header.sequenceNumber;
     packet->header.timestamp = rtp_header.header.timestamp;
     packet->header.ssrc = rtp_header.header.ssrc;
     packet->header.numCSRCs = 0;
     packet->payload_length = payload.size();
     packet->primary = true;
-    packet->waiting_time = 0;
+    // Waiting time will be set upon inserting the packet in the buffer.
+    RTC_DCHECK(!packet->waiting_time);
     packet->payload = new uint8_t[packet->payload_length];
     packet->sync_packet = is_sync_packet;
     if (!packet->payload) {
       LOG_F(LS_ERROR) << "Payload pointer is NULL.";
     }
     assert(!payload.empty());  // Already checked above.
     memcpy(packet->payload, payload.data(), packet->payload_length);
     // Insert packet in a packet list.
     packet_list.push_back(packet);
     // Save main payloads header for later.
@@ skipping 235 matching lines @@
     delay_manager_->ResetPacketIatCount();
   }
   return 0;
 }
 
 int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame) {
   PacketList packet_list;
   DtmfEvent dtmf_event;
   Operations operation;
   bool play_dtmf;
+  tick_timer_->Increment();
+  stats_.IncreaseCounter(output_size_samples_, fs_hz_);
   int return_value = GetDecision(&operation, &packet_list, &dtmf_event,
                                  &play_dtmf);
   if (return_value != 0) {
     last_mode_ = kModeError;
     return return_value;
   }
 
   AudioDecoder::SpeechType speech_type;
   int length = 0;
   int decode_return_value = Decode(&packet_list, &operation,
                                    &length, &speech_type);
 
   assert(vad_.get());
   bool sid_frame_available =
       (operation == kRfc3389Cng && !packet_list.empty());
   vad_->Update(decoded_buffer_.get(), static_cast<size_t>(length), speech_type,
                sid_frame_available, fs_hz_);
 
+  if (sid_frame_available || speech_type == AudioDecoder::kComfortNoise) {
+    // Start a new stopwatch since we are decoding a new CNG packet.
+    generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch();
+  }
+
   algorithm_buffer_->Clear();
   switch (operation) {
     case kNormal: {
       DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf);
       break;
     }
     case kMerge: {
       DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf);
       break;
     }
@@ skipping 152 matching lines @@
   // Set the timestamp in the audio frame to zero before the first packet has
   // been inserted. Otherwise, subtract the frame size in samples to get the
   // timestamp of the first sample in the frame (playout_timestamp_ is the
   // last + 1).
   audio_frame->timestamp_ =
       first_packet_
           ? 0
           : timestamp_scaler_->ToExternal(playout_timestamp_) -
                 static_cast<uint32_t>(audio_frame->samples_per_channel_);
 
+  if (!(last_mode_ == kModeRfc3389Cng ||
+      last_mode_ == kModeCodecInternalCng ||
+      last_mode_ == kModeExpand)) {
+    generated_noise_stopwatch_.reset();
+  }
+
   if (decode_return_value) return decode_return_value;
   return return_value;
 }
 
 int NetEqImpl::GetDecision(Operations* operation,
                            PacketList* packet_list,
                            DtmfEvent* dtmf_event,
                            bool* play_dtmf) {
   // Initialize output variables.
   *play_dtmf = false;
   *operation = kUndefined;
 
-  // Increment time counters.
-  packet_buffer_->IncrementWaitingTimes();
-  stats_.IncreaseCounter(output_size_samples_, fs_hz_);
-
   assert(sync_buffer_.get());
   uint32_t end_timestamp = sync_buffer_->end_timestamp();
   if (!new_codec_) {
     const uint32_t five_seconds_samples = 5 * fs_hz_;
     packet_buffer_->DiscardOldPackets(end_timestamp, five_seconds_samples);
   }
   const RTPHeader* header = packet_buffer_->NextRtpHeader();
 
+  RTC_DCHECK(!generated_noise_stopwatch_ ||
+             generated_noise_stopwatch_->ElapsedTicks() >= 1);
+  uint64_t generated_noise_samples =
+      generated_noise_stopwatch_
+          ? (generated_noise_stopwatch_->ElapsedTicks() - 1) *
+                    output_size_samples_ +
+                decision_logic_->noise_fast_forward()
+          : 0;
+
   if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) {
     // Because of timestamp peculiarities, we have to "manually" disallow using
     // a CNG packet with the same timestamp as the one that was last played.
     // This can happen when using redundancy and will cause the timing to shift.
     while (header && decoder_database_->IsComfortNoise(header->payloadType) &&
            (end_timestamp >= header->timestamp ||
-            end_timestamp + decision_logic_->generated_noise_samples() >
-                header->timestamp)) {
+            end_timestamp + generated_noise_samples > header->timestamp)) {
       // Don't use this packet, discard it.
       if (packet_buffer_->DiscardNextPacket() != PacketBuffer::kOK) {
         assert(false);  // Must be ok by design.
       }
       // Check buffer again.
       if (!new_codec_) {
         packet_buffer_->DiscardOldPackets(end_timestamp, 5 * fs_hz_);
       }
       header = packet_buffer_->NextRtpHeader();
     }
   }
 
   assert(expand_.get());
   const int samples_left = static_cast<int>(sync_buffer_->FutureLength() -
       expand_->overlap_length());
   if (last_mode_ == kModeAccelerateSuccess ||
       last_mode_ == kModeAccelerateLowEnergy ||
       last_mode_ == kModePreemptiveExpandSuccess ||
       last_mode_ == kModePreemptiveExpandLowEnergy) {
     // Subtract (samples_left + output_size_samples_) from sampleMemory.
     decision_logic_->AddSampleMemory(
         -(samples_left + rtc::checked_cast<int>(output_size_samples_)));
   }
 
   // Check if it is time to play a DTMF event.
   if (dtmf_buffer_->GetEvent(
       static_cast<uint32_t>(
-          end_timestamp + decision_logic_->generated_noise_samples()),
+          end_timestamp + generated_noise_samples),
       dtmf_event)) {
     *play_dtmf = true;
   }
 
   // Get instruction.
   assert(sync_buffer_.get());
   assert(expand_.get());
-  *operation = decision_logic_->GetDecision(*sync_buffer_,
-                                            *expand_,
-                                            decoder_frame_length_,
-                                            header,
-                                            last_mode_,
-                                            *play_dtmf,
-                                            &reset_decoder_);
+  generated_noise_samples =
+      generated_noise_stopwatch_
+          ? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ +
+                decision_logic_->noise_fast_forward()
+          : 0;
+  *operation = decision_logic_->GetDecision(
+      *sync_buffer_, *expand_, decoder_frame_length_, header, last_mode_,
+      *play_dtmf, generated_noise_samples, &reset_decoder_);
 
   // Check if we already have enough samples in the |sync_buffer_|. If so,
   // change decision to normal, unless the decision was merge, accelerate, or
   // preemptive expand.
   if (samples_left >= rtc::checked_cast<int>(output_size_samples_) &&
       *operation != kMerge &&
       *operation != kAccelerate &&
       *operation != kFastAccelerate &&
       *operation != kPreemptiveExpand) {
     *operation = kNormal;
@@ skipping 52 matching lines @@
       return 0;
     }
     case kRfc3389CngNoPacket:
     case kCodecInternalCng: {
       return 0;
     }
     case kDtmf: {
       // TODO(hlundin): Write test for this.
       // Update timestamp.
       timestamp_ = end_timestamp;
-      if (decision_logic_->generated_noise_samples() > 0 &&
-          last_mode_ != kModeDtmf) {
+      const uint64_t generated_noise_samples =
+          generated_noise_stopwatch_
+              ? generated_noise_stopwatch_->ElapsedTicks() *
+                        output_size_samples_ +
+                    decision_logic_->noise_fast_forward()
+              : 0;
+      if (generated_noise_samples > 0 && last_mode_ != kModeDtmf) {
         // Make a jump in timestamp due to the recently played comfort noise.
         uint32_t timestamp_jump =
-            static_cast<uint32_t>(decision_logic_->generated_noise_samples());
+            static_cast<uint32_t>(generated_noise_samples);
         sync_buffer_->IncreaseEndTimestamp(timestamp_jump);
         timestamp_ += timestamp_jump;
       }
-      decision_logic_->set_generated_noise_samples(0);
       return 0;
     }
     case kAccelerate:
     case kFastAccelerate: {
       // In order to do an accelerate we need at least 30 ms of audio data.
       if (samples_left >= static_cast<int>(samples_30_ms)) {
         // Already have enough data, so we do not need to extract any more.
         decision_logic_->set_sample_memory(samples_left);
         decision_logic_->set_prev_time_scale(true);
         return 0;
@@ skipping 62 matching lines @@
       // if comfort noise is not played. If comfort noise was just played,
       // this adjustment of timestamp is only done to get back in sync with the
       // stream timestamp; no loss to report.
       stats_.LostSamples(header->timestamp - end_timestamp);
     }
 
     if (*operation != kRfc3389Cng) {
       // We are about to decode and use a non-CNG packet.
       decision_logic_->SetCngOff();
     }
-    // Reset CNG timestamp as a new packet will be delivered.
-    // (Also if this is a CNG packet, since playedOutTS is updated.)
-    decision_logic_->set_generated_noise_samples(0);
 
     extracted_samples = ExtractPackets(required_samples, packet_list);
     if (extracted_samples < 0) {
       return kPacketBufferCorruption;
     }
   }
 
   if (*operation == kAccelerate || *operation == kFastAccelerate ||
       *operation == kPreemptiveExpand) {
     decision_logic_->set_sample_memory(samples_left + extracted_samples);
@@ skipping 312 matching lines @@
     if (return_value < 0) {
       return return_value;
     }
 
     sync_buffer_->PushBack(*algorithm_buffer_);
     algorithm_buffer_->Clear();
   }
   if (!play_dtmf) {
     dtmf_tone_generator_->Reset();
   }
+
+  if (!generated_noise_stopwatch_) {
+    // Start a new stopwatch since we may be covering for a lost CNG packet.
+    generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch();
+  }
+
   return 0;
 }
 
 int NetEqImpl::DoAccelerate(int16_t* decoded_buffer,
                             size_t decoded_length,
                             AudioDecoder::SpeechType speech_type,
                             bool play_dtmf,
                             bool fast_accelerate) {
   const size_t required_samples =
       static_cast<size_t>(240 * fs_mult_);  // Must have 30 ms.
@@ skipping 350 matching lines @@
     size_t discard_count = 0;
     Packet* packet = packet_buffer_->GetNextPacket(&discard_count);
     // |header| may be invalid after the |packet_buffer_| operation.
     header = NULL;
     if (!packet) {
       LOG(LS_ERROR) << "Should always be able to extract a packet here";
       assert(false);  // Should always be able to extract a packet here.
       return -1;
     }
     stats_.PacketsDiscarded(discard_count);
-    // Store waiting time in ms; packets->waiting_time is in "output blocks".
-    stats_.StoreWaitingTime(packet->waiting_time * kOutputSizeMs);
+    stats_.StoreWaitingTime(packet->waiting_time->ElapsedMs());
     assert(packet->payload_length > 0);
     packet_list->push_back(packet);  // Store packet in list.
 
     if (first_packet) {
       first_packet = false;
       if (nack_enabled_) {
         RTC_DCHECK(nack_);
         // TODO(henrik.lundin): Should we update this for all decoded packets?
         nack_->UpdateLastDecodedPacket(packet->header.sequenceNumber,
                                        packet->header.timestamp);
@@ skipping 152 matching lines @@
   } else if (last_mode_ == kModeExpand) {
     return OutputType::kPLC;
   } else if (vad_->running() && !vad_->active_speech()) {
     return OutputType::kVadPassive;
   } else {
     return OutputType::kNormalSpeech;
   }
 }
 
 void NetEqImpl::CreateDecisionLogic() {
-  decision_logic_.reset(DecisionLogic::Create(fs_hz_, output_size_samples_,
-                                              playout_mode_,
-                                              decoder_database_.get(),
-                                              *packet_buffer_.get(),
-                                              delay_manager_.get(),
-                                              buffer_level_filter_.get()));
+  decision_logic_.reset(DecisionLogic::Create(
+      fs_hz_, output_size_samples_, playout_mode_, decoder_database_.get(),
+      *packet_buffer_.get(), delay_manager_.get(), buffer_level_filter_.get(),
+      *tick_timer_));
 }
 }  // namespace webrtc