Moves channel-dependent audio input processing to separate encoder task queue

webrtc/voice_engine/channel.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/voice_engine/channel.h"
 
 #include <algorithm>
 #include <utility>
 
 #include "webrtc/audio/utility/audio_frame_operations.h"
 #include "webrtc/base/array_view.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/format_macros.h"
 #include "webrtc/base/location.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/rate_limiter.h"
+#include "webrtc/base/task_queue.h"
+#include "webrtc/base/thread_checker.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/config.h"
 #include "webrtc/logging/rtc_event_log/rtc_event_log.h"
 #include "webrtc/modules/audio_coding/codecs/audio_format_conversion.h"
 #include "webrtc/modules/audio_device/include/audio_device.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
 #include "webrtc/modules/include/module_common_types.h"
 #include "webrtc/modules/pacing/packet_router.h"
 #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_packet_received.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_receiver_strategy.h"
 #include "webrtc/modules/utility/include/process_thread.h"
 #include "webrtc/system_wrappers/include/trace.h"
 #include "webrtc/voice_engine/include/voe_rtp_rtcp.h"
 #include "webrtc/voice_engine/output_mixer.h"
 #include "webrtc/voice_engine/statistics.h"
 #include "webrtc/voice_engine/utility.h"
 
 namespace webrtc {
 namespace voe {
 
 namespace {
 
 constexpr int64_t kMaxRetransmissionWindowMs = 1000;
 constexpr int64_t kMinRetransmissionWindowMs = 30;
+// Number of preallocated audio frames in the pool of audio frames.
+// Local tests on Android devices have shown that we never reduce the size of
+// the pool below 5.
+constexpr size_t kAudioFramePoolSize = 10;
 
 }  // namespace
 
 const int kTelephoneEventAttenuationdB = 10;
 
 class RtcEventLogProxy final : public webrtc::RtcEventLog {
  public:
   RtcEventLogProxy() : event_log_(nullptr) {}
 
   bool StartLogging(const std::string& file_name,
@@ skipping 338 matching lines @@
   }
 
  private:
   Channel* owner_;
   // Maps remote side ssrc to extended highest sequence number received.
   std::map<uint32_t, uint32_t> extended_max_sequence_number_;
   rtc::CriticalSection crit_;
   RtcpBandwidthObserver* bandwidth_observer_ GUARDED_BY(crit_);
 };
 
+class Channel::ProcessAndEncodeAudioTask : public rtc::QueuedTask {
+ public:
+  ProcessAndEncodeAudioTask(std::unique_ptr<AudioFrame> audio_input,
+                            Channel* channel,
+                            AudioFramePool* audio_frame_pool)
+      : audio_input_(std::move(audio_input)),
+        channel_(channel),
+        audio_frame_pool_(audio_frame_pool) {}
+
+  ~ProcessAndEncodeAudioTask() override {
+    // Return the utilized audio frame to the pool so it can be used again.
+    audio_frame_pool_->Push(std::move(audio_input_));
+  }
+
+ private:
+  bool Run() override {
+    RTC_DCHECK_RUN_ON(channel_->encoder_queue_);
+    RTC_DCHECK(channel_);
+    channel_->ProcessAndEncodeAudioOnTaskQueue(audio_input_.get());
+    return true;
+  }
+
+  std::unique_ptr<AudioFrame> audio_input_;
+  Channel* const channel_;
+  AudioFramePool* audio_frame_pool_;
+};

[aleloi, 2017/03/23 13:33:21]

Since the task operates on a pool and channel pointer, these objects must not be
destroyed before the task is run. Tasks can potentially be alive for as long as
the task queue is not destroyed. I know only of these ways to accomplish that:

1. Make the task queue be destroyed before the channel and pool (but after the
last attempt to post anything on it)
2. Use reference counting to delay the destruction of channel and pool (maybe
infeasible if Channel itself has to be refcounted)
3. Delay destroying the channel until the last task has finished running with
some kind of messaging between task and channel d-tor.

[henrika_webrtc, 2017/03/23 14:02:17]

Thanks! The pool is now removed. Will take your comments into account and
discuss with Fredrik as well.

+
 int32_t Channel::SendData(FrameType frameType,
                           uint8_t payloadType,
                           uint32_t timeStamp,
                           const uint8_t* payloadData,
                           size_t payloadSize,
                           const RTPFragmentationHeader* fragmentation) {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
   WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::SendData(frameType=%u, payloadType=%u, timeStamp=%u,"
                " payloadSize=%" PRIuS ", fragmentation=0x%x)",
                frameType, payloadType, timeStamp, payloadSize, fragmentation);
 
   if (_includeAudioLevelIndication) {
     // Store current audio level in the RTP/RTCP module.
     // The level will be used in combination with voice-activity state
     // (frameType) to add an RTP header extension
     _rtpRtcpModule->SetAudioLevel(rms_level_.Average());
   }
 
   // Push data from ACM to RTP/RTCP-module to deliver audio frame for
   // packetization.
   // This call will trigger Transport::SendPacket() from the RTP/RTCP module.
   if (!_rtpRtcpModule->SendOutgoingData(
           (FrameType&)frameType, payloadType, timeStamp,
           // Leaving the time when this frame was
           // received from the capture device as
           // undefined for voice for now.
           -1, payloadData, payloadSize, fragmentation, nullptr, nullptr)) {
     _engineStatisticsPtr->SetLastError(
         VE_RTP_RTCP_MODULE_ERROR, kTraceWarning,
         "Channel::SendData() failed to send data to RTP/RTCP module");
     return -1;
   }
 
   _lastLocalTimeStamp = timeStamp;
   _lastPayloadType = payloadType;
-
   return 0;
 }
 
 bool Channel::SendRtp(const uint8_t* data,
                       size_t len,
                       const PacketOptions& options) {
   WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::SendPacket(channel=%d, len=%" PRIuS ")", len);
 
   rtc::CritScope cs(&_callbackCritSect);
@@ skipping 420 matching lines @@
       send_sequence_number_(0),
       rtp_ts_wraparound_handler_(new rtc::TimestampWrapAroundHandler()),
       capture_start_rtp_time_stamp_(-1),
       capture_start_ntp_time_ms_(-1),
       _engineStatisticsPtr(NULL),
       _outputMixerPtr(NULL),
       _moduleProcessThreadPtr(NULL),
       _audioDeviceModulePtr(NULL),
       _voiceEngineObserverPtr(NULL),
       _callbackCritSectPtr(NULL),
+      encoder_queue_(nullptr),
       _transportPtr(NULL),
       input_mute_(false),
       previous_frame_muted_(false),
       _outputGain(1.0f),
       _mixFileWithMicrophone(false),
       _lastLocalTimeStamp(0),
       _lastPayloadType(0),
       _includeAudioLevelIndication(false),
       transport_overhead_per_packet_(0),
       rtp_overhead_per_packet_(0),
       _outputSpeechType(AudioFrame::kNormalSpeech),
       restored_packet_in_use_(false),
       rtcp_observer_(new VoERtcpObserver(this)),
       associate_send_channel_(ChannelOwner(nullptr)),
       pacing_enabled_(config.enable_voice_pacing),
       feedback_observer_proxy_(new TransportFeedbackProxy()),
       seq_num_allocator_proxy_(new TransportSequenceNumberProxy()),
       rtp_packet_sender_proxy_(new RtpPacketSenderProxy()),
       retransmission_rate_limiter_(new RateLimiter(Clock::GetRealTimeClock(),
                                                    kMaxRetransmissionWindowMs)),
+      audio_frame_pool_(kAudioFramePoolSize),
       decoder_factory_(config.acm_config.decoder_factory) {
   WEBRTC_TRACE(kTraceMemory, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::Channel() - ctor");
   AudioCodingModule::Config acm_config(config.acm_config);
   acm_config.id = VoEModuleId(instanceId, channelId);
   acm_config.neteq_config.enable_muted_state = true;
   audio_coding_.reset(AudioCodingModule::Create(acm_config));
 
   _outputAudioLevel.Clear();
 
@@ skipping 175 matching lines @@
     _moduleProcessThreadPtr->DeRegisterModule(_rtpRtcpModule.get());
 
   // End of modules shutdown
 }
 
 int32_t Channel::SetEngineInformation(Statistics& engineStatistics,
                                       OutputMixer& outputMixer,
                                       ProcessThread& moduleProcessThread,
                                       AudioDeviceModule& audioDeviceModule,
                                       VoiceEngineObserver* voiceEngineObserver,
-                                      rtc::CriticalSection* callbackCritSect) {
+                                      rtc::CriticalSection* callbackCritSect,
+                                      rtc::TaskQueue* encoder_queue) {
   WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_instanceId, _channelId),
                "Channel::SetEngineInformation()");
+  RTC_DCHECK(encoder_queue);
   _engineStatisticsPtr = &engineStatistics;
   _outputMixerPtr = &outputMixer;
   _moduleProcessThreadPtr = &moduleProcessThread;
   _audioDeviceModulePtr = &audioDeviceModule;
   _voiceEngineObserverPtr = voiceEngineObserver;
   _callbackCritSectPtr = callbackCritSect;
+  encoder_queue_ = encoder_queue;
   return 0;
 }
 
-int32_t Channel::UpdateLocalTimeStamp() {
-  _timeStamp += static_cast<uint32_t>(_audioFrame.samples_per_channel_);
-  return 0;
-}
-
 void Channel::SetSink(std::unique_ptr<AudioSinkInterface> sink) {
   rtc::CritScope cs(&_callbackCritSect);
   audio_sink_ = std::move(sink);
 }
 
 const rtc::scoped_refptr<AudioDecoderFactory>&
 Channel::GetAudioDecoderFactory() const {
   return decoder_factory_;
 }
 
@@ skipping 1455 matching lines @@
     audio_coding_->EnableNack(maxNumberOfPackets);
   else
     audio_coding_->DisableNack();
 }
 
 // Called when we are missing one or more packets.
 int Channel::ResendPackets(const uint16_t* sequence_numbers, int length) {
   return _rtpRtcpModule->SendNACK(sequence_numbers, length);
 }
 
-uint32_t Channel::Demultiplex(const AudioFrame& audioFrame) {
-  WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
-               "Channel::Demultiplex()");
-  _audioFrame.CopyFrom(audioFrame);
-  _audioFrame.id_ = _channelId;
-  return 0;
+void Channel::ProcessAndEncodeAudio(const AudioFrame& audio_input) {
+  RTC_DCHECK(encoder_queue_);
+  std::unique_ptr<AudioFrame> audio_frame = audio_frame_pool_.Pop();
+  RTC_DCHECK(audio_frame) << "Pool of audio frames is empty";
+  if (audio_frame) {
+    audio_frame->CopyFrom(audio_input);
+    audio_frame->id_ = _channelId;
+    PostTask(std::move(audio_frame));
+  }
 }
 
-void Channel::Demultiplex(const int16_t* audio_data,
-                          int sample_rate,
-                          size_t number_of_frames,
-                          size_t number_of_channels) {
+void Channel::ProcessAndEncodeAudio(const int16_t* audio_data,
+                                    int sample_rate,
+                                    size_t number_of_frames,
+                                    size_t number_of_channels) {
+  RTC_DCHECK(encoder_queue_);
   CodecInst codec;
   GetSendCodec(codec);
-
-  // Never upsample or upmix the capture signal here. This should be done at the
-  // end of the send chain.
-  _audioFrame.sample_rate_hz_ = std::min(codec.plfreq, sample_rate);
-  _audioFrame.num_channels_ = std::min(number_of_channels, codec.channels);
-  RemixAndResample(audio_data, number_of_frames, number_of_channels,
-                   sample_rate, &input_resampler_, &_audioFrame);
+  std::unique_ptr<AudioFrame> audio_frame = audio_frame_pool_.Pop();
+  RTC_DCHECK(audio_frame) << "Pool of audio frames is empty";
+  if (audio_frame) {
+    audio_frame->sample_rate_hz_ = std::min(codec.plfreq, sample_rate);
+    audio_frame->num_channels_ = std::min(number_of_channels, codec.channels);
+    RemixAndResample(audio_data, number_of_frames, number_of_channels,

[aleloi, 2017/03/23 13:33:21]

Perhaps resampling could also be done on the task queue?

[henrika_webrtc, 2017/03/23 14:02:17]

Goof point. Will have to create unique method for it.

+                     sample_rate, &input_resampler_, audio_frame.get());
+    audio_frame->id_ = _channelId;
+    PostTask(std::move(audio_frame));
+  }
 }
 
-uint32_t Channel::PrepareEncodeAndSend(int mixingFrequency) {
-  WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
-               "Channel::PrepareEncodeAndSend()");
+void Channel::ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input) {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  PrepareEncodeAndSend(audio_input);
+  EncodeAndSend(audio_input);
+}
 
-  if (_audioFrame.samples_per_channel_ == 0) {
-    WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
-                 "Channel::PrepareEncodeAndSend() invalid audio frame");
-    return 0xFFFFFFFF;
-  }
+uint32_t Channel::PrepareEncodeAndSend(AudioFrame* audio_input) {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  RTC_DCHECK(audio_input->samples_per_channel_);
 
   if (channel_state_.Get().input_file_playing) {
-    MixOrReplaceAudioWithFile(mixingFrequency);
+    MixOrReplaceAudioWithFile(audio_input);
   }
 
   bool is_muted = InputMute();  // Cache locally as InputMute() takes a lock.
-  AudioFrameOperations::Mute(&_audioFrame, previous_frame_muted_, is_muted);
+  AudioFrameOperations::Mute(audio_input, previous_frame_muted_, is_muted);
 
   if (_includeAudioLevelIndication) {
     size_t length =
-        _audioFrame.samples_per_channel_ * _audioFrame.num_channels_;
-    RTC_CHECK_LE(length, sizeof(_audioFrame.data_));
+        audio_input->samples_per_channel_ * audio_input->num_channels_;
+    RTC_CHECK_LE(length, sizeof(audio_input->data_));
     if (is_muted && previous_frame_muted_) {
       rms_level_.AnalyzeMuted(length);
     } else {
       rms_level_.Analyze(
-          rtc::ArrayView<const int16_t>(_audioFrame.data_, length));
+          rtc::ArrayView<const int16_t>(audio_input->data_, length));
     }
   }
   previous_frame_muted_ = is_muted;
-
   return 0;
 }
 
-uint32_t Channel::EncodeAndSend() {
-  WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_instanceId, _channelId),
-               "Channel::EncodeAndSend()");
+uint32_t Channel::EncodeAndSend(AudioFrame* audio_input) {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  RTC_DCHECK_LE(audio_input->num_channels_, 2);
+  RTC_DCHECK(audio_input->samples_per_channel_);
 
-  assert(_audioFrame.num_channels_ <= 2);
-  if (_audioFrame.samples_per_channel_ == 0) {
-    WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
-                 "Channel::EncodeAndSend() invalid audio frame");
-    return 0xFFFFFFFF;
-  }
-
-  _audioFrame.id_ = _channelId;
+  audio_input->id_ = _channelId;
 
   // --- Add 10ms of raw (PCM) audio data to the encoder @ 32kHz.
 
   // The ACM resamples internally.
-  _audioFrame.timestamp_ = _timeStamp;
+  audio_input->timestamp_ = _timeStamp;
   // This call will trigger AudioPacketizationCallback::SendData if encoding
   // is done and payload is ready for packetization and transmission.
   // Otherwise, it will return without invoking the callback.
-  if (audio_coding_->Add10MsData((AudioFrame&)_audioFrame) < 0) {
-    WEBRTC_TRACE(kTraceError, kTraceVoice, VoEId(_instanceId, _channelId),
-                 "Channel::EncodeAndSend() ACM encoding failed");
+  if (audio_coding_->Add10MsData(*audio_input) < 0) {
+    LOG(LS_ERROR) << "ACM::Add10MsData() failed for channel " << _channelId;
     return 0xFFFFFFFF;
   }
 
-  _timeStamp += static_cast<uint32_t>(_audioFrame.samples_per_channel_);
+  _timeStamp += static_cast<uint32_t>(audio_input->samples_per_channel_);
   return 0;
 }
 
 void Channel::set_associate_send_channel(const ChannelOwner& channel) {
   RTC_DCHECK(!channel.channel() ||
              channel.channel()->ChannelId() != _channelId);
   rtc::CritScope lock(&assoc_send_channel_lock_);
   associate_send_channel_ = channel;
 }
 
@@ skipping 89 matching lines @@
 
 int Channel::GetRtpRtcp(RtpRtcp** rtpRtcpModule,
                         RtpReceiver** rtp_receiver) const {
   *rtpRtcpModule = _rtpRtcpModule.get();
   *rtp_receiver = rtp_receiver_.get();
   return 0;
 }
 
 // TODO(andrew): refactor Mix functions here and in transmit_mixer.cc to use
 // a shared helper.
-int32_t Channel::MixOrReplaceAudioWithFile(int mixingFrequency) {
+int32_t Channel::MixOrReplaceAudioWithFile(AudioFrame* audio_input) {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
   std::unique_ptr<int16_t[]> fileBuffer(new int16_t[640]);
   size_t fileSamples(0);
+  const int mixingFrequency = audio_input->sample_rate_hz_;
 
-  {
-    rtc::CritScope cs(&_fileCritSect);
-
-    if (!input_file_player_) {
-      WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
-                   "Channel::MixOrReplaceAudioWithFile() fileplayer"
-                   " doesnt exist");
-      return -1;
-    }
-
-    if (input_file_player_->Get10msAudioFromFile(fileBuffer.get(), &fileSamples,
-                                                 mixingFrequency) == -1) {
-      WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
-                   "Channel::MixOrReplaceAudioWithFile() file mixing "
-                   "failed");
-      return -1;
-    }
-    if (fileSamples == 0) {
-      WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
-                   "Channel::MixOrReplaceAudioWithFile() file is ended");
-      return 0;
-    }
+  if (!input_file_player_) {
+    WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
+                 "Channel::MixOrReplaceAudioWithFile() fileplayer"
+                 " doesnt exist");
+    return -1;
   }
 
-  assert(_audioFrame.samples_per_channel_ == fileSamples);
+  if (input_file_player_->Get10msAudioFromFile(fileBuffer.get(), &fileSamples,
+                                               mixingFrequency) == -1) {
+    WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
+                 "Channel::MixOrReplaceAudioWithFile() file mixing "
+                 "failed");
+    return -1;
+  }
+  if (fileSamples == 0) {
+    WEBRTC_TRACE(kTraceWarning, kTraceVoice, VoEId(_instanceId, _channelId),
+                 "Channel::MixOrReplaceAudioWithFile() file is ended");
+    return 0;
+  }
+
+  assert(audio_input->samples_per_channel_ == fileSamples);
 
   if (_mixFileWithMicrophone) {
     // Currently file stream is always mono.
     // TODO(xians): Change the code when FilePlayer supports real stereo.
-    MixWithSat(_audioFrame.data_, _audioFrame.num_channels_, fileBuffer.get(),
+    MixWithSat(audio_input->data_, audio_input->num_channels_, fileBuffer.get(),
                1, fileSamples);
   } else {
     // Replace ACM audio with file.
     // Currently file stream is always mono.
     // TODO(xians): Change the code when FilePlayer supports real stereo.
-    _audioFrame.UpdateFrame(
+    audio_input->UpdateFrame(
         _channelId, 0xFFFFFFFF, fileBuffer.get(), fileSamples, mixingFrequency,
         AudioFrame::kNormalSpeech, AudioFrame::kVadUnknown, 1);
   }
   return 0;
 }
 
 int32_t Channel::MixAudioWithFile(AudioFrame& audioFrame, int mixingFrequency) {
   assert(mixingFrequency <= 48000);
 
   std::unique_ptr<int16_t[]> fileBuffer(new int16_t[960]);
@@ skipping 167 matching lines @@
   int64_t avg_rtt = 0;
   int64_t max_rtt = 0;
   int64_t min_rtt = 0;
   if (_rtpRtcpModule->RTT(remoteSSRC, &rtt, &avg_rtt, &min_rtt, &max_rtt) !=
       0) {
     return 0;
   }
   return rtt;
 }
 
+void Channel::PostTask(std::unique_ptr<AudioFrame> audio_frame) {
+  encoder_queue_->PostTask(
+      std::unique_ptr<rtc::QueuedTask>(new ProcessAndEncodeAudioTask(
+          std::move(audio_frame), this, &audio_frame_pool_)));
+}
+
 }  // namespace voe
 }  // namespace webrtc