Refactor the AudioDevice for iOS and improve the performance and stability

webrtc/modules/audio_device/ios/audio_device_ios.mm

Index: webrtc/modules/audio_device/ios/audio_device_ios.mm
diff --git a/webrtc/modules/audio_device/ios/audio_device_ios.mm b/webrtc/modules/audio_device/ios/audio_device_ios.mm
index 6f610d7afcc7758fddeccd01e4434b0b391dde4f..245b47826e81c2fb3dba0ba97d21b24182c7eccc 100644
--- a/webrtc/modules/audio_device/ios/audio_device_ios.mm
+++ b/webrtc/modules/audio_device/ios/audio_device_ios.mm
@@ -16,100 +16,139 @@
 #import <Foundation/Foundation.h>
 
 #include "webrtc/modules/audio_device/ios/audio_device_ios.h"
-#include "webrtc/modules/utility/interface/helpers_ios.h"
 
+#include "webrtc/base/atomicops.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
-#include "webrtc/system_wrappers/interface/trace.h"
+#include "webrtc/modules/audio_device/fine_audio_buffer.h"
+#include "webrtc/modules/utility/interface/helpers_ios.h"
 
 namespace webrtc {
 
 #define LOGI() LOG(LS_INFO) << "AudioDeviceIOS::"
 
+// Preferred hardware sample rate (unit is in Hertz). The client sample rate
+// will be set to this value as well to avoid resampling the the audio unit's
+// format converter.
+const double kPreferredSampleRate = 48000.0;
+// Use a hardware I/O buffer size (unit is in seconds) that matches the 10ms
+// size used by WebRTC. The exact actual size will differ between devices.
+// Example: using 48kHz on iPhone 6 results in a native buffer size of
+// ~10.6667ms or 512 audio frames per buffer. The FineAudioBuffer instance will
+// take care of any buffering required to convert between native buffers and
+// buffers used by WebRTC. It is beneficial for the performance if the native
+// size is as close to 10ms as possible since it results in "clean" callback
+// sequence without bursts of callbacks back to back.
+const double kPreferredIOBufferDuration = 0.01;
+// Try to use mono to save resources. Also avoids channel format conversion
+// in the I/O audio unit. Seems to work only for built-in microphones.

[tkchin_webrtc, 2015/09/01 20:54:50]

Did you try this with bluetooth microphone? Wondering how you found out it only
works for built-in mic.

[henrika_webrtc, 2015/09/03 13:44:41]

I'll see if I can test with a BT device as well. My test shows that you can't
ask for mono and get mono as native format on the built-in mic. When we plug in
a mic, we always get stereo as native input but will be converted in the format
converted in the audio unit to match mono. Perhaps the comment is badly written.

OK, I was able to connect a Plantronics BT device and it also supports native
mono input (same as built-in).

However, we will crash today since it does not support 48kHz _only_ 8kHz (hence,
the existing code does not work either since it tries to open in 16kHz).

I will add a TODO to fix BT support in an upcoming CL. Hope that is OK. I have
verified that it works in loopback with a BT device using this code, but we will
fail when we check that we got the sample rate that was asked for. I will redo
this scheme and add a fallback but would like to do that in a separate CL.

[tkchin_webrtc, 2015/09/04 05:29:16]

That's fine since this is active work. We should finish it before next branch
cut though since I think some external people take webrtc cuts then and we don't
want to break everyone. At the very least we shouldn't crash on assert.

[henrika_webrtc, 2015/09/04 09:51:18]

Will avoid crash on assert.

+const int kPreferredNumberOfChannels = 1;
+// Number of bytes per audio sample for 16-bit signed integer representation.
+const UInt32 kBytesPerSample = 2;
+// Hardcoded delay estimates based on real measurements.
+// TODO(henrika): these value is not used in combination with built-in AEC.
+// Can most likely be removed.
+const UInt16 kFixedPlayoutDelayEstimate = 30;
+const UInt16 kFixedRecordDelayEstimate = 30;
+
 using ios::CheckAndLogError;
 
+// Activates an audio session suitable for full duplex VoIP sessions when
+// |activate| is true. Also sets the preferred sample rate and IO buffer
+// duration. Deactivates an active audio session if |activate| is set to false.
 static void ActivateAudioSession(AVAudioSession* session, bool activate) {
   LOG(LS_INFO) << "ActivateAudioSession(" << activate << ")";
   @autoreleasepool {
     NSError* error = nil;
     BOOL success = NO;
+    // Deactivate the audio session and return if |activate| is false.
     if (!activate) {
-      // Deactivate the audio session.
       success = [session setActive:NO error:&error];
       DCHECK(CheckAndLogError(success, error));
       return;
     }
-    // Activate an audio session and set category and mode. Only make changes
-    // if needed since setting them to the value they already have will clear
-    // transient properties (such as PortOverride) that some other component
-    // have set up.
+    // Use a category which supports simultaneous recording and playback.
+    // By default, using this category implies that our app’s audio is
+    // nonmixable, hence activating the session will interrupt any other
+    // audio sessions which are also nonmixable.
     if (session.category != AVAudioSessionCategoryPlayAndRecord) {
       error = nil;
       success = [session setCategory:AVAudioSessionCategoryPlayAndRecord
                                error:&error];
       DCHECK(CheckAndLogError(success, error));
     }
+    // Specify mode for two-way voice communication (e.g. VoIP).
     if (session.mode != AVAudioSessionModeVoiceChat) {
       error = nil;
       success = [session setMode:AVAudioSessionModeVoiceChat error:&error];
       DCHECK(CheckAndLogError(success, error));
     }
+    // Set the session's sample rate or the hardware sample rate.
+    // It is essential that we use the same sample rate as stream format
+    // to ensure that the I/O unit does not have to do sample rate conversion.
+    error = nil;
+    success =
+        [session setPreferredSampleRate:kPreferredSampleRate error:&error];
+    DCHECK(CheckAndLogError(success, error));
+    // Set the preferred audio I/O buffer duration, in seconds.
+    // TODO(henrika): add more comments here.
+    error = nil;
+    success = [session setPreferredIOBufferDuration:kPreferredIOBufferDuration
+                                              error:&error];
+    DCHECK(CheckAndLogError(success, error));
+
+    // TODO(henrika): add observers here...
+
+    // Activate the audio session. Activation can fail if another active audio
+    // session (e.g. phone call) has higher priority than ours.
     error = nil;
     success = [session setActive:YES error:&error];
     DCHECK(CheckAndLogError(success, error));
+    CHECK(session.isInputAvailable) << "No input path is available!";
     // Ensure that category and mode are actually activated.
     DCHECK(
         [session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]);
     DCHECK([session.mode isEqualToString:AVAudioSessionModeVoiceChat]);
+    // Try to set the preferred number of hardware audio channels. These calls
+    // must be done after setting the audio session’s category and mode and
+    // activating the session.
+    // We try to use mono in both directions to save resources and format
+    // conversions in the audio unit. Some devices does only support stereo;
+    // e.g. wired headset on iPhone 6.
+    // TODO(henrika): add support for stereo if needed.
+    error = nil;
+    success =
+        [session setPreferredInputNumberOfChannels:kPreferredNumberOfChannels
+                                             error:&error];
+    DCHECK(CheckAndLogError(success, error));
+    error = nil;
+    success =
+        [session setPreferredOutputNumberOfChannels:kPreferredNumberOfChannels
+                                              error:&error];
+    DCHECK(CheckAndLogError(success, error));
   }
 }
 
-// Query hardware characteristics, such as input and output latency, input and
-// output channel count, hardware sample rate, hardware volume setting, and
-// whether audio input is available. To obtain meaningful values for hardware
-// characteristics,the audio session must be initialized and active before we
-// query the values.
-// TODO(henrika): Note that these characteristics can change at runtime. For
-// instance, input sample rate may change when a user plugs in a headset.
-static void GetHardwareAudioParameters(AudioParameters* playout_parameters,
-                                       AudioParameters* record_parameters) {
-  LOG(LS_INFO) << "GetHardwareAudioParameters";
-  @autoreleasepool {
-    // Implicit initialization happens when we obtain a reference to the
-    // AVAudioSession object.
-    AVAudioSession* session = [AVAudioSession sharedInstance];
-    // Always get values when the audio session is active.
-    ActivateAudioSession(session, true);
-    CHECK(session.isInputAvailable) << "No input path is available!";
-    // Get current hardware parameters.
-    double sample_rate = (double)session.sampleRate;
-    double io_buffer_duration = (double)session.IOBufferDuration;
-    int output_channels = (int)session.outputNumberOfChannels;
-    int input_channels = (int)session.inputNumberOfChannels;
-    size_t frames_per_buffer =
-        static_cast<size_t>(sample_rate * io_buffer_duration + 0.5);
-    // Copy hardware parameters to output parameters.
-    playout_parameters->reset(sample_rate, output_channels, frames_per_buffer);
-    record_parameters->reset(sample_rate, input_channels, frames_per_buffer);
-    // Add logging for debugging purposes.
-    LOG(LS_INFO) << " sample rate: " << sample_rate;
-    LOG(LS_INFO) << " IO buffer duration: " << io_buffer_duration;
-    LOG(LS_INFO) << " frames_per_buffer: " << frames_per_buffer;
-    LOG(LS_INFO) << " output channels: " << output_channels;
-    LOG(LS_INFO) << " input channels: " << input_channels;
-    LOG(LS_INFO) << " output latency: " << (double)session.outputLatency;
-    LOG(LS_INFO) << " input latency: " << (double)session.inputLatency;
-    // Don't keep the audio session active. Instead, deactivate when needed.
-    ActivateAudioSession(session, false);
-    // TODO(henrika): to be extra safe, we can do more here. E.g., set
-    // preferred values for sample rate, channels etc., re-activate an audio
-    // session and verify the actual values again. Then we know for sure that
-    // the current values will in fact be correct. Or, we can skip all this
-    // and check setting when audio is started. Probably better.
-  }
+#if !defined(NDEBUG)
+// Helper method for printing out an AudioStreamBasicDescription structure.
+static void LogABSD(AudioStreamBasicDescription absd) {
+  char formatIDString[5];
+  UInt32 formatID = CFSwapInt32HostToBig(absd.mFormatID);
+  bcopy(&formatID, formatIDString, 4);
+  formatIDString[4] = '\0';
+  LOG(LS_INFO) << "LogABSD";
+  LOG(LS_INFO) << " sample rate: " << absd.mSampleRate;
+  LOG(LS_INFO) << " format ID: " << formatIDString;
+  LOG(LS_INFO) << " format flags: " << std::hex << absd.mFormatFlags;
+  LOG(LS_INFO) << " bytes per packet: " << absd.mBytesPerPacket;
+  LOG(LS_INFO) << " frames per packet: " << absd.mFramesPerPacket;
+  LOG(LS_INFO) << " bytes per frame: " << absd.mBytesPerFrame;
+  LOG(LS_INFO) << " channels per packet: " << absd.mChannelsPerFrame;
+  LOG(LS_INFO) << " bits per channel: " << absd.mBitsPerChannel;
+  LOG(LS_INFO) << " reserved: " << absd.mReserved;
 }
 
-#if !defined(NDEBUG)
+// Helper method that logs essential device information strings.
 static void LogDeviceInfo() {
   LOG(LS_INFO) << "LogDeviceInfo";
   @autoreleasepool {
@@ -119,127 +158,76 @@ static void LogDeviceInfo() {
     LOG(LS_INFO) << " device name: " << ios::GetDeviceName();
   }
 }
-#endif
+#endif  // !defined(NDEBUG)
 
 AudioDeviceIOS::AudioDeviceIOS()
-    : audio_device_buffer_(nullptr),
-      _critSect(*CriticalSectionWrapper::CreateCriticalSection()),
-      _auVoiceProcessing(nullptr),
-      _audioInterruptionObserver(nullptr),
+    : _audioDeviceBuffer(nullptr),
+      _vpioUnit(nullptr),
+      _recording(0),
+      _playing(0),
       _initialized(false),
-      _isShutDown(false),
-      _recording(false),
-      _playing(false),
       _recIsInitialized(false),
       _playIsInitialized(false),
-      _adbSampFreq(0),
-      _recordingDelay(0),
-      _playoutDelay(0),
-      _playoutDelayMeasurementCounter(9999),
-      _recordingDelayHWAndOS(0),
-      _recordingDelayMeasurementCounter(9999),
-      _playoutBufferUsed(0),
-      _recordingCurrentSeq(0),
-      _recordingBufferTotalSize(0) {
+      _audioInterruptionObserver(nullptr) {
   LOGI() << "ctor" << ios::GetCurrentThreadDescription();
-  memset(_playoutBuffer, 0, sizeof(_playoutBuffer));
-  memset(_recordingBuffer, 0, sizeof(_recordingBuffer));
-  memset(_recordingLength, 0, sizeof(_recordingLength));
-  memset(_recordingSeqNumber, 0, sizeof(_recordingSeqNumber));
 }
 
 AudioDeviceIOS::~AudioDeviceIOS() {
   LOGI() << "~dtor";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   Terminate();
-  delete &_critSect;
 }
 
 void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
   LOGI() << "AttachAudioBuffer";
   DCHECK(audioBuffer);
-  DCHECK(thread_checker_.CalledOnValidThread());
-  audio_device_buffer_ = audioBuffer;
+  DCHECK(_threadChecker.CalledOnValidThread());
+  _audioDeviceBuffer = audioBuffer;
 }
 
 int32_t AudioDeviceIOS::Init() {
   LOGI() << "Init";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   if (_initialized) {
     return 0;
   }
 #if !defined(NDEBUG)
   LogDeviceInfo();
 #endif
-  // Query hardware audio parameters and cache the results. These parameters
-  // will be used as preferred values later when streaming starts.
-  // Note that I override these "optimal" value below since I don't want to
-  // modify the existing behavior yet.
-  GetHardwareAudioParameters(&playout_parameters_, &record_parameters_);
-  // TODO(henrika): these parameters are currently hard coded to match the
-  // existing implementation where we always use 16kHz as preferred sample
-  // rate and mono only. Goal is to improve this scheme and make it more
-  // flexible. In addition, a better native buffer size shall be derived.
-  // Using 10ms as default here (only used by unit test so far).
-  // We should also implemented observers for notification of any change in
-  // these parameters.
-  playout_parameters_.reset(16000, 1, 160);
-  record_parameters_.reset(16000, 1, 160);
-
-  // AttachAudioBuffer() is called at construction by the main class but check
-  // just in case.
-  DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first";
-  // Inform the audio device buffer (ADB) about the new audio format.
-  // TODO(henrika): try to improve this section.
-  audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate());
-  audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels());
-  audio_device_buffer_->SetRecordingSampleRate(
-      record_parameters_.sample_rate());
-  audio_device_buffer_->SetRecordingChannels(record_parameters_.channels());
-
-  DCHECK(!_captureWorkerThread);
-  // Create and start the capture thread.
-  // TODO(henrika): do we need this thread?
-  _isShutDown = false;
-  _captureWorkerThread =
-      ThreadWrapper::CreateThread(RunCapture, this, "CaptureWorkerThread");
-  if (!_captureWorkerThread->Start()) {
-    LOG_F(LS_ERROR) << "Failed to start CaptureWorkerThread!";
-    return -1;
-  }
-  _captureWorkerThread->SetPriority(kRealtimePriority);
+  // Store the preferred sample rate and preferred number of channels already
+  // here. They have not been set and confirmed yet since ActivateAudioSession()
+  // is not called until audio is about to start. However, it makes sense to
+  // store the parameters now and then verify at a later stage.
+  _playoutParameters.reset(kPreferredSampleRate, kPreferredNumberOfChannels);
+  _recordParameters.reset(kPreferredSampleRate, kPreferredNumberOfChannels);
+  // Ensure that the audio device buffer (ADB) knows about the internal audio
+  // parameters. Note that, even if we are unable to get a mono audio session,
+  // we will always tell the I/O audio unit to do a channel format conversion
+  // to guarantee mono on the "input side" of the audio unit.
+  UpdateAudioDeviceBuffer();
   _initialized = true;
   return 0;
 }
 
 int32_t AudioDeviceIOS::Terminate() {
   LOGI() << "Terminate";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   if (!_initialized) {
     return 0;
   }
-  // Stop the capture thread.
-  if (_captureWorkerThread) {
-    if (!_captureWorkerThread->Stop()) {
-      LOG_F(LS_ERROR) << "Failed to stop CaptureWorkerThread!";
-      return -1;
-    }
-    _captureWorkerThread.reset();
-  }
   ShutdownPlayOrRecord();
-  _isShutDown = true;
   _initialized = false;
   return 0;
 }
 
 int32_t AudioDeviceIOS::InitPlayout() {
   LOGI() << "InitPlayout";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   DCHECK(_initialized);
   DCHECK(!_playIsInitialized);
   DCHECK(!_playing);
   if (!_recIsInitialized) {
-    if (InitPlayOrRecord() == -1) {
+    if (!InitPlayOrRecord()) {
       LOG_F(LS_ERROR) << "InitPlayOrRecord failed!";
       return -1;
     }
@@ -250,12 +238,12 @@ int32_t AudioDeviceIOS::InitPlayout() {
 
 int32_t AudioDeviceIOS::InitRecording() {
   LOGI() << "InitRecording";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   DCHECK(_initialized);
   DCHECK(!_recIsInitialized);
   DCHECK(!_recording);
   if (!_playIsInitialized) {
-    if (InitPlayOrRecord() == -1) {
+    if (!InitPlayOrRecord()) {
       LOG_F(LS_ERROR) << "InitPlayOrRecord failed!";
       return -1;
     }
@@ -266,92 +254,63 @@ int32_t AudioDeviceIOS::InitRecording() {
 
 int32_t AudioDeviceIOS::StartPlayout() {
   LOGI() << "StartPlayout";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   DCHECK(_playIsInitialized);
   DCHECK(!_playing);
-
-  CriticalSectionScoped lock(&_critSect);
-
-  memset(_playoutBuffer, 0, sizeof(_playoutBuffer));
-  _playoutBufferUsed = 0;
-  _playoutDelay = 0;
-  // Make sure first call to update delay function will update delay
-  _playoutDelayMeasurementCounter = 9999;
-
+  _fineAudioBuffer->ResetPlayout();
   if (!_recording) {
-    OSStatus result = AudioOutputUnitStart(_auVoiceProcessing);
+    OSStatus result = AudioOutputUnitStart(_vpioUnit);
     if (result != noErr) {
       LOG_F(LS_ERROR) << "AudioOutputUnitStart failed: " << result;
       return -1;
     }
   }
-  _playing = true;
+  rtc::AtomicOps::ReleaseStore(&_playing, 1);
   return 0;
 }
 
 int32_t AudioDeviceIOS::StopPlayout() {
   LOGI() << "StopPlayout";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   if (!_playIsInitialized || !_playing) {
     return 0;
   }
-
-  CriticalSectionScoped lock(&_critSect);
-
   if (!_recording) {
-    // Both playout and recording has stopped, shutdown the device.
     ShutdownPlayOrRecord();
   }
   _playIsInitialized = false;
-  _playing = false;
+  rtc::AtomicOps::ReleaseStore(&_playing, 0);
   return 0;
 }
 
 int32_t AudioDeviceIOS::StartRecording() {
   LOGI() << "StartRecording";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   DCHECK(_recIsInitialized);
   DCHECK(!_recording);
-
-  CriticalSectionScoped lock(&_critSect);
-
-  memset(_recordingBuffer, 0, sizeof(_recordingBuffer));
-  memset(_recordingLength, 0, sizeof(_recordingLength));
-  memset(_recordingSeqNumber, 0, sizeof(_recordingSeqNumber));
-
-  _recordingCurrentSeq = 0;
-  _recordingBufferTotalSize = 0;
-  _recordingDelay = 0;
-  _recordingDelayHWAndOS = 0;
-  // Make sure first call to update delay function will update delay
-  _recordingDelayMeasurementCounter = 9999;
-
+  _fineAudioBuffer->ResetRecord();
   if (!_playing) {
-    OSStatus result = AudioOutputUnitStart(_auVoiceProcessing);
+    OSStatus result = AudioOutputUnitStart(_vpioUnit);
     if (result != noErr) {
       LOG_F(LS_ERROR) << "AudioOutputUnitStart failed: " << result;
       return -1;
     }
   }
-  _recording = true;
+  rtc::AtomicOps::ReleaseStore(&_recording, 1);
   return 0;
 }
 
 int32_t AudioDeviceIOS::StopRecording() {
   LOGI() << "StopRecording";
-  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(_threadChecker.CalledOnValidThread());
   if (!_recIsInitialized || !_recording) {
     return 0;
   }
-
-  CriticalSectionScoped lock(&_critSect);
-
   if (!_playing) {
-    // Both playout and recording has stopped, shutdown the device.
     ShutdownPlayOrRecord();
   }
   _recIsInitialized = false;
-  _recording = false;
+  rtc::AtomicOps::ReleaseStore(&_recording, 0);
   return 0;
 }
 
@@ -391,240 +350,216 @@ int32_t AudioDeviceIOS::GetLoudspeakerStatus(bool& enabled) const {
 }
 
 int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const {
-  delayMS = _playoutDelay;
+  delayMS = kFixedPlayoutDelayEstimate;
   return 0;
 }
 
 int32_t AudioDeviceIOS::RecordingDelay(uint16_t& delayMS) const {
-  delayMS = _recordingDelay;
-  return 0;
-}
-
-int32_t AudioDeviceIOS::PlayoutBuffer(AudioDeviceModule::BufferType& type,
-                                      uint16_t& sizeMS) const {
-  type = AudioDeviceModule::kAdaptiveBufferSize;
-  sizeMS = _playoutDelay;
+  delayMS = kFixedRecordDelayEstimate;
   return 0;
 }
 
 int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const {
-  CHECK(playout_parameters_.is_valid());
-  DCHECK(thread_checker_.CalledOnValidThread());
-  *params = playout_parameters_;
+  DCHECK(_playoutParameters.is_valid());
+  DCHECK(_threadChecker.CalledOnValidThread());
+  *params = _playoutParameters;
   return 0;
 }
 
 int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const {
-  CHECK(record_parameters_.is_valid());
-  DCHECK(thread_checker_.CalledOnValidThread());
-  *params = record_parameters_;
+  DCHECK(_recordParameters.is_valid());
+  DCHECK(_threadChecker.CalledOnValidThread());
+  *params = _recordParameters;
   return 0;
 }
 
-// ============================================================================
-//                                 Private Methods
-// ============================================================================
-
-int32_t AudioDeviceIOS::InitPlayOrRecord() {
-  LOGI() << "AudioDeviceIOS::InitPlayOrRecord";
-  DCHECK(!_auVoiceProcessing);
-
-  OSStatus result = -1;
-
-  // Create Voice Processing Audio Unit
-  AudioComponentDescription desc;
-  AudioComponent comp;
-
-  desc.componentType = kAudioUnitType_Output;
-  desc.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
-  desc.componentManufacturer = kAudioUnitManufacturer_Apple;
-  desc.componentFlags = 0;
-  desc.componentFlagsMask = 0;
-
-  comp = AudioComponentFindNext(nullptr, &desc);
-  if (nullptr == comp) {
-    LOG_F(LS_ERROR) << "Could not find audio component for Audio Unit";
-    return -1;
-  }
-
-  result = AudioComponentInstanceNew(comp, &_auVoiceProcessing);
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to create Audio Unit instance: " << result;
-    return -1;
-  }
-
-  // TODO(henrika): I think we should set the preferred channel configuration
-  // in both directions as well to be safe.
+void AudioDeviceIOS::UpdateAudioDeviceBuffer() {
+  LOGI() << "UpdateAudioDevicebuffer";
+  // AttachAudioBuffer() is called at construction by the main class but check
+  // just in case.
+  DCHECK(_audioDeviceBuffer) << "AttachAudioBuffer must be called first";
+  // Inform the audio device buffer (ADB) about the new audio format.
+  _audioDeviceBuffer->SetPlayoutSampleRate(_playoutParameters.sample_rate());
+  _audioDeviceBuffer->SetPlayoutChannels(_playoutParameters.channels());
+  _audioDeviceBuffer->SetRecordingSampleRate(_recordParameters.sample_rate());
+  _audioDeviceBuffer->SetRecordingChannels(_recordParameters.channels());
+}
 
-  // Set preferred hardware sample rate to 16 kHz.
-  // TODO(henrika): improve this selection of sample rate. Why do we currently
-  // use a hard coded value? How can we fail and still continue?
-  NSError* error = nil;
+void AudioDeviceIOS::VerifyAudioParametersForActiveAudioSession() {

[tkchin_webrtc, 2015/09/01 20:54:50]

This does a bit more than verifying parameters, consider naming it something
like SetupAudioBuffersForActiveAudioSession

[henrika_webrtc, 2015/09/03 13:44:41]

Done.

+  LOGI() << "VerifyAudioParametersForActiveAudioSession";
   AVAudioSession* session = [AVAudioSession sharedInstance];
-  Float64 preferredSampleRate(playout_parameters_.sample_rate());
-  [session setPreferredSampleRate:preferredSampleRate error:&error];
-  if (error != nil) {
-    const char* errorString = [[error localizedDescription] UTF8String];
-    LOG_F(LS_ERROR) << "setPreferredSampleRate failed: " << errorString;
-  }
-
-  // TODO(henrika): we can reduce latency by setting the IOBufferDuration
-  // here. Default size for 16kHz is 0.016 sec or 16 msec on an iPhone 6.
-
-  // Activate the audio session.
-  ActivateAudioSession(session, true);
-
-  UInt32 enableIO = 1;
-  result = AudioUnitSetProperty(_auVoiceProcessing,
-                                kAudioOutputUnitProperty_EnableIO,
-                                kAudioUnitScope_Input,
-                                1,  // input bus
-                                &enableIO, sizeof(enableIO));
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to enable IO on input: " << result;
-  }
-
-  result = AudioUnitSetProperty(_auVoiceProcessing,
-                                kAudioOutputUnitProperty_EnableIO,
-                                kAudioUnitScope_Output,
-                                0,  // output bus
-                                &enableIO, sizeof(enableIO));
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to enable IO on output: " << result;
-  }
+  // Verify the current values once the audio session has been activated.

[tkchin_webrtc, 2015/09/01 20:54:50]

minor minor nit: line break after comment + block for comment to make things
easier to read in this function

e.g.
// comment about XYZ
XYZ

// comment about PQR
PQR

[henrika_webrtc, 2015/09/03 13:44:41]

Done.

+  LOG(LS_INFO) << " sample rate: " << session.sampleRate;
+  LOG(LS_INFO) << " IO buffer duration: " << session.IOBufferDuration;
+  LOG(LS_INFO) << " output channels: " << session.outputNumberOfChannels;
+  LOG(LS_INFO) << " input channels: " << session.inputNumberOfChannels;
+  LOG(LS_INFO) << " output latency: " << session.outputLatency;
+  LOG(LS_INFO) << " input latency: " << session.inputLatency;
+  // TODO(henrika): would it be possible to continue even for the case when
+  // we are unable to set the preferred hardware sample rate.
+  CHECK_EQ(static_cast<int>(session.sampleRate),

[tkchin_webrtc, 2015/09/01 20:54:50]

It seems overkill to crash if we don't get what we need. It seems better to just
have no audio. (Video may still be usable).

Leaving as TODO is fine though.

[henrika_webrtc, 2015/09/03 13:44:41]

Agree. Let me keep the TODO and fix this in combination with adding support for
BT. I'll try to ensure that we fall back to the rate that we got. I think it
will be possible with the new structure. I.e., audio should work even when 48 is
not supported. 

[tkchin_webrtc, 2015/09/04 05:29:16]

For next CL, I would just have a simple one that removes the crashing behavior.
If it's easy to add the BT stuff to it too then sure, but let's get the
non-crashing behavior back in ASAP.

[henrika_webrtc, 2015/09/04 09:51:18]

I'll add the non-crashing parts to this CL and BT support if I can make it
simple. If not, add in a follow-up.

+           _playoutParameters.sample_rate())
+      << "Failed to enable an audio session with the preferred sample rate!";
+  // At this stage, we also know the exact IO buffer duration and can add
+  // that info to the existing audio parameters where it is converted into
+  // number of audio frames.
+  // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz.
+  // Hence, 128 is the size we expect to see in upcoming render callbacks.
+  _playoutParameters.reset(_playoutParameters.sample_rate(),
+                           _playoutParameters.channels(),
+                           session.IOBufferDuration);
+  DCHECK(_playoutParameters.is_complete());
+  _recordParameters.reset(_recordParameters.sample_rate(),
+                          _recordParameters.channels(),
+                          session.IOBufferDuration);
+  DCHECK(_recordParameters.is_complete());
+  LOG(LS_INFO) << " frames per I/O buffer: "
+               << _playoutParameters.frames_per_buffer();
+  LOG(LS_INFO) << " bytes per I/O buffer: "
+               << _playoutParameters.GetBytesPerBuffer();
+  DCHECK_EQ(_playoutParameters.GetBytesPerBuffer(),
+            _recordParameters.GetBytesPerBuffer());
+  // Create a modified audio buffer class which allows us to ask for any number
+  // of samples (and not only multiple of 10ms) to match the native audio unit
+  // buffer size.
+  DCHECK(_audioDeviceBuffer);
+  _fineAudioBuffer.reset(new FineAudioBuffer(
+      _audioDeviceBuffer, _playoutParameters.GetBytesPerBuffer(),
+      _playoutParameters.sample_rate()));
+  // The extra/temporary playoutbuffer must be of this size to avoid
+  // unnecessary memcpy while caching data between successive callbacks.
+  const int requiredPlayoutBufferSize =
+      _fineAudioBuffer->RequiredPlayoutBufferSizeBytes();
+  LOG(LS_INFO) << " required playout buffer size: "
+               << requiredPlayoutBufferSize;
+  _playoutAudioBuffer.reset(new SInt8[requiredPlayoutBufferSize]);
+  // Allocate AudioBuffers to be used as storage for the received audio.
+  // The AudioBufferList structure works as a placeholder for the
+  // AudioBuffer structure, which holds a pointer to the actual data buffer
+  // in |_recordAudioBuffer|. Recorded audio will be rendered into this memory
+  // at each input callback when calling AudioUnitRender().
+  const int dataByteSize = _recordParameters.GetBytesPerBuffer();
+  _recordAudioBuffer.reset(new SInt8[dataByteSize]);
+  _audioRecordBufferList.mNumberBuffers = 1;
+  AudioBuffer* audioBuffer = &_audioRecordBufferList.mBuffers[0];
+  audioBuffer->mNumberChannels = _recordParameters.channels();
+  audioBuffer->mDataByteSize = dataByteSize;
+  audioBuffer->mData = _recordAudioBuffer.get();
+}
 
+void AudioDeviceIOS::SetupAndInitializeVoiceProcessingAudioUnit() {
+  LOGI() << "SetupAndInitializeVoiceProcessingAudioUnit";
+  DCHECK(!_vpioUnit);
+  // Create an audio component description to identify the Voice-Processing
+  // I/O audio unit.
+  AudioComponentDescription vpioUnitDescription;
+  vpioUnitDescription.componentType = kAudioUnitType_Output;
+  vpioUnitDescription.componentSubType = kAudioUnitSubType_VoiceProcessingIO;
+  vpioUnitDescription.componentManufacturer = kAudioUnitManufacturer_Apple;
+  vpioUnitDescription.componentFlags = 0;
+  vpioUnitDescription.componentFlagsMask = 0;
+  // Obtain an audio unit instance given the description.
+  AudioComponent foundVpioUnitRef =
+      AudioComponentFindNext(nullptr, &vpioUnitDescription);
+  // Create a Voice-Processing IO audio unit.
+  CHECK_EQ(AudioComponentInstanceNew(foundVpioUnitRef, &_vpioUnit), noErr)

[tkchin_webrtc, 2015/09/01 20:54:50]

ditto system call failures shouldn't crash applications, here and below

probably return bool

[henrika_webrtc, 2015/09/03 13:44:41]

OK; I will fix that. Must leave soon but will fix it tomorrow.

And, I assume you want me to add new-line before new comments here as well.
Correct?

+      << "Failed to create a VoiceProcessingIO audio unit";
+  // A VP I/O unit's bus 1 connects to input hardware (microphone). Enable
+  // input on the input scope of the input element.
+  AudioUnitElement inputBus = 1;
+  UInt32 enableInput = 1;
+  CHECK_EQ(AudioUnitSetProperty(_vpioUnit, kAudioOutputUnitProperty_EnableIO,
+                                kAudioUnitScope_Input, inputBus, &enableInput,
+                                sizeof(enableInput)),
+           noErr)
+      << "Failed to enable input on input scope of input element";
+  // A VP I/O unit's bus 0 connects to output hardware (speaker). Enable
+  // output on the output scope of the output element.
+  AudioUnitElement outputBus = 0;
+  UInt32 enableOutput = 1;
+  CHECK_EQ(AudioUnitSetProperty(_vpioUnit, kAudioOutputUnitProperty_EnableIO,
+                                kAudioUnitScope_Output, outputBus,
+                                &enableOutput, sizeof(enableOutput)),
+           noErr)
+      << "Failed to enable output on output scope of output element";
+  // Set the application formats for input and output:
+  // - use same format in both directions
+  // - avoid resampling in the I/O unit by using the hardware sample rate
+  // - linear PCM => noncompressed audio data format with one frame per packet
+  // - no need to specify interleaving since only mono is supported
+  AudioStreamBasicDescription applicationFormat = {0};
+  UInt32 size = sizeof(applicationFormat);
+  DCHECK_EQ(_playoutParameters.sample_rate(), _recordParameters.sample_rate());
+  DCHECK_EQ(1, kPreferredNumberOfChannels);
+  applicationFormat.mSampleRate = _playoutParameters.sample_rate();
+  applicationFormat.mFormatID = kAudioFormatLinearPCM;
+  applicationFormat.mFormatFlags =
+      kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked;
+  applicationFormat.mBytesPerPacket = kBytesPerSample;
+  applicationFormat.mFramesPerPacket = 1;  // uncompressed
+  applicationFormat.mBytesPerFrame = kBytesPerSample;
+  applicationFormat.mChannelsPerFrame = kPreferredNumberOfChannels;
+  applicationFormat.mBitsPerChannel = 8 * kBytesPerSample;
+#if !defined(NDEBUG)
+  LogABSD(applicationFormat);
+#endif
+  // Set the application format on the output scope of the input element/bus.
+  CHECK_EQ(AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_StreamFormat,
+                                kAudioUnitScope_Output, inputBus,
+                                &applicationFormat, size),
+
+           noErr)
+      << "Failed to set application format on output scope of input element";
+  // Set the application format on the input scope of the output element/bus.
+  CHECK_EQ(AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_StreamFormat,
+                                kAudioUnitScope_Input, outputBus,
+                                &applicationFormat, size),
+           noErr)
+      << "Failed to set application format on input scope of output element";
+  // Specify the callback function that provides audio samples to the audio
+  // unit.
+  AURenderCallbackStruct renderCallback;
+  renderCallback.inputProc = GetPlayoutData;
+  renderCallback.inputProcRefCon = this;
+  CHECK_EQ(AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_SetRenderCallback,
+                                kAudioUnitScope_Input, outputBus,
+                                &renderCallback, sizeof(renderCallback)),
+           noErr)
+      << "Failed to specify the render callback on the output element";
   // Disable AU buffer allocation for the recorder, we allocate our own.
-  // TODO(henrika): understand this part better.
+  // TODO(henrika): not sure that it actually saves resource to make this call.
   UInt32 flag = 0;
-  result = AudioUnitSetProperty(_auVoiceProcessing,
-                                kAudioUnitProperty_ShouldAllocateBuffer,
-                                kAudioUnitScope_Output, 1, &flag, sizeof(flag));
-  if (0 != result) {
-    LOG_F(LS_WARNING) << "Failed to disable AU buffer allocation: " << result;
-    // Should work anyway
-  }
-
-  // Set recording callback.
-  AURenderCallbackStruct auCbS;
-  memset(&auCbS, 0, sizeof(auCbS));
-  auCbS.inputProc = RecordProcess;
-  auCbS.inputProcRefCon = this;
-  result = AudioUnitSetProperty(
-      _auVoiceProcessing, kAudioOutputUnitProperty_SetInputCallback,
-      kAudioUnitScope_Global, 1, &auCbS, sizeof(auCbS));
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to set AU record callback: " << result;
-  }
-
-  // Set playout callback.
-  memset(&auCbS, 0, sizeof(auCbS));
-  auCbS.inputProc = PlayoutProcess;
-  auCbS.inputProcRefCon = this;
-  result = AudioUnitSetProperty(
-      _auVoiceProcessing, kAudioUnitProperty_SetRenderCallback,
-      kAudioUnitScope_Global, 0, &auCbS, sizeof(auCbS));
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to set AU output callback: " << result;
-  }
-
-  // Get stream format for out/0
-  AudioStreamBasicDescription playoutDesc;
-  UInt32 size = sizeof(playoutDesc);
-  result =
-      AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat,
-                           kAudioUnitScope_Output, 0, &playoutDesc, &size);
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to get AU output stream format: " << result;
-  }
-
-  playoutDesc.mSampleRate = preferredSampleRate;
-  LOG(LS_INFO) << "Audio Unit playout opened in sampling rate: "
-               << playoutDesc.mSampleRate;
-
-  // Store the sampling frequency to use towards the Audio Device Buffer
-  // todo: Add 48 kHz (increase buffer sizes). Other fs?
-  // TODO(henrika): Figure out if we really need this complex handling.
-  if ((playoutDesc.mSampleRate > 44090.0) &&
-      (playoutDesc.mSampleRate < 44110.0)) {
-    _adbSampFreq = 44100;
-  } else if ((playoutDesc.mSampleRate > 15990.0) &&
-             (playoutDesc.mSampleRate < 16010.0)) {
-    _adbSampFreq = 16000;
-  } else if ((playoutDesc.mSampleRate > 7990.0) &&
-             (playoutDesc.mSampleRate < 8010.0)) {
-    _adbSampFreq = 8000;
-  } else {
-    _adbSampFreq = 0;
-    FATAL() << "Invalid sample rate";
-  }
-
-  // Set the audio device buffer sampling rates (use same for play and record).
-  // TODO(henrika): this is not a good place to set these things up.
-  DCHECK(audio_device_buffer_);
-  DCHECK_EQ(_adbSampFreq, playout_parameters_.sample_rate());
-  audio_device_buffer_->SetRecordingSampleRate(_adbSampFreq);
-  audio_device_buffer_->SetPlayoutSampleRate(_adbSampFreq);
-
-  // Set stream format for out/0.
-  playoutDesc.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger |
-                             kLinearPCMFormatFlagIsPacked |
-                             kLinearPCMFormatFlagIsNonInterleaved;
-  playoutDesc.mBytesPerPacket = 2;
-  playoutDesc.mFramesPerPacket = 1;
-  playoutDesc.mBytesPerFrame = 2;
-  playoutDesc.mChannelsPerFrame = 1;
-  playoutDesc.mBitsPerChannel = 16;
-  result =
-      AudioUnitSetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat,
-                           kAudioUnitScope_Input, 0, &playoutDesc, size);
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to set AU stream format for out/0";
-  }
-
-  // Get stream format for in/1.
-  AudioStreamBasicDescription recordingDesc;
-  size = sizeof(recordingDesc);
-  result =
-      AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat,
-                           kAudioUnitScope_Input, 1, &recordingDesc, &size);
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to get AU stream format for in/1";
-  }
-
-  recordingDesc.mSampleRate = preferredSampleRate;
-  LOG(LS_INFO) << "Audio Unit recording opened in sampling rate: "
-               << recordingDesc.mSampleRate;
-
-  // Set stream format for out/1 (use same sampling frequency as for in/1).
-  recordingDesc.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger |
-                               kLinearPCMFormatFlagIsPacked |
-                               kLinearPCMFormatFlagIsNonInterleaved;
-  recordingDesc.mBytesPerPacket = 2;
-  recordingDesc.mFramesPerPacket = 1;
-  recordingDesc.mBytesPerFrame = 2;
-  recordingDesc.mChannelsPerFrame = 1;
-  recordingDesc.mBitsPerChannel = 16;
-  result =
-      AudioUnitSetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat,
-                           kAudioUnitScope_Output, 1, &recordingDesc, size);
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "Failed to set AU stream format for out/1";
-  }
-
-  // Initialize here already to be able to get/set stream properties.
-  result = AudioUnitInitialize(_auVoiceProcessing);
-  if (0 != result) {
-    LOG_F(LS_ERROR) << "AudioUnitInitialize failed: " << result;
-  }
-
-  // Get hardware sample rate for logging (see if we get what we asked for).
-  // TODO(henrika): what if we don't get what we ask for?
-  double sampleRate = session.sampleRate;
-  LOG(LS_INFO) << "Current HW sample rate is: " << sampleRate
-               << ", ADB sample rate is: " << _adbSampFreq;
-  LOG(LS_INFO) << "Current HW IO buffer size is: " <<
-      [session IOBufferDuration];
+  CHECK_EQ(AudioUnitSetProperty(
+               _vpioUnit, kAudioUnitProperty_ShouldAllocateBuffer,
+               kAudioUnitScope_Output, inputBus, &flag, sizeof(flag)),
+           noErr)
+      << "Failed to disable buffer allocation on the input element";
+  // Specify the callback to be called by the I/O thread to us when input audio
+  // is available. The recorded samples can then be obtained by calling the
+  // AudioUnitRender() method.
+  AURenderCallbackStruct inputCallback;
+  inputCallback.inputProc = RecordedDataIsAvailable;
+  inputCallback.inputProcRefCon = this;
+  CHECK_EQ(
+      AudioUnitSetProperty(_vpioUnit, kAudioOutputUnitProperty_SetInputCallback,
+                           kAudioUnitScope_Global, inputBus, &inputCallback,
+                           sizeof(inputCallback)),
+      noErr)
+      << "Failed to specify the input callback on the input element";
+  // Initialize the Voice-Processing I/O unit instance.
+  CHECK_EQ(AudioUnitInitialize(_vpioUnit), noErr)
+      << "Failed to initialize the Voice-Processing I/O unit";
+}
 
+bool AudioDeviceIOS::InitPlayOrRecord() {
+  LOGI() << "InitPlayOrRecord";
+  AVAudioSession* session = [AVAudioSession sharedInstance];
+  // Activate the audio session and ask for a set of preferred audio parameters.
+  ActivateAudioSession(session, true);
+  // Ensure that we got what what we asked for in our active audio session.
+  VerifyAudioParametersForActiveAudioSession();
+  // Create, setup and initialize a new Voice-Processing I/O unit.
+  SetupAndInitializeVoiceProcessingAudioUnit();
   // Listen to audio interruptions.
   // TODO(henrika): learn this area better.
   NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
@@ -655,8 +590,8 @@ int32_t AudioDeviceIOS::InitPlayOrRecord() {
                     // Post interruption the audio unit render callbacks don't
                     // automatically continue, so we restart the unit manually
                     // here.
-                    AudioOutputUnitStop(_auVoiceProcessing);
-                    AudioOutputUnitStart(_auVoiceProcessing);
+                    AudioOutputUnitStop(_vpioUnit);
+                    AudioOutputUnitStart(_vpioUnit);
                     break;
                   }
                 }
@@ -665,13 +600,11 @@ int32_t AudioDeviceIOS::InitPlayOrRecord() {
   // void* instead of an id because header is included in other pure C++
   // files.
   _audioInterruptionObserver = (__bridge_retained void*)observer;
-
-  return 0;
+  return true;
 }
 
-int32_t AudioDeviceIOS::ShutdownPlayOrRecord() {
+bool AudioDeviceIOS::ShutdownPlayOrRecord() {
   LOGI() << "ShutdownPlayOrRecord";
-
   if (_audioInterruptionObserver != nullptr) {
     NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
     // Transfer ownership of observer back to ARC, which will dealloc the
@@ -680,375 +613,112 @@ int32_t AudioDeviceIOS::ShutdownPlayOrRecord() {
     [center removeObserver:observer];
     _audioInterruptionObserver = nullptr;
   }
-
-  // Close and delete AU.
+  // Close and delete the voice-processing I/O unit.
   OSStatus result = -1;
-  if (nullptr != _auVoiceProcessing) {
-    result = AudioOutputUnitStop(_auVoiceProcessing);
-    if (0 != result) {
+  if (nullptr != _vpioUnit) {
+    result = AudioOutputUnitStop(_vpioUnit);
+    if (result != noErr) {
       LOG_F(LS_ERROR) << "AudioOutputUnitStop failed: " << result;
     }
-    result = AudioComponentInstanceDispose(_auVoiceProcessing);
-    if (0 != result) {
+    result = AudioComponentInstanceDispose(_vpioUnit);
+    if (result != noErr) {
       LOG_F(LS_ERROR) << "AudioComponentInstanceDispose failed: " << result;
     }
-    _auVoiceProcessing = nullptr;
+    _vpioUnit = nullptr;
   }
-
   // All I/O should be stopped or paused prior to deactivating the audio
   // session, hence we deactivate as last action.
   AVAudioSession* session = [AVAudioSession sharedInstance];
   ActivateAudioSession(session, false);
-  return 0;
+  return true;
 }
 
-// ============================================================================
-//                                  Thread Methods
-// ============================================================================
-
-OSStatus AudioDeviceIOS::RecordProcess(
+OSStatus AudioDeviceIOS::RecordedDataIsAvailable(
     void* inRefCon,
     AudioUnitRenderActionFlags* ioActionFlags,
     const AudioTimeStamp* inTimeStamp,
     UInt32 inBusNumber,
     UInt32 inNumberFrames,
     AudioBufferList* ioData) {
-  AudioDeviceIOS* ptrThis = static_cast<AudioDeviceIOS*>(inRefCon);
-  return ptrThis->RecordProcessImpl(ioActionFlags, inTimeStamp, inBusNumber,
-                                    inNumberFrames);
+  DCHECK_EQ(1u, inBusNumber);
+  DCHECK(!ioData);  // no buffer should be allocated for input at this stage
+  AudioDeviceIOS* thiz = static_cast<AudioDeviceIOS*>(inRefCon);

[tkchin_webrtc, 2015/09/01 20:54:50]

nit: would just call it audio_device_ios instead of thiz

[henrika_webrtc, 2015/09/03 13:44:41]

Done.

+  return thiz->OnRecordedDataIsAvailable(ioActionFlags, inTimeStamp,
+                                         inBusNumber, inNumberFrames);
 }
 
-OSStatus AudioDeviceIOS::RecordProcessImpl(
+OSStatus AudioDeviceIOS::OnRecordedDataIsAvailable(
     AudioUnitRenderActionFlags* ioActionFlags,
     const AudioTimeStamp* inTimeStamp,
-    uint32_t inBusNumber,
-    uint32_t inNumberFrames) {
-  // Setup some basic stuff
-  // Use temp buffer not to lock up recording buffer more than necessary
-  // todo: Make dataTmp a member variable with static size that holds
-  //       max possible frames?
-  int16_t* dataTmp = new int16_t[inNumberFrames];
-  memset(dataTmp, 0, 2 * inNumberFrames);
-
-  AudioBufferList abList;
-  abList.mNumberBuffers = 1;
-  abList.mBuffers[0].mData = dataTmp;
-  abList.mBuffers[0].mDataByteSize = 2 * inNumberFrames;  // 2 bytes/sample
-  abList.mBuffers[0].mNumberChannels = 1;
-
-  // Get data from mic
-  OSStatus res = AudioUnitRender(_auVoiceProcessing, ioActionFlags, inTimeStamp,
-                                 inBusNumber, inNumberFrames, &abList);
-  if (res != 0) {
-    // TODO(henrika): improve error handling.
-    delete[] dataTmp;
-    return 0;
-  }
-
-  if (_recording) {
-    // Insert all data in temp buffer into recording buffers
-    // There is zero or one buffer partially full at any given time,
-    // all others are full or empty
-    // Full means filled with noSamp10ms samples.
-
-    const unsigned int noSamp10ms = _adbSampFreq / 100;
-    unsigned int dataPos = 0;
-    uint16_t bufPos = 0;
-    int16_t insertPos = -1;
-    unsigned int nCopy = 0;  // Number of samples to copy
-
-    while (dataPos < inNumberFrames) {
-      // Loop over all recording buffers or
-      // until we find the partially full buffer
-      // First choice is to insert into partially full buffer,
-      // second choice is to insert into empty buffer
-      bufPos = 0;
-      insertPos = -1;
-      nCopy = 0;
-      while (bufPos < N_REC_BUFFERS) {
-        if ((_recordingLength[bufPos] > 0) &&
-            (_recordingLength[bufPos] < noSamp10ms)) {
-          // Found the partially full buffer
-          insertPos = static_cast<int16_t>(bufPos);
-          // Don't need to search more, quit loop
-          bufPos = N_REC_BUFFERS;
-        } else if ((-1 == insertPos) && (0 == _recordingLength[bufPos])) {
-          // Found an empty buffer
-          insertPos = static_cast<int16_t>(bufPos);
-        }
-        ++bufPos;
-      }
-
-      // Insert data into buffer
-      if (insertPos > -1) {
-        // We found a non-full buffer, copy data to it
-        unsigned int dataToCopy = inNumberFrames - dataPos;
-        unsigned int currentRecLen = _recordingLength[insertPos];
-        unsigned int roomInBuffer = noSamp10ms - currentRecLen;
-        nCopy = (dataToCopy < roomInBuffer ? dataToCopy : roomInBuffer);
-
-        memcpy(&_recordingBuffer[insertPos][currentRecLen], &dataTmp[dataPos],
-               nCopy * sizeof(int16_t));
-        if (0 == currentRecLen) {
-          _recordingSeqNumber[insertPos] = _recordingCurrentSeq;
-          ++_recordingCurrentSeq;
-        }
-        _recordingBufferTotalSize += nCopy;
-        // Has to be done last to avoid interrupt problems between threads.
-        _recordingLength[insertPos] += nCopy;
-        dataPos += nCopy;
-      } else {
-        // Didn't find a non-full buffer
-        // TODO(henrika): improve error handling
-        dataPos = inNumberFrames;  // Don't try to insert more
-      }
-    }
-  }
-  delete[] dataTmp;
-  return 0;
+    UInt32 inBusNumber,
+    UInt32 inNumberFrames) {
+  DCHECK_EQ(_recordParameters.frames_per_buffer(), inNumberFrames);
+  OSStatus result = noErr;
+  // Simply return if recording is not enabled.
+  if (!rtc::AtomicOps::AcquireLoad(&_recording))
+    return result;
+  // Obtain the recorded audio samples by initiating a rendering cycle.
+  // Since it happens on he input bus, the |ioData| parameter is a reference

[tkchin_webrtc, 2015/09/01 20:54:50]

nit: the input bus

[henrika_webrtc, 2015/09/03 13:44:41]

Done.

+  // to the preallocated audio buffer list that the audio unit renders into.
+  // TODO(henrika): should error handling be improved?
+  AudioBufferList* ioData = &_audioRecordBufferList;
+  result = AudioUnitRender(_vpioUnit, ioActionFlags, inTimeStamp, inBusNumber,
+                           inNumberFrames, ioData);
+  if (result != noErr) {
+    LOG_F(LS_ERROR) << "AudioOutputUnitStart failed: " << result;

[tkchin_webrtc, 2015/09/01 20:54:50]

Is this only called during start? I read it as a callback that happens each time
there is data.

[henrika_webrtc, 2015/09/03 13:44:41]

It is not a callback on the capture side. The callback is
RecordedDataIsAvailable() and once we are there we can ask for the actual
recorded data using AudioUnitRender(). It feels as if the scheme is written for
playout only initially and when recording was added, the naming did not become
perfect. But no, it is not a callback.

[tkchin_webrtc, 2015/09/04 05:29:16]

Ah I gotcha.

[henrika_webrtc, 2015/09/04 09:51:18]

Acknowledged.

+    return result;
+  }
+  // Get a pointer to the recorded audio and send it to the WebRTC ADB.
+  // Use the FineAudioBuffer instance to convert between native buffer size
+  // and the 10ms buffer size used by WebRTC.
+  const UInt32 dataSizeInBytes = ioData->mBuffers[0].mDataByteSize;
+  CHECK_EQ(dataSizeInBytes / kBytesPerSample, inNumberFrames);
+  SInt8* data = static_cast<SInt8*>(ioData->mBuffers[0].mData);
+  _fineAudioBuffer->DeliverRecordedData(data, dataSizeInBytes,

[tkchin_webrtc, 2015/09/01 20:54:50]

What threading requirements are there for DeliverRecordedData? Is there an
eventual thread dispatch / copy to something else for encoding?

[henrika_webrtc, 2015/09/03 13:44:41]

The data is copied to an internal buffer inside the platform independent code in
AudioDeviceBuffer. FineAudioBuffer will ensure that this copy is sent to WebRTC
on this thread (the high-prio IO thread). I must admit that I don't know the
exact details of the latest APM version and the threading model it uses, but I
assume it has a separate encoding and sending thread. I must ask around to
improve this feedback.

+                                        kFixedPlayoutDelayEstimate,
+                                        kFixedRecordDelayEstimate);
+  return noErr;
 }
 
-OSStatus AudioDeviceIOS::PlayoutProcess(
+OSStatus AudioDeviceIOS::GetPlayoutData(
     void* inRefCon,
     AudioUnitRenderActionFlags* ioActionFlags,
     const AudioTimeStamp* inTimeStamp,
     UInt32 inBusNumber,
     UInt32 inNumberFrames,
     AudioBufferList* ioData) {
-  AudioDeviceIOS* ptrThis = static_cast<AudioDeviceIOS*>(inRefCon);
-  return ptrThis->PlayoutProcessImpl(inNumberFrames, ioData);
-}
-
-OSStatus AudioDeviceIOS::PlayoutProcessImpl(uint32_t inNumberFrames,
-                                            AudioBufferList* ioData) {
-  int16_t* data = static_cast<int16_t*>(ioData->mBuffers[0].mData);
-  unsigned int dataSizeBytes = ioData->mBuffers[0].mDataByteSize;
-  unsigned int dataSize = dataSizeBytes / 2;  // Number of samples
-  CHECK_EQ(dataSize, inNumberFrames);
-  memset(data, 0, dataSizeBytes);  // Start with empty buffer
-
-  // Get playout data from Audio Device Buffer
-
-  if (_playing) {
-    unsigned int noSamp10ms = _adbSampFreq / 100;
-    // todo: Member variable and allocate when samp freq is determined
-    int16_t* dataTmp = new int16_t[noSamp10ms];
-    memset(dataTmp, 0, 2 * noSamp10ms);
-    unsigned int dataPos = 0;
-    int noSamplesOut = 0;
-    unsigned int nCopy = 0;
-
-    // First insert data from playout buffer if any
-    if (_playoutBufferUsed > 0) {
-      nCopy = (dataSize < _playoutBufferUsed) ? dataSize : _playoutBufferUsed;
-      DCHECK_EQ(nCopy, _playoutBufferUsed);
-      memcpy(data, _playoutBuffer, 2 * nCopy);
-      dataPos = nCopy;
-      memset(_playoutBuffer, 0, sizeof(_playoutBuffer));
-      _playoutBufferUsed = 0;
-    }
-
-    // Now get the rest from Audio Device Buffer.
-    while (dataPos < dataSize) {
-      // Update playout delay
-      UpdatePlayoutDelay();
-
-      // Ask for new PCM data to be played out using the AudioDeviceBuffer
-      noSamplesOut = audio_device_buffer_->RequestPlayoutData(noSamp10ms);
-
-      // Get data from Audio Device Buffer
-      noSamplesOut = audio_device_buffer_->GetPlayoutData(
-          reinterpret_cast<int8_t*>(dataTmp));
-      CHECK_EQ(noSamp10ms, (unsigned int)noSamplesOut);
-
-      // Insert as much as fits in data buffer
-      nCopy =
-          (dataSize - dataPos) > noSamp10ms ? noSamp10ms : (dataSize - dataPos);
-      memcpy(&data[dataPos], dataTmp, 2 * nCopy);
-
-      // Save rest in playout buffer if any
-      if (nCopy < noSamp10ms) {
-        memcpy(_playoutBuffer, &dataTmp[nCopy], 2 * (noSamp10ms - nCopy));
-        _playoutBufferUsed = noSamp10ms - nCopy;
-      }
-
-      // Update loop/index counter, if we copied less than noSamp10ms
-      // samples we shall quit loop anyway
-      dataPos += noSamp10ms;
-    }
-    delete[] dataTmp;
-  }
-  return 0;
-}
-
-// TODO(henrika): can either be removed or simplified.
-void AudioDeviceIOS::UpdatePlayoutDelay() {
-  ++_playoutDelayMeasurementCounter;
-
-  if (_playoutDelayMeasurementCounter >= 100) {
-    // Update HW and OS delay every second, unlikely to change
-
-    // Since this is eventually rounded to integral ms, add 0.5ms
-    // here to get round-to-nearest-int behavior instead of
-    // truncation.
-    double totalDelaySeconds = 0.0005;
-
-    // HW output latency
-    AVAudioSession* session = [AVAudioSession sharedInstance];
-    double latency = session.outputLatency;
-    assert(latency >= 0);
-    totalDelaySeconds += latency;
-
-    // HW buffer duration
-    double ioBufferDuration = session.IOBufferDuration;
-    assert(ioBufferDuration >= 0);
-    totalDelaySeconds += ioBufferDuration;
-
-    // AU latency
-    Float64 f64(0);
-    UInt32 size = sizeof(f64);
-    OSStatus result =
-        AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_Latency,
-                             kAudioUnitScope_Global, 0, &f64, &size);
-    if (0 != result) {
-      LOG_F(LS_ERROR) << "AU latency error: " << result;
-    }
-    assert(f64 >= 0);
-    totalDelaySeconds += f64;
-
-    // To ms
-    _playoutDelay = static_cast<uint32_t>(totalDelaySeconds * 1000);
-
-    // Reset counter
-    _playoutDelayMeasurementCounter = 0;
-  }
-
-  // todo: Add playout buffer?
-}
-
-void AudioDeviceIOS::UpdateRecordingDelay() {
-  ++_recordingDelayMeasurementCounter;
-
-  if (_recordingDelayMeasurementCounter >= 100) {
-    // Update HW and OS delay every second, unlikely to change
-
-    // Since this is eventually rounded to integral ms, add 0.5ms
-    // here to get round-to-nearest-int behavior instead of
-    // truncation.
-    double totalDelaySeconds = 0.0005;
-
-    // HW input latency
-    AVAudioSession* session = [AVAudioSession sharedInstance];
-    double latency = session.inputLatency;
-    assert(latency >= 0);
-    totalDelaySeconds += latency;
-
-    // HW buffer duration
-    double ioBufferDuration = session.IOBufferDuration;
-    assert(ioBufferDuration >= 0);
-    totalDelaySeconds += ioBufferDuration;
-
-    // AU latency
-    Float64 f64(0);
-    UInt32 size = sizeof(f64);
-    OSStatus result =
-        AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_Latency,
-                             kAudioUnitScope_Global, 0, &f64, &size);
-    if (0 != result) {
-      LOG_F(LS_ERROR) << "AU latency error: " << result;
-    }
-    assert(f64 >= 0);
-    totalDelaySeconds += f64;
-
-    // To ms
-    _recordingDelayHWAndOS = static_cast<uint32_t>(totalDelaySeconds / 1000);
-
-    // Reset counter
-    _recordingDelayMeasurementCounter = 0;
-  }
-
-  _recordingDelay = _recordingDelayHWAndOS;
-
-  // ADB recording buffer size, update every time
-  // Don't count the one next 10 ms to be sent, then convert samples => ms
-  const uint32_t noSamp10ms = _adbSampFreq / 100;
-  if (_recordingBufferTotalSize > noSamp10ms) {
-    _recordingDelay +=
-        (_recordingBufferTotalSize - noSamp10ms) / (_adbSampFreq / 1000);
-  }
+  DCHECK_EQ(0u, inBusNumber);
+  DCHECK(ioData);
+  AudioDeviceIOS* thiz = static_cast<AudioDeviceIOS*>(inRefCon);
+  return thiz->OnGetPlayoutData(ioActionFlags, inNumberFrames, ioData);

[tkchin_webrtc, 2015/09/01 20:54:50]

ditto audio_device_ios

[henrika_webrtc, 2015/09/03 13:44:41]

Done.

 }
 
-bool AudioDeviceIOS::RunCapture(void* ptrThis) {
-  return static_cast<AudioDeviceIOS*>(ptrThis)->CaptureWorkerThread();
-}
-
-bool AudioDeviceIOS::CaptureWorkerThread() {
-  if (_recording) {
-    int bufPos = 0;
-    unsigned int lowestSeq = 0;
-    int lowestSeqBufPos = 0;
-    bool foundBuf = true;
-    const unsigned int noSamp10ms = _adbSampFreq / 100;
-
-    while (foundBuf) {
-      // Check if we have any buffer with data to insert
-      // into the Audio Device Buffer,
-      // and find the one with the lowest seq number
-      foundBuf = false;
-      for (bufPos = 0; bufPos < N_REC_BUFFERS; ++bufPos) {
-        if (noSamp10ms == _recordingLength[bufPos]) {
-          if (!foundBuf) {
-            lowestSeq = _recordingSeqNumber[bufPos];
-            lowestSeqBufPos = bufPos;
-            foundBuf = true;
-          } else if (_recordingSeqNumber[bufPos] < lowestSeq) {
-            lowestSeq = _recordingSeqNumber[bufPos];
-            lowestSeqBufPos = bufPos;
-          }
-        }
-      }
-
-      // Insert data into the Audio Device Buffer if found any
-      if (foundBuf) {
-        // Update recording delay
-        UpdateRecordingDelay();
-
-        // Set the recorded buffer
-        audio_device_buffer_->SetRecordedBuffer(
-            reinterpret_cast<int8_t*>(_recordingBuffer[lowestSeqBufPos]),
-            _recordingLength[lowestSeqBufPos]);
-
-        // Don't need to set the current mic level in ADB since we only
-        // support digital AGC,
-        // and besides we cannot get or set the IOS mic level anyway.
-
-        // Set VQE info, use clockdrift == 0
-        audio_device_buffer_->SetVQEData(_playoutDelay, _recordingDelay, 0);
-
-        // Deliver recorded samples at specified sample rate, mic level
-        // etc. to the observer using callback
-        audio_device_buffer_->DeliverRecordedData();
-
-        // Make buffer available
-        _recordingSeqNumber[lowestSeqBufPos] = 0;
-        _recordingBufferTotalSize -= _recordingLength[lowestSeqBufPos];
-        // Must be done last to avoid interrupt problems between threads
-        _recordingLength[lowestSeqBufPos] = 0;
-      }
-    }
-  }
-
-  {
-    // Normal case
-    // Sleep thread (5ms) to let other threads get to work
-    // todo: Is 5 ms optimal? Sleep shorter if inserted into the Audio
-    //       Device Buffer?
-    timespec t;
-    t.tv_sec = 0;
-    t.tv_nsec = 5 * 1000 * 1000;
-    nanosleep(&t, nullptr);
-  }
-  return true;
+OSStatus AudioDeviceIOS::OnGetPlayoutData(
+    AudioUnitRenderActionFlags* ioActionFlags,
+    UInt32 inNumberFrames,
+    AudioBufferList* ioData) {
+  // Verify 16-bit, noninterleaved mono PCM signal format.
+  DCHECK_EQ(1u, ioData->mNumberBuffers);
+  DCHECK_EQ(1u, ioData->mBuffers[0].mNumberChannels);
+  // Get pointer to internal audio buffer to which new audio data shall be
+  // written.
+  const UInt32 dataSizeInBytes = ioData->mBuffers[0].mDataByteSize;
+  CHECK_EQ(dataSizeInBytes / kBytesPerSample, inNumberFrames);
+  SInt8* destination = static_cast<SInt8*>(ioData->mBuffers[0].mData);
+  // Produce silence and give audio unit a hint about it if playout is not
+  // activated.
+  if (!rtc::AtomicOps::AcquireLoad(&_playing)) {
+    *ioActionFlags |= kAudioUnitRenderAction_OutputIsSilence;
+    memset(destination, 0, dataSizeInBytes);
+    return noErr;
+  }
+  // Read decoded 16-bit PCM samples from WebRTC (using a size that matches
+  // the native I/O audio unit) to a preallocated intermediate buffer and
+  // copy the result to the audio buffer in the |ioData| destination.
+  SInt8* source = _playoutAudioBuffer.get();
+  _fineAudioBuffer->GetPlayoutData(source);
+  memcpy(destination, source, dataSizeInBytes);
+  return noErr;
 }
 
 }  // namespace webrtc