Index: webrtc/modules/audio_device/ios/audio_device_ios.mm |
diff --git a/webrtc/modules/audio_device/ios/audio_device_ios.mm b/webrtc/modules/audio_device/ios/audio_device_ios.mm |
index 6f610d7afcc7758fddeccd01e4434b0b391dde4f..5a6047c79802d902cac7235a35c1c54223d320c9 100644 |
--- a/webrtc/modules/audio_device/ios/audio_device_ios.mm |
+++ b/webrtc/modules/audio_device/ios/audio_device_ios.mm |
@@ -16,100 +16,155 @@ |
#import <Foundation/Foundation.h> |
#include "webrtc/modules/audio_device/ios/audio_device_ios.h" |
-#include "webrtc/modules/utility/interface/helpers_ios.h" |
+#include "webrtc/base/atomicops.h" |
#include "webrtc/base/checks.h" |
#include "webrtc/base/logging.h" |
-#include "webrtc/system_wrappers/interface/trace.h" |
+#include "webrtc/modules/audio_device/fine_audio_buffer.h" |
+#include "webrtc/modules/utility/interface/helpers_ios.h" |
namespace webrtc { |
#define LOGI() LOG(LS_INFO) << "AudioDeviceIOS::" |
+#define LOG_AND_RETURN_IF_ERROR(error, message) \ |
+ do { \ |
+ OSStatus err = error; \ |
+ if (err) { \ |
+ LOG(LS_ERROR) << message << ": " << err; \ |
+ return false; \ |
+ } \ |
+ } while (0) |
+ |
+// Preferred hardware sample rate (unit is in Hertz). The client sample rate |
+// will be set to this value as well to avoid resampling the the audio unit's |
+// format converter. Note that, some devices, e.g. BT headsets, only supports |
+// 8000Hz as native sample rate. |
+const double kPreferredSampleRate = 48000.0; |
+// Use a hardware I/O buffer size (unit is in seconds) that matches the 10ms |
+// size used by WebRTC. The exact actual size will differ between devices. |
+// Example: using 48kHz on iPhone 6 results in a native buffer size of |
+// ~10.6667ms or 512 audio frames per buffer. The FineAudioBuffer instance will |
+// take care of any buffering required to convert between native buffers and |
+// buffers used by WebRTC. It is beneficial for the performance if the native |
+// size is as close to 10ms as possible since it results in "clean" callback |
+// sequence without bursts of callbacks back to back. |
+const double kPreferredIOBufferDuration = 0.01; |
+// Try to use mono to save resources. Also avoids channel format conversion |
+// in the I/O audio unit. Initial tests have shown that it is possible to use |
+// mono natively for built-in microphones and for BT headsets but not for |
+// wired headsets. Wired headsets only support stereo as native channel format |
+// but it is a low cost operation to do a format conversion to mono in the |
+// audio unit. Hence, we will not hit a CHECK in |
+// VerifyAudioParametersForActiveAudioSession() for a mismatch between the |
+// preferred number of channels and the actual number of channels. |
+const int kPreferredNumberOfChannels = 1; |
+// Number of bytes per audio sample for 16-bit signed integer representation. |
+const UInt32 kBytesPerSample = 2; |
+// Hardcoded delay estimates based on real measurements. |
+// TODO(henrika): these value is not used in combination with built-in AEC. |
+// Can most likely be removed. |
+const UInt16 kFixedPlayoutDelayEstimate = 30; |
+const UInt16 kFixedRecordDelayEstimate = 30; |
+ |
using ios::CheckAndLogError; |
+// Activates an audio session suitable for full duplex VoIP sessions when |
+// |activate| is true. Also sets the preferred sample rate and IO buffer |
+// duration. Deactivates an active audio session if |activate| is set to false. |
static void ActivateAudioSession(AVAudioSession* session, bool activate) { |
LOG(LS_INFO) << "ActivateAudioSession(" << activate << ")"; |
@autoreleasepool { |
NSError* error = nil; |
BOOL success = NO; |
+ // Deactivate the audio session and return if |activate| is false. |
if (!activate) { |
- // Deactivate the audio session. |
success = [session setActive:NO error:&error]; |
DCHECK(CheckAndLogError(success, error)); |
return; |
} |
- // Activate an audio session and set category and mode. Only make changes |
- // if needed since setting them to the value they already have will clear |
- // transient properties (such as PortOverride) that some other component |
- // have set up. |
+ // Use a category which supports simultaneous recording and playback. |
+ // By default, using this category implies that our app’s audio is |
+ // nonmixable, hence activating the session will interrupt any other |
+ // audio sessions which are also nonmixable. |
if (session.category != AVAudioSessionCategoryPlayAndRecord) { |
error = nil; |
success = [session setCategory:AVAudioSessionCategoryPlayAndRecord |
error:&error]; |
DCHECK(CheckAndLogError(success, error)); |
} |
+ // Specify mode for two-way voice communication (e.g. VoIP). |
if (session.mode != AVAudioSessionModeVoiceChat) { |
error = nil; |
success = [session setMode:AVAudioSessionModeVoiceChat error:&error]; |
DCHECK(CheckAndLogError(success, error)); |
} |
+ // Set the session's sample rate or the hardware sample rate. |
+ // It is essential that we use the same sample rate as stream format |
+ // to ensure that the I/O unit does not have to do sample rate conversion. |
+ error = nil; |
+ success = |
+ [session setPreferredSampleRate:kPreferredSampleRate error:&error]; |
+ DCHECK(CheckAndLogError(success, error)); |
+ // Set the preferred audio I/O buffer duration, in seconds. |
+ // TODO(henrika): add more comments here. |
+ error = nil; |
+ success = [session setPreferredIOBufferDuration:kPreferredIOBufferDuration |
+ error:&error]; |
+ DCHECK(CheckAndLogError(success, error)); |
+ |
+ // TODO(henrika): add observers here... |
+ |
+ // Activate the audio session. Activation can fail if another active audio |
+ // session (e.g. phone call) has higher priority than ours. |
error = nil; |
success = [session setActive:YES error:&error]; |
DCHECK(CheckAndLogError(success, error)); |
+ CHECK(session.isInputAvailable) << "No input path is available!"; |
// Ensure that category and mode are actually activated. |
DCHECK( |
[session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]); |
DCHECK([session.mode isEqualToString:AVAudioSessionModeVoiceChat]); |
+ // Try to set the preferred number of hardware audio channels. These calls |
+ // must be done after setting the audio session’s category and mode and |
+ // activating the session. |
+ // We try to use mono in both directions to save resources and format |
+ // conversions in the audio unit. Some devices does only support stereo; |
+ // e.g. wired headset on iPhone 6. |
+ // TODO(henrika): add support for stereo if needed. |
+ error = nil; |
+ success = |
+ [session setPreferredInputNumberOfChannels:kPreferredNumberOfChannels |
+ error:&error]; |
+ DCHECK(CheckAndLogError(success, error)); |
+ error = nil; |
+ success = |
+ [session setPreferredOutputNumberOfChannels:kPreferredNumberOfChannels |
+ error:&error]; |
+ DCHECK(CheckAndLogError(success, error)); |
} |
} |
-// Query hardware characteristics, such as input and output latency, input and |
-// output channel count, hardware sample rate, hardware volume setting, and |
-// whether audio input is available. To obtain meaningful values for hardware |
-// characteristics,the audio session must be initialized and active before we |
-// query the values. |
-// TODO(henrika): Note that these characteristics can change at runtime. For |
-// instance, input sample rate may change when a user plugs in a headset. |
-static void GetHardwareAudioParameters(AudioParameters* playout_parameters, |
- AudioParameters* record_parameters) { |
- LOG(LS_INFO) << "GetHardwareAudioParameters"; |
- @autoreleasepool { |
- // Implicit initialization happens when we obtain a reference to the |
- // AVAudioSession object. |
- AVAudioSession* session = [AVAudioSession sharedInstance]; |
- // Always get values when the audio session is active. |
- ActivateAudioSession(session, true); |
- CHECK(session.isInputAvailable) << "No input path is available!"; |
- // Get current hardware parameters. |
- double sample_rate = (double)session.sampleRate; |
- double io_buffer_duration = (double)session.IOBufferDuration; |
- int output_channels = (int)session.outputNumberOfChannels; |
- int input_channels = (int)session.inputNumberOfChannels; |
- size_t frames_per_buffer = |
- static_cast<size_t>(sample_rate * io_buffer_duration + 0.5); |
- // Copy hardware parameters to output parameters. |
- playout_parameters->reset(sample_rate, output_channels, frames_per_buffer); |
- record_parameters->reset(sample_rate, input_channels, frames_per_buffer); |
- // Add logging for debugging purposes. |
- LOG(LS_INFO) << " sample rate: " << sample_rate; |
- LOG(LS_INFO) << " IO buffer duration: " << io_buffer_duration; |
- LOG(LS_INFO) << " frames_per_buffer: " << frames_per_buffer; |
- LOG(LS_INFO) << " output channels: " << output_channels; |
- LOG(LS_INFO) << " input channels: " << input_channels; |
- LOG(LS_INFO) << " output latency: " << (double)session.outputLatency; |
- LOG(LS_INFO) << " input latency: " << (double)session.inputLatency; |
- // Don't keep the audio session active. Instead, deactivate when needed. |
- ActivateAudioSession(session, false); |
- // TODO(henrika): to be extra safe, we can do more here. E.g., set |
- // preferred values for sample rate, channels etc., re-activate an audio |
- // session and verify the actual values again. Then we know for sure that |
- // the current values will in fact be correct. Or, we can skip all this |
- // and check setting when audio is started. Probably better. |
- } |
+#if !defined(NDEBUG) |
+// Helper method for printing out an AudioStreamBasicDescription structure. |
+static void LogABSD(AudioStreamBasicDescription absd) { |
+ char formatIDString[5]; |
+ UInt32 formatID = CFSwapInt32HostToBig(absd.mFormatID); |
+ bcopy(&formatID, formatIDString, 4); |
+ formatIDString[4] = '\0'; |
+ LOG(LS_INFO) << "LogABSD"; |
+ LOG(LS_INFO) << " sample rate: " << absd.mSampleRate; |
+ LOG(LS_INFO) << " format ID: " << formatIDString; |
+ LOG(LS_INFO) << " format flags: " << std::hex << absd.mFormatFlags; |
+ LOG(LS_INFO) << " bytes per packet: " << absd.mBytesPerPacket; |
+ LOG(LS_INFO) << " frames per packet: " << absd.mFramesPerPacket; |
+ LOG(LS_INFO) << " bytes per frame: " << absd.mBytesPerFrame; |
+ LOG(LS_INFO) << " channels per packet: " << absd.mChannelsPerFrame; |
+ LOG(LS_INFO) << " bits per channel: " << absd.mBitsPerChannel; |
+ LOG(LS_INFO) << " reserved: " << absd.mReserved; |
} |
-#if !defined(NDEBUG) |
+// Helper method that logs essential device information strings. |
static void LogDeviceInfo() { |
LOG(LS_INFO) << "LogDeviceInfo"; |
@autoreleasepool { |
@@ -119,127 +174,76 @@ static void LogDeviceInfo() { |
LOG(LS_INFO) << " device name: " << ios::GetDeviceName(); |
} |
} |
-#endif |
+#endif // !defined(NDEBUG) |
AudioDeviceIOS::AudioDeviceIOS() |
- : audio_device_buffer_(nullptr), |
- _critSect(*CriticalSectionWrapper::CreateCriticalSection()), |
- _auVoiceProcessing(nullptr), |
- _audioInterruptionObserver(nullptr), |
+ : _audioDeviceBuffer(nullptr), |
+ _vpioUnit(nullptr), |
+ _recording(0), |
+ _playing(0), |
_initialized(false), |
- _isShutDown(false), |
- _recording(false), |
- _playing(false), |
_recIsInitialized(false), |
_playIsInitialized(false), |
- _adbSampFreq(0), |
- _recordingDelay(0), |
- _playoutDelay(0), |
- _playoutDelayMeasurementCounter(9999), |
- _recordingDelayHWAndOS(0), |
- _recordingDelayMeasurementCounter(9999), |
- _playoutBufferUsed(0), |
- _recordingCurrentSeq(0), |
- _recordingBufferTotalSize(0) { |
+ _audioInterruptionObserver(nullptr) { |
LOGI() << "ctor" << ios::GetCurrentThreadDescription(); |
- memset(_playoutBuffer, 0, sizeof(_playoutBuffer)); |
- memset(_recordingBuffer, 0, sizeof(_recordingBuffer)); |
- memset(_recordingLength, 0, sizeof(_recordingLength)); |
- memset(_recordingSeqNumber, 0, sizeof(_recordingSeqNumber)); |
} |
AudioDeviceIOS::~AudioDeviceIOS() { |
LOGI() << "~dtor"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
Terminate(); |
- delete &_critSect; |
} |
void AudioDeviceIOS::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) { |
LOGI() << "AttachAudioBuffer"; |
DCHECK(audioBuffer); |
- DCHECK(thread_checker_.CalledOnValidThread()); |
- audio_device_buffer_ = audioBuffer; |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
+ _audioDeviceBuffer = audioBuffer; |
} |
int32_t AudioDeviceIOS::Init() { |
LOGI() << "Init"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
if (_initialized) { |
return 0; |
} |
#if !defined(NDEBUG) |
LogDeviceInfo(); |
#endif |
- // Query hardware audio parameters and cache the results. These parameters |
- // will be used as preferred values later when streaming starts. |
- // Note that I override these "optimal" value below since I don't want to |
- // modify the existing behavior yet. |
- GetHardwareAudioParameters(&playout_parameters_, &record_parameters_); |
- // TODO(henrika): these parameters are currently hard coded to match the |
- // existing implementation where we always use 16kHz as preferred sample |
- // rate and mono only. Goal is to improve this scheme and make it more |
- // flexible. In addition, a better native buffer size shall be derived. |
- // Using 10ms as default here (only used by unit test so far). |
- // We should also implemented observers for notification of any change in |
- // these parameters. |
- playout_parameters_.reset(16000, 1, 160); |
- record_parameters_.reset(16000, 1, 160); |
- |
- // AttachAudioBuffer() is called at construction by the main class but check |
- // just in case. |
- DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first"; |
- // Inform the audio device buffer (ADB) about the new audio format. |
- // TODO(henrika): try to improve this section. |
- audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate()); |
- audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels()); |
- audio_device_buffer_->SetRecordingSampleRate( |
- record_parameters_.sample_rate()); |
- audio_device_buffer_->SetRecordingChannels(record_parameters_.channels()); |
- |
- DCHECK(!_captureWorkerThread); |
- // Create and start the capture thread. |
- // TODO(henrika): do we need this thread? |
- _isShutDown = false; |
- _captureWorkerThread = |
- ThreadWrapper::CreateThread(RunCapture, this, "CaptureWorkerThread"); |
- if (!_captureWorkerThread->Start()) { |
- LOG_F(LS_ERROR) << "Failed to start CaptureWorkerThread!"; |
- return -1; |
- } |
- _captureWorkerThread->SetPriority(kRealtimePriority); |
+ // Store the preferred sample rate and preferred number of channels already |
+ // here. They have not been set and confirmed yet since ActivateAudioSession() |
+ // is not called until audio is about to start. However, it makes sense to |
+ // store the parameters now and then verify at a later stage. |
+ _playoutParameters.reset(kPreferredSampleRate, kPreferredNumberOfChannels); |
+ _recordParameters.reset(kPreferredSampleRate, kPreferredNumberOfChannels); |
+ // Ensure that the audio device buffer (ADB) knows about the internal audio |
+ // parameters. Note that, even if we are unable to get a mono audio session, |
+ // we will always tell the I/O audio unit to do a channel format conversion |
+ // to guarantee mono on the "input side" of the audio unit. |
+ UpdateAudioDeviceBuffer(); |
_initialized = true; |
return 0; |
} |
int32_t AudioDeviceIOS::Terminate() { |
LOGI() << "Terminate"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
if (!_initialized) { |
return 0; |
} |
- // Stop the capture thread. |
- if (_captureWorkerThread) { |
- if (!_captureWorkerThread->Stop()) { |
- LOG_F(LS_ERROR) << "Failed to stop CaptureWorkerThread!"; |
- return -1; |
- } |
- _captureWorkerThread.reset(); |
- } |
ShutdownPlayOrRecord(); |
- _isShutDown = true; |
_initialized = false; |
return 0; |
} |
int32_t AudioDeviceIOS::InitPlayout() { |
LOGI() << "InitPlayout"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
DCHECK(_initialized); |
DCHECK(!_playIsInitialized); |
DCHECK(!_playing); |
if (!_recIsInitialized) { |
- if (InitPlayOrRecord() == -1) { |
+ if (!InitPlayOrRecord()) { |
LOG_F(LS_ERROR) << "InitPlayOrRecord failed!"; |
return -1; |
} |
@@ -250,12 +254,12 @@ int32_t AudioDeviceIOS::InitPlayout() { |
int32_t AudioDeviceIOS::InitRecording() { |
LOGI() << "InitRecording"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
DCHECK(_initialized); |
DCHECK(!_recIsInitialized); |
DCHECK(!_recording); |
if (!_playIsInitialized) { |
- if (InitPlayOrRecord() == -1) { |
+ if (!InitPlayOrRecord()) { |
LOG_F(LS_ERROR) << "InitPlayOrRecord failed!"; |
return -1; |
} |
@@ -266,92 +270,63 @@ int32_t AudioDeviceIOS::InitRecording() { |
int32_t AudioDeviceIOS::StartPlayout() { |
LOGI() << "StartPlayout"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
DCHECK(_playIsInitialized); |
DCHECK(!_playing); |
- |
- CriticalSectionScoped lock(&_critSect); |
- |
- memset(_playoutBuffer, 0, sizeof(_playoutBuffer)); |
- _playoutBufferUsed = 0; |
- _playoutDelay = 0; |
- // Make sure first call to update delay function will update delay |
- _playoutDelayMeasurementCounter = 9999; |
- |
+ _fineAudioBuffer->ResetPlayout(); |
if (!_recording) { |
- OSStatus result = AudioOutputUnitStart(_auVoiceProcessing); |
+ OSStatus result = AudioOutputUnitStart(_vpioUnit); |
if (result != noErr) { |
LOG_F(LS_ERROR) << "AudioOutputUnitStart failed: " << result; |
return -1; |
} |
} |
- _playing = true; |
+ rtc::AtomicOps::ReleaseStore(&_playing, 1); |
return 0; |
} |
int32_t AudioDeviceIOS::StopPlayout() { |
LOGI() << "StopPlayout"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
if (!_playIsInitialized || !_playing) { |
return 0; |
} |
- |
- CriticalSectionScoped lock(&_critSect); |
- |
if (!_recording) { |
- // Both playout and recording has stopped, shutdown the device. |
ShutdownPlayOrRecord(); |
} |
_playIsInitialized = false; |
- _playing = false; |
+ rtc::AtomicOps::ReleaseStore(&_playing, 0); |
return 0; |
} |
int32_t AudioDeviceIOS::StartRecording() { |
LOGI() << "StartRecording"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
DCHECK(_recIsInitialized); |
DCHECK(!_recording); |
- |
- CriticalSectionScoped lock(&_critSect); |
- |
- memset(_recordingBuffer, 0, sizeof(_recordingBuffer)); |
- memset(_recordingLength, 0, sizeof(_recordingLength)); |
- memset(_recordingSeqNumber, 0, sizeof(_recordingSeqNumber)); |
- |
- _recordingCurrentSeq = 0; |
- _recordingBufferTotalSize = 0; |
- _recordingDelay = 0; |
- _recordingDelayHWAndOS = 0; |
- // Make sure first call to update delay function will update delay |
- _recordingDelayMeasurementCounter = 9999; |
- |
+ _fineAudioBuffer->ResetRecord(); |
if (!_playing) { |
- OSStatus result = AudioOutputUnitStart(_auVoiceProcessing); |
+ OSStatus result = AudioOutputUnitStart(_vpioUnit); |
if (result != noErr) { |
LOG_F(LS_ERROR) << "AudioOutputUnitStart failed: " << result; |
return -1; |
} |
} |
- _recording = true; |
+ rtc::AtomicOps::ReleaseStore(&_recording, 1); |
return 0; |
} |
int32_t AudioDeviceIOS::StopRecording() { |
LOGI() << "StopRecording"; |
- DCHECK(thread_checker_.CalledOnValidThread()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
if (!_recIsInitialized || !_recording) { |
return 0; |
} |
- |
- CriticalSectionScoped lock(&_critSect); |
- |
if (!_playing) { |
- // Both playout and recording has stopped, shutdown the device. |
ShutdownPlayOrRecord(); |
} |
_recIsInitialized = false; |
- _recording = false; |
+ rtc::AtomicOps::ReleaseStore(&_recording, 0); |
return 0; |
} |
@@ -391,240 +366,240 @@ int32_t AudioDeviceIOS::GetLoudspeakerStatus(bool& enabled) const { |
} |
int32_t AudioDeviceIOS::PlayoutDelay(uint16_t& delayMS) const { |
- delayMS = _playoutDelay; |
+ delayMS = kFixedPlayoutDelayEstimate; |
return 0; |
} |
int32_t AudioDeviceIOS::RecordingDelay(uint16_t& delayMS) const { |
- delayMS = _recordingDelay; |
- return 0; |
-} |
- |
-int32_t AudioDeviceIOS::PlayoutBuffer(AudioDeviceModule::BufferType& type, |
- uint16_t& sizeMS) const { |
- type = AudioDeviceModule::kAdaptiveBufferSize; |
- sizeMS = _playoutDelay; |
+ delayMS = kFixedRecordDelayEstimate; |
return 0; |
} |
int AudioDeviceIOS::GetPlayoutAudioParameters(AudioParameters* params) const { |
- CHECK(playout_parameters_.is_valid()); |
- DCHECK(thread_checker_.CalledOnValidThread()); |
- *params = playout_parameters_; |
+ LOGI() << "GetPlayoutAudioParameters"; |
+ DCHECK(_playoutParameters.is_valid()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
+ *params = _playoutParameters; |
return 0; |
} |
int AudioDeviceIOS::GetRecordAudioParameters(AudioParameters* params) const { |
- CHECK(record_parameters_.is_valid()); |
- DCHECK(thread_checker_.CalledOnValidThread()); |
- *params = record_parameters_; |
+ LOGI() << "GetRecordAudioParameters"; |
+ DCHECK(_recordParameters.is_valid()); |
+ DCHECK(_threadChecker.CalledOnValidThread()); |
+ *params = _recordParameters; |
return 0; |
} |
-// ============================================================================ |
-// Private Methods |
-// ============================================================================ |
- |
-int32_t AudioDeviceIOS::InitPlayOrRecord() { |
- LOGI() << "AudioDeviceIOS::InitPlayOrRecord"; |
- DCHECK(!_auVoiceProcessing); |
- |
- OSStatus result = -1; |
- |
- // Create Voice Processing Audio Unit |
- AudioComponentDescription desc; |
- AudioComponent comp; |
- |
- desc.componentType = kAudioUnitType_Output; |
- desc.componentSubType = kAudioUnitSubType_VoiceProcessingIO; |
- desc.componentManufacturer = kAudioUnitManufacturer_Apple; |
- desc.componentFlags = 0; |
- desc.componentFlagsMask = 0; |
- |
- comp = AudioComponentFindNext(nullptr, &desc); |
- if (nullptr == comp) { |
- LOG_F(LS_ERROR) << "Could not find audio component for Audio Unit"; |
- return -1; |
- } |
- |
- result = AudioComponentInstanceNew(comp, &_auVoiceProcessing); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to create Audio Unit instance: " << result; |
- return -1; |
- } |
- |
- // TODO(henrika): I think we should set the preferred channel configuration |
- // in both directions as well to be safe. |
+void AudioDeviceIOS::UpdateAudioDeviceBuffer() { |
+ LOGI() << "UpdateAudioDevicebuffer"; |
+ // AttachAudioBuffer() is called at construction by the main class but check |
+ // just in case. |
+ DCHECK(_audioDeviceBuffer) << "AttachAudioBuffer must be called first"; |
+ // Inform the audio device buffer (ADB) about the new audio format. |
+ _audioDeviceBuffer->SetPlayoutSampleRate(_playoutParameters.sample_rate()); |
+ _audioDeviceBuffer->SetPlayoutChannels(_playoutParameters.channels()); |
+ _audioDeviceBuffer->SetRecordingSampleRate(_recordParameters.sample_rate()); |
+ _audioDeviceBuffer->SetRecordingChannels(_recordParameters.channels()); |
+} |
- // Set preferred hardware sample rate to 16 kHz. |
- // TODO(henrika): improve this selection of sample rate. Why do we currently |
- // use a hard coded value? How can we fail and still continue? |
- NSError* error = nil; |
+void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() { |
+ LOGI() << "SetupAudioBuffersForActiveAudioSession"; |
AVAudioSession* session = [AVAudioSession sharedInstance]; |
- Float64 preferredSampleRate(playout_parameters_.sample_rate()); |
- [session setPreferredSampleRate:preferredSampleRate error:&error]; |
- if (error != nil) { |
- const char* errorString = [[error localizedDescription] UTF8String]; |
- LOG_F(LS_ERROR) << "setPreferredSampleRate failed: " << errorString; |
- } |
- |
- // TODO(henrika): we can reduce latency by setting the IOBufferDuration |
- // here. Default size for 16kHz is 0.016 sec or 16 msec on an iPhone 6. |
- |
- // Activate the audio session. |
- ActivateAudioSession(session, true); |
+ // Verify the current values once the audio session has been activated. |
+ LOG(LS_INFO) << " sample rate: " << session.sampleRate; |
+ LOG(LS_INFO) << " IO buffer duration: " << session.IOBufferDuration; |
+ LOG(LS_INFO) << " output channels: " << session.outputNumberOfChannels; |
+ LOG(LS_INFO) << " input channels: " << session.inputNumberOfChannels; |
+ LOG(LS_INFO) << " output latency: " << session.outputLatency; |
+ LOG(LS_INFO) << " input latency: " << session.inputLatency; |
+ // Log a warning message for the case when we are unable to set the preferred |
+ // hardware sample rate but continue and use the non-ideal sample rate after |
+ // reinitializing the audio parameters. |
+ if (session.sampleRate != _playoutParameters.sample_rate()) { |
+ LOG(LS_WARNING) |
+ << "Failed to enable an audio session with the preferred sample rate!"; |
+ } |
+ |
+ // At this stage, we also know the exact IO buffer duration and can add |
+ // that info to the existing audio parameters where it is converted into |
+ // number of audio frames. |
+ // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz. |
+ // Hence, 128 is the size we expect to see in upcoming render callbacks. |
+ _playoutParameters.reset(session.sampleRate, _playoutParameters.channels(), |
+ session.IOBufferDuration); |
+ DCHECK(_playoutParameters.is_complete()); |
+ _recordParameters.reset(session.sampleRate, _recordParameters.channels(), |
+ session.IOBufferDuration); |
+ DCHECK(_recordParameters.is_complete()); |
+ LOG(LS_INFO) << " frames per I/O buffer: " |
+ << _playoutParameters.frames_per_buffer(); |
+ LOG(LS_INFO) << " bytes per I/O buffer: " |
+ << _playoutParameters.GetBytesPerBuffer(); |
+ DCHECK_EQ(_playoutParameters.GetBytesPerBuffer(), |
+ _recordParameters.GetBytesPerBuffer()); |
+ |
+ // Update the ADB parameters since the sample rate might have changed. |
+ UpdateAudioDeviceBuffer(); |
+ |
+ // Create a modified audio buffer class which allows us to ask for, |
+ // or deliver, any number of samples (and not only multiple of 10ms) to match |
+ // the native audio unit buffer size. |
+ DCHECK(_audioDeviceBuffer); |
+ _fineAudioBuffer.reset(new FineAudioBuffer( |
+ _audioDeviceBuffer, _playoutParameters.GetBytesPerBuffer(), |
+ _playoutParameters.sample_rate())); |
+ |
+ // The extra/temporary playoutbuffer must be of this size to avoid |
+ // unnecessary memcpy while caching data between successive callbacks. |
+ const int requiredPlayoutBufferSize = |
+ _fineAudioBuffer->RequiredPlayoutBufferSizeBytes(); |
+ LOG(LS_INFO) << " required playout buffer size: " |
+ << requiredPlayoutBufferSize; |
+ _playoutAudioBuffer.reset(new SInt8[requiredPlayoutBufferSize]); |
+ |
+ // Allocate AudioBuffers to be used as storage for the received audio. |
+ // The AudioBufferList structure works as a placeholder for the |
+ // AudioBuffer structure, which holds a pointer to the actual data buffer |
+ // in |_recordAudioBuffer|. Recorded audio will be rendered into this memory |
+ // at each input callback when calling AudioUnitRender(). |
+ const int dataByteSize = _recordParameters.GetBytesPerBuffer(); |
+ _recordAudioBuffer.reset(new SInt8[dataByteSize]); |
+ _audioRecordBufferList.mNumberBuffers = 1; |
+ AudioBuffer* audioBuffer = &_audioRecordBufferList.mBuffers[0]; |
+ audioBuffer->mNumberChannels = _recordParameters.channels(); |
+ audioBuffer->mDataByteSize = dataByteSize; |
+ audioBuffer->mData = _recordAudioBuffer.get(); |
+} |
- UInt32 enableIO = 1; |
- result = AudioUnitSetProperty(_auVoiceProcessing, |
- kAudioOutputUnitProperty_EnableIO, |
- kAudioUnitScope_Input, |
- 1, // input bus |
- &enableIO, sizeof(enableIO)); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to enable IO on input: " << result; |
- } |
+bool AudioDeviceIOS::SetupAndInitializeVoiceProcessingAudioUnit() { |
+ LOGI() << "SetupAndInitializeVoiceProcessingAudioUnit"; |
+ DCHECK(!_vpioUnit); |
+ // Create an audio component description to identify the Voice-Processing |
+ // I/O audio unit. |
+ AudioComponentDescription vpioUnitDescription; |
+ vpioUnitDescription.componentType = kAudioUnitType_Output; |
+ vpioUnitDescription.componentSubType = kAudioUnitSubType_VoiceProcessingIO; |
+ vpioUnitDescription.componentManufacturer = kAudioUnitManufacturer_Apple; |
+ vpioUnitDescription.componentFlags = 0; |
+ vpioUnitDescription.componentFlagsMask = 0; |
+ // Obtain an audio unit instance given the description. |
+ AudioComponent foundVpioUnitRef = |
+ AudioComponentFindNext(nullptr, &vpioUnitDescription); |
+ |
+ // Create a Voice-Processing IO audio unit. |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioComponentInstanceNew(foundVpioUnitRef, &_vpioUnit), |
+ "Failed to create a VoiceProcessingIO audio unit"); |
+ |
+ // A VP I/O unit's bus 1 connects to input hardware (microphone). Enable |
+ // input on the input scope of the input element. |
+ AudioUnitElement inputBus = 1; |
+ UInt32 enableInput = 1; |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioOutputUnitProperty_EnableIO, |
+ kAudioUnitScope_Input, inputBus, &enableInput, |
+ sizeof(enableInput)), |
+ "Failed to enable input on input scope of input element"); |
+ |
+ // A VP I/O unit's bus 0 connects to output hardware (speaker). Enable |
+ // output on the output scope of the output element. |
+ AudioUnitElement outputBus = 0; |
+ UInt32 enableOutput = 1; |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioOutputUnitProperty_EnableIO, |
+ kAudioUnitScope_Output, outputBus, &enableOutput, |
+ sizeof(enableOutput)), |
+ "Failed to enable output on output scope of output element"); |
+ |
+ // Set the application formats for input and output: |
+ // - use same format in both directions |
+ // - avoid resampling in the I/O unit by using the hardware sample rate |
+ // - linear PCM => noncompressed audio data format with one frame per packet |
+ // - no need to specify interleaving since only mono is supported |
+ AudioStreamBasicDescription applicationFormat = {0}; |
+ UInt32 size = sizeof(applicationFormat); |
+ DCHECK_EQ(_playoutParameters.sample_rate(), _recordParameters.sample_rate()); |
+ DCHECK_EQ(1, kPreferredNumberOfChannels); |
+ applicationFormat.mSampleRate = _playoutParameters.sample_rate(); |
+ applicationFormat.mFormatID = kAudioFormatLinearPCM; |
+ applicationFormat.mFormatFlags = |
+ kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked; |
+ applicationFormat.mBytesPerPacket = kBytesPerSample; |
+ applicationFormat.mFramesPerPacket = 1; // uncompressed |
+ applicationFormat.mBytesPerFrame = kBytesPerSample; |
+ applicationFormat.mChannelsPerFrame = kPreferredNumberOfChannels; |
+ applicationFormat.mBitsPerChannel = 8 * kBytesPerSample; |
+#if !defined(NDEBUG) |
+ LogABSD(applicationFormat); |
+#endif |
- result = AudioUnitSetProperty(_auVoiceProcessing, |
- kAudioOutputUnitProperty_EnableIO, |
- kAudioUnitScope_Output, |
- 0, // output bus |
- &enableIO, sizeof(enableIO)); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to enable IO on output: " << result; |
- } |
+ // Set the application format on the output scope of the input element/bus. |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_StreamFormat, |
+ kAudioUnitScope_Output, inputBus, &applicationFormat, |
+ size), |
+ "Failed to set application format on output scope of input element"); |
+ |
+ // Set the application format on the input scope of the output element/bus. |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_StreamFormat, |
+ kAudioUnitScope_Input, outputBus, &applicationFormat, |
+ size), |
+ "Failed to set application format on input scope of output element"); |
+ |
+ // Specify the callback function that provides audio samples to the audio |
+ // unit. |
+ AURenderCallbackStruct renderCallback; |
+ renderCallback.inputProc = GetPlayoutData; |
+ renderCallback.inputProcRefCon = this; |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_SetRenderCallback, |
+ kAudioUnitScope_Input, outputBus, &renderCallback, |
+ sizeof(renderCallback)), |
+ "Failed to specify the render callback on the output element"); |
// Disable AU buffer allocation for the recorder, we allocate our own. |
- // TODO(henrika): understand this part better. |
+ // TODO(henrika): not sure that it actually saves resource to make this call. |
UInt32 flag = 0; |
- result = AudioUnitSetProperty(_auVoiceProcessing, |
- kAudioUnitProperty_ShouldAllocateBuffer, |
- kAudioUnitScope_Output, 1, &flag, sizeof(flag)); |
- if (0 != result) { |
- LOG_F(LS_WARNING) << "Failed to disable AU buffer allocation: " << result; |
- // Should work anyway |
- } |
- |
- // Set recording callback. |
- AURenderCallbackStruct auCbS; |
- memset(&auCbS, 0, sizeof(auCbS)); |
- auCbS.inputProc = RecordProcess; |
- auCbS.inputProcRefCon = this; |
- result = AudioUnitSetProperty( |
- _auVoiceProcessing, kAudioOutputUnitProperty_SetInputCallback, |
- kAudioUnitScope_Global, 1, &auCbS, sizeof(auCbS)); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to set AU record callback: " << result; |
- } |
- |
- // Set playout callback. |
- memset(&auCbS, 0, sizeof(auCbS)); |
- auCbS.inputProc = PlayoutProcess; |
- auCbS.inputProcRefCon = this; |
- result = AudioUnitSetProperty( |
- _auVoiceProcessing, kAudioUnitProperty_SetRenderCallback, |
- kAudioUnitScope_Global, 0, &auCbS, sizeof(auCbS)); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to set AU output callback: " << result; |
- } |
- |
- // Get stream format for out/0 |
- AudioStreamBasicDescription playoutDesc; |
- UInt32 size = sizeof(playoutDesc); |
- result = |
- AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat, |
- kAudioUnitScope_Output, 0, &playoutDesc, &size); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to get AU output stream format: " << result; |
- } |
- |
- playoutDesc.mSampleRate = preferredSampleRate; |
- LOG(LS_INFO) << "Audio Unit playout opened in sampling rate: " |
- << playoutDesc.mSampleRate; |
- |
- // Store the sampling frequency to use towards the Audio Device Buffer |
- // todo: Add 48 kHz (increase buffer sizes). Other fs? |
- // TODO(henrika): Figure out if we really need this complex handling. |
- if ((playoutDesc.mSampleRate > 44090.0) && |
- (playoutDesc.mSampleRate < 44110.0)) { |
- _adbSampFreq = 44100; |
- } else if ((playoutDesc.mSampleRate > 15990.0) && |
- (playoutDesc.mSampleRate < 16010.0)) { |
- _adbSampFreq = 16000; |
- } else if ((playoutDesc.mSampleRate > 7990.0) && |
- (playoutDesc.mSampleRate < 8010.0)) { |
- _adbSampFreq = 8000; |
- } else { |
- _adbSampFreq = 0; |
- FATAL() << "Invalid sample rate"; |
- } |
- |
- // Set the audio device buffer sampling rates (use same for play and record). |
- // TODO(henrika): this is not a good place to set these things up. |
- DCHECK(audio_device_buffer_); |
- DCHECK_EQ(_adbSampFreq, playout_parameters_.sample_rate()); |
- audio_device_buffer_->SetRecordingSampleRate(_adbSampFreq); |
- audio_device_buffer_->SetPlayoutSampleRate(_adbSampFreq); |
- |
- // Set stream format for out/0. |
- playoutDesc.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | |
- kLinearPCMFormatFlagIsPacked | |
- kLinearPCMFormatFlagIsNonInterleaved; |
- playoutDesc.mBytesPerPacket = 2; |
- playoutDesc.mFramesPerPacket = 1; |
- playoutDesc.mBytesPerFrame = 2; |
- playoutDesc.mChannelsPerFrame = 1; |
- playoutDesc.mBitsPerChannel = 16; |
- result = |
- AudioUnitSetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat, |
- kAudioUnitScope_Input, 0, &playoutDesc, size); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to set AU stream format for out/0"; |
- } |
- |
- // Get stream format for in/1. |
- AudioStreamBasicDescription recordingDesc; |
- size = sizeof(recordingDesc); |
- result = |
- AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat, |
- kAudioUnitScope_Input, 1, &recordingDesc, &size); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to get AU stream format for in/1"; |
- } |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioUnitProperty_ShouldAllocateBuffer, |
+ kAudioUnitScope_Output, inputBus, &flag, |
+ sizeof(flag)), |
+ "Failed to disable buffer allocation on the input element"); |
+ |
+ // Specify the callback to be called by the I/O thread to us when input audio |
+ // is available. The recorded samples can then be obtained by calling the |
+ // AudioUnitRender() method. |
+ AURenderCallbackStruct inputCallback; |
+ inputCallback.inputProc = RecordedDataIsAvailable; |
+ inputCallback.inputProcRefCon = this; |
+ LOG_AND_RETURN_IF_ERROR( |
+ AudioUnitSetProperty(_vpioUnit, kAudioOutputUnitProperty_SetInputCallback, |
+ kAudioUnitScope_Global, inputBus, &inputCallback, |
+ sizeof(inputCallback)), |
+ "Failed to specify the input callback on the input element"); |
+ |
+ // Initialize the Voice-Processing I/O unit instance. |
+ LOG_AND_RETURN_IF_ERROR(AudioUnitInitialize(_vpioUnit), |
+ "Failed to initialize the Voice-Processing I/O unit"); |
+ return true; |
+} |
- recordingDesc.mSampleRate = preferredSampleRate; |
- LOG(LS_INFO) << "Audio Unit recording opened in sampling rate: " |
- << recordingDesc.mSampleRate; |
+bool AudioDeviceIOS::InitPlayOrRecord() { |
+ LOGI() << "InitPlayOrRecord"; |
+ AVAudioSession* session = [AVAudioSession sharedInstance]; |
+ // Activate the audio session and ask for a set of preferred audio parameters. |
+ ActivateAudioSession(session, true); |
- // Set stream format for out/1 (use same sampling frequency as for in/1). |
- recordingDesc.mFormatFlags = kLinearPCMFormatFlagIsSignedInteger | |
- kLinearPCMFormatFlagIsPacked | |
- kLinearPCMFormatFlagIsNonInterleaved; |
- recordingDesc.mBytesPerPacket = 2; |
- recordingDesc.mFramesPerPacket = 1; |
- recordingDesc.mBytesPerFrame = 2; |
- recordingDesc.mChannelsPerFrame = 1; |
- recordingDesc.mBitsPerChannel = 16; |
- result = |
- AudioUnitSetProperty(_auVoiceProcessing, kAudioUnitProperty_StreamFormat, |
- kAudioUnitScope_Output, 1, &recordingDesc, size); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "Failed to set AU stream format for out/1"; |
- } |
+ // Ensure that we got what what we asked for in our active audio session. |
+ SetupAudioBuffersForActiveAudioSession(); |
- // Initialize here already to be able to get/set stream properties. |
- result = AudioUnitInitialize(_auVoiceProcessing); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "AudioUnitInitialize failed: " << result; |
+ // Create, setup and initialize a new Voice-Processing I/O unit. |
+ if (!SetupAndInitializeVoiceProcessingAudioUnit()) { |
+ return false; |
} |
- // Get hardware sample rate for logging (see if we get what we asked for). |
- // TODO(henrika): what if we don't get what we ask for? |
- double sampleRate = session.sampleRate; |
- LOG(LS_INFO) << "Current HW sample rate is: " << sampleRate |
- << ", ADB sample rate is: " << _adbSampFreq; |
- LOG(LS_INFO) << "Current HW IO buffer size is: " << |
- [session IOBufferDuration]; |
- |
// Listen to audio interruptions. |
// TODO(henrika): learn this area better. |
NSNotificationCenter* center = [NSNotificationCenter defaultCenter]; |
@@ -655,8 +630,8 @@ int32_t AudioDeviceIOS::InitPlayOrRecord() { |
// Post interruption the audio unit render callbacks don't |
// automatically continue, so we restart the unit manually |
// here. |
- AudioOutputUnitStop(_auVoiceProcessing); |
- AudioOutputUnitStart(_auVoiceProcessing); |
+ AudioOutputUnitStop(_vpioUnit); |
+ AudioOutputUnitStart(_vpioUnit); |
break; |
} |
} |
@@ -665,13 +640,11 @@ int32_t AudioDeviceIOS::InitPlayOrRecord() { |
// void* instead of an id because header is included in other pure C++ |
// files. |
_audioInterruptionObserver = (__bridge_retained void*)observer; |
- |
- return 0; |
+ return true; |
} |
-int32_t AudioDeviceIOS::ShutdownPlayOrRecord() { |
+bool AudioDeviceIOS::ShutdownPlayOrRecord() { |
LOGI() << "ShutdownPlayOrRecord"; |
- |
if (_audioInterruptionObserver != nullptr) { |
NSNotificationCenter* center = [NSNotificationCenter defaultCenter]; |
// Transfer ownership of observer back to ARC, which will dealloc the |
@@ -680,375 +653,113 @@ int32_t AudioDeviceIOS::ShutdownPlayOrRecord() { |
[center removeObserver:observer]; |
_audioInterruptionObserver = nullptr; |
} |
- |
- // Close and delete AU. |
+ // Close and delete the voice-processing I/O unit. |
OSStatus result = -1; |
- if (nullptr != _auVoiceProcessing) { |
- result = AudioOutputUnitStop(_auVoiceProcessing); |
- if (0 != result) { |
+ if (nullptr != _vpioUnit) { |
+ result = AudioOutputUnitStop(_vpioUnit); |
+ if (result != noErr) { |
LOG_F(LS_ERROR) << "AudioOutputUnitStop failed: " << result; |
} |
- result = AudioComponentInstanceDispose(_auVoiceProcessing); |
- if (0 != result) { |
+ result = AudioComponentInstanceDispose(_vpioUnit); |
+ if (result != noErr) { |
LOG_F(LS_ERROR) << "AudioComponentInstanceDispose failed: " << result; |
} |
- _auVoiceProcessing = nullptr; |
+ _vpioUnit = nullptr; |
} |
- |
// All I/O should be stopped or paused prior to deactivating the audio |
// session, hence we deactivate as last action. |
AVAudioSession* session = [AVAudioSession sharedInstance]; |
ActivateAudioSession(session, false); |
- return 0; |
+ return true; |
} |
-// ============================================================================ |
-// Thread Methods |
-// ============================================================================ |
- |
-OSStatus AudioDeviceIOS::RecordProcess( |
+OSStatus AudioDeviceIOS::RecordedDataIsAvailable( |
void* inRefCon, |
AudioUnitRenderActionFlags* ioActionFlags, |
const AudioTimeStamp* inTimeStamp, |
UInt32 inBusNumber, |
UInt32 inNumberFrames, |
AudioBufferList* ioData) { |
- AudioDeviceIOS* ptrThis = static_cast<AudioDeviceIOS*>(inRefCon); |
- return ptrThis->RecordProcessImpl(ioActionFlags, inTimeStamp, inBusNumber, |
- inNumberFrames); |
+ DCHECK_EQ(1u, inBusNumber); |
+ DCHECK(!ioData); // no buffer should be allocated for input at this stage |
+ AudioDeviceIOS* audio_device_ios = static_cast<AudioDeviceIOS*>(inRefCon); |
+ return audio_device_ios->OnRecordedDataIsAvailable( |
+ ioActionFlags, inTimeStamp, inBusNumber, inNumberFrames); |
} |
-OSStatus AudioDeviceIOS::RecordProcessImpl( |
+OSStatus AudioDeviceIOS::OnRecordedDataIsAvailable( |
AudioUnitRenderActionFlags* ioActionFlags, |
const AudioTimeStamp* inTimeStamp, |
- uint32_t inBusNumber, |
- uint32_t inNumberFrames) { |
- // Setup some basic stuff |
- // Use temp buffer not to lock up recording buffer more than necessary |
- // todo: Make dataTmp a member variable with static size that holds |
- // max possible frames? |
- int16_t* dataTmp = new int16_t[inNumberFrames]; |
- memset(dataTmp, 0, 2 * inNumberFrames); |
- |
- AudioBufferList abList; |
- abList.mNumberBuffers = 1; |
- abList.mBuffers[0].mData = dataTmp; |
- abList.mBuffers[0].mDataByteSize = 2 * inNumberFrames; // 2 bytes/sample |
- abList.mBuffers[0].mNumberChannels = 1; |
- |
- // Get data from mic |
- OSStatus res = AudioUnitRender(_auVoiceProcessing, ioActionFlags, inTimeStamp, |
- inBusNumber, inNumberFrames, &abList); |
- if (res != 0) { |
- // TODO(henrika): improve error handling. |
- delete[] dataTmp; |
- return 0; |
- } |
- |
- if (_recording) { |
- // Insert all data in temp buffer into recording buffers |
- // There is zero or one buffer partially full at any given time, |
- // all others are full or empty |
- // Full means filled with noSamp10ms samples. |
- |
- const unsigned int noSamp10ms = _adbSampFreq / 100; |
- unsigned int dataPos = 0; |
- uint16_t bufPos = 0; |
- int16_t insertPos = -1; |
- unsigned int nCopy = 0; // Number of samples to copy |
- |
- while (dataPos < inNumberFrames) { |
- // Loop over all recording buffers or |
- // until we find the partially full buffer |
- // First choice is to insert into partially full buffer, |
- // second choice is to insert into empty buffer |
- bufPos = 0; |
- insertPos = -1; |
- nCopy = 0; |
- while (bufPos < N_REC_BUFFERS) { |
- if ((_recordingLength[bufPos] > 0) && |
- (_recordingLength[bufPos] < noSamp10ms)) { |
- // Found the partially full buffer |
- insertPos = static_cast<int16_t>(bufPos); |
- // Don't need to search more, quit loop |
- bufPos = N_REC_BUFFERS; |
- } else if ((-1 == insertPos) && (0 == _recordingLength[bufPos])) { |
- // Found an empty buffer |
- insertPos = static_cast<int16_t>(bufPos); |
- } |
- ++bufPos; |
- } |
- |
- // Insert data into buffer |
- if (insertPos > -1) { |
- // We found a non-full buffer, copy data to it |
- unsigned int dataToCopy = inNumberFrames - dataPos; |
- unsigned int currentRecLen = _recordingLength[insertPos]; |
- unsigned int roomInBuffer = noSamp10ms - currentRecLen; |
- nCopy = (dataToCopy < roomInBuffer ? dataToCopy : roomInBuffer); |
- |
- memcpy(&_recordingBuffer[insertPos][currentRecLen], &dataTmp[dataPos], |
- nCopy * sizeof(int16_t)); |
- if (0 == currentRecLen) { |
- _recordingSeqNumber[insertPos] = _recordingCurrentSeq; |
- ++_recordingCurrentSeq; |
- } |
- _recordingBufferTotalSize += nCopy; |
- // Has to be done last to avoid interrupt problems between threads. |
- _recordingLength[insertPos] += nCopy; |
- dataPos += nCopy; |
- } else { |
- // Didn't find a non-full buffer |
- // TODO(henrika): improve error handling |
- dataPos = inNumberFrames; // Don't try to insert more |
- } |
- } |
- } |
- delete[] dataTmp; |
- return 0; |
+ UInt32 inBusNumber, |
+ UInt32 inNumberFrames) { |
+ DCHECK_EQ(_recordParameters.frames_per_buffer(), inNumberFrames); |
+ OSStatus result = noErr; |
+ // Simply return if recording is not enabled. |
+ if (!rtc::AtomicOps::AcquireLoad(&_recording)) |
+ return result; |
+ // Obtain the recorded audio samples by initiating a rendering cycle. |
+ // Since it happens on the input bus, the |ioData| parameter is a reference |
+ // to the preallocated audio buffer list that the audio unit renders into. |
+ // TODO(henrika): should error handling be improved? |
+ AudioBufferList* ioData = &_audioRecordBufferList; |
+ result = AudioUnitRender(_vpioUnit, ioActionFlags, inTimeStamp, inBusNumber, |
+ inNumberFrames, ioData); |
+ if (result != noErr) { |
+ LOG_F(LS_ERROR) << "AudioOutputUnitStart failed: " << result; |
+ return result; |
+ } |
+ // Get a pointer to the recorded audio and send it to the WebRTC ADB. |
+ // Use the FineAudioBuffer instance to convert between native buffer size |
+ // and the 10ms buffer size used by WebRTC. |
+ const UInt32 dataSizeInBytes = ioData->mBuffers[0].mDataByteSize; |
+ CHECK_EQ(dataSizeInBytes / kBytesPerSample, inNumberFrames); |
+ SInt8* data = static_cast<SInt8*>(ioData->mBuffers[0].mData); |
+ _fineAudioBuffer->DeliverRecordedData(data, dataSizeInBytes, |
+ kFixedPlayoutDelayEstimate, |
+ kFixedRecordDelayEstimate); |
+ return noErr; |
} |
-OSStatus AudioDeviceIOS::PlayoutProcess( |
+OSStatus AudioDeviceIOS::GetPlayoutData( |
void* inRefCon, |
AudioUnitRenderActionFlags* ioActionFlags, |
const AudioTimeStamp* inTimeStamp, |
UInt32 inBusNumber, |
UInt32 inNumberFrames, |
AudioBufferList* ioData) { |
- AudioDeviceIOS* ptrThis = static_cast<AudioDeviceIOS*>(inRefCon); |
- return ptrThis->PlayoutProcessImpl(inNumberFrames, ioData); |
+ DCHECK_EQ(0u, inBusNumber); |
+ DCHECK(ioData); |
+ AudioDeviceIOS* audio_device_ios = static_cast<AudioDeviceIOS*>(inRefCon); |
+ return audio_device_ios->OnGetPlayoutData(ioActionFlags, inNumberFrames, |
+ ioData); |
} |
-OSStatus AudioDeviceIOS::PlayoutProcessImpl(uint32_t inNumberFrames, |
- AudioBufferList* ioData) { |
- int16_t* data = static_cast<int16_t*>(ioData->mBuffers[0].mData); |
- unsigned int dataSizeBytes = ioData->mBuffers[0].mDataByteSize; |
- unsigned int dataSize = dataSizeBytes / 2; // Number of samples |
- CHECK_EQ(dataSize, inNumberFrames); |
- memset(data, 0, dataSizeBytes); // Start with empty buffer |
- |
- // Get playout data from Audio Device Buffer |
- |
- if (_playing) { |
- unsigned int noSamp10ms = _adbSampFreq / 100; |
- // todo: Member variable and allocate when samp freq is determined |
- int16_t* dataTmp = new int16_t[noSamp10ms]; |
- memset(dataTmp, 0, 2 * noSamp10ms); |
- unsigned int dataPos = 0; |
- int noSamplesOut = 0; |
- unsigned int nCopy = 0; |
- |
- // First insert data from playout buffer if any |
- if (_playoutBufferUsed > 0) { |
- nCopy = (dataSize < _playoutBufferUsed) ? dataSize : _playoutBufferUsed; |
- DCHECK_EQ(nCopy, _playoutBufferUsed); |
- memcpy(data, _playoutBuffer, 2 * nCopy); |
- dataPos = nCopy; |
- memset(_playoutBuffer, 0, sizeof(_playoutBuffer)); |
- _playoutBufferUsed = 0; |
- } |
- |
- // Now get the rest from Audio Device Buffer. |
- while (dataPos < dataSize) { |
- // Update playout delay |
- UpdatePlayoutDelay(); |
- |
- // Ask for new PCM data to be played out using the AudioDeviceBuffer |
- noSamplesOut = audio_device_buffer_->RequestPlayoutData(noSamp10ms); |
- |
- // Get data from Audio Device Buffer |
- noSamplesOut = audio_device_buffer_->GetPlayoutData( |
- reinterpret_cast<int8_t*>(dataTmp)); |
- CHECK_EQ(noSamp10ms, (unsigned int)noSamplesOut); |
- |
- // Insert as much as fits in data buffer |
- nCopy = |
- (dataSize - dataPos) > noSamp10ms ? noSamp10ms : (dataSize - dataPos); |
- memcpy(&data[dataPos], dataTmp, 2 * nCopy); |
- |
- // Save rest in playout buffer if any |
- if (nCopy < noSamp10ms) { |
- memcpy(_playoutBuffer, &dataTmp[nCopy], 2 * (noSamp10ms - nCopy)); |
- _playoutBufferUsed = noSamp10ms - nCopy; |
- } |
- |
- // Update loop/index counter, if we copied less than noSamp10ms |
- // samples we shall quit loop anyway |
- dataPos += noSamp10ms; |
- } |
- delete[] dataTmp; |
- } |
- return 0; |
-} |
- |
-// TODO(henrika): can either be removed or simplified. |
-void AudioDeviceIOS::UpdatePlayoutDelay() { |
- ++_playoutDelayMeasurementCounter; |
- |
- if (_playoutDelayMeasurementCounter >= 100) { |
- // Update HW and OS delay every second, unlikely to change |
- |
- // Since this is eventually rounded to integral ms, add 0.5ms |
- // here to get round-to-nearest-int behavior instead of |
- // truncation. |
- double totalDelaySeconds = 0.0005; |
- |
- // HW output latency |
- AVAudioSession* session = [AVAudioSession sharedInstance]; |
- double latency = session.outputLatency; |
- assert(latency >= 0); |
- totalDelaySeconds += latency; |
- |
- // HW buffer duration |
- double ioBufferDuration = session.IOBufferDuration; |
- assert(ioBufferDuration >= 0); |
- totalDelaySeconds += ioBufferDuration; |
- |
- // AU latency |
- Float64 f64(0); |
- UInt32 size = sizeof(f64); |
- OSStatus result = |
- AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_Latency, |
- kAudioUnitScope_Global, 0, &f64, &size); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "AU latency error: " << result; |
- } |
- assert(f64 >= 0); |
- totalDelaySeconds += f64; |
- |
- // To ms |
- _playoutDelay = static_cast<uint32_t>(totalDelaySeconds * 1000); |
- |
- // Reset counter |
- _playoutDelayMeasurementCounter = 0; |
- } |
- |
- // todo: Add playout buffer? |
-} |
- |
-void AudioDeviceIOS::UpdateRecordingDelay() { |
- ++_recordingDelayMeasurementCounter; |
- |
- if (_recordingDelayMeasurementCounter >= 100) { |
- // Update HW and OS delay every second, unlikely to change |
- |
- // Since this is eventually rounded to integral ms, add 0.5ms |
- // here to get round-to-nearest-int behavior instead of |
- // truncation. |
- double totalDelaySeconds = 0.0005; |
- |
- // HW input latency |
- AVAudioSession* session = [AVAudioSession sharedInstance]; |
- double latency = session.inputLatency; |
- assert(latency >= 0); |
- totalDelaySeconds += latency; |
- |
- // HW buffer duration |
- double ioBufferDuration = session.IOBufferDuration; |
- assert(ioBufferDuration >= 0); |
- totalDelaySeconds += ioBufferDuration; |
- |
- // AU latency |
- Float64 f64(0); |
- UInt32 size = sizeof(f64); |
- OSStatus result = |
- AudioUnitGetProperty(_auVoiceProcessing, kAudioUnitProperty_Latency, |
- kAudioUnitScope_Global, 0, &f64, &size); |
- if (0 != result) { |
- LOG_F(LS_ERROR) << "AU latency error: " << result; |
- } |
- assert(f64 >= 0); |
- totalDelaySeconds += f64; |
- |
- // To ms |
- _recordingDelayHWAndOS = static_cast<uint32_t>(totalDelaySeconds / 1000); |
- |
- // Reset counter |
- _recordingDelayMeasurementCounter = 0; |
- } |
- |
- _recordingDelay = _recordingDelayHWAndOS; |
- |
- // ADB recording buffer size, update every time |
- // Don't count the one next 10 ms to be sent, then convert samples => ms |
- const uint32_t noSamp10ms = _adbSampFreq / 100; |
- if (_recordingBufferTotalSize > noSamp10ms) { |
- _recordingDelay += |
- (_recordingBufferTotalSize - noSamp10ms) / (_adbSampFreq / 1000); |
- } |
-} |
- |
-bool AudioDeviceIOS::RunCapture(void* ptrThis) { |
- return static_cast<AudioDeviceIOS*>(ptrThis)->CaptureWorkerThread(); |
-} |
- |
-bool AudioDeviceIOS::CaptureWorkerThread() { |
- if (_recording) { |
- int bufPos = 0; |
- unsigned int lowestSeq = 0; |
- int lowestSeqBufPos = 0; |
- bool foundBuf = true; |
- const unsigned int noSamp10ms = _adbSampFreq / 100; |
- |
- while (foundBuf) { |
- // Check if we have any buffer with data to insert |
- // into the Audio Device Buffer, |
- // and find the one with the lowest seq number |
- foundBuf = false; |
- for (bufPos = 0; bufPos < N_REC_BUFFERS; ++bufPos) { |
- if (noSamp10ms == _recordingLength[bufPos]) { |
- if (!foundBuf) { |
- lowestSeq = _recordingSeqNumber[bufPos]; |
- lowestSeqBufPos = bufPos; |
- foundBuf = true; |
- } else if (_recordingSeqNumber[bufPos] < lowestSeq) { |
- lowestSeq = _recordingSeqNumber[bufPos]; |
- lowestSeqBufPos = bufPos; |
- } |
- } |
- } |
- |
- // Insert data into the Audio Device Buffer if found any |
- if (foundBuf) { |
- // Update recording delay |
- UpdateRecordingDelay(); |
- |
- // Set the recorded buffer |
- audio_device_buffer_->SetRecordedBuffer( |
- reinterpret_cast<int8_t*>(_recordingBuffer[lowestSeqBufPos]), |
- _recordingLength[lowestSeqBufPos]); |
- |
- // Don't need to set the current mic level in ADB since we only |
- // support digital AGC, |
- // and besides we cannot get or set the IOS mic level anyway. |
- |
- // Set VQE info, use clockdrift == 0 |
- audio_device_buffer_->SetVQEData(_playoutDelay, _recordingDelay, 0); |
- |
- // Deliver recorded samples at specified sample rate, mic level |
- // etc. to the observer using callback |
- audio_device_buffer_->DeliverRecordedData(); |
- |
- // Make buffer available |
- _recordingSeqNumber[lowestSeqBufPos] = 0; |
- _recordingBufferTotalSize -= _recordingLength[lowestSeqBufPos]; |
- // Must be done last to avoid interrupt problems between threads |
- _recordingLength[lowestSeqBufPos] = 0; |
- } |
- } |
- } |
- |
- { |
- // Normal case |
- // Sleep thread (5ms) to let other threads get to work |
- // todo: Is 5 ms optimal? Sleep shorter if inserted into the Audio |
- // Device Buffer? |
- timespec t; |
- t.tv_sec = 0; |
- t.tv_nsec = 5 * 1000 * 1000; |
- nanosleep(&t, nullptr); |
- } |
- return true; |
+OSStatus AudioDeviceIOS::OnGetPlayoutData( |
+ AudioUnitRenderActionFlags* ioActionFlags, |
+ UInt32 inNumberFrames, |
+ AudioBufferList* ioData) { |
+ // Verify 16-bit, noninterleaved mono PCM signal format. |
+ DCHECK_EQ(1u, ioData->mNumberBuffers); |
+ DCHECK_EQ(1u, ioData->mBuffers[0].mNumberChannels); |
+ // Get pointer to internal audio buffer to which new audio data shall be |
+ // written. |
+ const UInt32 dataSizeInBytes = ioData->mBuffers[0].mDataByteSize; |
+ CHECK_EQ(dataSizeInBytes / kBytesPerSample, inNumberFrames); |
+ SInt8* destination = static_cast<SInt8*>(ioData->mBuffers[0].mData); |
+ // Produce silence and give audio unit a hint about it if playout is not |
+ // activated. |
+ if (!rtc::AtomicOps::AcquireLoad(&_playing)) { |
+ *ioActionFlags |= kAudioUnitRenderAction_OutputIsSilence; |
+ memset(destination, 0, dataSizeInBytes); |
+ return noErr; |
+ } |
+ // Read decoded 16-bit PCM samples from WebRTC (using a size that matches |
+ // the native I/O audio unit) to a preallocated intermediate buffer and |
+ // copy the result to the audio buffer in the |ioData| destination. |
+ SInt8* source = _playoutAudioBuffer.get(); |
+ _fineAudioBuffer->GetPlayoutData(source); |
+ memcpy(destination, source, dataSizeInBytes); |
+ return noErr; |
} |
} // namespace webrtc |