Adds support for Bluetooth headsets to the iOS audio layer

webrtc/modules/audio_device/ios/audio_device_ios.mm

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 59 matching lines @@
 using ios::CheckAndLogError;
 
 // Activates an audio session suitable for full duplex VoIP sessions when
 // |activate| is true. Also sets the preferred sample rate and IO buffer
 // duration. Deactivates an active audio session if |activate| is set to false.
 static void ActivateAudioSession(AVAudioSession* session, bool activate) {
   LOG(LS_INFO) << "ActivateAudioSession(" << activate << ")";
   @autoreleasepool {
     NSError* error = nil;
     BOOL success = NO;
+
     // Deactivate the audio session and return if |activate| is false.
     if (!activate) {
       success = [session setActive:NO error:&error];
       RTC_DCHECK(CheckAndLogError(success, error));
       return;
     }
+
     // Use a category which supports simultaneous recording and playback.
     // By default, using this category implies that our app’s audio is
     // nonmixable, hence activating the session will interrupt any other
     // audio sessions which are also nonmixable.
     if (session.category != AVAudioSessionCategoryPlayAndRecord) {
       error = nil;
       success = [session setCategory:AVAudioSessionCategoryPlayAndRecord
                                error:&error];
       RTC_DCHECK(CheckAndLogError(success, error));
     }
+
     // Specify mode for two-way voice communication (e.g. VoIP).
     if (session.mode != AVAudioSessionModeVoiceChat) {
       error = nil;
       success = [session setMode:AVAudioSessionModeVoiceChat error:&error];
       RTC_DCHECK(CheckAndLogError(success, error));
     }
+
     // Set the session's sample rate or the hardware sample rate.
     // It is essential that we use the same sample rate as stream format
     // to ensure that the I/O unit does not have to do sample rate conversion.
     error = nil;
     success =
         [session setPreferredSampleRate:kPreferredSampleRate error:&error];
     RTC_DCHECK(CheckAndLogError(success, error));
+
     // Set the preferred audio I/O buffer duration, in seconds.
     // TODO(henrika): add more comments here.
     error = nil;
     success = [session setPreferredIOBufferDuration:kPreferredIOBufferDuration
                                               error:&error];
     RTC_DCHECK(CheckAndLogError(success, error));
 
-    // TODO(henrika): add observers here...
-
     // Activate the audio session. Activation can fail if another active audio
     // session (e.g. phone call) has higher priority than ours.
     error = nil;
     success = [session setActive:YES error:&error];
     RTC_DCHECK(CheckAndLogError(success, error));
     RTC_CHECK(session.isInputAvailable) << "No input path is available!";
+
     // Ensure that category and mode are actually activated.
     RTC_DCHECK(
         [session.category isEqualToString:AVAudioSessionCategoryPlayAndRecord]);
     RTC_DCHECK([session.mode isEqualToString:AVAudioSessionModeVoiceChat]);
+
     // Try to set the preferred number of hardware audio channels. These calls
     // must be done after setting the audio session’s category and mode and
     // activating the session.
     // We try to use mono in both directions to save resources and format
     // conversions in the audio unit. Some devices does only support stereo;
     // e.g. wired headset on iPhone 6.
     // TODO(henrika): add support for stereo if needed.
     error = nil;
     success =
         [session setPreferredInputNumberOfChannels:kPreferredNumberOfChannels
@@ skipping 259 matching lines @@
   // just in case.
   RTC_DCHECK(audio_device_buffer_) << "AttachAudioBuffer must be called first";
   // Inform the audio device buffer (ADB) about the new audio format.
   audio_device_buffer_->SetPlayoutSampleRate(playout_parameters_.sample_rate());
   audio_device_buffer_->SetPlayoutChannels(playout_parameters_.channels());
   audio_device_buffer_->SetRecordingSampleRate(
       record_parameters_.sample_rate());
   audio_device_buffer_->SetRecordingChannels(record_parameters_.channels());
 }
 
+void AudioDeviceIOS::RegisterNotificationObservers() {
+  LOGI() << "RegisterNotificationObservers";
+  // Get the default notification center of the current process.
+  NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
+
+  // Add AVAudioSessionInterruptionNotification observer.
+  // TODO(henrika): improve this section and try to merge actions with actions

[tkchin_webrtc, 2015/10/16 16:23:32]

imo we should make changes to depot_tools to allow 100 lines for objc/objc++
files

instead, to make things here more readable, suggest making a var for the block:

void (^interruption_block)(NSNotification *) = ^(NSNotification *) {
 ...
};

id interruption_observer =
    [center addObserverForName:AVAudioSessionInterruptionNotification
                        object:nil
                         queue:
                    usingBlock:interruption_block];

likewise for route_change_observer

[henrika_webrtc, 2015/10/20 10:18:16]

No action on adding 100 lines support. 
Thanks for the tip on improving the block structure.

+  // for the detected route change.
+  id interruption_observer = [center
+      addObserverForName:AVAudioSessionInterruptionNotification
+                  object:nil
+                   queue:[NSOperationQueue mainQueue]
+              usingBlock:^(NSNotification* notification) {
+                NSNumber* typeNumber =
+                    [notification userInfo][AVAudioSessionInterruptionTypeKey];

[tkchin_webrtc, 2015/10/16 16:23:32]

style:
notification.userInfo[AVAudioSessionInterruptionKey]
typeNumber.unsignedIntegerValue

[henrika_webrtc, 2015/10/20 10:18:16]

Done.

+                AVAudioSessionInterruptionType type =
+                    (AVAudioSessionInterruptionType)[typeNumber
+                                                         unsignedIntegerValue];
+                switch (type) {
+                  case AVAudioSessionInterruptionTypeBegan:
+                    // At this point our audio session has been deactivated and
+                    // the audio unit render callbacks no longer occur.
+                    // Nothing to do.
+                    break;
+                  case AVAudioSessionInterruptionTypeEnded: {
+                    NSError* error = nil;
+                    AVAudioSession* session = [AVAudioSession sharedInstance];
+                    [session setActive:YES error:&error];
+                    if (error != nil) {
+                      LOG_F(LS_ERROR) << "Failed to active audio session";
+                    }
+                    // Post interruption the audio unit render callbacks don't

[tkchin_webrtc, 2015/10/16 16:23:32]

Is this still true today?

[henrika_webrtc, 2015/10/20 10:18:16]

See TODO above. I will check and make changes in this CL if required.

+                    // automatically continue, so we restart the unit manually
+                    // here.
+                    AudioOutputUnitStop(vpio_unit_);
+                    AudioOutputUnitStart(vpio_unit_);
+                    break;
+                  }
+                }
+              }];
+
+  // Add AVAudioSessionRouteChangeNotification observer.
+  id route_change_observer = [center
+      addObserverForName:AVAudioSessionRouteChangeNotification
+                  object:nil
+                   queue:[NSOperationQueue mainQueue]
+              usingBlock:^(NSNotification* notification) {
+                // Get reason for current route change.
+                NSUInteger reason_value = [[notification.userInfo

[tkchin_webrtc, 2015/10/16 16:23:32]

style:
NSNumber *reason_number =
notification.userInfo[AVAudioSessionRouteChangeReasonKey];

Cast to the enum so that switch won't need default:

AVAudioSessionRouteChangeReason reason =
(AVAudioSessionRouteChangeReason)reason_number.unsignedIntegerValue;
switch (reason) {
  ...
}

[henrika_webrtc, 2015/10/20 10:18:16]

Done.

+                    valueForKey:AVAudioSessionRouteChangeReasonKey]
+                    unsignedIntegerValue];
+                bool valid_route_change = true;
+                LOG(LS_INFO) << "Route change:";
+                switch (reason_value) {
+                  case AVAudioSessionRouteChangeReasonNewDeviceAvailable:
+                    LOG(LS_INFO) << " NewDeviceAvailable";
+                    break;
+                  case AVAudioSessionRouteChangeReasonOldDeviceUnavailable:
+                    LOG(LS_INFO) << " OldDeviceUnavailable";
+                    break;
+                  case AVAudioSessionRouteChangeReasonCategoryChange:
+                    LOG(LS_INFO) << " CategoryChange";
+                    LOG(LS_INFO) << " New category: "
+                                 << ios::GetAudioSessionCategory();
+                    break;
+                  case AVAudioSessionRouteChangeReasonOverride:
+                    LOG(LS_INFO) << " Override";
+                    break;
+                  case AVAudioSessionRouteChangeReasonWakeFromSleep:
+                    LOG(LS_INFO) << " WakeFromSleep";
+                    break;
+                  case AVAudioSessionRouteChangeReasonRouteConfigurationChange:
+                    // Ignore this type of route change since we are focusing
+                    // on detecting headset changes.
+                    LOG(LS_INFO) << " RouteConfigurationChange";
+                    valid_route_change = false;
+                    break;
+                  default:
+                    LOG(LS_INFO) << " ReasonUnknown";
+                }
+
+                if (valid_route_change) {
+                  // Log previous route configuration.
+                  AVAudioSessionRouteDescription* prev_route = [notification
+                      userInfo][AVAudioSessionRouteChangePreviousRouteKey];

[tkchin_webrtc, 2015/10/16 16:23:32]

ditto .userInfo

[henrika_webrtc, 2015/10/20 10:18:16]

Done.

+                  LOG(LS_INFO) << "Previous route:";
+                  LOG(LS_INFO) << ios::StdStringFromNSString(
+                      [NSString stringWithFormat:@"%@", prev_route]);
+
+                  // Only restart audio for a valid route change and if the
+                  // session sample rate has changed.
+                  const double session_sample_rate =
+                      [[AVAudioSession sharedInstance] sampleRate];

[tkchin_webrtc, 2015/10/16 16:23:32]

ditto dot syntax for properties

[henrika_webrtc, 2015/10/20 10:18:16]

Done.

+                  if (playout_parameters_.sample_rate() !=
+                      session_sample_rate) {
+                    if (!RestartAudioUnitWithNewFormat(session_sample_rate)) {
+                      LOG(LS_ERROR) << "Audio restart failed";
+                    }
+                  }
+                }
+
+              }];
+
+  // Increment refcount on observers using ARC bridge. Instance variable is a
+  // void* instead of an id because header is included in other pure C++
+  // files.
+  audio_interruption_observer_ = (__bridge_retained void*)interruption_observer;
+  route_change_observer_ = (__bridge_retained void*)route_change_observer;
+}
+
+void AudioDeviceIOS::UnregisterNotificationObservers() {
+  LOGI() << "UnregisterNotificationObservers";
+  // Transfer ownership of observer back to ARC, which will deallocate the
+  // observer once it exits this scope.
+  NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
+  if (audio_interruption_observer_ != nullptr) {
+    id observer = (__bridge_transfer id)audio_interruption_observer_;
+    [center removeObserver:observer];
+    audio_interruption_observer_ = nullptr;
+  }
+  if (route_change_observer_ != nullptr) {
+    id observer = (__bridge_transfer id)route_change_observer_;
+    [center removeObserver:observer];
+    route_change_observer_ = nullptr;
+  }
+}
+
 void AudioDeviceIOS::SetupAudioBuffersForActiveAudioSession() {
   LOGI() << "SetupAudioBuffersForActiveAudioSession";
+  // Verify the current values once the audio session has been activated.
   AVAudioSession* session = [AVAudioSession sharedInstance];
-  // Verify the current values once the audio session has been activated.
   LOG(LS_INFO) << " sample rate: " << session.sampleRate;
   LOG(LS_INFO) << " IO buffer duration: " << session.IOBufferDuration;
   LOG(LS_INFO) << " output channels: " << session.outputNumberOfChannels;
   LOG(LS_INFO) << " input channels: " << session.inputNumberOfChannels;
   LOG(LS_INFO) << " output latency: " << session.outputLatency;
   LOG(LS_INFO) << " input latency: " << session.inputLatency;
+
   // Log a warning message for the case when we are unable to set the preferred
   // hardware sample rate but continue and use the non-ideal sample rate after
-  // reinitializing the audio parameters.
-  if (session.sampleRate != playout_parameters_.sample_rate()) {
-    LOG(LS_WARNING)
-        << "Failed to enable an audio session with the preferred sample rate!";
+  // reinitializing the audio parameters. Most BT headsets only support 8kHz or
+  // 16kHz.
+  if (session.sampleRate != kPreferredSampleRate) {
+    LOG(LS_WARNING) << "Unable to set the preferred sample rate";
   }
 
   // At this stage, we also know the exact IO buffer duration and can add
   // that info to the existing audio parameters where it is converted into
   // number of audio frames.
   // Example: IO buffer size = 0.008 seconds <=> 128 audio frames at 16kHz.
   // Hence, 128 is the size we expect to see in upcoming render callbacks.
   playout_parameters_.reset(session.sampleRate, playout_parameters_.channels(),
                             session.IOBufferDuration);
   RTC_DCHECK(playout_parameters_.is_complete());
@@ skipping 92 matching lines @@
   RTC_DCHECK_EQ(1, kPreferredNumberOfChannels);
   application_format.mSampleRate = playout_parameters_.sample_rate();
   application_format.mFormatID = kAudioFormatLinearPCM;
   application_format.mFormatFlags =
       kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked;
   application_format.mBytesPerPacket = kBytesPerSample;
   application_format.mFramesPerPacket = 1;  // uncompressed
   application_format.mBytesPerFrame = kBytesPerSample;
   application_format.mChannelsPerFrame = kPreferredNumberOfChannels;
   application_format.mBitsPerChannel = 8 * kBytesPerSample;
+  // Store the new format.
+  application_format_ = application_format;
 #if !defined(NDEBUG)
-  LogABSD(application_format);
+  LogABSD(application_format_);
 #endif
 
   // Set the application format on the output scope of the input element/bus.
   LOG_AND_RETURN_IF_ERROR(
       AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
                            kAudioUnitScope_Output, input_bus,
                            &application_format, size),
       "Failed to set application format on output scope of input element");
 
   // Set the application format on the input scope of the output element/bus.
@@ skipping 35 matching lines @@
                            kAudioUnitScope_Global, input_bus, &input_callback,
                            sizeof(input_callback)),
       "Failed to specify the input callback on the input element");
 
   // Initialize the Voice-Processing I/O unit instance.
   LOG_AND_RETURN_IF_ERROR(AudioUnitInitialize(vpio_unit_),
                           "Failed to initialize the Voice-Processing I/O unit");
   return true;
 }
 
+bool AudioDeviceIOS::RestartAudioUnitWithNewFormat(float sample_rate) {
+  LOGI() << "RestartAudioUnitWithNewFormat(sample_rate=" << sample_rate << ")";
+  // Stop the active audio unit.
+  LOG_AND_RETURN_IF_ERROR(AudioOutputUnitStop(vpio_unit_),
+                          "Failed to stop the the Voice-Processing I/O unit");
+
+  // The stream format is about to be changed and it requires that we first
+  // uninitialize it to deallocate its resources.
+  LOG_AND_RETURN_IF_ERROR(
+      AudioUnitUninitialize(vpio_unit_),
+      "Failed to uninitialize the the Voice-Processing I/O unit");
+
+  // Allocate new buffers given the new stream format.
+  SetupAudioBuffersForActiveAudioSession();
+
+  // Update the existing application format using the new sample rate.
+  application_format_.mSampleRate = playout_parameters_.sample_rate();
+  UInt32 size = sizeof(application_format_);
+  AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
+                       kAudioUnitScope_Output, 1, &application_format_, size);
+  AudioUnitSetProperty(vpio_unit_, kAudioUnitProperty_StreamFormat,
+                       kAudioUnitScope_Input, 0, &application_format_, size);
+
+  // Prepare the audio unit to render audio again.
+  LOG_AND_RETURN_IF_ERROR(AudioUnitInitialize(vpio_unit_),
+                          "Failed to initialize the Voice-Processing I/O unit");
+
+  // Start rendering audio using the new format.
+  LOG_AND_RETURN_IF_ERROR(AudioOutputUnitStart(vpio_unit_),
+                          "Failed to start the Voice-Processing I/O unit");
+  return true;
+}
+
 bool AudioDeviceIOS::InitPlayOrRecord() {
   LOGI() << "InitPlayOrRecord";
   AVAudioSession* session = [AVAudioSession sharedInstance];
   // Activate the audio session and ask for a set of preferred audio parameters.
   ActivateAudioSession(session, true);
 
+  // Start observing audio session interruptions and route changes.
+  RegisterNotificationObservers();
+
   // Ensure that we got what what we asked for in our active audio session.
   SetupAudioBuffersForActiveAudioSession();
 
   // Create, setup and initialize a new Voice-Processing I/O unit.
   if (!SetupAndInitializeVoiceProcessingAudioUnit()) {
     return false;
   }
-
-  // Listen to audio interruptions.
-  // TODO(henrika): learn this area better.
-  NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
-  id observer = [center
-      addObserverForName:AVAudioSessionInterruptionNotification
-                  object:nil
-                   queue:[NSOperationQueue mainQueue]
-              usingBlock:^(NSNotification* notification) {
-                NSNumber* typeNumber =
-                    [notification userInfo][AVAudioSessionInterruptionTypeKey];
-                AVAudioSessionInterruptionType type =
-                    (AVAudioSessionInterruptionType)[typeNumber
-                                                         unsignedIntegerValue];
-                switch (type) {
-                  case AVAudioSessionInterruptionTypeBegan:
-                    // At this point our audio session has been deactivated and
-                    // the audio unit render callbacks no longer occur.
-                    // Nothing to do.
-                    break;
-                  case AVAudioSessionInterruptionTypeEnded: {
-                    NSError* error = nil;
-                    AVAudioSession* session = [AVAudioSession sharedInstance];
-                    [session setActive:YES error:&error];
-                    if (error != nil) {
-                      LOG_F(LS_ERROR) << "Failed to active audio session";
-                    }
-                    // Post interruption the audio unit render callbacks don't
-                    // automatically continue, so we restart the unit manually
-                    // here.
-                    AudioOutputUnitStop(vpio_unit_);
-                    AudioOutputUnitStart(vpio_unit_);
-                    break;
-                  }
-                }
-              }];
-  // Increment refcount on observer using ARC bridge. Instance variable is a
-  // void* instead of an id because header is included in other pure C++
-  // files.
-  audio_interruption_observer_ = (__bridge_retained void*)observer;
   return true;
 }
 
 bool AudioDeviceIOS::ShutdownPlayOrRecord() {
   LOGI() << "ShutdownPlayOrRecord";
-  if (audio_interruption_observer_ != nullptr) {
-    NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
-    // Transfer ownership of observer back to ARC, which will dealloc the
-    // observer once it exits this scope.
-    id observer = (__bridge_transfer id)audio_interruption_observer_;
-    [center removeObserver:observer];
-    audio_interruption_observer_ = nullptr;
-  }
+  // Remove audio session notification observers.
+  UnregisterNotificationObservers();
+
   // Close and delete the voice-processing I/O unit.
   OSStatus result = -1;
   if (nullptr != vpio_unit_) {
     result = AudioOutputUnitStop(vpio_unit_);
     if (result != noErr) {
       LOG_F(LS_ERROR) << "AudioOutputUnitStop failed: " << result;
     }
+    result = AudioUnitUninitialize(vpio_unit_);

[tkchin_webrtc, 2015/10/16 16:23:32]

oh dear, were we really not doing this before? Thx for fixing.

[henrika_webrtc, 2015/10/20 10:18:16]

You are welcome ;-)

+    if (result != noErr) {
+      LOG_F(LS_ERROR) << "AudioUnitUninitialize failed: " << result;
+    }
     result = AudioComponentInstanceDispose(vpio_unit_);
     if (result != noErr) {
       LOG_F(LS_ERROR) << "AudioComponentInstanceDispose failed: " << result;
     }
     vpio_unit_ = nullptr;
   }
+
   // All I/O should be stopped or paused prior to deactivating the audio
   // session, hence we deactivate as last action.
   AVAudioSession* session = [AVAudioSession sharedInstance];
   ActivateAudioSession(session, false);
   return true;
 }
 
 OSStatus AudioDeviceIOS::RecordedDataIsAvailable(
     void* in_ref_con,
     AudioUnitRenderActionFlags* io_action_flags,
     const AudioTimeStamp* in_time_stamp,
     UInt32 in_bus_number,
     UInt32 in_number_frames,
     AudioBufferList* io_data) {
   RTC_DCHECK_EQ(1u, in_bus_number);
   RTC_DCHECK(
       !io_data);  // no buffer should be allocated for input at this stage
   AudioDeviceIOS* audio_device_ios = static_cast<AudioDeviceIOS*>(in_ref_con);
   return audio_device_ios->OnRecordedDataIsAvailable(
       io_action_flags, in_time_stamp, in_bus_number, in_number_frames);
 }
 
 OSStatus AudioDeviceIOS::OnRecordedDataIsAvailable(
     AudioUnitRenderActionFlags* io_action_flags,
     const AudioTimeStamp* in_time_stamp,
     UInt32 in_bus_number,
     UInt32 in_number_frames) {
-  RTC_DCHECK_EQ(record_parameters_.frames_per_buffer(), in_number_frames);
   OSStatus result = noErr;
   // Simply return if recording is not enabled.
   if (!rtc::AtomicOps::AcquireLoad(&recording_))
     return result;
   RTC_DCHECK_EQ(record_parameters_.frames_per_buffer(), in_number_frames);
   // Obtain the recorded audio samples by initiating a rendering cycle.
   // Since it happens on the input bus, the |io_data| parameter is a reference
   // to the preallocated audio buffer list that the audio unit renders into.
   // TODO(henrika): should error handling be improved?
   AudioBufferList* io_data = &audio_record_buffer_list_;
@@ skipping 51 matching lines @@
   // Read decoded 16-bit PCM samples from WebRTC (using a size that matches
   // the native I/O audio unit) to a preallocated intermediate buffer and
   // copy the result to the audio buffer in the |io_data| destination.
   SInt8* source = playout_audio_buffer_.get();
   fine_audio_buffer_->GetPlayoutData(source);
   memcpy(destination, source, dataSizeInBytes);
   return noErr;
 }
 
 }  // namespace webrtc