Adds support for OpenSL ES based audio capture on Android

webrtc/modules/audio_device/android/opensles_player.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 26 matching lines @@
   } while (0)
 
 namespace webrtc {
 
 OpenSLESPlayer::OpenSLESPlayer(AudioManager* audio_manager)
     : audio_manager_(audio_manager),
       audio_parameters_(audio_manager->GetPlayoutAudioParameters()),
       audio_device_buffer_(nullptr),
       initialized_(false),
       playing_(false),
-      bytes_per_buffer_(0),
       buffer_index_(0),
       engine_(nullptr),
       player_(nullptr),
       simple_buffer_queue_(nullptr),
       volume_(nullptr),
       last_play_time_(0) {
   ALOGD("ctor%s", GetThreadInfo().c_str());
   // Use native audio output parameters provided by the audio manager and
   // define the PCM format structure.
   pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
@@ skipping 29 matching lines @@
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   StopPlayout();
   return 0;
 }
 
 int OpenSLESPlayer::InitPlayout() {
   ALOGD("InitPlayout%s", GetThreadInfo().c_str());
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   RTC_DCHECK(!initialized_);
   RTC_DCHECK(!playing_);
-  ObtainEngineInterface();
+  if (!ObtainEngineInterface()) {
+    ALOGE("Failed to obtain SL Engine interface");
+    return -1;
+  }
   CreateMix();
   initialized_ = true;
   buffer_index_ = 0;
-  last_play_time_ = rtc::Time();
   return 0;
 }
 
 int OpenSLESPlayer::StartPlayout() {
   ALOGD("StartPlayout%s", GetThreadInfo().c_str());
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   RTC_DCHECK(initialized_);
   RTC_DCHECK(!playing_);
+  if (fine_audio_buffer_) {
+    fine_audio_buffer_->ResetPlayout();
+  }
   // The number of lower latency audio players is limited, hence we create the
   // audio player in Start() and destroy it in Stop().
   CreateAudioPlayer();
   // Fill up audio buffers to avoid initial glitch and to ensure that playback
   // starts when mode is later changed to SL_PLAYSTATE_PLAYING.
   // TODO(henrika): we can save some delay by only making one call to
   // EnqueuePlayoutData. Most likely not worth the risk of adding a glitch.
+  last_play_time_ = rtc::Time();
   for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
     EnqueuePlayoutData();
   }
   // Start streaming data by setting the play state to SL_PLAYSTATE_PLAYING.
   // For a player object, when the object is in the SL_PLAYSTATE_PLAYING
   // state, adding buffers will implicitly start playback.
   RETURN_ON_ERROR((*player_)->SetPlayState(player_, SL_PLAYSTATE_PLAYING), -1);
   playing_ = (GetPlayState() == SL_PLAYSTATE_PLAYING);
   RTC_DCHECK(playing_);
   return 0;
@@ skipping 53 matching lines @@
   const int sample_rate_hz = audio_parameters_.sample_rate();
   ALOGD("SetPlayoutSampleRate(%d)", sample_rate_hz);
   audio_device_buffer_->SetPlayoutSampleRate(sample_rate_hz);
   const size_t channels = audio_parameters_.channels();
   ALOGD("SetPlayoutChannels(%" PRIuS ")", channels);
   audio_device_buffer_->SetPlayoutChannels(channels);
   RTC_CHECK(audio_device_buffer_);
   AllocateDataBuffers();
 }
 
-SLDataFormat_PCM OpenSLESPlayer::CreatePCMConfiguration(
-    size_t channels,
-    int sample_rate,
-    size_t bits_per_sample) {
-  ALOGD("CreatePCMConfiguration");
-  RTC_CHECK_EQ(bits_per_sample, SL_PCMSAMPLEFORMAT_FIXED_16);
-  SLDataFormat_PCM format;
-  format.formatType = SL_DATAFORMAT_PCM;
-  format.numChannels = static_cast<SLuint32>(channels);
-  // Note that, the unit of sample rate is actually in milliHertz and not Hertz.
-  switch (sample_rate) {
-    case 8000:
-      format.samplesPerSec = SL_SAMPLINGRATE_8;
-      break;
-    case 16000:
-      format.samplesPerSec = SL_SAMPLINGRATE_16;
-      break;
-    case 22050:
-      format.samplesPerSec = SL_SAMPLINGRATE_22_05;
-      break;
-    case 32000:
-      format.samplesPerSec = SL_SAMPLINGRATE_32;
-      break;
-    case 44100:
-      format.samplesPerSec = SL_SAMPLINGRATE_44_1;
-      break;
-    case 48000:
-      format.samplesPerSec = SL_SAMPLINGRATE_48;
-      break;
-    default:
-      RTC_CHECK(false) << "Unsupported sample rate: " << sample_rate;
-  }
-  format.bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_16;
-  format.containerSize = SL_PCMSAMPLEFORMAT_FIXED_16;
-  format.endianness = SL_BYTEORDER_LITTLEENDIAN;
-  if (format.numChannels == 1)
-    format.channelMask = SL_SPEAKER_FRONT_CENTER;
-  else if (format.numChannels == 2)
-    format.channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
-  else
-    RTC_CHECK(false) << "Unsupported number of channels: "
-                     << format.numChannels;
-  return format;
-}
-
 void OpenSLESPlayer::AllocateDataBuffers() {
   ALOGD("AllocateDataBuffers");
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   RTC_DCHECK(!simple_buffer_queue_);
   RTC_CHECK(audio_device_buffer_);
-  // Don't use the lowest possible size as native buffer size. Instead,
-  // use 10ms to better match the frame size that WebRTC uses. It will result
-  // in a reduced risk for audio glitches and also in a more "clean" sequence
-  // of callbacks from the OpenSL ES thread in to WebRTC when asking for audio
-  // to render.
-  ALOGD("lowest possible buffer size: %" PRIuS,
-      audio_parameters_.GetBytesPerBuffer());
-  bytes_per_buffer_ = audio_parameters_.GetBytesPerFrame() *
-      audio_parameters_.frames_per_10ms_buffer();
-  RTC_DCHECK_GE(bytes_per_buffer_, audio_parameters_.GetBytesPerBuffer());
-  ALOGD("native buffer size: %" PRIuS, bytes_per_buffer_);
   // Create a modified audio buffer class which allows us to ask for any number
   // of samples (and not only multiple of 10ms) to match the native OpenSL ES
-  // buffer size.
-  fine_buffer_.reset(new FineAudioBuffer(audio_device_buffer_,
-                                         bytes_per_buffer_,
-                                         audio_parameters_.sample_rate()));
+  // buffer size. The native buffer size corresponds to the
+  // PROPERTY_OUTPUT_FRAMES_PER_BUFFER property which is the number of audio
+  // frames that the HAL (Hardware Abstraction Layer) buffer can hold. It is
+  // recommended to construct audio buffers so that they contain an exact
+  // multiple of this number. If so, callbacks will occur at regular intervals,
+  // which reduces jitter.
+  ALOGD("native buffer size: %" PRIuS, audio_parameters_.GetBytesPerBuffer());
+  ALOGD("native buffer size in ms: %.2f",
+        audio_parameters_.GetBufferSizeInMilliseconds());
+  fine_audio_buffer_.reset(new FineAudioBuffer(
+      audio_device_buffer_, audio_parameters_.GetBytesPerBuffer(),
+      audio_parameters_.sample_rate()));
   // Each buffer must be of this size to avoid unnecessary memcpy while caching
   // data between successive callbacks.
   const size_t required_buffer_size =
-      fine_buffer_->RequiredPlayoutBufferSizeBytes();
+      fine_audio_buffer_->RequiredPlayoutBufferSizeBytes();
   ALOGD("required buffer size: %" PRIuS, required_buffer_size);
   for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
     audio_buffers_[i].reset(new SLint8[required_buffer_size]);
   }
 }
 
 bool OpenSLESPlayer::ObtainEngineInterface() {
   ALOGD("ObtainEngineInterface");
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
-  RTC_DCHECK(!engine_);
+  if (engine_)

[tommi, 2016/09/15 09:34:13]

is ObtainEngineInterface always called on the same thread?  Does it make sense
to start using thread checkers to assert this?

[henrika_webrtc, 2016/09/16 13:30:47]

Yes Sir, it is...and I do have a thread checker to ensure it :-)

+    return true;
   // Get access to (or create if not already existing) the global OpenSL Engine
   // object.
   SLObjectItf engine_object = audio_manager_->GetOpenSLEngine();
   if (engine_object == nullptr) {
     ALOGE("Failed to access the global OpenSL engine");
     return false;
   }
   // Get the SL Engine Interface which is implicit.
   RETURN_ON_ERROR(
       (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine_),
@@ skipping 107 matching lines @@
   // RETURN_ON_ERROR((*volume_)->SetVolumeLevel(volume_, 0), false);
 
   return true;
 }
 
 void OpenSLESPlayer::DestroyAudioPlayer() {
   ALOGD("DestroyAudioPlayer");
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   if (!player_object_.Get())
     return;
+  (*simple_buffer_queue_)
+      ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr);
   player_object_.Reset();
   player_ = nullptr;
   simple_buffer_queue_ = nullptr;
   volume_ = nullptr;
 }
 
 // static
 void OpenSLESPlayer::SimpleBufferQueueCallback(
     SLAndroidSimpleBufferQueueItf caller,
     void* context) {
   OpenSLESPlayer* stream = reinterpret_cast<OpenSLESPlayer*>(context);
   stream->FillBufferQueue();
 }
 
 void OpenSLESPlayer::FillBufferQueue() {
   RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread());
   SLuint32 state = GetPlayState();
   if (state != SL_PLAYSTATE_PLAYING) {
     ALOGW("Buffer callback in non-playing state!");
     return;
   }
   EnqueuePlayoutData();
 }
 
 void OpenSLESPlayer::EnqueuePlayoutData() {
   // Check delta time between two successive callbacks and provide a warning
   // if it becomes very large.
-  // TODO(henrika): using 100ms as upper limit but this value is rather random.
+  // TODO(henrika): using 150ms as upper limit but this value is rather random.
   const uint32_t current_time = rtc::Time();
   const uint32_t diff = current_time - last_play_time_;
-  if (diff > 100) {
+  if (diff > 150) {
     ALOGW("Bad OpenSL ES playout timing, dT=%u [ms]", diff);
   }
   last_play_time_ = current_time;
   // Read audio data from the WebRTC source using the FineAudioBuffer object
   // to adjust for differences in buffer size between WebRTC (10ms) and native
   // OpenSL ES.
   SLint8* audio_ptr = audio_buffers_[buffer_index_].get();
-  fine_buffer_->GetPlayoutData(audio_ptr);
+  fine_audio_buffer_->GetPlayoutData(audio_ptr);
   // Enqueue the decoded audio buffer for playback.
-  SLresult err =
-      (*simple_buffer_queue_)
-          ->Enqueue(simple_buffer_queue_, audio_ptr, bytes_per_buffer_);
+  SLresult err = (*simple_buffer_queue_)
+                     ->Enqueue(simple_buffer_queue_, audio_ptr,
+                               audio_parameters_.GetBytesPerBuffer());
   if (SL_RESULT_SUCCESS != err) {
     ALOGE("Enqueue failed: %d", err);
   }
   buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
 }
 
 SLuint32 OpenSLESPlayer::GetPlayState() const {
   RTC_DCHECK(player_);
   SLuint32 state;
   SLresult err = (*player_)->GetPlayState(player_, &state);
   if (SL_RESULT_SUCCESS != err) {
     ALOGE("GetPlayState failed: %d", err);
   }
   return state;
 }
 
 }  // namespace webrtc