Adds support for OpenSL ES based audio capture on Android

webrtc/modules/audio_device/android/opensles_recorder.cc

Index: webrtc/modules/audio_device/android/opensles_recorder.cc
diff --git a/webrtc/modules/audio_device/android/opensles_recorder.cc b/webrtc/modules/audio_device/android/opensles_recorder.cc
new file mode 100644
index 0000000000000000000000000000000000000000..67d42b9d687bb02f6628d2aa7edaffa4aa5c572a
--- /dev/null
+++ b/webrtc/modules/audio_device/android/opensles_recorder.cc
@@ -0,0 +1,405 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/audio_device/android/opensles_recorder.h"
+
+#include <android/log.h>
+
+#include "webrtc/base/arraysize.h"
+#include "webrtc/base/checks.h"
+#include "webrtc/base/format_macros.h"
+#include "webrtc/base/timeutils.h"
+#include "webrtc/modules/audio_device/android/audio_common.h"
+#include "webrtc/modules/audio_device/android/audio_manager.h"
+#include "webrtc/modules/audio_device/fine_audio_buffer.h"
+
+#define TAG "OpenSLESRecorder"
+#define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
+#define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
+#define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
+#define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
+#define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
+
+#define RETURN_ON_ERROR(op, ...)                          \

[tommi, 2016/09/15 09:34:13]

I really prefer not to have macros with return statements :-/  also, macros
should in general be avoided.
Can the code be refactored to avoid this?

[henrika_webrtc, 2016/09/16 13:30:47]

Fixed and discussed offline.

+  do {                                                    \
+    SLresult err = (op);                                  \
+    if (err != SL_RESULT_SUCCESS) {                       \
+      ALOGE("%s failed: %s", #op, GetSLErrorString(err)); \
+      return __VA_ARGS__;                                 \

[tommi, 2016/09/15 09:34:13]

why __VA_ARGS__?

[henrika_webrtc, 2016/09/16 13:30:47]

Now removed.

+    }                                                     \
+  } while (0)
+
+namespace webrtc {
+
+OpenSLESRecorder::OpenSLESRecorder(AudioManager* audio_manager)
+    : audio_manager_(audio_manager),
+      audio_parameters_(audio_manager->GetRecordAudioParameters()),
+      audio_device_buffer_(nullptr),
+      initialized_(false),
+      recording_(false),
+      engine_(nullptr),
+      recorder_(nullptr),
+      simple_buffer_queue_(nullptr),
+      buffer_index_(0),
+      last_rec_time_(0) {
+  ALOGD("ctor%s", GetThreadInfo().c_str());
+  // Use native audio output parameters provided by the audio manager and
+  // define the PCM format structure.
+  pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
+                                       audio_parameters_.sample_rate(),
+                                       audio_parameters_.bits_per_sample());
+  // Detach from this thread since we want to use the checker to verify calls
+  // from the internal  audio thread.
+  thread_checker_opensles_.DetachFromThread();

[tommi, 2016/09/15 09:34:13]

nit: do this at the top of the ctor

[henrika_webrtc, 2016/09/16 13:30:47]

Done.

+}
+
+OpenSLESRecorder::~OpenSLESRecorder() {
+  ALOGD("dtor%s", GetThreadInfo().c_str());
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  Terminate();
+  DestroyAudioRecorder();
+  engine_ = nullptr;
+  RTC_DCHECK(!engine_);
+  RTC_DCHECK(!recorder_);
+  RTC_DCHECK(!simple_buffer_queue_);
+}
+
+int OpenSLESRecorder::Init() {
+  ALOGD("Init%s", GetThreadInfo().c_str());
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  return 0;
+}
+
+int OpenSLESRecorder::Terminate() {
+  ALOGD("Terminate%s", GetThreadInfo().c_str());
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  StopRecording();
+  return 0;
+}
+
+int OpenSLESRecorder::InitRecording() {
+  ALOGD("InitRecording%s", GetThreadInfo().c_str());
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  RTC_DCHECK(!initialized_);
+  RTC_DCHECK(!recording_);
+  if (!ObtainEngineInterface()) {
+    ALOGE("Failed to obtain SL Engine interface");
+    return -1;
+  }
+  CreateAudioRecorder();
+  initialized_ = true;
+  buffer_index_ = 0;
+  return 0;
+}
+
+int OpenSLESRecorder::StartRecording() {
+  ALOGD("StartRecording%s", GetThreadInfo().c_str());
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  RTC_DCHECK(initialized_);
+  RTC_DCHECK(!recording_);
+  if (fine_audio_buffer_) {
+    fine_audio_buffer_->ResetRecord();
+  }
+  // Add buffers to the queue before changing state to SL_RECORDSTATE_RECORDING
+  // to ensure that recording starts as soon as the state is modified. On some
+  // devices, SLAndroidSimpleBufferQueue::Clear() used in Stop() does not flush
+  // the buffers as intended and we therefore check the number of buffers
+  // already queued first. Enqueue() can return SL_RESULT_BUFFER_INSUFFICIENT
+  // otherwise.
+  int num_buffers_in_queue = GetBufferCount();
+  for (int i = 0; i < kNumOfOpenSLESBuffers - num_buffers_in_queue; ++i) {
+    if (!EnqueueAudioBuffer()) {
+      recording_ = false;
+      return -1;
+    }
+  }
+  num_buffers_in_queue = GetBufferCount();
+  RTC_DCHECK_EQ(num_buffers_in_queue, kNumOfOpenSLESBuffers);
+  LogBufferState();
+  // Start audio recording by changing the state to SL_RECORDSTATE_RECORDING.
+  // Given that buffers are already enqueued, recording should start at once.
+  last_rec_time_ = rtc::Time();
+  RETURN_ON_ERROR(
+      (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_RECORDING), -1);

[tommi, 2016/09/15 09:34:13]

it's not clear from reading the code what the -1 value is

[henrika_webrtc, 2016/09/16 13:30:48]

It is the returned value and we use 0/-1 to signal OK/ERROR. Not sure how to
make that more clear. Adding a comment as well.

+  recording_ = (GetRecordState() == SL_RECORDSTATE_RECORDING);
+  RTC_DCHECK(recording_);
+  return 0;
+}
+
+int OpenSLESRecorder::StopRecording() {
+  ALOGD("StopRecording%s", GetThreadInfo().c_str());
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  if (!initialized_ || !recording_) {
+    return 0;
+  }
+  // Stop recording by setting the record state to SL_RECORDSTATE_STOPPED.
+  RETURN_ON_ERROR(
+      (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_STOPPED), -1);
+  // Clear the buffer queue to get rid of old data when resuming recording.
+  RETURN_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_), -1);
+  thread_checker_opensles_.DetachFromThread();
+  initialized_ = false;
+  recording_ = false;
+  return 0;
+}
+
+void OpenSLESRecorder::AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) {
+  ALOGD("AttachAudioBuffer");

[tommi, 2016/09/15 09:34:13]

nit: the function looks like a wall of code. If there are logical parts that
belong together with a log statement, it would make it a little more readable if
there's whitespace that separates those parts.

[henrika_webrtc, 2016/09/16 13:30:47]

Thanks. Will improve.

+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  audio_device_buffer_ = audioBuffer;
+  const int sample_rate_hz = audio_parameters_.sample_rate();
+  ALOGD("SetRecordingSampleRate(%d)", sample_rate_hz);
+  audio_device_buffer_->SetRecordingSampleRate(sample_rate_hz);
+  const size_t channels = audio_parameters_.channels();
+  ALOGD("SetRecordingChannels(%" PRIuS ")", channels);
+  audio_device_buffer_->SetRecordingChannels(channels);
+  RTC_CHECK(audio_device_buffer_);
+  AllocateDataBuffers();
+}
+
+int OpenSLESRecorder::EnableBuiltInAEC(bool enable) {
+  ALOGD("EnableBuiltInAEC(%d)", enable);
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  ALOGE("Not implemented");
+  return 0;
+}
+
+int OpenSLESRecorder::EnableBuiltInAGC(bool enable) {
+  ALOGD("EnableBuiltInAGC(%d)", enable);
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  ALOGE("Not implemented");
+  return 0;
+}
+
+int OpenSLESRecorder::EnableBuiltInNS(bool enable) {
+  ALOGD("EnableBuiltInNS(%d)", enable);
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  ALOGE("Not implemented");
+  return 0;
+}
+
+bool OpenSLESRecorder::ObtainEngineInterface() {
+  ALOGD("ObtainEngineInterface");
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  if (engine_)
+    return true;
+  // Get access to (or create if not already existing) the global OpenSL Engine
+  // object.
+  SLObjectItf engine_object = audio_manager_->GetOpenSLEngine();
+  if (engine_object == nullptr) {
+    ALOGE("Failed to access the global OpenSL engine");
+    return false;
+  }
+  // Get the SL Engine Interface which is implicit.
+  RETURN_ON_ERROR(
+      (*engine_object)->GetInterface(engine_object, SL_IID_ENGINE, &engine_),
+      false);
+  return true;
+}
+
+bool OpenSLESRecorder::CreateAudioRecorder() {
+  ALOGD("CreateAudioRecorder");
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  if (recorder_object_.Get())
+    return true;
+  RTC_DCHECK(!recorder_);
+  RTC_DCHECK(!simple_buffer_queue_);
+
+  // Audio source configuration.
+  SLDataLocator_IODevice mic_locator = {SL_DATALOCATOR_IODEVICE,
+                                        SL_IODEVICE_AUDIOINPUT,
+                                        SL_DEFAULTDEVICEID_AUDIOINPUT, NULL};
+  SLDataSource audio_source = {&mic_locator, NULL};
+
+  // Audio sink configuration.
+  SLDataLocator_AndroidSimpleBufferQueue buffer_queue = {
+      SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
+      static_cast<SLuint32>(kNumOfOpenSLESBuffers)};
+  SLDataSink audio_sink = {&buffer_queue, &pcm_format_};
+
+  // Create the audio recorder object (requires the RECORD_AUDIO permission).
+  // Do not realize the recorder yet. Set the configuration first.
+  const SLInterfaceID interface_id[] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
+                                        SL_IID_ANDROIDCONFIGURATION};
+  const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
+  RETURN_ON_ERROR(
+      (*engine_)->CreateAudioRecorder(
+          engine_, recorder_object_.Receive(), &audio_source, &audio_sink,
+          arraysize(interface_id), interface_id, interface_required),
+      false);
+
+  // Configure the audio recorder (before it is realized).
+  SLAndroidConfigurationItf recorder_config;
+  RETURN_ON_ERROR(recorder_object_->GetInterface(recorder_object_.Get(),
+                                                 SL_IID_ANDROIDCONFIGURATION,
+                                                 &recorder_config),
+                  false);
+
+  // Uses the default microphone tuned for audio communication.
+  // Note that, SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION leads to a fast
+  // track but also excludes usage of required effects like AEC, AGC and NS.
+  // SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION
+  SLint32 stream_type = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
+  RETURN_ON_ERROR(
+      (*recorder_config)
+          ->SetConfiguration(recorder_config, SL_ANDROID_KEY_RECORDING_PRESET,
+                             &stream_type, sizeof(SLint32)),
+      false);
+
+  // The audio recorder can now be realized (in synchronous mode).
+  RETURN_ON_ERROR(
+      recorder_object_->Realize(recorder_object_.Get(), SL_BOOLEAN_FALSE),
+      false);
+
+  // Get the implicit recorder interface (SL_IID_RECORD).
+  RETURN_ON_ERROR(recorder_object_->GetInterface(recorder_object_.Get(),
+                                                 SL_IID_RECORD, &recorder_),
+                  false);
+
+  // Get the simple buffer queue interface (SL_IID_ANDROIDSIMPLEBUFFERQUEUE).
+  // It was explicitly requested.
+  RETURN_ON_ERROR(recorder_object_->GetInterface(
+                      recorder_object_.Get(), SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
+                      &simple_buffer_queue_),
+                  false);
+
+  // Register the input callback for the simple buffer queue.
+  // This callback will be called when receiving new data from the device.
+  RETURN_ON_ERROR((*simple_buffer_queue_)
+                      ->RegisterCallback(simple_buffer_queue_,
+                                         SimpleBufferQueueCallback, this),
+                  false);
+  return true;
+}
+
+void OpenSLESRecorder::DestroyAudioRecorder() {
+  ALOGD("DestroyAudioRecorder");
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  if (!recorder_object_.Get())
+    return;
+  (*simple_buffer_queue_)
+      ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr);
+  recorder_object_.Reset();
+  recorder_ = nullptr;
+  simple_buffer_queue_ = nullptr;
+}
+
+void OpenSLESRecorder::SimpleBufferQueueCallback(
+    SLAndroidSimpleBufferQueueItf buffer_queue,
+    void* context) {
+  OpenSLESRecorder* stream = reinterpret_cast<OpenSLESRecorder*>(context);

[tommi, 2016/09/15 09:34:13]

nit: use static_cast

[henrika_webrtc, 2016/09/16 13:30:47]

Done.

+  stream->ReadBufferQueue();
+}
+
+void OpenSLESRecorder::AllocateDataBuffers() {
+  ALOGD("AllocateDataBuffers");
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  RTC_DCHECK(!simple_buffer_queue_);
+  RTC_CHECK(audio_device_buffer_);
+  // Create a modified audio buffer class which allows us to deliver any number
+  // of samples (and not only multiple of 10ms) to match the native audio unit
+  // buffer size.
+  ALOGD("frames per native buffer: %" PRIuS,
+        audio_parameters_.frames_per_buffer());
+  ALOGD("frames per 10ms buffer: %" PRIuS,
+        audio_parameters_.frames_per_10ms_buffer());
+  ALOGD("bytes per native buffer: %" PRIuS,
+        audio_parameters_.GetBytesPerBuffer());
+  ALOGD("native sample rate: %d", audio_parameters_.sample_rate());
+  RTC_DCHECK(audio_device_buffer_);
+  fine_audio_buffer_.reset(new FineAudioBuffer(
+      audio_device_buffer_, audio_parameters_.GetBytesPerBuffer(),
+      audio_parameters_.sample_rate()));
+  // Allocate queue of audio buffers that stores recorded audio samples.
+  const int data_size_bytes = audio_parameters_.GetBytesPerBuffer();
+  audio_buffers_.reset(new std::unique_ptr<SLint8[]>[kNumOfOpenSLESBuffers]);
+  for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
+    audio_buffers_[i].reset(new SLint8[data_size_bytes]);
+  }
+}
+
+void OpenSLESRecorder::ReadBufferQueue() {
+  RTC_DCHECK(thread_checker_opensles_.CalledOnValidThread());
+  SLuint32 state = GetRecordState();
+  if (state != SL_RECORDSTATE_RECORDING) {
+    ALOGW("Buffer callback in non-recording state!");
+    return;
+  }
+  // Check delta time between two successive callbacks and provide a warning
+  // if it becomes very large.
+  // TODO(henrika): using 150ms as upper limit but this value is rather random.
+  const uint32_t current_time = rtc::Time();
+  const uint32_t diff = current_time - last_rec_time_;
+  if (diff > 150) {
+    ALOGW("Bad OpenSL ES record timing, dT=%u [ms]", diff);
+  }
+  last_rec_time_ = current_time;
+  // Send recorded audio data to the WebRTC sink.
+  // TODO(henrika): fix delay estimates. It is OK to use fixed values for now
+  // since there is no support to turn off built-in EC in combination with
+  // OpenSL ES anyhow. Hence, as is, the WebRTC based AEC (which would use
+  // these estimates) will never be active.
+  const size_t size_in_bytes =
+      static_cast<size_t>(audio_parameters_.GetBytesPerBuffer());
+  const int8_t* data =
+      static_cast<const int8_t*>(audio_buffers_[buffer_index_].get());
+  fine_audio_buffer_->DeliverRecordedData(data, size_in_bytes, 25, 25);
+  // Enqueue the utilized audio buffer and use if for recording again.
+  EnqueueAudioBuffer();
+}
+
+bool OpenSLESRecorder::EnqueueAudioBuffer() {
+  SLresult err =
+      (*simple_buffer_queue_)
+          ->Enqueue(simple_buffer_queue_, audio_buffers_[buffer_index_].get(),
+                    audio_parameters_.GetBytesPerBuffer());
+  if (SL_RESULT_SUCCESS != err) {
+    ALOGE("Enqueue failed: %s", GetSLErrorString(err));
+    return false;
+  }
+  buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
+  return true;
+}
+
+SLuint32 OpenSLESRecorder::GetRecordState() const {
+  RTC_DCHECK(recorder_);
+  SLuint32 state;
+  SLresult err = (*recorder_)->GetRecordState(recorder_, &state);
+  if (SL_RESULT_SUCCESS != err) {
+    ALOGE("GetRecordState failed: %s", GetSLErrorString(err));
+  }
+  return state;
+}
+
+SLAndroidSimpleBufferQueueState OpenSLESRecorder::GetBufferQueueState() const {
+  RTC_DCHECK(simple_buffer_queue_);
+  // state.count: Number of buffers currently in the queue.
+  // state.index: Index of the currently filling buffer. This is a linear index
+  // that keeps a cumulative count of the number of buffers recorded.
+  SLAndroidSimpleBufferQueueState state;
+  SLresult err =
+      (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &state);
+  if (SL_RESULT_SUCCESS != err) {
+    ALOGE("GetState failed: %s", GetSLErrorString(err));
+  }
+  return state;
+}
+
+void OpenSLESRecorder::LogBufferState() const {
+  SLAndroidSimpleBufferQueueState state = GetBufferQueueState();
+  ALOGD("state.count:%d state.index:%d", state.count, state.index);
+}
+
+SLuint32 OpenSLESRecorder::GetBufferCount() {
+  SLAndroidSimpleBufferQueueState state = GetBufferQueueState();
+  return state.count;
+}
+
+}  // namespace webrtc