Refactoring of the buffering of the nearend signal done inside the AEC

webrtc/modules/audio_processing/aec/aec_core.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 1087 matching lines @@
   } else {
     aec->overdrive_scaling =
         0.9f * aec->overdrive_scaling + 0.1f * aec->overDrive;
   }
 
   // Apply the overdrive.
   WebRtcAec_Overdrive(aec->overdrive_scaling, hNlFb, hNl);
 }
 
 static void EchoSuppression(AecCore* aec,
+                            float* nearend_extended_block_lowest_band,
                             float farend[PART_LEN2],
                             float* echo_subtractor_output,
                             float* output,
                             float* const* outputH) {
   float efw[2][PART_LEN1];
   float xfw[2][PART_LEN1];
   float dfw[2][PART_LEN1];
   float comfortNoiseHband[2][PART_LEN1];
   float fft[PART_LEN2];
   float nlpGainHband;
   int i;
   size_t j;
 
   // Coherence and non-linear filter
   float cohde[PART_LEN1], cohxd[PART_LEN1];
   float hNl[PART_LEN1];
 
   // Filter energy
   const int delayEstInterval = 10 * aec->mult;
 
   float* xfw_ptr = NULL;
 
   // Update eBuf with echo subtractor output.
   memcpy(aec->eBuf + PART_LEN, echo_subtractor_output,
          sizeof(float) * PART_LEN);
 
   // Analysis filter banks for the echo suppressor.
   // Windowed near-end ffts.
-  WindowData(fft, aec->dBuf);
+  WindowData(fft, nearend_extended_block_lowest_band);
   aec_rdft_forward_128(fft);
   StoreAsComplex(fft, dfw);
 
   // Windowed echo suppressor output ffts.
   WindowData(fft, aec->eBuf);
   aec_rdft_forward_128(fft);
   StoreAsComplex(fft, efw);
 
   // NLP
 
@@ skipping 60 matching lines @@
     // average nlp over low band: average over second half of freq spectrum
     // (4->8khz)
     GetHighbandGain(hNl, &nlpGainHband);
 
     // Inverse comfort_noise
     ScaledInverseFft(comfortNoiseHband, fft, 2.0f, 0);
 
     // compute gain factor
     for (j = 0; j < aec->num_bands - 1; ++j) {
       for (i = 0; i < PART_LEN; i++) {
-        outputH[j][i] = aec->dBufH[j][i] * nlpGainHband;
+        outputH[j][i] = aec->previous_nearend_block[j + 1][i] * nlpGainHband;
       }
     }
 
     // Add some comfort noise where Hband is attenuated.
     for (i = 0; i < PART_LEN; i++) {
       outputH[0][i] += cnScaleHband * fft[i];
     }
 
     // Saturate output to keep it in the allowed range.
     for (j = 0; j < aec->num_bands - 1; ++j) {
       for (i = 0; i < PART_LEN; i++) {
         outputH[j][i] = WEBRTC_SPL_SAT(WEBRTC_SPL_WORD16_MAX, outputH[j][i],
                                        WEBRTC_SPL_WORD16_MIN);
       }
     }
   }
 
   // Copy the current block to the old position.
-  memcpy(aec->dBuf, aec->dBuf + PART_LEN, sizeof(float) * PART_LEN);
   memcpy(aec->eBuf, aec->eBuf + PART_LEN, sizeof(float) * PART_LEN);
 
-  // Copy the current block to the old position for H band
-  for (j = 0; j < aec->num_bands - 1; ++j) {
-    memcpy(aec->dBufH[j], aec->dBufH[j] + PART_LEN, sizeof(float) * PART_LEN);
-  }
-
   memmove(aec->xfwBuf + PART_LEN1, aec->xfwBuf,
           sizeof(aec->xfwBuf) - sizeof(complex_t) * PART_LEN1);
 }
 
-static void ProcessBlock(AecCore* aec) {
+static void ProcessBlock(AecCore* aec,
+                         float nearend_block[NUM_HIGH_BANDS_MAX + 1]
+                                            [PART_LEN]) {
   size_t i;
 
   float fft[PART_LEN2];
+  float nearend_extended_block_lowest_band[PART_LEN2];
   float x_fft[2][PART_LEN1];
   float df[2][PART_LEN1];
   float far_spectrum = 0.0f;
   float near_spectrum = 0.0f;
   float abs_far_spectrum[PART_LEN1];
   float abs_near_spectrum[PART_LEN1];
 
   const float gPow[2] = {0.9f, 0.1f};
 
   // Noise estimate constants.
   const int noiseInitBlocks = 500 * aec->mult;
   const float step = 0.1f;
   const float ramp = 1.0002f;
   const float gInitNoise[2] = {0.999f, 0.001f};
 
-  float nearend[PART_LEN];
-  float* nearend_ptr = NULL;
   float farend[PART_LEN2];
   float* farend_ptr = NULL;
   float echo_subtractor_output[PART_LEN];
   float output[PART_LEN];
   float outputH[NUM_HIGH_BANDS_MAX][PART_LEN];
   float* outputH_ptr[NUM_HIGH_BANDS_MAX];
   float* x_fft_ptr = NULL;
 
   for (i = 0; i < NUM_HIGH_BANDS_MAX; ++i) {
     outputH_ptr[i] = outputH[i];
   }
 
-  // Concatenate old and new nearend blocks.
-  for (i = 0; i < aec->num_bands - 1; ++i) {
-    WebRtc_ReadBuffer(aec->nearFrBufH[i],
-                      reinterpret_cast<void**>(&nearend_ptr),
-                      nearend, PART_LEN);
-    memcpy(aec->dBufH[i] + PART_LEN, nearend_ptr, sizeof(nearend));
-  }
-  WebRtc_ReadBuffer(aec->nearFrBuf, reinterpret_cast<void**>(&nearend_ptr),
-                    nearend, PART_LEN);
-  memcpy(aec->dBuf + PART_LEN, nearend_ptr, sizeof(nearend));
-
   // We should always have at least one element stored in |far_buf|.
   assert(WebRtc_available_read(aec->far_time_buf) > 0);
   WebRtc_ReadBuffer(aec->far_time_buf, reinterpret_cast<void**>(&farend_ptr),
                     farend, 1);
 
   aec->data_dumper->DumpWav("aec_far", PART_LEN, &farend_ptr[PART_LEN],
                             std::min(aec->sampFreq, 16000), 1);
-  aec->data_dumper->DumpWav("aec_near", PART_LEN, nearend_ptr,
+  aec->data_dumper->DumpWav("aec_near", PART_LEN, &nearend_block[0][0],
                             std::min(aec->sampFreq, 16000), 1);
 
   if (aec->metricsMode == 1) {
     // Update power levels
     UpdateLevel(&aec->farlevel,
                 CalculatePower(&farend_ptr[PART_LEN], PART_LEN));
-    UpdateLevel(&aec->nearlevel, CalculatePower(nearend_ptr, PART_LEN));
+    UpdateLevel(&aec->nearlevel,
+                CalculatePower(&nearend_block[0][0], PART_LEN));
   }
 
   // Convert far-end signal to the frequency domain.
   memcpy(fft, farend_ptr, sizeof(float) * PART_LEN2);
   Fft(fft, x_fft);
   x_fft_ptr = &x_fft[0][0];
 
+  // Form extended nearend frame.
+  memcpy(&nearend_extended_block_lowest_band[0],
+         &aec->previous_nearend_block[0][0], sizeof(float) * PART_LEN);
+  memcpy(&nearend_extended_block_lowest_band[PART_LEN], &nearend_block[0][0],
+         sizeof(float) * PART_LEN);
+
   // Near fft
-  memcpy(fft, aec->dBuf, sizeof(float) * PART_LEN2);
+  memcpy(fft, nearend_extended_block_lowest_band, sizeof(float) * PART_LEN2);
   Fft(fft, df);
 
   // Power smoothing.
   if (aec->refined_adaptive_filter_enabled) {
     for (i = 0; i < PART_LEN1; ++i) {
       far_spectrum = (x_fft_ptr[i] * x_fft_ptr[i]) +
                      (x_fft_ptr[PART_LEN1 + i] * x_fft_ptr[PART_LEN1 + i]);
       // Calculate the magnitude spectrum.
       abs_far_spectrum[i] = sqrtf(far_spectrum);
     }
@@ skipping 61 matching lines @@
           aec->num_delay_values >= kDelayMetricsAggregationWindow) {
         UpdateDelayMetrics(aec);
       }
     }
   }
 
   // Perform echo subtraction.
   EchoSubtraction(aec->num_partitions, aec->extended_filter_enabled,
                   &aec->extreme_filter_divergence, aec->filter_step_size,
                   aec->error_threshold, &x_fft[0][0], &aec->xfBufBlockPos,
-                  aec->xfBuf, nearend_ptr, aec->xPow, aec->wfBuf,
+                  aec->xfBuf, &nearend_block[0][0], aec->xPow, aec->wfBuf,
                   echo_subtractor_output);
   aec->data_dumper->DumpRaw("aec_h_fft", PART_LEN1 * aec->num_partitions,
                             &aec->wfBuf[0][0]);
   aec->data_dumper->DumpRaw("aec_h_fft", PART_LEN1 * aec->num_partitions,
                             &aec->wfBuf[1][0]);
 
   aec->data_dumper->DumpWav("aec_out_linear", PART_LEN, echo_subtractor_output,
                             std::min(aec->sampFreq, 16000), 1);
 
   if (aec->metricsMode == 1) {
     UpdateLevel(&aec->linoutlevel,
                 CalculatePower(echo_subtractor_output, PART_LEN));
   }
 
   // Perform echo suppression.
-  EchoSuppression(aec, farend_ptr, echo_subtractor_output, output, outputH_ptr);
+  EchoSuppression(aec, nearend_extended_block_lowest_band, farend_ptr,
+                  echo_subtractor_output, output, outputH_ptr);
 
   if (aec->metricsMode == 1) {
     UpdateLevel(&aec->nlpoutlevel, CalculatePower(output, PART_LEN));
     UpdateMetrics(aec);
   }
 
+  // Store the nearend signal until the next frame.
+  for (i = 0; i < aec->num_bands; ++i) {
+    memcpy(&aec->previous_nearend_block[i][0], &nearend_block[i][0],
+           sizeof(float) * PART_LEN);
+  }
+
   // Store the output block.
   WebRtc_WriteBuffer(aec->outFrBuf, output, PART_LEN);
   // For high bands
   for (i = 0; i < aec->num_bands - 1; ++i) {
     WebRtc_WriteBuffer(aec->outFrBufH[i], outputH[i], PART_LEN);
   }
 
   aec->data_dumper->DumpWav("aec_out", PART_LEN, output,
                             std::min(aec->sampFreq, 16000), 1);
 }
 
 AecCore* WebRtcAec_CreateAec(int instance_count) {
   int i;
   AecCore* aec = new AecCore(instance_count);
 
   if (!aec) {
     return NULL;
   }
-
-  aec->nearFrBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
-  if (!aec->nearFrBuf) {
-    WebRtcAec_FreeAec(aec);
-    return NULL;
-  }
+  aec->nearend_buffer_size = 0;
+  memset(&aec->nearend_buffer[0], 0, sizeof(aec->nearend_buffer));
 
   aec->outFrBuf = WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
   if (!aec->outFrBuf) {
     WebRtcAec_FreeAec(aec);
     return NULL;
   }
 
   for (i = 0; i < NUM_HIGH_BANDS_MAX; ++i) {
-    aec->nearFrBufH[i] =
-        WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
-    if (!aec->nearFrBufH[i]) {
-      WebRtcAec_FreeAec(aec);
-      return NULL;
-    }
     aec->outFrBufH[i] =
         WebRtc_CreateBuffer(FRAME_LEN + PART_LEN, sizeof(float));
     if (!aec->outFrBufH[i]) {
       WebRtcAec_FreeAec(aec);
       return NULL;
     }
   }
 
   // Create far-end buffers.
   // For bit exactness with legacy code, each element in |far_time_buf| is
@@ skipping 64 matching lines @@
 
   return aec;
 }
 
 void WebRtcAec_FreeAec(AecCore* aec) {
   int i;
   if (aec == NULL) {
     return;
   }
 
-  WebRtc_FreeBuffer(aec->nearFrBuf);
   WebRtc_FreeBuffer(aec->outFrBuf);
 
   for (i = 0; i < NUM_HIGH_BANDS_MAX; ++i) {
-    WebRtc_FreeBuffer(aec->nearFrBufH[i]);
     WebRtc_FreeBuffer(aec->outFrBufH[i]);
   }
 
   WebRtc_FreeBuffer(aec->far_time_buf);
 
   WebRtc_FreeDelayEstimator(aec->delay_estimator);
   WebRtc_FreeDelayEstimatorFarend(aec->delay_estimator_farend);
 
   delete aec;
 }
@@ skipping 42 matching lines @@
 
   SetAdaptiveFilterStepSize(aec);
   SetErrorThreshold(aec);
 
   if (sampFreq == 8000) {
     aec->num_bands = 1;
   } else {
     aec->num_bands = (size_t)(sampFreq / 16000);
   }
 
-  WebRtc_InitBuffer(aec->nearFrBuf);
+  aec->nearend_buffer_size = 0;
+  memset(&aec->nearend_buffer[0], 0, sizeof(aec->nearend_buffer));
+
   WebRtc_InitBuffer(aec->outFrBuf);
   for (i = 0; i < NUM_HIGH_BANDS_MAX; ++i) {
-    WebRtc_InitBuffer(aec->nearFrBufH[i]);
     WebRtc_InitBuffer(aec->outFrBufH[i]);
   }
 
   // Initialize far-end buffers.
   WebRtc_InitBuffer(aec->far_time_buf);
 
   aec->system_delay = 0;
 
   if (WebRtc_InitDelayEstimatorFarend(aec->delay_estimator_farend) != 0) {
     return -1;
@@ skipping 42 matching lines @@
   }
 
   aec->farBufWritePos = 0;
   aec->farBufReadPos = 0;
 
   aec->inSamples = 0;
   aec->outSamples = 0;
   aec->knownDelay = 0;
 
   // Initialize buffers
-  memset(aec->dBuf, 0, sizeof(aec->dBuf));
+  memset(aec->previous_nearend_block, 0, sizeof(aec->previous_nearend_block));
   memset(aec->eBuf, 0, sizeof(aec->eBuf));
-  // For H bands
-  for (i = 0; i < NUM_HIGH_BANDS_MAX; ++i) {
-    memset(aec->dBufH[i], 0, sizeof(aec->dBufH[i]));
-  }
 
   memset(aec->xPow, 0, sizeof(aec->xPow));
   memset(aec->dPow, 0, sizeof(aec->dPow));
   memset(aec->dInitMinPow, 0, sizeof(aec->dInitMinPow));
   aec->noisePow = aec->dInitMinPow;
   aec->noiseEstCtr = 0;
 
   // Initial comfort noise power
   for (i = 0; i < PART_LEN1; i++) {
     aec->dMinPow[i] = 1.0e6f;
@@ skipping 64 matching lines @@
   aec->system_delay -= elements_moved * PART_LEN;
   return elements_moved;
 }
 
 void WebRtcAec_ProcessFrames(AecCore* aec,
                              const float* const* nearend,
                              size_t num_bands,
                              size_t num_samples,
                              int knownDelay,
                              float* const* out) {
-  size_t i, j;
   int out_elements = 0;
 
+  RTC_DCHECK(num_samples == 80 || num_samples == 160);
+
   aec->frame_count++;
   // For each frame the process is as follows:
   // 1) If the system_delay indicates on being too small for processing a
   //    frame we stuff the buffer with enough data for 10 ms.
   // 2 a) Adjust the buffer to the system delay, by moving the read pointer.
   //   b) Apply signal based delay correction, if we have detected poor AEC
   //    performance.
   // 3) TODO(bjornv): Investigate if we need to add this:
   //    If we can't move read pointer due to buffer size limitations we
   //    flush/stuff the buffer.
@@ skipping 10 matching lines @@
   // give a guess on how much we need to shift far-end buffers to align with
   // the near-end signal.  The other delay estimation algorithm uses the
   // far- and near-end signals to find the offset between them.  This one
   // (called "signal delay") is then used to fine tune the alignment, or
   // simply compensate for errors in the system based one.
   // Note that the two algorithms operate independently.  Currently, we only
   // allow one algorithm to be turned on.
 
   assert(aec->num_bands == num_bands);
 
-  for (j = 0; j < num_samples; j += FRAME_LEN) {
-    // TODO(bjornv): Change the near-end buffer handling to be the same as for
-    // far-end, that is, with a near_pre_buf.
-    // Buffer the near-end frame.
-    WebRtc_WriteBuffer(aec->nearFrBuf, &nearend[0][j], FRAME_LEN);
-    // For H band
-    for (i = 1; i < num_bands; ++i) {
-      WebRtc_WriteBuffer(aec->nearFrBufH[i - 1], &nearend[i][j], FRAME_LEN);
-    }
-
+  for (size_t j = 0; j < num_samples; j += FRAME_LEN) {
     // 1) At most we process |aec->mult|+1 partitions in 10 ms. Make sure we
     // have enough far-end data for that by stuffing the buffer if the
     // |system_delay| indicates others.
     if (aec->system_delay < FRAME_LEN) {
       // We don't have enough data so we rewind 10 ms.
       WebRtcAec_MoveFarReadPtr(aec, -(aec->mult + 1));
     }
 
     if (!aec->delay_agnostic_enabled) {
       // 2 a) Compensate for a possible change in the system delay.
@@ skipping 11 matching lines @@
                               DelaySource::kSystemDelay);
       aec->knownDelay -= moved_elements * PART_LEN;
     } else {
       // 2 b) Apply signal based delay correction.
       int move_elements = SignalBasedDelayCorrection(aec);
       int moved_elements = WebRtc_MoveReadPtr(aec->far_time_buf, move_elements);
       MaybeLogDelayAdjustment(moved_elements * (aec->sampFreq == 8000 ? 8 : 4),
                               DelaySource::kDelayAgnostic);
       int far_near_buffer_diff =
           WebRtc_available_read(aec->far_time_buf) -
-          WebRtc_available_read(aec->nearFrBuf) / PART_LEN;
+          (aec->nearend_buffer_size + FRAME_LEN) / PART_LEN;
       WebRtc_SoftResetDelayEstimator(aec->delay_estimator, moved_elements);
       WebRtc_SoftResetDelayEstimatorFarend(aec->delay_estimator_farend,
                                            moved_elements);
       aec->signal_delay_correction += moved_elements;
       // If we rely on reported system delay values only, a buffer underrun here
       // can never occur since we've taken care of that in 1) above.  Here, we
       // apply signal based delay correction and can therefore end up with
       // buffer underruns since the delay estimation can be wrong.  We therefore
       // stuff the buffer with enough elements if needed.
       if (far_near_buffer_diff < 0) {
         WebRtcAec_MoveFarReadPtr(aec, far_near_buffer_diff);
       }
     }
 
-    // 4) Process as many blocks as possible.
-    while (WebRtc_available_read(aec->nearFrBuf) >= PART_LEN) {
-      ProcessBlock(aec);
+    // Form a process a block of samples.
+    RTC_DCHECK_EQ(16, FRAME_LEN - PART_LEN);
+    float nearend_block[NUM_HIGH_BANDS_MAX + 1][PART_LEN];
+    const int num_samples_to_block = PART_LEN - aec->nearend_buffer_size;
+    const int num_samples_to_buffer = FRAME_LEN - num_samples_to_block;
+    for (size_t i = 0; i < num_bands; ++i) {
+      memcpy(&nearend_block[i][0], &aec->nearend_buffer[i][0],
+             aec->nearend_buffer_size * sizeof(float));
+      memcpy(&nearend_block[i][aec->nearend_buffer_size], &nearend[i][j],
+             num_samples_to_block * sizeof(float));
+    }
+    ProcessBlock(aec, nearend_block);
+
+    if (num_samples_to_buffer == PART_LEN) {
+      // If possible form and process a second block of samples.
+      for (size_t i = 0; i < num_bands; ++i) {
+        memcpy(&nearend_block[i][0], &nearend[i][j + num_samples_to_block],
+               num_samples_to_buffer * sizeof(float));
+      }
+      ProcessBlock(aec, nearend_block);
+      aec->nearend_buffer_size = 0;
+    } else {
+      // Buffer the remaining samples in the frame.
+      for (size_t i = 0; i < num_bands; ++i) {
+        memcpy(&aec->nearend_buffer[i][0],
+               &nearend[i][j + num_samples_to_block],
+               num_samples_to_buffer * sizeof(float));
+      }
+      aec->nearend_buffer_size = num_samples_to_buffer;
     }
 
     // 5) Update system delay with respect to the entire frame.
     aec->system_delay -= FRAME_LEN;
 
     // 6) Update output frame.
     // Stuff the out buffer if we have less than a frame to output.
     // This should only happen for the first frame.
     out_elements = static_cast<int>(WebRtc_available_read(aec->outFrBuf));
     if (out_elements < FRAME_LEN) {
       WebRtc_MoveReadPtr(aec->outFrBuf, out_elements - FRAME_LEN);
-      for (i = 0; i < num_bands - 1; ++i) {
+      for (size_t i = 0; i < num_bands - 1; ++i) {
         WebRtc_MoveReadPtr(aec->outFrBufH[i], out_elements - FRAME_LEN);
       }
     }
     // Obtain an output frame.
     WebRtc_ReadBuffer(aec->outFrBuf, NULL, &out[0][j], FRAME_LEN);
     // For H bands.
-    for (i = 1; i < num_bands; ++i) {
+    for (size_t i = 1; i < num_bands; ++i) {
       WebRtc_ReadBuffer(aec->outFrBufH[i - 1], NULL, &out[i][j], FRAME_LEN);
     }
   }
 }
 
 int WebRtcAec_GetDelayMetricsCore(AecCore* self,
                                   int* median,
                                   int* std,
                                   float* fraction_poor_delays) {
   assert(self != NULL);
@@ skipping 95 matching lines @@
 
 int WebRtcAec_system_delay(AecCore* self) {
   return self->system_delay;
 }
 
 void WebRtcAec_SetSystemDelay(AecCore* self, int delay) {
   assert(delay >= 0);
   self->system_delay = delay;
 }
 }  // namespace webrtc