Injectable audio encoders: BuiltinAudioEncoderFactory

webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
 
 #include <algorithm>
 #include <iterator>
 
+#include "webrtc/base/arraysize.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/numerics/exp_filter.h"
 #include "webrtc/base/safe_conversions.h"
+#include "webrtc/base/string_to_number.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/common_types.h"
 #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
 #include "webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h"
 #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 
 namespace webrtc {
 
 namespace {
 
+// Codec parameters for Opus.
+// draft-spittka-payload-rtp-opus-03
+
+// Recommended bitrates:
+// 8-12 kb/s for NB speech,
+// 16-20 kb/s for WB speech,
+// 28-40 kb/s for FB speech,
+// 48-64 kb/s for FB mono music, and
+// 64-128 kb/s for FB stereo music.
+// The current implementation applies the following values to mono signals,
+// and multiplies them by 2 for stereo.
+constexpr int kOpusBitrateNbBps = 12000;
+constexpr int kOpusBitrateWbBps = 20000;
+constexpr int kOpusBitrateFbBps = 32000;
+
+// Opus API allows a min bitrate of 500bps, but Opus documentation suggests
+// bitrate should be in the range of 6000 to 510000, inclusive.
+constexpr int kOpusMinBitrateBps = 6000;
+constexpr int kOpusMaxBitrateBps = 510000;
+
 constexpr int kSampleRateHz = 48000;
 
-// Opus API allows a min bitrate of 500bps, but Opus documentation suggests
-// a minimum bitrate of 6kbps.
-constexpr int kMinBitrateBps = 6000;
-
-constexpr int kMaxBitrateBps = 512000;
-
+// These two lists must be sorted from low to high
 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
-constexpr int kSupportedFrameLengths[] = {20, 60, 120};
+constexpr int kANASupportedFrameLengths[] = {20, 60, 120};

[minyue-webrtc, 2017/03/21 08:29:44]

I think ANA should do the filtering internally. I will check it, if not, we
shall implement it. Anyways, I will follow this in a separate CL.

[ossu, 2017/03/21 16:15:31]

That makes sense. This is currently a bit confusing, having two separate sets of
supported frame lengths next to each other.

+constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
 #else
-constexpr int kSupportedFrameLengths[] = {20, 60};
+constexpr int kANASupportedFrameLengths[] = {20, 60};
+constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
 #endif
 
 // PacketLossFractionSmoother uses an exponential filter with a time constant
 // of -1.0 / ln(0.9999) = 10000 ms.
 constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
 
-AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
-  AudioEncoderOpus::Config config;
-  config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
-  config.num_channels = codec_inst.channels;
-  config.bitrate_bps = rtc::Optional<int>(codec_inst.rate);
-  config.payload_type = codec_inst.pltype;
-  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
-                                                : AudioEncoderOpus::kAudio;
-  config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
-#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
-  config.low_rate_complexity = 9;
-#endif
-  return config;
-}
-
 // Optimize the loss rate to configure Opus. Basically, optimized loss rate is
 // the input loss rate rounded down to various levels, because a robustly good
 // audio quality is achieved by lowering the packet loss down.
 // Additionally, to prevent toggling, margins are used, i.e., when jumping to
 // a loss rate from below, a higher threshold is used than jumping to the same
 // level from above.
 float OptimizePacketLossRate(float new_loss_rate, float old_loss_rate) {
   RTC_DCHECK_GE(new_loss_rate, 0.0f);
   RTC_DCHECK_LE(new_loss_rate, 1.0f);
   RTC_DCHECK_GE(old_loss_rate, 0.0f);
@@ skipping 20 matching lines @@
                  kLossRate5Margin *
                      (kPacketLossRate5 - old_loss_rate > 0 ? 1 : -1)) {
     return kPacketLossRate5;
   } else if (new_loss_rate >= kPacketLossRate1) {
     return kPacketLossRate1;
   } else {
     return 0.0f;
   }
 }
 
+rtc::Optional<std::string> GetFormatParameter(const SdpAudioFormat& format,
+                                              const std::string& param) {
+  auto it = format.parameters.find(param);
+  return (it == format.parameters.end())
+             ? rtc::Optional<std::string>()
+             : rtc::Optional<std::string>(it->second);
+}
+
+template <typename T>
+rtc::Optional<T> GetFormatParameter(const SdpAudioFormat& format,
+                                    const std::string& param) {
+  return rtc::StringToNumber<T>(GetFormatParameter(format, param).value_or(""));
+};
+
+int CalculateDefaultBitrate(int max_playback_rate, int num_channels) {
+  const int bitrate = [&] {

[minyue-webrtc, 2017/03/21 08:29:44]

just curious, what is benefit of lambda here?

[ossu, 2017/03/21 16:15:31]

Nothing major. It allows us to make bitrate const by initializing up front. If
it weren't for the DCHECKs, below, we could just have returned
CalculateDefaultBitrate directly.

For background, see kwiberg@s comments here:
https://codereview.webrtc.org/2695243005/diff/100001/webrtc/modules/audio_cod...

IMO it's not necessary, but slightly better if the reader isn't put off by
lambdas. (and they shouldn't be - lambdas are great!)

+    if (max_playback_rate <= 8000) {
+      return kOpusBitrateNbBps * num_channels;
+    } else if (max_playback_rate <= 16000) {
+      return kOpusBitrateWbBps * num_channels;
+    } else {
+      return kOpusBitrateFbBps * num_channels;
+    }
+  }();
+  RTC_DCHECK_GE(bitrate, kOpusMinBitrateBps);
+  RTC_DCHECK_LE(bitrate, kOpusMaxBitrateBps);

[minyue-webrtc, 2017/03/21 08:29:44]

These are more of a compile time check, i.e.,

kOpusMinBitrateBps < kOpusBitrateNbBps < kOpusMaxBitrateBps / 2, etc

something like static_assert would fit best, but I am not sure if we can use it.

[ossu, 2017/03/21 16:15:31]

Yeah. I bet these could only hit if something majorly strange was going on, and
num_channels was biig.

+  return bitrate;
+}
+
+// Get the maxaveragebitrate parameter in string-form, so we can properly figure
+// out how invalid it is and accurately log invalid values.
+int CalculateBitrate(int max_playback_rate_hz,
+                     int num_channels,
+                     rtc::Optional<std::string> bitrate_param) {
+  const int default_bitrate =
+      CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
+
+  if (bitrate_param) {
+    const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
+    if (bitrate) {
+      if (*bitrate >= kOpusMinBitrateBps && *bitrate <= kOpusMaxBitrateBps) {
+        return *bitrate;

[minyue-webrtc, 2017/03/21 08:29:44]

148 - 157 is fairly ugly. can we refactor this part?

how about 

int chosen_bitrate = min(max(*bitrate, kOpusMinBitrateBps), kOpusMaxBitrateBps);
if (*bitrate != chosen_bitrate ) {
  LOG(LS_WARNING) <<
}

[ossu, 2017/03/21 16:15:31]

Alright. Makes sense.

+      }
+
+      const int new_bitrate = (*bitrate < kOpusMinBitrateBps)
+                                  ? kOpusMinBitrateBps
+                                  : kOpusMaxBitrateBps;
+      LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
+                      << " clamped to " << new_bitrate;
+      return new_bitrate;
+    }
+
+    LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
+                    << "\" replaced by default bitrate " << default_bitrate;
+  }
+
+  return default_bitrate;
+}
+
+rtc::Optional<int> GetChannelCount(const SdpAudioFormat& format) {
+  const auto param = GetFormatParameter(format, "stereo");
+  if (!param || param == "0") {
+    return rtc::Optional<int>(1);
+  } else if (param == "1") {
+    return rtc::Optional<int>(2);
+  }
+  return rtc::Optional<int>();
+}
+
+rtc::Optional<int> GetMaxPlaybackRate(const SdpAudioFormat& format) {
+  const auto param = GetFormatParameter(format, "maxplaybackrate");
+  if (!param) {
+    return rtc::Optional<int>(48000);

[minyue-webrtc, 2017/03/21 08:29:44]

can we

constexpr kDefaultMaxPlaybackRate = 48000 

[ossu, 2017/03/21 16:15:31]

There is a kSampleRateHz above. I could use that!

+  }
+  const auto parsed = rtc::StringToNumber<int>(*param);
+  if (parsed && *parsed >= 8000) {
+    return rtc::Optional<int>(*parsed);
+  }
+  return rtc::Optional<int>();

[minyue-webrtc, 2017/03/21 08:29:44]

It is hard for me to understand rtc::Optional<int>()? does that mean wrong
parsed value, then should the Get functions decide to use the default value?

Or we can get rid of all Get functions?

[ossu, 2017/03/21 16:15:31]

Yes, it means the "maxplaybackrate" option was invalid. It is used to reject bad
formats in QueryAudioFormat, but I'm not sure that's completely necessary. It
could just as well return 48 kHz in that case. If so, I'd just have it return an
int instead. WDYT?

+}
+
+void FindSupportedFrameLengths(int min_frame_length_ms, int max_frame_length_ms,

[ossu, 2017/03/20 18:18:53]

Decided to reuse the code from SetReceiverFrameLengthRange instead. It does not
fall back to using the whole list if the result is empty.

+                               std::vector<int>* out) {
+  out->clear();
+  std::copy_if(std::begin(kANASupportedFrameLengths),
+               std::end(kANASupportedFrameLengths),
+               std::back_inserter(*out),
+               [&](int frame_length_ms) {
+                 return frame_length_ms >= min_frame_length_ms &&
+                        frame_length_ms <= max_frame_length_ms;
+               });
+  RTC_DCHECK(std::is_sorted(out->begin(), out->end()));
+}
+
 }  // namespace
 
+rtc::Optional<AudioCodecInfo> AudioEncoderOpus::QueryAudioEncoder(
+    const SdpAudioFormat& format) {
+  if (STR_CASE_CMP(format.name.c_str(), GetPayloadName()) == 0 &&
+      format.clockrate_hz == 48000 && format.num_channels == 2) {
+
+    const rtc::Optional<int> num_channels = GetChannelCount(format);
+    const rtc::Optional<int> max_playback_rate = GetMaxPlaybackRate(format);
+    if (num_channels && max_playback_rate) {
+      const int bitrate =
+          CalculateBitrate(*max_playback_rate, *num_channels,
+                           GetFormatParameter(format, "maxaveragebitrate"));
+      AudioCodecInfo info(48000, *num_channels, bitrate, kOpusMinBitrateBps,
+                           kOpusMaxBitrateBps);
+      info.allow_comfort_noise = false;
+      info.supports_network_adaption = true;
+
+      return rtc::Optional<AudioCodecInfo>(info);
+    }
+  }
+  return rtc::Optional<AudioCodecInfo>();
+}
+
+AudioEncoderOpus::Config AudioEncoderOpus::CreateConfig(
+    const CodecInst& codec_inst) {
+  AudioEncoderOpus::Config config;
+  config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
+  config.num_channels = codec_inst.channels;
+  config.bitrate_bps = rtc::Optional<int>(codec_inst.rate);
+  config.payload_type = codec_inst.pltype;
+  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
+                                                : AudioEncoderOpus::kAudio;
+  config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+  config.low_rate_complexity = 9;
+#endif
+  return config;
+}
+
+AudioEncoderOpus::Config AudioEncoderOpus::CreateConfig(
+    int payload_type,
+    const SdpAudioFormat& format) {
+  AudioEncoderOpus::Config config;
+
+  // Normally, the first two parameters should already have been checked by
+  // QueryAudioEncoder. At this point, we might as well fall back to something
+  // reasonable.
+  config.num_channels = GetChannelCount(format).value_or(1);
+  config.max_playback_rate_hz = GetMaxPlaybackRate(format).value_or(48000);
+  config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
+  config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
+  config.bitrate_bps = rtc::Optional<int>(
+      CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
+                       GetFormatParameter(format, "maxaveragebitrate")));
+  config.payload_type = payload_type;
+  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
+                                                : AudioEncoderOpus::kAudio;
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+  config.low_rate_complexity = 9;
+#endif
+
+  constexpr int kMinSupportedFrameLength = kOpusSupportedFrameLengths[0];
+  constexpr int kMaxSupportedFrameLength =
+      kOpusSupportedFrameLengths[arraysize(kOpusSupportedFrameLengths) - 1];
+
+  const auto ptime = GetFormatParameter<int>(format, "ptime");

[minyue-webrtc, 2017/03/21 08:29:44]

add a helper function on pTime?

[ossu, 2017/03/21 16:15:31]

Sure!

+  if (ptime) {
+    // Pick the next highest supported frame length from
+    // kOpusSupportedFrameLengths. Default to the largest, if we find none.
+    config.frame_size_ms = kMaxSupportedFrameLength;
+    for (const int supported_frame_length : kOpusSupportedFrameLengths) {
+      if (supported_frame_length >= *ptime) {
+        config.frame_size_ms = supported_frame_length;
+        break;
+      }
+    }
+  }
+
+  // For now, minptime and maxptime are only used with ANA. If ptime is outside
+  // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know
+  // if ANA was to be used when setting up the config, and adjust accordingly.
+  const int min_frame_length_ms =
+      std::min(std::max(GetFormatParameter<int>(format, "minptime")
+                            .value_or(kMinSupportedFrameLength),
+                        kMinSupportedFrameLength),
+               kMaxSupportedFrameLength);
+  const int max_frame_length_ms =
+      std::min(std::max(GetFormatParameter<int>(format, "maxptime")
+                            .value_or(kMaxSupportedFrameLength),
+                        kMinSupportedFrameLength),
+               kMaxSupportedFrameLength);
+
+  FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
+                            &config.supported_frame_lengths_ms);
+
+  return config;
+}
+
 class AudioEncoderOpus::PacketLossFractionSmoother {
  public:
   explicit PacketLossFractionSmoother(const Clock* clock)
       : clock_(clock),
         last_sample_time_ms_(clock_->TimeInMilliseconds()),
         smoother_(kAlphaForPacketLossFractionSmoother) {}
 
   // Gets the smoothed packet loss fraction.
   float GetAverage() const {
     float value = smoother_.filtered();
@@ skipping 24 matching lines @@
 AudioEncoderOpus::Config::Config(const Config&) = default;
 AudioEncoderOpus::Config::~Config() = default;
 auto AudioEncoderOpus::Config::operator=(const Config&) -> Config& = default;
 
 bool AudioEncoderOpus::Config::IsOk() const {
   if (frame_size_ms <= 0 || frame_size_ms % 10 != 0)
     return false;
   if (num_channels != 1 && num_channels != 2)
     return false;
   if (bitrate_bps &&
-      (*bitrate_bps < kMinBitrateBps || *bitrate_bps > kMaxBitrateBps))
+      (*bitrate_bps < kOpusMinBitrateBps || *bitrate_bps > kOpusMaxBitrateBps))
     return false;
   if (complexity < 0 || complexity > 10)
     return false;
   if (low_rate_complexity < 0 || low_rate_complexity > 10)
     return false;
   return true;
 }
 
 int AudioEncoderOpus::Config::GetBitrateBps() const {
   RTC_DCHECK(IsOk());
@@ skipping 40 matching lines @@
       bitrate_smoother_(bitrate_smoother
           ? std::move(bitrate_smoother) : std::unique_ptr<SmoothingFilter>(
               // We choose 5sec as initial time constant due to empirical data.
               new SmoothingFilterImpl(5000, config.clock))) {
   RTC_CHECK(RecreateEncoderInstance(config));
 }
 
 AudioEncoderOpus::AudioEncoderOpus(const CodecInst& codec_inst)
     : AudioEncoderOpus(CreateConfig(codec_inst), nullptr) {}
 
+AudioEncoderOpus::AudioEncoderOpus(int payload_type,
+                                   const SdpAudioFormat& format)
+    : AudioEncoderOpus(CreateConfig(payload_type, format), nullptr) {}
+
 AudioEncoderOpus::~AudioEncoderOpus() {
   RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
 }
 
 int AudioEncoderOpus::SampleRateHz() const {
   return kSampleRateHz;
 }
 
 size_t AudioEncoderOpus::NumChannels() const {
   return config_.num_channels;
@@ skipping 107 matching lines @@
   } else if (send_side_bwe_with_overhead_) {
     if (!overhead_bytes_per_packet_) {
       LOG(LS_INFO)
           << "AudioEncoderOpus: Overhead unknown, target audio bitrate "
           << target_audio_bitrate_bps << " bps is ignored.";
       return;
     }
     const int overhead_bps = static_cast<int>(
         *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket());
     SetTargetBitrate(std::min(
-        kMaxBitrateBps,
-        std::max(kMinBitrateBps, target_audio_bitrate_bps - overhead_bps)));
+        kOpusMaxBitrateBps,
+        std::max(kOpusMinBitrateBps, target_audio_bitrate_bps - overhead_bps)));
   } else {
     SetTargetBitrate(target_audio_bitrate_bps);
   }
 }
 
 void AudioEncoderOpus::OnReceivedRtt(int rtt_ms) {
   if (!audio_network_adaptor_)
     return;
   audio_network_adaptor_->SetRtt(rtt_ms);
   ApplyAudioNetworkAdaptor();
 }
 
 void AudioEncoderOpus::OnReceivedOverhead(size_t overhead_bytes_per_packet) {
   if (audio_network_adaptor_) {
     audio_network_adaptor_->SetOverhead(overhead_bytes_per_packet);
     ApplyAudioNetworkAdaptor();
   } else {
     overhead_bytes_per_packet_ =
         rtc::Optional<size_t>(overhead_bytes_per_packet);
   }
 }
 
 void AudioEncoderOpus::SetReceiverFrameLengthRange(int min_frame_length_ms,
                                                    int max_frame_length_ms) {
   // Ensure that |SetReceiverFrameLengthRange| is called before
   // |EnableAudioNetworkAdaptor|, otherwise we need to recreate
   // |audio_network_adaptor_|, which is not a needed use case.
   RTC_DCHECK(!audio_network_adaptor_);
-
-  config_.supported_frame_lengths_ms.clear();
-  std::copy_if(std::begin(kSupportedFrameLengths),
-               std::end(kSupportedFrameLengths),
-               std::back_inserter(config_.supported_frame_lengths_ms),
-               [&](int frame_length_ms) {
-                 return frame_length_ms >= min_frame_length_ms &&
-                        frame_length_ms <= max_frame_length_ms;
-               });
-  RTC_DCHECK(std::is_sorted(config_.supported_frame_lengths_ms.begin(),
-                            config_.supported_frame_lengths_ms.end()));
+  FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
+                            &config_.supported_frame_lengths_ms);
 }
 
 AudioEncoder::EncodedInfo AudioEncoderOpus::EncodeImpl(
     uint32_t rtp_timestamp,
     rtc::ArrayView<const int16_t> audio,
     rtc::Buffer* encoded) {
   MaybeUpdateUplinkBandwidth();
 
   if (input_buffer_.empty())
     first_timestamp_in_buffer_ = rtp_timestamp;
@@ skipping 111 matching lines @@
   float opt_loss_rate = OptimizePacketLossRate(fraction, packet_loss_rate_);
   if (packet_loss_rate_ != opt_loss_rate) {
     packet_loss_rate_ = opt_loss_rate;
     RTC_CHECK_EQ(
         0, WebRtcOpus_SetPacketLossRate(
                inst_, static_cast<int32_t>(packet_loss_rate_ * 100 + .5)));
   }
 }
 
 void AudioEncoderOpus::SetTargetBitrate(int bits_per_second) {
-  config_.bitrate_bps = rtc::Optional<int>(
-      std::max(std::min(bits_per_second, kMaxBitrateBps), kMinBitrateBps));
+  config_.bitrate_bps = rtc::Optional<int>(std::max(
+      std::min(bits_per_second, kOpusMaxBitrateBps), kOpusMinBitrateBps));
   RTC_DCHECK(config_.IsOk());
   RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps()));
   const auto new_complexity = config_.GetNewComplexity();
   if (new_complexity && complexity_ != *new_complexity) {
     complexity_ = *new_complexity;
     RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
   }
 }
 
 void AudioEncoderOpus::ApplyAudioNetworkAdaptor() {
@@ skipping 20 matching lines @@
     const std::string& config_string,
     RtcEventLog* event_log,
     const Clock* clock) const {
   AudioNetworkAdaptorImpl::Config config;
   config.clock = clock;
   config.event_log = event_log;
   return std::unique_ptr<AudioNetworkAdaptor>(new AudioNetworkAdaptorImpl(
       config,
       ControllerManagerImpl::Create(
           config_string, NumChannels(), supported_frame_lengths_ms(),
-          kMinBitrateBps, num_channels_to_encode_, next_frame_length_ms_,
+          kOpusMinBitrateBps, num_channels_to_encode_, next_frame_length_ms_,
           GetTargetBitrate(), config_.fec_enabled, GetDtx(), clock)));
 }
 
 void AudioEncoderOpus::MaybeUpdateUplinkBandwidth() {
   if (audio_network_adaptor_) {
     int64_t now_ms = rtc::TimeMillis();
     if (!bitrate_smoother_last_update_time_ ||
         now_ms - *bitrate_smoother_last_update_time_ >=
             config_.uplink_bandwidth_update_interval_ms) {
       rtc::Optional<float> smoothed_bitrate = bitrate_smoother_->GetAverage();
       if (smoothed_bitrate)
         audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate);
       bitrate_smoother_last_update_time_ = rtc::Optional<int64_t>(now_ms);
     }
   }
 }
 
 }  // namespace webrtc