Injectable audio encoders: BuiltinAudioEncoderFactory

webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc

Index: webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
index bac963f4e8e2de9ff047ac78d889e9b187c6469e..1d350376c9bfb1a9657dec7261f7f7ff822299f5 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
@@ -13,10 +13,12 @@
 #include <algorithm>
 #include <iterator>
 
+#include "webrtc/base/arraysize.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/numerics/exp_filter.h"
 #include "webrtc/base/safe_conversions.h"
+#include "webrtc/base/string_to_number.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/common_types.h"
 #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
@@ -28,39 +30,41 @@ namespace webrtc {
 
 namespace {
 
-constexpr int kSampleRateHz = 48000;
+// Codec parameters for Opus.
+// draft-spittka-payload-rtp-opus-03
+
+// Recommended bitrates:
+// 8-12 kb/s for NB speech,
+// 16-20 kb/s for WB speech,
+// 28-40 kb/s for FB speech,
+// 48-64 kb/s for FB mono music, and
+// 64-128 kb/s for FB stereo music.
+// The current implementation applies the following values to mono signals,
+// and multiplies them by 2 for stereo.
+constexpr int kOpusBitrateNbBps = 12000;
+constexpr int kOpusBitrateWbBps = 20000;
+constexpr int kOpusBitrateFbBps = 32000;
 
 // Opus API allows a min bitrate of 500bps, but Opus documentation suggests
-// a minimum bitrate of 6kbps.
-constexpr int kMinBitrateBps = 6000;
+// bitrate should be in the range of 6000 to 510000, inclusive.
+constexpr int kOpusMinBitrateBps = 6000;
+constexpr int kOpusMaxBitrateBps = 510000;
 
-constexpr int kMaxBitrateBps = 512000;
+constexpr int kSampleRateHz = 48000;
 
+// These two lists must be sorted from low to high
 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
-constexpr int kSupportedFrameLengths[] = {20, 60, 120};
+constexpr int kANASupportedFrameLengths[] = {20, 60, 120};

[minyue-webrtc, 2017/03/21 08:29:44]

I think ANA should do the filtering internally. I will check it, if not, we
shall implement it. Anyways, I will follow this in a separate CL.

[ossu, 2017/03/21 16:15:31]

That makes sense. This is currently a bit confusing, having two separate sets of
supported frame lengths next to each other.

+constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
 #else
-constexpr int kSupportedFrameLengths[] = {20, 60};
+constexpr int kANASupportedFrameLengths[] = {20, 60};
+constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
 #endif
 
 // PacketLossFractionSmoother uses an exponential filter with a time constant
 // of -1.0 / ln(0.9999) = 10000 ms.
 constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
 
-AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
-  AudioEncoderOpus::Config config;
-  config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
-  config.num_channels = codec_inst.channels;
-  config.bitrate_bps = rtc::Optional<int>(codec_inst.rate);
-  config.payload_type = codec_inst.pltype;
-  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
-                                                : AudioEncoderOpus::kAudio;
-  config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
-#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
-  config.low_rate_complexity = 9;
-#endif
-  return config;
-}
-
 // Optimize the loss rate to configure Opus. Basically, optimized loss rate is
 // the input loss rate rounded down to various levels, because a robustly good
 // audio quality is achieved by lowering the packet loss down.
@@ -101,8 +105,199 @@ float OptimizePacketLossRate(float new_loss_rate, float old_loss_rate) {
   }
 }
 
+rtc::Optional<std::string> GetFormatParameter(const SdpAudioFormat& format,
+                                              const std::string& param) {
+  auto it = format.parameters.find(param);
+  return (it == format.parameters.end())
+             ? rtc::Optional<std::string>()
+             : rtc::Optional<std::string>(it->second);
+}
+
+template <typename T>
+rtc::Optional<T> GetFormatParameter(const SdpAudioFormat& format,
+                                    const std::string& param) {
+  return rtc::StringToNumber<T>(GetFormatParameter(format, param).value_or(""));
+};
+
+int CalculateDefaultBitrate(int max_playback_rate, int num_channels) {
+  const int bitrate = [&] {

[minyue-webrtc, 2017/03/21 08:29:44]

just curious, what is benefit of lambda here?

[ossu, 2017/03/21 16:15:31]

Nothing major. It allows us to make bitrate const by initializing up front. If
it weren't for the DCHECKs, below, we could just have returned
CalculateDefaultBitrate directly.

For background, see kwiberg@s comments here:
https://codereview.webrtc.org/2695243005/diff/100001/webrtc/modules/audio_cod...

IMO it's not necessary, but slightly better if the reader isn't put off by
lambdas. (and they shouldn't be - lambdas are great!)

+    if (max_playback_rate <= 8000) {
+      return kOpusBitrateNbBps * num_channels;
+    } else if (max_playback_rate <= 16000) {
+      return kOpusBitrateWbBps * num_channels;
+    } else {
+      return kOpusBitrateFbBps * num_channels;
+    }
+  }();
+  RTC_DCHECK_GE(bitrate, kOpusMinBitrateBps);
+  RTC_DCHECK_LE(bitrate, kOpusMaxBitrateBps);

[minyue-webrtc, 2017/03/21 08:29:44]

These are more of a compile time check, i.e.,

kOpusMinBitrateBps < kOpusBitrateNbBps < kOpusMaxBitrateBps / 2, etc

something like static_assert would fit best, but I am not sure if we can use it.

[ossu, 2017/03/21 16:15:31]

Yeah. I bet these could only hit if something majorly strange was going on, and
num_channels was biig.

+  return bitrate;
+}
+
+// Get the maxaveragebitrate parameter in string-form, so we can properly figure
+// out how invalid it is and accurately log invalid values.
+int CalculateBitrate(int max_playback_rate_hz,
+                     int num_channels,
+                     rtc::Optional<std::string> bitrate_param) {
+  const int default_bitrate =
+      CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
+
+  if (bitrate_param) {
+    const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
+    if (bitrate) {
+      if (*bitrate >= kOpusMinBitrateBps && *bitrate <= kOpusMaxBitrateBps) {
+        return *bitrate;

[minyue-webrtc, 2017/03/21 08:29:44]

148 - 157 is fairly ugly. can we refactor this part?

how about 

int chosen_bitrate = min(max(*bitrate, kOpusMinBitrateBps), kOpusMaxBitrateBps);
if (*bitrate != chosen_bitrate ) {
  LOG(LS_WARNING) <<
}

[ossu, 2017/03/21 16:15:31]

Alright. Makes sense.

+      }
+
+      const int new_bitrate = (*bitrate < kOpusMinBitrateBps)
+                                  ? kOpusMinBitrateBps
+                                  : kOpusMaxBitrateBps;
+      LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
+                      << " clamped to " << new_bitrate;
+      return new_bitrate;
+    }
+
+    LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
+                    << "\" replaced by default bitrate " << default_bitrate;
+  }
+
+  return default_bitrate;
+}
+
+rtc::Optional<int> GetChannelCount(const SdpAudioFormat& format) {
+  const auto param = GetFormatParameter(format, "stereo");
+  if (!param || param == "0") {
+    return rtc::Optional<int>(1);
+  } else if (param == "1") {
+    return rtc::Optional<int>(2);
+  }
+  return rtc::Optional<int>();
+}
+
+rtc::Optional<int> GetMaxPlaybackRate(const SdpAudioFormat& format) {
+  const auto param = GetFormatParameter(format, "maxplaybackrate");
+  if (!param) {
+    return rtc::Optional<int>(48000);

[minyue-webrtc, 2017/03/21 08:29:44]

can we

constexpr kDefaultMaxPlaybackRate = 48000 

[ossu, 2017/03/21 16:15:31]

There is a kSampleRateHz above. I could use that!

+  }
+  const auto parsed = rtc::StringToNumber<int>(*param);
+  if (parsed && *parsed >= 8000) {
+    return rtc::Optional<int>(*parsed);
+  }
+  return rtc::Optional<int>();

[minyue-webrtc, 2017/03/21 08:29:44]

It is hard for me to understand rtc::Optional<int>()? does that mean wrong
parsed value, then should the Get functions decide to use the default value?

Or we can get rid of all Get functions?

[ossu, 2017/03/21 16:15:31]

Yes, it means the "maxplaybackrate" option was invalid. It is used to reject bad
formats in QueryAudioFormat, but I'm not sure that's completely necessary. It
could just as well return 48 kHz in that case. If so, I'd just have it return an
int instead. WDYT?

+}
+
+void FindSupportedFrameLengths(int min_frame_length_ms, int max_frame_length_ms,

[ossu, 2017/03/20 18:18:53]

Decided to reuse the code from SetReceiverFrameLengthRange instead. It does not
fall back to using the whole list if the result is empty.

+                               std::vector<int>* out) {
+  out->clear();
+  std::copy_if(std::begin(kANASupportedFrameLengths),
+               std::end(kANASupportedFrameLengths),
+               std::back_inserter(*out),
+               [&](int frame_length_ms) {
+                 return frame_length_ms >= min_frame_length_ms &&
+                        frame_length_ms <= max_frame_length_ms;
+               });
+  RTC_DCHECK(std::is_sorted(out->begin(), out->end()));
+}
+
 }  // namespace
 
+rtc::Optional<AudioCodecInfo> AudioEncoderOpus::QueryAudioEncoder(
+    const SdpAudioFormat& format) {
+  if (STR_CASE_CMP(format.name.c_str(), GetPayloadName()) == 0 &&
+      format.clockrate_hz == 48000 && format.num_channels == 2) {
+
+    const rtc::Optional<int> num_channels = GetChannelCount(format);
+    const rtc::Optional<int> max_playback_rate = GetMaxPlaybackRate(format);
+    if (num_channels && max_playback_rate) {
+      const int bitrate =
+          CalculateBitrate(*max_playback_rate, *num_channels,
+                           GetFormatParameter(format, "maxaveragebitrate"));
+      AudioCodecInfo info(48000, *num_channels, bitrate, kOpusMinBitrateBps,
+                           kOpusMaxBitrateBps);
+      info.allow_comfort_noise = false;
+      info.supports_network_adaption = true;
+
+      return rtc::Optional<AudioCodecInfo>(info);
+    }
+  }
+  return rtc::Optional<AudioCodecInfo>();
+}
+
+AudioEncoderOpus::Config AudioEncoderOpus::CreateConfig(
+    const CodecInst& codec_inst) {
+  AudioEncoderOpus::Config config;
+  config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
+  config.num_channels = codec_inst.channels;
+  config.bitrate_bps = rtc::Optional<int>(codec_inst.rate);
+  config.payload_type = codec_inst.pltype;
+  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
+                                                : AudioEncoderOpus::kAudio;
+  config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+  config.low_rate_complexity = 9;
+#endif
+  return config;
+}
+
+AudioEncoderOpus::Config AudioEncoderOpus::CreateConfig(
+    int payload_type,
+    const SdpAudioFormat& format) {
+  AudioEncoderOpus::Config config;
+
+  // Normally, the first two parameters should already have been checked by
+  // QueryAudioEncoder. At this point, we might as well fall back to something
+  // reasonable.
+  config.num_channels = GetChannelCount(format).value_or(1);
+  config.max_playback_rate_hz = GetMaxPlaybackRate(format).value_or(48000);
+  config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
+  config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
+  config.bitrate_bps = rtc::Optional<int>(
+      CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
+                       GetFormatParameter(format, "maxaveragebitrate")));
+  config.payload_type = payload_type;
+  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
+                                                : AudioEncoderOpus::kAudio;
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+  config.low_rate_complexity = 9;
+#endif
+
+  constexpr int kMinSupportedFrameLength = kOpusSupportedFrameLengths[0];
+  constexpr int kMaxSupportedFrameLength =
+      kOpusSupportedFrameLengths[arraysize(kOpusSupportedFrameLengths) - 1];
+
+  const auto ptime = GetFormatParameter<int>(format, "ptime");

[minyue-webrtc, 2017/03/21 08:29:44]

add a helper function on pTime?

[ossu, 2017/03/21 16:15:31]

Sure!

+  if (ptime) {
+    // Pick the next highest supported frame length from
+    // kOpusSupportedFrameLengths. Default to the largest, if we find none.
+    config.frame_size_ms = kMaxSupportedFrameLength;
+    for (const int supported_frame_length : kOpusSupportedFrameLengths) {
+      if (supported_frame_length >= *ptime) {
+        config.frame_size_ms = supported_frame_length;
+        break;
+      }
+    }
+  }
+
+  // For now, minptime and maxptime are only used with ANA. If ptime is outside
+  // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know
+  // if ANA was to be used when setting up the config, and adjust accordingly.
+  const int min_frame_length_ms =
+      std::min(std::max(GetFormatParameter<int>(format, "minptime")
+                            .value_or(kMinSupportedFrameLength),
+                        kMinSupportedFrameLength),
+               kMaxSupportedFrameLength);
+  const int max_frame_length_ms =
+      std::min(std::max(GetFormatParameter<int>(format, "maxptime")
+                            .value_or(kMaxSupportedFrameLength),
+                        kMinSupportedFrameLength),
+               kMaxSupportedFrameLength);
+
+  FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
+                            &config.supported_frame_lengths_ms);
+
+  return config;
+}
+
 class AudioEncoderOpus::PacketLossFractionSmoother {
  public:
   explicit PacketLossFractionSmoother(const Clock* clock)
@@ -147,7 +342,7 @@ bool AudioEncoderOpus::Config::IsOk() const {
   if (num_channels != 1 && num_channels != 2)
     return false;
   if (bitrate_bps &&
-      (*bitrate_bps < kMinBitrateBps || *bitrate_bps > kMaxBitrateBps))
+      (*bitrate_bps < kOpusMinBitrateBps || *bitrate_bps > kOpusMaxBitrateBps))
     return false;
   if (complexity < 0 || complexity > 10)
     return false;
@@ -208,6 +403,10 @@ AudioEncoderOpus::AudioEncoderOpus(
 AudioEncoderOpus::AudioEncoderOpus(const CodecInst& codec_inst)
     : AudioEncoderOpus(CreateConfig(codec_inst), nullptr) {}
 
+AudioEncoderOpus::AudioEncoderOpus(int payload_type,
+                                   const SdpAudioFormat& format)
+    : AudioEncoderOpus(CreateConfig(payload_type, format), nullptr) {}
+
 AudioEncoderOpus::~AudioEncoderOpus() {
   RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
 }
@@ -335,8 +534,8 @@ void AudioEncoderOpus::OnReceivedUplinkBandwidth(
     const int overhead_bps = static_cast<int>(
         *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket());
     SetTargetBitrate(std::min(
-        kMaxBitrateBps,
-        std::max(kMinBitrateBps, target_audio_bitrate_bps - overhead_bps)));
+        kOpusMaxBitrateBps,
+        std::max(kOpusMinBitrateBps, target_audio_bitrate_bps - overhead_bps)));
   } else {
     SetTargetBitrate(target_audio_bitrate_bps);
   }
@@ -365,17 +564,8 @@ void AudioEncoderOpus::SetReceiverFrameLengthRange(int min_frame_length_ms,
   // |EnableAudioNetworkAdaptor|, otherwise we need to recreate
   // |audio_network_adaptor_|, which is not a needed use case.
   RTC_DCHECK(!audio_network_adaptor_);
-
-  config_.supported_frame_lengths_ms.clear();
-  std::copy_if(std::begin(kSupportedFrameLengths),
-               std::end(kSupportedFrameLengths),
-               std::back_inserter(config_.supported_frame_lengths_ms),
-               [&](int frame_length_ms) {
-                 return frame_length_ms >= min_frame_length_ms &&
-                        frame_length_ms <= max_frame_length_ms;
-               });
-  RTC_DCHECK(std::is_sorted(config_.supported_frame_lengths_ms.begin(),
-                            config_.supported_frame_lengths_ms.end()));
+  FindSupportedFrameLengths(min_frame_length_ms, max_frame_length_ms,
+                            &config_.supported_frame_lengths_ms);
 }
 
 AudioEncoder::EncodedInfo AudioEncoderOpus::EncodeImpl(
@@ -507,8 +697,8 @@ void AudioEncoderOpus::SetProjectedPacketLossRate(float fraction) {
 }
 
 void AudioEncoderOpus::SetTargetBitrate(int bits_per_second) {
-  config_.bitrate_bps = rtc::Optional<int>(
-      std::max(std::min(bits_per_second, kMaxBitrateBps), kMinBitrateBps));
+  config_.bitrate_bps = rtc::Optional<int>(std::max(
+      std::min(bits_per_second, kOpusMaxBitrateBps), kOpusMinBitrateBps));
   RTC_DCHECK(config_.IsOk());
   RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps()));
   const auto new_complexity = config_.GetNewComplexity();
@@ -549,7 +739,7 @@ AudioEncoderOpus::DefaultAudioNetworkAdaptorCreator(
       config,
       ControllerManagerImpl::Create(
           config_string, NumChannels(), supported_frame_lengths_ms(),
-          kMinBitrateBps, num_channels_to_encode_, next_frame_length_ms_,
+          kOpusMinBitrateBps, num_channels_to_encode_, next_frame_length_ms_,
           GetTargetBitrate(), config_.fec_enabled, GetDtx(), clock)));
 }