Injectable audio encoders: BuiltinAudioEncoderFactory

webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
 
+#include <stdlib.h>

[the sun, 2017/03/17 09:07:31]

nit: what for?

[ossu, 2017/03/17 09:55:39]

Not sure. Will look into it. May be a hold-over from an older version.

+
 #include <algorithm>
 #include <iterator>
 
+#include "webrtc/base/arraysize.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/numerics/exp_filter.h"
 #include "webrtc/base/safe_conversions.h"
+#include "webrtc/base/string_to_number.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/common_types.h"
 #include "webrtc/modules/audio_coding/audio_network_adaptor/audio_network_adaptor_impl.h"
 #include "webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.h"
 #include "webrtc/modules/audio_coding/codecs/opus/opus_interface.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 
 namespace webrtc {
 
 namespace {
 
+// Codec parameters for Opus.
+// draft-spittka-payload-rtp-opus-03
+
+// Recommended bitrates:
+// 8-12 kb/s for NB speech,
+// 16-20 kb/s for WB speech,
+// 28-40 kb/s for FB speech,
+// 48-64 kb/s for FB mono music, and
+// 64-128 kb/s for FB stereo music.
+// The current implementation applies the following values to mono signals,
+// and multiplies them by 2 for stereo.
+constexpr int kOpusBitrateNbBps = 12000;
+constexpr int kOpusBitrateWbBps = 20000;
+constexpr int kOpusBitrateFbBps = 32000;
+
+// Opus API allows a min bitrate of 500bps, but Opus documentation suggests
+// bitrate should be in the range of 6000 to 510000, inclusive.
+constexpr int kOpusMinBitrateBps = 6000;
+constexpr int kOpusMaxBitrateBps = 510000;
+
 constexpr int kSampleRateHz = 48000;
 
-// Opus API allows a min bitrate of 500bps, but Opus documentation suggests
-// a minimum bitrate of 6kbps.
-constexpr int kMinBitrateBps = 6000;
-
-constexpr int kMaxBitrateBps = 512000;
-
+// These two lists must be sorted from low to high
 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
-constexpr int kSupportedFrameLengths[] = {20, 60, 120};
+constexpr int kANASupportedFrameLengths[] = {20, 60, 120};
+constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
 #else
-constexpr int kSupportedFrameLengths[] = {20, 60};
+constexpr int kANASupportedFrameLengths[] = {20, 60};
+constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
 #endif
 
 // PacketLossFractionSmoother uses an exponential filter with a time constant
 // of -1.0 / ln(0.9999) = 10000 ms.
 constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f;
 
-AudioEncoderOpus::Config CreateConfig(const CodecInst& codec_inst) {
-  AudioEncoderOpus::Config config;
-  config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
-  config.num_channels = codec_inst.channels;
-  config.bitrate_bps = rtc::Optional<int>(codec_inst.rate);
-  config.payload_type = codec_inst.pltype;
-  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
-                                                : AudioEncoderOpus::kAudio;
-  config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
-#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
-  config.low_rate_complexity = 9;
-#endif
-  return config;
-}
-
 // Optimize the loss rate to configure Opus. Basically, optimized loss rate is
 // the input loss rate rounded down to various levels, because a robustly good
 // audio quality is achieved by lowering the packet loss down.
 // Additionally, to prevent toggling, margins are used, i.e., when jumping to
 // a loss rate from below, a higher threshold is used than jumping to the same
 // level from above.
 float OptimizePacketLossRate(float new_loss_rate, float old_loss_rate) {
   RTC_DCHECK_GE(new_loss_rate, 0.0f);
   RTC_DCHECK_LE(new_loss_rate, 1.0f);
   RTC_DCHECK_GE(old_loss_rate, 0.0f);
@@ skipping 20 matching lines @@
                  kLossRate5Margin *
                      (kPacketLossRate5 - old_loss_rate > 0 ? 1 : -1)) {
     return kPacketLossRate5;
   } else if (new_loss_rate >= kPacketLossRate1) {
     return kPacketLossRate1;
   } else {
     return 0.0f;
   }
 }
 
+rtc::Optional<std::string> GetFormatParameter(const SdpAudioFormat& format,
+                                              const std::string& param) {
+  auto it = format.parameters.find(param);
+  return (it == format.parameters.end())
+             ? rtc::Optional<std::string>()
+             : rtc::Optional<std::string>(it->second);
+}
+
+template <typename T>
+rtc::Optional<T> GetFormatParameter(const SdpAudioFormat& format,
+                                    const std::string& param) {
+  return rtc::StringToNumber<T>(GetFormatParameter(format, param).value_or(""));
+};
+
+int CalculateDefaultBitrate(int max_playback_rate, int num_channels) {
+  const int bitrate = [&] {
+    if (max_playback_rate <= 8000) {
+      return kOpusBitrateNbBps * num_channels;
+    } else if (max_playback_rate <= 16000) {
+      return kOpusBitrateWbBps * num_channels;
+    } else {
+      return kOpusBitrateFbBps * num_channels;
+    }
+  }();
+  RTC_DCHECK_GE(bitrate, kOpusMinBitrateBps);
+  RTC_DCHECK_LE(bitrate, kOpusMaxBitrateBps);
+  return bitrate;
+}
+
+// Get the maxaveragebitrate parameter in string-form, so we can properly figure
+// out how invalid it is and accurately log invalid values.
+int CalculateBitrate(int max_playback_rate_hz,
+                     int num_channels,
+                     rtc::Optional<std::string> bitrate_param) {
+  const int default_bitrate =
+      CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
+
+  if (bitrate_param) {
+    const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
+    if (bitrate) {
+      if (*bitrate >= kOpusMinBitrateBps && *bitrate <= kOpusMaxBitrateBps) {
+        return *bitrate;
+      }
+
+      const int new_bitrate = (*bitrate < kOpusMinBitrateBps)
+                                  ? kOpusMinBitrateBps
+                                  : kOpusMaxBitrateBps;
+      LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
+                      << " clamped to " << new_bitrate;
+      return new_bitrate;
+    }
+
+    LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
+                    << "\" replaced by default bitrate " << default_bitrate;
+  }
+
+  return default_bitrate;
+}
+
+rtc::Optional<int> GetChannelCount(const SdpAudioFormat& format) {
+  const auto param = GetFormatParameter(format, "stereo");
+  if (!param || param == "0") {
+    return rtc::Optional<int>(1);
+  } else if (param == "1") {
+    return rtc::Optional<int>(2);
+  }
+  return rtc::Optional<int>();
+}
+
+rtc::Optional<int> GetMaxPlaybackRate(const SdpAudioFormat& format) {
+  const auto param = GetFormatParameter(format, "maxplaybackrate");
+  if (!param) {
+    return rtc::Optional<int>(48000);
+  }
+  const auto parsed = rtc::StringToNumber<int>(*param);
+  if (parsed && *parsed >= 8000) {
+    return rtc::Optional<int>(*parsed);
+  }
+  return rtc::Optional<int>();
+}
+
 }  // namespace
 
+rtc::Optional<AudioCodecInfo> AudioEncoderOpus::QueryAudioEncoder(
+    const SdpAudioFormat& format) {
+  if (STR_CASE_CMP(format.name.c_str(), GetPayloadName()) == 0 &&
+      format.clockrate_hz == 48000 && format.num_channels == 2) {
+
+    const rtc::Optional<int> num_channels = GetChannelCount(format);
+    const rtc::Optional<int> max_playback_rate = GetMaxPlaybackRate(format);
+    if (num_channels && max_playback_rate) {
+      const int bitrate =
+          CalculateBitrate(*max_playback_rate, *num_channels,
+                           GetFormatParameter(format, "maxaveragebitrate"));
+      AudioCodecInfo info(48000, *num_channels, bitrate, kOpusMinBitrateBps,
+                           kOpusMaxBitrateBps);
+      info.allow_comfort_noise = false;
+      info.supports_network_adaption = true;
+
+      return rtc::Optional<AudioCodecInfo>(info);
+    }
+  }
+  return rtc::Optional<AudioCodecInfo>();
+}
+
+AudioEncoderOpus::Config AudioEncoderOpus::CreateConfig(
+    const CodecInst& codec_inst) {
+  AudioEncoderOpus::Config config;
+  config.frame_size_ms = rtc::CheckedDivExact(codec_inst.pacsize, 48);
+  config.num_channels = codec_inst.channels;
+  config.bitrate_bps = rtc::Optional<int>(codec_inst.rate);
+  config.payload_type = codec_inst.pltype;
+  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
+                                                : AudioEncoderOpus::kAudio;
+  config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+  config.low_rate_complexity = 9;
+#endif
+  return config;
+}
+
+AudioEncoderOpus::Config AudioEncoderOpus::CreateConfig(
+    int payload_type,
+    const SdpAudioFormat& format) {
+  AudioEncoderOpus::Config config;
+
+  // Normally, the first two parameters should already have been checked by
+  // QueryAudioEncoder. At this point, we might as well fall back to something
+  // reasonable.
+  config.num_channels = GetChannelCount(format).value_or(1);
+  config.max_playback_rate_hz = GetMaxPlaybackRate(format).value_or(48000);
+  config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
+  config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
+  config.bitrate_bps = rtc::Optional<int>(
+      CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
+                       GetFormatParameter(format, "maxaveragebitrate")));
+  config.payload_type = payload_type;
+  config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
+                                                : AudioEncoderOpus::kAudio;
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+  config.low_rate_complexity = 9;
+#endif
+
+  constexpr int kMinSupportedFrameLength = kOpusSupportedFrameLengths[0];
+  constexpr int kMaxSupportedFrameLength =
+      kOpusSupportedFrameLengths[arraysize(kOpusSupportedFrameLengths) - 1];
+
+  const auto ptime = GetFormatParameter<int>(format, "ptime");
+  if (ptime) {
+    // Pick the next highest supported frame length from
+    // kOpusSupportedFrameLengths. Default to the largest, if we find none.
+    config.frame_size_ms = kMaxSupportedFrameLength;
+    for (const int supported_frame_length : kOpusSupportedFrameLengths) {
+      if (supported_frame_length >= *ptime) {
+        config.frame_size_ms = supported_frame_length;
+        break;
+      }
+    }
+  }
+
+  // For now, minptime and maxptime are only used with ANA. If ptime is outside
+  // of this range, it will get adjusted once ANA takes hold. Ideally, we'd know
+  // if ANA was to be used when setting up the config, and adjust accordingly.
+  const int min_frame_length_ms =
+      std::min(std::max(GetFormatParameter<int>(format, "minptime")
+                            .value_or(kMinSupportedFrameLength),
+                        kMinSupportedFrameLength),
+               kMaxSupportedFrameLength);
+  const int max_frame_length_ms =
+      std::min(std::max(GetFormatParameter<int>(format, "maxptime")
+                            .value_or(kMaxSupportedFrameLength),
+                        kMinSupportedFrameLength),
+               kMaxSupportedFrameLength);
+
+  for (const int frame_length_ms : kANASupportedFrameLengths) {
+    if (frame_length_ms >= min_frame_length_ms &&
+        frame_length_ms <= max_frame_length_ms) {
+      config.supported_frame_lengths_ms.push_back(frame_length_ms);
+    }
+  }
+
+  // As a fallback, just pick the whole set of supported frame lengths.
+  if (config.supported_frame_lengths_ms.empty()) {
+    config.supported_frame_lengths_ms.assign(
+        std::begin(kANASupportedFrameLengths),
+        std::end(kANASupportedFrameLengths));
+  }
+
+  RTC_DCHECK(std::is_sorted(config.supported_frame_lengths_ms.begin(),
+                            config.supported_frame_lengths_ms.end()));
+
+  return config;
+}
+
 class AudioEncoderOpus::PacketLossFractionSmoother {
  public:
   explicit PacketLossFractionSmoother(const Clock* clock)
       : clock_(clock),
         last_sample_time_ms_(clock_->TimeInMilliseconds()),
         smoother_(kAlphaForPacketLossFractionSmoother) {}
 
   // Gets the smoothed packet loss fraction.
   float GetAverage() const {
     float value = smoother_.filtered();
@@ skipping 24 matching lines @@
 AudioEncoderOpus::Config::Config(const Config&) = default;
 AudioEncoderOpus::Config::~Config() = default;
 auto AudioEncoderOpus::Config::operator=(const Config&) -> Config& = default;
 
 bool AudioEncoderOpus::Config::IsOk() const {
   if (frame_size_ms <= 0 || frame_size_ms % 10 != 0)
     return false;
   if (num_channels != 1 && num_channels != 2)
     return false;
   if (bitrate_bps &&
-      (*bitrate_bps < kMinBitrateBps || *bitrate_bps > kMaxBitrateBps))
+      (*bitrate_bps < kOpusMinBitrateBps || *bitrate_bps > kOpusMaxBitrateBps))
     return false;
   if (complexity < 0 || complexity > 10)
     return false;
   if (low_rate_complexity < 0 || low_rate_complexity > 10)
     return false;
   return true;
 }
 
 int AudioEncoderOpus::Config::GetBitrateBps() const {
   RTC_DCHECK(IsOk());
@@ skipping 40 matching lines @@
       bitrate_smoother_(bitrate_smoother
           ? std::move(bitrate_smoother) : std::unique_ptr<SmoothingFilter>(
               // We choose 5sec as initial time constant due to empirical data.
               new SmoothingFilterImpl(5000, config.clock))) {
   RTC_CHECK(RecreateEncoderInstance(config));
 }
 
 AudioEncoderOpus::AudioEncoderOpus(const CodecInst& codec_inst)
     : AudioEncoderOpus(CreateConfig(codec_inst), nullptr) {}
 
+AudioEncoderOpus::AudioEncoderOpus(int payload_type,
+                                   const SdpAudioFormat& format)
+    : AudioEncoderOpus(CreateConfig(payload_type, format), nullptr) {}
+
 AudioEncoderOpus::~AudioEncoderOpus() {
   RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
 }
 
 int AudioEncoderOpus::SampleRateHz() const {
   return kSampleRateHz;
 }
 
 size_t AudioEncoderOpus::NumChannels() const {
   return config_.num_channels;
@@ skipping 107 matching lines @@
   } else if (send_side_bwe_with_overhead_) {
     if (!overhead_bytes_per_packet_) {
       LOG(LS_INFO)
           << "AudioEncoderOpus: Overhead unknown, target audio bitrate "
           << target_audio_bitrate_bps << " bps is ignored.";
       return;
     }
     const int overhead_bps = static_cast<int>(
         *overhead_bytes_per_packet_ * 8 * 100 / Num10MsFramesInNextPacket());
     SetTargetBitrate(std::min(
-        kMaxBitrateBps,
-        std::max(kMinBitrateBps, target_audio_bitrate_bps - overhead_bps)));
+        kOpusMaxBitrateBps,
+        std::max(kOpusMinBitrateBps, target_audio_bitrate_bps - overhead_bps)));
   } else {
     SetTargetBitrate(target_audio_bitrate_bps);
   }
 }
 
 void AudioEncoderOpus::OnReceivedRtt(int rtt_ms) {
   if (!audio_network_adaptor_)
     return;
   audio_network_adaptor_->SetRtt(rtt_ms);
   ApplyAudioNetworkAdaptor();
@@ skipping 10 matching lines @@
 }
 
 void AudioEncoderOpus::SetReceiverFrameLengthRange(int min_frame_length_ms,
                                                    int max_frame_length_ms) {
   // Ensure that |SetReceiverFrameLengthRange| is called before
   // |EnableAudioNetworkAdaptor|, otherwise we need to recreate
   // |audio_network_adaptor_|, which is not a needed use case.
   RTC_DCHECK(!audio_network_adaptor_);
 
   config_.supported_frame_lengths_ms.clear();
-  std::copy_if(std::begin(kSupportedFrameLengths),
-               std::end(kSupportedFrameLengths),
+  std::copy_if(std::begin(kANASupportedFrameLengths),
+               std::end(kANASupportedFrameLengths),
                std::back_inserter(config_.supported_frame_lengths_ms),
                [&](int frame_length_ms) {
                  return frame_length_ms >= min_frame_length_ms &&
                         frame_length_ms <= max_frame_length_ms;
                });
   RTC_DCHECK(std::is_sorted(config_.supported_frame_lengths_ms.begin(),
                             config_.supported_frame_lengths_ms.end()));
 }
 
 AudioEncoder::EncodedInfo AudioEncoderOpus::EncodeImpl(
@@ skipping 118 matching lines @@
   float opt_loss_rate = OptimizePacketLossRate(fraction, packet_loss_rate_);
   if (packet_loss_rate_ != opt_loss_rate) {
     packet_loss_rate_ = opt_loss_rate;
     RTC_CHECK_EQ(
         0, WebRtcOpus_SetPacketLossRate(
                inst_, static_cast<int32_t>(packet_loss_rate_ * 100 + .5)));
   }
 }
 
 void AudioEncoderOpus::SetTargetBitrate(int bits_per_second) {
-  config_.bitrate_bps = rtc::Optional<int>(
-      std::max(std::min(bits_per_second, kMaxBitrateBps), kMinBitrateBps));
+  config_.bitrate_bps = rtc::Optional<int>(std::max(
+      std::min(bits_per_second, kOpusMaxBitrateBps), kOpusMinBitrateBps));
   RTC_DCHECK(config_.IsOk());
   RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps()));
   const auto new_complexity = config_.GetNewComplexity();
   if (new_complexity && complexity_ != *new_complexity) {
     complexity_ = *new_complexity;
     RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
   }
 }
 
 void AudioEncoderOpus::ApplyAudioNetworkAdaptor() {
@@ skipping 20 matching lines @@
     const std::string& config_string,
     RtcEventLog* event_log,
     const Clock* clock) const {
   AudioNetworkAdaptorImpl::Config config;
   config.clock = clock;
   config.event_log = event_log;
   return std::unique_ptr<AudioNetworkAdaptor>(new AudioNetworkAdaptorImpl(
       config,
       ControllerManagerImpl::Create(
           config_string, NumChannels(), supported_frame_lengths_ms(),
-          kMinBitrateBps, num_channels_to_encode_, next_frame_length_ms_,
+          kOpusMinBitrateBps, num_channels_to_encode_, next_frame_length_ms_,
           GetTargetBitrate(), config_.fec_enabled, GetDtx(), clock)));
 }
 
 void AudioEncoderOpus::MaybeUpdateUplinkBandwidth() {
   if (audio_network_adaptor_) {
     int64_t now_ms = rtc::TimeMillis();
     if (!bitrate_smoother_last_update_time_ ||
         now_ms - *bitrate_smoother_last_update_time_ >=
             config_.uplink_bandwidth_update_interval_ms) {
       rtc::Optional<float> smoothed_bitrate = bitrate_smoother_->GetAverage();
       if (smoothed_bitrate)
         audio_network_adaptor_->SetUplinkBandwidth(*smoothed_bitrate);
       bitrate_smoother_last_update_time_ = rtc::Optional<int64_t>(now_ms);
     }
   }
 }
 
 }  // namespace webrtc