Wire up FlexFEC in VideoEngine2.

webrtc/media/engine/webrtcvideoengine2.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 21 matching lines @@
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/video_decoder.h"
 #include "webrtc/video_encoder.h"
 
 namespace cricket {
 namespace {
 
+// Three things happen when the FlexFEC field trial is enabled:
+// 1) FlexFEC is exposed in the default codec list, eventually showing up
+//    in the default SDP.
+// 2) FlexFEC send parameters are set in the VideoSendStream config.
+// 3) FlexFEC receive parameters are set in the FlexfecReceiveStream config,
+//    and the corresponding object is instantiated.
+const char kFlexfecFieldTrialName[] = "WebRTC-FlexFEC-03";
+
+bool IsFlexfecEnabled() {
+  return webrtc::field_trial::FindFullName(kFlexfecFieldTrialName) == "Enabled";
+}
+
 // Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory.
 class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory {
  public:
   // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned
   // by e.g. PeerConnectionFactory.
   explicit EncoderFactoryAdapter(cricket::WebRtcVideoEncoderFactory* factory)
       : factory_(factory) {}
   virtual ~EncoderFactoryAdapter() {}
 
   // Implement webrtc::VideoEncoderFactory.
@@ skipping 381 matching lines @@
   AddCodecAndMaybeRtxCodec(
       MakeVideoCodecWithDefaultFeedbackParams(kDefaultVp8PlType, kVp8CodecName),
       &codecs);
   if (webrtc::VP9Encoder::IsSupported() && webrtc::VP9Decoder::IsSupported()) {
     AddCodecAndMaybeRtxCodec(MakeVideoCodecWithDefaultFeedbackParams(
                                  kDefaultVp9PlType, kVp9CodecName),
                              &codecs);
   }
   if (webrtc::H264Encoder::IsSupported() &&
       webrtc::H264Decoder::IsSupported()) {
-    VideoCodec codec = MakeVideoCodecWithDefaultFeedbackParams(
+    VideoCodec h264_codec = MakeVideoCodecWithDefaultFeedbackParams(
         kDefaultH264PlType, kH264CodecName);
     // TODO(hta): Move all parameter generation for SDP into the codec
     // implementation, for all codecs and parameters.
     // TODO(hta): Move selection of profile-level-id to H.264 codec
     // implementation.
     // TODO(hta): Set FMTP parameters for all codecs of type H264.
-    codec.SetParam(kH264FmtpProfileLevelId,
-                   kH264ProfileLevelConstrainedBaseline);
-    codec.SetParam(kH264FmtpLevelAsymmetryAllowed, "1");
-    codec.SetParam(kH264FmtpPacketizationMode, "1");
-    AddCodecAndMaybeRtxCodec(codec, &codecs);
+    h264_codec.SetParam(kH264FmtpProfileLevelId,
+                        kH264ProfileLevelConstrainedBaseline);
+    h264_codec.SetParam(kH264FmtpLevelAsymmetryAllowed, "1");
+    h264_codec.SetParam(kH264FmtpPacketizationMode, "1");
+    AddCodecAndMaybeRtxCodec(h264_codec, &codecs);
   }
   AddCodecAndMaybeRtxCodec(VideoCodec(kDefaultRedPlType, kRedCodecName),
                            &codecs);
   codecs.push_back(VideoCodec(kDefaultUlpfecType, kUlpfecCodecName));
+
+  if (IsFlexfecEnabled()) {
+    VideoCodec flexfec_codec(kDefaultFlexfecPlType, kFlexfecCodecName);
+    flexfec_codec.SetParam(kFlexfecFmtpRepairWindow, "10000000");

[stefan-webrtc, 2016/11/17 12:48:59]

Comment on how this window size was chosen.

[brandtr, 2016/11/17 13:23:36]

Done.

+    codecs.push_back(flexfec_codec);
+  }
+
   return codecs;
 }
 
 static std::vector<VideoCodec> GetSupportedCodecs(
     const WebRtcVideoEncoderFactory* external_encoder_factory);
 
 rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings>
 WebRtcVideoChannel2::WebRtcVideoSendStream::ConfigureVideoEncoderSettings(
     const VideoCodec& codec) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
@@ skipping 758 matching lines @@
     receive_streams_.erase(prev_stream);
   }
 
   if (!ValidateReceiveSsrcAvailability(sp))
     return false;
 
   for (uint32_t used_ssrc : sp.ssrcs)
     receive_ssrcs_.insert(used_ssrc);
 
   webrtc::VideoReceiveStream::Config config(this);
-  ConfigureReceiverRtp(&config, sp);
+  webrtc::FlexfecConfig flexfec_config;
+  ConfigureReceiverRtp(&config, &flexfec_config, sp);
 
   // Set up A/V sync group based on sync label.
   config.sync_group = sp.sync_label;
 
   config.rtp.remb = send_codec_ ? HasRemb(send_codec_->codec) : false;
   config.rtp.transport_cc =
       send_codec_ ? HasTransportCc(send_codec_->codec) : false;
   config.disable_prerenderer_smoothing =
       video_config_.disable_prerenderer_smoothing;
 
   receive_streams_[ssrc] = new WebRtcVideoReceiveStream(
       call_, sp, std::move(config), external_decoder_factory_, default_stream,
-      recv_codecs_);
+      recv_codecs_, flexfec_config);
 
   return true;
 }
 
 void WebRtcVideoChannel2::ConfigureReceiverRtp(
     webrtc::VideoReceiveStream::Config* config,
+    webrtc::FlexfecConfig* flexfec_config,
     const StreamParams& sp) const {
   uint32_t ssrc = sp.first_ssrc();
 
   config->rtp.remote_ssrc = ssrc;
   config->rtp.local_ssrc = rtcp_receiver_report_ssrc_;
 
   config->rtp.extensions = recv_rtp_extensions_;
   // Whether or not the receive stream sends reduced size RTCP is determined
   // by the send params.
   // TODO(deadbeef): Once we change "send_params" to "sender_params" and
@@ skipping 18 matching lines @@
   for (size_t i = 0; i < recv_codecs_.size(); ++i) {
     uint32_t rtx_ssrc;
     if (recv_codecs_[i].rtx_payload_type != -1 &&
         sp.GetFidSsrc(ssrc, &rtx_ssrc)) {
       webrtc::VideoReceiveStream::Config::Rtp::Rtx& rtx =
           config->rtp.rtx[recv_codecs_[i].codec.id];
       rtx.ssrc = rtx_ssrc;
       rtx.payload_type = recv_codecs_[i].rtx_payload_type;
     }
   }
+
+  // TODO(brandtr): This code needs to be generalized when we add support for
+  // multistream protection.
+  uint32_t flexfec_ssrc;
+  if (sp.GetFecFrSsrc(ssrc, &flexfec_ssrc)) {
+    flexfec_config->flexfec_ssrc = flexfec_ssrc;
+    flexfec_config->protected_media_ssrcs = {ssrc};
+  }
 }
 
 bool WebRtcVideoChannel2::RemoveRecvStream(uint32_t ssrc) {
   LOG(LS_INFO) << "RemoveRecvStream: " << ssrc;
   if (ssrc == 0) {
     LOG(LS_ERROR) << "RemoveRecvStream with 0 ssrc is not supported.";
     return false;
   }
 
   rtc::CritScope stream_lock(&stream_crit_);
@@ skipping 137 matching lines @@
   if (!GetRtpSsrc(packet->cdata(), packet->size(), &ssrc)) {
     return;
   }
 
   int payload_type = 0;
   if (!GetRtpPayloadType(packet->cdata(), packet->size(), &payload_type)) {
     return;
   }
 
   // See if this payload_type is registered as one that usually gets its own
-  // SSRC (RTX) or at least is safe to drop either way (ULPFEC). If it is, and
+  // SSRC (RTX) or at least is safe to drop either way (FEC). If it is, and
   // it wasn't handled above by DeliverPacket, that means we don't know what
   // stream it associates with, and we shouldn't ever create an implicit channel
   // for these.
   for (auto& codec : recv_codecs_) {
     if (payload_type == codec.rtx_payload_type ||
         payload_type == codec.ulpfec.red_rtx_payload_type ||
-        payload_type == codec.ulpfec.ulpfec_payload_type) {
+        payload_type == codec.ulpfec.ulpfec_payload_type ||
+        payload_type == codec.flexfec.flexfec_payload_type) {
       return;
     }
   }
 
   switch (unsignalled_ssrc_handler_->OnUnsignalledSsrc(this, ssrc)) {
     case UnsignalledSsrcHandler::kDropPacket:
       return;
     case UnsignalledSsrcHandler::kDeliverPacket:
       break;
   }
@@ skipping 122 matching lines @@
       encoder_sink_(nullptr),
       parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
       rtp_parameters_(CreateRtpParametersWithOneEncoding()),
       allocated_encoder_(nullptr, webrtc::kVideoCodecUnknown, false),
       sending_(false),
       last_frame_timestamp_us_(0) {
   parameters_.config.rtp.max_packet_size = kVideoMtu;
   parameters_.conference_mode = send_params.conference_mode;
 
   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
+
+  // RTX.
   sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
                  &parameters_.config.rtp.rtx.ssrcs);
+
+  // FlexFEC.
+  // TODO(brandtr): This code needs to be generalized when we add support for
+  // multistream protection.
+  if (IsFlexfecEnabled()) {
+    uint32_t flexfec_ssrc;
+    bool flexfec_enabled = false;
+    for (uint32_t primary_ssrc : parameters_.config.rtp.ssrcs) {
+      if (sp.GetFecFrSsrc(primary_ssrc, &flexfec_ssrc)) {
+        if (flexfec_enabled) {
+          LOG(LS_INFO) << "Multiple FlexFEC streams proposed by remote, but "
+                          "our implementation only supports a single FlexFEC "
+                          "stream. Will not enable FlexFEC for proposed "
+                          "stream with SSRC: "
+                       << flexfec_ssrc << ".";
+          continue;
+        }
+
+        flexfec_enabled = true;
+        parameters_.config.rtp.flexfec.flexfec_ssrc = flexfec_ssrc;
+        parameters_.config.rtp.flexfec.protected_media_ssrcs = {primary_ssrc};
+      }
+    }
+  }
+
   parameters_.config.rtp.c_name = sp.cname;
   if (rtp_extensions) {
     parameters_.config.rtp.extensions = *rtp_extensions;
   }
   parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
                                          ? webrtc::RtcpMode::kReducedSize
                                          : webrtc::RtcpMode::kCompound;
   if (codec_settings) {
     SetCodec(*codec_settings);
   }
@@ skipping 164 matching lines @@
   parameters_.config.encoder_settings.encoder = new_encoder.encoder;
   parameters_.config.encoder_settings.full_overuse_time = new_encoder.external;
   parameters_.config.encoder_settings.payload_name = codec_settings.codec.name;
   parameters_.config.encoder_settings.payload_type = codec_settings.codec.id;
   if (new_encoder.external) {
     webrtc::VideoCodecType type = CodecTypeFromName(codec_settings.codec.name);
     parameters_.config.encoder_settings.internal_source =
         external_encoder_factory_->EncoderTypeHasInternalSource(type);
   }
   parameters_.config.rtp.ulpfec = codec_settings.ulpfec;
+  parameters_.config.rtp.flexfec.flexfec_payload_type =
+      codec_settings.flexfec.flexfec_payload_type;
 
   // Set RTX payload type if RTX is enabled.
   if (!parameters_.config.rtp.rtx.ssrcs.empty()) {
     if (codec_settings.rtx_payload_type == -1) {
       LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX "
                          "payload type. Ignoring.";
       parameters_.config.rtp.rtx.ssrcs.clear();
     } else {
       parameters_.config.rtp.rtx.payload_type = codec_settings.rtx_payload_type;
     }
@@ skipping 355 matching lines @@
   // Call stream_->Start() if necessary conditions are met.
   UpdateSendState();
 }
 
 WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
     webrtc::Call* call,
     const StreamParams& sp,
     webrtc::VideoReceiveStream::Config config,
     WebRtcVideoDecoderFactory* external_decoder_factory,
     bool default_stream,
-    const std::vector<VideoCodecSettings>& recv_codecs)
+    const std::vector<VideoCodecSettings>& recv_codecs,
+    webrtc::FlexfecConfig flexfec_config)

[magjed_webrtc, 2016/11/17 13:37:19]

Make this const-ref instead of by-value.

[brandtr, 2016/11/17 17:28:58]

Done.

     : call_(call),
       stream_params_(sp),
       stream_(NULL),
       default_stream_(default_stream),
       config_(std::move(config)),
+      flexfec_config_(flexfec_config),
+      flexfec_stream_(nullptr),
       external_decoder_factory_(external_decoder_factory),
       sink_(NULL),
       first_frame_timestamp_(-1),
       estimated_remote_start_ntp_time_ms_(0) {
   config_.renderer = this;
   std::vector<AllocatedDecoder> old_decoders;
   ConfigureCodecs(recv_codecs, &old_decoders);
   RecreateWebRtcStream();
   RTC_DCHECK(old_decoders.empty());
 }
@@ skipping 108 matching lines @@
     webrtc::VideoReceiveStream::Decoder decoder;
     decoder.decoder = allocated_decoder.decoder;
     decoder.payload_type = recv_codecs[i].codec.id;
     decoder.payload_name = recv_codecs[i].codec.name;
     ConfigureDecoderSpecifics(&decoder, recv_codecs[i].codec);
     config_.decoders.push_back(decoder);
   }
 
   // TODO(pbos): Reconfigure RTX based on incoming recv_codecs.
   config_.rtp.ulpfec = recv_codecs.front().ulpfec;
+  flexfec_config_.flexfec_payload_type =
+      recv_codecs.front().flexfec.flexfec_payload_type;
   config_.rtp.nack.rtp_history_ms =
       HasNack(recv_codecs.begin()->codec) ? kNackHistoryMs : 0;
 }
 
 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::SetLocalSsrc(
     uint32_t local_ssrc) {
   // TODO(pbos): Consider turning this sanity check into a RTC_DCHECK. You
   // should not be able to create a sender with the same SSRC as a receiver, but
   // right now this can't be done due to unittests depending on receiving what
   // they are sending from the same MediaChannel.
@@ skipping 51 matching lines @@
     needs_recreation = true;
   }
   if (needs_recreation) {
     LOG(LS_INFO) << "RecreateWebRtcStream (recv) because of SetRecvParameters";
     RecreateWebRtcStream();
     ClearDecoders(&old_decoders);
   }
 }
 
 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::RecreateWebRtcStream() {
-  if (stream_ != NULL) {
+  if (flexfec_stream_) {
+    call_->DestroyFlexfecReceiveStream(flexfec_stream_);
+    flexfec_stream_ = nullptr;
+  }
+  if (stream_) {
     call_->DestroyVideoReceiveStream(stream_);
   }
   stream_ = call_->CreateVideoReceiveStream(config_.Copy());
   stream_->Start();
+  if (IsFlexfecEnabled() && flexfec_config_.IsValid()) {
+    flexfec_stream_ = call_->CreateFlexfecReceiveStream(flexfec_config_);
+    flexfec_stream_->Start();
+  }
 }
 
 void WebRtcVideoChannel2::WebRtcVideoReceiveStream::ClearDecoders(
     std::vector<AllocatedDecoder>* allocated_decoders) {
   for (size_t i = 0; i < allocated_decoders->size(); ++i) {
     if ((*allocated_decoders)[i].external) {
       external_decoder_factory_->DestroyVideoDecoder(
           (*allocated_decoders)[i].external_decoder);
     }
     delete (*allocated_decoders)[i].decoder;
@@ skipping 98 matching lines @@
 
 WebRtcVideoChannel2::VideoCodecSettings::VideoCodecSettings()
     : rtx_payload_type(-1) {}
 
 bool WebRtcVideoChannel2::VideoCodecSettings::operator==(
     const WebRtcVideoChannel2::VideoCodecSettings& other) const {
   return codec == other.codec &&
          ulpfec.ulpfec_payload_type == other.ulpfec.ulpfec_payload_type &&
          ulpfec.red_payload_type == other.ulpfec.red_payload_type &&
          ulpfec.red_rtx_payload_type == other.ulpfec.red_rtx_payload_type &&
+         flexfec.flexfec_payload_type == other.flexfec.flexfec_payload_type &&

[magjed_webrtc, 2016/11/17 13:37:19]

Maybe you should implement operator== for FlexfecConfig?

[brandtr, 2016/11/17 17:28:58]

Done. Also implemented it for UlpfecConfig, to further clean up this function.

+         flexfec.flexfec_ssrc == other.flexfec.flexfec_ssrc &&
+         flexfec.protected_media_ssrcs == other.flexfec.protected_media_ssrcs &&
          rtx_payload_type == other.rtx_payload_type;
 }
 
 bool WebRtcVideoChannel2::VideoCodecSettings::operator!=(
     const WebRtcVideoChannel2::VideoCodecSettings& other) const {
   return !(*this == other);
 }
 
 std::vector<WebRtcVideoChannel2::VideoCodecSettings>
 WebRtcVideoChannel2::MapCodecs(const std::vector<VideoCodec>& codecs) {
   RTC_DCHECK(!codecs.empty());
 
   std::vector<VideoCodecSettings> video_codecs;
   std::map<int, bool> payload_used;
   std::map<int, VideoCodec::CodecType> payload_codec_type;
   // |rtx_mapping| maps video payload type to rtx payload type.
   std::map<int, int> rtx_mapping;
 
   webrtc::UlpfecConfig ulpfec_config;
+  int flexfec_payload_type = -1;
 
   for (size_t i = 0; i < codecs.size(); ++i) {
     const VideoCodec& in_codec = codecs[i];
     int payload_type = in_codec.id;
 
     if (payload_used[payload_type]) {
       LOG(LS_ERROR) << "Payload type already registered: "
                     << in_codec.ToString();
       return std::vector<VideoCodecSettings>();
     }
     payload_used[payload_type] = true;
     payload_codec_type[payload_type] = in_codec.GetCodecType();
 
     switch (in_codec.GetCodecType()) {
       case VideoCodec::CODEC_RED: {
         // RED payload type, should not have duplicates.
         RTC_DCHECK(ulpfec_config.red_payload_type == -1);
         ulpfec_config.red_payload_type = in_codec.id;
         continue;
       }
 
       case VideoCodec::CODEC_ULPFEC: {
         // ULPFEC payload type, should not have duplicates.
         RTC_DCHECK(ulpfec_config.ulpfec_payload_type == -1);
         ulpfec_config.ulpfec_payload_type = in_codec.id;
         continue;
       }
 
       case VideoCodec::CODEC_FLEXFEC: {
-        // TODO(brandtr): To be implemented.
+        // FlexFEC payload type, should not have duplicates.
+        RTC_DCHECK(flexfec_payload_type == -1);

[magjed_webrtc, 2016/11/17 13:37:19]

Use RTC_DCHECK_EQ(-1, flexfec_payload_type) instead.

[brandtr, 2016/11/17 17:28:58]

Done.

+        flexfec_payload_type = in_codec.id;
         continue;
       }
 
       case VideoCodec::CODEC_RTX: {
         int associated_payload_type;
         if (!in_codec.GetParam(kCodecParamAssociatedPayloadType,
                                &associated_payload_type) ||
             !IsValidRtpPayloadType(associated_payload_type)) {
           LOG(LS_ERROR)
               << "RTX codec with invalid or no associated payload type: "
@@ skipping 29 matching lines @@
       return std::vector<VideoCodecSettings>();
     }
 
     if (it->first == ulpfec_config.red_payload_type) {
       ulpfec_config.red_rtx_payload_type = it->second;
     }
   }
 
   for (size_t i = 0; i < video_codecs.size(); ++i) {
     video_codecs[i].ulpfec = ulpfec_config;
+    video_codecs[i].flexfec.flexfec_payload_type = flexfec_payload_type;
     if (rtx_mapping[video_codecs[i].codec.id] != 0 &&
         rtx_mapping[video_codecs[i].codec.id] !=
             ulpfec_config.red_payload_type) {
       video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id];
     }
   }
 
   return video_codecs;
 }
 
 }  // namespace cricket