Index: media/engine/webrtcvideoengine.cc |
diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc |
index 29fc442cb1f2742a88ba882107abec1fa5c88eca..1e591928bf7bb1468b2835ded559135d95050740 100644 |
--- a/media/engine/webrtcvideoengine.cc |
+++ b/media/engine/webrtcvideoengine.cc |
@@ -24,6 +24,7 @@ |
#include "api/video_codecs/video_encoder_factory.h" |
#include "call/call.h" |
#include "common_video/h264/profile_level_id.h" |
+#include "media/base/codec.h" |
#include "media/engine/constants.h" |
#include "media/engine/internaldecoderfactory.h" |
#include "media/engine/internalencoderfactory.h" |
@@ -36,6 +37,8 @@ |
#include "media/engine/webrtcmediaengine.h" |
#include "media/engine/webrtcvideoencoderfactory.h" |
#include "media/engine/webrtcvoiceengine.h" |
+#include "modules/video_coding/codecs/stereo/include/stereo_decoder_adapter.h" |
+#include "modules/video_coding/codecs/stereo/include/stereo_encoder_adapter.h" |
#include "rtc_base/copyonwritebuffer.h" |
#include "rtc_base/logging.h" |
#include "rtc_base/stringutils.h" |
@@ -70,6 +73,9 @@ class EncoderFactoryAdapter { |
const VideoCodec& codec, |
bool is_conference_mode_screenshare) const = 0; |
+ virtual AllocatedEncoder CreateStereoVideoEncoder( |
+ const VideoCodec& codec) = 0; |
+ |
virtual std::vector<VideoCodec> GetSupportedCodecs() const = 0; |
}; |
@@ -80,6 +86,10 @@ class DecoderFactoryAdapter { |
virtual std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder( |
const VideoCodec& codec, |
const VideoDecoderParams& decoder_params) const = 0; |
+ |
+ virtual std::unique_ptr<webrtc::VideoDecoder> CreateStereoVideoDecoder( |
+ const VideoCodec& codec, |
+ VideoDecoderParams params) = 0; |
}; |
namespace { |
@@ -107,10 +117,13 @@ class CricketEncoderFactoryAdapter : public EncoderFactoryAdapter { |
const VideoCodec& codec, |
bool is_conference_mode_screenshare) const override; |
+ AllocatedEncoder CreateStereoVideoEncoder(const VideoCodec& codec) override; |
+ |
std::vector<VideoCodec> GetSupportedCodecs() const override; |
const std::unique_ptr<WebRtcVideoEncoderFactory> internal_encoder_factory_; |
WebRtcVideoEncoderFactory* const external_encoder_factory_; |
+ std::unique_ptr<WebRtcVideoEncoderFactory> stereo_encoder_factory_; |
}; |
class CricketDecoderFactoryAdapter : public DecoderFactoryAdapter { |
@@ -129,8 +142,13 @@ class CricketDecoderFactoryAdapter : public DecoderFactoryAdapter { |
const VideoCodec& codec, |
const VideoDecoderParams& decoder_params) const override; |
+ std::unique_ptr<webrtc::VideoDecoder> CreateStereoVideoDecoder( |
+ const VideoCodec& codec, |
+ VideoDecoderParams params) override; |
+ |
const std::unique_ptr<WebRtcVideoDecoderFactory> internal_decoder_factory_; |
WebRtcVideoDecoderFactory* const external_decoder_factory_; |
+ std::unique_ptr<WebRtcVideoDecoderFactory> stereo_decoder_factory_; |
}; |
// Wraps webrtc::VideoEncoderFactory into common EncoderFactoryAdapter |
@@ -155,6 +173,10 @@ class WebRtcEncoderFactoryAdapter : public EncoderFactoryAdapter { |
info.has_internal_source); |
} |
+ AllocatedEncoder CreateStereoVideoEncoder(const VideoCodec& codec) override { |
+ return AllocatedEncoder(); |
+ } |
+ |
std::vector<VideoCodec> GetSupportedCodecs() const override { |
if (!encoder_factory_) |
return std::vector<VideoCodec>(); |
@@ -189,10 +211,137 @@ class WebRtcDecoderFactoryAdapter : public DecoderFactoryAdapter { |
webrtc::SdpVideoFormat(codec.name, codec.params)) |
: nullptr; |
} |
+ std::unique_ptr<webrtc::VideoDecoder> CreateStereoVideoDecoder( |
+ const VideoCodec& codec, |
+ VideoDecoderParams params) override { |
+ return nullptr; |
+ } |
std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_; |
}; |
+// Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory. |
+class StereoEncoderFactoryAdapter : public webrtc::VideoEncoderFactoryEx { |
+ public: |
+ // StereoEncoderFactoryAdapter doesn't take ownership of |factory|, which is |
+ // owned by e.g. PeerConnectionFactory. |
+ explicit StereoEncoderFactoryAdapter( |
+ cricket::WebRtcVideoEncoderFactory* factory, |
+ const cricket::VideoCodec& codec) |
+ : factory_(factory), codec_(codec) {} |
+ virtual ~StereoEncoderFactoryAdapter() {} |
+ |
+ // Implement webrtc::VideoEncoderFactory. |
+ webrtc::VideoEncoder* Create() override { |
+ return factory_->CreateVideoEncoder(codec_); |
+ } |
+ |
+ void Destroy(webrtc::VideoEncoder* encoder) override { |
+ return factory_->DestroyVideoEncoder(encoder); |
+ } |
+ |
+ private: |
+ cricket::WebRtcVideoEncoderFactory* const factory_; |
+ const cricket::VideoCodec codec_; |
+}; |
+ |
+// An encoder factory that wraps Create requests for all codec types |
+// with a webrtc::StereoEncoderAdapter. |
+class WebRtcStereoEncoderFactory : public cricket::WebRtcVideoEncoderFactory { |
+ public: |
+ explicit WebRtcStereoEncoderFactory( |
+ cricket::WebRtcVideoEncoderFactory* factory) |
+ : factory_(factory) {} |
+ |
+ webrtc::VideoEncoder* CreateVideoEncoder( |
+ const cricket::VideoCodec& codec) override { |
+ RTC_DCHECK(factory_ != NULL); |
+ return new webrtc::StereoEncoderAdapter( |
+ new StereoEncoderFactoryAdapter(factory_, codec)); |
+ } |
+ |
+ const std::vector<cricket::VideoCodec>& supported_codecs() const override { |
+ return factory_->supported_codecs(); |
+ } |
+ |
+ bool EncoderTypeHasInternalSource( |
+ webrtc::VideoCodecType type) const override { |
+ return factory_->EncoderTypeHasInternalSource(type); |
+ } |
+ |
+ void DestroyVideoEncoder(webrtc::VideoEncoder* encoder) override { |
+ delete encoder; |
+ } |
+ |
+ private: |
+ cricket::WebRtcVideoEncoderFactory* factory_; |
+}; |
+ |
+// Wrap cricket::WebRtcVideoDecoderFactory as a webrtc::VideoDecoderFactory. |
+class StereoDecoderFactoryAdapter : public webrtc::VideoDecoderFactoryEx { |
+ public: |
+ // StereoDecoderFactoryAdapter doesn't take ownership of |factory|. |
+ explicit StereoDecoderFactoryAdapter( |
+ cricket::WebRtcVideoDecoderFactory* factory, |
+ webrtc::VideoCodecType type, |
+ VideoDecoderParams* params) |
+ : factory_(factory), type_(type) { |
+ if (params) { |
+ params_.reset(new VideoDecoderParams(*params)); |
+ } |
+ } |
+ virtual ~StereoDecoderFactoryAdapter() {} |
+ |
+ // Implement webrtc::VideoDecoderFactory. |
+ webrtc::VideoDecoder* Create() override { |
+ RTC_DCHECK(factory_); |
+ if (params_) |
+ return factory_->CreateVideoDecoderWithParams(type_, *params_); |
+ else |
+ return factory_->CreateVideoDecoder(type_); |
+ } |
+ |
+ void Destroy(webrtc::VideoDecoder* decoder) override { |
+ return factory_->DestroyVideoDecoder(decoder); |
+ } |
+ |
+ private: |
+ cricket::WebRtcVideoDecoderFactory* const factory_; |
+ const webrtc::VideoCodecType type_; |
+ std::unique_ptr<VideoDecoderParams> params_; |
+}; |
+ |
+// A decoder factory that wraps Create requests for all codec types |
+// with a webrtc::StereoDecoderAdapter. |
+class WebRtcStereoDecoderFactory : public cricket::WebRtcVideoDecoderFactory { |
+ public: |
+ explicit WebRtcStereoDecoderFactory( |
+ cricket::WebRtcVideoDecoderFactory* factory) |
+ : factory_(factory) {} |
+ |
+ webrtc::VideoDecoder* CreateVideoDecoder( |
+ webrtc::VideoCodecType type) override { |
+ RTC_DCHECK(factory_ != NULL); |
+ return new webrtc::StereoDecoderAdapter(new StereoDecoderFactoryAdapter( |
+ factory_, webrtc::kVideoCodecVP9, nullptr)); |
+ } |
+ |
+ webrtc::VideoDecoder* CreateVideoDecoderWithParams( |
+ webrtc::VideoCodecType type, |
+ VideoDecoderParams params) override { |
+ RTC_DCHECK(factory_ != NULL); |
+ return new webrtc::StereoDecoderAdapter(new StereoDecoderFactoryAdapter( |
+ factory_, webrtc::kVideoCodecVP9, ¶ms)); |
+ } |
+ |
+ void DestroyVideoDecoder(webrtc::VideoDecoder* decoder) override { |
+ delete decoder; |
+ } |
+ |
+ private: |
+ cricket::WebRtcVideoDecoderFactory* factory_; |
+}; |
+ |
// If this field trial is enabled, we will enable sending FlexFEC and disable |
// sending ULPFEC whenever the former has been negotiated in the SDPs. |
bool IsFlexfecFieldTrialEnabled() { |
@@ -556,6 +705,14 @@ std::vector<VideoCodec> AssignPayloadTypesAndAddAssociatedRtxCodecs( |
if (payload_type > kLastDynamicPayloadType) |
break; |
} |
+ |
+ if (CodecNamesEq(codec.name, kVp9CodecName)) { |
+ output_codecs.push_back( |
+ VideoCodec::CreateStereoCodec(payload_type, codec)); |
+ ++payload_type; |
+ if (payload_type > kLastDynamicPayloadType) |
+ break; |
+ } |
} |
return output_codecs; |
} |
@@ -583,12 +740,11 @@ std::vector<VideoCodec> CricketEncoderFactoryAdapter::GetSupportedCodecs() |
return AssignPayloadTypesAndAddAssociatedRtxCodecs(codecs); |
} |
-WebRtcVideoChannel::WebRtcVideoChannel( |
- webrtc::Call* call, |
- const MediaConfig& config, |
- const VideoOptions& options, |
- const EncoderFactoryAdapter* encoder_factory, |
- const DecoderFactoryAdapter* decoder_factory) |
+WebRtcVideoChannel::WebRtcVideoChannel(webrtc::Call* call, |
+ const MediaConfig& config, |
+ const VideoOptions& options, |
+ EncoderFactoryAdapter* encoder_factory, |
+ DecoderFactoryAdapter* decoder_factory) |
: VideoMediaChannel(config), |
call_(call), |
unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_), |
@@ -619,6 +775,9 @@ WebRtcVideoChannel::SelectSendVideoCodec( |
encoder_factory_->GetSupportedCodecs(); |
// Select the first remote codec that is supported locally. |
for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) { |
+ // HARDCODE TO ALPHA |
+ if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec)) |
+ continue; |
// For H264, we will limit the encode level to the remote offered level |
// regardless if level asymmetry is allowed or not. This is strictly not |
// following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2 |
@@ -631,6 +790,24 @@ WebRtcVideoChannel::SelectSendVideoCodec( |
return rtc::Optional<VideoCodecSettings>(); |
} |
+rtc::Optional<WebRtcVideoChannel::VideoCodecSettings> |
+WebRtcVideoChannel::SelectStereoAssociatedVideoCodec( |
+ const std::vector<VideoCodecSettings>& remote_mapped_codecs) const { |
+ const std::vector<VideoCodec> local_supported_codecs = |
+ encoder_factory_->GetSupportedCodecs(); |
+ |
+ // Select the first remote codec that is supported locally. |
+ for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) { |
+ // HARDCODE TO VP9 |
+ if (!CodecNamesEq(remote_mapped_codec.codec.name.c_str(), kVp9CodecName)) |
+ continue; |
+ if (!cricket::VideoCodec::IsStereoCodec(remote_mapped_codec.codec) && |
+ FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec)) |
+ return rtc::Optional<VideoCodecSettings>(remote_mapped_codec); |
+ } |
+ return rtc::Optional<VideoCodecSettings>(); |
+} |
+ |
bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged( |
std::vector<VideoCodecSettings> before, |
std::vector<VideoCodecSettings> after) { |
@@ -677,6 +854,19 @@ bool WebRtcVideoChannel::GetChangedSendParameters( |
return false; |
} |
+ if (VideoCodec::IsStereoCodec(selected_send_codec->codec)) { |
+ rtc::Optional<VideoCodecSettings> associated_codec_settings = |
+ SelectStereoAssociatedVideoCodec(MapCodecs(params.codecs)); |
+ LOG(LS_ERROR) << __func__ << associated_codec_settings->codec.ToString(); |
+ if (!associated_codec_settings) { |
+ LOG(LS_ERROR) |
+ << "Stereo codec is not associated with any supported codec."; |
+ return false; |
+ } |
+ associated_codec_settings->stereo_codec.emplace(selected_send_codec->codec); |
+ selected_send_codec = associated_codec_settings; |
+ } |
+ |
// Never enable sending FlexFEC, unless we are in the experiment. |
if (!IsFlexfecFieldTrialEnabled()) { |
if (selected_send_codec->flexfec_payload_type != -1) { |
@@ -1549,7 +1739,7 @@ WebRtcVideoChannel::WebRtcVideoSendStream::WebRtcVideoSendStream( |
const StreamParams& sp, |
webrtc::VideoSendStream::Config config, |
const VideoOptions& options, |
- const EncoderFactoryAdapter* encoder_factory, |
+ EncoderFactoryAdapter* encoder_factory, |
bool enable_cpu_overuse_detection, |
int max_bitrate_bps, |
const rtc::Optional<VideoCodecSettings>& codec_settings, |
@@ -1750,6 +1940,17 @@ CricketEncoderFactoryAdapter::CreateVideoEncoder( |
return AllocatedEncoder(); |
} |
+EncoderFactoryAdapter::AllocatedEncoder |
+CricketEncoderFactoryAdapter::CreateStereoVideoEncoder( |
+ const VideoCodec& codec) { |
+ WebRtcStereoEncoderFactory* stereo_factory = |
+ new WebRtcStereoEncoderFactory(internal_encoder_factory_.get()); |
+ stereo_encoder_factory_.reset(stereo_factory); |
+ return AllocatedEncoder(std::unique_ptr<webrtc::VideoEncoder>( |
+ stereo_factory->CreateVideoEncoder(codec)), |
+ false, false /* is_external */); |
+} |
+ |
void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( |
const VideoCodecSettings& codec_settings, |
bool force_encoder_allocation) { |
@@ -1757,6 +1958,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( |
parameters_.encoder_config = CreateVideoEncoderConfig(codec_settings.codec); |
RTC_DCHECK_GT(parameters_.encoder_config.number_of_streams, 0); |
+ const bool is_stereo_codec = codec_settings.stereo_codec.has_value(); |
+ |
// Do not re-create encoders of the same type. We can't overwrite |
// |allocated_encoder_| immediately, because we need to release it after the |
// RecreateWebRtcStream() call. |
@@ -1768,8 +1971,10 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( |
webrtc::VideoEncoderConfig::ContentType::kScreen && |
parameters_.conference_mode; |
EncoderFactoryAdapter::AllocatedEncoder new_allocated_encoder = |
- encoder_factory_->CreateVideoEncoder(codec_settings.codec, |
- is_conference_mode_screenshare); |
+ is_stereo_codec |
+ ? encoder_factory_->CreateStereoVideoEncoder(codec_settings.codec) |
+ : encoder_factory_->CreateVideoEncoder( |
+ codec_settings.codec, is_conference_mode_screenshare); |
new_encoder = std::unique_ptr<webrtc::VideoEncoder>( |
std::move(new_allocated_encoder.encoder)); |
parameters_.config.encoder_settings.encoder = new_encoder.get(); |
@@ -1780,8 +1985,13 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( |
} else { |
new_encoder = std::move(allocated_encoder_); |
} |
- parameters_.config.encoder_settings.payload_name = codec_settings.codec.name; |
- parameters_.config.encoder_settings.payload_type = codec_settings.codec.id; |
+ VideoCodec payload_codec = is_stereo_codec |
+ ? codec_settings.stereo_codec.value() |
+ : codec_settings.codec; |
+ parameters_.config.encoder_settings.payload_name = payload_codec.name; |
+ parameters_.config.encoder_settings.payload_type = payload_codec.id; |
+ parameters_.config.encoder_settings.stereo_associated_payload_name = |
+ codec_settings.codec.name; |
parameters_.config.rtp.ulpfec = codec_settings.ulpfec; |
parameters_.config.rtp.flexfec.payload_type = |
codec_settings.flexfec_payload_type; |
@@ -2024,6 +2234,7 @@ VideoSenderInfo WebRtcVideoChannel::WebRtcVideoSendStream::GetVideoSenderInfo( |
for (uint32_t ssrc : parameters_.config.rtp.ssrcs) |
info.add_ssrc(ssrc); |
+ // TODO(emircan): Add stereo codec case. |
if (parameters_.codec_settings) { |
info.codec_name = parameters_.codec_settings->codec.name; |
info.codec_payload_type = rtc::Optional<int>( |
@@ -2150,7 +2361,7 @@ WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream( |
webrtc::Call* call, |
const StreamParams& sp, |
webrtc::VideoReceiveStream::Config config, |
- const DecoderFactoryAdapter* decoder_factory, |
+ DecoderFactoryAdapter* decoder_factory, |
bool default_stream, |
const std::vector<VideoCodecSettings>& recv_codecs, |
const webrtc::FlexfecReceiveStream::Config& flexfec_config) |
@@ -2225,6 +2436,17 @@ CricketDecoderFactoryAdapter::CreateVideoDecoder( |
return internal_decoder; |
} |
+std::unique_ptr<webrtc::VideoDecoder> |
+CricketDecoderFactoryAdapter::CreateStereoVideoDecoder( |
+ const VideoCodec& codec, |
+ VideoDecoderParams params) { |
+ webrtc::VideoCodecType type = webrtc::PayloadStringToCodecType(codec.name); |
+ stereo_decoder_factory_.reset( |
+ new WebRtcStereoDecoderFactory(internal_decoder_factory_.get())); |
+ return std::unique_ptr<webrtc::VideoDecoder>( |
+ stereo_decoder_factory_->CreateVideoDecoderWithParams(type, params)); |
+} |
+ |
void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( |
const std::vector<VideoCodecSettings>& recv_codecs, |
DecoderMap* old_decoders) { |
@@ -2232,6 +2454,8 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( |
config_.decoders.clear(); |
config_.rtp.rtx_associated_payload_types.clear(); |
for (const auto& recv_codec : recv_codecs) { |
+ const bool is_stereo_codec = |
+ cricket::VideoCodec::IsStereoCodec(recv_codec.codec); |
webrtc::SdpVideoFormat video_format(recv_codec.codec.name, |
recv_codec.codec.params); |
std::unique_ptr<webrtc::VideoDecoder> new_decoder; |
@@ -2243,8 +2467,11 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( |
} |
if (!new_decoder) { |
- new_decoder = decoder_factory_->CreateVideoDecoder(recv_codec.codec, |
- {stream_params_.id}); |
+ new_decoder = is_stereo_codec |
+ ? decoder_factory_->CreateStereoVideoDecoder( |
+ recv_codec.codec, {stream_params_.id}) |
+ : decoder_factory_->CreateVideoDecoder( |
+ recv_codec.codec, {stream_params_.id}); |
} |
webrtc::VideoReceiveStream::Decoder decoder; |