| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 */ | |
| 10 | |
| 11 #include "webrtc/common_types.h" | |
| 12 | |
| 13 #include <algorithm> // std::max | |
| 14 | |
| 15 #include "webrtc/base/checks.h" | |
| 16 #include "webrtc/base/logging.h" | |
| 17 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" | |
| 18 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" | |
| 19 #include "webrtc/modules/video_coding/main/source/encoded_frame.h" | |
| 20 #include "webrtc/modules/video_coding/main/source/video_coding_impl.h" | |
| 21 #include "webrtc/modules/video_coding/utility/include/quality_scaler.h" | |
| 22 #include "webrtc/system_wrappers/include/clock.h" | |
| 23 | |
| 24 namespace webrtc { | |
| 25 namespace vcm { | |
| 26 | |
| 27 VideoSender::VideoSender(Clock* clock, | |
| 28 EncodedImageCallback* post_encode_callback, | |
| 29 VideoEncoderRateObserver* encoder_rate_observer, | |
| 30 VCMQMSettingsCallback* qm_settings_callback) | |
| 31 : clock_(clock), | |
| 32 process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), | |
| 33 _encoder(nullptr), | |
| 34 _encodedFrameCallback(post_encode_callback), | |
| 35 _nextFrameTypes(1, kVideoFrameDelta), | |
| 36 _mediaOpt(clock_), | |
| 37 _sendStatsCallback(nullptr), | |
| 38 _codecDataBase(encoder_rate_observer, &_encodedFrameCallback), | |
| 39 frame_dropper_enabled_(true), | |
| 40 _sendStatsTimer(1000, clock_), | |
| 41 current_codec_(), | |
| 42 qm_settings_callback_(qm_settings_callback), | |
| 43 protection_callback_(nullptr), | |
| 44 encoder_params_({0, 0, 0, 0}) { | |
| 45 // Allow VideoSender to be created on one thread but used on another, post | |
| 46 // construction. This is currently how this class is being used by at least | |
| 47 // one external project (diffractor). | |
| 48 _mediaOpt.EnableQM(qm_settings_callback_ != nullptr); | |
| 49 _mediaOpt.Reset(); | |
| 50 main_thread_.DetachFromThread(); | |
| 51 } | |
| 52 | |
| 53 VideoSender::~VideoSender() {} | |
| 54 | |
| 55 int32_t VideoSender::Process() { | |
| 56 int32_t returnValue = VCM_OK; | |
| 57 | |
| 58 if (_sendStatsTimer.TimeUntilProcess() == 0) { | |
| 59 _sendStatsTimer.Processed(); | |
| 60 CriticalSectionScoped cs(process_crit_sect_.get()); | |
| 61 if (_sendStatsCallback != nullptr) { | |
| 62 uint32_t bitRate = _mediaOpt.SentBitRate(); | |
| 63 uint32_t frameRate = _mediaOpt.SentFrameRate(); | |
| 64 _sendStatsCallback->SendStatistics(bitRate, frameRate); | |
| 65 } | |
| 66 } | |
| 67 | |
| 68 { | |
| 69 rtc::CritScope cs(¶ms_lock_); | |
| 70 // Force an encoder parameters update, so that incoming frame rate is | |
| 71 // updated even if bandwidth hasn't changed. | |
| 72 encoder_params_.input_frame_rate = _mediaOpt.InputFrameRate(); | |
| 73 } | |
| 74 | |
| 75 return returnValue; | |
| 76 } | |
| 77 | |
| 78 int64_t VideoSender::TimeUntilNextProcess() { | |
| 79 return _sendStatsTimer.TimeUntilProcess(); | |
| 80 } | |
| 81 | |
| 82 // Register the send codec to be used. | |
| 83 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, | |
| 84 uint32_t numberOfCores, | |
| 85 uint32_t maxPayloadSize) { | |
| 86 RTC_DCHECK(main_thread_.CalledOnValidThread()); | |
| 87 rtc::CritScope lock(&send_crit_); | |
| 88 if (sendCodec == nullptr) { | |
| 89 return VCM_PARAMETER_ERROR; | |
| 90 } | |
| 91 | |
| 92 bool ret = | |
| 93 _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize); | |
| 94 | |
| 95 // Update encoder regardless of result to make sure that we're not holding on | |
| 96 // to a deleted instance. | |
| 97 _encoder = _codecDataBase.GetEncoder(); | |
| 98 // Cache the current codec here so they can be fetched from this thread | |
| 99 // without requiring the _sendCritSect lock. | |
| 100 current_codec_ = *sendCodec; | |
| 101 | |
| 102 if (!ret) { | |
| 103 LOG(LS_ERROR) << "Failed to initialize set encoder with payload name '" | |
| 104 << sendCodec->plName << "'."; | |
| 105 return VCM_CODEC_ERROR; | |
| 106 } | |
| 107 | |
| 108 int numLayers; | |
| 109 if (sendCodec->codecType == kVideoCodecVP8) { | |
| 110 numLayers = sendCodec->codecSpecific.VP8.numberOfTemporalLayers; | |
| 111 } else if (sendCodec->codecType == kVideoCodecVP9) { | |
| 112 numLayers = sendCodec->codecSpecific.VP9.numberOfTemporalLayers; | |
| 113 } else { | |
| 114 numLayers = 1; | |
| 115 } | |
| 116 | |
| 117 // If we have screensharing and we have layers, we disable frame dropper. | |
| 118 bool disable_frame_dropper = | |
| 119 numLayers > 1 && sendCodec->mode == kScreensharing; | |
| 120 if (disable_frame_dropper) { | |
| 121 _mediaOpt.EnableFrameDropper(false); | |
| 122 } else if (frame_dropper_enabled_) { | |
| 123 _mediaOpt.EnableFrameDropper(true); | |
| 124 } | |
| 125 _nextFrameTypes.clear(); | |
| 126 _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), | |
| 127 kVideoFrameDelta); | |
| 128 | |
| 129 _mediaOpt.SetEncodingData(sendCodec->codecType, | |
| 130 sendCodec->maxBitrate * 1000, | |
| 131 sendCodec->startBitrate * 1000, | |
| 132 sendCodec->width, | |
| 133 sendCodec->height, | |
| 134 sendCodec->maxFramerate, | |
| 135 numLayers, | |
| 136 maxPayloadSize); | |
| 137 return VCM_OK; | |
| 138 } | |
| 139 | |
| 140 const VideoCodec& VideoSender::GetSendCodec() const { | |
| 141 RTC_DCHECK(main_thread_.CalledOnValidThread()); | |
| 142 return current_codec_; | |
| 143 } | |
| 144 | |
| 145 int32_t VideoSender::SendCodecBlocking(VideoCodec* currentSendCodec) const { | |
| 146 rtc::CritScope lock(&send_crit_); | |
| 147 if (currentSendCodec == nullptr) { | |
| 148 return VCM_PARAMETER_ERROR; | |
| 149 } | |
| 150 return _codecDataBase.SendCodec(currentSendCodec) ? 0 : -1; | |
| 151 } | |
| 152 | |
| 153 VideoCodecType VideoSender::SendCodecBlocking() const { | |
| 154 rtc::CritScope lock(&send_crit_); | |
| 155 return _codecDataBase.SendCodec(); | |
| 156 } | |
| 157 | |
| 158 // Register an external decoder object. | |
| 159 // This can not be used together with external decoder callbacks. | |
| 160 int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder, | |
| 161 uint8_t payloadType, | |
| 162 bool internalSource /*= false*/) { | |
| 163 RTC_DCHECK(main_thread_.CalledOnValidThread()); | |
| 164 | |
| 165 rtc::CritScope lock(&send_crit_); | |
| 166 | |
| 167 if (externalEncoder == nullptr) { | |
| 168 bool wasSendCodec = false; | |
| 169 const bool ret = | |
| 170 _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec); | |
| 171 if (wasSendCodec) { | |
| 172 // Make sure the VCM doesn't use the de-registered codec | |
| 173 _encoder = nullptr; | |
| 174 } | |
| 175 return ret ? 0 : -1; | |
| 176 } | |
| 177 _codecDataBase.RegisterExternalEncoder( | |
| 178 externalEncoder, payloadType, internalSource); | |
| 179 return 0; | |
| 180 } | |
| 181 | |
| 182 // Get encode bitrate | |
| 183 int VideoSender::Bitrate(unsigned int* bitrate) const { | |
| 184 RTC_DCHECK(main_thread_.CalledOnValidThread()); | |
| 185 // Since we're running on the thread that's the only thread known to modify | |
| 186 // the value of _encoder, we don't need to grab the lock here. | |
| 187 | |
| 188 if (!_encoder) | |
| 189 return VCM_UNINITIALIZED; | |
| 190 *bitrate = _encoder->GetEncoderParameters().target_bitrate; | |
| 191 return 0; | |
| 192 } | |
| 193 | |
| 194 // Get encode frame rate | |
| 195 int VideoSender::FrameRate(unsigned int* framerate) const { | |
| 196 RTC_DCHECK(main_thread_.CalledOnValidThread()); | |
| 197 // Since we're running on the thread that's the only thread known to modify | |
| 198 // the value of _encoder, we don't need to grab the lock here. | |
| 199 | |
| 200 if (!_encoder) | |
| 201 return VCM_UNINITIALIZED; | |
| 202 | |
| 203 *framerate = _encoder->GetEncoderParameters().input_frame_rate; | |
| 204 return 0; | |
| 205 } | |
| 206 | |
| 207 int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, | |
| 208 uint8_t lossRate, | |
| 209 int64_t rtt) { | |
| 210 uint32_t target_rate = | |
| 211 _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt, | |
| 212 protection_callback_, qm_settings_callback_); | |
| 213 | |
| 214 uint32_t input_frame_rate = _mediaOpt.InputFrameRate(); | |
| 215 | |
| 216 rtc::CritScope cs(¶ms_lock_); | |
| 217 encoder_params_ = {target_rate, lossRate, rtt, input_frame_rate}; | |
| 218 | |
| 219 return VCM_OK; | |
| 220 } | |
| 221 | |
| 222 void VideoSender::SetEncoderParameters(EncoderParameters params) { | |
| 223 if (params.target_bitrate == 0) | |
| 224 return; | |
| 225 | |
| 226 if (params.input_frame_rate == 0) { | |
| 227 // No frame rate estimate available, use default. | |
| 228 params.input_frame_rate = current_codec_.maxFramerate; | |
| 229 } | |
| 230 if (_encoder != nullptr) | |
| 231 _encoder->SetEncoderParameters(params); | |
| 232 } | |
| 233 | |
| 234 int32_t VideoSender::RegisterTransportCallback( | |
| 235 VCMPacketizationCallback* transport) { | |
| 236 rtc::CritScope lock(&send_crit_); | |
| 237 _encodedFrameCallback.SetMediaOpt(&_mediaOpt); | |
| 238 _encodedFrameCallback.SetTransportCallback(transport); | |
| 239 return VCM_OK; | |
| 240 } | |
| 241 | |
| 242 // Register video output information callback which will be called to deliver | |
| 243 // information about the video stream produced by the encoder, for instance the | |
| 244 // average frame rate and bit rate. | |
| 245 int32_t VideoSender::RegisterSendStatisticsCallback( | |
| 246 VCMSendStatisticsCallback* sendStats) { | |
| 247 CriticalSectionScoped cs(process_crit_sect_.get()); | |
| 248 _sendStatsCallback = sendStats; | |
| 249 return VCM_OK; | |
| 250 } | |
| 251 | |
| 252 // Register a video protection callback which will be called to deliver the | |
| 253 // requested FEC rate and NACK status (on/off). | |
| 254 // Note: this callback is assumed to only be registered once and before it is | |
| 255 // used in this class. | |
| 256 int32_t VideoSender::RegisterProtectionCallback( | |
| 257 VCMProtectionCallback* protection_callback) { | |
| 258 RTC_DCHECK(protection_callback == nullptr || protection_callback_ == nullptr); | |
| 259 protection_callback_ = protection_callback; | |
| 260 return VCM_OK; | |
| 261 } | |
| 262 | |
| 263 // Enable or disable a video protection method. | |
| 264 void VideoSender::SetVideoProtection(VCMVideoProtection videoProtection) { | |
| 265 rtc::CritScope lock(&send_crit_); | |
| 266 switch (videoProtection) { | |
| 267 case kProtectionNone: | |
| 268 _mediaOpt.SetProtectionMethod(media_optimization::kNone); | |
| 269 break; | |
| 270 case kProtectionNack: | |
| 271 _mediaOpt.SetProtectionMethod(media_optimization::kNack); | |
| 272 break; | |
| 273 case kProtectionNackFEC: | |
| 274 _mediaOpt.SetProtectionMethod(media_optimization::kNackFec); | |
| 275 break; | |
| 276 case kProtectionFEC: | |
| 277 _mediaOpt.SetProtectionMethod(media_optimization::kFec); | |
| 278 break; | |
| 279 } | |
| 280 } | |
| 281 // Add one raw video frame to the encoder, blocking. | |
| 282 int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, | |
| 283 const VideoContentMetrics* contentMetrics, | |
| 284 const CodecSpecificInfo* codecSpecificInfo) { | |
| 285 EncoderParameters encoder_params; | |
| 286 { | |
| 287 rtc::CritScope lock(¶ms_lock_); | |
| 288 encoder_params = encoder_params_; | |
| 289 } | |
| 290 rtc::CritScope lock(&send_crit_); | |
| 291 if (_encoder == nullptr) | |
| 292 return VCM_UNINITIALIZED; | |
| 293 SetEncoderParameters(encoder_params); | |
| 294 // TODO(holmer): Add support for dropping frames per stream. Currently we | |
| 295 // only have one frame dropper for all streams. | |
| 296 if (_nextFrameTypes[0] == kEmptyFrame) { | |
| 297 return VCM_OK; | |
| 298 } | |
| 299 if (_mediaOpt.DropFrame()) { | |
| 300 _encoder->OnDroppedFrame(); | |
| 301 return VCM_OK; | |
| 302 } | |
| 303 _mediaOpt.UpdateContentData(contentMetrics); | |
| 304 // TODO(pbos): Make sure setting send codec is synchronized with video | |
| 305 // processing so frame size always matches. | |
| 306 if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(), | |
| 307 videoFrame.height())) { | |
| 308 LOG(LS_ERROR) << "Incoming frame doesn't match set resolution. Dropping."; | |
| 309 return VCM_PARAMETER_ERROR; | |
| 310 } | |
| 311 VideoFrame converted_frame = videoFrame; | |
| 312 if (converted_frame.native_handle() && !_encoder->SupportsNativeHandle()) { | |
| 313 // This module only supports software encoding. | |
| 314 // TODO(pbos): Offload conversion from the encoder thread. | |
| 315 converted_frame = converted_frame.ConvertNativeToI420Frame(); | |
| 316 RTC_CHECK(!converted_frame.IsZeroSize()) | |
| 317 << "Frame conversion failed, won't be able to encode frame."; | |
| 318 } | |
| 319 int32_t ret = | |
| 320 _encoder->Encode(converted_frame, codecSpecificInfo, _nextFrameTypes); | |
| 321 if (ret < 0) { | |
| 322 LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret; | |
| 323 return ret; | |
| 324 } | |
| 325 for (size_t i = 0; i < _nextFrameTypes.size(); ++i) { | |
| 326 _nextFrameTypes[i] = kVideoFrameDelta; // Default frame type. | |
| 327 } | |
| 328 if (qm_settings_callback_) | |
| 329 qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate()); | |
| 330 return VCM_OK; | |
| 331 } | |
| 332 | |
| 333 int32_t VideoSender::IntraFrameRequest(int stream_index) { | |
| 334 rtc::CritScope lock(&send_crit_); | |
| 335 if (stream_index < 0 || | |
| 336 static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) { | |
| 337 return -1; | |
| 338 } | |
| 339 _nextFrameTypes[stream_index] = kVideoFrameKey; | |
| 340 if (_encoder != nullptr && _encoder->InternalSource()) { | |
| 341 // Try to request the frame if we have an external encoder with | |
| 342 // internal source since AddVideoFrame never will be called. | |
| 343 if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) { | |
| 344 _nextFrameTypes[stream_index] = kVideoFrameDelta; | |
| 345 } | |
| 346 } | |
| 347 return VCM_OK; | |
| 348 } | |
| 349 | |
| 350 int32_t VideoSender::EnableFrameDropper(bool enable) { | |
| 351 rtc::CritScope lock(&send_crit_); | |
| 352 frame_dropper_enabled_ = enable; | |
| 353 _mediaOpt.EnableFrameDropper(enable); | |
| 354 return VCM_OK; | |
| 355 } | |
| 356 | |
| 357 void VideoSender::SuspendBelowMinBitrate() { | |
| 358 RTC_DCHECK(main_thread_.CalledOnValidThread()); | |
| 359 int threshold_bps; | |
| 360 if (current_codec_.numberOfSimulcastStreams == 0) { | |
| 361 threshold_bps = current_codec_.minBitrate * 1000; | |
| 362 } else { | |
| 363 threshold_bps = current_codec_.simulcastStream[0].minBitrate * 1000; | |
| 364 } | |
| 365 // Set the hysteresis window to be at 10% of the threshold, but at least | |
| 366 // 10 kbps. | |
| 367 int window_bps = std::max(threshold_bps / 10, 10000); | |
| 368 _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps); | |
| 369 } | |
| 370 | |
| 371 bool VideoSender::VideoSuspended() const { | |
| 372 return _mediaOpt.IsVideoSuspended(); | |
| 373 } | |
| 374 } // namespace vcm | |
| 375 } // namespace webrtc | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1417283007:
modules/video_coding refactorings (Closed)
Base URL: https://chromium.googlesource.com/external/webrtc.git@master