| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
| 12 | 12 |
| 13 #include <algorithm> | 13 #include <algorithm> |
| 14 #include <limits> | 14 #include <limits> |
| 15 | 15 |
| 16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
| 17 #include "webrtc/base/logging.h" | 17 #include "webrtc/base/logging.h" |
| 18 #include "webrtc/base/trace_event.h" | 18 #include "webrtc/base/trace_event.h" |
| 19 #include "webrtc/base/timeutils.h" | 19 #include "webrtc/base/timeutils.h" |
| 20 #include "webrtc/modules/pacing/paced_sender.h" | 20 #include "webrtc/modules/pacing/paced_sender.h" |
| 21 #include "webrtc/modules/video_coding/include/video_coding.h" | 21 #include "webrtc/modules/video_coding/include/video_coding.h" |
| 22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| 23 #include "webrtc/system_wrappers/include/metrics.h" | 23 #include "webrtc/system_wrappers/include/metrics.h" |
| 24 #include "webrtc/video/overuse_frame_detector.h" | 24 #include "webrtc/video/overuse_frame_detector.h" |
| 25 #include "webrtc/video/send_statistics_proxy.h" | 25 #include "webrtc/video/send_statistics_proxy.h" |
| 26 #include "webrtc/video_frame.h" | 26 #include "webrtc/video_frame.h" |
| 27 | 27 |
| 28 namespace webrtc { | 28 namespace webrtc { |
| 29 | 29 |
| 30 namespace { | |
| 31 | |
| 32 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { | |
| 33 if (payload_name == "VP8") | |
| 34 return kVideoCodecVP8; | |
| 35 if (payload_name == "VP9") | |
| 36 return kVideoCodecVP9; | |
| 37 if (payload_name == "H264") | |
| 38 return kVideoCodecH264; | |
| 39 return kVideoCodecGeneric; | |
| 40 } | |
| 41 | |
| 42 VideoCodec VideoEncoderConfigToVideoCodec(const VideoEncoderConfig& config, | |
| 43 const std::string& payload_name, | |
| 44 int payload_type) { | |
| 45 const std::vector<VideoStream>& streams = config.streams; | |
| 46 static const int kEncoderMinBitrateKbps = 30; | |
| 47 RTC_DCHECK(!streams.empty()); | |
| 48 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); | |
| 49 | |
| 50 VideoCodec video_codec; | |
| 51 memset(&video_codec, 0, sizeof(video_codec)); | |
| 52 video_codec.codecType = PayloadNameToCodecType(payload_name); | |
| 53 | |
| 54 switch (config.content_type) { | |
| 55 case VideoEncoderConfig::ContentType::kRealtimeVideo: | |
| 56 video_codec.mode = kRealtimeVideo; | |
| 57 break; | |
| 58 case VideoEncoderConfig::ContentType::kScreen: | |
| 59 video_codec.mode = kScreensharing; | |
| 60 if (config.streams.size() == 1 && | |
| 61 config.streams[0].temporal_layer_thresholds_bps.size() == 1) { | |
| 62 video_codec.targetBitrate = | |
| 63 config.streams[0].temporal_layer_thresholds_bps[0] / 1000; | |
| 64 } | |
| 65 break; | |
| 66 } | |
| 67 | |
| 68 switch (video_codec.codecType) { | |
| 69 case kVideoCodecVP8: { | |
| 70 if (config.encoder_specific_settings) { | |
| 71 video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>( | |
| 72 config.encoder_specific_settings); | |
| 73 } else { | |
| 74 video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); | |
| 75 } | |
| 76 video_codec.codecSpecific.VP8.numberOfTemporalLayers = | |
| 77 static_cast<unsigned char>( | |
| 78 streams.back().temporal_layer_thresholds_bps.size() + 1); | |
| 79 break; | |
| 80 } | |
| 81 case kVideoCodecVP9: { | |
| 82 if (config.encoder_specific_settings) { | |
| 83 video_codec.codecSpecific.VP9 = *reinterpret_cast<const VideoCodecVP9*>( | |
| 84 config.encoder_specific_settings); | |
| 85 if (video_codec.mode == kScreensharing) { | |
| 86 video_codec.codecSpecific.VP9.flexibleMode = true; | |
| 87 // For now VP9 screensharing use 1 temporal and 2 spatial layers. | |
| 88 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, | |
| 89 1); | |
| 90 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2); | |
| 91 } | |
| 92 } else { | |
| 93 video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); | |
| 94 } | |
| 95 video_codec.codecSpecific.VP9.numberOfTemporalLayers = | |
| 96 static_cast<unsigned char>( | |
| 97 streams.back().temporal_layer_thresholds_bps.size() + 1); | |
| 98 break; | |
| 99 } | |
| 100 case kVideoCodecH264: { | |
| 101 if (config.encoder_specific_settings) { | |
| 102 video_codec.codecSpecific.H264 = | |
| 103 *reinterpret_cast<const VideoCodecH264*>( | |
| 104 config.encoder_specific_settings); | |
| 105 } else { | |
| 106 video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); | |
| 107 } | |
| 108 break; | |
| 109 } | |
| 110 default: | |
| 111 // TODO(pbos): Support encoder_settings codec-agnostically. | |
| 112 RTC_DCHECK(!config.encoder_specific_settings) | |
| 113 << "Encoder-specific settings for codec type not wired up."; | |
| 114 break; | |
| 115 } | |
| 116 | |
| 117 strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); | |
| 118 video_codec.plName[kPayloadNameSize - 1] = '\0'; | |
| 119 video_codec.plType = payload_type; | |
| 120 video_codec.numberOfSimulcastStreams = | |
| 121 static_cast<unsigned char>(streams.size()); | |
| 122 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; | |
| 123 if (video_codec.minBitrate < kEncoderMinBitrateKbps) | |
| 124 video_codec.minBitrate = kEncoderMinBitrateKbps; | |
| 125 RTC_DCHECK_LE(streams.size(), static_cast<size_t>(kMaxSimulcastStreams)); | |
| 126 if (video_codec.codecType == kVideoCodecVP9) { | |
| 127 // If the vector is empty, bitrates will be configured automatically. | |
| 128 RTC_DCHECK(config.spatial_layers.empty() || | |
| 129 config.spatial_layers.size() == | |
| 130 video_codec.codecSpecific.VP9.numberOfSpatialLayers); | |
| 131 RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, | |
| 132 kMaxSimulcastStreams); | |
| 133 for (size_t i = 0; i < config.spatial_layers.size(); ++i) | |
| 134 video_codec.spatialLayers[i] = config.spatial_layers[i]; | |
| 135 } | |
| 136 for (size_t i = 0; i < streams.size(); ++i) { | |
| 137 SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; | |
| 138 RTC_DCHECK_GT(streams[i].width, 0u); | |
| 139 RTC_DCHECK_GT(streams[i].height, 0u); | |
| 140 RTC_DCHECK_GT(streams[i].max_framerate, 0); | |
| 141 // Different framerates not supported per stream at the moment. | |
| 142 RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); | |
| 143 RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); | |
| 144 RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); | |
| 145 RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); | |
| 146 RTC_DCHECK_GE(streams[i].max_qp, 0); | |
| 147 | |
| 148 sim_stream->width = static_cast<uint16_t>(streams[i].width); | |
| 149 sim_stream->height = static_cast<uint16_t>(streams[i].height); | |
| 150 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; | |
| 151 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; | |
| 152 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; | |
| 153 sim_stream->qpMax = streams[i].max_qp; | |
| 154 sim_stream->numberOfTemporalLayers = static_cast<unsigned char>( | |
| 155 streams[i].temporal_layer_thresholds_bps.size() + 1); | |
| 156 | |
| 157 video_codec.width = | |
| 158 std::max(video_codec.width, static_cast<uint16_t>(streams[i].width)); | |
| 159 video_codec.height = | |
| 160 std::max(video_codec.height, static_cast<uint16_t>(streams[i].height)); | |
| 161 video_codec.minBitrate = | |
| 162 std::min(static_cast<uint16_t>(video_codec.minBitrate), | |
| 163 static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000)); | |
| 164 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; | |
| 165 video_codec.qpMax = std::max(video_codec.qpMax, | |
| 166 static_cast<unsigned int>(streams[i].max_qp)); | |
| 167 } | |
| 168 | |
| 169 if (video_codec.maxBitrate == 0) { | |
| 170 // Unset max bitrate -> cap to one bit per pixel. | |
| 171 video_codec.maxBitrate = | |
| 172 (video_codec.width * video_codec.height * video_codec.maxFramerate) / | |
| 173 1000; | |
| 174 } | |
| 175 if (video_codec.maxBitrate < kEncoderMinBitrateKbps) | |
| 176 video_codec.maxBitrate = kEncoderMinBitrateKbps; | |
| 177 | |
| 178 RTC_DCHECK_GT(streams[0].max_framerate, 0); | |
| 179 video_codec.maxFramerate = streams[0].max_framerate; | |
| 180 video_codec.expect_encode_from_texture = config.expect_encode_from_texture; | |
| 181 | |
| 182 return video_codec; | |
| 183 } | |
| 184 | |
| 185 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle | |
| 186 // pipelining encoders better (multiple input frames before something comes | |
| 187 // out). This should effectively turn off CPU adaptations for systems that | |
| 188 // remotely cope with the load right now. | |
| 189 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { | |
| 190 CpuOveruseOptions options; | |
| 191 if (full_overuse_time) { | |
| 192 options.low_encode_usage_threshold_percent = 150; | |
| 193 options.high_encode_usage_threshold_percent = 200; | |
| 194 } | |
| 195 return options; | |
| 196 } | |
| 197 | |
| 198 } // namespace | |
| 199 | |
| 200 class ViEEncoder::EncodeTask : public rtc::QueuedTask { | |
| 201 public: | |
| 202 EncodeTask(const VideoFrame& frame, ViEEncoder* vie_encoder) | |
| 203 : vie_encoder_(vie_encoder) { | |
| 204 frame_.ShallowCopy(frame); | |
| 205 ++vie_encoder_->posted_frames_waiting_for_encode_; | |
| 206 } | |
| 207 | |
| 208 private: | |
| 209 bool Run() override { | |
| 210 RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0); | |
| 211 if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) { | |
| 212 vie_encoder_->EncodeVideoFrame(frame_); | |
| 213 } else { | |
| 214 // There is a newer frame in flight. Do not encode this frame. | |
| 215 LOG(LS_VERBOSE) | |
| 216 << "Incoming frame dropped due to that the encoder is blocked."; | |
| 217 } | |
| 218 return true; | |
| 219 } | |
| 220 VideoFrame frame_; | |
| 221 ViEEncoder* vie_encoder_; | |
| 222 }; | |
| 223 | |
| 224 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | 30 ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
| 31 ProcessThread* module_process_thread, |
| 225 SendStatisticsProxy* stats_proxy, | 32 SendStatisticsProxy* stats_proxy, |
| 226 const VideoSendStream::Config::EncoderSettings& settings, | 33 OveruseFrameDetector* overuse_detector, |
| 227 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | 34 EncodedImageCallback* sink) |
| 228 LoadObserver* overuse_callback, | 35 : number_of_cores_(number_of_cores), |
| 229 EncodedFrameObserver* encoder_timing) | 36 sink_(sink), |
| 230 : shutdown_event_(true /* manual_reset */, false), | |
| 231 number_of_cores_(number_of_cores), | |
| 232 settings_(settings), | |
| 233 vp_(VideoProcessing::Create()), | 37 vp_(VideoProcessing::Create()), |
| 234 video_sender_(Clock::GetRealTimeClock(), this, this), | 38 video_sender_(Clock::GetRealTimeClock(), this, this), |
| 235 overuse_detector_(Clock::GetRealTimeClock(), | |
| 236 GetCpuOveruseOptions(settings.full_overuse_time), | |
| 237 this, | |
| 238 encoder_timing, | |
| 239 stats_proxy), | |
| 240 load_observer_(overuse_callback), | |
| 241 stats_proxy_(stats_proxy), | 39 stats_proxy_(stats_proxy), |
| 242 pre_encode_callback_(pre_encode_callback), | 40 overuse_detector_(overuse_detector), |
| 243 module_process_thread_(nullptr), | 41 time_of_last_frame_activity_ms_(std::numeric_limits<int64_t>::max()), |
| 244 encoder_config_(), | 42 encoder_config_(), |
| 245 encoder_start_bitrate_bps_(0), | |
| 246 last_observed_bitrate_bps_(0), | 43 last_observed_bitrate_bps_(0), |
| 247 encoder_paused_and_dropped_frame_(false), | 44 encoder_paused_and_dropped_frame_(false), |
| 45 module_process_thread_(module_process_thread), |
| 248 has_received_sli_(false), | 46 has_received_sli_(false), |
| 249 picture_id_sli_(0), | 47 picture_id_sli_(0), |
| 250 has_received_rpsi_(false), | 48 has_received_rpsi_(false), |
| 251 picture_id_rpsi_(0), | 49 picture_id_rpsi_(0), |
| 252 clock_(Clock::GetRealTimeClock()), | 50 video_suspended_(false) { |
| 253 last_captured_timestamp_(0), | 51 module_process_thread_->RegisterModule(&video_sender_); |
| 254 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 52 vp_->EnableTemporalDecimation(true); |
| 255 clock_->TimeInMilliseconds()), | 53 } |
| 256 encoder_queue_("EncoderQueue") { | |
| 257 vp_->EnableTemporalDecimation(false); | |
| 258 | 54 |
| 259 encoder_queue_.PostTask([this] { | 55 vcm::VideoSender* ViEEncoder::video_sender() { |
| 260 RTC_DCHECK_RUN_ON(&encoder_queue_); | 56 return &video_sender_; |
| 261 video_sender_.RegisterExternalEncoder( | |
| 262 settings_.encoder, settings_.payload_type, settings_.internal_source); | |
| 263 }); | |
| 264 } | 57 } |
| 265 | 58 |
| 266 ViEEncoder::~ViEEncoder() { | 59 ViEEncoder::~ViEEncoder() { |
| 267 RTC_DCHECK(shutdown_event_.Wait(0)) | |
| 268 << "Must call ::Stop() before destruction."; | |
| 269 } | |
| 270 | |
| 271 void ViEEncoder::Stop() { | |
| 272 if (!encoder_queue_.IsCurrent()) { | |
| 273 encoder_queue_.PostTask([this] { Stop(); }); | |
| 274 shutdown_event_.Wait(rtc::Event::kForever); | |
| 275 return; | |
| 276 } | |
| 277 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 278 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); | |
| 279 shutdown_event_.Set(); | |
| 280 } | |
| 281 | |
| 282 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | |
| 283 RTC_DCHECK(!module_process_thread_); | |
| 284 module_process_thread_ = module_process_thread; | |
| 285 module_process_thread_->RegisterModule(&overuse_detector_); | |
| 286 module_process_thread_->RegisterModule(&video_sender_); | |
| 287 module_process_thread_checker_.DetachFromThread(); | |
| 288 } | |
| 289 | |
| 290 void ViEEncoder::DeRegisterProcessThread() { | |
| 291 module_process_thread_->DeRegisterModule(&overuse_detector_); | |
| 292 module_process_thread_->DeRegisterModule(&video_sender_); | 60 module_process_thread_->DeRegisterModule(&video_sender_); |
| 293 } | 61 } |
| 294 | 62 |
| 295 void ViEEncoder::SetSink(EncodedImageCallback* sink) { | 63 int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, |
| 296 encoder_queue_.PostTask([this, sink] { | 64 uint8_t pl_type, |
| 297 RTC_DCHECK_RUN_ON(&encoder_queue_); | 65 bool internal_source) { |
| 298 sink_ = sink; | 66 video_sender_.RegisterExternalEncoder(encoder, pl_type, internal_source); |
| 299 }); | 67 return 0; |
| 300 } | 68 } |
| 301 | 69 |
| 302 void ViEEncoder::SetStartBitrate(int start_bitrate_bps) { | 70 int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { |
| 303 encoder_queue_.PostTask([this, start_bitrate_bps] { | 71 video_sender_.RegisterExternalEncoder(nullptr, pl_type, false); |
| 304 RTC_DCHECK_RUN_ON(&encoder_queue_); | 72 return 0; |
| 305 encoder_start_bitrate_bps_ = start_bitrate_bps; | |
| 306 }); | |
| 307 } | 73 } |
| 308 | 74 |
| 309 void ViEEncoder::ConfigureEncoder(const VideoEncoderConfig& config, | 75 void ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec, |
| 310 size_t max_data_payload_length) { | 76 size_t max_data_payload_length) { |
| 311 VideoCodec video_codec = VideoEncoderConfigToVideoCodec( | |
| 312 config, settings_.payload_name, settings_.payload_type); | |
| 313 encoder_queue_.PostTask([this, video_codec, max_data_payload_length] { | |
| 314 ConfigureEncoderInternal(video_codec, max_data_payload_length); | |
| 315 }); | |
| 316 return; | |
| 317 } | |
| 318 | |
| 319 void ViEEncoder::ConfigureEncoderInternal(const VideoCodec& video_codec, | |
| 320 size_t max_data_payload_length) { | |
| 321 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 322 RTC_DCHECK_GE(encoder_start_bitrate_bps_, 0); | |
| 323 RTC_DCHECK(sink_); | |
| 324 | |
| 325 // Setting target width and height for VPM. | 77 // Setting target width and height for VPM. |
| 326 RTC_CHECK_EQ(VPM_OK, | 78 RTC_CHECK_EQ(VPM_OK, |
| 327 vp_->SetTargetResolution(video_codec.width, video_codec.height, | 79 vp_->SetTargetResolution(video_codec.width, video_codec.height, |
| 328 video_codec.maxFramerate)); | 80 video_codec.maxFramerate)); |
| 329 | 81 { |
| 330 encoder_config_ = video_codec; | 82 rtc::CritScope lock(&data_cs_); |
| 331 encoder_config_.startBitrate = encoder_start_bitrate_bps_ / 1000; | 83 encoder_config_ = video_codec; |
| 332 encoder_config_.startBitrate = | 84 } |
| 333 std::max(encoder_config_.startBitrate, video_codec.minBitrate); | |
| 334 encoder_config_.startBitrate = | |
| 335 std::min(encoder_config_.startBitrate, video_codec.maxBitrate); | |
| 336 | 85 |
| 337 bool success = video_sender_.RegisterSendCodec( | 86 bool success = video_sender_.RegisterSendCodec( |
| 338 &encoder_config_, number_of_cores_, | 87 &video_codec, number_of_cores_, |
| 339 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; | 88 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
| 340 | 89 |
| 341 if (!success) { | 90 if (!success) { |
| 342 LOG(LS_ERROR) << "Failed to configure encoder."; | 91 LOG(LS_ERROR) << "Failed to configure encoder."; |
| 343 RTC_DCHECK(success); | 92 RTC_DCHECK(success); |
| 344 } | 93 } |
| 345 | 94 |
| 346 if (stats_proxy_) { | 95 if (stats_proxy_) { |
| 347 VideoEncoderConfig::ContentType content_type = | 96 VideoEncoderConfig::ContentType content_type = |
| 348 VideoEncoderConfig::ContentType::kRealtimeVideo; | 97 VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 349 switch (video_codec.mode) { | 98 switch (video_codec.mode) { |
| 350 case kRealtimeVideo: | 99 case kRealtimeVideo: |
| 351 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; | 100 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 352 break; | 101 break; |
| 353 case kScreensharing: | 102 case kScreensharing: |
| 354 content_type = VideoEncoderConfig::ContentType::kScreen; | 103 content_type = VideoEncoderConfig::ContentType::kScreen; |
| 355 break; | 104 break; |
| 356 default: | 105 default: |
| 357 RTC_NOTREACHED(); | 106 RTC_NOTREACHED(); |
| 358 break; | 107 break; |
| 359 } | 108 } |
| 360 stats_proxy_->SetContentType(content_type); | 109 stats_proxy_->SetContentType(content_type); |
| 361 } | 110 } |
| 362 } | 111 } |
| 363 | 112 |
| 364 void ViEEncoder::IncomingCapturedFrame(const VideoFrame& video_frame) { | |
| 365 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | |
| 366 stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height()); | |
| 367 | |
| 368 VideoFrame incoming_frame = video_frame; | |
| 369 | |
| 370 // Local time in webrtc time base. | |
| 371 int64_t current_time = clock_->TimeInMilliseconds(); | |
| 372 incoming_frame.set_render_time_ms(current_time); | |
| 373 | |
| 374 // Capture time may come from clock with an offset and drift from clock_. | |
| 375 int64_t capture_ntp_time_ms; | |
| 376 if (video_frame.ntp_time_ms() != 0) { | |
| 377 capture_ntp_time_ms = video_frame.ntp_time_ms(); | |
| 378 } else if (video_frame.render_time_ms() != 0) { | |
| 379 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; | |
| 380 } else { | |
| 381 capture_ntp_time_ms = current_time + delta_ntp_internal_ms_; | |
| 382 } | |
| 383 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); | |
| 384 | |
| 385 // Convert NTP time, in ms, to RTP timestamp. | |
| 386 const int kMsToRtpTimestamp = 90; | |
| 387 incoming_frame.set_timestamp( | |
| 388 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms())); | |
| 389 | |
| 390 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { | |
| 391 // We don't allow the same capture time for two frames, drop this one. | |
| 392 LOG(LS_WARNING) << "Same/old NTP timestamp (" | |
| 393 << incoming_frame.ntp_time_ms() | |
| 394 << " <= " << last_captured_timestamp_ | |
| 395 << ") for incoming frame. Dropping."; | |
| 396 return; | |
| 397 } | |
| 398 | |
| 399 last_captured_timestamp_ = incoming_frame.ntp_time_ms(); | |
| 400 overuse_detector_.FrameCaptured(incoming_frame); | |
| 401 encoder_queue_.PostTask( | |
| 402 std::unique_ptr<rtc::QueuedTask>(new EncodeTask(incoming_frame, this))); | |
| 403 } | |
| 404 | |
| 405 bool ViEEncoder::EncoderPaused() const { | 113 bool ViEEncoder::EncoderPaused() const { |
| 406 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 407 // Pause video if paused by caller or as long as the network is down or the | 114 // Pause video if paused by caller or as long as the network is down or the |
| 408 // pacer queue has grown too large in buffered mode. | 115 // pacer queue has grown too large in buffered mode. |
| 409 // If the pacer queue has grown too large or the network is down, | 116 // If the pacer queue has grown too large or the network is down, |
| 410 // last_observed_bitrate_bps_ will be 0. | 117 // last_observed_bitrate_bps_ will be 0. |
| 411 return last_observed_bitrate_bps_ == 0; | 118 return video_suspended_ || last_observed_bitrate_bps_ == 0; |
| 412 } | 119 } |
| 413 | 120 |
| 414 void ViEEncoder::TraceFrameDropStart() { | 121 void ViEEncoder::TraceFrameDropStart() { |
| 415 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 416 // Start trace event only on the first frame after encoder is paused. | 122 // Start trace event only on the first frame after encoder is paused. |
| 417 if (!encoder_paused_and_dropped_frame_) { | 123 if (!encoder_paused_and_dropped_frame_) { |
| 418 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); | 124 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); |
| 419 } | 125 } |
| 420 encoder_paused_and_dropped_frame_ = true; | 126 encoder_paused_and_dropped_frame_ = true; |
| 421 return; | 127 return; |
| 422 } | 128 } |
| 423 | 129 |
| 424 void ViEEncoder::TraceFrameDropEnd() { | 130 void ViEEncoder::TraceFrameDropEnd() { |
| 425 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 426 // End trace event on first frame after encoder resumes, if frame was dropped. | 131 // End trace event on first frame after encoder resumes, if frame was dropped. |
| 427 if (encoder_paused_and_dropped_frame_) { | 132 if (encoder_paused_and_dropped_frame_) { |
| 428 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | 133 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
| 429 } | 134 } |
| 430 encoder_paused_and_dropped_frame_ = false; | 135 encoder_paused_and_dropped_frame_ = false; |
| 431 } | 136 } |
| 432 | 137 |
| 433 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { | 138 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { |
| 434 RTC_DCHECK_RUN_ON(&encoder_queue_); | 139 VideoCodecType codec_type; |
| 435 if (pre_encode_callback_) | 140 { |
| 436 pre_encode_callback_->OnFrame(video_frame); | 141 rtc::CritScope lock(&data_cs_); |
| 437 | 142 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| 438 if (EncoderPaused()) { | 143 if (EncoderPaused()) { |
| 439 TraceFrameDropStart(); | 144 TraceFrameDropStart(); |
| 440 return; | 145 return; |
| 146 } |
| 147 TraceFrameDropEnd(); |
| 148 codec_type = encoder_config_.codecType; |
| 441 } | 149 } |
| 442 TraceFrameDropEnd(); | |
| 443 | 150 |
| 444 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), | 151 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
| 445 "Encode"); | 152 "Encode"); |
| 446 const VideoFrame* frame_to_send = &video_frame; | 153 const VideoFrame* frame_to_send = &video_frame; |
| 447 // TODO(wuchengli): support texture frames. | 154 // TODO(wuchengli): support texture frames. |
| 448 if (!video_frame.video_frame_buffer()->native_handle()) { | 155 if (!video_frame.video_frame_buffer()->native_handle()) { |
| 449 // Pass frame via preprocessor. | 156 // Pass frame via preprocessor. |
| 450 frame_to_send = vp_->PreprocessFrame(video_frame); | 157 frame_to_send = vp_->PreprocessFrame(video_frame); |
| 451 if (!frame_to_send) { | 158 if (!frame_to_send) { |
| 452 // Drop this frame, or there was an error processing it. | 159 // Drop this frame, or there was an error processing it. |
| 453 return; | 160 return; |
| 454 } | 161 } |
| 455 } | 162 } |
| 456 | 163 |
| 457 if (encoder_config_.codecType == webrtc::kVideoCodecVP8) { | 164 if (codec_type == webrtc::kVideoCodecVP8) { |
| 458 webrtc::CodecSpecificInfo codec_specific_info; | 165 webrtc::CodecSpecificInfo codec_specific_info; |
| 459 codec_specific_info.codecType = webrtc::kVideoCodecVP8; | 166 codec_specific_info.codecType = webrtc::kVideoCodecVP8; |
| 460 | 167 { |
| 168 rtc::CritScope lock(&data_cs_); |
| 461 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = | 169 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = |
| 462 has_received_rpsi_; | 170 has_received_rpsi_; |
| 463 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = | 171 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = |
| 464 has_received_sli_; | 172 has_received_sli_; |
| 465 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = | 173 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = |
| 466 picture_id_rpsi_; | 174 picture_id_rpsi_; |
| 467 codec_specific_info.codecSpecific.VP8.pictureIdSLI = | 175 codec_specific_info.codecSpecific.VP8.pictureIdSLI = |
| 468 picture_id_sli_; | 176 picture_id_sli_; |
| 469 has_received_sli_ = false; | 177 has_received_sli_ = false; |
| 470 has_received_rpsi_ = false; | 178 has_received_rpsi_ = false; |
| 179 } |
| 471 | 180 |
| 472 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); | 181 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); |
| 473 return; | 182 return; |
| 474 } | 183 } |
| 475 video_sender_.AddVideoFrame(*frame_to_send, nullptr); | 184 video_sender_.AddVideoFrame(*frame_to_send, nullptr); |
| 476 } | 185 } |
| 477 | 186 |
| 478 void ViEEncoder::SendKeyFrame() { | 187 void ViEEncoder::SendKeyFrame() { |
| 479 if (!encoder_queue_.IsCurrent()) { | |
| 480 encoder_queue_.PostTask([this] { SendKeyFrame(); }); | |
| 481 return; | |
| 482 } | |
| 483 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 484 video_sender_.IntraFrameRequest(0); | 188 video_sender_.IntraFrameRequest(0); |
| 485 } | 189 } |
| 486 | 190 |
| 191 int64_t ViEEncoder::time_of_last_frame_activity_ms() { |
| 192 rtc::CritScope lock(&data_cs_); |
| 193 return time_of_last_frame_activity_ms_; |
| 194 } |
| 195 |
| 487 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( | 196 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( |
| 488 const EncodedImage& encoded_image, | 197 const EncodedImage& encoded_image, |
| 489 const CodecSpecificInfo* codec_specific_info, | 198 const CodecSpecificInfo* codec_specific_info, |
| 490 const RTPFragmentationHeader* fragmentation) { | 199 const RTPFragmentationHeader* fragmentation) { |
| 491 // Encoded is called on whatever thread the real encoder implementation run | 200 { |
| 492 // on. In the case of hardware encoders, there might be several encoders | 201 rtc::CritScope lock(&data_cs_); |
| 493 // running in parallel on different threads. | 202 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| 203 } |
| 494 if (stats_proxy_) { | 204 if (stats_proxy_) { |
| 495 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 205 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
| 496 } | 206 } |
| 497 | 207 |
| 498 EncodedImageCallback::Result result = | 208 EncodedImageCallback::Result result = |
| 499 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | 209 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); |
| 500 | 210 |
| 501 overuse_detector_.FrameSent(encoded_image._timeStamp); | 211 overuse_detector_->FrameSent(encoded_image._timeStamp); |
| 502 return result; | 212 return result; |
| 503 } | 213 } |
| 504 | 214 |
| 505 void ViEEncoder::SendStatistics(uint32_t bit_rate, | 215 void ViEEncoder::SendStatistics(uint32_t bit_rate, |
| 506 uint32_t frame_rate, | 216 uint32_t frame_rate, |
| 507 const std::string& encoder_name) { | 217 const std::string& encoder_name) { |
| 508 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | |
| 509 if (stats_proxy_) | 218 if (stats_proxy_) |
| 510 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); | 219 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); |
| 511 } | 220 } |
| 512 | 221 |
| 513 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { | 222 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
| 514 if (!encoder_queue_.IsCurrent()) { | 223 rtc::CritScope lock(&data_cs_); |
| 515 encoder_queue_.PostTask([this, picture_id] { OnReceivedSLI(picture_id); }); | |
| 516 return; | |
| 517 } | |
| 518 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 519 picture_id_sli_ = picture_id; | 224 picture_id_sli_ = picture_id; |
| 520 has_received_sli_ = true; | 225 has_received_sli_ = true; |
| 521 } | 226 } |
| 522 | 227 |
| 523 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { | 228 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { |
| 524 if (!encoder_queue_.IsCurrent()) { | 229 rtc::CritScope lock(&data_cs_); |
| 525 encoder_queue_.PostTask([this, picture_id] { OnReceivedRPSI(picture_id); }); | |
| 526 return; | |
| 527 } | |
| 528 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 529 picture_id_rpsi_ = picture_id; | 230 picture_id_rpsi_ = picture_id; |
| 530 has_received_rpsi_ = true; | 231 has_received_rpsi_ = true; |
| 531 } | 232 } |
| 532 | 233 |
| 533 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { | 234 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { |
| 534 if (!encoder_queue_.IsCurrent()) { | |
| 535 encoder_queue_.PostTask( | |
| 536 [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); }); | |
| 537 return; | |
| 538 } | |
| 539 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 540 // Key frame request from remote side, signal to VCM. | 235 // Key frame request from remote side, signal to VCM. |
| 541 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | 236 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
| 542 video_sender_.IntraFrameRequest(stream_index); | 237 video_sender_.IntraFrameRequest(stream_index); |
| 543 } | 238 } |
| 544 | 239 |
| 545 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | 240 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, |
| 546 uint8_t fraction_lost, | 241 uint8_t fraction_lost, |
| 547 int64_t round_trip_time_ms) { | 242 int64_t round_trip_time_ms) { |
| 548 if (!encoder_queue_.IsCurrent()) { | |
| 549 encoder_queue_.PostTask( | |
| 550 [this, bitrate_bps, fraction_lost, round_trip_time_ms] { | |
| 551 OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms); | |
| 552 }); | |
| 553 return; | |
| 554 } | |
| 555 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
| 556 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; | |
| 557 | |
| 558 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps | 243 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps |
| 559 << " packet loss " << static_cast<int>(fraction_lost) | 244 << " packet loss " << static_cast<int>(fraction_lost) |
| 560 << " rtt " << round_trip_time_ms; | 245 << " rtt " << round_trip_time_ms; |
| 561 | |
| 562 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | 246 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, |
| 563 round_trip_time_ms); | 247 round_trip_time_ms); |
| 564 | 248 bool video_suspension_changed; |
| 565 encoder_start_bitrate_bps_ = | |
| 566 bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; | |
| 567 bool video_is_suspended = bitrate_bps == 0; | 249 bool video_is_suspended = bitrate_bps == 0; |
| 568 bool video_suspension_changed = | 250 { |
| 569 video_is_suspended != (last_observed_bitrate_bps_ == 0); | 251 rtc::CritScope lock(&data_cs_); |
| 570 last_observed_bitrate_bps_ = bitrate_bps; | 252 last_observed_bitrate_bps_ = bitrate_bps; |
| 253 video_suspension_changed = video_suspended_ != video_is_suspended; |
| 254 video_suspended_ = video_is_suspended; |
| 255 // Set |time_of_last_frame_activity_ms_| to now if this is the first time |
| 256 // the encoder is supposed to produce encoded frames. |
| 257 // TODO(perkj): Remove this hack. It is here to avoid a race that the |
| 258 // encoder report that it has timed out before it has processed the first |
| 259 // frame. |
| 260 if (last_observed_bitrate_bps_ != 0 && |
| 261 time_of_last_frame_activity_ms_ == |
| 262 std::numeric_limits<int64_t>::max()) { |
| 263 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| 264 } |
| 265 } |
| 571 | 266 |
| 572 if (stats_proxy_ && video_suspension_changed) { | 267 if (stats_proxy_ && video_suspension_changed) { |
| 573 LOG(LS_INFO) << "Video suspend state changed to: " | 268 LOG(LS_INFO) << "Video suspend state changed to: " |
| 574 << (video_is_suspended ? "suspended" : "not suspended"); | 269 << (video_is_suspended ? "suspended" : "not suspended"); |
| 575 stats_proxy_->OnSuspendChange(video_is_suspended); | 270 stats_proxy_->OnSuspendChange(video_is_suspended); |
| 576 } | 271 } |
| 577 } | 272 } |
| 578 | 273 |
| 579 void ViEEncoder::OveruseDetected() { | |
| 580 RTC_DCHECK_RUN_ON(&module_process_thread_checker_); | |
| 581 // TODO(perkj): When ViEEncoder inherit rtc::VideoSink instead of | |
| 582 // VideoCaptureInput |load_observer_| should be removed and overuse be | |
| 583 // expressed as rtc::VideoSinkWants instead. | |
| 584 if (load_observer_) | |
| 585 load_observer_->OnLoadUpdate(LoadObserver::kOveruse); | |
| 586 } | |
| 587 | |
| 588 void ViEEncoder::NormalUsage() { | |
| 589 RTC_DCHECK_RUN_ON(&module_process_thread_checker_); | |
| 590 if (load_observer_) | |
| 591 load_observer_->OnLoadUpdate(LoadObserver::kUnderuse); | |
| 592 } | |
| 593 | |
| 594 } // namespace webrtc | 274 } // namespace webrtc |
| OLD | NEW |