OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
12 | 12 |
13 #include <algorithm> | 13 #include <algorithm> |
14 #include <limits> | 14 #include <limits> |
15 | 15 |
16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/logging.h" | 17 #include "webrtc/base/logging.h" |
18 #include "webrtc/base/trace_event.h" | 18 #include "webrtc/base/trace_event.h" |
19 #include "webrtc/base/timeutils.h" | 19 #include "webrtc/base/timeutils.h" |
20 #include "webrtc/modules/pacing/paced_sender.h" | 20 #include "webrtc/modules/pacing/paced_sender.h" |
21 #include "webrtc/modules/video_coding/include/video_coding.h" | 21 #include "webrtc/modules/video_coding/include/video_coding.h" |
22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
23 #include "webrtc/system_wrappers/include/metrics.h" | 23 #include "webrtc/system_wrappers/include/metrics.h" |
24 #include "webrtc/video/overuse_frame_detector.h" | 24 #include "webrtc/video/overuse_frame_detector.h" |
25 #include "webrtc/video/send_statistics_proxy.h" | 25 #include "webrtc/video/send_statistics_proxy.h" |
26 #include "webrtc/video_frame.h" | 26 #include "webrtc/video_frame.h" |
27 | 27 |
28 namespace webrtc { | 28 namespace webrtc { |
29 | 29 |
30 namespace { | |
31 | |
32 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { | |
33 if (payload_name == "VP8") | |
34 return kVideoCodecVP8; | |
35 if (payload_name == "VP9") | |
36 return kVideoCodecVP9; | |
37 if (payload_name == "H264") | |
38 return kVideoCodecH264; | |
39 return kVideoCodecGeneric; | |
40 } | |
41 | |
42 VideoCodec VideoEncoderConfigToVideoCodec(const VideoEncoderConfig& config, | |
43 const std::string& payload_name, | |
44 int payload_type) { | |
45 const std::vector<VideoStream>& streams = config.streams; | |
46 static const int kEncoderMinBitrateKbps = 30; | |
47 RTC_DCHECK(!streams.empty()); | |
48 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); | |
49 | |
50 VideoCodec video_codec; | |
51 memset(&video_codec, 0, sizeof(video_codec)); | |
52 video_codec.codecType = PayloadNameToCodecType(payload_name); | |
53 | |
54 switch (config.content_type) { | |
55 case VideoEncoderConfig::ContentType::kRealtimeVideo: | |
56 video_codec.mode = kRealtimeVideo; | |
57 break; | |
58 case VideoEncoderConfig::ContentType::kScreen: | |
59 video_codec.mode = kScreensharing; | |
60 if (config.streams.size() == 1 && | |
61 config.streams[0].temporal_layer_thresholds_bps.size() == 1) { | |
62 video_codec.targetBitrate = | |
63 config.streams[0].temporal_layer_thresholds_bps[0] / 1000; | |
64 } | |
65 break; | |
66 } | |
67 | |
68 switch (video_codec.codecType) { | |
69 case kVideoCodecVP8: { | |
70 if (config.encoder_specific_settings) { | |
71 video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>( | |
72 config.encoder_specific_settings); | |
73 } else { | |
74 video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); | |
75 } | |
76 video_codec.codecSpecific.VP8.numberOfTemporalLayers = | |
77 static_cast<unsigned char>( | |
78 streams.back().temporal_layer_thresholds_bps.size() + 1); | |
79 break; | |
80 } | |
81 case kVideoCodecVP9: { | |
82 if (config.encoder_specific_settings) { | |
83 video_codec.codecSpecific.VP9 = *reinterpret_cast<const VideoCodecVP9*>( | |
84 config.encoder_specific_settings); | |
85 if (video_codec.mode == kScreensharing) { | |
86 video_codec.codecSpecific.VP9.flexibleMode = true; | |
87 // For now VP9 screensharing use 1 temporal and 2 spatial layers. | |
88 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, | |
89 1); | |
90 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2); | |
91 } | |
92 } else { | |
93 video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); | |
94 } | |
95 video_codec.codecSpecific.VP9.numberOfTemporalLayers = | |
96 static_cast<unsigned char>( | |
97 streams.back().temporal_layer_thresholds_bps.size() + 1); | |
98 break; | |
99 } | |
100 case kVideoCodecH264: { | |
101 if (config.encoder_specific_settings) { | |
102 video_codec.codecSpecific.H264 = | |
103 *reinterpret_cast<const VideoCodecH264*>( | |
104 config.encoder_specific_settings); | |
105 } else { | |
106 video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); | |
107 } | |
108 break; | |
109 } | |
110 default: | |
111 // TODO(pbos): Support encoder_settings codec-agnostically. | |
112 RTC_DCHECK(!config.encoder_specific_settings) | |
113 << "Encoder-specific settings for codec type not wired up."; | |
114 break; | |
115 } | |
116 | |
117 strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); | |
118 video_codec.plName[kPayloadNameSize - 1] = '\0'; | |
119 video_codec.plType = payload_type; | |
120 video_codec.numberOfSimulcastStreams = | |
121 static_cast<unsigned char>(streams.size()); | |
122 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; | |
123 if (video_codec.minBitrate < kEncoderMinBitrateKbps) | |
124 video_codec.minBitrate = kEncoderMinBitrateKbps; | |
125 RTC_DCHECK_LE(streams.size(), static_cast<size_t>(kMaxSimulcastStreams)); | |
126 if (video_codec.codecType == kVideoCodecVP9) { | |
127 // If the vector is empty, bitrates will be configured automatically. | |
128 RTC_DCHECK(config.spatial_layers.empty() || | |
129 config.spatial_layers.size() == | |
130 video_codec.codecSpecific.VP9.numberOfSpatialLayers); | |
131 RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, | |
132 kMaxSimulcastStreams); | |
133 for (size_t i = 0; i < config.spatial_layers.size(); ++i) | |
134 video_codec.spatialLayers[i] = config.spatial_layers[i]; | |
135 } | |
136 for (size_t i = 0; i < streams.size(); ++i) { | |
137 SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; | |
138 RTC_DCHECK_GT(streams[i].width, 0u); | |
139 RTC_DCHECK_GT(streams[i].height, 0u); | |
140 RTC_DCHECK_GT(streams[i].max_framerate, 0); | |
141 // Different framerates not supported per stream at the moment. | |
142 RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); | |
143 RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); | |
144 RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); | |
145 RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); | |
146 RTC_DCHECK_GE(streams[i].max_qp, 0); | |
147 | |
148 sim_stream->width = static_cast<uint16_t>(streams[i].width); | |
149 sim_stream->height = static_cast<uint16_t>(streams[i].height); | |
150 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; | |
151 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; | |
152 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; | |
153 sim_stream->qpMax = streams[i].max_qp; | |
154 sim_stream->numberOfTemporalLayers = static_cast<unsigned char>( | |
155 streams[i].temporal_layer_thresholds_bps.size() + 1); | |
156 | |
157 video_codec.width = | |
158 std::max(video_codec.width, static_cast<uint16_t>(streams[i].width)); | |
159 video_codec.height = | |
160 std::max(video_codec.height, static_cast<uint16_t>(streams[i].height)); | |
161 video_codec.minBitrate = | |
162 std::min(static_cast<uint16_t>(video_codec.minBitrate), | |
163 static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000)); | |
164 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; | |
165 video_codec.qpMax = std::max(video_codec.qpMax, | |
166 static_cast<unsigned int>(streams[i].max_qp)); | |
167 } | |
168 | |
169 if (video_codec.maxBitrate == 0) { | |
170 // Unset max bitrate -> cap to one bit per pixel. | |
171 video_codec.maxBitrate = | |
172 (video_codec.width * video_codec.height * video_codec.maxFramerate) / | |
173 1000; | |
174 } | |
175 if (video_codec.maxBitrate < kEncoderMinBitrateKbps) | |
176 video_codec.maxBitrate = kEncoderMinBitrateKbps; | |
177 | |
178 RTC_DCHECK_GT(streams[0].max_framerate, 0); | |
179 video_codec.maxFramerate = streams[0].max_framerate; | |
180 video_codec.expect_encode_from_texture = config.expect_encode_from_texture; | |
181 | |
182 return video_codec; | |
183 } | |
184 | |
185 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle | |
186 // pipelining encoders better (multiple input frames before something comes | |
187 // out). This should effectively turn off CPU adaptations for systems that | |
188 // remotely cope with the load right now. | |
189 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { | |
190 CpuOveruseOptions options; | |
191 if (full_overuse_time) { | |
192 options.low_encode_usage_threshold_percent = 150; | |
193 options.high_encode_usage_threshold_percent = 200; | |
194 } | |
195 return options; | |
196 } | |
197 | |
198 } // namespace | |
199 | |
200 class ViEEncoder::EncodeTask : public rtc::QueuedTask { | |
201 public: | |
202 EncodeTask(const VideoFrame& frame, ViEEncoder* vie_encoder) | |
203 : vie_encoder_(vie_encoder) { | |
204 frame_.ShallowCopy(frame); | |
205 ++vie_encoder_->posted_frames_waiting_for_encode_; | |
206 } | |
207 | |
208 private: | |
209 bool Run() override { | |
210 RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0); | |
211 if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) { | |
212 vie_encoder_->EncodeVideoFrame(frame_); | |
213 } else { | |
214 // There is a newer frame in flight. Do not encode this frame. | |
215 LOG(LS_VERBOSE) | |
216 << "Incoming frame dropped due to that the encoder is blocked."; | |
217 } | |
218 return true; | |
219 } | |
220 VideoFrame frame_; | |
221 ViEEncoder* vie_encoder_; | |
222 }; | |
223 | |
30 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | 224 ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
31 ProcessThread* module_process_thread, | |
32 SendStatisticsProxy* stats_proxy, | 225 SendStatisticsProxy* stats_proxy, |
33 OveruseFrameDetector* overuse_detector, | 226 const VideoSendStream::Config::EncoderSettings& settings, |
34 EncodedImageCallback* sink) | 227 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, |
35 : number_of_cores_(number_of_cores), | 228 LoadObserver* overuse_callback, |
36 sink_(sink), | 229 EncodedFrameObserver* encoder_timing) |
230 : shutdown_event_(true /* manual_reset */, false), | |
231 number_of_cores_(number_of_cores), | |
232 settings_(settings), | |
37 vp_(VideoProcessing::Create()), | 233 vp_(VideoProcessing::Create()), |
38 video_sender_(Clock::GetRealTimeClock(), this, this), | 234 video_sender_(Clock::GetRealTimeClock(), this, this), |
235 overuse_detector_(Clock::GetRealTimeClock(), | |
236 GetCpuOveruseOptions(settings.full_overuse_time), | |
237 this, | |
238 encoder_timing, | |
239 stats_proxy), | |
240 load_observer_(overuse_callback), | |
39 stats_proxy_(stats_proxy), | 241 stats_proxy_(stats_proxy), |
40 overuse_detector_(overuse_detector), | 242 pre_encode_callback_(pre_encode_callback), |
41 time_of_last_frame_activity_ms_(std::numeric_limits<int64_t>::max()), | 243 module_process_thread_(nullptr), |
42 encoder_config_(), | 244 encoder_config_(), |
245 encoder_start_bitrate_bps_(0), | |
43 last_observed_bitrate_bps_(0), | 246 last_observed_bitrate_bps_(0), |
44 encoder_paused_and_dropped_frame_(false), | 247 encoder_paused_and_dropped_frame_(false), |
45 module_process_thread_(module_process_thread), | |
46 has_received_sli_(false), | 248 has_received_sli_(false), |
47 picture_id_sli_(0), | 249 picture_id_sli_(0), |
48 has_received_rpsi_(false), | 250 has_received_rpsi_(false), |
49 picture_id_rpsi_(0), | 251 picture_id_rpsi_(0), |
50 video_suspended_(false) { | 252 clock_(Clock::GetRealTimeClock()), |
253 last_captured_timestamp_(0), | |
254 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | |
255 clock_->TimeInMilliseconds()), | |
256 encoder_queue_("EncoderQueue") { | |
257 vp_->EnableTemporalDecimation(false); | |
258 encoded_sequenced_checker_.Detach(); | |
259 | |
260 encoder_queue_.PostTask([this] { | |
261 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
262 video_sender_.RegisterExternalEncoder( | |
263 settings_.encoder, settings_.payload_type, settings_.internal_source); | |
264 }); | |
265 } | |
266 | |
267 ViEEncoder::~ViEEncoder() { | |
268 RTC_DCHECK(shutdown_event_.Wait(0)) | |
269 << "Must call ::Stop() before destruction."; | |
270 } | |
271 | |
272 void ViEEncoder::Stop() { | |
273 if (!encoder_queue_.IsCurrent()) { | |
274 encoder_queue_.PostTask([this] { Stop(); }); | |
275 shutdown_event_.Wait(rtc::Event::kForever); | |
276 return; | |
277 } | |
278 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
279 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); | |
280 shutdown_event_.Set(); | |
281 } | |
282 | |
283 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | |
284 RTC_DCHECK(!module_process_thread_); | |
285 module_process_thread_ = module_process_thread; | |
286 module_process_thread_->RegisterModule(&overuse_detector_); | |
51 module_process_thread_->RegisterModule(&video_sender_); | 287 module_process_thread_->RegisterModule(&video_sender_); |
52 vp_->EnableTemporalDecimation(true); | 288 module_process_thread_checker_.DetachFromThread(); |
53 } | 289 } |
54 | 290 |
55 vcm::VideoSender* ViEEncoder::video_sender() { | 291 void ViEEncoder::DeRegisterProcessThread() { |
56 return &video_sender_; | 292 module_process_thread_->DeRegisterModule(&overuse_detector_); |
57 } | |
58 | |
59 ViEEncoder::~ViEEncoder() { | |
60 module_process_thread_->DeRegisterModule(&video_sender_); | 293 module_process_thread_->DeRegisterModule(&video_sender_); |
61 } | 294 } |
62 | 295 |
63 int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, | 296 void ViEEncoder::SetSink(EncodedImageCallback* sink) { |
64 uint8_t pl_type, | 297 encoder_queue_.PostTask([this, sink] { |
65 bool internal_source) { | 298 RTC_DCHECK_RUN_ON(&encoder_queue_); |
66 video_sender_.RegisterExternalEncoder(encoder, pl_type, internal_source); | 299 sink_ = sink; |
67 return 0; | 300 }); |
68 } | 301 } |
69 | 302 |
70 int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { | 303 void ViEEncoder::SetStartBitrate(int start_bitrate_bps) { |
71 video_sender_.RegisterExternalEncoder(nullptr, pl_type, false); | 304 encoder_queue_.PostTask([this, start_bitrate_bps] { |
72 return 0; | 305 RTC_DCHECK_RUN_ON(&encoder_queue_); |
73 } | 306 encoder_start_bitrate_bps_ = start_bitrate_bps; |
74 | 307 }); |
75 void ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec, | 308 } |
76 size_t max_data_payload_length) { | 309 |
310 void ViEEncoder::ConfigureEncoder(const VideoEncoderConfig& config, | |
311 size_t max_data_payload_length) { | |
312 VideoCodec video_codec = VideoEncoderConfigToVideoCodec( | |
313 config, settings_.payload_name, settings_.payload_type); | |
314 encoder_queue_.PostTask([this, video_codec, max_data_payload_length] { | |
315 ConfigureEncoderInternal(video_codec, max_data_payload_length); | |
316 }); | |
317 return; | |
318 } | |
319 | |
320 void ViEEncoder::ConfigureEncoderInternal(const VideoCodec& video_codec, | |
321 size_t max_data_payload_length) { | |
322 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
323 RTC_DCHECK_GE(encoder_start_bitrate_bps_, 0); | |
324 RTC_DCHECK(sink_); | |
325 | |
77 // Setting target width and height for VPM. | 326 // Setting target width and height for VPM. |
78 RTC_CHECK_EQ(VPM_OK, | 327 RTC_CHECK_EQ(VPM_OK, |
79 vp_->SetTargetResolution(video_codec.width, video_codec.height, | 328 vp_->SetTargetResolution(video_codec.width, video_codec.height, |
80 video_codec.maxFramerate)); | 329 video_codec.maxFramerate)); |
81 { | 330 |
82 rtc::CritScope lock(&data_cs_); | 331 encoder_config_ = video_codec; |
83 encoder_config_ = video_codec; | 332 encoder_config_.startBitrate = encoder_start_bitrate_bps_ / 1000; |
84 } | |
85 | 333 |
86 bool success = video_sender_.RegisterSendCodec( | 334 bool success = video_sender_.RegisterSendCodec( |
87 &video_codec, number_of_cores_, | 335 &encoder_config_, number_of_cores_, |
88 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; | 336 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
89 | 337 |
90 if (!success) { | 338 if (!success) { |
91 LOG(LS_ERROR) << "Failed to configure encoder."; | 339 LOG(LS_ERROR) << "Failed to configure encoder."; |
92 RTC_DCHECK(success); | 340 RTC_DCHECK(success); |
93 } | 341 } |
94 | 342 |
95 if (stats_proxy_) { | 343 if (stats_proxy_) { |
96 VideoEncoderConfig::ContentType content_type = | 344 VideoEncoderConfig::ContentType content_type = |
97 VideoEncoderConfig::ContentType::kRealtimeVideo; | 345 VideoEncoderConfig::ContentType::kRealtimeVideo; |
98 switch (video_codec.mode) { | 346 switch (video_codec.mode) { |
99 case kRealtimeVideo: | 347 case kRealtimeVideo: |
100 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; | 348 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; |
101 break; | 349 break; |
102 case kScreensharing: | 350 case kScreensharing: |
103 content_type = VideoEncoderConfig::ContentType::kScreen; | 351 content_type = VideoEncoderConfig::ContentType::kScreen; |
104 break; | 352 break; |
105 default: | 353 default: |
106 RTC_NOTREACHED(); | 354 RTC_NOTREACHED(); |
107 break; | 355 break; |
108 } | 356 } |
109 stats_proxy_->SetContentType(content_type); | 357 stats_proxy_->SetContentType(content_type); |
110 } | 358 } |
111 } | 359 } |
112 | 360 |
361 void ViEEncoder::IncomingCapturedFrame(const VideoFrame& video_frame) { | |
362 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | |
363 stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height()); | |
364 | |
365 VideoFrame incoming_frame = video_frame; | |
366 | |
367 // Local time in webrtc time base. | |
368 int64_t current_time = clock_->TimeInMilliseconds(); | |
369 incoming_frame.set_render_time_ms(current_time); | |
370 | |
371 // Capture time may come from clock with an offset and drift from clock_. | |
372 int64_t capture_ntp_time_ms; | |
373 if (video_frame.ntp_time_ms() != 0) { | |
374 capture_ntp_time_ms = video_frame.ntp_time_ms(); | |
375 } else if (video_frame.render_time_ms() != 0) { | |
376 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; | |
377 } else { | |
378 capture_ntp_time_ms = current_time + delta_ntp_internal_ms_; | |
379 } | |
380 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); | |
381 | |
382 // Convert NTP time, in ms, to RTP timestamp. | |
383 const int kMsToRtpTimestamp = 90; | |
384 incoming_frame.set_timestamp( | |
385 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms())); | |
386 | |
387 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { | |
388 // We don't allow the same capture time for two frames, drop this one. | |
389 LOG(LS_WARNING) << "Same/old NTP timestamp (" | |
390 << incoming_frame.ntp_time_ms() | |
391 << " <= " << last_captured_timestamp_ | |
392 << ") for incoming frame. Dropping."; | |
393 return; | |
394 } | |
395 | |
396 last_captured_timestamp_ = incoming_frame.ntp_time_ms(); | |
397 overuse_detector_.FrameCaptured(incoming_frame); | |
398 encoder_queue_.PostTask( | |
399 std::unique_ptr<rtc::QueuedTask>(new EncodeTask(incoming_frame, this))); | |
400 } | |
401 | |
113 bool ViEEncoder::EncoderPaused() const { | 402 bool ViEEncoder::EncoderPaused() const { |
403 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
114 // Pause video if paused by caller or as long as the network is down or the | 404 // Pause video if paused by caller or as long as the network is down or the |
115 // pacer queue has grown too large in buffered mode. | 405 // pacer queue has grown too large in buffered mode. |
116 // If the pacer queue has grown too large or the network is down, | 406 // If the pacer queue has grown too large or the network is down, |
117 // last_observed_bitrate_bps_ will be 0. | 407 // last_observed_bitrate_bps_ will be 0. |
118 return video_suspended_ || last_observed_bitrate_bps_ == 0; | 408 return last_observed_bitrate_bps_ == 0; |
119 } | 409 } |
120 | 410 |
121 void ViEEncoder::TraceFrameDropStart() { | 411 void ViEEncoder::TraceFrameDropStart() { |
412 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
122 // Start trace event only on the first frame after encoder is paused. | 413 // Start trace event only on the first frame after encoder is paused. |
123 if (!encoder_paused_and_dropped_frame_) { | 414 if (!encoder_paused_and_dropped_frame_) { |
124 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); | 415 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); |
125 } | 416 } |
126 encoder_paused_and_dropped_frame_ = true; | 417 encoder_paused_and_dropped_frame_ = true; |
127 return; | 418 return; |
128 } | 419 } |
129 | 420 |
130 void ViEEncoder::TraceFrameDropEnd() { | 421 void ViEEncoder::TraceFrameDropEnd() { |
422 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
131 // End trace event on first frame after encoder resumes, if frame was dropped. | 423 // End trace event on first frame after encoder resumes, if frame was dropped. |
132 if (encoder_paused_and_dropped_frame_) { | 424 if (encoder_paused_and_dropped_frame_) { |
133 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | 425 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
134 } | 426 } |
135 encoder_paused_and_dropped_frame_ = false; | 427 encoder_paused_and_dropped_frame_ = false; |
136 } | 428 } |
137 | 429 |
138 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { | 430 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { |
139 VideoCodecType codec_type; | 431 RTC_DCHECK_RUN_ON(&encoder_queue_); |
140 { | 432 if (pre_encode_callback_) |
141 rtc::CritScope lock(&data_cs_); | 433 pre_encode_callback_->OnFrame(video_frame); |
142 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); | 434 |
143 if (EncoderPaused()) { | 435 if (EncoderPaused()) { |
144 TraceFrameDropStart(); | 436 TraceFrameDropStart(); |
145 return; | 437 return; |
146 } | |
147 TraceFrameDropEnd(); | |
148 codec_type = encoder_config_.codecType; | |
149 } | 438 } |
439 TraceFrameDropEnd(); | |
150 | 440 |
151 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), | 441 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
152 "Encode"); | 442 "Encode"); |
153 const VideoFrame* frame_to_send = &video_frame; | 443 const VideoFrame* frame_to_send = &video_frame; |
154 // TODO(wuchengli): support texture frames. | 444 // TODO(wuchengli): support texture frames. |
155 if (!video_frame.video_frame_buffer()->native_handle()) { | 445 if (!video_frame.video_frame_buffer()->native_handle()) { |
156 // Pass frame via preprocessor. | 446 // Pass frame via preprocessor. |
157 frame_to_send = vp_->PreprocessFrame(video_frame); | 447 frame_to_send = vp_->PreprocessFrame(video_frame); |
158 if (!frame_to_send) { | 448 if (!frame_to_send) { |
159 // Drop this frame, or there was an error processing it. | 449 // Drop this frame, or there was an error processing it. |
160 return; | 450 return; |
161 } | 451 } |
162 } | 452 } |
163 | 453 |
164 if (codec_type == webrtc::kVideoCodecVP8) { | 454 if (encoder_config_.codecType == webrtc::kVideoCodecVP8) { |
165 webrtc::CodecSpecificInfo codec_specific_info; | 455 webrtc::CodecSpecificInfo codec_specific_info; |
166 codec_specific_info.codecType = webrtc::kVideoCodecVP8; | 456 codec_specific_info.codecType = webrtc::kVideoCodecVP8; |
167 { | 457 |
168 rtc::CritScope lock(&data_cs_); | |
169 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = | 458 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = |
170 has_received_rpsi_; | 459 has_received_rpsi_; |
171 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = | 460 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = |
172 has_received_sli_; | 461 has_received_sli_; |
173 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = | 462 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = |
174 picture_id_rpsi_; | 463 picture_id_rpsi_; |
175 codec_specific_info.codecSpecific.VP8.pictureIdSLI = | 464 codec_specific_info.codecSpecific.VP8.pictureIdSLI = |
176 picture_id_sli_; | 465 picture_id_sli_; |
177 has_received_sli_ = false; | 466 has_received_sli_ = false; |
178 has_received_rpsi_ = false; | 467 has_received_rpsi_ = false; |
179 } | |
180 | 468 |
181 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); | 469 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); |
182 return; | 470 return; |
183 } | 471 } |
184 video_sender_.AddVideoFrame(*frame_to_send, nullptr); | 472 video_sender_.AddVideoFrame(*frame_to_send, nullptr); |
185 } | 473 } |
186 | 474 |
187 void ViEEncoder::SendKeyFrame() { | 475 void ViEEncoder::SendKeyFrame() { |
476 if (!encoder_queue_.IsCurrent()) { | |
477 encoder_queue_.PostTask([this] { SendKeyFrame(); }); | |
478 return; | |
479 } | |
480 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
188 video_sender_.IntraFrameRequest(0); | 481 video_sender_.IntraFrameRequest(0); |
189 } | 482 } |
190 | 483 |
191 int64_t ViEEncoder::time_of_last_frame_activity_ms() { | |
192 rtc::CritScope lock(&data_cs_); | |
193 return time_of_last_frame_activity_ms_; | |
194 } | |
195 | |
196 int32_t ViEEncoder::Encoded(const EncodedImage& encoded_image, | 484 int32_t ViEEncoder::Encoded(const EncodedImage& encoded_image, |
197 const CodecSpecificInfo* codec_specific_info, | 485 const CodecSpecificInfo* codec_specific_info, |
198 const RTPFragmentationHeader* fragmentation) { | 486 const RTPFragmentationHeader* fragmentation) { |
199 { | 487 // Encoded is called on whatever thread the real encoder implementation run |
200 rtc::CritScope lock(&data_cs_); | 488 // on. |
201 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); | 489 RTC_DCHECK(encoded_sequenced_checker_.CalledSequentially()); |
pbos-webrtc
2016/07/13 12:35:39
This can be called in parallel for simulcast encod
perkj_webrtc
2016/07/14 10:11:28
ok, thanks. updated the comment.
| |
202 } | |
203 if (stats_proxy_) { | 490 if (stats_proxy_) { |
204 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 491 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
205 } | 492 } |
206 | 493 |
207 int success = | 494 int success = |
208 sink_->Encoded(encoded_image, codec_specific_info, fragmentation); | 495 sink_->Encoded(encoded_image, codec_specific_info, fragmentation); |
209 | 496 |
210 overuse_detector_->FrameSent(encoded_image._timeStamp); | 497 overuse_detector_.FrameSent(encoded_image._timeStamp); |
211 return success; | 498 return success; |
212 } | 499 } |
213 | 500 |
214 void ViEEncoder::SendStatistics(uint32_t bit_rate, | 501 void ViEEncoder::SendStatistics(uint32_t bit_rate, |
215 uint32_t frame_rate, | 502 uint32_t frame_rate, |
216 const std::string& encoder_name) { | 503 const std::string& encoder_name) { |
504 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | |
217 if (stats_proxy_) | 505 if (stats_proxy_) |
218 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); | 506 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); |
219 } | 507 } |
220 | 508 |
221 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { | 509 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
222 rtc::CritScope lock(&data_cs_); | 510 if (!encoder_queue_.IsCurrent()) { |
511 encoder_queue_.PostTask([this, picture_id] { OnReceivedSLI(picture_id); }); | |
512 return; | |
513 } | |
514 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
223 picture_id_sli_ = picture_id; | 515 picture_id_sli_ = picture_id; |
224 has_received_sli_ = true; | 516 has_received_sli_ = true; |
225 } | 517 } |
226 | 518 |
227 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { | 519 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { |
228 rtc::CritScope lock(&data_cs_); | 520 if (!encoder_queue_.IsCurrent()) { |
521 encoder_queue_.PostTask([this, picture_id] { OnReceivedRPSI(picture_id); }); | |
522 return; | |
523 } | |
524 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
229 picture_id_rpsi_ = picture_id; | 525 picture_id_rpsi_ = picture_id; |
230 has_received_rpsi_ = true; | 526 has_received_rpsi_ = true; |
231 } | 527 } |
232 | 528 |
233 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { | 529 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { |
530 if (!encoder_queue_.IsCurrent()) { | |
531 encoder_queue_.PostTask( | |
532 [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); }); | |
533 return; | |
534 } | |
535 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
234 // Key frame request from remote side, signal to VCM. | 536 // Key frame request from remote side, signal to VCM. |
235 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | 537 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
236 video_sender_.IntraFrameRequest(stream_index); | 538 video_sender_.IntraFrameRequest(stream_index); |
237 } | 539 } |
238 | 540 |
239 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | 541 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, |
240 uint8_t fraction_lost, | 542 uint8_t fraction_lost, |
241 int64_t round_trip_time_ms) { | 543 int64_t round_trip_time_ms) { |
544 if (!encoder_queue_.IsCurrent()) { | |
545 encoder_queue_.PostTask( | |
546 [this, bitrate_bps, fraction_lost, round_trip_time_ms] { | |
547 OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms); | |
548 }); | |
549 return; | |
550 } | |
551 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
552 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; | |
553 | |
242 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps | 554 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps |
243 << " packet loss " << static_cast<int>(fraction_lost) | 555 << " packet loss " << static_cast<int>(fraction_lost) |
244 << " rtt " << round_trip_time_ms; | 556 << " rtt " << round_trip_time_ms; |
557 | |
245 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | 558 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, |
246 round_trip_time_ms); | 559 round_trip_time_ms); |
247 bool video_suspension_changed; | 560 |
561 encoder_start_bitrate_bps_ = | |
562 bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; | |
248 bool video_is_suspended = bitrate_bps == 0; | 563 bool video_is_suspended = bitrate_bps == 0; |
249 { | 564 bool video_suspension_changed = |
250 rtc::CritScope lock(&data_cs_); | 565 video_is_suspended != (last_observed_bitrate_bps_ == 0); |
251 last_observed_bitrate_bps_ = bitrate_bps; | 566 last_observed_bitrate_bps_ = bitrate_bps; |
252 video_suspension_changed = video_suspended_ != video_is_suspended; | |
253 video_suspended_ = video_is_suspended; | |
254 // Set |time_of_last_frame_activity_ms_| to now if this is the first time | |
255 // the encoder is supposed to produce encoded frames. | |
256 // TODO(perkj): Remove this hack. It is here to avoid a race that the | |
257 // encoder report that it has timed out before it has processed the first | |
258 // frame. | |
259 if (last_observed_bitrate_bps_ != 0 && | |
260 time_of_last_frame_activity_ms_ == | |
261 std::numeric_limits<int64_t>::max()) { | |
262 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); | |
263 } | |
264 } | |
265 | 567 |
266 if (stats_proxy_ && video_suspension_changed) { | 568 if (stats_proxy_ && video_suspension_changed) { |
267 LOG(LS_INFO) << "Video suspend state changed to: " | 569 LOG(LS_INFO) << "Video suspend state changed to: " |
268 << (video_is_suspended ? "suspended" : "not suspended"); | 570 << (video_is_suspended ? "suspended" : "not suspended"); |
269 stats_proxy_->OnSuspendChange(video_is_suspended); | 571 stats_proxy_->OnSuspendChange(video_is_suspended); |
270 } | 572 } |
271 } | 573 } |
272 | 574 |
575 void ViEEncoder::OveruseDetected() { | |
576 RTC_DCHECK_RUN_ON(&module_process_thread_checker_); | |
577 // TODO(perkj): When ViEEncoder inherit rtc::VideoSink instead of | |
578 // VideoCaptureInput |load_observer_| should be removed and overuse be | |
579 // expressed as rtc::VideoSinkWants instead. | |
580 if (load_observer_) | |
581 load_observer_->OnLoadUpdate(LoadObserver::kOveruse); | |
582 } | |
583 | |
584 void ViEEncoder::NormalUsage() { | |
585 RTC_DCHECK_RUN_ON(&module_process_thread_checker_); | |
586 if (load_observer_) | |
587 load_observer_->OnLoadUpdate(LoadObserver::kUnderuse); | |
588 } | |
589 | |
273 } // namespace webrtc | 590 } // namespace webrtc |
OLD | NEW |