OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
12 | 12 |
13 #include <algorithm> | 13 #include <algorithm> |
14 #include <limits> | 14 #include <limits> |
15 | 15 |
16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/logging.h" | 17 #include "webrtc/base/logging.h" |
18 #include "webrtc/base/trace_event.h" | 18 #include "webrtc/base/trace_event.h" |
19 #include "webrtc/base/timeutils.h" | 19 #include "webrtc/base/timeutils.h" |
20 #include "webrtc/modules/pacing/paced_sender.h" | 20 #include "webrtc/modules/pacing/paced_sender.h" |
21 #include "webrtc/modules/video_coding/include/video_coding.h" | 21 #include "webrtc/modules/video_coding/include/video_coding.h" |
22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
23 #include "webrtc/system_wrappers/include/metrics.h" | 23 #include "webrtc/system_wrappers/include/metrics.h" |
24 #include "webrtc/video/overuse_frame_detector.h" | 24 #include "webrtc/video/overuse_frame_detector.h" |
25 #include "webrtc/video/send_statistics_proxy.h" | 25 #include "webrtc/video/send_statistics_proxy.h" |
26 #include "webrtc/video_frame.h" | 26 #include "webrtc/video_frame.h" |
27 | 27 |
28 namespace webrtc { | 28 namespace webrtc { |
29 | 29 |
30 namespace { | |
31 | |
32 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { | |
33 if (payload_name == "VP8") | |
34 return kVideoCodecVP8; | |
35 if (payload_name == "VP9") | |
36 return kVideoCodecVP9; | |
37 if (payload_name == "H264") | |
38 return kVideoCodecH264; | |
39 return kVideoCodecGeneric; | |
40 } | |
41 | |
42 VideoCodec VideoEncoderConfigToVideoCodec(const VideoEncoderConfig& config, | |
43 const std::string& payload_name, | |
44 int payload_type) { | |
45 const std::vector<VideoStream>& streams = config.streams; | |
46 static const int kEncoderMinBitrateKbps = 30; | |
47 RTC_DCHECK(!streams.empty()); | |
48 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); | |
49 | |
50 VideoCodec video_codec; | |
51 memset(&video_codec, 0, sizeof(video_codec)); | |
52 video_codec.codecType = PayloadNameToCodecType(payload_name); | |
53 | |
54 switch (config.content_type) { | |
55 case VideoEncoderConfig::ContentType::kRealtimeVideo: | |
56 video_codec.mode = kRealtimeVideo; | |
57 break; | |
58 case VideoEncoderConfig::ContentType::kScreen: | |
59 video_codec.mode = kScreensharing; | |
60 if (config.streams.size() == 1 && | |
61 config.streams[0].temporal_layer_thresholds_bps.size() == 1) { | |
62 video_codec.targetBitrate = | |
63 config.streams[0].temporal_layer_thresholds_bps[0] / 1000; | |
64 } | |
65 break; | |
66 } | |
67 | |
68 switch (video_codec.codecType) { | |
69 case kVideoCodecVP8: { | |
70 if (config.encoder_specific_settings) { | |
71 video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>( | |
72 config.encoder_specific_settings); | |
73 } else { | |
74 video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); | |
75 } | |
76 video_codec.codecSpecific.VP8.numberOfTemporalLayers = | |
77 static_cast<unsigned char>( | |
78 streams.back().temporal_layer_thresholds_bps.size() + 1); | |
79 break; | |
80 } | |
81 case kVideoCodecVP9: { | |
82 if (config.encoder_specific_settings) { | |
83 video_codec.codecSpecific.VP9 = *reinterpret_cast<const VideoCodecVP9*>( | |
84 config.encoder_specific_settings); | |
85 if (video_codec.mode == kScreensharing) { | |
86 video_codec.codecSpecific.VP9.flexibleMode = true; | |
87 // For now VP9 screensharing use 1 temporal and 2 spatial layers. | |
88 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfTemporalLayers, | |
89 1); | |
90 RTC_DCHECK_EQ(video_codec.codecSpecific.VP9.numberOfSpatialLayers, 2); | |
91 } | |
92 } else { | |
93 video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); | |
94 } | |
95 video_codec.codecSpecific.VP9.numberOfTemporalLayers = | |
96 static_cast<unsigned char>( | |
97 streams.back().temporal_layer_thresholds_bps.size() + 1); | |
98 break; | |
99 } | |
100 case kVideoCodecH264: { | |
101 if (config.encoder_specific_settings) { | |
102 video_codec.codecSpecific.H264 = | |
103 *reinterpret_cast<const VideoCodecH264*>( | |
104 config.encoder_specific_settings); | |
105 } else { | |
106 video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); | |
107 } | |
108 break; | |
109 } | |
110 default: | |
111 // TODO(pbos): Support encoder_settings codec-agnostically. | |
112 RTC_DCHECK(!config.encoder_specific_settings) | |
113 << "Encoder-specific settings for codec type not wired up."; | |
114 break; | |
115 } | |
116 | |
117 strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); | |
118 video_codec.plName[kPayloadNameSize - 1] = '\0'; | |
119 video_codec.plType = payload_type; | |
120 video_codec.numberOfSimulcastStreams = | |
121 static_cast<unsigned char>(streams.size()); | |
122 video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; | |
123 if (video_codec.minBitrate < kEncoderMinBitrateKbps) | |
124 video_codec.minBitrate = kEncoderMinBitrateKbps; | |
125 RTC_DCHECK_LE(streams.size(), static_cast<size_t>(kMaxSimulcastStreams)); | |
126 if (video_codec.codecType == kVideoCodecVP9) { | |
127 // If the vector is empty, bitrates will be configured automatically. | |
128 RTC_DCHECK(config.spatial_layers.empty() || | |
129 config.spatial_layers.size() == | |
130 video_codec.codecSpecific.VP9.numberOfSpatialLayers); | |
131 RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, | |
132 kMaxSimulcastStreams); | |
133 for (size_t i = 0; i < config.spatial_layers.size(); ++i) | |
134 video_codec.spatialLayers[i] = config.spatial_layers[i]; | |
135 } | |
136 for (size_t i = 0; i < streams.size(); ++i) { | |
137 SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; | |
138 RTC_DCHECK_GT(streams[i].width, 0u); | |
139 RTC_DCHECK_GT(streams[i].height, 0u); | |
140 RTC_DCHECK_GT(streams[i].max_framerate, 0); | |
141 // Different framerates not supported per stream at the moment. | |
142 RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); | |
143 RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); | |
144 RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); | |
145 RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); | |
146 RTC_DCHECK_GE(streams[i].max_qp, 0); | |
147 | |
148 sim_stream->width = static_cast<uint16_t>(streams[i].width); | |
149 sim_stream->height = static_cast<uint16_t>(streams[i].height); | |
150 sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; | |
151 sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; | |
152 sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; | |
153 sim_stream->qpMax = streams[i].max_qp; | |
154 sim_stream->numberOfTemporalLayers = static_cast<unsigned char>( | |
155 streams[i].temporal_layer_thresholds_bps.size() + 1); | |
156 | |
157 video_codec.width = | |
158 std::max(video_codec.width, static_cast<uint16_t>(streams[i].width)); | |
159 video_codec.height = | |
160 std::max(video_codec.height, static_cast<uint16_t>(streams[i].height)); | |
161 video_codec.minBitrate = | |
162 std::min(static_cast<uint16_t>(video_codec.minBitrate), | |
163 static_cast<uint16_t>(streams[i].min_bitrate_bps / 1000)); | |
164 video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; | |
165 video_codec.qpMax = std::max(video_codec.qpMax, | |
166 static_cast<unsigned int>(streams[i].max_qp)); | |
167 } | |
168 | |
169 if (video_codec.maxBitrate == 0) { | |
170 // Unset max bitrate -> cap to one bit per pixel. | |
171 video_codec.maxBitrate = | |
172 (video_codec.width * video_codec.height * video_codec.maxFramerate) / | |
173 1000; | |
174 } | |
175 if (video_codec.maxBitrate < kEncoderMinBitrateKbps) | |
176 video_codec.maxBitrate = kEncoderMinBitrateKbps; | |
177 | |
178 RTC_DCHECK_GT(streams[0].max_framerate, 0); | |
179 video_codec.maxFramerate = streams[0].max_framerate; | |
180 video_codec.expect_encode_from_texture = config.expect_encode_from_texture; | |
181 | |
182 return video_codec; | |
183 } | |
184 | |
185 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle | |
186 // pipelining encoders better (multiple input frames before something comes | |
187 // out). This should effectively turn off CPU adaptations for systems that | |
188 // remotely cope with the load right now. | |
189 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { | |
190 CpuOveruseOptions options; | |
191 if (full_overuse_time) { | |
192 options.low_encode_usage_threshold_percent = 150; | |
193 options.high_encode_usage_threshold_percent = 200; | |
194 } | |
195 return options; | |
196 } | |
197 | |
198 } // namespace | |
199 | |
200 class ViEEncoder::EncodeTask : public rtc::QueuedTask { | |
201 public: | |
202 EncodeTask(const VideoFrame& frame, ViEEncoder* vie_encoder) | |
203 : vie_encoder_(vie_encoder) { | |
204 frame_.ShallowCopy(frame); | |
205 ++vie_encoder_->posted_frames_waiting_for_encode_; | |
206 } | |
207 | |
208 private: | |
209 bool Run() override { | |
210 RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0); | |
211 if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) { | |
212 vie_encoder_->EncodeVideoFrame(frame_); | |
213 } else { | |
214 // There is a newer frame in flight. Do not encode this frame. | |
215 LOG(LS_VERBOSE) | |
216 << "Incoming frame dropped due to that the encoder is blocked."; | |
217 } | |
218 return true; | |
219 } | |
220 VideoFrame frame_; | |
221 ViEEncoder* vie_encoder_; | |
222 }; | |
223 | |
224 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | 30 ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
| 31 ProcessThread* module_process_thread, |
225 SendStatisticsProxy* stats_proxy, | 32 SendStatisticsProxy* stats_proxy, |
226 const VideoSendStream::Config::EncoderSettings& settings, | 33 OveruseFrameDetector* overuse_detector, |
227 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | 34 EncodedImageCallback* sink) |
228 LoadObserver* overuse_callback, | 35 : number_of_cores_(number_of_cores), |
229 EncodedFrameObserver* encoder_timing) | 36 sink_(sink), |
230 : shutdown_event_(true /* manual_reset */, false), | |
231 number_of_cores_(number_of_cores), | |
232 settings_(settings), | |
233 vp_(VideoProcessing::Create()), | 37 vp_(VideoProcessing::Create()), |
234 video_sender_(Clock::GetRealTimeClock(), this, this), | 38 video_sender_(Clock::GetRealTimeClock(), this, this), |
235 overuse_detector_(Clock::GetRealTimeClock(), | |
236 GetCpuOveruseOptions(settings.full_overuse_time), | |
237 this, | |
238 encoder_timing, | |
239 stats_proxy), | |
240 load_observer_(overuse_callback), | |
241 stats_proxy_(stats_proxy), | 39 stats_proxy_(stats_proxy), |
242 pre_encode_callback_(pre_encode_callback), | 40 overuse_detector_(overuse_detector), |
243 module_process_thread_(nullptr), | 41 time_of_last_frame_activity_ms_(std::numeric_limits<int64_t>::max()), |
244 encoder_config_(), | 42 encoder_config_(), |
245 encoder_start_bitrate_bps_(0), | |
246 last_observed_bitrate_bps_(0), | 43 last_observed_bitrate_bps_(0), |
247 encoder_paused_and_dropped_frame_(false), | 44 encoder_paused_and_dropped_frame_(false), |
| 45 module_process_thread_(module_process_thread), |
248 has_received_sli_(false), | 46 has_received_sli_(false), |
249 picture_id_sli_(0), | 47 picture_id_sli_(0), |
250 has_received_rpsi_(false), | 48 has_received_rpsi_(false), |
251 picture_id_rpsi_(0), | 49 picture_id_rpsi_(0), |
252 clock_(Clock::GetRealTimeClock()), | 50 video_suspended_(false) { |
253 last_captured_timestamp_(0), | 51 module_process_thread_->RegisterModule(&video_sender_); |
254 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 52 vp_->EnableTemporalDecimation(true); |
255 clock_->TimeInMilliseconds()), | 53 } |
256 encoder_queue_("EncoderQueue") { | |
257 vp_->EnableTemporalDecimation(false); | |
258 | 54 |
259 encoder_queue_.PostTask([this] { | 55 vcm::VideoSender* ViEEncoder::video_sender() { |
260 RTC_DCHECK_RUN_ON(&encoder_queue_); | 56 return &video_sender_; |
261 video_sender_.RegisterExternalEncoder( | |
262 settings_.encoder, settings_.payload_type, settings_.internal_source); | |
263 }); | |
264 } | 57 } |
265 | 58 |
266 ViEEncoder::~ViEEncoder() { | 59 ViEEncoder::~ViEEncoder() { |
267 RTC_DCHECK(shutdown_event_.Wait(0)) | |
268 << "Must call ::Stop() before destruction."; | |
269 } | |
270 | |
271 void ViEEncoder::Stop() { | |
272 if (!encoder_queue_.IsCurrent()) { | |
273 encoder_queue_.PostTask([this] { Stop(); }); | |
274 shutdown_event_.Wait(rtc::Event::kForever); | |
275 return; | |
276 } | |
277 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
278 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); | |
279 shutdown_event_.Set(); | |
280 } | |
281 | |
282 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) { | |
283 RTC_DCHECK(!module_process_thread_); | |
284 module_process_thread_ = module_process_thread; | |
285 module_process_thread_->RegisterModule(&overuse_detector_); | |
286 module_process_thread_->RegisterModule(&video_sender_); | |
287 module_process_thread_checker_.DetachFromThread(); | |
288 } | |
289 | |
290 void ViEEncoder::DeRegisterProcessThread() { | |
291 module_process_thread_->DeRegisterModule(&overuse_detector_); | |
292 module_process_thread_->DeRegisterModule(&video_sender_); | 60 module_process_thread_->DeRegisterModule(&video_sender_); |
293 } | 61 } |
294 | 62 |
295 void ViEEncoder::SetSink(EncodedImageCallback* sink) { | 63 int32_t ViEEncoder::RegisterExternalEncoder(webrtc::VideoEncoder* encoder, |
296 encoder_queue_.PostTask([this, sink] { | 64 uint8_t pl_type, |
297 RTC_DCHECK_RUN_ON(&encoder_queue_); | 65 bool internal_source) { |
298 sink_ = sink; | 66 video_sender_.RegisterExternalEncoder(encoder, pl_type, internal_source); |
299 }); | 67 return 0; |
300 } | 68 } |
301 | 69 |
302 void ViEEncoder::SetStartBitrate(int start_bitrate_bps) { | 70 int32_t ViEEncoder::DeRegisterExternalEncoder(uint8_t pl_type) { |
303 encoder_queue_.PostTask([this, start_bitrate_bps] { | 71 video_sender_.RegisterExternalEncoder(nullptr, pl_type, false); |
304 RTC_DCHECK_RUN_ON(&encoder_queue_); | 72 return 0; |
305 encoder_start_bitrate_bps_ = start_bitrate_bps; | |
306 }); | |
307 } | 73 } |
308 | 74 |
309 void ViEEncoder::ConfigureEncoder(const VideoEncoderConfig& config, | 75 void ViEEncoder::SetEncoder(const webrtc::VideoCodec& video_codec, |
310 size_t max_data_payload_length) { | 76 size_t max_data_payload_length) { |
311 VideoCodec video_codec = VideoEncoderConfigToVideoCodec( | |
312 config, settings_.payload_name, settings_.payload_type); | |
313 encoder_queue_.PostTask([this, video_codec, max_data_payload_length] { | |
314 ConfigureEncoderInternal(video_codec, max_data_payload_length); | |
315 }); | |
316 return; | |
317 } | |
318 | |
319 void ViEEncoder::ConfigureEncoderInternal(const VideoCodec& video_codec, | |
320 size_t max_data_payload_length) { | |
321 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
322 RTC_DCHECK_GE(encoder_start_bitrate_bps_, 0); | |
323 RTC_DCHECK(sink_); | |
324 | |
325 // Setting target width and height for VPM. | 77 // Setting target width and height for VPM. |
326 RTC_CHECK_EQ(VPM_OK, | 78 RTC_CHECK_EQ(VPM_OK, |
327 vp_->SetTargetResolution(video_codec.width, video_codec.height, | 79 vp_->SetTargetResolution(video_codec.width, video_codec.height, |
328 video_codec.maxFramerate)); | 80 video_codec.maxFramerate)); |
329 | 81 { |
330 encoder_config_ = video_codec; | 82 rtc::CritScope lock(&data_cs_); |
331 encoder_config_.startBitrate = encoder_start_bitrate_bps_ / 1000; | 83 encoder_config_ = video_codec; |
332 encoder_config_.startBitrate = | 84 } |
333 std::max(encoder_config_.startBitrate, video_codec.minBitrate); | |
334 encoder_config_.startBitrate = | |
335 std::min(encoder_config_.startBitrate, video_codec.maxBitrate); | |
336 | 85 |
337 bool success = video_sender_.RegisterSendCodec( | 86 bool success = video_sender_.RegisterSendCodec( |
338 &encoder_config_, number_of_cores_, | 87 &video_codec, number_of_cores_, |
339 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; | 88 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
340 | 89 |
341 if (!success) { | 90 if (!success) { |
342 LOG(LS_ERROR) << "Failed to configure encoder."; | 91 LOG(LS_ERROR) << "Failed to configure encoder."; |
343 RTC_DCHECK(success); | 92 RTC_DCHECK(success); |
344 } | 93 } |
345 | 94 |
346 if (stats_proxy_) { | 95 if (stats_proxy_) { |
347 VideoEncoderConfig::ContentType content_type = | 96 VideoEncoderConfig::ContentType content_type = |
348 VideoEncoderConfig::ContentType::kRealtimeVideo; | 97 VideoEncoderConfig::ContentType::kRealtimeVideo; |
349 switch (video_codec.mode) { | 98 switch (video_codec.mode) { |
350 case kRealtimeVideo: | 99 case kRealtimeVideo: |
351 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; | 100 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; |
352 break; | 101 break; |
353 case kScreensharing: | 102 case kScreensharing: |
354 content_type = VideoEncoderConfig::ContentType::kScreen; | 103 content_type = VideoEncoderConfig::ContentType::kScreen; |
355 break; | 104 break; |
356 default: | 105 default: |
357 RTC_NOTREACHED(); | 106 RTC_NOTREACHED(); |
358 break; | 107 break; |
359 } | 108 } |
360 stats_proxy_->SetContentType(content_type); | 109 stats_proxy_->SetContentType(content_type); |
361 } | 110 } |
362 } | 111 } |
363 | 112 |
364 void ViEEncoder::IncomingCapturedFrame(const VideoFrame& video_frame) { | |
365 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | |
366 stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height()); | |
367 | |
368 VideoFrame incoming_frame = video_frame; | |
369 | |
370 // Local time in webrtc time base. | |
371 int64_t current_time = clock_->TimeInMilliseconds(); | |
372 incoming_frame.set_render_time_ms(current_time); | |
373 | |
374 // Capture time may come from clock with an offset and drift from clock_. | |
375 int64_t capture_ntp_time_ms; | |
376 if (video_frame.ntp_time_ms() != 0) { | |
377 capture_ntp_time_ms = video_frame.ntp_time_ms(); | |
378 } else if (video_frame.render_time_ms() != 0) { | |
379 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; | |
380 } else { | |
381 capture_ntp_time_ms = current_time + delta_ntp_internal_ms_; | |
382 } | |
383 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); | |
384 | |
385 // Convert NTP time, in ms, to RTP timestamp. | |
386 const int kMsToRtpTimestamp = 90; | |
387 incoming_frame.set_timestamp( | |
388 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms())); | |
389 | |
390 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { | |
391 // We don't allow the same capture time for two frames, drop this one. | |
392 LOG(LS_WARNING) << "Same/old NTP timestamp (" | |
393 << incoming_frame.ntp_time_ms() | |
394 << " <= " << last_captured_timestamp_ | |
395 << ") for incoming frame. Dropping."; | |
396 return; | |
397 } | |
398 | |
399 last_captured_timestamp_ = incoming_frame.ntp_time_ms(); | |
400 overuse_detector_.FrameCaptured(incoming_frame); | |
401 encoder_queue_.PostTask( | |
402 std::unique_ptr<rtc::QueuedTask>(new EncodeTask(incoming_frame, this))); | |
403 } | |
404 | |
405 bool ViEEncoder::EncoderPaused() const { | 113 bool ViEEncoder::EncoderPaused() const { |
406 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
407 // Pause video if paused by caller or as long as the network is down or the | 114 // Pause video if paused by caller or as long as the network is down or the |
408 // pacer queue has grown too large in buffered mode. | 115 // pacer queue has grown too large in buffered mode. |
409 // If the pacer queue has grown too large or the network is down, | 116 // If the pacer queue has grown too large or the network is down, |
410 // last_observed_bitrate_bps_ will be 0. | 117 // last_observed_bitrate_bps_ will be 0. |
411 return last_observed_bitrate_bps_ == 0; | 118 return video_suspended_ || last_observed_bitrate_bps_ == 0; |
412 } | 119 } |
413 | 120 |
414 void ViEEncoder::TraceFrameDropStart() { | 121 void ViEEncoder::TraceFrameDropStart() { |
415 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
416 // Start trace event only on the first frame after encoder is paused. | 122 // Start trace event only on the first frame after encoder is paused. |
417 if (!encoder_paused_and_dropped_frame_) { | 123 if (!encoder_paused_and_dropped_frame_) { |
418 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); | 124 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); |
419 } | 125 } |
420 encoder_paused_and_dropped_frame_ = true; | 126 encoder_paused_and_dropped_frame_ = true; |
421 return; | 127 return; |
422 } | 128 } |
423 | 129 |
424 void ViEEncoder::TraceFrameDropEnd() { | 130 void ViEEncoder::TraceFrameDropEnd() { |
425 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
426 // End trace event on first frame after encoder resumes, if frame was dropped. | 131 // End trace event on first frame after encoder resumes, if frame was dropped. |
427 if (encoder_paused_and_dropped_frame_) { | 132 if (encoder_paused_and_dropped_frame_) { |
428 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); | 133 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
429 } | 134 } |
430 encoder_paused_and_dropped_frame_ = false; | 135 encoder_paused_and_dropped_frame_ = false; |
431 } | 136 } |
432 | 137 |
433 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { | 138 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) { |
434 RTC_DCHECK_RUN_ON(&encoder_queue_); | 139 VideoCodecType codec_type; |
435 if (pre_encode_callback_) | 140 { |
436 pre_encode_callback_->OnFrame(video_frame); | 141 rtc::CritScope lock(&data_cs_); |
437 | 142 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
438 if (EncoderPaused()) { | 143 if (EncoderPaused()) { |
439 TraceFrameDropStart(); | 144 TraceFrameDropStart(); |
440 return; | 145 return; |
| 146 } |
| 147 TraceFrameDropEnd(); |
| 148 codec_type = encoder_config_.codecType; |
441 } | 149 } |
442 TraceFrameDropEnd(); | |
443 | 150 |
444 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), | 151 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
445 "Encode"); | 152 "Encode"); |
446 const VideoFrame* frame_to_send = &video_frame; | 153 const VideoFrame* frame_to_send = &video_frame; |
447 // TODO(wuchengli): support texture frames. | 154 // TODO(wuchengli): support texture frames. |
448 if (!video_frame.video_frame_buffer()->native_handle()) { | 155 if (!video_frame.video_frame_buffer()->native_handle()) { |
449 // Pass frame via preprocessor. | 156 // Pass frame via preprocessor. |
450 frame_to_send = vp_->PreprocessFrame(video_frame); | 157 frame_to_send = vp_->PreprocessFrame(video_frame); |
451 if (!frame_to_send) { | 158 if (!frame_to_send) { |
452 // Drop this frame, or there was an error processing it. | 159 // Drop this frame, or there was an error processing it. |
453 return; | 160 return; |
454 } | 161 } |
455 } | 162 } |
456 | 163 |
457 if (encoder_config_.codecType == webrtc::kVideoCodecVP8) { | 164 if (codec_type == webrtc::kVideoCodecVP8) { |
458 webrtc::CodecSpecificInfo codec_specific_info; | 165 webrtc::CodecSpecificInfo codec_specific_info; |
459 codec_specific_info.codecType = webrtc::kVideoCodecVP8; | 166 codec_specific_info.codecType = webrtc::kVideoCodecVP8; |
460 | 167 { |
| 168 rtc::CritScope lock(&data_cs_); |
461 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = | 169 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = |
462 has_received_rpsi_; | 170 has_received_rpsi_; |
463 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = | 171 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = |
464 has_received_sli_; | 172 has_received_sli_; |
465 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = | 173 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = |
466 picture_id_rpsi_; | 174 picture_id_rpsi_; |
467 codec_specific_info.codecSpecific.VP8.pictureIdSLI = | 175 codec_specific_info.codecSpecific.VP8.pictureIdSLI = |
468 picture_id_sli_; | 176 picture_id_sli_; |
469 has_received_sli_ = false; | 177 has_received_sli_ = false; |
470 has_received_rpsi_ = false; | 178 has_received_rpsi_ = false; |
| 179 } |
471 | 180 |
472 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); | 181 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); |
473 return; | 182 return; |
474 } | 183 } |
475 video_sender_.AddVideoFrame(*frame_to_send, nullptr); | 184 video_sender_.AddVideoFrame(*frame_to_send, nullptr); |
476 } | 185 } |
477 | 186 |
478 void ViEEncoder::SendKeyFrame() { | 187 void ViEEncoder::SendKeyFrame() { |
479 if (!encoder_queue_.IsCurrent()) { | |
480 encoder_queue_.PostTask([this] { SendKeyFrame(); }); | |
481 return; | |
482 } | |
483 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
484 video_sender_.IntraFrameRequest(0); | 188 video_sender_.IntraFrameRequest(0); |
485 } | 189 } |
486 | 190 |
| 191 int64_t ViEEncoder::time_of_last_frame_activity_ms() { |
| 192 rtc::CritScope lock(&data_cs_); |
| 193 return time_of_last_frame_activity_ms_; |
| 194 } |
| 195 |
487 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( | 196 EncodedImageCallback::Result ViEEncoder::OnEncodedImage( |
488 const EncodedImage& encoded_image, | 197 const EncodedImage& encoded_image, |
489 const CodecSpecificInfo* codec_specific_info, | 198 const CodecSpecificInfo* codec_specific_info, |
490 const RTPFragmentationHeader* fragmentation) { | 199 const RTPFragmentationHeader* fragmentation) { |
491 // Encoded is called on whatever thread the real encoder implementation run | 200 { |
492 // on. In the case of hardware encoders, there might be several encoders | 201 rtc::CritScope lock(&data_cs_); |
493 // running in parallel on different threads. | 202 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| 203 } |
494 if (stats_proxy_) { | 204 if (stats_proxy_) { |
495 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 205 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
496 } | 206 } |
497 | 207 |
498 EncodedImageCallback::Result result = | 208 EncodedImageCallback::Result result = |
499 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); | 209 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); |
500 | 210 |
501 overuse_detector_.FrameSent(encoded_image._timeStamp); | 211 overuse_detector_->FrameSent(encoded_image._timeStamp); |
502 return result; | 212 return result; |
503 } | 213 } |
504 | 214 |
505 void ViEEncoder::SendStatistics(uint32_t bit_rate, | 215 void ViEEncoder::SendStatistics(uint32_t bit_rate, |
506 uint32_t frame_rate, | 216 uint32_t frame_rate, |
507 const std::string& encoder_name) { | 217 const std::string& encoder_name) { |
508 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread()); | |
509 if (stats_proxy_) | 218 if (stats_proxy_) |
510 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); | 219 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); |
511 } | 220 } |
512 | 221 |
513 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { | 222 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
514 if (!encoder_queue_.IsCurrent()) { | 223 rtc::CritScope lock(&data_cs_); |
515 encoder_queue_.PostTask([this, picture_id] { OnReceivedSLI(picture_id); }); | |
516 return; | |
517 } | |
518 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
519 picture_id_sli_ = picture_id; | 224 picture_id_sli_ = picture_id; |
520 has_received_sli_ = true; | 225 has_received_sli_ = true; |
521 } | 226 } |
522 | 227 |
523 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { | 228 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { |
524 if (!encoder_queue_.IsCurrent()) { | 229 rtc::CritScope lock(&data_cs_); |
525 encoder_queue_.PostTask([this, picture_id] { OnReceivedRPSI(picture_id); }); | |
526 return; | |
527 } | |
528 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
529 picture_id_rpsi_ = picture_id; | 230 picture_id_rpsi_ = picture_id; |
530 has_received_rpsi_ = true; | 231 has_received_rpsi_ = true; |
531 } | 232 } |
532 | 233 |
533 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { | 234 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { |
534 if (!encoder_queue_.IsCurrent()) { | |
535 encoder_queue_.PostTask( | |
536 [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); }); | |
537 return; | |
538 } | |
539 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
540 // Key frame request from remote side, signal to VCM. | 235 // Key frame request from remote side, signal to VCM. |
541 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | 236 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
542 video_sender_.IntraFrameRequest(stream_index); | 237 video_sender_.IntraFrameRequest(stream_index); |
543 } | 238 } |
544 | 239 |
545 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | 240 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, |
546 uint8_t fraction_lost, | 241 uint8_t fraction_lost, |
547 int64_t round_trip_time_ms) { | 242 int64_t round_trip_time_ms) { |
548 if (!encoder_queue_.IsCurrent()) { | |
549 encoder_queue_.PostTask( | |
550 [this, bitrate_bps, fraction_lost, round_trip_time_ms] { | |
551 OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms); | |
552 }); | |
553 return; | |
554 } | |
555 RTC_DCHECK_RUN_ON(&encoder_queue_); | |
556 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; | |
557 | |
558 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps | 243 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps |
559 << " packet loss " << static_cast<int>(fraction_lost) | 244 << " packet loss " << static_cast<int>(fraction_lost) |
560 << " rtt " << round_trip_time_ms; | 245 << " rtt " << round_trip_time_ms; |
561 | |
562 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | 246 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, |
563 round_trip_time_ms); | 247 round_trip_time_ms); |
564 | 248 bool video_suspension_changed; |
565 encoder_start_bitrate_bps_ = | |
566 bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; | |
567 bool video_is_suspended = bitrate_bps == 0; | 249 bool video_is_suspended = bitrate_bps == 0; |
568 bool video_suspension_changed = | 250 { |
569 video_is_suspended != (last_observed_bitrate_bps_ == 0); | 251 rtc::CritScope lock(&data_cs_); |
570 last_observed_bitrate_bps_ = bitrate_bps; | 252 last_observed_bitrate_bps_ = bitrate_bps; |
| 253 video_suspension_changed = video_suspended_ != video_is_suspended; |
| 254 video_suspended_ = video_is_suspended; |
| 255 // Set |time_of_last_frame_activity_ms_| to now if this is the first time |
| 256 // the encoder is supposed to produce encoded frames. |
| 257 // TODO(perkj): Remove this hack. It is here to avoid a race that the |
| 258 // encoder report that it has timed out before it has processed the first |
| 259 // frame. |
| 260 if (last_observed_bitrate_bps_ != 0 && |
| 261 time_of_last_frame_activity_ms_ == |
| 262 std::numeric_limits<int64_t>::max()) { |
| 263 time_of_last_frame_activity_ms_ = rtc::TimeMillis(); |
| 264 } |
| 265 } |
571 | 266 |
572 if (stats_proxy_ && video_suspension_changed) { | 267 if (stats_proxy_ && video_suspension_changed) { |
573 LOG(LS_INFO) << "Video suspend state changed to: " | 268 LOG(LS_INFO) << "Video suspend state changed to: " |
574 << (video_is_suspended ? "suspended" : "not suspended"); | 269 << (video_is_suspended ? "suspended" : "not suspended"); |
575 stats_proxy_->OnSuspendChange(video_is_suspended); | 270 stats_proxy_->OnSuspendChange(video_is_suspended); |
576 } | 271 } |
577 } | 272 } |
578 | 273 |
579 void ViEEncoder::OveruseDetected() { | |
580 RTC_DCHECK_RUN_ON(&module_process_thread_checker_); | |
581 // TODO(perkj): When ViEEncoder inherit rtc::VideoSink instead of | |
582 // VideoCaptureInput |load_observer_| should be removed and overuse be | |
583 // expressed as rtc::VideoSinkWants instead. | |
584 if (load_observer_) | |
585 load_observer_->OnLoadUpdate(LoadObserver::kOveruse); | |
586 } | |
587 | |
588 void ViEEncoder::NormalUsage() { | |
589 RTC_DCHECK_RUN_ON(&module_process_thread_checker_); | |
590 if (load_observer_) | |
591 load_observer_->OnLoadUpdate(LoadObserver::kUnderuse); | |
592 } | |
593 | |
594 } // namespace webrtc | 274 } // namespace webrtc |
OLD | NEW |