OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
12 | 12 |
13 #include <assert.h> | 13 #include <assert.h> |
14 | 14 |
15 #include <algorithm> | 15 #include <algorithm> |
16 | 16 |
17 #include "webrtc/base/checks.h" | 17 #include "webrtc/base/checks.h" |
18 #include "webrtc/base/logging.h" | 18 #include "webrtc/base/logging.h" |
19 #include "webrtc/base/trace_event.h" | 19 #include "webrtc/base/trace_event.h" |
20 #include "webrtc/common_video/include/frame_callback.h" | |
21 #include "webrtc/common_video/include/video_image.h" | |
22 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" | |
23 #include "webrtc/modules/pacing/paced_sender.h" | 20 #include "webrtc/modules/pacing/paced_sender.h" |
24 #include "webrtc/modules/utility/include/process_thread.h" | |
25 #include "webrtc/modules/video_coding/include/video_codec_interface.h" | |
26 #include "webrtc/modules/video_coding/include/video_coding.h" | 21 #include "webrtc/modules/video_coding/include/video_coding.h" |
27 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
28 #include "webrtc/system_wrappers/include/clock.h" | |
29 #include "webrtc/system_wrappers/include/metrics.h" | 23 #include "webrtc/system_wrappers/include/metrics.h" |
30 #include "webrtc/system_wrappers/include/tick_util.h" | 24 #include "webrtc/system_wrappers/include/tick_util.h" |
31 #include "webrtc/video/overuse_frame_detector.h" | 25 #include "webrtc/video/overuse_frame_detector.h" |
32 #include "webrtc/video/payload_router.h" | |
33 #include "webrtc/video/send_statistics_proxy.h" | 26 #include "webrtc/video/send_statistics_proxy.h" |
34 #include "webrtc/video_frame.h" | 27 #include "webrtc/video_frame.h" |
35 | 28 |
36 namespace webrtc { | 29 namespace webrtc { |
37 | 30 |
38 static const float kStopPaddingThresholdMs = 2000; | 31 static const float kStopPaddingThresholdMs = 2000; |
39 static const int kMinKeyFrameRequestIntervalMs = 300; | |
40 | 32 |
41 class QMVideoSettingsCallback : public VCMQMSettingsCallback { | 33 class QMVideoSettingsCallback : public VCMQMSettingsCallback { |
42 public: | 34 public: |
43 explicit QMVideoSettingsCallback(VideoProcessing* vpm); | 35 explicit QMVideoSettingsCallback(VideoProcessing* vpm); |
44 | 36 |
45 ~QMVideoSettingsCallback(); | 37 ~QMVideoSettingsCallback(); |
46 | 38 |
47 // Update VPM with QM (quality modes: frame size & frame rate) settings. | 39 // Update VPM with QM (quality modes: frame size & frame rate) settings. |
48 int32_t SetVideoQMSettings(const uint32_t frame_rate, | 40 int32_t SetVideoQMSettings(const uint32_t frame_rate, |
49 const uint32_t width, | 41 const uint32_t width, |
50 const uint32_t height); | 42 const uint32_t height); |
51 | 43 |
52 private: | 44 private: |
53 VideoProcessing* vp_; | 45 VideoProcessing* vp_; |
54 }; | 46 }; |
55 | 47 |
56 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | 48 ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
57 const std::vector<uint32_t>& ssrcs, | |
58 ProcessThread* module_process_thread, | 49 ProcessThread* module_process_thread, |
59 SendStatisticsProxy* stats_proxy, | 50 SendStatisticsProxy* stats_proxy, |
60 OveruseFrameDetector* overuse_detector) | 51 OveruseFrameDetector* overuse_detector) |
61 : number_of_cores_(number_of_cores), | 52 : number_of_cores_(number_of_cores), |
62 ssrcs_(ssrcs), | |
63 vp_(VideoProcessing::Create()), | 53 vp_(VideoProcessing::Create()), |
64 qm_callback_(new QMVideoSettingsCallback(vp_.get())), | 54 qm_callback_(new QMVideoSettingsCallback(vp_.get())), |
65 video_sender_(Clock::GetRealTimeClock(), | 55 video_sender_(Clock::GetRealTimeClock(), |
66 this, | 56 this, |
67 this, | 57 this, |
68 qm_callback_.get(), | 58 qm_callback_.get(), |
69 this), | 59 this), |
70 stats_proxy_(stats_proxy), | 60 stats_proxy_(stats_proxy), |
71 overuse_detector_(overuse_detector), | 61 overuse_detector_(overuse_detector), |
72 time_of_last_frame_activity_ms_(0), | 62 time_of_last_frame_activity_ms_(0), |
73 encoder_config_(), | 63 encoder_config_(), |
74 min_transmit_bitrate_bps_(0), | 64 min_transmit_bitrate_bps_(0), |
75 last_observed_bitrate_bps_(0), | 65 last_observed_bitrate_bps_(0), |
76 network_is_transmitting_(true), | 66 network_is_transmitting_(true), |
77 encoder_paused_(true), | 67 encoder_paused_(true), |
78 encoder_paused_and_dropped_frame_(false), | 68 encoder_paused_and_dropped_frame_(false), |
79 time_last_intra_request_ms_(ssrcs.size(), -1), | |
80 module_process_thread_(module_process_thread), | 69 module_process_thread_(module_process_thread), |
81 has_received_sli_(false), | 70 has_received_sli_(false), |
82 picture_id_sli_(0), | 71 picture_id_sli_(0), |
83 has_received_rpsi_(false), | 72 has_received_rpsi_(false), |
84 picture_id_rpsi_(0), | 73 picture_id_rpsi_(0), |
85 video_suspended_(false) { | 74 video_suspended_(false) { |
86 module_process_thread_->RegisterModule(&video_sender_); | 75 module_process_thread_->RegisterModule(&video_sender_); |
87 vp_->EnableTemporalDecimation(true); | 76 vp_->EnableTemporalDecimation(true); |
88 | 77 |
89 // Enable/disable content analysis: off by default for now. | 78 // Enable/disable content analysis: off by default for now. |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
149 | 138 |
150 bool success = video_sender_.RegisterSendCodec( | 139 bool success = video_sender_.RegisterSendCodec( |
151 &video_codec, number_of_cores_, | 140 &video_codec, number_of_cores_, |
152 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; | 141 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
153 if (!success) { | 142 if (!success) { |
154 LOG(LS_ERROR) << "Failed to configure encoder."; | 143 LOG(LS_ERROR) << "Failed to configure encoder."; |
155 RTC_DCHECK(success); | 144 RTC_DCHECK(success); |
156 } | 145 } |
157 | 146 |
158 if (stats_proxy_) { | 147 if (stats_proxy_) { |
159 // Clear stats for disabled layers. | |
160 for (size_t i = video_codec.numberOfSimulcastStreams; i < ssrcs_.size(); | |
161 ++i) { | |
162 stats_proxy_->OnInactiveSsrc(ssrcs_[i]); | |
163 } | |
164 VideoEncoderConfig::ContentType content_type = | 148 VideoEncoderConfig::ContentType content_type = |
165 VideoEncoderConfig::ContentType::kRealtimeVideo; | 149 VideoEncoderConfig::ContentType::kRealtimeVideo; |
166 switch (video_codec.mode) { | 150 switch (video_codec.mode) { |
167 case kRealtimeVideo: | 151 case kRealtimeVideo: |
168 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; | 152 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; |
169 break; | 153 break; |
170 case kScreensharing: | 154 case kScreensharing: |
171 content_type = VideoEncoderConfig::ContentType::kScreen; | 155 content_type = VideoEncoderConfig::ContentType::kScreen; |
172 break; | 156 break; |
173 default: | 157 default: |
(...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
346 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 330 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
347 } | 331 } |
348 | 332 |
349 int success = 0; | 333 int success = 0; |
350 { | 334 { |
351 rtc::CritScope lock(&sink_cs_); | 335 rtc::CritScope lock(&sink_cs_); |
352 success = sink_->Encoded(encoded_image, codec_specific_info, fragmentation); | 336 success = sink_->Encoded(encoded_image, codec_specific_info, fragmentation); |
353 } | 337 } |
354 | 338 |
355 overuse_detector_->FrameSent(encoded_image._timeStamp); | 339 overuse_detector_->FrameSent(encoded_image._timeStamp); |
356 if (kEnableFrameRecording) { | |
357 int layer = codec_specific_info->codecType == kVideoCodecVP8 | |
358 ? codec_specific_info->codecSpecific.VP8.simulcastIdx | |
359 : 0; | |
360 IvfFileWriter* file_writer; | |
361 { | |
362 rtc::CritScope lock(&data_cs_); | |
363 if (file_writers_[layer] == nullptr) { | |
364 std::ostringstream oss; | |
365 oss << "send_bitstream_ssrc"; | |
366 for (uint32_t ssrc : ssrcs_) | |
367 oss << "_" << ssrc; | |
368 oss << "_layer" << layer << ".ivf"; | |
369 file_writers_[layer] = | |
370 IvfFileWriter::Open(oss.str(), codec_specific_info->codecType); | |
371 } | |
372 file_writer = file_writers_[layer].get(); | |
373 } | |
374 if (file_writer) { | |
375 bool ok = file_writer->WriteFrame(encoded_image); | |
376 RTC_DCHECK(ok); | |
377 } | |
378 } | |
379 | |
380 return success; | 340 return success; |
381 } | 341 } |
382 | 342 |
383 void ViEEncoder::SendStatistics(uint32_t bit_rate, | 343 void ViEEncoder::SendStatistics(uint32_t bit_rate, |
384 uint32_t frame_rate, | 344 uint32_t frame_rate, |
385 const std::string& encoder_name) { | 345 const std::string& encoder_name) { |
386 if (stats_proxy_) | 346 if (stats_proxy_) |
387 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); | 347 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate, encoder_name); |
388 } | 348 } |
389 | 349 |
390 void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/, | 350 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
391 uint8_t picture_id) { | |
392 rtc::CritScope lock(&data_cs_); | 351 rtc::CritScope lock(&data_cs_); |
393 picture_id_sli_ = picture_id; | 352 picture_id_sli_ = picture_id; |
394 has_received_sli_ = true; | 353 has_received_sli_ = true; |
395 } | 354 } |
396 | 355 |
397 void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/, | 356 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { |
398 uint64_t picture_id) { | |
399 rtc::CritScope lock(&data_cs_); | 357 rtc::CritScope lock(&data_cs_); |
400 picture_id_rpsi_ = picture_id; | 358 picture_id_rpsi_ = picture_id; |
401 has_received_rpsi_ = true; | 359 has_received_rpsi_ = true; |
402 } | 360 } |
403 | 361 |
404 void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) { | 362 void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { |
405 // Key frame request from remote side, signal to VCM. | 363 // Key frame request from remote side, signal to VCM. |
406 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | 364 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
407 | 365 video_sender_.IntraFrameRequest(stream_index); |
408 for (size_t i = 0; i < ssrcs_.size(); ++i) { | |
409 if (ssrcs_[i] != ssrc) | |
410 continue; | |
411 int64_t now_ms = TickTime::MillisecondTimestamp(); | |
412 { | |
413 rtc::CritScope lock(&data_cs_); | |
414 if (time_last_intra_request_ms_[i] + kMinKeyFrameRequestIntervalMs > | |
415 now_ms) { | |
416 return; | |
417 } | |
418 time_last_intra_request_ms_[i] = now_ms; | |
419 } | |
420 video_sender_.IntraFrameRequest(static_cast<int>(i)); | |
421 return; | |
422 } | |
423 RTC_NOTREACHED() << "Should not receive keyframe requests on unknown SSRCs."; | |
424 } | 366 } |
425 | 367 |
426 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | 368 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, |
427 uint8_t fraction_lost, | 369 uint8_t fraction_lost, |
428 int64_t round_trip_time_ms) { | 370 int64_t round_trip_time_ms) { |
429 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps | 371 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps |
430 << " packet loss " << static_cast<int>(fraction_lost) | 372 << " packet loss " << static_cast<int>(fraction_lost) |
431 << " rtt " << round_trip_time_ms; | 373 << " rtt " << round_trip_time_ms; |
432 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | 374 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, |
433 round_trip_time_ms); | 375 round_trip_time_ms); |
434 bool video_is_suspended = video_sender_.VideoSuspended(); | 376 bool video_is_suspended = video_sender_.VideoSuspended(); |
435 bool video_suspension_changed; | 377 bool video_suspension_changed; |
436 { | 378 { |
437 rtc::CritScope lock(&data_cs_); | 379 rtc::CritScope lock(&data_cs_); |
438 last_observed_bitrate_bps_ = bitrate_bps; | 380 last_observed_bitrate_bps_ = bitrate_bps; |
439 video_suspension_changed = video_suspended_ != video_is_suspended; | 381 video_suspension_changed = video_suspended_ != video_is_suspended; |
440 video_suspended_ = video_is_suspended; | 382 video_suspended_ = video_is_suspended; |
441 } | 383 } |
442 | 384 |
443 if (!video_suspension_changed) | 385 if (!video_suspension_changed) |
444 return; | 386 return; |
445 // Video suspend-state changed, inform codec observer. | 387 // Video suspend-state changed, inform codec observer. |
446 LOG(LS_INFO) << "Video suspend state changed " << video_is_suspended | 388 LOG(LS_INFO) << "Video suspend state changed " << video_is_suspended; |
447 << " for ssrc " << ssrcs_[0]; | 389 |
448 if (stats_proxy_) | 390 if (stats_proxy_) |
449 stats_proxy_->OnSuspendChange(video_is_suspended); | 391 stats_proxy_->OnSuspendChange(video_is_suspended); |
450 } | 392 } |
451 | 393 |
452 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessing* vpm) | 394 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessing* vpm) |
453 : vp_(vpm) { | 395 : vp_(vpm) { |
454 } | 396 } |
455 | 397 |
456 QMVideoSettingsCallback::~QMVideoSettingsCallback() { | 398 QMVideoSettingsCallback::~QMVideoSettingsCallback() { |
457 } | 399 } |
458 | 400 |
459 int32_t QMVideoSettingsCallback::SetVideoQMSettings( | 401 int32_t QMVideoSettingsCallback::SetVideoQMSettings( |
460 const uint32_t frame_rate, | 402 const uint32_t frame_rate, |
461 const uint32_t width, | 403 const uint32_t width, |
462 const uint32_t height) { | 404 const uint32_t height) { |
463 return vp_->SetTargetResolution(width, height, frame_rate); | 405 return vp_->SetTargetResolution(width, height, frame_rate); |
464 } | 406 } |
465 | 407 |
466 } // namespace webrtc | 408 } // namespace webrtc |
OLD | NEW |