Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/video/vie_encoder.h" | 11 #include "webrtc/video/vie_encoder.h" |
| 12 | 12 |
| 13 #include <assert.h> | 13 #include <assert.h> |
| 14 | 14 |
| 15 #include <algorithm> | 15 #include <algorithm> |
| 16 | 16 |
| 17 #include "webrtc/base/checks.h" | 17 #include "webrtc/base/checks.h" |
| 18 #include "webrtc/base/logging.h" | 18 #include "webrtc/base/logging.h" |
| 19 #include "webrtc/base/trace_event.h" | 19 #include "webrtc/base/trace_event.h" |
| 20 #include "webrtc/common_video/include/frame_callback.h" | |
| 21 #include "webrtc/common_video/include/video_image.h" | |
| 22 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" | |
| 23 #include "webrtc/modules/pacing/paced_sender.h" | 20 #include "webrtc/modules/pacing/paced_sender.h" |
| 24 #include "webrtc/modules/utility/include/process_thread.h" | |
| 25 #include "webrtc/modules/video_coding/include/video_codec_interface.h" | |
| 26 #include "webrtc/modules/video_coding/include/video_coding.h" | 21 #include "webrtc/modules/video_coding/include/video_coding.h" |
| 27 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 22 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| 28 #include "webrtc/system_wrappers/include/clock.h" | |
| 29 #include "webrtc/system_wrappers/include/metrics.h" | 23 #include "webrtc/system_wrappers/include/metrics.h" |
| 30 #include "webrtc/system_wrappers/include/tick_util.h" | 24 #include "webrtc/system_wrappers/include/tick_util.h" |
| 31 #include "webrtc/video/overuse_frame_detector.h" | 25 #include "webrtc/video/overuse_frame_detector.h" |
| 32 #include "webrtc/video/payload_router.h" | |
| 33 #include "webrtc/video/send_statistics_proxy.h" | 26 #include "webrtc/video/send_statistics_proxy.h" |
| 34 #include "webrtc/video_frame.h" | 27 #include "webrtc/video_frame.h" |
| 35 | 28 |
| 36 namespace webrtc { | 29 namespace webrtc { |
| 37 | 30 |
| 38 static const float kStopPaddingThresholdMs = 2000; | 31 static const float kStopPaddingThresholdMs = 2000; |
| 39 static const int kMinKeyFrameRequestIntervalMs = 300; | |
| 40 | 32 |
| 41 class QMVideoSettingsCallback : public VCMQMSettingsCallback { | 33 class QMVideoSettingsCallback : public VCMQMSettingsCallback { |
| 42 public: | 34 public: |
| 43 explicit QMVideoSettingsCallback(VideoProcessing* vpm); | 35 explicit QMVideoSettingsCallback(VideoProcessing* vpm); |
| 44 | 36 |
| 45 ~QMVideoSettingsCallback(); | 37 ~QMVideoSettingsCallback(); |
| 46 | 38 |
| 47 // Update VPM with QM (quality modes: frame size & frame rate) settings. | 39 // Update VPM with QM (quality modes: frame size & frame rate) settings. |
| 48 int32_t SetVideoQMSettings(const uint32_t frame_rate, | 40 int32_t SetVideoQMSettings(const uint32_t frame_rate, |
| 49 const uint32_t width, | 41 const uint32_t width, |
| 50 const uint32_t height); | 42 const uint32_t height); |
| 51 | 43 |
| 52 private: | 44 private: |
| 53 VideoProcessing* vp_; | 45 VideoProcessing* vp_; |
| 54 }; | 46 }; |
| 55 | 47 |
| 56 ViEEncoder::ViEEncoder(uint32_t number_of_cores, | 48 ViEEncoder::ViEEncoder(uint32_t number_of_cores, |
| 57 const std::vector<uint32_t>& ssrcs, | |
| 58 ProcessThread* module_process_thread, | 49 ProcessThread* module_process_thread, |
| 59 SendStatisticsProxy* stats_proxy, | 50 SendStatisticsProxy* stats_proxy, |
| 60 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | 51 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, |
| 61 OveruseFrameDetector* overuse_detector, | 52 OveruseFrameDetector* overuse_detector, |
| 62 PacedSender* pacer) | 53 PacedSender* pacer) |
| 63 : number_of_cores_(number_of_cores), | 54 : number_of_cores_(number_of_cores), |
| 64 ssrcs_(ssrcs), | |
| 65 vp_(VideoProcessing::Create()), | 55 vp_(VideoProcessing::Create()), |
| 66 qm_callback_(new QMVideoSettingsCallback(vp_.get())), | 56 qm_callback_(new QMVideoSettingsCallback(vp_.get())), |
| 67 video_sender_(Clock::GetRealTimeClock(), this, this, qm_callback_.get()), | 57 video_sender_(Clock::GetRealTimeClock(), this, this, qm_callback_.get()), |
| 68 stats_proxy_(stats_proxy), | 58 stats_proxy_(stats_proxy), |
| 69 pre_encode_callback_(pre_encode_callback), | 59 pre_encode_callback_(pre_encode_callback), |
| 70 overuse_detector_(overuse_detector), | 60 overuse_detector_(overuse_detector), |
| 71 pacer_(pacer), | 61 pacer_(pacer), |
| 72 time_of_last_frame_activity_ms_(0), | 62 time_of_last_frame_activity_ms_(0), |
| 73 encoder_config_(), | 63 encoder_config_(), |
| 74 min_transmit_bitrate_bps_(0), | 64 min_transmit_bitrate_bps_(0), |
| 75 last_observed_bitrate_bps_(0), | 65 last_observed_bitrate_bps_(0), |
| 76 network_is_transmitting_(true), | 66 network_is_transmitting_(true), |
| 77 encoder_paused_(true), | 67 encoder_paused_(true), |
| 78 encoder_paused_and_dropped_frame_(false), | 68 encoder_paused_and_dropped_frame_(false), |
| 79 time_last_intra_request_ms_(ssrcs.size(), -1), | |
| 80 module_process_thread_(module_process_thread), | 69 module_process_thread_(module_process_thread), |
| 81 has_received_sli_(false), | 70 has_received_sli_(false), |
| 82 picture_id_sli_(0), | 71 picture_id_sli_(0), |
| 83 has_received_rpsi_(false), | 72 has_received_rpsi_(false), |
| 84 picture_id_rpsi_(0), | 73 picture_id_rpsi_(0), |
| 85 video_suspended_(false) { | 74 video_suspended_(false) { |
| 86 module_process_thread_->RegisterModule(&video_sender_); | 75 module_process_thread_->RegisterModule(&video_sender_); |
| 87 } | 76 } |
| 88 | 77 |
| 89 bool ViEEncoder::Init() { | 78 bool ViEEncoder::Init() { |
| (...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 162 | 151 |
| 163 bool success = video_sender_.RegisterSendCodec( | 152 bool success = video_sender_.RegisterSendCodec( |
| 164 &video_codec, number_of_cores_, | 153 &video_codec, number_of_cores_, |
| 165 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; | 154 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
| 166 if (!success) { | 155 if (!success) { |
| 167 LOG(LS_ERROR) << "Failed to configure encoder."; | 156 LOG(LS_ERROR) << "Failed to configure encoder."; |
| 168 RTC_DCHECK(success); | 157 RTC_DCHECK(success); |
| 169 } | 158 } |
| 170 | 159 |
| 171 if (stats_proxy_) { | 160 if (stats_proxy_) { |
| 172 // Clear stats for disabled layers. | |
| 173 for (size_t i = video_codec.numberOfSimulcastStreams; i < ssrcs_.size(); | |
| 174 ++i) { | |
| 175 stats_proxy_->OnInactiveSsrc(ssrcs_[i]); | |
| 176 } | |
| 177 VideoEncoderConfig::ContentType content_type = | 161 VideoEncoderConfig::ContentType content_type = |
| 178 VideoEncoderConfig::ContentType::kRealtimeVideo; | 162 VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 179 switch (video_codec.mode) { | 163 switch (video_codec.mode) { |
| 180 case kRealtimeVideo: | 164 case kRealtimeVideo: |
| 181 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; | 165 content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; |
| 182 break; | 166 break; |
| 183 case kScreensharing: | 167 case kScreensharing: |
| 184 content_type = VideoEncoderConfig::ContentType::kScreen; | 168 content_type = VideoEncoderConfig::ContentType::kScreen; |
| 185 break; | 169 break; |
| 186 default: | 170 default: |
| (...skipping 182 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 369 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); | 353 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); |
| 370 } | 354 } |
| 371 | 355 |
| 372 int success = 0; | 356 int success = 0; |
| 373 { | 357 { |
| 374 rtc::CritScope lock(&sink_cs_); | 358 rtc::CritScope lock(&sink_cs_); |
| 375 success = sink_->Encoded(encoded_image, codec_specific_info, fragmentation); | 359 success = sink_->Encoded(encoded_image, codec_specific_info, fragmentation); |
| 376 } | 360 } |
| 377 | 361 |
| 378 overuse_detector_->FrameSent(encoded_image._timeStamp); | 362 overuse_detector_->FrameSent(encoded_image._timeStamp); |
| 379 if (kEnableFrameRecording) { | |
| 380 int layer = codec_specific_info->codecType == kVideoCodecVP8 | |
| 381 ? codec_specific_info->codecSpecific.VP8.simulcastIdx | |
| 382 : 0; | |
| 383 IvfFileWriter* file_writer; | |
| 384 { | |
| 385 rtc::CritScope lock(&data_cs_); | |
| 386 if (file_writers_[layer] == nullptr) { | |
| 387 std::ostringstream oss; | |
| 388 oss << "send_bitstream_ssrc"; | |
| 389 for (uint32_t ssrc : ssrcs_) | |
| 390 oss << "_" << ssrc; | |
| 391 oss << "_layer" << layer << ".ivf"; | |
| 392 file_writers_[layer] = | |
| 393 IvfFileWriter::Open(oss.str(), codec_specific_info->codecType); | |
| 394 } | |
| 395 file_writer = file_writers_[layer].get(); | |
| 396 } | |
| 397 if (file_writer) { | |
| 398 bool ok = file_writer->WriteFrame(encoded_image); | |
| 399 RTC_DCHECK(ok); | |
| 400 } | |
| 401 } | |
| 402 | |
| 403 return success; | 363 return success; |
| 404 } | 364 } |
| 405 | 365 |
| 406 int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate, | 366 int32_t ViEEncoder::SendStatistics(const uint32_t bit_rate, |
| 407 const uint32_t frame_rate) { | 367 const uint32_t frame_rate) { |
| 408 if (stats_proxy_) | 368 if (stats_proxy_) |
| 409 stats_proxy_->OnOutgoingRate(frame_rate, bit_rate); | 369 stats_proxy_->OnOutgoingRate(frame_rate, bit_rate); |
| 410 return 0; | 370 return 0; |
| 411 } | 371 } |
| 412 | 372 |
| 413 void ViEEncoder::OnReceivedSLI(uint32_t /*ssrc*/, | 373 void ViEEncoder::OnReceivedSLI(uint8_t picture_id) { |
| 414 uint8_t picture_id) { | |
| 415 rtc::CritScope lock(&data_cs_); | 374 rtc::CritScope lock(&data_cs_); |
| 416 picture_id_sli_ = picture_id; | 375 picture_id_sli_ = picture_id; |
| 417 has_received_sli_ = true; | 376 has_received_sli_ = true; |
| 418 } | 377 } |
| 419 | 378 |
| 420 void ViEEncoder::OnReceivedRPSI(uint32_t /*ssrc*/, | 379 void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) { |
| 421 uint64_t picture_id) { | |
| 422 rtc::CritScope lock(&data_cs_); | 380 rtc::CritScope lock(&data_cs_); |
| 423 picture_id_rpsi_ = picture_id; | 381 picture_id_rpsi_ = picture_id; |
| 424 has_received_rpsi_ = true; | 382 has_received_rpsi_ = true; |
| 425 } | 383 } |
| 426 | 384 |
| 427 void ViEEncoder::OnReceivedIntraFrameRequest(uint32_t ssrc) { | 385 void ViEEncoder::OnReceivedIntraFrameRequest(int stream_index) { |
| 428 // Key frame request from remote side, signal to VCM. | 386 // Key frame request from remote side, signal to VCM. |
| 429 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); | 387 TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
| 430 | 388 video_sender_.IntraFrameRequest(stream_index); |
| 431 for (size_t i = 0; i < ssrcs_.size(); ++i) { | 389 return; |
|
pbos-webrtc
2016/05/02 01:05:27
remove
| |
| 432 if (ssrcs_[i] != ssrc) | |
| 433 continue; | |
| 434 int64_t now_ms = TickTime::MillisecondTimestamp(); | |
| 435 { | |
| 436 rtc::CritScope lock(&data_cs_); | |
| 437 if (time_last_intra_request_ms_[i] + kMinKeyFrameRequestIntervalMs > | |
|
pbos-webrtc
2016/05/02 01:05:27
Is this logic handled somewhere else as well or is
perkj_webrtc
2016/05/03 10:22:40
no- unintentinonally actually. I misread what this
| |
| 438 now_ms) { | |
| 439 return; | |
| 440 } | |
| 441 time_last_intra_request_ms_[i] = now_ms; | |
| 442 } | |
| 443 video_sender_.IntraFrameRequest(static_cast<int>(i)); | |
| 444 return; | |
| 445 } | |
| 446 RTC_NOTREACHED() << "Should not receive keyframe requests on unknown SSRCs."; | |
| 447 } | 390 } |
| 448 | 391 |
| 449 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, | 392 void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, |
| 450 uint8_t fraction_lost, | 393 uint8_t fraction_lost, |
| 451 int64_t round_trip_time_ms) { | 394 int64_t round_trip_time_ms) { |
| 452 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate" << bitrate_bps | 395 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate" << bitrate_bps |
| 453 << " packet loss " << static_cast<int>(fraction_lost) | 396 << " packet loss " << static_cast<int>(fraction_lost) |
| 454 << " rtt " << round_trip_time_ms; | 397 << " rtt " << round_trip_time_ms; |
| 455 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, | 398 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, |
| 456 round_trip_time_ms); | 399 round_trip_time_ms); |
| 457 bool video_is_suspended = video_sender_.VideoSuspended(); | 400 bool video_is_suspended = video_sender_.VideoSuspended(); |
| 458 bool video_suspension_changed; | 401 bool video_suspension_changed; |
| 459 { | 402 { |
| 460 rtc::CritScope lock(&data_cs_); | 403 rtc::CritScope lock(&data_cs_); |
| 461 last_observed_bitrate_bps_ = bitrate_bps; | 404 last_observed_bitrate_bps_ = bitrate_bps; |
| 462 video_suspension_changed = video_suspended_ != video_is_suspended; | 405 video_suspension_changed = video_suspended_ != video_is_suspended; |
| 463 video_suspended_ = video_is_suspended; | 406 video_suspended_ = video_is_suspended; |
| 464 } | 407 } |
| 465 | 408 |
| 466 if (!video_suspension_changed) | 409 if (!video_suspension_changed) |
| 467 return; | 410 return; |
| 468 // Video suspend-state changed, inform codec observer. | 411 // Video suspend-state changed, inform codec observer. |
| 469 LOG(LS_INFO) << "Video suspend state changed " << video_is_suspended | 412 LOG(LS_INFO) << "Video suspend state changed " << video_is_suspended; |
| 470 << " for ssrc " << ssrcs_[0]; | 413 |
| 471 if (stats_proxy_) | 414 if (stats_proxy_) |
| 472 stats_proxy_->OnSuspendChange(video_is_suspended); | 415 stats_proxy_->OnSuspendChange(video_is_suspended); |
| 473 } | 416 } |
| 474 | 417 |
| 475 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessing* vpm) | 418 QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessing* vpm) |
| 476 : vp_(vpm) { | 419 : vp_(vpm) { |
| 477 } | 420 } |
| 478 | 421 |
| 479 QMVideoSettingsCallback::~QMVideoSettingsCallback() { | 422 QMVideoSettingsCallback::~QMVideoSettingsCallback() { |
| 480 } | 423 } |
| 481 | 424 |
| 482 int32_t QMVideoSettingsCallback::SetVideoQMSettings( | 425 int32_t QMVideoSettingsCallback::SetVideoQMSettings( |
| 483 const uint32_t frame_rate, | 426 const uint32_t frame_rate, |
| 484 const uint32_t width, | 427 const uint32_t width, |
| 485 const uint32_t height) { | 428 const uint32_t height) { |
| 486 return vp_->SetTargetResolution(width, height, frame_rate); | 429 return vp_->SetTargetResolution(width, height, frame_rate); |
| 487 } | 430 } |
| 488 | 431 |
| 489 } // namespace webrtc | 432 } // namespace webrtc |
| OLD | NEW |