OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 23 matching lines...) Expand all Loading... |
34 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { | 34 VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { |
35 if (payload_name == "VP8") | 35 if (payload_name == "VP8") |
36 return kVideoCodecVP8; | 36 return kVideoCodecVP8; |
37 if (payload_name == "VP9") | 37 if (payload_name == "VP9") |
38 return kVideoCodecVP9; | 38 return kVideoCodecVP9; |
39 if (payload_name == "H264") | 39 if (payload_name == "H264") |
40 return kVideoCodecH264; | 40 return kVideoCodecH264; |
41 return kVideoCodecGeneric; | 41 return kVideoCodecGeneric; |
42 } | 42 } |
43 | 43 |
44 VideoCodec VideoEncoderConfigToVideoCodec( | 44 VideoCodec VideoEncoderConfigToVideoCodec(const VideoEncoderConfig& config, |
45 const VideoEncoderConfig& config, | 45 const std::string& payload_name, |
46 const std::vector<VideoStream>& streams, | 46 int payload_type) { |
47 const std::string& payload_name, | 47 const std::vector<VideoStream>& streams = config.streams; |
48 int payload_type) { | |
49 static const int kEncoderMinBitrateKbps = 30; | 48 static const int kEncoderMinBitrateKbps = 30; |
50 RTC_DCHECK(!streams.empty()); | 49 RTC_DCHECK(!streams.empty()); |
51 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); | 50 RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); |
52 | 51 |
53 VideoCodec video_codec; | 52 VideoCodec video_codec; |
54 memset(&video_codec, 0, sizeof(video_codec)); | 53 memset(&video_codec, 0, sizeof(video_codec)); |
55 video_codec.codecType = PayloadNameToCodecType(payload_name); | 54 video_codec.codecType = PayloadNameToCodecType(payload_name); |
56 | 55 |
57 switch (config.content_type) { | 56 switch (config.content_type) { |
58 case VideoEncoderConfig::ContentType::kRealtimeVideo: | 57 case VideoEncoderConfig::ContentType::kRealtimeVideo: |
59 video_codec.mode = kRealtimeVideo; | 58 video_codec.mode = kRealtimeVideo; |
60 break; | 59 break; |
61 case VideoEncoderConfig::ContentType::kScreen: | 60 case VideoEncoderConfig::ContentType::kScreen: |
62 video_codec.mode = kScreensharing; | 61 video_codec.mode = kScreensharing; |
63 if (streams.size() == 1 && | 62 if (config.streams.size() == 1 && |
64 streams[0].temporal_layer_thresholds_bps.size() == 1) { | 63 config.streams[0].temporal_layer_thresholds_bps.size() == 1) { |
65 video_codec.targetBitrate = | 64 video_codec.targetBitrate = |
66 streams[0].temporal_layer_thresholds_bps[0] / 1000; | 65 config.streams[0].temporal_layer_thresholds_bps[0] / 1000; |
67 } | 66 } |
68 break; | 67 break; |
69 } | 68 } |
70 | 69 |
71 if (config.encoder_specific_settings) | 70 if (config.encoder_specific_settings) |
72 config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); | 71 config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); |
73 | 72 |
74 switch (video_codec.codecType) { | 73 switch (video_codec.codecType) { |
75 case kVideoCodecVP8: { | 74 case kVideoCodecVP8: { |
76 if (!config.encoder_specific_settings) | 75 if (!config.encoder_specific_settings) |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
163 // Unset max bitrate -> cap to one bit per pixel. | 162 // Unset max bitrate -> cap to one bit per pixel. |
164 video_codec.maxBitrate = | 163 video_codec.maxBitrate = |
165 (video_codec.width * video_codec.height * video_codec.maxFramerate) / | 164 (video_codec.width * video_codec.height * video_codec.maxFramerate) / |
166 1000; | 165 1000; |
167 } | 166 } |
168 if (video_codec.maxBitrate < kEncoderMinBitrateKbps) | 167 if (video_codec.maxBitrate < kEncoderMinBitrateKbps) |
169 video_codec.maxBitrate = kEncoderMinBitrateKbps; | 168 video_codec.maxBitrate = kEncoderMinBitrateKbps; |
170 | 169 |
171 RTC_DCHECK_GT(streams[0].max_framerate, 0); | 170 RTC_DCHECK_GT(streams[0].max_framerate, 0); |
172 video_codec.maxFramerate = streams[0].max_framerate; | 171 video_codec.maxFramerate = streams[0].max_framerate; |
| 172 video_codec.expect_encode_from_texture = config.expect_encode_from_texture; |
| 173 |
173 return video_codec; | 174 return video_codec; |
174 } | 175 } |
175 | 176 |
176 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle | 177 // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle |
177 // pipelining encoders better (multiple input frames before something comes | 178 // pipelining encoders better (multiple input frames before something comes |
178 // out). This should effectively turn off CPU adaptations for systems that | 179 // out). This should effectively turn off CPU adaptations for systems that |
179 // remotely cope with the load right now. | 180 // remotely cope with the load right now. |
180 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { | 181 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) { |
181 CpuOveruseOptions options; | 182 CpuOveruseOptions options; |
182 if (full_overuse_time) { | 183 if (full_overuse_time) { |
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
298 const VideoSendStream::Config::EncoderSettings& settings, | 299 const VideoSendStream::Config::EncoderSettings& settings, |
299 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | 300 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, |
300 LoadObserver* overuse_callback, | 301 LoadObserver* overuse_callback, |
301 EncodedFrameObserver* encoder_timing) | 302 EncodedFrameObserver* encoder_timing) |
302 : shutdown_event_(true /* manual_reset */, false), | 303 : shutdown_event_(true /* manual_reset */, false), |
303 number_of_cores_(number_of_cores), | 304 number_of_cores_(number_of_cores), |
304 source_proxy_(new VideoSourceProxy(this)), | 305 source_proxy_(new VideoSourceProxy(this)), |
305 sink_(nullptr), | 306 sink_(nullptr), |
306 settings_(settings), | 307 settings_(settings), |
307 codec_type_(PayloadNameToCodecType(settings.payload_name)), | 308 codec_type_(PayloadNameToCodecType(settings.payload_name)), |
| 309 vp_(VideoProcessing::Create()), |
308 video_sender_(Clock::GetRealTimeClock(), this, this), | 310 video_sender_(Clock::GetRealTimeClock(), this, this), |
309 overuse_detector_(Clock::GetRealTimeClock(), | 311 overuse_detector_(Clock::GetRealTimeClock(), |
310 GetCpuOveruseOptions(settings.full_overuse_time), | 312 GetCpuOveruseOptions(settings.full_overuse_time), |
311 this, | 313 this, |
312 encoder_timing, | 314 encoder_timing, |
313 stats_proxy), | 315 stats_proxy), |
314 load_observer_(overuse_callback), | 316 load_observer_(overuse_callback), |
315 stats_proxy_(stats_proxy), | 317 stats_proxy_(stats_proxy), |
316 pre_encode_callback_(pre_encode_callback), | 318 pre_encode_callback_(pre_encode_callback), |
317 module_process_thread_(nullptr), | 319 module_process_thread_(nullptr), |
318 pending_encoder_reconfiguration_(false), | 320 encoder_config_(), |
319 encoder_start_bitrate_bps_(0), | 321 encoder_start_bitrate_bps_(0), |
320 max_data_payload_length_(0), | 322 max_data_payload_length_(0), |
321 last_observed_bitrate_bps_(0), | 323 last_observed_bitrate_bps_(0), |
322 encoder_paused_and_dropped_frame_(false), | 324 encoder_paused_and_dropped_frame_(false), |
323 has_received_sli_(false), | 325 has_received_sli_(false), |
324 picture_id_sli_(0), | 326 picture_id_sli_(0), |
325 has_received_rpsi_(false), | 327 has_received_rpsi_(false), |
326 picture_id_rpsi_(0), | 328 picture_id_rpsi_(0), |
327 clock_(Clock::GetRealTimeClock()), | 329 clock_(Clock::GetRealTimeClock()), |
328 last_captured_timestamp_(0), | 330 last_captured_timestamp_(0), |
329 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - | 331 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - |
330 clock_->TimeInMilliseconds()), | 332 clock_->TimeInMilliseconds()), |
331 last_frame_log_ms_(clock_->TimeInMilliseconds()), | 333 last_frame_log_ms_(clock_->TimeInMilliseconds()), |
332 captured_frame_count_(0), | 334 captured_frame_count_(0), |
333 dropped_frame_count_(0), | 335 dropped_frame_count_(0), |
334 encoder_queue_("EncoderQueue") { | 336 encoder_queue_("EncoderQueue") { |
| 337 vp_->EnableTemporalDecimation(false); |
| 338 |
335 encoder_queue_.PostTask([this, encoder_timing] { | 339 encoder_queue_.PostTask([this, encoder_timing] { |
336 RTC_DCHECK_RUN_ON(&encoder_queue_); | 340 RTC_DCHECK_RUN_ON(&encoder_queue_); |
337 video_sender_.RegisterExternalEncoder( | 341 video_sender_.RegisterExternalEncoder( |
338 settings_.encoder, settings_.payload_type, settings_.internal_source); | 342 settings_.encoder, settings_.payload_type, settings_.internal_source); |
339 overuse_detector_.StartCheckForOveruse(); | 343 overuse_detector_.StartCheckForOveruse(); |
340 }); | 344 }); |
341 } | 345 } |
342 | 346 |
343 ViEEncoder::~ViEEncoder() { | 347 ViEEncoder::~ViEEncoder() { |
344 RTC_DCHECK_RUN_ON(&thread_checker_); | 348 RTC_DCHECK_RUN_ON(&thread_checker_); |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
396 size_t max_data_payload_length) { | 400 size_t max_data_payload_length) { |
397 encoder_queue_.PostTask( | 401 encoder_queue_.PostTask( |
398 std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask( | 402 std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask( |
399 this, std::move(config), max_data_payload_length))); | 403 this, std::move(config), max_data_payload_length))); |
400 } | 404 } |
401 | 405 |
402 void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config, | 406 void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config, |
403 size_t max_data_payload_length) { | 407 size_t max_data_payload_length) { |
404 RTC_DCHECK_RUN_ON(&encoder_queue_); | 408 RTC_DCHECK_RUN_ON(&encoder_queue_); |
405 RTC_DCHECK(sink_); | 409 RTC_DCHECK(sink_); |
406 LOG(LS_INFO) << "ConfigureEncoder requested."; | 410 LOG(LS_INFO) << "ConfigureEncoderOnTaskQueue"; |
407 | 411 |
408 max_data_payload_length_ = max_data_payload_length; | 412 max_data_payload_length_ = max_data_payload_length; |
409 encoder_config_ = std::move(config); | 413 encoder_config_ = std::move(config); |
410 pending_encoder_reconfiguration_ = true; | |
411 | 414 |
412 // Reconfigure the encoder now if the encoder has an internal source or | 415 VideoCodec video_codec = VideoEncoderConfigToVideoCodec( |
413 // if this is the first time the encoder is configured. | 416 encoder_config_, settings_.payload_name, settings_.payload_type); |
414 // Otherwise, the reconfiguration is deferred until the next frame to minimize | |
415 // the number of reconfigurations. The codec configuration depends on incoming | |
416 // video frame size. | |
417 if (!last_frame_info_ || settings_.internal_source) { | |
418 if (!last_frame_info_) { | |
419 last_frame_info_ = rtc::Optional<VideoFrameInfo>( | |
420 VideoFrameInfo(176, 144, kVideoRotation_0, false)); | |
421 } | |
422 ReconfigureEncoder(); | |
423 } | |
424 } | |
425 | 417 |
426 void ViEEncoder::ReconfigureEncoder() { | 418 // Setting target width and height for VPM. |
427 RTC_DCHECK_RUN_ON(&encoder_queue_); | 419 RTC_CHECK_EQ(VPM_OK, |
428 RTC_DCHECK(pending_encoder_reconfiguration_); | 420 vp_->SetTargetResolution(video_codec.width, video_codec.height, |
429 std::vector<VideoStream> streams = | 421 video_codec.maxFramerate)); |
430 encoder_config_.video_stream_factory->CreateEncoderStreams( | |
431 last_frame_info_->width, last_frame_info_->height, encoder_config_); | |
432 | 422 |
433 VideoCodec codec = VideoEncoderConfigToVideoCodec( | 423 video_codec.startBitrate = |
434 encoder_config_, streams, settings_.payload_name, settings_.payload_type); | 424 std::max(encoder_start_bitrate_bps_ / 1000, video_codec.minBitrate); |
435 | 425 video_codec.startBitrate = |
436 codec.startBitrate = | 426 std::min(video_codec.startBitrate, video_codec.maxBitrate); |
437 std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); | |
438 codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); | |
439 codec.expect_encode_from_texture = last_frame_info_->is_texture; | |
440 | 427 |
441 bool success = video_sender_.RegisterSendCodec( | 428 bool success = video_sender_.RegisterSendCodec( |
442 &codec, number_of_cores_, | 429 &video_codec, number_of_cores_, |
443 static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK; | 430 static_cast<uint32_t>(max_data_payload_length)) == VCM_OK; |
| 431 |
444 if (!success) { | 432 if (!success) { |
445 LOG(LS_ERROR) << "Failed to configure encoder."; | 433 LOG(LS_ERROR) << "Failed to configure encoder."; |
446 RTC_DCHECK(success); | 434 RTC_DCHECK(success); |
447 } | 435 } |
448 | 436 |
449 rate_allocator_.reset(new SimulcastRateAllocator(codec)); | 437 rate_allocator_.reset(new SimulcastRateAllocator(video_codec)); |
450 if (stats_proxy_) { | 438 if (stats_proxy_) { |
451 stats_proxy_->OnEncoderReconfigured(encoder_config_, | 439 stats_proxy_->OnEncoderReconfigured(encoder_config_, |
452 rate_allocator_->GetPreferedBitrate()); | 440 rate_allocator_->GetPreferedBitrate()); |
453 } | 441 } |
454 | 442 |
455 pending_encoder_reconfiguration_ = false; | |
456 if (stats_proxy_) { | |
457 stats_proxy_->OnEncoderReconfigured(encoder_config_, | |
458 rate_allocator_->GetPreferedBitrate()); | |
459 } | |
460 sink_->OnEncoderConfigurationChanged( | 443 sink_->OnEncoderConfigurationChanged( |
461 std::move(streams), encoder_config_.min_transmit_bitrate_bps); | 444 encoder_config_.streams, encoder_config_.min_transmit_bitrate_bps); |
462 } | 445 } |
463 | 446 |
464 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { | 447 void ViEEncoder::OnFrame(const VideoFrame& video_frame) { |
465 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); | 448 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
466 stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height()); | 449 stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height()); |
467 | 450 |
468 VideoFrame incoming_frame = video_frame; | 451 VideoFrame incoming_frame = video_frame; |
469 | 452 |
470 // Local time in webrtc time base. | 453 // Local time in webrtc time base. |
471 int64_t current_time = clock_->TimeInMilliseconds(); | 454 int64_t current_time = clock_->TimeInMilliseconds(); |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
534 } | 517 } |
535 encoder_paused_and_dropped_frame_ = false; | 518 encoder_paused_and_dropped_frame_ = false; |
536 } | 519 } |
537 | 520 |
538 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, | 521 void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame, |
539 int64_t time_when_posted_in_ms) { | 522 int64_t time_when_posted_in_ms) { |
540 RTC_DCHECK_RUN_ON(&encoder_queue_); | 523 RTC_DCHECK_RUN_ON(&encoder_queue_); |
541 if (pre_encode_callback_) | 524 if (pre_encode_callback_) |
542 pre_encode_callback_->OnFrame(video_frame); | 525 pre_encode_callback_->OnFrame(video_frame); |
543 | 526 |
544 if (video_frame.width() != last_frame_info_->width || | |
545 video_frame.height() != last_frame_info_->height || | |
546 video_frame.rotation() != last_frame_info_->rotation || | |
547 video_frame.is_texture() != last_frame_info_->is_texture) { | |
548 pending_encoder_reconfiguration_ = true; | |
549 last_frame_info_ = rtc::Optional<VideoFrameInfo>( | |
550 VideoFrameInfo(video_frame.width(), video_frame.height(), | |
551 video_frame.rotation(), video_frame.is_texture())); | |
552 LOG(LS_INFO) << "Video frame parameters changed: dimensions=" | |
553 << last_frame_info_->width << "x" << last_frame_info_->height | |
554 << ", rotation=" << last_frame_info_->rotation | |
555 << ", texture=" << last_frame_info_->is_texture; | |
556 } | |
557 | |
558 if (pending_encoder_reconfiguration_) { | |
559 ReconfigureEncoder(); | |
560 } | |
561 | |
562 if (EncoderPaused()) { | 527 if (EncoderPaused()) { |
563 TraceFrameDropStart(); | 528 TraceFrameDropStart(); |
564 return; | 529 return; |
565 } | 530 } |
566 TraceFrameDropEnd(); | 531 TraceFrameDropEnd(); |
567 | 532 |
568 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), | 533 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
569 "Encode"); | 534 "Encode"); |
| 535 const VideoFrame* frame_to_send = &video_frame; |
| 536 // TODO(wuchengli): support texture frames. |
| 537 if (!video_frame.video_frame_buffer()->native_handle()) { |
| 538 // Pass frame via preprocessor. |
| 539 frame_to_send = vp_->PreprocessFrame(video_frame); |
| 540 if (!frame_to_send) { |
| 541 // Drop this frame, or there was an error processing it. |
| 542 return; |
| 543 } |
| 544 } |
570 | 545 |
571 overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms); | 546 overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms); |
572 | 547 |
573 if (codec_type_ == webrtc::kVideoCodecVP8) { | 548 if (codec_type_ == webrtc::kVideoCodecVP8) { |
574 webrtc::CodecSpecificInfo codec_specific_info; | 549 webrtc::CodecSpecificInfo codec_specific_info; |
575 codec_specific_info.codecType = webrtc::kVideoCodecVP8; | 550 codec_specific_info.codecType = webrtc::kVideoCodecVP8; |
576 | 551 |
577 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = | 552 codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = |
578 has_received_rpsi_; | 553 has_received_rpsi_; |
579 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = | 554 codec_specific_info.codecSpecific.VP8.hasReceivedSLI = |
580 has_received_sli_; | 555 has_received_sli_; |
581 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = | 556 codec_specific_info.codecSpecific.VP8.pictureIdRPSI = |
582 picture_id_rpsi_; | 557 picture_id_rpsi_; |
583 codec_specific_info.codecSpecific.VP8.pictureIdSLI = | 558 codec_specific_info.codecSpecific.VP8.pictureIdSLI = |
584 picture_id_sli_; | 559 picture_id_sli_; |
585 has_received_sli_ = false; | 560 has_received_sli_ = false; |
586 has_received_rpsi_ = false; | 561 has_received_rpsi_ = false; |
587 | 562 |
588 video_sender_.AddVideoFrame(video_frame, &codec_specific_info); | 563 video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info); |
589 return; | 564 return; |
590 } | 565 } |
591 video_sender_.AddVideoFrame(video_frame, nullptr); | 566 video_sender_.AddVideoFrame(*frame_to_send, nullptr); |
592 } | 567 } |
593 | 568 |
594 void ViEEncoder::SendKeyFrame() { | 569 void ViEEncoder::SendKeyFrame() { |
595 if (!encoder_queue_.IsCurrent()) { | 570 if (!encoder_queue_.IsCurrent()) { |
596 encoder_queue_.PostTask([this] { SendKeyFrame(); }); | 571 encoder_queue_.PostTask([this] { SendKeyFrame(); }); |
597 return; | 572 return; |
598 } | 573 } |
599 RTC_DCHECK_RUN_ON(&encoder_queue_); | 574 RTC_DCHECK_RUN_ON(&encoder_queue_); |
600 video_sender_.IntraFrameRequest(0); | 575 video_sender_.IntraFrameRequest(0); |
601 } | 576 } |
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
704 load_observer_->OnLoadUpdate(LoadObserver::kOveruse); | 679 load_observer_->OnLoadUpdate(LoadObserver::kOveruse); |
705 } | 680 } |
706 | 681 |
707 void ViEEncoder::NormalUsage() { | 682 void ViEEncoder::NormalUsage() { |
708 RTC_DCHECK_RUN_ON(&encoder_queue_); | 683 RTC_DCHECK_RUN_ON(&encoder_queue_); |
709 if (load_observer_) | 684 if (load_observer_) |
710 load_observer_->OnLoadUpdate(LoadObserver::kUnderuse); | 685 load_observer_->OnLoadUpdate(LoadObserver::kUnderuse); |
711 } | 686 } |
712 | 687 |
713 } // namespace webrtc | 688 } // namespace webrtc |
OLD | NEW |