| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 46 key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); | 46 key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); |
| 47 | 47 |
| 48 // Set the FEC packet mask type. |kFecMaskBursty| is more effective for | 48 // Set the FEC packet mask type. |kFecMaskBursty| is more effective for |
| 49 // consecutive losses and little/no packet re-ordering. As we currently | 49 // consecutive losses and little/no packet re-ordering. As we currently |
| 50 // do not have feedback data on the degree of correlated losses and packet | 50 // do not have feedback data on the degree of correlated losses and packet |
| 51 // re-ordering, we keep default setting to |kFecMaskRandom| for now. | 51 // re-ordering, we keep default setting to |kFecMaskRandom| for now. |
| 52 delta_fec_params.fec_mask_type = kFecMaskRandom; | 52 delta_fec_params.fec_mask_type = kFecMaskRandom; |
| 53 key_fec_params.fec_mask_type = kFecMaskRandom; | 53 key_fec_params.fec_mask_type = kFecMaskRandom; |
| 54 | 54 |
| 55 // TODO(Marco): Pass FEC protection values per layer. | 55 // TODO(Marco): Pass FEC protection values per layer. |
| 56 video_protection_callback->ProtectionRequest(&delta_fec_params, | 56 video_protection_callback->ProtectionRequest( |
| 57 &key_fec_params, | 57 &delta_fec_params, &key_fec_params, video_rate_bps, |
| 58 video_rate_bps, | 58 nack_overhead_rate_bps, fec_overhead_rate_bps); |
| 59 nack_overhead_rate_bps, | |
| 60 fec_overhead_rate_bps); | |
| 61 } | 59 } |
| 62 } // namespace | 60 } // namespace |
| 63 | 61 |
| 64 struct MediaOptimization::EncodedFrameSample { | 62 struct MediaOptimization::EncodedFrameSample { |
| 65 EncodedFrameSample(size_t size_bytes, | 63 EncodedFrameSample(size_t size_bytes, |
| 66 uint32_t timestamp, | 64 uint32_t timestamp, |
| 67 int64_t time_complete_ms) | 65 int64_t time_complete_ms) |
| 68 : size_bytes(size_bytes), | 66 : size_bytes(size_bytes), |
| 69 timestamp(timestamp), | 67 timestamp(timestamp), |
| 70 time_complete_ms(time_complete_ms) {} | 68 time_complete_ms(time_complete_ms) {} |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 108 memset(send_statistics_, 0, sizeof(send_statistics_)); | 106 memset(send_statistics_, 0, sizeof(send_statistics_)); |
| 109 memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); | 107 memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); |
| 110 } | 108 } |
| 111 | 109 |
| 112 MediaOptimization::~MediaOptimization(void) { | 110 MediaOptimization::~MediaOptimization(void) { |
| 113 loss_prot_logic_->Release(); | 111 loss_prot_logic_->Release(); |
| 114 } | 112 } |
| 115 | 113 |
| 116 void MediaOptimization::Reset() { | 114 void MediaOptimization::Reset() { |
| 117 CriticalSectionScoped lock(crit_sect_.get()); | 115 CriticalSectionScoped lock(crit_sect_.get()); |
| 118 SetEncodingDataInternal( | 116 SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, |
| 119 kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, max_payload_size_); | 117 max_payload_size_); |
| 120 memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); | 118 memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); |
| 121 incoming_frame_rate_ = 0.0; | 119 incoming_frame_rate_ = 0.0; |
| 122 frame_dropper_->Reset(); | 120 frame_dropper_->Reset(); |
| 123 loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); | 121 loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); |
| 124 frame_dropper_->SetRates(0, 0); | 122 frame_dropper_->SetRates(0, 0); |
| 125 content_->Reset(); | 123 content_->Reset(); |
| 126 qm_resolution_->Reset(); | 124 qm_resolution_->Reset(); |
| 127 loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); | 125 loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); |
| 128 loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); | 126 loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); |
| 129 send_statistics_zero_encode_ = 0; | 127 send_statistics_zero_encode_ = 0; |
| (...skipping 12 matching lines...) Expand all Loading... |
| 142 | 140 |
| 143 void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, | 141 void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, |
| 144 int32_t max_bit_rate, | 142 int32_t max_bit_rate, |
| 145 uint32_t target_bitrate, | 143 uint32_t target_bitrate, |
| 146 uint16_t width, | 144 uint16_t width, |
| 147 uint16_t height, | 145 uint16_t height, |
| 148 uint32_t frame_rate, | 146 uint32_t frame_rate, |
| 149 int num_layers, | 147 int num_layers, |
| 150 int32_t mtu) { | 148 int32_t mtu) { |
| 151 CriticalSectionScoped lock(crit_sect_.get()); | 149 CriticalSectionScoped lock(crit_sect_.get()); |
| 152 SetEncodingDataInternal(send_codec_type, | 150 SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate, |
| 153 max_bit_rate, | 151 target_bitrate, width, height, num_layers, mtu); |
| 154 frame_rate, | |
| 155 target_bitrate, | |
| 156 width, | |
| 157 height, | |
| 158 num_layers, | |
| 159 mtu); | |
| 160 } | 152 } |
| 161 | 153 |
| 162 void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, | 154 void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, |
| 163 int32_t max_bit_rate, | 155 int32_t max_bit_rate, |
| 164 uint32_t frame_rate, | 156 uint32_t frame_rate, |
| 165 uint32_t target_bitrate, | 157 uint32_t target_bitrate, |
| 166 uint16_t width, | 158 uint16_t width, |
| 167 uint16_t height, | 159 uint16_t height, |
| 168 int num_layers, | 160 int num_layers, |
| 169 int32_t mtu) { | 161 int32_t mtu) { |
| (...skipping 13 matching lines...) Expand all Loading... |
| 183 loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate)); | 175 loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate)); |
| 184 loss_prot_logic_->UpdateFrameSize(width, height); | 176 loss_prot_logic_->UpdateFrameSize(width, height); |
| 185 loss_prot_logic_->UpdateNumLayers(num_layers); | 177 loss_prot_logic_->UpdateNumLayers(num_layers); |
| 186 frame_dropper_->Reset(); | 178 frame_dropper_->Reset(); |
| 187 frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate)); | 179 frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate)); |
| 188 user_frame_rate_ = static_cast<float>(frame_rate); | 180 user_frame_rate_ = static_cast<float>(frame_rate); |
| 189 codec_width_ = width; | 181 codec_width_ = width; |
| 190 codec_height_ = height; | 182 codec_height_ = height; |
| 191 num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. | 183 num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. |
| 192 max_payload_size_ = mtu; | 184 max_payload_size_ = mtu; |
| 193 qm_resolution_->Initialize(target_bitrate_kbps, | 185 qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_, |
| 194 user_frame_rate_, | 186 codec_width_, codec_height_, num_layers_); |
| 195 codec_width_, | |
| 196 codec_height_, | |
| 197 num_layers_); | |
| 198 } | 187 } |
| 199 | 188 |
| 200 uint32_t MediaOptimization::SetTargetRates( | 189 uint32_t MediaOptimization::SetTargetRates( |
| 201 uint32_t target_bitrate, | 190 uint32_t target_bitrate, |
| 202 uint8_t fraction_lost, | 191 uint8_t fraction_lost, |
| 203 int64_t round_trip_time_ms, | 192 int64_t round_trip_time_ms, |
| 204 VCMProtectionCallback* protection_callback, | 193 VCMProtectionCallback* protection_callback, |
| 205 VCMQMSettingsCallback* qmsettings_callback) { | 194 VCMQMSettingsCallback* qmsettings_callback) { |
| 206 CriticalSectionScoped lock(crit_sect_.get()); | 195 CriticalSectionScoped lock(crit_sect_.get()); |
| 207 VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); | 196 VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); |
| (...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 249 loss_prot_logic_->UpdateMethod(); | 238 loss_prot_logic_->UpdateMethod(); |
| 250 | 239 |
| 251 // Update protection callback with protection settings. | 240 // Update protection callback with protection settings. |
| 252 uint32_t sent_video_rate_bps = 0; | 241 uint32_t sent_video_rate_bps = 0; |
| 253 uint32_t sent_nack_rate_bps = 0; | 242 uint32_t sent_nack_rate_bps = 0; |
| 254 uint32_t sent_fec_rate_bps = 0; | 243 uint32_t sent_fec_rate_bps = 0; |
| 255 // Get the bit cost of protection method, based on the amount of | 244 // Get the bit cost of protection method, based on the amount of |
| 256 // overhead data actually transmitted (including headers) the last | 245 // overhead data actually transmitted (including headers) the last |
| 257 // second. | 246 // second. |
| 258 if (protection_callback) { | 247 if (protection_callback) { |
| 259 UpdateProtectionCallback(selected_method, | 248 UpdateProtectionCallback(selected_method, &sent_video_rate_bps, |
| 260 &sent_video_rate_bps, | 249 &sent_nack_rate_bps, &sent_fec_rate_bps, |
| 261 &sent_nack_rate_bps, | |
| 262 &sent_fec_rate_bps, | |
| 263 protection_callback); | 250 protection_callback); |
| 264 } | 251 } |
| 265 uint32_t sent_total_rate_bps = | 252 uint32_t sent_total_rate_bps = |
| 266 sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; | 253 sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; |
| 267 // Estimate the overhead costs of the next second as staying the same | 254 // Estimate the overhead costs of the next second as staying the same |
| 268 // wrt the source bitrate. | 255 // wrt the source bitrate. |
| 269 if (sent_total_rate_bps > 0) { | 256 if (sent_total_rate_bps > 0) { |
| 270 protection_overhead_rate = | 257 protection_overhead_rate = |
| 271 static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) / | 258 static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) / |
| 272 sent_total_rate_bps; | 259 sent_total_rate_bps; |
| (...skipping 16 matching lines...) Expand all Loading... |
| 289 video_target_bitrate_ = max_bit_rate_; | 276 video_target_bitrate_ = max_bit_rate_; |
| 290 } | 277 } |
| 291 | 278 |
| 292 // Update encoding rates following protection settings. | 279 // Update encoding rates following protection settings. |
| 293 float target_video_bitrate_kbps = | 280 float target_video_bitrate_kbps = |
| 294 static_cast<float>(video_target_bitrate_) / 1000.0f; | 281 static_cast<float>(video_target_bitrate_) / 1000.0f; |
| 295 frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); | 282 frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); |
| 296 | 283 |
| 297 if (enable_qm_ && qmsettings_callback) { | 284 if (enable_qm_ && qmsettings_callback) { |
| 298 // Update QM with rates. | 285 // Update QM with rates. |
| 299 qm_resolution_->UpdateRates(target_video_bitrate_kbps, | 286 qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps, |
| 300 sent_video_rate_kbps, | 287 incoming_frame_rate_, fraction_lost_); |
| 301 incoming_frame_rate_, | |
| 302 fraction_lost_); | |
| 303 // Check for QM selection. | 288 // Check for QM selection. |
| 304 bool select_qm = CheckStatusForQMchange(); | 289 bool select_qm = CheckStatusForQMchange(); |
| 305 if (select_qm) { | 290 if (select_qm) { |
| 306 SelectQuality(qmsettings_callback); | 291 SelectQuality(qmsettings_callback); |
| 307 } | 292 } |
| 308 // Reset the short-term averaged content data. | 293 // Reset the short-term averaged content data. |
| 309 content_->ResetShortTermAvgData(); | 294 content_->ResetShortTermAvgData(); |
| 310 } | 295 } |
| 311 | 296 |
| 312 CheckSuspendConditions(); | 297 CheckSuspendConditions(); |
| (...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 507 } | 492 } |
| 508 } | 493 } |
| 509 | 494 |
| 510 void MediaOptimization::UpdateSentBitrate(int64_t now_ms) { | 495 void MediaOptimization::UpdateSentBitrate(int64_t now_ms) { |
| 511 if (encoded_frame_samples_.empty()) { | 496 if (encoded_frame_samples_.empty()) { |
| 512 avg_sent_bit_rate_bps_ = 0; | 497 avg_sent_bit_rate_bps_ = 0; |
| 513 return; | 498 return; |
| 514 } | 499 } |
| 515 size_t framesize_sum = 0; | 500 size_t framesize_sum = 0; |
| 516 for (FrameSampleList::iterator it = encoded_frame_samples_.begin(); | 501 for (FrameSampleList::iterator it = encoded_frame_samples_.begin(); |
| 517 it != encoded_frame_samples_.end(); | 502 it != encoded_frame_samples_.end(); ++it) { |
| 518 ++it) { | |
| 519 framesize_sum += it->size_bytes; | 503 framesize_sum += it->size_bytes; |
| 520 } | 504 } |
| 521 float denom = static_cast<float>( | 505 float denom = static_cast<float>( |
| 522 now_ms - encoded_frame_samples_.front().time_complete_ms); | 506 now_ms - encoded_frame_samples_.front().time_complete_ms); |
| 523 if (denom >= 1.0f) { | 507 if (denom >= 1.0f) { |
| 524 avg_sent_bit_rate_bps_ = | 508 avg_sent_bit_rate_bps_ = |
| 525 static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f); | 509 static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f); |
| 526 } else { | 510 } else { |
| 527 avg_sent_bit_rate_bps_ = framesize_sum * 8; | 511 avg_sent_bit_rate_bps_ = framesize_sum * 8; |
| 528 } | 512 } |
| (...skipping 29 matching lines...) Expand all Loading... |
| 558 memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); | 542 memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); |
| 559 } | 543 } |
| 560 | 544 |
| 561 // Check for change in frame size. | 545 // Check for change in frame size. |
| 562 if (qm->change_resolution_spatial) { | 546 if (qm->change_resolution_spatial) { |
| 563 codec_width_ = qm->codec_width; | 547 codec_width_ = qm->codec_width; |
| 564 codec_height_ = qm->codec_height; | 548 codec_height_ = qm->codec_height; |
| 565 } | 549 } |
| 566 | 550 |
| 567 LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed " | 551 LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed " |
| 568 "to " << qm->codec_width << "x" << qm->codec_height << "@" | 552 "to " |
| 553 << qm->codec_width << "x" << qm->codec_height << "@" |
| 569 << qm->frame_rate; | 554 << qm->frame_rate; |
| 570 | 555 |
| 571 // Update VPM with new target frame rate and frame size. | 556 // Update VPM with new target frame rate and frame size. |
| 572 // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating | 557 // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating |
| 573 // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate| | 558 // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate| |
| 574 // will vary/fluctuate, and since we don't want to change the state of the | 559 // will vary/fluctuate, and since we don't want to change the state of the |
| 575 // VPM frame dropper, unless a temporal action was selected, we use the | 560 // VPM frame dropper, unless a temporal action was selected, we use the |
| 576 // quantity |qm->frame_rate| for updating. | 561 // quantity |qm->frame_rate| for updating. |
| 577 video_qmsettings_callback->SetVideoQMSettings( | 562 video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_, |
| 578 qm->frame_rate, codec_width_, codec_height_); | 563 codec_height_); |
| 579 content_->UpdateFrameRate(qm->frame_rate); | 564 content_->UpdateFrameRate(qm->frame_rate); |
| 580 qm_resolution_->UpdateCodecParameters( | 565 qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_, |
| 581 qm->frame_rate, codec_width_, codec_height_); | 566 codec_height_); |
| 582 return true; | 567 return true; |
| 583 } | 568 } |
| 584 | 569 |
| 585 // Check timing constraints and look for significant change in: | 570 // Check timing constraints and look for significant change in: |
| 586 // (1) scene content, | 571 // (1) scene content, |
| 587 // (2) target bit rate. | 572 // (2) target bit rate. |
| 588 bool MediaOptimization::CheckStatusForQMchange() { | 573 bool MediaOptimization::CheckStatusForQMchange() { |
| 589 bool status = true; | 574 bool status = true; |
| 590 | 575 |
| 591 // Check that we do not call QMSelect too often, and that we waited some time | 576 // Check that we do not call QMSelect too often, and that we waited some time |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 639 if (video_target_bitrate_ > | 624 if (video_target_bitrate_ > |
| 640 suspension_threshold_bps_ + suspension_window_bps_) { | 625 suspension_threshold_bps_ + suspension_window_bps_) { |
| 641 video_suspended_ = false; | 626 video_suspended_ = false; |
| 642 } | 627 } |
| 643 } | 628 } |
| 644 } | 629 } |
| 645 } | 630 } |
| 646 | 631 |
| 647 } // namespace media_optimization | 632 } // namespace media_optimization |
| 648 } // namespace webrtc | 633 } // namespace webrtc |
| OLD | NEW |