| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/video/stream_synchronization.h" | 11 #include "webrtc/video/stream_synchronization.h" |
| 12 | 12 |
| 13 #include <assert.h> | 13 #include <assert.h> |
| 14 #include <math.h> | 14 #include <math.h> |
| 15 #include <stdlib.h> | 15 #include <stdlib.h> |
| 16 | 16 |
| 17 #include <algorithm> | 17 #include <algorithm> |
| 18 | 18 |
| 19 #include "webrtc/base/logging.h" | 19 #include "webrtc/base/logging.h" |
| 20 | 20 |
| 21 namespace webrtc { | 21 namespace webrtc { |
| 22 | 22 |
| 23 static const int kMaxChangeMs = 80; | 23 static const int kMaxChangeMs = 80; |
| 24 static const int kMaxDeltaDelayMs = 10000; | 24 static const int kMaxDeltaDelayMs = 10000; |
| 25 static const int kFilterLength = 4; | 25 static const int kFilterLength = 4; |
| 26 // Minimum difference between audio and video to warrant a change. | 26 // Minimum difference between audio and video to warrant a change. |
| 27 static const int kMinDeltaMs = 30; | 27 static const int kMinDeltaMs = 30; |
| 28 | 28 |
| 29 struct ViESyncDelay { | |
| 30 ViESyncDelay() { | |
| 31 extra_video_delay_ms = 0; | |
| 32 last_video_delay_ms = 0; | |
| 33 extra_audio_delay_ms = 0; | |
| 34 last_audio_delay_ms = 0; | |
| 35 network_delay = 120; | |
| 36 } | |
| 37 | |
| 38 int extra_video_delay_ms; | |
| 39 int last_video_delay_ms; | |
| 40 int extra_audio_delay_ms; | |
| 41 int last_audio_delay_ms; | |
| 42 int network_delay; | |
| 43 }; | |
| 44 | |
| 45 StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc, | 29 StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc, |
| 46 int audio_channel_id) | 30 int audio_channel_id) |
| 47 : channel_delay_(new ViESyncDelay), | 31 : video_primary_ssrc_(video_primary_ssrc), |
| 48 video_primary_ssrc_(video_primary_ssrc), | |
| 49 audio_channel_id_(audio_channel_id), | 32 audio_channel_id_(audio_channel_id), |
| 50 base_target_delay_ms_(0), | 33 base_target_delay_ms_(0), |
| 51 avg_diff_ms_(0) { | 34 avg_diff_ms_(0) { |
| 52 } | 35 } |
| 53 | 36 |
| 54 StreamSynchronization::~StreamSynchronization() { | |
| 55 delete channel_delay_; | |
| 56 } | |
| 57 | |
| 58 bool StreamSynchronization::ComputeRelativeDelay( | 37 bool StreamSynchronization::ComputeRelativeDelay( |
| 59 const Measurements& audio_measurement, | 38 const Measurements& audio_measurement, |
| 60 const Measurements& video_measurement, | 39 const Measurements& video_measurement, |
| 61 int* relative_delay_ms) { | 40 int* relative_delay_ms) { |
| 62 assert(relative_delay_ms); | 41 assert(relative_delay_ms); |
| 63 int64_t audio_last_capture_time_ms; | 42 int64_t audio_last_capture_time_ms; |
| 64 if (!RtpToNtpMs(audio_measurement.latest_timestamp, | 43 if (!RtpToNtpMs(audio_measurement.latest_timestamp, |
| 65 audio_measurement.rtcp, | 44 audio_measurement.rtcp, |
| 66 &audio_last_capture_time_ms)) { | 45 &audio_last_capture_time_ms)) { |
| 67 return false; | 46 return false; |
| (...skipping 19 matching lines...) Expand all Loading... |
| 87 } | 66 } |
| 88 | 67 |
| 89 bool StreamSynchronization::ComputeDelays(int relative_delay_ms, | 68 bool StreamSynchronization::ComputeDelays(int relative_delay_ms, |
| 90 int current_audio_delay_ms, | 69 int current_audio_delay_ms, |
| 91 int* total_audio_delay_target_ms, | 70 int* total_audio_delay_target_ms, |
| 92 int* total_video_delay_target_ms) { | 71 int* total_video_delay_target_ms) { |
| 93 assert(total_audio_delay_target_ms && total_video_delay_target_ms); | 72 assert(total_audio_delay_target_ms && total_video_delay_target_ms); |
| 94 | 73 |
| 95 int current_video_delay_ms = *total_video_delay_target_ms; | 74 int current_video_delay_ms = *total_video_delay_target_ms; |
| 96 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms | 75 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms |
| 97 << ", network delay diff: " << channel_delay_->network_delay | |
| 98 << " current diff: " << relative_delay_ms | 76 << " current diff: " << relative_delay_ms |
| 99 << " for channel " << audio_channel_id_; | 77 << " for channel " << audio_channel_id_; |
| 100 // Calculate the difference between the lowest possible video delay and | 78 // Calculate the difference between the lowest possible video delay and |
| 101 // the current audio delay. | 79 // the current audio delay. |
| 102 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + | 80 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + |
| 103 relative_delay_ms; | 81 relative_delay_ms; |
| 104 | 82 |
| 105 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + | 83 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + |
| 106 current_diff_ms) / kFilterLength; | 84 current_diff_ms) / kFilterLength; |
| 107 if (abs(avg_diff_ms_) < kMinDeltaMs) { | 85 if (abs(avg_diff_ms_) < kMinDeltaMs) { |
| 108 // Don't adjust if the diff is within our margin. | 86 // Don't adjust if the diff is within our margin. |
| 109 return false; | 87 return false; |
| 110 } | 88 } |
| 111 | 89 |
| 112 // Make sure we don't move too fast. | 90 // Make sure we don't move too fast. |
| 113 int diff_ms = avg_diff_ms_ / 2; | 91 int diff_ms = avg_diff_ms_ / 2; |
| 114 diff_ms = std::min(diff_ms, kMaxChangeMs); | 92 diff_ms = std::min(diff_ms, kMaxChangeMs); |
| 115 diff_ms = std::max(diff_ms, -kMaxChangeMs); | 93 diff_ms = std::max(diff_ms, -kMaxChangeMs); |
| 116 | 94 |
| 117 // Reset the average after a move to prevent overshooting reaction. | 95 // Reset the average after a move to prevent overshooting reaction. |
| 118 avg_diff_ms_ = 0; | 96 avg_diff_ms_ = 0; |
| 119 | 97 |
| 120 if (diff_ms > 0) { | 98 if (diff_ms > 0) { |
| 121 // The minimum video delay is longer than the current audio delay. | 99 // The minimum video delay is longer than the current audio delay. |
| 122 // We need to decrease extra video delay, or add extra audio delay. | 100 // We need to decrease extra video delay, or add extra audio delay. |
| 123 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { | 101 if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) { |
| 124 // We have extra delay added to ViE. Reduce this delay before adding | 102 // We have extra delay added to ViE. Reduce this delay before adding |
| 125 // extra delay to VoE. | 103 // extra delay to VoE. |
| 126 channel_delay_->extra_video_delay_ms -= diff_ms; | 104 channel_delay_.extra_video_delay_ms -= diff_ms; |
| 127 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; | 105 channel_delay_.extra_audio_delay_ms = base_target_delay_ms_; |
| 128 } else { // channel_delay_->extra_video_delay_ms > 0 | 106 } else { // channel_delay_.extra_video_delay_ms > 0 |
| 129 // We have no extra video delay to remove, increase the audio delay. | 107 // We have no extra video delay to remove, increase the audio delay. |
| 130 channel_delay_->extra_audio_delay_ms += diff_ms; | 108 channel_delay_.extra_audio_delay_ms += diff_ms; |
| 131 channel_delay_->extra_video_delay_ms = base_target_delay_ms_; | 109 channel_delay_.extra_video_delay_ms = base_target_delay_ms_; |
| 132 } | 110 } |
| 133 } else { // if (diff_ms > 0) | 111 } else { // if (diff_ms > 0) |
| 134 // The video delay is lower than the current audio delay. | 112 // The video delay is lower than the current audio delay. |
| 135 // We need to decrease extra audio delay, or add extra video delay. | 113 // We need to decrease extra audio delay, or add extra video delay. |
| 136 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { | 114 if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) { |
| 137 // We have extra delay in VoiceEngine. | 115 // We have extra delay in VoiceEngine. |
| 138 // Start with decreasing the voice delay. | 116 // Start with decreasing the voice delay. |
| 139 // Note: diff_ms is negative; add the negative difference. | 117 // Note: diff_ms is negative; add the negative difference. |
| 140 channel_delay_->extra_audio_delay_ms += diff_ms; | 118 channel_delay_.extra_audio_delay_ms += diff_ms; |
| 141 channel_delay_->extra_video_delay_ms = base_target_delay_ms_; | 119 channel_delay_.extra_video_delay_ms = base_target_delay_ms_; |
| 142 } else { // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_ | 120 } else { // channel_delay_.extra_audio_delay_ms > base_target_delay_ms_ |
| 143 // We have no extra delay in VoiceEngine, increase the video delay. | 121 // We have no extra delay in VoiceEngine, increase the video delay. |
| 144 // Note: diff_ms is negative; subtract the negative difference. | 122 // Note: diff_ms is negative; subtract the negative difference. |
| 145 channel_delay_->extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. | 123 channel_delay_.extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. |
| 146 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; | 124 channel_delay_.extra_audio_delay_ms = base_target_delay_ms_; |
| 147 } | 125 } |
| 148 } | 126 } |
| 149 | 127 |
| 150 // Make sure that video is never below our target. | 128 // Make sure that video is never below our target. |
| 151 channel_delay_->extra_video_delay_ms = std::max( | 129 channel_delay_.extra_video_delay_ms = std::max( |
| 152 channel_delay_->extra_video_delay_ms, base_target_delay_ms_); | 130 channel_delay_.extra_video_delay_ms, base_target_delay_ms_); |
| 153 | 131 |
| 154 int new_video_delay_ms; | 132 int new_video_delay_ms; |
| 155 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { | 133 if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) { |
| 156 new_video_delay_ms = channel_delay_->extra_video_delay_ms; | 134 new_video_delay_ms = channel_delay_.extra_video_delay_ms; |
| 157 } else { | 135 } else { |
| 158 // No change to the extra video delay. We are changing audio and we only | 136 // No change to the extra video delay. We are changing audio and we only |
| 159 // allow to change one at the time. | 137 // allow to change one at the time. |
| 160 new_video_delay_ms = channel_delay_->last_video_delay_ms; | 138 new_video_delay_ms = channel_delay_.last_video_delay_ms; |
| 161 } | 139 } |
| 162 | 140 |
| 163 // Make sure that we don't go below the extra video delay. | 141 // Make sure that we don't go below the extra video delay. |
| 164 new_video_delay_ms = std::max( | 142 new_video_delay_ms = std::max( |
| 165 new_video_delay_ms, channel_delay_->extra_video_delay_ms); | 143 new_video_delay_ms, channel_delay_.extra_video_delay_ms); |
| 166 | 144 |
| 167 // Verify we don't go above the maximum allowed video delay. | 145 // Verify we don't go above the maximum allowed video delay. |
| 168 new_video_delay_ms = | 146 new_video_delay_ms = |
| 169 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); | 147 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
| 170 | 148 |
| 171 int new_audio_delay_ms; | 149 int new_audio_delay_ms; |
| 172 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { | 150 if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) { |
| 173 new_audio_delay_ms = channel_delay_->extra_audio_delay_ms; | 151 new_audio_delay_ms = channel_delay_.extra_audio_delay_ms; |
| 174 } else { | 152 } else { |
| 175 // No change to the audio delay. We are changing video and we only | 153 // No change to the audio delay. We are changing video and we only |
| 176 // allow to change one at the time. | 154 // allow to change one at the time. |
| 177 new_audio_delay_ms = channel_delay_->last_audio_delay_ms; | 155 new_audio_delay_ms = channel_delay_.last_audio_delay_ms; |
| 178 } | 156 } |
| 179 | 157 |
| 180 // Make sure that we don't go below the extra audio delay. | 158 // Make sure that we don't go below the extra audio delay. |
| 181 new_audio_delay_ms = std::max( | 159 new_audio_delay_ms = std::max( |
| 182 new_audio_delay_ms, channel_delay_->extra_audio_delay_ms); | 160 new_audio_delay_ms, channel_delay_.extra_audio_delay_ms); |
| 183 | 161 |
| 184 // Verify we don't go above the maximum allowed audio delay. | 162 // Verify we don't go above the maximum allowed audio delay. |
| 185 new_audio_delay_ms = | 163 new_audio_delay_ms = |
| 186 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); | 164 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
| 187 | 165 |
| 188 // Remember our last audio and video delays. | 166 // Remember our last audio and video delays. |
| 189 channel_delay_->last_video_delay_ms = new_video_delay_ms; | 167 channel_delay_.last_video_delay_ms = new_video_delay_ms; |
| 190 channel_delay_->last_audio_delay_ms = new_audio_delay_ms; | 168 channel_delay_.last_audio_delay_ms = new_audio_delay_ms; |
| 191 | 169 |
| 192 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms | 170 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms |
| 193 << " for video primary SSRC " << video_primary_ssrc_ | 171 << " for video primary SSRC " << video_primary_ssrc_ |
| 194 << " and audio delay " << channel_delay_->extra_audio_delay_ms | 172 << " and audio delay " << channel_delay_.extra_audio_delay_ms |
| 195 << " for audio channel " << audio_channel_id_; | 173 << " for audio channel " << audio_channel_id_; |
| 196 | 174 |
| 197 // Return values. | 175 // Return values. |
| 198 *total_video_delay_target_ms = new_video_delay_ms; | 176 *total_video_delay_target_ms = new_video_delay_ms; |
| 199 *total_audio_delay_target_ms = new_audio_delay_ms; | 177 *total_audio_delay_target_ms = new_audio_delay_ms; |
| 200 return true; | 178 return true; |
| 201 } | 179 } |
| 202 | 180 |
| 203 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { | 181 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { |
| 204 // Initial extra delay for audio (accounting for existing extra delay). | 182 // Initial extra delay for audio (accounting for existing extra delay). |
| 205 channel_delay_->extra_audio_delay_ms += | 183 channel_delay_.extra_audio_delay_ms += |
| 206 target_delay_ms - base_target_delay_ms_; | 184 target_delay_ms - base_target_delay_ms_; |
| 207 channel_delay_->last_audio_delay_ms += | 185 channel_delay_.last_audio_delay_ms += |
| 208 target_delay_ms - base_target_delay_ms_; | 186 target_delay_ms - base_target_delay_ms_; |
| 209 | 187 |
| 210 // The video delay is compared to the last value (and how much we can update | 188 // The video delay is compared to the last value (and how much we can update |
| 211 // is limited by that as well). | 189 // is limited by that as well). |
| 212 channel_delay_->last_video_delay_ms += | 190 channel_delay_.last_video_delay_ms += |
| 213 target_delay_ms - base_target_delay_ms_; | 191 target_delay_ms - base_target_delay_ms_; |
| 214 | 192 |
| 215 channel_delay_->extra_video_delay_ms += | 193 channel_delay_.extra_video_delay_ms += |
| 216 target_delay_ms - base_target_delay_ms_; | 194 target_delay_ms - base_target_delay_ms_; |
| 217 | 195 |
| 218 // Video is already delayed by the desired amount. | 196 // Video is already delayed by the desired amount. |
| 219 base_target_delay_ms_ = target_delay_ms; | 197 base_target_delay_ms_ = target_delay_ms; |
| 220 } | 198 } |
| 221 | 199 |
| 222 } // namespace webrtc | 200 } // namespace webrtc |
| OLD | NEW |