OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/video/stream_synchronization.h" | 11 #include "webrtc/video/stream_synchronization.h" |
12 | 12 |
13 #include <assert.h> | 13 #include <assert.h> |
14 #include <math.h> | 14 #include <math.h> |
15 #include <stdlib.h> | 15 #include <stdlib.h> |
16 | 16 |
17 #include <algorithm> | 17 #include <algorithm> |
18 | 18 |
19 #include "webrtc/base/logging.h" | 19 #include "webrtc/base/logging.h" |
20 | 20 |
21 namespace webrtc { | 21 namespace webrtc { |
22 | 22 |
23 static const int kMaxChangeMs = 80; | 23 static const int kMaxChangeMs = 80; |
24 static const int kMaxDeltaDelayMs = 10000; | 24 static const int kMaxDeltaDelayMs = 10000; |
25 static const int kFilterLength = 4; | 25 static const int kFilterLength = 4; |
26 // Minimum difference between audio and video to warrant a change. | 26 // Minimum difference between audio and video to warrant a change. |
27 static const int kMinDeltaMs = 30; | 27 static const int kMinDeltaMs = 30; |
28 | 28 |
29 struct ViESyncDelay { | |
30 ViESyncDelay() { | |
31 extra_video_delay_ms = 0; | |
32 last_video_delay_ms = 0; | |
33 extra_audio_delay_ms = 0; | |
34 last_audio_delay_ms = 0; | |
35 network_delay = 120; | |
36 } | |
37 | |
38 int extra_video_delay_ms; | |
39 int last_video_delay_ms; | |
40 int extra_audio_delay_ms; | |
41 int last_audio_delay_ms; | |
42 int network_delay; | |
43 }; | |
44 | |
45 StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc, | 29 StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc, |
46 int audio_channel_id) | 30 int audio_channel_id) |
47 : channel_delay_(new ViESyncDelay), | 31 : video_primary_ssrc_(video_primary_ssrc), |
48 video_primary_ssrc_(video_primary_ssrc), | |
49 audio_channel_id_(audio_channel_id), | 32 audio_channel_id_(audio_channel_id), |
50 base_target_delay_ms_(0), | 33 base_target_delay_ms_(0), |
51 avg_diff_ms_(0) { | 34 avg_diff_ms_(0) { |
52 } | 35 } |
53 | 36 |
54 StreamSynchronization::~StreamSynchronization() { | |
55 delete channel_delay_; | |
56 } | |
57 | |
58 bool StreamSynchronization::ComputeRelativeDelay( | 37 bool StreamSynchronization::ComputeRelativeDelay( |
59 const Measurements& audio_measurement, | 38 const Measurements& audio_measurement, |
60 const Measurements& video_measurement, | 39 const Measurements& video_measurement, |
61 int* relative_delay_ms) { | 40 int* relative_delay_ms) { |
62 assert(relative_delay_ms); | 41 assert(relative_delay_ms); |
63 int64_t audio_last_capture_time_ms; | 42 int64_t audio_last_capture_time_ms; |
64 if (!RtpToNtpMs(audio_measurement.latest_timestamp, | 43 if (!RtpToNtpMs(audio_measurement.latest_timestamp, |
65 audio_measurement.rtcp, | 44 audio_measurement.rtcp, |
66 &audio_last_capture_time_ms)) { | 45 &audio_last_capture_time_ms)) { |
67 return false; | 46 return false; |
(...skipping 19 matching lines...) Expand all Loading... |
87 } | 66 } |
88 | 67 |
89 bool StreamSynchronization::ComputeDelays(int relative_delay_ms, | 68 bool StreamSynchronization::ComputeDelays(int relative_delay_ms, |
90 int current_audio_delay_ms, | 69 int current_audio_delay_ms, |
91 int* total_audio_delay_target_ms, | 70 int* total_audio_delay_target_ms, |
92 int* total_video_delay_target_ms) { | 71 int* total_video_delay_target_ms) { |
93 assert(total_audio_delay_target_ms && total_video_delay_target_ms); | 72 assert(total_audio_delay_target_ms && total_video_delay_target_ms); |
94 | 73 |
95 int current_video_delay_ms = *total_video_delay_target_ms; | 74 int current_video_delay_ms = *total_video_delay_target_ms; |
96 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms | 75 LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms |
97 << ", network delay diff: " << channel_delay_->network_delay | |
98 << " current diff: " << relative_delay_ms | 76 << " current diff: " << relative_delay_ms |
99 << " for channel " << audio_channel_id_; | 77 << " for channel " << audio_channel_id_; |
100 // Calculate the difference between the lowest possible video delay and | 78 // Calculate the difference between the lowest possible video delay and |
101 // the current audio delay. | 79 // the current audio delay. |
102 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + | 80 int current_diff_ms = current_video_delay_ms - current_audio_delay_ms + |
103 relative_delay_ms; | 81 relative_delay_ms; |
104 | 82 |
105 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + | 83 avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ + |
106 current_diff_ms) / kFilterLength; | 84 current_diff_ms) / kFilterLength; |
107 if (abs(avg_diff_ms_) < kMinDeltaMs) { | 85 if (abs(avg_diff_ms_) < kMinDeltaMs) { |
108 // Don't adjust if the diff is within our margin. | 86 // Don't adjust if the diff is within our margin. |
109 return false; | 87 return false; |
110 } | 88 } |
111 | 89 |
112 // Make sure we don't move too fast. | 90 // Make sure we don't move too fast. |
113 int diff_ms = avg_diff_ms_ / 2; | 91 int diff_ms = avg_diff_ms_ / 2; |
114 diff_ms = std::min(diff_ms, kMaxChangeMs); | 92 diff_ms = std::min(diff_ms, kMaxChangeMs); |
115 diff_ms = std::max(diff_ms, -kMaxChangeMs); | 93 diff_ms = std::max(diff_ms, -kMaxChangeMs); |
116 | 94 |
117 // Reset the average after a move to prevent overshooting reaction. | 95 // Reset the average after a move to prevent overshooting reaction. |
118 avg_diff_ms_ = 0; | 96 avg_diff_ms_ = 0; |
119 | 97 |
120 if (diff_ms > 0) { | 98 if (diff_ms > 0) { |
121 // The minimum video delay is longer than the current audio delay. | 99 // The minimum video delay is longer than the current audio delay. |
122 // We need to decrease extra video delay, or add extra audio delay. | 100 // We need to decrease extra video delay, or add extra audio delay. |
123 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { | 101 if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) { |
124 // We have extra delay added to ViE. Reduce this delay before adding | 102 // We have extra delay added to ViE. Reduce this delay before adding |
125 // extra delay to VoE. | 103 // extra delay to VoE. |
126 channel_delay_->extra_video_delay_ms -= diff_ms; | 104 channel_delay_.extra_video_delay_ms -= diff_ms; |
127 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; | 105 channel_delay_.extra_audio_delay_ms = base_target_delay_ms_; |
128 } else { // channel_delay_->extra_video_delay_ms > 0 | 106 } else { // channel_delay_.extra_video_delay_ms > 0 |
129 // We have no extra video delay to remove, increase the audio delay. | 107 // We have no extra video delay to remove, increase the audio delay. |
130 channel_delay_->extra_audio_delay_ms += diff_ms; | 108 channel_delay_.extra_audio_delay_ms += diff_ms; |
131 channel_delay_->extra_video_delay_ms = base_target_delay_ms_; | 109 channel_delay_.extra_video_delay_ms = base_target_delay_ms_; |
132 } | 110 } |
133 } else { // if (diff_ms > 0) | 111 } else { // if (diff_ms > 0) |
134 // The video delay is lower than the current audio delay. | 112 // The video delay is lower than the current audio delay. |
135 // We need to decrease extra audio delay, or add extra video delay. | 113 // We need to decrease extra audio delay, or add extra video delay. |
136 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { | 114 if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) { |
137 // We have extra delay in VoiceEngine. | 115 // We have extra delay in VoiceEngine. |
138 // Start with decreasing the voice delay. | 116 // Start with decreasing the voice delay. |
139 // Note: diff_ms is negative; add the negative difference. | 117 // Note: diff_ms is negative; add the negative difference. |
140 channel_delay_->extra_audio_delay_ms += diff_ms; | 118 channel_delay_.extra_audio_delay_ms += diff_ms; |
141 channel_delay_->extra_video_delay_ms = base_target_delay_ms_; | 119 channel_delay_.extra_video_delay_ms = base_target_delay_ms_; |
142 } else { // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_ | 120 } else { // channel_delay_.extra_audio_delay_ms > base_target_delay_ms_ |
143 // We have no extra delay in VoiceEngine, increase the video delay. | 121 // We have no extra delay in VoiceEngine, increase the video delay. |
144 // Note: diff_ms is negative; subtract the negative difference. | 122 // Note: diff_ms is negative; subtract the negative difference. |
145 channel_delay_->extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. | 123 channel_delay_.extra_video_delay_ms -= diff_ms; // X - (-Y) = X + Y. |
146 channel_delay_->extra_audio_delay_ms = base_target_delay_ms_; | 124 channel_delay_.extra_audio_delay_ms = base_target_delay_ms_; |
147 } | 125 } |
148 } | 126 } |
149 | 127 |
150 // Make sure that video is never below our target. | 128 // Make sure that video is never below our target. |
151 channel_delay_->extra_video_delay_ms = std::max( | 129 channel_delay_.extra_video_delay_ms = std::max( |
152 channel_delay_->extra_video_delay_ms, base_target_delay_ms_); | 130 channel_delay_.extra_video_delay_ms, base_target_delay_ms_); |
153 | 131 |
154 int new_video_delay_ms; | 132 int new_video_delay_ms; |
155 if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) { | 133 if (channel_delay_.extra_video_delay_ms > base_target_delay_ms_) { |
156 new_video_delay_ms = channel_delay_->extra_video_delay_ms; | 134 new_video_delay_ms = channel_delay_.extra_video_delay_ms; |
157 } else { | 135 } else { |
158 // No change to the extra video delay. We are changing audio and we only | 136 // No change to the extra video delay. We are changing audio and we only |
159 // allow to change one at the time. | 137 // allow to change one at the time. |
160 new_video_delay_ms = channel_delay_->last_video_delay_ms; | 138 new_video_delay_ms = channel_delay_.last_video_delay_ms; |
161 } | 139 } |
162 | 140 |
163 // Make sure that we don't go below the extra video delay. | 141 // Make sure that we don't go below the extra video delay. |
164 new_video_delay_ms = std::max( | 142 new_video_delay_ms = std::max( |
165 new_video_delay_ms, channel_delay_->extra_video_delay_ms); | 143 new_video_delay_ms, channel_delay_.extra_video_delay_ms); |
166 | 144 |
167 // Verify we don't go above the maximum allowed video delay. | 145 // Verify we don't go above the maximum allowed video delay. |
168 new_video_delay_ms = | 146 new_video_delay_ms = |
169 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); | 147 std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
170 | 148 |
171 int new_audio_delay_ms; | 149 int new_audio_delay_ms; |
172 if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) { | 150 if (channel_delay_.extra_audio_delay_ms > base_target_delay_ms_) { |
173 new_audio_delay_ms = channel_delay_->extra_audio_delay_ms; | 151 new_audio_delay_ms = channel_delay_.extra_audio_delay_ms; |
174 } else { | 152 } else { |
175 // No change to the audio delay. We are changing video and we only | 153 // No change to the audio delay. We are changing video and we only |
176 // allow to change one at the time. | 154 // allow to change one at the time. |
177 new_audio_delay_ms = channel_delay_->last_audio_delay_ms; | 155 new_audio_delay_ms = channel_delay_.last_audio_delay_ms; |
178 } | 156 } |
179 | 157 |
180 // Make sure that we don't go below the extra audio delay. | 158 // Make sure that we don't go below the extra audio delay. |
181 new_audio_delay_ms = std::max( | 159 new_audio_delay_ms = std::max( |
182 new_audio_delay_ms, channel_delay_->extra_audio_delay_ms); | 160 new_audio_delay_ms, channel_delay_.extra_audio_delay_ms); |
183 | 161 |
184 // Verify we don't go above the maximum allowed audio delay. | 162 // Verify we don't go above the maximum allowed audio delay. |
185 new_audio_delay_ms = | 163 new_audio_delay_ms = |
186 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); | 164 std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs); |
187 | 165 |
188 // Remember our last audio and video delays. | 166 // Remember our last audio and video delays. |
189 channel_delay_->last_video_delay_ms = new_video_delay_ms; | 167 channel_delay_.last_video_delay_ms = new_video_delay_ms; |
190 channel_delay_->last_audio_delay_ms = new_audio_delay_ms; | 168 channel_delay_.last_audio_delay_ms = new_audio_delay_ms; |
191 | 169 |
192 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms | 170 LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms |
193 << " for video primary SSRC " << video_primary_ssrc_ | 171 << " for video primary SSRC " << video_primary_ssrc_ |
194 << " and audio delay " << channel_delay_->extra_audio_delay_ms | 172 << " and audio delay " << channel_delay_.extra_audio_delay_ms |
195 << " for audio channel " << audio_channel_id_; | 173 << " for audio channel " << audio_channel_id_; |
196 | 174 |
197 // Return values. | 175 // Return values. |
198 *total_video_delay_target_ms = new_video_delay_ms; | 176 *total_video_delay_target_ms = new_video_delay_ms; |
199 *total_audio_delay_target_ms = new_audio_delay_ms; | 177 *total_audio_delay_target_ms = new_audio_delay_ms; |
200 return true; | 178 return true; |
201 } | 179 } |
202 | 180 |
203 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { | 181 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) { |
204 // Initial extra delay for audio (accounting for existing extra delay). | 182 // Initial extra delay for audio (accounting for existing extra delay). |
205 channel_delay_->extra_audio_delay_ms += | 183 channel_delay_.extra_audio_delay_ms += |
206 target_delay_ms - base_target_delay_ms_; | 184 target_delay_ms - base_target_delay_ms_; |
207 channel_delay_->last_audio_delay_ms += | 185 channel_delay_.last_audio_delay_ms += |
208 target_delay_ms - base_target_delay_ms_; | 186 target_delay_ms - base_target_delay_ms_; |
209 | 187 |
210 // The video delay is compared to the last value (and how much we can update | 188 // The video delay is compared to the last value (and how much we can update |
211 // is limited by that as well). | 189 // is limited by that as well). |
212 channel_delay_->last_video_delay_ms += | 190 channel_delay_.last_video_delay_ms += |
213 target_delay_ms - base_target_delay_ms_; | 191 target_delay_ms - base_target_delay_ms_; |
214 | 192 |
215 channel_delay_->extra_video_delay_ms += | 193 channel_delay_.extra_video_delay_ms += |
216 target_delay_ms - base_target_delay_ms_; | 194 target_delay_ms - base_target_delay_ms_; |
217 | 195 |
218 // Video is already delayed by the desired amount. | 196 // Video is already delayed by the desired amount. |
219 base_target_delay_ms_ = target_delay_ms; | 197 base_target_delay_ms_ = target_delay_ms; |
220 } | 198 } |
221 | 199 |
222 } // namespace webrtc | 200 } // namespace webrtc |
OLD | NEW |