OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/video_coding/frame_buffer2.h" | 11 #include "webrtc/modules/video_coding/frame_buffer2.h" |
12 | 12 |
13 #include <algorithm> | 13 #include <algorithm> |
14 #include <cstring> | 14 #include <cstring> |
15 #include <queue> | 15 #include <queue> |
16 | 16 |
| 17 #include "webrtc/base/atomicops.h" |
17 #include "webrtc/base/checks.h" | 18 #include "webrtc/base/checks.h" |
18 #include "webrtc/base/logging.h" | 19 #include "webrtc/base/logging.h" |
19 #include "webrtc/base/trace_event.h" | 20 #include "webrtc/base/trace_event.h" |
20 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 21 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
21 #include "webrtc/modules/video_coding/jitter_estimator.h" | 22 #include "webrtc/modules/video_coding/jitter_estimator.h" |
22 #include "webrtc/modules/video_coding/timing.h" | 23 #include "webrtc/modules/video_coding/timing.h" |
23 #include "webrtc/system_wrappers/include/clock.h" | 24 #include "webrtc/system_wrappers/include/clock.h" |
24 #include "webrtc/system_wrappers/include/metrics.h" | 25 #include "webrtc/system_wrappers/include/metrics.h" |
25 | 26 |
26 namespace webrtc { | 27 namespace webrtc { |
27 namespace video_coding { | 28 namespace video_coding { |
28 | 29 |
29 namespace { | 30 namespace { |
30 // Max number of frames the buffer will hold. | 31 // Max number of frames the buffer will hold. |
31 constexpr int kMaxFramesBuffered = 600; | 32 constexpr int kMaxFramesBuffered = 600; |
32 | 33 |
33 // Max number of decoded frame info that will be saved. | 34 // Max number of decoded frame info that will be saved. |
34 constexpr int kMaxFramesHistory = 50; | 35 constexpr int kMaxFramesHistory = 50; |
35 } // namespace | 36 } // namespace |
36 | 37 |
37 FrameBuffer::FrameBuffer(Clock* clock, | 38 FrameBuffer::FrameBuffer(Clock* clock, |
38 VCMJitterEstimator* jitter_estimator, | 39 VCMJitterEstimator* jitter_estimator, |
39 VCMTiming* timing, | 40 VCMTiming* timing, |
40 VCMReceiveStatisticsCallback* stats_callback) | 41 VCMReceiveStatisticsCallback* stats_callback) |
41 : clock_(clock), | 42 : clock_(clock), |
42 new_countinuous_frame_event_(false, false), | 43 new_continuous_frame_event_(false, false), |
43 jitter_estimator_(jitter_estimator), | 44 jitter_estimator_(jitter_estimator), |
44 timing_(timing), | 45 timing_(timing), |
45 inter_frame_delay_(clock_->TimeInMilliseconds()), | 46 inter_frame_delay_(clock_->TimeInMilliseconds()), |
46 last_decoded_frame_it_(frames_.end()), | 47 last_decoded_frame_it_(frames_.end()), |
47 last_continuous_frame_it_(frames_.end()), | 48 last_continuous_frame_it_(frames_.end()), |
48 num_frames_history_(0), | 49 num_frames_history_(0), |
49 num_frames_buffered_(0), | 50 num_frames_buffered_(0), |
50 stopped_(false), | 51 stopped_(0), |
51 protection_mode_(kProtectionNack), | 52 protection_mode_(kProtectionNack), |
52 stats_callback_(stats_callback) {} | 53 stats_callback_(stats_callback) {} |
53 | 54 |
54 FrameBuffer::~FrameBuffer() {} | 55 FrameBuffer::~FrameBuffer() {} |
55 | 56 |
56 FrameBuffer::ReturnReason FrameBuffer::NextFrame( | 57 FrameBuffer::ReturnReason FrameBuffer::NextFrame( |
57 int64_t max_wait_time_ms, | 58 int64_t max_wait_time_ms, |
58 std::unique_ptr<FrameObject>* frame_out) { | 59 std::unique_ptr<FrameObject>* frame_out) { |
59 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); | 60 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); |
60 int64_t latest_return_time_ms = | 61 int64_t latest_return_time_ms = |
61 clock_->TimeInMilliseconds() + max_wait_time_ms; | 62 clock_->TimeInMilliseconds() + max_wait_time_ms; |
62 int64_t wait_ms = max_wait_time_ms; | 63 int64_t wait_ms = max_wait_time_ms; |
63 | 64 |
64 do { | 65 do { |
| 66 if (rtc::AtomicOps::AcquireLoad(&stopped_)) |
| 67 return kStopped; |
| 68 |
65 int64_t now_ms = clock_->TimeInMilliseconds(); | 69 int64_t now_ms = clock_->TimeInMilliseconds(); |
| 70 wait_ms = max_wait_time_ms; |
66 { | 71 { |
67 rtc::CritScope lock(&crit_); | 72 rtc::CritScope lock(&crit_); |
68 new_countinuous_frame_event_.Reset(); | 73 new_continuous_frame_event_.Reset(); |
69 if (stopped_) | |
70 return kStopped; | |
71 | |
72 wait_ms = max_wait_time_ms; | |
73 | |
74 // Need to hold |crit_| in order to use |frames_|, therefore we | 74 // Need to hold |crit_| in order to use |frames_|, therefore we |
75 // set it here in the loop instead of outside the loop in order to not | 75 // set it here in the loop instead of outside the loop in order to not |
76 // acquire the lock unnecesserily. | 76 // acquire the lock unnecesserily. |
77 next_frame_it_ = frames_.end(); | 77 next_frame_it_ = frames_.end(); |
78 | 78 |
79 // |frame_it| points to the first frame after the | 79 // |frame_it| points to the first frame after the |
80 // |last_decoded_frame_it_|. | 80 // |last_decoded_frame_it_|. |
81 auto frame_it = frames_.end(); | 81 auto frame_it = frames_.end(); |
82 if (last_decoded_frame_it_ == frames_.end()) { | 82 if (last_decoded_frame_it_ == frames_.end()) { |
83 frame_it = frames_.begin(); | 83 frame_it = frames_.begin(); |
(...skipping 25 matching lines...) Expand all Loading... |
109 // enough and the stream has multiple spatial and temporal layers. | 109 // enough and the stream has multiple spatial and temporal layers. |
110 if (wait_ms == 0) | 110 if (wait_ms == 0) |
111 continue; | 111 continue; |
112 | 112 |
113 break; | 113 break; |
114 } | 114 } |
115 } // rtc::Critscope lock(&crit_); | 115 } // rtc::Critscope lock(&crit_); |
116 | 116 |
117 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms); | 117 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms); |
118 wait_ms = std::max<int64_t>(wait_ms, 0); | 118 wait_ms = std::max<int64_t>(wait_ms, 0); |
119 } while (new_countinuous_frame_event_.Wait(wait_ms)); | 119 } while (new_continuous_frame_event_.Wait(wait_ms)); |
120 | 120 |
121 rtc::CritScope lock(&crit_); | 121 rtc::CritScope lock(&crit_); |
122 int64_t now_ms = clock_->TimeInMilliseconds(); | 122 int64_t now_ms = clock_->TimeInMilliseconds(); |
123 if (next_frame_it_ != frames_.end()) { | 123 if (next_frame_it_ != frames_.end()) { |
124 std::unique_ptr<FrameObject> frame = | 124 std::unique_ptr<FrameObject> frame = |
125 std::move(next_frame_it_->second.frame); | 125 std::move(next_frame_it_->second.frame); |
126 | 126 |
127 if (!frame->delayed_by_retransmission()) { | 127 if (!frame->delayed_by_retransmission()) { |
128 int64_t frame_delay; | 128 int64_t frame_delay; |
129 | 129 |
130 if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay, | 130 if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay, |
131 frame->ReceivedTime())) { | 131 frame->ReceivedTime())) { |
132 jitter_estimator_->UpdateEstimate(frame_delay, frame->size()); | 132 jitter_estimator_->UpdateEstimate(frame_delay, frame->size()); |
133 } | 133 } |
134 | 134 |
135 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; | 135 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; |
136 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); | 136 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); |
137 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms); | 137 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms); |
138 } | 138 } |
139 | 139 |
140 UpdateJitterDelay(); | 140 UpdateJitterDelay(); |
141 | 141 |
142 PropagateDecodability(next_frame_it_->second); | 142 PropagateDecodability(next_frame_it_->second); |
143 AdvanceLastDecodedFrame(next_frame_it_); | 143 AdvanceLastDecodedFrame(next_frame_it_); |
144 last_decoded_frame_timestamp_ = frame->timestamp; | 144 last_decoded_frame_timestamp_ = frame->timestamp; |
145 *frame_out = std::move(frame); | 145 *frame_out = std::move(frame); |
146 return kFrameFound; | 146 return kFrameFound; |
147 } else if (latest_return_time_ms - now_ms > 0) { | 147 } |
| 148 |
| 149 if (latest_return_time_ms - now_ms > 0) { |
148 // If |next_frame_it_ == frames_.end()| and there is still time left, it | 150 // If |next_frame_it_ == frames_.end()| and there is still time left, it |
149 // means that the frame buffer was cleared as the thread in this function | 151 // means that the frame buffer was cleared as the thread in this function |
150 // was waiting to acquire |crit_| in order to return. Wait for the | 152 // was waiting to acquire |crit_| in order to return. Wait for the |
151 // remaining time and then return. | 153 // remaining time and then return. |
152 return NextFrame(latest_return_time_ms - now_ms, frame_out); | 154 return NextFrame(latest_return_time_ms - now_ms, frame_out); |
153 } else { | |
154 return kTimeout; | |
155 } | 155 } |
| 156 |
| 157 return kTimeout; |
156 } | 158 } |
157 | 159 |
158 void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { | 160 void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { |
159 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); | 161 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); |
160 rtc::CritScope lock(&crit_); | 162 rtc::CritScope lock(&crit_); |
161 protection_mode_ = mode; | 163 protection_mode_ = mode; |
162 } | 164 } |
163 | 165 |
| 166 // Start() and Stop() must be called on the same thread. |
| 167 // The value of stopped_ can only be changed on this thread. |
164 void FrameBuffer::Start() { | 168 void FrameBuffer::Start() { |
165 TRACE_EVENT0("webrtc", "FrameBuffer::Start"); | 169 TRACE_EVENT0("webrtc", "FrameBuffer::Start"); |
166 rtc::CritScope lock(&crit_); | 170 rtc::AtomicOps::ReleaseStore(&stopped_, 0); |
167 stopped_ = false; | |
168 } | 171 } |
169 | 172 |
170 void FrameBuffer::Stop() { | 173 void FrameBuffer::Stop() { |
171 TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); | 174 TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); |
172 rtc::CritScope lock(&crit_); | 175 // TODO(tommi,philipel): Previously, we grabbed the |crit_| lock when decoding |
173 stopped_ = true; | 176 // was being stopped. On Android, with captured frames being delivered on the |
174 new_countinuous_frame_event_.Set(); | 177 // decoder thread, this consistently caused the 'busy loop' checks in the |
| 178 // PlatformThread to trigger on "VoiceProcessThread", "ModuleProcessThread" |
| 179 // and occasionally on "PacerThread". |
| 180 // Look into what was going on and why |Stop()| wasn't able to grab the lock |
| 181 // and stop the FrameBuffer while that happened. |
| 182 // See bug: https://bugs.chromium.org/p/webrtc/issues/detail?id=7331 |
| 183 rtc::AtomicOps::Increment(&stopped_); |
| 184 new_continuous_frame_event_.Set(); |
175 } | 185 } |
176 | 186 |
177 int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) { | 187 int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) { |
178 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); | 188 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); |
179 rtc::CritScope lock(&crit_); | |
180 RTC_DCHECK(frame); | 189 RTC_DCHECK(frame); |
181 | 190 |
| 191 if (rtc::AtomicOps::AcquireLoad(&stopped_)) |
| 192 return -1; |
| 193 |
182 if (stats_callback_) | 194 if (stats_callback_) |
183 stats_callback_->OnCompleteFrame(frame->num_references == 0, frame->size()); | 195 stats_callback_->OnCompleteFrame(frame->num_references == 0, frame->size()); |
184 | 196 |
185 FrameKey key(frame->picture_id, frame->spatial_layer); | 197 FrameKey key(frame->picture_id, frame->spatial_layer); |
| 198 |
| 199 rtc::CritScope lock(&crit_); |
186 int last_continuous_picture_id = | 200 int last_continuous_picture_id = |
187 last_continuous_frame_it_ == frames_.end() | 201 last_continuous_frame_it_ == frames_.end() |
188 ? -1 | 202 ? -1 |
189 : last_continuous_frame_it_->first.picture_id; | 203 : last_continuous_frame_it_->first.picture_id; |
190 | 204 |
191 if (num_frames_buffered_ >= kMaxFramesBuffered) { | 205 if (num_frames_buffered_ >= kMaxFramesBuffered) { |
192 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id | 206 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id |
193 << ":" << static_cast<int>(key.spatial_layer) | 207 << ":" << static_cast<int>(key.spatial_layer) |
194 << ") could not be inserted due to the frame " | 208 << ") could not be inserted due to the frame " |
195 << "buffer being full, dropping frame."; | 209 << "buffer being full, dropping frame."; |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
244 ++num_frames_buffered_; | 258 ++num_frames_buffered_; |
245 | 259 |
246 if (info->second.num_missing_continuous == 0) { | 260 if (info->second.num_missing_continuous == 0) { |
247 info->second.continuous = true; | 261 info->second.continuous = true; |
248 PropagateContinuity(info); | 262 PropagateContinuity(info); |
249 last_continuous_picture_id = last_continuous_frame_it_->first.picture_id; | 263 last_continuous_picture_id = last_continuous_frame_it_->first.picture_id; |
250 | 264 |
251 // Since we now have new continuous frames there might be a better frame | 265 // Since we now have new continuous frames there might be a better frame |
252 // to return from NextFrame. Signal that thread so that it again can choose | 266 // to return from NextFrame. Signal that thread so that it again can choose |
253 // which frame to return. | 267 // which frame to return. |
254 new_countinuous_frame_event_.Set(); | 268 new_continuous_frame_event_.Set(); |
255 } | 269 } |
256 | 270 |
257 return last_continuous_picture_id; | 271 return last_continuous_picture_id; |
258 } | 272 } |
259 | 273 |
260 void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { | 274 void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { |
261 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); | 275 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); |
262 RTC_DCHECK(start->second.continuous); | 276 RTC_DCHECK(start->second.continuous); |
263 if (last_continuous_frame_it_ == frames_.end()) | 277 if (last_continuous_frame_it_ == frames_.end()) |
264 last_continuous_frame_it_ = start; | 278 last_continuous_frame_it_ = start; |
(...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
423 frames_.clear(); | 437 frames_.clear(); |
424 last_decoded_frame_it_ = frames_.end(); | 438 last_decoded_frame_it_ = frames_.end(); |
425 last_continuous_frame_it_ = frames_.end(); | 439 last_continuous_frame_it_ = frames_.end(); |
426 next_frame_it_ = frames_.end(); | 440 next_frame_it_ = frames_.end(); |
427 num_frames_history_ = 0; | 441 num_frames_history_ = 0; |
428 num_frames_buffered_ = 0; | 442 num_frames_buffered_ = 0; |
429 } | 443 } |
430 | 444 |
431 } // namespace video_coding | 445 } // namespace video_coding |
432 } // namespace webrtc | 446 } // namespace webrtc |
OLD | NEW |