| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/modules/video_coding/frame_buffer2.h" | 11 #include "webrtc/modules/video_coding/frame_buffer2.h" |
| 12 | 12 |
| 13 #include <algorithm> | 13 #include <algorithm> |
| 14 #include <cstring> | 14 #include <cstring> |
| 15 #include <queue> | 15 #include <queue> |
| 16 | 16 |
| 17 #include "webrtc/base/atomicops.h" |
| 17 #include "webrtc/base/checks.h" | 18 #include "webrtc/base/checks.h" |
| 18 #include "webrtc/base/logging.h" | 19 #include "webrtc/base/logging.h" |
| 19 #include "webrtc/base/trace_event.h" | 20 #include "webrtc/base/trace_event.h" |
| 20 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 21 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| 21 #include "webrtc/modules/video_coding/jitter_estimator.h" | 22 #include "webrtc/modules/video_coding/jitter_estimator.h" |
| 22 #include "webrtc/modules/video_coding/timing.h" | 23 #include "webrtc/modules/video_coding/timing.h" |
| 23 #include "webrtc/system_wrappers/include/clock.h" | 24 #include "webrtc/system_wrappers/include/clock.h" |
| 24 #include "webrtc/system_wrappers/include/metrics.h" | 25 #include "webrtc/system_wrappers/include/metrics.h" |
| 25 | 26 |
| 26 namespace webrtc { | 27 namespace webrtc { |
| 27 namespace video_coding { | 28 namespace video_coding { |
| 28 | 29 |
| 29 namespace { | 30 namespace { |
| 30 // Max number of frames the buffer will hold. | 31 // Max number of frames the buffer will hold. |
| 31 constexpr int kMaxFramesBuffered = 600; | 32 constexpr int kMaxFramesBuffered = 600; |
| 32 | 33 |
| 33 // Max number of decoded frame info that will be saved. | 34 // Max number of decoded frame info that will be saved. |
| 34 constexpr int kMaxFramesHistory = 50; | 35 constexpr int kMaxFramesHistory = 50; |
| 35 } // namespace | 36 } // namespace |
| 36 | 37 |
| 37 FrameBuffer::FrameBuffer(Clock* clock, | 38 FrameBuffer::FrameBuffer(Clock* clock, |
| 38 VCMJitterEstimator* jitter_estimator, | 39 VCMJitterEstimator* jitter_estimator, |
| 39 VCMTiming* timing, | 40 VCMTiming* timing, |
| 40 VCMReceiveStatisticsCallback* stats_callback) | 41 VCMReceiveStatisticsCallback* stats_callback) |
| 41 : clock_(clock), | 42 : clock_(clock), |
| 42 new_countinuous_frame_event_(false, false), | 43 new_continuous_frame_event_(false, false), |
| 43 jitter_estimator_(jitter_estimator), | 44 jitter_estimator_(jitter_estimator), |
| 44 timing_(timing), | 45 timing_(timing), |
| 45 inter_frame_delay_(clock_->TimeInMilliseconds()), | 46 inter_frame_delay_(clock_->TimeInMilliseconds()), |
| 46 last_decoded_frame_it_(frames_.end()), | 47 last_decoded_frame_it_(frames_.end()), |
| 47 last_continuous_frame_it_(frames_.end()), | 48 last_continuous_frame_it_(frames_.end()), |
| 48 num_frames_history_(0), | 49 num_frames_history_(0), |
| 49 num_frames_buffered_(0), | 50 num_frames_buffered_(0), |
| 50 stopped_(false), | 51 stopped_(0), |
| 51 protection_mode_(kProtectionNack), | 52 protection_mode_(kProtectionNack), |
| 52 stats_callback_(stats_callback) {} | 53 stats_callback_(stats_callback) {} |
| 53 | 54 |
| 54 FrameBuffer::~FrameBuffer() {} | 55 FrameBuffer::~FrameBuffer() {} |
| 55 | 56 |
| 56 FrameBuffer::ReturnReason FrameBuffer::NextFrame( | 57 FrameBuffer::ReturnReason FrameBuffer::NextFrame( |
| 57 int64_t max_wait_time_ms, | 58 int64_t max_wait_time_ms, |
| 58 std::unique_ptr<FrameObject>* frame_out) { | 59 std::unique_ptr<FrameObject>* frame_out) { |
| 59 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); | 60 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); |
| 60 int64_t latest_return_time_ms = | 61 int64_t latest_return_time_ms = |
| 61 clock_->TimeInMilliseconds() + max_wait_time_ms; | 62 clock_->TimeInMilliseconds() + max_wait_time_ms; |
| 62 int64_t wait_ms = max_wait_time_ms; | 63 int64_t wait_ms = max_wait_time_ms; |
| 63 | 64 |
| 64 do { | 65 do { |
| 66 if (rtc::AtomicOps::AcquireLoad(&stopped_)) |
| 67 return kStopped; |
| 68 |
| 65 int64_t now_ms = clock_->TimeInMilliseconds(); | 69 int64_t now_ms = clock_->TimeInMilliseconds(); |
| 70 wait_ms = max_wait_time_ms; |
| 66 { | 71 { |
| 67 rtc::CritScope lock(&crit_); | 72 rtc::CritScope lock(&crit_); |
| 68 new_countinuous_frame_event_.Reset(); | 73 new_continuous_frame_event_.Reset(); |
| 69 if (stopped_) | |
| 70 return kStopped; | |
| 71 | |
| 72 wait_ms = max_wait_time_ms; | |
| 73 | |
| 74 // Need to hold |crit_| in order to use |frames_|, therefore we | 74 // Need to hold |crit_| in order to use |frames_|, therefore we |
| 75 // set it here in the loop instead of outside the loop in order to not | 75 // set it here in the loop instead of outside the loop in order to not |
| 76 // acquire the lock unnecesserily. | 76 // acquire the lock unnecesserily. |
| 77 next_frame_it_ = frames_.end(); | 77 next_frame_it_ = frames_.end(); |
| 78 | 78 |
| 79 // |frame_it| points to the first frame after the | 79 // |frame_it| points to the first frame after the |
| 80 // |last_decoded_frame_it_|. | 80 // |last_decoded_frame_it_|. |
| 81 auto frame_it = frames_.end(); | 81 auto frame_it = frames_.end(); |
| 82 if (last_decoded_frame_it_ == frames_.end()) { | 82 if (last_decoded_frame_it_ == frames_.end()) { |
| 83 frame_it = frames_.begin(); | 83 frame_it = frames_.begin(); |
| (...skipping 25 matching lines...) Expand all Loading... |
| 109 // enough and the stream has multiple spatial and temporal layers. | 109 // enough and the stream has multiple spatial and temporal layers. |
| 110 if (wait_ms == 0) | 110 if (wait_ms == 0) |
| 111 continue; | 111 continue; |
| 112 | 112 |
| 113 break; | 113 break; |
| 114 } | 114 } |
| 115 } // rtc::Critscope lock(&crit_); | 115 } // rtc::Critscope lock(&crit_); |
| 116 | 116 |
| 117 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms); | 117 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms); |
| 118 wait_ms = std::max<int64_t>(wait_ms, 0); | 118 wait_ms = std::max<int64_t>(wait_ms, 0); |
| 119 } while (new_countinuous_frame_event_.Wait(wait_ms)); | 119 } while (new_continuous_frame_event_.Wait(wait_ms)); |
| 120 | 120 |
| 121 rtc::CritScope lock(&crit_); | 121 rtc::CritScope lock(&crit_); |
| 122 int64_t now_ms = clock_->TimeInMilliseconds(); | 122 int64_t now_ms = clock_->TimeInMilliseconds(); |
| 123 if (next_frame_it_ != frames_.end()) { | 123 if (next_frame_it_ != frames_.end()) { |
| 124 std::unique_ptr<FrameObject> frame = | 124 std::unique_ptr<FrameObject> frame = |
| 125 std::move(next_frame_it_->second.frame); | 125 std::move(next_frame_it_->second.frame); |
| 126 | 126 |
| 127 if (!frame->delayed_by_retransmission()) { | 127 if (!frame->delayed_by_retransmission()) { |
| 128 int64_t frame_delay; | 128 int64_t frame_delay; |
| 129 | 129 |
| 130 if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay, | 130 if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay, |
| 131 frame->ReceivedTime())) { | 131 frame->ReceivedTime())) { |
| 132 jitter_estimator_->UpdateEstimate(frame_delay, frame->size()); | 132 jitter_estimator_->UpdateEstimate(frame_delay, frame->size()); |
| 133 } | 133 } |
| 134 | 134 |
| 135 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; | 135 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; |
| 136 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); | 136 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); |
| 137 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms); | 137 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms); |
| 138 } | 138 } |
| 139 | 139 |
| 140 UpdateJitterDelay(); | 140 UpdateJitterDelay(); |
| 141 | 141 |
| 142 PropagateDecodability(next_frame_it_->second); | 142 PropagateDecodability(next_frame_it_->second); |
| 143 AdvanceLastDecodedFrame(next_frame_it_); | 143 AdvanceLastDecodedFrame(next_frame_it_); |
| 144 last_decoded_frame_timestamp_ = frame->timestamp; | 144 last_decoded_frame_timestamp_ = frame->timestamp; |
| 145 *frame_out = std::move(frame); | 145 *frame_out = std::move(frame); |
| 146 return kFrameFound; | 146 return kFrameFound; |
| 147 } else if (latest_return_time_ms - now_ms > 0) { | 147 } |
| 148 |
| 149 if (latest_return_time_ms - now_ms > 0) { |
| 148 // If |next_frame_it_ == frames_.end()| and there is still time left, it | 150 // If |next_frame_it_ == frames_.end()| and there is still time left, it |
| 149 // means that the frame buffer was cleared as the thread in this function | 151 // means that the frame buffer was cleared as the thread in this function |
| 150 // was waiting to acquire |crit_| in order to return. Wait for the | 152 // was waiting to acquire |crit_| in order to return. Wait for the |
| 151 // remaining time and then return. | 153 // remaining time and then return. |
| 152 return NextFrame(latest_return_time_ms - now_ms, frame_out); | 154 return NextFrame(latest_return_time_ms - now_ms, frame_out); |
| 153 } else { | |
| 154 return kTimeout; | |
| 155 } | 155 } |
| 156 |
| 157 return kTimeout; |
| 156 } | 158 } |
| 157 | 159 |
| 158 void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { | 160 void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { |
| 159 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); | 161 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); |
| 160 rtc::CritScope lock(&crit_); | 162 rtc::CritScope lock(&crit_); |
| 161 protection_mode_ = mode; | 163 protection_mode_ = mode; |
| 162 } | 164 } |
| 163 | 165 |
| 166 // Start() and Stop() must be called on the same thread. |
| 167 // The value of stopped_ can only be changed on this thread. |
| 164 void FrameBuffer::Start() { | 168 void FrameBuffer::Start() { |
| 165 TRACE_EVENT0("webrtc", "FrameBuffer::Start"); | 169 TRACE_EVENT0("webrtc", "FrameBuffer::Start"); |
| 166 rtc::CritScope lock(&crit_); | 170 rtc::AtomicOps::ReleaseStore(&stopped_, 0); |
| 167 stopped_ = false; | |
| 168 } | 171 } |
| 169 | 172 |
| 170 void FrameBuffer::Stop() { | 173 void FrameBuffer::Stop() { |
| 171 TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); | 174 TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); |
| 172 rtc::CritScope lock(&crit_); | 175 // TODO(tommi,philipel): Previously, we grabbed the |crit_| lock when decoding |
| 173 stopped_ = true; | 176 // was being stopped. On Android, with captured frames being delivered on the |
| 174 new_countinuous_frame_event_.Set(); | 177 // decoder thread, this consistently caused the 'busy loop' checks in the |
| 178 // PlatformThread to trigger on "VoiceProcessThread", "ModuleProcessThread" |
| 179 // and occasionally on "PacerThread". |
| 180 // Look into what was going on and why |Stop()| wasn't able to grab the lock |
| 181 // and stop the FrameBuffer while that happened. |
| 182 // See bug: https://bugs.chromium.org/p/webrtc/issues/detail?id=7331 |
| 183 rtc::AtomicOps::Increment(&stopped_); |
| 184 new_continuous_frame_event_.Set(); |
| 175 } | 185 } |
| 176 | 186 |
| 177 int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) { | 187 int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) { |
| 178 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); | 188 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); |
| 179 rtc::CritScope lock(&crit_); | |
| 180 RTC_DCHECK(frame); | 189 RTC_DCHECK(frame); |
| 181 | 190 |
| 191 if (rtc::AtomicOps::AcquireLoad(&stopped_)) |
| 192 return -1; |
| 193 |
| 182 if (stats_callback_) | 194 if (stats_callback_) |
| 183 stats_callback_->OnCompleteFrame(frame->num_references == 0, frame->size()); | 195 stats_callback_->OnCompleteFrame(frame->num_references == 0, frame->size()); |
| 184 | 196 |
| 185 FrameKey key(frame->picture_id, frame->spatial_layer); | 197 FrameKey key(frame->picture_id, frame->spatial_layer); |
| 198 |
| 199 rtc::CritScope lock(&crit_); |
| 186 int last_continuous_picture_id = | 200 int last_continuous_picture_id = |
| 187 last_continuous_frame_it_ == frames_.end() | 201 last_continuous_frame_it_ == frames_.end() |
| 188 ? -1 | 202 ? -1 |
| 189 : last_continuous_frame_it_->first.picture_id; | 203 : last_continuous_frame_it_->first.picture_id; |
| 190 | 204 |
| 191 if (num_frames_buffered_ >= kMaxFramesBuffered) { | 205 if (num_frames_buffered_ >= kMaxFramesBuffered) { |
| 192 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id | 206 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id |
| 193 << ":" << static_cast<int>(key.spatial_layer) | 207 << ":" << static_cast<int>(key.spatial_layer) |
| 194 << ") could not be inserted due to the frame " | 208 << ") could not be inserted due to the frame " |
| 195 << "buffer being full, dropping frame."; | 209 << "buffer being full, dropping frame."; |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 244 ++num_frames_buffered_; | 258 ++num_frames_buffered_; |
| 245 | 259 |
| 246 if (info->second.num_missing_continuous == 0) { | 260 if (info->second.num_missing_continuous == 0) { |
| 247 info->second.continuous = true; | 261 info->second.continuous = true; |
| 248 PropagateContinuity(info); | 262 PropagateContinuity(info); |
| 249 last_continuous_picture_id = last_continuous_frame_it_->first.picture_id; | 263 last_continuous_picture_id = last_continuous_frame_it_->first.picture_id; |
| 250 | 264 |
| 251 // Since we now have new continuous frames there might be a better frame | 265 // Since we now have new continuous frames there might be a better frame |
| 252 // to return from NextFrame. Signal that thread so that it again can choose | 266 // to return from NextFrame. Signal that thread so that it again can choose |
| 253 // which frame to return. | 267 // which frame to return. |
| 254 new_countinuous_frame_event_.Set(); | 268 new_continuous_frame_event_.Set(); |
| 255 } | 269 } |
| 256 | 270 |
| 257 return last_continuous_picture_id; | 271 return last_continuous_picture_id; |
| 258 } | 272 } |
| 259 | 273 |
| 260 void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { | 274 void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { |
| 261 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); | 275 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); |
| 262 RTC_DCHECK(start->second.continuous); | 276 RTC_DCHECK(start->second.continuous); |
| 263 if (last_continuous_frame_it_ == frames_.end()) | 277 if (last_continuous_frame_it_ == frames_.end()) |
| 264 last_continuous_frame_it_ = start; | 278 last_continuous_frame_it_ = start; |
| (...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 423 frames_.clear(); | 437 frames_.clear(); |
| 424 last_decoded_frame_it_ = frames_.end(); | 438 last_decoded_frame_it_ = frames_.end(); |
| 425 last_continuous_frame_it_ = frames_.end(); | 439 last_continuous_frame_it_ = frames_.end(); |
| 426 next_frame_it_ = frames_.end(); | 440 next_frame_it_ = frames_.end(); |
| 427 num_frames_history_ = 0; | 441 num_frames_history_ = 0; |
| 428 num_frames_buffered_ = 0; | 442 num_frames_buffered_ = 0; |
| 429 } | 443 } |
| 430 | 444 |
| 431 } // namespace video_coding | 445 } // namespace video_coding |
| 432 } // namespace webrtc | 446 } // namespace webrtc |
| OLD | NEW |