OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/video_coding/rtp_frame_reference_finder.h" | 11 #include "webrtc/modules/video_coding/rtp_frame_reference_finder.h" |
12 | 12 |
13 #include <algorithm> | 13 #include <algorithm> |
14 #include <limits> | 14 #include <limits> |
15 | 15 |
16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/logging.h" | 17 #include "webrtc/base/logging.h" |
18 #include "webrtc/modules/video_coding/frame_object.h" | 18 #include "webrtc/modules/video_coding/frame_object.h" |
19 #include "webrtc/modules/video_coding/packet_buffer.h" | 19 #include "webrtc/modules/video_coding/packet_buffer.h" |
20 | 20 |
21 namespace webrtc { | 21 namespace webrtc { |
22 namespace video_coding { | 22 namespace video_coding { |
23 | 23 |
24 RtpFrameReferenceFinder::RtpFrameReferenceFinder( | 24 RtpFrameReferenceFinder::RtpFrameReferenceFinder( |
25 OnCompleteFrameCallback* frame_callback) | 25 OnCompleteFrameCallback* frame_callback) |
26 : last_picture_id_(-1), | 26 : last_picture_id_(-1), |
27 last_unwrap_(-1), | 27 last_unwrap_(-1), |
28 current_ss_idx_(0), | 28 current_ss_idx_(0), |
| 29 cleared_to_seq_num_(-1), |
29 frame_callback_(frame_callback) {} | 30 frame_callback_(frame_callback) {} |
30 | 31 |
31 void RtpFrameReferenceFinder::ManageFrame( | 32 void RtpFrameReferenceFinder::ManageFrame( |
32 std::unique_ptr<RtpFrameObject> frame) { | 33 std::unique_ptr<RtpFrameObject> frame) { |
33 rtc::CritScope lock(&crit_); | 34 rtc::CritScope lock(&crit_); |
| 35 |
| 36 // If we have cleared past this frame, drop it. |
| 37 if (cleared_to_seq_num_ != -1 && |
| 38 AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) { |
| 39 return; |
| 40 } |
| 41 |
34 switch (frame->codec_type()) { | 42 switch (frame->codec_type()) { |
35 case kVideoCodecULPFEC: | 43 case kVideoCodecULPFEC: |
36 case kVideoCodecRED: | 44 case kVideoCodecRED: |
37 case kVideoCodecUnknown: | 45 case kVideoCodecUnknown: |
38 RTC_NOTREACHED(); | 46 RTC_NOTREACHED(); |
39 break; | 47 break; |
40 case kVideoCodecVP8: | 48 case kVideoCodecVP8: |
41 ManageFrameVp8(std::move(frame)); | 49 ManageFrameVp8(std::move(frame)); |
42 break; | 50 break; |
43 case kVideoCodecVP9: | 51 case kVideoCodecVP9: |
(...skipping 10 matching lines...) Expand all Loading... |
54 void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) { | 62 void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) { |
55 rtc::CritScope lock(&crit_); | 63 rtc::CritScope lock(&crit_); |
56 auto clean_padding_to = | 64 auto clean_padding_to = |
57 stashed_padding_.lower_bound(seq_num - kMaxPaddingAge); | 65 stashed_padding_.lower_bound(seq_num - kMaxPaddingAge); |
58 stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to); | 66 stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to); |
59 stashed_padding_.insert(seq_num); | 67 stashed_padding_.insert(seq_num); |
60 UpdateLastPictureIdWithPadding(seq_num); | 68 UpdateLastPictureIdWithPadding(seq_num); |
61 RetryStashedFrames(); | 69 RetryStashedFrames(); |
62 } | 70 } |
63 | 71 |
| 72 void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) { |
| 73 rtc::CritScope lock(&crit_); |
| 74 cleared_to_seq_num_ = seq_num; |
| 75 |
| 76 auto it = stashed_frames_.begin(); |
| 77 while (it != stashed_frames_.end()) { |
| 78 if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) { |
| 79 it = stashed_frames_.erase(it); |
| 80 } else { |
| 81 ++it; |
| 82 } |
| 83 } |
| 84 } |
| 85 |
64 void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) { | 86 void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) { |
65 auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num); | 87 auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num); |
66 | 88 |
67 // If this padding packet "belongs" to a group of pictures that we don't track | 89 // If this padding packet "belongs" to a group of pictures that we don't track |
68 // anymore, do nothing. | 90 // anymore, do nothing. |
69 if (gop_seq_num_it == last_seq_num_gop_.begin()) | 91 if (gop_seq_num_it == last_seq_num_gop_.begin()) |
70 return; | 92 return; |
71 --gop_seq_num_it; | 93 --gop_seq_num_it; |
72 | 94 |
73 // Calculate the next contiuous sequence number and search for it in | 95 // Calculate the next contiuous sequence number and search for it in |
(...skipping 11 matching lines...) Expand all Loading... |
85 ++next_seq_num_with_padding; | 107 ++next_seq_num_with_padding; |
86 padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); | 108 padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); |
87 } | 109 } |
88 } | 110 } |
89 | 111 |
90 void RtpFrameReferenceFinder::RetryStashedFrames() { | 112 void RtpFrameReferenceFinder::RetryStashedFrames() { |
91 size_t num_stashed_frames = stashed_frames_.size(); | 113 size_t num_stashed_frames = stashed_frames_.size(); |
92 | 114 |
93 // Clean up stashed frames if there are too many. | 115 // Clean up stashed frames if there are too many. |
94 while (stashed_frames_.size() > kMaxStashedFrames) | 116 while (stashed_frames_.size() > kMaxStashedFrames) |
95 stashed_frames_.pop(); | 117 stashed_frames_.pop_front(); |
96 | 118 |
97 // Since frames are stashed if there is not enough data to determine their | 119 // Since frames are stashed if there is not enough data to determine their |
98 // frame references we should at most check |stashed_frames_.size()| in | 120 // frame references we should at most check |stashed_frames_.size()| in |
99 // order to not pop and push frames in and endless loop. | 121 // order to not pop and push frames in and endless loop. |
| 122 // NOTE! This function may be called recursively, hence the |
| 123 // "!stashed_frames_.empty()" condition. |
100 for (size_t i = 0; i < num_stashed_frames && !stashed_frames_.empty(); ++i) { | 124 for (size_t i = 0; i < num_stashed_frames && !stashed_frames_.empty(); ++i) { |
101 std::unique_ptr<RtpFrameObject> frame = std::move(stashed_frames_.front()); | 125 std::unique_ptr<RtpFrameObject> frame = std::move(stashed_frames_.front()); |
102 stashed_frames_.pop(); | 126 stashed_frames_.pop_front(); |
103 ManageFrame(std::move(frame)); | 127 ManageFrame(std::move(frame)); |
104 } | 128 } |
105 } | 129 } |
106 | 130 |
107 void RtpFrameReferenceFinder::ManageFrameGeneric( | 131 void RtpFrameReferenceFinder::ManageFrameGeneric( |
108 std::unique_ptr<RtpFrameObject> frame, | 132 std::unique_ptr<RtpFrameObject> frame, |
109 int picture_id) { | 133 int picture_id) { |
110 // If |picture_id| is specified then we use that to set the frame references, | 134 // If |picture_id| is specified then we use that to set the frame references, |
111 // otherwise we use sequence number. | 135 // otherwise we use sequence number. |
112 if (picture_id != kNoPictureId) { | 136 if (picture_id != kNoPictureId) { |
113 if (last_unwrap_ == -1) | 137 if (last_unwrap_ == -1) |
114 last_unwrap_ = picture_id; | 138 last_unwrap_ = picture_id; |
115 | 139 |
116 frame->picture_id = UnwrapPictureId(picture_id % kPicIdLength); | 140 frame->picture_id = UnwrapPictureId(picture_id % kPicIdLength); |
117 frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1; | 141 frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1; |
118 frame->references[0] = frame->picture_id - 1; | 142 frame->references[0] = frame->picture_id - 1; |
119 frame_callback_->OnCompleteFrame(std::move(frame)); | 143 frame_callback_->OnCompleteFrame(std::move(frame)); |
120 return; | 144 return; |
121 } | 145 } |
122 | 146 |
123 if (frame->frame_type() == kVideoFrameKey) { | 147 if (frame->frame_type() == kVideoFrameKey) { |
124 last_seq_num_gop_.insert(std::make_pair( | 148 last_seq_num_gop_.insert(std::make_pair( |
125 frame->last_seq_num(), | 149 frame->last_seq_num(), |
126 std::make_pair(frame->last_seq_num(), frame->last_seq_num()))); | 150 std::make_pair(frame->last_seq_num(), frame->last_seq_num()))); |
127 } | 151 } |
128 | 152 |
129 // We have received a frame but not yet a keyframe, stash this frame. | 153 // We have received a frame but not yet a keyframe, stash this frame. |
130 if (last_seq_num_gop_.empty()) { | 154 if (last_seq_num_gop_.empty()) { |
131 stashed_frames_.emplace(std::move(frame)); | 155 stashed_frames_.push_back(std::move(frame)); |
132 return; | 156 return; |
133 } | 157 } |
134 | 158 |
135 // Clean up info for old keyframes but make sure to keep info | 159 // Clean up info for old keyframes but make sure to keep info |
136 // for the last keyframe. | 160 // for the last keyframe. |
137 auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100); | 161 auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100); |
138 if (clean_to != last_seq_num_gop_.end()) | 162 if (clean_to != last_seq_num_gop_.end()) |
139 last_seq_num_gop_.erase(last_seq_num_gop_.begin(), clean_to); | 163 last_seq_num_gop_.erase(last_seq_num_gop_.begin(), clean_to); |
140 | 164 |
141 // Find the last sequence number of the last frame for the keyframe | 165 // Find the last sequence number of the last frame for the keyframe |
142 // that this frame indirectly references. | 166 // that this frame indirectly references. |
143 auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num()); | 167 auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num()); |
144 if (seq_num_it == last_seq_num_gop_.begin()) { | 168 if (seq_num_it == last_seq_num_gop_.begin()) { |
145 LOG(LS_WARNING) << "Generic frame with packet range [" | 169 LOG(LS_WARNING) << "Generic frame with packet range [" |
146 << frame->first_seq_num() << ", " << frame->last_seq_num() | 170 << frame->first_seq_num() << ", " << frame->last_seq_num() |
147 << "] has no Gop, dropping frame."; | 171 << "] has no Gop, dropping frame."; |
148 return; | 172 return; |
149 } | 173 } |
150 seq_num_it--; | 174 seq_num_it--; |
151 | 175 |
152 // Make sure the packet sequence numbers are continuous, otherwise stash | 176 // Make sure the packet sequence numbers are continuous, otherwise stash |
153 // this frame. | 177 // this frame. |
154 uint16_t last_picture_id_gop = seq_num_it->second.first; | 178 uint16_t last_picture_id_gop = seq_num_it->second.first; |
155 uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second; | 179 uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second; |
156 if (frame->frame_type() == kVideoFrameDelta) { | 180 if (frame->frame_type() == kVideoFrameDelta) { |
157 uint16_t prev_seq_num = frame->first_seq_num() - 1; | 181 uint16_t prev_seq_num = frame->first_seq_num() - 1; |
158 if (prev_seq_num != last_picture_id_with_padding_gop) { | 182 if (prev_seq_num != last_picture_id_with_padding_gop) { |
159 stashed_frames_.emplace(std::move(frame)); | 183 stashed_frames_.push_back(std::move(frame)); |
160 return; | 184 return; |
161 } | 185 } |
162 } | 186 } |
163 | 187 |
164 RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first)); | 188 RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first)); |
165 | 189 |
166 // Since keyframes can cause reordering we can't simply assign the | 190 // Since keyframes can cause reordering we can't simply assign the |
167 // picture id according to some incrementing counter. | 191 // picture id according to some incrementing counter. |
168 frame->picture_id = frame->last_seq_num(); | 192 frame->picture_id = frame->last_seq_num(); |
169 frame->num_references = frame->frame_type() == kVideoFrameDelta; | 193 frame->num_references = frame->frame_type() == kVideoFrameDelta; |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
230 CompletedFrameVp8(std::move(frame)); | 254 CompletedFrameVp8(std::move(frame)); |
231 return; | 255 return; |
232 } | 256 } |
233 | 257 |
234 auto layer_info_it = layer_info_.find(codec_header.temporalIdx == 0 | 258 auto layer_info_it = layer_info_.find(codec_header.temporalIdx == 0 |
235 ? codec_header.tl0PicIdx - 1 | 259 ? codec_header.tl0PicIdx - 1 |
236 : codec_header.tl0PicIdx); | 260 : codec_header.tl0PicIdx); |
237 | 261 |
238 // If we don't have the base layer frame yet, stash this frame. | 262 // If we don't have the base layer frame yet, stash this frame. |
239 if (layer_info_it == layer_info_.end()) { | 263 if (layer_info_it == layer_info_.end()) { |
240 stashed_frames_.emplace(std::move(frame)); | 264 stashed_frames_.push_back(std::move(frame)); |
241 return; | 265 return; |
242 } | 266 } |
243 | 267 |
244 // A non keyframe base layer frame has been received, copy the layer info | 268 // A non keyframe base layer frame has been received, copy the layer info |
245 // from the previous base layer frame and set a reference to the previous | 269 // from the previous base layer frame and set a reference to the previous |
246 // base layer frame. | 270 // base layer frame. |
247 if (codec_header.temporalIdx == 0) { | 271 if (codec_header.temporalIdx == 0) { |
248 layer_info_it = | 272 layer_info_it = |
249 layer_info_ | 273 layer_info_ |
250 .insert(make_pair(codec_header.tl0PicIdx, layer_info_it->second)) | 274 .insert(make_pair(codec_header.tl0PicIdx, layer_info_it->second)) |
(...skipping 18 matching lines...) Expand all Loading... |
269 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { | 293 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { |
270 RTC_DCHECK_NE(-1, layer_info_it->second[layer]); | 294 RTC_DCHECK_NE(-1, layer_info_it->second[layer]); |
271 | 295 |
272 // If we have not yet received a frame between this frame and the referenced | 296 // If we have not yet received a frame between this frame and the referenced |
273 // frame then we have to wait for that frame to be completed first. | 297 // frame then we have to wait for that frame to be completed first. |
274 auto not_received_frame_it = | 298 auto not_received_frame_it = |
275 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); | 299 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); |
276 if (not_received_frame_it != not_yet_received_frames_.end() && | 300 if (not_received_frame_it != not_yet_received_frames_.end() && |
277 AheadOf<uint16_t, kPicIdLength>(frame->picture_id, | 301 AheadOf<uint16_t, kPicIdLength>(frame->picture_id, |
278 *not_received_frame_it)) { | 302 *not_received_frame_it)) { |
279 stashed_frames_.emplace(std::move(frame)); | 303 stashed_frames_.push_back(std::move(frame)); |
280 return; | 304 return; |
281 } | 305 } |
282 | 306 |
283 ++frame->num_references; | 307 ++frame->num_references; |
284 frame->references[layer] = layer_info_it->second[layer]; | 308 frame->references[layer] = layer_info_it->second[layer]; |
285 } | 309 } |
286 | 310 |
287 CompletedFrameVp8(std::move(frame)); | 311 CompletedFrameVp8(std::move(frame)); |
288 } | 312 } |
289 | 313 |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
391 return; | 415 return; |
392 } | 416 } |
393 | 417 |
394 auto gof_info_it = gof_info_.find( | 418 auto gof_info_it = gof_info_.find( |
395 (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) | 419 (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) |
396 ? codec_header.tl0_pic_idx - 1 | 420 ? codec_header.tl0_pic_idx - 1 |
397 : codec_header.tl0_pic_idx); | 421 : codec_header.tl0_pic_idx); |
398 | 422 |
399 // Gof info for this frame is not available yet, stash this frame. | 423 // Gof info for this frame is not available yet, stash this frame. |
400 if (gof_info_it == gof_info_.end()) { | 424 if (gof_info_it == gof_info_.end()) { |
401 stashed_frames_.emplace(std::move(frame)); | 425 stashed_frames_.push_back(std::move(frame)); |
402 return; | 426 return; |
403 } | 427 } |
404 | 428 |
405 GofInfo* info = &gof_info_it->second; | 429 GofInfo* info = &gof_info_it->second; |
406 FrameReceivedVp9(frame->picture_id, info); | 430 FrameReceivedVp9(frame->picture_id, info); |
407 | 431 |
408 // Make sure we don't miss any frame that could potentially have the | 432 // Make sure we don't miss any frame that could potentially have the |
409 // up switch flag set. | 433 // up switch flag set. |
410 if (MissingRequiredFrameVp9(frame->picture_id, *info)) { | 434 if (MissingRequiredFrameVp9(frame->picture_id, *info)) { |
411 stashed_frames_.emplace(std::move(frame)); | 435 stashed_frames_.push_back(std::move(frame)); |
412 return; | 436 return; |
413 } | 437 } |
414 | 438 |
415 if (codec_header.temporal_up_switch) { | 439 if (codec_header.temporal_up_switch) { |
416 auto pid_tidx = | 440 auto pid_tidx = |
417 std::make_pair(frame->picture_id, codec_header.temporal_idx); | 441 std::make_pair(frame->picture_id, codec_header.temporal_idx); |
418 up_switch_.insert(pid_tidx); | 442 up_switch_.insert(pid_tidx); |
419 } | 443 } |
420 | 444 |
421 // If this is a base layer frame that contains a scalability structure | 445 // If this is a base layer frame that contains a scalability structure |
(...skipping 118 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
540 if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated)) | 564 if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated)) |
541 last_unwrap_ = Add<1 << 16>(last_unwrap_, diff); | 565 last_unwrap_ = Add<1 << 16>(last_unwrap_, diff); |
542 else | 566 else |
543 last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff); | 567 last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff); |
544 | 568 |
545 return last_unwrap_; | 569 return last_unwrap_; |
546 } | 570 } |
547 | 571 |
548 } // namespace video_coding | 572 } // namespace video_coding |
549 } // namespace webrtc | 573 } // namespace webrtc |
OLD | NEW |