OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/video_coding/rtp_frame_reference_finder.h" | 11 #include "webrtc/modules/video_coding/rtp_frame_reference_finder.h" |
12 | 12 |
13 #include <algorithm> | 13 #include <algorithm> |
14 #include <limits> | 14 #include <limits> |
15 | 15 |
16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/logging.h" | 17 #include "webrtc/base/logging.h" |
18 #include "webrtc/modules/video_coding/frame_object.h" | 18 #include "webrtc/modules/video_coding/frame_object.h" |
19 #include "webrtc/modules/video_coding/packet_buffer.h" | 19 #include "webrtc/modules/video_coding/packet_buffer.h" |
20 | 20 |
21 namespace webrtc { | 21 namespace webrtc { |
22 namespace video_coding { | 22 namespace video_coding { |
23 | 23 |
24 RtpFrameReferenceFinder::RtpFrameReferenceFinder( | 24 RtpFrameReferenceFinder::RtpFrameReferenceFinder( |
25 OnCompleteFrameCallback* frame_callback) | 25 OnCompleteFrameCallback* frame_callback) |
26 : last_picture_id_(-1), | 26 : last_picture_id_(-1), |
27 last_unwrap_(-1), | 27 last_unwrap_(-1), |
28 current_ss_idx_(0), | 28 current_ss_idx_(0), |
29 cleared_to_seq_num_(-1), | |
29 frame_callback_(frame_callback) {} | 30 frame_callback_(frame_callback) {} |
30 | 31 |
31 void RtpFrameReferenceFinder::ManageFrame( | 32 void RtpFrameReferenceFinder::ManageFrame( |
32 std::unique_ptr<RtpFrameObject> frame) { | 33 std::unique_ptr<RtpFrameObject> frame) { |
33 rtc::CritScope lock(&crit_); | 34 rtc::CritScope lock(&crit_); |
35 | |
36 // If we have cleared past this frame, drop it. | |
37 if (cleared_to_seq_num_ != -1 && | |
38 AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) { | |
39 return; | |
40 } | |
41 | |
34 switch (frame->codec_type()) { | 42 switch (frame->codec_type()) { |
35 case kVideoCodecULPFEC: | 43 case kVideoCodecULPFEC: |
36 case kVideoCodecRED: | 44 case kVideoCodecRED: |
37 case kVideoCodecUnknown: | 45 case kVideoCodecUnknown: |
38 RTC_NOTREACHED(); | 46 RTC_NOTREACHED(); |
39 break; | 47 break; |
40 case kVideoCodecVP8: | 48 case kVideoCodecVP8: |
41 ManageFrameVp8(std::move(frame)); | 49 ManageFrameVp8(std::move(frame)); |
42 break; | 50 break; |
43 case kVideoCodecVP9: | 51 case kVideoCodecVP9: |
(...skipping 10 matching lines...) Expand all Loading... | |
54 void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) { | 62 void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) { |
55 rtc::CritScope lock(&crit_); | 63 rtc::CritScope lock(&crit_); |
56 auto clean_padding_to = | 64 auto clean_padding_to = |
57 stashed_padding_.lower_bound(seq_num - kMaxPaddingAge); | 65 stashed_padding_.lower_bound(seq_num - kMaxPaddingAge); |
58 stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to); | 66 stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to); |
59 stashed_padding_.insert(seq_num); | 67 stashed_padding_.insert(seq_num); |
60 UpdateLastPictureIdWithPadding(seq_num); | 68 UpdateLastPictureIdWithPadding(seq_num); |
61 RetryStashedFrames(); | 69 RetryStashedFrames(); |
62 } | 70 } |
63 | 71 |
72 void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) { | |
73 rtc::CritScope lock(&crit_); | |
74 cleared_to_seq_num_ = seq_num; | |
75 | |
76 auto it = stashed_frames_.begin(); | |
77 while (it != stashed_frames_.end()) { | |
78 if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) | |
stefan-webrtc
2016/09/08 12:18:28
{} since it's a multiline if.
| |
79 it = stashed_frames_.erase(it); | |
80 else | |
81 ++it; | |
82 } | |
83 } | |
84 | |
64 void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) { | 85 void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) { |
65 auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num); | 86 auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num); |
66 | 87 |
67 // If this padding packet "belongs" to a group of pictures that we don't track | 88 // If this padding packet "belongs" to a group of pictures that we don't track |
68 // anymore, do nothing. | 89 // anymore, do nothing. |
69 if (gop_seq_num_it == last_seq_num_gop_.begin()) | 90 if (gop_seq_num_it == last_seq_num_gop_.begin()) |
70 return; | 91 return; |
71 --gop_seq_num_it; | 92 --gop_seq_num_it; |
72 | 93 |
73 // Calculate the next contiuous sequence number and search for it in | 94 // Calculate the next contiuous sequence number and search for it in |
(...skipping 11 matching lines...) Expand all Loading... | |
85 ++next_seq_num_with_padding; | 106 ++next_seq_num_with_padding; |
86 padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); | 107 padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); |
87 } | 108 } |
88 } | 109 } |
89 | 110 |
90 void RtpFrameReferenceFinder::RetryStashedFrames() { | 111 void RtpFrameReferenceFinder::RetryStashedFrames() { |
91 size_t num_stashed_frames = stashed_frames_.size(); | 112 size_t num_stashed_frames = stashed_frames_.size(); |
92 | 113 |
93 // Clean up stashed frames if there are too many. | 114 // Clean up stashed frames if there are too many. |
94 while (stashed_frames_.size() > kMaxStashedFrames) | 115 while (stashed_frames_.size() > kMaxStashedFrames) |
95 stashed_frames_.pop(); | 116 stashed_frames_.pop_front(); |
96 | 117 |
97 // Since frames are stashed if there is not enough data to determine their | 118 // Since frames are stashed if there is not enough data to determine their |
98 // frame references we should at most check |stashed_frames_.size()| in | 119 // frame references we should at most check |stashed_frames_.size()| in |
99 // order to not pop and push frames in and endless loop. | 120 // order to not pop and push frames in and endless loop. |
121 // NOTE! This function may be called recursively, hence the | |
122 // "!stashed_frames_.empty()" condition. | |
100 for (size_t i = 0; i < num_stashed_frames && !stashed_frames_.empty(); ++i) { | 123 for (size_t i = 0; i < num_stashed_frames && !stashed_frames_.empty(); ++i) { |
101 std::unique_ptr<RtpFrameObject> frame = std::move(stashed_frames_.front()); | 124 std::unique_ptr<RtpFrameObject> frame = std::move(stashed_frames_.front()); |
102 stashed_frames_.pop(); | 125 stashed_frames_.pop_front(); |
103 ManageFrame(std::move(frame)); | 126 ManageFrame(std::move(frame)); |
104 } | 127 } |
105 } | 128 } |
106 | 129 |
107 void RtpFrameReferenceFinder::ManageFrameGeneric( | 130 void RtpFrameReferenceFinder::ManageFrameGeneric( |
108 std::unique_ptr<RtpFrameObject> frame, | 131 std::unique_ptr<RtpFrameObject> frame, |
109 int picture_id) { | 132 int picture_id) { |
110 // If |picture_id| is specified then we use that to set the frame references, | 133 // If |picture_id| is specified then we use that to set the frame references, |
111 // otherwise we use sequence number. | 134 // otherwise we use sequence number. |
112 if (picture_id != kNoPictureId) { | 135 if (picture_id != kNoPictureId) { |
113 if (last_unwrap_ == -1) | 136 if (last_unwrap_ == -1) |
114 last_unwrap_ = picture_id; | 137 last_unwrap_ = picture_id; |
115 | 138 |
116 frame->picture_id = UnwrapPictureId(picture_id % kPicIdLength); | 139 frame->picture_id = UnwrapPictureId(picture_id % kPicIdLength); |
117 frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1; | 140 frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1; |
118 frame->references[0] = frame->picture_id - 1; | 141 frame->references[0] = frame->picture_id - 1; |
119 frame_callback_->OnCompleteFrame(std::move(frame)); | 142 frame_callback_->OnCompleteFrame(std::move(frame)); |
120 return; | 143 return; |
121 } | 144 } |
122 | 145 |
123 if (frame->frame_type() == kVideoFrameKey) { | 146 if (frame->frame_type() == kVideoFrameKey) { |
124 last_seq_num_gop_.insert(std::make_pair( | 147 last_seq_num_gop_.insert(std::make_pair( |
125 frame->last_seq_num(), | 148 frame->last_seq_num(), |
126 std::make_pair(frame->last_seq_num(), frame->last_seq_num()))); | 149 std::make_pair(frame->last_seq_num(), frame->last_seq_num()))); |
127 } | 150 } |
128 | 151 |
129 // We have received a frame but not yet a keyframe, stash this frame. | 152 // We have received a frame but not yet a keyframe, stash this frame. |
130 if (last_seq_num_gop_.empty()) { | 153 if (last_seq_num_gop_.empty()) { |
131 stashed_frames_.emplace(std::move(frame)); | 154 stashed_frames_.push_back(std::move(frame)); |
132 return; | 155 return; |
133 } | 156 } |
134 | 157 |
135 // Clean up info for old keyframes but make sure to keep info | 158 // Clean up info for old keyframes but make sure to keep info |
136 // for the last keyframe. | 159 // for the last keyframe. |
137 auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100); | 160 auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100); |
138 if (clean_to != last_seq_num_gop_.end()) | 161 if (clean_to != last_seq_num_gop_.end()) |
139 last_seq_num_gop_.erase(last_seq_num_gop_.begin(), clean_to); | 162 last_seq_num_gop_.erase(last_seq_num_gop_.begin(), clean_to); |
140 | 163 |
141 // Find the last sequence number of the last frame for the keyframe | 164 // Find the last sequence number of the last frame for the keyframe |
142 // that this frame indirectly references. | 165 // that this frame indirectly references. |
143 auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num()); | 166 auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num()); |
144 if (seq_num_it == last_seq_num_gop_.begin()) { | 167 if (seq_num_it == last_seq_num_gop_.begin()) { |
145 LOG(LS_WARNING) << "Generic frame with packet range [" | 168 LOG(LS_WARNING) << "Generic frame with packet range [" |
146 << frame->first_seq_num() << ", " << frame->last_seq_num() | 169 << frame->first_seq_num() << ", " << frame->last_seq_num() |
147 << "] has no Gop, dropping frame."; | 170 << "] has no Gop, dropping frame."; |
148 return; | 171 return; |
149 } | 172 } |
150 seq_num_it--; | 173 seq_num_it--; |
151 | 174 |
152 // Make sure the packet sequence numbers are continuous, otherwise stash | 175 // Make sure the packet sequence numbers are continuous, otherwise stash |
153 // this frame. | 176 // this frame. |
154 uint16_t last_picture_id_gop = seq_num_it->second.first; | 177 uint16_t last_picture_id_gop = seq_num_it->second.first; |
155 uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second; | 178 uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second; |
156 if (frame->frame_type() == kVideoFrameDelta) { | 179 if (frame->frame_type() == kVideoFrameDelta) { |
157 uint16_t prev_seq_num = frame->first_seq_num() - 1; | 180 uint16_t prev_seq_num = frame->first_seq_num() - 1; |
158 if (prev_seq_num != last_picture_id_with_padding_gop) { | 181 if (prev_seq_num != last_picture_id_with_padding_gop) { |
159 stashed_frames_.emplace(std::move(frame)); | 182 stashed_frames_.push_back(std::move(frame)); |
160 return; | 183 return; |
161 } | 184 } |
162 } | 185 } |
163 | 186 |
164 RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first)); | 187 RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first)); |
165 | 188 |
166 // Since keyframes can cause reordering we can't simply assign the | 189 // Since keyframes can cause reordering we can't simply assign the |
167 // picture id according to some incrementing counter. | 190 // picture id according to some incrementing counter. |
168 frame->picture_id = frame->last_seq_num(); | 191 frame->picture_id = frame->last_seq_num(); |
169 frame->num_references = frame->frame_type() == kVideoFrameDelta; | 192 frame->num_references = frame->frame_type() == kVideoFrameDelta; |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
230 CompletedFrameVp8(std::move(frame)); | 253 CompletedFrameVp8(std::move(frame)); |
231 return; | 254 return; |
232 } | 255 } |
233 | 256 |
234 auto layer_info_it = layer_info_.find(codec_header.temporalIdx == 0 | 257 auto layer_info_it = layer_info_.find(codec_header.temporalIdx == 0 |
235 ? codec_header.tl0PicIdx - 1 | 258 ? codec_header.tl0PicIdx - 1 |
236 : codec_header.tl0PicIdx); | 259 : codec_header.tl0PicIdx); |
237 | 260 |
238 // If we don't have the base layer frame yet, stash this frame. | 261 // If we don't have the base layer frame yet, stash this frame. |
239 if (layer_info_it == layer_info_.end()) { | 262 if (layer_info_it == layer_info_.end()) { |
240 stashed_frames_.emplace(std::move(frame)); | 263 stashed_frames_.push_back(std::move(frame)); |
241 return; | 264 return; |
242 } | 265 } |
243 | 266 |
244 // A non keyframe base layer frame has been received, copy the layer info | 267 // A non keyframe base layer frame has been received, copy the layer info |
245 // from the previous base layer frame and set a reference to the previous | 268 // from the previous base layer frame and set a reference to the previous |
246 // base layer frame. | 269 // base layer frame. |
247 if (codec_header.temporalIdx == 0) { | 270 if (codec_header.temporalIdx == 0) { |
248 layer_info_it = | 271 layer_info_it = |
249 layer_info_ | 272 layer_info_ |
250 .insert(make_pair(codec_header.tl0PicIdx, layer_info_it->second)) | 273 .insert(make_pair(codec_header.tl0PicIdx, layer_info_it->second)) |
(...skipping 18 matching lines...) Expand all Loading... | |
269 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { | 292 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { |
270 RTC_DCHECK_NE(-1, layer_info_it->second[layer]); | 293 RTC_DCHECK_NE(-1, layer_info_it->second[layer]); |
271 | 294 |
272 // If we have not yet received a frame between this frame and the referenced | 295 // If we have not yet received a frame between this frame and the referenced |
273 // frame then we have to wait for that frame to be completed first. | 296 // frame then we have to wait for that frame to be completed first. |
274 auto not_received_frame_it = | 297 auto not_received_frame_it = |
275 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); | 298 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); |
276 if (not_received_frame_it != not_yet_received_frames_.end() && | 299 if (not_received_frame_it != not_yet_received_frames_.end() && |
277 AheadOf<uint16_t, kPicIdLength>(frame->picture_id, | 300 AheadOf<uint16_t, kPicIdLength>(frame->picture_id, |
278 *not_received_frame_it)) { | 301 *not_received_frame_it)) { |
279 stashed_frames_.emplace(std::move(frame)); | 302 stashed_frames_.push_back(std::move(frame)); |
280 return; | 303 return; |
281 } | 304 } |
282 | 305 |
283 ++frame->num_references; | 306 ++frame->num_references; |
284 frame->references[layer] = layer_info_it->second[layer]; | 307 frame->references[layer] = layer_info_it->second[layer]; |
285 } | 308 } |
286 | 309 |
287 CompletedFrameVp8(std::move(frame)); | 310 CompletedFrameVp8(std::move(frame)); |
288 } | 311 } |
289 | 312 |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
391 return; | 414 return; |
392 } | 415 } |
393 | 416 |
394 auto gof_info_it = gof_info_.find( | 417 auto gof_info_it = gof_info_.find( |
395 (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) | 418 (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) |
396 ? codec_header.tl0_pic_idx - 1 | 419 ? codec_header.tl0_pic_idx - 1 |
397 : codec_header.tl0_pic_idx); | 420 : codec_header.tl0_pic_idx); |
398 | 421 |
399 // Gof info for this frame is not available yet, stash this frame. | 422 // Gof info for this frame is not available yet, stash this frame. |
400 if (gof_info_it == gof_info_.end()) { | 423 if (gof_info_it == gof_info_.end()) { |
401 stashed_frames_.emplace(std::move(frame)); | 424 stashed_frames_.push_back(std::move(frame)); |
402 return; | 425 return; |
403 } | 426 } |
404 | 427 |
405 GofInfo* info = &gof_info_it->second; | 428 GofInfo* info = &gof_info_it->second; |
406 FrameReceivedVp9(frame->picture_id, info); | 429 FrameReceivedVp9(frame->picture_id, info); |
407 | 430 |
408 // Make sure we don't miss any frame that could potentially have the | 431 // Make sure we don't miss any frame that could potentially have the |
409 // up switch flag set. | 432 // up switch flag set. |
410 if (MissingRequiredFrameVp9(frame->picture_id, *info)) { | 433 if (MissingRequiredFrameVp9(frame->picture_id, *info)) { |
411 stashed_frames_.emplace(std::move(frame)); | 434 stashed_frames_.push_back(std::move(frame)); |
412 return; | 435 return; |
413 } | 436 } |
414 | 437 |
415 if (codec_header.temporal_up_switch) { | 438 if (codec_header.temporal_up_switch) { |
416 auto pid_tidx = | 439 auto pid_tidx = |
417 std::make_pair(frame->picture_id, codec_header.temporal_idx); | 440 std::make_pair(frame->picture_id, codec_header.temporal_idx); |
418 up_switch_.insert(pid_tidx); | 441 up_switch_.insert(pid_tidx); |
419 } | 442 } |
420 | 443 |
421 // If this is a base layer frame that contains a scalability structure | 444 // If this is a base layer frame that contains a scalability structure |
(...skipping 118 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
540 if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated)) | 563 if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated)) |
541 last_unwrap_ = Add<1 << 16>(last_unwrap_, diff); | 564 last_unwrap_ = Add<1 << 16>(last_unwrap_, diff); |
542 else | 565 else |
543 last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff); | 566 last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff); |
544 | 567 |
545 return last_unwrap_; | 568 return last_unwrap_; |
546 } | 569 } |
547 | 570 |
548 } // namespace video_coding | 571 } // namespace video_coding |
549 } // namespace webrtc | 572 } // namespace webrtc |
OLD | NEW |