Convert Vp9 Rtp headers to frame references.

webrtc/modules/video_coding/packet_buffer.cc

Index: webrtc/modules/video_coding/packet_buffer.cc
diff --git a/webrtc/modules/video_coding/packet_buffer.cc b/webrtc/modules/video_coding/packet_buffer.cc
index 8cbfc0c66f417c2cd6063f12ef41cd5d22a909fd..8de403934236adf6ed826d520f677cd7d127a31a 100644
--- a/webrtc/modules/video_coding/packet_buffer.cc
+++ b/webrtc/modules/video_coding/packet_buffer.cc
@@ -31,7 +31,8 @@ PacketBuffer::PacketBuffer(size_t start_buffer_size,
       sequence_buffer_(start_buffer_size),
       frame_callback_(frame_callback),
       last_picture_id_(-1),
-      last_unwrap_(-1) {
+      last_unwrap_(-1),
+      current_ss_idx_(0) {
   RTC_DCHECK_LE(start_buffer_size, max_buffer_size);
   // Buffer size must always be a power of 2.
   RTC_DCHECK((start_buffer_size & (start_buffer_size - 1)) == 0);
@@ -217,7 +218,7 @@ void PacketBuffer::ManageFrame(std::unique_ptr<RtpFrameObject> frame) {
       break;
     }
     case kVideoCodecVP9 : {
-      // TODO(philipel): ManageFrameVp9(std::move(frame));
+      ManageFrameVp9(std::move(frame));
       break;
     }
     case kVideoCodecH264 :
@@ -319,7 +320,7 @@ void PacketBuffer::ManageFrameVp8(std::unique_ptr<RtpFrameObject> frame) {
 
   // Find if there has been a gap in fully received frames and save the picture
   // id of those frames in |not_yet_received_frames_|.
-  if (AheadOf<uint8_t, kPicIdLength>(frame->picture_id, last_picture_id_)) {
+  if (AheadOf<uint16_t, kPicIdLength>(frame->picture_id, last_picture_id_)) {

[stefan-webrtc, 2016/04/26 08:15:12]

Do we have a unittest which would have caught this? Otherwise please add one.

Or maybe this wasn't a bug, but a change of internal representation?

[philipel, 2016/04/28 09:40:42]

This was not a bug, I simply wanted to test what would happen if we used 9 or 13
bits instead of 7 bits for picture ids.

If we ever want to use only 15 bit picture id we can simply change kPicIdLength
to 1 << 15.

     last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
     while (last_picture_id_ != frame->picture_id) {
       not_yet_received_frames_.insert(last_picture_id_);
@@ -389,7 +390,7 @@ void PacketBuffer::ManageFrameVp8(std::unique_ptr<RtpFrameObject> frame) {
     auto not_received_frame_it =
              not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
     if (not_received_frame_it != not_yet_received_frames_.end() &&
-        AheadOf<uint8_t, kPicIdLength>(frame->picture_id,
+        AheadOf<uint16_t, kPicIdLength>(frame->picture_id,
                                        *not_received_frame_it)) {
           stashed_frames_.emplace(std::move(frame));
           return;
@@ -436,14 +437,134 @@ void PacketBuffer::CompletedFrameVp8(std::unique_ptr<RtpFrameObject> frame) {
   RetryStashedFrames();
 }
 
-uint16_t PacketBuffer::UnwrapPictureId(uint16_t picture_id) {
+void PacketBuffer::ManageFrameVp9(std::unique_ptr<RtpFrameObject> frame) {
+  size_t index = frame->first_seq_num() % size_;
+  const VCMPacket& packet = data_buffer_[index];
+  const RTPVideoHeaderVP9& codec_header =
+      packet.codecSpecificHeader.codecHeader.VP9;
+
+  if (codec_header.picture_id == kNoPictureId) {
+    ManageFrameGeneric(std::move(frame));
+    return;
+  }
+
+  frame->spatial_layer = codec_header.spatial_idx;
+  frame->inter_layer_predicted = codec_header.inter_layer_predicted;
+  frame->picture_id = codec_header.picture_id % kPicIdLength;

[stefan-webrtc, 2016/04/26 08:15:12]

kPictureIdLength

[philipel, 2016/04/28 09:40:42]

I prefer not, there will be many lines that are already hard to format that will
become longer that 80 chars.

+
   if (last_unwrap_ == -1)
-    last_unwrap_ = picture_id;
+    last_unwrap_ = codec_header.picture_id;
+
+  if (last_picture_id_ == -1)
+    last_picture_id_ = frame->picture_id;
+
+  // GLORIUS flexible mode!

[stefan-webrtc, 2016/04/26 08:15:12]

I don't think we need this comment... :)

[philipel, 2016/04/28 09:40:42]

Removed :P

+  if (codec_header.flexible_mode) {
+    frame->num_references = codec_header.num_ref_pics;
+    for (size_t r = 0; r < frame->num_references; ++r) {

[stefan-webrtc, 2016/04/26 08:15:12]

use index i rather than r.

[philipel, 2016/04/28 09:40:42]

Done.

+      frame->references[r] = Subtract<1 << 16>(frame->picture_id,
+                                               codec_header.pid_diff[r]);
+    }
+
+    CompletedFrameVp9(std::move(frame));
+    return;
+  }
+
+  if (codec_header.ss_data_available) {
+    // Scalability structures can only be sent with tl0 frames.
+    RTC_DCHECK_EQ(0, codec_header.spatial_idx);

[stefan-webrtc, 2016/04/26 08:15:12]

I don't think we'd want to crash if someone gives us an incorrect bitstream with
an ss in a non tl0 frame, so please just log a warning. Also, you're actually
checking spatial_idx and not temporal_idx.

[philipel, 2016/04/28 09:40:42]

SS are now ignored if they are not sent on a base layer frame. Also log a
warning if that happens.

+
+    current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1);
+    scalability_structures_[current_ss_idx_] = codec_header.gof;
+    scalability_structures_[current_ss_idx_].pid_start = frame->picture_id;
+
+    auto pid_and_gof = std::make_pair(frame->picture_id,
+                                   &scalability_structures_[current_ss_idx_]);
+    gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof));
+  }
+
+  // Clean up info for base layers that are to old.

[stefan-webrtc, 2016/04/26 08:15:12]

too old

[philipel, 2016/04/28 09:40:41]

Done.

+  uint8_t old_tl0_pic_idx = codec_header.tl0_pic_idx - kMaxGofSaved;
+  auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx);
+  gof_info_.erase(gof_info_.begin(), clean_gof_info_to);
+
+  if (packet.frameType == kVideoFrameKey) {
+    // When using GOF all keyframes must include the scalability structure.
+    RTC_DCHECK(codec_header.ss_data_available);
+
+    frame->num_references = 0;
+    CompletedFrameVp9(std::move(frame));
+    return;
+  }
+
+  auto gof_info_it = gof_info_.find((codec_header.temporal_idx == 0 &&
+                                     !codec_header.ss_data_available)
+                                        ? codec_header.tl0_pic_idx - 1
+                                        : codec_header.tl0_pic_idx);
+
+  // Gof info for this frame is not available yet, stash this frame.
+  if (gof_info_it == gof_info_.end()) {
+    stashed_frames_.emplace(std::move(frame));

[stefan-webrtc, 2016/04/26 08:15:12]

What happens if we have already deleted the ss that this frame depends on? Will
stashed_frames_ ever be emptied and a key frame requested?

[philipel, 2016/04/28 09:40:42]

RetryStashedFrames have built in clean up logic which prevents frames from
building up.

+    return;
+  }
+
+  uint16_t picture_id_tl0 = gof_info_it->second.first;
+  GofInfoVP9* gof = gof_info_it->second.second;
+
+  // If this is a base layer frame that contains a scalability structure
+  // then gof info has already been inserted earlier, so we only want to
+  // insert if we haven't done so already.
+  if (codec_header.temporal_idx == 0 &&
+      !codec_header.ss_data_available) {
+    auto pid_and_gof = std::make_pair(frame->picture_id, gof);
+    gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof));
+  }
+
+  // Sync frames only depend on their tl0_pic_idx.W

[stefan-webrtc, 2016/04/26 08:15:12]

Erase W.

And is this comment true? I think a frame with the U flag set and temporal idx T
means that it's possible to switch up T+N, but it doesn't mean that following
frames only will depend on tl0, only that they will be limited to depending on
frames with temporal idx <= T.

[philipel, 2016/04/28 09:40:42]

I was thinking of sync frames for Vp8 for some reason. Fixed now.

+  if (codec_header.temporal_up_switch) {
+    frame->num_references = 1;
+    frame->references[0] = picture_id_tl0;
+    CompletedFrameVp9(std::move(frame));
+    return;
+  }
+
+  RTC_DCHECK((AheadOrAt<uint16_t, kPicIdLength>(frame->picture_id,

[stefan-webrtc, 2016/04/26 08:15:12]

Is this a safe DCHECK if someone produces a bit stream just to make us crash?

[philipel, 2016/04/28 09:40:41]

What this check do is to make sure we don't select a SS that is newer than the
SS that this frame belongs to, so it's safe (programming error).

+                                                picture_id_tl0)));
+
+  uint8_t diff = ForwardDiff<uint16_t, kPicIdLength>(gof->pid_start,
+                                                    frame->picture_id);

[stefan-webrtc, 2016/04/26 08:15:12]

May not be correctly indented

[philipel, 2016/04/28 09:40:41]

Done.

+  uint8_t gof_idx = diff % gof->num_frames_in_gof;
+
+  // Populate references according to the scalability structure.
+  frame->num_references = gof->num_ref_pics[gof_idx];
+  for (size_t r = 0; r < frame->num_references; ++r) {

[stefan-webrtc, 2016/04/26 08:15:12]

index i

[philipel, 2016/04/28 09:40:42]

Done.

+    diff = ForwardDiff<uint16_t, kPicIdLength>(gof->pid_start,
+                                              frame->picture_id);

[stefan-webrtc, 2016/04/26 08:15:12]

Indentation.

And actually, you have already computed this diff, right?

[philipel, 2016/04/28 09:40:42]

Done.

+    gof_idx = diff % gof->num_frames_in_gof;

[stefan-webrtc, 2016/04/26 08:15:12]

This too, if I'm not mistaken

[philipel, 2016/04/28 09:40:42]

Done.

+    frame->references[r] = Subtract<1 << 16>(frame->picture_id,
+                              gof->pid_diff[gof_idx][r]);
+  }
+
+  CompletedFrameVp9(std::move(frame));
+}
+
+void PacketBuffer::CompletedFrameVp9(std::unique_ptr<RtpFrameObject> frame) {
+  for (size_t r = 0; r < frame->num_references; ++r)
+    frame->references[r] = UnwrapPictureId(frame->references[r]);
+  frame->picture_id = UnwrapPictureId(frame->picture_id);
+
+  frame_callback_->OnCompleteFrame(std::move(frame));
+  RetryStashedFrames();
+}
+
+
+uint16_t PacketBuffer::UnwrapPictureId(uint16_t picture_id) {
+  RTC_DCHECK_NE(-1, last_unwrap_);
 
   uint16_t unwrap_truncated = last_unwrap_ % kPicIdLength;
-  uint16_t diff = MinDiff<uint8_t, kPicIdLength>(unwrap_truncated, picture_id);
+  uint16_t diff = MinDiff<uint16_t, kPicIdLength>(unwrap_truncated, picture_id);
 
-  if (AheadOf<uint8_t, kPicIdLength>(picture_id, unwrap_truncated))
+  if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated))
     last_unwrap_ = Add<1 << 16>(last_unwrap_, diff);
   else
     last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff);