Convert Vp9 Rtp headers to frame references.

webrtc/modules/video_coding/packet_buffer.cc

 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
                            OnCompleteFrameCallback* frame_callback)
     : size_(start_buffer_size),
       max_size_(max_buffer_size),
       first_seq_num_(0),
       last_seq_num_(0),
       first_packet_received_(false),
       data_buffer_(start_buffer_size),
       sequence_buffer_(start_buffer_size),
       frame_callback_(frame_callback),
       last_picture_id_(-1),
-      last_unwrap_(-1) {
+      last_unwrap_(-1),
+      current_ss_idx_(0) {
   RTC_DCHECK_LE(start_buffer_size, max_buffer_size);
   // Buffer size must always be a power of 2.
   RTC_DCHECK((start_buffer_size & (start_buffer_size - 1)) == 0);
   RTC_DCHECK((max_buffer_size & (max_buffer_size - 1)) == 0);
 }
 
 bool PacketBuffer::InsertPacket(const VCMPacket& packet) {
   rtc::CritScope lock(&crit_);
   uint16_t seq_num = packet.seqNum;
   size_t index = seq_num % size_;
@@ skipping 165 matching lines @@
     case kVideoCodecULPFEC :
     case kVideoCodecRED :
     case kVideoCodecUnknown : {
       RTC_NOTREACHED();
     }
     case kVideoCodecVP8 : {
       ManageFrameVp8(std::move(frame));
       break;
     }
     case kVideoCodecVP9 : {
-      // TODO(philipel): ManageFrameVp9(std::move(frame));
+      ManageFrameVp9(std::move(frame));
       break;
     }
     case kVideoCodecH264 :
     case kVideoCodecI420 :
     case kVideoCodecGeneric :
     default : {
       ManageFrameGeneric(std::move(frame));
     }
   }
 }
@@ skipping 81 matching lines @@
   frame->picture_id = codec_header.pictureId % kPicIdLength;
 
   if (last_unwrap_ == -1)
     last_unwrap_ = codec_header.pictureId;
 
   if (last_picture_id_ == -1)
     last_picture_id_ = frame->picture_id;
 
   // Find if there has been a gap in fully received frames and save the picture
   // id of those frames in |not_yet_received_frames_|.
-  if (AheadOf<uint8_t, kPicIdLength>(frame->picture_id, last_picture_id_)) {
+  if (AheadOf<uint16_t, kPicIdLength>(frame->picture_id, last_picture_id_)) {

[stefan-webrtc, 2016/04/26 08:15:12]

Do we have a unittest which would have caught this? Otherwise please add one.

Or maybe this wasn't a bug, but a change of internal representation?

[philipel, 2016/04/28 09:40:42]

This was not a bug, I simply wanted to test what would happen if we used 9 or 13
bits instead of 7 bits for picture ids.

If we ever want to use only 15 bit picture id we can simply change kPicIdLength
to 1 << 15.

     last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
     while (last_picture_id_ != frame->picture_id) {
       not_yet_received_frames_.insert(last_picture_id_);
       last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
     }
   }
 
   // Clean up info for base layers that are too old.
   uint8_t old_tl0_pic_idx = codec_header.tl0PicIdx - kMaxLayerInfo;
   auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
@@ skipping 49 matching lines @@
   // Find all references for this frame.
   frame->num_references = 0;
   for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
     RTC_DCHECK_NE(-1, layer_info_it->second[layer]);
 
     // If we have not yet received a frame between this frame and the referenced
     // frame then we have to wait for that frame to be completed first.
     auto not_received_frame_it =
              not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
     if (not_received_frame_it != not_yet_received_frames_.end() &&
-        AheadOf<uint8_t, kPicIdLength>(frame->picture_id,
+        AheadOf<uint16_t, kPicIdLength>(frame->picture_id,
                                        *not_received_frame_it)) {
           stashed_frames_.emplace(std::move(frame));
           return;
     }
 
     ++frame->num_references;
     frame->references[layer] = layer_info_it->second[layer];
   }
 
   CompletedFrameVp8(std::move(frame));
@@ skipping 26 matching lines @@
   not_yet_received_frames_.erase(frame->picture_id);
 
   for (size_t r = 0; r < frame->num_references; ++r)
     frame->references[r] = UnwrapPictureId(frame->references[r]);
   frame->picture_id = UnwrapPictureId(frame->picture_id);
 
   frame_callback_->OnCompleteFrame(std::move(frame));
   RetryStashedFrames();
 }
 
+void PacketBuffer::ManageFrameVp9(std::unique_ptr<RtpFrameObject> frame) {
+  size_t index = frame->first_seq_num() % size_;
+  const VCMPacket& packet = data_buffer_[index];
+  const RTPVideoHeaderVP9& codec_header =
+      packet.codecSpecificHeader.codecHeader.VP9;
+
+  if (codec_header.picture_id == kNoPictureId) {
+    ManageFrameGeneric(std::move(frame));
+    return;
+  }
+
+  frame->spatial_layer = codec_header.spatial_idx;
+  frame->inter_layer_predicted = codec_header.inter_layer_predicted;
+  frame->picture_id = codec_header.picture_id % kPicIdLength;

[stefan-webrtc, 2016/04/26 08:15:12]

kPictureIdLength

[philipel, 2016/04/28 09:40:42]

I prefer not, there will be many lines that are already hard to format that will
become longer that 80 chars.

+
+  if (last_unwrap_ == -1)
+    last_unwrap_ = codec_header.picture_id;
+
+  if (last_picture_id_ == -1)
+    last_picture_id_ = frame->picture_id;
+
+  // GLORIUS flexible mode!

[stefan-webrtc, 2016/04/26 08:15:12]

I don't think we need this comment... :)

[philipel, 2016/04/28 09:40:42]

Removed :P

+  if (codec_header.flexible_mode) {
+    frame->num_references = codec_header.num_ref_pics;
+    for (size_t r = 0; r < frame->num_references; ++r) {

[stefan-webrtc, 2016/04/26 08:15:12]

use index i rather than r.

[philipel, 2016/04/28 09:40:42]

Done.

+      frame->references[r] = Subtract<1 << 16>(frame->picture_id,
+                                               codec_header.pid_diff[r]);
+    }
+
+    CompletedFrameVp9(std::move(frame));
+    return;
+  }
+
+  if (codec_header.ss_data_available) {
+    // Scalability structures can only be sent with tl0 frames.
+    RTC_DCHECK_EQ(0, codec_header.spatial_idx);

[stefan-webrtc, 2016/04/26 08:15:12]

I don't think we'd want to crash if someone gives us an incorrect bitstream with
an ss in a non tl0 frame, so please just log a warning. Also, you're actually
checking spatial_idx and not temporal_idx.

[philipel, 2016/04/28 09:40:42]

SS are now ignored if they are not sent on a base layer frame. Also log a
warning if that happens.

+
+    current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1);
+    scalability_structures_[current_ss_idx_] = codec_header.gof;
+    scalability_structures_[current_ss_idx_].pid_start = frame->picture_id;
+
+    auto pid_and_gof = std::make_pair(frame->picture_id,
+                                   &scalability_structures_[current_ss_idx_]);
+    gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof));
+  }
+
+  // Clean up info for base layers that are to old.

[stefan-webrtc, 2016/04/26 08:15:12]

too old

[philipel, 2016/04/28 09:40:41]

Done.

+  uint8_t old_tl0_pic_idx = codec_header.tl0_pic_idx - kMaxGofSaved;
+  auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx);
+  gof_info_.erase(gof_info_.begin(), clean_gof_info_to);
+
+  if (packet.frameType == kVideoFrameKey) {
+    // When using GOF all keyframes must include the scalability structure.
+    RTC_DCHECK(codec_header.ss_data_available);
+
+    frame->num_references = 0;
+    CompletedFrameVp9(std::move(frame));
+    return;
+  }
+
+  auto gof_info_it = gof_info_.find((codec_header.temporal_idx == 0 &&
+                                     !codec_header.ss_data_available)
+                                        ? codec_header.tl0_pic_idx - 1
+                                        : codec_header.tl0_pic_idx);
+
+  // Gof info for this frame is not available yet, stash this frame.
+  if (gof_info_it == gof_info_.end()) {
+    stashed_frames_.emplace(std::move(frame));

[stefan-webrtc, 2016/04/26 08:15:12]

What happens if we have already deleted the ss that this frame depends on? Will
stashed_frames_ ever be emptied and a key frame requested?

[philipel, 2016/04/28 09:40:42]

RetryStashedFrames have built in clean up logic which prevents frames from
building up.

+    return;
+  }
+
+  uint16_t picture_id_tl0 = gof_info_it->second.first;
+  GofInfoVP9* gof = gof_info_it->second.second;
+
+  // If this is a base layer frame that contains a scalability structure
+  // then gof info has already been inserted earlier, so we only want to
+  // insert if we haven't done so already.
+  if (codec_header.temporal_idx == 0 &&
+      !codec_header.ss_data_available) {
+    auto pid_and_gof = std::make_pair(frame->picture_id, gof);
+    gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof));
+  }
+
+  // Sync frames only depend on their tl0_pic_idx.W

[stefan-webrtc, 2016/04/26 08:15:12]

Erase W.

And is this comment true? I think a frame with the U flag set and temporal idx T
means that it's possible to switch up T+N, but it doesn't mean that following
frames only will depend on tl0, only that they will be limited to depending on
frames with temporal idx <= T.

[philipel, 2016/04/28 09:40:42]

I was thinking of sync frames for Vp8 for some reason. Fixed now.

+  if (codec_header.temporal_up_switch) {
+    frame->num_references = 1;
+    frame->references[0] = picture_id_tl0;
+    CompletedFrameVp9(std::move(frame));
+    return;
+  }
+
+  RTC_DCHECK((AheadOrAt<uint16_t, kPicIdLength>(frame->picture_id,

[stefan-webrtc, 2016/04/26 08:15:12]

Is this a safe DCHECK if someone produces a bit stream just to make us crash?

[philipel, 2016/04/28 09:40:41]

What this check do is to make sure we don't select a SS that is newer than the
SS that this frame belongs to, so it's safe (programming error).

+                                                picture_id_tl0)));
+
+  uint8_t diff = ForwardDiff<uint16_t, kPicIdLength>(gof->pid_start,
+                                                    frame->picture_id);

[stefan-webrtc, 2016/04/26 08:15:12]

May not be correctly indented

[philipel, 2016/04/28 09:40:41]

Done.

+  uint8_t gof_idx = diff % gof->num_frames_in_gof;
+
+  // Populate references according to the scalability structure.
+  frame->num_references = gof->num_ref_pics[gof_idx];
+  for (size_t r = 0; r < frame->num_references; ++r) {

[stefan-webrtc, 2016/04/26 08:15:12]

index i

[philipel, 2016/04/28 09:40:42]

Done.

+    diff = ForwardDiff<uint16_t, kPicIdLength>(gof->pid_start,
+                                              frame->picture_id);

[stefan-webrtc, 2016/04/26 08:15:12]

Indentation.

And actually, you have already computed this diff, right?

[philipel, 2016/04/28 09:40:42]

Done.

+    gof_idx = diff % gof->num_frames_in_gof;

[stefan-webrtc, 2016/04/26 08:15:12]

This too, if I'm not mistaken

[philipel, 2016/04/28 09:40:42]

Done.

+    frame->references[r] = Subtract<1 << 16>(frame->picture_id,
+                              gof->pid_diff[gof_idx][r]);
+  }
+
+  CompletedFrameVp9(std::move(frame));
+}
+
+void PacketBuffer::CompletedFrameVp9(std::unique_ptr<RtpFrameObject> frame) {
+  for (size_t r = 0; r < frame->num_references; ++r)
+    frame->references[r] = UnwrapPictureId(frame->references[r]);
+  frame->picture_id = UnwrapPictureId(frame->picture_id);
+
+  frame_callback_->OnCompleteFrame(std::move(frame));
+  RetryStashedFrames();
+}
+
+
 uint16_t PacketBuffer::UnwrapPictureId(uint16_t picture_id) {
-  if (last_unwrap_ == -1)
-    last_unwrap_ = picture_id;
+  RTC_DCHECK_NE(-1, last_unwrap_);
 
   uint16_t unwrap_truncated = last_unwrap_ % kPicIdLength;
-  uint16_t diff = MinDiff<uint8_t, kPicIdLength>(unwrap_truncated, picture_id);
+  uint16_t diff = MinDiff<uint16_t, kPicIdLength>(unwrap_truncated, picture_id);
 
-  if (AheadOf<uint8_t, kPicIdLength>(picture_id, unwrap_truncated))
+  if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated))
     last_unwrap_ = Add<1 << 16>(last_unwrap_, diff);
   else
     last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff);
 
   return last_unwrap_;
 }
 
 void PacketBuffer::Flush() {
   rtc::CritScope lock(&crit_);
   for (size_t i = 0; i < size_; ++i)
     sequence_buffer_[i].used = false;
 
   last_seq_num_gop_.clear();
   while (!stashed_frames_.empty())
     stashed_frames_.pop();
   not_yet_received_frames_.clear();
 
   first_packet_received_ = false;
 }
 
 }  // namespace video_coding
 }  // namespace webrtc