Convert Vp8 Rtp headers to frame references.

webrtc/modules/video_coding/packet_buffer.cc

Index: webrtc/modules/video_coding/packet_buffer.cc
diff --git a/webrtc/modules/video_coding/packet_buffer.cc b/webrtc/modules/video_coding/packet_buffer.cc
index 0a05baa16abca1e88f96176dc4126b84a2ae6e52..ff564116e1cd2ba8510fa983837c2b4932d7cf46 100644
--- a/webrtc/modules/video_coding/packet_buffer.cc
+++ b/webrtc/modules/video_coding/packet_buffer.cc
@@ -15,7 +15,6 @@
 
 #include "webrtc/base/checks.h"
 #include "webrtc/modules/video_coding/frame_object.h"
-#include "webrtc/modules/video_coding/sequence_number_util.h"
 
 namespace webrtc {
 namespace video_coding {
@@ -25,12 +24,14 @@ PacketBuffer::PacketBuffer(size_t start_buffer_size,
                            OnCompleteFrameCallback* frame_callback)
     : size_(start_buffer_size),
       max_size_(max_buffer_size),
-      last_seq_num_(0),
       first_seq_num_(0),
-      initialized_(false),
+      last_seq_num_(0),
       data_buffer_(start_buffer_size),
       sequence_buffer_(start_buffer_size),
-      frame_callback_(frame_callback) {
+      frame_callback_(frame_callback),
+      last_picture_id_(-1),
+      last_unwrap_(-1),
+      initialized_(false) {
   RTC_DCHECK_LE(start_buffer_size, max_buffer_size);
   // Buffer size must always be a power of 2.
   RTC_DCHECK((start_buffer_size & (start_buffer_size - 1)) == 0);
@@ -70,10 +71,11 @@ bool PacketBuffer::InsertPacket(const VCMPacket& packet) {
   sequence_buffer_[index].frame_end = packet.markerBit;
   sequence_buffer_[index].seq_num = packet.seqNum;
   sequence_buffer_[index].continuous = false;
+  sequence_buffer_[index].frame_created = false;
   sequence_buffer_[index].used = true;
   data_buffer_[index] = packet;
 
-  FindCompleteFrames(seq_num);
+  FindFrames(seq_num);
   return true;
 }
 
@@ -110,8 +112,11 @@ bool PacketBuffer::ExpandBufferSize() {
 bool PacketBuffer::IsContinuous(uint16_t seq_num) const {
   int index = seq_num % size_;
   int prev_index = index > 0 ? index - 1 : size_ - 1;
+
   if (!sequence_buffer_[index].used)
     return false;
+  if (sequence_buffer_[index].frame_created)
+    return false;
   if (sequence_buffer_[index].frame_begin)
     return true;
   if (!sequence_buffer_[prev_index].used)
@@ -122,24 +127,27 @@ bool PacketBuffer::IsContinuous(uint16_t seq_num) const {
   return false;
 }
 
-void PacketBuffer::FindCompleteFrames(uint16_t seq_num) {
+void PacketBuffer::FindFrames(uint16_t seq_num) {
   int index = seq_num % size_;
   while (IsContinuous(seq_num)) {
     sequence_buffer_[index].continuous = true;
 
-    // If the frame is complete, find the first packet of the frame and
-    // create a FrameObject.
+    // If all packets of the frame is contiuous, find the first packet of the

[pbos-webrtc, 2016/04/01 14:24:44]

continuous

+    // frame and create an RtpFrameObject.
     if (sequence_buffer_[index].frame_end) {
       int rindex = index;

[pbos-webrtc, 2016/04/01 14:24:44]

can you rename rindex to something I can understand? :)

[philipel, 2016/04/05 12:40:49]

Renamed it to |start_index|.

       uint16_t start_seq_num = seq_num;
+
       while (!sequence_buffer_[rindex].frame_begin) {
+        sequence_buffer_[rindex].frame_created = true;
         rindex = rindex > 0 ? rindex - 1 : size_ - 1;
         start_seq_num--;
       }
+      sequence_buffer_[rindex].frame_created = true;
 
-      std::unique_ptr<FrameObject> frame(
-          new RtpFrameObject(this, 1, start_seq_num, seq_num));
-      frame_callback_->OnCompleteFrame(std::move(frame));
+      std::unique_ptr<RtpFrameObject> frame(

[pbos-webrtc, 2016/04/01 14:24:44]

Follow-up maybe, but can these be reused so we don't have to heap allocate on
every incoming frame? Maybe a TODO?

[philipel, 2016/04/05 12:40:49]

I think if we do that we add a lot of complexity for no measurable gain. A frame
object is very small since it does not hold any packets.

+          new RtpFrameObject(this, start_seq_num, seq_num));
+      FindReferences(std::move(frame));
     }
 
     index = (index + 1) % size_;
@@ -153,10 +161,9 @@ void PacketBuffer::ReturnFrame(RtpFrameObject* frame) {
   int end = (frame->last_packet() + 1) % size_;
   uint16_t seq_num = frame->first_packet();
   while (index != end) {
-    if (sequence_buffer_[index].seq_num == seq_num) {
+    if (sequence_buffer_[index].seq_num == seq_num)
       sequence_buffer_[index].used = false;
-      sequence_buffer_[index].continuous = false;
-    }
+
     index = (index + 1) % size_;
     ++seq_num;
   }
@@ -192,12 +199,225 @@ bool PacketBuffer::GetBitstream(const RtpFrameObject& frame,
   return true;
 }
 
+void PacketBuffer::FindReferences(std::unique_ptr<RtpFrameObject> frame) {
+  int start_index = frame->first_packet() % size_;

[pbos-webrtc, 2016/04/01 14:24:44]

size_t

[philipel, 2016/04/05 12:40:49]

Done.

+  VideoCodecType codec_type = data_buffer_[start_index].codec;
+
+  if (codec_type == kVideoCodecVP8) {
+    FindReferencesVp8(std::move(frame));
+  } else if (codec_type == kVideoCodecVP9) {
+    // TODO(philipel): FindReferencesVp9(std::move(frame));
+  } else {
+    FindReferencesGeneric(std::move(frame));
+  }
+}
+
+void PacketBuffer::RetryStashedFrames() {
+  int num_stashed_frames = stashed_frames_.size();

[pbos-webrtc, 2016/04/01 14:24:44]

size_t for indexes and sizes

[philipel, 2016/04/05 12:40:49]

Done.

+
+  for (int f = 0; f < num_stashed_frames && !stashed_frames_.empty(); ++f) {

[pbos-webrtc, 2016/04/01 14:24:44]

size_t

[pbos-webrtc, 2016/04/01 14:24:44]

Comment on why this is not the same as 

while(!stashed_frames_.empty())

Also f as index? Pref i for generic indices, otherwise something meaningful as a
name.

[philipel, 2016/04/05 12:40:49]

Done.

[philipel, 2016/04/05 12:40:49]

Done.

+    // Pop ever frame from the queue, and if the references could still not
+    // be determined then they will be put in the queue again.
+    std::unique_ptr<RtpFrameObject> frame = std::move(stashed_frames_.front());
+    stashed_frames_.pop();
+    FindReferences(std::move(frame));
+  }
+}
+
+void PacketBuffer::FindReferencesGeneric(
+    std::unique_ptr<RtpFrameObject> frame) {
+  int index = frame->first_packet() % size_;

[pbos-webrtc, 2016/04/01 14:24:44]

size_t

[philipel, 2016/04/05 12:40:49]

Done.

+  const VCMPacket& packet = data_buffer_[index];
+
+  if (packet.frameType == kVideoFrameKey) {
+    // Clean up keyframes if we have to many or if they are to old.
+    if (!last_seq_num_for_kf_.empty()) {
+      if (last_seq_num_for_kf_.size() > 5 ||

[pbos-webrtc, 2016/04/01 14:24:44]

s/5/kSomeConstantName

[philipel, 2016/04/05 12:40:49]

Done.

+         (MinDiff<uint16_t>(last_seq_num_for_kf_.begin()->second,
+                            frame->last_packet()) > 10000 &&
+          AheadOf<uint16_t>(last_seq_num_for_kf_.begin()->second,
+                            frame->last_packet()))) {
+        last_seq_num_for_kf_.erase(last_seq_num_for_kf_.begin()->second);
+      }
+    }
+
+    last_seq_num_for_kf_[frame->last_packet()] = frame->last_packet();
+  }
+
+  // Find the last sequence number of the last frame for the keyframe
+  // that this frame indirectly references.
+  auto seq_num_it = last_seq_num_for_kf_.upper_bound(frame->last_packet());
+  seq_num_it--;
+
+  // Make sure the packet sequence numbers are continuous, otherwise stash
+  // this frame.
+  if (packet.frameType == kVideoFrameDelta) {
+    if (seq_num_it->second !=
+        static_cast<uint16_t>(frame->first_packet() - 1)) {
+      stashed_frames_.emplace(std::move(frame));
+      return;
+    }
+  }
+
+  RTC_DCHECK(AheadOrAt(frame->last_packet(), seq_num_it->first));
+
+  frame->picture_id = frame->last_packet();
+  frame->num_references = packet.frameType == kVideoFrameDelta;
+  frame->references[0] = seq_num_it->second;
+  seq_num_it->second = frame->last_packet();
+
+  frame_callback_->OnCompleteFrame(std::move(frame));
+  RetryStashedFrames();
+}
+
+void PacketBuffer::FindReferencesVp8(std::unique_ptr<RtpFrameObject> frame) {
+  int index = frame->first_packet() % size_;

[pbos-webrtc, 2016/04/01 14:24:44]

size_t

[philipel, 2016/04/05 12:40:49]

Done.

+  const VCMPacket& packet = data_buffer_[index];
+  const RTPVideoHeaderVP8& codec_header =
+      packet.codecSpecificHeader.codecHeader.VP8;
+
+  if (codec_header.pictureId == kNoPictureId ||
+      codec_header.temporalIdx == kNoTemporalIdx ||
+      codec_header.tl0PicIdx == kNoTl0PicIdx) {
+    FindReferencesGeneric(std::move(frame));
+    return;
+  }
+
+  frame->picture_id = codec_header.pictureId % kPicIdLength;
+
+  if (last_picture_id_ == -1)
+    last_picture_id_ = frame->picture_id;
+
+  // Find if there has been a gap in fully received frames and save the picture
+  // id of those frames in |not_yet_received_frames_|.
+  if (AheadOf<uint8_t, kPicIdLength>(frame->picture_id, last_picture_id_)) {
+    last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
+    while (last_picture_id_ != frame->picture_id) {
+      not_yet_received_frames_.insert(last_picture_id_);
+      last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
+    }
+  }
+
+  if (packet.frameType == kVideoFrameKey) {
+    frame->num_references = 0;

[pbos-webrtc, 2016/04/01 14:24:44]

Shouldn't this line be true for every (generic) frame as well?

Are there parts we can deduplicate? (Not obvious to me, just throwing it out
there)

[philipel, 2016/04/05 12:40:49]

No, generic frames can have 0 or 1 reference.

+    layer_info_[codec_header.tl0PicIdx].fill(-1);
+    CompletedFrameVp8(std::move(frame));
+    return;
+  }
+
+  auto layer_info_it = layer_info_.find(codec_header.temporalIdx == 0
+                                            ? codec_header.tl0PicIdx - 1
+                                            : codec_header.tl0PicIdx);
+
+  // If we don't have the base layer frame yet, stash this frame.
+  if (layer_info_it == layer_info_.end()) {
+    stashed_frames_.emplace(std::move(frame));
+    return;
+  }
+
+  // A non keyframe base layer frame has been received, copy the layer info
+  // from the previous base layer frame and set a reference to the previous
+  // base layer frame.
+  if (codec_header.temporalIdx == 0) {

[pbos-webrtc, 2016/04/01 14:24:44]

Should parts of this be split out to non-VP8 code that can be shared? This
function is long and if handling temporal_idx will be the same for VP9 then it'd
make sense to split this into generic functions

[philipel, 2016/04/05 12:40:49]

VP9 wont need this functionality at all.

+    layer_info_it =
+        layer_info_
+            .insert(make_pair(codec_header.tl0PicIdx, layer_info_it->second))
+            .first;
+    frame->num_references = 1;
+    frame->references[0] = layer_info_it->second[0];
+    CompletedFrameVp8(std::move(frame));
+    return;
+  }
+
+  // Layer sync frame, this frame only references its base layer frame.
+  if (codec_header.layerSync) {
+    frame->num_references = 1;
+    frame->references[0] = layer_info_it->second[0];
+
+    CompletedFrameVp8(std::move(frame));
+    return;
+  }
+
+  // Find all references for this frame.
+  frame->num_references = 0;
+  for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
+    ++frame->num_references;
+    frame->references[layer] = layer_info_it->second[layer];
+
+    // If we have not yet received a frame between this frame and the referenced
+    // frame then we have to wait for that frame to be completed first.
+    auto not_received_frame_it =
+                 not_yet_received_frames_.upper_bound(frame->references[layer]);
+    if (not_received_frame_it != not_yet_received_frames_.end() &&
+        AheadOf<uint8_t, kPicIdLength>(frame->picture_id,
+                                       *not_received_frame_it)) {
+          stashed_frames_.emplace(std::move(frame));
+          return;
+    }
+  }
+
+  CompletedFrameVp8(std::move(frame));
+}
+
+void PacketBuffer::CompletedFrameVp8(std::unique_ptr<RtpFrameObject> frame) {
+  int index = frame->first_packet() % size_;

[pbos-webrtc, 2016/04/01 14:24:44]

size_t

[philipel, 2016/04/05 12:40:49]

Done.

+  const VCMPacket& packet = data_buffer_[index];
+  const RTPVideoHeaderVP8& codec_header =
+      packet.codecSpecificHeader.codecHeader.VP8;
+
+  uint8_t tl0_pic_idx = codec_header.tl0PicIdx;
+  uint8_t temporal_index = codec_header.temporalIdx;
+  auto layer_info_it = layer_info_.find(tl0_pic_idx);
+
+  // Update this layer info and newer.
+  while (layer_info_it != layer_info_.end()) {
+    if (layer_info_it->second[temporal_index] != -1 &&
+        AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_index],
+                                        frame->picture_id)) {
+      // The frame was not newer, then no subsequent layer info have to be
+      // update.
+      break;
+    }
+
+    layer_info_it->second[codec_header.temporalIdx] = frame->picture_id;
+    ++tl0_pic_idx;
+    layer_info_it = layer_info_.find(tl0_pic_idx);
+  }
+  not_yet_received_frames_.erase(frame->picture_id);
+
+  for (int r = 0; r < frame->num_references; ++r) {
+    frame->references[r] = UnwrapPictureId(frame->references[r]);
+  }
+  frame->picture_id = UnwrapPictureId(frame->picture_id);
+
+  frame_callback_->OnCompleteFrame(std::move(frame));
+  RetryStashedFrames();
+}
+
+uint16_t PacketBuffer::UnwrapPictureId(uint16_t picture_id) {
+  if (last_unwrap_ == -1)
+    last_unwrap_ = picture_id;
+
+  uint16_t unwrap_truncated = last_unwrap_ % kPicIdLength;
+  uint16_t diff = MinDiff<uint8_t, kPicIdLength>(unwrap_truncated, picture_id);
+
+  if (AheadOf<uint8_t, kPicIdLength>(picture_id, unwrap_truncated))
+    last_unwrap_ = Add<1 << 16>(last_unwrap_, diff);
+  else
+    last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff);
+
+  return last_unwrap_;
+}
+
 void PacketBuffer::Flush() {
   rtc::CritScope lock(&crit_);
-  for (size_t i = 0; i < size_; ++i) {
+  for (size_t i = 0; i < size_; ++i)
     sequence_buffer_[i].used = false;
-    sequence_buffer_[i].continuous = false;
-  }
+
+  last_seq_num_for_kf_.clear();
+  while (!stashed_frames_.empty())
+    stashed_frames_.pop();
+  not_yet_received_frames_.clear();
 }
 
 }  // namespace video_coding