H264SpsPpsTracker class which keep tracks of SPS/PPS.

webrtc/modules/video_coding/h264_sps_pps_tracker.cc

+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h"
+
+#include <string>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/h264/h264_common.h"
+#include "webrtc/modules/video_coding/frame_object.h"
+#include "webrtc/modules/video_coding/packet_buffer.h"
+
+namespace webrtc {
+namespace video_coding {
+
+namespace {
+const uint8_t start_code_h264[] = {0, 0, 0, 1};
+}  // namespace
+
+bool H264SpsPpsTracker::CopyAndFixBitstream(VCMPacket* packet) {
+  RTC_DCHECK(packet->codec == kVideoCodecH264);
+
+  const uint8_t* data = packet->dataPtr;
+  const size_t data_size = packet->sizeBytes;
+  const RTPVideoHeader& video_header = packet->video_header;
+  const RTPVideoHeaderH264& codec_header = video_header.codecHeader.H264;
+
+  // If we have nalus in this packet then we only want to insert this packet
+  // if it contains anything other than just an SPS/PPS nalu.

[stefan-webrtc, 2016/11/02 13:54:17]

It would be better if this comment explained why that is the case.

[philipel, 2016/11/02 14:39:43]

Done.

+  bool insert_packet = codec_header.nalus_length == 0 ? true : false;
+
+  int pps_id = -1;
+  size_t required_size = 0;
+  for (size_t i = 0; i < codec_header.nalus_length; ++i) {
+    const NaluInfo& nalu = codec_header.nalus[i];
+    switch (nalu.type) {
+      case H264::NaluType::kSps: {
+        // Save SPS.
+        sps_data_[nalu.sps_id].size = nalu.size;
+        sps_data_[nalu.sps_id].data.reset(new uint8_t[nalu.size]);
+        memcpy(sps_data_[nalu.sps_id].data.get(), data + nalu.offset,
+               nalu.size);
+        break;
+      }
+      case H264::NaluType::kPps: {
+        // Save PPS.
+        pps_data_[nalu.pps_id].sps_id = nalu.sps_id;
+        pps_data_[nalu.pps_id].size = nalu.size;
+        pps_data_[nalu.pps_id].data.reset(new uint8_t[nalu.size]);
+        memcpy(pps_data_[nalu.pps_id].data.get(), data + nalu.offset,
+               nalu.size);
+        break;
+      }
+      case H264::NaluType::kIdr: {
+        // If this is the first packet of an IDR, make sure we have the required
+        // SPS/PPS and also calculate how much extra space we need in the buffer
+        // to prepend the SPS/PPS to the bitstream with start codes.
+        if (video_header.isFirstPacket) {
+          if (nalu.pps_id == -1) {
+            LOG(LS_WARNING) << "No PPS id in IDR nalu.";
+            return false;
+          }
+
+          if (!pps_data_[nalu.pps_id].data) {

[stefan-webrtc, 2016/11/02 13:54:17]

I think a find would be more appropriate here, or?

[philipel, 2016/11/02 14:39:43]

Done.

+            LOG(LS_WARNING) << "No PPS with id << " << nalu.pps_id
+                            << " received";
+            return false;
+          }
+
+          if (!sps_data_[pps_data_[nalu.pps_id].sps_id].data) {
+            LOG(LS_WARNING) << "No SPS with id << "
+                            << pps_data_[nalu.pps_id].sps_id << " received";
+            return false;
+          }
+
+          pps_id = nalu.pps_id;
+          required_size += pps_data_[pps_id].size + sizeof(start_code_h264);
+          required_size += sps_data_[pps_data_[pps_id].sps_id].size +
+                           sizeof(start_code_h264);
+        }
+        FALLTHROUGH();
+      }
+      default: {

[stefan-webrtc, 2016/11/02 13:54:17]

What if this is a non-idr but referring to a pps which we don't have?

[philipel, 2016/11/02 14:39:43]

Then that packet will be inserted into the PacketBuffer. The thing is if we only
check the first packet of an IDR frame and then decides to drop that packet (due
to missing SPS/PPS) then the PacketBuffer will never create an RtpFrameObject
since it is missing its first packet.

In short, if we drop the first packet of a keyframe then we ensure that the rest
of the stream is non-decodable, and if we have received a keyframe then we
require all packets in the stream to be received in order to decode it, so we
can't miss any mid-stream SPS/PPS.

+        // Something other than an SPS/PPS nalu in this packet, then it should
+        // be inserted into the PacketBuffer.
+        insert_packet = true;
+      }
+    }
+  }
+
+  if (!insert_packet)
+    return false;
+
+  // Calculate how much space we need for the rest of the bitstream.
+  if (codec_header.packetization_type == kH264StapA) {
+    const uint8_t* nalu_ptr = data + 1;
+    while (nalu_ptr < data + data_size) {
+      // StapA is always the first packet so start codes should always be
+      // inserted.

[stefan-webrtc, 2016/11/02 13:54:17]

I think this comment is a bit weird. We should always prepend start codes on all
nal units. It has nothing to do with isFirstPacket, right?

FuA is the special case where it's only the start of a FuA that should get a
start code.

[philipel, 2016/11/02 14:39:43]

Right, removed confusing comment.

+      RTC_DCHECK(video_header.isFirstPacket);
+      required_size += sizeof(start_code_h264);
+
+      // The first two bytes describe the length of a segment.
+      uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+      nalu_ptr += 2;
+
+      required_size += segment_length;
+      nalu_ptr += segment_length;
+    }
+  } else {
+    if (video_header.isFirstPacket)
+      required_size += sizeof(start_code_h264);
+    required_size += data_size;
+  }
+
+  // Then we copy to the new buffer.
+  uint8_t* buffer = new uint8_t[required_size];
+  uint8_t* insert_at = buffer;
+
+  // If pps_id != -1 then we have the SPS/PPS and they should be prepended
+  // to the bitstream with start codes inserted.

[stefan-webrtc, 2016/11/02 13:54:17]

So here we basically always add pps/sps in front of all packets? Is it enough to
do it for key frames?

[philipel, 2016/11/02 14:39:43]

We only do it for the first packet of an IDR. If you look at line 66 then you
can see that pps_id is only set if this packet is the first packet and it
contains an IDR nalu.

[stefan-webrtc, 2016/11/02 14:54:38]

Right, maybe add a comment that we rely on sequence numbers being in sequence
for delta frames to be decodable?

+  if (pps_id != -1) {
+    // Insert SPS.
+    memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+    insert_at += sizeof(start_code_h264);
+    memcpy(insert_at, sps_data_[pps_data_[pps_id].sps_id].data.get(),
+           sps_data_[pps_data_[pps_id].sps_id].size);
+    insert_at += sps_data_[pps_data_[pps_id].sps_id].size;
+
+    // Insert PPS.
+    memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+    insert_at += sizeof(start_code_h264);
+    memcpy(insert_at, pps_data_[pps_id].data.get(), pps_data_[pps_id].size);
+    insert_at += pps_data_[pps_id].size;
+  }
+
+  // Copy the rest of the bitstream and insert start codes.
+  if (codec_header.packetization_type == kH264StapA) {
+    const uint8_t* nalu_ptr = data + 1;
+    while (nalu_ptr < data + data_size) {
+      memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+      insert_at += sizeof(start_code_h264);
+
+      // The first two bytes describes the length of a segment.
+      uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+      nalu_ptr += 2;
+
+      memcpy(insert_at, nalu_ptr, segment_length);
+      insert_at += segment_length;
+      nalu_ptr += segment_length;
+    }
+  } else {
+    // Might be a FuA, which may or may not be the first packet.
+    if (video_header.isFirstPacket) {
+      memcpy(insert_at, start_code_h264, sizeof(start_code_h264));
+      insert_at += sizeof(start_code_h264);
+    }
+
+    memcpy(insert_at, data, data_size);
+  }
+
+  packet->dataPtr = buffer;
+  packet->sizeBytes = required_size;
+  return true;
+}
+
+}  // namespace video_coding
+}  // namespace webrtc