Use sps and pps to determine decodability of H.264 frames.

webrtc/modules/video_coding/decoding_state.cc

 /*
  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/video_coding/decoding_state.h"
 
+#include "webrtc/base/logging.h"
+#include "webrtc/common_video/h264/h264_common.h"
 #include "webrtc/modules/include/module_common_types.h"
 #include "webrtc/modules/video_coding/frame_buffer.h"
 #include "webrtc/modules/video_coding/jitter_buffer_common.h"
 #include "webrtc/modules/video_coding/packet.h"
 
 namespace webrtc {
 
 VCMDecodingState::VCMDecodingState()
     : sequence_num_(0),
       time_stamp_(0),
@@ skipping 10 matching lines @@
 void VCMDecodingState::Reset() {
   // TODO(mikhal): Verify - not always would want to reset the sync
   sequence_num_ = 0;
   time_stamp_ = 0;
   picture_id_ = kNoPictureId;
   temporal_id_ = kNoTemporalIdx;
   tl0_pic_id_ = kNoTl0PicIdx;
   full_sync_ = true;
   in_initial_state_ = true;
   memset(frame_decoded_, 0, sizeof(frame_decoded_));
+  received_sps_.clear();
+  received_pps_.clear();
 }
 
 uint32_t VCMDecodingState::time_stamp() const {
   return time_stamp_;
 }
 
 uint16_t VCMDecodingState::sequence_num() const {
   return sequence_num_;
 }
 
@@ skipping 14 matching lines @@
 void VCMDecodingState::SetState(const VCMFrameBuffer* frame) {
   assert(frame != NULL && frame->GetHighSeqNum() >= 0);
   if (!UsingFlexibleMode(frame))
     UpdateSyncState(frame);
   sequence_num_ = static_cast<uint16_t>(frame->GetHighSeqNum());
   time_stamp_ = frame->TimeStamp();
   picture_id_ = frame->PictureId();
   temporal_id_ = frame->TemporalId();
   tl0_pic_id_ = frame->Tl0PicId();
 
+  std::vector<NaluInfo> nalus = frame->GetNaluInfos();
+  for (const NaluInfo& nalu : nalus) {

[philipel, 2016/09/15 12:52:46]

Maybe clean up old sps/pps? How many sps/pps can we expect the decoder to keep
track of?

[philipel, 2016/09/15 12:52:46]

"const NaluInfo& nalu : frame->GetNaluInfos()" to avoid unnecessary copy. 

[stefan-webrtc, 2016/09/30 09:36:02]

It must be able to track all of them according to the standard.

[stefan-webrtc, 2016/09/30 09:36:02]

Done.

[philipel, 2016/09/30 10:52:52]

Acknowledged.

+    if (nalu.type == H264::NaluType::kPps) {
+      if (nalu.pps_id < 0) {
+        LOG(LS_WARNING) << "Received pps without pps id.";
+      } else if (nalu.sps_id < 0) {
+        LOG(LS_WARNING) << "Received pps without sps id.";
+      } else {
+        received_pps_[nalu.pps_id] = nalu.sps_id;
+      }
+    } else if (nalu.type == H264::NaluType::kSps) {
+      if (nalu.sps_id < 0) {
+        LOG(LS_WARNING) << "Received sps without sps id.";
+      } else {
+        received_sps_.insert(nalu.sps_id);
+      }
+    }
+  }
+
   if (UsingFlexibleMode(frame)) {
     uint16_t frame_index = picture_id_ % kFrameDecodedLength;
     if (in_initial_state_) {
       frame_decoded_cleared_to_ = frame_index;
     } else if (frame->FrameType() == kVideoFrameKey) {
       memset(frame_decoded_, 0, sizeof(frame_decoded_));
       frame_decoded_cleared_to_ = frame_index;
     } else {
       if (AheadOfFramesDecodedClearedTo(frame_index)) {
         while (frame_decoded_cleared_to_ != frame_index) {
@@ skipping 12 matching lines @@
 void VCMDecodingState::CopyFrom(const VCMDecodingState& state) {
   sequence_num_ = state.sequence_num_;
   time_stamp_ = state.time_stamp_;
   picture_id_ = state.picture_id_;
   temporal_id_ = state.temporal_id_;
   tl0_pic_id_ = state.tl0_pic_id_;
   full_sync_ = state.full_sync_;
   in_initial_state_ = state.in_initial_state_;
   frame_decoded_cleared_to_ = state.frame_decoded_cleared_to_;
   memcpy(frame_decoded_, state.frame_decoded_, sizeof(frame_decoded_));
+  received_sps_ = state.received_sps_;
+  received_pps_ = state.received_pps_;
 }
 
 bool VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) {
   bool empty_packet = frame->GetHighSeqNum() == frame->GetLowSeqNum();
   if (in_initial_state_ && empty_packet) {
     // Drop empty packets as long as we are in the initial state.
     return true;
   }
   if ((empty_packet && ContinuousSeqNum(frame->GetHighSeqNum())) ||
       ContinuousFrame(frame)) {
@@ skipping 57 matching lines @@
   // Check continuity based on the following hierarchy:
   // - Temporal layers (stop here if out of sync).
   // - Picture Id when available.
   // - Sequence numbers.
   // Return true when in initial state.
   // Note that when a method is not applicable it will return false.
   assert(frame != NULL);
   // A key frame is always considered continuous as it doesn't refer to any
   // frames and therefore won't introduce any errors even if prior frames are
   // missing.
-  if (frame->FrameType() == kVideoFrameKey)
+  if (frame->FrameType() == kVideoFrameKey &&
+      HaveSpsAndPps(frame->GetNaluInfos())) {
     return true;
+  }
   // When in the initial state we always require a key frame to start decoding.
   if (in_initial_state_)
     return false;
   if (ContinuousLayer(frame->TemporalId(), frame->Tl0PicId()))
     return true;
   // tl0picId is either not used, or should remain unchanged.
   if (frame->Tl0PicId() != tl0_pic_id_)
     return false;
   // Base layers are not continuous or temporal layers are inactive.
   // In the presence of temporal layers, check for Picture ID/sequence number
   // continuity if sync can be restored by this frame.
   if (!full_sync_ && !frame->LayerSync())
     return false;
   if (UsingPictureId(frame)) {
     if (UsingFlexibleMode(frame)) {
       return ContinuousFrameRefs(frame);
     } else {
       return ContinuousPictureId(frame->PictureId());
     }
   } else {
-    return ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum()));
+    return ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum())) &&
+           HaveSpsAndPps(frame->GetNaluInfos());
   }
 }
 
 bool VCMDecodingState::ContinuousPictureId(int picture_id) const {
   int next_picture_id = picture_id_ + 1;
   if (picture_id < picture_id_) {
     // Wrap
     if (picture_id_ >= 0x80) {
       // 15 bits used for picture id
       return ((next_picture_id & 0x7FFF) == picture_id);
@@ skipping 56 matching lines @@
   // frame_decoded_cleared_to_. We just make the assumption
   // that we are not trying to reference back to a very old
   // index, but instead are referencing a newer index.
   uint16_t diff =
       index > frame_decoded_cleared_to_
           ? kFrameDecodedLength - (index - frame_decoded_cleared_to_)
           : frame_decoded_cleared_to_ - index;
   return diff > kFrameDecodedLength / 2;
 }
 
+bool VCMDecodingState::HaveSpsAndPps(const std::vector<NaluInfo>& nalus) const {
+  std::set<int> new_sps;

[philipel, 2016/09/15 12:52:46]

Remove |new_sps| and insert new sps into |received_sps_| instead.

[stefan-webrtc, 2016/09/30 09:36:02]

We can't insert them now since the frame hasn't been decoded yet, so we can't
considered these sps/pps received until then. We should however consider them
for this frame as they're part of it.

[philipel, 2016/09/30 10:52:52]

Acknowledged.

+  std::map<int, int> new_pps;

[philipel, 2016/09/15 12:52:46]

Remove |new_pps| and insert new pps into |received_pps_| instead.

+  for (const NaluInfo& nalu : nalus) {
+    switch (nalu.type) {
+      case H264::NaluType::kPps:
+        if (nalu.pps_id < 0) {
+          LOG(LS_WARNING) << "Received pps without pps id.";
+        } else if (nalu.sps_id < 0) {
+          LOG(LS_WARNING) << "Received pps without sps id.";
+        } else {
+          new_pps[nalu.pps_id] = nalu.sps_id;
+        }
+        break;
+      case H264::NaluType::kSps:
+        if (nalu.sps_id < 0) {
+          LOG(LS_WARNING) << "Received sps without sps id.";
+        } else {
+          new_sps.insert(nalu.sps_id);
+        }
+        break;
+      default: {
+        int sps_needed = -1;
+        auto pps_it = new_pps.find(nalu.pps_id);
+        if (pps_it != new_pps.end()) {
+          sps_needed = pps_it->second;
+        } else {
+          auto pps_it2 = received_pps_.find(nalu.pps_id);
+          if (pps_it2 == received_pps_.end()) {
+            return false;
+          }
+          sps_needed = pps_it2->second;
+        }
+        if (new_sps.find(sps_needed) == new_sps.end() &&
+            received_sps_.find(sps_needed) == received_sps_.end()) {
+          return false;
+        }
+        break;
+      }
+    }
+  }
+  return true;
+}
+
 }  // namespace webrtc