Request keyframe if too many packets are missing and NACK is disabled.

webrtc/modules/video_coding/main/source/jitter_buffer.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/modules/video_coding/main/source/jitter_buffer.h"
@@ skipping 15 matching lines @@
 #include "webrtc/system_wrappers/interface/event_wrapper.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 #include "webrtc/system_wrappers/interface/metrics.h"
 #include "webrtc/system_wrappers/interface/trace_event.h"
 
 namespace webrtc {
 
 // Use this rtt if no value has been reported.
 static const int64_t kDefaultRtt = 200;
 
+// Request a keyframe if no continuous frame has been received for this
+// number of milliseconds and NACKs are disabled.
+static const int64_t kMaxDiscontinuousFramesTime = 10000;
+
 typedef std::pair<uint32_t, VCMFrameBuffer*> FrameListPair;
 
 bool IsKeyFrame(FrameListPair pair) {
   return pair.second->FrameType() == kVideoFrameKey;
 }
 
 bool HasNonEmptyState(FrameListPair pair) {
   return pair.second->GetState() != kStateEmpty;
 }
 
@@ skipping 84 matching lines @@
       stats_callback_(NULL),
       incoming_frame_rate_(0),
       incoming_frame_count_(0),
       time_last_incoming_frame_count_(0),
       incoming_bit_count_(0),
       incoming_bit_rate_(0),
       num_consecutive_old_packets_(0),
       num_packets_(0),
       num_duplicated_packets_(0),
       num_discarded_packets_(0),
+      time_last_decodable_frame_(-1),
       time_first_packet_ms_(0),
       jitter_estimate_(clock),
       inter_frame_delay_(clock_->TimeInMilliseconds()),
       rtt_ms_(kDefaultRtt),
       nack_mode_(kNoNack),
       low_rtt_nack_threshold_ms_(-1),
       high_rtt_nack_threshold_ms_(-1),
       missing_sequence_numbers_(SequenceNumberLessThan()),
       max_nack_list_size_(0),
       max_packet_age_to_nack_(0),
@@ skipping 56 matching lines @@
   incoming_frame_rate_ = 0;
   incoming_bit_count_ = 0;
   incoming_bit_rate_ = 0;
   time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
   receive_statistics_ = FrameCounts();
 
   num_consecutive_old_packets_ = 0;
   num_packets_ = 0;
   num_duplicated_packets_ = 0;
   num_discarded_packets_ = 0;
+  time_last_decodable_frame_ = -1;
   time_first_packet_ms_ = 0;
 
   // Start in a non-signaled state.
   waiting_for_completion_.frame_size = 0;
   waiting_for_completion_.timestamp = 0;
   waiting_for_completion_.latest_packet_time = -1;
   first_packet_since_reset_ = true;
   rtt_ms_ = kDefaultRtt;
   last_decoded_state_.Reset();
   last_gof_valid_ = false;
@@ skipping 31 matching lines @@
   return running_;
 }
 
 void VCMJitterBuffer::Flush() {
   CriticalSectionScoped cs(crit_sect_);
   decodable_frames_.Reset(&free_frames_);
   incomplete_frames_.Reset(&free_frames_);
   last_decoded_state_.Reset();  // TODO(mikhal): sync reset.
   last_gof_valid_ = false;
   num_consecutive_old_packets_ = 0;
+  time_last_decodable_frame_ = -1;
   // Also reset the jitter and delay estimates
   jitter_estimate_.Reset();
   inter_frame_delay_.Reset(clock_->TimeInMilliseconds());
   waiting_for_completion_.frame_size = 0;
   waiting_for_completion_.timestamp = 0;
   waiting_for_completion_.latest_packet_time = -1;
   first_packet_since_reset_ = true;
   missing_sequence_numbers_.clear();
 }
 
@@ skipping 408 matching lines @@
           packet.frameType != kVideoFrameKey) {
         buffer_state = kFlushIndicator;
       }
 
       latest_received_sequence_number_ = LatestSequenceNumber(
           latest_received_sequence_number_, packet.seqNum);
     }
   }
 
   // Is the frame already in the decodable list?
-  bool continuous = IsContinuous(*frame);
+  bool continuous = IsContinuous(*frame, false);
+  bool continuous_for_decoding;
+  if (continuous &&

[stefan-webrtc, 2015/09/24 07:05:48]

I think I'm starting to see what you want to do here. You want to see if there
are too many non-decodable frames, and in that case flush and request a key
frame?

If you look at line 983, I think we have code that can do that for you. 

I think it's equally valid to request a key frame if the number of non-decodable
frames has grown too large even if NACK is on, or?

[joachim, 2015/09/24 08:52:46]

Thanks, however with NACKs disabled and "decode_error_mode_ = kWithErrors", no
frames will be added to "incomplete_frames_", so
"NonContinuousOrIncompleteDuration" as used in line 983 always returns "0".
Isn't this already handled in "GetNackList" and "UpdateNackList"?

[stefan-webrtc, 2015/09/28 13:49:24]

I see. In that case, would it make sense to only put complete and continuous
frames in decodable_frames_ also for the decode_error_mode_ and allow picking
frames from incomplete_frames_ for decoding when in that mode? I'd prefer if we
could reuse the NonContinuousOrIncompleteDuration code in some way since we
already have that functionality.

[joachim, 2015/11/05 23:48:01]

I updated the code as you suggested, looks a lot cleaner now imho.

+      (decode_error_mode_ != kWithErrors || nack_mode_ != kNoNack)) {
+    // If NACKs are enabled missing data will be handled through them so it's
+    // fine to always assume frames that are continuous for decoding below.
+    continuous_for_decoding = true;
+    time_last_decodable_frame_ = now_ms;
+  } else {
+    continuous_for_decoding = IsContinuous(*frame, true);
+    if (continuous_for_decoding) {
+      time_last_decodable_frame_ = now_ms;
+    }
+  }
   switch (buffer_state) {
     case kGeneralError:
     case kTimeStampError:
     case kSizeError: {
       free_frames_.push_back(frame);
       break;
     }
     case kCompleteSession: {
       if (previous_state != kStateDecodable &&
           previous_state != kStateComplete) {
         CountFrame(*frame);
         if (continuous) {
           // Signal that we have a complete session.
           frame_event_->Set();
         }
       }
       FALLTHROUGH();
     }
     // Note: There is no break here - continuing to kDecodableSession.
     case kDecodableSession: {
       *retransmitted = (frame->GetNackCount() > 0);
       if (continuous) {
         decodable_frames_.InsertFrame(frame);
         FindAndInsertContinuousFrames(*frame);
       } else {
         incomplete_frames_.InsertFrame(frame);
       }
+      if (continuous &&
+          !continuous_for_decoding &&
+          TooManyDiscontinuousFrames(now_ms)) {
+        return kFlushIndicator;
+      }
       break;
     }
     case kIncomplete: {
       if (frame->GetState() == kStateEmpty &&
           last_decoded_state_.UpdateEmptyFrame(frame)) {
         free_frames_.push_back(frame);
         return kNoError;
       } else {
         incomplete_frames_.InsertFrame(frame);
       }
+      if (TooManyDiscontinuousFrames(now_ms)) {
+        return kFlushIndicator;
+      }
       break;
     }
     case kNoError:
     case kOutOfBoundsPacket:
     case kDuplicatePacket: {
       // Put back the frame where it came from.
       if (frame_list != NULL) {
         frame_list->InsertFrame(frame);
       } else {
         free_frames_.push_back(frame);
       }
       ++num_duplicated_packets_;
       break;
     }
     case kFlushIndicator:
       free_frames_.push_back(frame);
       return kFlushIndicator;
     default: assert(false);
   }
   return buffer_state;
 }
 
 bool VCMJitterBuffer::IsContinuousInState(const VCMFrameBuffer& frame,
-    const VCMDecodingState& decoding_state) const {
-  if (decode_error_mode_ == kWithErrors)
+    const VCMDecodingState& decoding_state, bool ignore_error_mode) const {
+  if (!ignore_error_mode && decode_error_mode_ == kWithErrors) {
     return true;
+  }
+
   // Is this frame (complete or decodable) and continuous?
   // kStateDecodable will never be set when decode_error_mode_ is false
   // as SessionInfo determines this state based on the error mode (and frame
   // completeness).
   return (frame.GetState() == kStateComplete ||
           frame.GetState() == kStateDecodable) &&
          decoding_state.ContinuousFrame(&frame);
 }
 
-bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame) const {
-  if (IsContinuousInState(frame, last_decoded_state_)) {
+bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame,
+    bool ignore_error_mode) const {
+  if (IsContinuousInState(frame, last_decoded_state_, ignore_error_mode)) {
     return true;
   }
   VCMDecodingState decoding_state;
   decoding_state.CopyFrom(last_decoded_state_);
   for (FrameList::const_iterator it = decodable_frames_.begin();
        it != decodable_frames_.end(); ++it)  {
     VCMFrameBuffer* decodable_frame = it->second;
     if (IsNewerTimestamp(decodable_frame->TimeStamp(), frame.TimeStamp())) {
       break;
     }
     decoding_state.SetState(decodable_frame);
-    if (IsContinuousInState(frame, decoding_state)) {
+    if (IsContinuousInState(frame, decoding_state, ignore_error_mode)) {
       return true;
     }
   }
   return false;
 }
 
 void VCMJitterBuffer::FindAndInsertContinuousFrames(
     const VCMFrameBuffer& new_frame) {
   VCMDecodingState decoding_state;
   decoding_state.CopyFrom(last_decoded_state_);
@@ skipping 13 matching lines @@
   // 1. Continuous base or sync layer.
   // 2. The end of the list was reached.
   for (FrameList::iterator it = incomplete_frames_.begin();
        it != incomplete_frames_.end();)  {
     VCMFrameBuffer* frame = it->second;
     if (IsNewerTimestamp(original_decoded_state.time_stamp(),
                          frame->TimeStamp())) {
       ++it;
       continue;
     }
-    if (IsContinuousInState(*frame, decoding_state)) {
+    if (IsContinuousInState(*frame, decoding_state, false)) {
       decodable_frames_.InsertFrame(frame);
       incomplete_frames_.erase(it++);
       decoding_state.SetState(frame);
     } else if (frame->TemporalId() <= 0) {
       break;
     } else {
       ++it;
     }
   }
 }
 
+bool VCMJitterBuffer::TooManyDiscontinuousFrames(int64_t now_ms) {
+  // If we didn't receive a continuous decodable frame for too long and
+  // can't report through NACKs to the sender, request a keyframe.
+  if (nack_mode_ == kNoNack &&
+      time_last_decodable_frame_ >= 0 &&
+      now_ms - time_last_decodable_frame_ > kMaxDiscontinuousFramesTime) {
+    // Update to current time so we don't request keyframes too often.
+    time_last_decodable_frame_ = now_ms;
+    return true;
+  }
+  return false;
+}
+
 uint32_t VCMJitterBuffer::EstimatedJitterMs() {
   CriticalSectionScoped cs(crit_sect_);
   // Compute RTT multiplier for estimation.
   // low_rtt_nackThresholdMs_ == -1 means no FEC.
   double rtt_mult = 1.0f;
   if (low_rtt_nack_threshold_ms_ >= 0 &&
       rtt_ms_ >= low_rtt_nack_threshold_ms_) {
     // For RTTs above low_rtt_nack_threshold_ms_ we don't apply extra delay
     // when waiting for retransmissions.
     rtt_mult = 0.0f;
@@ skipping 92 matching lines @@
       bool found_key_frame = RecycleFramesUntilKeyFrame();
       if (!found_key_frame) {
         *request_key_frame = have_non_empty_frame;
         return std::vector<uint16_t>();
       }
     }
   }
   if (TooLargeNackList()) {
     *request_key_frame = !HandleTooLargeNackList();
   }
   if (max_incomplete_time_ms_ > 0) {

[stefan-webrtc, 2015/09/24 07:05:47]

I wonder if what you really want to do is something like what we are doing here?

[joachim, 2015/09/24 08:52:46]

Yes, that looks similar, see my other comment regarding
"NonContinuousOrIncompleteDuration" in my scenario.

     int non_continuous_incomplete_duration =
         NonContinuousOrIncompleteDuration();
     if (non_continuous_incomplete_duration > 90 * max_incomplete_time_ms_) {
       LOG_F(LS_WARNING) << "Too long non-decodable duration: "
                         << non_continuous_incomplete_duration << " > "
                         << 90 * max_incomplete_time_ms_;
       FrameList::reverse_iterator rit = find_if(incomplete_frames_.rbegin(),
           incomplete_frames_.rend(), IsKeyFrame);
       if (rit == incomplete_frames_.rend()) {
         // Request a key frame if we don't have one already.
@@ skipping 303 matching lines @@
   }
   // Evaluate if the RTT is higher than |high_rtt_nack_threshold_ms_|, and in
   // that case we don't wait for retransmissions.
   if (high_rtt_nack_threshold_ms_ >= 0 &&
       rtt_ms_ >= high_rtt_nack_threshold_ms_) {
     return false;
   }
   return true;
 }
 }  // namespace webrtc