VideoAdapter: Drop frames based on actual fps instead of expected fps

webrtc/media/base/videoadapter.cc

 /*
  *  Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/media/base/videoadapter.h"
 
 #include <algorithm>
+#include <cstdlib>
 #include <limits>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/media/base/mediaconstants.h"
 #include "webrtc/media/base/videocommon.h"
 
 namespace {
 
 struct Fraction {
@@ skipping 87 matching lines @@
 
 namespace cricket {
 
 VideoAdapter::VideoAdapter()
     : frames_in_(0),
       frames_out_(0),
       frames_scaled_(0),
       adaption_changes_(0),
       previous_width_(0),
       previous_height_(0),
-      input_interval_(0),
-      interval_next_frame_(0),
       resolution_request_max_pixel_count_(std::numeric_limits<int>::max()),
       resolution_request_max_pixel_count_step_up_(0) {}
 
 VideoAdapter::~VideoAdapter() {}
 
-void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) {
-  // TODO(perkj): Consider measuring input frame rate instead.
-  // Frame rate typically varies depending on lighting.
+bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) {
   rtc::CritScope cs(&critical_section_);
-  input_interval_ = interval;
+  if (!requested_format_ || requested_format_->interval == 0)
+    return true;
+
+  if (next_frame_timestamp_ns_) {
+    // Time until next frame should be outputted.
+    const int64_t time_until_next_frame_ns =
+        (*next_frame_timestamp_ns_ - in_timestamp_ns);
+
+    // Continue if timestamp is withing expected range.
+    if (std::abs(time_until_next_frame_ns) < 2 * requested_format_->interval) {
+      // Drop if a frame shouldn't be outputted yet.
+      if (time_until_next_frame_ns > 0)
+        return false;
+      // Time to output new frame.
+      *next_frame_timestamp_ns_ += requested_format_->interval;
+      return true;
+    }
+  }
+
+  // First timestamp received or timestamp is way outside expected range, so
+  // reset. Set first timestamp target to just half the interval to prefer
+  // keeping frames in case of jitter.
+  next_frame_timestamp_ns_ =
+      rtc::Optional<int64_t>(in_timestamp_ns + requested_format_->interval / 2);
+  return true;
 }
 
 void VideoAdapter::AdaptFrameResolution(int in_width,
                                         int in_height,
+                                        int64_t in_timestamp_ns,
                                         int* cropped_width,
                                         int* cropped_height,
                                         int* out_width,
                                         int* out_height) {
   rtc::CritScope cs(&critical_section_);
   ++frames_in_;
 
   // The max output pixel count is the minimum of the requests from
   // OnOutputFormatRequest and OnResolutionRequest.
   int max_pixel_count = resolution_request_max_pixel_count_;
   if (requested_format_) {
     max_pixel_count = std::min(
         max_pixel_count, requested_format_->width * requested_format_->height);
   }
 
   // Drop the input frame if necessary.
-  bool should_drop = false;
-  if (max_pixel_count == 0) {
-    // Drop all frames as the output format is 0x0.
-    should_drop = true;
-  } else if (requested_format_ && requested_format_->interval > 0) {
-    // Drop some frames based on input fps and output fps.
-    // Normally output fps is less than input fps.
-    interval_next_frame_ += input_interval_;
-    if (interval_next_frame_ >= requested_format_->interval) {
-      interval_next_frame_ -= requested_format_->interval;
-      // Reset |interval_next_frame_| if it accumulates too much to avoid
-      // "catching up" behaviour.
-      if (interval_next_frame_ >= requested_format_->interval)
-        interval_next_frame_ = 0;
-    } else {
-      should_drop = true;
-    }
-  }
-  if (should_drop) {
+  if (max_pixel_count == 0 || !KeepFrame(in_timestamp_ns)) {
     // Show VAdapt log every 90 frames dropped. (3 seconds)
     if ((frames_in_ - frames_out_) % 90 == 0) {
       // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
       // in default calls.
       LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
                    << " / out " << frames_out_
                    << " / in " << frames_in_
                    << " Changes: " << adaption_changes_
                    << " Input: " << in_width
                    << "x" << in_height
-                   << " i" << input_interval_
+                   << " timestamp: " << in_timestamp_ns
                    << " Output: i"
                    << (requested_format_ ? requested_format_->interval : 0);
     }
 
     // Drop frame.
     *cropped_width = 0;
     *cropped_height = 0;
     *out_width = 0;
     *out_height = 0;
     return;
@@ skipping 38 matching lines @@
   ++frames_out_;
   if (scale.numerator != scale.denominator)
     ++frames_scaled_;
 
   if (previous_width_ && (previous_width_ != *out_width ||
                           previous_height_ != *out_height)) {
     ++adaption_changes_;
     LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out "
                  << frames_out_ << " / in " << frames_in_
                  << " Changes: " << adaption_changes_ << " Input: " << in_width
-                 << "x" << in_height << " i" << input_interval_
+                 << "x" << in_height
                  << " Scale: " << scale.numerator << "/" << scale.denominator
                  << " Output: " << *out_width << "x" << *out_height << " i"
                  << (requested_format_ ? requested_format_->interval : 0);
   }
 
   previous_width_ = *out_width;
   previous_height_ = *out_height;
 }
 
 void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
   rtc::CritScope cs(&critical_section_);
   requested_format_ = rtc::Optional<VideoFormat>(format);
-  interval_next_frame_ = 0;
+  next_frame_timestamp_ns_ = rtc::Optional<int64_t>();
 }
 
 void VideoAdapter::OnResolutionRequest(
     rtc::Optional<int> max_pixel_count,
     rtc::Optional<int> max_pixel_count_step_up) {
   rtc::CritScope cs(&critical_section_);
   resolution_request_max_pixel_count_ =
       max_pixel_count.value_or(std::numeric_limits<int>::max());
   resolution_request_max_pixel_count_step_up_ =
       max_pixel_count_step_up.value_or(0);
 }
 
 }  // namespace cricket