Refactor how padding is calculated.

webrtc/video/video_send_stream.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/video/video_send_stream.h"
 
 #include <algorithm>
 #include <sstream>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/trace_event.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
 #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
 #include "webrtc/modules/pacing/packet_router.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
 #include "webrtc/modules/utility/include/process_thread.h"
 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
 #include "webrtc/video/call_stats.h"
 #include "webrtc/video/video_capture_input.h"
 #include "webrtc/video/vie_remb.h"
 #include "webrtc/video_send_stream.h"
 
 namespace webrtc {
 
 class RtcpIntraFrameObserver;
 class TransportFeedbackObserver;
 
 static const int kMinSendSidePacketHistorySize = 600;
+static const int kEncoderTimeOutMs = 2000;
 
 namespace {
 
 std::vector<RtpRtcp*> CreateRtpRtcpModules(
     Transport* outgoing_transport,
     RtcpIntraFrameObserver* intra_frame_callback,
     RtcpBandwidthObserver* bandwidth_callback,
     TransportFeedbackObserver* transport_feedback_callback,
     RtcpRttStats* rtt_stats,
     RtpPacketSender* paced_sender,
@@ skipping 288 matching lines @@
   }
   if (video_codec.maxBitrate < kEncoderMinBitrateKbps)
     video_codec.maxBitrate = kEncoderMinBitrateKbps;
 
   RTC_DCHECK_GT(streams[0].max_framerate, 0);
   video_codec.maxFramerate = streams[0].max_framerate;
 
   return video_codec;
 }
 
+int CalulcateMaxPadBitrateBps(const VideoEncoderConfig& config,
+                              bool pad_to_min_bitrate) {
+  int pad_up_to_bitrate_bps = 0;
+  // Calculate max padding bitrate for a multi layer codec.
+  if (config.streams.size() > 1) {
+    // Pad to min bitrate of the highest layer.
+    pad_up_to_bitrate_bps =
+        config.streams[config.streams.size() - 1].min_bitrate_bps;
+    // Add target_bitrate_bps of the lower layers.
+    for (size_t i = 0; i < config.streams.size() - 1; ++i)
+      pad_up_to_bitrate_bps += config.streams[i].target_bitrate_bps;
+  } else if (pad_to_min_bitrate) {
+    pad_up_to_bitrate_bps = config.streams[0].min_bitrate_bps;
+  }
+
+  pad_up_to_bitrate_bps =
+      std::max(pad_up_to_bitrate_bps, config.min_transmit_bitrate_bps);
+
+  return pad_up_to_bitrate_bps;
+}
+
 }  // namespace
 
 namespace internal {
 VideoSendStream::VideoSendStream(
     int num_cpu_cores,
     ProcessThread* module_process_thread,
     CallStats* call_stats,
     CongestionController* congestion_controller,
     BitrateAllocator* bitrate_allocator,
     SendDelayStats* send_delay_stats,
     VieRemb* remb,
     RtcEventLog* event_log,
     const VideoSendStream::Config& config,
     const VideoEncoderConfig& encoder_config,
     const std::map<uint32_t, RtpState>& suspended_ssrcs)
     : stats_proxy_(Clock::GetRealTimeClock(),
                    config,
                    encoder_config.content_type),
       encoded_frame_proxy_(config.post_encode_callback),
       config_(config),
       suspended_ssrcs_(suspended_ssrcs),
       module_process_thread_(module_process_thread),
       call_stats_(call_stats),
       congestion_controller_(congestion_controller),
       bitrate_allocator_(bitrate_allocator),
       remb_(remb),
       encoder_thread_(EncoderThreadFunction, this, "EncoderThread"),
       encoder_wakeup_event_(false, false),
       stop_encoder_thread_(0),
+      send_stream_registered_as_observer_(false),
       overuse_detector_(
           Clock::GetRealTimeClock(),
           GetCpuOveruseOptions(config.encoder_settings.full_overuse_time),
           this,
           config.post_encode_callback,
           &stats_proxy_),
       vie_encoder_(num_cpu_cores,
                    module_process_thread_,
                    &stats_proxy_,
                    &overuse_detector_,
@@ skipping 143 matching lines @@
   return false;
 }
 
 void VideoSendStream::EncoderProcess() {
   RTC_CHECK_EQ(0, vie_encoder_.RegisterExternalEncoder(
                       config_.encoder_settings.encoder,
                       config_.encoder_settings.payload_type,
                       config_.encoder_settings.internal_source));
 
   while (true) {
-    encoder_wakeup_event_.Wait(rtc::Event::kForever);
+    // Wake up every kEncodeCheckForActivityPeriodMs to check if the encoder is
+    // active. If not, deregiser as BitrateAllocatorObserver.

[stefan-webrtc, 2016/06/10 13:54:13]

deregister

[perkj_webrtc, 2016/06/14 10:58:02]

Done.

+    const int kEncodeCheckForActivityPeriodMs = 1000;
+    encoder_wakeup_event_.Wait(kEncodeCheckForActivityPeriodMs);
     if (rtc::AtomicOps::AcquireLoad(&stop_encoder_thread_))
       break;
-    rtc::Optional<EncoderSettings> encoder_settings;
+    bool change_settings = false;
     {
       rtc::CritScope lock(&encoder_settings_crit_);
       if (pending_encoder_settings_) {
-        encoder_settings = pending_encoder_settings_;
-        pending_encoder_settings_ = rtc::Optional<EncoderSettings>();
+        std::swap(current_encoder_settings_, pending_encoder_settings_);
+        pending_encoder_settings_.reset();
+        change_settings = true;
       }
     }
-    if (encoder_settings) {
-      encoder_settings->video_codec.startBitrate =
+    if (change_settings) {
+      current_encoder_settings_->video_codec.startBitrate =
           bitrate_allocator_->AddObserver(
-              this, encoder_settings->video_codec.minBitrate * 1000,
-              encoder_settings->video_codec.maxBitrate * 1000,
+              this, current_encoder_settings_->video_codec.minBitrate * 1000,
+              current_encoder_settings_->video_codec.maxBitrate * 1000,
+              CalulcateMaxPadBitrateBps(current_encoder_settings_->config,
+                                        config_.suspend_below_min_bitrate),
               !config_.suspend_below_min_bitrate) /
           1000;
+      send_stream_registered_as_observer_ = true;
 
-      payload_router_.SetSendStreams(encoder_settings->streams);
-      vie_encoder_.SetEncoder(encoder_settings->video_codec,
-                              encoder_settings->min_transmit_bitrate_bps,
+      payload_router_.SetSendStreams(current_encoder_settings_->config.streams);
+      vie_encoder_.SetEncoder(current_encoder_settings_->video_codec,
                               payload_router_.MaxPayloadLength());
 
       // vie_encoder_.SetEncoder must be called before this.
       if (config_.suspend_below_min_bitrate)
         video_sender_->SuspendBelowMinBitrate();
 
       // Clear stats for disabled layers.
-      for (size_t i = encoder_settings->streams.size();
+      for (size_t i = current_encoder_settings_->config.streams.size();
            i < config_.rtp.ssrcs.size(); ++i) {
         stats_proxy_.OnInactiveSsrc(config_.rtp.ssrcs[i]);
       }
 
       size_t number_of_temporal_layers =
-          encoder_settings->streams.back()
+          current_encoder_settings_->config.streams.back()
               .temporal_layer_thresholds_bps.size() +
           1;
       protection_bitrate_calculator_.SetEncodingData(
-          encoder_settings->video_codec.startBitrate * 1000,
-          encoder_settings->video_codec.width,
-          encoder_settings->video_codec.height,
-          encoder_settings->video_codec.maxFramerate, number_of_temporal_layers,
-          payload_router_.MaxPayloadLength());
+          current_encoder_settings_->video_codec.startBitrate * 1000,
+          current_encoder_settings_->video_codec.width,
+          current_encoder_settings_->video_codec.height,
+          current_encoder_settings_->video_codec.maxFramerate,
+          number_of_temporal_layers, payload_router_.MaxPayloadLength());
 
       // We might've gotten new settings while configuring the encoder settings,
       // restart from the top to see if that's the case before trying to encode
       // a frame (which might correspond to the last frame size).
       encoder_wakeup_event_.Set();
       continue;
     }
 
     VideoFrame frame;
     if (input_.GetVideoFrame(&frame)) {
       // TODO(perkj): |pre_encode_callback| is only used by tests. Tests should
       // register as a sink to the VideoSource instead.
       if (config_.pre_encode_callback) {
         config_.pre_encode_callback->OnFrame(frame);
       }
       vie_encoder_.EncodeVideoFrame(frame);
     }
+
+    // Check if the encoder has produced anything the last kEncoderTimeOutMs.
+    // If not, deregister as BitrateAllocatorObserver.
+    if (send_stream_registered_as_observer_ &&

[mflodman, 2016/06/13 05:02:43]

Instead of having 'send_stream_registered_as_observer_' as a class member, can
we just add a local 'bool frame_encoded = false; at line 621, then set to true
if we did encode a frame in the if statement below that line?

[perkj_webrtc, 2016/06/14 10:58:02]

discussed offline.

+        vie_encoder_.time_of_last_frame_activity_ms() <
+            rtc::TimeMillis() - kEncoderTimeOutMs) {
+      // The encoder has timed out.
+      LOG(LS_INFO) << "Encoder timed out.";

[stefan-webrtc, 2016/06/10 13:54:13]

Do LOG_F or make it more clear why it has timed out.

[perkj_webrtc, 2016/06/14 10:58:02]

Done.

+      bitrate_allocator_->RemoveObserver(this);
+      send_stream_registered_as_observer_ = false;
+    }
+    if (!send_stream_registered_as_observer_ &&
+        vie_encoder_.time_of_last_frame_activity_ms() >
+            rtc::TimeMillis() - kEncoderTimeOutMs) {
+      LOG(LS_INFO) << "Encoder is active.";

[stefan-webrtc, 2016/06/10 13:54:13]

Same here.

[perkj_webrtc, 2016/06/14 10:58:02]

Done.

+      bitrate_allocator_->AddObserver(
+          this, current_encoder_settings_->video_codec.minBitrate * 1000,
+          current_encoder_settings_->video_codec.maxBitrate * 1000,
+          CalulcateMaxPadBitrateBps(current_encoder_settings_->config,
+                                    config_.suspend_below_min_bitrate),
+          !config_.suspend_below_min_bitrate);
+      send_stream_registered_as_observer_ = true;
+    }
   }
   vie_encoder_.DeRegisterExternalEncoder(config_.encoder_settings.payload_type);
 }
 
 void VideoSendStream::ReconfigureVideoEncoder(
     const VideoEncoderConfig& config) {
   TRACE_EVENT0("webrtc", "VideoSendStream::(Re)configureVideoEncoder");
   LOG(LS_INFO) << "(Re)configureVideoEncoder: " << config.ToString();
   RTC_DCHECK_GE(config_.rtp.ssrcs.size(), config.streams.size());
   VideoCodec video_codec = VideoEncoderConfigToVideoCodec(
       config, config_.encoder_settings.payload_name,
       config_.encoder_settings.payload_type);
   {
     rtc::CritScope lock(&encoder_settings_crit_);
-    pending_encoder_settings_ = rtc::Optional<EncoderSettings>(
-        {video_codec, config.min_transmit_bitrate_bps, config.streams});
+    pending_encoder_settings_.reset(new EncoderSettings({video_codec, config}));
   }
   encoder_wakeup_event_.Set();
 }
 
 VideoSendStream::Stats VideoSendStream::GetStats() {
   return stats_proxy_.GetStats();
 }
 
 void VideoSendStream::OveruseDetected() {
   if (config_.overuse_callback)
@@ skipping 158 matching lines @@
   return rtp_states;
 }
 
 void VideoSendStream::SignalNetworkState(NetworkState state) {
   for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
     rtp_rtcp->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode
                                                 : RtcpMode::kOff);
   }
 }
 
-int VideoSendStream::GetPaddingNeededBps() const {
-  return vie_encoder_.GetPaddingNeededBps();
-}
-
 void VideoSendStream::OnBitrateUpdated(uint32_t bitrate_bps,
                                        uint8_t fraction_loss,
                                        int64_t rtt) {
   payload_router_.SetTargetSendBitrate(bitrate_bps);
   // Get the encoder target rate. It is the estimated network rate -
   // protection overhead.
   uint32_t encoder_target_rate = protection_bitrate_calculator_.SetTargetRates(
       bitrate_bps, stats_proxy_.GetSendFrameRate(), fraction_loss, rtt);
 
   vie_encoder_.OnBitrateUpdated(encoder_target_rate, fraction_loss, rtt);
@@ skipping 17 matching lines @@
                           &module_nack_rate);
     *sent_video_rate_bps += module_video_rate;
     *sent_nack_rate_bps += module_nack_rate;
     *sent_fec_rate_bps += module_fec_rate;
   }
   return 0;
 }
 
 }  // namespace internal
 }  // namespace webrtc