Extract bitrate allocation of spatial/temporal layers out of codec impl.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/video/vie_encoder.h"
 
 #include <algorithm>
 #include <limits>
 #include <utility>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/trace_event.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/modules/pacing/paced_sender.h"
+#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
 #include "webrtc/modules/video_coding/include/video_coding.h"
 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
+#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h"
+#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "webrtc/video/overuse_frame_detector.h"
 #include "webrtc/video/send_statistics_proxy.h"
 #include "webrtc/video_frame.h"
 
 namespace webrtc {
 
 namespace {
 // Time interval for logging frame counts.
 const int64_t kFrameLogIntervalMs = 60000;
 
@@ skipping 145 matching lines @@
 // remotely cope with the load right now.
 CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
   CpuOveruseOptions options;
   if (full_overuse_time) {
     options.low_encode_usage_threshold_percent = 150;
     options.high_encode_usage_threshold_percent = 200;
   }
   return options;
 }
 
+struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
+  ScreenshareTemporalLayersFactory() {}
+  virtual ~ScreenshareTemporalLayersFactory() {}
+
+  virtual webrtc::TemporalLayers* Create(int simulcast_id,
+                                         int num_temporal_layers,
+                                         uint8_t initial_tl0_pic_idx) const {
+    webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers(
+        num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock());
+    if (listener_)
+      listener_->OnTemporalLayersCreated(simulcast_id, tl);
+    return tl;
+  }
+};
+
 }  //  namespace
 
 class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask {
  public:
   ConfigureEncoderTask(ViEEncoder* vie_encoder,
                        VideoEncoderConfig config,
                        size_t max_data_payload_length)
       : vie_encoder_(vie_encoder),
         config_(std::move(config)),
         max_data_payload_length_(max_data_payload_length) {}
@@ skipping 145 matching lines @@
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(shutdown_event_.Wait(0))
       << "Must call ::Stop() before destruction.";
 }
 
 void ViEEncoder::Stop() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(nullptr);
   encoder_queue_.PostTask([this] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
+    rate_allocator_.reset();
     video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
                                           false);
     overuse_detector_.StopCheckForOveruse();
     shutdown_event_.Set();
   });
 
   shutdown_event_.Wait(rtc::Event::kForever);
 }
 
 void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
@@ skipping 67 matching lines @@
           last_frame_info_->width, last_frame_info_->height, encoder_config_);
 
   VideoCodec codec = VideoEncoderConfigToVideoCodec(
       encoder_config_, streams, settings_.payload_name, settings_.payload_type);
 
   codec.startBitrate =
       std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
   codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
   codec.expect_encode_from_texture = last_frame_info_->is_texture;
 
+  switch (codec.codecType) {

[perkj_webrtc, 2016/10/21 08:24:29]

Can we try to keep ViEEncoder codec agnostic? 
Ie- create a static Create method in the BitrateAllocator that creates the
correct instance of a BitrateAllocator given the codecType.

[sprang_webrtc, 2016/10/25 10:44:25]

I created a separate factory class for this, since it couldn't place it directly
in the interface.

+    case kVideoCodecVP8: {
+      SimulcastRateAllocator* rate_allocator =
+          new SimulcastRateAllocator(codec);
+      rate_allocator_.reset(rate_allocator);

[perkj_webrtc, 2016/10/21 08:24:29]

rate_allocator_.reset(new SimulcastRateAllocator(codec));

[sprang_webrtc, 2016/10/25 10:44:25]

Done. This was to avoid a few static_cast when accessing simulcast-specific
fields, but since there's only one usage left of those I'll bring the cast back.

+
+      // Set up default VP8 temporal layer factory, if not provided.
+      if (!codec.codecSpecific.VP8.tl_factory) {
+        if (codec.mode == kScreensharing &&
+            codec.numberOfSimulcastStreams == 1 &&
+            codec.codecSpecific.VP8.numberOfTemporalLayers == 2) {
+          // Conference mode temporal layering for screen content.
+          vp8_tl_factory_.reset(new ScreenshareTemporalLayersFactory());
+        } else {
+          // Standard video temporal layers.
+          vp8_tl_factory_.reset(new TemporalLayersFactory());
+        }
+        codec.codecSpecific.VP8.tl_factory = vp8_tl_factory_.get();
+      }
+      vp8_tl_factory_->SetListener(rate_allocator);

[perkj_webrtc, 2016/10/21 08:24:29]

Can we hide this from ViEEncoder by hiding it in SimulcastRateAllocator? 
SimulcastRateAllocator can own and setup the tl_factory. 
You can add a method on the rate allocator that lets it modify VideoCodec.

[sprang_webrtc, 2016/10/25 10:44:25]

I can do it so it can optionally hold the tl factory, but they can still be
propagated from the outside so can't always rely on that.

+    } break;
+    default:
+      rate_allocator_.reset(new DefaultVideoBitrateAllocator(codec));
+  }
+
   bool success = video_sender_.RegisterSendCodec(
                      &codec, number_of_cores_,
                      static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
   if (!success) {
     LOG(LS_ERROR) << "Failed to configure encoder.";
     RTC_DCHECK(success);
   }
 
-  rate_allocator_.reset(new SimulcastRateAllocator(codec));
   if (stats_proxy_) {
-    stats_proxy_->OnEncoderReconfigured(encoder_config_,
-                                        rate_allocator_->GetPreferedBitrate());
+    int frame_rate = stats_proxy_->GetSendFrameRate();
+    if (frame_rate == 0)
+      frame_rate = codec.maxFramerate;

[perkj_webrtc, 2016/10/21 08:24:29]

codec.maxFramerate does not have a real meaning. It is just a fake hard coded
value.

It seems a little bit weird to require a  frame rate in order to get the
preferedbitrate. The frame rate is hard to know. 

[sprang_webrtc, 2016/10/25 10:44:25]

I know, this is just a placeholder until we have valid stats. 
The rate allocator wants the frame rate because if it is too low we may change
the temporal layer layout. With the current implementations that may not change
the preferred bitrate, but we will still update the registered TemporalLayers
instances though - so it can impact the encoder.

+    stats_proxy_->OnEncoderReconfigured(
+        encoder_config_, rate_allocator_->GetPreferedBitrate(frame_rate));
   }
 
   pending_encoder_reconfiguration_ = false;
-  if (stats_proxy_) {
-    stats_proxy_->OnEncoderReconfigured(encoder_config_,
-                                        rate_allocator_->GetPreferedBitrate());
-  }
+
   sink_->OnEncoderConfigurationChanged(
       std::move(streams), encoder_config_.min_transmit_bitrate_bps);
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   stats_proxy_->OnIncomingFrame(video_frame.width(), video_frame.height());
 
   VideoFrame incoming_frame = video_frame;
 
@@ skipping 202 matching lines @@
     return;
   }
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active.";
 
   LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps
                   << " packet loss " << static_cast<int>(fraction_lost)
                   << " rtt " << round_trip_time_ms;
 
   video_sender_.SetChannelParameters(bitrate_bps, fraction_lost,
-                                     round_trip_time_ms);
+                                     round_trip_time_ms, rate_allocator_.get());
 
   encoder_start_bitrate_bps_ =
       bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_;
   bool video_is_suspended = bitrate_bps == 0;
   bool video_suspension_changed =
       video_is_suspended != (last_observed_bitrate_bps_ == 0);
   last_observed_bitrate_bps_ = bitrate_bps;
 
   if (stats_proxy_ && video_suspension_changed) {
     LOG(LS_INFO) << "Video suspend state changed to: "
@@ skipping 11 matching lines @@
     load_observer_->OnLoadUpdate(LoadObserver::kOveruse);
 }
 
 void ViEEncoder::NormalUsage() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   if (load_observer_)
     load_observer_->OnLoadUpdate(LoadObserver::kUnderuse);
 }
 
 }  // namespace webrtc