Let ViEEncoder tell VideoSendStream about reconfigurations.

webrtc/video/video_send_stream.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_send_stream.h"
@@ skipping 184 matching lines @@
 namespace {
 
 bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
   if (payload_name == "VP8" || payload_name == "VP9")
     return true;
   RTC_DCHECK(payload_name == "H264" || payload_name == "FAKE")
       << "unknown payload_name " << payload_name;
   return false;
 }
 
-int CalculateMaxPadBitrateBps(const VideoEncoderConfig& config,
+int CalculateMaxPadBitrateBps(std::vector<VideoStream> streams,
+                              int min_transmit_bitrate_bps,
                               bool pad_to_min_bitrate) {
   int pad_up_to_bitrate_bps = 0;
   // Calculate max padding bitrate for a multi layer codec.
-  if (config.streams.size() > 1) {
+  if (streams.size() > 1) {
     // Pad to min bitrate of the highest layer.
-    pad_up_to_bitrate_bps =
-        config.streams[config.streams.size() - 1].min_bitrate_bps;
+    pad_up_to_bitrate_bps = streams[streams.size() - 1].min_bitrate_bps;
     // Add target_bitrate_bps of the lower layers.
-    for (size_t i = 0; i < config.streams.size() - 1; ++i)
-      pad_up_to_bitrate_bps += config.streams[i].target_bitrate_bps;
+    for (size_t i = 0; i < streams.size() - 1; ++i)
+      pad_up_to_bitrate_bps += streams[i].target_bitrate_bps;
   } else if (pad_to_min_bitrate) {
-    pad_up_to_bitrate_bps = config.streams[0].min_bitrate_bps;
+    pad_up_to_bitrate_bps = streams[0].min_bitrate_bps;
   }
 
   pad_up_to_bitrate_bps =
-      std::max(pad_up_to_bitrate_bps, config.min_transmit_bitrate_bps);
+      std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps);
 
   return pad_up_to_bitrate_bps;
 }
 
 }  // namespace
 
 namespace internal {
 
 // VideoSendStreamImpl implements internal::VideoSendStream.
 // It is created and destroyed on |worker_queue|. The intent is to decrease the
 // need for locking and to ensure methods are called in sequence.
 // Public methods except |DeliverRtcp| must be called on |worker_queue|.
 // DeliverRtcp is called on the libjingle worker thread or a network thread.
 // An encoder may deliver frames through the EncodedImageCallback on an
 // arbitrary thread.
 class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
                             public webrtc::VCMProtectionCallback,
-                            public EncodedImageCallback {
+                            public ViEEncoder::EncoderSink {
  public:
   VideoSendStreamImpl(SendStatisticsProxy* stats_proxy,
                       rtc::TaskQueue* worker_queue,
                       CallStats* call_stats,
                       CongestionController* congestion_controller,
                       BitrateAllocator* bitrate_allocator,
                       SendDelayStats* send_delay_stats,
                       VieRemb* remb,
                       ViEEncoder* vie_encoder,
                       RtcEventLog* event_log,
                       const VideoSendStream::Config* config,
+                      int initial_encoder_max_bitrate,
                       std::map<uint32_t, RtpState> suspended_ssrcs);
   ~VideoSendStreamImpl() override;
 
   // RegisterProcessThread register |module_process_thread| with those objects
   // that use it. Registration has to happen on the thread were
   // |module_process_thread| was created (libjingle's worker thread).
   // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue,
   // maybe |worker_queue|.
   void RegisterProcessThread(ProcessThread* module_process_thread);
   void DeRegisterProcessThread();
 
   void SignalNetworkState(NetworkState state);
   bool DeliverRtcp(const uint8_t* packet, size_t length);
   void Start();
   void Stop();
 
-  void SignalEncoderConfigurationChanged(const VideoEncoderConfig& config);
   VideoSendStream::RtpStateMap GetRtpStates() const;
 
  private:
   class CheckEncoderActivityTask;
+  class EncoderReconfiguredTask;
 
   // Implements BitrateAllocatorObserver.
   uint32_t OnBitrateUpdated(uint32_t bitrate_bps,
                             uint8_t fraction_loss,
                             int64_t rtt) override;
 
   // Implements webrtc::VCMProtectionCallback.
   int ProtectionRequest(const FecProtectionParams* delta_params,
                         const FecProtectionParams* key_params,
                         uint32_t* sent_video_rate_bps,
                         uint32_t* sent_nack_rate_bps,
                         uint32_t* sent_fec_rate_bps) override;
 
+  void OnEncoderConfigurationChanged(std::vector<VideoStream> streams,
+                                     int min_transmit_bitrate_bps) override;
+
   // Implements EncodedImageCallback. The implementation routes encoded frames
   // to the |payload_router_| and |config.pre_encode_callback| if set.
   // Called on an arbitrary encoder callback thread.
   EncodedImageCallback::Result OnEncodedImage(
       const EncodedImage& encoded_image,
       const CodecSpecificInfo* codec_specific_info,
       const RTPFragmentationHeader* fragmentation) override;
 
   void ConfigureProtection();
   void ConfigureSsrcs();
   void SignalEncoderTimedOut();
   void SignalEncoderActive();
 
   SendStatisticsProxy* const stats_proxy_;
   const VideoSendStream::Config* const config_;
   std::map<uint32_t, RtpState> suspended_ssrcs_;
 
   ProcessThread* module_process_thread_;
   rtc::ThreadChecker module_process_thread_checker_;
   rtc::TaskQueue* const worker_queue_;
 
   rtc::CriticalSection encoder_activity_crit_sect_;
   CheckEncoderActivityTask* check_encoder_activity_task_
       GUARDED_BY(encoder_activity_crit_sect_);
+
   CallStats* const call_stats_;
   CongestionController* const congestion_controller_;
   BitrateAllocator* const bitrate_allocator_;
   VieRemb* const remb_;
 
   static const bool kEnableFrameRecording = false;
   static const int kMaxLayers = 3;
   std::unique_ptr<IvfFileWriter> file_writers_[kMaxLayers];
 
   int max_padding_bitrate_;
@@ skipping 20 matching lines @@
                    SendStatisticsProxy* stats_proxy,
                    ViEEncoder* vie_encoder,
                    ProcessThread* module_process_thread,
                    CallStats* call_stats,
                    CongestionController* congestion_controller,
                    BitrateAllocator* bitrate_allocator,
                    SendDelayStats* send_delay_stats,
                    VieRemb* remb,
                    RtcEventLog* event_log,
                    const VideoSendStream::Config* config,
+                   int initial_encoder_max_bitrate,
                    const std::map<uint32_t, RtpState>& suspended_ssrcs)
       : send_stream_(send_stream),
         done_event_(done_event),
         stats_proxy_(stats_proxy),
         vie_encoder_(vie_encoder),
         call_stats_(call_stats),
         congestion_controller_(congestion_controller),
         bitrate_allocator_(bitrate_allocator),
         send_delay_stats_(send_delay_stats),
         remb_(remb),
         event_log_(event_log),
         config_(config),
+        initial_encoder_max_bitrate_(initial_encoder_max_bitrate),
         suspended_ssrcs_(suspended_ssrcs) {}
 
   ~ConstructionTask() override { done_event_->Set(); }
 
  private:
   bool Run() override {
     send_stream_->reset(new VideoSendStreamImpl(
         stats_proxy_, rtc::TaskQueue::Current(), call_stats_,
         congestion_controller_, bitrate_allocator_, send_delay_stats_, remb_,
-        vie_encoder_, event_log_, config_, std::move(suspended_ssrcs_)));
+        vie_encoder_, event_log_, config_, initial_encoder_max_bitrate_,
+        std::move(suspended_ssrcs_)));
     return true;
   }
 
   std::unique_ptr<VideoSendStreamImpl>* const send_stream_;
   rtc::Event* const done_event_;
   SendStatisticsProxy* const stats_proxy_;
   ViEEncoder* const vie_encoder_;
   CallStats* const call_stats_;
   CongestionController* const congestion_controller_;
   BitrateAllocator* const bitrate_allocator_;
   SendDelayStats* const send_delay_stats_;
   VieRemb* const remb_;
   RtcEventLog* const event_log_;
   const VideoSendStream::Config* config_;
+  int initial_encoder_max_bitrate_;
   std::map<uint32_t, RtpState> suspended_ssrcs_;
 };
 
 class VideoSendStream::DestructAndGetRtpStateTask : public rtc::QueuedTask {
  public:
   DestructAndGetRtpStateTask(VideoSendStream::RtpStateMap* state_map,
                              std::unique_ptr<VideoSendStreamImpl> send_stream,
                              rtc::Event* done_event)
       : state_map_(state_map),
         send_stream_(std::move(send_stream)),
@@ skipping 59 matching lines @@
     // transferred to the task queue when PostDelayedTask was called.
     return false;
   }
   volatile int activity_;
 
   rtc::SequencedTaskChecker task_checker_;
   VideoSendStreamImpl* send_stream_;
   bool timed_out_;
 };
 
-class ReconfigureVideoEncoderTask : public rtc::QueuedTask {
+class VideoSendStreamImpl::EncoderReconfiguredTask : public rtc::QueuedTask {
  public:
-  ReconfigureVideoEncoderTask(VideoSendStreamImpl* send_stream,
-                              VideoEncoderConfig config)
-      : send_stream_(send_stream), config_(std::move(config)) {}
+  EncoderReconfiguredTask(VideoSendStreamImpl* send_stream,
+                          std::vector<VideoStream> streams,
+                          int min_transmit_bitrate_bps)
+      : send_stream_(send_stream),
+        streams_(std::move(streams)),
+        min_transmit_bitrate_bps_(min_transmit_bitrate_bps) {}
 
  private:
   bool Run() override {
-    send_stream_->SignalEncoderConfigurationChanged(std::move(config_));
+    send_stream_->OnEncoderConfigurationChanged(std::move(streams_),
+                                                min_transmit_bitrate_bps_);
     return true;
   }
 
   VideoSendStreamImpl* send_stream_;
-  VideoEncoderConfig config_;
+  std::vector<VideoStream> streams_;
+  int min_transmit_bitrate_bps_;
 };
 
 VideoSendStream::VideoSendStream(
     int num_cpu_cores,
     ProcessThread* module_process_thread,
     rtc::TaskQueue* worker_queue,
     CallStats* call_stats,
     CongestionController* congestion_controller,
     BitrateAllocator* bitrate_allocator,
     SendDelayStats* send_delay_stats,
     VieRemb* remb,
     RtcEventLog* event_log,
     VideoSendStream::Config config,
     VideoEncoderConfig encoder_config,
     const std::map<uint32_t, RtpState>& suspended_ssrcs)
     : worker_queue_(worker_queue),
       thread_sync_event_(false /* manual_reset */, false),
       stats_proxy_(Clock::GetRealTimeClock(),
                    config,
                    encoder_config.content_type),
       config_(std::move(config)) {
   vie_encoder_.reset(
       new ViEEncoder(num_cpu_cores, &stats_proxy_, config_.encoder_settings,
                      config_.pre_encode_callback, config_.overuse_callback,
                      config_.post_encode_callback));
 
+  // TODO(perkj): Remove vector<VideoStreams> from VideoEncoderConfig and
+  // replace with max_bitrate. The VideoStream should be created by ViEEncoder
+  // when the video resolution is known.
+  int initial_max_encoder_bitrate = 0;
+  for (const auto& stream : encoder_config.streams)
+    initial_max_encoder_bitrate += stream.max_bitrate_bps;
+
   worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(new ConstructionTask(
       &send_stream_, &thread_sync_event_, &stats_proxy_, vie_encoder_.get(),
       module_process_thread, call_stats, congestion_controller,
       bitrate_allocator, send_delay_stats, remb, event_log, &config_,
-      suspended_ssrcs)));
+      initial_max_encoder_bitrate, suspended_ssrcs)));
 
   // Wait for ConstructionTask to complete so that |send_stream_| can be used.
   // |module_process_thread| must be registered and deregistered on the thread
   // it was created on.
   thread_sync_event_.Wait(rtc::Event::kForever);
   send_stream_->RegisterProcessThread(module_process_thread);
 
   vie_encoder_->RegisterProcessThread(module_process_thread);
 
   ReconfigureVideoEncoder(std::move(encoder_config));
@@ skipping 32 matching lines @@
   vie_encoder_->SetSource(source);
 }
 
 void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) {
   // ReconfigureVideoEncoder will be called on the thread that deliverers video
   // frames. We must change the encoder settings immediately so that
   // the codec settings matches the next frame.
   // TODO(perkj): Move logic for reconfiguration the encoder due to frame size
   // change from WebRtcVideoChannel2::WebRtcVideoSendStream::OnFrame to
   // be internally handled by ViEEncoder.
-  vie_encoder_->ConfigureEncoder(config, config_.rtp.max_packet_size);
-
-  worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(
-      new ReconfigureVideoEncoderTask(send_stream_.get(), std::move(config))));
+  vie_encoder_->ConfigureEncoder(std::move(config),
+                                 config_.rtp.max_packet_size);
 }
 
 VideoSendStream::Stats VideoSendStream::GetStats() {
   // TODO(perkj, solenberg): Some test cases in EndToEndTest call GetStats from
   // a network thread. See comment in Call::GetStats().
   // RTC_DCHECK_RUN_ON(&thread_checker_);
   return stats_proxy_.GetStats();
 }
 
 void VideoSendStream::SignalNetworkState(NetworkState state) {
@@ skipping 25 matching lines @@
     SendStatisticsProxy* stats_proxy,
     rtc::TaskQueue* worker_queue,
     CallStats* call_stats,
     CongestionController* congestion_controller,
     BitrateAllocator* bitrate_allocator,
     SendDelayStats* send_delay_stats,
     VieRemb* remb,
     ViEEncoder* vie_encoder,
     RtcEventLog* event_log,
     const VideoSendStream::Config* config,
+    int initial_encoder_max_bitrate,
     std::map<uint32_t, RtpState> suspended_ssrcs)
     : stats_proxy_(stats_proxy),
       config_(config),
       suspended_ssrcs_(std::move(suspended_ssrcs)),
       module_process_thread_(nullptr),
       worker_queue_(worker_queue),
       check_encoder_activity_task_(nullptr),
       call_stats_(call_stats),
       congestion_controller_(congestion_controller),
       bitrate_allocator_(bitrate_allocator),
       remb_(remb),
       max_padding_bitrate_(0),
       encoder_min_bitrate_bps_(0),
-      encoder_max_bitrate_bps_(0),
+      encoder_max_bitrate_bps_(initial_encoder_max_bitrate),
       encoder_target_rate_bps_(0),
       vie_encoder_(vie_encoder),
       encoder_feedback_(Clock::GetRealTimeClock(),
                         config_->rtp.ssrcs,
                         vie_encoder),
       protection_bitrate_calculator_(Clock::GetRealTimeClock(), this),
       bandwidth_observer_(congestion_controller_->GetBitrateController()
                               ->CreateRtcpBandwidthObserver()),
       rtp_rtcp_modules_(CreateRtpRtcpModules(
           config_->send_transport,
@@ skipping 160 matching lines @@
 }
 
 void VideoSendStreamImpl::SignalEncoderActive() {
   RTC_DCHECK_RUN_ON(worker_queue_);
   LOG(LS_INFO) << "SignalEncoderActive, Encoder is active.";
   bitrate_allocator_->AddObserver(
       this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
       max_padding_bitrate_, !config_->suspend_below_min_bitrate);
 }
 
-void VideoSendStreamImpl::SignalEncoderConfigurationChanged(
-    const VideoEncoderConfig& config) {
-  RTC_DCHECK_GE(config_->rtp.ssrcs.size(), config.streams.size());
-  TRACE_EVENT0("webrtc", "VideoSendStream::SignalEncoderConfigurationChanged");
-  LOG(LS_INFO) << "SignalEncoderConfigurationChanged: " << config.ToString();
-  RTC_DCHECK_GE(config_->rtp.ssrcs.size(), config.streams.size());
+void VideoSendStreamImpl::OnEncoderConfigurationChanged(
+    std::vector<VideoStream> streams,
+    int min_transmit_bitrate_bps) {
+  if (!worker_queue_->IsCurrent()) {
+    // TODO(perkj): Using |this| in post is safe for now since destruction of
+    // VideoSendStreamImpl is synchronized in
+    // VideoSendStream::StopPermanentlyAndGetRtpStates. But we should really
+    // use some kind of weak_ptr to guarantee that VideoSendStreamImpl is still
+    // alive when this task runs.
+    worker_queue_->PostTask(
+        std::unique_ptr<rtc::QueuedTask>(new EncoderReconfiguredTask(
+            this, std::move(streams), min_transmit_bitrate_bps)));
+    return;
+  }
+  RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
+  TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged");
+  RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
   RTC_DCHECK_RUN_ON(worker_queue_);
 
   const int kEncoderMinBitrateBps = 30000;
   encoder_min_bitrate_bps_ =
-      std::max(config.streams[0].min_bitrate_bps, kEncoderMinBitrateBps);
+      std::max(streams[0].min_bitrate_bps, kEncoderMinBitrateBps);
   encoder_max_bitrate_bps_ = 0;
-  for (const auto& stream : config.streams)
+  for (const auto& stream : streams)
     encoder_max_bitrate_bps_ += stream.max_bitrate_bps;
-  max_padding_bitrate_ =
-      CalculateMaxPadBitrateBps(config, config_->suspend_below_min_bitrate);
-
-  payload_router_.SetSendStreams(config.streams);
+  max_padding_bitrate_ = CalculateMaxPadBitrateBps(
+      streams, min_transmit_bitrate_bps, config_->suspend_below_min_bitrate);
 
   // Clear stats for disabled layers.
-  for (size_t i = config.streams.size(); i < config_->rtp.ssrcs.size(); ++i) {
+  for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) {
     stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]);
   }
 
   size_t number_of_temporal_layers =
-      config.streams.back().temporal_layer_thresholds_bps.size() + 1;
+      streams.back().temporal_layer_thresholds_bps.size() + 1;
   protection_bitrate_calculator_.SetEncodingData(
-      config.streams[0].width, config.streams[0].height,
-      number_of_temporal_layers, config_->rtp.max_packet_size);
+      streams[0].width, streams[0].height, number_of_temporal_layers,
+      config_->rtp.max_packet_size);
 
   if (payload_router_.active()) {
     // The send stream is started already. Update the allocator with new bitrate
     // limits.
     bitrate_allocator_->AddObserver(
         this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
         max_padding_bitrate_, !config_->suspend_below_min_bitrate);
   }
 }
 
@@ skipping 212 matching lines @@
                           &module_nack_rate);
     *sent_video_rate_bps += module_video_rate;
     *sent_nack_rate_bps += module_nack_rate;
     *sent_fec_rate_bps += module_fec_rate;
   }
   return 0;
 }
 
 }  // namespace internal
 }  // namespace webrtc