Base A/V synchronization on sync_labels.

webrtc/video/call.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 47 matching lines @@
   void NormalUsage() override {
     rtc::CritScope lock(&crit_);
     overuse_callback_->OnLoadUpdate(LoadObserver::kUnderuse);
   }
 
  private:
   rtc::CriticalSection crit_;
   LoadObserver* overuse_callback_ GUARDED_BY(crit_);
 };
 
+class SyncGroup {

[the sun, 2015/06/11 11:52:04]

Simplify as discussed offline.

[pbos-webrtc, 2015/06/11 14:48:53]

Done.

+ public:
+  explicit SyncGroup(VoiceEngine* voice_engine) : voice_engine_(voice_engine) {
+    DCHECK(voice_engine != nullptr);
+  }
+
+  ~SyncGroup() {
+    DCHECK(audio_channel_map_.empty());
+    DCHECK(video_streams_.empty());
+  }
+
+  void AddAudioReceiveStream(AudioReceiveStream* receive_stream,
+                             int channel_id) {
+    if (sync_channel_ == -1)
+      SetSyncChannel(channel_id);
+    audio_channel_map_[receive_stream] = channel_id;
+  }
+
+  void RemoveAudioReceiveStream(AudioReceiveStream* receive_stream) {
+    DCHECK(audio_channel_map_.find(receive_stream) != audio_channel_map_.end());
+    int old_channel = audio_channel_map_[receive_stream];
+    int current_channel = sync_channel_;
+    audio_channel_map_.erase(receive_stream);
+    if (old_channel != current_channel)
+      return;
+    SetSyncChannel(
+        audio_channel_map_.empty() ? -1 : audio_channel_map_.begin()->second);
+  }
+
+  void AddVideoReceiveStream(VideoReceiveStream* receive_stream) {
+    video_streams_.insert(receive_stream);
+    if (sync_channel_ == -1)
+      return;
+    receive_stream->SetSyncChannel(voice_engine_, sync_channel_);
+  }
+
+  void RemoveVideoReceiveStream(VideoReceiveStream* receive_stream) {
+    receive_stream->SetSyncChannel(voice_engine_, -1);
+    video_streams_.erase(receive_stream);
+  }
+
+ private:
+  void SetSyncChannel(int channel_id) {
+    sync_channel_ = channel_id;
+    for (VideoReceiveStream* video_stream : video_streams_)
+      video_stream->SetSyncChannel(voice_engine_, channel_id);
+  }
+
+  VoiceEngine* const voice_engine_;
+  int sync_channel_ = -1;
+  std::map<AudioReceiveStream*, int> audio_channel_map_;
+  std::set<VideoReceiveStream*> video_streams_;
+};
+
 class Call : public webrtc::Call, public PacketReceiver {
  public:
   explicit Call(const Call::Config& config);
   virtual ~Call();
 
   PacketReceiver* Receiver() override;
 
   webrtc::AudioSendStream* CreateAudioSendStream(
       const webrtc::AudioSendStream::Config& config) override;
   void DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) override;
@@ skipping 24 matching lines @@
 
  private:
   DeliveryStatus DeliverRtcp(MediaType media_type, const uint8_t* packet,
                              size_t length);
   DeliveryStatus DeliverRtp(MediaType media_type, const uint8_t* packet,
                             size_t length);
 
   void SetBitrateControllerConfig(
       const webrtc::Call::Config::BitrateConfig& bitrate_config);
 
+  void AddAudioReceiveStreamSync(AudioReceiveStream* stream,
+                                 const std::string& sync_group,
+                                 int channel_id);
+  void RemoveAudioReceiveStreamSync(AudioReceiveStream* stream);
+
+  void AddVideoReceiveStreamSync(VideoReceiveStream* stream,
+                                 const std::string& sync_group);
+  void RemoveVideoReceiveStreamSync(VideoReceiveStream* stream);
+
+  SyncGroup* FindSyncGroup(const std::string& sync_group)
+      EXCLUSIVE_LOCKS_REQUIRED(sync_group_crit_);
+
   const int num_cpu_cores_;
   const rtc::scoped_ptr<ProcessThread> module_process_thread_;
   const rtc::scoped_ptr<ChannelGroup> channel_group_;
   const int base_channel_id_;
   volatile int next_channel_id_;
   Call::Config config_;
 
   // Needs to be held while write-locking |receive_crit_| or |send_crit_|. This
   // ensures that we have a consistent network state signalled to all senders
   // and receivers.
   rtc::CriticalSection network_enabled_crit_;
   bool network_enabled_ GUARDED_BY(network_enabled_crit_);
   TransportAdapter transport_adapter_;
 
   rtc::scoped_ptr<RWLockWrapper> receive_crit_;
   std::map<uint32_t, AudioReceiveStream*> audio_receive_ssrcs_
       GUARDED_BY(receive_crit_);
   std::map<uint32_t, VideoReceiveStream*> video_receive_ssrcs_
       GUARDED_BY(receive_crit_);
   std::set<VideoReceiveStream*> video_receive_streams_
       GUARDED_BY(receive_crit_);
 
   rtc::scoped_ptr<RWLockWrapper> send_crit_;
   std::map<uint32_t, VideoSendStream*> video_send_ssrcs_ GUARDED_BY(send_crit_);
   std::set<VideoSendStream*> video_send_streams_ GUARDED_BY(send_crit_);
 
+  rtc::CriticalSection sync_group_crit_;
+  std::map<VideoReceiveStream*, std::string> video_sync_groups_
+      GUARDED_BY(sync_group_crit_);
+  std::map<AudioReceiveStream*, std::string> audio_sync_groups_
+      GUARDED_BY(sync_group_crit_);
+  std::map<std::string, SyncGroup*> sync_groups_ GUARDED_BY(sync_group_crit_);
+
   rtc::scoped_ptr<CpuOveruseObserverProxy> overuse_observer_proxy_;
 
   VideoSendStream::RtpStateMap suspended_video_send_ssrcs_;
 
   DISALLOW_COPY_AND_ASSIGN(Call);
 };
 }  // namespace internal
 
 Call* Call::Create(const Call::Config& config) {
   return new internal::Call(config);
@@ skipping 37 matching lines @@
 
   SetBitrateControllerConfig(config_.bitrate_config);
 }
 
 Call::~Call() {
   CHECK_EQ(0u, video_send_ssrcs_.size());
   CHECK_EQ(0u, video_send_streams_.size());
   CHECK_EQ(0u, audio_receive_ssrcs_.size());
   CHECK_EQ(0u, video_receive_ssrcs_.size());
   CHECK_EQ(0u, video_receive_streams_.size());
+  for (auto& kv : sync_groups_)
+    delete kv.second;
 
   channel_group_->DeleteChannel(base_channel_id_);
   module_process_thread_->Stop();
   Trace::ReturnTrace();
 }
 
 PacketReceiver* Call::Receiver() { return this; }
 
 webrtc::AudioSendStream* Call::CreateAudioSendStream(
     const webrtc::AudioSendStream::Config& config) {
   return nullptr;
 }
 
 void Call::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) {
 }
 
 webrtc::AudioReceiveStream* Call::CreateAudioReceiveStream(
     const webrtc::AudioReceiveStream::Config& config) {
   TRACE_EVENT0("webrtc", "Call::CreateAudioReceiveStream");
   LOG(LS_INFO) << "CreateAudioReceiveStream: " << config.ToString();
   AudioReceiveStream* receive_stream = new AudioReceiveStream(
       channel_group_->GetRemoteBitrateEstimator(), config);
+  AddAudioReceiveStreamSync(receive_stream, config.sync_group,
+                            config.channel_id);
   {
     WriteLockScoped write_lock(*receive_crit_);
     DCHECK(audio_receive_ssrcs_.find(config.rtp.remote_ssrc) ==
         audio_receive_ssrcs_.end());
     audio_receive_ssrcs_[config.rtp.remote_ssrc] = receive_stream;
   }
   return receive_stream;
 }
 
 void Call::DestroyAudioReceiveStream(
     webrtc::AudioReceiveStream* receive_stream) {
   TRACE_EVENT0("webrtc", "Call::DestroyAudioReceiveStream");
   DCHECK(receive_stream != nullptr);
   AudioReceiveStream* audio_receive_stream =
       static_cast<AudioReceiveStream*>(receive_stream);
+  RemoveAudioReceiveStreamSync(audio_receive_stream);
   {
     WriteLockScoped write_lock(*receive_crit_);
     size_t num_deleted = audio_receive_ssrcs_.erase(
         audio_receive_stream->config().rtp.remote_ssrc);
     DCHECK(num_deleted == 1);
   }
   delete audio_receive_stream;
 }
 
 webrtc::VideoSendStream* Call::CreateVideoSendStream(
@@ skipping 60 matching lines @@
 }
 
 webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream(
     const webrtc::VideoReceiveStream::Config& config) {
   TRACE_EVENT0("webrtc", "Call::CreateVideoReceiveStream");
   LOG(LS_INFO) << "CreateVideoReceiveStream: " << config.ToString();
   VideoReceiveStream* receive_stream = new VideoReceiveStream(
       num_cpu_cores_, base_channel_id_, channel_group_.get(),
       rtc::AtomicOps::Increment(&next_channel_id_), config,
       config_.send_transport, config_.voice_engine);
+  AddVideoReceiveStreamSync(receive_stream, config.sync_group);
 
   // This needs to be taken before receive_crit_ as both locks need to be held
   // while changing network state.
   rtc::CritScope lock(&network_enabled_crit_);
   WriteLockScoped write_lock(*receive_crit_);
   DCHECK(video_receive_ssrcs_.find(config.rtp.remote_ssrc) ==
       video_receive_ssrcs_.end());
   video_receive_ssrcs_[config.rtp.remote_ssrc] = receive_stream;
   // TODO(pbos): Configure different RTX payloads per receive payload.
   VideoReceiveStream::Config::Rtp::RtxMap::const_iterator it =
       config.rtp.rtx.begin();
   if (it != config.rtp.rtx.end())
     video_receive_ssrcs_[it->second.ssrc] = receive_stream;
   video_receive_streams_.insert(receive_stream);
 
   if (!network_enabled_)
     receive_stream->SignalNetworkState(kNetworkDown);
   return receive_stream;
 }
 
 void Call::DestroyVideoReceiveStream(
     webrtc::VideoReceiveStream* receive_stream) {
   TRACE_EVENT0("webrtc", "Call::DestroyVideoReceiveStream");
   DCHECK(receive_stream != nullptr);
-
   VideoReceiveStream* receive_stream_impl = nullptr;
   {
     WriteLockScoped write_lock(*receive_crit_);
     // Remove all ssrcs pointing to a receive stream. As RTX retransmits on a
     // separate SSRC there can be either one or two.
     auto it = video_receive_ssrcs_.begin();
     while (it != video_receive_ssrcs_.end()) {
       if (it->second == static_cast<VideoReceiveStream*>(receive_stream)) {
         if (receive_stream_impl != nullptr)
           DCHECK(receive_stream_impl == it->second);
         receive_stream_impl = it->second;
         video_receive_ssrcs_.erase(it++);
       } else {
         ++it;
       }
     }
     video_receive_streams_.erase(receive_stream_impl);
   }
   CHECK(receive_stream_impl != nullptr);
+  RemoveVideoReceiveStreamSync(receive_stream_impl);
   delete receive_stream_impl;
 }
 
 Call::Stats Call::GetStats() const {
   Stats stats;
   // Fetch available send/receive bitrates.
   uint32_t send_bandwidth = 0;
   channel_group_->GetBitrateController()->AvailableBandwidth(&send_bandwidth);
   std::vector<unsigned int> ssrcs;
   uint32_t recv_bandwidth = 0;
@@ skipping 55 matching lines @@
     }
   }
   {
     ReadLockScoped write_lock(*receive_crit_);
     for (auto& kv : video_receive_ssrcs_) {
       kv.second->SignalNetworkState(state);
     }
   }
 }
 
+void Call::AddAudioReceiveStreamSync(AudioReceiveStream* stream,
+                                     const std::string& sync_group,
+                                     int channel_id) {
+  if (!config_.voice_engine || sync_group.empty())
+    return;
+  rtc::CritScope lock(&sync_group_crit_);
+  audio_sync_groups_[stream] = sync_group;
+  SyncGroup* group = FindSyncGroup(sync_group);
+  group->AddAudioReceiveStream(stream, channel_id);
+}
+
+void Call::RemoveAudioReceiveStreamSync(AudioReceiveStream* stream) {
+  if (!config_.voice_engine)
+    return;
+  rtc::CritScope lock(&sync_group_crit_);
+  SyncGroup* group = FindSyncGroup(audio_sync_groups_[stream]);
+  group->RemoveAudioReceiveStream(stream);
+}
+
+void Call::AddVideoReceiveStreamSync(VideoReceiveStream* stream,
+                                     const std::string& sync_group) {
+  if (!config_.voice_engine || sync_group.empty())
+    return;
+  rtc::CritScope lock(&sync_group_crit_);
+  video_sync_groups_[stream] = sync_group;
+  SyncGroup* group = FindSyncGroup(sync_group);
+  group->AddVideoReceiveStream(stream);
+}
+
+void Call::RemoveVideoReceiveStreamSync(VideoReceiveStream* stream) {
+  if (!config_.voice_engine)
+    return;
+  rtc::CritScope lock(&sync_group_crit_);
+  SyncGroup* group = FindSyncGroup(video_sync_groups_[stream]);
+  group->RemoveVideoReceiveStream(stream);
+}
+
+SyncGroup* Call::FindSyncGroup(const std::string& sync_group) {
+  DCHECK(!sync_group.empty());
+
+  if (sync_groups_.find(sync_group) == sync_groups_.end())
+    sync_groups_[sync_group] = new SyncGroup(config_.voice_engine);
+
+  SyncGroup* group = sync_groups_[sync_group];
+  return group;
+}
+
 PacketReceiver::DeliveryStatus Call::DeliverRtcp(MediaType media_type,
                                                  const uint8_t* packet,
                                                  size_t length) {
   // TODO(pbos): Figure out what channel needs it actually.
   //             Do NOT broadcast! Also make sure it's a valid packet.
   //             Return DELIVERY_UNKNOWN_SSRC if it can be determined that
   //             there's no receiver of the packet.
   bool rtcp_delivered = false;
   if (media_type == MediaType::ANY || media_type == MediaType::VIDEO) {
     ReadLockScoped read_lock(*receive_crit_);
@@ skipping 43 matching lines @@
                                                    const uint8_t* packet,
                                                    size_t length) {
   if (RtpHeaderParser::IsRtcp(packet, length))
     return DeliverRtcp(media_type, packet, length);
 
   return DeliverRtp(media_type, packet, length);
 }
 
 }  // namespace internal
 }  // namespace webrtc