Base A/V synchronization on sync_labels.

webrtc/video/call.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 91 matching lines @@
 
  private:
   DeliveryStatus DeliverRtcp(MediaType media_type, const uint8_t* packet,
                              size_t length);
   DeliveryStatus DeliverRtp(MediaType media_type, const uint8_t* packet,
                             size_t length);
 
   void SetBitrateControllerConfig(
       const webrtc::Call::Config::BitrateConfig& bitrate_config);
 
+  void ConfigureSync(const std::string& sync_group)
+      EXCLUSIVE_LOCKS_REQUIRED(receive_crit_);
+
   const int num_cpu_cores_;
   const rtc::scoped_ptr<ProcessThread> module_process_thread_;
   const rtc::scoped_ptr<ChannelGroup> channel_group_;
   const int base_channel_id_;
   volatile int next_channel_id_;
   Call::Config config_;
 
   // Needs to be held while write-locking |receive_crit_| or |send_crit_|. This
   // ensures that we have a consistent network state signalled to all senders
   // and receivers.
   rtc::CriticalSection network_enabled_crit_;
   bool network_enabled_ GUARDED_BY(network_enabled_crit_);
   TransportAdapter transport_adapter_;
 
   rtc::scoped_ptr<RWLockWrapper> receive_crit_;
   std::map<uint32_t, AudioReceiveStream*> audio_receive_ssrcs_
       GUARDED_BY(receive_crit_);
   std::map<uint32_t, VideoReceiveStream*> video_receive_ssrcs_
       GUARDED_BY(receive_crit_);
   std::set<VideoReceiveStream*> video_receive_streams_
       GUARDED_BY(receive_crit_);
+  std::map<std::string, AudioReceiveStream*> sync_stream_mapping_
+      GUARDED_BY(receive_crit_);
 
   rtc::scoped_ptr<RWLockWrapper> send_crit_;
   std::map<uint32_t, VideoSendStream*> video_send_ssrcs_ GUARDED_BY(send_crit_);
   std::set<VideoSendStream*> video_send_streams_ GUARDED_BY(send_crit_);
 
   rtc::scoped_ptr<CpuOveruseObserverProxy> overuse_observer_proxy_;
 
   VideoSendStream::RtpStateMap suspended_video_send_ssrcs_;
 
   DISALLOW_COPY_AND_ASSIGN(Call);
@@ skipping 69 matching lines @@
     const webrtc::AudioReceiveStream::Config& config) {
   TRACE_EVENT0("webrtc", "Call::CreateAudioReceiveStream");
   LOG(LS_INFO) << "CreateAudioReceiveStream: " << config.ToString();
   AudioReceiveStream* receive_stream = new AudioReceiveStream(
       channel_group_->GetRemoteBitrateEstimator(), config);
   {
     WriteLockScoped write_lock(*receive_crit_);
     DCHECK(audio_receive_ssrcs_.find(config.rtp.remote_ssrc) ==
         audio_receive_ssrcs_.end());
     audio_receive_ssrcs_[config.rtp.remote_ssrc] = receive_stream;
+    ConfigureSync(config.sync_group);
   }
   return receive_stream;
 }
 
 void Call::DestroyAudioReceiveStream(
     webrtc::AudioReceiveStream* receive_stream) {
   TRACE_EVENT0("webrtc", "Call::DestroyAudioReceiveStream");
   DCHECK(receive_stream != nullptr);
   AudioReceiveStream* audio_receive_stream =
       static_cast<AudioReceiveStream*>(receive_stream);
   {
     WriteLockScoped write_lock(*receive_crit_);
     size_t num_deleted = audio_receive_ssrcs_.erase(
         audio_receive_stream->config().rtp.remote_ssrc);
     DCHECK(num_deleted == 1);
+    const std::string& sync_group = audio_receive_stream->config().sync_group;
+    const auto it = sync_stream_mapping_.find(sync_group);
+    if (it != sync_stream_mapping_.end() &&
+        it->second == audio_receive_stream) {
+      sync_stream_mapping_.erase(it);
+      ConfigureSync(sync_group);
+    }
   }
   delete audio_receive_stream;
 }
 
 webrtc::VideoSendStream* Call::CreateVideoSendStream(
     const webrtc::VideoSendStream::Config& config,
     const VideoEncoderConfig& encoder_config) {
   TRACE_EVENT0("webrtc", "Call::CreateVideoSendStream");
   LOG(LS_INFO) << "CreateVideoSendStream: " << config.ToString();
   DCHECK(!config.rtp.ssrcs.empty());
@@ skipping 70 matching lines @@
   DCHECK(video_receive_ssrcs_.find(config.rtp.remote_ssrc) ==
       video_receive_ssrcs_.end());
   video_receive_ssrcs_[config.rtp.remote_ssrc] = receive_stream;
   // TODO(pbos): Configure different RTX payloads per receive payload.
   VideoReceiveStream::Config::Rtp::RtxMap::const_iterator it =
       config.rtp.rtx.begin();
   if (it != config.rtp.rtx.end())
     video_receive_ssrcs_[it->second.ssrc] = receive_stream;
   video_receive_streams_.insert(receive_stream);
 
+  ConfigureSync(config.sync_group);
+
   if (!network_enabled_)
     receive_stream->SignalNetworkState(kNetworkDown);
+
   return receive_stream;
 }
 
 void Call::DestroyVideoReceiveStream(
     webrtc::VideoReceiveStream* receive_stream) {
   TRACE_EVENT0("webrtc", "Call::DestroyVideoReceiveStream");
   DCHECK(receive_stream != nullptr);
-
   VideoReceiveStream* receive_stream_impl = nullptr;
   {
     WriteLockScoped write_lock(*receive_crit_);
     // Remove all ssrcs pointing to a receive stream. As RTX retransmits on a
     // separate SSRC there can be either one or two.
     auto it = video_receive_ssrcs_.begin();
     while (it != video_receive_ssrcs_.end()) {
       if (it->second == static_cast<VideoReceiveStream*>(receive_stream)) {
         if (receive_stream_impl != nullptr)
           DCHECK(receive_stream_impl == it->second);
         receive_stream_impl = it->second;
         video_receive_ssrcs_.erase(it++);
       } else {
         ++it;
       }
     }
     video_receive_streams_.erase(receive_stream_impl);
+    CHECK(receive_stream_impl != nullptr);
+    ConfigureSync(receive_stream_impl->config().sync_group);
   }
-  CHECK(receive_stream_impl != nullptr);
   delete receive_stream_impl;
 }
 
 Call::Stats Call::GetStats() const {
   Stats stats;
   // Fetch available send/receive bitrates.
   uint32_t send_bandwidth = 0;
   channel_group_->GetBitrateController()->AvailableBandwidth(&send_bandwidth);
   std::vector<unsigned int> ssrcs;
   uint32_t recv_bandwidth = 0;
@@ skipping 55 matching lines @@
     }
   }
   {
     ReadLockScoped write_lock(*receive_crit_);
     for (auto& kv : video_receive_ssrcs_) {
       kv.second->SignalNetworkState(state);
     }
   }
 }
 
+void Call::ConfigureSync(const std::string& sync_group) {
+  // Set sync only if there was no previous one.
+  if (config_.voice_engine == nullptr || sync_group.empty())
+    return;
+
+  AudioReceiveStream* sync_audio_stream = nullptr;
+  // Find existing audio stream.
+  const auto it = sync_stream_mapping_.find(sync_group);
+  if (it != sync_stream_mapping_.end()) {
+    sync_audio_stream = it->second;
+  } else {
+    // No configured audio stream, see if we can find one.
+    for (const auto& kv : audio_receive_ssrcs_) {
+      if (kv.second->config().sync_group == sync_group) {
+        if (sync_audio_stream != nullptr) {
+          LOG(LS_WARNING) << "Attempting to sync more than one audio stream "
+                             "within the same sync group. This is not "
+                             "supported in the current implementation.";
+          break;
+        }
+        sync_audio_stream = kv.second;
+      }
+    }
+  }
+  if (sync_audio_stream)
+    sync_stream_mapping_[sync_group] = sync_audio_stream;
+  size_t num_synced_streams = 0;
+  for (VideoReceiveStream* video_stream : video_receive_streams_) {
+    if (video_stream->config().sync_group != sync_group)
+      continue;
+    ++num_synced_streams;
+    if (num_synced_streams > 1) {
+      LOG(LS_WARNING) << "Attempting to sync more than one audio/video pair "
+                         "within the same sync group. This is not supported in "
+                         "the current implementation.";

[hta-webrtc, 2015/07/06 19:04:00]

Should there be a bug number filed to track this restriction?

[pbos-webrtc, 2015/07/09 10:21:49]

https://code.google.com/p/webrtc/issues/detail?id=4762

Put it in a comment as well.

+    }
+    // Only sync the first A/V pair within this sync group.
+    if (sync_audio_stream != nullptr && num_synced_streams == 1) {
+      video_stream->SetSyncChannel(config_.voice_engine,
+                                   sync_audio_stream->config().voe_channel_id);
+    } else {
+      video_stream->SetSyncChannel(config_.voice_engine, -1);
+    }
+  }
+}
+
 PacketReceiver::DeliveryStatus Call::DeliverRtcp(MediaType media_type,
                                                  const uint8_t* packet,
                                                  size_t length) {
   // TODO(pbos): Figure out what channel needs it actually.
   //             Do NOT broadcast! Also make sure it's a valid packet.
   //             Return DELIVERY_UNKNOWN_SSRC if it can be determined that
   //             there's no receiver of the packet.
   bool rtcp_delivered = false;
   if (media_type == MediaType::ANY || media_type == MediaType::VIDEO) {
     ReadLockScoped read_lock(*receive_crit_);
@@ skipping 43 matching lines @@
                                                    const uint8_t* packet,
                                                    size_t length) {
   if (RtpHeaderParser::IsRtcp(packet, length))
     return DeliverRtcp(media_type, packet, length);
 
   return DeliverRtp(media_type, packet, length);
 }
 
 }  // namespace internal
 }  // namespace webrtc