| OLD | NEW | 
|    1 /* |    1 /* | 
|    2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |    2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 
|    3  * |    3  * | 
|    4  *  Use of this source code is governed by a BSD-style license |    4  *  Use of this source code is governed by a BSD-style license | 
|    5  *  that can be found in the LICENSE file in the root of the source |    5  *  that can be found in the LICENSE file in the root of the source | 
|    6  *  tree. An additional intellectual property rights grant can be found |    6  *  tree. An additional intellectual property rights grant can be found | 
|    7  *  in the file PATENTS.  All contributing project authors may |    7  *  in the file PATENTS.  All contributing project authors may | 
|    8  *  be found in the AUTHORS file in the root of the source tree. |    8  *  be found in the AUTHORS file in the root of the source tree. | 
|    9  */ |    9  */ | 
|   10  |   10  | 
|   11 #include "webrtc/video/vie_sync_module.h" |   11 #include "webrtc/video/rtp_streams_synchronizer.h" | 
|   12  |   12  | 
|   13 #include "webrtc/base/checks.h" |   13 #include "webrtc/base/checks.h" | 
|   14 #include "webrtc/base/logging.h" |   14 #include "webrtc/base/logging.h" | 
|   15 #include "webrtc/base/timeutils.h" |   15 #include "webrtc/base/timeutils.h" | 
|   16 #include "webrtc/base/trace_event.h" |   16 #include "webrtc/base/trace_event.h" | 
|   17 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" |   17 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h" | 
|   18 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" |   18 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h" | 
|   19 #include "webrtc/modules/video_coding/video_coding_impl.h" |   19 #include "webrtc/modules/video_coding/video_coding_impl.h" | 
|   20 #include "webrtc/system_wrappers/include/clock.h" |   20 #include "webrtc/system_wrappers/include/clock.h" | 
|   21 #include "webrtc/video/stream_synchronization.h" |   21 #include "webrtc/video/stream_synchronization.h" | 
|   22 #include "webrtc/video_frame.h" |   22 #include "webrtc/video_frame.h" | 
|   23 #include "webrtc/voice_engine/include/voe_video_sync.h" |   23 #include "webrtc/voice_engine/include/voe_video_sync.h" | 
|   24  |   24  | 
|   25 namespace webrtc { |   25 namespace webrtc { | 
|   26 namespace { |   26 namespace { | 
|   27 int UpdateMeasurements(StreamSynchronization::Measurements* stream, |   27 int UpdateMeasurements(StreamSynchronization::Measurements* stream, | 
|   28                        const RtpRtcp& rtp_rtcp, const RtpReceiver& receiver) { |   28                        RtpRtcp* rtp_rtcp, RtpReceiver* receiver) { | 
|   29   if (!receiver.Timestamp(&stream->latest_timestamp)) |   29   if (!receiver->Timestamp(&stream->latest_timestamp)) | 
|   30     return -1; |   30     return -1; | 
|   31   if (!receiver.LastReceivedTimeMs(&stream->latest_receive_time_ms)) |   31   if (!receiver->LastReceivedTimeMs(&stream->latest_receive_time_ms)) | 
|   32     return -1; |   32     return -1; | 
|   33  |   33  | 
|   34   uint32_t ntp_secs = 0; |   34   uint32_t ntp_secs = 0; | 
|   35   uint32_t ntp_frac = 0; |   35   uint32_t ntp_frac = 0; | 
|   36   uint32_t rtp_timestamp = 0; |   36   uint32_t rtp_timestamp = 0; | 
|   37   if (rtp_rtcp.RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr, |   37   if (rtp_rtcp->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr, | 
|   38                          &rtp_timestamp) != 0) { |   38                           &rtp_timestamp) != 0) { | 
|   39     return -1; |   39     return -1; | 
|   40   } |   40   } | 
|   41  |   41  | 
|   42   bool new_rtcp_sr = false; |   42   bool new_rtcp_sr = false; | 
|   43   if (!UpdateRtcpList( |   43   if (!UpdateRtcpList( | 
|   44       ntp_secs, ntp_frac, rtp_timestamp, &stream->rtcp, &new_rtcp_sr)) { |   44       ntp_secs, ntp_frac, rtp_timestamp, &stream->rtcp, &new_rtcp_sr)) { | 
|   45     return -1; |   45     return -1; | 
|   46   } |   46   } | 
|   47  |   47  | 
|   48   return 0; |   48   return 0; | 
|   49 } |   49 } | 
|   50 }  // namespace |   50 }  // namespace | 
|   51  |   51  | 
|   52 ViESyncModule::ViESyncModule(vcm::VideoReceiver* video_receiver) |   52 RtpStreamsSynchronizer::RtpStreamsSynchronizer( | 
|   53     : video_receiver_(video_receiver), |   53     vcm::VideoReceiver* video_receiver, | 
|   54       clock_(Clock::GetRealTimeClock()), |   54     RtpStreamReceiver* rtp_stream_receiver) | 
|   55       rtp_receiver_(nullptr), |   55     : clock_(Clock::GetRealTimeClock()), | 
|   56       video_rtp_rtcp_(nullptr), |   56       video_receiver_(video_receiver), | 
 |   57       video_rtp_receiver_(rtp_stream_receiver->GetRtpReceiver()), | 
 |   58       video_rtp_rtcp_(rtp_stream_receiver->rtp_rtcp()), | 
|   57       voe_channel_id_(-1), |   59       voe_channel_id_(-1), | 
|   58       voe_sync_interface_(nullptr), |   60       voe_sync_interface_(nullptr), | 
|   59       last_sync_time_(rtc::TimeNanos()), |   61       audio_rtp_receiver_(nullptr), | 
|   60       sync_() {} |   62       audio_rtp_rtcp_(nullptr), | 
|   61  |   63       sync_(), | 
|   62 ViESyncModule::~ViESyncModule() { |   64       last_sync_time_(rtc::TimeNanos()) { | 
 |   65   process_thread_checker_.DetachFromThread(); | 
|   63 } |   66 } | 
|   64  |   67  | 
|   65 void ViESyncModule::ConfigureSync(int voe_channel_id, |   68 void RtpStreamsSynchronizer::ConfigureSync(int voe_channel_id, | 
|   66                                   VoEVideoSync* voe_sync_interface, |   69                                            VoEVideoSync* voe_sync_interface) { | 
|   67                                   RtpRtcp* video_rtcp_module, |  | 
|   68                                   RtpReceiver* rtp_receiver) { |  | 
|   69   if (voe_channel_id != -1) |   70   if (voe_channel_id != -1) | 
|   70     RTC_DCHECK(voe_sync_interface); |   71     RTC_DCHECK(voe_sync_interface); | 
|   71   rtc::CritScope lock(&data_cs_); |   72  | 
|   72   // Prevent expensive no-ops. |   73   rtc::CritScope lock(&crit_); | 
|   73   if (voe_channel_id_ == voe_channel_id && |   74   if (voe_channel_id_ == voe_channel_id && | 
|   74       voe_sync_interface_ == voe_sync_interface && |   75       voe_sync_interface_ == voe_sync_interface) { | 
|   75       rtp_receiver_ == rtp_receiver && video_rtp_rtcp_ == video_rtcp_module) { |   76     // This prevents expensive no-ops. | 
|   76     return; |   77     return; | 
|   77   } |   78   } | 
|   78   voe_channel_id_ = voe_channel_id; |   79   voe_channel_id_ = voe_channel_id; | 
|   79   voe_sync_interface_ = voe_sync_interface; |   80   voe_sync_interface_ = voe_sync_interface; | 
|   80   rtp_receiver_ = rtp_receiver; |   81  | 
|   81   video_rtp_rtcp_ = video_rtcp_module; |   82   audio_rtp_rtcp_ = nullptr; | 
|   82   sync_.reset( |   83   audio_rtp_receiver_ = nullptr; | 
|   83       new StreamSynchronization(video_rtp_rtcp_->SSRC(), voe_channel_id)); |   84   sync_.reset(nullptr); | 
 |   85  | 
 |   86   if (voe_channel_id_ != -1) { | 
 |   87     voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &audio_rtp_rtcp_, | 
 |   88                                     &audio_rtp_receiver_); | 
 |   89     RTC_DCHECK(audio_rtp_rtcp_); | 
 |   90     RTC_DCHECK(audio_rtp_receiver_); | 
 |   91     sync_.reset(new StreamSynchronization(video_rtp_rtcp_->SSRC(), | 
 |   92                                           voe_channel_id_)); | 
 |   93   } | 
|   84 } |   94 } | 
|   85  |   95  | 
|   86 int64_t ViESyncModule::TimeUntilNextProcess() { |   96 int64_t RtpStreamsSynchronizer::TimeUntilNextProcess() { | 
 |   97   RTC_DCHECK_RUN_ON(&process_thread_checker_); | 
|   87   const int64_t kSyncIntervalMs = 1000; |   98   const int64_t kSyncIntervalMs = 1000; | 
|   88   return kSyncIntervalMs - |   99   return kSyncIntervalMs - | 
|   89       (rtc::TimeNanos() - last_sync_time_) / rtc::kNumNanosecsPerMillisec; |  100       (rtc::TimeNanos() - last_sync_time_) / rtc::kNumNanosecsPerMillisec; | 
|   90 } |  101 } | 
|   91  |  102  | 
|   92 void ViESyncModule::Process() { |  103 void RtpStreamsSynchronizer::Process() { | 
|   93   rtc::CritScope lock(&data_cs_); |  104   RTC_DCHECK_RUN_ON(&process_thread_checker_); | 
 |  105  | 
 |  106   const int current_video_delay_ms = video_receiver_->Delay(); | 
|   94   last_sync_time_ = rtc::TimeNanos(); |  107   last_sync_time_ = rtc::TimeNanos(); | 
|   95  |  108  | 
|   96   const int current_video_delay_ms = video_receiver_->Delay(); |  109   rtc::CritScope lock(&crit_); | 
|   97  |  | 
|   98   if (voe_channel_id_ == -1) { |  110   if (voe_channel_id_ == -1) { | 
|   99     return; |  111     return; | 
|  100   } |  112   } | 
|  101   assert(video_rtp_rtcp_ && voe_sync_interface_); |  113   RTC_DCHECK(voe_sync_interface_); | 
|  102   assert(sync_.get()); |  114   RTC_DCHECK(sync_.get()); | 
|  103  |  115  | 
|  104   int audio_jitter_buffer_delay_ms = 0; |  116   int audio_jitter_buffer_delay_ms = 0; | 
|  105   int playout_buffer_delay_ms = 0; |  117   int playout_buffer_delay_ms = 0; | 
|  106   if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_, |  118   if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_, | 
|  107                                             &audio_jitter_buffer_delay_ms, |  119                                             &audio_jitter_buffer_delay_ms, | 
|  108                                             &playout_buffer_delay_ms) != 0) { |  120                                             &playout_buffer_delay_ms) != 0) { | 
|  109     return; |  121     return; | 
|  110   } |  122   } | 
|  111   const int current_audio_delay_ms = audio_jitter_buffer_delay_ms + |  123   const int current_audio_delay_ms = audio_jitter_buffer_delay_ms + | 
|  112       playout_buffer_delay_ms; |  124       playout_buffer_delay_ms; | 
|  113  |  125  | 
|  114   RtpRtcp* voice_rtp_rtcp = nullptr; |  126   if (UpdateMeasurements(&video_measurement_, video_rtp_rtcp_, | 
|  115   RtpReceiver* voice_receiver = nullptr; |  127                          video_rtp_receiver_) != 0) { | 
|  116   if (voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &voice_rtp_rtcp, |  | 
|  117                                       &voice_receiver) != 0) { |  | 
|  118     return; |  | 
|  119   } |  | 
|  120   assert(voice_rtp_rtcp); |  | 
|  121   assert(voice_receiver); |  | 
|  122  |  | 
|  123   if (UpdateMeasurements(&video_measurement_, *video_rtp_rtcp_, |  | 
|  124                          *rtp_receiver_) != 0) { |  | 
|  125     return; |  128     return; | 
|  126   } |  129   } | 
|  127  |  130  | 
|  128   if (UpdateMeasurements(&audio_measurement_, *voice_rtp_rtcp, |  131   if (UpdateMeasurements(&audio_measurement_, audio_rtp_rtcp_, | 
|  129                          *voice_receiver) != 0) { |  132                          audio_rtp_receiver_) != 0) { | 
|  130     return; |  133     return; | 
|  131   } |  134   } | 
|  132  |  135  | 
|  133   int relative_delay_ms; |  136   int relative_delay_ms; | 
|  134   // Calculate how much later or earlier the audio stream is compared to video. |  137   // Calculate how much later or earlier the audio stream is compared to video. | 
|  135   if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_, |  138   if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_, | 
|  136                                    &relative_delay_ms)) { |  139                                    &relative_delay_ms)) { | 
|  137     return; |  140     return; | 
|  138   } |  141   } | 
|  139  |  142  | 
| (...skipping 11 matching lines...) Expand all  Loading... | 
|  151     return; |  154     return; | 
|  152   } |  155   } | 
|  153  |  156  | 
|  154   if (voe_sync_interface_->SetMinimumPlayoutDelay( |  157   if (voe_sync_interface_->SetMinimumPlayoutDelay( | 
|  155       voe_channel_id_, target_audio_delay_ms) == -1) { |  158       voe_channel_id_, target_audio_delay_ms) == -1) { | 
|  156     LOG(LS_ERROR) << "Error setting voice delay."; |  159     LOG(LS_ERROR) << "Error setting voice delay."; | 
|  157   } |  160   } | 
|  158   video_receiver_->SetMinimumPlayoutDelay(target_video_delay_ms); |  161   video_receiver_->SetMinimumPlayoutDelay(target_video_delay_ms); | 
|  159 } |  162 } | 
|  160  |  163  | 
|  161 bool ViESyncModule::GetStreamSyncOffsetInMs(const VideoFrame& frame, |  164 bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs( | 
|  162                                             int64_t* stream_offset_ms) const { |  165     const VideoFrame& frame, int64_t* stream_offset_ms) const { | 
|  163   rtc::CritScope lock(&data_cs_); |  166   rtc::CritScope lock(&crit_); | 
|  164   if (voe_channel_id_ == -1) |  167   if (voe_channel_id_ == -1) | 
|  165     return false; |  168     return false; | 
|  166  |  169  | 
|  167   uint32_t playout_timestamp = 0; |  170   uint32_t playout_timestamp = 0; | 
|  168   if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_, |  171   if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_, | 
|  169                                                playout_timestamp) != 0) { |  172                                                playout_timestamp) != 0) { | 
|  170     return false; |  173     return false; | 
|  171   } |  174   } | 
|  172  |  175  | 
|  173   int64_t latest_audio_ntp; |  176   int64_t latest_audio_ntp; | 
| (...skipping 11 matching lines...) Expand all  Loading... | 
|  185   int64_t time_to_render_ms = |  188   int64_t time_to_render_ms = | 
|  186       frame.render_time_ms() - clock_->TimeInMilliseconds(); |  189       frame.render_time_ms() - clock_->TimeInMilliseconds(); | 
|  187   if (time_to_render_ms > 0) |  190   if (time_to_render_ms > 0) | 
|  188     latest_video_ntp += time_to_render_ms; |  191     latest_video_ntp += time_to_render_ms; | 
|  189  |  192  | 
|  190   *stream_offset_ms = latest_audio_ntp - latest_video_ntp; |  193   *stream_offset_ms = latest_audio_ntp - latest_video_ntp; | 
|  191   return true; |  194   return true; | 
|  192 } |  195 } | 
|  193  |  196  | 
|  194 }  // namespace webrtc |  197 }  // namespace webrtc | 
| OLD | NEW |