OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ |
| 10 |
| 11 #include "webrtc/modules/audio_mixer/source/new_audio_conference_mixer_impl.h" |
| 12 |
| 13 #include <algorithm> |
| 14 |
| 15 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h
" |
| 16 #include "webrtc/modules/audio_mixer/include/audio_mixer_defines.h" |
| 17 #include "webrtc/modules/audio_processing/include/audio_processing.h" |
| 18 #include "webrtc/modules/utility/include/audio_frame_operations.h" |
| 19 #include "webrtc/system_wrappers/include/critical_section_wrapper.h" |
| 20 #include "webrtc/system_wrappers/include/trace.h" |
| 21 |
| 22 namespace webrtc { |
| 23 namespace { |
| 24 |
| 25 struct ParticipantFrameStruct { |
| 26 ParticipantFrameStruct(MixerAudioSource* p, AudioFrame* a, bool m) |
| 27 : participant(p), audioFrame(a), muted(m) {} |
| 28 MixerAudioSource* participant; |
| 29 AudioFrame* audioFrame; |
| 30 bool muted; |
| 31 }; |
| 32 |
| 33 typedef std::list<ParticipantFrameStruct*> ParticipantFrameStructList; |
| 34 |
| 35 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing. |
| 36 // These effects are applied to |frame| itself prior to mixing. Assumes that |
| 37 // |mixed_frame| always has at least as many channels as |frame|. Supports |
| 38 // stereo at most. |
| 39 // |
| 40 // TODO(andrew): consider not modifying |frame| here. |
| 41 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) { |
| 42 assert(mixed_frame->num_channels_ >= frame->num_channels_); |
| 43 if (use_limiter) { |
| 44 // Divide by two to avoid saturation in the mixing. |
| 45 // This is only meaningful if the limiter will be used. |
| 46 *frame >>= 1; |
| 47 } |
| 48 if (mixed_frame->num_channels_ > frame->num_channels_) { |
| 49 // We only support mono-to-stereo. |
| 50 assert(mixed_frame->num_channels_ == 2 && frame->num_channels_ == 1); |
| 51 AudioFrameOperations::MonoToStereo(frame); |
| 52 } |
| 53 |
| 54 *mixed_frame += *frame; |
| 55 } |
| 56 |
| 57 // Return the max number of channels from a |list| composed of AudioFrames. |
| 58 size_t MaxNumChannels(const AudioFrameList* list) { |
| 59 size_t max_num_channels = 1; |
| 60 for (AudioFrameList::const_iterator iter = list->begin(); iter != list->end(); |
| 61 ++iter) { |
| 62 max_num_channels = std::max(max_num_channels, (*iter).frame->num_channels_); |
| 63 } |
| 64 return max_num_channels; |
| 65 } |
| 66 |
| 67 } // namespace |
| 68 |
| 69 MixerAudioSource::MixerAudioSource() : _mixHistory(new NewMixHistory()) {} |
| 70 |
| 71 MixerAudioSource::~MixerAudioSource() { |
| 72 delete _mixHistory; |
| 73 } |
| 74 |
| 75 bool MixerAudioSource::IsMixed() const { |
| 76 return _mixHistory->IsMixed(); |
| 77 } |
| 78 |
| 79 NewMixHistory::NewMixHistory() : _isMixed(0) {} |
| 80 |
| 81 NewMixHistory::~NewMixHistory() {} |
| 82 |
| 83 bool NewMixHistory::IsMixed() const { |
| 84 return _isMixed; |
| 85 } |
| 86 |
| 87 bool NewMixHistory::WasMixed() const { |
| 88 // Was mixed is the same as is mixed depending on perspective. This function |
| 89 // is for the perspective of NewAudioConferenceMixerImpl. |
| 90 return IsMixed(); |
| 91 } |
| 92 |
| 93 int32_t NewMixHistory::SetIsMixed(const bool mixed) { |
| 94 _isMixed = mixed; |
| 95 return 0; |
| 96 } |
| 97 |
| 98 void NewMixHistory::ResetMixedStatus() { |
| 99 _isMixed = false; |
| 100 } |
| 101 |
| 102 NewAudioConferenceMixer* NewAudioConferenceMixer::Create(int id) { |
| 103 NewAudioConferenceMixerImpl* mixer = new NewAudioConferenceMixerImpl(id); |
| 104 if (!mixer->Init()) { |
| 105 delete mixer; |
| 106 return NULL; |
| 107 } |
| 108 return mixer; |
| 109 } |
| 110 |
| 111 NewAudioConferenceMixerImpl::NewAudioConferenceMixerImpl(int id) |
| 112 : _id(id), |
| 113 _minimumMixingFreq(kLowestPossible), |
| 114 _mixReceiver(NULL), |
| 115 _outputFrequency(kDefaultFrequency), |
| 116 _sampleSize(0), |
| 117 _audioFramePool(NULL), |
| 118 _participantList(), |
| 119 _additionalParticipantList(), |
| 120 _numMixedParticipants(0), |
| 121 use_limiter_(true), |
| 122 _timeStamp(0), |
| 123 _timeScheduler(kProcessPeriodicityInMs), |
| 124 _processCalls(0) {} |
| 125 |
| 126 bool NewAudioConferenceMixerImpl::Init() { |
| 127 _crit.reset(CriticalSectionWrapper::CreateCriticalSection()); |
| 128 if (_crit.get() == NULL) |
| 129 return false; |
| 130 |
| 131 _cbCrit.reset(CriticalSectionWrapper::CreateCriticalSection()); |
| 132 if (_cbCrit.get() == NULL) |
| 133 return false; |
| 134 |
| 135 Config config; |
| 136 config.Set<ExperimentalAgc>(new ExperimentalAgc(false)); |
| 137 _limiter.reset(AudioProcessing::Create(config)); |
| 138 if (!_limiter.get()) |
| 139 return false; |
| 140 |
| 141 MemoryPool<AudioFrame>::CreateMemoryPool(_audioFramePool, |
| 142 DEFAULT_AUDIO_FRAME_POOLSIZE); |
| 143 if (_audioFramePool == NULL) |
| 144 return false; |
| 145 |
| 146 if (SetOutputFrequency(kDefaultFrequency) == -1) |
| 147 return false; |
| 148 |
| 149 if (_limiter->gain_control()->set_mode(GainControl::kFixedDigital) != |
| 150 _limiter->kNoError) |
| 151 return false; |
| 152 |
| 153 // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the |
| 154 // divide-by-2 but -7 is used instead to give a bit of headroom since the |
| 155 // AGC is not a hard limiter. |
| 156 if (_limiter->gain_control()->set_target_level_dbfs(7) != _limiter->kNoError) |
| 157 return false; |
| 158 |
| 159 if (_limiter->gain_control()->set_compression_gain_db(0) != |
| 160 _limiter->kNoError) |
| 161 return false; |
| 162 |
| 163 if (_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError) |
| 164 return false; |
| 165 |
| 166 if (_limiter->gain_control()->Enable(true) != _limiter->kNoError) |
| 167 return false; |
| 168 |
| 169 return true; |
| 170 } |
| 171 |
| 172 NewAudioConferenceMixerImpl::~NewAudioConferenceMixerImpl() { |
| 173 MemoryPool<AudioFrame>::DeleteMemoryPool(_audioFramePool); |
| 174 assert(_audioFramePool == NULL); |
| 175 } |
| 176 |
| 177 // Process should be called every kProcessPeriodicityInMs ms |
| 178 int64_t NewAudioConferenceMixerImpl::TimeUntilNextProcess() { |
| 179 int64_t timeUntilNextProcess = 0; |
| 180 CriticalSectionScoped cs(_crit.get()); |
| 181 if (_timeScheduler.TimeToNextUpdate(timeUntilNextProcess) != 0) { |
| 182 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, |
| 183 "failed in TimeToNextUpdate() call"); |
| 184 // Sanity check |
| 185 assert(false); |
| 186 return -1; |
| 187 } |
| 188 return timeUntilNextProcess; |
| 189 } |
| 190 |
| 191 void NewAudioConferenceMixerImpl::Process() { |
| 192 size_t remainingParticipantsAllowedToMix = kMaximumAmountOfMixedParticipants; |
| 193 { |
| 194 CriticalSectionScoped cs(_crit.get()); |
| 195 assert(_processCalls == 0); |
| 196 _processCalls++; |
| 197 |
| 198 // Let the scheduler know that we are running one iteration. |
| 199 _timeScheduler.UpdateScheduler(); |
| 200 } |
| 201 |
| 202 AudioFrameList mixList; |
| 203 AudioFrameList rampOutList; |
| 204 AudioFrameList additionalFramesList; |
| 205 std::map<int, MixerAudioSource*> mixedParticipantsMap; |
| 206 { |
| 207 CriticalSectionScoped cs(_cbCrit.get()); |
| 208 |
| 209 int32_t lowFreq = GetLowestMixingFrequency(); |
| 210 // SILK can run in 12 kHz and 24 kHz. These frequencies are not |
| 211 // supported so use the closest higher frequency to not lose any |
| 212 // information. |
| 213 // TODO(henrike): this is probably more appropriate to do in |
| 214 // GetLowestMixingFrequency(). |
| 215 if (lowFreq == 12000) { |
| 216 lowFreq = 16000; |
| 217 } else if (lowFreq == 24000) { |
| 218 lowFreq = 32000; |
| 219 } |
| 220 if (lowFreq <= 0) { |
| 221 CriticalSectionScoped cs(_crit.get()); |
| 222 _processCalls--; |
| 223 return; |
| 224 } else { |
| 225 switch (lowFreq) { |
| 226 case 8000: |
| 227 if (OutputFrequency() != kNbInHz) { |
| 228 SetOutputFrequency(kNbInHz); |
| 229 } |
| 230 break; |
| 231 case 16000: |
| 232 if (OutputFrequency() != kWbInHz) { |
| 233 SetOutputFrequency(kWbInHz); |
| 234 } |
| 235 break; |
| 236 case 32000: |
| 237 if (OutputFrequency() != kSwbInHz) { |
| 238 SetOutputFrequency(kSwbInHz); |
| 239 } |
| 240 break; |
| 241 case 48000: |
| 242 if (OutputFrequency() != kFbInHz) { |
| 243 SetOutputFrequency(kFbInHz); |
| 244 } |
| 245 break; |
| 246 default: |
| 247 assert(false); |
| 248 |
| 249 CriticalSectionScoped cs(_crit.get()); |
| 250 _processCalls--; |
| 251 return; |
| 252 } |
| 253 } |
| 254 |
| 255 UpdateToMix(&mixList, &rampOutList, &mixedParticipantsMap, |
| 256 &remainingParticipantsAllowedToMix); |
| 257 |
| 258 GetAdditionalAudio(&additionalFramesList); |
| 259 UpdateMixedStatus(mixedParticipantsMap); |
| 260 } |
| 261 |
| 262 // Get an AudioFrame for mixing from the memory pool. |
| 263 AudioFrame* mixedAudio = NULL; |
| 264 if (_audioFramePool->PopMemory(mixedAudio) == -1) { |
| 265 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, |
| 266 "failed PopMemory() call"); |
| 267 assert(false); |
| 268 return; |
| 269 } |
| 270 |
| 271 { |
| 272 CriticalSectionScoped cs(_crit.get()); |
| 273 |
| 274 // TODO(henrike): it might be better to decide the number of channels |
| 275 // with an API instead of dynamically. |
| 276 |
| 277 // Find the max channels over all mixing lists. |
| 278 const size_t num_mixed_channels = |
| 279 std::max(MaxNumChannels(&mixList), |
| 280 std::max(MaxNumChannels(&additionalFramesList), |
| 281 MaxNumChannels(&rampOutList))); |
| 282 |
| 283 mixedAudio->UpdateFrame(-1, _timeStamp, NULL, 0, _outputFrequency, |
| 284 AudioFrame::kNormalSpeech, AudioFrame::kVadPassive, |
| 285 num_mixed_channels); |
| 286 |
| 287 _timeStamp += static_cast<uint32_t>(_sampleSize); |
| 288 |
| 289 // We only use the limiter if it supports the output sample rate and |
| 290 // we're actually mixing multiple streams. |
| 291 use_limiter_ = _numMixedParticipants > 1 && |
| 292 _outputFrequency <= AudioProcessing::kMaxNativeSampleRateHz; |
| 293 |
| 294 MixFromList(mixedAudio, mixList); |
| 295 MixAnonomouslyFromList(mixedAudio, additionalFramesList); |
| 296 MixAnonomouslyFromList(mixedAudio, rampOutList); |
| 297 |
| 298 if (mixedAudio->samples_per_channel_ == 0) { |
| 299 // Nothing was mixed, set the audio samples to silence. |
| 300 mixedAudio->samples_per_channel_ = _sampleSize; |
| 301 mixedAudio->Mute(); |
| 302 } else { |
| 303 // Only call the limiter if we have something to mix. |
| 304 LimitMixedAudio(mixedAudio); |
| 305 } |
| 306 } |
| 307 |
| 308 { |
| 309 CriticalSectionScoped cs(_cbCrit.get()); |
| 310 if (_mixReceiver != NULL) { |
| 311 const AudioFrame** dummy = NULL; |
| 312 _mixReceiver->NewMixedAudio(_id, *mixedAudio, dummy, 0); |
| 313 } |
| 314 } |
| 315 |
| 316 // Reclaim all outstanding memory. |
| 317 _audioFramePool->PushMemory(mixedAudio); |
| 318 ClearAudioFrameList(&mixList); |
| 319 ClearAudioFrameList(&rampOutList); |
| 320 ClearAudioFrameList(&additionalFramesList); |
| 321 { |
| 322 CriticalSectionScoped cs(_crit.get()); |
| 323 _processCalls--; |
| 324 } |
| 325 return; |
| 326 } |
| 327 |
| 328 int32_t NewAudioConferenceMixerImpl::RegisterMixedStreamCallback( |
| 329 OldAudioMixerOutputReceiver* mixReceiver) { |
| 330 CriticalSectionScoped cs(_cbCrit.get()); |
| 331 if (_mixReceiver != NULL) { |
| 332 return -1; |
| 333 } |
| 334 _mixReceiver = mixReceiver; |
| 335 return 0; |
| 336 } |
| 337 |
| 338 int32_t NewAudioConferenceMixerImpl::UnRegisterMixedStreamCallback() { |
| 339 CriticalSectionScoped cs(_cbCrit.get()); |
| 340 if (_mixReceiver == NULL) { |
| 341 return -1; |
| 342 } |
| 343 _mixReceiver = NULL; |
| 344 return 0; |
| 345 } |
| 346 |
| 347 int32_t NewAudioConferenceMixerImpl::SetOutputFrequency( |
| 348 const Frequency& frequency) { |
| 349 CriticalSectionScoped cs(_crit.get()); |
| 350 |
| 351 _outputFrequency = frequency; |
| 352 _sampleSize = |
| 353 static_cast<size_t>((_outputFrequency * kProcessPeriodicityInMs) / 1000); |
| 354 |
| 355 return 0; |
| 356 } |
| 357 |
| 358 NewAudioConferenceMixer::Frequency |
| 359 NewAudioConferenceMixerImpl::OutputFrequency() const { |
| 360 CriticalSectionScoped cs(_crit.get()); |
| 361 return _outputFrequency; |
| 362 } |
| 363 |
| 364 int32_t NewAudioConferenceMixerImpl::SetMixabilityStatus( |
| 365 MixerAudioSource* participant, |
| 366 bool mixable) { |
| 367 if (!mixable) { |
| 368 // Anonymous participants are in a separate list. Make sure that the |
| 369 // participant is in the _participantList if it is being mixed. |
| 370 SetAnonymousMixabilityStatus(participant, false); |
| 371 } |
| 372 size_t numMixedParticipants; |
| 373 { |
| 374 CriticalSectionScoped cs(_cbCrit.get()); |
| 375 const bool isMixed = IsParticipantInList(*participant, _participantList); |
| 376 // API must be called with a new state. |
| 377 if (!(mixable ^ isMixed)) { |
| 378 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, |
| 379 "Mixable is aready %s", isMixed ? "ON" : "off"); |
| 380 return -1; |
| 381 } |
| 382 bool success = false; |
| 383 if (mixable) { |
| 384 success = AddParticipantToList(participant, &_participantList); |
| 385 } else { |
| 386 success = RemoveParticipantFromList(participant, &_participantList); |
| 387 } |
| 388 if (!success) { |
| 389 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, |
| 390 "failed to %s participant", mixable ? "add" : "remove"); |
| 391 assert(false); |
| 392 return -1; |
| 393 } |
| 394 |
| 395 size_t numMixedNonAnonymous = _participantList.size(); |
| 396 if (numMixedNonAnonymous > kMaximumAmountOfMixedParticipants) { |
| 397 numMixedNonAnonymous = kMaximumAmountOfMixedParticipants; |
| 398 } |
| 399 numMixedParticipants = |
| 400 numMixedNonAnonymous + _additionalParticipantList.size(); |
| 401 } |
| 402 // A MixerAudioSource was added or removed. Make sure the scratch |
| 403 // buffer is updated if necessary. |
| 404 // Note: The scratch buffer may only be updated in Process(). |
| 405 CriticalSectionScoped cs(_crit.get()); |
| 406 _numMixedParticipants = numMixedParticipants; |
| 407 return 0; |
| 408 } |
| 409 |
| 410 bool NewAudioConferenceMixerImpl::MixabilityStatus( |
| 411 const MixerAudioSource& participant) const { |
| 412 CriticalSectionScoped cs(_cbCrit.get()); |
| 413 return IsParticipantInList(participant, _participantList); |
| 414 } |
| 415 |
| 416 int32_t NewAudioConferenceMixerImpl::SetAnonymousMixabilityStatus( |
| 417 MixerAudioSource* participant, |
| 418 bool anonymous) { |
| 419 CriticalSectionScoped cs(_cbCrit.get()); |
| 420 if (IsParticipantInList(*participant, _additionalParticipantList)) { |
| 421 if (anonymous) { |
| 422 return 0; |
| 423 } |
| 424 if (!RemoveParticipantFromList(participant, &_additionalParticipantList)) { |
| 425 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, |
| 426 "unable to remove participant from anonymous list"); |
| 427 assert(false); |
| 428 return -1; |
| 429 } |
| 430 return AddParticipantToList(participant, &_participantList) ? 0 : -1; |
| 431 } |
| 432 if (!anonymous) { |
| 433 return 0; |
| 434 } |
| 435 const bool mixable = |
| 436 RemoveParticipantFromList(participant, &_participantList); |
| 437 if (!mixable) { |
| 438 WEBRTC_TRACE( |
| 439 kTraceWarning, kTraceAudioMixerServer, _id, |
| 440 "participant must be registered before turning it into anonymous"); |
| 441 // Setting anonymous status is only possible if MixerAudioSource is |
| 442 // already registered. |
| 443 return -1; |
| 444 } |
| 445 return AddParticipantToList(participant, &_additionalParticipantList) ? 0 |
| 446 : -1; |
| 447 } |
| 448 |
| 449 bool NewAudioConferenceMixerImpl::AnonymousMixabilityStatus( |
| 450 const MixerAudioSource& participant) const { |
| 451 CriticalSectionScoped cs(_cbCrit.get()); |
| 452 return IsParticipantInList(participant, _additionalParticipantList); |
| 453 } |
| 454 |
| 455 int32_t NewAudioConferenceMixerImpl::SetMinimumMixingFrequency(Frequency freq) { |
| 456 // Make sure that only allowed sampling frequencies are used. Use closest |
| 457 // higher sampling frequency to avoid losing information. |
| 458 if (static_cast<int>(freq) == 12000) { |
| 459 freq = kWbInHz; |
| 460 } else if (static_cast<int>(freq) == 24000) { |
| 461 freq = kSwbInHz; |
| 462 } |
| 463 |
| 464 if ((freq == kNbInHz) || (freq == kWbInHz) || (freq == kSwbInHz) || |
| 465 (freq == kLowestPossible)) { |
| 466 _minimumMixingFreq = freq; |
| 467 return 0; |
| 468 } else { |
| 469 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, |
| 470 "SetMinimumMixingFrequency incorrect frequency: %i", freq); |
| 471 assert(false); |
| 472 return -1; |
| 473 } |
| 474 } |
| 475 |
| 476 // Check all AudioFrames that are to be mixed. The highest sampling frequency |
| 477 // found is the lowest that can be used without losing information. |
| 478 int32_t NewAudioConferenceMixerImpl::GetLowestMixingFrequency() const { |
| 479 const int participantListFrequency = |
| 480 GetLowestMixingFrequencyFromList(_participantList); |
| 481 const int anonymousListFrequency = |
| 482 GetLowestMixingFrequencyFromList(_additionalParticipantList); |
| 483 const int highestFreq = (participantListFrequency > anonymousListFrequency) |
| 484 ? participantListFrequency |
| 485 : anonymousListFrequency; |
| 486 // Check if the user specified a lowest mixing frequency. |
| 487 if (_minimumMixingFreq != kLowestPossible) { |
| 488 if (_minimumMixingFreq > highestFreq) { |
| 489 return _minimumMixingFreq; |
| 490 } |
| 491 } |
| 492 return highestFreq; |
| 493 } |
| 494 |
| 495 int32_t NewAudioConferenceMixerImpl::GetLowestMixingFrequencyFromList( |
| 496 const MixerAudioSourceList& mixList) const { |
| 497 int32_t highestFreq = 8000; |
| 498 for (MixerAudioSourceList::const_iterator iter = mixList.begin(); |
| 499 iter != mixList.end(); ++iter) { |
| 500 const int32_t neededFrequency = (*iter)->NeededFrequency(_id); |
| 501 if (neededFrequency > highestFreq) { |
| 502 highestFreq = neededFrequency; |
| 503 } |
| 504 } |
| 505 return highestFreq; |
| 506 } |
| 507 |
| 508 void NewAudioConferenceMixerImpl::UpdateToMix( |
| 509 AudioFrameList* mixList, |
| 510 AudioFrameList* rampOutList, |
| 511 std::map<int, MixerAudioSource*>* mixParticipantList, |
| 512 size_t* maxAudioFrameCounter) const { |
| 513 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 514 "UpdateToMix(mixList,rampOutList,mixParticipantList,%d)", |
| 515 *maxAudioFrameCounter); |
| 516 const size_t mixListStartSize = mixList->size(); |
| 517 AudioFrameList activeList; |
| 518 // Struct needed by the passive lists to keep track of which AudioFrame |
| 519 // belongs to which MixerAudioSource. |
| 520 ParticipantFrameStructList passiveWasNotMixedList; |
| 521 ParticipantFrameStructList passiveWasMixedList; |
| 522 for (MixerAudioSourceList::const_iterator participant = |
| 523 _participantList.begin(); |
| 524 participant != _participantList.end(); ++participant) { |
| 525 // Stop keeping track of passive participants if there are already |
| 526 // enough participants available (they wont be mixed anyway). |
| 527 bool mustAddToPassiveList = |
| 528 (*maxAudioFrameCounter > |
| 529 (activeList.size() + passiveWasMixedList.size() + |
| 530 passiveWasNotMixedList.size())); |
| 531 |
| 532 bool wasMixed = false; |
| 533 wasMixed = (*participant)->_mixHistory->WasMixed(); |
| 534 AudioFrame* audioFrame = NULL; |
| 535 if (_audioFramePool->PopMemory(audioFrame) == -1) { |
| 536 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, |
| 537 "failed PopMemory() call"); |
| 538 assert(false); |
| 539 return; |
| 540 } |
| 541 audioFrame->sample_rate_hz_ = _outputFrequency; |
| 542 |
| 543 auto ret = (*participant)->GetAudioFrameWithMuted(_id, audioFrame); |
| 544 if (ret == MixerAudioSource::AudioFrameInfo::kError) { |
| 545 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, |
| 546 "failed to GetAudioFrameWithMuted() from participant"); |
| 547 _audioFramePool->PushMemory(audioFrame); |
| 548 continue; |
| 549 } |
| 550 const bool muted = (ret == MixerAudioSource::AudioFrameInfo::kMuted); |
| 551 if (_participantList.size() != 1) { |
| 552 // TODO(wu): Issue 3390, add support for multiple participants case. |
| 553 audioFrame->ntp_time_ms_ = -1; |
| 554 } |
| 555 |
| 556 // TODO(henrike): this assert triggers in some test cases where SRTP is |
| 557 // used which prevents NetEQ from making a VAD. Temporarily disable this |
| 558 // assert until the problem is fixed on a higher level. |
| 559 // assert(audioFrame->vad_activity_ != AudioFrame::kVadUnknown); |
| 560 if (audioFrame->vad_activity_ == AudioFrame::kVadUnknown) { |
| 561 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, |
| 562 "invalid VAD state from participant"); |
| 563 } |
| 564 |
| 565 if (audioFrame->vad_activity_ == AudioFrame::kVadActive) { |
| 566 if (!wasMixed && !muted) { |
| 567 RampIn(*audioFrame); |
| 568 } |
| 569 |
| 570 if (activeList.size() >= *maxAudioFrameCounter) { |
| 571 // There are already more active participants than should be |
| 572 // mixed. Only keep the ones with the highest energy. |
| 573 AudioFrameList::iterator replaceItem; |
| 574 uint32_t lowestEnergy = muted ? 0 : CalculateEnergy(*audioFrame); |
| 575 |
| 576 bool found_replace_item = false; |
| 577 for (AudioFrameList::iterator iter = activeList.begin(); |
| 578 iter != activeList.end(); ++iter) { |
| 579 const uint32_t energy = muted ? 0 : CalculateEnergy(*iter->frame); |
| 580 if (energy < lowestEnergy) { |
| 581 replaceItem = iter; |
| 582 lowestEnergy = energy; |
| 583 found_replace_item = true; |
| 584 } |
| 585 } |
| 586 if (found_replace_item) { |
| 587 RTC_DCHECK(!muted); // Cannot replace with a muted frame. |
| 588 FrameAndMuteInfo replaceFrame = *replaceItem; |
| 589 |
| 590 bool replaceWasMixed = false; |
| 591 std::map<int, MixerAudioSource*>::const_iterator it = |
| 592 mixParticipantList->find(replaceFrame.frame->id_); |
| 593 |
| 594 // When a frame is pushed to |activeList| it is also pushed |
| 595 // to mixParticipantList with the frame's id. This means |
| 596 // that the Find call above should never fail. |
| 597 assert(it != mixParticipantList->end()); |
| 598 replaceWasMixed = it->second->_mixHistory->WasMixed(); |
| 599 |
| 600 mixParticipantList->erase(replaceFrame.frame->id_); |
| 601 activeList.erase(replaceItem); |
| 602 |
| 603 activeList.push_front(FrameAndMuteInfo(audioFrame, muted)); |
| 604 (*mixParticipantList)[audioFrame->id_] = *participant; |
| 605 assert(mixParticipantList->size() <= |
| 606 kMaximumAmountOfMixedParticipants); |
| 607 |
| 608 if (replaceWasMixed) { |
| 609 if (!replaceFrame.muted) { |
| 610 RampOut(*replaceFrame.frame); |
| 611 } |
| 612 rampOutList->push_back(replaceFrame); |
| 613 assert(rampOutList->size() <= kMaximumAmountOfMixedParticipants); |
| 614 } else { |
| 615 _audioFramePool->PushMemory(replaceFrame.frame); |
| 616 } |
| 617 } else { |
| 618 if (wasMixed) { |
| 619 if (!muted) { |
| 620 RampOut(*audioFrame); |
| 621 } |
| 622 rampOutList->push_back(FrameAndMuteInfo(audioFrame, muted)); |
| 623 assert(rampOutList->size() <= kMaximumAmountOfMixedParticipants); |
| 624 } else { |
| 625 _audioFramePool->PushMemory(audioFrame); |
| 626 } |
| 627 } |
| 628 } else { |
| 629 activeList.push_front(FrameAndMuteInfo(audioFrame, muted)); |
| 630 (*mixParticipantList)[audioFrame->id_] = *participant; |
| 631 assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); |
| 632 } |
| 633 } else { |
| 634 if (wasMixed) { |
| 635 ParticipantFrameStruct* part_struct = |
| 636 new ParticipantFrameStruct(*participant, audioFrame, muted); |
| 637 passiveWasMixedList.push_back(part_struct); |
| 638 } else if (mustAddToPassiveList) { |
| 639 if (!muted) { |
| 640 RampIn(*audioFrame); |
| 641 } |
| 642 ParticipantFrameStruct* part_struct = |
| 643 new ParticipantFrameStruct(*participant, audioFrame, muted); |
| 644 passiveWasNotMixedList.push_back(part_struct); |
| 645 } else { |
| 646 _audioFramePool->PushMemory(audioFrame); |
| 647 } |
| 648 } |
| 649 } |
| 650 assert(activeList.size() <= *maxAudioFrameCounter); |
| 651 // At this point it is known which participants should be mixed. Transfer |
| 652 // this information to this functions output parameters. |
| 653 for (AudioFrameList::const_iterator iter = activeList.begin(); |
| 654 iter != activeList.end(); ++iter) { |
| 655 mixList->push_back(*iter); |
| 656 } |
| 657 activeList.clear(); |
| 658 // Always mix a constant number of AudioFrames. If there aren't enough |
| 659 // active participants mix passive ones. Starting with those that was mixed |
| 660 // last iteration. |
| 661 for (ParticipantFrameStructList::const_iterator iter = |
| 662 passiveWasMixedList.begin(); |
| 663 iter != passiveWasMixedList.end(); ++iter) { |
| 664 if (mixList->size() < *maxAudioFrameCounter + mixListStartSize) { |
| 665 mixList->push_back(FrameAndMuteInfo((*iter)->audioFrame, (*iter)->muted)); |
| 666 (*mixParticipantList)[(*iter)->audioFrame->id_] = (*iter)->participant; |
| 667 assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); |
| 668 } else { |
| 669 _audioFramePool->PushMemory((*iter)->audioFrame); |
| 670 } |
| 671 delete *iter; |
| 672 } |
| 673 // And finally the ones that have not been mixed for a while. |
| 674 for (ParticipantFrameStructList::const_iterator iter = |
| 675 passiveWasNotMixedList.begin(); |
| 676 iter != passiveWasNotMixedList.end(); ++iter) { |
| 677 if (mixList->size() < *maxAudioFrameCounter + mixListStartSize) { |
| 678 mixList->push_back(FrameAndMuteInfo((*iter)->audioFrame, (*iter)->muted)); |
| 679 (*mixParticipantList)[(*iter)->audioFrame->id_] = (*iter)->participant; |
| 680 assert(mixParticipantList->size() <= kMaximumAmountOfMixedParticipants); |
| 681 } else { |
| 682 _audioFramePool->PushMemory((*iter)->audioFrame); |
| 683 } |
| 684 delete *iter; |
| 685 } |
| 686 assert(*maxAudioFrameCounter + mixListStartSize >= mixList->size()); |
| 687 *maxAudioFrameCounter += mixListStartSize - mixList->size(); |
| 688 } |
| 689 |
| 690 void NewAudioConferenceMixerImpl::GetAdditionalAudio( |
| 691 AudioFrameList* additionalFramesList) const { |
| 692 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 693 "GetAdditionalAudio(additionalFramesList)"); |
| 694 // The GetAudioFrameWithMuted() callback may result in the participant being |
| 695 // removed from additionalParticipantList_. If that happens it will |
| 696 // invalidate any iterators. Create a copy of the participants list such |
| 697 // that the list of participants can be traversed safely. |
| 698 MixerAudioSourceList additionalParticipantList; |
| 699 additionalParticipantList.insert(additionalParticipantList.begin(), |
| 700 _additionalParticipantList.begin(), |
| 701 _additionalParticipantList.end()); |
| 702 |
| 703 for (MixerAudioSourceList::const_iterator participant = |
| 704 additionalParticipantList.begin(); |
| 705 participant != additionalParticipantList.end(); ++participant) { |
| 706 AudioFrame* audioFrame = NULL; |
| 707 if (_audioFramePool->PopMemory(audioFrame) == -1) { |
| 708 WEBRTC_TRACE(kTraceMemory, kTraceAudioMixerServer, _id, |
| 709 "failed PopMemory() call"); |
| 710 assert(false); |
| 711 return; |
| 712 } |
| 713 audioFrame->sample_rate_hz_ = _outputFrequency; |
| 714 auto ret = (*participant)->GetAudioFrameWithMuted(_id, audioFrame); |
| 715 if (ret == MixerAudioSource::AudioFrameInfo::kError) { |
| 716 WEBRTC_TRACE(kTraceWarning, kTraceAudioMixerServer, _id, |
| 717 "failed to GetAudioFrameWithMuted() from participant"); |
| 718 _audioFramePool->PushMemory(audioFrame); |
| 719 continue; |
| 720 } |
| 721 if (audioFrame->samples_per_channel_ == 0) { |
| 722 // Empty frame. Don't use it. |
| 723 _audioFramePool->PushMemory(audioFrame); |
| 724 continue; |
| 725 } |
| 726 additionalFramesList->push_back(FrameAndMuteInfo( |
| 727 audioFrame, ret == MixerAudioSource::AudioFrameInfo::kMuted)); |
| 728 } |
| 729 } |
| 730 |
| 731 void NewAudioConferenceMixerImpl::UpdateMixedStatus( |
| 732 const std::map<int, MixerAudioSource*>& mixedParticipantsMap) const { |
| 733 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 734 "UpdateMixedStatus(mixedParticipantsMap)"); |
| 735 assert(mixedParticipantsMap.size() <= kMaximumAmountOfMixedParticipants); |
| 736 |
| 737 // Loop through all participants. If they are in the mix map they |
| 738 // were mixed. |
| 739 for (MixerAudioSourceList::const_iterator participant = |
| 740 _participantList.begin(); |
| 741 participant != _participantList.end(); ++participant) { |
| 742 bool isMixed = false; |
| 743 for (std::map<int, MixerAudioSource*>::const_iterator it = |
| 744 mixedParticipantsMap.begin(); |
| 745 it != mixedParticipantsMap.end(); ++it) { |
| 746 if (it->second == *participant) { |
| 747 isMixed = true; |
| 748 break; |
| 749 } |
| 750 } |
| 751 (*participant)->_mixHistory->SetIsMixed(isMixed); |
| 752 } |
| 753 } |
| 754 |
| 755 void NewAudioConferenceMixerImpl::ClearAudioFrameList( |
| 756 AudioFrameList* audioFrameList) const { |
| 757 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 758 "ClearAudioFrameList(audioFrameList)"); |
| 759 for (AudioFrameList::iterator iter = audioFrameList->begin(); |
| 760 iter != audioFrameList->end(); ++iter) { |
| 761 _audioFramePool->PushMemory(iter->frame); |
| 762 } |
| 763 audioFrameList->clear(); |
| 764 } |
| 765 |
| 766 bool NewAudioConferenceMixerImpl::IsParticipantInList( |
| 767 const MixerAudioSource& participant, |
| 768 const MixerAudioSourceList& participantList) const { |
| 769 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 770 "IsParticipantInList(participant,participantList)"); |
| 771 for (MixerAudioSourceList::const_iterator iter = participantList.begin(); |
| 772 iter != participantList.end(); ++iter) { |
| 773 if (&participant == *iter) { |
| 774 return true; |
| 775 } |
| 776 } |
| 777 return false; |
| 778 } |
| 779 |
| 780 bool NewAudioConferenceMixerImpl::AddParticipantToList( |
| 781 MixerAudioSource* participant, |
| 782 MixerAudioSourceList* participantList) const { |
| 783 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 784 "AddParticipantToList(participant, participantList)"); |
| 785 participantList->push_back(participant); |
| 786 // Make sure that the mixed status is correct for new MixerAudioSource. |
| 787 participant->_mixHistory->ResetMixedStatus(); |
| 788 return true; |
| 789 } |
| 790 |
| 791 bool NewAudioConferenceMixerImpl::RemoveParticipantFromList( |
| 792 MixerAudioSource* participant, |
| 793 MixerAudioSourceList* participantList) const { |
| 794 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 795 "RemoveParticipantFromList(participant, participantList)"); |
| 796 for (MixerAudioSourceList::iterator iter = participantList->begin(); |
| 797 iter != participantList->end(); ++iter) { |
| 798 if (*iter == participant) { |
| 799 participantList->erase(iter); |
| 800 // Participant is no longer mixed, reset to default. |
| 801 participant->_mixHistory->ResetMixedStatus(); |
| 802 return true; |
| 803 } |
| 804 } |
| 805 return false; |
| 806 } |
| 807 |
| 808 int32_t NewAudioConferenceMixerImpl::MixFromList( |
| 809 AudioFrame* mixedAudio, |
| 810 const AudioFrameList& audioFrameList) const { |
| 811 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 812 "MixFromList(mixedAudio, audioFrameList)"); |
| 813 if (audioFrameList.empty()) |
| 814 return 0; |
| 815 |
| 816 uint32_t position = 0; |
| 817 |
| 818 if (_numMixedParticipants == 1) { |
| 819 mixedAudio->timestamp_ = audioFrameList.front().frame->timestamp_; |
| 820 mixedAudio->elapsed_time_ms_ = |
| 821 audioFrameList.front().frame->elapsed_time_ms_; |
| 822 } else { |
| 823 // TODO(wu): Issue 3390. |
| 824 // Audio frame timestamp is only supported in one channel case. |
| 825 mixedAudio->timestamp_ = 0; |
| 826 mixedAudio->elapsed_time_ms_ = -1; |
| 827 } |
| 828 |
| 829 for (AudioFrameList::const_iterator iter = audioFrameList.begin(); |
| 830 iter != audioFrameList.end(); ++iter) { |
| 831 if (position >= kMaximumAmountOfMixedParticipants) { |
| 832 WEBRTC_TRACE( |
| 833 kTraceMemory, kTraceAudioMixerServer, _id, |
| 834 "Trying to mix more than max amount of mixed participants:%d!", |
| 835 kMaximumAmountOfMixedParticipants); |
| 836 // Assert and avoid crash |
| 837 assert(false); |
| 838 position = 0; |
| 839 } |
| 840 if (!iter->muted) { |
| 841 MixFrames(mixedAudio, iter->frame, use_limiter_); |
| 842 } |
| 843 |
| 844 position++; |
| 845 } |
| 846 |
| 847 return 0; |
| 848 } |
| 849 |
| 850 // TODO(andrew): consolidate this function with MixFromList. |
| 851 int32_t NewAudioConferenceMixerImpl::MixAnonomouslyFromList( |
| 852 AudioFrame* mixedAudio, |
| 853 const AudioFrameList& audioFrameList) const { |
| 854 WEBRTC_TRACE(kTraceStream, kTraceAudioMixerServer, _id, |
| 855 "MixAnonomouslyFromList(mixedAudio, audioFrameList)"); |
| 856 |
| 857 if (audioFrameList.empty()) |
| 858 return 0; |
| 859 |
| 860 for (AudioFrameList::const_iterator iter = audioFrameList.begin(); |
| 861 iter != audioFrameList.end(); ++iter) { |
| 862 if (!iter->muted) { |
| 863 MixFrames(mixedAudio, iter->frame, use_limiter_); |
| 864 } |
| 865 } |
| 866 return 0; |
| 867 } |
| 868 |
| 869 bool NewAudioConferenceMixerImpl::LimitMixedAudio( |
| 870 AudioFrame* mixedAudio) const { |
| 871 if (!use_limiter_) { |
| 872 return true; |
| 873 } |
| 874 |
| 875 // Smoothly limit the mixed frame. |
| 876 const int error = _limiter->ProcessStream(mixedAudio); |
| 877 |
| 878 // And now we can safely restore the level. This procedure results in |
| 879 // some loss of resolution, deemed acceptable. |
| 880 // |
| 881 // It's possible to apply the gain in the AGC (with a target level of 0 dbFS |
| 882 // and compression gain of 6 dB). However, in the transition frame when this |
| 883 // is enabled (moving from one to two participants) it has the potential to |
| 884 // create discontinuities in the mixed frame. |
| 885 // |
| 886 // Instead we double the frame (with addition since left-shifting a |
| 887 // negative value is undefined). |
| 888 *mixedAudio += *mixedAudio; |
| 889 |
| 890 if (error != _limiter->kNoError) { |
| 891 WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id, |
| 892 "Error from AudioProcessing: %d", error); |
| 893 assert(false); |
| 894 return false; |
| 895 } |
| 896 return true; |
| 897 } |
| 898 } // namespace webrtc |
OLD | NEW |