OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 23 matching lines...) Expand all Loading... |
34 #include "webrtc/modules/audio_coding/neteq/delay_manager.h" | 34 #include "webrtc/modules/audio_coding/neteq/delay_manager.h" |
35 #include "webrtc/modules/audio_coding/neteq/delay_peak_detector.h" | 35 #include "webrtc/modules/audio_coding/neteq/delay_peak_detector.h" |
36 #include "webrtc/modules/audio_coding/neteq/dtmf_buffer.h" | 36 #include "webrtc/modules/audio_coding/neteq/dtmf_buffer.h" |
37 #include "webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h" | 37 #include "webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h" |
38 #include "webrtc/modules/audio_coding/neteq/expand.h" | 38 #include "webrtc/modules/audio_coding/neteq/expand.h" |
39 #include "webrtc/modules/audio_coding/neteq/merge.h" | 39 #include "webrtc/modules/audio_coding/neteq/merge.h" |
40 #include "webrtc/modules/audio_coding/neteq/nack_tracker.h" | 40 #include "webrtc/modules/audio_coding/neteq/nack_tracker.h" |
41 #include "webrtc/modules/audio_coding/neteq/normal.h" | 41 #include "webrtc/modules/audio_coding/neteq/normal.h" |
42 #include "webrtc/modules/audio_coding/neteq/packet_buffer.h" | 42 #include "webrtc/modules/audio_coding/neteq/packet_buffer.h" |
43 #include "webrtc/modules/audio_coding/neteq/packet.h" | 43 #include "webrtc/modules/audio_coding/neteq/packet.h" |
44 #include "webrtc/modules/audio_coding/neteq/payload_splitter.h" | 44 #include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h" |
45 #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h" | 45 #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h" |
46 #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h" | 46 #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h" |
47 #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" | 47 #include "webrtc/modules/audio_coding/neteq/sync_buffer.h" |
48 #include "webrtc/modules/audio_coding/neteq/tick_timer.h" | 48 #include "webrtc/modules/audio_coding/neteq/tick_timer.h" |
49 #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h" | 49 #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h" |
50 #include "webrtc/modules/include/module_common_types.h" | 50 #include "webrtc/modules/include/module_common_types.h" |
51 | 51 |
52 namespace webrtc { | 52 namespace webrtc { |
53 | 53 |
54 NetEqImpl::Dependencies::Dependencies( | 54 NetEqImpl::Dependencies::Dependencies( |
55 const NetEq::Config& config, | 55 const NetEq::Config& config, |
56 const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory) | 56 const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory) |
57 : tick_timer(new TickTimer), | 57 : tick_timer(new TickTimer), |
58 buffer_level_filter(new BufferLevelFilter), | 58 buffer_level_filter(new BufferLevelFilter), |
59 decoder_database(new DecoderDatabase(decoder_factory)), | 59 decoder_database(new DecoderDatabase(decoder_factory)), |
60 delay_peak_detector(new DelayPeakDetector(tick_timer.get())), | 60 delay_peak_detector(new DelayPeakDetector(tick_timer.get())), |
61 delay_manager(new DelayManager(config.max_packets_in_buffer, | 61 delay_manager(new DelayManager(config.max_packets_in_buffer, |
62 delay_peak_detector.get(), | 62 delay_peak_detector.get(), |
63 tick_timer.get())), | 63 tick_timer.get())), |
64 dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)), | 64 dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)), |
65 dtmf_tone_generator(new DtmfToneGenerator), | 65 dtmf_tone_generator(new DtmfToneGenerator), |
66 packet_buffer( | 66 packet_buffer( |
67 new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), | 67 new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())), |
68 payload_splitter(new PayloadSplitter), | 68 red_payload_splitter(new RedPayloadSplitter), |
69 timestamp_scaler(new TimestampScaler(*decoder_database)), | 69 timestamp_scaler(new TimestampScaler(*decoder_database)), |
70 accelerate_factory(new AccelerateFactory), | 70 accelerate_factory(new AccelerateFactory), |
71 expand_factory(new ExpandFactory), | 71 expand_factory(new ExpandFactory), |
72 preemptive_expand_factory(new PreemptiveExpandFactory) {} | 72 preemptive_expand_factory(new PreemptiveExpandFactory) {} |
73 | 73 |
74 NetEqImpl::Dependencies::~Dependencies() = default; | 74 NetEqImpl::Dependencies::~Dependencies() = default; |
75 | 75 |
76 NetEqImpl::NetEqImpl(const NetEq::Config& config, | 76 NetEqImpl::NetEqImpl(const NetEq::Config& config, |
77 Dependencies&& deps, | 77 Dependencies&& deps, |
78 bool create_components) | 78 bool create_components) |
79 : tick_timer_(std::move(deps.tick_timer)), | 79 : tick_timer_(std::move(deps.tick_timer)), |
80 buffer_level_filter_(std::move(deps.buffer_level_filter)), | 80 buffer_level_filter_(std::move(deps.buffer_level_filter)), |
81 decoder_database_(std::move(deps.decoder_database)), | 81 decoder_database_(std::move(deps.decoder_database)), |
82 delay_manager_(std::move(deps.delay_manager)), | 82 delay_manager_(std::move(deps.delay_manager)), |
83 delay_peak_detector_(std::move(deps.delay_peak_detector)), | 83 delay_peak_detector_(std::move(deps.delay_peak_detector)), |
84 dtmf_buffer_(std::move(deps.dtmf_buffer)), | 84 dtmf_buffer_(std::move(deps.dtmf_buffer)), |
85 dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), | 85 dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)), |
86 packet_buffer_(std::move(deps.packet_buffer)), | 86 packet_buffer_(std::move(deps.packet_buffer)), |
87 payload_splitter_(std::move(deps.payload_splitter)), | 87 red_payload_splitter_(std::move(deps.red_payload_splitter)), |
88 timestamp_scaler_(std::move(deps.timestamp_scaler)), | 88 timestamp_scaler_(std::move(deps.timestamp_scaler)), |
89 vad_(new PostDecodeVad()), | 89 vad_(new PostDecodeVad()), |
90 expand_factory_(std::move(deps.expand_factory)), | 90 expand_factory_(std::move(deps.expand_factory)), |
91 accelerate_factory_(std::move(deps.accelerate_factory)), | 91 accelerate_factory_(std::move(deps.accelerate_factory)), |
92 preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)), | 92 preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)), |
93 last_mode_(kModeNormal), | 93 last_mode_(kModeNormal), |
94 decoded_buffer_length_(kMaxFrameSize), | 94 decoded_buffer_length_(kMaxFrameSize), |
95 decoded_buffer_(new int16_t[decoded_buffer_length_]), | 95 decoded_buffer_(new int16_t[decoded_buffer_length_]), |
96 playout_timestamp_(0), | 96 playout_timestamp_(0), |
97 new_codec_(false), | 97 new_codec_(false), |
(...skipping 424 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
522 // directly once it's been inserted in the packet list. This way, |packet| | 522 // directly once it's been inserted in the packet list. This way, |packet| |
523 // is not defined outside of this block. | 523 // is not defined outside of this block. |
524 Packet* packet = new Packet; | 524 Packet* packet = new Packet; |
525 packet->header.markerBit = false; | 525 packet->header.markerBit = false; |
526 packet->header.payloadType = rtp_header.header.payloadType; | 526 packet->header.payloadType = rtp_header.header.payloadType; |
527 packet->header.sequenceNumber = rtp_header.header.sequenceNumber; | 527 packet->header.sequenceNumber = rtp_header.header.sequenceNumber; |
528 packet->header.timestamp = rtp_header.header.timestamp; | 528 packet->header.timestamp = rtp_header.header.timestamp; |
529 packet->header.ssrc = rtp_header.header.ssrc; | 529 packet->header.ssrc = rtp_header.header.ssrc; |
530 packet->header.numCSRCs = 0; | 530 packet->header.numCSRCs = 0; |
531 packet->payload.SetData(payload.data(), payload.size()); | 531 packet->payload.SetData(payload.data(), payload.size()); |
532 packet->primary = true; | |
533 // Waiting time will be set upon inserting the packet in the buffer. | 532 // Waiting time will be set upon inserting the packet in the buffer. |
534 RTC_DCHECK(!packet->waiting_time); | 533 RTC_DCHECK(!packet->waiting_time); |
535 // Insert packet in a packet list. | 534 // Insert packet in a packet list. |
536 packet_list.push_back(packet); | 535 packet_list.push_back(packet); |
537 // Save main payloads header for later. | 536 // Save main payloads header for later. |
538 memcpy(&main_header, &packet->header, sizeof(main_header)); | 537 memcpy(&main_header, &packet->header, sizeof(main_header)); |
539 } | 538 } |
540 | 539 |
541 bool update_sample_rate_and_channels = false; | 540 bool update_sample_rate_and_channels = false; |
542 // Reinitialize NetEq if it's needed (changed SSRC or first call). | 541 // Reinitialize NetEq if it's needed (changed SSRC or first call). |
(...skipping 21 matching lines...) Expand all Loading... |
564 | 563 |
565 // Trigger an update of sampling rate and the number of channels. | 564 // Trigger an update of sampling rate and the number of channels. |
566 update_sample_rate_and_channels = true; | 565 update_sample_rate_and_channels = true; |
567 } | 566 } |
568 | 567 |
569 // Update RTCP statistics, only for regular packets. | 568 // Update RTCP statistics, only for regular packets. |
570 rtcp_.Update(main_header, receive_timestamp); | 569 rtcp_.Update(main_header, receive_timestamp); |
571 | 570 |
572 // Check for RED payload type, and separate payloads into several packets. | 571 // Check for RED payload type, and separate payloads into several packets. |
573 if (decoder_database_->IsRed(main_header.payloadType)) { | 572 if (decoder_database_->IsRed(main_header.payloadType)) { |
574 if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) { | 573 if (!red_payload_splitter_->SplitRed(&packet_list)) { |
575 PacketBuffer::DeleteAllPackets(&packet_list); | 574 PacketBuffer::DeleteAllPackets(&packet_list); |
576 return kRedundancySplitError; | 575 return kRedundancySplitError; |
577 } | 576 } |
578 // Only accept a few RED payloads of the same type as the main data, | 577 // Only accept a few RED payloads of the same type as the main data, |
579 // DTMF events and CNG. | 578 // DTMF events and CNG. |
580 payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); | 579 red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_); |
581 // Update the stored main payload header since the main payload has now | 580 // Update the stored main payload header since the main payload has now |
582 // changed. | 581 // changed. |
583 memcpy(&main_header, &packet_list.front()->header, sizeof(main_header)); | 582 memcpy(&main_header, &packet_list.front()->header, sizeof(main_header)); |
584 } | 583 } |
585 | 584 |
586 // Check payload types. | 585 // Check payload types. |
587 if (decoder_database_->CheckPayloadTypes(packet_list) == | 586 if (decoder_database_->CheckPayloadTypes(packet_list) == |
588 DecoderDatabase::kDecoderNotFound) { | 587 DecoderDatabase::kDecoderNotFound) { |
589 PacketBuffer::DeleteAllPackets(&packet_list); | 588 PacketBuffer::DeleteAllPackets(&packet_list); |
590 return kUnknownRtpPayloadType; | 589 return kUnknownRtpPayloadType; |
(...skipping 22 matching lines...) Expand all Loading... |
613 PacketBuffer::DeleteAllPackets(&packet_list); | 612 PacketBuffer::DeleteAllPackets(&packet_list); |
614 return kDtmfInsertError; | 613 return kDtmfInsertError; |
615 } | 614 } |
616 delete current_packet; | 615 delete current_packet; |
617 it = packet_list.erase(it); | 616 it = packet_list.erase(it); |
618 } else { | 617 } else { |
619 ++it; | 618 ++it; |
620 } | 619 } |
621 } | 620 } |
622 | 621 |
623 // Check for FEC in packets, and separate payloads into several packets. | |
624 int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get()); | |
625 if (ret != PayloadSplitter::kOK) { | |
626 PacketBuffer::DeleteAllPackets(&packet_list); | |
627 switch (ret) { | |
628 case PayloadSplitter::kUnknownPayloadType: | |
629 return kUnknownRtpPayloadType; | |
630 default: | |
631 return kOtherError; | |
632 } | |
633 } | |
634 | |
635 // Update bandwidth estimate, if the packet is not comfort noise. | 622 // Update bandwidth estimate, if the packet is not comfort noise. |
636 if (!packet_list.empty() && | 623 if (!packet_list.empty() && |
637 !decoder_database_->IsComfortNoise(main_header.payloadType)) { | 624 !decoder_database_->IsComfortNoise(main_header.payloadType)) { |
638 // The list can be empty here if we got nothing but DTMF payloads. | 625 // The list can be empty here if we got nothing but DTMF payloads. |
639 AudioDecoder* decoder = | 626 AudioDecoder* decoder = |
640 decoder_database_->GetDecoder(main_header.payloadType); | 627 decoder_database_->GetDecoder(main_header.payloadType); |
641 assert(decoder); // Should always get a valid object, since we have | 628 assert(decoder); // Should always get a valid object, since we have |
642 // already checked that the payload types are known. | 629 // already checked that the payload types are known. |
643 decoder->IncomingPacket(packet_list.front()->payload.data(), | 630 decoder->IncomingPacket(packet_list.front()->payload.data(), |
644 packet_list.front()->payload.size(), | 631 packet_list.front()->payload.size(), |
(...skipping 12 matching lines...) Expand all Loading... |
657 LOG(LS_WARNING) << "SplitAudio unknown payload type"; | 644 LOG(LS_WARNING) << "SplitAudio unknown payload type"; |
658 return kUnknownRtpPayloadType; | 645 return kUnknownRtpPayloadType; |
659 } | 646 } |
660 | 647 |
661 if (info->IsComfortNoise()) { | 648 if (info->IsComfortNoise()) { |
662 // Carry comfort noise packets along. | 649 // Carry comfort noise packets along. |
663 parsed_packet_list.push_back(packet.release()); | 650 parsed_packet_list.push_back(packet.release()); |
664 } else { | 651 } else { |
665 std::vector<AudioDecoder::ParseResult> results = | 652 std::vector<AudioDecoder::ParseResult> results = |
666 info->GetDecoder()->ParsePayload(std::move(packet->payload), | 653 info->GetDecoder()->ParsePayload(std::move(packet->payload), |
667 packet->header.timestamp, | 654 packet->header.timestamp); |
668 packet->primary); | |
669 const RTPHeader& original_header = packet->header; | 655 const RTPHeader& original_header = packet->header; |
| 656 const Packet::Priority original_priority = packet->priority; |
670 for (auto& result : results) { | 657 for (auto& result : results) { |
671 RTC_DCHECK(result.frame); | 658 RTC_DCHECK(result.frame); |
672 // Reuse the packet if possible. | 659 // Reuse the packet if possible. |
673 if (!packet) { | 660 if (!packet) { |
674 packet.reset(new Packet); | 661 packet.reset(new Packet); |
675 packet->header = original_header; | 662 packet->header = original_header; |
676 } | 663 } |
677 packet->header.timestamp = result.timestamp; | 664 packet->header.timestamp = result.timestamp; |
678 // TODO(ossu): Move from primary to some sort of priority level. | 665 RTC_DCHECK_GE(result.priority, 0); |
679 packet->primary = result.primary; | 666 packet->priority.codec_level = result.priority; |
| 667 packet->priority.red_level = original_priority.red_level; |
680 packet->frame = std::move(result.frame); | 668 packet->frame = std::move(result.frame); |
681 parsed_packet_list.push_back(packet.release()); | 669 parsed_packet_list.push_back(packet.release()); |
682 } | 670 } |
683 } | 671 } |
684 } | 672 } |
685 | 673 |
686 if (nack_enabled_) { | 674 if (nack_enabled_) { |
687 RTC_DCHECK(nack_); | 675 RTC_DCHECK(nack_); |
688 if (update_sample_rate_and_channels) { | 676 if (update_sample_rate_and_channels) { |
689 nack_->Reset(); | 677 nack_->Reset(); |
690 } | 678 } |
691 nack_->UpdateLastReceivedPacket( | 679 nack_->UpdateLastReceivedPacket( |
692 parsed_packet_list.front()->header.sequenceNumber, | 680 parsed_packet_list.front()->header.sequenceNumber, |
693 parsed_packet_list.front()->header.timestamp); | 681 parsed_packet_list.front()->header.timestamp); |
694 } | 682 } |
695 | 683 |
696 // Insert packets in buffer. | 684 // Insert packets in buffer. |
697 const size_t buffer_length_before_insert = | 685 const size_t buffer_length_before_insert = |
698 packet_buffer_->NumPacketsInBuffer(); | 686 packet_buffer_->NumPacketsInBuffer(); |
699 ret = packet_buffer_->InsertPacketList( | 687 const int ret = packet_buffer_->InsertPacketList( |
700 &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_, | 688 &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_, |
701 ¤t_cng_rtp_payload_type_); | 689 ¤t_cng_rtp_payload_type_); |
702 if (ret == PacketBuffer::kFlushed) { | 690 if (ret == PacketBuffer::kFlushed) { |
703 // Reset DSP timestamp etc. if packet buffer flushed. | 691 // Reset DSP timestamp etc. if packet buffer flushed. |
704 new_codec_ = true; | 692 new_codec_ = true; |
705 update_sample_rate_and_channels = true; | 693 update_sample_rate_and_channels = true; |
706 } else if (ret != PacketBuffer::kOK) { | 694 } else if (ret != PacketBuffer::kOK) { |
707 PacketBuffer::DeleteAllPackets(&parsed_packet_list); | 695 PacketBuffer::DeleteAllPackets(&parsed_packet_list); |
708 return kOtherError; | 696 return kOtherError; |
709 } | 697 } |
(...skipping 1224 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1934 } | 1922 } |
1935 prev_sequence_number = packet->header.sequenceNumber; | 1923 prev_sequence_number = packet->header.sequenceNumber; |
1936 prev_timestamp = packet->header.timestamp; | 1924 prev_timestamp = packet->header.timestamp; |
1937 prev_payload_type = packet->header.payloadType; | 1925 prev_payload_type = packet->header.payloadType; |
1938 } | 1926 } |
1939 | 1927 |
1940 // Store number of extracted samples. | 1928 // Store number of extracted samples. |
1941 size_t packet_duration = 0; | 1929 size_t packet_duration = 0; |
1942 if (packet->frame) { | 1930 if (packet->frame) { |
1943 packet_duration = packet->frame->Duration(); | 1931 packet_duration = packet->frame->Duration(); |
1944 // TODO(ossu): Is this the correct way to track samples decoded from a | 1932 // TODO(ossu): Is this the correct way to track Opus FEC packets? |
1945 // redundant packet? | 1933 if (packet->priority.codec_level > 0) { |
1946 if (packet_duration > 0 && !packet->primary) { | |
1947 stats_.SecondaryDecodedSamples(rtc::checked_cast<int>(packet_duration)); | 1934 stats_.SecondaryDecodedSamples(rtc::checked_cast<int>(packet_duration)); |
1948 } | 1935 } |
1949 } else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) { | 1936 } else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) { |
1950 LOG(LS_WARNING) << "Unknown payload type " | 1937 LOG(LS_WARNING) << "Unknown payload type " |
1951 << static_cast<int>(packet->header.payloadType); | 1938 << static_cast<int>(packet->header.payloadType); |
1952 RTC_NOTREACHED(); | 1939 RTC_NOTREACHED(); |
1953 } | 1940 } |
1954 | 1941 |
1955 if (packet_duration == 0) { | 1942 if (packet_duration == 0) { |
1956 // Decoder did not return a packet duration. Assume that the packet | 1943 // Decoder did not return a packet duration. Assume that the packet |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2084 } | 2071 } |
2085 } | 2072 } |
2086 | 2073 |
2087 void NetEqImpl::CreateDecisionLogic() { | 2074 void NetEqImpl::CreateDecisionLogic() { |
2088 decision_logic_.reset(DecisionLogic::Create( | 2075 decision_logic_.reset(DecisionLogic::Create( |
2089 fs_hz_, output_size_samples_, playout_mode_, decoder_database_.get(), | 2076 fs_hz_, output_size_samples_, playout_mode_, decoder_database_.get(), |
2090 *packet_buffer_.get(), delay_manager_.get(), buffer_level_filter_.get(), | 2077 *packet_buffer_.get(), delay_manager_.get(), buffer_level_filter_.get(), |
2091 tick_timer_.get())); | 2078 tick_timer_.get())); |
2092 } | 2079 } |
2093 } // namespace webrtc | 2080 } // namespace webrtc |
OLD | NEW |