Moved Opus-specific payload splitting into AudioDecoderOpus.

webrtc/modules/audio_coding/neteq/neteq_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 23 matching lines @@
 #include "webrtc/modules/audio_coding/neteq/delay_manager.h"
 #include "webrtc/modules/audio_coding/neteq/delay_peak_detector.h"
 #include "webrtc/modules/audio_coding/neteq/dtmf_buffer.h"
 #include "webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h"
 #include "webrtc/modules/audio_coding/neteq/expand.h"
 #include "webrtc/modules/audio_coding/neteq/merge.h"
 #include "webrtc/modules/audio_coding/neteq/nack_tracker.h"
 #include "webrtc/modules/audio_coding/neteq/normal.h"
 #include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
 #include "webrtc/modules/audio_coding/neteq/packet.h"
-#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"
+#include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
 #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
 #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
 #include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
 #include "webrtc/modules/audio_coding/neteq/tick_timer.h"
 #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
 #include "webrtc/modules/include/module_common_types.h"
 
 namespace webrtc {
 
 NetEqImpl::Dependencies::Dependencies(
     const NetEq::Config& config,
     const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory)
     : tick_timer(new TickTimer),
       buffer_level_filter(new BufferLevelFilter),
       decoder_database(new DecoderDatabase(decoder_factory)),
       delay_peak_detector(new DelayPeakDetector(tick_timer.get())),
       delay_manager(new DelayManager(config.max_packets_in_buffer,
                                      delay_peak_detector.get(),
                                      tick_timer.get())),
       dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)),
       dtmf_tone_generator(new DtmfToneGenerator),
       packet_buffer(
           new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())),
-      payload_splitter(new PayloadSplitter),
+      red_payload_splitter(new RedPayloadSplitter),
       timestamp_scaler(new TimestampScaler(*decoder_database)),
       accelerate_factory(new AccelerateFactory),
       expand_factory(new ExpandFactory),
       preemptive_expand_factory(new PreemptiveExpandFactory) {}
 
 NetEqImpl::Dependencies::~Dependencies() = default;
 
 NetEqImpl::NetEqImpl(const NetEq::Config& config,
                      Dependencies&& deps,
                      bool create_components)
     : tick_timer_(std::move(deps.tick_timer)),
       buffer_level_filter_(std::move(deps.buffer_level_filter)),
       decoder_database_(std::move(deps.decoder_database)),
       delay_manager_(std::move(deps.delay_manager)),
       delay_peak_detector_(std::move(deps.delay_peak_detector)),
       dtmf_buffer_(std::move(deps.dtmf_buffer)),
       dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)),
       packet_buffer_(std::move(deps.packet_buffer)),
-      payload_splitter_(std::move(deps.payload_splitter)),
+      red_payload_splitter_(std::move(deps.red_payload_splitter)),
       timestamp_scaler_(std::move(deps.timestamp_scaler)),
       vad_(new PostDecodeVad()),
       expand_factory_(std::move(deps.expand_factory)),
       accelerate_factory_(std::move(deps.accelerate_factory)),
       preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)),
       last_mode_(kModeNormal),
       decoded_buffer_length_(kMaxFrameSize),
       decoded_buffer_(new int16_t[decoded_buffer_length_]),
       playout_timestamp_(0),
       new_codec_(false),
@@ skipping 424 matching lines @@
     // directly once it's been inserted in the packet list. This way, |packet|
     // is not defined outside of this block.
     Packet* packet = new Packet;
     packet->header.markerBit = false;
     packet->header.payloadType = rtp_header.header.payloadType;
     packet->header.sequenceNumber = rtp_header.header.sequenceNumber;
     packet->header.timestamp = rtp_header.header.timestamp;
     packet->header.ssrc = rtp_header.header.ssrc;
     packet->header.numCSRCs = 0;
     packet->payload.SetData(payload.data(), payload.size());
-    packet->primary = true;
     // Waiting time will be set upon inserting the packet in the buffer.
     RTC_DCHECK(!packet->waiting_time);
     // Insert packet in a packet list.
     packet_list.push_back(packet);
     // Save main payloads header for later.
     memcpy(&main_header, &packet->header, sizeof(main_header));
   }
 
   bool update_sample_rate_and_channels = false;
   // Reinitialize NetEq if it's needed (changed SSRC or first call).
@@ skipping 21 matching lines @@
 
     // Trigger an update of sampling rate and the number of channels.
     update_sample_rate_and_channels = true;
   }
 
   // Update RTCP statistics, only for regular packets.
   rtcp_.Update(main_header, receive_timestamp);
 
   // Check for RED payload type, and separate payloads into several packets.
   if (decoder_database_->IsRed(main_header.payloadType)) {
-    if (payload_splitter_->SplitRed(&packet_list) != PayloadSplitter::kOK) {
+    if (!red_payload_splitter_->SplitRed(&packet_list)) {
       PacketBuffer::DeleteAllPackets(&packet_list);
       return kRedundancySplitError;
     }
     // Only accept a few RED payloads of the same type as the main data,
     // DTMF events and CNG.
-    payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
+    red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
     // Update the stored main payload header since the main payload has now
     // changed.
     memcpy(&main_header, &packet_list.front()->header, sizeof(main_header));
   }
 
   // Check payload types.
   if (decoder_database_->CheckPayloadTypes(packet_list) ==
       DecoderDatabase::kDecoderNotFound) {
     PacketBuffer::DeleteAllPackets(&packet_list);
     return kUnknownRtpPayloadType;
@@ skipping 22 matching lines @@
         PacketBuffer::DeleteAllPackets(&packet_list);
         return kDtmfInsertError;
       }
       delete current_packet;
       it = packet_list.erase(it);
     } else {
       ++it;
     }
   }
 
-  // Check for FEC in packets, and separate payloads into several packets.
-  int ret = payload_splitter_->SplitFec(&packet_list, decoder_database_.get());
-  if (ret != PayloadSplitter::kOK) {
-    PacketBuffer::DeleteAllPackets(&packet_list);
-    switch (ret) {
-      case PayloadSplitter::kUnknownPayloadType:
-        return kUnknownRtpPayloadType;
-      default:
-        return kOtherError;
-    }
-  }
-
   // Update bandwidth estimate, if the packet is not comfort noise.
   if (!packet_list.empty() &&
       !decoder_database_->IsComfortNoise(main_header.payloadType)) {
     // The list can be empty here if we got nothing but DTMF payloads.
     AudioDecoder* decoder =
         decoder_database_->GetDecoder(main_header.payloadType);
     assert(decoder);  // Should always get a valid object, since we have
     // already checked that the payload types are known.
     decoder->IncomingPacket(packet_list.front()->payload.data(),
                             packet_list.front()->payload.size(),
@@ skipping 12 matching lines @@
       LOG(LS_WARNING) << "SplitAudio unknown payload type";
       return kUnknownRtpPayloadType;
     }
 
     if (info->IsComfortNoise()) {
       // Carry comfort noise packets along.
       parsed_packet_list.push_back(packet.release());
     } else {
       std::vector<AudioDecoder::ParseResult> results =
           info->GetDecoder()->ParsePayload(std::move(packet->payload),
-                                           packet->header.timestamp,
-                                           packet->primary);
+                                           packet->header.timestamp);
       const RTPHeader& original_header = packet->header;
+      const Packet::Priority original_priority = packet->priority;
       for (auto& result : results) {
         RTC_DCHECK(result.frame);
         // Reuse the packet if possible.
         if (!packet) {
           packet.reset(new Packet);
           packet->header = original_header;
         }
         packet->header.timestamp = result.timestamp;
-        // TODO(ossu): Move from primary to some sort of priority level.
-        packet->primary = result.primary;
+        RTC_DCHECK_GE(result.priority, 0);
+        packet->priority.codec_level = result.priority;
+        packet->priority.red_level = original_priority.red_level;
         packet->frame = std::move(result.frame);
         parsed_packet_list.push_back(packet.release());
       }
     }
   }
 
   if (nack_enabled_) {
     RTC_DCHECK(nack_);
     if (update_sample_rate_and_channels) {
       nack_->Reset();
     }
     nack_->UpdateLastReceivedPacket(
         parsed_packet_list.front()->header.sequenceNumber,
         parsed_packet_list.front()->header.timestamp);
   }
 
   // Insert packets in buffer.
   const size_t buffer_length_before_insert =
       packet_buffer_->NumPacketsInBuffer();
-  ret = packet_buffer_->InsertPacketList(
+  const int ret = packet_buffer_->InsertPacketList(
       &parsed_packet_list, *decoder_database_, &current_rtp_payload_type_,
       &current_cng_rtp_payload_type_);
   if (ret == PacketBuffer::kFlushed) {
     // Reset DSP timestamp etc. if packet buffer flushed.
     new_codec_ = true;
     update_sample_rate_and_channels = true;
   } else if (ret != PacketBuffer::kOK) {
     PacketBuffer::DeleteAllPackets(&parsed_packet_list);
     return kOtherError;
   }
@@ skipping 1224 matching lines @@
       }
       prev_sequence_number = packet->header.sequenceNumber;
       prev_timestamp = packet->header.timestamp;
       prev_payload_type = packet->header.payloadType;
     }
 
     // Store number of extracted samples.
     size_t packet_duration = 0;
     if (packet->frame) {
       packet_duration = packet->frame->Duration();
-      // TODO(ossu): Is this the correct way to track samples decoded from a
-      // redundant packet?
-      if (packet_duration > 0 && !packet->primary) {
+      // TODO(ossu): Is this the correct way to track Opus FEC packets?
+      if (packet->priority.codec_level > 0) {
         stats_.SecondaryDecodedSamples(rtc::checked_cast<int>(packet_duration));
       }
     } else if (!decoder_database_->IsComfortNoise(packet->header.payloadType)) {
       LOG(LS_WARNING) << "Unknown payload type "
                       << static_cast<int>(packet->header.payloadType);
       RTC_NOTREACHED();
     }
 
     if (packet_duration == 0) {
       // Decoder did not return a packet duration. Assume that the packet
@@ skipping 127 matching lines @@
   }
 }
 
 void NetEqImpl::CreateDecisionLogic() {
   decision_logic_.reset(DecisionLogic::Create(
       fs_hz_, output_size_samples_, playout_mode_, decoder_database_.get(),
       *packet_buffer_.get(), delay_manager_.get(), buffer_level_filter_.get(),
       tick_timer_.get()));
 }
 }  // namespace webrtc