Add NetEq delay plotting to event_log_visualizer

webrtc/tools/event_log_visualizer/analyzer.cc

 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/tools/event_log_visualizer/analyzer.h"
 
 #include <algorithm>
 #include <limits>
 #include <map>
 #include <sstream>
 #include <string>
 #include <utility>
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/format_macros.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/rate_statistics.h"
 #include "webrtc/call/audio_receive_stream.h"
 #include "webrtc/call/audio_send_stream.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_types.h"
+#include "webrtc/modules/audio_coding/neteq/tools/audio_sink.h"
+#include "webrtc/modules/audio_coding/neteq/tools/fake_decode_from_file.h"
+#include "webrtc/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h"
+#include "webrtc/modules/audio_coding/neteq/tools/neteq_replacement_input.h"
+#include "webrtc/modules/audio_coding/neteq/tools/neteq_test.h"
+#include "webrtc/modules/audio_coding/neteq/tools/resample_input_audio_file.h"
 #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
 #include "webrtc/modules/include/module_common_types.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/modules/rtp_rtcp/source/rtcp_packet/common_header.h"
 #include "webrtc/modules/rtp_rtcp/source/rtcp_packet/receiver_report.h"
 #include "webrtc/modules/rtp_rtcp/source/rtcp_packet/sender_report.h"
 #include "webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
@@ skipping 1349 matching lines @@
               static_cast<float>(*ana_event.config.num_channels));
         return rtc::Optional<float>();
       },
       audio_network_adaptation_events_, begin_time_, &time_series);
   plot->AppendTimeSeries(std::move(time_series));
   plot->SetXAxis(0, call_duration_s_, "Time (s)", kLeftMargin, kRightMargin);
   plot->SetSuggestedYAxis(0, 1, "Number of channels (1 (mono)/2 (stereo))",
                           kBottomMargin, kTopMargin);
   plot->SetTitle("Reported audio encoder number of channels");
 }
+
+class NetEqStreamInput : public test::NetEqInput {
+ public:
+  // Does not take any ownership, and all pointers must refer to valid objects
+  // that outlive the one constructed.
+  NetEqStreamInput(const std::vector<LoggedRtpPacket>* packet_stream,

[ivoc, 2017/05/16 13:25:50]

It looks like this class is not actually modifying the contents of packet_stream
or output_events_us, so wouldn't a const reference be nicer here?

[hlundin-webrtc, 2017/05/30 14:56:07]

This is a recommended pattern. It avoids the potential problem of dangling
references in the class.

[ivoc, 2017/05/30 16:29:45]

Acknowledged.

+                   const std::vector<uint64_t>* output_events_us)
+      : packet_stream_(*packet_stream),
+        output_events_us_(*output_events_us),
+        packet_stream_it_(packet_stream_.begin()),
+        output_events_us_it_(output_events_us_.begin()) {
+    RTC_DCHECK(packet_stream);
+    RTC_DCHECK(output_events_us);
+  }
+
+  rtc::Optional<int64_t> NextPacketTime() const override {
+    if (packet_stream_it_ == packet_stream_.end()) {
+      return rtc::Optional<int64_t>();
+    }
+    // Convert from us to ms.
+    return rtc::Optional<int64_t>(packet_stream_it_->timestamp / 1000);
+  }
+
+  rtc::Optional<int64_t> NextOutputEventTime() const override {
+    if (output_events_us_it_ == output_events_us_.end()) {
+      return rtc::Optional<int64_t>();
+    }
+    // Convert from us to ms.
+    return rtc::Optional<int64_t>(
+        rtc::checked_cast<int64_t>(*output_events_us_it_ / 1000));
+  }
+
+  std::unique_ptr<PacketData> PopPacket() override {
+    if (packet_stream_it_ == packet_stream_.end()) {
+      return std::unique_ptr<PacketData>();
+    }
+    std::unique_ptr<PacketData> packet_data(new PacketData());
+    packet_data->header = packet_stream_it_->header;
+    // Convert from us to ms.
+    packet_data->time_ms = packet_stream_it_->timestamp / 1000.0;
+
+    // This is a header-only "dummy" packet. Set the payload to all zeros, with
+    // length according to the virtual length.
+    packet_data->payload.SetSize(packet_stream_it_->total_length);
+    std::fill_n(packet_data->payload.data(), packet_data->payload.size(), 0);
+
+    ++packet_stream_it_;
+    return packet_data;
+  }
+
+  void AdvanceOutputEvent() override {
+    if (output_events_us_it_ != output_events_us_.end()) {
+      ++output_events_us_it_;
+    }
+  }
+
+  bool ended() const override {
+    return packet_stream_it_ == packet_stream_.end() ||
+           output_events_us_it_ == output_events_us_.end();
+  }
+
+  rtc::Optional<RTPHeader> NextHeader() const override {
+    if (packet_stream_it_ == packet_stream_.end()) {
+      return rtc::Optional<RTPHeader>();
+    }
+    return rtc::Optional<RTPHeader>(packet_stream_it_->header);
+  }
+
+ private:
+  const std::vector<LoggedRtpPacket>& packet_stream_;
+  const std::vector<uint64_t>& output_events_us_;

[ivoc, 2017/05/16 13:25:50]

Since all that is used is the end iterator (for both packet_stream_ and
output_events_us_), I think it makes sense to just store that instead of a
reference to the vector.

[hlundin-webrtc, 2017/05/30 14:56:07]

Done.

+  std::vector<LoggedRtpPacket>::const_iterator packet_stream_it_;
+  std::vector<uint64_t>::const_iterator output_events_us_it_;
+};
+
+// Plots the jitter buffer delay profile. This will plot only for the first
+// incoming audio SSRC. If the stream contains more than one incoming audio
+// SSRC, all but the first will be ignored.
+void EventLogAnalyzer::CreateAudioJitterBufferGraph(

[ivoc, 2017/05/16 13:25:50]

This function is a bit long, so I think it would be nice to extract one or
several parts into smaller, anonymous functions. Here are some ideas:
- Filling the vector of audio output events
- Setting up the NetEqTest
- Plotting the results

[hlundin-webrtc, 2017/05/30 14:56:07]

I broke out two parts but left the actual graph population in this function.
WDYT?

[ivoc, 2017/05/30 16:29:45]

Looks good! Nice work.

+    const std::string& replacement_file_name,
+    int file_sample_rate_hz,
+    Plot* plot) {
+  const auto& incoming_audio_kv = std::find_if(
+      rtp_packets_.begin(), rtp_packets_.end(),
+      [this](std::pair<StreamId, std::vector<LoggedRtpPacket>> kv) {
+        return kv.first.GetDirection() == kIncomingPacket &&
+               this->IsAudioSsrc(kv.first);
+      });
+  if (incoming_audio_kv == rtp_packets_.end()) {
+    // No incoming audio stream found.
+    return;
+  }
+  const StreamId stream_id = incoming_audio_kv->first;
+  const std::vector<LoggedRtpPacket>& packet_stream = incoming_audio_kv->second;
+
+  // Create a vector of all audio output events.
+  std::vector<uint64_t> output_events_us;
+  rtc::Optional<uint32_t> ssrc;
+  for (size_t i = 0; i < parsed_log_.GetNumberOfEvents(); ++i) {
+    if (parsed_log_.GetEventType(i) == ParsedRtcEventLog::AUDIO_PLAYOUT_EVENT) {
+      uint32_t this_ssrc;
+      parsed_log_.GetAudioPlayout(i, &this_ssrc);
+      if (!ssrc || *ssrc == 0) {

[ivoc, 2017/05/16 13:25:50]

Isn't it better to check that this_ssrc == stream_id.GetSsrc() ?

[hlundin-webrtc, 2017/05/30 14:56:07]

Won't we miss out on the events with SSRC == 0 then? I presume they are the ones
that happen before any incoming packets.

[ivoc, 2017/05/30 16:29:45]

Good point.

+        ssrc = rtc::Optional<uint32_t>(this_ssrc);
+      } else {
+        RTC_DCHECK_EQ(this_ssrc, *ssrc)
+            << "Audio output events from multiple SSRCs";
+      }
+      output_events_us.push_back(parsed_log_.GetTimestamp(i));
+    } else if (parsed_log_.GetEventType(i) == ParsedRtcEventLog::LOG_END) {
+      // End of first part of the log. The logging might restart after this, but
+      // it makes the plot hard to interpret if we include subsequent parts.
+      break;
+    }
+  }
+
+  // Create the NetEqInput object based on the packet and audio output events.
+  std::unique_ptr<test::NetEqInput> input(
+      new NetEqStreamInput(&packet_stream, &output_events_us));
+
+  constexpr int kReplacementPt = 127;
+  std::set<uint8_t> cn_types;
+  std::set<uint8_t> forbidden_types;
+  input.reset(new test::NetEqReplacementInput(std::move(input), kReplacementPt,
+                                              cn_types, forbidden_types));
+
+  NetEq::Config config;
+  config.max_packets_in_buffer = 200;
+  config.enable_fast_accelerate = true;
+
+  std::unique_ptr<test::VoidAudioSink> output(new test::VoidAudioSink());
+
+  test::NetEqTest::DecoderMap codecs;
+
+  // Create a "replacement decoder" that produces the decoded audio by reading
+  // from a file rather than from the encoded payloads.
+  std::unique_ptr<test::ResampleInputAudioFile> replacement_file(
+      new test::ResampleInputAudioFile(replacement_file_name,
+                                       file_sample_rate_hz));
+  replacement_file->set_output_rate_hz(48000);
+  std::unique_ptr<AudioDecoder> replacement_decoder(
+      new test::FakeDecodeFromFile(std::move(replacement_file), 48000, false));
+  test::NetEqTest::ExtDecoderMap ext_codecs;
+  ext_codecs[kReplacementPt] = {replacement_decoder.get(),
+                                NetEqDecoder::kDecoderArbitrary,
+                                "replacement codec"};
+
+  test::DefaultNetEqTestErrorCallback error_cb;
+  test::NetEqDelayAnalyzer delay_cb;
+  test::NetEqTest::Callbacks callbacks;
+  callbacks.error_callback = &error_cb;
+  callbacks.post_insert_packet = &delay_cb;
+  callbacks.get_audio_callback = &delay_cb;
+
+  test::NetEqTest test(config, codecs, ext_codecs, std::move(input),
+                       std::move(output), callbacks);
+  test.Run();
+
+  std::vector<float> send_times_s;
+  std::vector<float> arrival_delay_ms;
+  std::vector<float> corrected_arrival_delay_ms;
+  std::vector<rtc::Optional<float>> playout_delay_ms;
+  std::vector<rtc::Optional<float>> target_delay_ms;
+  delay_cb.CreateGraphs(&send_times_s, &arrival_delay_ms,
+                        &corrected_arrival_delay_ms, &playout_delay_ms,
+                        &target_delay_ms);
+  RTC_DCHECK_EQ(send_times_s.size(), arrival_delay_ms.size());
+  RTC_DCHECK_EQ(send_times_s.size(), corrected_arrival_delay_ms.size());
+  RTC_DCHECK_EQ(send_times_s.size(), playout_delay_ms.size());
+  RTC_DCHECK_EQ(send_times_s.size(), target_delay_ms.size());
+
+  std::map<StreamId, TimeSeries> time_series_packet_arrival;
+  std::map<StreamId, TimeSeries> time_series_relative_packet_arrival;
+  std::map<StreamId, TimeSeries> time_series_play_time;
+  std::map<StreamId, TimeSeries> time_series_target_time;
+  float min_y_axis = 0.f;
+  float max_y_axis = 0.f;
+  for (size_t i = 0; i < send_times_s.size(); ++i) {
+    time_series_packet_arrival[stream_id].points.emplace_back(
+        TimeSeriesPoint(send_times_s[i], arrival_delay_ms[i]));
+    time_series_relative_packet_arrival[stream_id].points.emplace_back(
+        TimeSeriesPoint(send_times_s[i], corrected_arrival_delay_ms[i]));
+    min_y_axis = std::min(min_y_axis, corrected_arrival_delay_ms[i]);
+    max_y_axis = std::max(max_y_axis, corrected_arrival_delay_ms[i]);
+    if (playout_delay_ms[i]) {
+      time_series_play_time[stream_id].points.emplace_back(
+          TimeSeriesPoint(send_times_s[i], *playout_delay_ms[i]));
+      min_y_axis = std::min(min_y_axis, *playout_delay_ms[i]);
+      max_y_axis = std::max(max_y_axis, *playout_delay_ms[i]);
+    }
+    if (target_delay_ms[i]) {
+      time_series_target_time[stream_id].points.emplace_back(
+          TimeSeriesPoint(send_times_s[i], *target_delay_ms[i]));
+      min_y_axis = std::min(min_y_axis, *target_delay_ms[i]);
+      max_y_axis = std::max(max_y_axis, *target_delay_ms[i]);
+    }
+  }
+
+  for (auto& series : time_series_relative_packet_arrival) {
+    series.second.label = "Relative packet arrival delay";
+    series.second.style = LINE_GRAPH;
+    plot->AppendTimeSeries(std::move(series.second));
+  }
+  for (auto& series : time_series_play_time) {
+    series.second.label = "Playout delay";
+    series.second.style = LINE_GRAPH;
+    plot->AppendTimeSeries(std::move(series.second));
+  }
+  for (auto& series : time_series_target_time) {
+    series.second.label = "Target delay";
+    series.second.style = LINE_DOT_GRAPH;
+    plot->AppendTimeSeries(std::move(series.second));
+  }
+
+  plot->SetXAxis(0, call_duration_s_, "Time (s)", kLeftMargin, kRightMargin);
+  plot->SetYAxis(min_y_axis, max_y_axis, "Relative delay (ms)", kBottomMargin,
+                 kTopMargin);
+  plot->SetTitle("NetEq timing");
+}
 }  // namespace plotting
 }  // namespace webrtc