webrtc/video/full_stack.cc - Issue 1289933003: Full stack graphs

Unified Diff: webrtc/video/full_stack.cc

Issue 1289933003: Full stack graphs (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master

Patch Set: Removing unused define + rebase master Created 5 years, 4 months ago

Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.

Jump to:

View side-by-side diff with in-line comments

Index: webrtc/video/full_stack.cc

diff --git a/webrtc/video/full_stack.cc b/webrtc/video/full_stack.cc

index 1220482a1dbf597f44c9fac31bf997852a06c735..e6ab1a61997caed4ae6d42673b7c440bf5dacc4d 100644

--- a/webrtc/video/full_stack.cc

+++ b/webrtc/video/full_stack.cc

@@ -25,8 +25,6 @@

#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"

#include "webrtc/system_wrappers/interface/event_wrapper.h"

#include "webrtc/system_wrappers/interface/sleep.h"

-#include "webrtc/test/call_test.h"

-#include "webrtc/test/direct_transport.h"

#include "webrtc/test/encoder_settings.h"

#include "webrtc/test/fake_encoder.h"

#include "webrtc/test/frame_generator.h"

@@ -34,35 +32,13 @@

#include "webrtc/test/statistics.h"

#include "webrtc/test/testsupport/fileutils.h"

#include "webrtc/typedefs.h"

+#include "webrtc/video/full_stack.h"

namespace webrtc {

static const int kFullStackTestDurationSecs = 60;

static const int kSendStatsPollingIntervalMs = 1000;

-struct FullStackTestParams {

- const char* test_label;

- struct {

- const char* name;

- size_t width, height;

- int fps;

- } clip;

- bool screenshare;

- int min_bitrate_bps;

- int target_bitrate_bps;

- int max_bitrate_bps;

- double avg_psnr_threshold;

- double avg_ssim_threshold;

- int test_durations_secs;

- std::string codec;

- FakeNetworkPipe::Config link;

-};

-class FullStackTest : public test::CallTest {

- protected:

- void RunTest(const FullStackTestParams& params);

-};

class VideoAnalyzer : public PacketReceiver,

public newapi::Transport,

public VideoRenderer,

@@ -74,12 +50,15 @@ class VideoAnalyzer : public PacketReceiver,

const char* test_label,

double avg_psnr_threshold,

double avg_ssim_threshold,

- int duration_frames)

+ int duration_frames,

+ const std::string& graph_data_filename)

: input_(input),

transport_(transport),

receiver_(nullptr),

send_stream_(nullptr),

test_label_(test_label),

+ save_samples_(!graph_data_filename.empty()),

+ graph_data_filename_(graph_data_filename),

frames_to_process_(duration_frames),

frames_recorded_(0),

frames_processed_(0),

@@ -148,12 +127,14 @@ class VideoAnalyzer : public PacketReceiver,

VideoFrame copy = video_frame;

copy.set_timestamp(copy.ntp_time_ms() * 90);

+ VideoSendStream::Stats stats = send_stream_->GetStats();

{

rtc::CritScope lock(&crit_);

if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)

first_send_frame_ = copy;

frames_.push_back(copy);

+ send_stats_.push_back(stats);

}

input_->IncomingCapturedFrame(video_frame);

@@ -171,8 +152,11 @@ class VideoAnalyzer : public PacketReceiver,

header.timestamp - first_send_frame_.timestamp();

first_send_frame_.Reset();

}

- send_times_[header.timestamp - rtp_timestamp_delta_] =

+ uint32_t timestamp = header.timestamp - rtp_timestamp_delta_;

+ send_times_[timestamp] =

Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();

+ encoded_frame_sizes_[timestamp] +=

+ length - (header.headerLength + header.paddingLength);

sprang_webrtc 2015/08/19 13:33:37 Rebase error? This is handles by EncodedFrameCallb

ivica 2015/08/20 13:25:59 For graphs I need encoded frame size for each fram

}

return transport_->SendRtp(packet, length);

@@ -198,8 +182,9 @@ class VideoAnalyzer : public PacketReceiver,

while (frames_.front().timestamp() < send_timestamp) {

AddFrameComparison(frames_.front(), last_rendered_frame_, true,

- render_time_ms);

+ render_time_ms, send_stats_.front());

frames_.pop_front();

+ send_stats_.pop_front();

}

VideoFrame reference_frame = frames_.front();

@@ -208,7 +193,9 @@ class VideoAnalyzer : public PacketReceiver,

EXPECT_EQ(reference_frame.timestamp(), send_timestamp);

assert(reference_frame.timestamp() == send_timestamp);

- AddFrameComparison(reference_frame, video_frame, false, render_time_ms);

+ AddFrameComparison(reference_frame, video_frame, false, render_time_ms,

+ send_stats_.front());

+ send_stats_.pop_front();

last_rendered_frame_ = video_frame;

}

@@ -253,20 +240,28 @@ class VideoAnalyzer : public PacketReceiver,

private:

struct FrameComparison {

FrameComparison()

- : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}

+ : dropped(false),

+ send_time_ms(0),

+ recv_time_ms(0),

+ render_time_ms(0),

+ encoded_frame_size(0) {}

FrameComparison(const VideoFrame& reference,

const VideoFrame& render,

bool dropped,

int64_t send_time_ms,

int64_t recv_time_ms,

- int64_t render_time_ms)

+ int64_t render_time_ms,

+ size_t encoded_frame_size,

+ const VideoSendStream::Stats& send_stats)

: reference(reference),

render(render),

dropped(dropped),

send_time_ms(send_time_ms),

recv_time_ms(recv_time_ms),

- render_time_ms(render_time_ms) {}

+ render_time_ms(render_time_ms),

+ encoded_frame_size(encoded_frame_size),

+ send_stats(send_stats) {}

VideoFrame reference;

VideoFrame render;

@@ -274,22 +269,73 @@ class VideoAnalyzer : public PacketReceiver,

int64_t send_time_ms;

int64_t recv_time_ms;

int64_t render_time_ms;

+ size_t encoded_frame_size;

+ VideoSendStream::Stats send_stats;

+ };

+ struct Sample {

+ Sample(double dropped,

+ double input_time_ms,

+ double send_time_ms,

+ double recv_time_ms,

+ double avg_encode_time_ms,

+ double encoded_frame_size,

+ double encode_frame_rate,

+ double encode_usage_percent,

+ double media_bitrate_bps,

+ double psnr,

+ double ssim,

+ double decoded_time_ms)

+ : dropped(dropped),

+ input_time_ms(input_time_ms),

+ send_time_ms(send_time_ms),

+ recv_time_ms(recv_time_ms),

+ avg_encode_time_ms(avg_encode_time_ms),

+ encoded_frame_size(encoded_frame_size),

+ encode_frame_rate(encode_frame_rate),

+ encode_usage_percent(encode_usage_percent),

+ media_bitrate_bps(media_bitrate_bps),

+ psnr(psnr),

+ ssim(ssim),

+ decoded_time_ms(decoded_time_ms) {}

+ double dropped;

+ double input_time_ms;

+ double send_time_ms;

+ double recv_time_ms;

+ double avg_encode_time_ms;

+ double encoded_frame_size;

+ double encode_frame_rate;

+ double encode_usage_percent;

+ double media_bitrate_bps;

+ double psnr;

+ double ssim;

+ double decoded_time_ms;

};

void AddFrameComparison(const VideoFrame& reference,

const VideoFrame& render,

bool dropped,

- int64_t render_time_ms)

+ int64_t render_time_ms,

+ const VideoSendStream::Stats& send_stats)

EXCLUSIVE_LOCKS_REQUIRED(crit_) {

int64_t send_time_ms = send_times_[reference.timestamp()];

send_times_.erase(reference.timestamp());

int64_t recv_time_ms = recv_times_[reference.timestamp()];

recv_times_.erase(reference.timestamp());

+ size_t encoded_size = encoded_frame_sizes_[reference.timestamp()];

+ encoded_frame_sizes_.erase(reference.timestamp());

+ VideoFrame reference_copy;

+ VideoFrame render_copy;

+ reference_copy.CopyFrame(reference);

+ render_copy.CopyFrame(render);

rtc::CritScope crit(&comparison_lock_);

- comparisons_.push_back(FrameComparison(reference, render, dropped,

- send_time_ms, recv_time_ms,

- render_time_ms));

+ comparisons_.push_back(FrameComparison(

+ reference_copy, render_copy, dropped, send_time_ms, recv_time_ms,

+ render_time_ms, encoded_size, send_stats));

comparison_available_event_->Set();

}

@@ -347,6 +393,9 @@ class VideoAnalyzer : public PacketReceiver,

if (FrameProcessed()) {

PrintResults();

+ if (save_samples_) {

+ PrintSamplesToFile();

+ }

sprang_webrtc 2015/08/19 13:33:37 Remove braces

ivica 2015/08/20 13:25:59 Done.

done_->Set();

comparison_available_event_->Set();

return false;

@@ -419,9 +468,22 @@ class VideoAnalyzer : public PacketReceiver,

double psnr = I420PSNR(&comparison.reference, &comparison.render);

double ssim = I420SSIM(&comparison.reference, &comparison.render);

+ int64_t input_time_ms = comparison.reference.ntp_time_ms();

+ Sample sample(

+ comparison.dropped, input_time_ms, comparison.send_time_ms,

+ comparison.recv_time_ms, comparison.send_stats.avg_encode_time_ms,

+ comparison.encoded_frame_size, comparison.send_stats.encode_frame_rate,

+ comparison.send_stats.encode_usage_percent,

+ comparison.send_stats.media_bitrate_bps, psnr, ssim,

+ comparison.render_time_ms); // decoded_time_ms

sprang_webrtc 2015/08/19 13:33:37 I don't think comparison.send_stats is available h

ivica 2015/08/20 13:25:59 Removed. Can be added later if necessary.

rtc::CritScope crit(&comparison_lock_);

+ if (save_samples_) {

+ samples_.push_back(sample);

+ }

psnr_.AddSample(psnr);

ssim_.AddSample(ssim);

if (comparison.dropped) {

++dropped_frames_;

return;

@@ -430,11 +492,11 @@ class VideoAnalyzer : public PacketReceiver,

rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);

last_render_time_ = comparison.render_time_ms;

- int64_t input_time_ms = comparison.reference.ntp_time_ms();

sender_time_.AddSample(comparison.send_time_ms - input_time_ms);

receiver_time_.AddSample(comparison.render_time_ms -

comparison.recv_time_ms);

end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);

+ encoded_frame_size_.AddSample(comparison.encoded_frame_size);

}

void PrintResult(const char* result_type,

@@ -448,7 +510,52 @@ class VideoAnalyzer : public PacketReceiver,

unit);

}

+ void PrintSamplesToFile(void) {

+ FILE* f = fopen(graph_data_filename_.c_str(), "w");

+ if (f == NULL) {

+ fprintf(stderr, "Couldn't open file %s\n", graph_data_filename_.c_str());

+ abort();

+ }

+ rtc::CritScope crit(&comparison_lock_);

+ std::sort(samples_.begin(), samples_.end(),

+ [](const Sample& A, const Sample& B)

+ -> bool { return A.input_time_ms < B.input_time_ms; });

+ fprintf(f, "%s\n", test_label_);

+ fprintf(f, "%d\n", (int)samples_.size());

sprang_webrtc 2015/08/19 13:33:37 Use static_cast rather than c-style cast. Or just

ivica 2015/08/20 13:25:59 Using %" PRIuS " as described in style guide...

+#define PRINT_SAMPLES(field) \

+ { \

+ fprintf(f, "%s ", #field); \

+ for (Sample & value : samples_) { \

+ fprintf(f, "%lf ", value.field); \

+ } \

+ fprintf(f, "\n"); \

+ }

sprang_webrtc 2015/08/19 13:33:37 Macros are unfortunately only allowed in some very

ivica 2015/08/20 13:25:59 Changed from one-field-one-row to one-field-one-co

+ PRINT_SAMPLES(dropped);

+ PRINT_SAMPLES(input_time_ms);

+ PRINT_SAMPLES(send_time_ms);

+ PRINT_SAMPLES(recv_time_ms);

+ PRINT_SAMPLES(avg_encode_time_ms);

+ PRINT_SAMPLES(encoded_frame_size);

+ PRINT_SAMPLES(encode_frame_rate);

+ PRINT_SAMPLES(encode_usage_percent);

+ PRINT_SAMPLES(media_bitrate_bps);

+ PRINT_SAMPLES(psnr);

+ PRINT_SAMPLES(ssim);

+ PRINT_SAMPLES(decoded_time_ms);

+#undef PRINT_SAMPLES

+ // Add an end specifier, to be able to add comments to the end of file if

+ // necessary.

+ fprintf(f, "--END--\n");

+ fclose(f);

+ }

const char* const test_label_;

+ bool save_samples_;

sprang_webrtc 2015/08/19 13:33:37 Not sure this is necessary. Calling graph_data_fil

ivica 2015/08/20 13:25:59 Removed.

+ std::string graph_data_filename_;

+ std::vector<Sample> samples_ GUARDED_BY(comparison_lock_);

test::Statistics sender_time_ GUARDED_BY(comparison_lock_);

test::Statistics receiver_time_ GUARDED_BY(comparison_lock_);

test::Statistics psnr_ GUARDED_BY(comparison_lock_);

@@ -470,9 +577,11 @@ class VideoAnalyzer : public PacketReceiver,

rtc::CriticalSection crit_;

std::deque<VideoFrame> frames_ GUARDED_BY(crit_);

+ std::deque<VideoSendStream::Stats> send_stats_ GUARDED_BY(crit_);

VideoFrame last_rendered_frame_ GUARDED_BY(crit_);

std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);

std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);

+ std::map<uint32_t, size_t> encoded_frame_sizes_ GUARDED_BY(crit_);

VideoFrame first_send_frame_ GUARDED_BY(crit_);

const double avg_psnr_threshold_;

const double avg_ssim_threshold_;

@@ -486,11 +595,16 @@ class VideoAnalyzer : public PacketReceiver,

};

void FullStackTest::RunTest(const FullStackTestParams& params) {

+ // TODO(ivica): Add num_temporal_layers as a param.

+ unsigned char num_temporal_layers =

+ params.graph_data_filename.empty() ? 2 : 1;

test::DirectTransport send_transport(params.link);

test::DirectTransport recv_transport(params.link);

VideoAnalyzer analyzer(nullptr, &send_transport, params.test_label,

params.avg_psnr_threshold, params.avg_ssim_threshold,

- params.test_durations_secs * params.clip.fps);

+ params.test_durations_secs * params.clip.fps,

+ params.graph_data_filename);

CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));

@@ -538,18 +652,21 @@ void FullStackTest::RunTest(const FullStackTestParams& params) {

vp8_settings = VideoEncoder::GetDefaultVp8Settings();

vp8_settings.denoisingOn = false;

vp8_settings.frameDroppingOn = false;

- vp8_settings.numberOfTemporalLayers = 2;

+ vp8_settings.numberOfTemporalLayers = num_temporal_layers;

encoder_config_.encoder_specific_settings = &vp8_settings;

} else if (params.codec == "VP9") {

vp9_settings = VideoEncoder::GetDefaultVp9Settings();

vp9_settings.denoisingOn = false;

vp9_settings.frameDroppingOn = false;

- vp9_settings.numberOfTemporalLayers = 2;

+ vp9_settings.numberOfTemporalLayers = num_temporal_layers;

encoder_config_.encoder_specific_settings = &vp9_settings;

}

stream->temporal_layer_thresholds_bps.clear();

- stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);

+ if (num_temporal_layers > 1) {

+ stream->temporal_layer_thresholds_bps.push_back(

+ stream->target_bitrate_bps);

+ }

}

CreateMatchingReceiveConfigs();

« webrtc/video/full_stack.h ('K') | « webrtc/video/full_stack.h ('k') | webrtc/video/full_stack_plot.py » ('j') | no next file with comments »