OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 #include "webrtc/video/video_quality_test.h" | 10 #include "webrtc/video/video_quality_test.h" |
(...skipping 27 matching lines...) Expand all Loading... | |
38 #include "webrtc/rtc_base/optional.h" | 38 #include "webrtc/rtc_base/optional.h" |
39 #include "webrtc/rtc_base/platform_file.h" | 39 #include "webrtc/rtc_base/platform_file.h" |
40 #include "webrtc/rtc_base/timeutils.h" | 40 #include "webrtc/rtc_base/timeutils.h" |
41 #include "webrtc/system_wrappers/include/cpu_info.h" | 41 #include "webrtc/system_wrappers/include/cpu_info.h" |
42 #include "webrtc/system_wrappers/include/field_trial.h" | 42 #include "webrtc/system_wrappers/include/field_trial.h" |
43 #include "webrtc/test/gtest.h" | 43 #include "webrtc/test/gtest.h" |
44 #include "webrtc/test/layer_filtering_transport.h" | 44 #include "webrtc/test/layer_filtering_transport.h" |
45 #include "webrtc/test/run_loop.h" | 45 #include "webrtc/test/run_loop.h" |
46 #include "webrtc/test/statistics.h" | 46 #include "webrtc/test/statistics.h" |
47 #include "webrtc/test/testsupport/fileutils.h" | 47 #include "webrtc/test/testsupport/fileutils.h" |
48 #include "webrtc/test/testsupport/frame_writer.h" | |
48 #include "webrtc/test/vcm_capturer.h" | 49 #include "webrtc/test/vcm_capturer.h" |
49 #include "webrtc/test/video_renderer.h" | 50 #include "webrtc/test/video_renderer.h" |
50 #include "webrtc/voice_engine/include/voe_base.h" | 51 #include "webrtc/voice_engine/include/voe_base.h" |
51 | 52 |
52 #include "webrtc/test/rtp_file_writer.h" | 53 #include "webrtc/test/rtp_file_writer.h" |
53 | 54 |
54 namespace { | 55 namespace { |
55 | 56 |
56 constexpr int kSendStatsPollingIntervalMs = 1000; | 57 constexpr int kSendStatsPollingIntervalMs = 1000; |
57 | 58 |
(...skipping 738 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
796 PrintResult("sender_time", sender_time_, " ms"); | 797 PrintResult("sender_time", sender_time_, " ms"); |
797 PrintResult("receiver_time", receiver_time_, " ms"); | 798 PrintResult("receiver_time", receiver_time_, " ms"); |
798 PrintResult("total_delay_incl_network", end_to_end_, " ms"); | 799 PrintResult("total_delay_incl_network", end_to_end_, " ms"); |
799 PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); | 800 PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); |
800 PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); | 801 PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); |
801 PrintResult("encode_time", encode_time_ms_, " ms"); | 802 PrintResult("encode_time", encode_time_ms_, " ms"); |
802 PrintResult("media_bitrate", media_bitrate_bps_, " bps"); | 803 PrintResult("media_bitrate", media_bitrate_bps_, " bps"); |
803 PrintResult("fec_bitrate", fec_bitrate_bps_, " bps"); | 804 PrintResult("fec_bitrate", fec_bitrate_bps_, " bps"); |
804 PrintResult("send_bandwidth", send_bandwidth_bps_, " bps"); | 805 PrintResult("send_bandwidth", send_bandwidth_bps_, " bps"); |
805 | 806 |
807 if (worst_frame_) { | |
808 printf("RESULT min_psnr: %s = %lf dB\n", test_label_.c_str(), | |
809 worst_frame_->psnr); | |
810 } | |
811 | |
806 if (receive_stream_ != nullptr) { | 812 if (receive_stream_ != nullptr) { |
807 PrintResult("decode_time", decode_time_ms_, " ms"); | 813 PrintResult("decode_time", decode_time_ms_, " ms"); |
808 } | 814 } |
809 | 815 |
810 printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(), | 816 printf("RESULT dropped_frames: %s = %d frames\n", test_label_.c_str(), |
811 dropped_frames_); | 817 dropped_frames_); |
812 printf("RESULT cpu_usage: %s = %lf %%\n", test_label_.c_str(), | 818 printf("RESULT cpu_usage: %s = %lf %%\n", test_label_.c_str(), |
813 GetCpuUsagePercent()); | 819 GetCpuUsagePercent()); |
814 | 820 |
815 #if defined(WEBRTC_WIN) | 821 #if defined(WEBRTC_WIN) |
816 // On Linux and Mac in Resident Set some unused pages may be counted. | 822 // On Linux and Mac in Resident Set some unused pages may be counted. |
817 // Therefore this metric will depend on order in which tests are run and | 823 // Therefore this metric will depend on order in which tests are run and |
818 // will be flaky. | 824 // will be flaky. |
819 PrintResult("memory_usage", memory_usage_, " bytes"); | 825 PrintResult("memory_usage", memory_usage_, " bytes"); |
820 #endif | 826 #endif |
827 // TODO(ilnik): enable frame writing for android, once jpeg frame writer | |
828 // is implemented. | |
829 | |
830 #if !defined(WEBRTC_ANDROID) | |
831 if (worst_frame_) { | |
832 test::Y4mFrameWriterImpl frame_writer(test_label_ + ".y4m", | |
stefan-webrtc
2017/08/22 11:32:18
Isn't it better to write this to some directory ot
ilnik
2017/08/22 12:07:42
Yes, this problem is acknowledged. As Erik stated,
| |
833 worst_frame_->frame.width(), | |
834 worst_frame_->frame.height(), 1); | |
835 bool res = frame_writer.Init(); | |
836 RTC_DCHECK(res); | |
837 size_t length = | |
838 CalcBufferSize(VideoType::kI420, worst_frame_->frame.width(), | |
839 worst_frame_->frame.height()); | |
840 rtc::Buffer extracted_buffer(length); | |
841 size_t extracted_length = | |
842 ExtractBuffer(worst_frame_->frame.video_frame_buffer()->ToI420(), | |
843 length, extracted_buffer.data()); | |
844 RTC_DCHECK_EQ(extracted_length, frame_writer.FrameLength()); | |
845 res = frame_writer.WriteFrame(extracted_buffer.data()); | |
846 RTC_DCHECK(res); | |
847 frame_writer.Close(); | |
848 } | |
849 #endif | |
821 | 850 |
822 // Disable quality check for quick test, as quality checks may fail | 851 // Disable quality check for quick test, as quality checks may fail |
823 // because too few samples were collected. | 852 // because too few samples were collected. |
824 if (!is_quick_test_enabled_) { | 853 if (!is_quick_test_enabled_) { |
825 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); | 854 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); |
826 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); | 855 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); |
827 } | 856 } |
828 } | 857 } |
829 | 858 |
830 void PerformFrameComparison(const FrameComparison& comparison) { | 859 void PerformFrameComparison(const FrameComparison& comparison) { |
831 // Perform expensive psnr and ssim calculations while not holding lock. | 860 // Perform expensive psnr and ssim calculations while not holding lock. |
832 double psnr = -1.0; | 861 double psnr = -1.0; |
833 double ssim = -1.0; | 862 double ssim = -1.0; |
834 if (comparison.reference && !comparison.dropped) { | 863 if (comparison.reference && !comparison.dropped) { |
835 psnr = I420PSNR(&*comparison.reference, &*comparison.render); | 864 psnr = I420PSNR(&*comparison.reference, &*comparison.render); |
836 ssim = I420SSIM(&*comparison.reference, &*comparison.render); | 865 ssim = I420SSIM(&*comparison.reference, &*comparison.render); |
837 } | 866 } |
838 | 867 |
839 rtc::CritScope crit(&comparison_lock_); | 868 rtc::CritScope crit(&comparison_lock_); |
869 | |
870 if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) { | |
871 worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render}); | |
872 } | |
873 | |
840 if (graph_data_output_file_) { | 874 if (graph_data_output_file_) { |
841 samples_.push_back(Sample( | 875 samples_.push_back(Sample( |
842 comparison.dropped, comparison.input_time_ms, comparison.send_time_ms, | 876 comparison.dropped, comparison.input_time_ms, comparison.send_time_ms, |
843 comparison.recv_time_ms, comparison.render_time_ms, | 877 comparison.recv_time_ms, comparison.render_time_ms, |
844 comparison.encoded_frame_size, psnr, ssim)); | 878 comparison.encoded_frame_size, psnr, ssim)); |
845 } | 879 } |
846 if (psnr >= 0.0) | 880 if (psnr >= 0.0) |
847 psnr_.AddSample(psnr); | 881 psnr_.AddSample(psnr); |
848 if (ssim >= 0.0) | 882 if (ssim >= 0.0) |
849 ssim_.AddSample(ssim); | 883 ssim_.AddSample(ssim); |
(...skipping 183 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1033 test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_); | 1067 test::Statistics encode_frame_rate_ GUARDED_BY(comparison_lock_); |
1034 test::Statistics encode_time_ms_ GUARDED_BY(comparison_lock_); | 1068 test::Statistics encode_time_ms_ GUARDED_BY(comparison_lock_); |
1035 test::Statistics encode_usage_percent_ GUARDED_BY(comparison_lock_); | 1069 test::Statistics encode_usage_percent_ GUARDED_BY(comparison_lock_); |
1036 test::Statistics decode_time_ms_ GUARDED_BY(comparison_lock_); | 1070 test::Statistics decode_time_ms_ GUARDED_BY(comparison_lock_); |
1037 test::Statistics decode_time_max_ms_ GUARDED_BY(comparison_lock_); | 1071 test::Statistics decode_time_max_ms_ GUARDED_BY(comparison_lock_); |
1038 test::Statistics media_bitrate_bps_ GUARDED_BY(comparison_lock_); | 1072 test::Statistics media_bitrate_bps_ GUARDED_BY(comparison_lock_); |
1039 test::Statistics fec_bitrate_bps_ GUARDED_BY(comparison_lock_); | 1073 test::Statistics fec_bitrate_bps_ GUARDED_BY(comparison_lock_); |
1040 test::Statistics send_bandwidth_bps_ GUARDED_BY(comparison_lock_); | 1074 test::Statistics send_bandwidth_bps_ GUARDED_BY(comparison_lock_); |
1041 test::Statistics memory_usage_ GUARDED_BY(comparison_lock_); | 1075 test::Statistics memory_usage_ GUARDED_BY(comparison_lock_); |
1042 | 1076 |
1077 struct FrameWithPsnr { | |
1078 double psnr; | |
1079 VideoFrame frame; | |
1080 }; | |
1081 | |
1082 // Rendered frame with worst PSNR is saved for further analysis. | |
1083 rtc::Optional<FrameWithPsnr> worst_frame_ GUARDED_BY(comparison_lock_); | |
1084 | |
1043 size_t last_fec_bytes_; | 1085 size_t last_fec_bytes_; |
1044 | 1086 |
1045 const int frames_to_process_; | 1087 const int frames_to_process_; |
1046 int frames_recorded_; | 1088 int frames_recorded_; |
1047 int frames_processed_; | 1089 int frames_processed_; |
1048 int dropped_frames_; | 1090 int dropped_frames_; |
1049 int dropped_frames_before_first_encode_; | 1091 int dropped_frames_before_first_encode_; |
1050 int dropped_frames_before_rendering_; | 1092 int dropped_frames_before_rendering_; |
1051 int64_t last_render_time_; | 1093 int64_t last_render_time_; |
1052 uint32_t rtp_timestamp_delta_; | 1094 uint32_t rtp_timestamp_delta_; |
(...skipping 1029 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2082 if (!params_.logging.encoded_frame_base_path.empty()) { | 2124 if (!params_.logging.encoded_frame_base_path.empty()) { |
2083 std::ostringstream str; | 2125 std::ostringstream str; |
2084 str << receive_logs_++; | 2126 str << receive_logs_++; |
2085 std::string path = | 2127 std::string path = |
2086 params_.logging.encoded_frame_base_path + "." + str.str() + ".recv.ivf"; | 2128 params_.logging.encoded_frame_base_path + "." + str.str() + ".recv.ivf"; |
2087 stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path), | 2129 stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path), |
2088 100000000); | 2130 100000000); |
2089 } | 2131 } |
2090 } | 2132 } |
2091 } // namespace webrtc | 2133 } // namespace webrtc |
OLD | NEW |