diff options
author | 2019-06-17 22:18:29 +0000 | |
---|---|---|
committer | 2019-06-17 22:18:29 +0000 | |
commit | 504b10ec5101b237e4c07e1f2de4b6c48138181e (patch) | |
tree | 979c9ce8ab11efd05e4413305758dc5d6bc76ab4 /lib/libcxx/utils/google-benchmark/test | |
parent | A bit more KNF no binary change (diff) | |
download | wireguard-openbsd-504b10ec5101b237e4c07e1f2de4b6c48138181e.tar.xz wireguard-openbsd-504b10ec5101b237e4c07e1f2de4b6c48138181e.zip |
Import libc++ 8.0.0.
Diffstat (limited to 'lib/libcxx/utils/google-benchmark/test')
15 files changed, 936 insertions, 228 deletions
diff --git a/lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake b/lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake new file mode 100644 index 00000000000..3d078586f1d --- /dev/null +++ b/lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake @@ -0,0 +1,46 @@ + +include(split_list) + +set(ASM_TEST_FLAGS "") +check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) +if (BENCHMARK_HAS_O3_FLAG) + list(APPEND ASM_TEST_FLAGS -O3) +endif() + +check_cxx_compiler_flag(-g0 BENCHMARK_HAS_G0_FLAG) +if (BENCHMARK_HAS_G0_FLAG) + list(APPEND ASM_TEST_FLAGS -g0) +endif() + +check_cxx_compiler_flag(-fno-stack-protector BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG) +if (BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG) + list(APPEND ASM_TEST_FLAGS -fno-stack-protector) +endif() + +split_list(ASM_TEST_FLAGS) +string(TOUPPER "${CMAKE_CXX_COMPILER_ID}" ASM_TEST_COMPILER) + +macro(add_filecheck_test name) + cmake_parse_arguments(ARG "" "" "CHECK_PREFIXES" ${ARGV}) + add_library(${name} OBJECT ${name}.cc) + set_target_properties(${name} PROPERTIES COMPILE_FLAGS "-S ${ASM_TEST_FLAGS}") + set(ASM_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${name}.s") + add_custom_target(copy_${name} ALL + COMMAND ${PROJECT_SOURCE_DIR}/tools/strip_asm.py + $<TARGET_OBJECTS:${name}> + ${ASM_OUTPUT_FILE} + BYPRODUCTS ${ASM_OUTPUT_FILE}) + add_dependencies(copy_${name} ${name}) + if (NOT ARG_CHECK_PREFIXES) + set(ARG_CHECK_PREFIXES "CHECK") + endif() + foreach(prefix ${ARG_CHECK_PREFIXES}) + add_test(NAME run_${name}_${prefix} + COMMAND + ${LLVM_FILECHECK_EXE} ${name}.cc + --input-file=${ASM_OUTPUT_FILE} + --check-prefixes=CHECK,CHECK-${ASM_TEST_COMPILER} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endforeach() +endmacro() + diff --git a/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt b/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt index f49ca5148f4..f15ce208189 100644 --- a/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt +++ b/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt @@ -125,9 +125,21 @@ add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01 compile_output_test(user_counters_test) add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) +compile_output_test(report_aggregates_only_test) +add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01) + +compile_output_test(display_aggregates_only_test) +add_test(display_aggregates_only_test display_aggregates_only_test --benchmark_min_time=0.01) + compile_output_test(user_counters_tabular_test) add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) +compile_output_test(user_counters_thousands_test) +add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01) + +compile_output_test(memory_manager_test) +add_test(memory_manager_test memory_manager_test --benchmark_min_time=0.01) + check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) if (BENCHMARK_HAS_CXX03_FLAG) compile_benchmark_test(cxx03_test) diff --git a/lib/libcxx/utils/google-benchmark/test/complexity_test.cc b/lib/libcxx/utils/google-benchmark/test/complexity_test.cc index 5f91660898b..323ddfe7ac5 100644 --- a/lib/libcxx/utils/google-benchmark/test/complexity_test.cc +++ b/lib/libcxx/utils/google-benchmark/test/complexity_test.cc @@ -12,9 +12,10 @@ namespace { #define ADD_COMPLEXITY_CASES(...) \ int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) -int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, - std::string big_o) { - SetSubstitutions({{"%bigo_name", big_o_test_name}, +int AddComplexityTest(std::string test_name, std::string big_o_test_name, + std::string rms_test_name, std::string big_o) { + SetSubstitutions({{"%name", test_name}, + {"%bigo_name", big_o_test_name}, {"%rms_name", rms_test_name}, {"%bigo_str", "[ ]* %float " + big_o}, {"%bigo", big_o}, @@ -25,12 +26,18 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. {"^%rms_name %rms %rms[ ]*$", MR_Next}}); AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, + {"\"run_name\": \"%name\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"BigO\",$", MR_Next}, {"\"cpu_coefficient\": %float,$", MR_Next}, {"\"real_coefficient\": %float,$", MR_Next}, {"\"big_o\": \"%bigo\",$", MR_Next}, {"\"time_unit\": \"ns\"$", MR_Next}, {"}", MR_Next}, {"\"name\": \"%rms_name\",$"}, + {"\"run_name\": \"%name\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"RMS\",$", MR_Next}, {"\"rms\": %float$", MR_Next}, {"}", MR_Next}}); AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, @@ -59,6 +66,7 @@ BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) { return 1.0; }); +const char *one_test_name = "BM_Complexity_O1"; const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; const char *enum_big_o_1 = "\\([0-9]+\\)"; @@ -69,13 +77,16 @@ const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; const char *lambda_big_o_1 = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1); +ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, + enum_big_o_1); // Add auto enum tests -ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1); +ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, + auto_big_o_1); // Add lambda tests -ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); +ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name, + lambda_big_o_1); // ========================================================================= // // --------------------------- Testing BigO O(N) --------------------------- // @@ -112,16 +123,19 @@ BENCHMARK(BM_Complexity_O_N) ->Range(1 << 10, 1 << 16) ->Complexity(); +const char *n_test_name = "BM_Complexity_O_N"; const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; const char *enum_auto_big_o_n = "N"; const char *lambda_big_o_n = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + enum_auto_big_o_n); // Add lambda tests -ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); +ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name, + lambda_big_o_n); // ========================================================================= // // ------------------------- Testing BigO O(N*lgN) ------------------------- // @@ -148,18 +162,19 @@ BENCHMARK(BM_Complexity_O_N_log_N) ->Range(1 << 10, 1 << 16) ->Complexity(); +const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N"; const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; const char *enum_auto_big_o_n_lg_n = "NlgN"; const char *lambda_big_o_n_lg_n = "f\\(N\\)"; // Add enum tests -ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, - enum_auto_big_o_n_lg_n); +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n); // Add lambda tests -ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, - lambda_big_o_n_lg_n); +ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name, + rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc b/lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc new file mode 100644 index 00000000000..3c36d3f03c1 --- /dev/null +++ b/lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc @@ -0,0 +1,43 @@ + +#undef NDEBUG +#include <cstdio> +#include <string> + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// Ok this test is super ugly. We want to check what happens with the file +// reporter in the presence of DisplayAggregatesOnly(). +// We do not care about console output, the normal tests check that already. + +void BM_SummaryRepeat(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly(); + +int main(int argc, char* argv[]) { + const std::string output = GetFileReporterOutput(argc, argv); + + if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 6 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3\"") != 3 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") != + 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") != + 1) { + std::cout << "Precondition mismatch. Expected to only find 6 " + "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" + "\"name\": \"BM_SummaryRepeat/repeats:3\", " + "\"name\": \"BM_SummaryRepeat/repeats:3\", " + "\"name\": \"BM_SummaryRepeat/repeats:3\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_median\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire " + "output:\n"; + std::cout << output; + return 1; + } + + return 0; +} diff --git a/lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc b/lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc new file mode 100644 index 00000000000..94be6083795 --- /dev/null +++ b/lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc @@ -0,0 +1,42 @@ +#include <memory> + +#include "../src/check.h" +#include "benchmark/benchmark.h" +#include "output_test.h" + +class TestMemoryManager : public benchmark::MemoryManager { + void Start() {} + void Stop(Result* result) { + result->num_allocs = 42; + result->max_bytes_used = 42000; + } +}; + +void BM_empty(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(state.iterations()); + } +} +BENCHMARK(BM_empty); + +ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"}, + {"\"run_name\": \"BM_empty\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"allocs_per_iter\": %float,$", MR_Next}, + {"\"max_bytes_used\": 42000$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}}); + + +int main(int argc, char *argv[]) { + std::unique_ptr<benchmark::MemoryManager> mm(new TestMemoryManager()); + + benchmark::RegisterMemoryManager(mm.get()); + RunOutputTests(argc, argv); + benchmark::RegisterMemoryManager(nullptr); +} diff --git a/lib/libcxx/utils/google-benchmark/test/output_test.h b/lib/libcxx/utils/google-benchmark/test/output_test.h index 31a919991f7..9385761b214 100644 --- a/lib/libcxx/utils/google-benchmark/test/output_test.h +++ b/lib/libcxx/utils/google-benchmark/test/output_test.h @@ -60,6 +60,13 @@ int SetSubstitutions( // Run all output tests. void RunOutputTests(int argc, char* argv[]); +// Count the number of 'pat' substrings in the 'haystack' string. +int SubstrCnt(const std::string& haystack, const std::string& pat); + +// Run registered benchmarks with file reporter enabled, and return the content +// outputted by the file reporter. +std::string GetFileReporterOutput(int argc, char* argv[]); + // ========================================================================= // // ------------------------- Results checking ------------------------------ // // ========================================================================= // diff --git a/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc b/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc index 394c4f5d1a2..5dc951d2bca 100644 --- a/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc +++ b/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc @@ -1,8 +1,12 @@ +#include <cstdio> #include <cstring> +#include <fstream> #include <iostream> #include <map> #include <memory> +#include <random> #include <sstream> +#include <streambuf> #include "../src/benchmark_api_internal.h" #include "../src/check.h" // NOTE: check.h is for internal use only! @@ -35,15 +39,18 @@ SubMap& GetSubstitutions() { // Don't use 'dec_re' from header because it may not yet be initialized. // clang-format off static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; + static std::string time_re = "([0-9]+[.])?[0-9]+"; static SubMap map = { {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, // human-readable float {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, {"%int", "[ ]*[0-9]+"}, {" %s ", "[ ]+"}, - {"%time", "[ ]*[0-9]{1,6} ns"}, - {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"}, - {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, + {"%time", "[ ]*" + time_re + "[ ]+ns"}, + {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, + {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, + {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, + {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, {"%csv_header", "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," "items_per_second,label,error_occurred,error_message"}, @@ -202,7 +209,7 @@ void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { void ResultsChecker::CheckResults(std::stringstream& output) { // first reset the stream to the start { - auto start = std::ios::streampos(0); + auto start = std::stringstream::pos_type(0); // clear before calling tellg() output.clear(); // seek to zero only when needed @@ -423,3 +430,76 @@ void RunOutputTests(int argc, char* argv[]) { CHECK(std::strcmp(csv.name, "CSVReporter") == 0); internal::GetResultsChecker().CheckResults(csv.out_stream); } + +int SubstrCnt(const std::string& haystack, const std::string& pat) { + if (pat.length() == 0) return 0; + int count = 0; + for (size_t offset = haystack.find(pat); offset != std::string::npos; + offset = haystack.find(pat, offset + pat.length())) + ++count; + return count; +} + +static char ToHex(int ch) { + return ch < 10 ? static_cast<char>('0' + ch) + : static_cast<char>('a' + (ch - 10)); +} + +static char RandomHexChar() { + static std::mt19937 rd{std::random_device{}()}; + static std::uniform_int_distribution<int> mrand{0, 15}; + return ToHex(mrand(rd)); +} + +static std::string GetRandomFileName() { + std::string model = "test.%%%%%%"; + for (auto & ch : model) { + if (ch == '%') + ch = RandomHexChar(); + } + return model; +} + +static bool FileExists(std::string const& name) { + std::ifstream in(name.c_str()); + return in.good(); +} + +static std::string GetTempFileName() { + // This function attempts to avoid race conditions where two tests + // create the same file at the same time. However, it still introduces races + // similar to tmpnam. + int retries = 3; + while (--retries) { + std::string name = GetRandomFileName(); + if (!FileExists(name)) + return name; + } + std::cerr << "Failed to create unique temporary file name" << std::endl; + std::abort(); +} + +std::string GetFileReporterOutput(int argc, char* argv[]) { + std::vector<char*> new_argv(argv, argv + argc); + assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size()); + + std::string tmp_file_name = GetTempFileName(); + std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n'; + + std::string tmp = "--benchmark_out="; + tmp += tmp_file_name; + new_argv.emplace_back(const_cast<char*>(tmp.c_str())); + + argc = int(new_argv.size()); + + benchmark::Initialize(&argc, new_argv.data()); + benchmark::RunSpecifiedBenchmarks(); + + // Read the output back from the file, and delete the file. + std::ifstream tmp_stream(tmp_file_name); + std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)), + std::istreambuf_iterator<char>()); + std::remove(tmp_file_name.c_str()); + + return output; +} diff --git a/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc b/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc index 18de6d68e21..3ac5b21fb34 100644 --- a/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc +++ b/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc @@ -30,8 +30,8 @@ struct TestCase { void CheckRun(Run const& run) const { // clang-format off - CHECK(name == run.benchmark_name) << "expected " << name << " got " - << run.benchmark_name; + CHECK(name == run.benchmark_name()) << "expected " << name << " got " + << run.benchmark_name(); if (label) { CHECK(run.report_label == label) << "expected " << label << " got " << run.report_label; diff --git a/lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc b/lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc new file mode 100644 index 00000000000..9646b9be534 --- /dev/null +++ b/lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc @@ -0,0 +1,39 @@ + +#undef NDEBUG +#include <cstdio> +#include <string> + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// Ok this test is super ugly. We want to check what happens with the file +// reporter in the presence of ReportAggregatesOnly(). +// We do not care about console output, the normal tests check that already. + +void BM_SummaryRepeat(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); + +int main(int argc, char* argv[]) { + const std::string output = GetFileReporterOutput(argc, argv); + + if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 3 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") != + 1 || + SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") != + 1) { + std::cout << "Precondition mismatch. Expected to only find three " + "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n" + "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_median\", " + "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire " + "output:\n"; + std::cout << output; + return 1; + } + + return 0; +} diff --git a/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc b/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc index 1662fcb8b54..ec6d51b3591 100644 --- a/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc +++ b/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc @@ -17,18 +17,21 @@ static int AddContextCases() { { {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default}, {"Running .*/reporter_output_test(\\.exe)?$", MR_Next}, - {"Run on \\(%int X %float MHz CPU s\\)", MR_Next}, + {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next}, }); AddCases(TC_JSONOut, {{"^\\{", MR_Default}, {"\"context\":", MR_Next}, {"\"date\": \"", MR_Next}, - {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next}, + {"\"host_name\":", MR_Next}, + {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",", + MR_Next}, {"\"num_cpus\": %int,$", MR_Next}, {"\"mhz_per_cpu\": %float,$", MR_Next}, {"\"cpu_scaling_enabled\": ", MR_Next}, {"\"caches\": \\[$", MR_Next}}); - auto const& Caches = benchmark::CPUInfo::Get().caches; + auto const& Info = benchmark::CPUInfo::Get(); + auto const& Caches = Info.caches; if (!Caches.empty()) { AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}}); } @@ -45,8 +48,13 @@ static int AddContextCases() { {"\"num_sharing\": %int$", MR_Next}, {"}[,]{0,1}$", MR_Next}}); } - AddCases(TC_JSONOut, {{"],$"}}); + auto const& LoadAvg = Info.load_avg; + if (!LoadAvg.empty()) { + AddCases(TC_ConsoleErr, + {{"Load Average: (%float, ){0,2}%float$", MR_Next}}); + } + AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}}); return 0; } int dummy_register = AddContextCases(); @@ -64,6 +72,8 @@ BENCHMARK(BM_basic); ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, + {"\"run_name\": \"BM_basic\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -82,9 +92,11 @@ void BM_bytes_per_second(benchmark::State& state) { } BENCHMARK(BM_bytes_per_second); -ADD_CASES(TC_ConsoleOut, - {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report " + "bytes_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, + {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -104,9 +116,11 @@ void BM_items_per_second(benchmark::State& state) { } BENCHMARK(BM_items_per_second); -ADD_CASES(TC_ConsoleOut, - {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report " + "items_per_second=%float[kM]{0,1}/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, + {"\"run_name\": \"BM_items_per_second\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -128,6 +142,8 @@ BENCHMARK(BM_label); ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, + {"\"run_name\": \"BM_label\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -149,6 +165,8 @@ void BM_error(benchmark::State& state) { BENCHMARK(BM_error); ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, + {"\"run_name\": \"BM_error\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"error_occurred\": true,$", MR_Next}, {"\"error_message\": \"message\",$", MR_Next}}); @@ -165,7 +183,9 @@ void BM_no_arg_name(benchmark::State& state) { } BENCHMARK(BM_no_arg_name)->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}, + {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); // ========================================================================= // @@ -178,7 +198,9 @@ void BM_arg_name(benchmark::State& state) { } BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}, + {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); // ========================================================================= // @@ -192,10 +214,25 @@ void BM_arg_names(benchmark::State& state) { BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); ADD_CASES(TC_ConsoleOut, {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}, + {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); // ========================================================================= // +// ------------------------ Testing Big Args Output ------------------------ // +// ========================================================================= // + +void BM_BigArgs(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U); +ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"}, + {"^BM_BigArgs/2147483648 %console_report$"}}); + +// ========================================================================= // // ----------------------- Testing Complexity Output ----------------------- // // ========================================================================= // @@ -221,16 +258,33 @@ void BM_Repeat(benchmark::State& state) { } // need two repetitions min to be able to output any aggregate output BENCHMARK(BM_Repeat)->Repetitions(2); -ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"}, - {"^BM_Repeat/repeats:2 %console_report$"}, - {"^BM_Repeat/repeats:2_mean %console_report$"}, - {"^BM_Repeat/repeats:2_median %console_report$"}, - {"^BM_Repeat/repeats:2_stddev %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Repeat/repeats:2 %console_report$"}, + {"^BM_Repeat/repeats:2 %console_report$"}, + {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"}, + {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"}, + {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:2_median\",$"}, - {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}}); + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, + {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"}, @@ -238,18 +292,37 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"}, {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}}); // but for two repetitions, mean and median is the same, so let's repeat.. BENCHMARK(BM_Repeat)->Repetitions(3); -ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, - {"^BM_Repeat/repeats:3 %console_report$"}, - {"^BM_Repeat/repeats:3 %console_report$"}, - {"^BM_Repeat/repeats:3_mean %console_report$"}, - {"^BM_Repeat/repeats:3_median %console_report$"}, - {"^BM_Repeat/repeats:3_stddev %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"}, + {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"}, + {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:3_median\",$"}, - {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3\",%csv_report$"}, @@ -258,20 +331,41 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); // median differs between even/odd number of repetitions, so just to be sure BENCHMARK(BM_Repeat)->Repetitions(4); -ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4 %console_report$"}, - {"^BM_Repeat/repeats:4_mean %console_report$"}, - {"^BM_Repeat/repeats:4_median %console_report$"}, - {"^BM_Repeat/repeats:4_stddev %console_report$"}}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4 %console_report$"}, + {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"}, + {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"}, + {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_mean\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 4,$", MR_Next}, {"\"name\": \"BM_Repeat/repeats:4_median\",$"}, - {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}}); + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 4,$", MR_Next}, + {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}, + {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 4,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, {"^\"BM_Repeat/repeats:4\",%csv_report$"}, @@ -288,7 +382,9 @@ void BM_RepeatOnce(benchmark::State& state) { } BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}, + {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); // Test that non-aggregate data is not reported @@ -297,20 +393,72 @@ void BM_SummaryRepeat(benchmark::State& state) { } } BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); -ADD_CASES(TC_ConsoleOut, +ADD_CASES( + TC_ConsoleOut, + {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"}, + {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"}, + {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}}); +ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, - {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, - {"^BM_SummaryRepeat/repeats:3_median %console_report$"}, - {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); -ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, - {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); + {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"}, + {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}, + {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}}); ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"}, {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); +// Test that non-aggregate data is not displayed. +// NOTE: this test is kinda bad. we are only testing the display output. +// But we don't check that the file output still contains everything... +void BM_SummaryDisplay(benchmark::State& state) { + for (auto _ : state) { + } +} +BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly(); +ADD_CASES( + TC_ConsoleOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"}, + {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"}, + {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}}); +ADD_CASES(TC_JSONOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"}, + {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, + {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"}, + {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}}); +ADD_CASES(TC_CSVOut, + {{".*BM_SummaryDisplay/repeats:2 ", MR_Not}, + {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"}, + {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"}, + {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}}); + +// Test repeats with custom time unit. void BM_RepeatTimeUnit(benchmark::State& state) { for (auto _ : state) { } @@ -319,18 +467,34 @@ BENCHMARK(BM_RepeatTimeUnit) ->Repetitions(3) ->ReportAggregatesOnly() ->Unit(benchmark::kMicrosecond); -ADD_CASES(TC_ConsoleOut, +ADD_CASES( + TC_ConsoleOut, + {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, + {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"}, + {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ " + "]*3$"}, + {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ " + "]*3$"}}); +ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, - {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, - {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"}, - {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); -ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, - {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, - {"\"time_unit\": \"us\",?$"}, - {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, - {"\"time_unit\": \"us\",?$"}, - {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, - {"\"time_unit\": \"us\",?$"}}); + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, + {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"time_unit\": \"us\",?$"}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"}, + {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"time_unit\": \"us\",?$"}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, + {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"time_unit\": \"us\",?$"}}); ADD_CASES(TC_CSVOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, @@ -346,37 +510,92 @@ const auto UserStatistics = [](const std::vector<double>& v) { }; void BM_UserStats(benchmark::State& state) { for (auto _ : state) { + state.SetIterationTime(150 / 10e8); } } // clang-format off BENCHMARK(BM_UserStats) ->Repetitions(3) + ->Iterations(5) + ->UseManualTime() ->ComputeStatistics("", UserStatistics); // clang-format on // check that user-provided stats is calculated, and is after the default-ones // empty string as name is intentional, it would sort before anything else -ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"}, - {"^BM_UserStats/repeats:3 %console_report$"}, - {"^BM_UserStats/repeats:3 %console_report$"}, - {"^BM_UserStats/repeats:3_mean %console_report$"}, - {"^BM_UserStats/repeats:3_median %console_report$"}, - {"^BM_UserStats/repeats:3_stddev %console_report$"}, - {"^BM_UserStats/repeats:3_ %console_report$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"}, - {"\"name\": \"BM_UserStats/repeats:3\",$"}, - {"\"name\": \"BM_UserStats/repeats:3\",$"}, - {"\"name\": \"BM_UserStats/repeats:3_mean\",$"}, - {"\"name\": \"BM_UserStats/repeats:3_median\",$"}, - {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"}, - {"\"name\": \"BM_UserStats/repeats:3_\",$"}}); -ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_median\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"}, - {"^\"BM_UserStats/repeats:3_\",%csv_report$"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ " + "]* 150 ns %time [ ]*5$"}, + {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " + "]* 150 ns %time [ ]*5$"}, + {"^BM_UserStats/iterations:5/repeats:3/manual_time [ " + "]* 150 ns %time [ ]*5$"}, + {"^BM_UserStats/iterations:5/repeats:3/" + "manual_time_mean [ ]* 150 ns %time [ ]*3$"}, + {"^BM_UserStats/iterations:5/repeats:3/" + "manual_time_median [ ]* 150 ns %time [ ]*3$"}, + {"^BM_UserStats/iterations:5/repeats:3/" + "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"}, + {"^BM_UserStats/iterations:5/repeats:3/manual_time_ " + "[ ]* 150 ns %time [ ]*3$"}}); +ADD_CASES( + TC_JSONOut, + {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": 5,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": 5,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": 5,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"}, + {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$", + MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"\",$", MR_Next}, + {"\"iterations\": 3,$", MR_Next}, + {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}}); +ADD_CASES( + TC_CSVOut, + {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/" + "manual_time_median\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/" + "manual_time_stddev\",%csv_report$"}, + {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}}); // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // diff --git a/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc b/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc index 39785fb7f6d..06579772ff7 100644 --- a/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc +++ b/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc @@ -33,8 +33,8 @@ struct TestCase { typedef benchmark::BenchmarkReporter::Run Run; void CheckRun(Run const& run) const { - CHECK(name == run.benchmark_name) - << "expected " << name << " got " << run.benchmark_name; + CHECK(name == run.benchmark_name()) + << "expected " << name << " got " << run.benchmark_name(); CHECK(error_occurred == run.error_occurred); CHECK(error_message == run.error_message); if (error_occurred) { diff --git a/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc b/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc index 4c81734cf8a..2c5d073f613 100644 --- a/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc +++ b/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc @@ -9,56 +9,56 @@ namespace { TEST(StringUtilTest, stoul) { { size_t pos = 0; - EXPECT_EQ(0, benchmark::stoul("0", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(0ul, benchmark::stoul("0", &pos)); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; - EXPECT_EQ(7, benchmark::stoul("7", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(7ul, benchmark::stoul("7", &pos)); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; - EXPECT_EQ(135, benchmark::stoul("135", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(135ul, benchmark::stoul("135", &pos)); + EXPECT_EQ(3ul, pos); } #if ULONG_MAX == 0xFFFFFFFFul { size_t pos = 0; EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul("4294967295", &pos)); - EXPECT_EQ(10, pos); + EXPECT_EQ(10ul, pos); } #elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul { size_t pos = 0; EXPECT_EQ(0xFFFFFFFFFFFFFFFFul, benchmark::stoul("18446744073709551615", &pos)); - EXPECT_EQ(20, pos); + EXPECT_EQ(20ul, pos); } #endif { size_t pos = 0; - EXPECT_EQ(10, benchmark::stoul("1010", &pos, 2)); - EXPECT_EQ(4, pos); + EXPECT_EQ(10ul, benchmark::stoul("1010", &pos, 2)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(520, benchmark::stoul("1010", &pos, 8)); - EXPECT_EQ(4, pos); + EXPECT_EQ(520ul, benchmark::stoul("1010", &pos, 8)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(1010, benchmark::stoul("1010", &pos, 10)); - EXPECT_EQ(4, pos); + EXPECT_EQ(1010ul, benchmark::stoul("1010", &pos, 10)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(4112, benchmark::stoul("1010", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4112ul, benchmark::stoul("1010", &pos, 16)); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; - EXPECT_EQ(0xBEEF, benchmark::stoul("BEEF", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16)); + EXPECT_EQ(4ul, pos); } { ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument); @@ -69,42 +69,42 @@ TEST(StringUtilTest, stoi) { { size_t pos = 0; EXPECT_EQ(0, benchmark::stoi("0", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; EXPECT_EQ(-17, benchmark::stoi("-17", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(3ul, pos); } { size_t pos = 0; EXPECT_EQ(1357, benchmark::stoi("1357", &pos)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument); @@ -115,28 +115,28 @@ TEST(StringUtilTest, stod) { { size_t pos = 0; EXPECT_EQ(0.0, benchmark::stod("0", &pos)); - EXPECT_EQ(1, pos); + EXPECT_EQ(1ul, pos); } { size_t pos = 0; EXPECT_EQ(-84.0, benchmark::stod("-84", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(3ul, pos); } { size_t pos = 0; EXPECT_EQ(1234.0, benchmark::stod("1234", &pos)); - EXPECT_EQ(4, pos); + EXPECT_EQ(4ul, pos); } { size_t pos = 0; EXPECT_EQ(1.5, benchmark::stod("1.5", &pos)); - EXPECT_EQ(3, pos); + EXPECT_EQ(3ul, pos); } { size_t pos = 0; /* Note: exactly representable as double */ EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos)); - EXPECT_EQ(8, pos); + EXPECT_EQ(8ul, pos); } { ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument); diff --git a/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc b/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc index 4f126b6d978..030e98916c3 100644 --- a/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc +++ b/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc @@ -69,18 +69,21 @@ void BM_Counters_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Bat\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float,$", MR_Next}, - {"\"Frob\": %float,$", MR_Next}, - {"\"Lob\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -113,18 +116,22 @@ void BM_CounterRates_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Bat\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float,$", MR_Next}, - {"\"Frob\": %float,$", MR_Next}, - {"\"Lob\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," "%float,%float,%float,%float,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -157,15 +164,18 @@ void BM_CounterSet0_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report," "%float,,%float,%float,,"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -189,15 +199,18 @@ void BM_CounterSet1_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bar\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report," "%float,,%float,%float,,"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -225,15 +238,18 @@ void BM_CounterSet2_Tabular(benchmark::State& state) { }); } BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"Bat\": %float,$", MR_Next}, - {"\"Baz\": %float,$", MR_Next}, - {"\"Foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, + {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report," ",%float,%float,%float,,"}}); // VS2013 does not allow this function to be passed as a lambda argument diff --git a/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc b/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc index 7f7ccb9f77a..bb0d6b4c5a9 100644 --- a/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc +++ b/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc @@ -32,6 +32,8 @@ BENCHMARK(BM_Counters_Simple); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, + {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -66,19 +68,22 @@ void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { state.SetItemsProcessed(150); } BENCHMARK(BM_Counters_WithBytesAndItemsPSec); -ADD_CASES(TC_ConsoleOut, - {{"^BM_Counters_WithBytesAndItemsPSec %console_report " - "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bytes_per_second\": %float,$", MR_Next}, - {"\"items_per_second\": %float,$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report " + "bar=%hrfloat bytes_per_second=%hrfloat/s " + "foo=%hrfloat items_per_second=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, + {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"bytes_per_second\": %float,$", MR_Next}, + {"\"foo\": %float,$", MR_Next}, + {"\"items_per_second\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," "%csv_bytes_items_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -110,6 +115,8 @@ ADD_CASES( TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, + {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -141,14 +148,17 @@ void BM_Counters_Threads(benchmark::State& state) { BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report " "bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES( TC_CSVOut, {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); @@ -174,14 +184,17 @@ void BM_Counters_AvgThreads(benchmark::State& state) { BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int " "%console_report bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES( TC_CSVOut, {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); @@ -210,6 +223,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -242,14 +258,17 @@ void BM_Counters_IterationInvariant(benchmark::State& state) { BENCHMARK(BM_Counters_IterationInvariant); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report " "bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_IterationInvariant\",$"}, + {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -281,6 +300,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"}, + {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$", + MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, {"\"iterations\": %int,$", MR_Next}, {"\"real_time\": %float,$", MR_Next}, {"\"cpu_time\": %float,$", MR_Next}, @@ -316,14 +338,17 @@ void BM_Counters_AvgIterations(benchmark::State& state) { BENCHMARK(BM_Counters_AvgIterations); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report " "bar=%hrfloat foo=%hrfloat$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_AvgIterations\",$"}, + {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument @@ -351,14 +376,17 @@ void BM_Counters_kAvgIterationsRate(benchmark::State& state) { BENCHMARK(BM_Counters_kAvgIterationsRate); ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate " "%console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); -ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, - {"\"iterations\": %int,$", MR_Next}, - {"\"real_time\": %float,$", MR_Next}, - {"\"cpu_time\": %float,$", MR_Next}, - {"\"time_unit\": \"ns\",$", MR_Next}, - {"\"bar\": %float,$", MR_Next}, - {"\"foo\": %float$", MR_Next}, - {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"}, + {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report," "%float,%float$"}}); // VS2013 does not allow this function to be passed as a lambda argument diff --git a/lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc b/lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc new file mode 100644 index 00000000000..fa0ef972047 --- /dev/null +++ b/lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc @@ -0,0 +1,161 @@ + +#undef NDEBUG + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// ========================================================================= // +// ------------------------ Thousands Customisation ------------------------ // +// ========================================================================= // + +void BM_Counters_Thousands(benchmark::State& state) { + for (auto _ : state) { + } + namespace bm = benchmark; + state.counters.insert({ + {"t0_1000000DefaultBase", + bm::Counter(1000 * 1000, bm::Counter::kDefaults)}, + {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1000)}, + {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024)}, + {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1000)}, + {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults, + benchmark::Counter::OneK::kIs1024)}, + }); +} +BENCHMARK(BM_Counters_Thousands)->Repetitions(2); +ADD_CASES( + TC_ConsoleOut, + { + {"^BM_Counters_Thousands/repeats:2 %console_report " + "t0_1000000DefaultBase=1000k " + "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " + "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2 %console_report " + "t0_1000000DefaultBase=1000k " + "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k " + "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2_mean %console_report " + "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " + "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " + "t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2_median %console_report " + "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k " + "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k " + "t4_1048576Base1024=1024k$"}, + {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ " + "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 " + "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"}, + }); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, + {"\"run_type\": \"iteration\",$", MR_Next}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"mean\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"median\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next}, + {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next}, + {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_JSONOut, + {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"}, + {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next}, + {"\"run_type\": \"aggregate\",$", MR_Next}, + {"\"aggregate_name\": \"stddev\",$", MR_Next}, + {"\"iterations\": 2,$", MR_Next}, + {"\"real_time\": %float,$", MR_Next}, + {"\"cpu_time\": %float,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"t0_1000000DefaultBase\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t1_1000000Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t2_1000000Base1024\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t3_1048576Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next}, + {"\"t4_1048576Base1024\": 0\\.(0)*e\\+(0)*$", MR_Next}, + {"}", MR_Next}}); + +ADD_CASES( + TC_CSVOut, + {{"^\"BM_Counters_Thousands/" + "repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+(" + "0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/" + "repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+(" + "0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/" + "repeats:2_mean\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\." + "04858e\\+(0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/" + "repeats:2_median\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\." + "04858e\\+(0)*6,1\\.04858e\\+(0)*6$"}, + {"^\"BM_Counters_Thousands/repeats:2_stddev\",%csv_report,0,0,0,0,0$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckThousands(Results const& e) { + if (e.name != "BM_Counters_Thousands/repeats:2") + return; // Do not check the aggregates! + + // check that the values are within 0.01% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "t0_1000000DefaultBase", EQ, 1000 * 1000, + 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t1_1000000Base1000", EQ, 1000 * 1000, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t2_1000000Base1024", EQ, 1000 * 1000, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t3_1048576Base1000", EQ, 1024 * 1024, 0.0001); + CHECK_FLOAT_COUNTER_VALUE(e, "t4_1048576Base1024", EQ, 1024 * 1024, 0.0001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Thousands", &CheckThousands); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } |