summaryrefslogtreecommitdiffstats
path: root/lib/libcxx/utils/google-benchmark/test
diff options
context:
space:
mode:
authorpatrick <patrick@openbsd.org>2019-06-17 22:18:29 +0000
committerpatrick <patrick@openbsd.org>2019-06-17 22:18:29 +0000
commit504b10ec5101b237e4c07e1f2de4b6c48138181e (patch)
tree979c9ce8ab11efd05e4413305758dc5d6bc76ab4 /lib/libcxx/utils/google-benchmark/test
parentA bit more KNF no binary change (diff)
downloadwireguard-openbsd-504b10ec5101b237e4c07e1f2de4b6c48138181e.tar.xz
wireguard-openbsd-504b10ec5101b237e4c07e1f2de4b6c48138181e.zip
Import libc++ 8.0.0.
Diffstat (limited to 'lib/libcxx/utils/google-benchmark/test')
-rw-r--r--lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake46
-rw-r--r--lib/libcxx/utils/google-benchmark/test/CMakeLists.txt12
-rw-r--r--lib/libcxx/utils/google-benchmark/test/complexity_test.cc39
-rw-r--r--lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc43
-rw-r--r--lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc42
-rw-r--r--lib/libcxx/utils/google-benchmark/test/output_test.h7
-rw-r--r--lib/libcxx/utils/google-benchmark/test/output_test_helper.cc88
-rw-r--r--lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc4
-rw-r--r--lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc39
-rw-r--r--lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc365
-rw-r--r--lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc4
-rw-r--r--lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc62
-rw-r--r--lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc118
-rw-r--r--lib/libcxx/utils/google-benchmark/test/user_counters_test.cc134
-rw-r--r--lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc161
15 files changed, 936 insertions, 228 deletions
diff --git a/lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake b/lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake
new file mode 100644
index 00000000000..3d078586f1d
--- /dev/null
+++ b/lib/libcxx/utils/google-benchmark/test/AssemblyTests.cmake
@@ -0,0 +1,46 @@
+
+include(split_list)
+
+set(ASM_TEST_FLAGS "")
+check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG)
+if (BENCHMARK_HAS_O3_FLAG)
+ list(APPEND ASM_TEST_FLAGS -O3)
+endif()
+
+check_cxx_compiler_flag(-g0 BENCHMARK_HAS_G0_FLAG)
+if (BENCHMARK_HAS_G0_FLAG)
+ list(APPEND ASM_TEST_FLAGS -g0)
+endif()
+
+check_cxx_compiler_flag(-fno-stack-protector BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG)
+if (BENCHMARK_HAS_FNO_STACK_PROTECTOR_FLAG)
+ list(APPEND ASM_TEST_FLAGS -fno-stack-protector)
+endif()
+
+split_list(ASM_TEST_FLAGS)
+string(TOUPPER "${CMAKE_CXX_COMPILER_ID}" ASM_TEST_COMPILER)
+
+macro(add_filecheck_test name)
+ cmake_parse_arguments(ARG "" "" "CHECK_PREFIXES" ${ARGV})
+ add_library(${name} OBJECT ${name}.cc)
+ set_target_properties(${name} PROPERTIES COMPILE_FLAGS "-S ${ASM_TEST_FLAGS}")
+ set(ASM_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${name}.s")
+ add_custom_target(copy_${name} ALL
+ COMMAND ${PROJECT_SOURCE_DIR}/tools/strip_asm.py
+ $<TARGET_OBJECTS:${name}>
+ ${ASM_OUTPUT_FILE}
+ BYPRODUCTS ${ASM_OUTPUT_FILE})
+ add_dependencies(copy_${name} ${name})
+ if (NOT ARG_CHECK_PREFIXES)
+ set(ARG_CHECK_PREFIXES "CHECK")
+ endif()
+ foreach(prefix ${ARG_CHECK_PREFIXES})
+ add_test(NAME run_${name}_${prefix}
+ COMMAND
+ ${LLVM_FILECHECK_EXE} ${name}.cc
+ --input-file=${ASM_OUTPUT_FILE}
+ --check-prefixes=CHECK,CHECK-${ASM_TEST_COMPILER}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+ endforeach()
+endmacro()
+
diff --git a/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt b/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt
index f49ca5148f4..f15ce208189 100644
--- a/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt
+++ b/lib/libcxx/utils/google-benchmark/test/CMakeLists.txt
@@ -125,9 +125,21 @@ add_test(templated_fixture_test templated_fixture_test --benchmark_min_time=0.01
compile_output_test(user_counters_test)
add_test(user_counters_test user_counters_test --benchmark_min_time=0.01)
+compile_output_test(report_aggregates_only_test)
+add_test(report_aggregates_only_test report_aggregates_only_test --benchmark_min_time=0.01)
+
+compile_output_test(display_aggregates_only_test)
+add_test(display_aggregates_only_test display_aggregates_only_test --benchmark_min_time=0.01)
+
compile_output_test(user_counters_tabular_test)
add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01)
+compile_output_test(user_counters_thousands_test)
+add_test(user_counters_thousands_test user_counters_thousands_test --benchmark_min_time=0.01)
+
+compile_output_test(memory_manager_test)
+add_test(memory_manager_test memory_manager_test --benchmark_min_time=0.01)
+
check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG)
if (BENCHMARK_HAS_CXX03_FLAG)
compile_benchmark_test(cxx03_test)
diff --git a/lib/libcxx/utils/google-benchmark/test/complexity_test.cc b/lib/libcxx/utils/google-benchmark/test/complexity_test.cc
index 5f91660898b..323ddfe7ac5 100644
--- a/lib/libcxx/utils/google-benchmark/test/complexity_test.cc
+++ b/lib/libcxx/utils/google-benchmark/test/complexity_test.cc
@@ -12,9 +12,10 @@ namespace {
#define ADD_COMPLEXITY_CASES(...) \
int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__)
-int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name,
- std::string big_o) {
- SetSubstitutions({{"%bigo_name", big_o_test_name},
+int AddComplexityTest(std::string test_name, std::string big_o_test_name,
+ std::string rms_test_name, std::string big_o) {
+ SetSubstitutions({{"%name", test_name},
+ {"%bigo_name", big_o_test_name},
{"%rms_name", rms_test_name},
{"%bigo_str", "[ ]* %float " + big_o},
{"%bigo", big_o},
@@ -25,12 +26,18 @@ int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name,
{"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name.
{"^%rms_name %rms %rms[ ]*$", MR_Next}});
AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"},
+ {"\"run_name\": \"%name\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"BigO\",$", MR_Next},
{"\"cpu_coefficient\": %float,$", MR_Next},
{"\"real_coefficient\": %float,$", MR_Next},
{"\"big_o\": \"%bigo\",$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next},
{"\"name\": \"%rms_name\",$"},
+ {"\"run_name\": \"%name\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"RMS\",$", MR_Next},
{"\"rms\": %float$", MR_Next},
{"}", MR_Next}});
AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"},
@@ -59,6 +66,7 @@ BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int64_t) {
return 1.0;
});
+const char *one_test_name = "BM_Complexity_O1";
const char *big_o_1_test_name = "BM_Complexity_O1_BigO";
const char *rms_o_1_test_name = "BM_Complexity_O1_RMS";
const char *enum_big_o_1 = "\\([0-9]+\\)";
@@ -69,13 +77,16 @@ const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)";
const char *lambda_big_o_1 = "f\\(N\\)";
// Add enum tests
-ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1);
+ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
+ enum_big_o_1);
// Add auto enum tests
-ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1);
+ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
+ auto_big_o_1);
// Add lambda tests
-ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1);
+ADD_COMPLEXITY_CASES(one_test_name, big_o_1_test_name, rms_o_1_test_name,
+ lambda_big_o_1);
// ========================================================================= //
// --------------------------- Testing BigO O(N) --------------------------- //
@@ -112,16 +123,19 @@ BENCHMARK(BM_Complexity_O_N)
->Range(1 << 10, 1 << 16)
->Complexity();
+const char *n_test_name = "BM_Complexity_O_N";
const char *big_o_n_test_name = "BM_Complexity_O_N_BigO";
const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS";
const char *enum_auto_big_o_n = "N";
const char *lambda_big_o_n = "f\\(N\\)";
// Add enum tests
-ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n);
+ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
+ enum_auto_big_o_n);
// Add lambda tests
-ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n);
+ADD_COMPLEXITY_CASES(n_test_name, big_o_n_test_name, rms_o_n_test_name,
+ lambda_big_o_n);
// ========================================================================= //
// ------------------------- Testing BigO O(N*lgN) ------------------------- //
@@ -148,18 +162,19 @@ BENCHMARK(BM_Complexity_O_N_log_N)
->Range(1 << 10, 1 << 16)
->Complexity();
+const char *n_lg_n_test_name = "BM_Complexity_O_N_log_N";
const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO";
const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS";
const char *enum_auto_big_o_n_lg_n = "NlgN";
const char *lambda_big_o_n_lg_n = "f\\(N\\)";
// Add enum tests
-ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name,
- enum_auto_big_o_n_lg_n);
+ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
+ rms_o_n_lg_n_test_name, enum_auto_big_o_n_lg_n);
// Add lambda tests
-ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name,
- lambda_big_o_n_lg_n);
+ADD_COMPLEXITY_CASES(n_lg_n_test_name, big_o_n_lg_n_test_name,
+ rms_o_n_lg_n_test_name, lambda_big_o_n_lg_n);
// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
diff --git a/lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc b/lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc
new file mode 100644
index 00000000000..3c36d3f03c1
--- /dev/null
+++ b/lib/libcxx/utils/google-benchmark/test/display_aggregates_only_test.cc
@@ -0,0 +1,43 @@
+
+#undef NDEBUG
+#include <cstdio>
+#include <string>
+
+#include "benchmark/benchmark.h"
+#include "output_test.h"
+
+// Ok this test is super ugly. We want to check what happens with the file
+// reporter in the presence of DisplayAggregatesOnly().
+// We do not care about console output, the normal tests check that already.
+
+void BM_SummaryRepeat(benchmark::State& state) {
+ for (auto _ : state) {
+ }
+}
+BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->DisplayAggregatesOnly();
+
+int main(int argc, char* argv[]) {
+ const std::string output = GetFileReporterOutput(argc, argv);
+
+ if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 6 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3\"") != 3 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") !=
+ 1 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") !=
+ 1) {
+ std::cout << "Precondition mismatch. Expected to only find 6 "
+ "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n"
+ "\"name\": \"BM_SummaryRepeat/repeats:3\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3_median\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire "
+ "output:\n";
+ std::cout << output;
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc b/lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc
new file mode 100644
index 00000000000..94be6083795
--- /dev/null
+++ b/lib/libcxx/utils/google-benchmark/test/memory_manager_test.cc
@@ -0,0 +1,42 @@
+#include <memory>
+
+#include "../src/check.h"
+#include "benchmark/benchmark.h"
+#include "output_test.h"
+
+class TestMemoryManager : public benchmark::MemoryManager {
+ void Start() {}
+ void Stop(Result* result) {
+ result->num_allocs = 42;
+ result->max_bytes_used = 42000;
+ }
+};
+
+void BM_empty(benchmark::State& state) {
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(state.iterations());
+ }
+}
+BENCHMARK(BM_empty);
+
+ADD_CASES(TC_ConsoleOut, {{"^BM_empty %console_report$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_empty\",$"},
+ {"\"run_name\": \"BM_empty\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"allocs_per_iter\": %float,$", MR_Next},
+ {"\"max_bytes_used\": 42000$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_CSVOut, {{"^\"BM_empty\",%csv_report$"}});
+
+
+int main(int argc, char *argv[]) {
+ std::unique_ptr<benchmark::MemoryManager> mm(new TestMemoryManager());
+
+ benchmark::RegisterMemoryManager(mm.get());
+ RunOutputTests(argc, argv);
+ benchmark::RegisterMemoryManager(nullptr);
+}
diff --git a/lib/libcxx/utils/google-benchmark/test/output_test.h b/lib/libcxx/utils/google-benchmark/test/output_test.h
index 31a919991f7..9385761b214 100644
--- a/lib/libcxx/utils/google-benchmark/test/output_test.h
+++ b/lib/libcxx/utils/google-benchmark/test/output_test.h
@@ -60,6 +60,13 @@ int SetSubstitutions(
// Run all output tests.
void RunOutputTests(int argc, char* argv[]);
+// Count the number of 'pat' substrings in the 'haystack' string.
+int SubstrCnt(const std::string& haystack, const std::string& pat);
+
+// Run registered benchmarks with file reporter enabled, and return the content
+// outputted by the file reporter.
+std::string GetFileReporterOutput(int argc, char* argv[]);
+
// ========================================================================= //
// ------------------------- Results checking ------------------------------ //
// ========================================================================= //
diff --git a/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc b/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc
index 394c4f5d1a2..5dc951d2bca 100644
--- a/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc
+++ b/lib/libcxx/utils/google-benchmark/test/output_test_helper.cc
@@ -1,8 +1,12 @@
+#include <cstdio>
#include <cstring>
+#include <fstream>
#include <iostream>
#include <map>
#include <memory>
+#include <random>
#include <sstream>
+#include <streambuf>
#include "../src/benchmark_api_internal.h"
#include "../src/check.h" // NOTE: check.h is for internal use only!
@@ -35,15 +39,18 @@ SubMap& GetSubstitutions() {
// Don't use 'dec_re' from header because it may not yet be initialized.
// clang-format off
static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
+ static std::string time_re = "([0-9]+[.])?[0-9]+";
static SubMap map = {
{"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
// human-readable float
{"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
{"%int", "[ ]*[0-9]+"},
{" %s ", "[ ]+"},
- {"%time", "[ ]*[0-9]{1,6} ns"},
- {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"},
- {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
+ {"%time", "[ ]*" + time_re + "[ ]+ns"},
+ {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"},
+ {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"},
+ {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"},
+ {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"},
{"%csv_header",
"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
"items_per_second,label,error_occurred,error_message"},
@@ -202,7 +209,7 @@ void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
void ResultsChecker::CheckResults(std::stringstream& output) {
// first reset the stream to the start
{
- auto start = std::ios::streampos(0);
+ auto start = std::stringstream::pos_type(0);
// clear before calling tellg()
output.clear();
// seek to zero only when needed
@@ -423,3 +430,76 @@ void RunOutputTests(int argc, char* argv[]) {
CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
internal::GetResultsChecker().CheckResults(csv.out_stream);
}
+
+int SubstrCnt(const std::string& haystack, const std::string& pat) {
+ if (pat.length() == 0) return 0;
+ int count = 0;
+ for (size_t offset = haystack.find(pat); offset != std::string::npos;
+ offset = haystack.find(pat, offset + pat.length()))
+ ++count;
+ return count;
+}
+
+static char ToHex(int ch) {
+ return ch < 10 ? static_cast<char>('0' + ch)
+ : static_cast<char>('a' + (ch - 10));
+}
+
+static char RandomHexChar() {
+ static std::mt19937 rd{std::random_device{}()};
+ static std::uniform_int_distribution<int> mrand{0, 15};
+ return ToHex(mrand(rd));
+}
+
+static std::string GetRandomFileName() {
+ std::string model = "test.%%%%%%";
+ for (auto & ch : model) {
+ if (ch == '%')
+ ch = RandomHexChar();
+ }
+ return model;
+}
+
+static bool FileExists(std::string const& name) {
+ std::ifstream in(name.c_str());
+ return in.good();
+}
+
+static std::string GetTempFileName() {
+ // This function attempts to avoid race conditions where two tests
+ // create the same file at the same time. However, it still introduces races
+ // similar to tmpnam.
+ int retries = 3;
+ while (--retries) {
+ std::string name = GetRandomFileName();
+ if (!FileExists(name))
+ return name;
+ }
+ std::cerr << "Failed to create unique temporary file name" << std::endl;
+ std::abort();
+}
+
+std::string GetFileReporterOutput(int argc, char* argv[]) {
+ std::vector<char*> new_argv(argv, argv + argc);
+ assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size());
+
+ std::string tmp_file_name = GetTempFileName();
+ std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n';
+
+ std::string tmp = "--benchmark_out=";
+ tmp += tmp_file_name;
+ new_argv.emplace_back(const_cast<char*>(tmp.c_str()));
+
+ argc = int(new_argv.size());
+
+ benchmark::Initialize(&argc, new_argv.data());
+ benchmark::RunSpecifiedBenchmarks();
+
+ // Read the output back from the file, and delete the file.
+ std::ifstream tmp_stream(tmp_file_name);
+ std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)),
+ std::istreambuf_iterator<char>());
+ std::remove(tmp_file_name.c_str());
+
+ return output;
+}
diff --git a/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc b/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc
index 18de6d68e21..3ac5b21fb34 100644
--- a/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc
+++ b/lib/libcxx/utils/google-benchmark/test/register_benchmark_test.cc
@@ -30,8 +30,8 @@ struct TestCase {
void CheckRun(Run const& run) const {
// clang-format off
- CHECK(name == run.benchmark_name) << "expected " << name << " got "
- << run.benchmark_name;
+ CHECK(name == run.benchmark_name()) << "expected " << name << " got "
+ << run.benchmark_name();
if (label) {
CHECK(run.report_label == label) << "expected " << label << " got "
<< run.report_label;
diff --git a/lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc b/lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc
new file mode 100644
index 00000000000..9646b9be534
--- /dev/null
+++ b/lib/libcxx/utils/google-benchmark/test/report_aggregates_only_test.cc
@@ -0,0 +1,39 @@
+
+#undef NDEBUG
+#include <cstdio>
+#include <string>
+
+#include "benchmark/benchmark.h"
+#include "output_test.h"
+
+// Ok this test is super ugly. We want to check what happens with the file
+// reporter in the presence of ReportAggregatesOnly().
+// We do not care about console output, the normal tests check that already.
+
+void BM_SummaryRepeat(benchmark::State& state) {
+ for (auto _ : state) {
+ }
+}
+BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
+
+int main(int argc, char* argv[]) {
+ const std::string output = GetFileReporterOutput(argc, argv);
+
+ if (SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3") != 3 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_mean\"") != 1 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_median\"") !=
+ 1 ||
+ SubstrCnt(output, "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"") !=
+ 1) {
+ std::cout << "Precondition mismatch. Expected to only find three "
+ "occurrences of \"BM_SummaryRepeat/repeats:3\" substring:\n"
+ "\"name\": \"BM_SummaryRepeat/repeats:3_mean\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3_median\", "
+ "\"name\": \"BM_SummaryRepeat/repeats:3_stddev\"\nThe entire "
+ "output:\n";
+ std::cout << output;
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc b/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc
index 1662fcb8b54..ec6d51b3591 100644
--- a/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc
+++ b/lib/libcxx/utils/google-benchmark/test/reporter_output_test.cc
@@ -17,18 +17,21 @@ static int AddContextCases() {
{
{"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default},
{"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
- {"Run on \\(%int X %float MHz CPU s\\)", MR_Next},
+ {"Run on \\(%int X %float MHz CPU s?\\)", MR_Next},
});
AddCases(TC_JSONOut,
{{"^\\{", MR_Default},
{"\"context\":", MR_Next},
{"\"date\": \"", MR_Next},
- {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next},
+ {"\"host_name\":", MR_Next},
+ {"\"executable\": \".*(/|\\\\)reporter_output_test(\\.exe)?\",",
+ MR_Next},
{"\"num_cpus\": %int,$", MR_Next},
{"\"mhz_per_cpu\": %float,$", MR_Next},
{"\"cpu_scaling_enabled\": ", MR_Next},
{"\"caches\": \\[$", MR_Next}});
- auto const& Caches = benchmark::CPUInfo::Get().caches;
+ auto const& Info = benchmark::CPUInfo::Get();
+ auto const& Caches = Info.caches;
if (!Caches.empty()) {
AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
}
@@ -45,8 +48,13 @@ static int AddContextCases() {
{"\"num_sharing\": %int$", MR_Next},
{"}[,]{0,1}$", MR_Next}});
}
-
AddCases(TC_JSONOut, {{"],$"}});
+ auto const& LoadAvg = Info.load_avg;
+ if (!LoadAvg.empty()) {
+ AddCases(TC_ConsoleErr,
+ {{"Load Average: (%float, ){0,2}%float$", MR_Next}});
+ }
+ AddCases(TC_JSONOut, {{"\"load_avg\": \\[(%float,?){0,3}],$", MR_Next}});
return 0;
}
int dummy_register = AddContextCases();
@@ -64,6 +72,8 @@ BENCHMARK(BM_basic);
ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
+ {"\"run_name\": \"BM_basic\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -82,9 +92,11 @@ void BM_bytes_per_second(benchmark::State& state) {
}
BENCHMARK(BM_bytes_per_second);
-ADD_CASES(TC_ConsoleOut,
- {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_bytes_per_second %console_report "
+ "bytes_per_second=%float[kM]{0,1}/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
+ {"\"run_name\": \"BM_bytes_per_second\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -104,9 +116,11 @@ void BM_items_per_second(benchmark::State& state) {
}
BENCHMARK(BM_items_per_second);
-ADD_CASES(TC_ConsoleOut,
- {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_items_per_second %console_report "
+ "items_per_second=%float[kM]{0,1}/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
+ {"\"run_name\": \"BM_items_per_second\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -128,6 +142,8 @@ BENCHMARK(BM_label);
ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
+ {"\"run_name\": \"BM_label\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -149,6 +165,8 @@ void BM_error(benchmark::State& state) {
BENCHMARK(BM_error);
ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
+ {"\"run_name\": \"BM_error\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"error_occurred\": true,$", MR_Next},
{"\"error_message\": \"message\",$", MR_Next}});
@@ -165,7 +183,9 @@ void BM_no_arg_name(benchmark::State& state) {
}
BENCHMARK(BM_no_arg_name)->Arg(3);
ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"},
+ {"\"run_name\": \"BM_no_arg_name/3\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
// ========================================================================= //
@@ -178,7 +198,9 @@ void BM_arg_name(benchmark::State& state) {
}
BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"},
+ {"\"run_name\": \"BM_arg_name/first:3\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
// ========================================================================= //
@@ -192,10 +214,25 @@ void BM_arg_names(benchmark::State& state) {
BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
ADD_CASES(TC_ConsoleOut,
{{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"},
+ {"\"run_name\": \"BM_arg_names/first:2/5/third:4\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
// ========================================================================= //
+// ------------------------ Testing Big Args Output ------------------------ //
+// ========================================================================= //
+
+void BM_BigArgs(benchmark::State& state) {
+ for (auto _ : state) {
+ }
+}
+BENCHMARK(BM_BigArgs)->RangeMultiplier(2)->Range(1U << 30U, 1U << 31U);
+ADD_CASES(TC_ConsoleOut, {{"^BM_BigArgs/1073741824 %console_report$"},
+ {"^BM_BigArgs/2147483648 %console_report$"}});
+
+// ========================================================================= //
// ----------------------- Testing Complexity Output ----------------------- //
// ========================================================================= //
@@ -221,16 +258,33 @@ void BM_Repeat(benchmark::State& state) {
}
// need two repetitions min to be able to output any aggregate output
BENCHMARK(BM_Repeat)->Repetitions(2);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
- {"^BM_Repeat/repeats:2 %console_report$"},
- {"^BM_Repeat/repeats:2_mean %console_report$"},
- {"^BM_Repeat/repeats:2_median %console_report$"},
- {"^BM_Repeat/repeats:2_stddev %console_report$"}});
+ADD_CASES(TC_ConsoleOut,
+ {{"^BM_Repeat/repeats:2 %console_report$"},
+ {"^BM_Repeat/repeats:2 %console_report$"},
+ {"^BM_Repeat/repeats:2_mean %console_time_only_report [ ]*2$"},
+ {"^BM_Repeat/repeats:2_median %console_time_only_report [ ]*2$"},
+ {"^BM_Repeat/repeats:2_stddev %console_time_only_report [ ]*2$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:2\"", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:2_median\",$"},
- {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}});
+ {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
+ {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
@@ -238,18 +292,37 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
{"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
// but for two repetitions, mean and median is the same, so let's repeat..
BENCHMARK(BM_Repeat)->Repetitions(3);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
- {"^BM_Repeat/repeats:3 %console_report$"},
- {"^BM_Repeat/repeats:3 %console_report$"},
- {"^BM_Repeat/repeats:3_mean %console_report$"},
- {"^BM_Repeat/repeats:3_median %console_report$"},
- {"^BM_Repeat/repeats:3_stddev %console_report$"}});
+ADD_CASES(TC_ConsoleOut,
+ {{"^BM_Repeat/repeats:3 %console_report$"},
+ {"^BM_Repeat/repeats:3 %console_report$"},
+ {"^BM_Repeat/repeats:3 %console_report$"},
+ {"^BM_Repeat/repeats:3_mean %console_time_only_report [ ]*3$"},
+ {"^BM_Repeat/repeats:3_median %console_time_only_report [ ]*3$"},
+ {"^BM_Repeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:3_median\",$"},
- {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
+ {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
@@ -258,20 +331,41 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
// median differs between even/odd number of repetitions, so just to be sure
BENCHMARK(BM_Repeat)->Repetitions(4);
-ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
- {"^BM_Repeat/repeats:4 %console_report$"},
- {"^BM_Repeat/repeats:4 %console_report$"},
- {"^BM_Repeat/repeats:4 %console_report$"},
- {"^BM_Repeat/repeats:4_mean %console_report$"},
- {"^BM_Repeat/repeats:4_median %console_report$"},
- {"^BM_Repeat/repeats:4_stddev %console_report$"}});
+ADD_CASES(TC_ConsoleOut,
+ {{"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4_mean %console_time_only_report [ ]*4$"},
+ {"^BM_Repeat/repeats:4_median %console_time_only_report [ ]*4$"},
+ {"^BM_Repeat/repeats:4_stddev %console_time_only_report [ ]*4$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 4,$", MR_Next},
{"\"name\": \"BM_Repeat/repeats:4_median\",$"},
- {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}});
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 4,$", MR_Next},
+ {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"},
+ {"\"run_name\": \"BM_Repeat/repeats:4\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 4,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"},
{"^\"BM_Repeat/repeats:4\",%csv_report$"},
@@ -288,7 +382,9 @@ void BM_RepeatOnce(benchmark::State& state) {
}
BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"},
+ {"\"run_name\": \"BM_RepeatOnce/repeats:1\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
// Test that non-aggregate data is not reported
@@ -297,20 +393,72 @@ void BM_SummaryRepeat(benchmark::State& state) {
}
}
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
-ADD_CASES(TC_ConsoleOut,
+ADD_CASES(
+ TC_ConsoleOut,
+ {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
+ {"^BM_SummaryRepeat/repeats:3_mean %console_time_only_report [ ]*3$"},
+ {"^BM_SummaryRepeat/repeats:3_median %console_time_only_report [ ]*3$"},
+ {"^BM_SummaryRepeat/repeats:3_stddev %console_time_only_report [ ]*3$"}});
+ADD_CASES(TC_JSONOut,
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
- {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
- {"^BM_SummaryRepeat/repeats:3_median %console_report$"},
- {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
-ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
- {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
- {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
- {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
+ {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
+ {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
+ {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"},
+ {"\"run_name\": \"BM_SummaryRepeat/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
+// Test that non-aggregate data is not displayed.
+// NOTE: this test is kinda bad. we are only testing the display output.
+// But we don't check that the file output still contains everything...
+void BM_SummaryDisplay(benchmark::State& state) {
+ for (auto _ : state) {
+ }
+}
+BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
+ADD_CASES(
+ TC_ConsoleOut,
+ {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
+ {"^BM_SummaryDisplay/repeats:2_mean %console_time_only_report [ ]*2$"},
+ {"^BM_SummaryDisplay/repeats:2_median %console_time_only_report [ ]*2$"},
+ {"^BM_SummaryDisplay/repeats:2_stddev %console_time_only_report [ ]*2$"}});
+ADD_CASES(TC_JSONOut,
+ {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
+ {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
+ {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
+ {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
+ {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
+ {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
+ {"\"run_name\": \"BM_SummaryDisplay/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next}});
+ADD_CASES(TC_CSVOut,
+ {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
+ {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
+ {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
+ {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
+
+// Test repeats with custom time unit.
void BM_RepeatTimeUnit(benchmark::State& state) {
for (auto _ : state) {
}
@@ -319,18 +467,34 @@ BENCHMARK(BM_RepeatTimeUnit)
->Repetitions(3)
->ReportAggregatesOnly()
->Unit(benchmark::kMicrosecond);
-ADD_CASES(TC_ConsoleOut,
+ADD_CASES(
+ TC_ConsoleOut,
+ {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
+ {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_time_only_report [ ]*3$"},
+ {"^BM_RepeatTimeUnit/repeats:3_median %console_us_time_only_report [ "
+ "]*3$"},
+ {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_time_only_report [ "
+ "]*3$"}});
+ADD_CASES(TC_JSONOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
- {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
- {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"},
- {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
-ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
- {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
- {"\"time_unit\": \"us\",?$"},
- {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
- {"\"time_unit\": \"us\",?$"},
- {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
- {"\"time_unit\": \"us\",?$"}});
+ {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
+ {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"time_unit\": \"us\",?$"},
+ {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
+ {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"time_unit\": \"us\",?$"},
+ {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
+ {"\"run_name\": \"BM_RepeatTimeUnit/repeats:3\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"time_unit\": \"us\",?$"}});
ADD_CASES(TC_CSVOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
@@ -346,37 +510,92 @@ const auto UserStatistics = [](const std::vector<double>& v) {
};
void BM_UserStats(benchmark::State& state) {
for (auto _ : state) {
+ state.SetIterationTime(150 / 10e8);
}
}
// clang-format off
BENCHMARK(BM_UserStats)
->Repetitions(3)
+ ->Iterations(5)
+ ->UseManualTime()
->ComputeStatistics("", UserStatistics);
// clang-format on
// check that user-provided stats is calculated, and is after the default-ones
// empty string as name is intentional, it would sort before anything else
-ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
- {"^BM_UserStats/repeats:3 %console_report$"},
- {"^BM_UserStats/repeats:3 %console_report$"},
- {"^BM_UserStats/repeats:3_mean %console_report$"},
- {"^BM_UserStats/repeats:3_median %console_report$"},
- {"^BM_UserStats/repeats:3_stddev %console_report$"},
- {"^BM_UserStats/repeats:3_ %console_report$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
- {"\"name\": \"BM_UserStats/repeats:3\",$"},
- {"\"name\": \"BM_UserStats/repeats:3\",$"},
- {"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
- {"\"name\": \"BM_UserStats/repeats:3_median\",$"},
- {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
- {"\"name\": \"BM_UserStats/repeats:3_\",$"}});
-ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
- {"^\"BM_UserStats/repeats:3\",%csv_report$"},
- {"^\"BM_UserStats/repeats:3\",%csv_report$"},
- {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"},
- {"^\"BM_UserStats/repeats:3_median\",%csv_report$"},
- {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
- {"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
+ "]* 150 ns %time [ ]*5$"},
+ {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
+ "]* 150 ns %time [ ]*5$"},
+ {"^BM_UserStats/iterations:5/repeats:3/manual_time [ "
+ "]* 150 ns %time [ ]*5$"},
+ {"^BM_UserStats/iterations:5/repeats:3/"
+ "manual_time_mean [ ]* 150 ns %time [ ]*3$"},
+ {"^BM_UserStats/iterations:5/repeats:3/"
+ "manual_time_median [ ]* 150 ns %time [ ]*3$"},
+ {"^BM_UserStats/iterations:5/repeats:3/"
+ "manual_time_stddev [ ]* 0.000 ns %time [ ]*3$"},
+ {"^BM_UserStats/iterations:5/repeats:3/manual_time_ "
+ "[ ]* 150 ns %time [ ]*3$"}});
+ADD_CASES(
+ TC_JSONOut,
+ {{"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": 5,$", MR_Next},
+ {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
+ {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": 5,$", MR_Next},
+ {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
+ {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": 5,$", MR_Next},
+ {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
+ {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
+ {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_median\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next},
+ {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_stddev\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"name\": \"BM_UserStats/iterations:5/repeats:3/manual_time_\",$"},
+ {"\"run_name\": \"BM_UserStats/iterations:5/repeats:3/manual_time\",$",
+ MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"\",$", MR_Next},
+ {"\"iterations\": 3,$", MR_Next},
+ {"\"real_time\": 1\\.5(0)*e\\+(0)*2,$", MR_Next}});
+ADD_CASES(
+ TC_CSVOut,
+ {{"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
+ {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
+ {"^\"BM_UserStats/iterations:5/repeats:3/manual_time\",%csv_report$"},
+ {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_mean\",%csv_report$"},
+ {"^\"BM_UserStats/iterations:5/repeats:3/"
+ "manual_time_median\",%csv_report$"},
+ {"^\"BM_UserStats/iterations:5/repeats:3/"
+ "manual_time_stddev\",%csv_report$"},
+ {"^\"BM_UserStats/iterations:5/repeats:3/manual_time_\",%csv_report$"}});
// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
diff --git a/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc b/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc
index 39785fb7f6d..06579772ff7 100644
--- a/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc
+++ b/lib/libcxx/utils/google-benchmark/test/skip_with_error_test.cc
@@ -33,8 +33,8 @@ struct TestCase {
typedef benchmark::BenchmarkReporter::Run Run;
void CheckRun(Run const& run) const {
- CHECK(name == run.benchmark_name)
- << "expected " << name << " got " << run.benchmark_name;
+ CHECK(name == run.benchmark_name())
+ << "expected " << name << " got " << run.benchmark_name();
CHECK(error_occurred == run.error_occurred);
CHECK(error_message == run.error_message);
if (error_occurred) {
diff --git a/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc b/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc
index 4c81734cf8a..2c5d073f613 100644
--- a/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc
+++ b/lib/libcxx/utils/google-benchmark/test/string_util_gtest.cc
@@ -9,56 +9,56 @@ namespace {
TEST(StringUtilTest, stoul) {
{
size_t pos = 0;
- EXPECT_EQ(0, benchmark::stoul("0", &pos));
- EXPECT_EQ(1, pos);
+ EXPECT_EQ(0ul, benchmark::stoul("0", &pos));
+ EXPECT_EQ(1ul, pos);
}
{
size_t pos = 0;
- EXPECT_EQ(7, benchmark::stoul("7", &pos));
- EXPECT_EQ(1, pos);
+ EXPECT_EQ(7ul, benchmark::stoul("7", &pos));
+ EXPECT_EQ(1ul, pos);
}
{
size_t pos = 0;
- EXPECT_EQ(135, benchmark::stoul("135", &pos));
- EXPECT_EQ(3, pos);
+ EXPECT_EQ(135ul, benchmark::stoul("135", &pos));
+ EXPECT_EQ(3ul, pos);
}
#if ULONG_MAX == 0xFFFFFFFFul
{
size_t pos = 0;
EXPECT_EQ(0xFFFFFFFFul, benchmark::stoul("4294967295", &pos));
- EXPECT_EQ(10, pos);
+ EXPECT_EQ(10ul, pos);
}
#elif ULONG_MAX == 0xFFFFFFFFFFFFFFFFul
{
size_t pos = 0;
EXPECT_EQ(0xFFFFFFFFFFFFFFFFul, benchmark::stoul("18446744073709551615", &pos));
- EXPECT_EQ(20, pos);
+ EXPECT_EQ(20ul, pos);
}
#endif
{
size_t pos = 0;
- EXPECT_EQ(10, benchmark::stoul("1010", &pos, 2));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(10ul, benchmark::stoul("1010", &pos, 2));
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
- EXPECT_EQ(520, benchmark::stoul("1010", &pos, 8));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(520ul, benchmark::stoul("1010", &pos, 8));
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
- EXPECT_EQ(1010, benchmark::stoul("1010", &pos, 10));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(1010ul, benchmark::stoul("1010", &pos, 10));
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
- EXPECT_EQ(4112, benchmark::stoul("1010", &pos, 16));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4112ul, benchmark::stoul("1010", &pos, 16));
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
- EXPECT_EQ(0xBEEF, benchmark::stoul("BEEF", &pos, 16));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(0xBEEFul, benchmark::stoul("BEEF", &pos, 16));
+ EXPECT_EQ(4ul, pos);
}
{
ASSERT_THROW(benchmark::stoul("this is a test"), std::invalid_argument);
@@ -69,42 +69,42 @@ TEST(StringUtilTest, stoi) {
{
size_t pos = 0;
EXPECT_EQ(0, benchmark::stoi("0", &pos));
- EXPECT_EQ(1, pos);
+ EXPECT_EQ(1ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(-17, benchmark::stoi("-17", &pos));
- EXPECT_EQ(3, pos);
+ EXPECT_EQ(3ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(1357, benchmark::stoi("1357", &pos));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(10, benchmark::stoi("1010", &pos, 2));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(520, benchmark::stoi("1010", &pos, 8));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(1010, benchmark::stoi("1010", &pos, 10));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(4112, benchmark::stoi("1010", &pos, 16));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(0xBEEF, benchmark::stoi("BEEF", &pos, 16));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
ASSERT_THROW(benchmark::stoi("this is a test"), std::invalid_argument);
@@ -115,28 +115,28 @@ TEST(StringUtilTest, stod) {
{
size_t pos = 0;
EXPECT_EQ(0.0, benchmark::stod("0", &pos));
- EXPECT_EQ(1, pos);
+ EXPECT_EQ(1ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(-84.0, benchmark::stod("-84", &pos));
- EXPECT_EQ(3, pos);
+ EXPECT_EQ(3ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(1234.0, benchmark::stod("1234", &pos));
- EXPECT_EQ(4, pos);
+ EXPECT_EQ(4ul, pos);
}
{
size_t pos = 0;
EXPECT_EQ(1.5, benchmark::stod("1.5", &pos));
- EXPECT_EQ(3, pos);
+ EXPECT_EQ(3ul, pos);
}
{
size_t pos = 0;
/* Note: exactly representable as double */
EXPECT_EQ(-1.25e+9, benchmark::stod("-1.25e+9", &pos));
- EXPECT_EQ(8, pos);
+ EXPECT_EQ(8ul, pos);
}
{
ASSERT_THROW(benchmark::stod("this is a test"), std::invalid_argument);
diff --git a/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc b/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc
index 4f126b6d978..030e98916c3 100644
--- a/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc
+++ b/lib/libcxx/utils/google-benchmark/test/user_counters_tabular_test.cc
@@ -69,18 +69,21 @@ void BM_Counters_Tabular(benchmark::State& state) {
});
}
BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"Bar\": %float,$", MR_Next},
- {"\"Bat\": %float,$", MR_Next},
- {"\"Baz\": %float,$", MR_Next},
- {"\"Foo\": %float,$", MR_Next},
- {"\"Frob\": %float,$", MR_Next},
- {"\"Lob\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
+ {"\"run_name\": \"BM_Counters_Tabular/threads:%int\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"Bar\": %float,$", MR_Next},
+ {"\"Bat\": %float,$", MR_Next},
+ {"\"Baz\": %float,$", MR_Next},
+ {"\"Foo\": %float,$", MR_Next},
+ {"\"Frob\": %float,$", MR_Next},
+ {"\"Lob\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -113,18 +116,22 @@ void BM_CounterRates_Tabular(benchmark::State& state) {
});
}
BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"Bar\": %float,$", MR_Next},
- {"\"Bat\": %float,$", MR_Next},
- {"\"Baz\": %float,$", MR_Next},
- {"\"Foo\": %float,$", MR_Next},
- {"\"Frob\": %float,$", MR_Next},
- {"\"Lob\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
+ {"\"run_name\": \"BM_CounterRates_Tabular/threads:%int\",$",
+ MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"Bar\": %float,$", MR_Next},
+ {"\"Bat\": %float,$", MR_Next},
+ {"\"Baz\": %float,$", MR_Next},
+ {"\"Foo\": %float,$", MR_Next},
+ {"\"Frob\": %float,$", MR_Next},
+ {"\"Lob\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
"%float,%float,%float,%float,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -157,15 +164,18 @@ void BM_CounterSet0_Tabular(benchmark::State& state) {
});
}
BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"Bar\": %float,$", MR_Next},
- {"\"Baz\": %float,$", MR_Next},
- {"\"Foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
+ {"\"run_name\": \"BM_CounterSet0_Tabular/threads:%int\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"Bar\": %float,$", MR_Next},
+ {"\"Baz\": %float,$", MR_Next},
+ {"\"Foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
"%float,,%float,%float,,"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -189,15 +199,18 @@ void BM_CounterSet1_Tabular(benchmark::State& state) {
});
}
BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"Bar\": %float,$", MR_Next},
- {"\"Baz\": %float,$", MR_Next},
- {"\"Foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
+ {"\"run_name\": \"BM_CounterSet1_Tabular/threads:%int\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"Bar\": %float,$", MR_Next},
+ {"\"Baz\": %float,$", MR_Next},
+ {"\"Foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
"%float,,%float,%float,,"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -225,15 +238,18 @@ void BM_CounterSet2_Tabular(benchmark::State& state) {
});
}
BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"Bat\": %float,$", MR_Next},
- {"\"Baz\": %float,$", MR_Next},
- {"\"Foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
+ {"\"run_name\": \"BM_CounterSet2_Tabular/threads:%int\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"Bat\": %float,$", MR_Next},
+ {"\"Baz\": %float,$", MR_Next},
+ {"\"Foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
",%float,%float,%float,,"}});
// VS2013 does not allow this function to be passed as a lambda argument
diff --git a/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc b/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc
index 7f7ccb9f77a..bb0d6b4c5a9 100644
--- a/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc
+++ b/lib/libcxx/utils/google-benchmark/test/user_counters_test.cc
@@ -32,6 +32,8 @@ BENCHMARK(BM_Counters_Simple);
ADD_CASES(TC_ConsoleOut,
{{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"},
+ {"\"run_name\": \"BM_Counters_Simple\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -66,19 +68,22 @@ void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) {
state.SetItemsProcessed(150);
}
BENCHMARK(BM_Counters_WithBytesAndItemsPSec);
-ADD_CASES(TC_ConsoleOut,
- {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
- "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bytes_per_second\": %float,$", MR_Next},
- {"\"items_per_second\": %float,$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_WithBytesAndItemsPSec %console_report "
+ "bar=%hrfloat bytes_per_second=%hrfloat/s "
+ "foo=%hrfloat items_per_second=%hrfloat/s$"}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"},
+ {"\"run_name\": \"BM_Counters_WithBytesAndItemsPSec\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"bytes_per_second\": %float,$", MR_Next},
+ {"\"foo\": %float,$", MR_Next},
+ {"\"items_per_second\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\","
"%csv_bytes_items_report,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -110,6 +115,8 @@ ADD_CASES(
TC_ConsoleOut,
{{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"},
+ {"\"run_name\": \"BM_Counters_Rate\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -141,14 +148,17 @@ void BM_Counters_Threads(benchmark::State& state) {
BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report "
"bar=%hrfloat foo=%hrfloat$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"},
+ {"\"run_name\": \"BM_Counters_Threads/threads:%int\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(
TC_CSVOut,
{{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}});
@@ -174,14 +184,17 @@ void BM_Counters_AvgThreads(benchmark::State& state) {
BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int "
"%console_report bar=%hrfloat foo=%hrfloat$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"},
+ {"\"run_name\": \"BM_Counters_AvgThreads/threads:%int\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(
TC_CSVOut,
{{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}});
@@ -210,6 +223,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"},
+ {"\"run_name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$",
+ MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -242,14 +258,17 @@ void BM_Counters_IterationInvariant(benchmark::State& state) {
BENCHMARK(BM_Counters_IterationInvariant);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_IterationInvariant %console_report "
"bar=%hrfloat foo=%hrfloat$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_IterationInvariant\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_IterationInvariant\",$"},
+ {"\"run_name\": \"BM_Counters_IterationInvariant\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_IterationInvariant\",%csv_report,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -281,6 +300,9 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kIsIterationInvariantRate "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
ADD_CASES(TC_JSONOut,
{{"\"name\": \"BM_Counters_kIsIterationInvariantRate\",$"},
+ {"\"run_name\": \"BM_Counters_kIsIterationInvariantRate\",$",
+ MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
{"\"iterations\": %int,$", MR_Next},
{"\"real_time\": %float,$", MR_Next},
{"\"cpu_time\": %float,$", MR_Next},
@@ -316,14 +338,17 @@ void BM_Counters_AvgIterations(benchmark::State& state) {
BENCHMARK(BM_Counters_AvgIterations);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgIterations %console_report "
"bar=%hrfloat foo=%hrfloat$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgIterations\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_AvgIterations\",$"},
+ {"\"run_name\": \"BM_Counters_AvgIterations\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut,
{{"^\"BM_Counters_AvgIterations\",%csv_report,%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument
@@ -351,14 +376,17 @@ void BM_Counters_kAvgIterationsRate(benchmark::State& state) {
BENCHMARK(BM_Counters_kAvgIterationsRate);
ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_kAvgIterationsRate "
"%console_report bar=%hrfloat/s foo=%hrfloat/s$"}});
-ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
- {"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %float,$", MR_Next},
- {"\"cpu_time\": %float,$", MR_Next},
- {"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bar\": %float,$", MR_Next},
- {"\"foo\": %float$", MR_Next},
- {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_kAvgIterationsRate\",$"},
+ {"\"run_name\": \"BM_Counters_kAvgIterationsRate\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"bar\": %float,$", MR_Next},
+ {"\"foo\": %float$", MR_Next},
+ {"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_kAvgIterationsRate\",%csv_report,"
"%float,%float$"}});
// VS2013 does not allow this function to be passed as a lambda argument
diff --git a/lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc b/lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc
new file mode 100644
index 00000000000..fa0ef972047
--- /dev/null
+++ b/lib/libcxx/utils/google-benchmark/test/user_counters_thousands_test.cc
@@ -0,0 +1,161 @@
+
+#undef NDEBUG
+
+#include "benchmark/benchmark.h"
+#include "output_test.h"
+
+// ========================================================================= //
+// ------------------------ Thousands Customisation ------------------------ //
+// ========================================================================= //
+
+void BM_Counters_Thousands(benchmark::State& state) {
+ for (auto _ : state) {
+ }
+ namespace bm = benchmark;
+ state.counters.insert({
+ {"t0_1000000DefaultBase",
+ bm::Counter(1000 * 1000, bm::Counter::kDefaults)},
+ {"t1_1000000Base1000", bm::Counter(1000 * 1000, bm::Counter::kDefaults,
+ benchmark::Counter::OneK::kIs1000)},
+ {"t2_1000000Base1024", bm::Counter(1000 * 1000, bm::Counter::kDefaults,
+ benchmark::Counter::OneK::kIs1024)},
+ {"t3_1048576Base1000", bm::Counter(1024 * 1024, bm::Counter::kDefaults,
+ benchmark::Counter::OneK::kIs1000)},
+ {"t4_1048576Base1024", bm::Counter(1024 * 1024, bm::Counter::kDefaults,
+ benchmark::Counter::OneK::kIs1024)},
+ });
+}
+BENCHMARK(BM_Counters_Thousands)->Repetitions(2);
+ADD_CASES(
+ TC_ConsoleOut,
+ {
+ {"^BM_Counters_Thousands/repeats:2 %console_report "
+ "t0_1000000DefaultBase=1000k "
+ "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k "
+ "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"},
+ {"^BM_Counters_Thousands/repeats:2 %console_report "
+ "t0_1000000DefaultBase=1000k "
+ "t1_1000000Base1000=1000k t2_1000000Base1024=976.56[23]k "
+ "t3_1048576Base1000=1048.58k t4_1048576Base1024=1024k$"},
+ {"^BM_Counters_Thousands/repeats:2_mean %console_report "
+ "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k "
+ "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k "
+ "t4_1048576Base1024=1024k$"},
+ {"^BM_Counters_Thousands/repeats:2_median %console_report "
+ "t0_1000000DefaultBase=1000k t1_1000000Base1000=1000k "
+ "t2_1000000Base1024=976.56[23]k t3_1048576Base1000=1048.58k "
+ "t4_1048576Base1024=1024k$"},
+ {"^BM_Counters_Thousands/repeats:2_stddev %console_time_only_report [ "
+ "]*2 t0_1000000DefaultBase=0 t1_1000000Base1000=0 "
+ "t2_1000000Base1024=0 t3_1048576Base1000=0 t4_1048576Base1024=0$"},
+ });
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
+ {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Thousands/repeats:2\",$"},
+ {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"iteration\",$", MR_Next},
+ {"\"iterations\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Thousands/repeats:2_mean\",$"},
+ {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"mean\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Thousands/repeats:2_median\",$"},
+ {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"median\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"t0_1000000DefaultBase\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t1_1000000Base1000\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t2_1000000Base1024\": 1\\.(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t3_1048576Base1000\": 1\\.048576(0)*e\\+(0)*6,$", MR_Next},
+ {"\"t4_1048576Base1024\": 1\\.048576(0)*e\\+(0)*6$", MR_Next},
+ {"}", MR_Next}});
+ADD_CASES(TC_JSONOut,
+ {{"\"name\": \"BM_Counters_Thousands/repeats:2_stddev\",$"},
+ {"\"run_name\": \"BM_Counters_Thousands/repeats:2\",$", MR_Next},
+ {"\"run_type\": \"aggregate\",$", MR_Next},
+ {"\"aggregate_name\": \"stddev\",$", MR_Next},
+ {"\"iterations\": 2,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
+ {"\"time_unit\": \"ns\",$", MR_Next},
+ {"\"t0_1000000DefaultBase\": 0\\.(0)*e\\+(0)*,$", MR_Next},
+ {"\"t1_1000000Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next},
+ {"\"t2_1000000Base1024\": 0\\.(0)*e\\+(0)*,$", MR_Next},
+ {"\"t3_1048576Base1000\": 0\\.(0)*e\\+(0)*,$", MR_Next},
+ {"\"t4_1048576Base1024\": 0\\.(0)*e\\+(0)*$", MR_Next},
+ {"}", MR_Next}});
+
+ADD_CASES(
+ TC_CSVOut,
+ {{"^\"BM_Counters_Thousands/"
+ "repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+("
+ "0)*6,1\\.04858e\\+(0)*6$"},
+ {"^\"BM_Counters_Thousands/"
+ "repeats:2\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\.04858e\\+("
+ "0)*6,1\\.04858e\\+(0)*6$"},
+ {"^\"BM_Counters_Thousands/"
+ "repeats:2_mean\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\."
+ "04858e\\+(0)*6,1\\.04858e\\+(0)*6$"},
+ {"^\"BM_Counters_Thousands/"
+ "repeats:2_median\",%csv_report,1e\\+(0)*6,1e\\+(0)*6,1e\\+(0)*6,1\\."
+ "04858e\\+(0)*6,1\\.04858e\\+(0)*6$"},
+ {"^\"BM_Counters_Thousands/repeats:2_stddev\",%csv_report,0,0,0,0,0$"}});
+// VS2013 does not allow this function to be passed as a lambda argument
+// to CHECK_BENCHMARK_RESULTS()
+void CheckThousands(Results const& e) {
+ if (e.name != "BM_Counters_Thousands/repeats:2")
+ return; // Do not check the aggregates!
+
+ // check that the values are within 0.01% of the expected values
+ CHECK_FLOAT_COUNTER_VALUE(e, "t0_1000000DefaultBase", EQ, 1000 * 1000,
+ 0.0001);
+ CHECK_FLOAT_COUNTER_VALUE(e, "t1_1000000Base1000", EQ, 1000 * 1000, 0.0001);
+ CHECK_FLOAT_COUNTER_VALUE(e, "t2_1000000Base1024", EQ, 1000 * 1000, 0.0001);
+ CHECK_FLOAT_COUNTER_VALUE(e, "t3_1048576Base1000", EQ, 1024 * 1024, 0.0001);
+ CHECK_FLOAT_COUNTER_VALUE(e, "t4_1048576Base1024", EQ, 1024 * 1024, 0.0001);
+}
+CHECK_BENCHMARK_RESULTS("BM_Counters_Thousands", &CheckThousands);
+
+// ========================================================================= //
+// --------------------------- TEST CASES END ------------------------------ //
+// ========================================================================= //
+
+int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }