diff options
Diffstat (limited to 'tools/testing/selftests/kselftest_harness.h')
-rw-r--r-- | tools/testing/selftests/kselftest_harness.h | 576 |
1 files changed, 439 insertions, 137 deletions
diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index c9f03ef93338..4fd735e48ee7 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -50,17 +50,25 @@ #ifndef __KSELFTEST_HARNESS_H #define __KSELFTEST_HARNESS_H +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include <asm/types.h> +#include <ctype.h> #include <errno.h> +#include <limits.h> #include <stdbool.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <sys/mman.h> #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> +#include <setjmp.h> + +#include "kselftest.h" #define TEST_TIMEOUT_DEFAULT 30 @@ -74,7 +82,7 @@ #endif /** - * TH_LOG(fmt, ...) + * TH_LOG() * * @fmt: format string * @...: optional arguments @@ -88,14 +96,6 @@ * E.g., #define TH_LOG_ENABLED 1 * * If no definition is provided, logging is enabled by default. - * - * If there is no way to print an error message for the process running the - * test (e.g. not allowed to write to stderr), it is still possible to get the - * ASSERT_* number for which the test failed. This behavior can be enabled by - * writing `_metadata->no_print = true;` before the check sequence that is - * unable to print. When an error occur, instead of printing an error message - * and calling `abort(3)`, the test process call `_exit(2)` with the assert - * number as argument, which is then printed by the parent process. */ #define TH_LOG(fmt, ...) do { \ if (TH_LOG_ENABLED) \ @@ -104,32 +104,37 @@ /* Unconditional logger for internal use. */ #define __TH_LOG(fmt, ...) \ - fprintf(TH_LOG_STREAM, "%s:%d:%s:" fmt "\n", \ + fprintf(TH_LOG_STREAM, "# %s:%d:%s:" fmt "\n", \ __FILE__, __LINE__, _metadata->name, ##__VA_ARGS__) /** - * XFAIL(statement, fmt, ...) + * SKIP() * - * @statement: statement to run after reporting XFAIL + * @statement: statement to run after reporting SKIP * @fmt: format string * @...: optional arguments * - * This forces a "pass" after reporting a failure with an XFAIL prefix, + * .. code-block:: c + * + * SKIP(statement, fmt, ...); + * + * This forces a "pass" after reporting why something is being skipped * and runs "statement", which is usually "return" or "goto skip". */ -#define XFAIL(statement, fmt, ...) do { \ +#define SKIP(statement, fmt, ...) do { \ + snprintf(_metadata->results->reason, \ + sizeof(_metadata->results->reason), fmt, ##__VA_ARGS__); \ if (TH_LOG_ENABLED) { \ - fprintf(TH_LOG_STREAM, "[ XFAIL! ] " fmt "\n", \ - ##__VA_ARGS__); \ + fprintf(TH_LOG_STREAM, "# SKIP %s\n", \ + _metadata->results->reason); \ } \ - /* TODO: find a way to pass xfail to test runner process. */ \ - _metadata->passed = 1; \ + _metadata->exit_code = KSFT_SKIP; \ _metadata->trigger = 0; \ statement; \ } while (0) /** - * TEST(test_name) - Defines the test function and creates the registration + * TEST() - Defines the test function and creates the registration * stub * * @test_name: test name @@ -148,7 +153,7 @@ #define TEST(test_name) __TEST_IMPL(test_name, -1) /** - * TEST_SIGNAL(test_name, signal) + * TEST_SIGNAL() * * @test_name: test name * @signal: signal number @@ -172,7 +177,10 @@ struct __test_metadata *_metadata, \ struct __fixture_variant_metadata *variant) \ { \ - test_name(_metadata); \ + _metadata->setup_completed = true; \ + if (setjmp(_metadata->env) == 0) \ + test_name(_metadata); \ + __test_check_assert(_metadata); \ } \ static struct __test_metadata _##test_name##_object = \ { .name = #test_name, \ @@ -188,15 +196,16 @@ struct __test_metadata __attribute__((unused)) *_metadata) /** - * FIXTURE_DATA(datatype_name) - Wraps the struct name so we have one less + * FIXTURE_DATA() - Wraps the struct name so we have one less * argument to pass around * * @datatype_name: datatype name * * .. code-block:: c * - * FIXTURE_DATA(datatype name) + * FIXTURE_DATA(datatype_name) * + * Almost always, you want just FIXTURE() instead (see below). * This call may be used when the type of the fixture data * is needed. In general, this should not be needed unless * the *self* is being passed to a helper directly. @@ -204,14 +213,14 @@ #define FIXTURE_DATA(datatype_name) struct _test_data_##datatype_name /** - * FIXTURE(fixture_name) - Called once per fixture to setup the data and + * FIXTURE() - Called once per fixture to setup the data and * register * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE(datatype name) { + * FIXTURE(fixture_name) { * type property1; * ... * }; @@ -231,14 +240,14 @@ FIXTURE_DATA(fixture_name) /** - * FIXTURE_SETUP(fixture_name) - Prepares the setup function for the fixture. - * *_metadata* is included so that EXPECT_* and ASSERT_* work correctly. + * FIXTURE_SETUP() - Prepares the setup function for the fixture. + * *_metadata* is included so that EXPECT_*, ASSERT_* etc. work correctly. * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE_SETUP(fixture name) { implementation } + * FIXTURE_SETUP(fixture_name) { implementation } * * Populates the required "setup" function for a fixture. An instance of the * datatype defined with FIXTURE_DATA() will be exposed as *self* for the @@ -257,14 +266,14 @@ __attribute__((unused)) *variant) /** - * FIXTURE_TEARDOWN(fixture_name) - * *_metadata* is included so that EXPECT_* and ASSERT_* work correctly. + * FIXTURE_TEARDOWN() + * *_metadata* is included so that EXPECT_*, ASSERT_* etc. work correctly. * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE_TEARDOWN(fixture name) { implementation } + * FIXTURE_TEARDOWN(fixture_name) { implementation } * * Populates the required "teardown" function for a fixture. An instance of the * datatype defined with FIXTURE_DATA() will be exposed as *self* for the @@ -275,29 +284,31 @@ #define FIXTURE_TEARDOWN(fixture_name) \ void fixture_name##_teardown( \ struct __test_metadata __attribute__((unused)) *_metadata, \ - FIXTURE_DATA(fixture_name) __attribute__((unused)) *self) + FIXTURE_DATA(fixture_name) __attribute__((unused)) *self, \ + const FIXTURE_VARIANT(fixture_name) \ + __attribute__((unused)) *variant) /** - * FIXTURE_VARIANT(fixture_name) - Optionally called once per fixture + * FIXTURE_VARIANT() - Optionally called once per fixture * to declare fixture variant * * @fixture_name: fixture name * * .. code-block:: c * - * FIXTURE_VARIANT(datatype name) { + * FIXTURE_VARIANT(fixture_name) { * type property1; * ... * }; * - * Defines type of constant parameters provided to FIXTURE_SETUP() and TEST_F() - * as *variant*. Variants allow the same tests to be run with different - * arguments. + * Defines type of constant parameters provided to FIXTURE_SETUP(), TEST_F() and + * FIXTURE_TEARDOWN as *variant*. Variants allow the same tests to be run with + * different arguments. */ #define FIXTURE_VARIANT(fixture_name) struct _fixture_variant_##fixture_name /** - * FIXTURE_VARIANT_ADD(fixture_name, variant_name) - Called once per fixture + * FIXTURE_VARIANT_ADD() - Called once per fixture * variant to setup and register the data * * @fixture_name: fixture name @@ -305,8 +316,8 @@ * * .. code-block:: c * - * FIXTURE_ADD(datatype name) { - * .property1 = val1; + * FIXTURE_VARIANT_ADD(fixture_name, variant_name) { + * .property1 = val1, * ... * }; * @@ -331,7 +342,7 @@ _##fixture_name##_##variant_name##_variant = /** - * TEST_F(fixture_name, test_name) - Emits test registration and helpers for + * TEST_F() - Emits test registration and helpers for * fixture-based test cases * * @fixture_name: fixture name @@ -345,9 +356,11 @@ * Very similar to TEST() except that *self* is the setup instance of fixture's * datatype exposed for use by the implementation. * - * Warning: use of ASSERT_* here will skip TEARDOWN. + * The @test_name code is run in a separate process sharing the same memory + * (i.e. vfork), which means that the test process can update its privileges + * without impacting the related FIXTURE_TEARDOWN() (e.g. to remove files from + * a directory where write access was dropped). */ -/* TODO(wad) register fixtures on dedicated test lists. */ #define TEST_F(fixture_name, test_name) \ __TEST_F_IMPL(fixture_name, test_name, -1, TEST_TIMEOUT_DEFAULT) @@ -368,13 +381,35 @@ { \ /* fixture data is alloced, setup, and torn down per call. */ \ FIXTURE_DATA(fixture_name) self; \ + pid_t child = 1; \ + int status = 0; \ memset(&self, 0, sizeof(FIXTURE_DATA(fixture_name))); \ - fixture_name##_setup(_metadata, &self, variant->data); \ - /* Let setup failure terminate early. */ \ - if (!_metadata->passed) \ - return; \ - fixture_name##_##test_name(_metadata, &self, variant->data); \ - fixture_name##_teardown(_metadata, &self); \ + if (setjmp(_metadata->env) == 0) { \ + /* Use the same _metadata. */ \ + child = vfork(); \ + if (child == 0) { \ + fixture_name##_setup(_metadata, &self, variant->data); \ + /* Let setup failure terminate early. */ \ + if (_metadata->exit_code) \ + _exit(0); \ + _metadata->setup_completed = true; \ + fixture_name##_##test_name(_metadata, &self, variant->data); \ + } else if (child < 0 || child != waitpid(child, &status, 0)) { \ + ksft_print_msg("ERROR SPAWNING TEST GRANDCHILD\n"); \ + _metadata->exit_code = KSFT_FAIL; \ + } \ + } \ + if (child == 0) { \ + if (_metadata->setup_completed && !_metadata->teardown_parent) \ + fixture_name##_teardown(_metadata, &self, variant->data); \ + _exit(0); \ + } \ + if (_metadata->setup_completed && _metadata->teardown_parent) \ + fixture_name##_teardown(_metadata, &self, variant->data); \ + if (!WIFEXITED(status) && WIFSIGNALED(status)) \ + /* Forward signal to __wait_for_test(). */ \ + kill(getpid(), WTERMSIG(status)); \ + __test_check_assert(_metadata); \ } \ static struct __test_metadata \ _##fixture_name##_##test_name##_object = { \ @@ -383,6 +418,7 @@ .fixture = &_##fixture_name##_fixture_object, \ .termsig = signal, \ .timeout = tmout, \ + .teardown_parent = false, \ }; \ static void __attribute__((constructor)) \ _register_##fixture_name##_##test_name(void) \ @@ -424,7 +460,7 @@ */ /** - * ASSERT_EQ(expected, seen) + * ASSERT_EQ() * * @expected: expected value * @seen: measured value @@ -435,7 +471,7 @@ __EXPECT(expected, #expected, seen, #seen, ==, 1) /** - * ASSERT_NE(expected, seen) + * ASSERT_NE() * * @expected: expected value * @seen: measured value @@ -446,7 +482,7 @@ __EXPECT(expected, #expected, seen, #seen, !=, 1) /** - * ASSERT_LT(expected, seen) + * ASSERT_LT() * * @expected: expected value * @seen: measured value @@ -457,7 +493,7 @@ __EXPECT(expected, #expected, seen, #seen, <, 1) /** - * ASSERT_LE(expected, seen) + * ASSERT_LE() * * @expected: expected value * @seen: measured value @@ -468,7 +504,7 @@ __EXPECT(expected, #expected, seen, #seen, <=, 1) /** - * ASSERT_GT(expected, seen) + * ASSERT_GT() * * @expected: expected value * @seen: measured value @@ -479,7 +515,7 @@ __EXPECT(expected, #expected, seen, #seen, >, 1) /** - * ASSERT_GE(expected, seen) + * ASSERT_GE() * * @expected: expected value * @seen: measured value @@ -490,7 +526,7 @@ __EXPECT(expected, #expected, seen, #seen, >=, 1) /** - * ASSERT_NULL(seen) + * ASSERT_NULL() * * @seen: measured value * @@ -500,7 +536,7 @@ __EXPECT(NULL, "NULL", seen, #seen, ==, 1) /** - * ASSERT_TRUE(seen) + * ASSERT_TRUE() * * @seen: measured value * @@ -510,7 +546,7 @@ __EXPECT(0, "0", seen, #seen, !=, 1) /** - * ASSERT_FALSE(seen) + * ASSERT_FALSE() * * @seen: measured value * @@ -520,7 +556,7 @@ __EXPECT(0, "0", seen, #seen, ==, 1) /** - * ASSERT_STREQ(expected, seen) + * ASSERT_STREQ() * * @expected: expected value * @seen: measured value @@ -531,7 +567,7 @@ __EXPECT_STR(expected, seen, ==, 1) /** - * ASSERT_STRNE(expected, seen) + * ASSERT_STRNE() * * @expected: expected value * @seen: measured value @@ -542,7 +578,7 @@ __EXPECT_STR(expected, seen, !=, 1) /** - * EXPECT_EQ(expected, seen) + * EXPECT_EQ() * * @expected: expected value * @seen: measured value @@ -553,7 +589,7 @@ __EXPECT(expected, #expected, seen, #seen, ==, 0) /** - * EXPECT_NE(expected, seen) + * EXPECT_NE() * * @expected: expected value * @seen: measured value @@ -564,7 +600,7 @@ __EXPECT(expected, #expected, seen, #seen, !=, 0) /** - * EXPECT_LT(expected, seen) + * EXPECT_LT() * * @expected: expected value * @seen: measured value @@ -575,7 +611,7 @@ __EXPECT(expected, #expected, seen, #seen, <, 0) /** - * EXPECT_LE(expected, seen) + * EXPECT_LE() * * @expected: expected value * @seen: measured value @@ -586,7 +622,7 @@ __EXPECT(expected, #expected, seen, #seen, <=, 0) /** - * EXPECT_GT(expected, seen) + * EXPECT_GT() * * @expected: expected value * @seen: measured value @@ -597,7 +633,7 @@ __EXPECT(expected, #expected, seen, #seen, >, 0) /** - * EXPECT_GE(expected, seen) + * EXPECT_GE() * * @expected: expected value * @seen: measured value @@ -608,7 +644,7 @@ __EXPECT(expected, #expected, seen, #seen, >=, 0) /** - * EXPECT_NULL(seen) + * EXPECT_NULL() * * @seen: measured value * @@ -618,7 +654,7 @@ __EXPECT(NULL, "NULL", seen, #seen, ==, 0) /** - * EXPECT_TRUE(seen) + * EXPECT_TRUE() * * @seen: measured value * @@ -628,7 +664,7 @@ __EXPECT(0, "0", seen, #seen, !=, 0) /** - * EXPECT_FALSE(seen) + * EXPECT_FALSE() * * @seen: measured value * @@ -638,7 +674,7 @@ __EXPECT(0, "0", seen, #seen, ==, 0) /** - * EXPECT_STREQ(expected, seen) + * EXPECT_STREQ() * * @expected: expected value * @seen: measured value @@ -649,7 +685,7 @@ __EXPECT_STR(expected, seen, ==, 0) /** - * EXPECT_STRNE(expected, seen) + * EXPECT_STRNE() * * @expected: expected value * @seen: measured value @@ -659,7 +695,9 @@ #define EXPECT_STRNE(expected, seen) \ __EXPECT_STR(expected, seen, !=, 0) +#ifndef ARRAY_SIZE #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) +#endif /* Support an optional handler after and ASSERT_* or EXPECT_*. The approach is * not thread-safe, but it should be fine in most sane test scenarios. @@ -669,24 +707,51 @@ */ #define OPTIONAL_HANDLER(_assert) \ for (; _metadata->trigger; _metadata->trigger = \ - __bail(_assert, _metadata->no_print, _metadata->step)) + __bail(_assert, _metadata)) -#define __INC_STEP(_metadata) \ - if (_metadata->passed && _metadata->step < 255) \ - _metadata->step++; +#define is_signed_type(var) (!!(((__typeof__(var))(-1)) < (__typeof__(var))1)) #define __EXPECT(_expected, _expected_str, _seen, _seen_str, _t, _assert) do { \ /* Avoid multiple evaluation of the cases */ \ __typeof__(_expected) __exp = (_expected); \ __typeof__(_seen) __seen = (_seen); \ - if (_assert) __INC_STEP(_metadata); \ if (!(__exp _t __seen)) { \ - unsigned long long __exp_print = (uintptr_t)__exp; \ - unsigned long long __seen_print = (uintptr_t)__seen; \ - __TH_LOG("Expected %s (%llu) %s %s (%llu)", \ - _expected_str, __exp_print, #_t, \ - _seen_str, __seen_print); \ - _metadata->passed = 0; \ + /* Report with actual signedness to avoid weird output. */ \ + switch (is_signed_type(__exp) * 2 + is_signed_type(__seen)) { \ + case 0: { \ + unsigned long long __exp_print = (uintptr_t)__exp; \ + unsigned long long __seen_print = (uintptr_t)__seen; \ + __TH_LOG("Expected %s (%llu) %s %s (%llu)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + case 1: { \ + unsigned long long __exp_print = (uintptr_t)__exp; \ + long long __seen_print = (intptr_t)__seen; \ + __TH_LOG("Expected %s (%llu) %s %s (%lld)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + case 2: { \ + long long __exp_print = (intptr_t)__exp; \ + unsigned long long __seen_print = (uintptr_t)__seen; \ + __TH_LOG("Expected %s (%lld) %s %s (%llu)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + case 3: { \ + long long __exp_print = (intptr_t)__exp; \ + long long __seen_print = (intptr_t)__seen; \ + __TH_LOG("Expected %s (%lld) %s %s (%lld)", \ + _expected_str, __exp_print, #_t, \ + _seen_str, __seen_print); \ + break; \ + } \ + } \ + _metadata->exit_code = KSFT_FAIL; \ /* Ensure the optional handler is triggered */ \ _metadata->trigger = 1; \ } \ @@ -695,10 +760,9 @@ #define __EXPECT_STR(_expected, _seen, _t, _assert) do { \ const char *__exp = (_expected); \ const char *__seen = (_seen); \ - if (_assert) __INC_STEP(_metadata); \ if (!(strcmp(__exp, __seen) _t 0)) { \ __TH_LOG("Expected '%s' %s '%s'.", __exp, #_t, __seen); \ - _metadata->passed = 0; \ + _metadata->exit_code = KSFT_FAIL; \ _metadata->trigger = 1; \ } \ } while (0); OPTIONAL_HANDLER(_assert) @@ -726,6 +790,10 @@ } \ } +struct __test_results { + char reason[1024]; /* Reason for test result */ +}; + struct __test_metadata; struct __fixture_variant_metadata; @@ -740,6 +808,37 @@ struct __fixture_metadata { .prev = &_fixture_global, }; +struct __test_xfail { + struct __fixture_metadata *fixture; + struct __fixture_variant_metadata *variant; + struct __test_metadata *test; + struct __test_xfail *prev, *next; +}; + +/** + * XFAIL_ADD() - mark variant + test case combination as expected to fail + * @fixture_name: name of the fixture + * @variant_name: name of the variant + * @test_name: name of the test case + * + * Mark a combination of variant + test case for a given fixture as expected + * to fail. Tests marked this way will report XPASS / XFAIL return codes, + * instead of PASS / FAIL,and use respective counters. + */ +#define XFAIL_ADD(fixture_name, variant_name, test_name) \ + static struct __test_xfail \ + _##fixture_name##_##variant_name##_##test_name##_xfail = \ + { \ + .fixture = &_##fixture_name##_fixture_object, \ + .variant = &_##fixture_name##_##variant_name##_object, \ + .test = &_##fixture_name##_##test_name##_object, \ + }; \ + static void __attribute__((constructor)) \ + _register_##fixture_name##_##variant_name##_##test_name##_xfail(void) \ + { \ + __register_xfail(&_##fixture_name##_##variant_name##_##test_name##_xfail); \ + } + static struct __fixture_metadata *__fixture_list = &_fixture_global; static int __constructor_order; @@ -754,6 +853,7 @@ static inline void __register_fixture(struct __fixture_metadata *f) struct __fixture_variant_metadata { const char *name; const void *data; + struct __test_xfail *xfails; struct __fixture_variant_metadata *prev, *next; }; @@ -772,15 +872,24 @@ struct __test_metadata { pid_t pid; /* pid of test when being run */ struct __fixture_metadata *fixture; int termsig; - int passed; + int exit_code; int trigger; /* extra handler after the evaluation */ int timeout; /* seconds to wait for test timeout */ bool timed_out; /* did this test timeout instead of exiting? */ - __u8 step; - bool no_print; /* manual trigger when TH_LOG_STREAM is not available */ + bool aborted; /* stopped test due to failed ASSERT */ + bool setup_completed; /* did setup finish? */ + bool teardown_parent; /* run teardown in a parent process */ + jmp_buf env; /* for exiting out of test early */ + struct __test_results *results; struct __test_metadata *prev, *next; }; +static inline bool __test_passed(struct __test_metadata *metadata) +{ + return metadata->exit_code != KSFT_FAIL && + metadata->exit_code <= KSFT_SKIP; +} + /* * Since constructors are called in reverse order, reverse the test * list so tests are run in source declaration order. @@ -795,16 +904,28 @@ static inline void __register_test(struct __test_metadata *t) __LIST_APPEND(t->fixture->tests, t); } -static inline int __bail(int for_realz, bool no_print, __u8 step) +static inline void __register_xfail(struct __test_xfail *xf) { + __LIST_APPEND(xf->variant->xfails, xf); +} + +static inline int __bail(int for_realz, struct __test_metadata *t) +{ + /* if this is ASSERT, return immediately. */ if (for_realz) { - if (no_print) - _exit(step); - abort(); + t->aborted = true; + longjmp(t->env, 1); } + /* otherwise, end the for loop and continue. */ return 0; } +static inline void __test_check_assert(struct __test_metadata *t) +{ + if (t->aborted) + abort(); +} + struct __test_metadata *__active_test; static void __timeout_handler(int sig, siginfo_t *info, void *ucontext) { @@ -813,18 +934,19 @@ static void __timeout_handler(int sig, siginfo_t *info, void *ucontext) /* Sanity check handler execution environment. */ if (!t) { fprintf(TH_LOG_STREAM, - "no active test in SIGALRM handler!?\n"); + "# no active test in SIGALRM handler!?\n"); abort(); } if (sig != SIGALRM || sig != info->si_signo) { fprintf(TH_LOG_STREAM, - "%s: SIGALRM handler caught signal %d!?\n", + "# %s: SIGALRM handler caught signal %d!?\n", t->name, sig != SIGALRM ? sig : info->si_signo); abort(); } t->timed_out = true; - kill(t->pid, SIGKILL); + // signal process group + kill(-(t->pid), SIGKILL); } void __wait_for_test(struct __test_metadata *t) @@ -837,9 +959,9 @@ void __wait_for_test(struct __test_metadata *t) int status; if (sigaction(SIGALRM, &action, &saved_action)) { - t->passed = 0; + t->exit_code = KSFT_FAIL; fprintf(TH_LOG_STREAM, - "%s: unable to install SIGALRM handler\n", + "# %s: unable to install SIGALRM handler\n", t->name); return; } @@ -849,120 +971,300 @@ void __wait_for_test(struct __test_metadata *t) waitpid(t->pid, &status, 0); alarm(0); if (sigaction(SIGALRM, &saved_action, NULL)) { - t->passed = 0; + t->exit_code = KSFT_FAIL; fprintf(TH_LOG_STREAM, - "%s: unable to uninstall SIGALRM handler\n", + "# %s: unable to uninstall SIGALRM handler\n", t->name); return; } __active_test = NULL; if (t->timed_out) { - t->passed = 0; + t->exit_code = KSFT_FAIL; fprintf(TH_LOG_STREAM, - "%s: Test terminated by timeout\n", t->name); + "# %s: Test terminated by timeout\n", t->name); } else if (WIFEXITED(status)) { - t->passed = t->termsig == -1 ? !WEXITSTATUS(status) : 0; - if (t->termsig != -1) { - fprintf(TH_LOG_STREAM, - "%s: Test exited normally " - "instead of by signal (code: %d)\n", - t->name, - WEXITSTATUS(status)); - } else if (!t->passed) { + if (WEXITSTATUS(status) == KSFT_SKIP || + WEXITSTATUS(status) == KSFT_XPASS || + WEXITSTATUS(status) == KSFT_XFAIL) { + t->exit_code = WEXITSTATUS(status); + } else if (t->termsig != -1) { + t->exit_code = KSFT_FAIL; fprintf(TH_LOG_STREAM, - "%s: Test failed at step #%d\n", + "# %s: Test exited normally instead of by signal (code: %d)\n", t->name, WEXITSTATUS(status)); + } else { + switch (WEXITSTATUS(status)) { + /* Success */ + case KSFT_PASS: + t->exit_code = KSFT_PASS; + break; + /* Failure */ + default: + t->exit_code = KSFT_FAIL; + fprintf(TH_LOG_STREAM, + "# %s: Test failed\n", + t->name); + } } } else if (WIFSIGNALED(status)) { - t->passed = 0; + t->exit_code = KSFT_FAIL; if (WTERMSIG(status) == SIGABRT) { fprintf(TH_LOG_STREAM, - "%s: Test terminated by assertion\n", + "# %s: Test terminated by assertion\n", t->name); } else if (WTERMSIG(status) == t->termsig) { - t->passed = 1; + t->exit_code = KSFT_PASS; } else { fprintf(TH_LOG_STREAM, - "%s: Test terminated unexpectedly " - "by signal %d\n", + "# %s: Test terminated unexpectedly by signal %d\n", t->name, WTERMSIG(status)); } } else { fprintf(TH_LOG_STREAM, - "%s: Test ended in some other way [%u]\n", + "# %s: Test ended in some other way [%u]\n", t->name, status); } } +static void test_harness_list_tests(void) +{ + struct __fixture_variant_metadata *v; + struct __fixture_metadata *f; + struct __test_metadata *t; + + for (f = __fixture_list; f; f = f->next) { + v = f->variant; + t = f->tests; + + if (f == __fixture_list) + fprintf(stderr, "%-20s %-25s %s\n", + "# FIXTURE", "VARIANT", "TEST"); + else + fprintf(stderr, "--------------------------------------------------------------------------------\n"); + + do { + fprintf(stderr, "%-20s %-25s %s\n", + t == f->tests ? f->name : "", + v ? v->name : "", + t ? t->name : ""); + + v = v ? v->next : NULL; + t = t ? t->next : NULL; + } while (v || t); + } +} + +static int test_harness_argv_check(int argc, char **argv) +{ + int opt; + + while ((opt = getopt(argc, argv, "hlF:f:V:v:t:T:r:")) != -1) { + switch (opt) { + case 'f': + case 'F': + case 'v': + case 'V': + case 't': + case 'T': + case 'r': + break; + case 'l': + test_harness_list_tests(); + return KSFT_SKIP; + case 'h': + default: + fprintf(stderr, + "Usage: %s [-h|-l] [-t|-T|-v|-V|-f|-F|-r name]\n" + "\t-h print help\n" + "\t-l list all tests\n" + "\n" + "\t-t name include test\n" + "\t-T name exclude test\n" + "\t-v name include variant\n" + "\t-V name exclude variant\n" + "\t-f name include fixture\n" + "\t-F name exclude fixture\n" + "\t-r name run specified test\n" + "\n" + "Test filter options can be specified " + "multiple times. The filtering stops\n" + "at the first match. For example to " + "include all tests from variant 'bla'\n" + "but not test 'foo' specify '-T foo -v bla'.\n" + "", argv[0]); + return opt == 'h' ? KSFT_SKIP : KSFT_FAIL; + } + } + + return KSFT_PASS; +} + +static bool test_enabled(int argc, char **argv, + struct __fixture_metadata *f, + struct __fixture_variant_metadata *v, + struct __test_metadata *t) +{ + unsigned int flen = 0, vlen = 0, tlen = 0; + bool has_positive = false; + int opt; + + optind = 1; + while ((opt = getopt(argc, argv, "F:f:V:v:t:T:r:")) != -1) { + has_positive |= islower(opt); + + switch (tolower(opt)) { + case 't': + if (!strcmp(t->name, optarg)) + return islower(opt); + break; + case 'f': + if (!strcmp(f->name, optarg)) + return islower(opt); + break; + case 'v': + if (!strcmp(v->name, optarg)) + return islower(opt); + break; + case 'r': + if (!tlen) { + flen = strlen(f->name); + vlen = strlen(v->name); + tlen = strlen(t->name); + } + if (strlen(optarg) == flen + 1 + vlen + !!vlen + tlen && + !strncmp(f->name, &optarg[0], flen) && + !strncmp(v->name, &optarg[flen + 1], vlen) && + !strncmp(t->name, &optarg[flen + 1 + vlen + !!vlen], tlen)) + return true; + break; + } + } + + /* + * If there are no positive tests then we assume user just wants + * exclusions and everything else is a pass. + */ + return !has_positive; +} + void __run_test(struct __fixture_metadata *f, struct __fixture_variant_metadata *variant, struct __test_metadata *t) { + struct __test_xfail *xfail; + char test_name[LINE_MAX]; + const char *diagnostic; + /* reset test struct */ - t->passed = 1; + t->exit_code = KSFT_PASS; t->trigger = 0; - t->step = 0; - t->no_print = 0; + memset(t->results->reason, 0, sizeof(t->results->reason)); + + snprintf(test_name, sizeof(test_name), "%s%s%s.%s", + f->name, variant->name[0] ? "." : "", variant->name, t->name); + + ksft_print_msg(" RUN %s ...\n", test_name); + + /* Make sure output buffers are flushed before fork */ + fflush(stdout); + fflush(stderr); - printf("[ RUN ] %s%s%s.%s\n", - f->name, variant->name[0] ? "." : "", variant->name, t->name); t->pid = fork(); if (t->pid < 0) { - printf("ERROR SPAWNING TEST CHILD\n"); - t->passed = 0; + ksft_print_msg("ERROR SPAWNING TEST CHILD\n"); + t->exit_code = KSFT_FAIL; } else if (t->pid == 0) { + setpgrp(); t->fn(t, variant); - /* return the step that failed or 0 */ - _exit(t->passed ? 0 : t->step); + _exit(t->exit_code); } else { __wait_for_test(t); } - printf("[ %4s ] %s%s%s.%s\n", (t->passed ? "OK" : "FAIL"), - f->name, variant->name[0] ? "." : "", variant->name, t->name); + ksft_print_msg(" %4s %s\n", + __test_passed(t) ? "OK" : "FAIL", test_name); + + /* Check if we're expecting this test to fail */ + for (xfail = variant->xfails; xfail; xfail = xfail->next) + if (xfail->test == t) + break; + if (xfail) + t->exit_code = __test_passed(t) ? KSFT_XPASS : KSFT_XFAIL; + + if (t->results->reason[0]) + diagnostic = t->results->reason; + else if (t->exit_code == KSFT_PASS || t->exit_code == KSFT_FAIL) + diagnostic = NULL; + else + diagnostic = "unknown"; + + ksft_test_result_code(t->exit_code, test_name, + diagnostic ? "%s" : "", diagnostic); } -static int test_harness_run(int __attribute__((unused)) argc, - char __attribute__((unused)) **argv) +static int test_harness_run(int argc, char **argv) { struct __fixture_variant_metadata no_variant = { .name = "", }; struct __fixture_variant_metadata *v; struct __fixture_metadata *f; + struct __test_results *results; struct __test_metadata *t; - int ret = 0; + int ret; unsigned int case_count = 0, test_count = 0; unsigned int count = 0; unsigned int pass_count = 0; + ret = test_harness_argv_check(argc, argv); + if (ret != KSFT_PASS) + return ret; + for (f = __fixture_list; f; f = f->next) { for (v = f->variant ?: &no_variant; v; v = v->next) { - case_count++; + unsigned int old_tests = test_count; + for (t = f->tests; t; t = t->next) - test_count++; + if (test_enabled(argc, argv, f, v, t)) + test_count++; + + if (old_tests != test_count) + case_count++; } } - /* TODO(wad) add optional arguments similar to gtest. */ - printf("[==========] Running %u tests from %u test cases.\n", + results = mmap(NULL, sizeof(*results), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + ksft_print_header(); + ksft_set_plan(test_count); + ksft_print_msg("Starting %u tests from %u test cases.\n", test_count, case_count); for (f = __fixture_list; f; f = f->next) { for (v = f->variant ?: &no_variant; v; v = v->next) { for (t = f->tests; t; t = t->next) { + if (!test_enabled(argc, argv, f, v, t)) + continue; count++; + t->results = results; __run_test(f, v, t); - if (t->passed) + t->results = NULL; + if (__test_passed(t)) pass_count++; else ret = 1; } } } - printf("[==========] %u / %u tests passed.\n", pass_count, count); - printf("[ %s ]\n", (ret ? "FAILED" : "PASSED")); - return ret; + munmap(results, sizeof(*results)); + + ksft_print_msg("%s: %u / %u tests passed.\n", ret ? "FAILED" : "PASSED", + pass_count, count); + ksft_exit(ret == 0); + + /* unreachable */ + return KSFT_FAIL; } static void __attribute__((constructor)) __constructor_order_first(void) |