diff options
Diffstat (limited to 'tools/testing/selftests')
48 files changed, 1615 insertions, 281 deletions
diff --git a/tools/testing/selftests/.gitignore b/tools/testing/selftests/.gitignore index 91750352459d..8059ce834247 100644 --- a/tools/testing/selftests/.gitignore +++ b/tools/testing/selftests/.gitignore @@ -1,4 +1,3 @@ -kselftest gpiogpio-event-mon gpiogpio-hammer gpioinclude/ diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index f2ebf8cf4686..9781ca79794a 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -71,6 +71,9 @@ override LDFLAGS = override MAKEFLAGS = endif +# Append kselftest to KBUILD_OUTPUT to avoid cluttering +# KBUILD_OUTPUT with selftest objects and headers installed +# by selftests Makefile or lib.mk. ifneq ($(KBUILD_SRC),) override LDFLAGS = endif @@ -79,19 +82,13 @@ ifneq ($(O),) BUILD := $(O) else ifneq ($(KBUILD_OUTPUT),) - BUILD := $(KBUILD_OUTPUT) + BUILD := $(KBUILD_OUTPUT)/kselftest else BUILD := $(shell pwd) DEFAULT_INSTALL_HDR_PATH := 1 endif endif -# KSFT_TAP_LEVEL is used from KSFT framework to prevent nested TAP header -# printing from tests. Applicable to run_tests case where run_tests adds -# TAP header prior running tests and when a test program invokes another -# with system() call. Export it here to cover override RUN_TESTS defines. -export KSFT_TAP_LEVEL=`echo 1` - # Prepare for headers install top_srcdir ?= ../../.. include $(top_srcdir)/scripts/subarch.include @@ -169,14 +166,22 @@ clean_hotplug: run_pstore_crash: make -C pstore run_crash -INSTALL_PATH ?= install +# Use $BUILD as the default install root. $BUILD points to the +# right output location for the following cases: +# 1. output_dir=kernel_src +# 2. a separate output directory is specified using O= KBUILD_OUTPUT +# 3. a separate output directory is specified using KBUILD_OUTPUT +# +INSTALL_PATH ?= $(BUILD)/install INSTALL_PATH := $(abspath $(INSTALL_PATH)) ALL_SCRIPT := $(INSTALL_PATH)/run_kselftest.sh -install: +install: all ifdef INSTALL_PATH @# Ask all targets to install their files - mkdir -p $(INSTALL_PATH) + mkdir -p $(INSTALL_PATH)/kselftest + install -m 744 kselftest/runner.sh $(INSTALL_PATH)/kselftest/ + install -m 744 kselftest/prefix.pl $(INSTALL_PATH)/kselftest/ @for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ make OUTPUT=$$BUILD_TARGET -C $$TARGET INSTALL_PATH=$(INSTALL_PATH)/$$TARGET install; \ @@ -186,24 +191,20 @@ ifdef INSTALL_PATH echo "#!/bin/sh" > $(ALL_SCRIPT) echo "BASE_DIR=\$$(realpath \$$(dirname \$$0))" >> $(ALL_SCRIPT) echo "cd \$$BASE_DIR" >> $(ALL_SCRIPT) + echo ". ./kselftest/runner.sh" >> $(ALL_SCRIPT) echo "ROOT=\$$PWD" >> $(ALL_SCRIPT) echo "if [ \"\$$1\" = \"--summary\" ]; then" >> $(ALL_SCRIPT) - echo " OUTPUT=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT) - echo " cat /dev/null > \$$OUTPUT" >> $(ALL_SCRIPT) - echo "else" >> $(ALL_SCRIPT) - echo " OUTPUT=/dev/stdout" >> $(ALL_SCRIPT) + echo " logfile=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT) + echo " cat /dev/null > \$$logfile" >> $(ALL_SCRIPT) echo "fi" >> $(ALL_SCRIPT) - echo "export KSFT_TAP_LEVEL=1" >> $(ALL_SCRIPT) - echo "export skip=4" >> $(ALL_SCRIPT) for TARGET in $(TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ - echo "echo ; echo TAP version 13" >> $(ALL_SCRIPT); \ - echo "echo Running tests in $$TARGET" >> $(ALL_SCRIPT); \ - echo "echo ========================================" >> $(ALL_SCRIPT); \ echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \ echo "cd $$TARGET" >> $(ALL_SCRIPT); \ + echo -n "run_many" >> $(ALL_SCRIPT); \ make -s --no-print-directory OUTPUT=$$BUILD_TARGET -C $$TARGET emit_tests >> $(ALL_SCRIPT); \ + echo "" >> $(ALL_SCRIPT); \ echo "cd \$$ROOT" >> $(ALL_SCRIPT); \ done; diff --git a/tools/testing/selftests/breakpoints/breakpoint_test.c b/tools/testing/selftests/breakpoints/breakpoint_test.c index 901b85ea6a59..8f3655e59020 100644 --- a/tools/testing/selftests/breakpoints/breakpoint_test.c +++ b/tools/testing/selftests/breakpoints/breakpoint_test.c @@ -21,6 +21,8 @@ #include "../kselftest.h" +#define COUNT_ISN_BPS 4 +#define COUNT_WPS 4 /* Breakpoint access modes */ enum { @@ -220,7 +222,7 @@ static void trigger_tests(void) if (!local && !global) continue; - for (i = 0; i < 4; i++) { + for (i = 0; i < COUNT_ISN_BPS; i++) { dummy_funcs[i](); check_trapped(); } @@ -292,7 +294,7 @@ static void launch_instruction_breakpoints(char *buf, int local, int global) { int i; - for (i = 0; i < 4; i++) { + for (i = 0; i < COUNT_ISN_BPS; i++) { set_breakpoint_addr(dummy_funcs[i], i); toggle_breakpoint(i, BP_X, 1, local, global, 1); ptrace(PTRACE_CONT, child_pid, NULL, 0); @@ -314,7 +316,7 @@ static void launch_watchpoints(char *buf, int mode, int len, else mode_str = "read"; - for (i = 0; i < 4; i++) { + for (i = 0; i < COUNT_WPS; i++) { set_breakpoint_addr(&dummy_var[i], i); toggle_breakpoint(i, mode, len, local, global, 1); ptrace(PTRACE_CONT, child_pid, NULL, 0); @@ -330,8 +332,15 @@ static void launch_watchpoints(char *buf, int mode, int len, static void launch_tests(void) { char buf[1024]; + unsigned int tests = 0; int len, local, global, i; + tests += 3 * COUNT_ISN_BPS; + tests += sizeof(long) / 2 * 3 * COUNT_WPS; + tests += sizeof(long) / 2 * 3 * COUNT_WPS; + tests += 2; + ksft_set_plan(tests); + /* Instruction breakpoints */ for (local = 0; local < 2; local++) { for (global = 0; global < 2; global++) { diff --git a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c index 2d95e5adde72..ab59d814341a 100644 --- a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c +++ b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c @@ -118,7 +118,7 @@ static bool set_watchpoint(pid_t pid, int size, int wp) return false; } -static bool run_test(int wr_size, int wp_size, int wr, int wp) +static bool arun_test(int wr_size, int wp_size, int wr, int wp) { int status; siginfo_t siginfo; @@ -214,6 +214,7 @@ int main(int argc, char **argv) bool result; ksft_print_header(); + ksft_set_plan(213); act.sa_handler = sigalrm; sigemptyset(&act.sa_mask); diff --git a/tools/testing/selftests/breakpoints/step_after_suspend_test.c b/tools/testing/selftests/breakpoints/step_after_suspend_test.c index f82dcc1f8841..cf868b5e00f7 100644 --- a/tools/testing/selftests/breakpoints/step_after_suspend_test.c +++ b/tools/testing/selftests/breakpoints/step_after_suspend_test.c @@ -173,6 +173,7 @@ int main(int argc, char **argv) int opt; bool do_suspend = true; bool succeeded = true; + unsigned int tests = 0; cpu_set_t available_cpus; int err; int cpu; @@ -191,6 +192,13 @@ int main(int argc, char **argv) } } + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &available_cpus)) + continue; + tests++; + } + ksft_set_plan(tests); + if (do_suspend) suspend(); diff --git a/tools/testing/selftests/capabilities/test_execve.c b/tools/testing/selftests/capabilities/test_execve.c index 3ab39a61b95b..df0ef02b4036 100644 --- a/tools/testing/selftests/capabilities/test_execve.c +++ b/tools/testing/selftests/capabilities/test_execve.c @@ -430,8 +430,6 @@ int main(int argc, char **argv) { char *tmp1, *tmp2, *our_path; - ksft_print_header(); - /* Find our path */ tmp1 = strdup(argv[0]); if (!tmp1) @@ -445,6 +443,8 @@ int main(int argc, char **argv) mpid = getpid(); if (fork_wait()) { + ksft_print_header(); + ksft_set_plan(12); ksft_print_msg("[RUN]\t+++ Tests with uid == 0 +++\n"); return do_tests(0, our_path); } @@ -452,6 +452,8 @@ int main(int argc, char **argv) ksft_print_msg("==================================================\n"); if (fork_wait()) { + ksft_print_header(); + ksft_set_plan(9); ksft_print_msg("[RUN]\t+++ Tests with uid != 0 +++\n"); return do_tests(1, our_path); } diff --git a/tools/testing/selftests/drivers/.gitignore b/tools/testing/selftests/drivers/.gitignore new file mode 100644 index 000000000000..f6aebcc27b76 --- /dev/null +++ b/tools/testing/selftests/drivers/.gitignore @@ -0,0 +1 @@ +/dma-buf/udmabuf diff --git a/tools/testing/selftests/exec/.gitignore b/tools/testing/selftests/exec/.gitignore index 64073e050c6a..b02279da6fa1 100644 --- a/tools/testing/selftests/exec/.gitignore +++ b/tools/testing/selftests/exec/.gitignore @@ -6,4 +6,5 @@ execveat.moved execveat.path.ephemeral execveat.ephemeral execveat.denatured -xxxxxxxx*
\ No newline at end of file +/recursion-depth +xxxxxxxx* diff --git a/tools/testing/selftests/exec/Makefile b/tools/testing/selftests/exec/Makefile index 427c41ba5151..33339e31e365 100644 --- a/tools/testing/selftests/exec/Makefile +++ b/tools/testing/selftests/exec/Makefile @@ -1,11 +1,15 @@ # SPDX-License-Identifier: GPL-2.0 CFLAGS = -Wall +CFLAGS += -Wno-nonnull +CFLAGS += -D_GNU_SOURCE TEST_GEN_PROGS := execveat TEST_GEN_FILES := execveat.symlink execveat.denatured script subdir # Makefile is a run-time dependency, since it's accessed by the execveat test TEST_FILES := Makefile +TEST_GEN_PROGS += recursion-depth + EXTRA_CLEAN := $(OUTPUT)/subdir.moved $(OUTPUT)/execveat.moved $(OUTPUT)/xxxxx* include ../lib.mk diff --git a/tools/testing/selftests/exec/recursion-depth.c b/tools/testing/selftests/exec/recursion-depth.c new file mode 100644 index 000000000000..2dbd5bc45b3e --- /dev/null +++ b/tools/testing/selftests/exec/recursion-depth.c @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 Alexey Dobriyan <adobriyan@gmail.com> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +/* Test that pointing #! script interpreter to self doesn't recurse. */ +#include <errno.h> +#include <sched.h> +#include <stdio.h> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <sys/mount.h> +#include <unistd.h> + +int main(void) +{ + if (unshare(CLONE_NEWNS) == -1) { + if (errno == ENOSYS || errno == EPERM) { + fprintf(stderr, "error: unshare, errno %d\n", errno); + return 4; + } + fprintf(stderr, "error: unshare, errno %d\n", errno); + return 1; + } + if (mount(NULL, "/", NULL, MS_PRIVATE|MS_REC, NULL) == -1) { + fprintf(stderr, "error: mount '/', errno %d\n", errno); + return 1; + } + /* Require "exec" filesystem. */ + if (mount(NULL, "/tmp", "ramfs", 0, NULL) == -1) { + fprintf(stderr, "error: mount ramfs, errno %d\n", errno); + return 1; + } + +#define FILENAME "/tmp/1" + + int fd = creat(FILENAME, 0700); + if (fd == -1) { + fprintf(stderr, "error: creat, errno %d\n", errno); + return 1; + } +#define S "#!" FILENAME "\n" + if (write(fd, S, strlen(S)) != strlen(S)) { + fprintf(stderr, "error: write, errno %d\n", errno); + return 1; + } + close(fd); + + int rv = execve(FILENAME, NULL, NULL); + if (rv == -1 && errno == ELOOP) { + return 0; + } + fprintf(stderr, "error: execve, rv %d, errno %d\n", rv, errno); + return 1; +} diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/tracing-error-log.tc b/tools/testing/selftests/ftrace/test.d/ftrace/tracing-error-log.tc new file mode 100644 index 000000000000..021c03fd885d --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/ftrace/tracing-error-log.tc @@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: ftrace - test tracing error log support + +fail() { #msg + echo $1 + exit_fail +} + +# event tracing is currently the only ftrace tracer that uses the +# tracing error_log, hence this check +if [ ! -f set_event ]; then + echo "event tracing is not supported" + exit_unsupported +fi + +ftrace_errlog_check 'event filter parse error' '((sig >= 10 && sig < 15) || dsig ^== 17) && comm != bash' 'events/signal/signal_generate/filter' + +exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/functions b/tools/testing/selftests/ftrace/test.d/functions index 7b96e80e6b8a..779ec11f61bd 100644 --- a/tools/testing/selftests/ftrace/test.d/functions +++ b/tools/testing/selftests/ftrace/test.d/functions @@ -109,3 +109,15 @@ LOCALHOST=127.0.0.1 yield() { ping $LOCALHOST -c 1 || sleep .001 || usleep 1 || sleep 1 } + +ftrace_errlog_check() { # err-prefix command-with-error-pos-by-^ command-file + pos=$(echo -n "${2%^*}" | wc -c) # error position + command=$(echo "$2" | tr -d ^) + echo "Test command: $command" + echo > error_log + (! echo "$command" > "$3" ) 2> /dev/null + grep "$1: error:" -A 3 error_log + N=$(tail -n 1 error_log | wc -c) + # " Command: " and "^\n" => 13 + test $(expr 13 + $pos) -eq $N +} diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc new file mode 100644 index 000000000000..29faaec942c6 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc @@ -0,0 +1,85 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Kprobe event parser error log check + +[ -f kprobe_events ] || exit_unsupported # this is configurable + +[ -f error_log ] || exit_unsupported + +check_error() { # command-with-error-pos-by-^ + ftrace_errlog_check 'trace_kprobe' "$1" 'kprobe_events' +} + +if grep -q 'r\[maxactive\]' README; then +check_error 'p^100 vfs_read' # MAXACT_NO_KPROBE +check_error 'r^1a111 vfs_read' # BAD_MAXACT +check_error 'r^100000 vfs_read' # MAXACT_TOO_BIG +fi + +check_error 'p ^non_exist_func' # BAD_PROBE_ADDR (enoent) +check_error 'p ^hoge-fuga' # BAD_PROBE_ADDR (bad syntax) +check_error 'p ^hoge+1000-1000' # BAD_PROBE_ADDR (bad syntax) +check_error 'r ^vfs_read+10' # BAD_RETPROBE +check_error 'p:^/bar vfs_read' # NO_GROUP_NAME +check_error 'p:^12345678901234567890123456789012345678901234567890123456789012345/bar vfs_read' # GROUP_TOO_LONG + +check_error 'p:^foo.1/bar vfs_read' # BAD_GROUP_NAME +check_error 'p:foo/^ vfs_read' # NO_EVENT_NAME +check_error 'p:foo/^12345678901234567890123456789012345678901234567890123456789012345 vfs_read' # EVENT_TOO_LONG +check_error 'p:foo/^bar.1 vfs_read' # BAD_EVENT_NAME + +check_error 'p vfs_read ^$retval' # RETVAL_ON_PROBE +check_error 'p vfs_read ^$stack10000' # BAD_STACK_NUM + +if grep -q '$arg<N>' README; then +check_error 'p vfs_read ^$arg10000' # BAD_ARG_NUM +fi + +check_error 'p vfs_read ^$none_var' # BAD_VAR + +check_error 'p vfs_read ^%none_reg' # BAD_REG_NAME +check_error 'p vfs_read ^@12345678abcde' # BAD_MEM_ADDR +check_error 'p vfs_read ^@+10' # FILE_ON_KPROBE + +check_error 'p vfs_read ^+0@0)' # DEREF_NEED_BRACE +check_error 'p vfs_read ^+0ab1(@0)' # BAD_DEREF_OFFS +check_error 'p vfs_read +0(+0(@0^)' # DEREF_OPEN_BRACE + +if grep -A1 "fetcharg:" README | grep -q '\$comm' ; then +check_error 'p vfs_read +0(^$comm)' # COMM_CANT_DEREF +fi + +check_error 'p vfs_read ^&1' # BAD_FETCH_ARG + + +# We've introduced this limitation with array support +if grep -q ' <type>\\\[<array-size>\\\]' README; then +check_error 'p vfs_read +0(^+0(+0(+0(+0(+0(+0(+0(+0(+0(+0(+0(+0(+0(@0))))))))))))))' # TOO_MANY_OPS? +check_error 'p vfs_read +0(@11):u8[10^' # ARRAY_NO_CLOSE +check_error 'p vfs_read +0(@11):u8[10]^a' # BAD_ARRAY_SUFFIX +check_error 'p vfs_read +0(@11):u8[^10a]' # BAD_ARRAY_NUM +check_error 'p vfs_read +0(@11):u8[^256]' # ARRAY_TOO_BIG +fi + +check_error 'p vfs_read @11:^unknown_type' # BAD_TYPE +check_error 'p vfs_read $stack0:^string' # BAD_STRING +check_error 'p vfs_read @11:^b10@a/16' # BAD_BITFIELD + +check_error 'p vfs_read ^arg123456789012345678901234567890=@11' # ARG_NAME_TOO_LOG +check_error 'p vfs_read ^=@11' # NO_ARG_NAME +check_error 'p vfs_read ^var.1=@11' # BAD_ARG_NAME +check_error 'p vfs_read var1=@11 ^var1=@12' # USED_ARG_NAME +check_error 'p vfs_read ^+1234567(+1234567(+1234567(+1234567(+1234567(+1234567(@1234))))))' # ARG_TOO_LONG +check_error 'p vfs_read arg1=^' # NO_ARG_BODY + +# instruction boundary check is valid on x86 (at this moment) +case $(uname -m) in + x86_64|i[3456]86) + echo 'p vfs_read' > kprobe_events + if grep -q FTRACE ../kprobes/list ; then + check_error 'p ^vfs_read+3' # BAD_INSN_BNDRY (only if function-tracer is enabled) + fi + ;; +esac + +exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc new file mode 100644 index 000000000000..14229d5778a0 --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc @@ -0,0 +1,23 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Uprobe event parser error log check + +[ -f uprobe_events ] || exit_unsupported # this is configurable + +[ -f error_log ] || exit_unsupported + +check_error() { # command-with-error-pos-by-^ + ftrace_errlog_check 'trace_uprobe' "$1" 'uprobe_events' +} + +check_error 'p ^/non_exist_file:100' # FILE_NOT_FOUND +check_error 'p ^/sys:100' # NO_REGULAR_FILE +check_error 'p /bin/sh:^10a' # BAD_UPROBE_OFFS +check_error 'p /bin/sh:10(^1a)' # BAD_REFCNT +check_error 'p /bin/sh:10(10^' # REFCNT_OPEN_BRACE +check_error 'p /bin/sh:10(10)^a' # BAD_REFCNT_SUFFIX + +check_error 'p /bin/sh:10 ^@+ab' # BAD_FILE_OFFS +check_error 'p /bin/sh:10 ^@symbol' # SYM_ON_UPROBE + +exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc deleted file mode 100644 index 9912616a8672..000000000000 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-extended-error-support.tc +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# description: event trigger - test extended error support - - -fail() { #msg - echo $1 - exit_fail -} - -if [ ! -f set_event ]; then - echo "event tracing is not supported" - exit_unsupported -fi - -if [ ! -f synthetic_events ]; then - echo "synthetic event is not supported" - exit_unsupported -fi - -echo "Test extended error support" -echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' > events/sched/sched_wakeup/trigger -! echo 'hist:keys=pid:ts0=common_timestamp.usecs if comm=="ping"' >> events/sched/sched_wakeup/trigger 2> /dev/null -if ! grep -q "ERROR:" events/sched/sched_wakeup/hist; then - fail "Failed to generate extended error in histogram" -fi - -exit 0 diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi.c b/tools/testing/selftests/futex/functional/futex_requeue_pi.c index 54cd5c414e82..8d20957f7586 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi.c @@ -395,6 +395,7 @@ int main(int argc, char *argv[]) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg("%s: Test requeue functionality\n", basename(argv[0])); ksft_print_msg( "\tArguments: broadcast=%d locked=%d owner=%d timeout=%ldns\n", diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c index 08187a16507f..742624c59ba7 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c @@ -79,6 +79,7 @@ int main(int argc, char *argv[]) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg("%s: Detect mismatched requeue_pi operations\n", basename(argv[0])); diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c index f0542a344d95..a0f5934707ff 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c @@ -144,6 +144,7 @@ int main(int argc, char *argv[]) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg("%s: Test signal handling during requeue_pi\n", basename(argv[0])); ksft_print_msg("\tArguments: <none>\n"); diff --git a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c index 6216de828093..a458d42ff86e 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c +++ b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c @@ -98,6 +98,7 @@ int main(int argc, char **argv) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg( "%s: Test the futex value of private file mappings in FUTEX_WAIT\n", basename(argv[0])); diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c index bab3dfe1787f..04b95478059c 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c +++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c @@ -69,6 +69,7 @@ int main(int argc, char *argv[]) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg("%s: Block on a futex and wait for timeout\n", basename(argv[0])); ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns); diff --git a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c index 26975322545b..3a1d12a14921 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c +++ b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c @@ -100,6 +100,7 @@ int main(int argc, char **argv) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg("%s: Test the uninitialized futex value in FUTEX_WAIT\n", basename(argv[0])); diff --git a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c index da15a63269b4..a34a6bbc30ce 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c +++ b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c @@ -65,6 +65,7 @@ int main(int argc, char *argv[]) } ksft_print_header(); + ksft_set_plan(1); ksft_print_msg("%s: Test the unexpected futex value in FUTEX_WAIT\n", basename(argv[0])); diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h index 47e1d995c182..ec15c4f6af55 100644 --- a/tools/testing/selftests/kselftest.h +++ b/tools/testing/selftests/kselftest.h @@ -33,6 +33,7 @@ struct ksft_count { }; static struct ksft_count ksft_cnt; +static unsigned int ksft_plan; static inline int ksft_test_num(void) { @@ -61,13 +62,21 @@ static inline void ksft_print_header(void) printf("TAP version 13\n"); } +static inline void ksft_set_plan(unsigned int plan) +{ + ksft_plan = plan; + printf("1..%d\n", ksft_plan); +} + static inline void ksft_print_cnts(void) { - printf("Pass %d Fail %d Xfail %d Xpass %d Skip %d Error %d\n", + if (ksft_plan != ksft_test_num()) + printf("# Planned tests != run tests (%u != %u)\n", + ksft_plan, ksft_test_num()); + printf("# Pass %d Fail %d Xfail %d Xpass %d Skip %d Error %d\n", ksft_cnt.ksft_pass, ksft_cnt.ksft_fail, ksft_cnt.ksft_xfail, ksft_cnt.ksft_xpass, ksft_cnt.ksft_xskip, ksft_cnt.ksft_error); - printf("1..%d\n", ksft_test_num()); } static inline void ksft_print_msg(const char *msg, ...) @@ -111,7 +120,7 @@ static inline void ksft_test_result_skip(const char *msg, ...) ksft_cnt.ksft_xskip++; va_start(args, msg); - printf("ok %d # skip ", ksft_test_num()); + printf("not ok %d # SKIP ", ksft_test_num()); vprintf(msg, args); va_end(args); } @@ -172,7 +181,7 @@ static inline int ksft_exit_skip(const char *msg, ...) va_list args; va_start(args, msg); - printf("1..%d # Skipped: ", ksft_test_num()); + printf("not ok %d # SKIP ", 1 + ksft_test_num()); vprintf(msg, args); va_end(args); } else { diff --git a/tools/testing/selftests/kselftest/prefix.pl b/tools/testing/selftests/kselftest/prefix.pl new file mode 100755 index 000000000000..ec7e48118183 --- /dev/null +++ b/tools/testing/selftests/kselftest/prefix.pl @@ -0,0 +1,23 @@ +#!/usr/bin/perl +# SPDX-License-Identifier: GPL-2.0 +# Prefix all lines with "# ", unbuffered. Command being piped in may need +# to have unbuffering forced with "stdbuf -i0 -o0 -e0 $cmd". +use strict; + +binmode STDIN; +binmode STDOUT; + +STDOUT->autoflush(1); + +my $needed = 1; +while (1) { + my $char; + my $bytes = sysread(STDIN, $char, 1); + exit 0 if ($bytes == 0); + if ($needed) { + print "# "; + $needed = 0; + } + print $char; + $needed = 1 if ($char eq "\n"); +} diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh new file mode 100644 index 000000000000..eff3ee303d0d --- /dev/null +++ b/tools/testing/selftests/kselftest/runner.sh @@ -0,0 +1,86 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Runs a set of tests in a given subdirectory. +export skip_rc=4 +export logfile=/dev/stdout +export per_test_logging= + +# There isn't a shell-agnostic way to find the path of a sourced file, +# so we must rely on BASE_DIR being set to find other tools. +if [ -z "$BASE_DIR" ]; then + echo "Error: BASE_DIR must be set before sourcing." >&2 + exit 1 +fi + +# If Perl is unavailable, we must fall back to line-at-a-time prefixing +# with sed instead of unbuffered output. +tap_prefix() +{ + if [ ! -x /usr/bin/perl ]; then + sed -e 's/^/# /' + else + "$BASE_DIR"/kselftest/prefix.pl + fi +} + +# If stdbuf is unavailable, we must fall back to line-at-a-time piping. +tap_unbuffer() +{ + if ! which stdbuf >/dev/null ; then + "$@" + else + stdbuf -i0 -o0 -e0 "$@" + fi +} + +run_one() +{ + DIR="$1" + TEST="$2" + NUM="$3" + + BASENAME_TEST=$(basename $TEST) + + TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST" + echo "# $TEST_HDR_MSG" + if [ ! -x "$TEST" ]; then + echo -n "# Warning: file $TEST is " + if [ ! -e "$TEST" ]; then + echo "missing!" + else + echo "not executable, correct this." + fi + echo "not ok $test_num $TEST_HDR_MSG" + else + cd `dirname $TEST` > /dev/null + (((((tap_unbuffer ./$BASENAME_TEST 2>&1; echo $? >&3) | + tap_prefix >&4) 3>&1) | + (read xs; exit $xs)) 4>>"$logfile" && + echo "ok $test_num $TEST_HDR_MSG") || + (if [ $? -eq $skip_rc ]; then \ + echo "not ok $test_num $TEST_HDR_MSG # SKIP" + else + echo "not ok $test_num $TEST_HDR_MSG" + fi) + cd - >/dev/null + fi +} + +run_many() +{ + echo "TAP version 13" + DIR=$(basename "$PWD") + test_num=0 + total=$(echo "$@" | wc -w) + echo "1..$total" + for TEST in "$@"; do + BASENAME_TEST=$(basename $TEST) + test_num=$(( test_num + 1 )) + if [ -n "$per_test_logging" ]; then + logfile="/tmp/$BASENAME_TEST" + cat /dev/null > "$logfile" + fi + run_one "$DIR" "$TEST" "$test_num" + done +} diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 2689d1ea6d7a..df1bf9230a74 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -1,9 +1,14 @@ /x86_64/cr4_cpuid_sync_test /x86_64/evmcs_test +/x86_64/hyperv_cpuid +/x86_64/kvm_create_max_vcpus /x86_64/platform_info_test /x86_64/set_sregs_test +/x86_64/smm_test +/x86_64/state_test /x86_64/sync_regs_test /x86_64/vmx_close_while_nested_test +/x86_64/vmx_set_nested_state_test /x86_64/vmx_tsc_adjust_test -/x86_64/state_test +/clear_dirty_log_test /dirty_log_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index f8588cca2bef..79c524395ebe 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -20,6 +20,8 @@ TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test TEST_GEN_PROGS_x86_64 += x86_64/smm_test +TEST_GEN_PROGS_x86_64 += x86_64/kvm_create_max_vcpus +TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += clear_dirty_log_test diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 93f99c6b7d79..f50a15c38f9b 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -314,7 +314,7 @@ static void run_test(enum vm_guest_mode mode, unsigned long iterations, #ifdef USE_CLEAR_DIRTY_LOG struct kvm_enable_cap cap = {}; - cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT; + cap.cap = KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2; cap.args[0] = 1; vm_enable_cap(vm, &cap); #endif @@ -430,7 +430,7 @@ int main(int argc, char *argv[]) int opt, i; #ifdef USE_CLEAR_DIRTY_LOG - if (!kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT)) { + if (!kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2)) { fprintf(stderr, "KVM_CLEAR_DIRTY_LOG not available, skipping tests\n"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 07b71ad9734a..8c6b9619797d 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -118,6 +118,10 @@ void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_events *events); void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_events *events); +void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid, + struct kvm_nested_state *state); +int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid, + struct kvm_nested_state *state, bool ignore_error); const char *exit_reason_str(unsigned int exit_reason); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 4ca96b228e46..e9113857f44e 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1250,6 +1250,38 @@ void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid, ret, errno); } +void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid, + struct kvm_nested_state *state) +{ + struct vcpu *vcpu = vcpu_find(vm, vcpuid); + int ret; + + TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid); + + ret = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, state); + TEST_ASSERT(ret == 0, + "KVM_SET_NESTED_STATE failed, ret: %i errno: %i", + ret, errno); +} + +int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid, + struct kvm_nested_state *state, bool ignore_error) +{ + struct vcpu *vcpu = vcpu_find(vm, vcpuid); + int ret; + + TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid); + + ret = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, state); + if (!ignore_error) { + TEST_ASSERT(ret == 0, + "KVM_SET_NESTED_STATE failed, ret: %i errno: %i", + ret, errno); + } + + return ret; +} + /* * VM VCPU System Regs Get * diff --git a/tools/testing/selftests/kvm/x86_64/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/x86_64/kvm_create_max_vcpus.c new file mode 100644 index 000000000000..50e92996f918 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/kvm_create_max_vcpus.c @@ -0,0 +1,70 @@ +/* + * kvm_create_max_vcpus + * + * Copyright (C) 2019, Google LLC. + * + * This work is licensed under the terms of the GNU GPL, version 2. + * + * Test for KVM_CAP_MAX_VCPUS and KVM_CAP_MAX_VCPU_ID. + */ + +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "test_util.h" + +#include "kvm_util.h" +#include "asm/kvm.h" +#include "linux/kvm.h" + +void test_vcpu_creation(int first_vcpu_id, int num_vcpus) +{ + struct kvm_vm *vm; + int i; + + printf("Testing creating %d vCPUs, with IDs %d...%d.\n", + num_vcpus, first_vcpu_id, first_vcpu_id + num_vcpus - 1); + + vm = vm_create(VM_MODE_P52V48_4K, DEFAULT_GUEST_PHY_PAGES, O_RDWR); + + for (i = 0; i < num_vcpus; i++) { + int vcpu_id = first_vcpu_id + i; + + /* This asserts that the vCPU was created. */ + vm_vcpu_add(vm, vcpu_id, 0, 0); + } + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + int kvm_max_vcpu_id = kvm_check_cap(KVM_CAP_MAX_VCPU_ID); + int kvm_max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); + + printf("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id); + printf("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus); + + /* + * Upstream KVM prior to 4.8 does not support KVM_CAP_MAX_VCPU_ID. + * Userspace is supposed to use KVM_CAP_MAX_VCPUS as the maximum ID + * in this case. + */ + if (!kvm_max_vcpu_id) + kvm_max_vcpu_id = kvm_max_vcpus; + + TEST_ASSERT(kvm_max_vcpu_id >= kvm_max_vcpus, + "KVM_MAX_VCPU_ID (%d) must be at least as large as KVM_MAX_VCPUS (%d).", + kvm_max_vcpu_id, kvm_max_vcpus); + + test_vcpu_creation(0, kvm_max_vcpus); + + if (kvm_max_vcpu_id > kvm_max_vcpus) + test_vcpu_creation( + kvm_max_vcpu_id - kvm_max_vcpus, kvm_max_vcpus); + + return 0; +} diff --git a/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c b/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c new file mode 100644 index 000000000000..61a2163cf9f1 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/vmx_set_nested_state_test.c @@ -0,0 +1,280 @@ +/* + * vmx_set_nested_state_test + * + * Copyright (C) 2019, Google LLC. + * + * This work is licensed under the terms of the GNU GPL, version 2. + * + * This test verifies the integrity of calling the ioctl KVM_SET_NESTED_STATE. + */ + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "vmx.h" + +#include <errno.h> +#include <linux/kvm.h> +#include <string.h> +#include <sys/ioctl.h> +#include <unistd.h> + +/* + * Mirror of VMCS12_REVISION in arch/x86/kvm/vmx/vmcs12.h. If that value + * changes this should be updated. + */ +#define VMCS12_REVISION 0x11e57ed0 +#define VCPU_ID 5 + +void test_nested_state(struct kvm_vm *vm, struct kvm_nested_state *state) +{ + volatile struct kvm_run *run; + + vcpu_nested_state_set(vm, VCPU_ID, state, false); + run = vcpu_state(vm, VCPU_ID); + vcpu_run(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN, + "Got exit_reason other than KVM_EXIT_SHUTDOWN: %u (%s),\n", + run->exit_reason, + exit_reason_str(run->exit_reason)); +} + +void test_nested_state_expect_errno(struct kvm_vm *vm, + struct kvm_nested_state *state, + int expected_errno) +{ + volatile struct kvm_run *run; + int rv; + + rv = vcpu_nested_state_set(vm, VCPU_ID, state, true); + TEST_ASSERT(rv == -1 && errno == expected_errno, + "Expected %s (%d) from vcpu_nested_state_set but got rv: %i errno: %s (%d)", + strerror(expected_errno), expected_errno, rv, strerror(errno), + errno); + run = vcpu_state(vm, VCPU_ID); + vcpu_run(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN, + "Got exit_reason other than KVM_EXIT_SHUTDOWN: %u (%s),\n", + run->exit_reason, + exit_reason_str(run->exit_reason)); +} + +void test_nested_state_expect_einval(struct kvm_vm *vm, + struct kvm_nested_state *state) +{ + test_nested_state_expect_errno(vm, state, EINVAL); +} + +void test_nested_state_expect_efault(struct kvm_vm *vm, + struct kvm_nested_state *state) +{ + test_nested_state_expect_errno(vm, state, EFAULT); +} + +void set_revision_id_for_vmcs12(struct kvm_nested_state *state, + u32 vmcs12_revision) +{ + /* Set revision_id in vmcs12 to vmcs12_revision. */ + *(u32 *)(state->data) = vmcs12_revision; +} + +void set_default_state(struct kvm_nested_state *state) +{ + memset(state, 0, sizeof(*state)); + state->flags = KVM_STATE_NESTED_RUN_PENDING | + KVM_STATE_NESTED_GUEST_MODE; + state->format = 0; + state->size = sizeof(*state); +} + +void set_default_vmx_state(struct kvm_nested_state *state, int size) +{ + memset(state, 0, size); + state->flags = KVM_STATE_NESTED_GUEST_MODE | + KVM_STATE_NESTED_RUN_PENDING | + KVM_STATE_NESTED_EVMCS; + state->format = 0; + state->size = size; + state->vmx.vmxon_pa = 0x1000; + state->vmx.vmcs_pa = 0x2000; + state->vmx.smm.flags = 0; + set_revision_id_for_vmcs12(state, VMCS12_REVISION); +} + +void test_vmx_nested_state(struct kvm_vm *vm) +{ + /* Add a page for VMCS12. */ + const int state_sz = sizeof(struct kvm_nested_state) + getpagesize(); + struct kvm_nested_state *state = + (struct kvm_nested_state *)malloc(state_sz); + + /* The format must be set to 0. 0 for VMX, 1 for SVM. */ + set_default_vmx_state(state, state_sz); + state->format = 1; + test_nested_state_expect_einval(vm, state); + + /* + * We cannot virtualize anything if the guest does not have VMX + * enabled. + */ + set_default_vmx_state(state, state_sz); + test_nested_state_expect_einval(vm, state); + + /* + * We cannot virtualize anything if the guest does not have VMX + * enabled. We expect KVM_SET_NESTED_STATE to return 0 if vmxon_pa + * is set to -1ull. + */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = -1ull; + test_nested_state(vm, state); + + /* Enable VMX in the guest CPUID. */ + vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); + + /* It is invalid to have vmxon_pa == -1ull and SMM flags non-zero. */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = -1ull; + state->vmx.smm.flags = 1; + test_nested_state_expect_einval(vm, state); + + /* It is invalid to have vmxon_pa == -1ull and vmcs_pa != -1ull. */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = -1ull; + state->vmx.vmcs_pa = 0; + test_nested_state_expect_einval(vm, state); + + /* + * Setting vmxon_pa == -1ull and vmcs_pa == -1ull exits early without + * setting the nested state. + */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = -1ull; + state->vmx.vmcs_pa = -1ull; + test_nested_state(vm, state); + + /* It is invalid to have vmxon_pa set to a non-page aligned address. */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = 1; + test_nested_state_expect_einval(vm, state); + + /* + * It is invalid to have KVM_STATE_NESTED_SMM_GUEST_MODE and + * KVM_STATE_NESTED_GUEST_MODE set together. + */ + set_default_vmx_state(state, state_sz); + state->flags = KVM_STATE_NESTED_GUEST_MODE | + KVM_STATE_NESTED_RUN_PENDING; + state->vmx.smm.flags = KVM_STATE_NESTED_SMM_GUEST_MODE; + test_nested_state_expect_einval(vm, state); + + /* + * It is invalid to have any of the SMM flags set besides: + * KVM_STATE_NESTED_SMM_GUEST_MODE + * KVM_STATE_NESTED_SMM_VMXON + */ + set_default_vmx_state(state, state_sz); + state->vmx.smm.flags = ~(KVM_STATE_NESTED_SMM_GUEST_MODE | + KVM_STATE_NESTED_SMM_VMXON); + test_nested_state_expect_einval(vm, state); + + /* Outside SMM, SMM flags must be zero. */ + set_default_vmx_state(state, state_sz); + state->flags = 0; + state->vmx.smm.flags = KVM_STATE_NESTED_SMM_GUEST_MODE; + test_nested_state_expect_einval(vm, state); + + /* Size must be large enough to fit kvm_nested_state and vmcs12. */ + set_default_vmx_state(state, state_sz); + state->size = sizeof(*state); + test_nested_state(vm, state); + + /* vmxon_pa cannot be the same address as vmcs_pa. */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = 0; + state->vmx.vmcs_pa = 0; + test_nested_state_expect_einval(vm, state); + + /* The revision id for vmcs12 must be VMCS12_REVISION. */ + set_default_vmx_state(state, state_sz); + set_revision_id_for_vmcs12(state, 0); + test_nested_state_expect_einval(vm, state); + + /* + * Test that if we leave nesting the state reflects that when we get + * it again. + */ + set_default_vmx_state(state, state_sz); + state->vmx.vmxon_pa = -1ull; + state->vmx.vmcs_pa = -1ull; + state->flags = 0; + test_nested_state(vm, state); + vcpu_nested_state_get(vm, VCPU_ID, state); + TEST_ASSERT(state->size >= sizeof(*state) && state->size <= state_sz, + "Size must be between %d and %d. The size returned was %d.", + sizeof(*state), state_sz, state->size); + TEST_ASSERT(state->vmx.vmxon_pa == -1ull, "vmxon_pa must be -1ull."); + TEST_ASSERT(state->vmx.vmcs_pa == -1ull, "vmcs_pa must be -1ull."); + + free(state); +} + +int main(int argc, char *argv[]) +{ + struct kvm_vm *vm; + struct kvm_nested_state state; + struct kvm_cpuid_entry2 *entry = kvm_get_supported_cpuid_entry(1); + + if (!kvm_check_cap(KVM_CAP_NESTED_STATE)) { + printf("KVM_CAP_NESTED_STATE not available, skipping test\n"); + exit(KSFT_SKIP); + } + + /* + * AMD currently does not implement set_nested_state, so for now we + * just early out. + */ + if (!(entry->ecx & CPUID_VMX)) { + fprintf(stderr, "nested VMX not enabled, skipping test\n"); + exit(KSFT_SKIP); + } + + vm = vm_create_default(VCPU_ID, 0, 0); + + /* Passing a NULL kvm_nested_state causes a EFAULT. */ + test_nested_state_expect_efault(vm, NULL); + + /* 'size' cannot be smaller than sizeof(kvm_nested_state). */ + set_default_state(&state); + state.size = 0; + test_nested_state_expect_einval(vm, &state); + + /* + * Setting the flags 0xf fails the flags check. The only flags that + * can be used are: + * KVM_STATE_NESTED_GUEST_MODE + * KVM_STATE_NESTED_RUN_PENDING + * KVM_STATE_NESTED_EVMCS + */ + set_default_state(&state); + state.flags = 0xf; + test_nested_state_expect_einval(vm, &state); + + /* + * If KVM_STATE_NESTED_RUN_PENDING is set then + * KVM_STATE_NESTED_GUEST_MODE has to be set as well. + */ + set_default_state(&state); + state.flags = KVM_STATE_NESTED_RUN_PENDING; + test_nested_state_expect_einval(vm, &state); + + /* + * TODO: When SVM support is added for KVM_SET_NESTED_STATE + * add tests here to support it like VMX. + */ + if (entry->ecx & CPUID_VMX) + test_vmx_nested_state(vm); + + kvm_vm_free(vm); + return 0; +} diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 5979fdc4f36c..077337195783 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -3,17 +3,12 @@ CC := $(CROSS_COMPILE)gcc ifeq (0,$(MAKELEVEL)) - ifneq ($(O),) - OUTPUT := $(O) - else - ifneq ($(KBUILD_OUTPUT),) - OUTPUT := $(KBUILD_OUTPUT) - else - OUTPUT := $(shell pwd) - DEFAULT_INSTALL_HDR_PATH := 1 - endif + ifeq ($(OUTPUT),) + OUTPUT := $(shell pwd) + DEFAULT_INSTALL_HDR_PATH := 1 endif endif +selfdir = $(realpath $(dir $(filter %/lib.mk,$(MAKEFILE_LIST)))) # The following are built by lib.mk common compile rules. # TEST_CUSTOM_PROGS should be used by tests that require @@ -65,44 +60,13 @@ all: $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED) $(TEST_GEN_FILES) endif .ONESHELL: -define RUN_TEST_PRINT_RESULT - TEST_HDR_MSG="selftests: "`basename $$PWD`:" $$BASENAME_TEST"; \ - echo $$TEST_HDR_MSG; \ - echo "========================================"; \ - if [ ! -x $$TEST ]; then \ - echo "$$TEST_HDR_MSG: Warning: file $$BASENAME_TEST is not executable, correct this.";\ - echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \ - else \ - cd `dirname $$TEST` > /dev/null; \ - if [ "X$(summary)" != "X" ]; then \ - (./$$BASENAME_TEST > /tmp/$$BASENAME_TEST 2>&1 && \ - echo "ok 1..$$test_num $$TEST_HDR_MSG [PASS]") || \ - (if [ $$? -eq $$skip ]; then \ - echo "not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]"; \ - else echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \ - fi;) \ - else \ - (./$$BASENAME_TEST && \ - echo "ok 1..$$test_num $$TEST_HDR_MSG [PASS]") || \ - (if [ $$? -eq $$skip ]; then \ - echo "not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]"; \ - else echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \ - fi;) \ - fi; \ - cd - > /dev/null; \ - fi; -endef - define RUN_TESTS - @export KSFT_TAP_LEVEL=`echo 1`; \ - test_num=`echo 0`; \ - skip=`echo 4`; \ - echo "TAP version 13"; \ - for TEST in $(1); do \ - BASENAME_TEST=`basename $$TEST`; \ - test_num=`echo $$test_num+1 | bc`; \ - $(call RUN_TEST_PRINT_RESULT,$(TEST),$(BASENAME_TEST),$(test_num),$(skip)) \ - done; + @BASE_DIR="$(selfdir)"; \ + . $(selfdir)/kselftest/runner.sh; \ + if [ "X$(summary)" != "X" ]; then \ + per_test_logging=1; \ + fi; \ + run_many $(1) endef run_tests: all @@ -139,24 +103,12 @@ else $(error Error: set INSTALL_PATH to use install) endif -define EMIT_TESTS - @test_num=`echo 0`; \ +emit_tests: for TEST in $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS); do \ BASENAME_TEST=`basename $$TEST`; \ - test_num=`echo $$test_num+1 | bc`; \ - TEST_HDR_MSG="selftests: "`basename $$PWD`:" $$BASENAME_TEST"; \ - echo "echo $$TEST_HDR_MSG"; \ - if [ ! -x $$TEST ]; then \ - echo "echo \"$$TEST_HDR_MSG: Warning: file $$BASENAME_TEST is not executable, correct this.\""; \ - echo "echo \"not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]\""; \ - else - echo "(./$$BASENAME_TEST >> \$$OUTPUT 2>&1 && echo \"ok 1..$$test_num $$TEST_HDR_MSG [PASS]\") || (if [ \$$? -eq \$$skip ]; then echo \"not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]\"; else echo \"not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]\"; fi;)"; \ - fi; \ - done; -endef - -emit_tests: - $(EMIT_TESTS) + echo " \\"; \ + echo -n " \"$$BASENAME_TEST\""; \ + done; \ # define if isn't already. It is undefined in make O= case. ifeq ($(RM),) diff --git a/tools/testing/selftests/membarrier/membarrier_test.c b/tools/testing/selftests/membarrier/membarrier_test.c index 6793f8ecc8e7..70b4ddbf126b 100644 --- a/tools/testing/selftests/membarrier/membarrier_test.c +++ b/tools/testing/selftests/membarrier/membarrier_test.c @@ -304,6 +304,7 @@ static int test_membarrier_query(void) int main(int argc, char **argv) { ksft_print_header(); + ksft_set_plan(13); test_membarrier_query(); test_membarrier(); diff --git a/tools/testing/selftests/pidfd/.gitignore b/tools/testing/selftests/pidfd/.gitignore new file mode 100644 index 000000000000..822a1e63d045 --- /dev/null +++ b/tools/testing/selftests/pidfd/.gitignore @@ -0,0 +1 @@ +pidfd_test diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c index d59378a93782..5bae1792e3d6 100644 --- a/tools/testing/selftests/pidfd/pidfd_test.c +++ b/tools/testing/selftests/pidfd/pidfd_test.c @@ -371,6 +371,7 @@ static int test_pidfd_send_signal_syscall_support(void) int main(int argc, char **argv) { ksft_print_header(); + ksft_set_plan(4); test_pidfd_send_signal_syscall_support(); test_pidfd_send_signal_simple_success(); diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile index c30c52e1d0d2..d6469535630a 100644 --- a/tools/testing/selftests/rseq/Makefile +++ b/tools/testing/selftests/rseq/Makefile @@ -1,5 +1,11 @@ # SPDX-License-Identifier: GPL-2.0+ OR MIT -CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L./ -Wl,-rpath=./ + +ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),) +CLANG_FLAGS += -no-integrated-as +endif + +CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L./ -Wl,-rpath=./ \ + $(CLANG_FLAGS) LDLIBS += -lpthread # Own dependencies because we only want to build against 1st prerequisite, but diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h index 3cea19877227..84f28f147fb6 100644 --- a/tools/testing/selftests/rseq/rseq-arm.h +++ b/tools/testing/selftests/rseq/rseq-arm.h @@ -5,7 +5,54 @@ * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ -#define RSEQ_SIG 0x53053053 +/* + * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand + * value 0x5de3. This traps if user-space reaches this instruction by mistake, + * and the uncommon operand ensures the kernel does not move the instruction + * pointer to attacker-controlled code on rseq abort. + * + * The instruction pattern in the A32 instruction set is: + * + * e7f5def3 udf #24035 ; 0x5de3 + * + * This translates to the following instruction pattern in the T16 instruction + * set: + * + * little endian: + * def3 udf #243 ; 0xf3 + * e7f5 b.n <7f5> + * + * pre-ARMv6 big endian code: + * e7f5 b.n <7f5> + * def3 udf #243 ; 0xf3 + * + * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian + * code and big-endian data. Ensure the RSEQ_SIG data signature matches code + * endianness. Prior to ARMv6, -mbig-endian generates big-endian code and data + * (which match), so there is no need to reverse the endianness of the data + * representation of the signature. However, the choice between BE32 and BE8 + * is done by the linker, so we cannot know whether code and data endianness + * will be mixed before the linker is invoked. + */ + +#define RSEQ_SIG_CODE 0xe7f5def3 + +#ifndef __ASSEMBLER__ + +#define RSEQ_SIG_DATA \ + ({ \ + int sig; \ + asm volatile ("b 2f\n\t" \ + "1: .inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \ + "2:\n\t" \ + "ldr %[sig], 1b\n\t" \ + : [sig] "=r" (sig)); \ + sig; \ + }) + +#define RSEQ_SIG RSEQ_SIG_DATA + +#endif #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") @@ -30,18 +77,35 @@ do { \ #include "rseq-skip.h" #else /* !RSEQ_SKIP_FASTPATH */ -#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ +#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ + __rseq_str(label) ":\n\t" \ ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".word " __rseq_str(label) "b, 0x0\n\t" \ ".popsection\n\t" -#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ - __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ +#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ + __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ (post_commit_ip - start_ip), abort_ip) +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ + ".popsection\n\t" + #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ "adr r0, " __rseq_str(cs_label) "\n\t" \ @@ -61,7 +125,8 @@ do { \ __rseq_str(table_label) ":\n\t" \ ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ - ".word " __rseq_str(RSEQ_SIG) "\n\t" \ + ".arm\n\t" \ + ".inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \ __rseq_str(label) ":\n\t" \ teardown \ "b %l[" __rseq_str(abort_label) "]\n\t" @@ -86,7 +151,12 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -148,7 +218,12 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -214,7 +289,10 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -266,7 +344,12 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -336,7 +419,12 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -407,7 +495,13 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -485,7 +579,12 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif "str %[src], %[rseq_scratch0]\n\t" "str %[dst], %[rseq_scratch1]\n\t" "str %[len], %[rseq_scratch2]\n\t" @@ -604,7 +703,12 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif "str %[src], %[rseq_scratch0]\n\t" "str %[dst], %[rseq_scratch1]\n\t" "str %[len], %[rseq_scratch2]\n\t" diff --git a/tools/testing/selftests/rseq/rseq-arm64.h b/tools/testing/selftests/rseq/rseq-arm64.h index 954f34671ca6..200dae9e4208 100644 --- a/tools/testing/selftests/rseq/rseq-arm64.h +++ b/tools/testing/selftests/rseq/rseq-arm64.h @@ -6,7 +6,20 @@ * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> */ -#define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0 */ +/* + * aarch64 -mbig-endian generates mixed endianness code vs data: + * little-endian code and big-endian data. Ensure the RSEQ_SIG signature + * matches code endianness. + */ +#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */ + +#ifdef __AARCH64EB__ +#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */ +#else +#define RSEQ_SIG_DATA RSEQ_SIG_CODE +#endif + +#define RSEQ_SIG RSEQ_SIG_DATA #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") @@ -82,19 +95,35 @@ do { \ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ post_commit_offset, abort_ip) \ - " .pushsection __rseq_table, \"aw\"\n" \ + " .pushsection __rseq_cs, \"aw\"\n" \ " .balign 32\n" \ __rseq_str(label) ":\n" \ " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ " .quad " __rseq_str(start_ip) ", " \ __rseq_str(post_commit_offset) ", " \ __rseq_str(abort_ip) "\n" \ + " .popsection\n\t" \ + " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \ + " .quad " __rseq_str(label) "b\n" \ " .popsection\n" #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ (post_commit_ip - start_ip), abort_ip) +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + " .pushsection __rseq_exit_point_array, \"aw\"\n" \ + " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ + " .popsection\n" + #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ @@ -105,7 +134,7 @@ do { \ #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ " b 222f\n" \ - " .inst " __rseq_str(RSEQ_SIG) "\n" \ + " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \ __rseq_str(label) ":\n" \ " b %l[" __rseq_str(abort_label) "]\n" \ "222:\n" @@ -182,6 +211,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -231,6 +265,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -282,6 +321,9 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -325,6 +367,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -379,6 +426,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -433,6 +485,12 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -490,6 +548,11 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) @@ -545,6 +608,11 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) diff --git a/tools/testing/selftests/rseq/rseq-mips.h b/tools/testing/selftests/rseq/rseq-mips.h index 7f48ecf46994..e989e7c14b09 100644 --- a/tools/testing/selftests/rseq/rseq-mips.h +++ b/tools/testing/selftests/rseq/rseq-mips.h @@ -7,7 +7,39 @@ * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ -#define RSEQ_SIG 0x53053053 +/* + * RSEQ_SIG uses the break instruction. The instruction pattern is: + * + * On MIPS: + * 0350000d break 0x350 + * + * On nanoMIPS: + * 00100350 break 0x350 + * + * On microMIPS: + * 0000d407 break 0x350 + * + * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit + * halfwords, so the signature halfwords need to be swapped accordingly for + * little-endian. + */ +#if defined(__nanomips__) +# ifdef __MIPSEL__ +# define RSEQ_SIG 0x03500010 +# else +# define RSEQ_SIG 0x00100350 +# endif +#elif defined(__mips_micromips) +# ifdef __MIPSEL__ +# define RSEQ_SIG 0xd4070000 +# else +# define RSEQ_SIG 0x0000d407 +# endif +#elif defined(__mips__) +# define RSEQ_SIG 0x0350000d +#else +/* Unknown MIPS architecture. */ +#endif #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory") #define rseq_smp_rmb() rseq_smp_mb() @@ -54,20 +86,38 @@ do { \ # error unsupported _MIPS_SZLONG #endif -#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ +#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ + __rseq_str(label) ":\n\t" \ ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \ ".popsection\n\t" -#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ - __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ +#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ + __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ (post_commit_ip - start_ip), abort_ip) +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ + LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \ + ".popsection\n\t" + #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ @@ -113,7 +163,12 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -173,7 +228,12 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -237,7 +297,10 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -289,7 +352,12 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -357,7 +425,12 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -426,7 +499,13 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -500,7 +579,12 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif LONG_S " %[src], %[rseq_scratch0]\n\t" LONG_S " %[dst], %[rseq_scratch1]\n\t" LONG_S " %[len], %[rseq_scratch2]\n\t" @@ -616,7 +700,12 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, rseq_workaround_gcc_asm_size_guess(); __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif LONG_S " %[src], %[rseq_scratch0]\n\t" LONG_S " %[dst], %[rseq_scratch1]\n\t" LONG_S " %[len], %[rseq_scratch2]\n\t" diff --git a/tools/testing/selftests/rseq/rseq-ppc.h b/tools/testing/selftests/rseq/rseq-ppc.h index 52630c9f42be..76be90196fe4 100644 --- a/tools/testing/selftests/rseq/rseq-ppc.h +++ b/tools/testing/selftests/rseq/rseq-ppc.h @@ -6,7 +6,15 @@ * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> */ -#define RSEQ_SIG 0x53053053 +/* + * RSEQ_SIG is used with the following trap instruction: + * + * powerpc-be: 0f e5 00 0b twui r5,11 + * powerpc64-le: 0b 00 e5 0f twui r5,11 + * powerpc64-be: 0f e5 00 0b twui r5,11 + */ + +#define RSEQ_SIG 0x0fe5000b #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") @@ -33,8 +41,8 @@ do { \ #else /* !RSEQ_SKIP_FASTPATH */ /* - * The __rseq_table section can be used by debuggers to better handle - * single-stepping through the restartable critical sections. + * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to + * better handle single-stepping through the restartable critical sections. */ #ifdef __PPC64__ @@ -46,11 +54,14 @@ do { \ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ __rseq_str(label) ":\n\t" \ ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".quad " __rseq_str(label) "b\n\t" \ ".popsection\n\t" #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ @@ -63,6 +74,19 @@ do { \ "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ __rseq_str(label) ":\n\t" +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ + ".popsection\n\t" + #else /* #ifdef __PPC64__ */ #define STORE_WORD "stw " @@ -72,12 +96,29 @@ do { \ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ __rseq_str(label) ":\n\t" \ ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ /* 32-bit only supported on BE */ \ ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".long 0x0, " __rseq_str(label) "b\n\t" \ + ".popsection\n\t" + +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + /* 32-bit only supported on BE */ \ + ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ ".popsection\n\t" #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ @@ -169,6 +210,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ @@ -224,6 +270,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ @@ -286,6 +337,9 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ @@ -337,6 +391,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ @@ -400,6 +459,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ @@ -465,6 +529,12 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ @@ -532,6 +602,11 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* setup for mempcy */ "mr %%r19, %[len]\n\t" "mr %%r20, %[src]\n\t" @@ -601,6 +676,11 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* setup for mempcy */ "mr %%r19, %[len]\n\t" "mr %%r20, %[src]\n\t" diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h index 0afdf7957974..8ef94ad1cbb4 100644 --- a/tools/testing/selftests/rseq/rseq-s390.h +++ b/tools/testing/selftests/rseq/rseq-s390.h @@ -44,22 +44,54 @@ do { \ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ __rseq_str(label) ":\n\t" \ ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".quad " __rseq_str(label) "b\n\t" \ + ".popsection\n\t" + +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ ".popsection\n\t" #elif __s390__ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ __rseq_str(label) ":\n\t" \ ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".long 0x0, " __rseq_str(label) "b\n\t" \ + ".popsection\n\t" + +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \ ".popsection\n\t" #define LONG_L "l" @@ -92,14 +124,14 @@ do { \ ".long " __rseq_str(RSEQ_SIG) "\n\t" \ __rseq_str(label) ":\n\t" \ teardown \ - "j %l[" __rseq_str(abort_label) "]\n\t" \ + "jg %l[" __rseq_str(abort_label) "]\n\t" \ ".popsection\n\t" #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ ".pushsection __rseq_failure, \"ax\"\n\t" \ __rseq_str(label) ":\n\t" \ teardown \ - "j %l[" __rseq_str(cmpfail_label) "]\n\t" \ + "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" static inline __attribute__((always_inline)) @@ -109,6 +141,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -167,6 +204,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -227,6 +269,9 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -275,6 +320,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -346,6 +396,12 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) @@ -414,6 +470,11 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif LONG_S " %[src], %[rseq_scratch0]\n\t" LONG_S " %[dst], %[rseq_scratch1]\n\t" LONG_S " %[len], %[rseq_scratch2]\n\t" diff --git a/tools/testing/selftests/rseq/rseq-x86.h b/tools/testing/selftests/rseq/rseq-x86.h index 089410a314e9..b2da6004fe30 100644 --- a/tools/testing/selftests/rseq/rseq-x86.h +++ b/tools/testing/selftests/rseq/rseq-x86.h @@ -7,8 +7,25 @@ #include <stdint.h> +/* + * RSEQ_SIG is used with the following reserved undefined instructions, which + * trap in user-space: + * + * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi + * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi + */ #define RSEQ_SIG 0x53053053 +/* + * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input + * operands, we cannot use "m" input operands, and rather pass the __rseq_abi + * address through a "r" input operand. + */ + +/* Offset of cpu_id and rseq_cs fields in struct rseq. */ +#define RSEQ_CPU_ID_OFFSET 4 +#define RSEQ_CS_OFFSET 8 + #ifdef __x86_64__ #define rseq_smp_mb() \ @@ -37,32 +54,49 @@ do { \ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ __rseq_str(label) ":\n\t" \ ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".quad " __rseq_str(label) "b\n\t" \ ".popsection\n\t" + #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ (post_commit_ip - start_ip), abort_ip) +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ + ".popsection\n\t" + #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ - "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t" \ + "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ __rseq_str(label) ":\n\t" #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ - "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ + "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ "jnz " __rseq_str(label) "\n\t" #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ ".pushsection __rseq_failure, \"ax\"\n\t" \ - /* Disassembler-friendly signature: nopl <sig>(%rip). */\ - ".byte 0x0f, 0x1f, 0x05\n\t" \ + /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \ + ".byte 0x0f, 0xb9, 0x3d\n\t" \ ".long " __rseq_str(RSEQ_SIG) "\n\t" \ __rseq_str(label) ":\n\t" \ teardown \ @@ -83,15 +117,20 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -102,8 +141,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -140,16 +178,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "movq %[v], %%rbx\n\t" "cmpq %%rbx, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "movq %[v], %%rbx\n\t" "cmpq %%rbx, %[expectnot]\n\t" "je %l[error2]\n\t" @@ -164,8 +207,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), @@ -199,12 +241,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) #endif /* final store */ "addq %[count], %[v]\n\t" @@ -213,8 +258,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [count] "er" (count) @@ -244,15 +288,20 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -266,8 +315,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -314,9 +362,15 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" @@ -325,7 +379,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" "cmpq %[v2], %[expect2]\n\t" @@ -338,8 +392,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), @@ -381,18 +434,23 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif "movq %[src], %[rseq_scratch0]\n\t" "movq %[dst], %[rseq_scratch1]\n\t" "movq %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) "cmpq %[v], %[expect]\n\t" "jnz 7f\n\t" #endif @@ -440,8 +498,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, #endif : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [expect] "r" (expect), @@ -520,31 +577,47 @@ do { \ */ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ - ".pushsection __rseq_table, \"aw\"\n\t" \ + ".pushsection __rseq_cs, \"aw\"\n\t" \ ".balign 32\n\t" \ __rseq_str(label) ":\n\t" \ ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \ + ".long " __rseq_str(label) "b, 0x0\n\t" \ ".popsection\n\t" #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ (post_commit_ip - start_ip), abort_ip) +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \ + ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ + ".popsection\n\t" + #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ - "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t" \ + "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ __rseq_str(label) ":\n\t" #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ - "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ + "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ "jnz " __rseq_str(label) "\n\t" #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ ".pushsection __rseq_failure, \"ax\"\n\t" \ - /* Disassembler-friendly signature: nopl <sig>. */ \ - ".byte 0x0f, 0x1f, 0x05\n\t" \ + /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \ + ".byte 0x0f, 0xb9, 0x3d\n\t" \ ".long " __rseq_str(RSEQ_SIG) "\n\t" \ __rseq_str(label) ":\n\t" \ teardown \ @@ -565,15 +638,20 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -584,8 +662,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -622,16 +699,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "movl %[v], %%ebx\n\t" "cmpl %%ebx, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "movl %[v], %%ebx\n\t" "cmpl %%ebx, %[expectnot]\n\t" "je %l[error2]\n\t" @@ -646,8 +728,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), @@ -681,12 +762,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) #endif /* final store */ "addl %[count], %[v]\n\t" @@ -695,8 +779,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [count] "ir" (count) @@ -726,15 +809,20 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -749,8 +837,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* try store input */ [v2] "m" (*v2), [newv2] "m" (newv2), @@ -788,16 +875,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %[v], %%eax\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "movl %[expect], %%eax\n\t" "cmpl %[v], %%eax\n\t" "jnz %l[error2]\n\t" @@ -813,8 +905,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -853,9 +944,15 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" @@ -864,7 +961,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" "cmpl %[expect2], %[v2]\n\t" @@ -878,8 +975,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), @@ -922,19 +1018,24 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif "movl %[src], %[rseq_scratch0]\n\t" "movl %[dst], %[rseq_scratch1]\n\t" "movl %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 7f\n\t" @@ -984,8 +1085,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, #endif : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [expect] "m" (expect), @@ -1030,19 +1130,24 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif "movl %[src], %[rseq_scratch0]\n\t" "movl %[dst], %[rseq_scratch1]\n\t" "movl %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 7f\n\t" @@ -1093,8 +1198,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, #endif : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), + [rseq_abi] "r" (&__rseq_abi), /* final store input */ [v] "m" (*v), [expect] "m" (expect), diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index 4847e97ed049..7159eb777fd3 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -25,18 +25,27 @@ #include <syscall.h> #include <assert.h> #include <signal.h> +#include <limits.h> #include "rseq.h" #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) -__attribute__((tls_model("initial-exec"))) __thread -volatile struct rseq __rseq_abi = { +__thread volatile struct rseq __rseq_abi = { .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, }; -static __attribute__((tls_model("initial-exec"))) __thread -volatile int refcount; +/* + * Shared with other libraries. This library may take rseq ownership if it is + * still 0 when executing the library constructor. Set to 1 by library + * constructor when handling rseq. Set to 0 in destructor if handling rseq. + */ +int __rseq_handled; + +/* Whether this library have ownership of rseq registration. */ +static int rseq_ownership; + +static __thread volatile uint32_t __rseq_refcount; static void signal_off_save(sigset_t *oldset) { @@ -69,8 +78,14 @@ int rseq_register_current_thread(void) int rc, ret = 0; sigset_t oldset; + if (!rseq_ownership) + return 0; signal_off_save(&oldset); - if (refcount++) + if (__rseq_refcount == UINT_MAX) { + ret = -1; + goto end; + } + if (__rseq_refcount++) goto end; rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG); if (!rc) { @@ -78,9 +93,9 @@ int rseq_register_current_thread(void) goto end; } if (errno != EBUSY) - __rseq_abi.cpu_id = -2; + __rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED; ret = -1; - refcount--; + __rseq_refcount--; end: signal_restore(oldset); return ret; @@ -91,13 +106,20 @@ int rseq_unregister_current_thread(void) int rc, ret = 0; sigset_t oldset; + if (!rseq_ownership) + return 0; signal_off_save(&oldset); - if (--refcount) + if (!__rseq_refcount) { + ret = -1; + goto end; + } + if (--__rseq_refcount) goto end; rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, RSEQ_SIG); if (!rc) goto end; + __rseq_refcount = 1; ret = -1; end: signal_restore(oldset); @@ -115,3 +137,20 @@ int32_t rseq_fallback_current_cpu(void) } return cpu; } + +void __attribute__((constructor)) rseq_init(void) +{ + /* Check whether rseq is handled by another library. */ + if (__rseq_handled) + return; + __rseq_handled = 1; + rseq_ownership = 1; +} + +void __attribute__((destructor)) rseq_fini(void) +{ + if (!rseq_ownership) + return; + __rseq_handled = 0; + rseq_ownership = 0; +} diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h index 6c1126e7f685..d40d60e7499e 100644 --- a/tools/testing/selftests/rseq/rseq.h +++ b/tools/testing/selftests/rseq/rseq.h @@ -44,6 +44,7 @@ #endif extern __thread volatile struct rseq __rseq_abi; +extern int __rseq_handled; #define rseq_likely(x) __builtin_expect(!!(x), 1) #define rseq_unlikely(x) __builtin_expect(!!(x), 0) diff --git a/tools/testing/selftests/sigaltstack/sas.c b/tools/testing/selftests/sigaltstack/sas.c index 228c2ae47687..ad0f8df2ca0a 100644 --- a/tools/testing/selftests/sigaltstack/sas.c +++ b/tools/testing/selftests/sigaltstack/sas.c @@ -109,6 +109,7 @@ int main(void) int err; ksft_print_header(); + ksft_set_plan(3); sigemptyset(&act.sa_mask); act.sa_flags = SA_ONSTACK | SA_SIGINFO; diff --git a/tools/testing/selftests/sync/sync_test.c b/tools/testing/selftests/sync/sync_test.c index 7f7938263c5c..3824b66f41a0 100644 --- a/tools/testing/selftests/sync/sync_test.c +++ b/tools/testing/selftests/sync/sync_test.c @@ -86,6 +86,7 @@ int main(void) int err; ksft_print_header(); + ksft_set_plan(3 + 7); sync_api_supported(); diff --git a/tools/testing/selftests/sysctl/sysctl.sh b/tools/testing/selftests/sysctl/sysctl.sh index 780ce7123374..6a970b127c9b 100755 --- a/tools/testing/selftests/sysctl/sysctl.sh +++ b/tools/testing/selftests/sysctl/sysctl.sh @@ -24,19 +24,21 @@ TEST_FILE=$(mktemp) # This represents # -# TEST_ID:TEST_COUNT:ENABLED +# TEST_ID:TEST_COUNT:ENABLED:TARGET # # TEST_ID: is the test id number # TEST_COUNT: number of times we should run the test # ENABLED: 1 if enabled, 0 otherwise +# TARGET: test target file required on the test_sysctl module # # Once these are enabled please leave them as-is. Write your own test, # we have tons of space. -ALL_TESTS="0001:1:1" -ALL_TESTS="$ALL_TESTS 0002:1:1" -ALL_TESTS="$ALL_TESTS 0003:1:1" -ALL_TESTS="$ALL_TESTS 0004:1:1" -ALL_TESTS="$ALL_TESTS 0005:3:1" +ALL_TESTS="0001:1:1:int_0001" +ALL_TESTS="$ALL_TESTS 0002:1:1:string_0001" +ALL_TESTS="$ALL_TESTS 0003:1:1:int_0002" +ALL_TESTS="$ALL_TESTS 0004:1:1:uint_0001" +ALL_TESTS="$ALL_TESTS 0005:3:1:int_0003" +ALL_TESTS="$ALL_TESTS 0006:50:1:bitmap_0001" test_modprobe() { @@ -149,6 +151,9 @@ reset_vals() string_0001) VAL="(none)" ;; + bitmap_0001) + VAL="" + ;; *) ;; esac @@ -157,8 +162,10 @@ reset_vals() set_orig() { - if [ ! -z $TARGET ]; then - echo "${ORIG}" > "${TARGET}" + if [ ! -z $TARGET ] && [ ! -z $ORIG ]; then + if [ -f ${TARGET} ]; then + echo "${ORIG}" > "${TARGET}" + fi fi } @@ -177,9 +184,25 @@ verify() return 0 } +# proc files get read a page at a time, which can confuse diff, +# and get you incorrect results on proc files with long data. To use +# diff against them you must first extract the output to a file, and +# then compare against that file. +verify_diff_proc_file() +{ + TMP_DUMP_FILE=$(mktemp) + cat $1 > $TMP_DUMP_FILE + + if ! diff -w -q $TMP_DUMP_FILE $2; then + return 1 + else + return 0 + fi +} + verify_diff_w() { - echo "$TEST_STR" | diff -q -w -u - $1 + echo "$TEST_STR" | diff -q -w -u - $1 > /dev/null return $? } @@ -600,9 +623,70 @@ run_stringtests() test_rc } +target_exists() +{ + TARGET="${SYSCTL}/$1" + TEST_ID="$2" + + if [ ! -f ${TARGET} ] ; then + echo "Target for test $TEST_ID: $TARGET not exist, skipping test ..." + return 0 + fi + return 1 +} + +run_bitmaptest() { + # Total length of bitmaps string to use, a bit under + # the maximum input size of the test node + LENGTH=$((RANDOM % 65000)) + + # First bit to set + BIT=$((RANDOM % 1024)) + + # String containing our list of bits to set + TEST_STR=$BIT + + # build up the string + while [ "${#TEST_STR}" -le "$LENGTH" ]; do + # Make sure next entry is discontiguous, + # skip ahead at least 2 + BIT=$((BIT + $((2 + RANDOM % 10)))) + + # Add new bit to the list + TEST_STR="${TEST_STR},${BIT}" + + # Randomly make it a range + if [ "$((RANDOM % 2))" -eq "1" ]; then + RANGE_END=$((BIT + $((1 + RANDOM % 10)))) + TEST_STR="${TEST_STR}-${RANGE_END}" + BIT=$RANGE_END + fi + done + + echo -n "Checking bitmap handler... " + TEST_FILE=$(mktemp) + echo -n "$TEST_STR" > $TEST_FILE + + cat $TEST_FILE > $TARGET 2> /dev/null + if [ $? -ne 0 ]; then + echo "FAIL" >&2 + rc=1 + test_rc + fi + + if ! verify_diff_proc_file "$TARGET" "$TEST_FILE"; then + echo "FAIL" >&2 + rc=1 + else + echo "ok" + rc=0 + fi + test_rc +} + sysctl_test_0001() { - TARGET="${SYSCTL}/int_0001" + TARGET="${SYSCTL}/$(get_test_target 0001)" reset_vals ORIG=$(cat "${TARGET}") TEST_STR=$(( $ORIG + 1 )) @@ -614,7 +698,7 @@ sysctl_test_0001() sysctl_test_0002() { - TARGET="${SYSCTL}/string_0001" + TARGET="${SYSCTL}/$(get_test_target 0002)" reset_vals ORIG=$(cat "${TARGET}") TEST_STR="Testing sysctl" @@ -627,7 +711,7 @@ sysctl_test_0002() sysctl_test_0003() { - TARGET="${SYSCTL}/int_0002" + TARGET="${SYSCTL}/$(get_test_target 0003)" reset_vals ORIG=$(cat "${TARGET}") TEST_STR=$(( $ORIG + 1 )) @@ -640,7 +724,7 @@ sysctl_test_0003() sysctl_test_0004() { - TARGET="${SYSCTL}/uint_0001" + TARGET="${SYSCTL}/$(get_test_target 0004)" reset_vals ORIG=$(cat "${TARGET}") TEST_STR=$(( $ORIG + 1 )) @@ -653,13 +737,21 @@ sysctl_test_0004() sysctl_test_0005() { - TARGET="${SYSCTL}/int_0003" + TARGET="${SYSCTL}/$(get_test_target 0005)" reset_vals ORIG=$(cat "${TARGET}") run_limit_digit_int_array } +sysctl_test_0006() +{ + TARGET="${SYSCTL}/bitmap_0001" + reset_vals + ORIG="" + run_bitmaptest +} + list_tests() { echo "Test ID list:" @@ -673,10 +765,9 @@ list_tests() echo "0003 x $(get_test_count 0003) - tests proc_dointvec()" echo "0004 x $(get_test_count 0004) - tests proc_douintvec()" echo "0005 x $(get_test_count 0005) - tests proc_douintvec() array" + echo "0006 x $(get_test_count 0006) - tests proc_do_large_bitmap()" } -test_reqs - usage() { NUM_TESTS=$(grep -o ' ' <<<"$ALL_TESTS" | grep -c .) @@ -724,25 +815,35 @@ function get_test_count() { test_num $1 TEST_DATA=$(echo $ALL_TESTS | awk '{print $'$1'}') - LAST_TWO=${TEST_DATA#*:*} - echo ${LAST_TWO%:*} + echo ${TEST_DATA} | awk -F":" '{print $2}' } function get_test_enabled() { test_num $1 TEST_DATA=$(echo $ALL_TESTS | awk '{print $'$1'}') - echo ${TEST_DATA#*:*:} + echo ${TEST_DATA} | awk -F":" '{print $3}' +} + +function get_test_target() +{ + test_num $1 + TEST_DATA=$(echo $ALL_TESTS | awk '{print $'$1'}') + echo ${TEST_DATA} | awk -F":" '{print $4}' } function run_all_tests() { for i in $ALL_TESTS ; do - TEST_ID=${i%:*:*} + TEST_ID=${i%:*:*:*} ENABLED=$(get_test_enabled $TEST_ID) TEST_COUNT=$(get_test_count $TEST_ID) + TEST_TARGET=$(get_test_target $TEST_ID) + if target_exists $TEST_TARGET $TEST_ID; then + continue + fi if [[ $ENABLED -eq "1" ]]; then - test_case $TEST_ID $TEST_COUNT + test_case $TEST_ID $TEST_COUNT $TEST_TARGET fi done } @@ -775,12 +876,14 @@ function watch_case() function test_case() { - NUM_TESTS=$DEFAULT_NUM_TESTS - if [ $# -eq 2 ]; then - NUM_TESTS=$2 - fi + NUM_TESTS=$2 i=0 + + if target_exists $3 $1; then + continue + fi + while [ $i -lt $NUM_TESTS ]; do test_num $1 watch_log $i ${TEST_NAME}_test_$1 noclear @@ -803,15 +906,15 @@ function parse_args() elif [[ "$1" = "-t" ]]; then shift test_num $1 - test_case $1 $(get_test_count $1) + test_case $1 $(get_test_count $1) $(get_test_target $1) elif [[ "$1" = "-c" ]]; then shift test_num $1 test_num $2 - test_case $1 $2 + test_case $1 $2 $(get_test_target $1) elif [[ "$1" = "-s" ]]; then shift - test_case $1 1 + test_case $1 1 $(get_test_target $1) elif [[ "$1" = "-l" ]]; then list_tests elif [[ "$1" = "-h" || "$1" = "--help" ]]; then @@ -825,8 +928,8 @@ function parse_args() test_reqs allow_user_defaults check_production_sysctl_writes_strict -test_modprobe load_req_mod +test_modprobe trap "test_finish" EXIT |