diff options
Diffstat (limited to 'tools/testing/selftests/rseq')
31 files changed, 5363 insertions, 4100 deletions
diff --git a/tools/testing/selftests/rseq/.gitignore b/tools/testing/selftests/rseq/.gitignore index 5910888ebfe1..16496de5f6ce 100644 --- a/tools/testing/selftests/rseq/.gitignore +++ b/tools/testing/selftests/rseq/.gitignore @@ -1,7 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-only basic_percpu_ops_test +basic_percpu_ops_mm_cid_test basic_test basic_rseq_op_test param_test param_test_benchmark param_test_compare_twice +param_test_mm_cid +param_test_mm_cid_benchmark +param_test_mm_cid_compare_twice diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile index 2af9d39a9716..5a3432fceb58 100644 --- a/tools/testing/selftests/rseq/Makefile +++ b/tools/testing/selftests/rseq/Makefile @@ -4,16 +4,19 @@ ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),) CLANG_FLAGS += -no-integrated-as endif -CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L$(OUTPUT) -Wl,-rpath=./ \ - $(CLANG_FLAGS) -LDLIBS += -lpthread +top_srcdir = ../../../.. + +CFLAGS += -O2 -Wall -g -I./ $(KHDR_INCLUDES) -L$(OUTPUT) -Wl,-rpath=./ \ + $(CLANG_FLAGS) -I$(top_srcdir)/tools/include +LDLIBS += -lpthread -ldl # Own dependencies because we only want to build against 1st prerequisite, but # still track changes to header files and depend on shared object. OVERRIDE_TARGETS = 1 -TEST_GEN_PROGS = basic_test basic_percpu_ops_test param_test \ - param_test_benchmark param_test_compare_twice +TEST_GEN_PROGS = basic_test basic_percpu_ops_test basic_percpu_ops_mm_cid_test param_test \ + param_test_benchmark param_test_compare_twice param_test_mm_cid \ + param_test_mm_cid_benchmark param_test_mm_cid_compare_twice TEST_GEN_PROGS_EXTENDED = librseq.so @@ -29,6 +32,9 @@ $(OUTPUT)/librseq.so: rseq.c rseq.h rseq-*.h $(OUTPUT)/%: %.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h $(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -o $@ +$(OUTPUT)/basic_percpu_ops_mm_cid_test: basic_percpu_ops_test.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID $< $(LDLIBS) -lrseq -o $@ + $(OUTPUT)/param_test_benchmark: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ rseq.h rseq-*.h $(CC) $(CFLAGS) -DBENCHMARK $< $(LDLIBS) -lrseq -o $@ @@ -36,3 +42,15 @@ $(OUTPUT)/param_test_benchmark: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ $(OUTPUT)/param_test_compare_twice: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ rseq.h rseq-*.h $(CC) $(CFLAGS) -DRSEQ_COMPARE_TWICE $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/param_test_mm_cid: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ + rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/param_test_mm_cid_benchmark: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ + rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID -DBENCHMARK $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/param_test_mm_cid_compare_twice: param_test.c $(TEST_GEN_PROGS_EXTENDED) \ + rseq.h rseq-*.h + $(CC) $(CFLAGS) -DBUILDOPT_RSEQ_PERCPU_MM_CID -DRSEQ_COMPARE_TWICE $< $(LDLIBS) -lrseq -o $@ diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.c b/tools/testing/selftests/rseq/basic_percpu_ops_test.c index eb3f6db36d36..2348d2c20d0a 100644 --- a/tools/testing/selftests/rseq/basic_percpu_ops_test.c +++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.c @@ -9,9 +9,44 @@ #include <string.h> #include <stddef.h> +#include "../kselftest.h" #include "rseq.h" -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID +# define RSEQ_PERCPU RSEQ_PERCPU_MM_CID +static +int get_current_cpu_id(void) +{ + return rseq_current_mm_cid(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_mm_cid_available(); +} +static +bool rseq_use_cpu_index(void) +{ + return false; /* Use mm_cid */ +} +#else +# define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID +static +int get_current_cpu_id(void) +{ + return rseq_cpu_start(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_current_cpu_raw() >= 0; +} +static +bool rseq_use_cpu_index(void) +{ + return true; /* Use cpu_id as index. */ +} +#endif struct percpu_lock_entry { intptr_t v; @@ -52,9 +87,9 @@ int rseq_this_cpu_lock(struct percpu_lock *lock) for (;;) { int ret; - cpu = rseq_cpu_start(); - ret = rseq_cmpeqv_storev(&lock->c[cpu].v, - 0, 1, cpu); + cpu = get_current_cpu_id(); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -142,13 +177,14 @@ void this_cpu_list_push(struct percpu_list *list, intptr_t *targetptr, newval, expect; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; - ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -168,15 +204,16 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, for (;;) { struct percpu_list_node *head; intptr_t *targetptr, expectnot, *load; - off_t offset; + long offset; int ret, cpu; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; - ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, + ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expectnot, offset, load, cpu); if (rseq_likely(!ret)) { if (_cpu) @@ -247,7 +284,7 @@ void test_percpu_list(void) /* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; for (j = 1; j <= 100; j++) { struct percpu_list_node *node; @@ -272,7 +309,7 @@ void test_percpu_list(void) for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_list_node *node; - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; while ((node = __percpu_list_pop(&list, i))) { @@ -296,6 +333,10 @@ int main(int argc, char **argv) errno, strerror(errno)); goto error; } + if (!rseq_validate_cpu_id()) { + fprintf(stderr, "Error: cpu id getter unavailable\n"); + goto error; + } printf("spinlock\n"); test_percpu_spinlock(); printf("percpu_list\n"); diff --git a/tools/testing/selftests/rseq/basic_test.c b/tools/testing/selftests/rseq/basic_test.c index d8efbfb89193..295eea16466f 100644 --- a/tools/testing/selftests/rseq/basic_test.c +++ b/tools/testing/selftests/rseq/basic_test.c @@ -22,6 +22,8 @@ void test_cpu_pointer(void) CPU_ZERO(&test_affinity); for (i = 0; i < CPU_SETSIZE; i++) { if (CPU_ISSET(i, &affinity)) { + int node; + CPU_SET(i, &test_affinity); sched_setaffinity(0, sizeof(test_affinity), &test_affinity); @@ -29,6 +31,8 @@ void test_cpu_pointer(void) assert(rseq_current_cpu() == i); assert(rseq_current_cpu_raw() == i); assert(rseq_cpu_start() == i); + node = rseq_fallback_current_node(); + assert(rseq_current_node_id() == node); CPU_CLR(i, &test_affinity); } } diff --git a/tools/testing/selftests/rseq/compiler.h b/tools/testing/selftests/rseq/compiler.h new file mode 100644 index 000000000000..49d62fbd6dda --- /dev/null +++ b/tools/testing/selftests/rseq/compiler.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ +/* + * rseq/compiler.h + * + * Work-around asm goto compiler bugs. + * + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifndef RSEQ_COMPILER_H +#define RSEQ_COMPILER_H + +/* + * gcc prior to 4.8.2 miscompiles asm goto. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 + * + * gcc prior to 8.1.0 miscompiles asm goto at O1. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103908 + * + * clang prior to version 13.0.1 miscompiles asm goto at O2. + * https://github.com/llvm/llvm-project/issues/52735 + * + * Work around these issues by adding a volatile inline asm with + * memory clobber in the fallthrough after the asm goto and at each + * label target. Emit this for all compilers in case other similar + * issues are found in the future. + */ +#define rseq_after_asm_goto() asm volatile ("" : : : "memory") + +/* Combine two tokens. */ +#define RSEQ__COMBINE_TOKENS(_tokena, _tokenb) \ + _tokena##_tokenb +#define RSEQ_COMBINE_TOKENS(_tokena, _tokenb) \ + RSEQ__COMBINE_TOKENS(_tokena, _tokenb) + +#ifdef __cplusplus +#define rseq_unqual_scalar_typeof(x) \ + std::remove_cv<std::remove_reference<decltype(x)>::type>::type +#else +#define rseq_scalar_type_to_expr(type) \ + unsigned type: (unsigned type)0, \ + signed type: (signed type)0 + +/* + * Use C11 _Generic to express unqualified type from expression. This removes + * volatile qualifier from expression type. + */ +#define rseq_unqual_scalar_typeof(x) \ + __typeof__( \ + _Generic((x), \ + char: (char)0, \ + rseq_scalar_type_to_expr(char), \ + rseq_scalar_type_to_expr(short), \ + rseq_scalar_type_to_expr(int), \ + rseq_scalar_type_to_expr(long), \ + rseq_scalar_type_to_expr(long long), \ + default: (x) \ + ) \ + ) +#endif + +#endif /* RSEQ_COMPILER_H_ */ diff --git a/tools/testing/selftests/rseq/param_test.c b/tools/testing/selftests/rseq/param_test.c index e8a657a5f48a..2f37961240ca 100644 --- a/tools/testing/selftests/rseq/param_test.c +++ b/tools/testing/selftests/rseq/param_test.c @@ -1,8 +1,10 @@ // SPDX-License-Identifier: LGPL-2.1 #define _GNU_SOURCE #include <assert.h> +#include <linux/membarrier.h> #include <pthread.h> #include <sched.h> +#include <stdatomic.h> #include <stdint.h> #include <stdio.h> #include <stdlib.h> @@ -14,6 +16,7 @@ #include <signal.h> #include <errno.h> #include <stddef.h> +#include <stdbool.h> static inline pid_t rseq_gettid(void) { @@ -34,13 +37,9 @@ static int opt_modulo, verbose; static int opt_yield, opt_signal, opt_sleep, opt_disable_rseq, opt_threads = 200, - opt_disable_mod = 0, opt_test = 's', opt_mb = 0; + opt_disable_mod = 0, opt_test = 's'; -#ifndef RSEQ_SKIP_FASTPATH static long long opt_reps = 5000; -#else -static long long opt_reps = 100; -#endif static __thread __attribute__((tls_model("initial-exec"))) unsigned int signals_delivered; @@ -159,7 +158,7 @@ unsigned int yield_mod_cnt, nr_abort; " cbnz " INJECT_ASM_REG ", 222b\n" \ "333:\n" -#elif __PPC__ +#elif defined(__PPC__) #define RSEQ_INJECT_INPUT \ , [loop_cnt_1]"m"(loop_cnt[1]) \ @@ -205,6 +204,29 @@ unsigned int yield_mod_cnt, nr_abort; "addiu " INJECT_ASM_REG ", -1\n\t" \ "bnez " INJECT_ASM_REG ", 222b\n\t" \ "333:\n\t" +#elif defined(__riscv) + +#define RSEQ_INJECT_INPUT \ + , [loop_cnt_1]"m"(loop_cnt[1]) \ + , [loop_cnt_2]"m"(loop_cnt[2]) \ + , [loop_cnt_3]"m"(loop_cnt[3]) \ + , [loop_cnt_4]"m"(loop_cnt[4]) \ + , [loop_cnt_5]"m"(loop_cnt[5]) \ + , [loop_cnt_6]"m"(loop_cnt[6]) + +#define INJECT_ASM_REG "t1" + +#define RSEQ_INJECT_CLOBBER \ + , INJECT_ASM_REG + +#define RSEQ_INJECT_ASM(n) \ + "lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \ + "beqz " INJECT_ASM_REG ", 333f\n\t" \ + "222:\n\t" \ + "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \ + "bnez " INJECT_ASM_REG ", 222b\n\t" \ + "333:\n\t" + #else #error unsupported target @@ -243,6 +265,73 @@ unsigned int yield_mod_cnt, nr_abort; #include "rseq.h" +static enum rseq_mo opt_mo = RSEQ_MO_RELAXED; + +#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +#define TEST_MEMBARRIER + +static int sys_membarrier(int cmd, int flags, int cpu_id) +{ + return syscall(__NR_membarrier, cmd, flags, cpu_id); +} +#endif + +#ifdef BUILDOPT_RSEQ_PERCPU_MM_CID +# define RSEQ_PERCPU RSEQ_PERCPU_MM_CID +static +int get_current_cpu_id(void) +{ + return rseq_current_mm_cid(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_mm_cid_available(); +} +static +bool rseq_use_cpu_index(void) +{ + return false; /* Use mm_cid */ +} +# ifdef TEST_MEMBARRIER +/* + * Membarrier does not currently support targeting a mm_cid, so + * issue the barrier on all cpus. + */ +static +int rseq_membarrier_expedited(int cpu) +{ + return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + 0, 0); +} +# endif /* TEST_MEMBARRIER */ +#else +# define RSEQ_PERCPU RSEQ_PERCPU_CPU_ID +static +int get_current_cpu_id(void) +{ + return rseq_cpu_start(); +} +static +bool rseq_validate_cpu_id(void) +{ + return rseq_current_cpu_raw() >= 0; +} +static +bool rseq_use_cpu_index(void) +{ + return true; /* Use cpu_id as index. */ +} +# ifdef TEST_MEMBARRIER +static +int rseq_membarrier_expedited(int cpu) +{ + return sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ, + MEMBARRIER_CMD_FLAG_CPU, cpu); +} +# endif /* TEST_MEMBARRIER */ +#endif + struct percpu_lock_entry { intptr_t v; } __attribute__((aligned(128))); @@ -330,8 +419,14 @@ static int rseq_this_cpu_lock(struct percpu_lock *lock) for (;;) { int ret; - cpu = rseq_cpu_start(); - ret = rseq_cmpeqv_storev(&lock->c[cpu].v, + cpu = get_current_cpu_id(); + if (cpu < 0) { + fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n", + getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu); + abort(); + } + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &lock->c[cpu].v, 0, 1, cpu); if (rseq_likely(!ret)) break; @@ -366,9 +461,7 @@ void *test_percpu_spinlock_thread(void *arg) abort(); reps = thread_data->reps; for (i = 0; i < reps; i++) { - int cpu = rseq_cpu_start(); - - cpu = rseq_this_cpu_lock(&data->lock); + int cpu = rseq_this_cpu_lock(&data->lock); data->c[cpu].count++; rseq_percpu_unlock(&data->lock, cpu); #ifndef BENCHMARK @@ -450,8 +543,9 @@ void *test_percpu_inc_thread(void *arg) do { int cpu; - cpu = rseq_cpu_start(); - ret = rseq_addv(&data->c[cpu].count, 1, cpu); + cpu = get_current_cpu_id(); + ret = rseq_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &data->c[cpu].count, 1, cpu); } while (rseq_unlikely(ret)); #ifndef BENCHMARK if (i != 0 && !(i % (reps / 10))) @@ -520,13 +614,14 @@ void this_cpu_list_push(struct percpu_list *list, intptr_t *targetptr, newval, expect; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load list->c[cpu].head with single-copy atomicity. */ expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); newval = (intptr_t)node; targetptr = (intptr_t *)&list->c[cpu].head; node->next = (struct percpu_list_node *)expect; - ret = rseq_cmpeqv_storev(targetptr, expect, newval, cpu); + ret = rseq_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expect, newval, cpu); if (rseq_likely(!ret)) break; /* Retry if comparison fails or rseq aborts. */ @@ -549,16 +644,17 @@ struct percpu_list_node *this_cpu_list_pop(struct percpu_list *list, for (;;) { struct percpu_list_node *head; intptr_t *targetptr, expectnot, *load; - off_t offset; + long offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); targetptr = (intptr_t *)&list->c[cpu].head; expectnot = (intptr_t)NULL; offset = offsetof(struct percpu_list_node, next); load = (intptr_t *)&head; - ret = rseq_cmpnev_storeoffp_load(targetptr, expectnot, - offset, load, cpu); + ret = rseq_cmpnev_storeoffp_load(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, expectnot, + offset, load, cpu); if (rseq_likely(!ret)) { node = head; break; @@ -629,7 +725,7 @@ void test_percpu_list(void) /* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; for (j = 1; j <= 100; j++) { struct percpu_list_node *node; @@ -666,7 +762,7 @@ void test_percpu_list(void) for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_list_node *node; - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; while ((node = __percpu_list_pop(&list, i))) { @@ -696,7 +792,7 @@ bool this_cpu_buffer_push(struct percpu_buffer *buffer, intptr_t offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) break; @@ -704,14 +800,9 @@ bool this_cpu_buffer_push(struct percpu_buffer *buffer, targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; - if (opt_mb) - ret = rseq_cmpeqv_trystorev_storev_release( - targetptr_final, offset, targetptr_spec, - newval_spec, newval_final, cpu); - else - ret = rseq_cmpeqv_trystorev_storev(targetptr_final, - offset, targetptr_spec, newval_spec, - newval_final, cpu); + ret = rseq_cmpeqv_trystorev_storev(opt_mo, RSEQ_PERCPU, + targetptr_final, offset, targetptr_spec, + newval_spec, newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; @@ -734,7 +825,7 @@ struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, intptr_t offset; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) { @@ -744,7 +835,8 @@ struct percpu_buffer_node *this_cpu_buffer_pop(struct percpu_buffer *buffer, head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); newval = offset - 1; targetptr = (intptr_t *)&buffer->c[cpu].offset; - ret = rseq_cmpeqv_cmpeqv_storev(targetptr, offset, + ret = rseq_cmpeqv_cmpeqv_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr, offset, (intptr_t *)&buffer->c[cpu].array[offset - 1], (intptr_t)head, newval, cpu); if (rseq_likely(!ret)) @@ -820,7 +912,7 @@ void test_percpu_buffer(void) /* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; /* Worse-case is every item in same CPU. */ buffer.c[i].array = @@ -870,7 +962,7 @@ void test_percpu_buffer(void) for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_buffer_node *node; - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; while ((node = __percpu_buffer_pop(&buffer, i))) { @@ -901,7 +993,7 @@ bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, size_t copylen; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == buffer->c[cpu].buflen) @@ -912,15 +1004,11 @@ bool this_cpu_memcpy_buffer_push(struct percpu_memcpy_buffer *buffer, copylen = sizeof(item); newval_final = offset + 1; targetptr_final = &buffer->c[cpu].offset; - if (opt_mb) - ret = rseq_cmpeqv_trymemcpy_storev_release( - targetptr_final, offset, - destptr, srcptr, copylen, - newval_final, cpu); - else - ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, - offset, destptr, srcptr, copylen, - newval_final, cpu); + ret = rseq_cmpeqv_trymemcpy_storev( + opt_mo, RSEQ_PERCPU, + targetptr_final, offset, + destptr, srcptr, copylen, + newval_final, cpu); if (rseq_likely(!ret)) { result = true; break; @@ -945,7 +1033,7 @@ bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, size_t copylen; int ret; - cpu = rseq_cpu_start(); + cpu = get_current_cpu_id(); /* Load offset with single-copy atomicity. */ offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); if (offset == 0) @@ -956,8 +1044,8 @@ bool this_cpu_memcpy_buffer_pop(struct percpu_memcpy_buffer *buffer, copylen = sizeof(*item); newval_final = offset - 1; targetptr_final = &buffer->c[cpu].offset; - ret = rseq_cmpeqv_trymemcpy_storev(targetptr_final, - offset, destptr, srcptr, copylen, + ret = rseq_cmpeqv_trymemcpy_storev(RSEQ_MO_RELAXED, RSEQ_PERCPU, + targetptr_final, offset, destptr, srcptr, copylen, newval_final, cpu); if (rseq_likely(!ret)) { result = true; @@ -1035,7 +1123,7 @@ void test_percpu_memcpy_buffer(void) /* Generate list entries for every usable cpu. */ sched_getaffinity(0, sizeof(allowed_cpus), &allowed_cpus); for (i = 0; i < CPU_SETSIZE; i++) { - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; /* Worse-case is every item in same CPU. */ buffer.c[i].array = @@ -1082,7 +1170,7 @@ void test_percpu_memcpy_buffer(void) for (i = 0; i < CPU_SETSIZE; i++) { struct percpu_memcpy_buffer_node item; - if (!CPU_ISSET(i, &allowed_cpus)) + if (rseq_use_cpu_index() && !CPU_ISSET(i, &allowed_cpus)) continue; while (__percpu_memcpy_buffer_pop(&buffer, &item, i)) { @@ -1131,6 +1219,214 @@ static int set_signal_handler(void) return ret; } +/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */ +#ifdef TEST_MEMBARRIER +struct test_membarrier_thread_args { + int stop; + intptr_t percpu_list_ptr; +}; + +/* Worker threads modify data in their "active" percpu lists. */ +void *test_membarrier_worker_thread(void *arg) +{ + struct test_membarrier_thread_args *args = + (struct test_membarrier_thread_args *)arg; + const int iters = opt_reps; + int i; + + if (rseq_register_current_thread()) { + fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + + /* Wait for initialization. */ + while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {} + + for (i = 0; i < iters; ++i) { + int ret; + + do { + int cpu = get_current_cpu_id(); + + ret = rseq_offset_deref_addv(RSEQ_MO_RELAXED, RSEQ_PERCPU, + &args->percpu_list_ptr, + sizeof(struct percpu_list_entry) * cpu, 1, cpu); + } while (rseq_unlikely(ret)); + } + + if (rseq_unregister_current_thread()) { + fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + return NULL; +} + +void test_membarrier_init_percpu_list(struct percpu_list *list) +{ + int i; + + memset(list, 0, sizeof(*list)); + for (i = 0; i < CPU_SETSIZE; i++) { + struct percpu_list_node *node; + + node = malloc(sizeof(*node)); + assert(node); + node->data = 0; + node->next = NULL; + list->c[i].head = node; + } +} + +void test_membarrier_free_percpu_list(struct percpu_list *list) +{ + int i; + + for (i = 0; i < CPU_SETSIZE; i++) + free(list->c[i].head); +} + +/* + * The manager thread swaps per-cpu lists that worker threads see, + * and validates that there are no unexpected modifications. + */ +void *test_membarrier_manager_thread(void *arg) +{ + struct test_membarrier_thread_args *args = + (struct test_membarrier_thread_args *)arg; + struct percpu_list list_a, list_b; + intptr_t expect_a = 0, expect_b = 0; + int cpu_a = 0, cpu_b = 0; + + if (rseq_register_current_thread()) { + fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + + /* Init lists. */ + test_membarrier_init_percpu_list(&list_a); + test_membarrier_init_percpu_list(&list_b); + + __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); + + while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) { + /* list_a is "active". */ + cpu_a = rand() % CPU_SETSIZE; + /* + * As list_b is "inactive", we should never see changes + * to list_b. + */ + if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) { + fprintf(stderr, "Membarrier test failed\n"); + abort(); + } + + /* Make list_b "active". */ + __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE); + if (rseq_membarrier_expedited(cpu_a) && + errno != ENXIO /* missing CPU */) { + perror("sys_membarrier"); + abort(); + } + /* + * Cpu A should now only modify list_b, so the values + * in list_a should be stable. + */ + expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE); + + cpu_b = rand() % CPU_SETSIZE; + /* + * As list_a is "inactive", we should never see changes + * to list_a. + */ + if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) { + fprintf(stderr, "Membarrier test failed\n"); + abort(); + } + + /* Make list_a "active". */ + __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); + if (rseq_membarrier_expedited(cpu_b) && + errno != ENXIO /* missing CPU*/) { + perror("sys_membarrier"); + abort(); + } + /* Remember a value from list_b. */ + expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE); + } + + test_membarrier_free_percpu_list(&list_a); + test_membarrier_free_percpu_list(&list_b); + + if (rseq_unregister_current_thread()) { + fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n", + errno, strerror(errno)); + abort(); + } + return NULL; +} + +void test_membarrier(void) +{ + const int num_threads = opt_threads; + struct test_membarrier_thread_args thread_args; + pthread_t worker_threads[num_threads]; + pthread_t manager_thread; + int i, ret; + + if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0)) { + perror("sys_membarrier"); + abort(); + } + + thread_args.stop = 0; + thread_args.percpu_list_ptr = 0; + ret = pthread_create(&manager_thread, NULL, + test_membarrier_manager_thread, &thread_args); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + + for (i = 0; i < num_threads; i++) { + ret = pthread_create(&worker_threads[i], NULL, + test_membarrier_worker_thread, &thread_args); + if (ret) { + errno = ret; + perror("pthread_create"); + abort(); + } + } + + + for (i = 0; i < num_threads; i++) { + ret = pthread_join(worker_threads[i], NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } + } + + __atomic_store_n(&thread_args.stop, 1, __ATOMIC_RELEASE); + ret = pthread_join(manager_thread, NULL); + if (ret) { + errno = ret; + perror("pthread_join"); + abort(); + } +} +#else /* TEST_MEMBARRIER */ +void test_membarrier(void) +{ + fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. " + "Skipping membarrier test.\n"); +} +#endif + static void show_usage(int argc, char **argv) { printf("Usage : %s <OPTIONS>\n", @@ -1153,7 +1449,7 @@ static void show_usage(int argc, char **argv) printf(" [-r N] Number of repetitions per thread (default 5000)\n"); printf(" [-d] Disable rseq system call (no initialization)\n"); printf(" [-D M] Disable rseq for each M threads\n"); - printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n"); + printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n"); printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); printf(" [-v] Verbose output.\n"); printf(" [-h] Show this help.\n"); @@ -1268,6 +1564,7 @@ int main(int argc, char **argv) case 'i': case 'b': case 'm': + case 'r': break; default: show_usage(argc, argv); @@ -1279,7 +1576,7 @@ int main(int argc, char **argv) verbose = 1; break; case 'M': - opt_mb = 1; + opt_mo = RSEQ_MO_RELEASE; break; default: show_usage(argc, argv); @@ -1299,6 +1596,10 @@ int main(int argc, char **argv) if (!opt_disable_rseq && rseq_register_current_thread()) goto error; + if (!opt_disable_rseq && !rseq_validate_cpu_id()) { + fprintf(stderr, "Error: cpu id getter unavailable\n"); + goto error; + } switch (opt_test) { case 's': printf_verbose("spinlock\n"); @@ -1320,6 +1621,10 @@ int main(int argc, char **argv) printf_verbose("counter increment\n"); test_percpu_inc(); break; + case 'r': + printf_verbose("membarrier\n"); + test_membarrier(); + break; } if (!opt_disable_rseq && rseq_unregister_current_thread()) abort(); diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h new file mode 100644 index 000000000000..fb4ec8a75dd4 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-abi.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +#ifndef _RSEQ_ABI_H +#define _RSEQ_ABI_H + +/* + * rseq-abi.h + * + * Restartable sequences system call API + * + * Copyright (c) 2015-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include <linux/types.h> +#include <asm/byteorder.h> + +enum rseq_abi_cpu_id_state { + RSEQ_ABI_CPU_ID_UNINITIALIZED = -1, + RSEQ_ABI_CPU_ID_REGISTRATION_FAILED = -2, +}; + +enum rseq_abi_flags { + RSEQ_ABI_FLAG_UNREGISTER = (1 << 0), +}; + +enum rseq_abi_cs_flags_bit { + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0, + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1, + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2, +}; + +enum rseq_abi_cs_flags { + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT = + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT), + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL = + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT), + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE = + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT), +}; + +/* + * struct rseq_abi_cs is aligned on 4 * 8 bytes to ensure it is always + * contained within a single cache-line. It is usually declared as + * link-time constant data. + */ +struct rseq_abi_cs { + /* Version of this structure. */ + __u32 version; + /* enum rseq_abi_cs_flags */ + __u32 flags; + __u64 start_ip; + /* Offset from start_ip. */ + __u64 post_commit_offset; + __u64 abort_ip; +} __attribute__((aligned(4 * sizeof(__u64)))); + +/* + * struct rseq_abi is aligned on 4 * 8 bytes to ensure it is always + * contained within a single cache-line. + * + * A single struct rseq_abi per thread is allowed. + */ +struct rseq_abi { + /* + * Restartable sequences cpu_id_start field. Updated by the + * kernel. Read by user-space with single-copy atomicity + * semantics. This field should only be read by the thread which + * registered this data structure. Aligned on 32-bit. Always + * contains a value in the range of possible CPUs, although the + * value may not be the actual current CPU (e.g. if rseq is not + * initialized). This CPU number value should always be compared + * against the value of the cpu_id field before performing a rseq + * commit or returning a value read from a data structure indexed + * using the cpu_id_start value. + */ + __u32 cpu_id_start; + /* + * Restartable sequences cpu_id field. Updated by the kernel. + * Read by user-space with single-copy atomicity semantics. This + * field should only be read by the thread which registered this + * data structure. Aligned on 32-bit. Values + * RSEQ_CPU_ID_UNINITIALIZED and RSEQ_CPU_ID_REGISTRATION_FAILED + * have a special semantic: the former means "rseq uninitialized", + * and latter means "rseq initialization failed". This value is + * meant to be read within rseq critical sections and compared + * with the cpu_id_start value previously read, before performing + * the commit instruction, or read and compared with the + * cpu_id_start value before returning a value loaded from a data + * structure indexed using the cpu_id_start value. + */ + __u32 cpu_id; + /* + * Restartable sequences rseq_cs field. + * + * Contains NULL when no critical section is active for the current + * thread, or holds a pointer to the currently active struct rseq_cs. + * + * Updated by user-space, which sets the address of the currently + * active rseq_cs at the beginning of assembly instruction sequence + * block, and set to NULL by the kernel when it restarts an assembly + * instruction sequence block, as well as when the kernel detects that + * it is preempting or delivering a signal outside of the range + * targeted by the rseq_cs. Also needs to be set to NULL by user-space + * before reclaiming memory that contains the targeted struct rseq_cs. + * + * Read and set by the kernel. Set by user-space with single-copy + * atomicity semantics. This field should only be updated by the + * thread which registered this data structure. Aligned on 64-bit. + */ + union { + __u64 ptr64; + + /* + * The "arch" field provides architecture accessor for + * the ptr field based on architecture pointer size and + * endianness. + */ + struct { +#ifdef __LP64__ + __u64 ptr; +#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN) + __u32 padding; /* Initialized to zero. */ + __u32 ptr; +#else + __u32 ptr; + __u32 padding; /* Initialized to zero. */ +#endif + } arch; + } rseq_cs; + + /* + * Restartable sequences flags field. + * + * This field should only be updated by the thread which + * registered this data structure. Read by the kernel. + * Mainly used for single-stepping through rseq critical sections + * with debuggers. + * + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT + * Inhibit instruction sequence block restart on preemption + * for this thread. + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL + * Inhibit instruction sequence block restart on signal + * delivery for this thread. + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE + * Inhibit instruction sequence block restart on migration for + * this thread. + */ + __u32 flags; + + /* + * Restartable sequences node_id field. Updated by the kernel. Read by + * user-space with single-copy atomicity semantics. This field should + * only be read by the thread which registered this data structure. + * Aligned on 32-bit. Contains the current NUMA node ID. + */ + __u32 node_id; + + /* + * Restartable sequences mm_cid field. Updated by the kernel. Read by + * user-space with single-copy atomicity semantics. This field should + * only be read by the thread which registered this data structure. + * Aligned on 32-bit. Contains the current thread's concurrency ID + * (allocated uniquely within a memory map). + */ + __u32 mm_cid; + + /* + * Flexible array member at end of structure, after last feature field. + */ + char end[]; +} __attribute__((aligned(4 * sizeof(__u64)))); + +#endif /* _RSEQ_ABI_H */ diff --git a/tools/testing/selftests/rseq/rseq-arm-bits.h b/tools/testing/selftests/rseq/rseq-arm-bits.h new file mode 100644 index 000000000000..4f03cb395462 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-arm-bits.h @@ -0,0 +1,505 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-arm-bits.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[error2]\n\t" +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expectnot], r0\n\t" + "beq %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expectnot], r0\n\t" + "beq %l[error2]\n\t" +#endif + "str r0, %[load]\n\t" + "add r0, %[voffp]\n\t" + "ldr r0, [r0]\n\t" + /* final store */ + "str r0, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "Ir" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + "ldr r0, %[v]\n\t" + "add r0, %[count]\n\t" + /* final store */ + "str r0, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [count] "Ir" (count) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + "ldr r0, %[v2]\n\t" + "cmp %[expect2], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[error2]\n\t" + "ldr r0, %[v2]\n\t" + "cmp %[expect2], r0\n\t" + "bne %l[error3]\n\t" +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne %l[error2]\n\t" +#endif + /* try store */ + "str %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "dmb\n\t" /* full mb provides store-release */ +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint32_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + "str %[src], %[rseq_scratch0]\n\t" + "str %[dst], %[rseq_scratch1]\n\t" + "str %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + "ldr r0, %[v]\n\t" + "cmp %[expect], r0\n\t" + "bne 7f\n\t" +#endif + /* try memcpy */ + "cmp %[len], #0\n\t" \ + "beq 333f\n\t" \ + "222:\n\t" \ + "ldrb %%r0, [%[src]]\n\t" \ + "strb %%r0, [%[dst]]\n\t" \ + "adds %[src], #1\n\t" \ + "adds %[dst], #1\n\t" \ + "subs %[len], #1\n\t" \ + "bne 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "dmb\n\t" /* full mb provides store-release */ +#endif + /* final store */ + "str %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t" + "b 8f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + abort, 1b, 2b, 4f) + RSEQ_ASM_DEFINE_CMPFAIL(5, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + /* teardown */ + "ldr %[len], %[rseq_scratch2]\n\t" + "ldr %[dst], %[rseq_scratch1]\n\t" + "ldr %[src], %[rseq_scratch0]\n\t", + error2) +#endif + "8:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + RSEQ_INJECT_INPUT + : "r0", "memory", "cc" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h index 5943c816c07c..d887b3bbe257 100644 --- a/tools/testing/selftests/rseq/rseq-arm.h +++ b/tools/testing/selftests/rseq/rseq-arm.h @@ -2,7 +2,7 @@ /* * rseq-arm.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ /* @@ -66,7 +66,7 @@ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ - __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ + rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ rseq_smp_mb(); \ ____p1; \ }) @@ -76,13 +76,9 @@ __extension__ ({ \ #define rseq_smp_store_release(p, v) \ do { \ rseq_smp_mb(); \ - RSEQ_WRITE_ONCE(*p, v); \ + RSEQ_WRITE_ONCE(*(p), v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -147,679 +143,34 @@ do { \ teardown \ "b %l[" __rseq_str(cmpfail_label) "]\n\t" -#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" -#endif - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expectnot], r0\n\t" - "beq %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expectnot], r0\n\t" - "beq %l[error2]\n\t" -#endif - "str r0, %[load]\n\t" - "add r0, %[voffp]\n\t" - "ldr r0, [r0]\n\t" - /* final store */ - "str r0, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "Ir" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +/* Per-cpu-id indexing. */ -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - "ldr r0, %[v]\n\t" - "add r0, %[count]\n\t" - /* final store */ - "str r0, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "m" (*v), - [count] "Ir" (count) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-mm-cid indexing. */ - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" -#endif - /* try store */ - "str %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" -#endif - /* try store */ - "str %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - "dmb\n\t" /* full mb provides store-release */ - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - "ldr r0, %[v2]\n\t" - "cmp %[expect2], r0\n\t" - "bne %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne %l[error2]\n\t" - "ldr r0, %[v2]\n\t" - "cmp %[expect2], r0\n\t" - "bne %l[error3]\n\t" -#endif - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "str %[src], %[rseq_scratch0]\n\t" - "str %[dst], %[rseq_scratch1]\n\t" - "str %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 7f\n\t" -#endif - /* try memcpy */ - "cmp %[len], #0\n\t" \ - "beq 333f\n\t" \ - "222:\n\t" \ - "ldrb %%r0, [%[src]]\n\t" \ - "strb %%r0, [%[dst]]\n\t" \ - "adds %[src], #1\n\t" \ - "adds %[dst], #1\n\t" \ - "subs %[len], #1\n\t" \ - "bne 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "str %[src], %[rseq_scratch0]\n\t" - "str %[dst], %[rseq_scratch1]\n\t" - "str %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 7f\n\t" -#endif - /* try memcpy */ - "cmp %[len], #0\n\t" \ - "beq 333f\n\t" \ - "222:\n\t" \ - "ldrb %%r0, [%[src]]\n\t" \ - "strb %%r0, [%[dst]]\n\t" \ - "adds %[src], #1\n\t" \ - "adds %[dst], #1\n\t" \ - "subs %[len], #1\n\t" \ - "bne 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "dmb\n\t" /* full mb provides store-release */ - /* final store */ - "str %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "r0", "memory", "cc" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("expected value comparison failed"); -#endif -} +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-arm64-bits.h b/tools/testing/selftests/rseq/rseq-arm64-bits.h new file mode 100644 index 000000000000..cc7226b1efe1 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-arm64-bits.h @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-arm64-bits.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_STORE(load) + RSEQ_ASM_OP_R_LOAD_OFF(voffp) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [expectnot] "r" (expectnot), + [load] "Qo" (*load), + [voffp] "r" (voffp) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_ADD(count) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "Qo" (*v), + [expect] "r" (expect), + [v2] "Qo" (*v2), + [expect2] "r" (expect2), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + RSEQ_ASM_OP_STORE(newv2, v2) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "Qo" (*v), + [newv] "r" (newv), + [v2] "Qo" (*v2), + [newv2] "r" (newv2) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "Qo" (*v), + [newv] "r" (newv), + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-arm64.h b/tools/testing/selftests/rseq/rseq-arm64.h index 200dae9e4208..21e1626a7235 100644 --- a/tools/testing/selftests/rseq/rseq-arm64.h +++ b/tools/testing/selftests/rseq/rseq-arm64.h @@ -2,7 +2,7 @@ /* * rseq-arm64.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> */ @@ -27,68 +27,66 @@ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ - __typeof(*p) ____p1; \ - switch (sizeof(*p)) { \ + union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \ + switch (sizeof(*(p))) { \ case 1: \ - asm volatile ("ldarb %w0, %1" \ - : "=r" (*(__u8 *)p) \ - : "Q" (*p) : "memory"); \ + __asm__ __volatile__ ("ldarb %w0, %1" \ + : "=r" (*(__u8 *)__u.__c) \ + : "Q" (*(p)) : "memory"); \ break; \ case 2: \ - asm volatile ("ldarh %w0, %1" \ - : "=r" (*(__u16 *)p) \ - : "Q" (*p) : "memory"); \ + __asm__ __volatile__ ("ldarh %w0, %1" \ + : "=r" (*(__u16 *)__u.__c) \ + : "Q" (*(p)) : "memory"); \ break; \ case 4: \ - asm volatile ("ldar %w0, %1" \ - : "=r" (*(__u32 *)p) \ - : "Q" (*p) : "memory"); \ + __asm__ __volatile__ ("ldar %w0, %1" \ + : "=r" (*(__u32 *)__u.__c) \ + : "Q" (*(p)) : "memory"); \ break; \ case 8: \ - asm volatile ("ldar %0, %1" \ - : "=r" (*(__u64 *)p) \ - : "Q" (*p) : "memory"); \ + __asm__ __volatile__ ("ldar %0, %1" \ + : "=r" (*(__u64 *)__u.__c) \ + : "Q" (*(p)) : "memory"); \ break; \ } \ - ____p1; \ + (rseq_unqual_scalar_typeof(*(p)))__u.__val; \ }) #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() #define rseq_smp_store_release(p, v) \ do { \ - switch (sizeof(*p)) { \ + union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \ + { .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \ + switch (sizeof(*(p))) { \ case 1: \ - asm volatile ("stlrb %w1, %0" \ - : "=Q" (*p) \ - : "r" ((__u8)v) \ + __asm__ __volatile__ ("stlrb %w1, %0" \ + : "=Q" (*(p)) \ + : "r" (*(__u8 *)__u.__c) \ : "memory"); \ break; \ case 2: \ - asm volatile ("stlrh %w1, %0" \ - : "=Q" (*p) \ - : "r" ((__u16)v) \ + __asm__ __volatile__ ("stlrh %w1, %0" \ + : "=Q" (*(p)) \ + : "r" (*(__u16 *)__u.__c) \ : "memory"); \ break; \ case 4: \ - asm volatile ("stlr %w1, %0" \ - : "=Q" (*p) \ - : "r" ((__u32)v) \ + __asm__ __volatile__ ("stlr %w1, %0" \ + : "=Q" (*(p)) \ + : "r" (*(__u32 *)__u.__c) \ : "memory"); \ break; \ case 8: \ - asm volatile ("stlr %1, %0" \ - : "=Q" (*p) \ - : "r" ((__u64)v) \ + __asm__ __volatile__ ("stlr %1, %0" \ + : "=Q" (*(p)) \ + : "r" (*(__u64 *)__u.__c) \ : "memory"); \ break; \ } \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define RSEQ_ASM_TMP_REG32 "w15" #define RSEQ_ASM_TMP_REG "x15" #define RSEQ_ASM_TMP_REG_2 "x14" @@ -204,459 +202,34 @@ do { \ " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ "333:\n" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "Qo" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) -#endif - RSEQ_ASM_OP_R_LOAD(v) - RSEQ_ASM_OP_R_STORE(load) - RSEQ_ASM_OP_R_LOAD_OFF(voffp) - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "Qo" (*v), - [expectnot] "r" (expectnot), - [load] "Qo" (*load), - [voffp] "r" (voffp) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +/* Per-cpu-id indexing. */ -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - RSEQ_ASM_OP_R_LOAD(v) - RSEQ_ASM_OP_R_ADD(count) - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "Qo" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [v2] "Qo" (*v2), - [newv2] "r" (newv2) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [v2] "Qo" (*v2), - [newv2] "r" (newv2) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) -#endif - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "Qo" (*v), - [expect] "r" (expect), - [v2] "Qo" (*v2), - [expect2] "r" (expect2), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) -#endif - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "Qo" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [expect] "r" (expect), - [v] "Qo" (*v), - [newv] "r" (newv), - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID + +/* Per-mm-cid indexing. */ + +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID + +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-arm64-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-bits-reset.h b/tools/testing/selftests/rseq/rseq-bits-reset.h new file mode 100644 index 000000000000..e8655089f9cb --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-bits-reset.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-bits-reset.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#undef RSEQ_TEMPLATE_IDENTIFIER +#undef RSEQ_TEMPLATE_CPU_ID_FIELD +#undef RSEQ_TEMPLATE_CPU_ID_OFFSET +#undef RSEQ_TEMPLATE_SUFFIX diff --git a/tools/testing/selftests/rseq/rseq-bits-template.h b/tools/testing/selftests/rseq/rseq-bits-template.h new file mode 100644 index 000000000000..65698d6a6cf9 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-bits-template.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-bits-template.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifdef RSEQ_TEMPLATE_CPU_ID +# define RSEQ_TEMPLATE_CPU_ID_OFFSET RSEQ_CPU_ID_OFFSET +# define RSEQ_TEMPLATE_CPU_ID_FIELD cpu_id +# ifdef RSEQ_TEMPLATE_MO_RELEASE +# define RSEQ_TEMPLATE_SUFFIX _release_cpu_id +# elif defined (RSEQ_TEMPLATE_MO_RELAXED) +# define RSEQ_TEMPLATE_SUFFIX _relaxed_cpu_id +# else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +# endif +#elif defined(RSEQ_TEMPLATE_MM_CID) +# define RSEQ_TEMPLATE_CPU_ID_OFFSET RSEQ_MM_CID_OFFSET +# define RSEQ_TEMPLATE_CPU_ID_FIELD mm_cid +# ifdef RSEQ_TEMPLATE_MO_RELEASE +# define RSEQ_TEMPLATE_SUFFIX _release_mm_cid +# elif defined (RSEQ_TEMPLATE_MO_RELAXED) +# define RSEQ_TEMPLATE_SUFFIX _relaxed_mm_cid +# else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +# endif +#elif defined (RSEQ_TEMPLATE_CPU_ID_NONE) +# ifdef RSEQ_TEMPLATE_MO_RELEASE +# define RSEQ_TEMPLATE_SUFFIX _release +# elif defined (RSEQ_TEMPLATE_MO_RELAXED) +# define RSEQ_TEMPLATE_SUFFIX _relaxed +# else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +# endif +#else +# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead." +#endif + +#define RSEQ_TEMPLATE_IDENTIFIER(x) RSEQ_COMBINE_TOKENS(x, RSEQ_TEMPLATE_SUFFIX) + diff --git a/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h b/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h new file mode 100644 index 000000000000..38c584661571 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-generic-thread-pointer.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ +/* + * rseq-generic-thread-pointer.h + * + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifndef _RSEQ_GENERIC_THREAD_POINTER +#define _RSEQ_GENERIC_THREAD_POINTER + +#ifdef __cplusplus +extern "C" { +#endif + +/* Use gcc builtin thread pointer. */ +static inline void *rseq_thread_pointer(void) +{ + return __builtin_thread_pointer(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tools/testing/selftests/rseq/rseq-mips-bits.h b/tools/testing/selftests/rseq/rseq-mips-bits.h new file mode 100644 index 000000000000..6c48af4d0944 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-mips-bits.h @@ -0,0 +1,462 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * Author: Paul Burton <paul.burton@mips.com> + * (C) Copyright 2018 MIPS Tech LLC + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[error2]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "beq $4, %[expectnot], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "beq $4, %[expectnot], %l[error2]\n\t" +#endif + LONG_S " $4, %[load]\n\t" + LONG_ADDI " $4, %[voffp]\n\t" + LONG_L " $4, 0($4)\n\t" + /* final store */ + LONG_S " $4, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "Ir" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + LONG_L " $4, %[v]\n\t" + LONG_ADDI " $4, %[count]\n\t" + /* final store */ + LONG_S " $4, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [count] "Ir" (count) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + LONG_L " $4, %[v2]\n\t" + "bne $4, %[expect2], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[error2]\n\t" + LONG_L " $4, %[v2]\n\t" + "bne $4, %[expect2], %l[error3]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("1st expected value comparison failed"); +error3: + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], %l[error2]\n\t" +#endif + /* try store */ + LONG_S " %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "sync\n\t" /* full sync provides store-release */ +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) + "5:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uintptr_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + LONG_S " %[src], %[rseq_scratch0]\n\t" + LONG_S " %[dst], %[rseq_scratch1]\n\t" + LONG_S " %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + LONG_L " $4, %[v]\n\t" + "bne $4, %[expect], 7f\n\t" +#endif + /* try memcpy */ + "beqz %[len], 333f\n\t" \ + "222:\n\t" \ + "lb $4, 0(%[src])\n\t" \ + "sb $4, 0(%[dst])\n\t" \ + LONG_ADDI " %[src], 1\n\t" \ + LONG_ADDI " %[dst], 1\n\t" \ + LONG_ADDI " %[len], -1\n\t" \ + "bnez %[len], 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "sync\n\t" /* full sync provides store-release */ +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t" + "b 8f\n\t" + RSEQ_ASM_DEFINE_ABORT(3, 4, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + abort, 1b, 2b, 4f) + RSEQ_ASM_DEFINE_CMPFAIL(5, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error2) +#endif + "8:\n\t" + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + RSEQ_INJECT_INPUT + : "$4", "memory" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-mips.h b/tools/testing/selftests/rseq/rseq-mips.h index e989e7c14b09..42ef8e946693 100644 --- a/tools/testing/selftests/rseq/rseq-mips.h +++ b/tools/testing/selftests/rseq/rseq-mips.h @@ -2,9 +2,7 @@ /* * Author: Paul Burton <paul.burton@mips.com> * (C) Copyright 2018 MIPS Tech LLC - * - * Based on rseq-arm.h: - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ /* @@ -47,7 +45,7 @@ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ - __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ + rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ rseq_smp_mb(); \ ____p1; \ }) @@ -57,13 +55,9 @@ __extension__ ({ \ #define rseq_smp_store_release(p, v) \ do { \ rseq_smp_mb(); \ - RSEQ_WRITE_ONCE(*p, v); \ + RSEQ_WRITE_ONCE(*(p), v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #if _MIPS_SZLONG == 64 # define LONG ".dword" # define LONG_LA "dla" @@ -154,661 +148,34 @@ do { \ teardown \ "b %l[" __rseq_str(cmpfail_label) "]\n\t" -#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("") - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "beq $4, %[expectnot], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "beq $4, %[expectnot], %l[error2]\n\t" -#endif - LONG_S " $4, %[load]\n\t" - LONG_ADDI " $4, %[voffp]\n\t" - LONG_L " $4, 0($4)\n\t" - /* final store */ - LONG_S " $4, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "Ir" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - LONG_L " $4, %[v]\n\t" - LONG_ADDI " $4, %[count]\n\t" - /* final store */ - LONG_S " $4, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "m" (*v), - [count] "Ir" (count) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" -#endif - /* try store */ - LONG_S " %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" -#endif - /* try store */ - LONG_S " %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - "sync\n\t" /* full sync provides store-release */ - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - LONG_L " $4, %[v2]\n\t" - "bne $4, %[expect2], %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], %l[error2]\n\t" - LONG_L " $4, %[v2]\n\t" - "bne $4, %[expect2], %l[error3]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - "b 5f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) - "5:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} +/* Per-cpu-id indexing. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uintptr_t rseq_scratch[3]; +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - LONG_S " %[src], %[rseq_scratch0]\n\t" - LONG_S " %[dst], %[rseq_scratch1]\n\t" - LONG_S " %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 7f\n\t" -#endif - /* try memcpy */ - "beqz %[len], 333f\n\t" \ - "222:\n\t" \ - "lb $4, 0(%[src])\n\t" \ - "sb $4, 0(%[dst])\n\t" \ - LONG_ADDI " %[src], 1\n\t" \ - LONG_ADDI " %[dst], 1\n\t" \ - LONG_ADDI " %[len], -1\n\t" \ - "bnez %[len], 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("expected value comparison failed"); -#endif -} +/* Per-mm-cid indexing. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uintptr_t rseq_scratch[3]; +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID - rseq_workaround_gcc_asm_size_guess(); - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - LONG_S " %[src], %[rseq_scratch0]\n\t" - LONG_S " %[dst], %[rseq_scratch1]\n\t" - LONG_S " %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - LONG_L " $4, %[v]\n\t" - "bne $4, %[expect], 7f\n\t" -#endif - /* try memcpy */ - "beqz %[len], 333f\n\t" \ - "222:\n\t" \ - "lb $4, 0(%[src])\n\t" \ - "sb $4, 0(%[dst])\n\t" \ - LONG_ADDI " %[src], 1\n\t" \ - LONG_ADDI " %[dst], 1\n\t" \ - LONG_ADDI " %[len], -1\n\t" \ - "bnez %[len], 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "sync\n\t" /* full sync provides store-release */ - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(3, 4, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - abort, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_CMPFAIL(5, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "$4", "memory" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - rseq_workaround_gcc_asm_size_guess(); - return 0; -abort: - rseq_workaround_gcc_asm_size_guess(); - RSEQ_INJECT_FAILED - return -1; -cmpfail: - rseq_workaround_gcc_asm_size_guess(); - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("cpu_id comparison failed"); -error2: - rseq_workaround_gcc_asm_size_guess(); - rseq_bug("expected value comparison failed"); -#endif -} +/* APIs which are not based on cpu ids. */ -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-mips-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-ppc-bits.h b/tools/testing/selftests/rseq/rseq-ppc-bits.h new file mode 100644 index 000000000000..98e69eae1e62 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-ppc-bits.h @@ -0,0 +1,454 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-ppc-bits.h + * + * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> + */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v not equal to @expectnot */ + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v not equal to @expectnot */ + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) +#endif + /* load the value of @v */ + RSEQ_ASM_OP_R_LOAD(v) + /* store it in @load */ + RSEQ_ASM_OP_R_STORE(load) + /* dereference voffp(v) */ + RSEQ_ASM_OP_R_LOADX(voffp) + /* final store the value at voffp(v) */ + RSEQ_ASM_OP_R_FINAL_STORE(v, 2) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "b" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + /* load the value of @v */ + RSEQ_ASM_OP_R_LOAD(v) + /* add @count to it */ + RSEQ_ASM_OP_R_ADD(count) + /* final store */ + RSEQ_ASM_OP_R_FINAL_STORE(v, 2) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) + /* cmp @v2 equal to @expct2 */ + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + /* cmp @v2 equal to @expct2 */ + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + /* try store */ + RSEQ_ASM_OP_STORE(newv2, v2) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + /* for 'release' */ + "lwsync\n\t" +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* setup for mempcy */ + "mr %%r19, %[len]\n\t" + "mr %%r20, %[src]\n\t" + "mr %%r21, %[dst]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + /* cmp cpuid */ + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + /* cmp @v equal to @expect */ + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) +#endif + /* try memcpy */ + RSEQ_ASM_OP_R_MEMCPY() + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + /* for 'release' */ + "lwsync\n\t" +#endif + /* final store */ + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) + RSEQ_INJECT_ASM(6) + /* teardown */ + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len) + RSEQ_INJECT_INPUT + : "memory", "cc", "r17", "r18", "r19", "r20", "r21" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h b/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h new file mode 100644 index 000000000000..263eee84fb76 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-ppc-thread-pointer.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ +/* + * rseq-ppc-thread-pointer.h + * + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifndef _RSEQ_PPC_THREAD_POINTER +#define _RSEQ_PPC_THREAD_POINTER + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void *rseq_thread_pointer(void) +{ +#ifdef __powerpc64__ + register void *__result asm ("r13"); +#else + register void *__result asm ("r2"); +#endif + asm ("" : "=r" (__result)); + return __result; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tools/testing/selftests/rseq/rseq-ppc.h b/tools/testing/selftests/rseq/rseq-ppc.h index 76be90196fe4..57b160597189 100644 --- a/tools/testing/selftests/rseq/rseq-ppc.h +++ b/tools/testing/selftests/rseq/rseq-ppc.h @@ -2,7 +2,7 @@ /* * rseq-ppc.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> */ @@ -23,7 +23,7 @@ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ - __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ + rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ rseq_smp_lwsync(); \ ____p1; \ }) @@ -33,13 +33,9 @@ __extension__ ({ \ #define rseq_smp_store_release(p, v) \ do { \ rseq_smp_lwsync(); \ - RSEQ_WRITE_ONCE(*p, v); \ + RSEQ_WRITE_ONCE(*(p), v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - /* * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to * better handle single-stepping through the restartable critical sections. @@ -47,10 +43,13 @@ do { \ #ifdef __PPC64__ -#define STORE_WORD "std " -#define LOAD_WORD "ld " -#define LOADX_WORD "ldx " -#define CMP_WORD "cmpd " +#define RSEQ_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ +#define RSEQ_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ +#define RSEQ_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ +#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ +#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */ +#define RSEQ_CMP_LONG "cmpd " +#define RSEQ_CMP_LONG_INT "cmpdi " #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ @@ -89,10 +88,13 @@ do { \ #else /* #ifdef __PPC64__ */ -#define STORE_WORD "stw " -#define LOAD_WORD "lwz " -#define LOADX_WORD "lwzx " -#define CMP_WORD "cmpw " +#define RSEQ_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */ +#define RSEQ_STORE_INT(arg) RSEQ_STORE_LONG(arg) /* To memory ("m" constraint) */ +#define RSEQ_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */ +#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */ +#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */ +#define RSEQ_CMP_LONG "cmpw " +#define RSEQ_CMP_LONG_INT "cmpwi " #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ @@ -125,7 +127,7 @@ do { \ RSEQ_INJECT_ASM(1) \ "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ - "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ + RSEQ_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ __rseq_str(label) ":\n\t" #endif /* #ifdef __PPC64__ */ @@ -136,7 +138,7 @@ do { \ #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ - "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ + RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ "bne- cr7, " __rseq_str(label) "\n\t" @@ -153,25 +155,25 @@ do { \ * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) */ #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ - CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ + RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ "bne- cr7, " __rseq_str(label) "\n\t" #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ - CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ + RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ "beq- cr7, " __rseq_str(label) "\n\t" #define RSEQ_ASM_OP_STORE(value, var) \ - STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" + RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" /* Load @var to r17 */ #define RSEQ_ASM_OP_R_LOAD(var) \ - LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" + RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" /* Store r17 to @var */ #define RSEQ_ASM_OP_R_STORE(var) \ - STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" + RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" /* Add @count to r17 */ #define RSEQ_ASM_OP_R_ADD(count) \ @@ -179,11 +181,11 @@ do { \ /* Load (r17 + voffp) to r17 */ #define RSEQ_ASM_OP_R_LOADX(voffp) \ - LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" + RSEQ_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" /* TODO: implement a faster memcpy. */ #define RSEQ_ASM_OP_R_MEMCPY() \ - "cmpdi %%r19, 0\n\t" \ + RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ "beq 333f\n\t" \ "addi %%r20, %%r20, -1\n\t" \ "addi %%r21, %%r21, -1\n\t" \ @@ -191,561 +193,46 @@ do { \ "lbzu %%r18, 1(%%r20)\n\t" \ "stbu %%r18, 1(%%r21)\n\t" \ "addi %%r19, %%r19, -1\n\t" \ - "cmpdi %%r19, 0\n\t" \ + RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \ "bne 222b\n\t" \ "333:\n\t" \ #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ - STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ + RSEQ_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ __rseq_str(post_commit_label) ":\n\t" #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ - STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ + RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ __rseq_str(post_commit_label) ":\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v not equal to @expectnot */ - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v not equal to @expectnot */ - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) -#endif - /* load the value of @v */ - RSEQ_ASM_OP_R_LOAD(v) - /* store it in @load */ - RSEQ_ASM_OP_R_STORE(load) - /* dereference voffp(v) */ - RSEQ_ASM_OP_R_LOADX(voffp) - /* final store the value at voffp(v) */ - RSEQ_ASM_OP_R_FINAL_STORE(v, 2) - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "b" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - /* load the value of @v */ - RSEQ_ASM_OP_R_LOAD(v) - /* add @count to it */ - RSEQ_ASM_OP_R_ADD(count) - /* final store */ - RSEQ_ASM_OP_R_FINAL_STORE(v, 2) - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try store */ - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try store */ - RSEQ_ASM_OP_STORE(newv2, v2) - RSEQ_INJECT_ASM(5) - /* for 'release' */ - "lwsync\n\t" - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) - /* cmp @v2 equal to @expct2 */ - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) - /* cmp @v2 equal to @expct2 */ - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) -#endif - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* setup for mempcy */ - "mr %%r19, %[len]\n\t" - "mr %%r20, %[src]\n\t" - "mr %%r21, %[dst]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try memcpy */ - RSEQ_ASM_OP_R_MEMCPY() - RSEQ_INJECT_ASM(5) - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - /* teardown */ - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17", "r18", "r19", "r20", "r21" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* setup for mempcy */ - "mr %%r19, %[len]\n\t" - "mr %%r20, %[src]\n\t" - "mr %%r21, %[dst]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) -#endif - /* try memcpy */ - RSEQ_ASM_OP_R_MEMCPY() - RSEQ_INJECT_ASM(5) - /* for 'release' */ - "lwsync\n\t" - /* final store */ - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) - RSEQ_INJECT_ASM(6) - /* teardown */ - RSEQ_ASM_DEFINE_ABORT(4, abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len) - RSEQ_INJECT_INPUT - : "memory", "cc", "r17", "r18", "r19", "r20", "r21" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -#undef STORE_WORD -#undef LOAD_WORD -#undef LOADX_WORD -#undef CMP_WORD - -#endif /* !RSEQ_SKIP_FASTPATH */ +/* Per-cpu-id indexing. */ + +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID + +/* Per-mm-cid indexing. */ + +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID + +/* APIs which are not based on cpu ids. */ + +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-ppc-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-riscv-bits.h b/tools/testing/selftests/rseq/rseq-riscv-bits.h new file mode 100644 index 000000000000..de31a0143139 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-riscv-bits.h @@ -0,0 +1,410 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + off_t voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_STORE(load) + RSEQ_ASM_OP_R_LOAD_OFF(voffp) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expectnot] "r" (expectnot), + [load] "m" (*load), + [voffp] "r" (voffp) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") +#endif + RSEQ_ASM_OP_R_LOAD(v) + RSEQ_ASM_OP_R_ADD(count) + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) + RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") + RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") +#endif + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [v2] "m" (*v2), + [expect2] "r" (expect2), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +error3: + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV + +/* + * pval = *(ptr+off) + * *pval += inc; + */ +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") +#endif + RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [ptr] "r" (ptr), + [off] "er" (off), + [inc] "er" (inc) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") +#endif + RSEQ_ASM_OP_STORE(newv2, v2) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "m" (*v), + [newv] "r" (newv), + [v2] "m" (*v2), + [newv2] "r" (newv2) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __always_inline +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]") +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") + RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") +#endif + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") +#endif + RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) +#else + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) +#endif + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [expect] "r" (expect), + [v] "m" (*v), + [newv] "r" (newv), + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len) + RSEQ_INJECT_INPUT + : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2, + RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4 + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +cmpfail: + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +error2: + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-riscv.h b/tools/testing/selftests/rseq/rseq-riscv.h new file mode 100644 index 000000000000..37e598d0a365 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-riscv.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike + * other architectures, the ebreak instruction has no immediate field for + * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG. + * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it + * is an uncommon instruction and will raise an illegal instruction + * exception when executed in all modes. + */ +#include <endian.h> + +#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN) +#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */ +#else +#error "Currently, RSEQ only supports Little-Endian version" +#endif + +#if __riscv_xlen == 64 +#define __REG_SEL(a, b) a +#elif __riscv_xlen == 32 +#define __REG_SEL(a, b) b +#endif + +#define REG_L __REG_SEL("ld ", "lw ") +#define REG_S __REG_SEL("sd ", "sw ") + +#define RISCV_FENCE(p, s) \ + __asm__ __volatile__ ("fence " #p "," #s : : : "memory") +#define rseq_smp_mb() RISCV_FENCE(rw, rw) +#define rseq_smp_rmb() RISCV_FENCE(r, r) +#define rseq_smp_wmb() RISCV_FENCE(w, w) +#define RSEQ_ASM_TMP_REG_1 "t6" +#define RSEQ_ASM_TMP_REG_2 "t5" +#define RSEQ_ASM_TMP_REG_3 "t4" +#define RSEQ_ASM_TMP_REG_4 "t3" + +#define rseq_smp_load_acquire(p) \ +__extension__ ({ \ + rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ + RISCV_FENCE(r, rw); \ + ____p1; \ +}) + +#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() + +#define rseq_smp_store_release(p, v) \ +do { \ + RISCV_FENCE(rw, w); \ + RSEQ_WRITE_ONCE(*(p), v); \ +} while (0) + +#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ + post_commit_offset, abort_ip) \ + ".pushsection __rseq_cs, \"aw\"\n" \ + ".balign 32\n" \ + __rseq_str(label) ":\n" \ + ".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ + ".quad " __rseq_str(start_ip) ", " \ + __rseq_str(post_commit_offset) ", " \ + __rseq_str(abort_ip) "\n" \ + ".popsection\n\t" \ + ".pushsection __rseq_cs_ptr_array, \"aw\"\n" \ + ".quad " __rseq_str(label) "b\n" \ + ".popsection\n" + +#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ + __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ + ((post_commit_ip) - (start_ip)), abort_ip) + +/* + * Exit points of a rseq critical section consist of all instructions outside + * of the critical section where a critical section can either branch to or + * reach through the normal course of its execution. The abort IP and the + * post-commit IP are already part of the __rseq_cs section and should not be + * explicitly defined as additional exit points. Knowing all exit points is + * useful to assist debuggers stepping over the critical section. + */ +#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \ + ".pushsection __rseq_exit_point_array, \"aw\"\n" \ + ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \ + ".popsection\n" + +#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ + RSEQ_INJECT_ASM(1) \ + "la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \ + REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \ + __rseq_str(label) ":\n" + +#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ + "j 222f\n" \ + ".balign 4\n" \ + ".long " __rseq_str(RSEQ_SIG) "\n" \ + __rseq_str(label) ":\n" \ + "j %l[" __rseq_str(abort_label) "]\n" \ + "222:\n" + +#define RSEQ_ASM_OP_STORE(value, var) \ + REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" + +#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ + REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ + "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ + __rseq_str(label) "\n" + +#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ + "lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ + "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ + __rseq_str(label) "\n" + +#define RSEQ_ASM_OP_CMPNE(var, expect, label) \ + REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ + "beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ + __rseq_str(label) "\n" + +#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ + RSEQ_INJECT_ASM(2) \ + RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) + +#define RSEQ_ASM_OP_R_LOAD(var) \ + REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" + +#define RSEQ_ASM_OP_R_STORE(var) \ + REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" + +#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \ + "add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \ + RSEQ_ASM_TMP_REG_1 "\n" \ + REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n" + +#define RSEQ_ASM_OP_R_ADD(count) \ + "add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \ + ", %[" __rseq_str(count) "]\n" + +#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ + RSEQ_ASM_OP_STORE(value, var) \ + __rseq_str(post_commit_label) ":\n" + +#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \ + "fence rw, w\n" \ + RSEQ_ASM_OP_STORE(value, var) \ + __rseq_str(post_commit_label) ":\n" + +#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ + REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ + __rseq_str(post_commit_label) ":\n" + +#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \ + "beqz %[" __rseq_str(len) "], 333f\n" \ + "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \ + "mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \ + "mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \ + "222:\n" \ + "lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \ + "sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \ + "addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \ + "addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \ + "addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \ + "bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \ + "333:\n" + +#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \ + "mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \ + RSEQ_ASM_OP_R_ADD(off) \ + REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \ + RSEQ_ASM_OP_R_ADD(inc) \ + __rseq_str(post_commit_label) ":\n" + +/* Per-cpu-id indexing. */ + +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID + +/* Per-mm-cid indexing. */ + +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED + +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID + +/* APIs which are not based on cpu ids. */ + +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-riscv-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-s390-bits.h b/tools/testing/selftests/rseq/rseq-s390-bits.h new file mode 100644 index 000000000000..0cf17d9f170a --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-s390-bits.h @@ -0,0 +1,474 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ + +#include "rseq-bits-template.h" + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[error2]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_L " %%r1, %[v]\n\t" + LONG_CMP_R " %%r1, %[expectnot]\n\t" + "je %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_L " %%r1, %[v]\n\t" + LONG_CMP_R " %%r1, %[expectnot]\n\t" + "je %l[error2]\n\t" +#endif + LONG_S " %%r1, %[load]\n\t" + LONG_ADD_R " %%r1, %[voffp]\n\t" + LONG_L " %%r1, 0(%%r1)\n\t" + /* final store */ + LONG_S " %%r1, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "r" (voffp), + [load] "m" (*load) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0", "r1" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) +#endif + LONG_L " %%r0, %[v]\n\t" + LONG_ADD_R " %%r0, %[count]\n\t" + /* final store */ + LONG_S " %%r0, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [count] "r" (count) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + LONG_CMP " %[expect2], %[v2]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[error2]\n\t" + LONG_CMP " %[expect2], %[v2]\n\t" + "jnz %l[error3]\n\t" +#endif + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +/* s390 is TSO. */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + LONG_CMP " %[expect], %[v]\n\t" + "jnz %l[error2]\n\t" +#endif + /* try store */ + LONG_S " %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* s390 is TSO. */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint64_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + LONG_S " %[src], %[rseq_scratch0]\n\t" + LONG_S " %[dst], %[rseq_scratch1]\n\t" + LONG_S " %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_INJECT_ASM(3) + LONG_CMP " %[expect], %[v]\n\t" + "jnz 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + LONG_CMP " %[expect], %[v]\n\t" + "jnz 7f\n\t" +#endif + /* try memcpy */ + LONG_LT_R " %[len], %[len]\n\t" + "jz 333f\n\t" + "222:\n\t" + "ic %%r0,0(%[src])\n\t" + "stc %%r0,0(%[dst])\n\t" + LONG_ADDI " %[src], 1\n\t" + LONG_ADDI " %[dst], 1\n\t" + LONG_ADDI " %[len], -1\n\t" + "jnz 222b\n\t" + "333:\n\t" + RSEQ_INJECT_ASM(5) + /* final store */ + LONG_S " %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t" + RSEQ_ASM_DEFINE_ABORT(4, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + abort) + RSEQ_ASM_DEFINE_CMPFAIL(5, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + LONG_L " %[len], %[rseq_scratch2]\n\t" + LONG_L " %[dst], %[rseq_scratch1]\n\t" + LONG_L " %[src], %[rseq_scratch0]\n\t", + error2) +#endif + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + RSEQ_INJECT_INPUT + : "memory", "cc", "r0" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h index 8ef94ad1cbb4..33baaa9f9997 100644 --- a/tools/testing/selftests/rseq/rseq-s390.h +++ b/tools/testing/selftests/rseq/rseq-s390.h @@ -15,7 +15,7 @@ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ - __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ + rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ rseq_barrier(); \ ____p1; \ }) @@ -25,13 +25,9 @@ __extension__ ({ \ #define rseq_smp_store_release(p, v) \ do { \ rseq_barrier(); \ - RSEQ_WRITE_ONCE(*p, v); \ + RSEQ_WRITE_ONCE(*(p), v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #ifdef __s390x__ #define LONG_L "lg" @@ -134,448 +130,34 @@ do { \ "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-cpu-id indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[error2]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -/* - * Compare @v against @expectnot. When it does _not_ match, load @v - * into @load, and store the content of *@v + voffp into @v. - */ -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_L " %%r1, %[v]\n\t" - LONG_CMP_R " %%r1, %[expectnot]\n\t" - "je %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_L " %%r1, %[v]\n\t" - LONG_CMP_R " %%r1, %[expectnot]\n\t" - "je %l[error2]\n\t" -#endif - LONG_S " %%r1, %[load]\n\t" - LONG_ADD_R " %%r1, %[voffp]\n\t" - LONG_L " %%r1, 0(%%r1)\n\t" - /* final store */ - LONG_S " %%r1, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "r" (voffp), - [load] "m" (*load) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0", "r1" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-mm-cid indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) -#endif - LONG_L " %%r0, %[v]\n\t" - LONG_ADD_R " %%r0, %[count]\n\t" - /* final store */ - LONG_S " %%r0, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [count] "r" (count) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - LONG_S " %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -/* s390 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - LONG_CMP " %[expect2], %[v2]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - LONG_CMP " %[expect], %[v]\n\t" - "jnz %l[error2]\n\t" - LONG_CMP " %[expect2], %[v2]\n\t" - "jnz %l[error3]\n\t" -#endif - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint64_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - LONG_S " %[src], %[rseq_scratch0]\n\t" - LONG_S " %[dst], %[rseq_scratch1]\n\t" - LONG_S " %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) - RSEQ_INJECT_ASM(3) - LONG_CMP " %[expect], %[v]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) - LONG_CMP " %[expect], %[v]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - LONG_LT_R " %[len], %[len]\n\t" - "jz 333f\n\t" - "222:\n\t" - "ic %%r0,0(%[src])\n\t" - "stc %%r0,0(%[dst])\n\t" - LONG_ADDI " %[src], 1\n\t" - LONG_ADDI " %[dst], 1\n\t" - LONG_ADDI " %[len], -1\n\t" - "jnz 222b\n\t" - "333:\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - LONG_S " %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - LONG_L " %[len], %[rseq_scratch2]\n\t" - LONG_L " %[dst], %[rseq_scratch1]\n\t" - LONG_L " %[src], %[rseq_scratch0]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [current_cpu_id] "m" (__rseq_abi.cpu_id), - [rseq_cs] "m" (__rseq_abi.rseq_cs), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - RSEQ_INJECT_INPUT - : "memory", "cc", "r0" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +/* APIs which are not based on cpu ids. */ -/* s390 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, - newv, cpu); -} -#endif /* !RSEQ_SKIP_FASTPATH */ +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-s390-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq-skip.h b/tools/testing/selftests/rseq/rseq-skip.h deleted file mode 100644 index 72750b5905a9..000000000000 --- a/tools/testing/selftests/rseq/rseq-skip.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * rseq-skip.h - * - * (C) Copyright 2017-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> - */ - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return -1; -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return -1; -} diff --git a/tools/testing/selftests/rseq/rseq-thread-pointer.h b/tools/testing/selftests/rseq/rseq-thread-pointer.h new file mode 100644 index 000000000000..977c25d758b2 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-thread-pointer.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ +/* + * rseq-thread-pointer.h + * + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifndef _RSEQ_THREAD_POINTER +#define _RSEQ_THREAD_POINTER + +#if defined(__x86_64__) || defined(__i386__) +#include "rseq-x86-thread-pointer.h" +#elif defined(__PPC__) +#include "rseq-ppc-thread-pointer.h" +#else +#include "rseq-generic-thread-pointer.h" +#endif + +#endif diff --git a/tools/testing/selftests/rseq/rseq-x86-bits.h b/tools/testing/selftests/rseq/rseq-x86-bits.h new file mode 100644 index 000000000000..8a9431eec467 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-x86-bits.h @@ -0,0 +1,993 @@ +/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ +/* + * rseq-x86-bits.h + * + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#include "rseq-bits-template.h" + +#ifdef __x86_64__ + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpq %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" +#endif + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movq %[v], %%rbx\n\t" + "cmpq %%rbx, %[expectnot]\n\t" + "je %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "movq %[v], %%rbx\n\t" + "cmpq %%rbx, %[expectnot]\n\t" + "je %l[error2]\n\t" +#endif + "movq %%rbx, %[load]\n\t" + "addq %[voffp], %%rbx\n\t" + "movq (%%rbx), %%rbx\n\t" + /* final store */ + "movq %%rbx, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "er" (voffp), + [load] "m" (*load) + : "memory", "cc", "rax", "rbx" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) +#endif + /* final store */ + "addq %[count], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [count] "er" (count) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV + +/* + * pval = *(ptr+off) + * *pval += inc; + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) +#endif + /* get p+v */ + "movq %[ptr], %%rbx\n\t" + "addq %[off], %%rbx\n\t" + /* get pv */ + "movq (%%rbx), %%rcx\n\t" + /* *pv += inc */ + "addq %[inc], (%%rcx)\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [ptr] "m" (*ptr), + [off] "er" (off), + [inc] "er" (inc) + : "memory", "cc", "rax", "rbx", "rcx" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + return 0; +abort: + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + "cmpq %[v2], %[expect2]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpq %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" + "cmpq %[v2], %[expect2]\n\t" + "jnz %l[error3]\n\t" +#endif + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpq %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" +#endif + /* try store */ + "movq %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint64_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + "movq %[src], %[rseq_scratch0]\n\t" + "movq %[dst], %[rseq_scratch1]\n\t" + "movq %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jnz 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) + "cmpq %[v], %[expect]\n\t" + "jnz 7f\n\t" +#endif + /* try memcpy */ + "test %[len], %[len]\n\t" \ + "jz 333f\n\t" \ + "222:\n\t" \ + "movb (%[src]), %%al\n\t" \ + "movb %%al, (%[dst])\n\t" \ + "inc %[src]\n\t" \ + "inc %[dst]\n\t" \ + "dec %[len]\n\t" \ + "jnz 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) + /* final store */ + "movq %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t" + RSEQ_ASM_DEFINE_ABORT(4, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + abort) + RSEQ_ASM_DEFINE_CMPFAIL(5, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + "movq %[rseq_scratch2], %[len]\n\t" + "movq %[rseq_scratch1], %[dst]\n\t" + "movq %[rseq_scratch0], %[src]\n\t", + error2) +#endif + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#elif defined(__i386__) + +#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpl %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpl %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" +#endif + /* final store */ + "movl %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + [v] "m" (*v), + [expect] "r" (expect), + [newv] "r" (newv) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, + long voffp, intptr_t *load, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movl %[v], %%ebx\n\t" + "cmpl %%ebx, %[expectnot]\n\t" + "je %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "movl %[v], %%ebx\n\t" + "cmpl %%ebx, %[expectnot]\n\t" + "je %l[error2]\n\t" +#endif + "movl %%ebx, %[load]\n\t" + "addl %[voffp], %%ebx\n\t" + "movl (%%ebx), %%ebx\n\t" + /* final store */ + "movl %%ebx, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expectnot] "r" (expectnot), + [voffp] "ir" (voffp), + [load] "m" (*load) + : "memory", "cc", "eax", "ebx" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) +#endif + /* final store */ + "addl %[count], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(4) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [count] "ir" (count) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpl %[v], %[expect]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) + "cmpl %[expect2], %[v2]\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "cmpl %[v], %[expect]\n\t" + "jnz %l[error2]\n\t" + "cmpl %[expect2], %[v2]\n\t" + "jnz %l[error3]\n\t" +#endif + "movl %[newv], %%eax\n\t" + /* final store */ + "movl %%eax, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* cmp2 input */ + [v2] "m" (*v2), + [expect2] "r" (expect2), + /* final store input */ + [v] "m" (*v), + [expect] "r" (expect), + [newv] "m" (newv) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2, error3 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("1st expected value comparison failed"); +error3: + rseq_after_asm_goto(); + rseq_bug("2nd expected value comparison failed"); +#endif +} + +#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) + +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movl %[expect], %%eax\n\t" + "cmpl %[v], %%eax\n\t" + "jnz %l[cmpfail]\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + "movl %[expect], %%eax\n\t" + "cmpl %[v], %%eax\n\t" + "jnz %l[error2]\n\t" +#endif + /* try store */ + "movl %[newv2], %[v2]\n\t" + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "lock; addl $0,-128(%%esp)\n\t" +#endif + /* final store */ + "movl %[newv], %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* try store input */ + [v2] "m" (*v2), + [newv2] "r" (newv2), + /* final store input */ + [v] "m" (*v), + [expect] "m" (expect), + [newv] "r" (newv) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif + +} + +/* TODO: implement a faster memcpy. */ +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + uint32_t rseq_scratch[3]; + + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) +#endif + "movl %[src], %[rseq_scratch0]\n\t" + "movl %[dst], %[rseq_scratch1]\n\t" + "movl %[len], %[rseq_scratch2]\n\t" + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "movl %[expect], %%eax\n\t" + "cmpl %%eax, %[v]\n\t" + "jnz 5f\n\t" + RSEQ_INJECT_ASM(4) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) + "movl %[expect], %%eax\n\t" + "cmpl %%eax, %[v]\n\t" + "jnz 7f\n\t" +#endif + /* try memcpy */ + "test %[len], %[len]\n\t" \ + "jz 333f\n\t" \ + "222:\n\t" \ + "movb (%[src]), %%al\n\t" \ + "movb %%al, (%[dst])\n\t" \ + "inc %[src]\n\t" \ + "inc %[dst]\n\t" \ + "dec %[len]\n\t" \ + "jnz 222b\n\t" \ + "333:\n\t" \ + RSEQ_INJECT_ASM(5) +#ifdef RSEQ_TEMPLATE_MO_RELEASE + "lock; addl $0,-128(%%esp)\n\t" +#endif + "movl %[newv], %%eax\n\t" + /* final store */ + "movl %%eax, %[v]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(6) + /* teardown */ + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t" + RSEQ_ASM_DEFINE_ABORT(4, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + abort) + RSEQ_ASM_DEFINE_CMPFAIL(5, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + cmpfail) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_CMPFAIL(6, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + error1) + RSEQ_ASM_DEFINE_CMPFAIL(7, + "movl %[rseq_scratch2], %[len]\n\t" + "movl %[rseq_scratch1], %[dst]\n\t" + "movl %[rseq_scratch0], %[src]\n\t", + error2) +#endif + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + /* final store input */ + [v] "m" (*v), + [expect] "m" (expect), + [newv] "m" (newv), + /* try memcpy input */ + [dst] "r" (dst), + [src] "r" (src), + [len] "r" (len), + [rseq_scratch0] "m" (rseq_scratch[0]), + [rseq_scratch1] "m" (rseq_scratch[1]), + [rseq_scratch2] "m" (rseq_scratch[2]) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, cmpfail +#ifdef RSEQ_COMPARE_TWICE + , error1, error2 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +cmpfail: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +error2: + rseq_after_asm_goto(); + rseq_bug("expected value comparison failed"); +#endif +} + +#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ + +#endif + +#include "rseq-bits-reset.h" diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h new file mode 100644 index 000000000000..d3133587d996 --- /dev/null +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */ +/* + * rseq-x86-thread-pointer.h + * + * (C) Copyright 2021 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + */ + +#ifndef _RSEQ_X86_THREAD_POINTER +#define _RSEQ_X86_THREAD_POINTER + +#include <features.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#if __GNUC_PREREQ (11, 1) +static inline void *rseq_thread_pointer(void) +{ + return __builtin_thread_pointer(); +} +#else +static inline void *rseq_thread_pointer(void) +{ + void *__result; + +# ifdef __x86_64__ + __asm__ ("mov %%fs:0, %0" : "=r" (__result)); +# else + __asm__ ("mov %%gs:0, %0" : "=r" (__result)); +# endif + return __result; +} +#endif /* !GCC 11 */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tools/testing/selftests/rseq/rseq-x86.h b/tools/testing/selftests/rseq/rseq-x86.h index b2da6004fe30..a2aa428ba151 100644 --- a/tools/testing/selftests/rseq/rseq-x86.h +++ b/tools/testing/selftests/rseq/rseq-x86.h @@ -2,9 +2,13 @@ /* * rseq-x86.h * - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ +#ifndef RSEQ_H +#error "Never use <rseq-x86.h> directly; include <rseq.h> instead." +#endif + #include <stdint.h> /* @@ -22,12 +26,15 @@ * address through a "r" input operand. */ -/* Offset of cpu_id and rseq_cs fields in struct rseq. */ +/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ #define RSEQ_CPU_ID_OFFSET 4 #define RSEQ_CS_OFFSET 8 +#define RSEQ_MM_CID_OFFSET 24 #ifdef __x86_64__ +#define RSEQ_ASM_TP_SEGMENT %%fs + #define rseq_smp_mb() \ __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") #define rseq_smp_rmb() rseq_barrier() @@ -35,7 +42,7 @@ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ - __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ + rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ rseq_barrier(); \ ____p1; \ }) @@ -45,13 +52,9 @@ __extension__ ({ \ #define rseq_smp_store_release(p, v) \ do { \ rseq_barrier(); \ - RSEQ_WRITE_ONCE(*p, v); \ + RSEQ_WRITE_ONCE(*(p), v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -110,440 +113,9 @@ do { \ "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "cmpq %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -/* - * Compare @v against @expectnot. When it does _not_ match, load @v - * into @load, and store the content of *@v + voffp into @v. - */ -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "movq %[v], %%rbx\n\t" - "cmpq %%rbx, %[expectnot]\n\t" - "je %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "movq %[v], %%rbx\n\t" - "cmpq %%rbx, %[expectnot]\n\t" - "je %l[error2]\n\t" -#endif - "movq %%rbx, %[load]\n\t" - "addq %[voffp], %%rbx\n\t" - "movq (%%rbx), %%rbx\n\t" - /* final store */ - "movq %%rbx, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "er" (voffp), - [load] "m" (*load) - : "memory", "cc", "rax", "rbx" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) -#endif - /* final store */ - "addq %[count], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [count] "er" (count) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} +#elif defined(__i386__) -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "cmpq %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - "movq %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -/* x86-64 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu); -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - "cmpq %[v2], %[expect2]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "cmpq %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" - "cmpq %[v2], %[expect2]\n\t" - "jnz %l[error3]\n\t" -#endif - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint64_t rseq_scratch[3]; - - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "movq %[src], %[rseq_scratch0]\n\t" - "movq %[dst], %[rseq_scratch1]\n\t" - "movq %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpq %[v], %[expect]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) - "cmpq %[v], %[expect]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - "test %[len], %[len]\n\t" \ - "jz 333f\n\t" \ - "222:\n\t" \ - "movb (%[src]), %%al\n\t" \ - "movb %%al, (%[dst])\n\t" \ - "inc %[src]\n\t" \ - "inc %[dst]\n\t" \ - "dec %[len]\n\t" \ - "jnz 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - /* final store */ - "movq %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - "movq %[rseq_scratch2], %[len]\n\t" - "movq %[rseq_scratch1], %[dst]\n\t" - "movq %[rseq_scratch0], %[src]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - : "memory", "cc", "rax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -/* x86-64 is TSO. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len, - newv, cpu); -} - -#endif /* !RSEQ_SKIP_FASTPATH */ - -#elif __i386__ +#define RSEQ_ASM_TP_SEGMENT %%gs #define rseq_smp_mb() \ __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc") @@ -567,10 +139,6 @@ do { \ RSEQ_WRITE_ONCE(*p, v); \ } while (0) -#ifdef RSEQ_SKIP_FASTPATH -#include "rseq-skip.h" -#else /* !RSEQ_SKIP_FASTPATH */ - /* * Use eax as scratch register and take memory operands as input to * lessen register pressure. Especially needed when compiling in O0. @@ -631,606 +199,36 @@ do { \ "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \ ".popsection\n\t" -static inline __attribute__((always_inline)) -int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpl %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "cmpl %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* final store */ - "movl %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -/* - * Compare @v against @expectnot. When it does _not_ match, load @v - * into @load, and store the content of *@v + voffp into @v. - */ -static inline __attribute__((always_inline)) -int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, - off_t voffp, intptr_t *load, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[v], %%ebx\n\t" - "cmpl %%ebx, %[expectnot]\n\t" - "je %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "movl %[v], %%ebx\n\t" - "cmpl %%ebx, %[expectnot]\n\t" - "je %l[error2]\n\t" -#endif - "movl %%ebx, %[load]\n\t" - "addl %[voffp], %%ebx\n\t" - "movl (%%ebx), %%ebx\n\t" - /* final store */ - "movl %%ebx, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(5) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [expectnot] "r" (expectnot), - [voffp] "ir" (voffp), - [load] "m" (*load) - : "memory", "cc", "eax", "ebx" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_addv(intptr_t *v, intptr_t count, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) -#endif - /* final store */ - "addl %[count], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(4) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [count] "ir" (count) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort -#ifdef RSEQ_COMPARE_TWICE - , error1 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -#endif -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpl %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "cmpl %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - "movl %[newv2], %%eax\n\t" - "movl %%eax, %[v2]\n\t" - RSEQ_INJECT_ASM(5) - /* final store */ - "movl %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* try store input */ - [v2] "m" (*v2), - [newv2] "m" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); #endif -} -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t newv2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) +/* Per-cpu-id indexing. */ - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[expect], %%eax\n\t" - "cmpl %[v], %%eax\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "movl %[expect], %%eax\n\t" - "cmpl %[v], %%eax\n\t" - "jnz %l[error2]\n\t" -#endif - /* try store */ - "movl %[newv2], %[v2]\n\t" - RSEQ_INJECT_ASM(5) - "lock; addl $0,-128(%%esp)\n\t" - /* final store */ - "movl %[newv], %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* try store input */ - [v2] "m" (*v2), - [newv2] "r" (newv2), - /* final store input */ - [v] "m" (*v), - [expect] "m" (expect), - [newv] "r" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif - -} - -static inline __attribute__((always_inline)) -int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, - intptr_t *v2, intptr_t expect2, - intptr_t newv, int cpu) -{ - RSEQ_INJECT_C(9) - - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) -#endif - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "cmpl %[v], %[expect]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(4) - "cmpl %[expect2], %[v2]\n\t" - "jnz %l[cmpfail]\n\t" - RSEQ_INJECT_ASM(5) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) - "cmpl %[v], %[expect]\n\t" - "jnz %l[error2]\n\t" - "cmpl %[expect2], %[v2]\n\t" - "jnz %l[error3]\n\t" -#endif - "movl %[newv], %%eax\n\t" - /* final store */ - "movl %%eax, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - RSEQ_ASM_DEFINE_ABORT(4, "", abort) - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* cmp2 input */ - [v2] "m" (*v2), - [expect2] "r" (expect2), - /* final store input */ - [v] "m" (*v), - [expect] "r" (expect), - [newv] "m" (newv) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2, error3 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("1st expected value comparison failed"); -error3: - rseq_bug("2nd expected value comparison failed"); -#endif -} - -/* TODO: implement a faster memcpy. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; +#define RSEQ_TEMPLATE_CPU_ID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_CPU_ID - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "movl %[src], %[rseq_scratch0]\n\t" - "movl %[dst], %[rseq_scratch1]\n\t" - "movl %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - "test %[len], %[len]\n\t" \ - "jz 333f\n\t" \ - "222:\n\t" \ - "movb (%[src]), %%al\n\t" \ - "movb %%al, (%[dst])\n\t" \ - "inc %[src]\n\t" \ - "inc %[dst]\n\t" \ - "dec %[len]\n\t" \ - "jnz 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "movl %[newv], %%eax\n\t" - /* final store */ - "movl %%eax, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [expect] "m" (expect), - [newv] "m" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} - -/* TODO: implement a faster memcpy. */ -static inline __attribute__((always_inline)) -int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, - void *dst, void *src, size_t len, - intptr_t newv, int cpu) -{ - uint32_t rseq_scratch[3]; +/* Per-mm-cid indexing. */ - RSEQ_INJECT_C(9) +#define RSEQ_TEMPLATE_MM_CID +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED - __asm__ __volatile__ goto ( - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) -#endif - "movl %[src], %[rseq_scratch0]\n\t" - "movl %[dst], %[rseq_scratch1]\n\t" - "movl %[len], %[rseq_scratch2]\n\t" - /* Start rseq by storing table entry pointer into rseq_cs. */ - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) - RSEQ_INJECT_ASM(3) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 5f\n\t" - RSEQ_INJECT_ASM(4) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) - "movl %[expect], %%eax\n\t" - "cmpl %%eax, %[v]\n\t" - "jnz 7f\n\t" -#endif - /* try memcpy */ - "test %[len], %[len]\n\t" \ - "jz 333f\n\t" \ - "222:\n\t" \ - "movb (%[src]), %%al\n\t" \ - "movb %%al, (%[dst])\n\t" \ - "inc %[src]\n\t" \ - "inc %[dst]\n\t" \ - "dec %[len]\n\t" \ - "jnz 222b\n\t" \ - "333:\n\t" \ - RSEQ_INJECT_ASM(5) - "lock; addl $0,-128(%%esp)\n\t" - "movl %[newv], %%eax\n\t" - /* final store */ - "movl %%eax, %[v]\n\t" - "2:\n\t" - RSEQ_INJECT_ASM(6) - /* teardown */ - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - abort) - RSEQ_ASM_DEFINE_CMPFAIL(5, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - cmpfail) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_CMPFAIL(6, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error1) - RSEQ_ASM_DEFINE_CMPFAIL(7, - "movl %[rseq_scratch2], %[len]\n\t" - "movl %[rseq_scratch1], %[dst]\n\t" - "movl %[rseq_scratch0], %[src]\n\t", - error2) -#endif - : /* gcc asm goto does not allow outputs */ - : [cpu_id] "r" (cpu), - [rseq_abi] "r" (&__rseq_abi), - /* final store input */ - [v] "m" (*v), - [expect] "m" (expect), - [newv] "m" (newv), - /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) - : "memory", "cc", "eax" - RSEQ_INJECT_CLOBBER - : abort, cmpfail -#ifdef RSEQ_COMPARE_TWICE - , error1, error2 -#endif - ); - return 0; -abort: - RSEQ_INJECT_FAILED - return -1; -cmpfail: - return 1; -#ifdef RSEQ_COMPARE_TWICE -error1: - rseq_bug("cpu_id comparison failed"); -error2: - rseq_bug("expected value comparison failed"); -#endif -} +#define RSEQ_TEMPLATE_MO_RELEASE +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELEASE +#undef RSEQ_TEMPLATE_MM_CID -#endif /* !RSEQ_SKIP_FASTPATH */ +/* APIs which are not based on cpu ids. */ -#endif +#define RSEQ_TEMPLATE_CPU_ID_NONE +#define RSEQ_TEMPLATE_MO_RELAXED +#include "rseq-x86-bits.h" +#undef RSEQ_TEMPLATE_MO_RELAXED +#undef RSEQ_TEMPLATE_CPU_ID_NONE diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index 7159eb777fd3..96e812bdf8a4 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -26,104 +26,191 @@ #include <assert.h> #include <signal.h> #include <limits.h> +#include <dlfcn.h> +#include <stddef.h> +#include <sys/auxv.h> +#include <linux/auxvec.h> +#include <linux/compiler.h> + +#include "../kselftest.h" #include "rseq.h" -#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +/* + * Define weak versions to play nice with binaries that are statically linked + * against a libc that doesn't support registering its own rseq. + */ +__weak ptrdiff_t __rseq_offset; +__weak unsigned int __rseq_size; +__weak unsigned int __rseq_flags; -__thread volatile struct rseq __rseq_abi = { - .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, -}; +static const ptrdiff_t *libc_rseq_offset_p = &__rseq_offset; +static const unsigned int *libc_rseq_size_p = &__rseq_size; +static const unsigned int *libc_rseq_flags_p = &__rseq_flags; + +/* Offset from the thread pointer to the rseq area. */ +ptrdiff_t rseq_offset; /* - * Shared with other libraries. This library may take rseq ownership if it is - * still 0 when executing the library constructor. Set to 1 by library - * constructor when handling rseq. Set to 0 in destructor if handling rseq. + * Size of the registered rseq area. 0 if the registration was + * unsuccessful. */ -int __rseq_handled; +unsigned int rseq_size = -1U; + +/* Flags used during rseq registration. */ +unsigned int rseq_flags; + +/* + * rseq feature size supported by the kernel. 0 if the registration was + * unsuccessful. + */ +unsigned int rseq_feature_size = -1U; -/* Whether this library have ownership of rseq registration. */ static int rseq_ownership; +static int rseq_reg_success; /* At least one rseq registration has succeded. */ -static __thread volatile uint32_t __rseq_refcount; +/* Allocate a large area for the TLS. */ +#define RSEQ_THREAD_AREA_ALLOC_SIZE 1024 -static void signal_off_save(sigset_t *oldset) -{ - sigset_t set; - int ret; +/* Original struct rseq feature size is 20 bytes. */ +#define ORIG_RSEQ_FEATURE_SIZE 20 - sigfillset(&set); - ret = pthread_sigmask(SIG_BLOCK, &set, oldset); - if (ret) - abort(); -} +/* Original struct rseq allocation size is 32 bytes. */ +#define ORIG_RSEQ_ALLOC_SIZE 32 + +static +__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"), aligned(RSEQ_THREAD_AREA_ALLOC_SIZE))) = { + .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED, +}; -static void signal_restore(sigset_t oldset) +static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len, + int flags, uint32_t sig) { - int ret; + return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); +} - ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL); - if (ret) - abort(); +static int sys_getcpu(unsigned *cpu, unsigned *node) +{ + return syscall(__NR_getcpu, cpu, node, NULL); } -static int sys_rseq(volatile struct rseq *rseq_abi, uint32_t rseq_len, - int flags, uint32_t sig) +int rseq_available(void) { - return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); + int rc; + + rc = sys_rseq(NULL, 0, 0, 0); + if (rc != -1) + abort(); + switch (errno) { + case ENOSYS: + return 0; + case EINVAL: + return 1; + default: + abort(); + } } int rseq_register_current_thread(void) { - int rc, ret = 0; - sigset_t oldset; + int rc; - if (!rseq_ownership) + if (!rseq_ownership) { + /* Treat libc's ownership as a successful registration. */ return 0; - signal_off_save(&oldset); - if (__rseq_refcount == UINT_MAX) { - ret = -1; - goto end; } - if (__rseq_refcount++) - goto end; - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG); - if (!rc) { - assert(rseq_current_cpu_raw() >= 0); - goto end; + rc = sys_rseq(&__rseq_abi, rseq_size, 0, RSEQ_SIG); + if (rc) { + if (RSEQ_READ_ONCE(rseq_reg_success)) { + /* Incoherent success/failure within process. */ + abort(); + } + return -1; } - if (errno != EBUSY) - __rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED; - ret = -1; - __rseq_refcount--; -end: - signal_restore(oldset); - return ret; + assert(rseq_current_cpu_raw() >= 0); + RSEQ_WRITE_ONCE(rseq_reg_success, 1); + return 0; } int rseq_unregister_current_thread(void) { - int rc, ret = 0; - sigset_t oldset; + int rc; - if (!rseq_ownership) + if (!rseq_ownership) { + /* Treat libc's ownership as a successful unregistration. */ return 0; - signal_off_save(&oldset); - if (!__rseq_refcount) { - ret = -1; - goto end; } - if (--__rseq_refcount) - goto end; - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), - RSEQ_FLAG_UNREGISTER, RSEQ_SIG); - if (!rc) - goto end; - __rseq_refcount = 1; - ret = -1; -end: - signal_restore(oldset); - return ret; + rc = sys_rseq(&__rseq_abi, rseq_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); + if (rc) + return -1; + return 0; +} + +static +unsigned int get_rseq_feature_size(void) +{ + unsigned long auxv_rseq_feature_size, auxv_rseq_align; + + auxv_rseq_align = getauxval(AT_RSEQ_ALIGN); + assert(!auxv_rseq_align || auxv_rseq_align <= RSEQ_THREAD_AREA_ALLOC_SIZE); + + auxv_rseq_feature_size = getauxval(AT_RSEQ_FEATURE_SIZE); + assert(!auxv_rseq_feature_size || auxv_rseq_feature_size <= RSEQ_THREAD_AREA_ALLOC_SIZE); + if (auxv_rseq_feature_size) + return auxv_rseq_feature_size; + else + return ORIG_RSEQ_FEATURE_SIZE; +} + +static __attribute__((constructor)) +void rseq_init(void) +{ + /* + * If the libc's registered rseq size isn't already valid, it may be + * because the binary is dynamically linked and not necessarily due to + * libc not having registered a restartable sequence. Try to find the + * symbols if that's the case. + */ + if (!*libc_rseq_size_p) { + libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset"); + libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size"); + libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags"); + } + if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p && + *libc_rseq_size_p != 0) { + /* rseq registration owned by glibc */ + rseq_offset = *libc_rseq_offset_p; + rseq_size = *libc_rseq_size_p; + rseq_flags = *libc_rseq_flags_p; + rseq_feature_size = get_rseq_feature_size(); + if (rseq_feature_size > rseq_size) + rseq_feature_size = rseq_size; + return; + } + rseq_ownership = 1; + if (!rseq_available()) { + rseq_size = 0; + rseq_feature_size = 0; + return; + } + rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer(); + rseq_flags = 0; + rseq_feature_size = get_rseq_feature_size(); + if (rseq_feature_size == ORIG_RSEQ_FEATURE_SIZE) + rseq_size = ORIG_RSEQ_ALLOC_SIZE; + else + rseq_size = RSEQ_THREAD_AREA_ALLOC_SIZE; +} + +static __attribute__((destructor)) +void rseq_exit(void) +{ + if (!rseq_ownership) + return; + rseq_offset = 0; + rseq_size = -1U; + rseq_feature_size = -1U; + rseq_ownership = 0; } int32_t rseq_fallback_current_cpu(void) @@ -138,19 +225,15 @@ int32_t rseq_fallback_current_cpu(void) return cpu; } -void __attribute__((constructor)) rseq_init(void) +int32_t rseq_fallback_current_node(void) { - /* Check whether rseq is handled by another library. */ - if (__rseq_handled) - return; - __rseq_handled = 1; - rseq_ownership = 1; -} + uint32_t cpu_id, node_id; + int ret; -void __attribute__((destructor)) rseq_fini(void) -{ - if (!rseq_ownership) - return; - __rseq_handled = 0; - rseq_ownership = 0; + ret = sys_getcpu(&cpu_id, &node_id); + if (ret) { + perror("sys_getcpu()"); + return ret; + } + return (int32_t) node_id; } diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h index 3f63eb362b92..d7364ea4d201 100644 --- a/tools/testing/selftests/rseq/rseq.h +++ b/tools/testing/selftests/rseq/rseq.h @@ -16,7 +16,18 @@ #include <errno.h> #include <stdio.h> #include <stdlib.h> -#include <linux/rseq.h> +#include <stddef.h> +#include "rseq-abi.h" +#include "compiler.h" + +#ifndef rseq_sizeof_field +#define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER)) +#endif + +#ifndef rseq_offsetofend +#define rseq_offsetofend(TYPE, MEMBER) \ + (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER)) +#endif /* * Empty code injection macros, override when testing. @@ -43,8 +54,44 @@ #define RSEQ_INJECT_FAILED #endif -extern __thread volatile struct rseq __rseq_abi; -extern int __rseq_handled; +#include "rseq-thread-pointer.h" + +/* Offset from the thread pointer to the rseq area. */ +extern ptrdiff_t rseq_offset; + +/* + * Size of the registered rseq area. 0 if the registration was + * unsuccessful. + */ +extern unsigned int rseq_size; + +/* Flags used during rseq registration. */ +extern unsigned int rseq_flags; + +/* + * rseq feature size supported by the kernel. 0 if the registration was + * unsuccessful. + */ +extern unsigned int rseq_feature_size; + +enum rseq_mo { + RSEQ_MO_RELAXED = 0, + RSEQ_MO_CONSUME = 1, /* Unused */ + RSEQ_MO_ACQUIRE = 2, /* Unused */ + RSEQ_MO_RELEASE = 3, + RSEQ_MO_ACQ_REL = 4, /* Unused */ + RSEQ_MO_SEQ_CST = 5, /* Unused */ +}; + +enum rseq_percpu_mode { + RSEQ_PERCPU_CPU_ID = 0, + RSEQ_PERCPU_MM_CID = 1, +}; + +static inline struct rseq_abi *rseq_get_abi(void) +{ + return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); +} #define rseq_likely(x) __builtin_expect(!!(x), 1) #define rseq_unlikely(x) __builtin_expect(!!(x), 0) @@ -79,6 +126,8 @@ extern int __rseq_handled; #include <rseq-mips.h> #elif defined(__s390__) #include <rseq-s390.h> +#elif defined(__riscv) +#include <rseq-riscv.h> #else #error unsupported target #endif @@ -103,12 +152,17 @@ int rseq_unregister_current_thread(void); int32_t rseq_fallback_current_cpu(void); /* + * Restartable sequence fallback for reading the current node number. + */ +int32_t rseq_fallback_current_node(void); + +/* * Values returned can be either the current CPU number, -1 (rseq is * uninitialized), or -2 (rseq initialization has failed). */ static inline int32_t rseq_current_cpu_raw(void) { - return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id); + return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id); } /* @@ -124,7 +178,7 @@ static inline int32_t rseq_current_cpu_raw(void) */ static inline uint32_t rseq_cpu_start(void) { - return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start); + return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start); } static inline uint32_t rseq_current_cpu(void) @@ -137,13 +191,33 @@ static inline uint32_t rseq_current_cpu(void) return cpu; } +static inline bool rseq_node_id_available(void) +{ + return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, node_id); +} + +/* + * Current NUMA node number. + */ +static inline uint32_t rseq_current_node_id(void) +{ + assert(rseq_node_id_available()); + return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id); +} + +static inline bool rseq_mm_cid_available(void) +{ + return (int) rseq_feature_size >= rseq_offsetofend(struct rseq_abi, mm_cid); +} + +static inline uint32_t rseq_current_mm_cid(void) +{ + return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid); +} + static inline void rseq_clear_rseq_cs(void) { -#ifdef __LP64__ - __rseq_abi.rseq_cs.ptr = 0; -#else - __rseq_abi.rseq_cs.ptr.ptr32 = 0; -#endif + RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); } /* @@ -162,4 +236,149 @@ static inline void rseq_prepare_unload(void) rseq_clear_rseq_cs(); } +static inline __attribute__((always_inline)) +int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + intptr_t newv, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu); + } + return -1; +} + +/* + * Compare @v against @expectnot. When it does _not_ match, load @v + * into @load, and store the content of *@v + voffp into @v. + */ +static inline __attribute__((always_inline)) +int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, + int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu); + } + return -1; +} + +static inline __attribute__((always_inline)) +int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t count, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_addv_relaxed_cpu_id(v, count, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_addv_relaxed_mm_cid(v, count, cpu); + } + return -1; +} + +#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV +/* + * pval = *(ptr+off) + * *pval += inc; + */ +static inline __attribute__((always_inline)) +int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *ptr, long off, intptr_t inc, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu); + } + return -1; +} +#endif + +static inline __attribute__((always_inline)) +int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t newv2, + intptr_t newv, int cpu) +{ + switch (rseq_mo) { + case RSEQ_MO_RELAXED: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); + } + return -1; + case RSEQ_MO_RELEASE: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu); + } + return -1; + default: + return -1; + } +} + +static inline __attribute__((always_inline)) +int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t expect2, + intptr_t newv, int cpu) +{ + if (rseq_mo != RSEQ_MO_RELAXED) + return -1; + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); + } + return -1; +} + +static inline __attribute__((always_inline)) +int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, + intptr_t *v, intptr_t expect, + void *dst, void *src, size_t len, + intptr_t newv, int cpu) +{ + switch (rseq_mo) { + case RSEQ_MO_RELAXED: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); + } + return -1; + case RSEQ_MO_RELEASE: + switch (percpu_mode) { + case RSEQ_PERCPU_CPU_ID: + return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu); + case RSEQ_PERCPU_MM_CID: + return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu); + } + return -1; + default: + return -1; + } +} + #endif /* RSEQ_H_ */ diff --git a/tools/testing/selftests/rseq/run_param_test.sh b/tools/testing/selftests/rseq/run_param_test.sh index e426304fd4a0..8d31426ab41f 100755 --- a/tools/testing/selftests/rseq/run_param_test.sh +++ b/tools/testing/selftests/rseq/run_param_test.sh @@ -15,6 +15,7 @@ TEST_LIST=( "-T m" "-T m -M" "-T i" + "-T r" ) TEST_NAME=( @@ -25,6 +26,7 @@ TEST_NAME=( "memcpy" "memcpy with barrier" "increment" + "membarrier" ) IFS="$OLDIFS" @@ -40,6 +42,11 @@ function do_tests() ./param_test ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 echo "Running compare-twice test ${TEST_NAME[$i]}" ./param_test_compare_twice ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 + + echo "Running mm_cid test ${TEST_NAME[$i]}" + ./param_test_mm_cid ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 + echo "Running mm_cid compare-twice test ${TEST_NAME[$i]}" + ./param_test_mm_cid_compare_twice ${TEST_LIST[$i]} -r ${REPS} -t ${NR_THREADS} ${@} ${EXTRA_ARGS} || exit 1 let "i++" done } |