diff options
Diffstat (limited to 'arch/arm/include')
| -rw-r--r-- | arch/arm/include/asm/assembler.h | 24 | ||||
| -rw-r--r-- | arch/arm/include/asm/current.h | 50 | ||||
| -rw-r--r-- | arch/arm/include/asm/switch_to.h | 2 | ||||
| -rw-r--r-- | arch/arm/include/asm/thread_info.h | 2 |
4 files changed, 78 insertions, 0 deletions
diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index e2b1fd558bf3..c1551dee28be 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -199,6 +199,30 @@ .endm .endr + .macro get_current, rd +#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO + mrc p15, 0, \rd, c13, c0, 3 @ get TPIDRURO register +#else + get_thread_info \rd + ldr \rd, [\rd, #TI_TASK] +#endif + .endm + + .macro set_current, rn +#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO + mcr p15, 0, \rn, c13, c0, 3 @ set TPIDRURO register +#endif + .endm + + .macro reload_current, t1:req, t2:req +#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO + adr_l \t1, __entry_task @ get __entry_task base address + mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset + ldr \t1, [\t1, \t2] @ load variable + mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO +#endif + .endm + /* * Get current thread_info. */ diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h new file mode 100644 index 000000000000..1d472fa7697b --- /dev/null +++ b/arch/arm/include/asm/current.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 Keith Packard <keithp@keithp.com> + * Copyright (c) 2021 Google, LLC <ardb@kernel.org> + */ + +#ifndef _ASM_ARM_CURRENT_H +#define _ASM_ARM_CURRENT_H + +#ifndef __ASSEMBLY__ + +struct task_struct; + +static inline void set_current(struct task_struct *cur) +{ + if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) + return; + + /* Set TPIDRURO */ + asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); +} + +#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO + +static inline struct task_struct *get_current(void) +{ + struct task_struct *cur; + +#if __has_builtin(__builtin_thread_pointer) + /* + * Use the __builtin helper when available - this results in better + * code, especially when using GCC in combination with the per-task + * stack protector, as the compiler will recognize that it needs to + * load the TLS register only once in every function. + */ + cur = __builtin_thread_pointer(); +#else + asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur)); +#endif + return cur; +} + +#define current get_current() +#else +#include <asm-generic/current.h> +#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */ + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_ARM_CURRENT_H */ diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h index 007d8fea7157..61e4a3c4ca6e 100644 --- a/arch/arm/include/asm/switch_to.h +++ b/arch/arm/include/asm/switch_to.h @@ -26,6 +26,8 @@ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info #define switch_to(prev,next,last) \ do { \ __complete_pending_tlbi(); \ + if (IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) \ + __this_cpu_write(__entry_task, next); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index f0cacc733231..76b6fbd5540c 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -29,6 +29,8 @@ struct task_struct; +DECLARE_PER_CPU(struct task_struct *, __entry_task); + #include <asm/types.h> struct cpu_context_save { |