diff options
Diffstat (limited to 'include/linux/instrumented.h')
| -rw-r--r-- | include/linux/instrumented.h | 133 |
1 files changed, 105 insertions, 28 deletions
diff --git a/include/linux/instrumented.h b/include/linux/instrumented.h index 42faebbaa202..711a1f0d1a73 100644 --- a/include/linux/instrumented.h +++ b/include/linux/instrumented.h @@ -2,7 +2,7 @@ /* * This header provides generic wrappers for memory access instrumentation that - * the compiler cannot emit for: KASAN, KCSAN. + * the compiler cannot emit for: KASAN, KCSAN, KMSAN. */ #ifndef _LINUX_INSTRUMENTED_H #define _LINUX_INSTRUMENTED_H @@ -10,16 +10,16 @@ #include <linux/compiler.h> #include <linux/kasan-checks.h> #include <linux/kcsan-checks.h> +#include <linux/kmsan-checks.h> #include <linux/types.h> /** * instrument_read - instrument regular read access + * @v: address of access + * @size: size of access * * Instrument a regular read access. The instrumentation should be inserted * before the actual read happens. - * - * @ptr address of access - * @size size of access */ static __always_inline void instrument_read(const volatile void *v, size_t size) { @@ -29,12 +29,11 @@ static __always_inline void instrument_read(const volatile void *v, size_t size) /** * instrument_write - instrument regular write access + * @v: address of access + * @size: size of access * * Instrument a regular write access. The instrumentation should be inserted * before the actual write happens. - * - * @ptr address of access - * @size size of access */ static __always_inline void instrument_write(const volatile void *v, size_t size) { @@ -44,12 +43,11 @@ static __always_inline void instrument_write(const volatile void *v, size_t size /** * instrument_read_write - instrument regular read-write access + * @v: address of access + * @size: size of access * * Instrument a regular write access. The instrumentation should be inserted * before the actual write happens. - * - * @ptr address of access - * @size size of access */ static __always_inline void instrument_read_write(const volatile void *v, size_t size) { @@ -59,12 +57,11 @@ static __always_inline void instrument_read_write(const volatile void *v, size_t /** * instrument_atomic_read - instrument atomic read access + * @v: address of access + * @size: size of access * * Instrument an atomic read access. The instrumentation should be inserted * before the actual read happens. - * - * @ptr address of access - * @size size of access */ static __always_inline void instrument_atomic_read(const volatile void *v, size_t size) { @@ -74,12 +71,11 @@ static __always_inline void instrument_atomic_read(const volatile void *v, size_ /** * instrument_atomic_write - instrument atomic write access + * @v: address of access + * @size: size of access * * Instrument an atomic write access. The instrumentation should be inserted * before the actual write happens. - * - * @ptr address of access - * @size size of access */ static __always_inline void instrument_atomic_write(const volatile void *v, size_t size) { @@ -89,12 +85,11 @@ static __always_inline void instrument_atomic_write(const volatile void *v, size /** * instrument_atomic_read_write - instrument atomic read-write access + * @v: address of access + * @size: size of access * * Instrument an atomic read-write access. The instrumentation should be * inserted before the actual write happens. - * - * @ptr address of access - * @size size of access */ static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size) { @@ -104,36 +99,118 @@ static __always_inline void instrument_atomic_read_write(const volatile void *v, /** * instrument_copy_to_user - instrument reads of copy_to_user + * @to: destination address + * @from: source address + * @n: number of bytes to copy * * Instrument reads from kernel memory, that are due to copy_to_user (and * variants). The instrumentation must be inserted before the accesses. - * - * @to destination address - * @from source address - * @n number of bytes to copy */ static __always_inline void instrument_copy_to_user(void __user *to, const void *from, unsigned long n) { kasan_check_read(from, n); kcsan_check_read(from, n); + kmsan_copy_to_user(to, from, n, 0); } /** - * instrument_copy_from_user - instrument writes of copy_from_user + * instrument_copy_from_user_before - add instrumentation before copy_from_user + * @to: destination address + * @from: source address + * @n: number of bytes to copy * * Instrument writes to kernel memory, that are due to copy_from_user (and * variants). The instrumentation should be inserted before the accesses. + */ +static __always_inline void +instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n) +{ + kasan_check_write(to, n); + kcsan_check_write(to, n); +} + +/** + * instrument_copy_from_user_after - add instrumentation after copy_from_user + * @to: destination address + * @from: source address + * @n: number of bytes to copy + * @left: number of bytes not copied (as returned by copy_from_user) * - * @to destination address - * @from source address - * @n number of bytes to copy + * Instrument writes to kernel memory, that are due to copy_from_user (and + * variants). The instrumentation should be inserted after the accesses. */ static __always_inline void -instrument_copy_from_user(const void *to, const void __user *from, unsigned long n) +instrument_copy_from_user_after(const void *to, const void __user *from, + unsigned long n, unsigned long left) +{ + kmsan_unpoison_memory(to, n - left); +} + +/** + * instrument_memcpy_before - add instrumentation before non-instrumented memcpy + * @to: destination address + * @from: source address + * @n: number of bytes to copy + * + * Instrument memory accesses that happen in custom memcpy implementations. The + * instrumentation should be inserted before the memcpy call. + */ +static __always_inline void instrument_memcpy_before(void *to, const void *from, + unsigned long n) { kasan_check_write(to, n); + kasan_check_read(from, n); kcsan_check_write(to, n); + kcsan_check_read(from, n); +} + +/** + * instrument_memcpy_after - add instrumentation after non-instrumented memcpy + * @to: destination address + * @from: source address + * @n: number of bytes to copy + * @left: number of bytes not copied (if known) + * + * Instrument memory accesses that happen in custom memcpy implementations. The + * instrumentation should be inserted after the memcpy call. + */ +static __always_inline void instrument_memcpy_after(void *to, const void *from, + unsigned long n, + unsigned long left) +{ + kmsan_memmove(to, from, n - left); } +/** + * instrument_get_user() - add instrumentation to get_user()-like macros + * @to: destination variable, may not be address-taken + * + * get_user() and friends are fragile, so it may depend on the implementation + * whether the instrumentation happens before or after the data is copied from + * the userspace. + */ +#define instrument_get_user(to) \ +({ \ + u64 __tmp = (u64)(to); \ + kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \ + to = __tmp; \ +}) + + +/** + * instrument_put_user() - add instrumentation to put_user()-like macros + * @from: source address + * @ptr: userspace pointer to copy to + * @size: number of bytes to copy + * + * put_user() and friends are fragile, so it may depend on the implementation + * whether the instrumentation happens before or after the data is copied from + * the userspace. + */ +#define instrument_put_user(from, ptr, size) \ +({ \ + kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \ +}) + #endif /* _LINUX_INSTRUMENTED_H */ |