diff options
Diffstat (limited to '')
| -rw-r--r-- | mm/maccess.c | 276 |
1 files changed, 111 insertions, 165 deletions
diff --git a/mm/maccess.c b/mm/maccess.c index 3ca8d97e5010..5f4d240f67ec 100644 --- a/mm/maccess.c +++ b/mm/maccess.c @@ -1,77 +1,108 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Access kernel memory without faulting. + * Access kernel or user memory without faulting. */ #include <linux/export.h> #include <linux/mm.h> #include <linux/uaccess.h> -static __always_inline long -probe_read_common(void *dst, const void __user *src, size_t size) +bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, + size_t size) { - long ret; - - pagefault_disable(); - ret = __copy_from_user_inatomic(dst, src, size); - pagefault_enable(); - - return ret ? -EFAULT : 0; + return true; } -static __always_inline long -probe_write_common(void __user *dst, const void *src, size_t size) +#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ + while (len >= sizeof(type)) { \ + __get_kernel_nofault(dst, src, type, err_label); \ + dst += sizeof(type); \ + src += sizeof(type); \ + len -= sizeof(type); \ + } + +long copy_from_kernel_nofault(void *dst, const void *src, size_t size) { - long ret; + unsigned long align = 0; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) + align = (unsigned long)dst | (unsigned long)src; + + if (!copy_from_kernel_nofault_allowed(src, size)) + return -ERANGE; pagefault_disable(); - ret = __copy_to_user_inatomic(dst, src, size); + if (!(align & 7)) + copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); + if (!(align & 3)) + copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); + if (!(align & 1)) + copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); + copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); pagefault_enable(); - - return ret ? -EFAULT : 0; + return 0; +Efault: + pagefault_enable(); + return -EFAULT; } +EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); + +#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ + while (len >= sizeof(type)) { \ + __put_kernel_nofault(dst, src, type, err_label); \ + dst += sizeof(type); \ + src += sizeof(type); \ + len -= sizeof(type); \ + } -/** - * probe_kernel_read(): safely attempt to read from a kernel-space location - * @dst: pointer to the buffer that shall take the data - * @src: address to read from - * @size: size of the data chunk - * - * Safely read from address @src to the buffer at @dst. If a kernel fault - * happens, handle that and return -EFAULT. - * - * We ensure that the copy_from_user is executed in atomic context so that - * do_page_fault() doesn't attempt to take mmap_sem. This makes - * probe_kernel_read() suitable for use within regions where the caller - * already holds mmap_sem, or other locks which nest inside mmap_sem. - * - * probe_kernel_read_strict() is the same as probe_kernel_read() except for - * the case where architectures have non-overlapping user and kernel address - * ranges: probe_kernel_read_strict() will additionally return -EFAULT for - * probing memory on a user address range where probe_user_read() is supposed - * to be used instead. - */ +long copy_to_kernel_nofault(void *dst, const void *src, size_t size) +{ + unsigned long align = 0; -long __weak probe_kernel_read(void *dst, const void *src, size_t size) - __attribute__((alias("__probe_kernel_read"))); + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) + align = (unsigned long)dst | (unsigned long)src; -long __weak probe_kernel_read_strict(void *dst, const void *src, size_t size) - __attribute__((alias("__probe_kernel_read"))); + pagefault_disable(); + if (!(align & 7)) + copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); + if (!(align & 3)) + copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); + if (!(align & 1)) + copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); + copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); + pagefault_enable(); + return 0; +Efault: + pagefault_enable(); + return -EFAULT; +} -long __probe_kernel_read(void *dst, const void *src, size_t size) +long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) { - long ret; - mm_segment_t old_fs = get_fs(); + const void *src = unsafe_addr; - set_fs(KERNEL_DS); - ret = probe_read_common(dst, (__force const void __user *)src, size); - set_fs(old_fs); + if (unlikely(count <= 0)) + return 0; + if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) + return -ERANGE; - return ret; + pagefault_disable(); + do { + __get_kernel_nofault(dst, src, u8, Efault); + dst++; + src++; + } while (dst[-1] && src - unsafe_addr < count); + pagefault_enable(); + + dst[-1] = '\0'; + return src - unsafe_addr; +Efault: + pagefault_enable(); + dst[-1] = '\0'; + return -EFAULT; } -EXPORT_SYMBOL_GPL(probe_kernel_read); /** - * probe_user_read(): safely attempt to read from a user-space location + * copy_from_user_nofault(): safely attempt to read from a user-space location * @dst: pointer to the buffer that shall take the data * @src: address to read from. This must be a user address. * @size: size of the data chunk @@ -79,52 +110,23 @@ EXPORT_SYMBOL_GPL(probe_kernel_read); * Safely read from user address @src to the buffer at @dst. If a kernel fault * happens, handle that and return -EFAULT. */ - -long __weak probe_user_read(void *dst, const void __user *src, size_t size) - __attribute__((alias("__probe_user_read"))); - -long __probe_user_read(void *dst, const void __user *src, size_t size) +long copy_from_user_nofault(void *dst, const void __user *src, size_t size) { long ret = -EFAULT; - mm_segment_t old_fs = get_fs(); - - set_fs(USER_DS); - if (access_ok(src, size)) - ret = probe_read_common(dst, src, size); - set_fs(old_fs); - - return ret; -} -EXPORT_SYMBOL_GPL(probe_user_read); - -/** - * probe_kernel_write(): safely attempt to write to a location - * @dst: address to write to - * @src: pointer to the data that shall be written - * @size: size of the data chunk - * - * Safely write to address @dst from the buffer at @src. If a kernel fault - * happens, handle that and return -EFAULT. - */ - -long __weak probe_kernel_write(void *dst, const void *src, size_t size) - __attribute__((alias("__probe_kernel_write"))); - -long __probe_kernel_write(void *dst, const void *src, size_t size) -{ - long ret; - mm_segment_t old_fs = get_fs(); - - set_fs(KERNEL_DS); - ret = probe_write_common((__force void __user *)dst, src, size); - set_fs(old_fs); + if (access_ok(src, size)) { + pagefault_disable(); + ret = __copy_from_user_inatomic(dst, src, size); + pagefault_enable(); + } - return ret; + if (ret) + return -EFAULT; + return 0; } -EXPORT_SYMBOL_GPL(probe_kernel_write); +EXPORT_SYMBOL_GPL(copy_from_user_nofault); /** - * probe_user_write(): safely attempt to write to a user-space location + * copy_to_user_nofault(): safely attempt to write to a user-space location * @dst: address to write to * @src: pointer to the data that shall be written * @size: size of the data chunk @@ -132,80 +134,24 @@ EXPORT_SYMBOL_GPL(probe_kernel_write); * Safely write to address @dst from the buffer at @src. If a kernel fault * happens, handle that and return -EFAULT. */ - -long __weak probe_user_write(void __user *dst, const void *src, size_t size) - __attribute__((alias("__probe_user_write"))); - -long __probe_user_write(void __user *dst, const void *src, size_t size) +long copy_to_user_nofault(void __user *dst, const void *src, size_t size) { long ret = -EFAULT; - mm_segment_t old_fs = get_fs(); - - set_fs(USER_DS); - if (access_ok(dst, size)) - ret = probe_write_common(dst, src, size); - set_fs(old_fs); - - return ret; -} -EXPORT_SYMBOL_GPL(probe_user_write); - -/** - * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address. - * @dst: Destination address, in kernel space. This buffer must be at - * least @count bytes long. - * @unsafe_addr: Unsafe address. - * @count: Maximum number of bytes to copy, including the trailing NUL. - * - * Copies a NUL-terminated string from unsafe address to kernel buffer. - * - * On success, returns the length of the string INCLUDING the trailing NUL. - * - * If access fails, returns -EFAULT (some data may have been copied - * and the trailing NUL added). - * - * If @count is smaller than the length of the string, copies @count-1 bytes, - * sets the last byte of @dst buffer to NUL and returns @count. - * - * strncpy_from_unsafe_strict() is the same as strncpy_from_unsafe() except - * for the case where architectures have non-overlapping user and kernel address - * ranges: strncpy_from_unsafe_strict() will additionally return -EFAULT for - * probing memory on a user address range where strncpy_from_unsafe_user() is - * supposed to be used instead. - */ - -long __weak strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) - __attribute__((alias("__strncpy_from_unsafe"))); -long __weak strncpy_from_unsafe_strict(char *dst, const void *unsafe_addr, - long count) - __attribute__((alias("__strncpy_from_unsafe"))); - -long __strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) -{ - mm_segment_t old_fs = get_fs(); - const void *src = unsafe_addr; - long ret; - - if (unlikely(count <= 0)) - return 0; - - set_fs(KERNEL_DS); - pagefault_disable(); - - do { - ret = __get_user(*dst++, (const char __user __force *)src++); - } while (dst[-1] && ret == 0 && src - unsafe_addr < count); - - dst[-1] = '\0'; - pagefault_enable(); - set_fs(old_fs); + if (access_ok(dst, size)) { + pagefault_disable(); + ret = __copy_to_user_inatomic(dst, src, size); + pagefault_enable(); + } - return ret ? -EFAULT : src - unsafe_addr; + if (ret) + return -EFAULT; + return 0; } +EXPORT_SYMBOL_GPL(copy_to_user_nofault); /** - * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user + * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user * address. * @dst: Destination address, in kernel space. This buffer must be at * least @count bytes long. @@ -222,20 +168,17 @@ long __strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count) * If @count is smaller than the length of the string, copies @count-1 bytes, * sets the last byte of @dst buffer to NUL and returns @count. */ -long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, +long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, long count) { - mm_segment_t old_fs = get_fs(); long ret; if (unlikely(count <= 0)) return 0; - set_fs(USER_DS); pagefault_disable(); ret = strncpy_from_user(dst, unsafe_addr, count); pagefault_enable(); - set_fs(old_fs); if (ret >= count) { ret = count; @@ -248,7 +191,7 @@ long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, } /** - * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL. + * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. * @unsafe_addr: The string to measure. * @count: Maximum count (including NUL) * @@ -263,16 +206,19 @@ long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr, * Unlike strnlen_user, this can be used from IRQ handler etc. because * it disables pagefaults. */ -long strnlen_unsafe_user(const void __user *unsafe_addr, long count) +long strnlen_user_nofault(const void __user *unsafe_addr, long count) { - mm_segment_t old_fs = get_fs(); int ret; - set_fs(USER_DS); pagefault_disable(); ret = strnlen_user(unsafe_addr, count); pagefault_enable(); - set_fs(old_fs); return ret; } + +void __copy_overflow(int size, unsigned long count) +{ + WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); +} +EXPORT_SYMBOL(__copy_overflow); |