diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/Makefile | 4 | ||||
| -rw-r--r-- | kernel/cfi.c | 329 | ||||
| -rw-r--r-- | kernel/kallsyms.c | 55 | ||||
| -rw-r--r-- | kernel/kthread.c | 3 | ||||
| -rw-r--r-- | kernel/module.c | 43 | ||||
| -rw-r--r-- | kernel/workqueue.c | 2 |
6 files changed, 429 insertions, 7 deletions
diff --git a/kernel/Makefile b/kernel/Makefile index 320f1f3941b7..e8a6715f38dc 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -41,6 +41,9 @@ KCSAN_SANITIZE_kcov.o := n UBSAN_SANITIZE_kcov.o := n CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector +# Don't instrument error handlers +CFLAGS_REMOVE_cfi.o := $(CC_FLAGS_CFI) + obj-y += sched/ obj-y += locking/ obj-y += power/ @@ -111,6 +114,7 @@ obj-$(CONFIG_BPF) += bpf/ obj-$(CONFIG_KCSAN) += kcsan/ obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o +obj-$(CONFIG_CFI_CLANG) += cfi.o obj-$(CONFIG_PERF_EVENTS) += events/ diff --git a/kernel/cfi.c b/kernel/cfi.c new file mode 100644 index 000000000000..e17a56639766 --- /dev/null +++ b/kernel/cfi.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Clang Control Flow Integrity (CFI) error and slowpath handling. + * + * Copyright (C) 2021 Google LLC + */ + +#include <linux/hardirq.h> +#include <linux/kallsyms.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/printk.h> +#include <linux/ratelimit.h> +#include <linux/rcupdate.h> +#include <linux/vmalloc.h> +#include <asm/cacheflush.h> +#include <asm/set_memory.h> + +/* Compiler-defined handler names */ +#ifdef CONFIG_CFI_PERMISSIVE +#define cfi_failure_handler __ubsan_handle_cfi_check_fail +#else +#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort +#endif + +static inline void handle_cfi_failure(void *ptr) +{ + if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) + WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr); + else + panic("CFI failure (target: %pS)\n", ptr); +} + +#ifdef CONFIG_MODULES +#ifdef CONFIG_CFI_CLANG_SHADOW +/* + * Index type. A 16-bit index can address at most (2^16)-2 pages (taking + * into account SHADOW_INVALID), i.e. ~256M with 4k pages. + */ +typedef u16 shadow_t; +#define SHADOW_INVALID ((shadow_t)~0UL) + +struct cfi_shadow { + /* Page index for the beginning of the shadow */ + unsigned long base; + /* An array of __cfi_check locations (as indices to the shadow) */ + shadow_t shadow[1]; +} __packed; + +/* + * The shadow covers ~128M from the beginning of the module region. If + * the region is larger, we fall back to __module_address for the rest. + */ +#define __SHADOW_RANGE (_UL(SZ_128M) >> PAGE_SHIFT) + +/* The in-memory size of struct cfi_shadow, always at least one page */ +#define __SHADOW_PAGES ((__SHADOW_RANGE * sizeof(shadow_t)) >> PAGE_SHIFT) +#define SHADOW_PAGES max(1UL, __SHADOW_PAGES) +#define SHADOW_SIZE (SHADOW_PAGES << PAGE_SHIFT) + +/* The actual size of the shadow array, minus metadata */ +#define SHADOW_ARR_SIZE (SHADOW_SIZE - offsetof(struct cfi_shadow, shadow)) +#define SHADOW_ARR_SLOTS (SHADOW_ARR_SIZE / sizeof(shadow_t)) + +static DEFINE_MUTEX(shadow_update_lock); +static struct cfi_shadow __rcu *cfi_shadow __read_mostly; + +/* Returns the index in the shadow for the given address */ +static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr) +{ + unsigned long index; + unsigned long page = ptr >> PAGE_SHIFT; + + if (unlikely(page < s->base)) + return -1; /* Outside of module area */ + + index = page - s->base; + + if (index >= SHADOW_ARR_SLOTS) + return -1; /* Cannot be addressed with shadow */ + + return (int)index; +} + +/* Returns the page address for an index in the shadow */ +static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s, + int index) +{ + if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS)) + return 0; + + return (s->base + index) << PAGE_SHIFT; +} + +/* Returns the __cfi_check function address for the given shadow location */ +static inline unsigned long shadow_to_check_fn(const struct cfi_shadow *s, + int index) +{ + if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS)) + return 0; + + if (unlikely(s->shadow[index] == SHADOW_INVALID)) + return 0; + + /* __cfi_check is always page aligned */ + return (s->base + s->shadow[index]) << PAGE_SHIFT; +} + +static void prepare_next_shadow(const struct cfi_shadow __rcu *prev, + struct cfi_shadow *next) +{ + int i, index, check; + + /* Mark everything invalid */ + memset(next->shadow, 0xFF, SHADOW_ARR_SIZE); + + if (!prev) + return; /* No previous shadow */ + + /* If the base address didn't change, an update is not needed */ + if (prev->base == next->base) { + memcpy(next->shadow, prev->shadow, SHADOW_ARR_SIZE); + return; + } + + /* Convert the previous shadow to the new address range */ + for (i = 0; i < SHADOW_ARR_SLOTS; ++i) { + if (prev->shadow[i] == SHADOW_INVALID) + continue; + + index = ptr_to_shadow(next, shadow_to_ptr(prev, i)); + if (index < 0) + continue; + + check = ptr_to_shadow(next, + shadow_to_check_fn(prev, prev->shadow[i])); + if (check < 0) + continue; + + next->shadow[index] = (shadow_t)check; + } +} + +static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod, + unsigned long min_addr, unsigned long max_addr) +{ + int check_index; + unsigned long check = (unsigned long)mod->cfi_check; + unsigned long ptr; + + if (unlikely(!PAGE_ALIGNED(check))) { + pr_warn("cfi: not using shadow for module %s\n", mod->name); + return; + } + + check_index = ptr_to_shadow(s, check); + if (check_index < 0) + return; /* Module not addressable with shadow */ + + /* For each page, store the check function index in the shadow */ + for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) { + int index = ptr_to_shadow(s, ptr); + + if (index >= 0) { + /* Each page must only contain one module */ + WARN_ON_ONCE(s->shadow[index] != SHADOW_INVALID); + s->shadow[index] = (shadow_t)check_index; + } + } +} + +static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod, + unsigned long min_addr, unsigned long max_addr) +{ + unsigned long ptr; + + for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) { + int index = ptr_to_shadow(s, ptr); + + if (index >= 0) + s->shadow[index] = SHADOW_INVALID; + } +} + +typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *, + unsigned long min_addr, unsigned long max_addr); + +static void update_shadow(struct module *mod, unsigned long base_addr, + update_shadow_fn fn) +{ + struct cfi_shadow *prev; + struct cfi_shadow *next; + unsigned long min_addr, max_addr; + + next = vmalloc(SHADOW_SIZE); + + mutex_lock(&shadow_update_lock); + prev = rcu_dereference_protected(cfi_shadow, + mutex_is_locked(&shadow_update_lock)); + + if (next) { + next->base = base_addr >> PAGE_SHIFT; + prepare_next_shadow(prev, next); + + min_addr = (unsigned long)mod->core_layout.base; + max_addr = min_addr + mod->core_layout.text_size; + fn(next, mod, min_addr & PAGE_MASK, max_addr & PAGE_MASK); + + set_memory_ro((unsigned long)next, SHADOW_PAGES); + } + + rcu_assign_pointer(cfi_shadow, next); + mutex_unlock(&shadow_update_lock); + synchronize_rcu(); + + if (prev) { + set_memory_rw((unsigned long)prev, SHADOW_PAGES); + vfree(prev); + } +} + +void cfi_module_add(struct module *mod, unsigned long base_addr) +{ + update_shadow(mod, base_addr, add_module_to_shadow); +} + +void cfi_module_remove(struct module *mod, unsigned long base_addr) +{ + update_shadow(mod, base_addr, remove_module_from_shadow); +} + +static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s, + unsigned long ptr) +{ + int index; + + if (unlikely(!s)) + return NULL; /* No shadow available */ + + index = ptr_to_shadow(s, ptr); + if (index < 0) + return NULL; /* Cannot be addressed with shadow */ + + return (cfi_check_fn)shadow_to_check_fn(s, index); +} + +static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr) +{ + cfi_check_fn fn; + + rcu_read_lock_sched(); + fn = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), ptr); + rcu_read_unlock_sched(); + + return fn; +} + +#else /* !CONFIG_CFI_CLANG_SHADOW */ + +static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr) +{ + return NULL; +} + +#endif /* CONFIG_CFI_CLANG_SHADOW */ + +static inline cfi_check_fn find_module_check_fn(unsigned long ptr) +{ + cfi_check_fn fn = NULL; + struct module *mod; + + rcu_read_lock_sched(); + mod = __module_address(ptr); + if (mod) + fn = mod->cfi_check; + rcu_read_unlock_sched(); + + return fn; +} + +static inline cfi_check_fn find_check_fn(unsigned long ptr) +{ + cfi_check_fn fn = NULL; + + if (is_kernel_text(ptr)) + return __cfi_check; + + /* + * Indirect call checks can happen when RCU is not watching. Both + * the shadow and __module_address use RCU, so we need to wake it + * up if necessary. + */ + RCU_NONIDLE({ + if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW)) + fn = find_shadow_check_fn(ptr); + + if (!fn) + fn = find_module_check_fn(ptr); + }); + + return fn; +} + +void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag) +{ + cfi_check_fn fn = find_check_fn((unsigned long)ptr); + + if (likely(fn)) + fn(id, ptr, diag); + else /* Don't allow unchecked modules */ + handle_cfi_failure(ptr); +} +EXPORT_SYMBOL(__cfi_slowpath_diag); + +#else /* !CONFIG_MODULES */ + +void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag) +{ + handle_cfi_failure(ptr); /* No modules */ +} +EXPORT_SYMBOL(__cfi_slowpath_diag); + +#endif /* CONFIG_MODULES */ + +void cfi_failure_handler(void *data, void *ptr, void *vtable) +{ + handle_cfi_failure(ptr); +} +EXPORT_SYMBOL(cfi_failure_handler); diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 8043a90aa50e..c851ca0ed357 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -161,6 +161,27 @@ static unsigned long kallsyms_sym_address(int idx) return kallsyms_relative_base - 1 - kallsyms_offsets[idx]; } +#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_LTO_CLANG_THIN) +/* + * LLVM appends a hash to static function names when ThinLTO and CFI are + * both enabled, i.e. foo() becomes foo$707af9a22804d33c81801f27dcfe489b. + * This causes confusion and potentially breaks user space tools, so we + * strip the suffix from expanded symbol names. + */ +static inline bool cleanup_symbol_name(char *s) +{ + char *res; + + res = strrchr(s, '$'); + if (res) + *res = '\0'; + + return res != NULL; +} +#else +static inline bool cleanup_symbol_name(char *s) { return false; } +#endif + /* Lookup the address for this symbol. Returns 0 if not found. */ unsigned long kallsyms_lookup_name(const char *name) { @@ -173,6 +194,9 @@ unsigned long kallsyms_lookup_name(const char *name) if (strcmp(namebuf, name) == 0) return kallsyms_sym_address(i); + + if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0) + return kallsyms_sym_address(i); } return module_kallsyms_lookup_name(name); } @@ -303,7 +327,9 @@ const char *kallsyms_lookup(unsigned long addr, namebuf, KSYM_NAME_LEN); if (modname) *modname = NULL; - return namebuf; + + ret = namebuf; + goto found; } /* See if it's in a module or a BPF JITed image. */ @@ -316,11 +342,16 @@ const char *kallsyms_lookup(unsigned long addr, if (!ret) ret = ftrace_mod_address_lookup(addr, symbolsize, offset, modname, namebuf); + +found: + cleanup_symbol_name(namebuf); return ret; } int lookup_symbol_name(unsigned long addr, char *symname) { + int res; + symname[0] = '\0'; symname[KSYM_NAME_LEN - 1] = '\0'; @@ -331,15 +362,23 @@ int lookup_symbol_name(unsigned long addr, char *symname) /* Grab name */ kallsyms_expand_symbol(get_symbol_offset(pos), symname, KSYM_NAME_LEN); - return 0; + goto found; } /* See if it's in a module. */ - return lookup_module_symbol_name(addr, symname); + res = lookup_module_symbol_name(addr, symname); + if (res) + return res; + +found: + cleanup_symbol_name(symname); + return 0; } int lookup_symbol_attrs(unsigned long addr, unsigned long *size, unsigned long *offset, char *modname, char *name) { + int res; + name[0] = '\0'; name[KSYM_NAME_LEN - 1] = '\0'; @@ -351,10 +390,16 @@ int lookup_symbol_attrs(unsigned long addr, unsigned long *size, kallsyms_expand_symbol(get_symbol_offset(pos), name, KSYM_NAME_LEN); modname[0] = '\0'; - return 0; + goto found; } /* See if it's in a module. */ - return lookup_module_symbol_attrs(addr, size, offset, modname, name); + res = lookup_module_symbol_attrs(addr, size, offset, modname, name); + if (res) + return res; + +found: + cleanup_symbol_name(name); + return 0; } /* Look up a kernel symbol and return it in a text buffer. */ diff --git a/kernel/kthread.c b/kernel/kthread.c index 1578973c5740..a1972eba2917 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -963,7 +963,8 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker, struct timer_list *timer = &dwork->timer; struct kthread_work *work = &dwork->work; - WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn); + WARN_ON_FUNCTION_MISMATCH(timer->function, + kthread_delayed_work_timer_fn); /* * If @delay is 0, queue @dwork->work immediately. This is for diff --git a/kernel/module.c b/kernel/module.c index 30479355ab85..20fb004e7d8d 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2146,6 +2146,8 @@ void __weak module_arch_freeing_init(struct module *mod) { } +static void cfi_cleanup(struct module *mod); + /* Free a module, remove from lists, etc. */ static void free_module(struct module *mod) { @@ -2187,6 +2189,9 @@ static void free_module(struct module *mod) synchronize_rcu(); mutex_unlock(&module_mutex); + /* Clean up CFI for the module. */ + cfi_cleanup(mod); + /* This may be empty, but that's OK */ module_arch_freeing_init(mod); module_memfree(mod->init_layout.base); @@ -3866,6 +3871,8 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname, return 0; } +static void cfi_init(struct module *mod); + /* * Allocate and load the module: note that size of section 0 is always * zero, and we rely on this for optional sections. @@ -3997,6 +4004,9 @@ static int load_module(struct load_info *info, const char __user *uargs, flush_module_icache(mod); + /* Setup CFI for the module. */ + cfi_init(mod); + /* Now copy in args */ mod->args = strndup_user(uargs, ~0UL >> 1); if (IS_ERR(mod->args)) { @@ -4070,6 +4080,7 @@ static int load_module(struct load_info *info, const char __user *uargs, synchronize_rcu(); kfree(mod->args); free_arch_cleanup: + cfi_cleanup(mod); module_arch_cleanup(mod); free_modinfo: free_modinfo(mod); @@ -4415,6 +4426,38 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, #endif /* CONFIG_LIVEPATCH */ #endif /* CONFIG_KALLSYMS */ +static void cfi_init(struct module *mod) +{ +#ifdef CONFIG_CFI_CLANG + initcall_t *init; + exitcall_t *exit; + + rcu_read_lock_sched(); + mod->cfi_check = (cfi_check_fn) + find_kallsyms_symbol_value(mod, "__cfi_check"); + init = (initcall_t *) + find_kallsyms_symbol_value(mod, "__cfi_jt_init_module"); + exit = (exitcall_t *) + find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module"); + rcu_read_unlock_sched(); + + /* Fix init/exit functions to point to the CFI jump table */ + if (init) + mod->init = *init; + if (exit) + mod->exit = *exit; + + cfi_module_add(mod, module_addr_min); +#endif +} + +static void cfi_cleanup(struct module *mod) +{ +#ifdef CONFIG_CFI_CLANG + cfi_module_remove(mod, module_addr_min); +#endif +} + /* Maximum number of characters written by module_flags() */ #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 79f2319543ce..b19d759e55a5 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1630,7 +1630,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, struct work_struct *work = &dwork->work; WARN_ON_ONCE(!wq); - WARN_ON_ONCE(timer->function != delayed_work_timer_fn); + WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn); WARN_ON_ONCE(timer_pending(timer)); WARN_ON_ONCE(!list_empty(&work->entry)); |