From 926a59b1dfe2580e1a00bb2ba1664e5472077cbd Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:35 +0930 Subject: module: Annotate module version magic Due to the new lockdep checks in the coming patch, we go: [ 9.759380] ------------[ cut here ]------------ [ 9.759389] WARNING: CPU: 31 PID: 597 at ../kernel/module.c:216 each_symbol_section+0x121/0x130() [ 9.759391] Modules linked in: [ 9.759393] CPU: 31 PID: 597 Comm: modprobe Not tainted 4.0.0-rc1+ #65 [ 9.759393] Hardware name: Intel Corporation S2600GZ/S2600GZ, BIOS SE5C600.86B.02.02.0002.122320131210 12/23/2013 [ 9.759396] ffffffff817d8676 ffff880424567ca8 ffffffff8157e98b 0000000000000001 [ 9.759398] 0000000000000000 ffff880424567ce8 ffffffff8105fbc7 ffff880424567cd8 [ 9.759400] 0000000000000000 ffffffff810ec160 ffff880424567d40 0000000000000000 [ 9.759400] Call Trace: [ 9.759407] [] dump_stack+0x4f/0x7b [ 9.759410] [] warn_slowpath_common+0x97/0xe0 [ 9.759412] [] ? section_objs+0x60/0x60 [ 9.759414] [] warn_slowpath_null+0x1a/0x20 [ 9.759415] [] each_symbol_section+0x121/0x130 [ 9.759417] [] find_symbol+0x31/0x70 [ 9.759420] [] load_module+0x20f/0x2660 [ 9.759422] [] ? __do_page_fault+0x190/0x4e0 [ 9.759426] [] ? retint_restore_args+0x13/0x13 [ 9.759427] [] ? retint_restore_args+0x13/0x13 [ 9.759433] [] ? trace_hardirqs_on_caller+0x11d/0x1e0 [ 9.759437] [] ? trace_hardirqs_on_thunk+0x3a/0x3f [ 9.759439] [] ? retint_restore_args+0x13/0x13 [ 9.759441] [] SyS_init_module+0xce/0x100 [ 9.759443] [] system_call_fastpath+0x12/0x17 [ 9.759445] ---[ end trace 9294429076a9c644 ]--- As per the comment this site should be fine, but lets wrap it in preempt_disable() anyhow to placate lockdep. Cc: Rusty Russell Acked-by: Paul E. McKenney Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- kernel/module.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/kernel/module.c b/kernel/module.c index 42a1d2afb217..1150d5239205 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1169,11 +1169,17 @@ static inline int check_modstruct_version(Elf_Shdr *sechdrs, { const unsigned long *crc; - /* Since this should be found in kernel (which can't be removed), - * no locking is necessary. */ + /* + * Since this should be found in kernel (which can't be removed), no + * locking is necessary -- use preempt_disable() to placate lockdep. + */ + preempt_disable(); if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL, - &crc, true, false)) + &crc, true, false)) { + preempt_enable(); BUG(); + } + preempt_enable(); return check_version(sechdrs, versindex, VMLINUX_SYMBOL_STR(module_layout), mod, crc, NULL); -- cgit v1.2.3-59-g8ed1b From bed831f9a251968272dae10a83b512c7db256ef0 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:35 +0930 Subject: module, jump_label: Fix module locking As per the module core lockdep annotations in the coming patch: [ 18.034047] ---[ end trace 9294429076a9c673 ]--- [ 18.047760] Hardware name: Intel Corporation S2600GZ/S2600GZ, BIOS SE5C600.86B.02.02.0002.122320131210 12/23/2013 [ 18.059228] ffffffff817d8676 ffff880036683c38 ffffffff8157e98b 0000000000000001 [ 18.067541] 0000000000000000 ffff880036683c78 ffffffff8105fbc7 ffff880036683c68 [ 18.075851] ffffffffa0046b08 0000000000000000 ffffffffa0046d00 ffffffffa0046cc8 [ 18.084173] Call Trace: [ 18.086906] [] dump_stack+0x4f/0x7b [ 18.092649] [] warn_slowpath_common+0x97/0xe0 [ 18.099361] [] warn_slowpath_null+0x1a/0x20 [ 18.105880] [] __module_address+0x1d2/0x1e0 [ 18.112400] [] jump_label_module_notify+0x143/0x1e0 [ 18.119710] [] notifier_call_chain+0x4f/0x70 [ 18.126326] [] __blocking_notifier_call_chain+0x5e/0x90 [ 18.134009] [] blocking_notifier_call_chain+0x16/0x20 [ 18.141490] [] load_module+0x1b50/0x2660 [ 18.147720] [] SyS_init_module+0xce/0x100 [ 18.154045] [] system_call_fastpath+0x12/0x17 [ 18.160748] ---[ end trace 9294429076a9c674 ]--- Jump labels is not doing it right; fix this. Cc: Rusty Russell Cc: Jason Baron Acked-by: Paul E. McKenney Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- kernel/jump_label.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 9019f15deab2..52ebaca1b9fc 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -302,7 +302,7 @@ static int jump_label_add_module(struct module *mod) continue; key = iterk; - if (__module_address(iter->key) == mod) { + if (within_module(iter->key, mod)) { /* * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. */ @@ -339,7 +339,7 @@ static void jump_label_del_module(struct module *mod) key = (struct static_key *)(unsigned long)iter->key; - if (__module_address(iter->key) == mod) + if (within_module(iter->key, mod)) continue; prev = &key->next; @@ -443,14 +443,16 @@ static void jump_label_update(struct static_key *key, int enable) { struct jump_entry *stop = __stop___jump_table; struct jump_entry *entry = jump_label_get_entries(key); - #ifdef CONFIG_MODULES - struct module *mod = __module_address((unsigned long)key); + struct module *mod; __jump_label_mod_update(key, enable); + preempt_disable(); + mod = __module_address((unsigned long)key); if (mod) stop = mod->jump_entries + mod->num_jump_entries; + preempt_enable(); #endif /* if there are no users, entry can be NULL */ if (entry) -- cgit v1.2.3-59-g8ed1b From 0be964be0d45084245673c971d72a4b51690231d Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:35 +0930 Subject: module: Sanitize RCU usage and locking Currently the RCU usage in module is an inconsistent mess of RCU and RCU-sched, this is broken for CONFIG_PREEMPT where synchronize_rcu() does not imply synchronize_sched(). Most usage sites use preempt_{dis,en}able() which is RCU-sched, but (most of) the modification sites use synchronize_rcu(). With the exception of the module bug list, which actually uses RCU. Convert everything over to RCU-sched. Furthermore add lockdep asserts to all sites, because it's not at all clear to me the required locking is observed, esp. on exported functions. Cc: Rusty Russell Acked-by: "Paul E. McKenney" Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/module.h | 12 ++++++++++-- kernel/module.c | 40 ++++++++++++++++++++++++++++++++-------- lib/bug.c | 7 +++++-- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/include/linux/module.h b/include/linux/module.h index c883b86ea964..fb56dd85a862 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -421,14 +421,22 @@ struct symsearch { bool unused; }; -/* Search for an exported symbol by name. */ +/* + * Search for an exported symbol by name. + * + * Must be called with module_mutex held or preemption disabled. + */ const struct kernel_symbol *find_symbol(const char *name, struct module **owner, const unsigned long **crc, bool gplok, bool warn); -/* Walk the exported symbol table */ +/* + * Walk the exported symbol table + * + * Must be called with module_mutex held or preemption disabled. + */ bool each_symbol_section(bool (*fn)(const struct symsearch *arr, struct module *owner, void *data), void *data); diff --git a/kernel/module.c b/kernel/module.c index 1150d5239205..a15899e00ca9 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -105,6 +105,22 @@ static LIST_HEAD(modules); struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */ #endif /* CONFIG_KGDB_KDB */ +static void module_assert_mutex(void) +{ + lockdep_assert_held(&module_mutex); +} + +static void module_assert_mutex_or_preempt(void) +{ +#ifdef CONFIG_LOCKDEP + if (unlikely(!debug_locks)) + return; + + WARN_ON(!rcu_read_lock_sched_held() && + !lockdep_is_held(&module_mutex)); +#endif +} + #ifdef CONFIG_MODULE_SIG #ifdef CONFIG_MODULE_SIG_FORCE static bool sig_enforce = true; @@ -318,6 +334,8 @@ bool each_symbol_section(bool (*fn)(const struct symsearch *arr, #endif }; + module_assert_mutex_or_preempt(); + if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data)) return true; @@ -457,6 +475,8 @@ static struct module *find_module_all(const char *name, size_t len, { struct module *mod; + module_assert_mutex(); + list_for_each_entry(mod, &modules, list) { if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) continue; @@ -1860,8 +1880,8 @@ static void free_module(struct module *mod) list_del_rcu(&mod->list); /* Remove this module from bug list, this uses list_del_rcu */ module_bug_cleanup(mod); - /* Wait for RCU synchronizing before releasing mod->list and buglist. */ - synchronize_rcu(); + /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */ + synchronize_sched(); mutex_unlock(&module_mutex); /* This may be NULL, but that's OK */ @@ -3133,11 +3153,11 @@ static noinline int do_init_module(struct module *mod) mod->init_text_size = 0; /* * We want to free module_init, but be aware that kallsyms may be - * walking this with preempt disabled. In all the failure paths, - * we call synchronize_rcu/synchronize_sched, but we don't want - * to slow down the success path, so use actual RCU here. + * walking this with preempt disabled. In all the failure paths, we + * call synchronize_sched(), but we don't want to slow down the success + * path, so use actual RCU here. */ - call_rcu(&freeinit->rcu, do_free_init); + call_rcu_sched(&freeinit->rcu, do_free_init); mutex_unlock(&module_mutex); wake_up_all(&module_wq); @@ -3395,8 +3415,8 @@ static int load_module(struct load_info *info, const char __user *uargs, /* Unlink carefully: kallsyms could be walking list. */ list_del_rcu(&mod->list); wake_up_all(&module_wq); - /* Wait for RCU synchronizing before releasing mod->list. */ - synchronize_rcu(); + /* Wait for RCU-sched synchronizing before releasing mod->list. */ + synchronize_sched(); mutex_unlock(&module_mutex); free_module: /* Free lock-classes; relies on the preceding sync_rcu() */ @@ -3663,6 +3683,8 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, unsigned int i; int ret; + module_assert_mutex(); + list_for_each_entry(mod, &modules, list) { if (mod->state == MODULE_STATE_UNFORMED) continue; @@ -3837,6 +3859,8 @@ struct module *__module_address(unsigned long addr) if (addr < module_addr_min || addr > module_addr_max) return NULL; + module_assert_mutex_or_preempt(); + list_for_each_entry_rcu(mod, &modules, list) { if (mod->state == MODULE_STATE_UNFORMED) continue; diff --git a/lib/bug.c b/lib/bug.c index 0c3bd9552b6f..cff145f032a5 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -66,7 +66,7 @@ static const struct bug_entry *module_find_bug(unsigned long bugaddr) struct module *mod; const struct bug_entry *bug = NULL; - rcu_read_lock(); + rcu_read_lock_sched(); list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { unsigned i; @@ -77,7 +77,7 @@ static const struct bug_entry *module_find_bug(unsigned long bugaddr) } bug = NULL; out: - rcu_read_unlock(); + rcu_read_unlock_sched(); return bug; } @@ -88,6 +88,8 @@ void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, char *secstrings; unsigned int i; + lockdep_assert_held(&module_mutex); + mod->bug_table = NULL; mod->num_bugs = 0; @@ -113,6 +115,7 @@ void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, void module_bug_cleanup(struct module *mod) { + lockdep_assert_held(&module_mutex); list_del_rcu(&mod->bug_list); } -- cgit v1.2.3-59-g8ed1b From d72da4a4d973d8a0a0d3c97e7cdebf287fbe3a99 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:36 +0930 Subject: rbtree: Make lockless searches non-fatal Change the insert and erase code such that lockless searches are non-fatal. In and of itself an rbtree cannot be correctly searched while in-modification, we can however provide weaker guarantees that will allow the rbtree to be used in conjunction with other techniques, such as latches; see 9b0fd802e8c0 ("seqcount: Add raw_write_seqcount_latch()"). For this to work we need the following guarantees from the rbtree code: 1) a lockless reader must not see partial stores, this would allow it to observe nodes that are invalid memory. 2) there must not be (temporary) loops in the tree structure in the modifier's program order, this would cause a lookup which interrupts the modifier to get stuck indefinitely. For 1) we must use WRITE_ONCE() for all updates to the tree structure; in particular this patch only does rb_{left,right} as those are the only element required for simple searches. It generates slightly worse code, probably because volatile. But in pointer chasing heavy code a few instructions more should not matter. For 2) I have carefully audited the code and drawn every intermediate link state and not found a loop. Cc: Mathieu Desnoyers Cc: "Paul E. McKenney" Cc: Oleg Nesterov Cc: Andrea Arcangeli Cc: David Woodhouse Cc: Rik van Riel Reviewed-by: Michel Lespinasse Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/rbtree.h | 16 +++++++-- include/linux/rbtree_augmented.h | 21 +++++++---- lib/rbtree.c | 76 ++++++++++++++++++++++++++++------------ 3 files changed, 81 insertions(+), 32 deletions(-) diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index fb31765e935a..830c4992088d 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -31,6 +31,7 @@ #include #include +#include struct rb_node { unsigned long __rb_parent_color; @@ -73,11 +74,11 @@ extern struct rb_node *rb_first_postorder(const struct rb_root *); extern struct rb_node *rb_next_postorder(const struct rb_node *); /* Fast replacement of a single node without remove/rebalance/add/rebalance */ -extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, +extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, struct rb_root *root); -static inline void rb_link_node(struct rb_node * node, struct rb_node * parent, - struct rb_node ** rb_link) +static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, + struct rb_node **rb_link) { node->__rb_parent_color = (unsigned long)parent; node->rb_left = node->rb_right = NULL; @@ -85,6 +86,15 @@ static inline void rb_link_node(struct rb_node * node, struct rb_node * parent, *rb_link = node; } +static inline void rb_link_node_rcu(struct rb_node *node, struct rb_node *parent, + struct rb_node **rb_link) +{ + node->__rb_parent_color = (unsigned long)parent; + node->rb_left = node->rb_right = NULL; + + rcu_assign_pointer(*rb_link, node); +} + #define rb_entry_safe(ptr, type, member) \ ({ typeof(ptr) ____ptr = (ptr); \ ____ptr ? rb_entry(____ptr, type, member) : NULL; \ diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 378c5ee75f78..14d7b831b63a 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -123,11 +123,11 @@ __rb_change_child(struct rb_node *old, struct rb_node *new, { if (parent) { if (parent->rb_left == old) - parent->rb_left = new; + WRITE_ONCE(parent->rb_left, new); else - parent->rb_right = new; + WRITE_ONCE(parent->rb_right, new); } else - root->rb_node = new; + WRITE_ONCE(root->rb_node, new); } extern void __rb_erase_color(struct rb_node *parent, struct rb_root *root, @@ -137,7 +137,8 @@ static __always_inline struct rb_node * __rb_erase_augmented(struct rb_node *node, struct rb_root *root, const struct rb_augment_callbacks *augment) { - struct rb_node *child = node->rb_right, *tmp = node->rb_left; + struct rb_node *child = node->rb_right; + struct rb_node *tmp = node->rb_left; struct rb_node *parent, *rebalance; unsigned long pc; @@ -167,6 +168,7 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root, tmp = parent; } else { struct rb_node *successor = child, *child2; + tmp = child->rb_left; if (!tmp) { /* @@ -180,6 +182,7 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root, */ parent = successor; child2 = successor->rb_right; + augment->copy(node, successor); } else { /* @@ -201,19 +204,23 @@ __rb_erase_augmented(struct rb_node *node, struct rb_root *root, successor = tmp; tmp = tmp->rb_left; } while (tmp); - parent->rb_left = child2 = successor->rb_right; - successor->rb_right = child; + child2 = successor->rb_right; + WRITE_ONCE(parent->rb_left, child2); + WRITE_ONCE(successor->rb_right, child); rb_set_parent(child, successor); + augment->copy(node, successor); augment->propagate(parent, successor); } - successor->rb_left = tmp = node->rb_left; + tmp = node->rb_left; + WRITE_ONCE(successor->rb_left, tmp); rb_set_parent(tmp, successor); pc = node->__rb_parent_color; tmp = __rb_parent(pc); __rb_change_child(node, successor, tmp, root); + if (child2) { successor->__rb_parent_color = pc; rb_set_parent_color(child2, parent, RB_BLACK); diff --git a/lib/rbtree.c b/lib/rbtree.c index c16c81a3d430..1356454e36de 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -44,6 +44,30 @@ * parentheses and have some accompanying text comment. */ +/* + * Notes on lockless lookups: + * + * All stores to the tree structure (rb_left and rb_right) must be done using + * WRITE_ONCE(). And we must not inadvertently cause (temporary) loops in the + * tree structure as seen in program order. + * + * These two requirements will allow lockless iteration of the tree -- not + * correct iteration mind you, tree rotations are not atomic so a lookup might + * miss entire subtrees. + * + * But they do guarantee that any such traversal will only see valid elements + * and that it will indeed complete -- does not get stuck in a loop. + * + * It also guarantees that if the lookup returns an element it is the 'correct' + * one. But not returning an element does _NOT_ mean it's not present. + * + * NOTE: + * + * Stores to __rb_parent_color are not important for simple lookups so those + * are left undone as of now. Nor did I check for loops involving parent + * pointers. + */ + static inline void rb_set_black(struct rb_node *rb) { rb->__rb_parent_color |= RB_BLACK; @@ -129,8 +153,9 @@ __rb_insert(struct rb_node *node, struct rb_root *root, * This still leaves us in violation of 4), the * continuation into Case 3 will fix that. */ - parent->rb_right = tmp = node->rb_left; - node->rb_left = parent; + tmp = node->rb_left; + WRITE_ONCE(parent->rb_right, tmp); + WRITE_ONCE(node->rb_left, parent); if (tmp) rb_set_parent_color(tmp, parent, RB_BLACK); @@ -149,8 +174,8 @@ __rb_insert(struct rb_node *node, struct rb_root *root, * / \ * n U */ - gparent->rb_left = tmp; /* == parent->rb_right */ - parent->rb_right = gparent; + WRITE_ONCE(gparent->rb_left, tmp); /* == parent->rb_right */ + WRITE_ONCE(parent->rb_right, gparent); if (tmp) rb_set_parent_color(tmp, gparent, RB_BLACK); __rb_rotate_set_parents(gparent, parent, root, RB_RED); @@ -171,8 +196,9 @@ __rb_insert(struct rb_node *node, struct rb_root *root, tmp = parent->rb_left; if (node == tmp) { /* Case 2 - right rotate at parent */ - parent->rb_left = tmp = node->rb_right; - node->rb_right = parent; + tmp = node->rb_right; + WRITE_ONCE(parent->rb_left, tmp); + WRITE_ONCE(node->rb_right, parent); if (tmp) rb_set_parent_color(tmp, parent, RB_BLACK); @@ -183,8 +209,8 @@ __rb_insert(struct rb_node *node, struct rb_root *root, } /* Case 3 - left rotate at gparent */ - gparent->rb_right = tmp; /* == parent->rb_left */ - parent->rb_left = gparent; + WRITE_ONCE(gparent->rb_right, tmp); /* == parent->rb_left */ + WRITE_ONCE(parent->rb_left, gparent); if (tmp) rb_set_parent_color(tmp, gparent, RB_BLACK); __rb_rotate_set_parents(gparent, parent, root, RB_RED); @@ -224,8 +250,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root, * / \ / \ * Sl Sr N Sl */ - parent->rb_right = tmp1 = sibling->rb_left; - sibling->rb_left = parent; + tmp1 = sibling->rb_left; + WRITE_ONCE(parent->rb_right, tmp1); + WRITE_ONCE(sibling->rb_left, parent); rb_set_parent_color(tmp1, parent, RB_BLACK); __rb_rotate_set_parents(parent, sibling, root, RB_RED); @@ -275,9 +302,10 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root, * \ * Sr */ - sibling->rb_left = tmp1 = tmp2->rb_right; - tmp2->rb_right = sibling; - parent->rb_right = tmp2; + tmp1 = tmp2->rb_right; + WRITE_ONCE(sibling->rb_left, tmp1); + WRITE_ONCE(tmp2->rb_right, sibling); + WRITE_ONCE(parent->rb_right, tmp2); if (tmp1) rb_set_parent_color(tmp1, sibling, RB_BLACK); @@ -297,8 +325,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root, * / \ / \ * (sl) sr N (sl) */ - parent->rb_right = tmp2 = sibling->rb_left; - sibling->rb_left = parent; + tmp2 = sibling->rb_left; + WRITE_ONCE(parent->rb_right, tmp2); + WRITE_ONCE(sibling->rb_left, parent); rb_set_parent_color(tmp1, sibling, RB_BLACK); if (tmp2) rb_set_parent(tmp2, parent); @@ -310,8 +339,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root, sibling = parent->rb_left; if (rb_is_red(sibling)) { /* Case 1 - right rotate at parent */ - parent->rb_left = tmp1 = sibling->rb_right; - sibling->rb_right = parent; + tmp1 = sibling->rb_right; + WRITE_ONCE(parent->rb_left, tmp1); + WRITE_ONCE(sibling->rb_right, parent); rb_set_parent_color(tmp1, parent, RB_BLACK); __rb_rotate_set_parents(parent, sibling, root, RB_RED); @@ -336,9 +366,10 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root, break; } /* Case 3 - right rotate at sibling */ - sibling->rb_right = tmp1 = tmp2->rb_left; - tmp2->rb_left = sibling; - parent->rb_left = tmp2; + tmp1 = tmp2->rb_left; + WRITE_ONCE(sibling->rb_right, tmp1); + WRITE_ONCE(tmp2->rb_left, sibling); + WRITE_ONCE(parent->rb_left, tmp2); if (tmp1) rb_set_parent_color(tmp1, sibling, RB_BLACK); @@ -347,8 +378,9 @@ ____rb_erase_color(struct rb_node *parent, struct rb_root *root, sibling = tmp2; } /* Case 4 - left rotate at parent + color flips */ - parent->rb_left = tmp2 = sibling->rb_right; - sibling->rb_right = parent; + tmp2 = sibling->rb_right; + WRITE_ONCE(parent->rb_left, tmp2); + WRITE_ONCE(sibling->rb_right, parent); rb_set_parent_color(tmp1, sibling, RB_BLACK); if (tmp2) rb_set_parent(tmp2, parent); -- cgit v1.2.3-59-g8ed1b From 6695b92a60bc7160c92d6dc5b17cc79673017c2f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:36 +0930 Subject: seqlock: Better document raw_write_seqcount_latch() Improve the documentation of the latch technique as used in the current timekeeping code, such that it can be readily employed elsewhere. Borrow from the comments in timekeeping and replace those with a reference to this more generic comment. Cc: Andrea Arcangeli Cc: David Woodhouse Cc: Rik van Riel Cc: "Paul E. McKenney" Cc: Oleg Nesterov Reviewed-by: Mathieu Desnoyers Acked-by: Michel Lespinasse Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/seqlock.h | 76 ++++++++++++++++++++++++++++++++++++++++++++++- kernel/time/timekeeping.c | 27 +---------------- 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 5f68d0a391ce..1c0cf3102fdc 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -233,9 +233,83 @@ static inline void raw_write_seqcount_end(seqcount_t *s) s->sequence++; } -/* +/** * raw_write_seqcount_latch - redirect readers to even/odd copy * @s: pointer to seqcount_t + * + * The latch technique is a multiversion concurrency control method that allows + * queries during non-atomic modifications. If you can guarantee queries never + * interrupt the modification -- e.g. the concurrency is strictly between CPUs + * -- you most likely do not need this. + * + * Where the traditional RCU/lockless data structures rely on atomic + * modifications to ensure queries observe either the old or the new state the + * latch allows the same for non-atomic updates. The trade-off is doubling the + * cost of storage; we have to maintain two copies of the entire data + * structure. + * + * Very simply put: we first modify one copy and then the other. This ensures + * there is always one copy in a stable state, ready to give us an answer. + * + * The basic form is a data structure like: + * + * struct latch_struct { + * seqcount_t seq; + * struct data_struct data[2]; + * }; + * + * Where a modification, which is assumed to be externally serialized, does the + * following: + * + * void latch_modify(struct latch_struct *latch, ...) + * { + * smp_wmb(); <- Ensure that the last data[1] update is visible + * latch->seq++; + * smp_wmb(); <- Ensure that the seqcount update is visible + * + * modify(latch->data[0], ...); + * + * smp_wmb(); <- Ensure that the data[0] update is visible + * latch->seq++; + * smp_wmb(); <- Ensure that the seqcount update is visible + * + * modify(latch->data[1], ...); + * } + * + * The query will have a form like: + * + * struct entry *latch_query(struct latch_struct *latch, ...) + * { + * struct entry *entry; + * unsigned seq, idx; + * + * do { + * seq = latch->seq; + * smp_rmb(); + * + * idx = seq & 0x01; + * entry = data_query(latch->data[idx], ...); + * + * smp_rmb(); + * } while (seq != latch->seq); + * + * return entry; + * } + * + * So during the modification, queries are first redirected to data[1]. Then we + * modify data[0]. When that is complete, we redirect queries back to data[0] + * and we can modify data[1]. + * + * NOTE: The non-requirement for atomic modifications does _NOT_ include + * the publishing of new entries in the case where data is a dynamic + * data structure. + * + * An iteration might start in data[0] and get suspended long enough + * to miss an entire modification sequence, once it resumes it might + * observe the new entry. + * + * NOTE: When data is a dynamic data structure; one should use regular RCU + * patterns to manage the lifetimes of the objects within. */ static inline void raw_write_seqcount_latch(seqcount_t *s) { diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 946acb72179f..cbfedddbf0cb 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -330,32 +330,7 @@ static inline s64 timekeeping_get_ns(struct tk_read_base *tkr) * We want to use this from any context including NMI and tracing / * instrumenting the timekeeping code itself. * - * So we handle this differently than the other timekeeping accessor - * functions which retry when the sequence count has changed. The - * update side does: - * - * smp_wmb(); <- Ensure that the last base[1] update is visible - * tkf->seq++; - * smp_wmb(); <- Ensure that the seqcount update is visible - * update(tkf->base[0], tkr); - * smp_wmb(); <- Ensure that the base[0] update is visible - * tkf->seq++; - * smp_wmb(); <- Ensure that the seqcount update is visible - * update(tkf->base[1], tkr); - * - * The reader side does: - * - * do { - * seq = tkf->seq; - * smp_rmb(); - * idx = seq & 0x01; - * now = now(tkf->base[idx]); - * smp_rmb(); - * } while (seq != tkf->seq) - * - * As long as we update base[0] readers are forced off to - * base[1]. Once base[0] is updated readers are redirected to base[0] - * and the base[1] update takes place. + * Employ the latch technique; see @raw_write_seqcount_latch. * * So if a NMI hits the update of base[0] then it will use base[1] * which is still consistent. In the worst case this can result is a -- cgit v1.2.3-59-g8ed1b From 0a04b0166929405cd833c1cc40f99e862b965ddc Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:36 +0930 Subject: rcu: Move lockless_dereference() out of rcupdate.h I want to use lockless_dereference() from seqlock.h, which would mean including rcupdate.h from it, however rcupdate.h already includes seqlock.h. Avoid this by moving lockless_dereference() into compiler.h. This is somewhat tricky since it uses smp_read_barrier_depends() which isn't available there, but its a CPP macro so we can get away with it. The alternative would be moving it into asm/barrier.h, but that would be updating each arch (I can do if people feel that is more appropriate). Cc: Paul McKenney Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/compiler.h | 15 +++++++++++++++ include/linux/rcupdate.h | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 867722591be2..eae42c21d5fd 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -457,6 +457,21 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s (volatile typeof(x) *)&(x); }) #define ACCESS_ONCE(x) (*__ACCESS_ONCE(x)) +/** + * lockless_dereference() - safely load a pointer for later dereference + * @p: The pointer to load + * + * Similar to rcu_dereference(), but for situations where the pointed-to + * object's lifetime is managed by something other than RCU. That + * "something other" might be reference counting or simple immortality. + */ +#define lockless_dereference(p) \ +({ \ + typeof(p) _________p1 = ACCESS_ONCE(p); \ + smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ + (_________p1); \ +}) + /* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */ #ifdef CONFIG_KPROBES # define __kprobes __attribute__((__section__(".kprobes.text"))) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 573a5afd5ed8..0356ad954ea5 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -649,21 +649,6 @@ static inline void rcu_preempt_sleep_check(void) */ #define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v) -/** - * lockless_dereference() - safely load a pointer for later dereference - * @p: The pointer to load - * - * Similar to rcu_dereference(), but for situations where the pointed-to - * object's lifetime is managed by something other than RCU. That - * "something other" might be reference counting or simple immortality. - */ -#define lockless_dereference(p) \ -({ \ - typeof(p) _________p1 = ACCESS_ONCE(p); \ - smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ - (_________p1); \ -}) - /** * rcu_assign_pointer() - assign to RCU-protected pointer * @p: pointer to assign to -- cgit v1.2.3-59-g8ed1b From 7fc26327b75685f37f58d64bdb061460f834f80d Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:36 +0930 Subject: seqlock: Introduce raw_read_seqcount_latch() Because with latches there is a strict data dependency on the seq load we can avoid the rmb in favour of a read_barrier_depends. Suggested-by: Ingo Molnar Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/seqlock.h | 9 +++++++-- kernel/time/timekeeping.c | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 1c0cf3102fdc..890c7ef709d5 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -35,6 +35,7 @@ #include #include #include +#include #include /* @@ -233,6 +234,11 @@ static inline void raw_write_seqcount_end(seqcount_t *s) s->sequence++; } +static inline int raw_read_seqcount_latch(seqcount_t *s) +{ + return lockless_dereference(s->sequence); +} + /** * raw_write_seqcount_latch - redirect readers to even/odd copy * @s: pointer to seqcount_t @@ -284,8 +290,7 @@ static inline void raw_write_seqcount_end(seqcount_t *s) * unsigned seq, idx; * * do { - * seq = latch->seq; - * smp_rmb(); + * seq = lockless_dereference(latch->seq); * * idx = seq & 0x01; * entry = data_query(latch->data[idx], ...); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index cbfedddbf0cb..266dafe8f015 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -393,7 +393,7 @@ static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) u64 now; do { - seq = raw_read_seqcount(&tkf->seq); + seq = raw_read_seqcount_latch(&tkf->seq); tkr = tkf->base + (seq & 0x01); now = ktime_to_ns(tkr->base) + timekeeping_get_ns(tkr); } while (read_seqcount_retry(&tkf->seq, seq)); -- cgit v1.2.3-59-g8ed1b From ade3f510f93a5613b672febe88eff8ea7f1c63b7 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:37 +0930 Subject: rbtree: Implement generic latch_tree Implement a latched RB-tree in order to get unconditional RCU/lockless lookups. Cc: Oleg Nesterov Cc: Michel Lespinasse Cc: Andrea Arcangeli Cc: David Woodhouse Cc: Rik van Riel Cc: Mathieu Desnoyers Cc: "Paul E. McKenney" Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/rbtree_latch.h | 212 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 include/linux/rbtree_latch.h diff --git a/include/linux/rbtree_latch.h b/include/linux/rbtree_latch.h new file mode 100644 index 000000000000..4f3432c61d12 --- /dev/null +++ b/include/linux/rbtree_latch.h @@ -0,0 +1,212 @@ +/* + * Latched RB-trees + * + * Copyright (C) 2015 Intel Corp., Peter Zijlstra + * + * Since RB-trees have non-atomic modifications they're not immediately suited + * for RCU/lockless queries. Even though we made RB-tree lookups non-fatal for + * lockless lookups; we cannot guarantee they return a correct result. + * + * The simplest solution is a seqlock + RB-tree, this will allow lockless + * lookups; but has the constraint (inherent to the seqlock) that read sides + * cannot nest in write sides. + * + * If we need to allow unconditional lookups (say as required for NMI context + * usage) we need a more complex setup; this data structure provides this by + * employing the latch technique -- see @raw_write_seqcount_latch -- to + * implement a latched RB-tree which does allow for unconditional lookups by + * virtue of always having (at least) one stable copy of the tree. + * + * However, while we have the guarantee that there is at all times one stable + * copy, this does not guarantee an iteration will not observe modifications. + * What might have been a stable copy at the start of the iteration, need not + * remain so for the duration of the iteration. + * + * Therefore, this does require a lockless RB-tree iteration to be non-fatal; + * see the comment in lib/rbtree.c. Note however that we only require the first + * condition -- not seeing partial stores -- because the latch thing isolates + * us from loops. If we were to interrupt a modification the lookup would be + * pointed at the stable tree and complete while the modification was halted. + */ + +#ifndef RB_TREE_LATCH_H +#define RB_TREE_LATCH_H + +#include +#include + +struct latch_tree_node { + struct rb_node node[2]; +}; + +struct latch_tree_root { + seqcount_t seq; + struct rb_root tree[2]; +}; + +/** + * latch_tree_ops - operators to define the tree order + * @less: used for insertion; provides the (partial) order between two elements. + * @comp: used for lookups; provides the order between the search key and an element. + * + * The operators are related like: + * + * comp(a->key,b) < 0 := less(a,b) + * comp(a->key,b) > 0 := less(b,a) + * comp(a->key,b) == 0 := !less(a,b) && !less(b,a) + * + * If these operators define a partial order on the elements we make no + * guarantee on which of the elements matching the key is found. See + * latch_tree_find(). + */ +struct latch_tree_ops { + bool (*less)(struct latch_tree_node *a, struct latch_tree_node *b); + int (*comp)(void *key, struct latch_tree_node *b); +}; + +static __always_inline struct latch_tree_node * +__lt_from_rb(struct rb_node *node, int idx) +{ + return container_of(node, struct latch_tree_node, node[idx]); +} + +static __always_inline void +__lt_insert(struct latch_tree_node *ltn, struct latch_tree_root *ltr, int idx, + bool (*less)(struct latch_tree_node *a, struct latch_tree_node *b)) +{ + struct rb_root *root = <r->tree[idx]; + struct rb_node **link = &root->rb_node; + struct rb_node *node = <n->node[idx]; + struct rb_node *parent = NULL; + struct latch_tree_node *ltp; + + while (*link) { + parent = *link; + ltp = __lt_from_rb(parent, idx); + + if (less(ltn, ltp)) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + + rb_link_node_rcu(node, parent, link); + rb_insert_color(node, root); +} + +static __always_inline void +__lt_erase(struct latch_tree_node *ltn, struct latch_tree_root *ltr, int idx) +{ + rb_erase(<n->node[idx], <r->tree[idx]); +} + +static __always_inline struct latch_tree_node * +__lt_find(void *key, struct latch_tree_root *ltr, int idx, + int (*comp)(void *key, struct latch_tree_node *node)) +{ + struct rb_node *node = rcu_dereference_raw(ltr->tree[idx].rb_node); + struct latch_tree_node *ltn; + int c; + + while (node) { + ltn = __lt_from_rb(node, idx); + c = comp(key, ltn); + + if (c < 0) + node = rcu_dereference_raw(node->rb_left); + else if (c > 0) + node = rcu_dereference_raw(node->rb_right); + else + return ltn; + } + + return NULL; +} + +/** + * latch_tree_insert() - insert @node into the trees @root + * @node: nodes to insert + * @root: trees to insert @node into + * @ops: operators defining the node order + * + * It inserts @node into @root in an ordered fashion such that we can always + * observe one complete tree. See the comment for raw_write_seqcount_latch(). + * + * The inserts use rcu_assign_pointer() to publish the element such that the + * tree structure is stored before we can observe the new @node. + * + * All modifications (latch_tree_insert, latch_tree_remove) are assumed to be + * serialized. + */ +static __always_inline void +latch_tree_insert(struct latch_tree_node *node, + struct latch_tree_root *root, + const struct latch_tree_ops *ops) +{ + raw_write_seqcount_latch(&root->seq); + __lt_insert(node, root, 0, ops->less); + raw_write_seqcount_latch(&root->seq); + __lt_insert(node, root, 1, ops->less); +} + +/** + * latch_tree_erase() - removes @node from the trees @root + * @node: nodes to remote + * @root: trees to remove @node from + * @ops: operators defining the node order + * + * Removes @node from the trees @root in an ordered fashion such that we can + * always observe one complete tree. See the comment for + * raw_write_seqcount_latch(). + * + * It is assumed that @node will observe one RCU quiescent state before being + * reused of freed. + * + * All modifications (latch_tree_insert, latch_tree_remove) are assumed to be + * serialized. + */ +static __always_inline void +latch_tree_erase(struct latch_tree_node *node, + struct latch_tree_root *root, + const struct latch_tree_ops *ops) +{ + raw_write_seqcount_latch(&root->seq); + __lt_erase(node, root, 0); + raw_write_seqcount_latch(&root->seq); + __lt_erase(node, root, 1); +} + +/** + * latch_tree_find() - find the node matching @key in the trees @root + * @key: search key + * @root: trees to search for @key + * @ops: operators defining the node order + * + * Does a lockless lookup in the trees @root for the node matching @key. + * + * It is assumed that this is called while holding the appropriate RCU read + * side lock. + * + * If the operators define a partial order on the elements (there are multiple + * elements which have the same key value) it is undefined which of these + * elements will be found. Nor is it possible to iterate the tree to find + * further elements with the same key value. + * + * Returns: a pointer to the node matching @key or NULL. + */ +static __always_inline struct latch_tree_node * +latch_tree_find(void *key, struct latch_tree_root *root, + const struct latch_tree_ops *ops) +{ + struct latch_tree_node *node; + unsigned int seq; + + do { + seq = raw_read_seqcount_latch(&root->seq); + node = __lt_find(key, root, seq & 1, ops->comp); + } while (read_seqcount_retry(&root->seq, seq)); + + return node; +} + +#endif /* RB_TREE_LATCH_H */ -- cgit v1.2.3-59-g8ed1b From 93c2e105f6bcee231c951ba0e56e84505c4b0483 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:37 +0930 Subject: module: Optimize __module_address() using a latched RB-tree Currently __module_address() is using a linear search through all modules in order to find the module corresponding to the provided address. With a lot of modules this can take a lot of time. One of the users of this is kernel_text_address() which is employed in many stack unwinders; which in turn are used by perf-callchain and ftrace (possibly from NMI context). So by optimizing __module_address() we optimize many stack unwinders which are used by both perf and tracing in performance sensitive code. Cc: Rusty Russell Cc: Steven Rostedt Cc: Mathieu Desnoyers Cc: Oleg Nesterov Cc: "Paul E. McKenney" Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/module.h | 29 +++++++++++-- kernel/module.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 8 deletions(-) diff --git a/include/linux/module.h b/include/linux/module.h index fb56dd85a862..ddf35a3368fb 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -210,6 +211,13 @@ enum module_state { MODULE_STATE_UNFORMED, /* Still setting it up. */ }; +struct module; + +struct mod_tree_node { + struct module *mod; + struct latch_tree_node node; +}; + struct module { enum module_state state; @@ -269,8 +277,15 @@ struct module { /* Startup function. */ int (*init)(void); - /* If this is non-NULL, vfree after init() returns */ - void *module_init; + /* + * If this is non-NULL, vfree() after init() returns. + * + * Cacheline align here, such that: + * module_init, module_core, init_size, core_size, + * init_text_size, core_text_size and ltn_core.node[0] + * are on the same cacheline. + */ + void *module_init ____cacheline_aligned; /* Here is the actual code + data, vfree'd on unload. */ void *module_core; @@ -281,6 +296,14 @@ struct module { /* The size of the executable code in each section. */ unsigned int init_text_size, core_text_size; + /* + * We want mtn_core::{mod,node[0]} to be in the same cacheline as the + * above entries such that a regular lookup will only touch one + * cacheline. + */ + struct mod_tree_node mtn_core; + struct mod_tree_node mtn_init; + /* Size of RO sections of the module (text+rodata) */ unsigned int init_ro_size, core_ro_size; @@ -367,7 +390,7 @@ struct module { ctor_fn_t *ctors; unsigned int num_ctors; #endif -}; +} ____cacheline_aligned; #ifndef MODULE_ARCH_INIT #define MODULE_ARCH_INIT {} #endif diff --git a/kernel/module.c b/kernel/module.c index a15899e00ca9..e0db5c31cb53 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -101,6 +101,108 @@ DEFINE_MUTEX(module_mutex); EXPORT_SYMBOL_GPL(module_mutex); static LIST_HEAD(modules); + +/* + * Use a latched RB-tree for __module_address(); this allows us to use + * RCU-sched lookups of the address from any context. + * + * Because modules have two address ranges: init and core, we need two + * latch_tree_nodes entries. Therefore we need the back-pointer from + * mod_tree_node. + * + * Because init ranges are short lived we mark them unlikely and have placed + * them outside the critical cacheline in struct module. + */ + +static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n) +{ + struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node); + struct module *mod = mtn->mod; + + if (unlikely(mtn == &mod->mtn_init)) + return (unsigned long)mod->module_init; + + return (unsigned long)mod->module_core; +} + +static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n) +{ + struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node); + struct module *mod = mtn->mod; + + if (unlikely(mtn == &mod->mtn_init)) + return (unsigned long)mod->init_size; + + return (unsigned long)mod->core_size; +} + +static __always_inline bool +mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b) +{ + return __mod_tree_val(a) < __mod_tree_val(b); +} + +static __always_inline int +mod_tree_comp(void *key, struct latch_tree_node *n) +{ + unsigned long val = (unsigned long)key; + unsigned long start, end; + + start = __mod_tree_val(n); + if (val < start) + return -1; + + end = start + __mod_tree_size(n); + if (val >= end) + return 1; + + return 0; +} + +static const struct latch_tree_ops mod_tree_ops = { + .less = mod_tree_less, + .comp = mod_tree_comp, +}; + +static struct latch_tree_root mod_tree __cacheline_aligned; + +/* + * These modifications: insert, remove_init and remove; are serialized by the + * module_mutex. + */ +static void mod_tree_insert(struct module *mod) +{ + mod->mtn_core.mod = mod; + mod->mtn_init.mod = mod; + + latch_tree_insert(&mod->mtn_core.node, &mod_tree, &mod_tree_ops); + if (mod->init_size) + latch_tree_insert(&mod->mtn_init.node, &mod_tree, &mod_tree_ops); +} + +static void mod_tree_remove_init(struct module *mod) +{ + if (mod->init_size) + latch_tree_erase(&mod->mtn_init.node, &mod_tree, &mod_tree_ops); +} + +static void mod_tree_remove(struct module *mod) +{ + latch_tree_erase(&mod->mtn_core.node, &mod_tree, &mod_tree_ops); + mod_tree_remove_init(mod); +} + +static struct module *mod_tree_find(unsigned long addr) +{ + struct latch_tree_node *ltn; + + ltn = latch_tree_find((void *)addr, &mod_tree, &mod_tree_ops); + if (!ltn) + return NULL; + + return container_of(ltn, struct mod_tree_node, node)->mod; +} + #ifdef CONFIG_KGDB_KDB struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */ #endif /* CONFIG_KGDB_KDB */ @@ -1878,6 +1980,7 @@ static void free_module(struct module *mod) mutex_lock(&module_mutex); /* Unlink carefully: kallsyms could be walking list. */ list_del_rcu(&mod->list); + mod_tree_remove(mod); /* Remove this module from bug list, this uses list_del_rcu */ module_bug_cleanup(mod); /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */ @@ -3145,6 +3248,7 @@ static noinline int do_init_module(struct module *mod) mod->symtab = mod->core_symtab; mod->strtab = mod->core_strtab; #endif + mod_tree_remove_init(mod); unset_module_init_ro_nx(mod); module_arch_freeing_init(mod); mod->module_init = NULL; @@ -3215,6 +3319,7 @@ again: goto out; } list_add_rcu(&mod->list, &modules); + mod_tree_insert(mod); err = 0; out: @@ -3861,13 +3966,13 @@ struct module *__module_address(unsigned long addr) module_assert_mutex_or_preempt(); - list_for_each_entry_rcu(mod, &modules, list) { + mod = mod_tree_find(addr); + if (mod) { + BUG_ON(!within_module(addr, mod)); if (mod->state == MODULE_STATE_UNFORMED) - continue; - if (within_module(addr, mod)) - return mod; + mod = NULL; } - return NULL; + return mod; } EXPORT_SYMBOL_GPL(__module_address); -- cgit v1.2.3-59-g8ed1b From 6c9692e2d6a2206d8fd75ea247daa47fb75e4a02 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:37 +0930 Subject: module: Make the mod_tree stuff conditional on PERF_EVENTS || TRACING Andrew worried about the overhead on small systems; only use the fancy code when either perf or tracing is enabled. Cc: Rusty Russell Cc: Steven Rostedt Requested-by: Andrew Morton Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- include/linux/module.h | 4 +++- init/Kconfig | 4 ++++ kernel/module.c | 30 ++++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/include/linux/module.h b/include/linux/module.h index ddf35a3368fb..4c1b02e1361d 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -282,7 +282,7 @@ struct module { * * Cacheline align here, such that: * module_init, module_core, init_size, core_size, - * init_text_size, core_text_size and ltn_core.node[0] + * init_text_size, core_text_size and mtn_core::{mod,node[0]} * are on the same cacheline. */ void *module_init ____cacheline_aligned; @@ -296,6 +296,7 @@ struct module { /* The size of the executable code in each section. */ unsigned int init_text_size, core_text_size; +#ifdef CONFIG_MODULES_TREE_LOOKUP /* * We want mtn_core::{mod,node[0]} to be in the same cacheline as the * above entries such that a regular lookup will only touch one @@ -303,6 +304,7 @@ struct module { */ struct mod_tree_node mtn_core; struct mod_tree_node mtn_init; +#endif /* Size of RO sections of the module (text+rodata) */ unsigned int init_ro_size, core_ro_size; diff --git a/init/Kconfig b/init/Kconfig index dc24dec60232..968a001790af 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1989,6 +1989,10 @@ endchoice endif # MODULES +config MODULES_TREE_LOOKUP + def_bool y + depends on PERF_EVENTS || TRACING + config INIT_ALL_POSSIBLE bool help diff --git a/kernel/module.c b/kernel/module.c index e0db5c31cb53..ac3044ceca3f 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -102,6 +102,8 @@ DEFINE_MUTEX(module_mutex); EXPORT_SYMBOL_GPL(module_mutex); static LIST_HEAD(modules); +#ifdef CONFIG_MODULES_TREE_LOOKUP + /* * Use a latched RB-tree for __module_address(); this allows us to use * RCU-sched lookups of the address from any context. @@ -112,6 +114,10 @@ static LIST_HEAD(modules); * * Because init ranges are short lived we mark them unlikely and have placed * them outside the critical cacheline in struct module. + * + * This is conditional on PERF_EVENTS || TRACING because those can really hit + * __module_address() hard by doing a lot of stack unwinding; potentially from + * NMI context. */ static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n) @@ -192,7 +198,7 @@ static void mod_tree_remove(struct module *mod) mod_tree_remove_init(mod); } -static struct module *mod_tree_find(unsigned long addr) +static struct module *mod_find(unsigned long addr) { struct latch_tree_node *ltn; @@ -203,6 +209,26 @@ static struct module *mod_tree_find(unsigned long addr) return container_of(ltn, struct mod_tree_node, node)->mod; } +#else /* MODULES_TREE_LOOKUP */ + +static void mod_tree_insert(struct module *mod) { } +static void mod_tree_remove_init(struct module *mod) { } +static void mod_tree_remove(struct module *mod) { } + +static struct module *mod_find(unsigned long addr) +{ + struct module *mod; + + list_for_each_entry_rcu(mod, &modules, list) { + if (within_module(addr, mod)) + return mod; + } + + return NULL; +} + +#endif /* MODULES_TREE_LOOKUP */ + #ifdef CONFIG_KGDB_KDB struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */ #endif /* CONFIG_KGDB_KDB */ @@ -3966,7 +3992,7 @@ struct module *__module_address(unsigned long addr) module_assert_mutex_or_preempt(); - mod = mod_tree_find(addr); + mod = mod_find(addr); if (mod) { BUG_ON(!within_module(addr, mod)); if (mod->state == MODULE_STATE_UNFORMED) -- cgit v1.2.3-59-g8ed1b From b7df4d1b23bfca830f1076412d21524686c5a441 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:37 +0930 Subject: module: Use __module_address() for module_address_lookup() Use the generic __module_address() addr to struct module lookup instead of open coding it once more. Cc: Rusty Russell Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- kernel/module.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/kernel/module.c b/kernel/module.c index ac3044ceca3f..293dfaf4ce52 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -3671,19 +3671,15 @@ const char *module_address_lookup(unsigned long addr, char **modname, char *namebuf) { - struct module *mod; const char *ret = NULL; + struct module *mod; preempt_disable(); - list_for_each_entry_rcu(mod, &modules, list) { - if (mod->state == MODULE_STATE_UNFORMED) - continue; - if (within_module(addr, mod)) { - if (modname) - *modname = mod->name; - ret = get_ksymbol(mod, addr, size, offset); - break; - } + mod = __module_address(addr); + if (mod) { + if (modname) + *modname = mod->name; + ret = get_ksymbol(mod, addr, size, offset); } /* Make a copy in here where it's safe */ if (ret) { @@ -3691,6 +3687,7 @@ const char *module_address_lookup(unsigned long addr, ret = namebuf; } preempt_enable(); + return ret; } -- cgit v1.2.3-59-g8ed1b From 4f666546d047752c17265f4641cc9470c1cbaed4 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 27 May 2015 11:09:38 +0930 Subject: module: Rework module_addr_{min,max} __module_address() does an initial bound check before doing the {list/tree} iteration to find the actual module. The bound variables are nowhere near the mod_tree cacheline, in fact they're nowhere near one another. module_addr_min lives in .data while module_addr_max lives in .bss (smarty pants GCC thinks the explicit 0 assignment is a mistake). Rectify this by moving the two variables into a structure together with the latch_tree_root to guarantee they all share the same cacheline and avoid hitting two extra cachelines for the lookup. While reworking the bounds code, move the bound update from allocation to insertion time, this avoids updating the bounds for a few error paths. Cc: Rusty Russell Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Rusty Russell --- kernel/module.c | 80 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/kernel/module.c b/kernel/module.c index 293dfaf4ce52..c8da2a59ebf7 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -170,7 +170,26 @@ static const struct latch_tree_ops mod_tree_ops = { .comp = mod_tree_comp, }; -static struct latch_tree_root mod_tree __cacheline_aligned; +static struct mod_tree_root { + struct latch_tree_root root; + unsigned long addr_min; + unsigned long addr_max; +} mod_tree __cacheline_aligned = { + .addr_min = -1UL, +}; + +#define module_addr_min mod_tree.addr_min +#define module_addr_max mod_tree.addr_max + +static noinline void __mod_tree_insert(struct mod_tree_node *node) +{ + latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops); +} + +static void __mod_tree_remove(struct mod_tree_node *node) +{ + latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops); +} /* * These modifications: insert, remove_init and remove; are serialized by the @@ -181,20 +200,20 @@ static void mod_tree_insert(struct module *mod) mod->mtn_core.mod = mod; mod->mtn_init.mod = mod; - latch_tree_insert(&mod->mtn_core.node, &mod_tree, &mod_tree_ops); + __mod_tree_insert(&mod->mtn_core); if (mod->init_size) - latch_tree_insert(&mod->mtn_init.node, &mod_tree, &mod_tree_ops); + __mod_tree_insert(&mod->mtn_init); } static void mod_tree_remove_init(struct module *mod) { if (mod->init_size) - latch_tree_erase(&mod->mtn_init.node, &mod_tree, &mod_tree_ops); + __mod_tree_remove(&mod->mtn_init); } static void mod_tree_remove(struct module *mod) { - latch_tree_erase(&mod->mtn_core.node, &mod_tree, &mod_tree_ops); + __mod_tree_remove(&mod->mtn_core); mod_tree_remove_init(mod); } @@ -202,7 +221,7 @@ static struct module *mod_find(unsigned long addr) { struct latch_tree_node *ltn; - ltn = latch_tree_find((void *)addr, &mod_tree, &mod_tree_ops); + ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops); if (!ltn) return NULL; @@ -211,6 +230,8 @@ static struct module *mod_find(unsigned long addr) #else /* MODULES_TREE_LOOKUP */ +static unsigned long module_addr_min = -1UL, module_addr_max = 0; + static void mod_tree_insert(struct module *mod) { } static void mod_tree_remove_init(struct module *mod) { } static void mod_tree_remove(struct module *mod) { } @@ -229,6 +250,28 @@ static struct module *mod_find(unsigned long addr) #endif /* MODULES_TREE_LOOKUP */ +/* + * Bounds of module text, for speeding up __module_address. + * Protected by module_mutex. + */ +static void __mod_update_bounds(void *base, unsigned int size) +{ + unsigned long min = (unsigned long)base; + unsigned long max = min + size; + + if (min < module_addr_min) + module_addr_min = min; + if (max > module_addr_max) + module_addr_max = max; +} + +static void mod_update_bounds(struct module *mod) +{ + __mod_update_bounds(mod->module_core, mod->core_size); + if (mod->init_size) + __mod_update_bounds(mod->module_init, mod->init_size); +} + #ifdef CONFIG_KGDB_KDB struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */ #endif /* CONFIG_KGDB_KDB */ @@ -297,10 +340,6 @@ static DECLARE_WAIT_QUEUE_HEAD(module_wq); static BLOCKING_NOTIFIER_HEAD(module_notify_list); -/* Bounds of module allocation, for speeding __module_address. - * Protected by module_mutex. */ -static unsigned long module_addr_min = -1UL, module_addr_max = 0; - int register_module_notifier(struct notifier_block *nb) { return blocking_notifier_chain_register(&module_notify_list, nb); @@ -2539,22 +2578,6 @@ void * __weak module_alloc(unsigned long size) return vmalloc_exec(size); } -static void *module_alloc_update_bounds(unsigned long size) -{ - void *ret = module_alloc(size); - - if (ret) { - mutex_lock(&module_mutex); - /* Update module bounds. */ - if ((unsigned long)ret < module_addr_min) - module_addr_min = (unsigned long)ret; - if ((unsigned long)ret + size > module_addr_max) - module_addr_max = (unsigned long)ret + size; - mutex_unlock(&module_mutex); - } - return ret; -} - #ifdef CONFIG_DEBUG_KMEMLEAK static void kmemleak_load_module(const struct module *mod, const struct load_info *info) @@ -2960,7 +2983,7 @@ static int move_module(struct module *mod, struct load_info *info) void *ptr; /* Do the allocs. */ - ptr = module_alloc_update_bounds(mod->core_size); + ptr = module_alloc(mod->core_size); /* * The pointer to this block is stored in the module structure * which is inside the block. Just mark it as not being a @@ -2974,7 +2997,7 @@ static int move_module(struct module *mod, struct load_info *info) mod->module_core = ptr; if (mod->init_size) { - ptr = module_alloc_update_bounds(mod->init_size); + ptr = module_alloc(mod->init_size); /* * The pointer to this block is stored in the module structure * which is inside the block. This block doesn't need to be @@ -3344,6 +3367,7 @@ again: err = -EEXIST; goto out; } + mod_update_bounds(mod); list_add_rcu(&mod->list, &modules); mod_tree_insert(mod); err = 0; -- cgit v1.2.3-59-g8ed1b From 28b8d0c8f560300836dff352348e513cdf328e50 Mon Sep 17 00:00:00 2001 From: Gobinda Charan Maji Date: Wed, 27 May 2015 11:09:38 +0930 Subject: sysfs: tightened sysfs permission checks There were some inconsistency in restriction to VERIFY_OCTAL_PERMISSIONS(). Previously the test was "User perms >= group perms >= other perms". The permission field of User, Group or Other consists of three bits. LSB is EXECUTE permission, MSB is READ permission and the middle bit is WRITE permission. But logically WRITE is "more privileged" than READ. Say for example, permission value is "0430". Here User has only READ permission whereas Group has both WRITE and EXECUTE permission. So, the checks could be tightened and the tests are separated to USER_READABLE >= GROUP_READABLE >= OTHER_READABLE, USER_WRITABLE >= GROUP_WRITABLE and OTHER_WRITABLE is not permitted. Signed-off-by: Gobinda Charan Maji Signed-off-by: Rusty Russell --- include/linux/kernel.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 3a5b48e52a9e..cd54b357c934 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -818,13 +818,15 @@ static inline void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) { } #endif /* Permissions on a sysfs file: you didn't miss the 0 prefix did you? */ -#define VERIFY_OCTAL_PERMISSIONS(perms) \ - (BUILD_BUG_ON_ZERO((perms) < 0) + \ - BUILD_BUG_ON_ZERO((perms) > 0777) + \ - /* User perms >= group perms >= other perms */ \ - BUILD_BUG_ON_ZERO(((perms) >> 6) < (((perms) >> 3) & 7)) + \ - BUILD_BUG_ON_ZERO((((perms) >> 3) & 7) < ((perms) & 7)) + \ - /* Other writable? Generally considered a bad idea. */ \ - BUILD_BUG_ON_ZERO((perms) & 2) + \ +#define VERIFY_OCTAL_PERMISSIONS(perms) \ + (BUILD_BUG_ON_ZERO((perms) < 0) + \ + BUILD_BUG_ON_ZERO((perms) > 0777) + \ + /* USER_READABLE >= GROUP_READABLE >= OTHER_READABLE */ \ + BUILD_BUG_ON_ZERO((((perms) >> 6) & 4) < (((perms) >> 3) & 4)) + \ + BUILD_BUG_ON_ZERO((((perms) >> 3) & 4) < ((perms) & 4)) + \ + /* USER_WRITABLE >= GROUP_WRITABLE */ \ + BUILD_BUG_ON_ZERO((((perms) >> 6) & 2) < (((perms) >> 3) & 2)) + \ + /* OTHER_WRITABLE? Generally considered a bad idea. */ \ + BUILD_BUG_ON_ZERO((perms) & 2) + \ (perms)) #endif -- cgit v1.2.3-59-g8ed1b From 9c27847dda9cfae7c273cde62becf364f9fa9ea3 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 27 May 2015 11:09:38 +0930 Subject: kernel/params: constify struct kernel_param_ops uses Most code already uses consts for the struct kernel_param_ops, sweep the kernel for the last offending stragglers. Other than include/linux/moduleparam.h and kernel/params.c all other changes were generated with the following Coccinelle SmPL patch. Merge conflicts between trees can be handled with Coccinelle. In the future git could get Coccinelle merge support to deal with patch --> fail --> grammar --> Coccinelle --> new patch conflicts automatically for us on patches where the grammar is available and the patch is of high confidence. Consider this a feature request. Test compiled on x86_64 against: * allnoconfig * allmodconfig * allyesconfig @ const_found @ identifier ops; @@ const struct kernel_param_ops ops = { }; @ const_not_found depends on !const_found @ identifier ops; @@ -struct kernel_param_ops ops = { +const struct kernel_param_ops ops = { }; Generated-by: Coccinelle SmPL Cc: Rusty Russell Cc: Junio C Hamano Cc: Andrew Morton Cc: Kees Cook Cc: Tejun Heo Cc: Ingo Molnar Cc: cocci@systeme.lip6.fr Cc: linux-kernel@vger.kernel.org Signed-off-by: Luis R. Rodriguez Signed-off-by: Rusty Russell --- arch/s390/kernel/perf_cpum_sf.c | 2 +- arch/x86/kvm/mmu_audit.c | 2 +- arch/x86/platform/uv/uv_nmi.c | 2 +- drivers/block/null_blk.c | 4 ++-- drivers/char/ipmi/ipmi_watchdog.c | 6 +++--- drivers/dma/dmatest.c | 4 ++-- drivers/ide/ide.c | 2 +- drivers/infiniband/ulp/srp/ib_srp.c | 4 ++-- drivers/input/misc/ati_remote2.c | 4 ++-- drivers/input/mouse/psmouse-base.c | 2 +- drivers/misc/lis3lv02d/lis3lv02d.c | 2 +- drivers/mtd/ubi/block.c | 2 +- drivers/net/wireless/ath/wil6210/main.c | 4 ++-- drivers/power/test_power.c | 16 ++++++++-------- drivers/thermal/intel_powerclamp.c | 4 ++-- drivers/tty/hvc/hvc_iucv.c | 2 +- drivers/tty/sysrq.c | 2 +- drivers/video/fbdev/uvesafb.c | 2 +- drivers/virtio/virtio_mmio.c | 2 +- fs/nfs/super.c | 2 +- include/linux/moduleparam.h | 30 +++++++++++++++--------------- kernel/params.c | 14 +++++++------- net/sunrpc/auth.c | 2 +- net/sunrpc/xprtsock.c | 6 +++--- security/apparmor/lsm.c | 6 +++--- security/integrity/ima/ima_crypto.c | 2 +- sound/pci/hda/hda_intel.c | 2 +- 27 files changed, 66 insertions(+), 66 deletions(-) diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index e6a1578fc000..afe05bfb7e00 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -1572,7 +1572,7 @@ static int param_set_sfb_size(const char *val, const struct kernel_param *kp) } #define param_check_sfb_size(name, p) __param_check(name, p, void) -static struct kernel_param_ops param_ops_sfb_size = { +static const struct kernel_param_ops param_ops_sfb_size = { .set = param_set_sfb_size, .get = param_get_sfb_size, }; diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c index 9ade5cfb5a4c..87393e3ae087 100644 --- a/arch/x86/kvm/mmu_audit.c +++ b/arch/x86/kvm/mmu_audit.c @@ -291,7 +291,7 @@ static int mmu_audit_set(const char *val, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops audit_param_ops = { +static const struct kernel_param_ops audit_param_ops = { .set = mmu_audit_set, .get = param_get_bool, }; diff --git a/arch/x86/platform/uv/uv_nmi.c b/arch/x86/platform/uv/uv_nmi.c index 7488cafab955..020c101c255f 100644 --- a/arch/x86/platform/uv/uv_nmi.c +++ b/arch/x86/platform/uv/uv_nmi.c @@ -104,7 +104,7 @@ static int param_set_local64(const char *val, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops param_ops_local64 = { +static const struct kernel_param_ops param_ops_local64 = { .get = param_get_local64, .set = param_set_local64, }; diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index 65cd61a4145e..0b4b25617bb7 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -99,7 +99,7 @@ static int null_set_queue_mode(const char *str, const struct kernel_param *kp) return null_param_store_val(str, &queue_mode, NULL_Q_BIO, NULL_Q_MQ); } -static struct kernel_param_ops null_queue_mode_param_ops = { +static const struct kernel_param_ops null_queue_mode_param_ops = { .set = null_set_queue_mode, .get = param_get_int, }; @@ -127,7 +127,7 @@ static int null_set_irqmode(const char *str, const struct kernel_param *kp) NULL_IRQ_TIMER); } -static struct kernel_param_ops null_irqmode_param_ops = { +static const struct kernel_param_ops null_irqmode_param_ops = { .set = null_set_irqmode, .get = param_get_int, }; diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c index 37b8be7cba95..0ac3bd1a5497 100644 --- a/drivers/char/ipmi/ipmi_watchdog.c +++ b/drivers/char/ipmi/ipmi_watchdog.c @@ -208,7 +208,7 @@ static int set_param_timeout(const char *val, const struct kernel_param *kp) return rv; } -static struct kernel_param_ops param_ops_timeout = { +static const struct kernel_param_ops param_ops_timeout = { .set = set_param_timeout, .get = param_get_int, }; @@ -270,14 +270,14 @@ static int set_param_wdog_ifnum(const char *val, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops param_ops_wdog_ifnum = { +static const struct kernel_param_ops param_ops_wdog_ifnum = { .set = set_param_wdog_ifnum, .get = param_get_int, }; #define param_check_wdog_ifnum param_check_int -static struct kernel_param_ops param_ops_str = { +static const struct kernel_param_ops param_ops_str = { .set = set_param_str, .get = get_param_str, }; diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 220ee49633e4..b8576fd6bd0e 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -120,7 +120,7 @@ static struct dmatest_info { static int dmatest_run_set(const char *val, const struct kernel_param *kp); static int dmatest_run_get(char *val, const struct kernel_param *kp); -static struct kernel_param_ops run_ops = { +static const struct kernel_param_ops run_ops = { .set = dmatest_run_set, .get = dmatest_run_get, }; @@ -195,7 +195,7 @@ static int dmatest_wait_get(char *val, const struct kernel_param *kp) return param_get_bool(val, kp); } -static struct kernel_param_ops wait_ops = { +static const struct kernel_param_ops wait_ops = { .get = dmatest_wait_get, .set = param_set_bool, }; diff --git a/drivers/ide/ide.c b/drivers/ide/ide.c index e29b02ca9e91..f086ef387475 100644 --- a/drivers/ide/ide.c +++ b/drivers/ide/ide.c @@ -199,7 +199,7 @@ static int ide_set_dev_param_mask(const char *s, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops param_ops_ide_dev_mask = { +static const struct kernel_param_ops param_ops_ide_dev_mask = { .set = ide_set_dev_param_mask }; diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 918814cd0f80..e4eec9da5ea9 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -98,7 +98,7 @@ module_param(register_always, bool, 0444); MODULE_PARM_DESC(register_always, "Use memory registration even for contiguous memory regions"); -static struct kernel_param_ops srp_tmo_ops; +static const struct kernel_param_ops srp_tmo_ops; static int srp_reconnect_delay = 10; module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, @@ -183,7 +183,7 @@ out: return res; } -static struct kernel_param_ops srp_tmo_ops = { +static const struct kernel_param_ops srp_tmo_ops = { .get = srp_tmo_get, .set = srp_tmo_set, }; diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c index f63341f20b91..cfd58e87da26 100644 --- a/drivers/input/misc/ati_remote2.c +++ b/drivers/input/misc/ati_remote2.c @@ -94,7 +94,7 @@ static int ati_remote2_get_mode_mask(char *buffer, static unsigned int channel_mask = ATI_REMOTE2_MAX_CHANNEL_MASK; #define param_check_channel_mask(name, p) __param_check(name, p, unsigned int) -static struct kernel_param_ops param_ops_channel_mask = { +static const struct kernel_param_ops param_ops_channel_mask = { .set = ati_remote2_set_channel_mask, .get = ati_remote2_get_channel_mask, }; @@ -103,7 +103,7 @@ MODULE_PARM_DESC(channel_mask, "Bitmask of channels to accept <15:Channel16>...< static unsigned int mode_mask = ATI_REMOTE2_MAX_MODE_MASK; #define param_check_mode_mask(name, p) __param_check(name, p, unsigned int) -static struct kernel_param_ops param_ops_mode_mask = { +static const struct kernel_param_ops param_ops_mode_mask = { .set = ati_remote2_set_mode_mask, .get = ati_remote2_get_mode_mask, }; diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c index 5bb1658f60c7..f8286b625ec4 100644 --- a/drivers/input/mouse/psmouse-base.c +++ b/drivers/input/mouse/psmouse-base.c @@ -47,7 +47,7 @@ MODULE_LICENSE("GPL"); static unsigned int psmouse_max_proto = PSMOUSE_AUTO; static int psmouse_set_maxproto(const char *val, const struct kernel_param *); static int psmouse_get_maxproto(char *buffer, const struct kernel_param *kp); -static struct kernel_param_ops param_ops_proto_abbrev = { +static const struct kernel_param_ops param_ops_proto_abbrev = { .set = psmouse_set_maxproto, .get = psmouse_get_maxproto, }; diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c index 4739689d23ad..fb8705fc3aca 100644 --- a/drivers/misc/lis3lv02d/lis3lv02d.c +++ b/drivers/misc/lis3lv02d/lis3lv02d.c @@ -115,7 +115,7 @@ static int param_set_axis(const char *val, const struct kernel_param *kp) return ret; } -static struct kernel_param_ops param_ops_axis = { +static const struct kernel_param_ops param_ops_axis = { .set = param_set_axis, .get = param_get_int, }; diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c index c9eb78f10a0d..4968c071f399 100644 --- a/drivers/mtd/ubi/block.c +++ b/drivers/mtd/ubi/block.c @@ -161,7 +161,7 @@ static int __init ubiblock_set_param(const char *val, return 0; } -static struct kernel_param_ops ubiblock_param_ops = { +static const struct kernel_param_ops ubiblock_param_ops = { .set = ubiblock_set_param, }; module_param_cb(block, &ubiblock_param_ops, NULL, 0); diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c index c2a238426425..790c3df0bb03 100644 --- a/drivers/net/wireless/ath/wil6210/main.c +++ b/drivers/net/wireless/ath/wil6210/main.c @@ -58,7 +58,7 @@ static int mtu_max_set(const char *val, const struct kernel_param *kp) return ret; } -static struct kernel_param_ops mtu_max_ops = { +static const struct kernel_param_ops mtu_max_ops = { .set = mtu_max_set, .get = param_get_uint, }; @@ -87,7 +87,7 @@ static int ring_order_set(const char *val, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops ring_order_ops = { +static const struct kernel_param_ops ring_order_ops = { .set = ring_order_set, .get = param_get_uint, }; diff --git a/drivers/power/test_power.c b/drivers/power/test_power.c index f986e0cca7ac..83c42ea88f2b 100644 --- a/drivers/power/test_power.c +++ b/drivers/power/test_power.c @@ -448,42 +448,42 @@ static int param_set_battery_voltage(const char *key, #define param_get_battery_voltage param_get_int -static struct kernel_param_ops param_ops_ac_online = { +static const struct kernel_param_ops param_ops_ac_online = { .set = param_set_ac_online, .get = param_get_ac_online, }; -static struct kernel_param_ops param_ops_usb_online = { +static const struct kernel_param_ops param_ops_usb_online = { .set = param_set_usb_online, .get = param_get_usb_online, }; -static struct kernel_param_ops param_ops_battery_status = { +static const struct kernel_param_ops param_ops_battery_status = { .set = param_set_battery_status, .get = param_get_battery_status, }; -static struct kernel_param_ops param_ops_battery_present = { +static const struct kernel_param_ops param_ops_battery_present = { .set = param_set_battery_present, .get = param_get_battery_present, }; -static struct kernel_param_ops param_ops_battery_technology = { +static const struct kernel_param_ops param_ops_battery_technology = { .set = param_set_battery_technology, .get = param_get_battery_technology, }; -static struct kernel_param_ops param_ops_battery_health = { +static const struct kernel_param_ops param_ops_battery_health = { .set = param_set_battery_health, .get = param_get_battery_health, }; -static struct kernel_param_ops param_ops_battery_capacity = { +static const struct kernel_param_ops param_ops_battery_capacity = { .set = param_set_battery_capacity, .get = param_get_battery_capacity, }; -static struct kernel_param_ops param_ops_battery_voltage = { +static const struct kernel_param_ops param_ops_battery_voltage = { .set = param_set_battery_voltage, .get = param_get_battery_voltage, }; diff --git a/drivers/thermal/intel_powerclamp.c b/drivers/thermal/intel_powerclamp.c index 725718e97a0b..500f8f551817 100644 --- a/drivers/thermal/intel_powerclamp.c +++ b/drivers/thermal/intel_powerclamp.c @@ -119,7 +119,7 @@ exit: return ret; } -static struct kernel_param_ops duration_ops = { +static const struct kernel_param_ops duration_ops = { .set = duration_set, .get = param_get_int, }; @@ -167,7 +167,7 @@ exit_win: return ret; } -static struct kernel_param_ops window_size_ops = { +static const struct kernel_param_ops window_size_ops = { .set = window_size_set, .get = param_get_int, }; diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c index f78a87b07872..bb809cf36617 100644 --- a/drivers/tty/hvc/hvc_iucv.c +++ b/drivers/tty/hvc/hvc_iucv.c @@ -1345,7 +1345,7 @@ static int param_get_vmidfilter(char *buffer, const struct kernel_param *kp) #define param_check_vmidfilter(name, p) __param_check(name, p, void) -static struct kernel_param_ops param_ops_vmidfilter = { +static const struct kernel_param_ops param_ops_vmidfilter = { .set = param_set_vmidfilter, .get = param_get_vmidfilter, }; diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c index 843f2cdc280b..03b02c37d247 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c @@ -1002,7 +1002,7 @@ static int sysrq_reset_seq_param_set(const char *buffer, return 0; } -static struct kernel_param_ops param_ops_sysrq_reset_seq = { +static const struct kernel_param_ops param_ops_sysrq_reset_seq = { .get = param_get_ushort, .set = sysrq_reset_seq_param_set, }; diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c index d32d1c4d1b99..178ae93b7ebd 100644 --- a/drivers/video/fbdev/uvesafb.c +++ b/drivers/video/fbdev/uvesafb.c @@ -1977,7 +1977,7 @@ static int param_set_scroll(const char *val, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops param_ops_scroll = { +static const struct kernel_param_ops param_ops_scroll = { .set = param_set_scroll, }; #define param_check_scroll(name, p) __param_check(name, p, void) diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 7a5e60dea6c5..10189b5b627f 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -691,7 +691,7 @@ static int vm_cmdline_get(char *buffer, const struct kernel_param *kp) return strlen(buffer) + 1; } -static struct kernel_param_ops vm_cmdline_param_ops = { +static const struct kernel_param_ops vm_cmdline_param_ops = { .set = vm_cmdline_set, .get = vm_cmdline_get, }; diff --git a/fs/nfs/super.c b/fs/nfs/super.c index f175b833b6ba..aa62004f1706 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -2847,7 +2847,7 @@ static int param_set_portnr(const char *val, const struct kernel_param *kp) *((unsigned int *)kp->arg) = num; return 0; } -static struct kernel_param_ops param_ops_portnr = { +static const struct kernel_param_ops param_ops_portnr = { .set = param_set_portnr, .get = param_get_uint, }; diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 1c9effa25e26..5d0f4d97997f 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -225,7 +225,7 @@ struct kparam_array /* Obsolete - use module_param_cb() */ #define module_param_call(name, set, get, arg, perm) \ - static struct kernel_param_ops __param_ops_##name = \ + static const struct kernel_param_ops __param_ops_##name = \ { .flags = 0, (void *)set, (void *)get }; \ __module_param_call(MODULE_PARAM_PREFIX, \ name, &__param_ops_##name, arg, \ @@ -376,64 +376,64 @@ static inline void destroy_params(const struct kernel_param *params, #define __param_check(name, p, type) \ static inline type __always_unused *__check_##name(void) { return(p); } -extern struct kernel_param_ops param_ops_byte; +extern const struct kernel_param_ops param_ops_byte; extern int param_set_byte(const char *val, const struct kernel_param *kp); extern int param_get_byte(char *buffer, const struct kernel_param *kp); #define param_check_byte(name, p) __param_check(name, p, unsigned char) -extern struct kernel_param_ops param_ops_short; +extern const struct kernel_param_ops param_ops_short; extern int param_set_short(const char *val, const struct kernel_param *kp); extern int param_get_short(char *buffer, const struct kernel_param *kp); #define param_check_short(name, p) __param_check(name, p, short) -extern struct kernel_param_ops param_ops_ushort; +extern const struct kernel_param_ops param_ops_ushort; extern int param_set_ushort(const char *val, const struct kernel_param *kp); extern int param_get_ushort(char *buffer, const struct kernel_param *kp); #define param_check_ushort(name, p) __param_check(name, p, unsigned short) -extern struct kernel_param_ops param_ops_int; +extern const struct kernel_param_ops param_ops_int; extern int param_set_int(const char *val, const struct kernel_param *kp); extern int param_get_int(char *buffer, const struct kernel_param *kp); #define param_check_int(name, p) __param_check(name, p, int) -extern struct kernel_param_ops param_ops_uint; +extern const struct kernel_param_ops param_ops_uint; extern int param_set_uint(const char *val, const struct kernel_param *kp); extern int param_get_uint(char *buffer, const struct kernel_param *kp); #define param_check_uint(name, p) __param_check(name, p, unsigned int) -extern struct kernel_param_ops param_ops_long; +extern const struct kernel_param_ops param_ops_long; extern int param_set_long(const char *val, const struct kernel_param *kp); extern int param_get_long(char *buffer, const struct kernel_param *kp); #define param_check_long(name, p) __param_check(name, p, long) -extern struct kernel_param_ops param_ops_ulong; +extern const struct kernel_param_ops param_ops_ulong; extern int param_set_ulong(const char *val, const struct kernel_param *kp); extern int param_get_ulong(char *buffer, const struct kernel_param *kp); #define param_check_ulong(name, p) __param_check(name, p, unsigned long) -extern struct kernel_param_ops param_ops_ullong; +extern const struct kernel_param_ops param_ops_ullong; extern int param_set_ullong(const char *val, const struct kernel_param *kp); extern int param_get_ullong(char *buffer, const struct kernel_param *kp); #define param_check_ullong(name, p) __param_check(name, p, unsigned long long) -extern struct kernel_param_ops param_ops_charp; +extern const struct kernel_param_ops param_ops_charp; extern int param_set_charp(const char *val, const struct kernel_param *kp); extern int param_get_charp(char *buffer, const struct kernel_param *kp); #define param_check_charp(name, p) __param_check(name, p, char *) /* We used to allow int as well as bool. We're taking that away! */ -extern struct kernel_param_ops param_ops_bool; +extern const struct kernel_param_ops param_ops_bool; extern int param_set_bool(const char *val, const struct kernel_param *kp); extern int param_get_bool(char *buffer, const struct kernel_param *kp); #define param_check_bool(name, p) __param_check(name, p, bool) -extern struct kernel_param_ops param_ops_invbool; +extern const struct kernel_param_ops param_ops_invbool; extern int param_set_invbool(const char *val, const struct kernel_param *kp); extern int param_get_invbool(char *buffer, const struct kernel_param *kp); #define param_check_invbool(name, p) __param_check(name, p, bool) /* An int, which can only be set like a bool (though it shows as an int). */ -extern struct kernel_param_ops param_ops_bint; +extern const struct kernel_param_ops param_ops_bint; extern int param_set_bint(const char *val, const struct kernel_param *kp); #define param_get_bint param_get_int #define param_check_bint param_check_int @@ -477,9 +477,9 @@ extern int param_set_bint(const char *val, const struct kernel_param *kp); perm, -1, 0); \ __MODULE_PARM_TYPE(name, "array of " #type) -extern struct kernel_param_ops param_array_ops; +extern const struct kernel_param_ops param_array_ops; -extern struct kernel_param_ops param_ops_string; +extern const struct kernel_param_ops param_ops_string; extern int param_set_copystring(const char *val, const struct kernel_param *); extern int param_get_string(char *buffer, const struct kernel_param *kp); diff --git a/kernel/params.c b/kernel/params.c index a22d6a759b1a..b7635c025e9b 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -251,7 +251,7 @@ char *parse_args(const char *doing, return scnprintf(buffer, PAGE_SIZE, format, \ *((type *)kp->arg)); \ } \ - struct kernel_param_ops param_ops_##name = { \ + const struct kernel_param_ops param_ops_##name = { \ .set = param_set_##name, \ .get = param_get_##name, \ }; \ @@ -303,7 +303,7 @@ static void param_free_charp(void *arg) maybe_kfree_parameter(*((char **)arg)); } -struct kernel_param_ops param_ops_charp = { +const struct kernel_param_ops param_ops_charp = { .set = param_set_charp, .get = param_get_charp, .free = param_free_charp, @@ -328,7 +328,7 @@ int param_get_bool(char *buffer, const struct kernel_param *kp) } EXPORT_SYMBOL(param_get_bool); -struct kernel_param_ops param_ops_bool = { +const struct kernel_param_ops param_ops_bool = { .flags = KERNEL_PARAM_OPS_FL_NOARG, .set = param_set_bool, .get = param_get_bool, @@ -356,7 +356,7 @@ int param_get_invbool(char *buffer, const struct kernel_param *kp) } EXPORT_SYMBOL(param_get_invbool); -struct kernel_param_ops param_ops_invbool = { +const struct kernel_param_ops param_ops_invbool = { .set = param_set_invbool, .get = param_get_invbool, }; @@ -379,7 +379,7 @@ int param_set_bint(const char *val, const struct kernel_param *kp) } EXPORT_SYMBOL(param_set_bint); -struct kernel_param_ops param_ops_bint = { +const struct kernel_param_ops param_ops_bint = { .flags = KERNEL_PARAM_OPS_FL_NOARG, .set = param_set_bint, .get = param_get_int, @@ -476,7 +476,7 @@ static void param_array_free(void *arg) arr->ops->free(arr->elem + arr->elemsize * i); } -struct kernel_param_ops param_array_ops = { +const struct kernel_param_ops param_array_ops = { .set = param_array_set, .get = param_array_get, .free = param_array_free, @@ -504,7 +504,7 @@ int param_get_string(char *buffer, const struct kernel_param *kp) } EXPORT_SYMBOL(param_get_string); -struct kernel_param_ops param_ops_string = { +const struct kernel_param_ops param_ops_string = { .set = param_set_copystring, .get = param_get_string, }; diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 47f38be4155f..02f53674dc39 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -72,7 +72,7 @@ static int param_get_hashtbl_sz(char *buffer, const struct kernel_param *kp) #define param_check_hashtbl_sz(name, p) __param_check(name, p, unsigned int); -static struct kernel_param_ops param_ops_hashtbl_sz = { +static const struct kernel_param_ops param_ops_hashtbl_sz = { .set = param_set_hashtbl_sz, .get = param_get_hashtbl_sz, }; diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 66891e32c5e3..b0517287075b 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -2982,7 +2982,7 @@ static int param_set_portnr(const char *val, const struct kernel_param *kp) RPC_MAX_RESVPORT); } -static struct kernel_param_ops param_ops_portnr = { +static const struct kernel_param_ops param_ops_portnr = { .set = param_set_portnr, .get = param_get_uint, }; @@ -3001,7 +3001,7 @@ static int param_set_slot_table_size(const char *val, RPC_MAX_SLOT_TABLE); } -static struct kernel_param_ops param_ops_slot_table_size = { +static const struct kernel_param_ops param_ops_slot_table_size = { .set = param_set_slot_table_size, .get = param_get_uint, }; @@ -3017,7 +3017,7 @@ static int param_set_max_slot_table_size(const char *val, RPC_MAX_SLOT_TABLE_LIMIT); } -static struct kernel_param_ops param_ops_max_slot_table_size = { +static const struct kernel_param_ops param_ops_max_slot_table_size = { .set = param_set_max_slot_table_size, .get = param_get_uint, }; diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index e5f1561439db..45eb96d1e1d9 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -667,7 +667,7 @@ static struct security_operations apparmor_ops = { static int param_set_aabool(const char *val, const struct kernel_param *kp); static int param_get_aabool(char *buffer, const struct kernel_param *kp); #define param_check_aabool param_check_bool -static struct kernel_param_ops param_ops_aabool = { +static const struct kernel_param_ops param_ops_aabool = { .flags = KERNEL_PARAM_OPS_FL_NOARG, .set = param_set_aabool, .get = param_get_aabool @@ -676,7 +676,7 @@ static struct kernel_param_ops param_ops_aabool = { static int param_set_aauint(const char *val, const struct kernel_param *kp); static int param_get_aauint(char *buffer, const struct kernel_param *kp); #define param_check_aauint param_check_uint -static struct kernel_param_ops param_ops_aauint = { +static const struct kernel_param_ops param_ops_aauint = { .set = param_set_aauint, .get = param_get_aauint }; @@ -684,7 +684,7 @@ static struct kernel_param_ops param_ops_aauint = { static int param_set_aalockpolicy(const char *val, const struct kernel_param *kp); static int param_get_aalockpolicy(char *buffer, const struct kernel_param *kp); #define param_check_aalockpolicy param_check_bool -static struct kernel_param_ops param_ops_aalockpolicy = { +static const struct kernel_param_ops param_ops_aalockpolicy = { .flags = KERNEL_PARAM_OPS_FL_NOARG, .set = param_set_aalockpolicy, .get = param_get_aalockpolicy diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c index 686355fea7fd..e24121afb2f2 100644 --- a/security/integrity/ima/ima_crypto.c +++ b/security/integrity/ima/ima_crypto.c @@ -55,7 +55,7 @@ static int param_set_bufsize(const char *val, const struct kernel_param *kp) return 0; } -static struct kernel_param_ops param_ops_bufsize = { +static const struct kernel_param_ops param_ops_bufsize = { .set = param_set_bufsize, .get = param_get_uint, }; diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index 34040d26c94f..23a1d2ffa7f3 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -166,7 +166,7 @@ MODULE_PARM_DESC(beep_mode, "Select HDA Beep registration mode " #ifdef CONFIG_PM static int param_set_xint(const char *val, const struct kernel_param *kp); -static struct kernel_param_ops param_ops_xint = { +static const struct kernel_param_ops param_ops_xint = { .set = param_set_xint, .get = param_get_int, }; -- cgit v1.2.3-59-g8ed1b From 05f408dddb013168759cdb4cbd0ba4e189a9504d Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 27 May 2015 11:09:38 +0930 Subject: kernel/module.c: use generic module param operaters for sig_enforce We're directly checking and modifying sig_enforce when needed instead of using the generic helpers. This prevents us from generalizing this helper so that others can use it. Use indirect helpers to allow us to generalize this code a bit and to make it a bit more clear what this is doing. Cc: Rusty Russell Cc: Jani Nikula Cc: Andrew Morton Cc: Kees Cook Cc: Tejun Heo Cc: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: cocci@systeme.lip6.fr Signed-off-by: Luis R. Rodriguez Signed-off-by: Rusty Russell --- kernel/module.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/kernel/module.c b/kernel/module.c index c8da2a59ebf7..9e8c9305bba9 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -301,23 +301,25 @@ static bool sig_enforce = false; static int param_set_bool_enable_only(const char *val, const struct kernel_param *kp) { - int err; - bool test; + int err = 0; + bool new_value; + bool orig_value = *(bool *)kp->arg; struct kernel_param dummy_kp = *kp; - dummy_kp.arg = &test; + dummy_kp.arg = &new_value; err = param_set_bool(val, &dummy_kp); if (err) return err; /* Don't let them unset it once it's set! */ - if (!test && sig_enforce) + if (!new_value && orig_value) return -EROFS; - if (test) - sig_enforce = true; - return 0; + if (new_value) + err = param_set_bool(val, kp); + + return err; } static const struct kernel_param_ops param_ops_bool_enable_only = { -- cgit v1.2.3-59-g8ed1b From d19f05d8a8fa221e5d5f4eaca0f3ca5874c990d3 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 27 May 2015 11:09:38 +0930 Subject: kernel/params.c: generalize bool_enable_only This takes out the bool_enable_only implementation from the module loading code and generalizes it so that others can make use of it. Cc: Rusty Russell Cc: Jani Nikula Cc: Andrew Morton Cc: Kees Cook Cc: Tejun Heo Cc: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: cocci@systeme.lip6.fr Signed-off-by: Luis R. Rodriguez Signed-off-by: Rusty Russell --- include/linux/moduleparam.h | 6 ++++++ kernel/module.c | 31 ------------------------------- kernel/params.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 5d0f4d97997f..7e0079936396 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -427,6 +427,12 @@ extern int param_set_bool(const char *val, const struct kernel_param *kp); extern int param_get_bool(char *buffer, const struct kernel_param *kp); #define param_check_bool(name, p) __param_check(name, p, bool) +extern const struct kernel_param_ops param_ops_bool_enable_only; +extern int param_set_bool_enable_only(const char *val, + const struct kernel_param *kp); +/* getter is the same as for the regular bool */ +#define param_check_bool_enable_only param_check_bool + extern const struct kernel_param_ops param_ops_invbool; extern int param_set_invbool(const char *val, const struct kernel_param *kp); extern int param_get_invbool(char *buffer, const struct kernel_param *kp); diff --git a/kernel/module.c b/kernel/module.c index 9e8c9305bba9..9b0e36145474 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -298,37 +298,6 @@ static bool sig_enforce = true; #else static bool sig_enforce = false; -static int param_set_bool_enable_only(const char *val, - const struct kernel_param *kp) -{ - int err = 0; - bool new_value; - bool orig_value = *(bool *)kp->arg; - struct kernel_param dummy_kp = *kp; - - dummy_kp.arg = &new_value; - - err = param_set_bool(val, &dummy_kp); - if (err) - return err; - - /* Don't let them unset it once it's set! */ - if (!new_value && orig_value) - return -EROFS; - - if (new_value) - err = param_set_bool(val, kp); - - return err; -} - -static const struct kernel_param_ops param_ops_bool_enable_only = { - .flags = KERNEL_PARAM_OPS_FL_NOARG, - .set = param_set_bool_enable_only, - .get = param_get_bool, -}; -#define param_check_bool_enable_only param_check_bool - module_param(sig_enforce, bool_enable_only, 0644); #endif /* !CONFIG_MODULE_SIG_FORCE */ #endif /* CONFIG_MODULE_SIG */ diff --git a/kernel/params.c b/kernel/params.c index b7635c025e9b..324624ed620f 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -335,6 +335,36 @@ const struct kernel_param_ops param_ops_bool = { }; EXPORT_SYMBOL(param_ops_bool); +int param_set_bool_enable_only(const char *val, const struct kernel_param *kp) +{ + int err = 0; + bool new_value; + bool orig_value = *(bool *)kp->arg; + struct kernel_param dummy_kp = *kp; + + dummy_kp.arg = &new_value; + + err = param_set_bool(val, &dummy_kp); + if (err) + return err; + + /* Don't let them unset it once it's set! */ + if (!new_value && orig_value) + return -EROFS; + + if (new_value) + err = param_set_bool(val, kp); + + return err; +} +EXPORT_SYMBOL_GPL(param_set_bool_enable_only); + +const struct kernel_param_ops param_ops_bool_enable_only = { + .flags = KERNEL_PARAM_OPS_FL_NOARG, + .set = param_set_bool_enable_only, + .get = param_get_bool, +}; + /* This one must be bool. */ int param_set_invbool(const char *val, const struct kernel_param *kp) { -- cgit v1.2.3-59-g8ed1b From 154be21c582857c468575e7cab488fe39dc1445b Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 27 May 2015 11:09:39 +0930 Subject: kernel/params.c: export param_ops_bool_enable_only This will grant access to this helper to code built as modules. Cc: Rusty Russell Cc: David Howells Cc: Ming Lei Cc: Seth Forshee Cc: Kyle McMartin Signed-off-by: Luis R. Rodriguez Signed-off-by: Rusty Russell --- kernel/params.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/params.c b/kernel/params.c index 324624ed620f..7edf31f2ce96 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -364,6 +364,7 @@ const struct kernel_param_ops param_ops_bool_enable_only = { .set = param_set_bool_enable_only, .get = param_get_bool, }; +EXPORT_SYMBOL_GPL(param_ops_bool_enable_only); /* This one must be bool. */ int param_set_invbool(const char *val, const struct kernel_param *kp) -- cgit v1.2.3-59-g8ed1b From 552f530cbc34072d824af021e3289fdd195c880d Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 27 May 2015 11:09:39 +0930 Subject: kernel/workqueue.c: remove ifdefs over wq_power_efficient We can avoid an ifdef over wq_power_efficient's declaration by just using IS_ENABLED(). Cc: Rusty Russell Cc: Jani Nikula Cc: Andrew Morton Cc: Kees Cook Cc: Tejun Heo Cc: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: cocci@systeme.lip6.fr Signed-off-by: Luis R. Rodriguez Signed-off-by: Rusty Russell --- kernel/workqueue.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 586ad91300b0..59bc24918655 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -280,12 +280,7 @@ static bool wq_disable_numa; module_param_named(disable_numa, wq_disable_numa, bool, 0444); /* see the comment above the definition of WQ_POWER_EFFICIENT */ -#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT -static bool wq_power_efficient = true; -#else -static bool wq_power_efficient; -#endif - +static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); module_param_named(power_efficient, wq_power_efficient, bool, 0444); static bool wq_numa_enabled; /* unbound NUMA affinity enabled */ -- cgit v1.2.3-59-g8ed1b From 6727bb9c6abe836d88191ce13bfdd7a53c245e15 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 27 May 2015 11:09:39 +0930 Subject: kernel/module.c: avoid ifdefs for sig_enforce declaration There's no need to require an ifdef over the declaration of sig_enforce as IS_ENABLED() can be used. While at it, there's no harm in exposing this kernel parameter outside of CONFIG_MODULE_SIG as it'd be a no-op on non module sig kernels. Now, technically we should in theory be able to remove the #ifdef'ery over the declaration of the module parameter as we are also trusting the bool_enable_only code for CONFIG_MODULE_SIG kernels but for now remain paranoid and keep it. With time if no one can put a bullet through bool_enable_only and if there are no technical requirements over not exposing CONFIG_MODULE_SIG_FORCE with the measures in place by bool_enable_only we could remove this last ifdef. Cc: Rusty Russell Cc: Andrew Morton Cc: Kees Cook Cc: Tejun Heo Cc: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: cocci@systeme.lip6.fr Signed-off-by: Luis R. Rodriguez Signed-off-by: Rusty Russell --- kernel/module.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/kernel/module.c b/kernel/module.c index 9b0e36145474..427b99f1a4b3 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -292,15 +292,10 @@ static void module_assert_mutex_or_preempt(void) #endif } -#ifdef CONFIG_MODULE_SIG -#ifdef CONFIG_MODULE_SIG_FORCE -static bool sig_enforce = true; -#else -static bool sig_enforce = false; - +static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE); +#ifndef CONFIG_MODULE_SIG_FORCE module_param(sig_enforce, bool_enable_only, 0644); #endif /* !CONFIG_MODULE_SIG_FORCE */ -#endif /* CONFIG_MODULE_SIG */ /* Block module loading/unloading? */ int modules_disabled = 0; -- cgit v1.2.3-59-g8ed1b From b6c09b512dc0c5f62a4cc40184f5de749e90fdac Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 16 Jun 2015 12:16:22 +0930 Subject: modules: clarify CONFIG_MODULE_COMPRESS help, suggest 'N'. Andreas turned this option on, only to find out Debian (and Ubuntu!) don't enable support in their kmod builds. Shorten the text, and suggest N at the bottom (at least for now). Reported-by: Andreas Mohr Signed-off-by: Rusty Russell --- init/Kconfig | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/init/Kconfig b/init/Kconfig index 968a001790af..bcb7e1e13f7d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1948,26 +1948,21 @@ config MODULE_COMPRESS bool "Compress modules on installation" depends on MODULES help - This option compresses the kernel modules when 'make - modules_install' is run. - The modules will be compressed either using gzip or xz depend on the - choice made in "Compression algorithm". + Compresses kernel modules when 'make modules_install' is run; gzip or + xz depending on "Compression algorithm" below. - module-init-tools has support for gzip format while kmod handle gzip - and xz compressed modules. + module-init-tools MAY support gzip, and kmod MAY support gzip and xz. - When a kernel module is installed from outside of the main kernel - source and uses the Kbuild system for installing modules then that - kernel module will also be compressed when it is installed. + Out-of-tree kernel modules installed using Kbuild will also be + compressed upon installation. - This option provides little benefit when the modules are to be used inside - an initrd or initramfs, it generally is more efficient to compress the whole - initrd or initramfs instead. + Note: for modules inside an initrd or initramfs, it's more efficient + to compress the whole initrd or initramfs instead. - This is fully compatible with signed modules while the signed module is - compressed. module-init-tools or kmod handles decompression and provide to - other layer the uncompressed but signed payload. + Note: This is fully compatible with signed modules. + + If in doubt, say N. choice prompt "Compression algorithm" -- cgit v1.2.3-59-g8ed1b From 74c3dea355245c17ee407a3ce3ea34f55b40f2eb Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 17 Jun 2015 06:16:52 +0930 Subject: params: suppress unused variable error, warn once just in case code changes. It shouldn't fail due to OOM (it's boot time), and already warns if we get two identical names. But you never know what the future holds, and WARN_ON_ONCE() keeps gcc happy with minimal code. Reported-by: Louis Langholtz Acked-by: Tejun Heo Signed-off-by: Rusty Russell --- kernel/params.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/params.c b/kernel/params.c index 7edf31f2ce96..e906874da5fc 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -884,6 +884,7 @@ static void __init version_sysfs_builtin(void) mk = locate_module_kobject(vattr->module_name); if (mk) { err = sysfs_create_file(&mk->kobj, &vattr->mattr.attr); + WARN_ON_ONCE(err); kobject_uevent(&mk->kobj, KOBJ_ADD); kobject_put(&mk->kobj); } -- cgit v1.2.3-59-g8ed1b From 5104b7d7678b0029417f6ac08243773a77259ac6 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 17 Jun 2015 06:17:52 +0930 Subject: module: make perm const Change the struct kernel_param.perm field to a const, as it should never be changed. Signed-off-by: Dan Streetman Signed-off-by: Rusty Russell (cut from larger patch) --- include/linux/moduleparam.h | 2 +- kernel/params.c | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 7e0079936396..ab5031453807 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -68,7 +68,7 @@ enum { struct kernel_param { const char *name; const struct kernel_param_ops *ops; - u16 perm; + const u16 perm; s8 level; u8 flags; union { diff --git a/kernel/params.c b/kernel/params.c index e906874da5fc..a8b09f6c87dc 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -395,12 +395,11 @@ EXPORT_SYMBOL(param_ops_invbool); int param_set_bint(const char *val, const struct kernel_param *kp) { - struct kernel_param boolkp; + /* Match bool exactly, by re-using it. */ + struct kernel_param boolkp = *kp; bool v; int ret; - /* Match bool exactly, by re-using it. */ - boolkp = *kp; boolkp.arg = &v; ret = param_set_bool(val, &boolkp); @@ -480,9 +479,8 @@ static int param_array_get(char *buffer, const struct kernel_param *kp) { int i, off, ret; const struct kparam_array *arr = kp->arr; - struct kernel_param p; + struct kernel_param p = *kp; - p = *kp; for (i = off = 0; i < (arr->num ? *arr->num : arr->max); i++) { if (i) buffer[off++] = ','; -- cgit v1.2.3-59-g8ed1b From b51d23e4e9fea6f264d39535c2a62d1f51e7ccc3 Mon Sep 17 00:00:00 2001 From: Dan Streetman Date: Wed, 17 Jun 2015 06:18:52 +0930 Subject: module: add per-module param_lock Add a "param_lock" mutex to each module, and update params.c to use the correct built-in or module mutex while locking kernel params. Remove the kparam_block_sysfs_r/w() macros, replace them with direct calls to kernel_param_[un]lock(module). The kernel param code currently uses a single mutex to protect modification of any and all kernel params. While this generally works, there is one specific problem with it; a module callback function cannot safely load another module, i.e. with request_module() or even with indirect calls such as crypto_has_alg(). If the module to be loaded has any of its params configured (e.g. with a /etc/modprobe.d/* config file), then the attempt will result in a deadlock between the first module param callback waiting for modprobe, and modprobe trying to lock the single kernel param mutex to set the new module's param. This fixes that by using per-module mutexes, so that each individual module is protected against concurrent changes in its own kernel params, but is not blocked by changes to other module params. All built-in modules continue to use the built-in mutex, since they will always be loaded at runtime and references (e.g. request_module(), crypto_has_alg()) to them will never cause load-time param changing. This also simplifies the interface used by modules to block sysfs access to their params; while there are currently functions to block and unblock sysfs param access which are split up by read and write and expect a single kernel param to be passed, their actual operation is identical and applies to all params, not just the one passed to them; they simply lock and unlock the global param mutex. They are replaced with direct calls to kernel_param_[un]lock(THIS_MODULE), which locks THIS_MODULE's param_lock, or if the module is built-in, it locks the built-in mutex. Suggested-by: Rusty Russell Signed-off-by: Dan Streetman Signed-off-by: Rusty Russell --- arch/um/drivers/hostaudio_kern.c | 20 ++++---- drivers/net/ethernet/myricom/myri10ge/myri10ge.c | 6 +-- drivers/net/wireless/libertas_tf/if_usb.c | 6 +-- drivers/usb/atm/ueagle-atm.c | 4 +- drivers/video/fbdev/vt8623fb.c | 4 +- include/linux/module.h | 1 + include/linux/moduleparam.h | 61 ++++-------------------- kernel/module.c | 2 + kernel/params.c | 50 +++++++++++-------- net/mac80211/rate.c | 4 +- 10 files changed, 65 insertions(+), 93 deletions(-) diff --git a/arch/um/drivers/hostaudio_kern.c b/arch/um/drivers/hostaudio_kern.c index 9b90fdc4b151..f6b911cc3923 100644 --- a/arch/um/drivers/hostaudio_kern.c +++ b/arch/um/drivers/hostaudio_kern.c @@ -185,9 +185,9 @@ static int hostaudio_open(struct inode *inode, struct file *file) int ret; #ifdef DEBUG - kparam_block_sysfs_write(dsp); + kernel_param_lock(THIS_MODULE); printk(KERN_DEBUG "hostaudio: open called (host: %s)\n", dsp); - kparam_unblock_sysfs_write(dsp); + kernel_param_unlock(THIS_MODULE); #endif state = kmalloc(sizeof(struct hostaudio_state), GFP_KERNEL); @@ -199,11 +199,11 @@ static int hostaudio_open(struct inode *inode, struct file *file) if (file->f_mode & FMODE_WRITE) w = 1; - kparam_block_sysfs_write(dsp); + kernel_param_lock(THIS_MODULE); mutex_lock(&hostaudio_mutex); ret = os_open_file(dsp, of_set_rw(OPENFLAGS(), r, w), 0); mutex_unlock(&hostaudio_mutex); - kparam_unblock_sysfs_write(dsp); + kernel_param_unlock(THIS_MODULE); if (ret < 0) { kfree(state); @@ -260,17 +260,17 @@ static int hostmixer_open_mixdev(struct inode *inode, struct file *file) if (file->f_mode & FMODE_WRITE) w = 1; - kparam_block_sysfs_write(mixer); + kernel_param_lock(THIS_MODULE); mutex_lock(&hostaudio_mutex); ret = os_open_file(mixer, of_set_rw(OPENFLAGS(), r, w), 0); mutex_unlock(&hostaudio_mutex); - kparam_unblock_sysfs_write(mixer); + kernel_param_unlock(THIS_MODULE); if (ret < 0) { - kparam_block_sysfs_write(dsp); + kernel_param_lock(THIS_MODULE); printk(KERN_ERR "hostaudio_open_mixdev failed to open '%s', " "err = %d\n", dsp, -ret); - kparam_unblock_sysfs_write(dsp); + kernel_param_unlock(THIS_MODULE); kfree(state); return ret; } @@ -326,10 +326,10 @@ MODULE_LICENSE("GPL"); static int __init hostaudio_init_module(void) { - __kernel_param_lock(); + kernel_param_lock(THIS_MODULE); printk(KERN_INFO "UML Audio Relay (host dsp = %s, host mixer = %s)\n", dsp, mixer); - __kernel_param_unlock(); + kernel_param_unlock(THIS_MODULE); module_data.dev_audio = register_sound_dsp(&hostaudio_fops, -1); if (module_data.dev_audio < 0) { diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 2bae50292dcd..83651ac8ddb9 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -279,7 +279,7 @@ MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); -/* Careful: must be accessed under kparam_block_sysfs_write */ +/* Careful: must be accessed under kernel_param_lock() */ static char *myri10ge_fw_name = NULL; module_param(myri10ge_fw_name, charp, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); @@ -3427,7 +3427,7 @@ static void myri10ge_select_firmware(struct myri10ge_priv *mgp) } } - kparam_block_sysfs_write(myri10ge_fw_name); + kernel_param_lock(THIS_MODULE); if (myri10ge_fw_name != NULL) { char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); if (fw_name) { @@ -3435,7 +3435,7 @@ static void myri10ge_select_firmware(struct myri10ge_priv *mgp) set_fw_name(mgp, fw_name, true); } } - kparam_unblock_sysfs_write(myri10ge_fw_name); + kernel_param_unlock(THIS_MODULE); if (mgp->board_number < MYRI10GE_MAX_BOARDS && myri10ge_fw_names[mgp->board_number] != NULL && diff --git a/drivers/net/wireless/libertas_tf/if_usb.c b/drivers/net/wireless/libertas_tf/if_usb.c index 1a20cee5febe..799a2efe5793 100644 --- a/drivers/net/wireless/libertas_tf/if_usb.c +++ b/drivers/net/wireless/libertas_tf/if_usb.c @@ -821,15 +821,15 @@ static int if_usb_prog_firmware(struct if_usb_card *cardp) lbtf_deb_enter(LBTF_DEB_USB); - kparam_block_sysfs_write(fw_name); + kernel_param_lock(THIS_MODULE); ret = request_firmware(&cardp->fw, lbtf_fw_name, &cardp->udev->dev); if (ret < 0) { pr_err("request_firmware() failed with %#x\n", ret); pr_err("firmware %s not found\n", lbtf_fw_name); - kparam_unblock_sysfs_write(fw_name); + kernel_param_unlock(THIS_MODULE); goto done; } - kparam_unblock_sysfs_write(fw_name); + kernel_param_unlock(THIS_MODULE); if (check_fwfile_format(cardp->fw->data, cardp->fw->size)) goto release_fw; diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c index 888998a7fe31..a2ae88dbda78 100644 --- a/drivers/usb/atm/ueagle-atm.c +++ b/drivers/usb/atm/ueagle-atm.c @@ -1599,7 +1599,7 @@ static void cmvs_file_name(struct uea_softc *sc, char *const cmv_name, int ver) char file_arr[] = "CMVxy.bin"; char *file; - kparam_block_sysfs_write(cmv_file); + kernel_param_lock(THIS_MODULE); /* set proper name corresponding modem version and line type */ if (cmv_file[sc->modem_index] == NULL) { if (UEA_CHIP_VERSION(sc) == ADI930) @@ -1618,7 +1618,7 @@ static void cmvs_file_name(struct uea_softc *sc, char *const cmv_name, int ver) strlcat(cmv_name, file, UEA_FW_NAME_MAX); if (ver == 2) strlcat(cmv_name, ".v2", UEA_FW_NAME_MAX); - kparam_unblock_sysfs_write(cmv_file); + kernel_param_unlock(THIS_MODULE); } static int request_cmvs_old(struct uea_softc *sc, diff --git a/drivers/video/fbdev/vt8623fb.c b/drivers/video/fbdev/vt8623fb.c index ea7f056ed5fe..8bac309c24b9 100644 --- a/drivers/video/fbdev/vt8623fb.c +++ b/drivers/video/fbdev/vt8623fb.c @@ -754,9 +754,9 @@ static int vt8623_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) /* Prepare startup mode */ - kparam_block_sysfs_write(mode_option); + kernel_param_lock(THIS_MODULE); rc = fb_find_mode(&(info->var), info, mode_option, NULL, 0, NULL, 8); - kparam_unblock_sysfs_write(mode_option); + kernel_param_unlock(THIS_MODULE); if (! ((rc == 1) || (rc == 2))) { rc = -EINVAL; dev_err(info->device, "mode %s not found\n", mode_option); diff --git a/include/linux/module.h b/include/linux/module.h index 4c1b02e1361d..6ba0e87fa804 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -240,6 +240,7 @@ struct module { unsigned int num_syms; /* Kernel parameters. */ + struct mutex param_lock; struct kernel_param *kp; unsigned int num_kp; diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index ab5031453807..f1fdc50520d8 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -67,6 +67,7 @@ enum { struct kernel_param { const char *name; + struct module *mod; const struct kernel_param_ops *ops; const u16 perm; s8 level; @@ -108,7 +109,7 @@ struct kparam_array * * @perm is 0 if the the variable is not to appear in sysfs, or 0444 * for world-readable, 0644 for root-writable, etc. Note that if it - * is writable, you may need to use kparam_block_sysfs_write() around + * is writable, you may need to use kernel_param_lock() around * accesses (esp. charp, which can be kfreed when it changes). * * The @type is simply pasted to refer to a param_ops_##type and a @@ -216,12 +217,12 @@ struct kparam_array parameters. */ #define __module_param_call(prefix, name, ops, arg, perm, level, flags) \ /* Default value instead of permissions? */ \ - static const char __param_str_##name[] = prefix #name; \ + static const char __param_str_##name[] = prefix #name; \ static struct kernel_param __moduleparam_const __param_##name \ __used \ __attribute__ ((unused,__section__ ("__param"),aligned(sizeof(void *)))) \ - = { __param_str_##name, ops, VERIFY_OCTAL_PERMISSIONS(perm), \ - level, flags, { arg } } + = { __param_str_##name, THIS_MODULE, ops, \ + VERIFY_OCTAL_PERMISSIONS(perm), level, flags, { arg } } /* Obsolete - use module_param_cb() */ #define module_param_call(name, set, get, arg, perm) \ @@ -238,58 +239,14 @@ __check_old_set_param(int (*oldset)(const char *, struct kernel_param *)) return 0; } -/** - * kparam_block_sysfs_write - make sure a parameter isn't written via sysfs. - * @name: the name of the parameter - * - * There's no point blocking write on a paramter that isn't writable via sysfs! - */ -#define kparam_block_sysfs_write(name) \ - do { \ - BUG_ON(!(__param_##name.perm & 0222)); \ - __kernel_param_lock(); \ - } while (0) - -/** - * kparam_unblock_sysfs_write - allows sysfs to write to a parameter again. - * @name: the name of the parameter - */ -#define kparam_unblock_sysfs_write(name) \ - do { \ - BUG_ON(!(__param_##name.perm & 0222)); \ - __kernel_param_unlock(); \ - } while (0) - -/** - * kparam_block_sysfs_read - make sure a parameter isn't read via sysfs. - * @name: the name of the parameter - * - * This also blocks sysfs writes. - */ -#define kparam_block_sysfs_read(name) \ - do { \ - BUG_ON(!(__param_##name.perm & 0444)); \ - __kernel_param_lock(); \ - } while (0) - -/** - * kparam_unblock_sysfs_read - allows sysfs to read a parameter again. - * @name: the name of the parameter - */ -#define kparam_unblock_sysfs_read(name) \ - do { \ - BUG_ON(!(__param_##name.perm & 0444)); \ - __kernel_param_unlock(); \ - } while (0) - #ifdef CONFIG_SYSFS -extern void __kernel_param_lock(void); -extern void __kernel_param_unlock(void); +extern void kernel_param_lock(struct module *mod); +extern void kernel_param_unlock(struct module *mod); #else -static inline void __kernel_param_lock(void) +static inline void kernel_param_lock(struct module *mod) { } -static inline void __kernel_param_unlock(void) +static inline void kernel_param_unlock(struct module *mod) { } #endif diff --git a/kernel/module.c b/kernel/module.c index 427b99f1a4b3..8ec33ce202a6 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -3442,6 +3442,8 @@ static int load_module(struct load_info *info, const char __user *uargs, if (err) goto unlink_mod; + mutex_init(&mod->param_lock); + /* Now we've got everything in the final locations, we can * find optional sections. */ err = find_module_sections(mod, info); diff --git a/kernel/params.c b/kernel/params.c index a8b09f6c87dc..8890d0b8dffc 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -25,15 +25,20 @@ #include #include -/* Protects all parameters, and incidentally kmalloced_param list. */ +/* Protects all built-in parameters, modules use their own param_lock */ static DEFINE_MUTEX(param_lock); +/* Use the module's mutex, or if built-in use the built-in mutex */ +#define KPARAM_MUTEX(mod) ((mod) ? &(mod)->param_lock : ¶m_lock) +#define KPARAM_IS_LOCKED(mod) mutex_is_locked(KPARAM_MUTEX(mod)) + /* This just allows us to keep track of which parameters are kmalloced. */ struct kmalloced_param { struct list_head list; char val[]; }; static LIST_HEAD(kmalloced_params); +static DEFINE_SPINLOCK(kmalloced_params_lock); static void *kmalloc_parameter(unsigned int size) { @@ -43,7 +48,10 @@ static void *kmalloc_parameter(unsigned int size) if (!p) return NULL; + spin_lock(&kmalloced_params_lock); list_add(&p->list, &kmalloced_params); + spin_unlock(&kmalloced_params_lock); + return p->val; } @@ -52,6 +60,7 @@ static void maybe_kfree_parameter(void *param) { struct kmalloced_param *p; + spin_lock(&kmalloced_params_lock); list_for_each_entry(p, &kmalloced_params, list) { if (p->val == param) { list_del(&p->list); @@ -59,6 +68,7 @@ static void maybe_kfree_parameter(void *param) break; } } + spin_unlock(&kmalloced_params_lock); } static char dash2underscore(char c) @@ -118,10 +128,10 @@ static int parse_one(char *param, return -EINVAL; pr_debug("handling %s with %p\n", param, params[i].ops->set); - mutex_lock(¶m_lock); + kernel_param_lock(params[i].mod); param_check_unsafe(¶ms[i]); err = params[i].ops->set(val, ¶ms[i]); - mutex_unlock(¶m_lock); + kernel_param_unlock(params[i].mod); return err; } } @@ -417,7 +427,8 @@ const struct kernel_param_ops param_ops_bint = { EXPORT_SYMBOL(param_ops_bint); /* We break the rule and mangle the string. */ -static int param_array(const char *name, +static int param_array(struct module *mod, + const char *name, const char *val, unsigned int min, unsigned int max, void *elem, int elemsize, @@ -448,7 +459,7 @@ static int param_array(const char *name, /* nul-terminate and parse */ save = val[len]; ((char *)val)[len] = '\0'; - BUG_ON(!mutex_is_locked(¶m_lock)); + BUG_ON(!KPARAM_IS_LOCKED(mod)); ret = set(val, &kp); if (ret != 0) @@ -470,7 +481,7 @@ static int param_array_set(const char *val, const struct kernel_param *kp) const struct kparam_array *arr = kp->arr; unsigned int temp_num; - return param_array(kp->name, val, 1, arr->max, arr->elem, + return param_array(kp->mod, kp->name, val, 1, arr->max, arr->elem, arr->elemsize, arr->ops->set, kp->level, arr->num ?: &temp_num); } @@ -485,7 +496,7 @@ static int param_array_get(char *buffer, const struct kernel_param *kp) if (i) buffer[off++] = ','; p.arg = arr->elem + arr->elemsize * i; - BUG_ON(!mutex_is_locked(¶m_lock)); + BUG_ON(!KPARAM_IS_LOCKED(p.mod)); ret = arr->ops->get(buffer + off, &p); if (ret < 0) return ret; @@ -568,9 +579,9 @@ static ssize_t param_attr_show(struct module_attribute *mattr, if (!attribute->param->ops->get) return -EPERM; - mutex_lock(¶m_lock); + kernel_param_lock(mk->mod); count = attribute->param->ops->get(buf, attribute->param); - mutex_unlock(¶m_lock); + kernel_param_unlock(mk->mod); if (count > 0) { strcat(buf, "\n"); ++count; @@ -580,7 +591,7 @@ static ssize_t param_attr_show(struct module_attribute *mattr, /* sysfs always hands a nul-terminated string in buf. We rely on that. */ static ssize_t param_attr_store(struct module_attribute *mattr, - struct module_kobject *km, + struct module_kobject *mk, const char *buf, size_t len) { int err; @@ -589,10 +600,10 @@ static ssize_t param_attr_store(struct module_attribute *mattr, if (!attribute->param->ops->set) return -EPERM; - mutex_lock(¶m_lock); + kernel_param_lock(mk->mod); param_check_unsafe(attribute->param); err = attribute->param->ops->set(buf, attribute->param); - mutex_unlock(¶m_lock); + kernel_param_unlock(mk->mod); if (!err) return len; return err; @@ -605,18 +616,19 @@ static ssize_t param_attr_store(struct module_attribute *mattr, #define __modinit __init #endif -#ifdef CONFIG_SYSFS -void __kernel_param_lock(void) +void kernel_param_lock(struct module *mod) { - mutex_lock(¶m_lock); + mutex_lock(KPARAM_MUTEX(mod)); } -EXPORT_SYMBOL(__kernel_param_lock); -void __kernel_param_unlock(void) +void kernel_param_unlock(struct module *mod) { - mutex_unlock(¶m_lock); + mutex_unlock(KPARAM_MUTEX(mod)); } -EXPORT_SYMBOL(__kernel_param_unlock); + +#ifdef CONFIG_SYSFS +EXPORT_SYMBOL(kernel_param_lock); +EXPORT_SYMBOL(kernel_param_unlock); /* * add_sysfs_param - add a parameter to sysfs diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c index d53355b011f5..8544e2eb570e 100644 --- a/net/mac80211/rate.c +++ b/net/mac80211/rate.c @@ -103,7 +103,7 @@ ieee80211_rate_control_ops_get(const char *name) const struct rate_control_ops *ops; const char *alg_name; - kparam_block_sysfs_write(ieee80211_default_rc_algo); + kernel_param_lock(THIS_MODULE); if (!name) alg_name = ieee80211_default_rc_algo; else @@ -117,7 +117,7 @@ ieee80211_rate_control_ops_get(const char *name) /* try built-in one if specific alg requested but not found */ if (!ops && strlen(CONFIG_MAC80211_RC_DEFAULT)) ops = ieee80211_try_rate_control_ops_get(CONFIG_MAC80211_RC_DEFAULT); - kparam_unblock_sysfs_write(ieee80211_default_rc_algo); + kernel_param_unlock(THIS_MODULE); return ops; } -- cgit v1.2.3-59-g8ed1b From 38183b9c31cf21d8996d6eee2e3a14508b20c418 Mon Sep 17 00:00:00 2001 From: Stephen Rothwell Date: Thu, 28 May 2015 17:20:58 +1000 Subject: rcu: merge fix for Convert ACCESS_ONCE() to READ_ONCE() and WRITE_ONCE() This mirrors the change introduced by 7d0ae8086b8 of same title in Linus' tree; it's not obvious as a merge resolution since we moved the function. Signed-off-by: Stephen Rothwell Signed-off-by: Rusty Russell --- include/linux/compiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index eae42c21d5fd..52bdec710ed7 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -467,7 +467,7 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s */ #define lockless_dereference(p) \ ({ \ - typeof(p) _________p1 = ACCESS_ONCE(p); \ + typeof(p) _________p1 = READ_ONCE(p); \ smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ (_________p1); \ }) -- cgit v1.2.3-59-g8ed1b From cf2fde7b39e9446e2af015215d7fb695781af0c1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 26 Jun 2015 06:44:38 +0930 Subject: param: fix module param locks when !CONFIG_SYSFS. As Dan Streetman points out, the entire point of locking for is to stop sysfs accesses, so they're elided entirely in the !SYSFS case. Reported-by: Stephen Rothwell Signed-off-by: Rusty Russell --- include/linux/module.h | 2 ++ kernel/module.c | 9 ++++++++- kernel/params.c | 18 ++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/include/linux/module.h b/include/linux/module.h index 6ba0e87fa804..46efa1c9de60 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -240,7 +240,9 @@ struct module { unsigned int num_syms; /* Kernel parameters. */ +#ifdef CONFIG_SYSFS struct mutex param_lock; +#endif struct kernel_param *kp; unsigned int num_kp; diff --git a/kernel/module.c b/kernel/module.c index 8ec33ce202a6..b4994adf7187 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -1820,6 +1820,10 @@ static void mod_sysfs_fini(struct module *mod) mod_kobject_put(mod); } +static void init_param_lock(struct module *mod) +{ + mutex_init(&mod->param_lock); +} #else /* !CONFIG_SYSFS */ static int mod_sysfs_setup(struct module *mod, @@ -1842,6 +1846,9 @@ static void del_usage_links(struct module *mod) { } +static void init_param_lock(struct module *mod) +{ +} #endif /* CONFIG_SYSFS */ static void mod_sysfs_teardown(struct module *mod) @@ -3442,7 +3449,7 @@ static int load_module(struct load_info *info, const char __user *uargs, if (err) goto unlink_mod; - mutex_init(&mod->param_lock); + init_param_lock(mod); /* Now we've got everything in the final locations, we can * find optional sections. */ diff --git a/kernel/params.c b/kernel/params.c index 8890d0b8dffc..faa461c16f12 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -25,12 +25,22 @@ #include #include +#ifdef CONFIG_SYSFS /* Protects all built-in parameters, modules use their own param_lock */ static DEFINE_MUTEX(param_lock); /* Use the module's mutex, or if built-in use the built-in mutex */ #define KPARAM_MUTEX(mod) ((mod) ? &(mod)->param_lock : ¶m_lock) -#define KPARAM_IS_LOCKED(mod) mutex_is_locked(KPARAM_MUTEX(mod)) + +static inline void check_kparam_locked(struct module *mod) +{ + BUG_ON(!mutex_is_locked(KPARAM_MUTEX(mod))); +} +#else +static inline void check_kparam_locked(struct module *mod) +{ +} +#endif /* !CONFIG_SYSFS */ /* This just allows us to keep track of which parameters are kmalloced. */ struct kmalloced_param { @@ -459,7 +469,7 @@ static int param_array(struct module *mod, /* nul-terminate and parse */ save = val[len]; ((char *)val)[len] = '\0'; - BUG_ON(!KPARAM_IS_LOCKED(mod)); + check_kparam_locked(mod); ret = set(val, &kp); if (ret != 0) @@ -496,7 +506,7 @@ static int param_array_get(char *buffer, const struct kernel_param *kp) if (i) buffer[off++] = ','; p.arg = arr->elem + arr->elemsize * i; - BUG_ON(!KPARAM_IS_LOCKED(p.mod)); + check_kparam_locked(p.mod); ret = arr->ops->get(buffer + off, &p); if (ret < 0) return ret; @@ -616,6 +626,7 @@ static ssize_t param_attr_store(struct module_attribute *mattr, #define __modinit __init #endif +#ifdef CONFIG_SYSFS void kernel_param_lock(struct module *mod) { mutex_lock(KPARAM_MUTEX(mod)); @@ -626,7 +637,6 @@ void kernel_param_unlock(struct module *mod) mutex_unlock(KPARAM_MUTEX(mod)); } -#ifdef CONFIG_SYSFS EXPORT_SYMBOL(kernel_param_lock); EXPORT_SYMBOL(kernel_param_unlock); -- cgit v1.2.3-59-g8ed1b From 20bdc2cfdbc484777b30b96fcdbb8994038f3ce1 Mon Sep 17 00:00:00 2001 From: Stephen Rothwell Date: Fri, 26 Jun 2015 13:19:19 +1000 Subject: modules: only use mod->param_lock if CONFIG_MODULES Signed-off-by: Stephen Rothwell Signed-off-by: Rusty Russell --- kernel/params.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/params.c b/kernel/params.c index faa461c16f12..adc0bbc06cc5 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -30,7 +30,11 @@ static DEFINE_MUTEX(param_lock); /* Use the module's mutex, or if built-in use the built-in mutex */ +#ifdef CONFIG_MODULES #define KPARAM_MUTEX(mod) ((mod) ? &(mod)->param_lock : ¶m_lock) +#else +#define KPARAM_MUTEX(mod) (¶m_lock) +#endif static inline void check_kparam_locked(struct module *mod) { -- cgit v1.2.3-59-g8ed1b