From 6733bab7bc09b67668028dab562caea1b4ff3c69 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 18 Aug 2017 10:59:16 -0700 Subject: irq_work: Map irq_work_on_queue() to irq_work_on() in !SMP Commit 478850160636 ("irq_work: Implement remote queueing") provides irq_work_on_queue() only for SMP builds. However, providing it simplifies code that submits irq_work to lists of CPUs, eliminating the !SMP special cases. This commit therefore maps irq_work_on_queue() to irq_work_on() in !SMP builds, but validating the specified CPU. Signed-off-by: Paul E. McKenney --- kernel/irq_work.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'kernel/irq_work.c') diff --git a/kernel/irq_work.c b/kernel/irq_work.c index bcf107ce0854..9f20f6c72579 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -56,7 +56,6 @@ void __weak arch_irq_work_raise(void) */ } -#ifdef CONFIG_SMP /* * Enqueue the irq_work @work on @cpu unless it's already pending * somewhere. @@ -68,6 +67,8 @@ bool irq_work_queue_on(struct irq_work *work, int cpu) /* All work should have been flushed before going offline */ WARN_ON_ONCE(cpu_is_offline(cpu)); +#ifdef CONFIG_SMP + /* Arch remote IPI send/receive backend aren't NMI safe */ WARN_ON_ONCE(in_nmi()); @@ -78,10 +79,12 @@ bool irq_work_queue_on(struct irq_work *work, int cpu) if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) arch_send_call_function_single_ipi(cpu); +#else /* #ifdef CONFIG_SMP */ + irq_work_queue(work); +#endif /* #else #ifdef CONFIG_SMP */ + return true; } -EXPORT_SYMBOL_GPL(irq_work_queue_on); -#endif /* Enqueue the irq work @work on the current CPU */ bool irq_work_queue(struct irq_work *work) -- cgit v1.3-8-gc7d7 From 16c0890dc66d258fdeccf7b15a133f3930b19143 Mon Sep 17 00:00:00 2001 From: Byungchul Park Date: Tue, 31 Oct 2017 02:46:54 +0100 Subject: irq/work: Don't reinvent the wheel but use existing llist API Use the proper llist APIs instead of open-coded variants of them. Signed-off-by: Byungchul Park Signed-off-by: Frederic Weisbecker Acked-by: Thomas Gleixner Cc: Linus Torvalds Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1509414414-14987-1-git-send-email-frederic@kernel.org Signed-off-by: Ingo Molnar --- kernel/irq_work.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'kernel/irq_work.c') diff --git a/kernel/irq_work.c b/kernel/irq_work.c index bcf107ce0854..e2ebe8c71e8f 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -138,11 +138,7 @@ static void irq_work_run_list(struct llist_head *list) return; llnode = llist_del_all(list); - while (llnode != NULL) { - work = llist_entry(llnode, struct irq_work, llnode); - - llnode = llist_next(llnode); - + llist_for_each_entry(work, llnode, llnode) { /* * Clear the PENDING bit, after this point the @work * can be re-used. -- cgit v1.3-8-gc7d7 From 3c7169a3bf8216a56761a8edf775072dd36a00a0 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 6 Nov 2017 16:01:26 +0100 Subject: irq_work: Use lockdep to assert IRQs are disabled/enabled Use lockdep to check that IRQs are enabled or disabled as expected. This way the sanity check only shows overhead when concurrency correctness debug code is enabled. Signed-off-by: Frederic Weisbecker Acked-by: Thomas Gleixner Cc: David S . Miller Cc: Lai Jiangshan Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Tejun Heo Link: http://lkml.kernel.org/r/1509980490-4285-11-git-send-email-frederic@kernel.org Signed-off-by: Ingo Molnar --- kernel/irq_work.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/irq_work.c') diff --git a/kernel/irq_work.c b/kernel/irq_work.c index bcf107ce0854..899579657a0a 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -188,7 +188,7 @@ void irq_work_tick(void) */ void irq_work_sync(struct irq_work *work) { - WARN_ON_ONCE(irqs_disabled()); + lockdep_assert_irqs_enabled(); while (work->flags & IRQ_WORK_BUSY) cpu_relax(); -- cgit v1.3-8-gc7d7 From d00a08cf9ee986ad6689ce8c6fd176aff679c106 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 12 Nov 2017 13:02:51 +0100 Subject: irq/work: Use llist_for_each_entry_safe The llist_for_each_entry() loop in irq_work_run_list() is unsafe because once the works PENDING bit is cleared it can be requeued on another CPU. Use llist_for_each_entry_safe() instead. Fixes: 16c0890dc66d ("irq/work: Don't reinvent the wheel but use existing llist API") Reported-by:Chris Wilson Signed-off-by: Thomas Gleixner Cc: Frederic Weisbecker Cc: Byungchul Park Cc: Peter Zijlstra Cc: Petri Latvala Link: http://lkml.kernel.org/r/151027307351.14762.4611888896020658384@mail.alporthouse.com --- kernel/irq_work.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel/irq_work.c') diff --git a/kernel/irq_work.c b/kernel/irq_work.c index e2ebe8c71e8f..6647b33f7eb0 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -128,9 +128,9 @@ bool irq_work_needs_cpu(void) static void irq_work_run_list(struct llist_head *list) { - unsigned long flags; - struct irq_work *work; + struct irq_work *work, *tmp; struct llist_node *llnode; + unsigned long flags; BUG_ON(!irqs_disabled()); @@ -138,7 +138,7 @@ static void irq_work_run_list(struct llist_head *list) return; llnode = llist_del_all(list); - llist_for_each_entry(work, llnode, llnode) { + llist_for_each_entry_safe(work, tmp, llnode, llnode) { /* * Clear the PENDING bit, after this point the @work * can be re-used. -- cgit v1.3-8-gc7d7