From 7d65f4a6553203da6a22097821d151fbbe7e4956 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 5 Sep 2013 15:49:45 +0200 Subject: irq: Consolidate do_softirq() arch overriden implementations All arch overriden implementations of do_softirq() share the following common code: disable irqs (to avoid races with the pending check), check if there are softirqs pending, then execute __do_softirq() on a specific stack. Consolidate the common parts such that archs only worry about the stack switch. Acked-by: Linus Torvalds Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Paul Mackerras Cc: James Hogan Cc: James E.J. Bottomley Cc: Helge Deller Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: David S. Miller Cc: Andrew Morton --- kernel/softirq.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index d7d498d8cc4f..26ee72725d29 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -29,7 +29,6 @@ #define CREATE_TRACE_POINTS #include -#include /* - No shared variables, all the data are CPU local. - If a softirq needs serialization, let it serialize itself @@ -283,7 +282,7 @@ restart: tsk_restore_flags(current, old_flags, PF_MEMALLOC); } -#ifndef __ARCH_HAS_DO_SOFTIRQ + asmlinkage void do_softirq(void) { @@ -298,13 +297,12 @@ asmlinkage void do_softirq(void) pending = local_softirq_pending(); if (pending) - __do_softirq(); + do_softirq_own_stack(); + WARN_ON_ONCE(softirq_count()); local_irq_restore(flags); } -#endif - /* * Enter an interrupt context. */ -- cgit v1.2.3-59-g8ed1b From be6e1016440860fc4ec098b2d0aed3d0397b5d6e Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 24 Sep 2013 16:39:41 +0200 Subject: irq: Optimize call to softirq on hardirq exit Before processing softirqs on hardirq exit, we already do the check for pending softirqs while hardirqs are guaranteed to be disabled. So we can take a shortcut and safely jump to the arch specific implementation directly. Acked-by: Linus Torvalds Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Paul Mackerras Cc: James Hogan Cc: James E.J. Bottomley Cc: Helge Deller Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: David S. Miller Cc: Andrew Morton --- kernel/softirq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index 26ee72725d29..17c5cd2e3dae 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -335,7 +335,7 @@ static inline void invoke_softirq(void) * in its own stack to prevent from any overrun on top * of a potentially deep task stack. */ - do_softirq(); + do_softirq_own_stack(); } else { wakeup_softirqd(); } -- cgit v1.2.3-59-g8ed1b From 5d60d3e7c08a46643e902e39d9743cf394382151 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 24 Sep 2013 04:11:35 +0200 Subject: irq: Improve a bit softirq debugging do_softirq() has a debug check that verifies that it is not nesting on softirqs processing, nor miscounting the softirq part of the preempt count. But making sure that softirqs processing don't nest is actually a more generic concern that applies to any caller of __do_softirq(). Do take it one step further and generalize that debug check to any softirq processing. Acked-by: Linus Torvalds Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Paul Mackerras Cc: James Hogan Cc: James E.J. Bottomley Cc: Helge Deller Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: David S. Miller Cc: Andrew Morton --- kernel/softirq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index 17c5cd2e3dae..9f8092b82a94 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -133,7 +133,6 @@ EXPORT_SYMBOL(local_bh_disable); static void __local_bh_enable(unsigned int cnt) { - WARN_ON_ONCE(in_irq()); WARN_ON_ONCE(!irqs_disabled()); if (softirq_count() == cnt) @@ -148,6 +147,7 @@ static void __local_bh_enable(unsigned int cnt) */ void _local_bh_enable(void) { + WARN_ON_ONCE(in_irq()); __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); } @@ -279,6 +279,7 @@ restart: account_irq_exit_time(current); __local_bh_enable(SOFTIRQ_OFFSET); + WARN_ON_ONCE(in_interrupt()); tsk_restore_flags(current, old_flags, PF_MEMALLOC); } @@ -299,7 +300,6 @@ asmlinkage void do_softirq(void) if (pending) do_softirq_own_stack(); - WARN_ON_ONCE(softirq_count()); local_irq_restore(flags); } -- cgit v1.2.3-59-g8ed1b From 0bed698a334766ed07bacd6cb33f0228003a7f61 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 5 Sep 2013 16:14:00 +0200 Subject: irq: Justify the various softirq stack choices For clarity, comment the various stack choices for softirqs processing, whether we execute them from ksoftirqd or local_irq_enable() calls. Their use on irq_exit() is already commented. Acked-by: Linus Torvalds Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Paul Mackerras Cc: James Hogan Cc: James E.J. Bottomley Cc: Helge Deller Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: David S. Miller Cc: Andrew Morton --- kernel/softirq.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/softirq.c b/kernel/softirq.c index 9f8092b82a94..2b4328ea769f 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -170,8 +170,13 @@ static inline void _local_bh_enable_ip(unsigned long ip) */ sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1); - if (unlikely(!in_interrupt() && local_softirq_pending())) + if (unlikely(!in_interrupt() && local_softirq_pending())) { + /* + * Run softirq if any pending. And do it in its own stack + * as we may be calling this deep in a task call stack already. + */ do_softirq(); + } dec_preempt_count(); #ifdef CONFIG_TRACE_IRQFLAGS @@ -769,6 +774,10 @@ static void run_ksoftirqd(unsigned int cpu) { local_irq_disable(); if (local_softirq_pending()) { + /* + * We can safely run softirq on inline stack, as we are not deep + * in the task stack here. + */ __do_softirq(); rcu_note_context_switch(cpu); local_irq_enable(); -- cgit v1.2.3-59-g8ed1b From cc1f027454929924471bea2f362431072e3c71be Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Tue, 24 Sep 2013 17:17:47 +0200 Subject: irq: Optimize softirq stack selection in irq exit If irq_exit() is called on the arch's specified irq stack, it should be safe to run softirqs inline under that same irq stack as it is near empty by the time we call irq_exit(). For example if we use the same stack for both hard and soft irqs here, the worst case scenario is: hardirq -> softirq -> hardirq. But then the softirq supersedes the first hardirq as the stack user since irq_exit() is called in a mostly empty stack. So the stack merge in this case looks acceptable. Stack overrun still have a chance to happen if hardirqs have more opportunities to nest, but then it's another problem to solve. So lets adapt the irq exit's softirq stack on top of a new Kconfig symbol that can be defined when irq_exit() runs on the irq stack. That way we can spare some stack switch on irq processing and all the cache issues that come along. Acked-by: Linus Torvalds Signed-off-by: Frederic Weisbecker Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Paul Mackerras Cc: James Hogan Cc: James E.J. Bottomley Cc: Helge Deller Cc: Martin Schwidefsky Cc: Heiko Carstens Cc: David S. Miller Cc: Andrew Morton --- arch/Kconfig | 10 ++++++++++ kernel/softirq.c | 14 ++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/arch/Kconfig b/arch/Kconfig index af2cc6eabcc7..ad95133f8fae 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -390,6 +390,16 @@ config HAVE_UNDERSCORE_SYMBOL_PREFIX Some architectures generate an _ in front of C symbols; things like module loading and assembly files need to know about this. +config HAVE_IRQ_EXIT_ON_IRQ_STACK + bool + help + Architecture doesn't only execute the irq handler on the irq stack + but also irq_exit(). This way we can process softirqs on this irq + stack instead of switching to a new one when we call __do_softirq() + in the end of an hardirq. + This spares a stack switch and improves cache usage on softirq + processing. + # # ABI hall of shame # diff --git a/kernel/softirq.c b/kernel/softirq.c index 2b4328ea769f..dacd0ab51df4 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -332,15 +332,21 @@ void irq_enter(void) static inline void invoke_softirq(void) { if (!force_irqthreads) { +#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK /* * We can safely execute softirq on the current stack if * it is the irq stack, because it should be near empty - * at this stage. But we have no way to know if the arch - * calls irq_exit() on the irq stack. So call softirq - * in its own stack to prevent from any overrun on top - * of a potentially deep task stack. + * at this stage. + */ + __do_softirq(); +#else + /* + * Otherwise, irq_exit() is called on the task stack that can + * be potentially deep already. So call softirq in its own stack + * to prevent from any overrun. */ do_softirq_own_stack(); +#endif } else { wakeup_softirqd(); } -- cgit v1.2.3-59-g8ed1b