From a3ff53167cef2b5c6c8948246172d6f9279f037f Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Tue, 19 Mar 2024 11:40:12 +0100 Subject: x86/asm: Remove %P operand modifier from altinstr asm templates The "P" asm operand modifier is a x86 target-specific modifier. For x86_64, when used with a symbol reference, the "%P" modifier emits "sym" instead of "sym(%rip)". This property is currently used to prevent %RIP-relative addressing in .altinstr sections. %RIP-relative addresses are nowadays correctly handled in .altinstr sections, so remove %P operand modifier from altinstr asm templates. Also note that unlike GCC, clang emits %rip-relative symbol reference with "P" asm operand modifier, so the patch also unifies symbol handling with both compilers. No functional changes intended. Signed-off-by: Uros Bizjak Signed-off-by: Ingo Molnar Cc: Linus Torvalds Cc: Josh Poimboeuf Cc: Ard Biesheuvel Cc: "H. Peter Anvin" Link: https://lore.kernel.org/r/20240319104418.284519-2-ubizjak@gmail.com --- arch/x86/include/asm/apic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/include/asm/apic.h') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 94ce0f7c9d3a..fa2e4244654e 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -91,7 +91,7 @@ static inline void native_apic_mem_write(u32 reg, u32 v) { volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg); - alternative_io("movl %0, %P1", "xchgl %0, %P1", X86_BUG_11AP, + alternative_io("movl %0, %1", "xchgl %0, %1", X86_BUG_11AP, ASM_OUTPUT2("=r" (v), "=m" (*addr)), ASM_OUTPUT2("0" (v), "m" (*addr))); } -- cgit v1.2.3-59-g8ed1b From 5ce344beaca688f4cdea07045e0b8f03dc537e74 Mon Sep 17 00:00:00 2001 From: Adam Dunlap Date: Mon, 18 Mar 2024 16:09:27 -0700 Subject: x86/apic: Force native_apic_mem_read() to use the MOV instruction When done from a virtual machine, instructions that touch APIC memory must be emulated. By convention, MMIO accesses are typically performed via io.h helpers such as readl() or writeq() to simplify instruction emulation/decoding (ex: in KVM hosts and SEV guests) [0]. Currently, native_apic_mem_read() does not follow this convention, allowing the compiler to emit instructions other than the MOV instruction generated by readl(). In particular, when the kernel is compiled with clang and run as a SEV-ES or SEV-SNP guest, the compiler would emit a TESTL instruction which is not supported by the SEV-ES emulator, causing a boot failure in that environment. It is likely the same problem would happen in a TDX guest as that uses the same instruction emulator as SEV-ES. To make sure all emulators can emulate APIC memory reads via MOV, use the readl() function in native_apic_mem_read(). It is expected that any emulator would support MOV in any addressing mode as it is the most generic and is what is usually emitted currently. The TESTL instruction is emitted when native_apic_mem_read() is inlined into apic_mem_wait_icr_idle(). The emulator comes from insn_decode_mmio() in arch/x86/lib/insn-eval.c. It's not worth it to extend insn_decode_mmio() to support more instructions since, in theory, the compiler could choose to output nearly any instruction for such reads which would bloat the emulator beyond reason. [0] https://lore.kernel.org/all/20220405232939.73860-12-kirill.shutemov@linux.intel.com/ [ bp: Massage commit message, fix typos. ] Signed-off-by: Adam Dunlap Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Thomas Gleixner Reviewed-by: Ard Biesheuvel Tested-by: Kevin Loughlin Cc: Link: https://lore.kernel.org/r/20240318230927.2191933-1-acdunlap@google.com --- arch/x86/include/asm/apic.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/x86/include/asm/apic.h') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 94ce0f7c9d3a..e6ab0cf15ed5 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -13,6 +13,7 @@ #include #include #include +#include #define ARCH_APICTIMER_STOPS_ON_C3 1 @@ -98,7 +99,7 @@ static inline void native_apic_mem_write(u32 reg, u32 v) static inline u32 native_apic_mem_read(u32 reg) { - return *((volatile u32 *)(APIC_BASE + reg)); + return readl((void __iomem *)(APIC_BASE + reg)); } static inline void native_apic_mem_eoi(void) -- cgit v1.2.3-59-g8ed1b From fef05a078b6fa1e9047e0486f1f6daf70664fd12 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 23 Apr 2024 10:41:11 -0700 Subject: x86/irq: Factor out common code for checking pending interrupts Use a common function for checking pending interrupt vector in APIC IRR instead of duplicated open coding them. Additional checks for posted MSI vectors can then be contained in this function. Signed-off-by: Jacob Pan Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240423174114.526704-10-jacob.jun.pan@linux.intel.com --- arch/x86/include/asm/apic.h | 11 +++++++++++ arch/x86/kernel/apic/vector.c | 5 ++--- arch/x86/kernel/irq.c | 5 ++--- 3 files changed, 15 insertions(+), 6 deletions(-) (limited to 'arch/x86/include/asm/apic.h') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index e6ab0cf15ed5..50f9781fa3ed 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -500,6 +500,17 @@ static inline bool lapic_vector_set_in_irr(unsigned int vector) return !!(irr & (1U << (vector % 32))); } +static inline bool is_vector_pending(unsigned int vector) +{ + unsigned int irr; + + irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); + if (irr & (1 << (vector % 32))) + return true; + + return false; +} + /* * Warm reset vector position: */ diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 185738c72766..9eec52925fa3 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -965,7 +965,7 @@ static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr) lockdep_assert_held(&vector_lock); hlist_for_each_entry_safe(apicd, tmp, &cl->head, clist) { - unsigned int irr, vector = apicd->prev_vector; + unsigned int vector = apicd->prev_vector; /* * Paranoia: Check if the vector that needs to be cleaned @@ -979,8 +979,7 @@ static void __vector_cleanup(struct vector_cleanup *cl, bool check_irr) * fixup_irqs() was just called to scan IRR for set bits and * forward them to new destination CPUs via IPIs. */ - irr = check_irr ? apic_read(APIC_IRR + (vector / 32 * 0x10)) : 0; - if (irr & (1U << (vector % 32))) { + if (check_irr && is_vector_pending(vector)) { pr_warn_once("Moved interrupt pending in old target APIC %u\n", apicd->irq); rearm = true; continue; diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 578e4f6a5080..385e3a5fc304 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -484,7 +484,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification) /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ void fixup_irqs(void) { - unsigned int irr, vector; + unsigned int vector; struct irq_desc *desc; struct irq_data *data; struct irq_chip *chip; @@ -511,8 +511,7 @@ void fixup_irqs(void) if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector]))) continue; - irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); - if (irr & (1 << (vector % 32))) { + if (is_vector_pending(vector)) { desc = __this_cpu_read(vector_irq[vector]); raw_spin_lock(&desc->lock); -- cgit v1.2.3-59-g8ed1b From ce0a92871179f8ca58ae8e3cf50e726a163bf831 Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Tue, 23 Apr 2024 10:41:12 -0700 Subject: x86/irq: Extend checks for pending vectors to posted interrupts During interrupt affinity change, it is possible to have interrupts delivered to the old CPU after the affinity has changed to the new one. To prevent lost interrupts, local APIC IRR is checked on the old CPU. Similar checks must be done for posted MSIs given the same reason. Consider the following scenario: Device system agent iommu memory CPU/LAPIC 1 FEEX_XXXX 2 Interrupt request 3 Fetch IRTE -> 4 ->Atomic Swap PID.PIR(vec) Push to Global Observable(GO) 5 if (ON*) done;* else 6 send a notification -> * ON: outstanding notification, 1 will suppress new notifications If the affinity change happens between 3 and 5 in the IOMMU, the old CPU's posted interrupt request (PIR) could have the pending bit set for the vector being moved. Add a helper function to check individual vector status. Then use the helper to check for pending interrupts on the source CPU's PID. Signed-off-by: Jacob Pan Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240423174114.526704-11-jacob.jun.pan@linux.intel.com --- arch/x86/include/asm/apic.h | 3 ++- arch/x86/include/asm/posted_intr.h | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) (limited to 'arch/x86/include/asm/apic.h') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 50f9781fa3ed..5644c396713e 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -14,6 +14,7 @@ #include #include #include +#include #define ARCH_APICTIMER_STOPS_ON_C3 1 @@ -508,7 +509,7 @@ static inline bool is_vector_pending(unsigned int vector) if (irr & (1 << (vector % 32))) return true; - return false; + return pi_pending_this_cpu(vector); } /* diff --git a/arch/x86/include/asm/posted_intr.h b/arch/x86/include/asm/posted_intr.h index 6f84f6739d99..de788b400fba 100644 --- a/arch/x86/include/asm/posted_intr.h +++ b/arch/x86/include/asm/posted_intr.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _X86_POSTED_INTR_H #define _X86_POSTED_INTR_H +#include #define POSTED_INTR_ON 0 #define POSTED_INTR_SN 1 @@ -92,8 +93,25 @@ static inline void __pi_clear_sn(struct pi_desc *pi_desc) } #ifdef CONFIG_X86_POSTED_MSI +/* + * Not all external vectors are subject to interrupt remapping, e.g. IOMMU's + * own interrupts. Here we do not distinguish them since those vector bits in + * PIR will always be zero. + */ +static inline bool pi_pending_this_cpu(unsigned int vector) +{ + struct pi_desc *pid = this_cpu_ptr(&posted_msi_pi_desc); + + if (WARN_ON_ONCE(vector > NR_VECTORS || vector < FIRST_EXTERNAL_VECTOR)) + return false; + + return test_bit(vector, (unsigned long *)pid->pir); +} + extern void intel_posted_msi_init(void); #else +static inline bool pi_pending_this_cpu(unsigned int vector) { return false; } + static inline void intel_posted_msi_init(void) {}; #endif /* X86_POSTED_MSI */ -- cgit v1.2.3-59-g8ed1b From 6ecc2e7932fe8f132d3b671685f9995785f19e9a Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Mon, 6 May 2024 10:56:12 -0700 Subject: x86/irq: Use existing helper for pending vector check lapic_vector_set_in_irr() is already available, use it for checking pending vectors at the local APIC. No functional change. Signed-off-by: Jacob Pan Signed-off-by: Thomas Gleixner Reviewed-by: Imran Khan Link: https://lore.kernel.org/r/20240506175612.1141095-1-jacob.jun.pan@linux.intel.com --- arch/x86/include/asm/apic.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'arch/x86/include/asm/apic.h') diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 5644c396713e..467532b3e070 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -503,13 +503,7 @@ static inline bool lapic_vector_set_in_irr(unsigned int vector) static inline bool is_vector_pending(unsigned int vector) { - unsigned int irr; - - irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); - if (irr & (1 << (vector % 32))) - return true; - - return pi_pending_this_cpu(vector); + return lapic_vector_set_in_irr(vector) || pi_pending_this_cpu(vector); } /* -- cgit v1.2.3-59-g8ed1b