diff options
Diffstat (limited to '')
| -rw-r--r-- | arch/arm/kvm/hyp/Makefile | 34 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/banked-sr.c | 70 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/cp15-sr.c | 72 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/entry.S | 121 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/hyp-entry.S | 295 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/s2-setup.c | 22 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/switch.c | 242 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/tlb.c | 68 | ||||
| -rw-r--r-- | arch/arm/kvm/hyp/vfp.S | 57 |
9 files changed, 0 insertions, 981 deletions
diff --git a/arch/arm/kvm/hyp/Makefile b/arch/arm/kvm/hyp/Makefile deleted file mode 100644 index ba88b1eca93c..000000000000 --- a/arch/arm/kvm/hyp/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for Kernel-based Virtual Machine module, HYP part -# - -ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING - -KVM=../../../../virt/kvm - -CFLAGS_ARMV7VE :=$(call cc-option, -march=armv7ve) - -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/vgic-v3-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/timer-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/aarch32.o - -obj-$(CONFIG_KVM_ARM_HOST) += tlb.o -obj-$(CONFIG_KVM_ARM_HOST) += cp15-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += vfp.o -obj-$(CONFIG_KVM_ARM_HOST) += banked-sr.o -CFLAGS_banked-sr.o += $(CFLAGS_ARMV7VE) - -obj-$(CONFIG_KVM_ARM_HOST) += entry.o -obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o -obj-$(CONFIG_KVM_ARM_HOST) += switch.o -CFLAGS_switch.o += $(CFLAGS_ARMV7VE) -obj-$(CONFIG_KVM_ARM_HOST) += s2-setup.o - -# KVM code is run at a different exception code with a different map, so -# compiler instrumentation that inserts callbacks or checks into the code may -# cause crashes. Just disable it. -GCOV_PROFILE := n -KASAN_SANITIZE := n -UBSAN_SANITIZE := n -KCOV_INSTRUMENT := n diff --git a/arch/arm/kvm/hyp/banked-sr.c b/arch/arm/kvm/hyp/banked-sr.c deleted file mode 100644 index c4632ed9e819..000000000000 --- a/arch/arm/kvm/hyp/banked-sr.c +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Original code: - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - * - * Mostly rewritten in C by Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <asm/kvm_hyp.h> - -/* - * gcc before 4.9 doesn't understand -march=armv7ve, so we have to - * trick the assembler. - */ -__asm__(".arch_extension virt"); - -void __hyp_text __banked_save_state(struct kvm_cpu_context *ctxt) -{ - ctxt->gp_regs.usr_regs.ARM_sp = read_special(SP_usr); - ctxt->gp_regs.usr_regs.ARM_pc = read_special(ELR_hyp); - ctxt->gp_regs.usr_regs.ARM_cpsr = read_special(SPSR); - ctxt->gp_regs.KVM_ARM_SVC_sp = read_special(SP_svc); - ctxt->gp_regs.KVM_ARM_SVC_lr = read_special(LR_svc); - ctxt->gp_regs.KVM_ARM_SVC_spsr = read_special(SPSR_svc); - ctxt->gp_regs.KVM_ARM_ABT_sp = read_special(SP_abt); - ctxt->gp_regs.KVM_ARM_ABT_lr = read_special(LR_abt); - ctxt->gp_regs.KVM_ARM_ABT_spsr = read_special(SPSR_abt); - ctxt->gp_regs.KVM_ARM_UND_sp = read_special(SP_und); - ctxt->gp_regs.KVM_ARM_UND_lr = read_special(LR_und); - ctxt->gp_regs.KVM_ARM_UND_spsr = read_special(SPSR_und); - ctxt->gp_regs.KVM_ARM_IRQ_sp = read_special(SP_irq); - ctxt->gp_regs.KVM_ARM_IRQ_lr = read_special(LR_irq); - ctxt->gp_regs.KVM_ARM_IRQ_spsr = read_special(SPSR_irq); - ctxt->gp_regs.KVM_ARM_FIQ_r8 = read_special(R8_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_r9 = read_special(R9_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_r10 = read_special(R10_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_fp = read_special(R11_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_ip = read_special(R12_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_sp = read_special(SP_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_lr = read_special(LR_fiq); - ctxt->gp_regs.KVM_ARM_FIQ_spsr = read_special(SPSR_fiq); -} - -void __hyp_text __banked_restore_state(struct kvm_cpu_context *ctxt) -{ - write_special(ctxt->gp_regs.usr_regs.ARM_sp, SP_usr); - write_special(ctxt->gp_regs.usr_regs.ARM_pc, ELR_hyp); - write_special(ctxt->gp_regs.usr_regs.ARM_cpsr, SPSR_cxsf); - write_special(ctxt->gp_regs.KVM_ARM_SVC_sp, SP_svc); - write_special(ctxt->gp_regs.KVM_ARM_SVC_lr, LR_svc); - write_special(ctxt->gp_regs.KVM_ARM_SVC_spsr, SPSR_svc); - write_special(ctxt->gp_regs.KVM_ARM_ABT_sp, SP_abt); - write_special(ctxt->gp_regs.KVM_ARM_ABT_lr, LR_abt); - write_special(ctxt->gp_regs.KVM_ARM_ABT_spsr, SPSR_abt); - write_special(ctxt->gp_regs.KVM_ARM_UND_sp, SP_und); - write_special(ctxt->gp_regs.KVM_ARM_UND_lr, LR_und); - write_special(ctxt->gp_regs.KVM_ARM_UND_spsr, SPSR_und); - write_special(ctxt->gp_regs.KVM_ARM_IRQ_sp, SP_irq); - write_special(ctxt->gp_regs.KVM_ARM_IRQ_lr, LR_irq); - write_special(ctxt->gp_regs.KVM_ARM_IRQ_spsr, SPSR_irq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_r8, R8_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_r9, R9_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_r10, R10_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_fp, R11_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_ip, R12_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_sp, SP_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_lr, LR_fiq); - write_special(ctxt->gp_regs.KVM_ARM_FIQ_spsr, SPSR_fiq); -} diff --git a/arch/arm/kvm/hyp/cp15-sr.c b/arch/arm/kvm/hyp/cp15-sr.c deleted file mode 100644 index e6923306f698..000000000000 --- a/arch/arm/kvm/hyp/cp15-sr.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Original code: - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - * - * Mostly rewritten in C by Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <asm/kvm_hyp.h> - -static u64 *cp15_64(struct kvm_cpu_context *ctxt, int idx) -{ - return (u64 *)(ctxt->cp15 + idx); -} - -void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt) -{ - ctxt->cp15[c0_CSSELR] = read_sysreg(CSSELR); - ctxt->cp15[c1_SCTLR] = read_sysreg(SCTLR); - ctxt->cp15[c1_CPACR] = read_sysreg(CPACR); - *cp15_64(ctxt, c2_TTBR0) = read_sysreg(TTBR0); - *cp15_64(ctxt, c2_TTBR1) = read_sysreg(TTBR1); - ctxt->cp15[c2_TTBCR] = read_sysreg(TTBCR); - ctxt->cp15[c3_DACR] = read_sysreg(DACR); - ctxt->cp15[c5_DFSR] = read_sysreg(DFSR); - ctxt->cp15[c5_IFSR] = read_sysreg(IFSR); - ctxt->cp15[c5_ADFSR] = read_sysreg(ADFSR); - ctxt->cp15[c5_AIFSR] = read_sysreg(AIFSR); - ctxt->cp15[c6_DFAR] = read_sysreg(DFAR); - ctxt->cp15[c6_IFAR] = read_sysreg(IFAR); - *cp15_64(ctxt, c7_PAR) = read_sysreg(PAR); - ctxt->cp15[c10_PRRR] = read_sysreg(PRRR); - ctxt->cp15[c10_NMRR] = read_sysreg(NMRR); - ctxt->cp15[c10_AMAIR0] = read_sysreg(AMAIR0); - ctxt->cp15[c10_AMAIR1] = read_sysreg(AMAIR1); - ctxt->cp15[c12_VBAR] = read_sysreg(VBAR); - ctxt->cp15[c13_CID] = read_sysreg(CID); - ctxt->cp15[c13_TID_URW] = read_sysreg(TID_URW); - ctxt->cp15[c13_TID_URO] = read_sysreg(TID_URO); - ctxt->cp15[c13_TID_PRIV] = read_sysreg(TID_PRIV); - ctxt->cp15[c14_CNTKCTL] = read_sysreg(CNTKCTL); -} - -void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt) -{ - write_sysreg(ctxt->cp15[c0_MPIDR], VMPIDR); - write_sysreg(ctxt->cp15[c0_CSSELR], CSSELR); - write_sysreg(ctxt->cp15[c1_SCTLR], SCTLR); - write_sysreg(ctxt->cp15[c1_CPACR], CPACR); - write_sysreg(*cp15_64(ctxt, c2_TTBR0), TTBR0); - write_sysreg(*cp15_64(ctxt, c2_TTBR1), TTBR1); - write_sysreg(ctxt->cp15[c2_TTBCR], TTBCR); - write_sysreg(ctxt->cp15[c3_DACR], DACR); - write_sysreg(ctxt->cp15[c5_DFSR], DFSR); - write_sysreg(ctxt->cp15[c5_IFSR], IFSR); - write_sysreg(ctxt->cp15[c5_ADFSR], ADFSR); - write_sysreg(ctxt->cp15[c5_AIFSR], AIFSR); - write_sysreg(ctxt->cp15[c6_DFAR], DFAR); - write_sysreg(ctxt->cp15[c6_IFAR], IFAR); - write_sysreg(*cp15_64(ctxt, c7_PAR), PAR); - write_sysreg(ctxt->cp15[c10_PRRR], PRRR); - write_sysreg(ctxt->cp15[c10_NMRR], NMRR); - write_sysreg(ctxt->cp15[c10_AMAIR0], AMAIR0); - write_sysreg(ctxt->cp15[c10_AMAIR1], AMAIR1); - write_sysreg(ctxt->cp15[c12_VBAR], VBAR); - write_sysreg(ctxt->cp15[c13_CID], CID); - write_sysreg(ctxt->cp15[c13_TID_URW], TID_URW); - write_sysreg(ctxt->cp15[c13_TID_URO], TID_URO); - write_sysreg(ctxt->cp15[c13_TID_PRIV], TID_PRIV); - write_sysreg(ctxt->cp15[c14_CNTKCTL], CNTKCTL); -} diff --git a/arch/arm/kvm/hyp/entry.S b/arch/arm/kvm/hyp/entry.S deleted file mode 100644 index 4bd1f6a74180..000000000000 --- a/arch/arm/kvm/hyp/entry.S +++ /dev/null @@ -1,121 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2016 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> -*/ - -#include <linux/linkage.h> -#include <asm/asm-offsets.h> -#include <asm/kvm_arm.h> -#include <asm/kvm_asm.h> - - .arch_extension virt - - .text - .pushsection .hyp.text, "ax" - -#define USR_REGS_OFFSET (CPU_CTXT_GP_REGS + GP_REGS_USR) - -/* int __guest_enter(struct kvm_vcpu *vcpu, struct kvm_cpu_context *host) */ -ENTRY(__guest_enter) - @ Save host registers - add r1, r1, #(USR_REGS_OFFSET + S_R4) - stm r1!, {r4-r12} - str lr, [r1, #4] @ Skip SP_usr (already saved) - - @ Restore guest registers - add r0, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R0) - ldr lr, [r0, #S_LR] - ldm r0, {r0-r12} - - clrex - eret -ENDPROC(__guest_enter) - -ENTRY(__guest_exit) - /* - * return convention: - * guest r0, r1, r2 saved on the stack - * r0: vcpu pointer - * r1: exception code - */ - - add r2, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R3) - stm r2!, {r3-r12} - str lr, [r2, #4] - add r2, r0, #(VCPU_GUEST_CTXT + USR_REGS_OFFSET + S_R0) - pop {r3, r4, r5} @ r0, r1, r2 - stm r2, {r3-r5} - - ldr r0, [r0, #VCPU_HOST_CTXT] - add r0, r0, #(USR_REGS_OFFSET + S_R4) - ldm r0!, {r4-r12} - ldr lr, [r0, #4] - - mov r0, r1 - mrs r1, SPSR - mrs r2, ELR_hyp - mrc p15, 4, r3, c5, c2, 0 @ HSR - - /* - * Force loads and stores to complete before unmasking aborts - * and forcing the delivery of the exception. This gives us a - * single instruction window, which the handler will try to - * match. - */ - dsb sy - cpsie a - - .global abort_guest_exit_start -abort_guest_exit_start: - - isb - - .global abort_guest_exit_end -abort_guest_exit_end: - - /* - * If we took an abort, r0[31] will be set, and cmp will set - * the N bit in PSTATE. - */ - cmp r0, #0 - msrmi SPSR_cxsf, r1 - msrmi ELR_hyp, r2 - mcrmi p15, 4, r3, c5, c2, 0 @ HSR - - bx lr -ENDPROC(__guest_exit) - -/* - * If VFPv3 support is not available, then we will not switch the VFP - * registers; however cp10 and cp11 accesses will still trap and fallback - * to the regular coprocessor emulation code, which currently will - * inject an undefined exception to the guest. - */ -#ifdef CONFIG_VFPv3 -ENTRY(__vfp_guest_restore) - push {r3, r4, lr} - - @ NEON/VFP used. Turn on VFP access. - mrc p15, 4, r1, c1, c1, 2 @ HCPTR - bic r1, r1, #(HCPTR_TCP(10) | HCPTR_TCP(11)) - mcr p15, 4, r1, c1, c1, 2 @ HCPTR - isb - - @ Switch VFP/NEON hardware state to the guest's - mov r4, r0 - ldr r0, [r0, #VCPU_HOST_CTXT] - add r0, r0, #CPU_CTXT_VFP - bl __vfp_save_state - add r0, r4, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP) - bl __vfp_restore_state - - pop {r3, r4, lr} - pop {r0, r1, r2} - clrex - eret -ENDPROC(__vfp_guest_restore) -#endif - - .popsection - diff --git a/arch/arm/kvm/hyp/hyp-entry.S b/arch/arm/kvm/hyp/hyp-entry.S deleted file mode 100644 index fe3d7811a908..000000000000 --- a/arch/arm/kvm/hyp/hyp-entry.S +++ /dev/null @@ -1,295 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - */ - -#include <linux/arm-smccc.h> -#include <linux/linkage.h> -#include <asm/kvm_arm.h> -#include <asm/kvm_asm.h> - - .arch_extension virt - - .text - .pushsection .hyp.text, "ax" - -.macro load_vcpu reg - mrc p15, 4, \reg, c13, c0, 2 @ HTPIDR -.endm - -/******************************************************************** - * Hypervisor exception vector and handlers - * - * - * The KVM/ARM Hypervisor ABI is defined as follows: - * - * Entry to Hyp mode from the host kernel will happen _only_ when an HVC - * instruction is issued since all traps are disabled when running the host - * kernel as per the Hyp-mode initialization at boot time. - * - * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc - * below) when the HVC instruction is called from SVC mode (i.e. a guest or the - * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC - * instructions are called from within Hyp-mode. - * - * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode): - * Switching to Hyp mode is done through a simple HVC #0 instruction. The - * exception vector code will check that the HVC comes from VMID==0. - * - r0 contains a pointer to a HYP function - * - r1, r2, and r3 contain arguments to the above function. - * - The HYP function will be called with its arguments in r0, r1 and r2. - * On HYP function return, we return directly to SVC. - * - * Note that the above is used to execute code in Hyp-mode from a host-kernel - * point of view, and is a different concept from performing a world-switch and - * executing guest code SVC mode (with a VMID != 0). - */ - - .align 5 -__kvm_hyp_vector: - .global __kvm_hyp_vector - - @ Hyp-mode exception vector - W(b) hyp_reset - W(b) hyp_undef - W(b) hyp_svc - W(b) hyp_pabt - W(b) hyp_dabt - W(b) hyp_hvc - W(b) hyp_irq - W(b) hyp_fiq - -#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR - .align 5 -__kvm_hyp_vector_ic_inv: - .global __kvm_hyp_vector_ic_inv - - /* - * We encode the exception entry in the bottom 3 bits of - * SP, and we have to guarantee to be 8 bytes aligned. - */ - W(add) sp, sp, #1 /* Reset 7 */ - W(add) sp, sp, #1 /* Undef 6 */ - W(add) sp, sp, #1 /* Syscall 5 */ - W(add) sp, sp, #1 /* Prefetch abort 4 */ - W(add) sp, sp, #1 /* Data abort 3 */ - W(add) sp, sp, #1 /* HVC 2 */ - W(add) sp, sp, #1 /* IRQ 1 */ - W(nop) /* FIQ 0 */ - - mcr p15, 0, r0, c7, c5, 0 /* ICIALLU */ - isb - - b decode_vectors - - .align 5 -__kvm_hyp_vector_bp_inv: - .global __kvm_hyp_vector_bp_inv - - /* - * We encode the exception entry in the bottom 3 bits of - * SP, and we have to guarantee to be 8 bytes aligned. - */ - W(add) sp, sp, #1 /* Reset 7 */ - W(add) sp, sp, #1 /* Undef 6 */ - W(add) sp, sp, #1 /* Syscall 5 */ - W(add) sp, sp, #1 /* Prefetch abort 4 */ - W(add) sp, sp, #1 /* Data abort 3 */ - W(add) sp, sp, #1 /* HVC 2 */ - W(add) sp, sp, #1 /* IRQ 1 */ - W(nop) /* FIQ 0 */ - - mcr p15, 0, r0, c7, c5, 6 /* BPIALL */ - isb - -decode_vectors: - -#ifdef CONFIG_THUMB2_KERNEL - /* - * Yet another silly hack: Use VPIDR as a temp register. - * Thumb2 is really a pain, as SP cannot be used with most - * of the bitwise instructions. The vect_br macro ensures - * things gets cleaned-up. - */ - mcr p15, 4, r0, c0, c0, 0 /* VPIDR */ - mov r0, sp - and r0, r0, #7 - sub sp, sp, r0 - push {r1, r2} - mov r1, r0 - mrc p15, 4, r0, c0, c0, 0 /* VPIDR */ - mrc p15, 0, r2, c0, c0, 0 /* MIDR */ - mcr p15, 4, r2, c0, c0, 0 /* VPIDR */ -#endif - -.macro vect_br val, targ -ARM( eor sp, sp, #\val ) -ARM( tst sp, #7 ) -ARM( eorne sp, sp, #\val ) - -THUMB( cmp r1, #\val ) -THUMB( popeq {r1, r2} ) - - beq \targ -.endm - - vect_br 0, hyp_fiq - vect_br 1, hyp_irq - vect_br 2, hyp_hvc - vect_br 3, hyp_dabt - vect_br 4, hyp_pabt - vect_br 5, hyp_svc - vect_br 6, hyp_undef - vect_br 7, hyp_reset -#endif - -.macro invalid_vector label, cause - .align -\label: mov r0, #\cause - b __hyp_panic -.endm - - invalid_vector hyp_reset ARM_EXCEPTION_RESET - invalid_vector hyp_undef ARM_EXCEPTION_UNDEFINED - invalid_vector hyp_svc ARM_EXCEPTION_SOFTWARE - invalid_vector hyp_pabt ARM_EXCEPTION_PREF_ABORT - invalid_vector hyp_fiq ARM_EXCEPTION_FIQ - -ENTRY(__hyp_do_panic) - mrs lr, cpsr - bic lr, lr, #MODE_MASK - orr lr, lr, #SVC_MODE -THUMB( orr lr, lr, #PSR_T_BIT ) - msr spsr_cxsf, lr - ldr lr, =panic - msr ELR_hyp, lr - ldr lr, =__kvm_call_hyp - clrex - eret -ENDPROC(__hyp_do_panic) - -hyp_hvc: - /* - * Getting here is either because of a trap from a guest, - * or from executing HVC from the host kernel, which means - * "do something in Hyp mode". - */ - push {r0, r1, r2} - - @ Check syndrome register - mrc p15, 4, r1, c5, c2, 0 @ HSR - lsr r0, r1, #HSR_EC_SHIFT - cmp r0, #HSR_EC_HVC - bne guest_trap @ Not HVC instr. - - /* - * Let's check if the HVC came from VMID 0 and allow simple - * switch to Hyp mode - */ - mrrc p15, 6, r0, r2, c2 - lsr r2, r2, #16 - and r2, r2, #0xff - cmp r2, #0 - bne guest_hvc_trap @ Guest called HVC - - /* - * Getting here means host called HVC, we shift parameters and branch - * to Hyp function. - */ - pop {r0, r1, r2} - - /* - * Check if we have a kernel function, which is guaranteed to be - * bigger than the maximum hyp stub hypercall - */ - cmp r0, #HVC_STUB_HCALL_NR - bhs 1f - - /* - * Not a kernel function, treat it as a stub hypercall. - * Compute the physical address for __kvm_handle_stub_hvc - * (as the code lives in the idmaped page) and branch there. - * We hijack ip (r12) as a tmp register. - */ - push {r1} - ldr r1, =kimage_voffset - ldr r1, [r1] - ldr ip, =__kvm_handle_stub_hvc - sub ip, ip, r1 - pop {r1} - - bx ip - -1: - /* - * Pushing r2 here is just a way of keeping the stack aligned to - * 8 bytes on any path that can trigger a HYP exception. Here, - * we may well be about to jump into the guest, and the guest - * exit would otherwise be badly decoded by our fancy - * "decode-exception-without-a-branch" code... - */ - push {r2, lr} - - mov lr, r0 - mov r0, r1 - mov r1, r2 - mov r2, r3 - -THUMB( orr lr, #1) - blx lr @ Call the HYP function - - pop {r2, lr} - eret - -guest_hvc_trap: - movw r2, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1 - movt r2, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1 - ldr r0, [sp] @ Guest's r0 - teq r0, r2 - bne guest_trap - add sp, sp, #12 - @ Returns: - @ r0 = 0 - @ r1 = HSR value (perfectly predictable) - @ r2 = ARM_SMCCC_ARCH_WORKAROUND_1 - mov r0, #0 - eret - -guest_trap: - load_vcpu r0 @ Load VCPU pointer to r0 - -#ifdef CONFIG_VFPv3 - @ Check for a VFP access - lsr r1, r1, #HSR_EC_SHIFT - cmp r1, #HSR_EC_CP_0_13 - beq __vfp_guest_restore -#endif - - mov r1, #ARM_EXCEPTION_HVC - b __guest_exit - -hyp_irq: - push {r0, r1, r2} - mov r1, #ARM_EXCEPTION_IRQ - load_vcpu r0 @ Load VCPU pointer to r0 - b __guest_exit - -hyp_dabt: - push {r0, r1} - mrs r0, ELR_hyp - ldr r1, =abort_guest_exit_start -THUMB( add r1, r1, #1) - cmp r0, r1 - ldrne r1, =abort_guest_exit_end -THUMB( addne r1, r1, #1) - cmpne r0, r1 - pop {r0, r1} - bne __hyp_panic - - orr r0, r0, #(1 << ARM_EXIT_WITH_ABORT_BIT) - eret - - .ltorg - - .popsection diff --git a/arch/arm/kvm/hyp/s2-setup.c b/arch/arm/kvm/hyp/s2-setup.c deleted file mode 100644 index 5dfbea5adf65..000000000000 --- a/arch/arm/kvm/hyp/s2-setup.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2016 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/types.h> -#include <asm/kvm_arm.h> -#include <asm/kvm_asm.h> -#include <asm/kvm_hyp.h> - -void __hyp_text __init_stage2_translation(void) -{ - u64 val; - - val = read_sysreg(VTCR) & ~VTCR_MASK; - - val |= read_sysreg(HTCR) & VTCR_HTCR_SH; - val |= KVM_VTCR_SL0 | KVM_VTCR_T0SZ | KVM_VTCR_S; - - write_sysreg(val, VTCR); -} diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c deleted file mode 100644 index 1efeef3fd0ee..000000000000 --- a/arch/arm/kvm/hyp/switch.c +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2015 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ -#include <linux/jump_label.h> - -#include <asm/kvm_asm.h> -#include <asm/kvm_hyp.h> -#include <asm/kvm_mmu.h> - -__asm__(".arch_extension virt"); - -/* - * Activate the traps, saving the host's fpexc register before - * overwriting it. We'll restore it on VM exit. - */ -static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu, u32 *fpexc_host) -{ - u32 val; - - /* - * We are about to set HCPTR.TCP10/11 to trap all floating point - * register accesses to HYP, however, the ARM ARM clearly states that - * traps are only taken to HYP if the operation would not otherwise - * trap to SVC. Therefore, always make sure that for 32-bit guests, - * we set FPEXC.EN to prevent traps to SVC, when setting the TCP bits. - */ - val = read_sysreg(VFP_FPEXC); - *fpexc_host = val; - if (!(val & FPEXC_EN)) { - write_sysreg(val | FPEXC_EN, VFP_FPEXC); - isb(); - } - - write_sysreg(vcpu->arch.hcr, HCR); - /* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */ - write_sysreg(HSTR_T(15), HSTR); - write_sysreg(HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11), HCPTR); - val = read_sysreg(HDCR); - val |= HDCR_TPM | HDCR_TPMCR; /* trap performance monitors */ - val |= HDCR_TDRA | HDCR_TDOSA | HDCR_TDA; /* trap debug regs */ - write_sysreg(val, HDCR); -} - -static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu) -{ - u32 val; - - /* - * If we pended a virtual abort, preserve it until it gets - * cleared. See B1.9.9 (Virtual Abort exception) for details, - * but the crucial bit is the zeroing of HCR.VA in the - * pseudocode. - */ - if (vcpu->arch.hcr & HCR_VA) - vcpu->arch.hcr = read_sysreg(HCR); - - write_sysreg(0, HCR); - write_sysreg(0, HSTR); - val = read_sysreg(HDCR); - write_sysreg(val & ~(HDCR_TPM | HDCR_TPMCR), HDCR); - write_sysreg(0, HCPTR); -} - -static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu) -{ - struct kvm *kvm = kern_hyp_va(vcpu->kvm); - write_sysreg(kvm_get_vttbr(kvm), VTTBR); - write_sysreg(vcpu->arch.midr, VPIDR); -} - -static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu) -{ - write_sysreg(0, VTTBR); - write_sysreg(read_sysreg(MIDR), VPIDR); -} - - -static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu) -{ - if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { - __vgic_v3_save_state(vcpu); - __vgic_v3_deactivate_traps(vcpu); - } -} - -static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu) -{ - if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { - __vgic_v3_activate_traps(vcpu); - __vgic_v3_restore_state(vcpu); - } -} - -static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) -{ - u32 hsr = read_sysreg(HSR); - u8 ec = hsr >> HSR_EC_SHIFT; - u32 hpfar, far; - - vcpu->arch.fault.hsr = hsr; - - if (ec == HSR_EC_IABT) - far = read_sysreg(HIFAR); - else if (ec == HSR_EC_DABT) - far = read_sysreg(HDFAR); - else - return true; - - /* - * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode: - * - * Abort on the stage 2 translation for a memory access from a - * Non-secure PL1 or PL0 mode: - * - * For any Access flag fault or Translation fault, and also for any - * Permission fault on the stage 2 translation of a memory access - * made as part of a translation table walk for a stage 1 translation, - * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR - * is UNKNOWN. - */ - if (!(hsr & HSR_DABT_S1PTW) && (hsr & HSR_FSC_TYPE) == FSC_PERM) { - u64 par, tmp; - - par = read_sysreg(PAR); - write_sysreg(far, ATS1CPR); - isb(); - - tmp = read_sysreg(PAR); - write_sysreg(par, PAR); - - if (unlikely(tmp & 1)) - return false; /* Translation failed, back to guest */ - - hpfar = ((tmp >> 12) & ((1UL << 28) - 1)) << 4; - } else { - hpfar = read_sysreg(HPFAR); - } - - vcpu->arch.fault.hxfar = far; - vcpu->arch.fault.hpfar = hpfar; - return true; -} - -int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu) -{ - struct kvm_cpu_context *host_ctxt; - struct kvm_cpu_context *guest_ctxt; - bool fp_enabled; - u64 exit_code; - u32 fpexc; - - vcpu = kern_hyp_va(vcpu); - write_sysreg(vcpu, HTPIDR); - - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); - guest_ctxt = &vcpu->arch.ctxt; - - __sysreg_save_state(host_ctxt); - __banked_save_state(host_ctxt); - - __activate_traps(vcpu, &fpexc); - __activate_vm(vcpu); - - __vgic_restore_state(vcpu); - __timer_enable_traps(vcpu); - - __sysreg_restore_state(guest_ctxt); - __banked_restore_state(guest_ctxt); - - /* Jump in the fire! */ -again: - exit_code = __guest_enter(vcpu, host_ctxt); - /* And we're baaack! */ - - if (exit_code == ARM_EXCEPTION_HVC && !__populate_fault_info(vcpu)) - goto again; - - fp_enabled = __vfp_enabled(); - - __banked_save_state(guest_ctxt); - __sysreg_save_state(guest_ctxt); - __timer_disable_traps(vcpu); - - __vgic_save_state(vcpu); - - __deactivate_traps(vcpu); - __deactivate_vm(vcpu); - - __banked_restore_state(host_ctxt); - __sysreg_restore_state(host_ctxt); - - if (fp_enabled) { - __vfp_save_state(&guest_ctxt->vfp); - __vfp_restore_state(&host_ctxt->vfp); - } - - write_sysreg(fpexc, VFP_FPEXC); - - return exit_code; -} - -static const char * const __hyp_panic_string[] = { - [ARM_EXCEPTION_RESET] = "\nHYP panic: RST PC:%08x CPSR:%08x", - [ARM_EXCEPTION_UNDEFINED] = "\nHYP panic: UNDEF PC:%08x CPSR:%08x", - [ARM_EXCEPTION_SOFTWARE] = "\nHYP panic: SVC PC:%08x CPSR:%08x", - [ARM_EXCEPTION_PREF_ABORT] = "\nHYP panic: PABRT PC:%08x CPSR:%08x", - [ARM_EXCEPTION_DATA_ABORT] = "\nHYP panic: DABRT PC:%08x ADDR:%08x", - [ARM_EXCEPTION_IRQ] = "\nHYP panic: IRQ PC:%08x CPSR:%08x", - [ARM_EXCEPTION_FIQ] = "\nHYP panic: FIQ PC:%08x CPSR:%08x", - [ARM_EXCEPTION_HVC] = "\nHYP panic: HVC PC:%08x CPSR:%08x", -}; - -void __hyp_text __noreturn __hyp_panic(int cause) -{ - u32 elr = read_special(ELR_hyp); - u32 val; - - if (cause == ARM_EXCEPTION_DATA_ABORT) - val = read_sysreg(HDFAR); - else - val = read_special(SPSR); - - if (read_sysreg(VTTBR)) { - struct kvm_vcpu *vcpu; - struct kvm_cpu_context *host_ctxt; - - vcpu = (struct kvm_vcpu *)read_sysreg(HTPIDR); - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); - __timer_disable_traps(vcpu); - __deactivate_traps(vcpu); - __deactivate_vm(vcpu); - __banked_restore_state(host_ctxt); - __sysreg_restore_state(host_ctxt); - } - - /* Call panic for real */ - __hyp_do_panic(__hyp_panic_string[cause], elr, val); - - unreachable(); -} diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c deleted file mode 100644 index 848f27bbad9d..000000000000 --- a/arch/arm/kvm/hyp/tlb.c +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Original code: - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - * - * Mostly rewritten in C by Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <asm/kvm_hyp.h> -#include <asm/kvm_mmu.h> - -/** - * Flush per-VMID TLBs - * - * __kvm_tlb_flush_vmid(struct kvm *kvm); - * - * We rely on the hardware to broadcast the TLB invalidation to all CPUs - * inside the inner-shareable domain (which is the case for all v7 - * implementations). If we come across a non-IS SMP implementation, we'll - * have to use an IPI based mechanism. Until then, we stick to the simple - * hardware assisted version. - * - * As v7 does not support flushing per IPA, just nuke the whole TLB - * instead, ignoring the ipa value. - */ -void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm) -{ - dsb(ishst); - - /* Switch to requested VMID */ - kvm = kern_hyp_va(kvm); - write_sysreg(kvm_get_vttbr(kvm), VTTBR); - isb(); - - write_sysreg(0, TLBIALLIS); - dsb(ish); - isb(); - - write_sysreg(0, VTTBR); -} - -void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa) -{ - __kvm_tlb_flush_vmid(kvm); -} - -void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu) -{ - struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm); - - /* Switch to requested VMID */ - write_sysreg(kvm_get_vttbr(kvm), VTTBR); - isb(); - - write_sysreg(0, TLBIALL); - dsb(nsh); - isb(); - - write_sysreg(0, VTTBR); -} - -void __hyp_text __kvm_flush_vm_context(void) -{ - write_sysreg(0, TLBIALLNSNHIS); - write_sysreg(0, ICIALLUIS); - dsb(ish); -} diff --git a/arch/arm/kvm/hyp/vfp.S b/arch/arm/kvm/hyp/vfp.S deleted file mode 100644 index 675a52348d8d..000000000000 --- a/arch/arm/kvm/hyp/vfp.S +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - */ - -#include <linux/linkage.h> -#include <asm/vfpmacros.h> - - .text - .pushsection .hyp.text, "ax" - -/* void __vfp_save_state(struct vfp_hard_struct *vfp); */ -ENTRY(__vfp_save_state) - push {r4, r5} - VFPFMRX r1, FPEXC - - @ Make sure *really* VFP is enabled so we can touch the registers. - orr r5, r1, #FPEXC_EN - tst r5, #FPEXC_EX @ Check for VFP Subarchitecture - bic r5, r5, #FPEXC_EX @ FPEXC_EX disable - VFPFMXR FPEXC, r5 - isb - - VFPFMRX r2, FPSCR - beq 1f - - @ If FPEXC_EX is 0, then FPINST/FPINST2 reads are upredictable, so - @ we only need to save them if FPEXC_EX is set. - VFPFMRX r3, FPINST - tst r5, #FPEXC_FP2V - VFPFMRX r4, FPINST2, ne @ vmrsne -1: - VFPFSTMIA r0, r5 @ Save VFP registers - stm r0, {r1-r4} @ Save FPEXC, FPSCR, FPINST, FPINST2 - pop {r4, r5} - bx lr -ENDPROC(__vfp_save_state) - -/* void __vfp_restore_state(struct vfp_hard_struct *vfp); - * Assume FPEXC_EN is on and FPEXC_EX is off */ -ENTRY(__vfp_restore_state) - VFPFLDMIA r0, r1 @ Load VFP registers - ldm r0, {r0-r3} @ Load FPEXC, FPSCR, FPINST, FPINST2 - - VFPFMXR FPSCR, r1 - tst r0, #FPEXC_EX @ Check for VFP Subarchitecture - beq 1f - VFPFMXR FPINST, r2 - tst r0, #FPEXC_FP2V - VFPFMXR FPINST2, r3, ne -1: - VFPFMXR FPEXC, r0 @ FPEXC (last, in case !EN) - bx lr -ENDPROC(__vfp_restore_state) - - .popsection |