diff options
Diffstat (limited to 'virt/kvm/arm/hyp')
| -rw-r--r-- | virt/kvm/arm/hyp/aarch32.c | 140 | ||||
| -rw-r--r-- | virt/kvm/arm/hyp/timer-sr.c | 49 | ||||
| -rw-r--r-- | virt/kvm/arm/hyp/vgic-v3-sr.c | 1130 |
3 files changed, 0 insertions, 1319 deletions
diff --git a/virt/kvm/arm/hyp/aarch32.c b/virt/kvm/arm/hyp/aarch32.c deleted file mode 100644 index 25c0e47d57cb..000000000000 --- a/virt/kvm/arm/hyp/aarch32.c +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Hyp portion of the (not much of an) Emulation layer for 32bit guests. - * - * Copyright (C) 2012,2013 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> - * - * based on arch/arm/kvm/emulate.c - * Copyright (C) 2012 - Virtual Open Systems and Columbia University - * Author: Christoffer Dall <c.dall@virtualopensystems.com> - */ - -#include <linux/kvm_host.h> -#include <asm/kvm_emulate.h> -#include <asm/kvm_hyp.h> - -/* - * stolen from arch/arm/kernel/opcodes.c - * - * condition code lookup table - * index into the table is test code: EQ, NE, ... LT, GT, AL, NV - * - * bit position in short is condition code: NZCV - */ -static const unsigned short cc_map[16] = { - 0xF0F0, /* EQ == Z set */ - 0x0F0F, /* NE */ - 0xCCCC, /* CS == C set */ - 0x3333, /* CC */ - 0xFF00, /* MI == N set */ - 0x00FF, /* PL */ - 0xAAAA, /* VS == V set */ - 0x5555, /* VC */ - 0x0C0C, /* HI == C set && Z clear */ - 0xF3F3, /* LS == C clear || Z set */ - 0xAA55, /* GE == (N==V) */ - 0x55AA, /* LT == (N!=V) */ - 0x0A05, /* GT == (!Z && (N==V)) */ - 0xF5FA, /* LE == (Z || (N!=V)) */ - 0xFFFF, /* AL always */ - 0 /* NV */ -}; - -/* - * Check if a trapped instruction should have been executed or not. - */ -bool __hyp_text kvm_condition_valid32(const struct kvm_vcpu *vcpu) -{ - unsigned long cpsr; - u32 cpsr_cond; - int cond; - - /* Top two bits non-zero? Unconditional. */ - if (kvm_vcpu_get_hsr(vcpu) >> 30) - return true; - - /* Is condition field valid? */ - cond = kvm_vcpu_get_condition(vcpu); - if (cond == 0xE) - return true; - - cpsr = *vcpu_cpsr(vcpu); - - if (cond < 0) { - /* This can happen in Thumb mode: examine IT state. */ - unsigned long it; - - it = ((cpsr >> 8) & 0xFC) | ((cpsr >> 25) & 0x3); - - /* it == 0 => unconditional. */ - if (it == 0) - return true; - - /* The cond for this insn works out as the top 4 bits. */ - cond = (it >> 4); - } - - cpsr_cond = cpsr >> 28; - - if (!((cc_map[cond] >> cpsr_cond) & 1)) - return false; - - return true; -} - -/** - * adjust_itstate - adjust ITSTATE when emulating instructions in IT-block - * @vcpu: The VCPU pointer - * - * When exceptions occur while instructions are executed in Thumb IF-THEN - * blocks, the ITSTATE field of the CPSR is not advanced (updated), so we have - * to do this little bit of work manually. The fields map like this: - * - * IT[7:0] -> CPSR[26:25],CPSR[15:10] - */ -static void __hyp_text kvm_adjust_itstate(struct kvm_vcpu *vcpu) -{ - unsigned long itbits, cond; - unsigned long cpsr = *vcpu_cpsr(vcpu); - bool is_arm = !(cpsr & PSR_AA32_T_BIT); - - if (is_arm || !(cpsr & PSR_AA32_IT_MASK)) - return; - - cond = (cpsr & 0xe000) >> 13; - itbits = (cpsr & 0x1c00) >> (10 - 2); - itbits |= (cpsr & (0x3 << 25)) >> 25; - - /* Perform ITAdvance (see page A2-52 in ARM DDI 0406C) */ - if ((itbits & 0x7) == 0) - itbits = cond = 0; - else - itbits = (itbits << 1) & 0x1f; - - cpsr &= ~PSR_AA32_IT_MASK; - cpsr |= cond << 13; - cpsr |= (itbits & 0x1c) << (10 - 2); - cpsr |= (itbits & 0x3) << 25; - *vcpu_cpsr(vcpu) = cpsr; -} - -/** - * kvm_skip_instr - skip a trapped instruction and proceed to the next - * @vcpu: The vcpu pointer - */ -void __hyp_text kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr) -{ - u32 pc = *vcpu_pc(vcpu); - bool is_thumb; - - is_thumb = !!(*vcpu_cpsr(vcpu) & PSR_AA32_T_BIT); - if (is_thumb && !is_wide_instr) - pc += 2; - else - pc += 4; - - *vcpu_pc(vcpu) = pc; - - kvm_adjust_itstate(vcpu); -} diff --git a/virt/kvm/arm/hyp/timer-sr.c b/virt/kvm/arm/hyp/timer-sr.c deleted file mode 100644 index ff76e6845fe4..000000000000 --- a/virt/kvm/arm/hyp/timer-sr.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012-2015 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <clocksource/arm_arch_timer.h> -#include <linux/compiler.h> -#include <linux/kvm_host.h> - -#include <asm/kvm_hyp.h> - -void __hyp_text __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high) -{ - u64 cntvoff = (u64)cntvoff_high << 32 | cntvoff_low; - write_sysreg(cntvoff, cntvoff_el2); -} - -/* - * Should only be called on non-VHE systems. - * VHE systems use EL2 timers and configure EL1 timers in kvm_timer_init_vhe(). - */ -void __hyp_text __timer_disable_traps(struct kvm_vcpu *vcpu) -{ - u64 val; - - /* Allow physical timer/counter access for the host */ - val = read_sysreg(cnthctl_el2); - val |= CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN; - write_sysreg(val, cnthctl_el2); -} - -/* - * Should only be called on non-VHE systems. - * VHE systems use EL2 timers and configure EL1 timers in kvm_timer_init_vhe(). - */ -void __hyp_text __timer_enable_traps(struct kvm_vcpu *vcpu) -{ - u64 val; - - /* - * Disallow physical timer access for the guest - * Physical counter access is allowed - */ - val = read_sysreg(cnthctl_el2); - val &= ~CNTHCTL_EL1PCEN; - val |= CNTHCTL_EL1PCTEN; - write_sysreg(val, cnthctl_el2); -} diff --git a/virt/kvm/arm/hyp/vgic-v3-sr.c b/virt/kvm/arm/hyp/vgic-v3-sr.c deleted file mode 100644 index ccf1fde9836c..000000000000 --- a/virt/kvm/arm/hyp/vgic-v3-sr.c +++ /dev/null @@ -1,1130 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2012-2015 - ARM Ltd - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/compiler.h> -#include <linux/irqchip/arm-gic-v3.h> -#include <linux/kvm_host.h> - -#include <asm/kvm_emulate.h> -#include <asm/kvm_hyp.h> -#include <asm/kvm_mmu.h> - -#define vtr_to_max_lr_idx(v) ((v) & 0xf) -#define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) -#define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) - -static u64 __hyp_text __gic_v3_get_lr(unsigned int lr) -{ - switch (lr & 0xf) { - case 0: - return read_gicreg(ICH_LR0_EL2); - case 1: - return read_gicreg(ICH_LR1_EL2); - case 2: - return read_gicreg(ICH_LR2_EL2); - case 3: - return read_gicreg(ICH_LR3_EL2); - case 4: - return read_gicreg(ICH_LR4_EL2); - case 5: - return read_gicreg(ICH_LR5_EL2); - case 6: - return read_gicreg(ICH_LR6_EL2); - case 7: - return read_gicreg(ICH_LR7_EL2); - case 8: - return read_gicreg(ICH_LR8_EL2); - case 9: - return read_gicreg(ICH_LR9_EL2); - case 10: - return read_gicreg(ICH_LR10_EL2); - case 11: - return read_gicreg(ICH_LR11_EL2); - case 12: - return read_gicreg(ICH_LR12_EL2); - case 13: - return read_gicreg(ICH_LR13_EL2); - case 14: - return read_gicreg(ICH_LR14_EL2); - case 15: - return read_gicreg(ICH_LR15_EL2); - } - - unreachable(); -} - -static void __hyp_text __gic_v3_set_lr(u64 val, int lr) -{ - switch (lr & 0xf) { - case 0: - write_gicreg(val, ICH_LR0_EL2); - break; - case 1: - write_gicreg(val, ICH_LR1_EL2); - break; - case 2: - write_gicreg(val, ICH_LR2_EL2); - break; - case 3: - write_gicreg(val, ICH_LR3_EL2); - break; - case 4: - write_gicreg(val, ICH_LR4_EL2); - break; - case 5: - write_gicreg(val, ICH_LR5_EL2); - break; - case 6: - write_gicreg(val, ICH_LR6_EL2); - break; - case 7: - write_gicreg(val, ICH_LR7_EL2); - break; - case 8: - write_gicreg(val, ICH_LR8_EL2); - break; - case 9: - write_gicreg(val, ICH_LR9_EL2); - break; - case 10: - write_gicreg(val, ICH_LR10_EL2); - break; - case 11: - write_gicreg(val, ICH_LR11_EL2); - break; - case 12: - write_gicreg(val, ICH_LR12_EL2); - break; - case 13: - write_gicreg(val, ICH_LR13_EL2); - break; - case 14: - write_gicreg(val, ICH_LR14_EL2); - break; - case 15: - write_gicreg(val, ICH_LR15_EL2); - break; - } -} - -static void __hyp_text __vgic_v3_write_ap0rn(u32 val, int n) -{ - switch (n) { - case 0: - write_gicreg(val, ICH_AP0R0_EL2); - break; - case 1: - write_gicreg(val, ICH_AP0R1_EL2); - break; - case 2: - write_gicreg(val, ICH_AP0R2_EL2); - break; - case 3: - write_gicreg(val, ICH_AP0R3_EL2); - break; - } -} - -static void __hyp_text __vgic_v3_write_ap1rn(u32 val, int n) -{ - switch (n) { - case 0: - write_gicreg(val, ICH_AP1R0_EL2); - break; - case 1: - write_gicreg(val, ICH_AP1R1_EL2); - break; - case 2: - write_gicreg(val, ICH_AP1R2_EL2); - break; - case 3: - write_gicreg(val, ICH_AP1R3_EL2); - break; - } -} - -static u32 __hyp_text __vgic_v3_read_ap0rn(int n) -{ - u32 val; - - switch (n) { - case 0: - val = read_gicreg(ICH_AP0R0_EL2); - break; - case 1: - val = read_gicreg(ICH_AP0R1_EL2); - break; - case 2: - val = read_gicreg(ICH_AP0R2_EL2); - break; - case 3: - val = read_gicreg(ICH_AP0R3_EL2); - break; - default: - unreachable(); - } - - return val; -} - -static u32 __hyp_text __vgic_v3_read_ap1rn(int n) -{ - u32 val; - - switch (n) { - case 0: - val = read_gicreg(ICH_AP1R0_EL2); - break; - case 1: - val = read_gicreg(ICH_AP1R1_EL2); - break; - case 2: - val = read_gicreg(ICH_AP1R2_EL2); - break; - case 3: - val = read_gicreg(ICH_AP1R3_EL2); - break; - default: - unreachable(); - } - - return val; -} - -void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu) -{ - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; - - /* - * Make sure stores to the GIC via the memory mapped interface - * are now visible to the system register interface when reading the - * LRs, and when reading back the VMCR on non-VHE systems. - */ - if (used_lrs || !has_vhe()) { - if (!cpu_if->vgic_sre) { - dsb(sy); - isb(); - } - } - - if (used_lrs || cpu_if->its_vpe.its_vm) { - int i; - u32 elrsr; - - elrsr = read_gicreg(ICH_ELRSR_EL2); - - write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2); - - for (i = 0; i < used_lrs; i++) { - if (elrsr & (1 << i)) - cpu_if->vgic_lr[i] &= ~ICH_LR_STATE; - else - cpu_if->vgic_lr[i] = __gic_v3_get_lr(i); - - __gic_v3_set_lr(0, i); - } - } -} - -void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu) -{ - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; - int i; - - if (used_lrs || cpu_if->its_vpe.its_vm) { - write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); - - for (i = 0; i < used_lrs; i++) - __gic_v3_set_lr(cpu_if->vgic_lr[i], i); - } - - /* - * Ensure that writes to the LRs, and on non-VHE systems ensure that - * the write to the VMCR in __vgic_v3_activate_traps(), will have - * reached the (re)distributors. This ensure the guest will read the - * correct values from the memory-mapped interface. - */ - if (used_lrs || !has_vhe()) { - if (!cpu_if->vgic_sre) { - isb(); - dsb(sy); - } - } -} - -void __hyp_text __vgic_v3_activate_traps(struct kvm_vcpu *vcpu) -{ - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - - /* - * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a - * Group0 interrupt (as generated in GICv2 mode) to be - * delivered as a FIQ to the guest, with potentially fatal - * consequences. So we must make sure that ICC_SRE_EL1 has - * been actually programmed with the value we want before - * starting to mess with the rest of the GIC, and VMCR_EL2 in - * particular. This logic must be called before - * __vgic_v3_restore_state(). - */ - if (!cpu_if->vgic_sre) { - write_gicreg(0, ICC_SRE_EL1); - isb(); - write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2); - - - if (has_vhe()) { - /* - * Ensure that the write to the VMCR will have reached - * the (re)distributors. This ensure the guest will - * read the correct values from the memory-mapped - * interface. - */ - isb(); - dsb(sy); - } - } - - /* - * Prevent the guest from touching the GIC system registers if - * SRE isn't enabled for GICv3 emulation. - */ - write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE, - ICC_SRE_EL2); - - /* - * If we need to trap system registers, we must write - * ICH_HCR_EL2 anyway, even if no interrupts are being - * injected, - */ - if (static_branch_unlikely(&vgic_v3_cpuif_trap) || - cpu_if->its_vpe.its_vm) - write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); -} - -void __hyp_text __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu) -{ - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - u64 val; - - if (!cpu_if->vgic_sre) { - cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); - } - - val = read_gicreg(ICC_SRE_EL2); - write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); - - if (!cpu_if->vgic_sre) { - /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ - isb(); - write_gicreg(1, ICC_SRE_EL1); - } - - /* - * If we were trapping system registers, we enabled the VGIC even if - * no interrupts were being injected, and we disable it again here. - */ - if (static_branch_unlikely(&vgic_v3_cpuif_trap) || - cpu_if->its_vpe.its_vm) - write_gicreg(0, ICH_HCR_EL2); -} - -void __hyp_text __vgic_v3_save_aprs(struct kvm_vcpu *vcpu) -{ - struct vgic_v3_cpu_if *cpu_if; - u64 val; - u32 nr_pre_bits; - - vcpu = kern_hyp_va(vcpu); - cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - - val = read_gicreg(ICH_VTR_EL2); - nr_pre_bits = vtr_to_nr_pre_bits(val); - - switch (nr_pre_bits) { - case 7: - cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3); - cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2); - /* Fall through */ - case 6: - cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1); - /* Fall through */ - default: - cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0); - } - - switch (nr_pre_bits) { - case 7: - cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3); - cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2); - /* Fall through */ - case 6: - cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1); - /* Fall through */ - default: - cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0); - } -} - -void __hyp_text __vgic_v3_restore_aprs(struct kvm_vcpu *vcpu) -{ - struct vgic_v3_cpu_if *cpu_if; - u64 val; - u32 nr_pre_bits; - - vcpu = kern_hyp_va(vcpu); - cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - - val = read_gicreg(ICH_VTR_EL2); - nr_pre_bits = vtr_to_nr_pre_bits(val); - - switch (nr_pre_bits) { - case 7: - __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3); - __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2); - /* Fall through */ - case 6: - __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1); - /* Fall through */ - default: - __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0); - } - - switch (nr_pre_bits) { - case 7: - __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3); - __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2); - /* Fall through */ - case 6: - __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1); - /* Fall through */ - default: - __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0); - } -} - -void __hyp_text __vgic_v3_init_lrs(void) -{ - int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); - int i; - - for (i = 0; i <= max_lr_idx; i++) - __gic_v3_set_lr(0, i); -} - -u64 __hyp_text __vgic_v3_get_ich_vtr_el2(void) -{ - return read_gicreg(ICH_VTR_EL2); -} - -u64 __hyp_text __vgic_v3_read_vmcr(void) -{ - return read_gicreg(ICH_VMCR_EL2); -} - -void __hyp_text __vgic_v3_write_vmcr(u32 vmcr) -{ - write_gicreg(vmcr, ICH_VMCR_EL2); -} - -#ifdef CONFIG_ARM64 - -static int __hyp_text __vgic_v3_bpr_min(void) -{ - /* See Pseudocode for VPriorityGroup */ - return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); -} - -static int __hyp_text __vgic_v3_get_group(struct kvm_vcpu *vcpu) -{ - u32 esr = kvm_vcpu_get_hsr(vcpu); - u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; - - return crm != 8; -} - -#define GICv3_IDLE_PRIORITY 0xff - -static int __hyp_text __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, - u32 vmcr, - u64 *lr_val) -{ - unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs; - u8 priority = GICv3_IDLE_PRIORITY; - int i, lr = -1; - - for (i = 0; i < used_lrs; i++) { - u64 val = __gic_v3_get_lr(i); - u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; - - /* Not pending in the state? */ - if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT) - continue; - - /* Group-0 interrupt, but Group-0 disabled? */ - if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK)) - continue; - - /* Group-1 interrupt, but Group-1 disabled? */ - if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK)) - continue; - - /* Not the highest priority? */ - if (lr_prio >= priority) - continue; - - /* This is a candidate */ - priority = lr_prio; - *lr_val = val; - lr = i; - } - - if (lr == -1) - *lr_val = ICC_IAR1_EL1_SPURIOUS; - - return lr; -} - -static int __hyp_text __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, - int intid, u64 *lr_val) -{ - unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs; - int i; - - for (i = 0; i < used_lrs; i++) { - u64 val = __gic_v3_get_lr(i); - - if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid && - (val & ICH_LR_ACTIVE_BIT)) { - *lr_val = val; - return i; - } - } - - *lr_val = ICC_IAR1_EL1_SPURIOUS; - return -1; -} - -static int __hyp_text __vgic_v3_get_highest_active_priority(void) -{ - u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); - u32 hap = 0; - int i; - - for (i = 0; i < nr_apr_regs; i++) { - u32 val; - - /* - * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers - * contain the active priority levels for this VCPU - * for the maximum number of supported priority - * levels, and we return the full priority level only - * if the BPR is programmed to its minimum, otherwise - * we return a combination of the priority level and - * subpriority, as determined by the setting of the - * BPR, but without the full subpriority. - */ - val = __vgic_v3_read_ap0rn(i); - val |= __vgic_v3_read_ap1rn(i); - if (!val) { - hap += 32; - continue; - } - - return (hap + __ffs(val)) << __vgic_v3_bpr_min(); - } - - return GICv3_IDLE_PRIORITY; -} - -static unsigned int __hyp_text __vgic_v3_get_bpr0(u32 vmcr) -{ - return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; -} - -static unsigned int __hyp_text __vgic_v3_get_bpr1(u32 vmcr) -{ - unsigned int bpr; - - if (vmcr & ICH_VMCR_CBPR_MASK) { - bpr = __vgic_v3_get_bpr0(vmcr); - if (bpr < 7) - bpr++; - } else { - bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; - } - - return bpr; -} - -/* - * Convert a priority to a preemption level, taking the relevant BPR - * into account by zeroing the sub-priority bits. - */ -static u8 __hyp_text __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) -{ - unsigned int bpr; - - if (!grp) - bpr = __vgic_v3_get_bpr0(vmcr) + 1; - else - bpr = __vgic_v3_get_bpr1(vmcr); - - return pri & (GENMASK(7, 0) << bpr); -} - -/* - * The priority value is independent of any of the BPR values, so we - * normalize it using the minumal BPR value. This guarantees that no - * matter what the guest does with its BPR, we can always set/get the - * same value of a priority. - */ -static void __hyp_text __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) -{ - u8 pre, ap; - u32 val; - int apr; - - pre = __vgic_v3_pri_to_pre(pri, vmcr, grp); - ap = pre >> __vgic_v3_bpr_min(); - apr = ap / 32; - - if (!grp) { - val = __vgic_v3_read_ap0rn(apr); - __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr); - } else { - val = __vgic_v3_read_ap1rn(apr); - __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr); - } -} - -static int __hyp_text __vgic_v3_clear_highest_active_priority(void) -{ - u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); - u32 hap = 0; - int i; - - for (i = 0; i < nr_apr_regs; i++) { - u32 ap0, ap1; - int c0, c1; - - ap0 = __vgic_v3_read_ap0rn(i); - ap1 = __vgic_v3_read_ap1rn(i); - if (!ap0 && !ap1) { - hap += 32; - continue; - } - - c0 = ap0 ? __ffs(ap0) : 32; - c1 = ap1 ? __ffs(ap1) : 32; - - /* Always clear the LSB, which is the highest priority */ - if (c0 < c1) { - ap0 &= ~BIT(c0); - __vgic_v3_write_ap0rn(ap0, i); - hap += c0; - } else { - ap1 &= ~BIT(c1); - __vgic_v3_write_ap1rn(ap1, i); - hap += c1; - } - - /* Rescale to 8 bits of priority */ - return hap << __vgic_v3_bpr_min(); - } - - return GICv3_IDLE_PRIORITY; -} - -static void __hyp_text __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - u64 lr_val; - u8 lr_prio, pmr; - int lr, grp; - - grp = __vgic_v3_get_group(vcpu); - - lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); - if (lr < 0) - goto spurious; - - if (grp != !!(lr_val & ICH_LR_GROUP)) - goto spurious; - - pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; - lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; - if (pmr <= lr_prio) - goto spurious; - - if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp)) - goto spurious; - - lr_val &= ~ICH_LR_STATE; - /* No active state for LPIs */ - if ((lr_val & ICH_LR_VIRTUAL_ID_MASK) <= VGIC_MAX_SPI) - lr_val |= ICH_LR_ACTIVE_BIT; - __gic_v3_set_lr(lr_val, lr); - __vgic_v3_set_active_priority(lr_prio, vmcr, grp); - vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); - return; - -spurious: - vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS); -} - -static void __hyp_text __vgic_v3_clear_active_lr(int lr, u64 lr_val) -{ - lr_val &= ~ICH_LR_ACTIVE_BIT; - if (lr_val & ICH_LR_HW) { - u32 pid; - - pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT; - gic_write_dir(pid); - } - - __gic_v3_set_lr(lr_val, lr); -} - -static void __hyp_text __vgic_v3_bump_eoicount(void) -{ - u32 hcr; - - hcr = read_gicreg(ICH_HCR_EL2); - hcr += 1 << ICH_HCR_EOIcount_SHIFT; - write_gicreg(hcr, ICH_HCR_EL2); -} - -static void __hyp_text __vgic_v3_write_dir(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - u32 vid = vcpu_get_reg(vcpu, rt); - u64 lr_val; - int lr; - - /* EOImode == 0, nothing to be done here */ - if (!(vmcr & ICH_VMCR_EOIM_MASK)) - return; - - /* No deactivate to be performed on an LPI */ - if (vid >= VGIC_MIN_LPI) - return; - - lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); - if (lr == -1) { - __vgic_v3_bump_eoicount(); - return; - } - - __vgic_v3_clear_active_lr(lr, lr_val); -} - -static void __hyp_text __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - u32 vid = vcpu_get_reg(vcpu, rt); - u64 lr_val; - u8 lr_prio, act_prio; - int lr, grp; - - grp = __vgic_v3_get_group(vcpu); - - /* Drop priority in any case */ - act_prio = __vgic_v3_clear_highest_active_priority(); - - /* If EOIing an LPI, no deactivate to be performed */ - if (vid >= VGIC_MIN_LPI) - return; - - /* EOImode == 1, nothing to be done here */ - if (vmcr & ICH_VMCR_EOIM_MASK) - return; - - lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); - if (lr == -1) { - __vgic_v3_bump_eoicount(); - return; - } - - lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; - - /* If priorities or group do not match, the guest has fscked-up. */ - if (grp != !!(lr_val & ICH_LR_GROUP) || - __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio) - return; - - /* Let's now perform the deactivation */ - __vgic_v3_clear_active_lr(lr, lr_val); -} - -static void __hyp_text __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK)); -} - -static void __hyp_text __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK)); -} - -static void __hyp_text __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - u64 val = vcpu_get_reg(vcpu, rt); - - if (val & 1) - vmcr |= ICH_VMCR_ENG0_MASK; - else - vmcr &= ~ICH_VMCR_ENG0_MASK; - - __vgic_v3_write_vmcr(vmcr); -} - -static void __hyp_text __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - u64 val = vcpu_get_reg(vcpu, rt); - - if (val & 1) - vmcr |= ICH_VMCR_ENG1_MASK; - else - vmcr &= ~ICH_VMCR_ENG1_MASK; - - __vgic_v3_write_vmcr(vmcr); -} - -static void __hyp_text __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr)); -} - -static void __hyp_text __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr)); -} - -static void __hyp_text __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - u64 val = vcpu_get_reg(vcpu, rt); - u8 bpr_min = __vgic_v3_bpr_min() - 1; - - /* Enforce BPR limiting */ - if (val < bpr_min) - val = bpr_min; - - val <<= ICH_VMCR_BPR0_SHIFT; - val &= ICH_VMCR_BPR0_MASK; - vmcr &= ~ICH_VMCR_BPR0_MASK; - vmcr |= val; - - __vgic_v3_write_vmcr(vmcr); -} - -static void __hyp_text __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) -{ - u64 val = vcpu_get_reg(vcpu, rt); - u8 bpr_min = __vgic_v3_bpr_min(); - - if (vmcr & ICH_VMCR_CBPR_MASK) - return; - - /* Enforce BPR limiting */ - if (val < bpr_min) - val = bpr_min; - - val <<= ICH_VMCR_BPR1_SHIFT; - val &= ICH_VMCR_BPR1_MASK; - vmcr &= ~ICH_VMCR_BPR1_MASK; - vmcr |= val; - - __vgic_v3_write_vmcr(vmcr); -} - -static void __hyp_text __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n) -{ - u32 val; - - if (!__vgic_v3_get_group(vcpu)) - val = __vgic_v3_read_ap0rn(n); - else - val = __vgic_v3_read_ap1rn(n); - - vcpu_set_reg(vcpu, rt, val); -} - -static void __hyp_text __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n) -{ - u32 val = vcpu_get_reg(vcpu, rt); - - if (!__vgic_v3_get_group(vcpu)) - __vgic_v3_write_ap0rn(val, n); - else - __vgic_v3_write_ap1rn(val, n); -} - -static void __hyp_text __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_read_apxrn(vcpu, rt, 0); -} - -static void __hyp_text __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_read_apxrn(vcpu, rt, 1); -} - -static void __hyp_text __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_read_apxrn(vcpu, rt, 2); -} - -static void __hyp_text __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_read_apxrn(vcpu, rt, 3); -} - -static void __hyp_text __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_write_apxrn(vcpu, rt, 0); -} - -static void __hyp_text __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_write_apxrn(vcpu, rt, 1); -} - -static void __hyp_text __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_write_apxrn(vcpu, rt, 2); -} - -static void __hyp_text __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - __vgic_v3_write_apxrn(vcpu, rt, 3); -} - -static void __hyp_text __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - u64 lr_val; - int lr, lr_grp, grp; - - grp = __vgic_v3_get_group(vcpu); - - lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); - if (lr == -1) - goto spurious; - - lr_grp = !!(lr_val & ICH_LR_GROUP); - if (lr_grp != grp) - lr_val = ICC_IAR1_EL1_SPURIOUS; - -spurious: - vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); -} - -static void __hyp_text __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - vmcr &= ICH_VMCR_PMR_MASK; - vmcr >>= ICH_VMCR_PMR_SHIFT; - vcpu_set_reg(vcpu, rt, vmcr); -} - -static void __hyp_text __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - u32 val = vcpu_get_reg(vcpu, rt); - - val <<= ICH_VMCR_PMR_SHIFT; - val &= ICH_VMCR_PMR_MASK; - vmcr &= ~ICH_VMCR_PMR_MASK; - vmcr |= val; - - write_gicreg(vmcr, ICH_VMCR_EL2); -} - -static void __hyp_text __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - u32 val = __vgic_v3_get_highest_active_priority(); - vcpu_set_reg(vcpu, rt, val); -} - -static void __hyp_text __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - u32 vtr, val; - - vtr = read_gicreg(ICH_VTR_EL2); - /* PRIbits */ - val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; - /* IDbits */ - val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; - /* SEIS */ - val |= ((vtr >> 22) & 1) << ICC_CTLR_EL1_SEIS_SHIFT; - /* A3V */ - val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; - /* EOImode */ - val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT; - /* CBPR */ - val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT; - - vcpu_set_reg(vcpu, rt, val); -} - -static void __hyp_text __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, - u32 vmcr, int rt) -{ - u32 val = vcpu_get_reg(vcpu, rt); - - if (val & ICC_CTLR_EL1_CBPR_MASK) - vmcr |= ICH_VMCR_CBPR_MASK; - else - vmcr &= ~ICH_VMCR_CBPR_MASK; - - if (val & ICC_CTLR_EL1_EOImode_MASK) - vmcr |= ICH_VMCR_EOIM_MASK; - else - vmcr &= ~ICH_VMCR_EOIM_MASK; - - write_gicreg(vmcr, ICH_VMCR_EL2); -} - -int __hyp_text __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) -{ - int rt; - u32 esr; - u32 vmcr; - void (*fn)(struct kvm_vcpu *, u32, int); - bool is_read; - u32 sysreg; - - esr = kvm_vcpu_get_hsr(vcpu); - if (vcpu_mode_is_32bit(vcpu)) { - if (!kvm_condition_valid(vcpu)) { - __kvm_skip_instr(vcpu); - return 1; - } - - sysreg = esr_cp15_to_sysreg(esr); - } else { - sysreg = esr_sys64_to_sysreg(esr); - } - - is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ; - - switch (sysreg) { - case SYS_ICC_IAR0_EL1: - case SYS_ICC_IAR1_EL1: - if (unlikely(!is_read)) - return 0; - fn = __vgic_v3_read_iar; - break; - case SYS_ICC_EOIR0_EL1: - case SYS_ICC_EOIR1_EL1: - if (unlikely(is_read)) - return 0; - fn = __vgic_v3_write_eoir; - break; - case SYS_ICC_IGRPEN1_EL1: - if (is_read) - fn = __vgic_v3_read_igrpen1; - else - fn = __vgic_v3_write_igrpen1; - break; - case SYS_ICC_BPR1_EL1: - if (is_read) - fn = __vgic_v3_read_bpr1; - else - fn = __vgic_v3_write_bpr1; - break; - case SYS_ICC_AP0Rn_EL1(0): - case SYS_ICC_AP1Rn_EL1(0): - if (is_read) - fn = __vgic_v3_read_apxr0; - else - fn = __vgic_v3_write_apxr0; - break; - case SYS_ICC_AP0Rn_EL1(1): - case SYS_ICC_AP1Rn_EL1(1): - if (is_read) - fn = __vgic_v3_read_apxr1; - else - fn = __vgic_v3_write_apxr1; - break; - case SYS_ICC_AP0Rn_EL1(2): - case SYS_ICC_AP1Rn_EL1(2): - if (is_read) - fn = __vgic_v3_read_apxr2; - else - fn = __vgic_v3_write_apxr2; - break; - case SYS_ICC_AP0Rn_EL1(3): - case SYS_ICC_AP1Rn_EL1(3): - if (is_read) - fn = __vgic_v3_read_apxr3; - else - fn = __vgic_v3_write_apxr3; - break; - case SYS_ICC_HPPIR0_EL1: - case SYS_ICC_HPPIR1_EL1: - if (unlikely(!is_read)) - return 0; - fn = __vgic_v3_read_hppir; - break; - case SYS_ICC_IGRPEN0_EL1: - if (is_read) - fn = __vgic_v3_read_igrpen0; - else - fn = __vgic_v3_write_igrpen0; - break; - case SYS_ICC_BPR0_EL1: - if (is_read) - fn = __vgic_v3_read_bpr0; - else - fn = __vgic_v3_write_bpr0; - break; - case SYS_ICC_DIR_EL1: - if (unlikely(is_read)) - return 0; - fn = __vgic_v3_write_dir; - break; - case SYS_ICC_RPR_EL1: - if (unlikely(!is_read)) - return 0; - fn = __vgic_v3_read_rpr; - break; - case SYS_ICC_CTLR_EL1: - if (is_read) - fn = __vgic_v3_read_ctlr; - else - fn = __vgic_v3_write_ctlr; - break; - case SYS_ICC_PMR_EL1: - if (is_read) - fn = __vgic_v3_read_pmr; - else - fn = __vgic_v3_write_pmr; - break; - default: - return 0; - } - - vmcr = __vgic_v3_read_vmcr(); - rt = kvm_vcpu_sys_get_rt(vcpu); - fn(vcpu, vmcr, rt); - - __kvm_skip_instr(vcpu); - - return 1; -} - -#endif |