diff options
Diffstat (limited to 'arch/x86/xen')
| -rw-r--r-- | arch/x86/xen/Kconfig | 2 | ||||
| -rw-r--r-- | arch/x86/xen/Makefile | 41 | ||||
| -rw-r--r-- | arch/x86/xen/efi.c | 14 | ||||
| -rw-r--r-- | arch/x86/xen/enlighten.c | 5 | ||||
| -rw-r--r-- | arch/x86/xen/enlighten_hvm.c | 2 | ||||
| -rw-r--r-- | arch/x86/xen/enlighten_pv.c | 34 | ||||
| -rw-r--r-- | arch/x86/xen/enlighten_pvh.c | 3 | ||||
| -rw-r--r-- | arch/x86/xen/grant-table.c | 25 | ||||
| -rw-r--r-- | arch/x86/xen/irq.c | 2 | ||||
| -rw-r--r-- | arch/x86/xen/mmu.c | 188 | ||||
| -rw-r--r-- | arch/x86/xen/mmu_hvm.c | 2 | ||||
| -rw-r--r-- | arch/x86/xen/mmu_pv.c | 170 | ||||
| -rw-r--r-- | arch/x86/xen/p2m.c | 8 | ||||
| -rw-r--r-- | arch/x86/xen/pci-swiotlb-xen.c | 2 | ||||
| -rw-r--r-- | arch/x86/xen/platform-pci-unplug.c | 23 | ||||
| -rw-r--r-- | arch/x86/xen/pmu.c | 13 | ||||
| -rw-r--r-- | arch/x86/xen/smp_pv.c | 2 | ||||
| -rw-r--r-- | arch/x86/xen/spinlock.c | 44 | ||||
| -rw-r--r-- | arch/x86/xen/time.c | 4 | ||||
| -rw-r--r-- | arch/x86/xen/vdso.h | 2 | ||||
| -rw-r--r-- | arch/x86/xen/xen-asm_64.S | 8 | ||||
| -rw-r--r-- | arch/x86/xen/xen-pvh.S | 17 |
22 files changed, 276 insertions, 335 deletions
diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index c1f98f32c45f..1ef391aa184d 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -18,6 +18,7 @@ config XEN_PV bool "Xen PV guest support" default y depends on XEN + select PARAVIRT_XXL select XEN_HAVE_PVMMU select XEN_HAVE_VPMU help @@ -68,7 +69,6 @@ config XEN_SAVE_RESTORE config XEN_DEBUG_FS bool "Enable Xen debug and tuning parameters in debugfs" depends on XEN && DEBUG_FS - default n help Enable statistics output and various tuning options in debugfs. Enabling this option may incur a significant performance overhead. diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile index d83cb5478f54..dd2550d33b38 100644 --- a/arch/x86/xen/Makefile +++ b/arch/x86/xen/Makefile @@ -12,25 +12,46 @@ endif # Make sure early boot has no stackprotector nostackp := $(call cc-option, -fno-stack-protector) CFLAGS_enlighten_pv.o := $(nostackp) -CFLAGS_mmu_pv.o := $(nostackp) +CFLAGS_mmu_pv.o := $(nostackp) -obj-y := enlighten.o multicalls.o mmu.o irq.o \ - time.o xen-asm.o xen-asm_$(BITS).o \ - grant-table.o suspend.o platform-pci-unplug.o +obj-y += enlighten.o +obj-y += mmu.o +obj-y += time.o +obj-y += grant-table.o +obj-y += suspend.o -obj-$(CONFIG_XEN_PVHVM) += enlighten_hvm.o mmu_hvm.o suspend_hvm.o -obj-$(CONFIG_XEN_PV) += setup.o apic.o pmu.o suspend_pv.o \ - p2m.o enlighten_pv.o mmu_pv.o -obj-$(CONFIG_XEN_PVH) += enlighten_pvh.o +obj-$(CONFIG_XEN_PVHVM) += enlighten_hvm.o +obj-$(CONFIG_XEN_PVHVM) += mmu_hvm.o +obj-$(CONFIG_XEN_PVHVM) += suspend_hvm.o +obj-$(CONFIG_XEN_PVHVM) += platform-pci-unplug.o -obj-$(CONFIG_EVENT_TRACING) += trace.o +obj-$(CONFIG_XEN_PV) += setup.o +obj-$(CONFIG_XEN_PV) += apic.o +obj-$(CONFIG_XEN_PV) += pmu.o +obj-$(CONFIG_XEN_PV) += suspend_pv.o +obj-$(CONFIG_XEN_PV) += p2m.o +obj-$(CONFIG_XEN_PV) += enlighten_pv.o +obj-$(CONFIG_XEN_PV) += mmu_pv.o +obj-$(CONFIG_XEN_PV) += irq.o +obj-$(CONFIG_XEN_PV) += multicalls.o +obj-$(CONFIG_XEN_PV) += xen-asm.o +obj-$(CONFIG_XEN_PV) += xen-asm_$(BITS).o + +obj-$(CONFIG_XEN_PVH) += enlighten_pvh.o +obj-$(CONFIG_XEN_PVH) += xen-pvh.o + +obj-$(CONFIG_EVENT_TRACING) += trace.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_XEN_PV_SMP) += smp_pv.o obj-$(CONFIG_XEN_PVHVM_SMP) += smp_hvm.o + obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o + obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o + obj-$(CONFIG_XEN_DOM0) += vga.o + obj-$(CONFIG_SWIOTLB_XEN) += pci-swiotlb-xen.o + obj-$(CONFIG_XEN_EFI) += efi.o -obj-$(CONFIG_XEN_PVH) += xen-pvh.o diff --git a/arch/x86/xen/efi.c b/arch/x86/xen/efi.c index 1804b27f9632..1fbb629a9d78 100644 --- a/arch/x86/xen/efi.c +++ b/arch/x86/xen/efi.c @@ -1,18 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2014 Oracle Co., Daniel Kiper - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program. If not, see <http://www.gnu.org/licenses/>. */ #include <linux/bitops.h> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 2eeddd814653..e996e8e744cb 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -1,10 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG -#include <linux/bootmem.h> +#include <linux/memblock.h> #endif #include <linux/cpu.h> #include <linux/kexec.h> #include <linux/slab.h> +#include <xen/xen.h> #include <xen/features.h> #include <xen/page.h> #include <xen/interface/memory.h> diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 19c1ff542387..0e75642d42a3 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + #include <linux/acpi.h> #include <linux/cpu.h> #include <linux/kexec.h> diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 52a7c3faee0c..2f6787fc7106 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -23,7 +23,7 @@ #include <linux/start_kernel.h> #include <linux/sched.h> #include <linux/kprobes.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/export.h> #include <linux/mm.h> #include <linux/page-flags.h> @@ -31,7 +31,6 @@ #include <linux/console.h> #include <linux/pci.h> #include <linux/gfp.h> -#include <linux/memblock.h> #include <linux/edd.h> #include <linux/frame.h> @@ -995,11 +994,14 @@ void __init xen_setup_vcpu_info_placement(void) * percpu area for all cpus, so make use of it. */ if (xen_have_vcpu_info_placement) { - pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); - pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct); - pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); - pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); - pv_mmu_ops.read_cr2 = xen_read_cr2_direct; + pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); + pv_ops.irq.restore_fl = + __PV_IS_CALLEE_SAVE(xen_restore_fl_direct); + pv_ops.irq.irq_disable = + __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); + pv_ops.irq.irq_enable = + __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); + pv_ops.mmu.read_cr2 = xen_read_cr2_direct; } } @@ -1174,14 +1176,14 @@ static void __init xen_boot_params_init_edd(void) */ static void __init xen_setup_gdt(int cpu) { - pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot; - pv_cpu_ops.load_gdt = xen_load_gdt_boot; + pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot; + pv_ops.cpu.load_gdt = xen_load_gdt_boot; setup_stack_canary_segment(cpu); switch_to_new_gdt(cpu); - pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry; - pv_cpu_ops.load_gdt = xen_load_gdt; + pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry; + pv_ops.cpu.load_gdt = xen_load_gdt; } static void __init xen_dom0_set_legacy_features(void) @@ -1206,8 +1208,8 @@ asmlinkage __visible void __init xen_start_kernel(void) /* Install Xen paravirt ops */ pv_info = xen_info; - pv_init_ops.patch = paravirt_patch_default; - pv_cpu_ops = xen_cpu_ops; + pv_ops.init.patch = paravirt_patch_default; + pv_ops.cpu = xen_cpu_ops; xen_init_irq_ops(); /* @@ -1276,8 +1278,10 @@ asmlinkage __visible void __init xen_start_kernel(void) #endif if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) { - pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start; - pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit; + pv_ops.mmu.ptep_modify_prot_start = + xen_ptep_modify_prot_start; + pv_ops.mmu.ptep_modify_prot_commit = + xen_ptep_modify_prot_commit; } machine_ops = xen_machine_ops; diff --git a/arch/x86/xen/enlighten_pvh.c b/arch/x86/xen/enlighten_pvh.c index c85d1a88f476..02e3ab7ff242 100644 --- a/arch/x86/xen/enlighten_pvh.c +++ b/arch/x86/xen/enlighten_pvh.c @@ -11,6 +11,7 @@ #include <asm/xen/interface.h> #include <asm/xen/hypercall.h> +#include <xen/xen.h> #include <xen/interface/memory.h> #include <xen/interface/hvm/start_info.h> @@ -75,7 +76,7 @@ static void __init init_pvh_bootparams(void) * Version 2.12 supports Xen entry point but we will use default x86/PC * environment (i.e. hardware_subarch 0). */ - pvh_bootparams.hdr.version = 0x212; + pvh_bootparams.hdr.version = (2 << 8) | 12; pvh_bootparams.hdr.type_of_loader = (9 << 4) | 0; /* Xen loader */ x86_init.acpi.get_root_pointer = pvh_get_root_pointer; diff --git a/arch/x86/xen/grant-table.c b/arch/x86/xen/grant-table.c index 92ccc718152d..ecb0d5450334 100644 --- a/arch/x86/xen/grant-table.c +++ b/arch/x86/xen/grant-table.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT /****************************************************************************** * grant_table.c * x86 specific part @@ -8,30 +9,6 @@ * Copyright (c) 2004-2005, K A Fraser * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp> * VA Linux Systems Japan. Split out x86 specific part. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 - * as published by the Free Software Foundation; or, when distributed - * separately from the Linux kernel or incorporated into other - * software packages, subject to the following license: - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this source file (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, modify, - * merge, publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. */ #include <linux/sched.h> diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c index 7515a19fd324..850c93f346c7 100644 --- a/arch/x86/xen/irq.c +++ b/arch/x86/xen/irq.c @@ -128,6 +128,6 @@ static const struct pv_irq_ops xen_irq_ops __initconst = { void __init xen_init_irq_ops(void) { - pv_irq_ops = xen_irq_ops; + pv_ops.irq = xen_irq_ops; x86_init.irqs.intr_init = xen_init_IRQ; } diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 96fc2f0fdbfe..60e9c37fd79f 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + #include <linux/pfn.h> #include <asm/xen/page.h> #include <asm/xen/hypercall.h> @@ -6,12 +8,6 @@ #include "multicalls.h" #include "mmu.h" -/* - * Protects atomic reservation decrease/increase against concurrent increases. - * Also protects non-atomic updates of current_pages and balloon lists. - */ -DEFINE_SPINLOCK(xen_reservation_lock); - unsigned long arbitrary_virt_to_mfn(void *vaddr) { xmaddr_t maddr = arbitrary_virt_to_machine(vaddr); @@ -42,186 +38,6 @@ xmaddr_t arbitrary_virt_to_machine(void *vaddr) } EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine); -static noinline void xen_flush_tlb_all(void) -{ - struct mmuext_op *op; - struct multicall_space mcs; - - preempt_disable(); - - mcs = xen_mc_entry(sizeof(*op)); - - op = mcs.args; - op->cmd = MMUEXT_TLB_FLUSH_ALL; - MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); - - xen_mc_issue(PARAVIRT_LAZY_MMU); - - preempt_enable(); -} - -#define REMAP_BATCH_SIZE 16 - -struct remap_data { - xen_pfn_t *pfn; - bool contiguous; - bool no_translate; - pgprot_t prot; - struct mmu_update *mmu_update; -}; - -static int remap_area_pfn_pte_fn(pte_t *ptep, pgtable_t token, - unsigned long addr, void *data) -{ - struct remap_data *rmd = data; - pte_t pte = pte_mkspecial(mfn_pte(*rmd->pfn, rmd->prot)); - - /* - * If we have a contiguous range, just update the pfn itself, - * else update pointer to be "next pfn". - */ - if (rmd->contiguous) - (*rmd->pfn)++; - else - rmd->pfn++; - - rmd->mmu_update->ptr = virt_to_machine(ptep).maddr; - rmd->mmu_update->ptr |= rmd->no_translate ? - MMU_PT_UPDATE_NO_TRANSLATE : - MMU_NORMAL_PT_UPDATE; - rmd->mmu_update->val = pte_val_ma(pte); - rmd->mmu_update++; - - return 0; -} - -static int do_remap_pfn(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t *pfn, int nr, - int *err_ptr, pgprot_t prot, - unsigned int domid, - bool no_translate, - struct page **pages) -{ - int err = 0; - struct remap_data rmd; - struct mmu_update mmu_update[REMAP_BATCH_SIZE]; - unsigned long range; - int mapped = 0; - - BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO))); - - rmd.pfn = pfn; - rmd.prot = prot; - /* - * We use the err_ptr to indicate if there we are doing a contiguous - * mapping or a discontigious mapping. - */ - rmd.contiguous = !err_ptr; - rmd.no_translate = no_translate; - - while (nr) { - int index = 0; - int done = 0; - int batch = min(REMAP_BATCH_SIZE, nr); - int batch_left = batch; - range = (unsigned long)batch << PAGE_SHIFT; - - rmd.mmu_update = mmu_update; - err = apply_to_page_range(vma->vm_mm, addr, range, - remap_area_pfn_pte_fn, &rmd); - if (err) - goto out; - - /* We record the error for each page that gives an error, but - * continue mapping until the whole set is done */ - do { - int i; - - err = HYPERVISOR_mmu_update(&mmu_update[index], - batch_left, &done, domid); - - /* - * @err_ptr may be the same buffer as @gfn, so - * only clear it after each chunk of @gfn is - * used. - */ - if (err_ptr) { - for (i = index; i < index + done; i++) - err_ptr[i] = 0; - } - if (err < 0) { - if (!err_ptr) - goto out; - err_ptr[i] = err; - done++; /* Skip failed frame. */ - } else - mapped += done; - batch_left -= done; - index += done; - } while (batch_left); - - nr -= batch; - addr += range; - if (err_ptr) - err_ptr += batch; - cond_resched(); - } -out: - - xen_flush_tlb_all(); - - return err < 0 ? err : mapped; -} - -int xen_remap_domain_gfn_range(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t gfn, int nr, - pgprot_t prot, unsigned domid, - struct page **pages) -{ - if (xen_feature(XENFEAT_auto_translated_physmap)) - return -EOPNOTSUPP; - - return do_remap_pfn(vma, addr, &gfn, nr, NULL, prot, domid, false, - pages); -} -EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range); - -int xen_remap_domain_gfn_array(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t *gfn, int nr, - int *err_ptr, pgprot_t prot, - unsigned domid, struct page **pages) -{ - if (xen_feature(XENFEAT_auto_translated_physmap)) - return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr, - prot, domid, pages); - - /* We BUG_ON because it's a programmer error to pass a NULL err_ptr, - * and the consequences later is quite hard to detect what the actual - * cause of "wrong memory was mapped in". - */ - BUG_ON(err_ptr == NULL); - return do_remap_pfn(vma, addr, gfn, nr, err_ptr, prot, domid, - false, pages); -} -EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array); - -int xen_remap_domain_mfn_array(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t *mfn, int nr, - int *err_ptr, pgprot_t prot, - unsigned int domid, struct page **pages) -{ - if (xen_feature(XENFEAT_auto_translated_physmap)) - return -EOPNOTSUPP; - - return do_remap_pfn(vma, addr, mfn, nr, err_ptr, prot, domid, - true, pages); -} -EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array); - /* Returns: 0 success */ int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, int nr, struct page **pages) diff --git a/arch/x86/xen/mmu_hvm.c b/arch/x86/xen/mmu_hvm.c index dd2ad82eee80..57409373750f 100644 --- a/arch/x86/xen/mmu_hvm.c +++ b/arch/x86/xen/mmu_hvm.c @@ -73,7 +73,7 @@ static int is_pagetable_dying_supported(void) void __init xen_hvm_init_mmu_ops(void) { if (is_pagetable_dying_supported()) - pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap; + pv_ops.mmu.exit_mmap = xen_hvm_exit_mmap; #ifdef CONFIG_PROC_VMCORE WARN_ON(register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram)); #endif diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index dd461c0167ef..0d7b3ae4960b 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + /* * Xen mmu operations * @@ -99,6 +101,12 @@ static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; #endif /* CONFIG_X86_64 */ /* + * Protects atomic reservation decrease/increase against concurrent increases. + * Also protects non-atomic updates of current_pages and balloon lists. + */ +static DEFINE_SPINLOCK(xen_reservation_lock); + +/* * Note about cr3 (pagetable base) values: * * xen_cr3 contains the current logical cr3 value; it contains the @@ -856,7 +864,7 @@ static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page, * The init_mm pagetable is really pinned as soon as its created, but * that's before we have page structures to store the bits. So do all * the book-keeping now once struct pages for allocated pages are - * initialized. This happens only after free_all_bootmem() is called. + * initialized. This happens only after memblock_free_all() is called. */ static void __init xen_after_bootmem(void) { @@ -2209,7 +2217,7 @@ static void __init xen_write_cr3_init(unsigned long cr3) set_page_prot(initial_page_table, PAGE_KERNEL); set_page_prot(initial_kernel_pmd, PAGE_KERNEL); - pv_mmu_ops.write_cr3 = &xen_write_cr3; + pv_ops.mmu.write_cr3 = &xen_write_cr3; } /* @@ -2358,27 +2366,27 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) static void __init xen_post_allocator_init(void) { - pv_mmu_ops.set_pte = xen_set_pte; - pv_mmu_ops.set_pmd = xen_set_pmd; - pv_mmu_ops.set_pud = xen_set_pud; + pv_ops.mmu.set_pte = xen_set_pte; + pv_ops.mmu.set_pmd = xen_set_pmd; + pv_ops.mmu.set_pud = xen_set_pud; #ifdef CONFIG_X86_64 - pv_mmu_ops.set_p4d = xen_set_p4d; + pv_ops.mmu.set_p4d = xen_set_p4d; #endif /* This will work as long as patching hasn't happened yet (which it hasn't) */ - pv_mmu_ops.alloc_pte = xen_alloc_pte; - pv_mmu_ops.alloc_pmd = xen_alloc_pmd; - pv_mmu_ops.release_pte = xen_release_pte; - pv_mmu_ops.release_pmd = xen_release_pmd; + pv_ops.mmu.alloc_pte = xen_alloc_pte; + pv_ops.mmu.alloc_pmd = xen_alloc_pmd; + pv_ops.mmu.release_pte = xen_release_pte; + pv_ops.mmu.release_pmd = xen_release_pmd; #ifdef CONFIG_X86_64 - pv_mmu_ops.alloc_pud = xen_alloc_pud; - pv_mmu_ops.release_pud = xen_release_pud; + pv_ops.mmu.alloc_pud = xen_alloc_pud; + pv_ops.mmu.release_pud = xen_release_pud; #endif - pv_mmu_ops.make_pte = PV_CALLEE_SAVE(xen_make_pte); + pv_ops.mmu.make_pte = PV_CALLEE_SAVE(xen_make_pte); #ifdef CONFIG_X86_64 - pv_mmu_ops.write_cr3 = &xen_write_cr3; + pv_ops.mmu.write_cr3 = &xen_write_cr3; #endif } @@ -2466,7 +2474,7 @@ void __init xen_init_mmu_ops(void) x86_init.paging.pagetable_init = xen_pagetable_init; x86_init.hyper.init_after_bootmem = xen_after_bootmem; - pv_mmu_ops = xen_mmu_ops; + pv_ops.mmu = xen_mmu_ops; memset(dummy_mapping, 0xff, PAGE_SIZE); } @@ -2666,6 +2674,138 @@ void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) } EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); +static noinline void xen_flush_tlb_all(void) +{ + struct mmuext_op *op; + struct multicall_space mcs; + + preempt_disable(); + + mcs = xen_mc_entry(sizeof(*op)); + + op = mcs.args; + op->cmd = MMUEXT_TLB_FLUSH_ALL; + MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); + + xen_mc_issue(PARAVIRT_LAZY_MMU); + + preempt_enable(); +} + +#define REMAP_BATCH_SIZE 16 + +struct remap_data { + xen_pfn_t *pfn; + bool contiguous; + bool no_translate; + pgprot_t prot; + struct mmu_update *mmu_update; +}; + +static int remap_area_pfn_pte_fn(pte_t *ptep, pgtable_t token, + unsigned long addr, void *data) +{ + struct remap_data *rmd = data; + pte_t pte = pte_mkspecial(mfn_pte(*rmd->pfn, rmd->prot)); + + /* + * If we have a contiguous range, just update the pfn itself, + * else update pointer to be "next pfn". + */ + if (rmd->contiguous) + (*rmd->pfn)++; + else + rmd->pfn++; + + rmd->mmu_update->ptr = virt_to_machine(ptep).maddr; + rmd->mmu_update->ptr |= rmd->no_translate ? + MMU_PT_UPDATE_NO_TRANSLATE : + MMU_NORMAL_PT_UPDATE; + rmd->mmu_update->val = pte_val_ma(pte); + rmd->mmu_update++; + + return 0; +} + +int xen_remap_pfn(struct vm_area_struct *vma, unsigned long addr, + xen_pfn_t *pfn, int nr, int *err_ptr, pgprot_t prot, + unsigned int domid, bool no_translate, struct page **pages) +{ + int err = 0; + struct remap_data rmd; + struct mmu_update mmu_update[REMAP_BATCH_SIZE]; + unsigned long range; + int mapped = 0; + + BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO))); + + rmd.pfn = pfn; + rmd.prot = prot; + /* + * We use the err_ptr to indicate if there we are doing a contiguous + * mapping or a discontigious mapping. + */ + rmd.contiguous = !err_ptr; + rmd.no_translate = no_translate; + + while (nr) { + int index = 0; + int done = 0; + int batch = min(REMAP_BATCH_SIZE, nr); + int batch_left = batch; + + range = (unsigned long)batch << PAGE_SHIFT; + + rmd.mmu_update = mmu_update; + err = apply_to_page_range(vma->vm_mm, addr, range, + remap_area_pfn_pte_fn, &rmd); + if (err) + goto out; + + /* + * We record the error for each page that gives an error, but + * continue mapping until the whole set is done + */ + do { + int i; + + err = HYPERVISOR_mmu_update(&mmu_update[index], + batch_left, &done, domid); + + /* + * @err_ptr may be the same buffer as @gfn, so + * only clear it after each chunk of @gfn is + * used. + */ + if (err_ptr) { + for (i = index; i < index + done; i++) + err_ptr[i] = 0; + } + if (err < 0) { + if (!err_ptr) + goto out; + err_ptr[i] = err; + done++; /* Skip failed frame. */ + } else + mapped += done; + batch_left -= done; + index += done; + } while (batch_left); + + nr -= batch; + addr += range; + if (err_ptr) + err_ptr += batch; + cond_resched(); + } +out: + + xen_flush_tlb_all(); + + return err < 0 ? err : mapped; +} +EXPORT_SYMBOL_GPL(xen_remap_pfn); + #ifdef CONFIG_KEXEC_CORE phys_addr_t paddr_vmcoreinfo_note(void) { diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 159a897151d6..b06731705529 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + /* * Xen leaves the responsibility for maintaining p2m mappings to the * guests themselves, but it must also access and update the p2m array @@ -65,7 +67,7 @@ #include <linux/hash.h> #include <linux/sched.h> #include <linux/seq_file.h> -#include <linux/bootmem.h> +#include <linux/memblock.h> #include <linux/slab.h> #include <linux/vmalloc.h> @@ -180,7 +182,7 @@ static void p2m_init_identity(unsigned long *p2m, unsigned long pfn) static void * __ref alloc_p2m_page(void) { if (unlikely(!slab_is_available())) - return alloc_bootmem_align(PAGE_SIZE, PAGE_SIZE); + return memblock_alloc(PAGE_SIZE, PAGE_SIZE); return (void *)__get_free_page(GFP_KERNEL); } @@ -188,7 +190,7 @@ static void * __ref alloc_p2m_page(void) static void __ref free_p2m_page(void *p) { if (unlikely(!slab_is_available())) { - free_bootmem((unsigned long)p, PAGE_SIZE); + memblock_free((unsigned long)p, PAGE_SIZE); return; } diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c index 37c6056a7bba..33293ce01d8d 100644 --- a/arch/x86/xen/pci-swiotlb-xen.c +++ b/arch/x86/xen/pci-swiotlb-xen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + /* Glue code to lib/swiotlb-xen.c */ #include <linux/dma-mapping.h> diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c index 33a783c77d96..96d7f7d39cb9 100644 --- a/arch/x86/xen/platform-pci-unplug.c +++ b/arch/x86/xen/platform-pci-unplug.c @@ -1,28 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0 + /****************************************************************************** * platform-pci-unplug.c * * Xen platform PCI device driver * Copyright (c) 2010, Citrix - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple - * Place - Suite 330, Boston, MA 02111-1307 USA. - * */ #include <linux/init.h> #include <linux/io.h> #include <linux/export.h> +#include <xen/xen.h> #include <xen/platform_pci.h> #include "xen-ops.h" @@ -30,7 +19,6 @@ #define XEN_PLATFORM_ERR_PROTOCOL -2 #define XEN_PLATFORM_ERR_BLACKLIST -3 -#ifdef CONFIG_XEN_PVHVM /* store the value of xen_emul_unplug after the unplug is done */ static int xen_platform_pci_unplug; static int xen_emul_unplug; @@ -146,6 +134,10 @@ void xen_unplug_emulated_devices(void) { int r; + /* PVH guests don't have emulated devices. */ + if (xen_pvh_domain()) + return; + /* user explicitly requested no unplug */ if (xen_emul_unplug & XEN_UNPLUG_NEVER) return; @@ -214,4 +206,3 @@ static int __init parse_xen_emul_unplug(char *arg) return 0; } early_param("xen_emul_unplug", parse_xen_emul_unplug); -#endif diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 95997e6c0696..e13b0b49fcdf 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -3,6 +3,7 @@ #include <linux/interrupt.h> #include <asm/xen/hypercall.h> +#include <xen/xen.h> #include <xen/page.h> #include <xen/interface/xen.h> #include <xen/interface/vcpu.h> @@ -90,6 +91,12 @@ static void xen_pmu_arch_init(void) k7_counters_mirrored = 0; break; } + } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) { + amd_num_counters = F10H_NUM_COUNTERS; + amd_counters_base = MSR_K7_PERFCTR0; + amd_ctrls_base = MSR_K7_EVNTSEL0; + amd_msr_step = 1; + k7_counters_mirrored = 0; } else { uint32_t eax, ebx, ecx, edx; @@ -285,7 +292,7 @@ static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err) { - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { if (is_amd_pmu_msr(msr)) { if (!xen_amd_pmu_emulate(msr, val, 1)) *val = native_read_msr_safe(msr, err); @@ -308,7 +315,7 @@ bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) { uint64_t val = ((uint64_t)high << 32) | low; - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { if (is_amd_pmu_msr(msr)) { if (!xen_amd_pmu_emulate(msr, &val, 0)) *err = native_write_msr_safe(msr, low, high); @@ -379,7 +386,7 @@ static unsigned long long xen_intel_read_pmc(int counter) unsigned long long xen_read_pmc(int counter) { - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) return xen_amd_read_pmc(counter); else return xen_intel_read_pmc(counter); diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index e3b18ad49889..145506f9fdbe 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -22,6 +22,7 @@ #include <linux/tick.h> #include <linux/nmi.h> #include <linux/cpuhotplug.h> +#include <linux/stackprotector.h> #include <asm/paravirt.h> #include <asm/desc.h> @@ -88,6 +89,7 @@ static void cpu_bringup(void) asmlinkage __visible void cpu_bringup_and_idle(void) { cpu_bringup(); + boot_init_stack_canary(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 973f10e05211..441c88262169 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c @@ -39,34 +39,25 @@ static void xen_qlock_kick(int cpu) */ static void xen_qlock_wait(u8 *byte, u8 val) { + unsigned long flags; int irq = __this_cpu_read(lock_kicker_irq); /* If kicker interrupts not initialized yet, just spin */ - if (irq == -1) + if (irq == -1 || in_nmi()) return; - /* clear pending */ - xen_clear_irq_pending(irq); - barrier(); - - /* - * We check the byte value after clearing pending IRQ to make sure - * that we won't miss a wakeup event because of the clearing. - * - * The sync_clear_bit() call in xen_clear_irq_pending() is atomic. - * So it is effectively a memory barrier for x86. - */ - if (READ_ONCE(*byte) != val) - return; + /* Guard against reentry. */ + local_irq_save(flags); - /* - * If an interrupt happens here, it will leave the wakeup irq - * pending, which will cause xen_poll_irq() to return - * immediately. - */ + /* If irq pending already clear it. */ + if (xen_test_irq_pending(irq)) { + xen_clear_irq_pending(irq); + } else if (READ_ONCE(*byte) == val) { + /* Block until irq becomes pending (or a spurious wakeup) */ + xen_poll_irq(irq); + } - /* Block until irq becomes pending (or perhaps a spurious wakeup) */ - xen_poll_irq(irq); + local_irq_restore(flags); } static irqreturn_t dummy_handler(int irq, void *dev_id) @@ -141,11 +132,12 @@ void __init xen_init_spinlocks(void) printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); __pv_init_lock_hash(); - pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; - pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); - pv_lock_ops.wait = xen_qlock_wait; - pv_lock_ops.kick = xen_qlock_kick; - pv_lock_ops.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); + pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; + pv_ops.lock.queued_spin_unlock = + PV_CALLEE_SAVE(__pv_queued_spin_unlock); + pv_ops.lock.wait = xen_qlock_wait; + pv_ops.lock.kick = xen_qlock_kick; + pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); } static __init int xen_parse_nopvspin(char *arg) diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index c84f1e039d84..72bf446c3fee 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c @@ -513,7 +513,7 @@ static void __init xen_time_init(void) void __init xen_init_time_ops(void) { xen_sched_clock_offset = xen_clocksource_read(); - pv_time_ops = xen_time_ops; + pv_ops.time = xen_time_ops; x86_init.timers.timer_init = xen_time_init; x86_init.timers.setup_percpu_clockev = x86_init_noop; @@ -555,7 +555,7 @@ void __init xen_hvm_init_time_ops(void) } xen_sched_clock_offset = xen_clocksource_read(); - pv_time_ops = xen_time_ops; + pv_ops.time = xen_time_ops; x86_init.timers.setup_percpu_clockev = xen_time_init; x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; diff --git a/arch/x86/xen/vdso.h b/arch/x86/xen/vdso.h index 861fedfe5230..873c54c488fe 100644 --- a/arch/x86/xen/vdso.h +++ b/arch/x86/xen/vdso.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* Bit used for the pseudo-hwcap for non-negative segments. We use bit 1 to avoid bugs in some versions of glibc when bit 0 is used; the choice is otherwise arbitrary. */ diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S index 417b339e5c8e..bb1c2da0381d 100644 --- a/arch/x86/xen/xen-asm_64.S +++ b/arch/x86/xen/xen-asm_64.S @@ -91,13 +91,15 @@ ENTRY(xen_iret) ENTRY(xen_sysret64) /* * We're already on the usermode stack at this point, but - * still with the kernel gs, so we can easily switch back + * still with the kernel gs, so we can easily switch back. + * + * tss.sp2 is scratch space. */ - movq %rsp, PER_CPU_VAR(rsp_scratch) + movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp pushq $__USER_DS - pushq PER_CPU_VAR(rsp_scratch) + pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) pushq %r11 pushq $__USER_CS pushq %rcx diff --git a/arch/x86/xen/xen-pvh.S b/arch/x86/xen/xen-pvh.S index ca2d3b2bf2af..1f8825bbaffb 100644 --- a/arch/x86/xen/xen-pvh.S +++ b/arch/x86/xen/xen-pvh.S @@ -1,18 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* * Copyright C 2016, Oracle and/or its affiliates. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program. If not, see <http://www.gnu.org/licenses/>. */ .code32 @@ -181,7 +170,7 @@ canary: .fill 48, 1, 0 early_stack: - .fill 256, 1, 0 + .fill BOOT_STACK_SIZE, 1, 0 early_stack_end: ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY, |