diff options
Diffstat (limited to 'arch/x86/platform/efi')
| -rw-r--r-- | arch/x86/platform/efi/Makefile | 1 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi-bgrt.c | 12 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi.c | 436 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi_32.c | 13 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi_64.c | 377 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi_stub_64.S | 166 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi_thunk_64.S | 65 |
7 files changed, 906 insertions, 164 deletions
diff --git a/arch/x86/platform/efi/Makefile b/arch/x86/platform/efi/Makefile index b7b0b35c1981..d51045afcaaf 100644 --- a/arch/x86/platform/efi/Makefile +++ b/arch/x86/platform/efi/Makefile @@ -1,3 +1,4 @@ obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o obj-$(CONFIG_ACPI_BGRT) += efi-bgrt.o obj-$(CONFIG_EARLY_PRINTK_EFI) += early_printk.o +obj-$(CONFIG_EFI_MIXED) += efi_thunk_$(BITS).o diff --git a/arch/x86/platform/efi/efi-bgrt.c b/arch/x86/platform/efi/efi-bgrt.c index 7145ec63c520..f15103dff4b4 100644 --- a/arch/x86/platform/efi/efi-bgrt.c +++ b/arch/x86/platform/efi/efi-bgrt.c @@ -42,14 +42,15 @@ void __init efi_bgrt_init(void) if (bgrt_tab->header.length < sizeof(*bgrt_tab)) return; - if (bgrt_tab->version != 1) + if (bgrt_tab->version != 1 || bgrt_tab->status != 1) return; if (bgrt_tab->image_type != 0 || !bgrt_tab->image_address) return; image = efi_lookup_mapped_addr(bgrt_tab->image_address); if (!image) { - image = ioremap(bgrt_tab->image_address, sizeof(bmp_header)); + image = early_memremap(bgrt_tab->image_address, + sizeof(bmp_header)); ioremapped = true; if (!image) return; @@ -57,7 +58,7 @@ void __init efi_bgrt_init(void) memcpy_fromio(&bmp_header, image, sizeof(bmp_header)); if (ioremapped) - iounmap(image); + early_iounmap(image, sizeof(bmp_header)); bgrt_image_size = bmp_header.size; bgrt_image = kmalloc(bgrt_image_size, GFP_KERNEL); @@ -65,7 +66,8 @@ void __init efi_bgrt_init(void) return; if (ioremapped) { - image = ioremap(bgrt_tab->image_address, bmp_header.size); + image = early_memremap(bgrt_tab->image_address, + bmp_header.size); if (!image) { kfree(bgrt_image); bgrt_image = NULL; @@ -75,5 +77,5 @@ void __init efi_bgrt_init(void) memcpy_fromio(bgrt_image, image, bgrt_image_size); if (ioremapped) - iounmap(image); + early_iounmap(image, bmp_header.size); } diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index d62ec87a2b26..3781dd39e8bd 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -52,6 +52,7 @@ #include <asm/tlbflush.h> #include <asm/x86_init.h> #include <asm/rtc.h> +#include <asm/uv/uv.h> #define EFI_DEBUG @@ -67,9 +68,7 @@ struct efi_memory_map memmap; static struct efi efi_phys __initdata; static efi_system_table_t efi_systab __initdata; -unsigned long x86_efi_facility; - -static __initdata efi_config_table_type_t arch_tables[] = { +static efi_config_table_type_t arch_tables[] __initdata = { #ifdef CONFIG_X86_UV {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab}, #endif @@ -78,16 +77,7 @@ static __initdata efi_config_table_type_t arch_tables[] = { u64 efi_setup; /* efi setup_data physical address */ -/* - * Returns 1 if 'facility' is enabled, 0 otherwise. - */ -int efi_enabled(int facility) -{ - return test_bit(facility, &x86_efi_facility) != 0; -} -EXPORT_SYMBOL(efi_enabled); - -static bool __initdata disable_runtime = false; +static bool disable_runtime __initdata = false; static int __init setup_noefi(char *arg) { disable_runtime = true; @@ -256,27 +246,12 @@ static efi_status_t __init phys_efi_set_virtual_address_map( return status; } -static efi_status_t __init phys_efi_get_time(efi_time_t *tm, - efi_time_cap_t *tc) -{ - unsigned long flags; - efi_status_t status; - - spin_lock_irqsave(&rtc_lock, flags); - efi_call_phys_prelog(); - status = efi_call_phys2(efi_phys.get_time, virt_to_phys(tm), - virt_to_phys(tc)); - efi_call_phys_epilog(); - spin_unlock_irqrestore(&rtc_lock, flags); - return status; -} - int efi_set_rtc_mmss(const struct timespec *now) { unsigned long nowtime = now->tv_sec; - efi_status_t status; - efi_time_t eft; - efi_time_cap_t cap; + efi_status_t status; + efi_time_t eft; + efi_time_cap_t cap; struct rtc_time tm; status = efi.get_time(&eft, &cap); @@ -294,9 +269,8 @@ int efi_set_rtc_mmss(const struct timespec *now) eft.second = tm.tm_sec; eft.nanosecond = 0; } else { - printk(KERN_ERR - "%s: Invalid EFI RTC value: write of %lx to EFI RTC failed\n", - __FUNCTION__, nowtime); + pr_err("%s: Invalid EFI RTC value: write of %lx to EFI RTC failed\n", + __func__, nowtime); return -1; } @@ -412,8 +386,7 @@ static void __init print_efi_memmap(void) p < memmap.map_end; p += memmap.desc_size, i++) { md = p; - pr_info("mem%02u: type=%u, attr=0x%llx, " - "range=[0x%016llx-0x%016llx) (%lluMB)\n", + pr_info("mem%02u: type=%u, attr=0x%llx, range=[0x%016llx-0x%016llx) (%lluMB)\n", i, md->type, md->attribute, md->phys_addr, md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT), (md->num_pages >> (20 - EFI_PAGE_SHIFT))); @@ -445,9 +418,8 @@ void __init efi_reserve_boot_services(void) memblock_is_region_reserved(start, size)) { /* Could not reserve, skip it */ md->num_pages = 0; - memblock_dbg("Could not reserve boot range " - "[0x%010llx-0x%010llx]\n", - start, start+size-1); + memblock_dbg("Could not reserve boot range [0x%010llx-0x%010llx]\n", + start, start+size-1); } else memblock_reserve(start, size); } @@ -455,7 +427,7 @@ void __init efi_reserve_boot_services(void) void __init efi_unmap_memmap(void) { - clear_bit(EFI_MEMMAP, &x86_efi_facility); + clear_bit(EFI_MEMMAP, &efi.flags); if (memmap.map) { early_iounmap(memmap.map, memmap.nr_map * memmap.desc_size); memmap.map = NULL; @@ -466,9 +438,6 @@ void __init efi_free_boot_services(void) { void *p; - if (!efi_is_native()) - return; - for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { efi_memory_desc_t *md = p; unsigned long long start = md->phys_addr; @@ -583,45 +552,82 @@ static int __init efi_systab_init(void *phys) return -EINVAL; } if ((efi.systab->hdr.revision >> 16) == 0) - pr_err("Warning: System table version " - "%d.%02d, expected 1.00 or greater!\n", + pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n", efi.systab->hdr.revision >> 16, efi.systab->hdr.revision & 0xffff); + set_bit(EFI_SYSTEM_TABLES, &efi.flags); + return 0; } -static int __init efi_runtime_init(void) +static int __init efi_runtime_init32(void) { - efi_runtime_services_t *runtime; + efi_runtime_services_32_t *runtime; + + runtime = early_ioremap((unsigned long)efi.systab->runtime, + sizeof(efi_runtime_services_32_t)); + if (!runtime) { + pr_err("Could not map the runtime service table!\n"); + return -ENOMEM; + } /* - * Check out the runtime services table. We need to map - * the runtime services table so that we can grab the physical - * address of several of the EFI runtime functions, needed to - * set the firmware into virtual mode. + * We will only need *early* access to the following two + * EFI runtime services before set_virtual_address_map + * is invoked. */ + efi_phys.set_virtual_address_map = + (efi_set_virtual_address_map_t *) + (unsigned long)runtime->set_virtual_address_map; + early_iounmap(runtime, sizeof(efi_runtime_services_32_t)); + + return 0; +} + +static int __init efi_runtime_init64(void) +{ + efi_runtime_services_64_t *runtime; + runtime = early_ioremap((unsigned long)efi.systab->runtime, - sizeof(efi_runtime_services_t)); + sizeof(efi_runtime_services_64_t)); if (!runtime) { pr_err("Could not map the runtime service table!\n"); return -ENOMEM; } + /* - * We will only need *early* access to the following - * two EFI runtime services before set_virtual_address_map + * We will only need *early* access to the following two + * EFI runtime services before set_virtual_address_map * is invoked. */ - efi_phys.get_time = (efi_get_time_t *)runtime->get_time; efi_phys.set_virtual_address_map = - (efi_set_virtual_address_map_t *) - runtime->set_virtual_address_map; + (efi_set_virtual_address_map_t *) + (unsigned long)runtime->set_virtual_address_map; + early_iounmap(runtime, sizeof(efi_runtime_services_64_t)); + + return 0; +} + +static int __init efi_runtime_init(void) +{ + int rv; + /* - * Make efi_get_time can be called before entering - * virtual mode. + * Check out the runtime services table. We need to map + * the runtime services table so that we can grab the physical + * address of several of the EFI runtime functions, needed to + * set the firmware into virtual mode. */ - efi.get_time = phys_efi_get_time; - early_iounmap(runtime, sizeof(efi_runtime_services_t)); + if (efi_enabled(EFI_64BIT)) + rv = efi_runtime_init64(); + else + rv = efi_runtime_init32(); + + if (rv) + return rv; + + set_bit(EFI_RUNTIME_SERVICES, &efi.flags); return 0; } @@ -640,6 +646,8 @@ static int __init efi_memmap_init(void) if (add_efi_memmap) do_add_efi_memmap(); + set_bit(EFI_MEMMAP, &efi.flags); + return 0; } @@ -722,7 +730,7 @@ void __init efi_init(void) if (efi_systab_init(efi_phys.systab)) return; - set_bit(EFI_SYSTEM_TABLES, &x86_efi_facility); + set_bit(EFI_SYSTEM_TABLES, &efi.flags); efi.config_table = (unsigned long)efi.systab->tables; efi.fw_vendor = (unsigned long)efi.systab->fw_vendor; @@ -750,24 +758,21 @@ void __init efi_init(void) if (efi_config_init(arch_tables)) return; - set_bit(EFI_CONFIG_TABLES, &x86_efi_facility); - /* * Note: We currently don't support runtime services on an EFI * that doesn't match the kernel 32/64-bit mode. */ - if (!efi_is_native()) + if (!efi_runtime_supported()) pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n"); else { if (disable_runtime || efi_runtime_init()) return; - set_bit(EFI_RUNTIME_SERVICES, &x86_efi_facility); } if (efi_memmap_init()) return; - set_bit(EFI_MEMMAP, &x86_efi_facility); + set_bit(EFI_MEMMAP, &efi.flags); print_efi_memmap(); } @@ -792,7 +797,7 @@ void __init efi_set_executable(efi_memory_desc_t *md, bool executable) set_memory_nx(addr, npages); } -static void __init runtime_code_page_mkexec(void) +void __init runtime_code_page_mkexec(void) { efi_memory_desc_t *md; void *p; @@ -844,6 +849,22 @@ void __init old_map_region(efi_memory_desc_t *md) (unsigned long long)md->phys_addr); } +static void native_runtime_setup(void) +{ + efi.get_time = virt_efi_get_time; + efi.set_time = virt_efi_set_time; + efi.get_wakeup_time = virt_efi_get_wakeup_time; + efi.set_wakeup_time = virt_efi_set_wakeup_time; + efi.get_variable = virt_efi_get_variable; + efi.get_next_variable = virt_efi_get_next_variable; + efi.set_variable = virt_efi_set_variable; + efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count; + efi.reset_system = virt_efi_reset_system; + efi.query_variable_info = virt_efi_query_variable_info; + efi.update_capsule = virt_efi_update_capsule; + efi.query_capsule_caps = virt_efi_query_capsule_caps; +} + /* Merge contiguous regions of the same type and attribute */ static void __init efi_merge_regions(void) { @@ -891,8 +912,9 @@ static void __init get_systab_virt_addr(efi_memory_desc_t *md) } } -static int __init save_runtime_map(void) +static void __init save_runtime_map(void) { +#ifdef CONFIG_KEXEC efi_memory_desc_t *md; void *tmp, *p, *q = NULL; int count = 0; @@ -914,38 +936,44 @@ static int __init save_runtime_map(void) } efi_runtime_map_setup(q, count, memmap.desc_size); + return; - return 0; out: kfree(q); - return -ENOMEM; + pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n"); +#endif } -/* - * Map efi regions which were passed via setup_data. The virt_addr is a fixed - * addr which was used in first kernel of a kexec boot. - */ -static void __init efi_map_regions_fixed(void) +static void *realloc_pages(void *old_memmap, int old_shift) { - void *p; - efi_memory_desc_t *md; + void *ret; - for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { - md = p; - efi_map_region_fixed(md); /* FIXME: add error handling */ - get_systab_virt_addr(md); - } + ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1); + if (!ret) + goto out; + + /* + * A first-time allocation doesn't have anything to copy. + */ + if (!old_memmap) + return ret; + memcpy(ret, old_memmap, PAGE_SIZE << old_shift); + +out: + free_pages((unsigned long)old_memmap, old_shift); + return ret; } /* - * Map efi memory ranges for runtime serivce and update new_memmap with virtual - * addresses. + * Map the efi memory ranges of the runtime services and update new_mmap with + * virtual addresses. */ -static void * __init efi_map_regions(int *count) +static void * __init efi_map_regions(int *count, int *pg_shift) { + void *p, *new_memmap = NULL; + unsigned long left = 0; efi_memory_desc_t *md; - void *p, *tmp, *new_memmap = NULL; for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { md = p; @@ -960,20 +988,80 @@ static void * __init efi_map_regions(int *count) efi_map_region(md); get_systab_virt_addr(md); - tmp = krealloc(new_memmap, (*count + 1) * memmap.desc_size, - GFP_KERNEL); - if (!tmp) - goto out; - new_memmap = tmp; + if (left < memmap.desc_size) { + new_memmap = realloc_pages(new_memmap, *pg_shift); + if (!new_memmap) + return NULL; + + left += PAGE_SIZE << *pg_shift; + (*pg_shift)++; + } + memcpy(new_memmap + (*count * memmap.desc_size), md, memmap.desc_size); + + left -= memmap.desc_size; (*count)++; } return new_memmap; -out: - kfree(new_memmap); - return NULL; +} + +static void __init kexec_enter_virtual_mode(void) +{ +#ifdef CONFIG_KEXEC + efi_memory_desc_t *md; + void *p; + + efi.systab = NULL; + + /* + * We don't do virtual mode, since we don't do runtime services, on + * non-native EFI + */ + if (!efi_is_native()) { + efi_unmap_memmap(); + return; + } + + /* + * Map efi regions which were passed via setup_data. The virt_addr is a + * fixed addr which was used in first kernel of a kexec boot. + */ + for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { + md = p; + efi_map_region_fixed(md); /* FIXME: add error handling */ + get_systab_virt_addr(md); + } + + save_runtime_map(); + + BUG_ON(!efi.systab); + + efi_sync_low_kernel_mappings(); + + /* + * Now that EFI is in virtual mode, update the function + * pointers in the runtime service table to the new virtual addresses. + * + * Call EFI services through wrapper functions. + */ + efi.runtime_version = efi_systab.hdr.revision; + + native_runtime_setup(); + + efi.set_virtual_address_map = NULL; + + if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX)) + runtime_code_page_mkexec(); + + /* clean DUMMY object */ + efi.set_variable(efi_dummy_name, &EFI_DUMMY_GUID, + EFI_VARIABLE_NON_VOLATILE | + EFI_VARIABLE_BOOTSERVICE_ACCESS | + EFI_VARIABLE_RUNTIME_ACCESS, + 0, NULL); +#endif } /* @@ -995,57 +1083,53 @@ out: * * Specially for kexec boot, efi runtime maps in previous kernel should * be passed in via setup_data. In that case runtime ranges will be mapped - * to the same virtual addresses as the first kernel. + * to the same virtual addresses as the first kernel, see + * kexec_enter_virtual_mode(). */ -void __init efi_enter_virtual_mode(void) +static void __init __efi_enter_virtual_mode(void) { - efi_status_t status; + int count = 0, pg_shift = 0; void *new_memmap = NULL; - int err, count = 0; + efi_status_t status; efi.systab = NULL; - /* - * We don't do virtual mode, since we don't do runtime services, on - * non-native EFI - */ - if (!efi_is_native()) { - efi_unmap_memmap(); + efi_merge_regions(); + new_memmap = efi_map_regions(&count, &pg_shift); + if (!new_memmap) { + pr_err("Error reallocating memory, EFI runtime non-functional!\n"); return; } - if (efi_setup) { - efi_map_regions_fixed(); - } else { - efi_merge_regions(); - new_memmap = efi_map_regions(&count); - if (!new_memmap) { - pr_err("Error reallocating memory, EFI runtime non-functional!\n"); - return; - } - } - - err = save_runtime_map(); - if (err) - pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n"); + save_runtime_map(); BUG_ON(!efi.systab); - efi_setup_page_tables(); + if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) + return; + efi_sync_low_kernel_mappings(); + efi_dump_pagetable(); - if (!efi_setup) { + if (efi_is_native()) { status = phys_efi_set_virtual_address_map( - memmap.desc_size * count, - memmap.desc_size, - memmap.desc_version, - (efi_memory_desc_t *)__pa(new_memmap)); - - if (status != EFI_SUCCESS) { - pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n", - status); - panic("EFI call to SetVirtualAddressMap() failed!"); - } + memmap.desc_size * count, + memmap.desc_size, + memmap.desc_version, + (efi_memory_desc_t *)__pa(new_memmap)); + } else { + status = efi_thunk_set_virtual_address_map( + efi_phys.set_virtual_address_map, + memmap.desc_size * count, + memmap.desc_size, + memmap.desc_version, + (efi_memory_desc_t *)__pa(new_memmap)); + } + + if (status != EFI_SUCCESS) { + pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n", + status); + panic("EFI call to SetVirtualAddressMap() failed!"); } /* @@ -1055,24 +1139,43 @@ void __init efi_enter_virtual_mode(void) * Call EFI services through wrapper functions. */ efi.runtime_version = efi_systab.hdr.revision; - efi.get_time = virt_efi_get_time; - efi.set_time = virt_efi_set_time; - efi.get_wakeup_time = virt_efi_get_wakeup_time; - efi.set_wakeup_time = virt_efi_set_wakeup_time; - efi.get_variable = virt_efi_get_variable; - efi.get_next_variable = virt_efi_get_next_variable; - efi.set_variable = virt_efi_set_variable; - efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count; - efi.reset_system = virt_efi_reset_system; + + if (efi_is_native()) + native_runtime_setup(); + else + efi_thunk_runtime_setup(); + efi.set_virtual_address_map = NULL; - efi.query_variable_info = virt_efi_query_variable_info; - efi.update_capsule = virt_efi_update_capsule; - efi.query_capsule_caps = virt_efi_query_capsule_caps; - if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX)) - runtime_code_page_mkexec(); + efi_runtime_mkexec(); - kfree(new_memmap); + /* + * We mapped the descriptor array into the EFI pagetable above but we're + * not unmapping it here. Here's why: + * + * We're copying select PGDs from the kernel page table to the EFI page + * table and when we do so and make changes to those PGDs like unmapping + * stuff from them, those changes appear in the kernel page table and we + * go boom. + * + * From setup_real_mode(): + * + * ... + * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd; + * + * In this particular case, our allocation is in PGD 0 of the EFI page + * table but we've copied that PGD from PGD[272] of the EFI page table: + * + * pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272 + * + * where the direct memory mapping in kernel space is. + * + * new_memmap's VA comes from that direct mapping and thus clearing it, + * it would get cleared in the kernel page table too. + * + * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift); + */ + free_pages((unsigned long)new_memmap, pg_shift); /* clean DUMMY object */ efi.set_variable(efi_dummy_name, &EFI_DUMMY_GUID, @@ -1082,6 +1185,14 @@ void __init efi_enter_virtual_mode(void) 0, NULL); } +void __init efi_enter_virtual_mode(void) +{ + if (efi_setup) + kexec_enter_virtual_mode(); + else + __efi_enter_virtual_mode(); +} + /* * Convenience functions to obtain memory types and attributes */ @@ -1119,9 +1230,8 @@ u64 efi_mem_attributes(unsigned long phys_addr) } /* - * Some firmware has serious problems when using more than 50% of the EFI - * variable store, i.e. it triggers bugs that can brick machines. Ensure that - * we never use more than this safe limit. + * Some firmware implementations refuse to boot if there's insufficient space + * in the variable store. Ensure that we never use more than a safe limit. * * Return EFI_SUCCESS if it is safe to write 'size' bytes to the variable * store. @@ -1140,10 +1250,9 @@ efi_status_t efi_query_variable_store(u32 attributes, unsigned long size) return status; /* - * Some firmware implementations refuse to boot if there's insufficient - * space in the variable store. We account for that by refusing the - * write if permitting it would reduce the available space to under - * 5KB. This figure was provided by Samsung, so should be safe. + * We account for that by refusing the write if permitting it would + * reduce the available space to under 5KB. This figure was provided by + * Samsung, so should be safe. */ if ((remaining_size - size < EFI_MIN_RESERVE) && !efi_no_storage_paranoia) { @@ -1206,8 +1315,27 @@ static int __init parse_efi_cmdline(char *str) str++; if (!strncmp(str, "old_map", 7)) - set_bit(EFI_OLD_MEMMAP, &x86_efi_facility); + set_bit(EFI_OLD_MEMMAP, &efi.flags); return 0; } early_param("efi", parse_efi_cmdline); + +void __init efi_apply_memmap_quirks(void) +{ + /* + * Once setup is done earlier, unmap the EFI memory map on mismatched + * firmware/kernel architectures since there is no support for runtime + * services. + */ + if (!efi_runtime_supported()) { + pr_info("efi: Setup done, disabling due to 32/64-bit mismatch\n"); + efi_unmap_memmap(); + } + + /* + * UV doesn't support the new EFI pagetable mapping yet. + */ + if (is_uv_system()) + set_bit(EFI_OLD_MEMMAP, &efi.flags); +} diff --git a/arch/x86/platform/efi/efi_32.c b/arch/x86/platform/efi/efi_32.c index 249b183cf417..9ee3491e31fb 100644 --- a/arch/x86/platform/efi/efi_32.c +++ b/arch/x86/platform/efi/efi_32.c @@ -40,7 +40,12 @@ static unsigned long efi_rt_eflags; void efi_sync_low_kernel_mappings(void) {} -void efi_setup_page_tables(void) {} +void __init efi_dump_pagetable(void) {} +int efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) +{ + return 0; +} +void efi_cleanup_page_tables(unsigned long pa_memmap, unsigned num_pages) {} void __init efi_map_region(efi_memory_desc_t *md) { @@ -77,3 +82,9 @@ void efi_call_phys_epilog(void) local_irq_restore(efi_rt_eflags); } + +void __init efi_runtime_mkexec(void) +{ + if (__supported_pte_mask & _PAGE_NX) + runtime_code_page_mkexec(); +} diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 6284f158a47d..290d397e1dd9 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -39,6 +39,7 @@ #include <asm/cacheflush.h> #include <asm/fixmap.h> #include <asm/realmode.h> +#include <asm/time.h> static pgd_t *save_pgd __initdata; static unsigned long efi_flags __initdata; @@ -58,7 +59,8 @@ struct efi_scratch { u64 prev_cr3; pgd_t *efi_pgt; bool use_pgd; -}; + u64 phys_stack; +} __packed; static void __init early_code_mapping_set_exec(int executable) { @@ -137,12 +139,64 @@ void efi_sync_low_kernel_mappings(void) sizeof(pgd_t) * num_pgds); } -void efi_setup_page_tables(void) +int efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) { + unsigned long text; + struct page *page; + unsigned npages; + pgd_t *pgd; + + if (efi_enabled(EFI_OLD_MEMMAP)) + return 0; + efi_scratch.efi_pgt = (pgd_t *)(unsigned long)real_mode_header->trampoline_pgd; + pgd = __va(efi_scratch.efi_pgt); - if (!efi_enabled(EFI_OLD_MEMMAP)) - efi_scratch.use_pgd = true; + /* + * It can happen that the physical address of new_memmap lands in memory + * which is not mapped in the EFI page table. Therefore we need to go + * and ident-map those pages containing the map before calling + * phys_efi_set_virtual_address_map(). + */ + if (kernel_map_pages_in_pgd(pgd, pa_memmap, pa_memmap, num_pages, _PAGE_NX)) { + pr_err("Error ident-mapping new memmap (0x%lx)!\n", pa_memmap); + return 1; + } + + efi_scratch.use_pgd = true; + + /* + * When making calls to the firmware everything needs to be 1:1 + * mapped and addressable with 32-bit pointers. Map the kernel + * text and allocate a new stack because we can't rely on the + * stack pointer being < 4GB. + */ + if (!IS_ENABLED(CONFIG_EFI_MIXED)) + return 0; + + page = alloc_page(GFP_KERNEL|__GFP_DMA32); + if (!page) + panic("Unable to allocate EFI runtime stack < 4GB\n"); + + efi_scratch.phys_stack = virt_to_phys(page_address(page)); + efi_scratch.phys_stack += PAGE_SIZE; /* stack grows down */ + + npages = (_end - _text) >> PAGE_SHIFT; + text = __pa(_text); + + if (kernel_map_pages_in_pgd(pgd, text >> PAGE_SHIFT, text, npages, 0)) { + pr_err("Failed to map kernel text 1:1\n"); + return 1; + } + + return 0; +} + +void efi_cleanup_page_tables(unsigned long pa_memmap, unsigned num_pages) +{ + pgd_t *pgd = (pgd_t *)__va(real_mode_header->trampoline_pgd); + + kernel_unmap_pages_in_pgd(pgd, pa_memmap, num_pages); } static void __init __map_region(efi_memory_desc_t *md, u64 va) @@ -173,6 +227,16 @@ void __init efi_map_region(efi_memory_desc_t *md) */ __map_region(md, md->phys_addr); + /* + * Enforce the 1:1 mapping as the default virtual address when + * booting in EFI mixed mode, because even though we may be + * running a 64-bit kernel, the firmware may only be 32-bit. + */ + if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) { + md->virt_addr = md->phys_addr; + return; + } + efi_va -= size; /* Is PA 2M-aligned? */ @@ -233,3 +297,308 @@ void __init parse_efi_setup(u64 phys_addr, u32 data_len) { efi_setup = phys_addr + sizeof(struct setup_data); } + +void __init efi_runtime_mkexec(void) +{ + if (!efi_enabled(EFI_OLD_MEMMAP)) + return; + + if (__supported_pte_mask & _PAGE_NX) + runtime_code_page_mkexec(); +} + +void __init efi_dump_pagetable(void) +{ +#ifdef CONFIG_EFI_PGT_DUMP + pgd_t *pgd = (pgd_t *)__va(real_mode_header->trampoline_pgd); + + ptdump_walk_pgd_level(NULL, pgd); +#endif +} + +#ifdef CONFIG_EFI_MIXED +extern efi_status_t efi64_thunk(u32, ...); + +#define runtime_service32(func) \ +({ \ + u32 table = (u32)(unsigned long)efi.systab; \ + u32 *rt, *___f; \ + \ + rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime)); \ + ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \ + *___f; \ +}) + +/* + * Switch to the EFI page tables early so that we can access the 1:1 + * runtime services mappings which are not mapped in any other page + * tables. This function must be called before runtime_service32(). + * + * Also, disable interrupts because the IDT points to 64-bit handlers, + * which aren't going to function correctly when we switch to 32-bit. + */ +#define efi_thunk(f, ...) \ +({ \ + efi_status_t __s; \ + unsigned long flags; \ + u32 func; \ + \ + efi_sync_low_kernel_mappings(); \ + local_irq_save(flags); \ + \ + efi_scratch.prev_cr3 = read_cr3(); \ + write_cr3((unsigned long)efi_scratch.efi_pgt); \ + __flush_tlb_all(); \ + \ + func = runtime_service32(f); \ + __s = efi64_thunk(func, __VA_ARGS__); \ + \ + write_cr3(efi_scratch.prev_cr3); \ + __flush_tlb_all(); \ + local_irq_restore(flags); \ + \ + __s; \ +}) + +efi_status_t efi_thunk_set_virtual_address_map( + void *phys_set_virtual_address_map, + unsigned long memory_map_size, + unsigned long descriptor_size, + u32 descriptor_version, + efi_memory_desc_t *virtual_map) +{ + efi_status_t status; + unsigned long flags; + u32 func; + + efi_sync_low_kernel_mappings(); + local_irq_save(flags); + + efi_scratch.prev_cr3 = read_cr3(); + write_cr3((unsigned long)efi_scratch.efi_pgt); + __flush_tlb_all(); + + func = (u32)(unsigned long)phys_set_virtual_address_map; + status = efi64_thunk(func, memory_map_size, descriptor_size, + descriptor_version, virtual_map); + + write_cr3(efi_scratch.prev_cr3); + __flush_tlb_all(); + local_irq_restore(flags); + + return status; +} + +static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc) +{ + efi_status_t status; + u32 phys_tm, phys_tc; + + spin_lock(&rtc_lock); + + phys_tm = virt_to_phys(tm); + phys_tc = virt_to_phys(tc); + + status = efi_thunk(get_time, phys_tm, phys_tc); + + spin_unlock(&rtc_lock); + + return status; +} + +static efi_status_t efi_thunk_set_time(efi_time_t *tm) +{ + efi_status_t status; + u32 phys_tm; + + spin_lock(&rtc_lock); + + phys_tm = virt_to_phys(tm); + + status = efi_thunk(set_time, phys_tm); + + spin_unlock(&rtc_lock); + + return status; +} + +static efi_status_t +efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending, + efi_time_t *tm) +{ + efi_status_t status; + u32 phys_enabled, phys_pending, phys_tm; + + spin_lock(&rtc_lock); + + phys_enabled = virt_to_phys(enabled); + phys_pending = virt_to_phys(pending); + phys_tm = virt_to_phys(tm); + + status = efi_thunk(get_wakeup_time, phys_enabled, + phys_pending, phys_tm); + + spin_unlock(&rtc_lock); + + return status; +} + +static efi_status_t +efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm) +{ + efi_status_t status; + u32 phys_tm; + + spin_lock(&rtc_lock); + + phys_tm = virt_to_phys(tm); + + status = efi_thunk(set_wakeup_time, enabled, phys_tm); + + spin_unlock(&rtc_lock); + + return status; +} + + +static efi_status_t +efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor, + u32 *attr, unsigned long *data_size, void *data) +{ + efi_status_t status; + u32 phys_name, phys_vendor, phys_attr; + u32 phys_data_size, phys_data; + + phys_data_size = virt_to_phys(data_size); + phys_vendor = virt_to_phys(vendor); + phys_name = virt_to_phys(name); + phys_attr = virt_to_phys(attr); + phys_data = virt_to_phys(data); + + status = efi_thunk(get_variable, phys_name, phys_vendor, + phys_attr, phys_data_size, phys_data); + + return status; +} + +static efi_status_t +efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor, + u32 attr, unsigned long data_size, void *data) +{ + u32 phys_name, phys_vendor, phys_data; + efi_status_t status; + + phys_name = virt_to_phys(name); + phys_vendor = virt_to_phys(vendor); + phys_data = virt_to_phys(data); + + /* If data_size is > sizeof(u32) we've got problems */ + status = efi_thunk(set_variable, phys_name, phys_vendor, + attr, data_size, phys_data); + + return status; +} + +static efi_status_t +efi_thunk_get_next_variable(unsigned long *name_size, + efi_char16_t *name, + efi_guid_t *vendor) +{ + efi_status_t status; + u32 phys_name_size, phys_name, phys_vendor; + + phys_name_size = virt_to_phys(name_size); + phys_vendor = virt_to_phys(vendor); + phys_name = virt_to_phys(name); + + status = efi_thunk(get_next_variable, phys_name_size, + phys_name, phys_vendor); + + return status; +} + +static efi_status_t +efi_thunk_get_next_high_mono_count(u32 *count) +{ + efi_status_t status; + u32 phys_count; + + phys_count = virt_to_phys(count); + status = efi_thunk(get_next_high_mono_count, phys_count); + + return status; +} + +static void +efi_thunk_reset_system(int reset_type, efi_status_t status, + unsigned long data_size, efi_char16_t *data) +{ + u32 phys_data; + + phys_data = virt_to_phys(data); + + efi_thunk(reset_system, reset_type, status, data_size, phys_data); +} + +static efi_status_t +efi_thunk_update_capsule(efi_capsule_header_t **capsules, + unsigned long count, unsigned long sg_list) +{ + /* + * To properly support this function we would need to repackage + * 'capsules' because the firmware doesn't understand 64-bit + * pointers. + */ + return EFI_UNSUPPORTED; +} + +static efi_status_t +efi_thunk_query_variable_info(u32 attr, u64 *storage_space, + u64 *remaining_space, + u64 *max_variable_size) +{ + efi_status_t status; + u32 phys_storage, phys_remaining, phys_max; + + if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION) + return EFI_UNSUPPORTED; + + phys_storage = virt_to_phys(storage_space); + phys_remaining = virt_to_phys(remaining_space); + phys_max = virt_to_phys(max_variable_size); + + status = efi_thunk(query_variable_info, attr, phys_storage, + phys_remaining, phys_max); + + return status; +} + +static efi_status_t +efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules, + unsigned long count, u64 *max_size, + int *reset_type) +{ + /* + * To properly support this function we would need to repackage + * 'capsules' because the firmware doesn't understand 64-bit + * pointers. + */ + return EFI_UNSUPPORTED; +} + +void efi_thunk_runtime_setup(void) +{ + efi.get_time = efi_thunk_get_time; + efi.set_time = efi_thunk_set_time; + efi.get_wakeup_time = efi_thunk_get_wakeup_time; + efi.set_wakeup_time = efi_thunk_set_wakeup_time; + efi.get_variable = efi_thunk_get_variable; + efi.get_next_variable = efi_thunk_get_next_variable; + efi.set_variable = efi_thunk_set_variable; + efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count; + efi.reset_system = efi_thunk_reset_system; + efi.query_variable_info = efi_thunk_query_variable_info; + efi.update_capsule = efi_thunk_update_capsule; + efi.query_capsule_caps = efi_thunk_query_capsule_caps; +} +#endif /* CONFIG_EFI_MIXED */ diff --git a/arch/x86/platform/efi/efi_stub_64.S b/arch/x86/platform/efi/efi_stub_64.S index 88073b140298..e0984ef0374b 100644 --- a/arch/x86/platform/efi/efi_stub_64.S +++ b/arch/x86/platform/efi/efi_stub_64.S @@ -7,6 +7,10 @@ */ #include <linux/linkage.h> +#include <asm/segment.h> +#include <asm/msr.h> +#include <asm/processor-flags.h> +#include <asm/page_types.h> #define SAVE_XMM \ mov %rsp, %rax; \ @@ -164,7 +168,169 @@ ENTRY(efi_call6) ret ENDPROC(efi_call6) +#ifdef CONFIG_EFI_MIXED + +/* + * We run this function from the 1:1 mapping. + * + * This function must be invoked with a 1:1 mapped stack. + */ +ENTRY(__efi64_thunk) + movl %ds, %eax + push %rax + movl %es, %eax + push %rax + movl %ss, %eax + push %rax + + subq $32, %rsp + movl %esi, 0x0(%rsp) + movl %edx, 0x4(%rsp) + movl %ecx, 0x8(%rsp) + movq %r8, %rsi + movl %esi, 0xc(%rsp) + movq %r9, %rsi + movl %esi, 0x10(%rsp) + + sgdt save_gdt(%rip) + + leaq 1f(%rip), %rbx + movq %rbx, func_rt_ptr(%rip) + + /* Switch to gdt with 32-bit segments */ + movl 64(%rsp), %eax + lgdt (%rax) + + leaq efi_enter32(%rip), %rax + pushq $__KERNEL_CS + pushq %rax + lretq + +1: addq $32, %rsp + + lgdt save_gdt(%rip) + + pop %rbx + movl %ebx, %ss + pop %rbx + movl %ebx, %es + pop %rbx + movl %ebx, %ds + + /* + * Convert 32-bit status code into 64-bit. + */ + test %rax, %rax + jz 1f + movl %eax, %ecx + andl $0x0fffffff, %ecx + andl $0xf0000000, %eax + shl $32, %rax + or %rcx, %rax +1: + ret +ENDPROC(__efi64_thunk) + +ENTRY(efi_exit32) + movq func_rt_ptr(%rip), %rax + push %rax + mov %rdi, %rax + ret +ENDPROC(efi_exit32) + + .code32 +/* + * EFI service pointer must be in %edi. + * + * The stack should represent the 32-bit calling convention. + */ +ENTRY(efi_enter32) + movl $__KERNEL_DS, %eax + movl %eax, %ds + movl %eax, %es + movl %eax, %ss + + /* Reload pgtables */ + movl %cr3, %eax + movl %eax, %cr3 + + /* Disable paging */ + movl %cr0, %eax + btrl $X86_CR0_PG_BIT, %eax + movl %eax, %cr0 + + /* Disable long mode via EFER */ + movl $MSR_EFER, %ecx + rdmsr + btrl $_EFER_LME, %eax + wrmsr + + call *%edi + + /* We must preserve return value */ + movl %eax, %edi + + /* + * Some firmware will return with interrupts enabled. Be sure to + * disable them before we switch GDTs. + */ + cli + + movl 68(%esp), %eax + movl %eax, 2(%eax) + lgdtl (%eax) + + movl %cr4, %eax + btsl $(X86_CR4_PAE_BIT), %eax + movl %eax, %cr4 + + movl %cr3, %eax + movl %eax, %cr3 + + movl $MSR_EFER, %ecx + rdmsr + btsl $_EFER_LME, %eax + wrmsr + + xorl %eax, %eax + lldt %ax + + movl 72(%esp), %eax + pushl $__KERNEL_CS + pushl %eax + + /* Enable paging */ + movl %cr0, %eax + btsl $X86_CR0_PG_BIT, %eax + movl %eax, %cr0 + lret +ENDPROC(efi_enter32) + + .data + .balign 8 + .global efi32_boot_gdt +efi32_boot_gdt: .word 0 + .quad 0 + +save_gdt: .word 0 + .quad 0 +func_rt_ptr: .quad 0 + + .global efi_gdt64 +efi_gdt64: + .word efi_gdt64_end - efi_gdt64 + .long 0 /* Filled out by user */ + .word 0 + .quad 0x0000000000000000 /* NULL descriptor */ + .quad 0x00af9a000000ffff /* __KERNEL_CS */ + .quad 0x00cf92000000ffff /* __KERNEL_DS */ + .quad 0x0080890000000000 /* TS descriptor */ + .quad 0x0000000000000000 /* TS continued */ +efi_gdt64_end: +#endif /* CONFIG_EFI_MIXED */ + .data ENTRY(efi_scratch) .fill 3,8,0 .byte 0 + .quad 0 diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S new file mode 100644 index 000000000000..8806fa73e6e6 --- /dev/null +++ b/arch/x86/platform/efi/efi_thunk_64.S @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2014 Intel Corporation; author Matt Fleming + */ + +#include <linux/linkage.h> +#include <asm/page_types.h> + + .text + .code64 +ENTRY(efi64_thunk) + push %rbp + push %rbx + + /* + * Switch to 1:1 mapped 32-bit stack pointer. + */ + movq %rsp, efi_saved_sp(%rip) + movq efi_scratch+25(%rip), %rsp + + /* + * Calculate the physical address of the kernel text. + */ + movq $__START_KERNEL_map, %rax + subq phys_base(%rip), %rax + + /* + * Push some physical addresses onto the stack. This is easier + * to do now in a code64 section while the assembler can address + * 64-bit values. Note that all the addresses on the stack are + * 32-bit. + */ + subq $16, %rsp + leaq efi_exit32(%rip), %rbx + subq %rax, %rbx + movl %ebx, 8(%rsp) + leaq efi_gdt64(%rip), %rbx + subq %rax, %rbx + movl %ebx, 2(%ebx) + movl %ebx, 4(%rsp) + leaq efi_gdt32(%rip), %rbx + subq %rax, %rbx + movl %ebx, 2(%ebx) + movl %ebx, (%rsp) + + leaq __efi64_thunk(%rip), %rbx + subq %rax, %rbx + call *%rbx + + movq efi_saved_sp(%rip), %rsp + pop %rbx + pop %rbp + retq +ENDPROC(efi64_thunk) + + .data +efi_gdt32: + .word efi_gdt32_end - efi_gdt32 + .long 0 /* Filled out above */ + .word 0 + .quad 0x0000000000000000 /* NULL descriptor */ + .quad 0x00cf9a000000ffff /* __KERNEL_CS */ + .quad 0x00cf93000000ffff /* __KERNEL_DS */ +efi_gdt32_end: + +efi_saved_sp: .quad 0 |