diff options
| author |   Linus Torvalds <torvalds@linux-foundation.org> | 2019-05-06 16:42:54 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2019-05-06 16:42:54 -0700 |
| commit | 14be4c61c205dcb0a72251c1e2790814181bd9ba (patch) | |
| tree | ff239c62505f16f594559756321fa46aee102225 | |
| parent | Merge tag 'm68k-for-v5.2-tag1' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k (diff) | |
| parent | s390/vdso: drop unnecessary cc-ldoption (diff) | |
| download | linux-dev-14be4c61c205dcb0a72251c1e2790814181bd9ba.tar.xz linux-dev-14be4c61c205dcb0a72251c1e2790814181bd9ba.zip | |
Merge tag 's390-5.2-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux
Pull s390 updates from Martin Schwidefsky:
- Support for kernel address space layout randomization
- Add support for kernel image signature verification
- Convert s390 to the generic get_user_pages_fast code
- Convert s390 to the stack unwind API analog to x86
- Add support for CPU directed interrupts for PCI devices
- Provide support for MIO instructions to the PCI base layer, this will
allow the use of direct PCI mappings in user space code
- Add the basic KVM guest ultravisor interface for protected VMs
- Add AT_HWCAP bits for several new hardware capabilities
- Update the CPU measurement facility counter definitions to SVN 6
- Arnds cleanup patches for his quest to get LLVM compiles working
- A vfio-ccw update with bug fixes and support for halt and clear
- Improvements for the hardware TRNG code
- Another round of cleanup for the QDIO layer
- Numerous cleanups and bug fixes
* tag 's390-5.2-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (98 commits)
s390/vdso: drop unnecessary cc-ldoption
s390: fix clang -Wpointer-sign warnigns in boot code
s390: drop CONFIG_VIRT_TO_BUS
s390: boot, purgatory: pass $(CLANG_FLAGS) where needed
s390: only build for new CPUs with clang
s390: simplify disabled_wait
s390/ftrace: use HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
s390/unwind: introduce stack unwind API
s390/opcodes: add missing instructions to the disassembler
s390/bug: add entry size to the __bug_table section
s390: use proper expoline sections for .dma code
s390/nospec: rename assembler generated expoline thunks
s390: add missing ENDPROC statements to assembler functions
locking/lockdep: check for freed initmem in static_obj()
s390/kernel: add support for kernel address space layout randomization (KASLR)
s390/kernel: introduce .dma sections
s390/sclp: do not use static sccbs
s390/kprobes: use static buffer for insn_page
s390/kernel: convert SYSCALL and PGM_CHECK handlers to .quad
s390/kernel: build a relocatable kernel
...
159 files changed, 4907 insertions, 1938 deletions
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 58e69ec8b7a0..3c0646e28488 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3429,6 +3429,8 @@ bridges without forcing it upstream. Note: this removes isolation between devices and may put more devices in an IOMMU group. + force_floating [S390] Force usage of floating interrupts. + nomio [S390] Do not use MIO instructions. pcie_aspm= [PCIE] Forcibly enable or disable PCIe Active State Power Management. diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 97b555e772d7..07485582d027 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -143,6 +143,7 @@ config S390 select HAVE_FUNCTION_TRACER select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS + select HAVE_GENERIC_GUP select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZ4 @@ -184,7 +185,6 @@ config S390 select TTY select VIRT_CPU_ACCOUNTING select ARCH_HAS_SCALED_CPUTIME - select VIRT_TO_BUS select HAVE_NMI @@ -236,6 +236,7 @@ choice config MARCH_Z900 bool "IBM zSeries model z800 and z900" + depends on !CC_IS_CLANG select HAVE_MARCH_Z900_FEATURES help Select this to enable optimizations for model z800/z900 (2064 and @@ -244,6 +245,7 @@ config MARCH_Z900 config MARCH_Z990 bool "IBM zSeries model z890 and z990" + depends on !CC_IS_CLANG select HAVE_MARCH_Z990_FEATURES help Select this to enable optimizations for model z890/z990 (2084 and @@ -252,6 +254,7 @@ config MARCH_Z990 config MARCH_Z9_109 bool "IBM System z9" + depends on !CC_IS_CLANG select HAVE_MARCH_Z9_109_FEATURES help Select this to enable optimizations for IBM System z9 (2094 and @@ -343,12 +346,15 @@ config TUNE_DEFAULT config TUNE_Z900 bool "IBM zSeries model z800 and z900" + depends on !CC_IS_CLANG config TUNE_Z990 bool "IBM zSeries model z890 and z990" + depends on !CC_IS_CLANG config TUNE_Z9_109 bool "IBM System z9" + depends on !CC_IS_CLANG config TUNE_Z10 bool "IBM System z10" @@ -384,6 +390,9 @@ config COMPAT (and some other stuff like libraries and such) is needed for executing 31 bit applications. It is safe to say "Y". +config COMPAT_VDSO + def_bool COMPAT && !CC_IS_CLANG + config SYSVIPC_COMPAT def_bool y if COMPAT && SYSVIPC @@ -545,6 +554,17 @@ config ARCH_HAS_KEXEC_PURGATORY def_bool y depends on KEXEC_FILE +config KEXEC_VERIFY_SIG + bool "Verify kernel signature during kexec_file_load() syscall" + depends on KEXEC_FILE && SYSTEM_DATA_VERIFICATION + help + This option makes kernel signature verification mandatory for + the kexec_file_load() syscall. + + In addition to that option, you need to enable signature + verification for the corresponding kernel image type being + loaded in order for this to work. + config ARCH_RANDOM def_bool y prompt "s390 architectural random number generation API" @@ -605,6 +625,29 @@ config EXPOLINE_FULL endchoice +config RELOCATABLE + bool "Build a relocatable kernel" + select MODULE_REL_CRCS if MODVERSIONS + default y + help + This builds a kernel image that retains relocation information + so it can be loaded at an arbitrary address. + The kernel is linked as a position-independent executable (PIE) + and contains dynamic relocations which are processed early in the + bootup process. + The relocations make the kernel image about 15% larger (compressed + 10%), but are discarded at runtime. + +config RANDOMIZE_BASE + bool "Randomize the address of the kernel image (KASLR)" + depends on RELOCATABLE + default y + help + In support of Kernel Address Space Layout Randomization (KASLR), + this randomizes the address at which the kernel image is loaded, + as a security feature that deters exploit attempts relying on + knowledge of the location of kernel internals. + endmenu menu "Memory setup" @@ -833,6 +876,17 @@ config HAVE_PNETID menu "Virtualization" +config PROTECTED_VIRTUALIZATION_GUEST + def_bool n + prompt "Protected virtualization guest support" + help + Select this option, if you want to be able to run this + kernel as a protected virtualization KVM guest. + Protected virtualization capable machines have a mini hypervisor + located at machine level (an ultravisor). With help of the + Ultravisor, KVM will be able to run "protected" VMs, special + VMs whose memory and management data are unavailable to KVM. + config PFAULT def_bool y prompt "Pseudo page fault support" diff --git a/arch/s390/Makefile b/arch/s390/Makefile index e21053e5e0da..df1d6a150f30 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -16,10 +16,14 @@ KBUILD_AFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC KBUILD_AFLAGS += -m64 KBUILD_CFLAGS += -m64 +ifeq ($(CONFIG_RELOCATABLE),y) +KBUILD_CFLAGS += -fPIE +LDFLAGS_vmlinux := -pie +endif aflags_dwarf := -Wa,-gdwarf-2 -KBUILD_AFLAGS_DECOMPRESSOR := -m64 -D__ASSEMBLY__ +KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ KBUILD_AFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),$(aflags_dwarf)) -KBUILD_CFLAGS_DECOMPRESSOR := -m64 -O2 +KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 KBUILD_CFLAGS_DECOMPRESSOR += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY KBUILD_CFLAGS_DECOMPRESSOR += -fno-delete-null-pointer-checks -msoft-float KBUILD_CFLAGS_DECOMPRESSOR += -fno-asynchronous-unwind-tables @@ -111,7 +115,7 @@ endif cfi := $(call as-instr,.cfi_startproc\n.cfi_val_offset 15$(comma)-160\n.cfi_endproc,-DCONFIG_AS_CFI_VAL_OFFSET=1) KBUILD_CFLAGS += -mbackchain -msoft-float $(cflags-y) -KBUILD_CFLAGS += -pipe -fno-strength-reduce -Wno-sign-compare +KBUILD_CFLAGS += -pipe -Wno-sign-compare KBUILD_CFLAGS += -fno-asynchronous-unwind-tables $(cfi) KBUILD_AFLAGS += $(aflags-y) $(cfi) export KBUILD_AFLAGS_DECOMPRESSOR diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index c844eaf24ed7..c51496bbac19 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -12,25 +12,35 @@ KBUILD_AFLAGS := $(KBUILD_AFLAGS_DECOMPRESSOR) KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) # -# Use -march=z900 for als.c to be able to print an error +# Use minimum architecture for als.c to be able to print an error # message if the kernel is started on a machine which is too old # -ifneq ($(CC_FLAGS_MARCH),-march=z900) +ifndef CONFIG_CC_IS_CLANG +CC_FLAGS_MARCH_MINIMUM := -march=z900 +else +CC_FLAGS_MARCH_MINIMUM := -march=z10 +endif + +ifneq ($(CC_FLAGS_MARCH),$(CC_FLAGS_MARCH_MINIMUM)) AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) -AFLAGS_head.o += -march=z900 +AFLAGS_head.o += $(CC_FLAGS_MARCH_MINIMUM) AFLAGS_REMOVE_mem.o += $(CC_FLAGS_MARCH) -AFLAGS_mem.o += -march=z900 +AFLAGS_mem.o += $(CC_FLAGS_MARCH_MINIMUM) CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) -CFLAGS_als.o += -march=z900 +CFLAGS_als.o += $(CC_FLAGS_MARCH_MINIMUM) CFLAGS_REMOVE_sclp_early_core.o += $(CC_FLAGS_MARCH) -CFLAGS_sclp_early_core.o += -march=z900 +CFLAGS_sclp_early_core.o += $(CC_FLAGS_MARCH_MINIMUM) endif CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char -obj-y := head.o als.o startup.o mem_detect.o ipl_parm.o string.o ebcdic.o -obj-y += sclp_early_core.o mem.o ipl_vmparm.o cmdline.o ctype.o -targets := bzImage startup.a section_cmp.boot.data $(obj-y) +obj-y := head.o als.o startup.o mem_detect.o ipl_parm.o ipl_report.o +obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o +obj-y += ctype.o text_dma.o +obj-$(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) += uv.o +obj-$(CONFIG_RELOCATABLE) += machine_kexec_reloc.o +obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o +targets := bzImage startup.a section_cmp.boot.data section_cmp.boot.preserved.data $(obj-y) subdir- := compressed OBJECTS := $(addprefix $(obj)/,$(obj-y)) @@ -48,7 +58,8 @@ define cmd_section_cmp touch $@ endef -$(obj)/bzImage: $(obj)/compressed/vmlinux $(obj)/section_cmp.boot.data FORCE +OBJCOPYFLAGS_bzImage := --pad-to $$(readelf -s $(obj)/compressed/vmlinux | awk '/\<_end\>/ {print or(strtonum("0x"$$2),4095)+1}') +$(obj)/bzImage: $(obj)/compressed/vmlinux $(obj)/section_cmp.boot.data $(obj)/section_cmp.boot.preserved.data FORCE $(call if_changed,objcopy) $(obj)/section_cmp%: vmlinux $(obj)/compressed/vmlinux FORCE diff --git a/arch/s390/boot/als.c b/arch/s390/boot/als.c index f902215e9cd9..ff6801d401c4 100644 --- a/arch/s390/boot/als.c +++ b/arch/s390/boot/als.c @@ -99,7 +99,7 @@ static void facility_mismatch(void) print_machine_type(); print_missing_facilities(); sclp_early_printk("See Principles of Operations for facility bits\n"); - disabled_wait(0x8badcccc); + disabled_wait(); } void verify_facilities(void) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 82bc06346e05..ad57c2205a71 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -9,5 +9,10 @@ void setup_boot_command_line(void); void parse_boot_command_line(void); void setup_memory_end(void); void print_missing_facilities(void); +unsigned long get_random_base(unsigned long safe_addr); + +extern int kaslr_enabled; + +unsigned long read_ipl_report(unsigned long safe_offset); #endif /* BOOT_BOOT_H */ diff --git a/arch/s390/boot/compressed/decompressor.h b/arch/s390/boot/compressed/decompressor.h index e1c1f2ec60f4..c15eb7114d83 100644 --- a/arch/s390/boot/compressed/decompressor.h +++ b/arch/s390/boot/compressed/decompressor.h @@ -17,6 +17,11 @@ struct vmlinux_info { unsigned long bss_size; /* uncompressed image .bss size */ unsigned long bootdata_off; unsigned long bootdata_size; + unsigned long bootdata_preserved_off; + unsigned long bootdata_preserved_size; + unsigned long dynsym_start; + unsigned long rela_dyn_start; + unsigned long rela_dyn_end; }; extern char _vmlinux_info[]; diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S index 7efc3938f595..112b8d9f1e4c 100644 --- a/arch/s390/boot/compressed/vmlinux.lds.S +++ b/arch/s390/boot/compressed/vmlinux.lds.S @@ -33,7 +33,29 @@ SECTIONS *(.data.*) _edata = . ; } + /* + * .dma section for code, data, ex_table that need to stay below 2 GB, + * even when the kernel is relocate: above 2 GB. + */ + _sdma = .; + .dma.text : { + . = ALIGN(PAGE_SIZE); + _stext_dma = .; + *(.dma.text) + . = ALIGN(PAGE_SIZE); + _etext_dma = .; + } + . = ALIGN(16); + .dma.ex_table : { + _start_dma_ex_table = .; + KEEP(*(.dma.ex_table)) + _stop_dma_ex_table = .; + } + .dma.data : { *(.dma.data) } + _edma = .; + BOOT_DATA + BOOT_DATA_PRESERVED /* * uncompressed image info used by the decompressor it should match diff --git a/arch/s390/boot/head.S b/arch/s390/boot/head.S index ce2cbbc41742..028aab03a9e7 100644 --- a/arch/s390/boot/head.S +++ b/arch/s390/boot/head.S @@ -305,7 +305,7 @@ ENTRY(startup_kdump) xc 0x300(256),0x300 xc 0xe00(256),0xe00 xc 0xf00(256),0xf00 - lctlg %c0,%c15,0x200(%r0) # initialize control registers + lctlg %c0,%c15,.Lctl-.LPG0(%r13) # load control registers stcke __LC_BOOT_CLOCK mvc __LC_LAST_UPDATE_CLOCK(8),__LC_BOOT_CLOCK+1 spt 6f-.LPG0(%r13) @@ -319,20 +319,54 @@ ENTRY(startup_kdump) .align 8 6: .long 0x7fffffff,0xffffffff +.Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space + .quad 0 # cr1: primary space segment table + .quad .Lduct # cr2: dispatchable unit control table + .quad 0 # cr3: instruction authorization + .quad 0xffff # cr4: instruction authorization + .quad .Lduct # cr5: primary-aste origin + .quad 0 # cr6: I/O interrupts + .quad 0 # cr7: secondary space segment table + .quad 0 # cr8: access registers translation + .quad 0 # cr9: tracing off + .quad 0 # cr10: tracing off + .quad 0 # cr11: tracing off + .quad 0 # cr12: tracing off + .quad 0 # cr13: home space segment table + .quad 0xc0000000 # cr14: machine check handling off + .quad .Llinkage_stack # cr15: linkage stack operations + + .section .dma.data,"aw",@progbits +.Lduct: .long 0,.Laste,.Laste,0,.Lduald,0,0,0 + .long 0,0,0,0,0,0,0,0 +.Llinkage_stack: + .long 0,0,0x89000000,0,0,0,0x8a000000,0 + .align 64 +.Laste: .quad 0,0xffffffffffffffff,0,0,0,0,0,0 + .align 128 +.Lduald:.rept 8 + .long 0x80000000,0,0,0 # invalid access-list entries + .endr + .previous + #include "head_kdump.S" # # params at 10400 (setup.h) +# Must be keept in sync with struct parmarea in setup.h # .org PARMAREA - .long 0,0 # IPL_DEVICE - .long 0,0 # INITRD_START - .long 0,0 # INITRD_SIZE - .long 0,0 # OLDMEM_BASE - .long 0,0 # OLDMEM_SIZE + .quad 0 # IPL_DEVICE + .quad 0 # INITRD_START + .quad 0 # INITRD_SIZE + .quad 0 # OLDMEM_BASE + .quad 0 # OLDMEM_SIZE .org COMMAND_LINE .byte "root=/dev/ram0 ro" .byte 0 - .org 0x11000 + .org EARLY_SCCB_OFFSET + .fill 4096 + + .org HEAD_END diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 36beb56de021..3c49bde8aa5e 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -7,16 +7,19 @@ #include <asm/sections.h> #include <asm/boot_data.h> #include <asm/facility.h> +#include <asm/uv.h> #include "boot.h" char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; -struct ipl_parameter_block __bootdata(early_ipl_block); -int __bootdata(early_ipl_block_valid); +struct ipl_parameter_block __bootdata_preserved(ipl_block); +int __bootdata_preserved(ipl_block_valid); unsigned long __bootdata(memory_end); int __bootdata(memory_end_set); int __bootdata(noexec_disabled); +int kaslr_enabled __section(.data); + static inline int __diag308(unsigned long subcode, void *addr) { register unsigned long _addr asm("0") = (unsigned long)addr; @@ -45,13 +48,15 @@ void store_ipl_parmblock(void) { int rc; - rc = __diag308(DIAG308_STORE, &early_ipl_block); + uv_set_shared(__pa(&ipl_block)); + rc = __diag308(DIAG308_STORE, &ipl_block); + uv_remove_shared(__pa(&ipl_block)); if (rc == DIAG308_RC_OK && - early_ipl_block.hdr.version <= IPL_MAX_SUPPORTED_VERSION) - early_ipl_block_valid = 1; + ipl_block.hdr.version <= IPL_MAX_SUPPORTED_VERSION) + ipl_block_valid = 1; } -static size_t scpdata_length(const char *buf, size_t count) +static size_t scpdata_length(const u8 *buf, size_t count) { while (count) { if (buf[count - 1] != '\0' && buf[count - 1] != ' ') @@ -68,26 +73,26 @@ static size_t ipl_block_get_ascii_scpdata(char *dest, size_t size, size_t i; int has_lowercase; - count = min(size - 1, scpdata_length(ipb->ipl_info.fcp.scp_data, - ipb->ipl_info.fcp.scp_data_len)); + count = min(size - 1, scpdata_length(ipb->fcp.scp_data, + ipb->fcp.scp_data_len)); if (!count) goto out; has_lowercase = 0; for (i = 0; i < count; i++) { - if (!isascii(ipb->ipl_info.fcp.scp_data[i])) { + if (!isascii(ipb->fcp.scp_data[i])) { count = 0; goto out; } - if (!has_lowercase && islower(ipb->ipl_info.fcp.scp_data[i])) + if (!has_lowercase && islower(ipb->fcp.scp_data[i])) has_lowercase = 1; } if (has_lowercase) - memcpy(dest, ipb->ipl_info.fcp.scp_data, count); + memcpy(dest, ipb->fcp.scp_data, count); else for (i = 0; i < count; i++) - dest[i] = tolower(ipb->ipl_info.fcp.scp_data[i]); + dest[i] = tolower(ipb->fcp.scp_data[i]); out: dest[count] = '\0'; return count; @@ -103,14 +108,14 @@ static void append_ipl_block_parm(void) delim = early_command_line + len; /* '\0' character position */ parm = early_command_line + len + 1; /* append right after '\0' */ - switch (early_ipl_block.hdr.pbt) { - case DIAG308_IPL_TYPE_CCW: + switch (ipl_block.pb0_hdr.pbt) { + case IPL_PBT_CCW: rc = ipl_block_get_ascii_vmparm( - parm, COMMAND_LINE_SIZE - len - 1, &early_ipl_block); + parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); break; - case DIAG308_IPL_TYPE_FCP: + case IPL_PBT_FCP: rc = ipl_block_get_ascii_scpdata( - parm, COMMAND_LINE_SIZE - len - 1, &early_ipl_block); + parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); break; } if (rc) { @@ -141,7 +146,7 @@ void setup_boot_command_line(void) strcpy(early_command_line, strim(COMMAND_LINE)); /* append IPL PARM data to the boot command line */ - if (early_ipl_block_valid) + if (!is_prot_virt_guest() && ipl_block_valid) append_ipl_block_parm(); } @@ -211,6 +216,7 @@ void parse_boot_command_line(void) char *args; int rc; + kaslr_enabled = IS_ENABLED(CONFIG_RANDOMIZE_BASE); args = strcpy(command_line_buf, early_command_line); while (*args) { args = next_arg(args, ¶m, &val); @@ -228,15 +234,21 @@ void parse_boot_command_line(void) if (!strcmp(param, "facilities")) modify_fac_list(val); + + if (!strcmp(param, "nokaslr")) + kaslr_enabled = 0; } } void setup_memory_end(void) { #ifdef CONFIG_CRASH_DUMP - if (!OLDMEM_BASE && early_ipl_block_valid && - early_ipl_block.hdr.pbt == DIAG308_IPL_TYPE_FCP && - early_ipl_block.ipl_info.fcp.opt == DIAG308_IPL_OPT_DUMP) { + if (OLDMEM_BASE) { + kaslr_enabled = 0; + } else if (ipl_block_valid && + ipl_block.pb0_hdr.pbt == IPL_PBT_FCP && + ipl_block.fcp.opt == IPL_PB0_FCP_OPT_DUMP) { + kaslr_enabled = 0; if (!sclp_early_get_hsa_size(&memory_end) && memory_end) memory_end_set = 1; } diff --git a/arch/s390/boot/ipl_report.c b/arch/s390/boot/ipl_report.c new file mode 100644 index 000000000000..0b4965573656 --- /dev/null +++ b/arch/s390/boot/ipl_report.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/init.h> +#include <linux/ctype.h> +#include <asm/ebcdic.h> +#include <asm/sclp.h> +#include <asm/sections.h> +#include <asm/boot_data.h> +#include <uapi/asm/ipl.h> +#include "boot.h" + +int __bootdata_preserved(ipl_secure_flag); + +unsigned long __bootdata_preserved(ipl_cert_list_addr); +unsigned long __bootdata_preserved(ipl_cert_list_size); + +unsigned long __bootdata(early_ipl_comp_list_addr); +unsigned long __bootdata(early_ipl_comp_list_size); + +#define for_each_rb_entry(entry, rb) \ + for (entry = rb->entries; \ + (void *) entry + sizeof(*entry) <= (void *) rb + rb->len; \ + entry++) + +static inline bool intersects(unsigned long addr0, unsigned long size0, + unsigned long addr1, unsigned long size1) +{ + return addr0 + size0 > addr1 && addr1 + size1 > addr0; +} + +static unsigned long find_bootdata_space(struct ipl_rb_components *comps, + struct ipl_rb_certificates *certs, + unsigned long safe_addr) +{ + struct ipl_rb_certificate_entry *cert; + struct ipl_rb_component_entry *comp; + size_t size; + + /* + * Find the length for the IPL report boot data + */ + early_ipl_comp_list_size = 0; + for_each_rb_entry(comp, comps) + early_ipl_comp_list_size += sizeof(*comp); + ipl_cert_list_size = 0; + for_each_rb_entry(cert, certs) + ipl_cert_list_size += sizeof(unsigned int) + cert->len; + size = ipl_cert_list_size + early_ipl_comp_list_size; + + /* + * Start from safe_addr to find a free memory area large + * enough for the IPL report boot data. This area is used + * for ipl_cert_list_addr/ipl_cert_list_size and + * early_ipl_comp_list_addr/early_ipl_comp_list_size. It must + * not overlap with any component or any certificate. + */ +repeat: + if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && INITRD_START && INITRD_SIZE && + intersects(INITRD_START, INITRD_SIZE, safe_addr, size)) + safe_addr = INITRD_START + INITRD_SIZE; + for_each_rb_entry(comp, comps) + if (intersects(safe_addr, size, comp->addr, comp->len)) { + safe_addr = comp->addr + comp->len; + goto repeat; + } + for_each_rb_entry(cert, certs) + if (intersects(safe_addr, size, cert->addr, cert->len)) { + safe_addr = cert->addr + cert->len; + goto repeat; + } + early_ipl_comp_list_addr = safe_addr; + ipl_cert_list_addr = safe_addr + early_ipl_comp_list_size; + + return safe_addr + size; +} + +static void copy_components_bootdata(struct ipl_rb_components *comps) +{ + struct ipl_rb_component_entry *comp, *ptr; + + ptr = (struct ipl_rb_component_entry *) early_ipl_comp_list_addr; + for_each_rb_entry(comp, comps) + memcpy(ptr++, comp, sizeof(*ptr)); +} + +static void copy_certificates_bootdata(struct ipl_rb_certificates *certs) +{ + struct ipl_rb_certificate_entry *cert; + void *ptr; + + ptr = (void *) ipl_cert_list_addr; + for_each_rb_entry(cert, certs) { + *(unsigned int *) ptr = cert->len; + ptr += sizeof(unsigned int); + memcpy(ptr, (void *) cert->addr, cert->len); + ptr += cert->len; + } +} + +unsigned long read_ipl_report(unsigned long safe_addr) +{ + struct ipl_rb_certificates *certs; + struct ipl_rb_components *comps; + struct ipl_pl_hdr *pl_hdr; + struct ipl_rl_hdr *rl_hdr; + struct ipl_rb_hdr *rb_hdr; + unsigned long tmp; + void *rl_end; + + /* + * Check if there is a IPL report by looking at the copy + * of the IPL parameter information block. + */ + if (!ipl_block_valid || + !(ipl_block.hdr.flags & IPL_PL_FLAG_IPLSR)) + return safe_addr; + ipl_secure_flag = !!(ipl_block.hdr.flags & IPL_PL_FLAG_SIPL); + /* + * There is an IPL report, to find it load the pointer to the + * IPL parameter information block from lowcore and skip past + * the IPL parameter list, then align the address to a double + * word boundary. + */ + tmp = (unsigned long) S390_lowcore.ipl_parmblock_ptr; + pl_hdr = (struct ipl_pl_hdr *) tmp; + tmp = (tmp + pl_hdr->len + 7) & -8UL; + rl_hdr = (struct ipl_rl_hdr *) tmp; + /* Walk through the IPL report blocks in the IPL Report list */ + certs = NULL; + comps = NULL; + rl_end = (void *) rl_hdr + rl_hdr->len; + rb_hdr = (void *) rl_hdr + sizeof(*rl_hdr); + while ((void *) rb_hdr + sizeof(*rb_hdr) < rl_end && + (void *) rb_hdr + rb_hdr->len <= rl_end) { + + switch (rb_hdr->rbt) { + case IPL_RBT_CERTIFICATES: + certs = (struct ipl_rb_certificates *) rb_hdr; + break; + case IPL_RBT_COMPONENTS: + comps = (struct ipl_rb_components *) rb_hdr; + break; + default: + break; + } + + rb_hdr = (void *) rb_hdr + rb_hdr->len; + } + + /* + * With either the component list or the certificate list + * missing the kernel will stay ignorant of secure IPL. + */ + if (!comps || !certs) + return safe_addr; + + /* + * Copy component and certificate list to a safe area + * where the decompressed kernel can find them. + */ + safe_addr = find_bootdata_space(comps, certs, safe_addr); + copy_components_bootdata(comps); + copy_certificates_bootdata(certs); + + return safe_addr; +} diff --git a/arch/s390/boot/kaslr.c b/arch/s390/boot/kaslr.c new file mode 100644 index 000000000000..3bdd8132e56b --- /dev/null +++ b/arch/s390/boot/kaslr.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 2019 + */ +#include <asm/mem_detect.h> +#include <asm/cpacf.h> +#include <asm/timex.h> +#include <asm/sclp.h> +#include "compressed/decompressor.h" + +#define PRNG_MODE_TDES 1 +#define PRNG_MODE_SHA512 2 +#define PRNG_MODE_TRNG 3 + +struct prno_parm { + u32 res; + u32 reseed_counter; + u64 stream_bytes; + u8 V[112]; + u8 C[112]; +}; + +struct prng_parm { + u8 parm_block[32]; + u32 reseed_counter; + u64 byte_counter; +}; + +static int check_prng(void) +{ + if (!cpacf_query_func(CPACF_KMC, CPACF_KMC_PRNG)) { + sclp_early_printk("KASLR disabled: CPU has no PRNG\n"); + return 0; + } + if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) + return PRNG_MODE_TRNG; + if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_SHA512_DRNG_GEN)) + return PRNG_MODE_SHA512; + else + return PRNG_MODE_TDES; +} + +static unsigned long get_random(unsigned long limit) +{ + struct prng_parm prng = { + /* initial parameter block for tdes mode, copied from libica */ + .parm_block = { + 0x0F, 0x2B, 0x8E, 0x63, 0x8C, 0x8E, 0xD2, 0x52, + 0x64, 0xB7, 0xA0, 0x7B, 0x75, 0x28, 0xB8, 0xF4, + 0x75, 0x5F, 0xD2, 0xA6, 0x8D, 0x97, 0x11, 0xFF, + 0x49, 0xD8, 0x23, 0xF3, 0x7E, 0x21, 0xEC, 0xA0 + }, + }; + unsigned long seed, random; + struct prno_parm prno; + __u64 entropy[4]; + int mode, i; + + mode = check_prng(); + seed = get_tod_clock_fast(); + switch (mode) { + case PRNG_MODE_TRNG: + cpacf_trng(NULL, 0, (u8 *) &random, sizeof(random)); + break; + case PRNG_MODE_SHA512: + cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, &prno, NULL, 0, + (u8 *) &seed, sizeof(seed)); + cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN, &prno, (u8 *) &random, + sizeof(random), NULL, 0); + break; + case PRNG_MODE_TDES: + /* add entropy */ + *(unsigned long *) prng.parm_block ^= seed; + for (i = 0; i < 16; i++) { + cpacf_kmc(CPACF_KMC_PRNG, prng.parm_block, + (char *) entropy, (char *) entropy, + sizeof(entropy)); + memcpy(prng.parm_block, entropy, sizeof(entropy)); + } + random = seed; + cpacf_kmc(CPACF_KMC_PRNG, prng.parm_block, (u8 *) &random, + (u8 *) &random, sizeof(random)); + break; + default: + random = 0; + } + return random % limit; +} + +unsigned long get_random_base(unsigned long safe_addr) +{ + unsigned long base, start, end, kernel_size; + unsigned long block_sum, offset; + int i; + + if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && INITRD_START && INITRD_SIZE) { + if (safe_addr < INITRD_START + INITRD_SIZE) + safe_addr = INITRD_START + INITRD_SIZE; + } + safe_addr = ALIGN(safe_addr, THREAD_SIZE); + + kernel_size = vmlinux.image_size + vmlinux.bss_size; + block_sum = 0; + for_each_mem_detect_block(i, &start, &end) { + if (memory_end_set) { + if (start >= memory_end) + break; + if (end > memory_end) + end = memory_end; + } + if (end - start < kernel_size) + continue; + block_sum += end - start - kernel_size; + } + if (!block_sum) { + sclp_early_printk("KASLR disabled: not enough memory\n"); + return 0; + } + + base = get_random(block_sum); + if (base == 0) + return 0; + if (base < safe_addr) + base = safe_addr; + block_sum = offset = 0; + for_each_mem_detect_block(i, &start, &end) { + if (memory_end_set) { + if (start >= memory_end) + break; + if (end > memory_end) + end = memory_end; + } + if (end - start < kernel_size) + continue; + block_sum += end - start - kernel_size; + if (base <= block_sum) { + base = start + base - offset; + base = ALIGN_DOWN(base, THREAD_SIZE); + break; + } + offset = block_sum; + } + return base; +} diff --git a/arch/s390/boot/machine_kexec_reloc.c b/arch/s390/boot/machine_kexec_reloc.c new file mode 100644 index 000000000000..b7a5d0f72097 --- /dev/null +++ b/arch/s390/boot/machine_kexec_reloc.c @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "../kernel/machine_kexec_reloc.c" diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index bdfc5549a299..7b0d05414618 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -1,11 +1,55 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/string.h> +#include <linux/elf.h> +#include <asm/sections.h> #include <asm/setup.h> +#include <asm/kexec.h> #include <asm/sclp.h> +#include <asm/diag.h> +#include <asm/uv.h> #include "compressed/decompressor.h" #include "boot.h" extern char __boot_data_start[], __boot_data_end[]; +extern char __boot_data_preserved_start[], __boot_data_preserved_end[]; +unsigned long __bootdata_preserved(__kaslr_offset); + +/* + * Some code and data needs to stay below 2 GB, even when the kernel would be + * relocated above 2 GB, because it has to use 31 bit addresses. + * Such code and data is part of the .dma section, and its location is passed + * over to the decompressed / relocated kernel via the .boot.preserved.data + * section. + */ +extern char _sdma[], _edma[]; +extern char _stext_dma[], _etext_dma[]; +extern struct exception_table_entry _start_dma_ex_table[]; +extern struct exception_table_entry _stop_dma_ex_table[]; +unsigned long __bootdata_preserved(__sdma) = __pa(&_sdma); +unsigned long __bootdata_preserved(__edma) = __pa(&_edma); +unsigned long __bootdata_preserved(__stext_dma) = __pa(&_stext_dma); +unsigned long __bootdata_preserved(__etext_dma) = __pa(&_etext_dma); +struct exception_table_entry * + __bootdata_preserved(__start_dma_ex_table) = _start_dma_ex_table; +struct exception_table_entry * + __bootdata_preserved(__stop_dma_ex_table) = _stop_dma_ex_table; + +int _diag210_dma(struct diag210 *addr); +int _diag26c_dma(void *req, void *resp, enum diag26c_sc subcode); +int _diag14_dma(unsigned long rx, unsigned long ry1, unsigned long subcode); +void _diag0c_dma(struct hypfs_diag0c_entry *entry); +void _diag308_reset_dma(void); +struct diag_ops __bootdata_preserved(diag_dma_ops) = { + .diag210 = _diag210_dma, + .diag26c = _diag26c_dma, + .diag14 = _diag14_dma, + .diag0c = _diag0c_dma, + .diag308_reset = _diag308_reset_dma +}; +static struct diag210 _diag210_tmp_dma __section(".dma.data"); +struct diag210 *__bootdata_preserved(__diag210_tmp_dma) = &_diag210_tmp_dma; +void _swsusp_reset_dma(void); +unsigned long __bootdata_preserved(__swsusp_reset_dma) = __pa(_swsusp_reset_dma); void error(char *x) { @@ -13,7 +57,7 @@ void error(char *x) sclp_early_printk(x); sclp_early_printk("\n\n -- System halted"); - disabled_wait(0xdeadbeef); + disabled_wait(); } #ifdef CONFIG_KERNEL_UNCOMPRESSED @@ -23,19 +67,16 @@ unsigned long mem_safe_offset(void) } #endif -static void rescue_initrd(void) +static void rescue_initrd(unsigned long addr) { - unsigned long min_initrd_addr; - if (!IS_ENABLED(CONFIG_BLK_DEV_INITRD)) return; if (!INITRD_START || !INITRD_SIZE) return; - min_initrd_addr = mem_safe_offset(); - if (min_initrd_addr <= INITRD_START) + if (addr <= INITRD_START) return; - memmove((void *)min_initrd_addr, (void *)INITRD_START, INITRD_SIZE); - INITRD_START = min_initrd_addr; + memmove((void *)addr, (void *)INITRD_START, INITRD_SIZE); + INITRD_START = addr; } static void copy_bootdata(void) @@ -43,23 +84,81 @@ static void copy_bootdata(void) if (__boot_data_end - __boot_data_start != vmlinux.bootdata_size) error(".boot.data section size mismatch"); memcpy((void *)vmlinux.bootdata_off, __boot_data_start, vmlinux.bootdata_size); + if (__boot_data_preserved_end - __boot_data_preserved_start != vmlinux.bootdata_preserved_size) + error(".boot.preserved.data section size mismatch"); + memcpy((void *)vmlinux.bootdata_preserved_off, __boot_data_preserved_start, vmlinux.bootdata_preserved_size); +} + +static void handle_relocs(unsigned long offset) +{ + Elf64_Rela *rela_start, *rela_end, *rela; + int r_type, r_sym, rc; + Elf64_Addr loc, val; + Elf64_Sym *dynsym; + + rela_start = (Elf64_Rela *) vmlinux.rela_dyn_start; + rela_end = (Elf64_Rela *) vmlinux.rela_dyn_end; + dynsym = (Elf64_Sym *) vmlinux.dynsym_start; + for (rela = rela_start; rela < rela_end; rela++) { + loc = rela->r_offset + offset; + val = rela->r_addend + offset; + r_sym = ELF64_R_SYM(rela->r_info); + if (r_sym) + val += dynsym[r_sym].st_value; + r_type = ELF64_R_TYPE(rela->r_info); + rc = arch_kexec_do_relocs(r_type, (void *) loc, val, 0); + if (rc) + error("Unknown relocation type"); + } } void startup_kernel(void) { + unsigned long random_lma; + unsigned long safe_addr; void *img; - rescue_initrd(); - sclp_early_read_info(); store_ipl_parmblock(); + safe_addr = mem_safe_offset(); + safe_addr = read_ipl_report(safe_addr); + uv_query_info(); + rescue_initrd(safe_addr); + sclp_early_read_info(); setup_boot_command_line(); parse_boot_command_line(); setup_memory_end(); detect_memory(); + + random_lma = __kaslr_offset = 0; + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && kaslr_enabled) { + random_lma = get_random_base(safe_addr); + if (random_lma) { + __kaslr_offset = random_lma - vmlinux.default_lma; + img = (void *)vmlinux.default_lma; + vmlinux.default_lma += __kaslr_offset; + vmlinux.entry += __kaslr_offset; + vmlinux.bootdata_off += __kaslr_offset; + vmlinux.bootdata_preserved_off += __kaslr_offset; + vmlinux.rela_dyn_start += __kaslr_offset; + vmlinux.rela_dyn_end += __kaslr_offset; + vmlinux.dynsym_start += __kaslr_offset; + } + } + if (!IS_ENABLED(CONFIG_KERNEL_UNCOMPRESSED)) { img = decompress_kernel(); memmove((void *)vmlinux.default_lma, img, vmlinux.image_size); - } + } else if (__kaslr_offset) + memcpy((void *)vmlinux.default_lma, img, vmlinux.image_size); + copy_bootdata(); + if (IS_ENABLED(CONFIG_RELOCATABLE)) + handle_relocs(__kaslr_offset); + + if (__kaslr_offset) { + /* Clear non-relocated kernel */ + if (IS_ENABLED(CONFIG_KERNEL_UNCOMPRESSED)) + memset(img, 0, vmlinux.image_size); + } vmlinux.entry(); } diff --git a/arch/s390/boot/text_dma.S b/arch/s390/boot/text_dma.S new file mode 100644 index 000000000000..9715715c4c28 --- /dev/null +++ b/arch/s390/boot/text_dma.S @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Code that needs to run below 2 GB. + * + * Copyright IBM Corp. 2019 + */ + +#include <linux/linkage.h> +#include <asm/errno.h> +#include <asm/sigp.h> + +#ifdef CC_USING_EXPOLINE + .pushsection .dma.text.__s390_indirect_jump_r14,"axG" +__dma__s390_indirect_jump_r14: + larl %r1,0f + ex 0,0(%r1) + j . +0: br %r14 + .popsection +#endif + + .section .dma.text,"ax" +/* + * Simplified version of expoline thunk. The normal thunks can not be used here, + * because they might be more than 2 GB away, and not reachable by the relative + * branch. No comdat, exrl, etc. optimizations used here, because it only + * affects a few functions that are not performance-relevant. + */ + .macro BR_EX_DMA_r14 +#ifdef CC_USING_EXPOLINE + jg __dma__s390_indirect_jump_r14 +#else + br %r14 +#endif + .endm + +/* + * int _diag14_dma(unsigned long rx, unsigned long ry1, unsigned long subcode) + */ +ENTRY(_diag14_dma) + lgr %r1,%r2 + lgr %r2,%r3 + lgr %r3,%r4 + lhi %r5,-EIO + sam31 + diag %r1,%r2,0x14 +.Ldiag14_ex: + ipm %r5 + srl %r5,28 +.Ldiag14_fault: + sam64 + lgfr %r2,%r5 + BR_EX_DMA_r14 + EX_TABLE_DMA(.Ldiag14_ex, .Ldiag14_fault) +ENDPROC(_diag14_dma) + +/* + * int _diag210_dma(struct diag210 *addr) + */ +ENTRY(_diag210_dma) + lgr %r1,%r2 + lhi %r2,-1 + sam31 + diag %r1,%r0,0x210 +.Ldiag210_ex: + ipm %r2 + srl %r2,28 +.Ldiag210_fault: + sam64 + lgfr %r2,%r2 + BR_EX_DMA_r14 + EX_TABLE_DMA(.Ldiag210_ex, .Ldiag210_fault) +ENDPROC(_diag210_dma) + +/* + * int _diag26c_dma(void *req, void *resp, enum diag26c_sc subcode) + */ +ENTRY(_diag26c_dma) + lghi %r5,-EOPNOTSUPP + sam31 + diag %r2,%r4,0x26c +.Ldiag26c_ex: + sam64 + lgfr %r2,%r5 + BR_EX_DMA_r14 + EX_TABLE_DMA(.Ldiag26c_ex, .Ldiag26c_ex) +ENDPROC(_diag26c_dma) + +/* + * void _diag0c_dma(struct hypfs_diag0c_entry *entry) + */ +ENTRY(_diag0c_dma) + sam31 + diag %r2,%r2,0x0c + sam64 + BR_EX_DMA_r14 +ENDPROC(_diag0c_dma) + +/* + * void _swsusp_reset_dma(void) + */ +ENTRY(_swsusp_reset_dma) + larl %r1,restart_entry + larl %r2,.Lrestart_diag308_psw + og %r1,0(%r2) + stg %r1,0(%r0) + lghi %r0,0 + diag %r0,%r0,0x308 +restart_entry: + lhi %r1,1 + sigp %r1,%r0,SIGP_SET_ARCHITECTURE + sam64 + BR_EX_DMA_r14 +ENDPROC(_swsusp_reset_dma) + +/* + * void _diag308_reset_dma(void) + * + * Calls diag 308 subcode 1 and continues execution + */ +ENTRY(_diag308_reset_dma) + larl %r4,.Lctlregs # Save control registers + stctg %c0,%c15,0(%r4) + lg %r2,0(%r4) # Disable lowcore protection + nilh %r2,0xefff + larl %r4,.Lctlreg0 + stg %r2,0(%r4) + lctlg %c0,%c0,0(%r4) + larl %r4,.Lfpctl # Floating point control register + stfpc 0(%r4) + larl %r4,.Lprefix # Save prefix register + stpx 0(%r4) + larl %r4,.Lprefix_zero # Set prefix register to 0 + spx 0(%r4) + larl %r4,.Lcontinue_psw # Save PSW flags + epsw %r2,%r3 + stm %r2,%r3,0(%r4) + larl %r4,restart_part2 # Setup restart PSW at absolute 0 + larl %r3,.Lrestart_diag308_psw + og %r4,0(%r3) # Save PSW + lghi %r3,0 + sturg %r4,%r3 # Use sturg, because of large pages + lghi %r1,1 + lghi %r0,0 + diag %r0,%r1,0x308 +restart_part2: + lhi %r0,0 # Load r0 with zero + lhi %r1,2 # Use mode 2 = ESAME (dump) + sigp %r1,%r0,SIGP_SET_ARCHITECTURE # Switch to ESAME mode + sam64 # Switch to 64 bit addressing mode + larl %r4,.Lctlregs # Restore control registers + lctlg %c0,%c15,0(%r4) + larl %r4,.Lfpctl # Restore floating point ctl register + lfpc 0(%r4) + larl %r4,.Lprefix # Restore prefix register + spx 0(%r4) + larl %r4,.Lcontinue_psw # Restore PSW flags + lpswe 0(%r4) +.Lcontinue: + BR_EX_DMA_r14 +ENDPROC(_diag308_reset_dma) + + .section .dma.data,"aw",@progbits +.align 8 +.Lrestart_diag308_psw: + .long 0x00080000,0x80000000 + +.align 8 +.Lcontinue_psw: + .quad 0,.Lcontinue + +.align 8 +.Lctlreg0: + .quad 0 +.Lctlregs: + .rept 16 + .quad 0 + .endr +.Lfpctl: + .long 0 +.Lprefix: + .long 0 +.Lprefix_zero: + .long 0 diff --git a/arch/s390/boot/uv.c b/arch/s390/boot/uv.c new file mode 100644 index 000000000000..ed007f4a6444 --- /dev/null +++ b/arch/s390/boot/uv.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <asm/uv.h> +#include <asm/facility.h> +#include <asm/sections.h> + +int __bootdata_preserved(prot_virt_guest); + +void uv_query_info(void) +{ + struct uv_cb_qui uvcb = { + .header.cmd = UVC_CMD_QUI, + .header.len = sizeof(uvcb) + }; + + if (!test_facility(158)) + return; + + if (uv_call(0, (uint64_t)&uvcb)) + return; + + if (test_bit_inv(BIT_UVC_CMD_SET_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list) && + test_bit_inv(BIT_UVC_CMD_REMOVE_SHARED_ACCESS, (unsigned long *)uvcb.inst_calls_list)) + prot_virt_guest = 1; +} diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 9824c7bad9d4..b0920b35f87b 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -64,6 +64,7 @@ CONFIG_NUMA=y CONFIG_PREEMPT=y CONFIG_HZ_100=y CONFIG_KEXEC_FILE=y +CONFIG_KEXEC_VERIFY_SIG=y CONFIG_EXPOLINE=y CONFIG_EXPOLINE_AUTO=y CONFIG_MEMORY_HOTPLUG=y diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig index 4fcbe5792744..09aa5cb14873 100644 --- a/arch/s390/configs/performance_defconfig +++ b/arch/s390/configs/performance_defconfig @@ -65,6 +65,7 @@ CONFIG_NR_CPUS=512 CONFIG_NUMA=y CONFIG_HZ_100=y CONFIG_KEXEC_FILE=y +CONFIG_KEXEC_VERIFY_SIG=y CONFIG_EXPOLINE=y CONFIG_EXPOLINE_AUTO=y CONFIG_MEMORY_HOTPLUG=y diff --git a/arch/s390/crypto/crc32be-vx.S b/arch/s390/crypto/crc32be-vx.S index 2bf01ba44107..0099044e2c86 100644 --- a/arch/s390/crypto/crc32be-vx.S +++ b/arch/s390/crypto/crc32be-vx.S @@ -207,5 +207,6 @@ ENTRY(crc32_be_vgfm_16) .Ldone: VLGVF %r2,%v2,3 BR_EX %r14 +ENDPROC(crc32_be_vgfm_16) .previous diff --git a/arch/s390/crypto/crc32le-vx.S b/arch/s390/crypto/crc32le-vx.S index 7d6f568bd3ad..71caf0f4ec08 100644 --- a/arch/s390/crypto/crc32le-vx.S +++ b/arch/s390/crypto/crc32le-vx.S @@ -105,13 +105,14 @@ ENTRY(crc32_le_vgfm_16) larl %r5,.Lconstants_CRC_32_LE j crc32_le_vgfm_generic +ENDPROC(crc32_le_vgfm_16) ENTRY(crc32c_le_vgfm_16) larl %r5,.Lconstants_CRC_32C_LE j crc32_le_vgfm_generic +ENDPROC(crc32c_le_vgfm_16) - -crc32_le_vgfm_generic: +ENTRY(crc32_le_vgfm_generic) /* Load CRC-32 constants */ VLM CONST_PERM_LE2BE,CONST_CRC_POLY,0,%r5 @@ -267,5 +268,6 @@ crc32_le_vgfm_generic: .Ldone: VLGVF %r2,%v2,2 BR_EX %r14 +ENDPROC(crc32_le_vgfm_generic) .previous diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index a97a1802cfb4..12cca467af7d 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -61,6 +61,7 @@ static unsigned int prng_reseed_limit; module_param_named(reseed_limit, prng_reseed_limit, int, 0); MODULE_PARM_DESC(prng_reseed_limit, "PRNG reseed limit"); +static bool trng_available; /* * Any one who considers arithmetical methods of producing random digits is, @@ -115,46 +116,68 @@ static const u8 initial_parm_block[32] __initconst = { /* * generate_entropy: - * This algorithm produces 64 bytes of entropy data based on 1024 - * individual stckf() invocations assuming that each stckf() value - * contributes 0.25 bits of entropy. So the caller gets 256 bit - * entropy per 64 byte or 4 bits entropy per byte. + * This function fills a given buffer with random bytes. The entropy within + * the random bytes given back is assumed to have at least 50% - meaning + * a 64 bytes buffer has at least 64 * 8 / 2 = 256 bits of entropy. + * Within the function the entropy generation is done in junks of 64 bytes. + * So the caller should also ask for buffer fill in multiples of 64 bytes. + * The generation of the entropy is based on the assumption that every stckf() + * invocation produces 0.5 bits of entropy. To accumulate 256 bits of entropy + * at least 512 stckf() values are needed. The entropy relevant part of the + * stckf value is bit 51 (counting starts at the left with bit nr 0) so + * here we use the lower 4 bytes and exor the values into 2k of bufferspace. + * To be on the save side, if there is ever a problem with stckf() the + * other half of the page buffer is filled with bytes from urandom via + * get_random_bytes(), so this function consumes 2k of urandom for each + * requested 64 bytes output data. Finally the buffer page is condensed into + * a 64 byte value by hashing with a SHA512 hash. */ static int generate_entropy(u8 *ebuf, size_t nbytes) { int n, ret = 0; - u8 *pg, *h, hash[64]; - - /* allocate 2 pages */ - pg = (u8 *) __get_free_pages(GFP_KERNEL, 1); + u8 *pg, pblock[80] = { + /* 8 x 64 bit init values */ + 0x6A, 0x09, 0xE6, 0x67, 0xF3, 0xBC, 0xC9, 0x08, + 0xBB, 0x67, 0xAE, 0x85, 0x84, 0xCA, 0xA7, 0x3B, + 0x3C, 0x6E, 0xF3, 0x72, 0xFE, 0x94, 0xF8, 0x2B, + 0xA5, 0x4F, 0xF5, 0x3A, 0x5F, 0x1D, 0x36, 0xF1, + 0x51, 0x0E, 0x52, 0x7F, 0xAD, 0xE6, 0x82, 0xD1, + 0x9B, 0x05, 0x68, 0x8C, 0x2B, 0x3E, 0x6C, 0x1F, + 0x1F, 0x83, 0xD9, 0xAB, 0xFB, 0x41, 0xBD, 0x6B, + 0x5B, 0xE0, 0xCD, 0x19, 0x13, 0x7E, 0x21, 0x79, + /* 128 bit counter total message bit length */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 }; + + /* allocate one page stckf buffer */ + pg = (u8 *) __get_free_page(GFP_KERNEL); if (!pg) { prng_errorflag = PRNG_GEN_ENTROPY_FAILED; return -ENOMEM; } + /* fill the ebuf in chunks of 64 byte each */ while (nbytes) { - /* fill pages with urandom bytes */ - get_random_bytes(pg, 2*PAGE_SIZE); - /* exor pages with 1024 stckf values */ - for (n = 0; n < 2 * PAGE_SIZE / sizeof(u64); n++) { - u64 *p = ((u64 *)pg) + n; + /* fill lower 2k with urandom bytes */ + get_random_bytes(pg, PAGE_SIZE / 2); + /* exor upper 2k with 512 stckf values, offset 4 bytes each */ + for (n = 0; n < 512; n++) { + int offset = (PAGE_SIZE / 2) + (n * 4) - 4; + u64 *p = (u64 *)(pg + offset); *p ^= get_tod_clock_fast(); } - n = (nbytes < sizeof(hash)) ? nbytes : sizeof(hash); - if (n < sizeof(hash)) - h = hash; - else - h = ebuf; - /* hash over the filled pages */ - cpacf_kimd(CPACF_KIMD_SHA_512, h, pg, 2*PAGE_SIZE); - if (n < sizeof(hash)) - memcpy(ebuf, hash, n); + /* hash over the filled page */ + cpacf_klmd(CPACF_KLMD_SHA_512, pblock, pg, PAGE_SIZE); + n = (nbytes < 64) ? nbytes : 64; + memcpy(ebuf, pblock, n); ret += n; ebuf += n; nbytes -= n; } - free_pages((unsigned long)pg, 1); + memzero_explicit(pblock, sizeof(pblock)); + memzero_explicit(pg, PAGE_SIZE); + free_page((unsigned long)pg); return ret; } @@ -344,8 +367,8 @@ static int __init prng_sha512_selftest(void) static int __init prng_sha512_instantiate(void) { - int ret, datalen; - u8 seed[64 + 32 + 16]; + int ret, datalen, seedlen; + u8 seed[128 + 16]; pr_debug("prng runs in SHA-512 mode " "with chunksize=%d and reseed_limit=%u\n", @@ -368,16 +391,36 @@ static int __init prng_sha512_instantiate(void) if (ret) goto outfree; - /* generate initial seed bytestring, with 256 + 128 bits entropy */ - ret = generate_entropy(seed, 64 + 32); - if (ret != 64 + 32) - goto outfree; - /* followed by 16 bytes of unique nonce */ - get_tod_clock_ext(seed + 64 + 32); + /* generate initial seed, we need at least 256 + 128 bits entropy. */ + if (trng_available) { + /* + * Trng available, so use it. The trng works in chunks of + * 32 bytes and produces 100% entropy. So we pull 64 bytes + * which gives us 512 bits entropy. + */ + seedlen = 2 * 32; + cpacf_trng(NULL, 0, seed, seedlen); + } else { + /* + * No trng available, so use the generate_entropy() function. + * This function works in 64 byte junks and produces + * 50% entropy. So we pull 2*64 bytes which gives us 512 bits + * of entropy. + */ + seedlen = 2 * 64; + ret = generate_entropy(seed, seedlen); + if (ret != seedlen) + goto outfree; + } + + /* append the seed by 16 bytes of unique nonce */ + get_tod_clock_ext(seed + seedlen); + seedlen += 16; - /* initial seed of the prno drng */ + /* now initial seed of the prno drng */ cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, - &prng_data->prnows, NULL, 0, seed, sizeof(seed)); + &prng_data->prnows, NULL, 0, seed, seedlen); + memzero_explicit(seed, sizeof(seed)); /* if fips mode is enabled, generate a first block of random bytes for the FIPS 140-2 Conditional Self Test */ @@ -405,17 +448,26 @@ static void prng_sha512_deinstantiate(void) static int prng_sha512_reseed(void) { - int ret; + int ret, seedlen; u8 seed[64]; - /* fetch 256 bits of fresh entropy */ - ret = generate_entropy(seed, sizeof(seed)); - if (ret != sizeof(seed)) - return ret; + /* We need at least 256 bits of fresh entropy for reseeding */ + if (trng_available) { + /* trng produces 256 bits entropy in 32 bytes */ + seedlen = 32; + cpacf_trng(NULL, 0, seed, seedlen); + } else { + /* generate_entropy() produces 256 bits entropy in 64 bytes */ + seedlen = 64; + ret = generate_entropy(seed, seedlen); + if (ret != sizeof(seed)) + return ret; + } /* do a reseed of the prno drng with this bytestring */ cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, - &prng_data->prnows, NULL, 0, seed, sizeof(seed)); + &prng_data->prnows, NULL, 0, seed, seedlen); + memzero_explicit(seed, sizeof(seed)); return 0; } @@ -592,6 +644,7 @@ static ssize_t prng_sha512_read(struct file *file, char __user *ubuf, ret = -EFAULT; break; } + memzero_explicit(p, n); ubuf += n; nbytes -= n; ret += n; @@ -773,6 +826,10 @@ static int __init prng_init(void) if (!cpacf_query_func(CPACF_KMC, CPACF_KMC_PRNG)) return -EOPNOTSUPP; + /* check if TRNG subfunction is available */ + if (cpacf_query_func(CPACF_PRNO, CPACF_PRNO_TRNG)) + trng_available = true; + /* choose prng mode */ if (prng_mode != PRNG_MODE_TDES) { /* check for MSA5 support for PRNO operations */ diff --git a/arch/s390/defconfig b/arch/s390/defconfig index 4d58a92b5d97..c59b922cb6c5 100644 --- a/arch/s390/defconfig +++ b/arch/s390/defconfig @@ -39,6 +39,7 @@ CONFIG_NR_CPUS=256 CONFIG_NUMA=y CONFIG_HZ_100=y CONFIG_KEXEC_FILE=y +CONFIG_KEXEC_VERIFY_SIG=y CONFIG_CRASH_DUMP=y CONFIG_HIBERNATION=y CONFIG_PM_DEBUG=y diff --git a/arch/s390/hypfs/hypfs_diag0c.c b/arch/s390/hypfs/hypfs_diag0c.c index 72e3140fafb5..3235e4d82f2d 100644 --- a/arch/s390/hypfs/hypfs_diag0c.c +++ b/arch/s390/hypfs/hypfs_diag0c.c @@ -16,26 +16,12 @@ #define DBFS_D0C_HDR_VERSION 0 /* - * Execute diagnose 0c in 31 bit mode - */ -static void diag0c(struct hypfs_diag0c_entry *entry) -{ - diag_stat_inc(DIAG_STAT_X00C); - asm volatile ( - " sam31\n" - " diag %0,%0,0x0c\n" - " sam64\n" - : /* no output register */ - : "a" (entry) - : "memory"); -} - -/* * Get hypfs_diag0c_entry from CPU vector and store diag0c data */ static void diag0c_fn(void *data) { - diag0c(((void **) data)[smp_processor_id()]); + diag_stat_inc(DIAG_STAT_X00C); + diag_dma_ops.diag0c(((void **) data)[smp_processor_id()]); } /* diff --git a/arch/s390/include/asm/airq.h b/arch/s390/include/asm/airq.h index fcf539efb32f..c10d2ee2dfda 100644 --- a/arch/s390/include/asm/airq.h +++ b/arch/s390/include/asm/airq.h @@ -14,7 +14,7 @@ struct airq_struct { struct hlist_node list; /* Handler queueing. */ - void (*handler)(struct airq_struct *); /* Thin-interrupt handler */ + void (*handler)(struct airq_struct *airq, bool floating); u8 *lsi_ptr; /* Local-Summary-Indicator pointer */ u8 lsi_mask; /* Local-Summary-Indicator mask */ u8 isc; /* Interrupt-subclass */ @@ -35,13 +35,15 @@ struct airq_iv { unsigned int *data; /* 32 bit value associated with each bit */ unsigned long bits; /* Number of bits in the vector */ unsigned long end; /* Number of highest allocated bit + 1 */ + unsigned long flags; /* Allocation flags */ spinlock_t lock; /* Lock to protect alloc & free */ }; -#define AIRQ_IV_ALLOC 1 /* Use an allocation bit mask */ -#define AIRQ_IV_BITLOCK 2 /* Allocate the lock bit mask */ -#define AIRQ_IV_PTR 4 /* Allocate the ptr array */ -#define AIRQ_IV_DATA 8 /* Allocate the data array */ +#define AIRQ_IV_ALLOC 1 /* Use an allocation bit mask */ +#define AIRQ_IV_BITLOCK 2 /* Allocate the lock bit mask */ +#define AIRQ_IV_PTR 4 /* Allocate the ptr array */ +#define AIRQ_IV_DATA 8 /* Allocate the data array */ +#define AIRQ_IV_CACHELINE 16 /* Cacheline alignment for the vector */ struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags); void airq_iv_release(struct airq_iv *iv); diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index d1f8a4d94cca..9900d655014c 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -73,7 +73,7 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *ptr) } #endif mask = 1UL << (nr & (BITS_PER_LONG - 1)); - __atomic64_or(mask, addr); + __atomic64_or(mask, (long *)addr); } static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr) @@ -94,7 +94,7 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr) } #endif mask = ~(1UL << (nr & (BITS_PER_LONG - 1))); - __atomic64_and(mask, addr); + __atomic64_and(mask, (long *)addr); } static inline void change_bit(unsigned long nr, volatile unsigned long *ptr) @@ -115,7 +115,7 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *ptr) } #endif mask = 1UL << (nr & (BITS_PER_LONG - 1)); - __atomic64_xor(mask, addr); + __atomic64_xor(mask, (long *)addr); } static inline int @@ -125,7 +125,7 @@ test_and_set_bit(unsigned long nr, volatile unsigned long *ptr) unsigned long old, mask; mask = 1UL << (nr & (BITS_PER_LONG - 1)); - old = __atomic64_or_barrier(mask, addr); + old = __atomic64_or_barrier(mask, (long *)addr); return (old & mask) != 0; } @@ -136,7 +136,7 @@ test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr) unsigned long old, mask; mask = ~(1UL << (nr & (BITS_PER_LONG - 1))); - old = __atomic64_and_barrier(mask, addr); + old = __atomic64_and_barrier(mask, (long *)addr); return (old & ~mask) != 0; } @@ -147,7 +147,7 @@ test_and_change_bit(unsigned long nr, volatile unsigned long *ptr) unsigned long old, mask; mask = 1UL << (nr & (BITS_PER_LONG - 1)); - old = __atomic64_xor_barrier(mask, addr); + old = __atomic64_xor_barrier(mask, (long *)addr); return (old & mask) != 0; } diff --git a/arch/s390/include/asm/boot_data.h b/arch/s390/include/asm/boot_data.h index 2d999ccb977a..f7eed27b3220 100644 --- a/arch/s390/include/asm/boot_data.h +++ b/arch/s390/include/asm/boot_data.h @@ -5,7 +5,14 @@ #include <asm/ipl.h> extern char early_command_line[COMMAND_LINE_SIZE]; -extern struct ipl_parameter_block early_ipl_block; -extern int early_ipl_block_valid; +extern struct ipl_parameter_block ipl_block; +extern int ipl_block_valid; +extern int ipl_secure_flag; + +extern unsigned long ipl_cert_list_addr; +extern unsigned long ipl_cert_list_size; + +extern unsigned long early_ipl_comp_list_addr; +extern unsigned long early_ipl_comp_list_size; #endif /* _ASM_S390_BOOT_DATA_H */ diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index 429f43a8a8e8..713fc9735ffb 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -15,7 +15,7 @@ ".section .rodata.str,\"aMS\",@progbits,1\n" \ "2: .asciz \""__FILE__"\"\n" \ ".previous\n" \ - ".section __bug_table,\"aw\"\n" \ + ".section __bug_table,\"awM\",@progbits,%2\n" \ "3: .long 1b-3b,2b-3b\n" \ " .short %0,%1\n" \ " .org 3b+%2\n" \ @@ -27,17 +27,17 @@ #else /* CONFIG_DEBUG_BUGVERBOSE */ -#define __EMIT_BUG(x) do { \ - asm volatile( \ - "0: j 0b+2\n" \ - "1:\n" \ - ".section __bug_table,\"aw\"\n" \ - "2: .long 1b-2b\n" \ - " .short %0\n" \ - " .org 2b+%1\n" \ - ".previous\n" \ - : : "i" (x), \ - "i" (sizeof(struct bug_entry))); \ +#define __EMIT_BUG(x) do { \ + asm volatile( \ + "0: j 0b+2\n" \ + "1:\n" \ + ".section __bug_table,\"awM\",@progbits,%1\n" \ + "2: .long 1b-2b\n" \ + " .short %0\n" \ + " .org 2b+%1\n" \ + ".previous\n" \ + : : "i" (x), \ + "i" (sizeof(struct bug_entry))); \ } while (0) #endif /* CONFIG_DEBUG_BUGVERBOSE */ diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h index 19562be22b7e..0036eab14391 100644 --- a/arch/s390/include/asm/diag.h +++ b/arch/s390/include/asm/diag.h @@ -308,4 +308,17 @@ union diag318_info { int diag204(unsigned long subcode, unsigned long size, void *addr); int diag224(void *ptr); int diag26c(void *req, void *resp, enum diag26c_sc subcode); + +struct hypfs_diag0c_entry; + +struct diag_ops { + int (*diag210)(struct diag210 *addr); + int (*diag26c)(void *req, void *resp, enum diag26c_sc subcode); + int (*diag14)(unsigned long rx, unsigned long ry1, unsigned long subcode); + void (*diag0c)(struct hypfs_diag0c_entry *entry); + void (*diag308_reset)(void); +}; + +extern struct diag_ops diag_dma_ops; +extern struct diag210 *__diag210_tmp_dma; #endif /* _ASM_S390_DIAG_H */ diff --git a/arch/s390/include/asm/ebcdic.h b/arch/s390/include/asm/ebcdic.h index 29441beb92e6..efb50fc6866c 100644 --- a/arch/s390/include/asm/ebcdic.h +++ b/arch/s390/include/asm/ebcdic.h @@ -20,7 +20,7 @@ extern __u8 _ebc_tolower[256]; /* EBCDIC -> lowercase */ extern __u8 _ebc_toupper[256]; /* EBCDIC -> uppercase */ static inline void -codepage_convert(const __u8 *codepage, volatile __u8 * addr, unsigned long nr) +codepage_convert(const __u8 *codepage, volatile char *addr, unsigned long nr) { if (nr-- <= 0) return; diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index f74639a05f0f..5775fc22f410 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -107,6 +107,10 @@ #define HWCAP_S390_VXRS_BCD 4096 #define HWCAP_S390_VXRS_EXT 8192 #define HWCAP_S390_GS 16384 +#define HWCAP_S390_VXRS_EXT2 32768 +#define HWCAP_S390_VXRS_PDE 65536 +#define HWCAP_S390_SORT 131072 +#define HWCAP_S390_DFLT 262144 /* Internal bits, not exposed via elf */ #define HWCAP_INT_SIE 1UL diff --git a/arch/s390/include/asm/extable.h b/arch/s390/include/asm/extable.h index 80a4e5a9cb46..ae27f756b409 100644 --- a/arch/s390/include/asm/extable.h +++ b/arch/s390/include/asm/extable.h @@ -19,6 +19,11 @@ struct exception_table_entry int insn, fixup; }; +extern struct exception_table_entry *__start_dma_ex_table; +extern struct exception_table_entry *__stop_dma_ex_table; + +const struct exception_table_entry *s390_search_extables(unsigned long addr); + static inline unsigned long extable_fixup(const struct exception_table_entry *x) { return (unsigned long)&x->fixup + x->fixup; diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 5a3c95b11952..68d362f8d6c1 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -11,9 +11,16 @@ #define MCOUNT_RETURN_FIXUP 18 #endif +#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR + #ifndef __ASSEMBLY__ +#ifdef CONFIG_CC_IS_CLANG +/* https://bugs.llvm.org/show_bug.cgi?id=41424 */ +#define ftrace_return_address(n) 0UL +#else #define ftrace_return_address(n) __builtin_return_address(n) +#endif void _mcount(void); void ftrace_caller(void); diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h index f34d729347e4..ca421614722f 100644 --- a/arch/s390/include/asm/io.h +++ b/arch/s390/include/asm/io.h @@ -30,14 +30,8 @@ void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr); #define ioremap_wc ioremap_nocache #define ioremap_wt ioremap_nocache -static inline void __iomem *ioremap(unsigned long offset, unsigned long size) -{ - return (void __iomem *) offset; -} - -static inline void iounmap(volatile void __iomem *addr) -{ -} +void __iomem *ioremap(unsigned long offset, unsigned long size); +void iounmap(volatile void __iomem *addr); static inline void __iomem *ioport_map(unsigned long port, unsigned int nr) { @@ -57,14 +51,17 @@ static inline void ioport_unmap(void __iomem *p) * the corresponding device and create the mapping cookie. */ #define pci_iomap pci_iomap +#define pci_iomap_range pci_iomap_range #define pci_iounmap pci_iounmap -#define pci_iomap_wc pci_iomap -#define pci_iomap_wc_range pci_iomap_range +#define pci_iomap_wc pci_iomap_wc +#define pci_iomap_wc_range pci_iomap_wc_range #define memcpy_fromio(dst, src, count) zpci_memcpy_fromio(dst, src, count) #define memcpy_toio(dst, src, count) zpci_memcpy_toio(dst, src, count) #define memset_io(dst, val, count) zpci_memset_io(dst, val, count) +#define mmiowb() zpci_barrier() + #define __raw_readb zpci_read_u8 #define __raw_readw zpci_read_u16 #define __raw_readl zpci_read_u32 diff --git a/arch/s390/include/asm/ipl.h b/arch/s390/include/asm/ipl.h index a8389e2d2f03..084e71b7272a 100644 --- a/arch/s390/include/asm/ipl.h +++ b/arch/s390/include/asm/ipl.h @@ -12,74 +12,36 @@ #include <asm/types.h> #include <asm/cio.h> #include <asm/setup.h> +#include <uapi/asm/ipl.h> -#define NSS_NAME_SIZE 8 - -#define IPL_PARM_BLK_FCP_LEN (sizeof(struct ipl_list_hdr) + \ - sizeof(struct ipl_block_fcp)) - -#define IPL_PARM_BLK0_FCP_LEN (sizeof(struct ipl_block_fcp) + 16) +struct ipl_parameter_block { + struct ipl_pl_hdr hdr; + union { + struct ipl_pb_hdr pb0_hdr; + struct ipl_pb0_common common; + struct ipl_pb0_fcp fcp; + struct ipl_pb0_ccw ccw; + char raw[PAGE_SIZE - sizeof(struct ipl_pl_hdr)]; + }; +} __packed __aligned(PAGE_SIZE); -#define IPL_PARM_BLK_CCW_LEN (sizeof(struct ipl_list_hdr) + \ - sizeof(struct ipl_block_ccw)) +#define NSS_NAME_SIZE 8 -#define IPL_PARM_BLK0_CCW_LEN (sizeof(struct ipl_block_ccw) + 16) +#define IPL_BP_FCP_LEN (sizeof(struct ipl_pl_hdr) + \ + sizeof(struct ipl_pb0_fcp)) +#define IPL_BP0_FCP_LEN (sizeof(struct ipl_pb0_fcp)) +#define IPL_BP_CCW_LEN (sizeof(struct ipl_pl_hdr) + \ + sizeof(struct ipl_pb0_ccw)) +#define IPL_BP0_CCW_LEN (sizeof(struct ipl_pb0_ccw)) #define IPL_MAX_SUPPORTED_VERSION (0) -struct ipl_list_hdr { - u32 len; - u8 reserved1[3]; - u8 version; - u32 blk0_len; - u8 pbt; - u8 flags; - u16 reserved2; - u8 loadparm[8]; -} __attribute__((packed)); - -struct ipl_block_fcp { - u8 reserved1[305-1]; - u8 opt; - u8 reserved2[3]; - u16 reserved3; - u16 devno; - u8 reserved4[4]; - u64 wwpn; - u64 lun; - u32 bootprog; - u8 reserved5[12]; - u64 br_lba; - u32 scp_data_len; - u8 reserved6[260]; - u8 scp_data[]; -} __attribute__((packed)); - -#define DIAG308_VMPARM_SIZE 64 -#define DIAG308_SCPDATA_SIZE (PAGE_SIZE - (sizeof(struct ipl_list_hdr) + \ - offsetof(struct ipl_block_fcp, scp_data))) - -struct ipl_block_ccw { - u8 reserved1[84]; - u16 reserved2 : 13; - u8 ssid : 3; - u16 devno; - u8 vm_flags; - u8 reserved3[3]; - u32 vm_parm_len; - u8 nss_name[8]; - u8 vm_parm[DIAG308_VMPARM_SIZE]; - u8 reserved4[8]; -} __attribute__((packed)); +#define IPL_RB_CERT_UNKNOWN ((unsigned short)-1) -struct ipl_parameter_block { - struct ipl_list_hdr hdr; - union { - struct ipl_block_fcp fcp; - struct ipl_block_ccw ccw; - char raw[PAGE_SIZE - sizeof(struct ipl_list_hdr)]; - } ipl_info; -} __packed __aligned(PAGE_SIZE); +#define DIAG308_VMPARM_SIZE (64) +#define DIAG308_SCPDATA_OFFSET offsetof(struct ipl_parameter_block, \ + fcp.scp_data) +#define DIAG308_SCPDATA_SIZE (PAGE_SIZE - DIAG308_SCPDATA_OFFSET) struct save_area; struct save_area * __init save_area_alloc(bool is_boot_cpu); @@ -88,7 +50,6 @@ void __init save_area_add_regs(struct save_area *, void *regs); void __init save_area_add_vxrs(struct save_area *, __vector128 *vxrs); extern void s390_reset_system(void); -extern void ipl_store_parameters(void); extern size_t ipl_block_get_ascii_vmparm(char *dest, size_t size, const struct ipl_parameter_block *ipb); @@ -122,6 +83,33 @@ extern struct ipl_info ipl_info; extern void setup_ipl(void); extern void set_os_info_reipl_block(void); +struct ipl_report { + struct ipl_parameter_block *ipib; + struct list_head components; + struct list_head certificates; + size_t size; +}; + +struct ipl_report_component { + struct list_head list; + struct ipl_rb_component_entry entry; +}; + +struct ipl_report_certificate { + struct list_head list; + struct ipl_rb_certificate_entry entry; + void *key; +}; + +struct kexec_buf; +struct ipl_report *ipl_report_init(struct ipl_parameter_block *ipib); +void *ipl_report_finish(struct ipl_report *report); +int ipl_report_free(struct ipl_report *report); +int ipl_report_add_component(struct ipl_report *report, struct kexec_buf *kbuf, + unsigned char flags, unsigned short cert); +int ipl_report_add_certificate(struct ipl_report *report, void *key, + unsigned long addr, unsigned long len); + /* * DIAG 308 support */ @@ -133,32 +121,12 @@ enum diag308_subcode { DIAG308_STORE = 6, }; -enum diag308_ipl_type { - DIAG308_IPL_TYPE_FCP = 0, - DIAG308_IPL_TYPE_CCW = 2, -}; - -enum diag308_opt { - DIAG308_IPL_OPT_IPL = 0x10, - DIAG308_IPL_OPT_DUMP = 0x20, -}; - -enum diag308_flags { - DIAG308_FLAGS_LP_VALID = 0x80, -}; - -enum diag308_vm_flags { - DIAG308_VM_FLAGS_NSS_VALID = 0x80, - DIAG308_VM_FLAGS_VP_VALID = 0x40, -}; - enum diag308_rc { DIAG308_RC_OK = 0x0001, DIAG308_RC_NOCONFIG = 0x0102, }; extern int diag308(unsigned long subcode, void *addr); -extern void diag308_reset(void); extern void store_status(void (*fn)(void *), void *data); extern void lgr_info_log(void); diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index afaf5e3c57fd..9f75d67b8c20 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -47,7 +47,6 @@ enum interruption_class { IRQEXT_CMC, IRQEXT_FTP, IRQIO_CIO, - IRQIO_QAI, IRQIO_DAS, IRQIO_C15, IRQIO_C70, @@ -55,12 +54,14 @@ enum interruption_class { IRQIO_VMR, IRQIO_LCS, IRQIO_CTC, - IRQIO_APB, IRQIO_ADM, IRQIO_CSC, - IRQIO_PCI, - IRQIO_MSI, IRQIO_VIR, + IRQIO_QAI, + IRQIO_APB, + IRQIO_PCF, + IRQIO_PCD, + IRQIO_MSI, IRQIO_VAI, IRQIO_GAL, NMI_NMI, diff --git a/arch/s390/include/asm/kexec.h b/arch/s390/include/asm/kexec.h index 825dd0f7f221..ea398a05f643 100644 --- a/arch/s390/include/asm/kexec.h +++ b/arch/s390/include/asm/kexec.h @@ -11,6 +11,7 @@ #include <asm/processor.h> #include <asm/page.h> +#include <asm/setup.h> /* * KEXEC_SOURCE_MEMORY_LIMIT maximum page get_free_page can return. * I.e. Maximum page that is mapped directly into kernel memory, @@ -42,6 +43,9 @@ /* The native architecture */ #define KEXEC_ARCH KEXEC_ARCH_S390 +/* Allow kexec_file to load a segment to 0 */ +#define KEXEC_BUF_MEM_UNKNOWN -1 + /* Provide a dummy definition to avoid build failures. */ static inline void crash_setup_regs(struct pt_regs *newregs, struct pt_regs *oldregs) { } @@ -51,20 +55,24 @@ struct s390_load_data { /* Pointer to the kernel buffer. Used to register cmdline etc.. */ void *kernel_buf; + /* Load address of the kernel_buf. */ + unsigned long kernel_mem; + + /* Parmarea in the kernel buffer. */ + struct parmarea *parm; + /* Total size of loaded segments in memory. Used as an offset. */ size_t memsz; - /* Load address of initrd. Used to register INITRD_START in kernel. */ - unsigned long initrd_load_addr; + struct ipl_report *report; }; -int kexec_file_add_purgatory(struct kimage *image, - struct s390_load_data *data); -int kexec_file_add_initrd(struct kimage *image, - struct s390_load_data *data, - char *initrd, unsigned long initrd_len); -int *kexec_file_update_kernel(struct kimage *iamge, - struct s390_load_data *data); +int s390_verify_sig(const char *kernel, unsigned long kernel_len); +void *kexec_file_add_components(struct kimage *image, + int (*add_kernel)(struct kimage *image, + struct s390_load_data *data)); +int arch_kexec_do_relocs(int r_type, void *loc, unsigned long val, + unsigned long addr); extern const struct kexec_file_ops s390_kexec_image_ops; extern const struct kexec_file_ops s390_kexec_elf_ops; diff --git a/arch/s390/include/asm/linkage.h b/arch/s390/include/asm/linkage.h index 1b95da3fdd64..7f22262b0e46 100644 --- a/arch/s390/include/asm/linkage.h +++ b/arch/s390/include/asm/linkage.h @@ -28,5 +28,12 @@ .long (_target) - . ; \ .previous +#define EX_TABLE_DMA(_fault, _target) \ + .section .dma.ex_table, "a" ; \ + .align 4 ; \ + .long (_fault) - . ; \ + .long (_target) - . ; \ + .previous + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 5b9f10b1e55d..237ee0c4169f 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -129,7 +129,7 @@ struct lowcore { /* SMP info area */ __u32 cpu_nr; /* 0x03a0 */ __u32 softirq_pending; /* 0x03a4 */ - __u32 preempt_count; /* 0x03a8 */ + __s32 preempt_count; /* 0x03a8 */ __u32 spinlock_lockval; /* 0x03ac */ __u32 spinlock_index; /* 0x03b0 */ __u32 fpu_flags; /* 0x03b4 */ diff --git a/arch/s390/include/asm/nospec-insn.h b/arch/s390/include/asm/nospec-insn.h index 123dac3717b3..0033dcd663b1 100644 --- a/arch/s390/include/asm/nospec-insn.h +++ b/arch/s390/include/asm/nospec-insn.h @@ -32,23 +32,23 @@ _LC_BR_R1 = __LC_BR_R1 .endm .macro __THUNK_PROLOG_BR r1,r2 - __THUNK_PROLOG_NAME __s390x_indirect_jump_r\r2\()use_r\r1 + __THUNK_PROLOG_NAME __s390_indirect_jump_r\r2\()use_r\r1 .endm .macro __THUNK_PROLOG_BC d0,r1,r2 - __THUNK_PROLOG_NAME __s390x_indirect_branch_\d0\()_\r2\()use_\r1 + __THUNK_PROLOG_NAME __s390_indirect_branch_\d0\()_\r2\()use_\r1 .endm .macro __THUNK_BR r1,r2 - jg __s390x_indirect_jump_r\r2\()use_r\r1 + jg __s390_indirect_jump_r\r2\()use_r\r1 .endm .macro __THUNK_BC d0,r1,r2 - jg __s390x_indirect_branch_\d0\()_\r2\()use_\r1 + jg __s390_indirect_branch_\d0\()_\r2\()use_\r1 .endm .macro __THUNK_BRASL r1,r2,r3 - brasl \r1,__s390x_indirect_jump_r\r3\()use_r\r2 + brasl \r1,__s390_indirect_jump_r\r3\()use_r\r2 .endm .macro __DECODE_RR expand,reg,ruse diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 4e0efebc56a9..305befd55326 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -26,6 +26,9 @@ int pci_proc_domain(struct pci_bus *); #define ZPCI_BUS_NR 0 /* default bus number */ #define ZPCI_DEVFN 0 /* default device number */ +#define ZPCI_NR_DMA_SPACES 1 +#define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS + /* PCI Function Controls */ #define ZPCI_FC_FN_ENABLED 0x80 #define ZPCI_FC_ERROR 0x40 @@ -83,6 +86,8 @@ enum zpci_state { struct zpci_bar_struct { struct resource *res; /* bus resource */ + void __iomem *mio_wb; + void __iomem *mio_wt; u32 val; /* bar start & 3 flag bits */ u16 map_idx; /* index into bar mapping array */ u8 size; /* order 2 exponent */ @@ -112,6 +117,8 @@ struct zpci_dev { /* IRQ stuff */ u64 msi_addr; /* MSI address */ unsigned int max_msi; /* maximum number of MSI's */ + unsigned int msi_first_bit; + unsigned int msi_nr_irqs; struct airq_iv *aibv; /* adapter interrupt bit vector */ unsigned long aisb; /* number of the summary bit */ @@ -130,6 +137,7 @@ struct zpci_dev { struct iommu_device iommu_dev; /* IOMMU core handle */ char res_name[16]; + bool mio_capable; struct zpci_bar_struct bars[PCI_BAR_COUNT]; u64 start_dma; /* Start of available DMA addresses */ @@ -158,6 +166,7 @@ static inline bool zdev_enabled(struct zpci_dev *zdev) } extern const struct attribute_group *zpci_attr_groups[]; +extern unsigned int s390_pci_force_floating __initdata; /* ----------------------------------------------------------------------------- Prototypes @@ -219,6 +228,9 @@ struct zpci_dev *get_zdev_by_fid(u32); int zpci_dma_init(void); void zpci_dma_exit(void); +int __init zpci_irq_init(void); +void __init zpci_irq_exit(void); + /* FMB */ int zpci_fmb_enable_device(struct zpci_dev *); int zpci_fmb_disable_device(struct zpci_dev *); diff --git a/arch/s390/include/asm/pci_clp.h b/arch/s390/include/asm/pci_clp.h index b3b31b31f0d3..3ec52a05d500 100644 --- a/arch/s390/include/asm/pci_clp.h +++ b/arch/s390/include/asm/pci_clp.h @@ -43,6 +43,8 @@ struct clp_fh_list_entry { #define CLP_SET_ENABLE_PCI_FN 0 /* Yes, 0 enables it */ #define CLP_SET_DISABLE_PCI_FN 1 /* Yes, 1 disables it */ +#define CLP_SET_ENABLE_MIO 2 +#define CLP_SET_DISABLE_MIO 3 #define CLP_UTIL_STR_LEN 64 #define CLP_PFIP_NR_SEGMENTS 4 @@ -80,7 +82,8 @@ struct clp_req_query_pci { struct clp_rsp_query_pci { struct clp_rsp_hdr hdr; u16 vfn; /* virtual fn number */ - u16 : 7; + u16 : 6; + u16 mio_addr_avail : 1; u16 util_str_avail : 1; /* utility string available? */ u16 pfgid : 8; /* pci function group id */ u32 fid; /* pci function id */ @@ -96,6 +99,15 @@ struct clp_rsp_query_pci { u32 reserved[11]; u32 uid; /* user defined id */ u8 util_str[CLP_UTIL_STR_LEN]; /* utility string */ + u32 reserved2[16]; + u32 mio_valid : 6; + u32 : 26; + u32 : 32; + struct { + u64 wb; + u64 wt; + } addr[PCI_BAR_COUNT]; + u32 reserved3[6]; } __packed; /* Query PCI function group request */ @@ -118,7 +130,11 @@ struct clp_rsp_query_pci_grp { u8 refresh : 1; /* TLB refresh mode */ u16 reserved2; u16 mui; - u64 reserved3; + u16 : 16; + u16 maxfaal; + u16 : 4; + u16 dnoi : 12; + u16 maxcpu; u64 dasm; /* dma address space mask */ u64 msia; /* MSI address */ u64 reserved4; diff --git a/arch/s390/include/asm/pci_insn.h b/arch/s390/include/asm/pci_insn.h index ba22a6ea51a1..ff81ed19c506 100644 --- a/arch/s390/include/asm/pci_insn.h +++ b/arch/s390/include/asm/pci_insn.h @@ -2,6 +2,8 @@ #ifndef _ASM_S390_PCI_INSN_H #define _ASM_S390_PCI_INSN_H +#include <linux/jump_label.h> + /* Load/Store status codes */ #define ZPCI_PCI_ST_FUNC_NOT_ENABLED 4 #define ZPCI_PCI_ST_FUNC_IN_ERR 8 @@ -38,6 +40,8 @@ #define ZPCI_MOD_FC_RESET_ERROR 7 #define ZPCI_MOD_FC_RESET_BLOCK 9 #define ZPCI_MOD_FC_SET_MEASURE 10 +#define ZPCI_MOD_FC_REG_INT_D 16 +#define ZPCI_MOD_FC_DEREG_INT_D 17 /* FIB function controls */ #define ZPCI_FIB_FC_ENABLED 0x80 @@ -51,16 +55,7 @@ #define ZPCI_FIB_FC_LS_BLOCKED 0x20 #define ZPCI_FIB_FC_DMAAS_REG 0x10 -/* Function Information Block */ -struct zpci_fib { - u32 fmt : 8; /* format */ - u32 : 24; - u32 : 32; - u8 fc; /* function controls */ - u64 : 56; - u64 pba; /* PCI base address */ - u64 pal; /* PCI address limit */ - u64 iota; /* I/O Translation Anchor */ +struct zpci_fib_fmt0 { u32 : 1; u32 isc : 3; /* Interrupt subclass */ u32 noi : 12; /* Number of interrupts */ @@ -72,16 +67,90 @@ struct zpci_fib { u32 : 32; u64 aibv; /* Adapter int bit vector address */ u64 aisb; /* Adapter int summary bit address */ +}; + +struct zpci_fib_fmt1 { + u32 : 4; + u32 noi : 12; + u32 : 16; + u32 dibvo : 16; + u32 : 16; + u64 : 64; + u64 : 64; +}; + +/* Function Information Block */ +struct zpci_fib { + u32 fmt : 8; /* format */ + u32 : 24; + u32 : 32; + u8 fc; /* function controls */ + u64 : 56; + u64 pba; /* PCI base address */ + u64 pal; /* PCI address limit */ + u64 iota; /* I/O Translation Anchor */ + union { + struct zpci_fib_fmt0 fmt0; + struct zpci_fib_fmt1 fmt1; + }; u64 fmb_addr; /* Function measurement block address and key */ u32 : 32; u32 gd; } __packed __aligned(8); +/* directed interruption information block */ +struct zpci_diib { + u32 : 1; + u32 isc : 3; + u32 : 28; + u16 : 16; + u16 nr_cpus; + u64 disb_addr; + u64 : 64; + u64 : 64; +} __packed __aligned(8); + +/* cpu directed interruption information block */ +struct zpci_cdiib { + u64 : 64; + u64 dibv_addr; + u64 : 64; + u64 : 64; + u64 : 64; +} __packed __aligned(8); + +union zpci_sic_iib { + struct zpci_diib diib; + struct zpci_cdiib cdiib; +}; + +DECLARE_STATIC_KEY_FALSE(have_mio); + u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status); int zpci_refresh_trans(u64 fn, u64 addr, u64 range); -int zpci_load(u64 *data, u64 req, u64 offset); -int zpci_store(u64 data, u64 req, u64 offset); -int zpci_store_block(const u64 *data, u64 req, u64 offset); -int zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc); +int __zpci_load(u64 *data, u64 req, u64 offset); +int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len); +int __zpci_store(u64 data, u64 req, u64 offset); +int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len); +int __zpci_store_block(const u64 *data, u64 req, u64 offset); +void zpci_barrier(void); +int __zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib); + +static inline int zpci_set_irq_ctrl(u16 ctl, u8 isc) +{ + union zpci_sic_iib iib = {{0}}; + + return __zpci_set_irq_ctrl(ctl, isc, &iib); +} + +#ifdef CONFIG_PCI +static inline void enable_mio_ctl(void) +{ + if (static_branch_likely(&have_mio)) + __ctl_set_bit(2, 5); +} +#else /* CONFIG_PCI */ +static inline void enable_mio_ctl(void) {} +#endif /* CONFIG_PCI */ #endif diff --git a/arch/s390/include/asm/pci_io.h b/arch/s390/include/asm/pci_io.h index cbb9cb9c6547..cd060b5dd8fd 100644 --- a/arch/s390/include/asm/pci_io.h +++ b/arch/s390/include/asm/pci_io.h @@ -37,12 +37,10 @@ extern struct zpci_iomap_entry *zpci_iomap_start; #define zpci_read(LENGTH, RETTYPE) \ static inline RETTYPE zpci_read_##RETTYPE(const volatile void __iomem *addr) \ { \ - struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; \ - u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, LENGTH); \ u64 data; \ int rc; \ \ - rc = zpci_load(&data, req, ZPCI_OFFSET(addr)); \ + rc = zpci_load(&data, addr, LENGTH); \ if (rc) \ data = -1ULL; \ return (RETTYPE) data; \ @@ -52,11 +50,9 @@ static inline RETTYPE zpci_read_##RETTYPE(const volatile void __iomem *addr) \ static inline void zpci_write_##VALTYPE(VALTYPE val, \ const volatile void __iomem *addr) \ { \ - struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; \ - u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, LENGTH); \ u64 data = (VALTYPE) val; \ \ - zpci_store(data, req, ZPCI_OFFSET(addr)); \ + zpci_store(addr, data, LENGTH); \ } zpci_read(8, u64) @@ -68,36 +64,38 @@ zpci_write(4, u32) zpci_write(2, u16) zpci_write(1, u8) -static inline int zpci_write_single(u64 req, const u64 *data, u64 offset, u8 len) +static inline int zpci_write_single(volatile void __iomem *dst, const void *src, + unsigned long len) { u64 val; switch (len) { case 1: - val = (u64) *((u8 *) data); + val = (u64) *((u8 *) src); break; case 2: - val = (u64) *((u16 *) data); + val = (u64) *((u16 *) src); break; case 4: - val = (u64) *((u32 *) data); + val = (u64) *((u32 *) src); break; case 8: - val = (u64) *((u64 *) data); + val = (u64) *((u64 *) src); break; default: val = 0; /* let FW report error */ break; } - return zpci_store(val, req, offset); + return zpci_store(dst, val, len); } -static inline int zpci_read_single(u64 req, u64 *dst, u64 offset, u8 len) +static inline int zpci_read_single(void *dst, const volatile void __iomem *src, + unsigned long len) { u64 data; int cc; - cc = zpci_load(&data, req, offset); + cc = zpci_load(&data, src, len); if (cc) goto out; @@ -119,10 +117,8 @@ out: return cc; } -static inline int zpci_write_block(u64 req, const u64 *data, u64 offset) -{ - return zpci_store_block(data, req, offset); -} +int zpci_write_block(volatile void __iomem *dst, const void *src, + unsigned long len); static inline u8 zpci_get_max_write_size(u64 src, u64 dst, int len, int max) { @@ -140,18 +136,15 @@ static inline int zpci_memcpy_fromio(void *dst, const volatile void __iomem *src, unsigned long n) { - struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(src)]; - u64 req, offset = ZPCI_OFFSET(src); int size, rc = 0; while (n > 0) { size = zpci_get_max_write_size((u64 __force) src, (u64) dst, n, 8); - req = ZPCI_CREATE_REQ(entry->fh, entry->bar, size); - rc = zpci_read_single(req, dst, offset, size); + rc = zpci_read_single(dst, src, size); if (rc) break; - offset += size; + src += size; dst += size; n -= size; } @@ -161,8 +154,6 @@ static inline int zpci_memcpy_fromio(void *dst, static inline int zpci_memcpy_toio(volatile void __iomem *dst, const void *src, unsigned long n) { - struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)]; - u64 req, offset = ZPCI_OFFSET(dst); int size, rc = 0; if (!src) @@ -171,16 +162,14 @@ static inline int zpci_memcpy_toio(volatile void __iomem *dst, while (n > 0) { size = zpci_get_max_write_size((u64 __force) dst, (u64) src, n, 128); - req = ZPCI_CREATE_REQ(entry->fh, entry->bar, size); - if (size > 8) /* main path */ - rc = zpci_write_block(req, src, offset); + rc = zpci_write_block(dst, src, size); else - rc = zpci_write_single(req, src, offset, size); + rc = zpci_write_single(dst, src, size); if (rc) break; - offset += size; src += size; + dst += size; n -= size; } return rc; diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 76dc344edb8c..9f0195d5fa16 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -238,7 +238,7 @@ static inline int is_module_addr(void *addr) #define _REGION_ENTRY_NOEXEC 0x100 /* region no-execute bit */ #define _REGION_ENTRY_OFFSET 0xc0 /* region table offset */ #define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */ -#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */ +#define _REGION_ENTRY_TYPE_MASK 0x0c /* region table type mask */ #define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */ #define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */ #define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */ @@ -277,6 +277,7 @@ static inline int is_module_addr(void *addr) #define _SEGMENT_ENTRY_PROTECT 0x200 /* segment protection bit */ #define _SEGMENT_ENTRY_NOEXEC 0x100 /* segment no-execute bit */ #define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */ +#define _SEGMENT_ENTRY_TYPE_MASK 0x0c /* segment table type mask */ #define _SEGMENT_ENTRY (0) #define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID) @@ -614,15 +615,9 @@ static inline int pgd_none(pgd_t pgd) static inline int pgd_bad(pgd_t pgd) { - /* - * With dynamic page table levels the pgd can be a region table - * entry or a segment table entry. Check for the bit that are - * invalid for either table entry. - */ - unsigned long mask = - ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID & - ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; - return (pgd_val(pgd) & mask) != 0; + if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1) + return 0; + return (pgd_val(pgd) & ~_REGION_ENTRY_BITS) != 0; } static inline unsigned long pgd_pfn(pgd_t pgd) @@ -703,6 +698,8 @@ static inline int pmd_large(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { + if ((pmd_val(pmd) & _SEGMENT_ENTRY_TYPE_MASK) > 0) + return 1; if (pmd_large(pmd)) return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0; return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0; @@ -710,8 +707,12 @@ static inline int pmd_bad(pmd_t pmd) static inline int pud_bad(pud_t pud) { - if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) - return pmd_bad(__pmd(pud_val(pud))); + unsigned long type = pud_val(pud) & _REGION_ENTRY_TYPE_MASK; + + if (type > _REGION_ENTRY_TYPE_R3) + return 1; + if (type < _REGION_ENTRY_TYPE_R3) + return 0; if (pud_large(pud)) return (pud_val(pud) & ~_REGION_ENTRY_BITS_LARGE) != 0; return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0; @@ -719,8 +720,12 @@ static inline int pud_bad(pud_t pud) static inline int p4d_bad(p4d_t p4d) { - if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) - return pud_bad(__pud(p4d_val(p4d))); + unsigned long type = p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK; + + if (type > _REGION_ENTRY_TYPE_R2) + return 1; + if (type < _REGION_ENTRY_TYPE_R2) + return 0; return (p4d_val(p4d) & ~_REGION_ENTRY_BITS) != 0; } @@ -1204,41 +1209,78 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) #define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) -#define pgd_offset_k(address) pgd_offset(&init_mm, address) -#define pgd_offset_raw(pgd, addr) ((pgd) + pgd_index(addr)) - #define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) #define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN) #define p4d_deref(pud) (p4d_val(pud) & _REGION_ENTRY_ORIGIN) #define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) -static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) +/* + * The pgd_offset function *always* adds the index for the top-level + * region/segment table. This is done to get a sequence like the + * following to work: + * pgdp = pgd_offset(current->mm, addr); + * pgd = READ_ONCE(*pgdp); + * p4dp = p4d_offset(&pgd, addr); + * ... + * The subsequent p4d_offset, pud_offset and pmd_offset functions + * only add an index if they dereferenced the pointer. + */ +static inline pgd_t *pgd_offset_raw(pgd_t *pgd, unsigned long address) { - p4d_t *p4d = (p4d_t *) pgd; + unsigned long rste; + unsigned int shift; - if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1) - p4d = (p4d_t *) pgd_deref(*pgd); - return p4d + p4d_index(address); + /* Get the first entry of the top level table */ + rste = pgd_val(*pgd); + /* Pick up the shift from the table type of the first entry */ + shift = ((rste & _REGION_ENTRY_TYPE_MASK) >> 2) * 11 + 20; + return pgd + ((address >> shift) & (PTRS_PER_PGD - 1)); } -static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) +#define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address) +#define pgd_offset_k(address) pgd_offset(&init_mm, address) + +static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { - pud_t *pud = (pud_t *) p4d; + if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R1) + return (p4d_t *) pgd_deref(*pgd) + p4d_index(address); + return (p4d_t *) pgd; +} - if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) - pud = (pud_t *) p4d_deref(*p4d); - return pud + pud_index(address); +static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) +{ + if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2) + return (pud_t *) p4d_deref(*p4d) + pud_index(address); + return (pud_t *) p4d; } static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) { - pmd_t *pmd = (pmd_t *) pud; + if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R3) + return (pmd_t *) pud_deref(*pud) + pmd_index(address); + return (pmd_t *) pud; +} - if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) - pmd = (pmd_t *) pud_deref(*pud); - return pmd + pmd_index(address); +static inline pte_t *pte_offset(pmd_t *pmd, unsigned long address) +{ + return (pte_t *) pmd_deref(*pmd) + pte_index(address); +} + +#define pte_offset_kernel(pmd, address) pte_offset(pmd, address) +#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) +#define pte_unmap(pte) do { } while (0) + +static inline bool gup_fast_permitted(unsigned long start, int nr_pages) +{ + unsigned long len, end; + + len = (unsigned long) nr_pages << PAGE_SHIFT; + end = start + len; + if (end < start) + return false; + return end <= current->mm->context.asce_limit; } +#define gup_fast_permitted gup_fast_permitted #define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot)) #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) @@ -1249,12 +1291,6 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) #define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd)) -/* Find an entry in the lowest level page table.. */ -#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr)) -#define pte_offset_kernel(pmd, address) pte_offset(pmd,address) -#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) -#define pte_unmap(pte) do { } while (0) - static inline pmd_t pmd_wrprotect(pmd_t pmd) { pmd_val(pmd) &= ~_SEGMENT_ENTRY_WRITE; diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 81038ab357ce..b0fcbc37b637 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -156,25 +156,6 @@ struct thread_struct { typedef struct thread_struct thread_struct; -/* - * Stack layout of a C stack frame. - */ -#ifndef __PACK_STACK -struct stack_frame { - unsigned long back_chain; - unsigned long empty1[5]; - unsigned long gprs[10]; - unsigned int empty2[8]; -}; -#else -struct stack_frame { - unsigned long empty1[5]; - unsigned int empty2[8]; - unsigned long gprs[10]; - unsigned long back_chain; -}; -#endif - #define ARCH_MIN_TASKALIGN 8 #define INIT_THREAD { \ @@ -206,11 +187,7 @@ struct mm_struct; struct seq_file; struct pt_regs; -typedef int (*dump_trace_func_t)(void *data, unsigned long address, int reliable); -void dump_trace(dump_trace_func_t func, void *data, - struct task_struct *task, unsigned long sp); void show_registers(struct pt_regs *regs); - void show_cacheinfo(struct seq_file *m); /* Free all resources held by a thread. */ @@ -244,55 +221,6 @@ static __no_kasan_or_inline unsigned short stap(void) return cpu_address; } -#define CALL_ARGS_0() \ - register unsigned long r2 asm("2") -#define CALL_ARGS_1(arg1) \ - register unsigned long r2 asm("2") = (unsigned long)(arg1) -#define CALL_ARGS_2(arg1, arg2) \ - CALL_ARGS_1(arg1); \ - register unsigned long r3 asm("3") = (unsigned long)(arg2) -#define CALL_ARGS_3(arg1, arg2, arg3) \ - CALL_ARGS_2(arg1, arg2); \ - register unsigned long r4 asm("4") = (unsigned long)(arg3) -#define CALL_ARGS_4(arg1, arg2, arg3, arg4) \ - CALL_ARGS_3(arg1, arg2, arg3); \ - register unsigned long r4 asm("5") = (unsigned long)(arg4) -#define CALL_ARGS_5(arg1, arg2, arg3, arg4, arg5) \ - CALL_ARGS_4(arg1, arg2, arg3, arg4); \ - register unsigned long r4 asm("6") = (unsigned long)(arg5) - -#define CALL_FMT_0 -#define CALL_FMT_1 CALL_FMT_0, "0" (r2) -#define CALL_FMT_2 CALL_FMT_1, "d" (r3) -#define CALL_FMT_3 CALL_FMT_2, "d" (r4) -#define CALL_FMT_4 CALL_FMT_3, "d" (r5) -#define CALL_FMT_5 CALL_FMT_4, "d" (r6) - -#define CALL_CLOBBER_5 "0", "1", "14", "cc", "memory" -#define CALL_CLOBBER_4 CALL_CLOBBER_5 -#define CALL_CLOBBER_3 CALL_CLOBBER_4, "5" -#define CALL_CLOBBER_2 CALL_CLOBBER_3, "4" -#define CALL_CLOBBER_1 CALL_CLOBBER_2, "3" -#define CALL_CLOBBER_0 CALL_CLOBBER_1 - -#define CALL_ON_STACK(fn, stack, nr, args...) \ -({ \ - CALL_ARGS_##nr(args); \ - unsigned long prev; \ - \ - asm volatile( \ - " la %[_prev],0(15)\n" \ - " la 15,0(%[_stack])\n" \ - " stg %[_prev],%[_bc](15)\n" \ - " brasl 14,%[_fn]\n" \ - " la 15,0(%[_prev])\n" \ - : "+&d" (r2), [_prev] "=&a" (prev) \ - : [_stack] "a" (stack), \ - [_bc] "i" (offsetof(struct stack_frame, back_chain)), \ - [_fn] "X" (fn) CALL_FMT_##nr : CALL_CLOBBER_##nr); \ - r2; \ -}) - /* * Give up the time slice of the virtual PU. */ @@ -339,10 +267,10 @@ static __no_kasan_or_inline void __load_psw_mask(unsigned long mask) asm volatile( " larl %0,1f\n" - " stg %0,%O1+8(%R1)\n" - " lpswe %1\n" + " stg %0,%1\n" + " lpswe %2\n" "1:" - : "=&d" (addr), "=Q" (psw) : "Q" (psw) : "memory", "cc"); + : "=&d" (addr), "=Q" (psw.addr) : "Q" (psw) : "memory", "cc"); } /* @@ -387,12 +315,12 @@ void enabled_wait(void); /* * Function to drop a processor into disabled wait state */ -static inline void __noreturn disabled_wait(unsigned long code) +static inline void __noreturn disabled_wait(void) { psw_t psw; psw.mask = PSW_MASK_BASE | PSW_MASK_WAIT | PSW_MASK_BA | PSW_MASK_EA; - psw.addr = code; + psw.addr = _THIS_IP_; __load_psw(psw); while (1); } diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index ef4c9dec06a4..f577c5f6031a 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -79,6 +79,9 @@ struct sclp_info { unsigned char has_kss : 1; unsigned char has_gisaf : 1; unsigned char has_diag318 : 1; + unsigned char has_sipl : 1; + unsigned char has_sipl_g2 : 1; + unsigned char has_dirq : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/arch/s390/include/asm/sections.h b/arch/s390/include/asm/sections.h index 7afe4620685c..42de04ad9c07 100644 --- a/arch/s390/include/asm/sections.h +++ b/arch/s390/include/asm/sections.h @@ -2,8 +2,20 @@ #ifndef _S390_SECTIONS_H #define _S390_SECTIONS_H +#define arch_is_kernel_initmem_freed arch_is_kernel_initmem_freed + #include <asm-generic/sections.h> +extern bool initmem_freed; + +static inline int arch_is_kernel_initmem_freed(unsigned long addr) +{ + if (!initmem_freed) + return 0; + return addr >= (unsigned long)__init_begin && + addr < (unsigned long)__init_end; +} + /* * .boot.data section contains variables "shared" between the decompressor and * the decompressed kernel. The decompressor will store values in them, and @@ -16,4 +28,14 @@ */ #define __bootdata(var) __section(.boot.data.var) var +/* + * .boot.preserved.data is similar to .boot.data, but it is not part of the + * .init section and thus will be preserved for later use in the decompressed + * kernel. + */ +#define __bootdata_preserved(var) __section(.boot.preserved.data.var) var + +extern unsigned long __sdma, __edma; +extern unsigned long __stext_dma, __etext_dma; + #endif diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index efda97804aa4..925889d360c1 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -12,7 +12,10 @@ #define EP_OFFSET 0x10008 #define EP_STRING "S390EP" #define PARMAREA 0x10400 -#define PARMAREA_END 0x11000 +#define EARLY_SCCB_OFFSET 0x11000 +#define HEAD_END 0x12000 + +#define EARLY_SCCB_SIZE PAGE_SIZE /* * Machine features detected in early.c @@ -65,6 +68,16 @@ #define OLDMEM_SIZE (*(unsigned long *) (OLDMEM_SIZE_OFFSET)) #define COMMAND_LINE ((char *) (COMMAND_LINE_OFFSET)) +struct parmarea { + unsigned long ipl_device; /* 0x10400 */ + unsigned long initrd_start; /* 0x10408 */ + unsigned long initrd_size; /* 0x10410 */ + unsigned long oldmem_base; /* 0x10418 */ + unsigned long oldmem_size; /* 0x10420 */ + char pad1[0x10480 - 0x10428]; /* 0x10428 - 0x10480 */ + char command_line[ARCH_COMMAND_LINE_SIZE]; /* 0x10480 */ +}; + extern int noexec_disabled; extern int memory_end_set; extern unsigned long memory_end; @@ -134,6 +147,12 @@ extern void (*_machine_restart)(char *command); extern void (*_machine_halt)(void); extern void (*_machine_power_off)(void); +extern unsigned long __kaslr_offset; +static inline unsigned long kaslr_offset(void) +{ + return __kaslr_offset; +} + #else /* __ASSEMBLY__ */ #define IPL_DEVICE (IPL_DEVICE_OFFSET) diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h new file mode 100644 index 000000000000..49634bfbecdd --- /dev/null +++ b/arch/s390/include/asm/stacktrace.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_STACKTRACE_H +#define _ASM_S390_STACKTRACE_H + +#include <linux/uaccess.h> +#include <linux/ptrace.h> +#include <asm/switch_to.h> + +enum stack_type { + STACK_TYPE_UNKNOWN, + STACK_TYPE_TASK, + STACK_TYPE_IRQ, + STACK_TYPE_NODAT, + STACK_TYPE_RESTART, +}; + +struct stack_info { + enum stack_type type; + unsigned long begin, end; +}; + +const char *stack_type_name(enum stack_type type); +int get_stack_info(unsigned long sp, struct task_struct *task, + struct stack_info *info, unsigned long *visit_mask); + +static inline bool on_stack(struct stack_info *info, + unsigned long addr, size_t len) +{ + if (info->type == STACK_TYPE_UNKNOWN) + return false; + if (addr + len < addr) + return false; + return addr >= info->begin && addr + len < info->end; +} + +static inline unsigned long get_stack_pointer(struct task_struct *task, + struct pt_regs *regs) +{ + if (regs) + return (unsigned long) kernel_stack_pointer(regs); + if (task == current) + return current_stack_pointer(); + return (unsigned long) task->thread.ksp; +} + +/* + * Stack layout of a C stack frame. + */ +#ifndef __PACK_STACK +struct stack_frame { + unsigned long back_chain; + unsigned long empty1[5]; + unsigned long gprs[10]; + unsigned int empty2[8]; +}; +#else +struct stack_frame { + unsigned long empty1[5]; + unsigned int empty2[8]; + unsigned long gprs[10]; + unsigned long back_chain; +}; +#endif + +#define CALL_ARGS_0() \ + register unsigned long r2 asm("2") +#define CALL_ARGS_1(arg1) \ + register unsigned long r2 asm("2") = (unsigned long)(arg1) +#define CALL_ARGS_2(arg1, arg2) \ + CALL_ARGS_1(arg1); \ + register unsigned long r3 asm("3") = (unsigned long)(arg2) +#define CALL_ARGS_3(arg1, arg2, arg3) \ + CALL_ARGS_2(arg1, arg2); \ + register unsigned long r4 asm("4") = (unsigned long)(arg3) +#define CALL_ARGS_4(arg1, arg2, arg3, arg4) \ + CALL_ARGS_3(arg1, arg2, arg3); \ + register unsigned long r4 asm("5") = (unsigned long)(arg4) +#define CALL_ARGS_5(arg1, arg2, arg3, arg4, arg5) \ + CALL_ARGS_4(arg1, arg2, arg3, arg4); \ + register unsigned long r4 asm("6") = (unsigned long)(arg5) + +#define CALL_FMT_0 "=&d" (r2) : +#define CALL_FMT_1 "+&d" (r2) : +#define CALL_FMT_2 CALL_FMT_1 "d" (r3), +#define CALL_FMT_3 CALL_FMT_2 "d" (r4), +#define CALL_FMT_4 CALL_FMT_3 "d" (r5), +#define CALL_FMT_5 CALL_FMT_4 "d" (r6), + +#define CALL_CLOBBER_5 "0", "1", "14", "cc", "memory" +#define CALL_CLOBBER_4 CALL_CLOBBER_5 +#define CALL_CLOBBER_3 CALL_CLOBBER_4, "5" +#define CALL_CLOBBER_2 CALL_CLOBBER_3, "4" +#define CALL_CLOBBER_1 CALL_CLOBBER_2, "3" +#define CALL_CLOBBER_0 CALL_CLOBBER_1 + +#define CALL_ON_STACK(fn, stack, nr, args...) \ +({ \ + CALL_ARGS_##nr(args); \ + unsigned long prev; \ + \ + asm volatile( \ + " la %[_prev],0(15)\n" \ + " la 15,0(%[_stack])\n" \ + " stg %[_prev],%[_bc](15)\n" \ + " brasl 14,%[_fn]\n" \ + " la 15,0(%[_prev])\n" \ + : [_prev] "=&a" (prev), CALL_FMT_##nr \ + [_stack] "a" (stack), \ + [_bc] "i" (offsetof(struct stack_frame, back_chain)), \ + [_fn] "X" (fn) : CALL_CLOBBER_##nr); \ + r2; \ +}) + +#endif /* _ASM_S390_STACKTRACE_H */ diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h index 59c3e91f2cdb..ab3407aa4fd8 100644 --- a/arch/s390/include/asm/syscall.h +++ b/arch/s390/include/asm/syscall.h @@ -14,13 +14,8 @@ #include <linux/err.h> #include <asm/ptrace.h> -/* - * The syscall table always contains 32 bit pointers since we know that the - * address of the function to be called is (way) below 4GB. So the "int" - * type here is what we want [need] for both 32 bit and 64 bit systems. - */ -extern const unsigned int sys_call_table[]; -extern const unsigned int sys_call_table_emu[]; +extern const unsigned long sys_call_table[]; +extern const unsigned long sys_call_table_emu[]; static inline long syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/s390/include/asm/syscall_wrapper.h b/arch/s390/include/asm/syscall_wrapper.h index 5596c5c625d2..3c3d6fe8e2f0 100644 --- a/arch/s390/include/asm/syscall_wrapper.h +++ b/arch/s390/include/asm/syscall_wrapper.h @@ -119,8 +119,8 @@ "Type aliasing is used to sanitize syscall arguments");\ asmlinkage long __s390x_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) \ __attribute__((alias(__stringify(__se_sys##name)))); \ - ALLOW_ERROR_INJECTION(__s390x_sys##name, ERRNO); \ - static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ + ALLOW_ERROR_INJECTION(__s390x_sys##name, ERRNO); \ + long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ __S390_SYS_STUBx(x, name, __VA_ARGS__) \ asmlinkage long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 007fcb9aeeb8..bd2fd9a7821d 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -55,8 +55,10 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n); unsigned long __must_check raw_copy_to_user(void __user *to, const void *from, unsigned long n); +#ifndef CONFIG_KASAN #define INLINE_COPY_FROM_USER #define INLINE_COPY_TO_USER +#endif #ifdef CONFIG_HAVE_MARCH_Z10_FEATURES diff --git a/arch/s390/include/asm/unwind.h b/arch/s390/include/asm/unwind.h new file mode 100644 index 000000000000..6eb2ef105d87 --- /dev/null +++ b/arch/s390/include/asm/unwind.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_UNWIND_H +#define _ASM_S390_UNWIND_H + +#include <linux/sched.h> +#include <linux/ftrace.h> +#include <asm/ptrace.h> +#include <asm/stacktrace.h> + +/* + * To use the stack unwinder it has to be initialized with unwind_start. + * There four combinations for task and regs: + * 1) task==NULL, regs==NULL: the unwind starts for the task that is currently + * running, sp/ip picked up from the CPU registers + * 2) task==NULL, regs!=NULL: the unwind starts from the sp/ip found in + * the struct pt_regs of an interrupt frame for the current task + * 3) task!=NULL, regs==NULL: the unwind starts for an inactive task with + * the sp picked up from task->thread.ksp and the ip picked up from the + * return address stored by __switch_to + * 4) task!=NULL, regs!=NULL: the sp/ip are picked up from the interrupt + * frame 'regs' of a inactive task + * If 'first_frame' is not zero unwind_start skips unwind frames until it + * reaches the specified stack pointer. + * The end of the unwinding is indicated with unwind_done, this can be true + * right after unwind_start, e.g. with first_frame!=0 that can not be found. + * unwind_next_frame skips to the next frame. + * Once the unwind is completed unwind_error() can be used to check if there + * has been a situation where the unwinder could not correctly understand + * the tasks call chain. + */ + +struct unwind_state { + struct stack_info stack_info; + unsigned long stack_mask; + struct task_struct *task; + struct pt_regs *regs; + unsigned long sp, ip; + int graph_idx; + bool reliable; + bool error; +}; + +void __unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs, unsigned long first_frame); +bool unwind_next_frame(struct unwind_state *state); +unsigned long unwind_get_return_address(struct unwind_state *state); + +static inline bool unwind_done(struct unwind_state *state) +{ + return state->stack_info.type == STACK_TYPE_UNKNOWN; +} + +static inline bool unwind_error(struct unwind_state *state) +{ + return state->error; +} + +static inline void unwind_start(struct unwind_state *state, + struct task_struct *task, + struct pt_regs *regs, + unsigned long sp) +{ + sp = sp ? : get_stack_pointer(task, regs); + __unwind_start(state, task, regs, sp); +} + +static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state) +{ + return unwind_done(state) ? NULL : state->regs; +} + +#define unwind_for_each_frame(state, task, regs, first_frame) \ + for (unwind_start(state, task, regs, first_frame); \ + !unwind_done(state); \ + unwind_next_frame(state)) + +static inline void unwind_init(void) {} +static inline void unwind_module_init(struct module *mod, void *orc_ip, + size_t orc_ip_size, void *orc, + size_t orc_size) {} + +#ifdef CONFIG_KASAN +/* + * This disables KASAN checking when reading a value from another task's stack, + * since the other task could be running on another CPU and could have poisoned + * the stack in the meantime. + */ +#define READ_ONCE_TASK_STACK(task, x) \ +({ \ + unsigned long val; \ + if (task == current) \ + val = READ_ONCE(x); \ + else \ + val = READ_ONCE_NOCHECK(x); \ + val; \ +}) +#else +#define READ_ONCE_TASK_STACK(task, x) READ_ONCE(x) +#endif + +#endif /* _ASM_S390_UNWIND_H */ diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h new file mode 100644 index 000000000000..ef3c00b049ab --- /dev/null +++ b/arch/s390/include/asm/uv.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Ultravisor Interfaces + * + * Copyright IBM Corp. 2019 + * + * Author(s): + * Vasily Gorbik <gor@linux.ibm.com> + * Janosch Frank <frankja@linux.ibm.com> + */ +#ifndef _ASM_S390_UV_H +#define _ASM_S390_UV_H + +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/bug.h> +#include <asm/page.h> + +#define UVC_RC_EXECUTED 0x0001 +#define UVC_RC_INV_CMD 0x0002 +#define UVC_RC_INV_STATE 0x0003 +#define UVC_RC_INV_LEN 0x0005 +#define UVC_RC_NO_RESUME 0x0007 + +#define UVC_CMD_QUI 0x0001 +#define UVC_CMD_SET_SHARED_ACCESS 0x1000 +#define UVC_CMD_REMOVE_SHARED_ACCESS 0x1001 + +/* Bits in installed uv calls */ +enum uv_cmds_inst { + BIT_UVC_CMD_QUI = 0, + BIT_UVC_CMD_SET_SHARED_ACCESS = 8, + BIT_UVC_CMD_REMOVE_SHARED_ACCESS = 9, +}; + +struct uv_cb_header { + u16 len; + u16 cmd; /* Command Code */ + u16 rc; /* Response Code */ + u16 rrc; /* Return Reason Code */ +} __packed __aligned(8); + +struct uv_cb_qui { + struct uv_cb_header header; + u64 reserved08; + u64 inst_calls_list[4]; + u64 reserved30[15]; +} __packed __aligned(8); + +struct uv_cb_share { + struct uv_cb_header header; + u64 reserved08[3]; + u64 paddr; + u64 reserved28; +} __packed __aligned(8); + +static inline int uv_call(unsigned long r1, unsigned long r2) +{ + int cc; + + asm volatile( + "0: .insn rrf,0xB9A40000,%[r1],%[r2],0,0\n" + " brc 3,0b\n" + " ipm %[cc]\n" + " srl %[cc],28\n" + : [cc] "=d" (cc) + : [r1] "a" (r1), [r2] "a" (r2) + : "memory", "cc"); + return cc; +} + +#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST +extern int prot_virt_guest; + +static inline int is_prot_virt_guest(void) +{ + return prot_virt_guest; +} + +static inline int share(unsigned long addr, u16 cmd) +{ + struct uv_cb_share uvcb = { + .header.cmd = cmd, + .header.len = sizeof(uvcb), + .paddr = addr + }; + + if (!is_prot_virt_guest()) + return -ENOTSUPP; + /* + * Sharing is page wise, if we encounter addresses that are + * not page aligned, we assume something went wrong. If + * malloced structs are passed to this function, we could leak + * data to the hypervisor. + */ + BUG_ON(addr & ~PAGE_MASK); + + if (!uv_call(0, (u64)&uvcb)) + return 0; + return -EINVAL; +} + +/* + * Guest 2 request to the Ultravisor to make a page shared with the + * hypervisor for IO. + * + * @addr: Real or absolute address of the page to be shared + */ +static inline int uv_set_shared(unsigned long addr) +{ + return share(addr, UVC_CMD_SET_SHARED_ACCESS); +} + +/* + * Guest 2 request to the Ultravisor to make a page unshared. + * + * @addr: Real or absolute address of the page to be unshared + */ +static inline int uv_remove_shared(unsigned long addr) +{ + return share(addr, UVC_CMD_REMOVE_SHARED_ACCESS); +} + +void uv_query_info(void); +#else +#define is_prot_virt_guest() 0 +static inline int uv_set_shared(unsigned long addr) { return 0; } +static inline int uv_remove_shared(unsigned long addr) { return 0; } +static inline void uv_query_info(void) {} +#endif + +#endif /* _ASM_S390_UV_H */ diff --git a/arch/s390/include/asm/vmlinux.lds.h b/arch/s390/include/asm/vmlinux.lds.h index 2d127f900352..cbe670a6861b 100644 --- a/arch/s390/include/asm/vmlinux.lds.h +++ b/arch/s390/include/asm/vmlinux.lds.h @@ -18,3 +18,16 @@ *(SORT_BY_ALIGNMENT(SORT_BY_NAME(.boot.data*))) \ __boot_data_end = .; \ } + +/* + * .boot.preserved.data is similar to .boot.data, but it is not part of the + * .init section and thus will be preserved for later use in the decompressed + * kernel. + */ +#define BOOT_DATA_PRESERVED \ + . = ALIGN(PAGE_SIZE); \ + .boot.preserved.data : { \ + __boot_data_preserved_start = .; \ + *(SORT_BY_ALIGNMENT(SORT_BY_NAME(.boot.preserved.data*))) \ + __boot_data_preserved_end = .; \ + } diff --git a/arch/s390/include/uapi/asm/ipl.h b/arch/s390/include/uapi/asm/ipl.h new file mode 100644 index 000000000000..fd32b1cd80d2 --- /dev/null +++ b/arch/s390/include/uapi/asm/ipl.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_UAPI_IPL_H +#define _ASM_S390_UAPI_IPL_H + +#include <linux/types.h> + +/* IPL Parameter List header */ +struct ipl_pl_hdr { + __u32 len; + __u8 flags; + __u8 reserved1[2]; + __u8 version; +} __packed; + +#define IPL_PL_FLAG_IPLPS 0x80 +#define IPL_PL_FLAG_SIPL 0x40 +#define IPL_PL_FLAG_IPLSR 0x20 + +/* IPL Parameter Block header */ +struct ipl_pb_hdr { + __u32 len; + __u8 pbt; +} __packed; + +/* IPL Parameter Block types */ +enum ipl_pbt { + IPL_PBT_FCP = 0, + IPL_PBT_SCP_DATA = 1, + IPL_PBT_CCW = 2, +}; + +/* IPL Parameter Block 0 with common fields */ +struct ipl_pb0_common { + __u32 len; + __u8 pbt; + __u8 flags; + __u8 reserved1[2]; + __u8 loadparm[8]; + __u8 reserved2[84]; +} __packed; + +#define IPL_PB0_FLAG_LOADPARM 0x80 + +/* IPL Parameter Block 0 for FCP */ +struct ipl_pb0_fcp { + __u32 len; + __u8 pbt; + __u8 reserved1[3]; + __u8 loadparm[8]; + __u8 reserved2[304]; + __u8 opt; + __u8 reserved3[3]; + __u8 cssid; + __u8 reserved4[1]; + __u16 devno; + __u8 reserved5[4]; + __u64 wwpn; + __u64 lun; + __u32 bootprog; + __u8 reserved6[12]; + __u64 br_lba; + __u32 scp_data_len; + __u8 reserved7[260]; + __u8 scp_data[]; +} __packed; + +#define IPL_PB0_FCP_OPT_IPL 0x10 +#define IPL_PB0_FCP_OPT_DUMP 0x20 + +/* IPL Parameter Block 0 for CCW */ +struct ipl_pb0_ccw { + __u32 len; + __u8 pbt; + __u8 flags; + __u8 reserved1[2]; + __u8 loadparm[8]; + __u8 reserved2[84]; + __u16 reserved3 : 13; + __u8 ssid : 3; + __u16 devno; + __u8 vm_flags; + __u8 reserved4[3]; + __u32 vm_parm_len; + __u8 nss_name[8]; + __u8 vm_parm[64]; + __u8 reserved5[8]; +} __packed; + +#define IPL_PB0_CCW_VM_FLAG_NSS 0x80 +#define IPL_PB0_CCW_VM_FLAG_VP 0x40 + +/* IPL Parameter Block 1 for additional SCP data */ +struct ipl_pb1_scp_data { + __u32 len; + __u8 pbt; + __u8 scp_data[]; +} __packed; + +/* IPL Report List header */ +struct ipl_rl_hdr { + __u32 len; + __u8 flags; + __u8 reserved1[2]; + __u8 version; + __u8 reserved2[8]; +} __packed; + +/* IPL Report Block header */ +struct ipl_rb_hdr { + __u32 len; + __u8 rbt; + __u8 reserved1[11]; +} __packed; + +/* IPL Report Block types */ +enum ipl_rbt { + IPL_RBT_CERTIFICATES = 1, + IPL_RBT_COMPONENTS = 2, +}; + +/* IPL Report Block for the certificate list */ +struct ipl_rb_certificate_entry { + __u64 addr; + __u64 len; +} __packed; + +struct ipl_rb_certificates { + __u32 len; + __u8 rbt; + __u8 reserved1[11]; + struct ipl_rb_certificate_entry entries[]; +} __packed; + +/* IPL Report Block for the component list */ +struct ipl_rb_component_entry { + __u64 addr; + __u64 len; + __u8 flags; + __u8 reserved1[5]; + __u16 certificate_index; + __u8 reserved2[8]; +}; + +#define IPL_RB_COMPONENT_FLAG_SIGNED 0x80 +#define IPL_RB_COMPONENT_FLAG_VERIFIED 0x40 + +struct ipl_rb_components { + __u32 len; + __u8 rbt; + __u8 reserved1[11]; + struct ipl_rb_component_entry entries[]; +} __packed; + +#endif diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 8a62c7f72e1b..b0478d01a0c5 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -39,6 +39,7 @@ CFLAGS_smp.o := -Wno-nonnull # CFLAGS_stacktrace.o += -fno-optimize-sibling-calls CFLAGS_dumpstack.o += -fno-optimize-sibling-calls +CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls # # Pass UTS_MACHINE for user_regset definition @@ -51,7 +52,7 @@ obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o early_nobss.o obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o pgm_check.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o -obj-y += nospec-branch.o ipl_vmparm.o +obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o extra-y += head64.o vmlinux.lds @@ -77,6 +78,8 @@ obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o +obj-$(CONFIG_IMA) += ima_arch.o + obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf_common.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_events.o perf_regs.o @@ -86,7 +89,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace.o # vdso obj-y += vdso64/ -obj-$(CONFIG_COMPAT) += vdso32/ +obj-$(CONFIG_COMPAT_VDSO) += vdso32/ chkbss := head64.o early_nobss.o include $(srctree)/arch/s390/scripts/Makefile.chkbss diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 164bec175628..41ac4ad21311 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -16,6 +16,7 @@ #include <asm/pgtable.h> #include <asm/gmap.h> #include <asm/nmi.h> +#include <asm/stacktrace.h> int main(void) { diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/base.S index f268fca67e82..2f39ea57f358 100644 --- a/arch/s390/kernel/base.S +++ b/arch/s390/kernel/base.S @@ -28,6 +28,7 @@ ENTRY(s390_base_mcck_handler) 1: la %r1,4095 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1) lpswe __LC_MCK_OLD_PSW +ENDPROC(s390_base_mcck_handler) .section .bss .align 8 @@ -48,6 +49,7 @@ ENTRY(s390_base_ext_handler) 1: lmg %r0,%r15,__LC_SAVE_AREA_ASYNC ni __LC_EXT_OLD_PSW+1,0xfd # clear wait state bit lpswe __LC_EXT_OLD_PSW +ENDPROC(s390_base_ext_handler) .section .bss .align 8 @@ -68,6 +70,7 @@ ENTRY(s390_base_pgm_handler) lmg %r0,%r15,__LC_SAVE_AREA_SYNC lpswe __LC_PGM_OLD_PSW 1: lpswe disabled_wait_psw-0b(%r13) +ENDPROC(s390_base_pgm_handler) .align 8 disabled_wait_psw: @@ -79,71 +82,3 @@ disabled_wait_psw: s390_base_pgm_handler_fn: .quad 0 .previous - -# -# Calls diag 308 subcode 1 and continues execution -# -ENTRY(diag308_reset) - larl %r4,.Lctlregs # Save control registers - stctg %c0,%c15,0(%r4) - lg %r2,0(%r4) # Disable lowcore protection - nilh %r2,0xefff - larl %r4,.Lctlreg0 - stg %r2,0(%r4) - lctlg %c0,%c0,0(%r4) - larl %r4,.Lfpctl # Floating point control register - stfpc 0(%r4) - larl %r4,.Lprefix # Save prefix register - stpx 0(%r4) - larl %r4,.Lprefix_zero # Set prefix register to 0 - spx 0(%r4) - larl %r4,.Lcontinue_psw # Save PSW flags - epsw %r2,%r3 - stm %r2,%r3,0(%r4) - larl %r4,.Lrestart_psw # Setup restart PSW at absolute 0 - lghi %r3,0 - lg %r4,0(%r4) # Save PSW - sturg %r4,%r3 # Use sturg, because of large pages - lghi %r1,1 - lghi %r0,0 - diag %r0,%r1,0x308 -.Lrestart_part2: - lhi %r0,0 # Load r0 with zero - lhi %r1,2 # Use mode 2 = ESAME (dump) - sigp %r1,%r0,SIGP_SET_ARCHITECTURE # Switch to ESAME mode - sam64 # Switch to 64 bit addressing mode - larl %r4,.Lctlregs # Restore control registers - lctlg %c0,%c15,0(%r4) - larl %r4,.Lfpctl # Restore floating point ctl register - lfpc 0(%r4) - larl %r4,.Lprefix # Restore prefix register - spx 0(%r4) - larl %r4,.Lcontinue_psw # Restore PSW flags - lpswe 0(%r4) -.Lcontinue: - BR_EX %r14 -.align 16 -.Lrestart_psw: - .long 0x00080000,0x80000000 + .Lrestart_part2 - - .section .data..nosave,"aw",@progbits -.align 8 -.Lcontinue_psw: - .quad 0,.Lcontinue - .previous - - .section .bss -.align 8 -.Lctlreg0: - .quad 0 -.Lctlregs: - .rept 16 - .quad 0 - .endr -.Lfpctl: - .long 0 -.Lprefix: - .long 0 -.Lprefix_zero: - .long 0 - .previous diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index 7edaa733a77f..e9dac9a24d3f 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -13,6 +13,7 @@ #include <linux/debugfs.h> #include <asm/diag.h> #include <asm/trace/diag.h> +#include <asm/sections.h> struct diag_stat { unsigned int counter[NR_DIAG_STAT]; @@ -49,6 +50,9 @@ static const struct diag_desc diag_map[NR_DIAG_STAT] = { [DIAG_STAT_X500] = { .code = 0x500, .name = "Virtio Service" }, }; +struct diag_ops __bootdata_preserved(diag_dma_ops); +struct diag210 *__bootdata_preserved(__diag210_tmp_dma); + static int show_diag_stat(struct seq_file *m, void *v) { struct diag_stat *stat; @@ -139,30 +143,10 @@ EXPORT_SYMBOL(diag_stat_inc_norecursion); /* * Diagnose 14: Input spool file manipulation */ -static inline int __diag14(unsigned long rx, unsigned long ry1, - unsigned long subcode) -{ - register unsigned long _ry1 asm("2") = ry1; - register unsigned long _ry2 asm("3") = subcode; - int rc = 0; - - asm volatile( - " sam31\n" - " diag %2,2,0x14\n" - " sam64\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (rc), "+d" (_ry2) - : "d" (rx), "d" (_ry1) - : "cc"); - - return rc; -} - int diag14(unsigned long rx, unsigned long ry1, unsigned long subcode) { diag_stat_inc(DIAG_STAT_X014); - return __diag14(rx, ry1, subcode); + return diag_dma_ops.diag14(rx, ry1, subcode); } EXPORT_SYMBOL(diag14); @@ -195,30 +179,17 @@ EXPORT_SYMBOL(diag204); */ int diag210(struct diag210 *addr) { - /* - * diag 210 needs its data below the 2GB border, so we - * use a static data area to be sure - */ - static struct diag210 diag210_tmp; static DEFINE_SPINLOCK(diag210_lock); unsigned long flags; int ccode; spin_lock_irqsave(&diag210_lock, flags); - diag210_tmp = *addr; + *__diag210_tmp_dma = *addr; diag_stat_inc(DIAG_STAT_X210); - asm volatile( - " lhi %0,-1\n" - " sam31\n" - " diag %1,0,0x210\n" - "0: ipm %0\n" - " srl %0,28\n" - "1: sam64\n" - EX_TABLE(0b, 1b) - : "=&d" (ccode) : "a" (&diag210_tmp) : "cc", "memory"); - - *addr = diag210_tmp; + ccode = diag_dma_ops.diag210(__diag210_tmp_dma); + + *addr = *__diag210_tmp_dma; spin_unlock_irqrestore(&diag210_lock, flags); return ccode; @@ -243,27 +214,9 @@ EXPORT_SYMBOL(diag224); /* * Diagnose 26C: Access Certain System Information */ -static inline int __diag26c(void *req, void *resp, enum diag26c_sc subcode) -{ - register unsigned long _req asm("2") = (addr_t) req; - register unsigned long _resp asm("3") = (addr_t) resp; - register unsigned long _subcode asm("4") = subcode; - register unsigned long _rc asm("5") = -EOPNOTSUPP; - - asm volatile( - " sam31\n" - " diag %[rx],%[ry],0x26c\n" - "0: sam64\n" - EX_TABLE(0b,0b) - : "+d" (_rc) - : [rx] "d" (_req), "d" (_resp), [ry] "d" (_subcode) - : "cc", "memory"); - return _rc; -} - int diag26c(void *req, void *resp, enum diag26c_sc subcode) { diag_stat_inc(DIAG_STAT_X26C); - return __diag26c(req, resp, subcode); + return diag_dma_ops.diag26c(req, resp, subcode); } EXPORT_SYMBOL(diag26c); diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index cb7f55bbe06e..9e87b68be21c 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -21,95 +21,124 @@ #include <asm/debug.h> #include <asm/dis.h> #include <asm/ipl.h> +#include <asm/unwind.h> -/* - * For dump_trace we have tree different stack to consider: - * - the panic stack which is used if the kernel stack has overflown - * - the asynchronous interrupt stack (cpu related) - * - the synchronous kernel stack (process related) - * The stack trace can start at any of the three stacks and can potentially - * touch all of them. The order is: panic stack, async stack, sync stack. - */ -static unsigned long __no_sanitize_address -__dump_trace(dump_trace_func_t func, void *data, unsigned long sp, - unsigned long low, unsigned long high) +const char *stack_type_name(enum stack_type type) { - struct stack_frame *sf; - struct pt_regs *regs; - - while (1) { - if (sp < low || sp > high - sizeof(*sf)) - return sp; - sf = (struct stack_frame *) sp; - if (func(data, sf->gprs[8], 0)) - return sp; - /* Follow the backchain. */ - while (1) { - low = sp; - sp = sf->back_chain; - if (!sp) - break; - if (sp <= low || sp > high - sizeof(*sf)) - return sp; - sf = (struct stack_frame *) sp; - if (func(data, sf->gprs[8], 1)) - return sp; - } - /* Zero backchain detected, check for interrupt frame. */ - sp = (unsigned long) (sf + 1); - if (sp <= low || sp > high - sizeof(*regs)) - return sp; - regs = (struct pt_regs *) sp; - if (!user_mode(regs)) { - if (func(data, regs->psw.addr, 1)) - return sp; - } - low = sp; - sp = regs->gprs[15]; + switch (type) { + case STACK_TYPE_TASK: + return "task"; + case STACK_TYPE_IRQ: + return "irq"; + case STACK_TYPE_NODAT: + return "nodat"; + case STACK_TYPE_RESTART: + return "restart"; + default: + return "unknown"; } } -void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, - unsigned long sp) +static inline bool in_stack(unsigned long sp, struct stack_info *info, + enum stack_type type, unsigned long low, + unsigned long high) +{ + if (sp < low || sp >= high) + return false; + info->type = type; + info->begin = low; + info->end = high; + return true; +} + +static bool in_task_stack(unsigned long sp, struct task_struct *task, + struct stack_info *info) +{ + unsigned long stack; + + stack = (unsigned long) task_stack_page(task); + return in_stack(sp, info, STACK_TYPE_TASK, stack, stack + THREAD_SIZE); +} + +static bool in_irq_stack(unsigned long sp, struct stack_info *info) { - unsigned long frame_size; + unsigned long frame_size, top; frame_size = STACK_FRAME_OVERHEAD + sizeof(struct pt_regs); -#ifdef CONFIG_CHECK_STACK - sp = __dump_trace(func, data, sp, - S390_lowcore.nodat_stack + frame_size - THREAD_SIZE, - S390_lowcore.nodat_stack + frame_size); -#endif - sp = __dump_trace(func, data, sp, - S390_lowcore.async_stack + frame_size - THREAD_SIZE, - S390_lowcore.async_stack + frame_size); - task = task ?: current; - __dump_trace(func, data, sp, - (unsigned long)task_stack_page(task), - (unsigned long)task_stack_page(task) + THREAD_SIZE); + top = S390_lowcore.async_stack + frame_size; + return in_stack(sp, info, STACK_TYPE_IRQ, top - THREAD_SIZE, top); +} + +static bool in_nodat_stack(unsigned long sp, struct stack_info *info) +{ + unsigned long frame_size, top; + + frame_size = STACK_FRAME_OVERHEAD + sizeof(struct pt_regs); + top = S390_lowcore.nodat_stack + frame_size; + return in_stack(sp, info, STACK_TYPE_NODAT, top - THREAD_SIZE, top); } -EXPORT_SYMBOL_GPL(dump_trace); -static int show_address(void *data, unsigned long address, int reliable) +static bool in_restart_stack(unsigned long sp, struct stack_info *info) { - if (reliable) - printk(" [<%016lx>] %pSR \n", address, (void *)address); - else - printk("([<%016lx>] %pSR)\n", address, (void *)address); + unsigned long frame_size, top; + + frame_size = STACK_FRAME_OVERHEAD + sizeof(struct pt_regs); + top = S390_lowcore.restart_stack + frame_size; + return in_stack(sp, info, STACK_TYPE_RESTART, top - THREAD_SIZE, top); +} + +int get_stack_info(unsigned long sp, struct task_struct *task, + struct stack_info *info, unsigned long *visit_mask) +{ + if (!sp) + goto unknown; + + task = task ? : current; + + /* Check per-task stack */ + if (in_task_stack(sp, task, info)) + goto recursion_check; + + if (task != current) + goto unknown; + + /* Check per-cpu stacks */ + if (!in_irq_stack(sp, info) && + !in_nodat_stack(sp, info) && + !in_restart_stack(sp, info)) + goto unknown; + +recursion_check: + /* + * Make sure we don't iterate through any given stack more than once. + * If it comes up a second time then there's something wrong going on: + * just break out and report an unknown stack type. + */ + if (*visit_mask & (1UL << info->type)) { + printk_deferred_once(KERN_WARNING + "WARNING: stack recursion on stack type %d\n", + info->type); + goto unknown; + } + *visit_mask |= 1UL << info->type; return 0; +unknown: + info->type = STACK_TYPE_UNKNOWN; + return -EINVAL; } void show_stack(struct task_struct *task, unsigned long *stack) { - unsigned long sp = (unsigned long) stack; + struct unwind_state state; - if (!sp) - sp = task ? task->thread.ksp : current_stack_pointer(); printk("Call Trace:\n"); - dump_trace(show_address, NULL, task, sp); if (!task) task = current; - debug_show_held_locks(task); + unwind_for_each_frame(&state, task, NULL, (unsigned long) stack) + printk(state.reliable ? " [<%016lx>] %pSR \n" : + "([<%016lx>] %pSR)\n", + state.ip, (void *) state.ip); + debug_show_held_locks(task ? : current); } static void show_last_breaking_event(struct pt_regs *regs) diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index d6edf45f93b9..629f173f60cd 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -30,6 +30,7 @@ #include <asm/sclp.h> #include <asm/facility.h> #include <asm/boot_data.h> +#include <asm/pci_insn.h> #include "entry.h" /* @@ -138,9 +139,9 @@ static void early_pgm_check_handler(void) unsigned long addr; addr = S390_lowcore.program_old_psw.addr; - fixup = search_exception_tables(addr); + fixup = s390_search_extables(addr); if (!fixup) - disabled_wait(0); + disabled_wait(); /* Disable low address protection before storing into lowcore. */ __ctl_store(cr0, 0, 0); cr0_new = cr0 & ~(1UL << 28); @@ -235,6 +236,7 @@ static __init void detect_machine_facilities(void) clock_comparator_max = -1ULL >> 1; __ctl_set_bit(0, 53); } + enable_mio_ctl(); } static inline void save_vector_registers(void) @@ -296,7 +298,7 @@ static void __init check_image_bootable(void) sclp_early_printk("Linux kernel boot failure: An attempt to boot a vmlinux ELF image failed.\n"); sclp_early_printk("This image does not contain all parts necessary for starting up. Use\n"); sclp_early_printk("bzImage or arch/s390/boot/compressed/vmlinux instead.\n"); - disabled_wait(0xbadb007); + disabled_wait(); } void __init startup_init(void) @@ -309,7 +311,6 @@ void __init startup_init(void) setup_facility_list(); detect_machine_type(); setup_arch_string(); - ipl_store_parameters(); setup_boot_command_line(); detect_diag9c(); detect_diag44(); diff --git a/arch/s390/kernel/early_nobss.c b/arch/s390/kernel/early_nobss.c index 8d73f7fae16e..52a3ef959341 100644 --- a/arch/s390/kernel/early_nobss.c +++ b/arch/s390/kernel/early_nobss.c @@ -25,7 +25,7 @@ static void __init reset_tod_clock(void) return; /* TOD clock not running. Set the clock to Unix Epoch. */ if (set_tod_clock(TOD_UNIX_EPOCH) != 0 || store_tod_clock(&time) != 0) - disabled_wait(0); + disabled_wait(); memset(tod_clock_base, 0, 16); *(__u64 *) &tod_clock_base[1] = TOD_UNIX_EPOCH; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 583d65ef5007..3f4d272577d3 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -224,6 +224,7 @@ ENTRY(__bpon) .globl __bpon BPON BR_EX %r14 +ENDPROC(__bpon) /* * Scheduler resume function, called by switch_to @@ -248,6 +249,7 @@ ENTRY(__switch_to) lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task ALTERNATIVE "", ".insn s,0xb2800000,_LPP_OFFSET", 40 BR_EX %r14 +ENDPROC(__switch_to) .L__critical_start: @@ -324,6 +326,7 @@ sie_exit: EX_TABLE(.Lrewind_pad4,.Lsie_fault) EX_TABLE(.Lrewind_pad2,.Lsie_fault) EX_TABLE(sie_exit,.Lsie_fault) +ENDPROC(sie64a) EXPORT_SYMBOL(sie64a) EXPORT_SYMBOL(sie_exit) #endif @@ -358,19 +361,19 @@ ENTRY(system_call) # load address of system call table lg %r10,__THREAD_sysc_table(%r13,%r12) llgh %r8,__PT_INT_CODE+2(%r11) - slag %r8,%r8,2 # shift and test for svc 0 + slag %r8,%r8,3 # shift and test for svc 0 jnz .Lsysc_nr_ok # svc 0: system call number in %r1 llgfr %r1,%r1 # clear high word in r1 cghi %r1,NR_syscalls jnl .Lsysc_nr_ok sth %r1,__PT_INT_CODE+2(%r11) - slag %r8,%r1,2 + slag %r8,%r1,3 .Lsysc_nr_ok: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) stg %r2,__PT_ORIG_GPR2(%r11) stg %r7,STACK_FRAME_OVERHEAD(%r15) - lgf %r9,0(%r8,%r10) # get system call add. + lg %r9,0(%r8,%r10) # get system call add. TSTMSK __TI_flags(%r12),_TIF_TRACE jnz .Lsysc_tracesys BASR_EX %r14,%r9 # call sys_xxxx @@ -556,8 +559,8 @@ ENTRY(system_call) lghi %r0,NR_syscalls clgr %r0,%r2 jnh .Lsysc_tracenogo - sllg %r8,%r2,2 - lgf %r9,0(%r8,%r10) + sllg %r8,%r2,3 + lg %r9,0(%r8,%r10) .Lsysc_tracego: lmg %r3,%r7,__PT_R3(%r11) stg %r7,STACK_FRAME_OVERHEAD(%r15) @@ -570,6 +573,7 @@ ENTRY(system_call) lgr %r2,%r11 # pass pointer to pt_regs larl %r14,.Lsysc_return jg do_syscall_trace_exit +ENDPROC(system_call) # # a new process exits the kernel with ret_from_fork @@ -584,10 +588,16 @@ ENTRY(ret_from_fork) jne .Lsysc_tracenogo # it's a kernel thread lmg %r9,%r10,__PT_R9(%r11) # load gprs + la %r2,0(%r10) + BASR_EX %r14,%r9 + j .Lsysc_tracenogo +ENDPROC(ret_from_fork) + ENTRY(kernel_thread_starter) la %r2,0(%r10) BASR_EX %r14,%r9 j .Lsysc_tracenogo +ENDPROC(kernel_thread_starter) /* * Program check handler routine @@ -665,9 +675,9 @@ ENTRY(pgm_check_handler) larl %r1,pgm_check_table llgh %r10,__PT_INT_CODE+2(%r11) nill %r10,0x007f - sll %r10,2 + sll %r10,3 je .Lpgm_return - lgf %r9,0(%r10,%r1) # load address of handler routine + lg %r9,0(%r10,%r1) # load address of handler routine lgr %r2,%r11 # pass pointer to pt_regs BASR_EX %r14,%r9 # branch to interrupt-handler .Lpgm_return: @@ -698,6 +708,7 @@ ENTRY(pgm_check_handler) stg %r14,__LC_RETURN_PSW+8 lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs +ENDPROC(pgm_check_handler) /* * IO interrupt handler routine @@ -926,6 +937,7 @@ ENTRY(io_int_handler) ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts TRACE_IRQS_OFF j .Lio_return +ENDPROC(io_int_handler) /* * External interrupt handler routine @@ -965,6 +977,7 @@ ENTRY(ext_int_handler) lghi %r3,EXT_INTERRUPT brasl %r14,do_IRQ j .Lio_return +ENDPROC(ext_int_handler) /* * Load idle PSW. The second "half" of this function is in .Lcleanup_idle. @@ -989,6 +1002,7 @@ ENTRY(psw_idle) lpswe __SF_EMPTY(%r15) BR_EX %r14 .Lpsw_idle_end: +ENDPROC(psw_idle) /* * Store floating-point controls and floating-point or vector register @@ -1031,6 +1045,7 @@ ENTRY(save_fpu_regs) .Lsave_fpu_regs_exit: BR_EX %r14 .Lsave_fpu_regs_end: +ENDPROC(save_fpu_regs) EXPORT_SYMBOL(save_fpu_regs) /* @@ -1077,6 +1092,7 @@ load_fpu_regs: .Lload_fpu_regs_exit: BR_EX %r14 .Lload_fpu_regs_end: +ENDPROC(load_fpu_regs) .L__critical_end: @@ -1206,6 +1222,7 @@ ENTRY(mcck_int_handler) lg %r15,__LC_NODAT_STACK la %r11,STACK_FRAME_OVERHEAD(%r15) j .Lmcck_skip +ENDPROC(mcck_int_handler) # # PSW restart interrupt handler @@ -1232,6 +1249,7 @@ ENTRY(restart_int_handler) 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu brc 2,2b 3: j 3b +ENDPROC(restart_int_handler) .section .kprobes.text, "ax" @@ -1241,7 +1259,7 @@ ENTRY(restart_int_handler) * No need to properly save the registers, we are going to panic anyway. * Setup a pt_regs so that show_trace can provide a good call trace. */ -stack_overflow: +ENTRY(stack_overflow) lg %r15,__LC_NODAT_STACK # change to panic stack la %r11,STACK_FRAME_OVERHEAD(%r15) stmg %r0,%r7,__PT_R0(%r11) @@ -1251,9 +1269,10 @@ stack_overflow: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs jg kernel_stack_overflow +ENDPROC(stack_overflow) #endif -cleanup_critical: +ENTRY(cleanup_critical) #if IS_ENABLED(CONFIG_KVM) clg %r9,BASED(.Lcleanup_table_sie) # .Lsie_gmap jl 0f @@ -1289,6 +1308,7 @@ cleanup_critical: clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end jl .Lcleanup_load_fpu_regs 0: BR_EX %r14,%r11 +ENDPROC(cleanup_critical) .align 8 .Lcleanup_table: @@ -1512,7 +1532,7 @@ cleanup_critical: .quad .Lsie_skip - .Lsie_entry #endif .section .rodata, "a" -#define SYSCALL(esame,emu) .long __s390x_ ## esame +#define SYSCALL(esame,emu) .quad __s390x_ ## esame .globl sys_call_table sys_call_table: #include "asm/syscall_table.h" @@ -1520,7 +1540,7 @@ sys_call_table: #ifdef CONFIG_COMPAT -#define SYSCALL(esame,emu) .long __s390_ ## emu +#define SYSCALL(esame,emu) .quad __s390_ ## emu .globl sys_call_table_emu sys_call_table_emu: #include "asm/syscall_table.h" diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index c3816ae108b0..20420c2b8a14 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -65,7 +65,7 @@ int setup_profiling_timer(unsigned int multiplier); void __init time_init(void); int pfn_is_nosave(unsigned long); void s390_early_resume(void); -unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip); +unsigned long prepare_ftrace_return(unsigned long parent, unsigned long sp, unsigned long ip); struct s390_mmap_arg_struct; struct fadvise64_64_args; diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 39b13d71a8fe..1bb85f60c0dd 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -201,17 +201,18 @@ device_initcall(ftrace_plt_init); * Hook the return address and push it in the stack of return addresses * in current thread info. */ -unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) +unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp, + unsigned long ip) { if (unlikely(ftrace_graph_is_dead())) goto out; if (unlikely(atomic_read(¤t->tracing_graph_pause))) goto out; ip -= MCOUNT_INSN_SIZE; - if (!function_graph_enter(parent, ip, 0, NULL)) - parent = (unsigned long) return_to_handler; + if (!function_graph_enter(ra, ip, 0, (void *) sp)) + ra = (unsigned long) return_to_handler; out: - return parent; + return ra; } NOKPROBE_SYMBOL(prepare_ftrace_return); diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index 56491e636eab..5aea1a527443 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -26,7 +26,6 @@ ENTRY(startup_continue) 0: larl %r1,tod_clock_base mvc 0(16,%r1),__LC_BOOT_CLOCK larl %r13,.LPG1 # get base - lctlg %c0,%c15,.Lctl-.LPG1(%r13) # load control registers larl %r0,boot_vdso_data stg %r0,__LC_VDSO_PER_CPU # @@ -61,22 +60,6 @@ ENTRY(startup_continue) .align 16 .LPG1: -.Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space - .quad 0 # cr1: primary space segment table - .quad .Lduct # cr2: dispatchable unit control table - .quad 0 # cr3: instruction authorization - .quad 0xffff # cr4: instruction authorization - .quad .Lduct # cr5: primary-aste origin - .quad 0 # cr6: I/O interrupts - .quad 0 # cr7: secondary space segment table - .quad 0 # cr8: access registers translation - .quad 0 # cr9: tracing off - .quad 0 # cr10: tracing off - .quad 0 # cr11: tracing off - .quad 0 # cr12: tracing off - .quad 0 # cr13: home space segment table - .quad 0xc0000000 # cr14: machine check handling off - .quad .Llinkage_stack # cr15: linkage stack operations .Lpcmsk:.quad 0x0000000180000000 .L4malign:.quad 0xffffffffffc00000 .Lscan2g:.quad 0x80000000 + 0x20000 - 8 # 2GB + 128K - 8 @@ -84,14 +67,5 @@ ENTRY(startup_continue) .Lparmaddr: .quad PARMAREA .align 64 -.Lduct: .long 0,.Laste,.Laste,0,.Lduald,0,0,0 - .long 0,0,0,0,0,0,0,0 -.Laste: .quad 0,0xffffffffffffffff,0,0,0,0,0,0 - .align 128 -.Lduald:.rept 8 - .long 0x80000000,0,0,0 # invalid access-list entries - .endr -.Llinkage_stack: - .long 0,0,0x89000000,0,0,0,0x8a000000,0 .Ldw: .quad 0x0002000180000000,0x0000000000000000 .Laregs:.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/arch/s390/kernel/ima_arch.c b/arch/s390/kernel/ima_arch.c new file mode 100644 index 000000000000..f3c3e6e1c5d3 --- /dev/null +++ b/arch/s390/kernel/ima_arch.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/ima.h> +#include <asm/boot_data.h> + +bool arch_ima_get_secureboot(void) +{ + return ipl_secure_flag; +} + +const char * const *arch_get_ima_policy(void) +{ + return NULL; +} diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 18a5d6317acc..d836af3ccc38 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -31,6 +31,7 @@ #include <asm/os_info.h> #include <asm/sections.h> #include <asm/boot_data.h> +#include <asm/uv.h> #include "entry.h" #define IPL_PARM_BLOCK_VERSION 0 @@ -119,11 +120,15 @@ static char *dump_type_str(enum dump_type type) } } -struct ipl_parameter_block __bootdata(early_ipl_block); -int __bootdata(early_ipl_block_valid); +int __bootdata_preserved(ipl_block_valid); +struct ipl_parameter_block __bootdata_preserved(ipl_block); +int __bootdata_preserved(ipl_secure_flag); -static int ipl_block_valid; -static struct ipl_parameter_block ipl_block; +unsigned long __bootdata_preserved(ipl_cert_list_addr); +unsigned long __bootdata_preserved(ipl_cert_list_size); + +unsigned long __bootdata(early_ipl_comp_list_addr); +unsigned long __bootdata(early_ipl_comp_list_size); static int reipl_capabilities = IPL_TYPE_UNKNOWN; @@ -246,11 +251,11 @@ static __init enum ipl_type get_ipl_type(void) if (!ipl_block_valid) return IPL_TYPE_UNKNOWN; - switch (ipl_block.hdr.pbt) { - case DIAG308_IPL_TYPE_CCW: + switch (ipl_block.pb0_hdr.pbt) { + case IPL_PBT_CCW: return IPL_TYPE_CCW; - case DIAG308_IPL_TYPE_FCP: - if (ipl_block.ipl_info.fcp.opt == DIAG308_IPL_OPT_DUMP) + case IPL_PBT_FCP: + if (ipl_block.fcp.opt == IPL_PB0_FCP_OPT_DUMP) return IPL_TYPE_FCP_DUMP; else return IPL_TYPE_FCP; @@ -269,12 +274,35 @@ static ssize_t ipl_type_show(struct kobject *kobj, struct kobj_attribute *attr, static struct kobj_attribute sys_ipl_type_attr = __ATTR_RO(ipl_type); +static ssize_t ipl_secure_show(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + return sprintf(page, "%i\n", !!ipl_secure_flag); +} + +static struct kobj_attribute sys_ipl_secure_attr = + __ATTR(secure, 0444, ipl_secure_show, NULL); + +static ssize_t ipl_has_secure_show(struct kobject *kobj, + struct kobj_attribute *attr, char *page) +{ + if (MACHINE_IS_LPAR) + return sprintf(page, "%i\n", !!sclp.has_sipl); + else if (MACHINE_IS_VM) + return sprintf(page, "%i\n", !!sclp.has_sipl_g2); + else + return sprintf(page, "%i\n", 0); +} + +static struct kobj_attribute sys_ipl_has_secure_attr = + __ATTR(has_secure, 0444, ipl_has_secure_show, NULL); + static ssize_t ipl_vm_parm_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) { char parm[DIAG308_VMPARM_SIZE + 1] = {}; - if (ipl_block_valid && (ipl_block.hdr.pbt == DIAG308_IPL_TYPE_CCW)) + if (ipl_block_valid && (ipl_block.pb0_hdr.pbt == IPL_PBT_CCW)) ipl_block_get_ascii_vmparm(parm, sizeof(parm), &ipl_block); return sprintf(page, "%s\n", parm); } @@ -287,12 +315,11 @@ static ssize_t sys_ipl_device_show(struct kobject *kobj, { switch (ipl_info.type) { case IPL_TYPE_CCW: - return sprintf(page, "0.%x.%04x\n", ipl_block.ipl_info.ccw.ssid, - ipl_block.ipl_info.ccw.devno); + return sprintf(page, "0.%x.%04x\n", ipl_block.ccw.ssid, + ipl_block.ccw.devno); case IPL_TYPE_FCP: case IPL_TYPE_FCP_DUMP: - return sprintf(page, "0.0.%04x\n", - ipl_block.ipl_info.fcp.devno); + return sprintf(page, "0.0.%04x\n", ipl_block.fcp.devno); default: return 0; } @@ -316,8 +343,8 @@ static ssize_t ipl_scp_data_read(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, char *buf, loff_t off, size_t count) { - unsigned int size = ipl_block.ipl_info.fcp.scp_data_len; - void *scp_data = &ipl_block.ipl_info.fcp.scp_data; + unsigned int size = ipl_block.fcp.scp_data_len; + void *scp_data = &ipl_block.fcp.scp_data; return memory_read_from_buffer(buf, count, &off, scp_data, size); } @@ -333,13 +360,13 @@ static struct bin_attribute *ipl_fcp_bin_attrs[] = { /* FCP ipl device attributes */ DEFINE_IPL_ATTR_RO(ipl_fcp, wwpn, "0x%016llx\n", - (unsigned long long)ipl_block.ipl_info.fcp.wwpn); + (unsigned long long)ipl_block.fcp.wwpn); DEFINE_IPL_ATTR_RO(ipl_fcp, lun, "0x%016llx\n", - (unsigned long long)ipl_block.ipl_info.fcp.lun); + (unsigned long long)ipl_block.fcp.lun); DEFINE_IPL_ATTR_RO(ipl_fcp, bootprog, "%lld\n", - (unsigned long long)ipl_block.ipl_info.fcp.bootprog); + (unsigned long long)ipl_block.fcp.bootprog); DEFINE_IPL_ATTR_RO(ipl_fcp, br_lba, "%lld\n", - (unsigned long long)ipl_block.ipl_info.fcp.br_lba); + (unsigned long long)ipl_block.fcp.br_lba); static ssize_t ipl_ccw_loadparm_show(struct kobject *kobj, struct kobj_attribute *attr, char *page) @@ -365,6 +392,8 @@ static struct attribute *ipl_fcp_attrs[] = { &sys_ipl_fcp_bootprog_attr.attr, &sys_ipl_fcp_br_lba_attr.attr, &sys_ipl_ccw_loadparm_attr.attr, + &sys_ipl_secure_attr.attr, + &sys_ipl_has_secure_attr.attr, NULL, }; @@ -380,6 +409,8 @@ static struct attribute *ipl_ccw_attrs_vm[] = { &sys_ipl_device_attr.attr, &sys_ipl_ccw_loadparm_attr.attr, &sys_ipl_vm_parm_attr.attr, + &sys_ipl_secure_attr.attr, + &sys_ipl_has_secure_attr.attr, NULL, }; @@ -387,6 +418,8 @@ static struct attribute *ipl_ccw_attrs_lpar[] = { &sys_ipl_type_attr.attr, &sys_ipl_device_attr.attr, &sys_ipl_ccw_loadparm_attr.attr, + &sys_ipl_secure_attr.attr, + &sys_ipl_has_secure_attr.attr, NULL, }; @@ -495,14 +528,14 @@ static ssize_t reipl_generic_vmparm_store(struct ipl_parameter_block *ipb, if (!(isalnum(buf[i]) || isascii(buf[i]) || isprint(buf[i]))) return -EINVAL; - memset(ipb->ipl_info.ccw.vm_parm, 0, DIAG308_VMPARM_SIZE); - ipb->ipl_info.ccw.vm_parm_len = ip_len; + memset(ipb->ccw.vm_parm, 0, DIAG308_VMPARM_SIZE); + ipb->ccw.vm_parm_len = ip_len; if (ip_len > 0) { - ipb->ipl_info.ccw.vm_flags |= DIAG308_VM_FLAGS_VP_VALID; - memcpy(ipb->ipl_info.ccw.vm_parm, buf, ip_len); - ASCEBC(ipb->ipl_info.ccw.vm_parm, ip_len); + ipb->ccw.vm_flags |= IPL_PB0_CCW_VM_FLAG_VP; + memcpy(ipb->ccw.vm_parm, buf, ip_len); + ASCEBC(ipb->ccw.vm_parm, ip_len); } else { - ipb->ipl_info.ccw.vm_flags &= ~DIAG308_VM_FLAGS_VP_VALID; + ipb->ccw.vm_flags &= ~IPL_PB0_CCW_VM_FLAG_VP; } return len; @@ -549,8 +582,8 @@ static ssize_t reipl_fcp_scpdata_read(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, char *buf, loff_t off, size_t count) { - size_t size = reipl_block_fcp->ipl_info.fcp.scp_data_len; - void *scp_data = reipl_block_fcp->ipl_info.fcp.scp_data; + size_t size = reipl_block_fcp->fcp.scp_data_len; + void *scp_data = reipl_block_fcp->fcp.scp_data; return memory_read_from_buffer(buf, count, &off, scp_data, size); } @@ -566,17 +599,17 @@ static ssize_t reipl_fcp_scpdata_write(struct file *filp, struct kobject *kobj, if (off) return -EINVAL; - memcpy(reipl_block_fcp->ipl_info.fcp.scp_data, buf, count); + memcpy(reipl_block_fcp->fcp.scp_data, buf, count); if (scpdata_len % 8) { padding = 8 - (scpdata_len % 8); - memset(reipl_block_fcp->ipl_info.fcp.scp_data + scpdata_len, + memset(reipl_block_fcp->fcp.scp_data + scpdata_len, 0, padding); scpdata_len += padding; } - reipl_block_fcp->ipl_info.fcp.scp_data_len = scpdata_len; - reipl_block_fcp->hdr.len = IPL_PARM_BLK_FCP_LEN + scpdata_len; - reipl_block_fcp->hdr.blk0_len = IPL_PARM_BLK0_FCP_LEN + scpdata_len; + reipl_block_fcp->hdr.len = IPL_BP_FCP_LEN + scpdata_len; + reipl_block_fcp->fcp.len = IPL_BP0_FCP_LEN + scpdata_len; + reipl_block_fcp->fcp.scp_data_len = scpdata_len; return count; } @@ -590,20 +623,20 @@ static struct bin_attribute *reipl_fcp_bin_attrs[] = { }; DEFINE_IPL_ATTR_RW(reipl_fcp, wwpn, "0x%016llx\n", "%llx\n", - reipl_block_fcp->ipl_info.fcp.wwpn); + reipl_block_fcp->fcp.wwpn); DEFINE_IPL_ATTR_RW(reipl_fcp, lun, "0x%016llx\n", "%llx\n", - reipl_block_fcp->ipl_info.fcp.lun); + reipl_block_fcp->fcp.lun); DEFINE_IPL_ATTR_RW(reipl_fcp, bootprog, "%lld\n", "%lld\n", - reipl_block_fcp->ipl_info.fcp.bootprog); + reipl_block_fcp->fcp.bootprog); DEFINE_IPL_ATTR_RW(reipl_fcp, br_lba, "%lld\n", "%lld\n", - reipl_block_fcp->ipl_info.fcp.br_lba); + reipl_block_fcp->fcp.br_lba); DEFINE_IPL_ATTR_RW(reipl_fcp, device, "0.0.%04llx\n", "0.0.%llx\n", - reipl_block_fcp->ipl_info.fcp.devno); + reipl_block_fcp->fcp.devno); static void reipl_get_ascii_loadparm(char *loadparm, struct ipl_parameter_block *ibp) { - memcpy(loadparm, ibp->hdr.loadparm, LOADPARM_LEN); + memcpy(loadparm, ibp->common.loadparm, LOADPARM_LEN); EBCASC(loadparm, LOADPARM_LEN); loadparm[LOADPARM_LEN] = 0; strim(loadparm); @@ -638,11 +671,11 @@ static ssize_t reipl_generic_loadparm_store(struct ipl_parameter_block *ipb, return -EINVAL; } /* initialize loadparm with blanks */ - memset(ipb->hdr.loadparm, ' ', LOADPARM_LEN); + memset(ipb->common.loadparm, ' ', LOADPARM_LEN); /* copy and convert to ebcdic */ - memcpy(ipb->hdr.loadparm, buf, lp_len); - ASCEBC(ipb->hdr.loadparm, LOADPARM_LEN); - ipb->hdr.flags |= DIAG308_FLAGS_LP_VALID; + memcpy(ipb->common.loadparm, buf, lp_len); + ASCEBC(ipb->common.loadparm, LOADPARM_LEN); + ipb->common.flags |= IPL_PB0_FLAG_LOADPARM; return len; } @@ -680,7 +713,7 @@ static struct attribute_group reipl_fcp_attr_group = { }; /* CCW reipl device attributes */ -DEFINE_IPL_CCW_ATTR_RW(reipl_ccw, device, reipl_block_ccw->ipl_info.ccw); +DEFINE_IPL_CCW_ATTR_RW(reipl_ccw, device, reipl_block_ccw->ccw); /* NSS wrapper */ static ssize_t reipl_nss_loadparm_show(struct kobject *kobj, @@ -742,7 +775,7 @@ static struct attribute_group reipl_ccw_attr_group_lpar = { static void reipl_get_ascii_nss_name(char *dst, struct ipl_parameter_block *ipb) { - memcpy(dst, ipb->ipl_info.ccw.nss_name, NSS_NAME_SIZE); + memcpy(dst, ipb->ccw.nss_name, NSS_NAME_SIZE); EBCASC(dst, NSS_NAME_SIZE); dst[NSS_NAME_SIZE] = 0; } @@ -770,16 +803,14 @@ static ssize_t reipl_nss_name_store(struct kobject *kobj, if (nss_len > NSS_NAME_SIZE) return -EINVAL; - memset(reipl_block_nss->ipl_info.ccw.nss_name, 0x40, NSS_NAME_SIZE); + memset(reipl_block_nss->ccw.nss_name, 0x40, NSS_NAME_SIZE); if (nss_len > 0) { - reipl_block_nss->ipl_info.ccw.vm_flags |= - DIAG308_VM_FLAGS_NSS_VALID; - memcpy(reipl_block_nss->ipl_info.ccw.nss_name, buf, nss_len); - ASCEBC(reipl_block_nss->ipl_info.ccw.nss_name, nss_len); - EBC_TOUPPER(reipl_block_nss->ipl_info.ccw.nss_name, nss_len); + reipl_block_nss->ccw.vm_flags |= IPL_PB0_CCW_VM_FLAG_NSS; + memcpy(reipl_block_nss->ccw.nss_name, buf, nss_len); + ASCEBC(reipl_block_nss->ccw.nss_name, nss_len); + EBC_TOUPPER(reipl_block_nss->ccw.nss_name, nss_len); } else { - reipl_block_nss->ipl_info.ccw.vm_flags &= - ~DIAG308_VM_FLAGS_NSS_VALID; + reipl_block_nss->ccw.vm_flags &= ~IPL_PB0_CCW_VM_FLAG_NSS; } return len; @@ -866,15 +897,21 @@ static void __reipl_run(void *unused) { switch (reipl_type) { case IPL_TYPE_CCW: + uv_set_shared(__pa(reipl_block_ccw)); diag308(DIAG308_SET, reipl_block_ccw); + uv_remove_shared(__pa(reipl_block_ccw)); diag308(DIAG308_LOAD_CLEAR, NULL); break; case IPL_TYPE_FCP: + uv_set_shared(__pa(reipl_block_fcp)); diag308(DIAG308_SET, reipl_block_fcp); + uv_remove_shared(__pa(reipl_block_fcp)); diag308(DIAG308_LOAD_CLEAR, NULL); break; case IPL_TYPE_NSS: + uv_set_shared(__pa(reipl_block_nss)); diag308(DIAG308_SET, reipl_block_nss); + uv_remove_shared(__pa(reipl_block_nss)); diag308(DIAG308_LOAD_CLEAR, NULL); break; case IPL_TYPE_UNKNOWN: @@ -883,7 +920,7 @@ static void __reipl_run(void *unused) case IPL_TYPE_FCP_DUMP: break; } - disabled_wait((unsigned long) __builtin_return_address(0)); + disabled_wait(); } static void reipl_run(struct shutdown_trigger *trigger) @@ -893,10 +930,10 @@ static void reipl_run(struct shutdown_trigger *trigger) static void reipl_block_ccw_init(struct ipl_parameter_block *ipb) { - ipb->hdr.len = IPL_PARM_BLK_CCW_LEN; + ipb->hdr.len = IPL_BP_CCW_LEN; ipb->hdr.version = IPL_PARM_BLOCK_VERSION; - ipb->hdr.blk0_len = IPL_PARM_BLK0_CCW_LEN; - ipb->hdr.pbt = DIAG308_IPL_TYPE_CCW; + ipb->pb0_hdr.len = IPL_BP0_CCW_LEN; + ipb->pb0_hdr.pbt = IPL_PBT_CCW; } static void reipl_block_ccw_fill_parms(struct ipl_parameter_block *ipb) @@ -904,21 +941,20 @@ static void reipl_block_ccw_fill_parms(struct ipl_parameter_block *ipb) /* LOADPARM */ /* check if read scp info worked and set loadparm */ if (sclp_ipl_info.is_valid) - memcpy(ipb->hdr.loadparm, &sclp_ipl_info.loadparm, LOADPARM_LEN); + memcpy(ipb->ccw.loadparm, &sclp_ipl_info.loadparm, LOADPARM_LEN); else /* read scp info failed: set empty loadparm (EBCDIC blanks) */ - memset(ipb->hdr.loadparm, 0x40, LOADPARM_LEN); - ipb->hdr.flags = DIAG308_FLAGS_LP_VALID; + memset(ipb->ccw.loadparm, 0x40, LOADPARM_LEN); + ipb->ccw.flags = IPL_PB0_FLAG_LOADPARM; /* VM PARM */ if (MACHINE_IS_VM && ipl_block_valid && - (ipl_block.ipl_info.ccw.vm_flags & DIAG308_VM_FLAGS_VP_VALID)) { + (ipl_block.ccw.vm_flags & IPL_PB0_CCW_VM_FLAG_VP)) { - ipb->ipl_info.ccw.vm_flags |= DIAG308_VM_FLAGS_VP_VALID; - ipb->ipl_info.ccw.vm_parm_len = - ipl_block.ipl_info.ccw.vm_parm_len; - memcpy(ipb->ipl_info.ccw.vm_parm, - ipl_block.ipl_info.ccw.vm_parm, DIAG308_VMPARM_SIZE); + ipb->ccw.vm_flags |= IPL_PB0_CCW_VM_FLAG_VP; + ipb->ccw.vm_parm_len = ipl_block.ccw.vm_parm_len; + memcpy(ipb->ccw.vm_parm, + ipl_block.ccw.vm_parm, DIAG308_VMPARM_SIZE); } } @@ -958,8 +994,8 @@ static int __init reipl_ccw_init(void) reipl_block_ccw_init(reipl_block_ccw); if (ipl_info.type == IPL_TYPE_CCW) { - reipl_block_ccw->ipl_info.ccw.ssid = ipl_block.ipl_info.ccw.ssid; - reipl_block_ccw->ipl_info.ccw.devno = ipl_block.ipl_info.ccw.devno; + reipl_block_ccw->ccw.ssid = ipl_block.ccw.ssid; + reipl_block_ccw->ccw.devno = ipl_block.ccw.devno; reipl_block_ccw_fill_parms(reipl_block_ccw); } @@ -997,14 +1033,14 @@ static int __init reipl_fcp_init(void) * is invalid in the SCSI IPL parameter block, so take it * always from sclp_ipl_info. */ - memcpy(reipl_block_fcp->hdr.loadparm, sclp_ipl_info.loadparm, + memcpy(reipl_block_fcp->fcp.loadparm, sclp_ipl_info.loadparm, LOADPARM_LEN); } else { - reipl_block_fcp->hdr.len = IPL_PARM_BLK_FCP_LEN; + reipl_block_fcp->hdr.len = IPL_BP_FCP_LEN; reipl_block_fcp->hdr.version = IPL_PARM_BLOCK_VERSION; - reipl_block_fcp->hdr.blk0_len = IPL_PARM_BLK0_FCP_LEN; - reipl_block_fcp->hdr.pbt = DIAG308_IPL_TYPE_FCP; - reipl_block_fcp->ipl_info.fcp.opt = DIAG308_IPL_OPT_IPL; + reipl_block_fcp->fcp.len = IPL_BP0_FCP_LEN; + reipl_block_fcp->fcp.pbt = IPL_PBT_FCP; + reipl_block_fcp->fcp.opt = IPL_PB0_FCP_OPT_IPL; } reipl_capabilities |= IPL_TYPE_FCP; return 0; @@ -1022,10 +1058,10 @@ static int __init reipl_type_init(void) /* * If we have an OS info reipl block, this will be used */ - if (reipl_block->hdr.pbt == DIAG308_IPL_TYPE_FCP) { + if (reipl_block->pb0_hdr.pbt == IPL_PBT_FCP) { memcpy(reipl_block_fcp, reipl_block, size); reipl_type = IPL_TYPE_FCP; - } else if (reipl_block->hdr.pbt == DIAG308_IPL_TYPE_CCW) { + } else if (reipl_block->pb0_hdr.pbt == IPL_PBT_CCW) { memcpy(reipl_block_ccw, reipl_block, size); reipl_type = IPL_TYPE_CCW; } @@ -1070,15 +1106,15 @@ static struct shutdown_action __refdata reipl_action = { /* FCP dump device attributes */ DEFINE_IPL_ATTR_RW(dump_fcp, wwpn, "0x%016llx\n", "%llx\n", - dump_block_fcp->ipl_info.fcp.wwpn); + dump_block_fcp->fcp.wwpn); DEFINE_IPL_ATTR_RW(dump_fcp, lun, "0x%016llx\n", "%llx\n", - dump_block_fcp->ipl_info.fcp.lun); + dump_block_fcp->fcp.lun); DEFINE_IPL_ATTR_RW(dump_fcp, bootprog, "%lld\n", "%lld\n", - dump_block_fcp->ipl_info.fcp.bootprog); + dump_block_fcp->fcp.bootprog); DEFINE_IPL_ATTR_RW(dump_fcp, br_lba, "%lld\n", "%lld\n", - dump_block_fcp->ipl_info.fcp.br_lba); + dump_block_fcp->fcp.br_lba); DEFINE_IPL_ATTR_RW(dump_fcp, device, "0.0.%04llx\n", "0.0.%llx\n", - dump_block_fcp->ipl_info.fcp.devno); + dump_block_fcp->fcp.devno); static struct attribute *dump_fcp_attrs[] = { &sys_dump_fcp_device_attr.attr, @@ -1095,7 +1131,7 @@ static struct attribute_group dump_fcp_attr_group = { }; /* CCW dump device attributes */ -DEFINE_IPL_CCW_ATTR_RW(dump_ccw, device, dump_block_ccw->ipl_info.ccw); +DEFINE_IPL_CCW_ATTR_RW(dump_ccw, device, dump_block_ccw->ccw); static struct attribute *dump_ccw_attrs[] = { &sys_dump_ccw_device_attr.attr, @@ -1145,7 +1181,9 @@ static struct kset *dump_kset; static void diag308_dump(void *dump_block) { + uv_set_shared(__pa(dump_block)); diag308(DIAG308_SET, dump_block); + uv_remove_shared(__pa(dump_block)); while (1) { if (diag308(DIAG308_LOAD_NORMAL_DUMP, NULL) != 0x302) break; @@ -1187,10 +1225,10 @@ static int __init dump_ccw_init(void) free_page((unsigned long)dump_block_ccw); return rc; } - dump_block_ccw->hdr.len = IPL_PARM_BLK_CCW_LEN; + dump_block_ccw->hdr.len = IPL_BP_CCW_LEN; dump_block_ccw->hdr.version = IPL_PARM_BLOCK_VERSION; - dump_block_ccw->hdr.blk0_len = IPL_PARM_BLK0_CCW_LEN; - dump_block_ccw->hdr.pbt = DIAG308_IPL_TYPE_CCW; + dump_block_ccw->ccw.len = IPL_BP0_CCW_LEN; + dump_block_ccw->ccw.pbt = IPL_PBT_CCW; dump_capabilities |= DUMP_TYPE_CCW; return 0; } @@ -1209,11 +1247,11 @@ static int __init dump_fcp_init(void) free_page((unsigned long)dump_block_fcp); return rc; } - dump_block_fcp->hdr.len = IPL_PARM_BLK_FCP_LEN; + dump_block_fcp->hdr.len = IPL_BP_FCP_LEN; dump_block_fcp->hdr.version = IPL_PARM_BLOCK_VERSION; - dump_block_fcp->hdr.blk0_len = IPL_PARM_BLK0_FCP_LEN; - dump_block_fcp->hdr.pbt = DIAG308_IPL_TYPE_FCP; - dump_block_fcp->ipl_info.fcp.opt = DIAG308_IPL_OPT_DUMP; + dump_block_fcp->fcp.len = IPL_BP0_FCP_LEN; + dump_block_fcp->fcp.pbt = IPL_PBT_FCP; + dump_block_fcp->fcp.opt = IPL_PB0_FCP_OPT_DUMP; dump_capabilities |= DUMP_TYPE_FCP; return 0; } @@ -1337,7 +1375,7 @@ static void stop_run(struct shutdown_trigger *trigger) { if (strcmp(trigger->name, ON_PANIC_STR) == 0 || strcmp(trigger->name, ON_RESTART_STR) == 0) - disabled_wait((unsigned long) __builtin_return_address(0)); + disabled_wait(); smp_stop_cpu(); } @@ -1572,7 +1610,7 @@ static int __init s390_ipl_init(void) * READ SCP info provides the correct value. */ if (memcmp(sclp_ipl_info.loadparm, str, sizeof(str)) == 0 && ipl_block_valid) - memcpy(sclp_ipl_info.loadparm, ipl_block.hdr.loadparm, LOADPARM_LEN); + memcpy(sclp_ipl_info.loadparm, ipl_block.ccw.loadparm, LOADPARM_LEN); shutdown_actions_init(); shutdown_triggers_init(); return 0; @@ -1657,15 +1695,15 @@ void __init setup_ipl(void) ipl_info.type = get_ipl_type(); switch (ipl_info.type) { case IPL_TYPE_CCW: - ipl_info.data.ccw.dev_id.ssid = ipl_block.ipl_info.ccw.ssid; - ipl_info.data.ccw.dev_id.devno = ipl_block.ipl_info.ccw.devno; + ipl_info.data.ccw.dev_id.ssid = ipl_block.ccw.ssid; + ipl_info.data.ccw.dev_id.devno = ipl_block.ccw.devno; break; case IPL_TYPE_FCP: case IPL_TYPE_FCP_DUMP: ipl_info.data.fcp.dev_id.ssid = 0; - ipl_info.data.fcp.dev_id.devno = ipl_block.ipl_info.fcp.devno; - ipl_info.data.fcp.wwpn = ipl_block.ipl_info.fcp.wwpn; - ipl_info.data.fcp.lun = ipl_block.ipl_info.fcp.lun; + ipl_info.data.fcp.dev_id.devno = ipl_block.fcp.devno; + ipl_info.data.fcp.wwpn = ipl_block.fcp.wwpn; + ipl_info.data.fcp.lun = ipl_block.fcp.lun; break; case IPL_TYPE_NSS: case IPL_TYPE_UNKNOWN: @@ -1675,14 +1713,6 @@ void __init setup_ipl(void) atomic_notifier_chain_register(&panic_notifier_list, &on_panic_nb); } -void __init ipl_store_parameters(void) -{ - if (early_ipl_block_valid) { - memcpy(&ipl_block, &early_ipl_block, sizeof(ipl_block)); - ipl_block_valid = 1; - } -} - void s390_reset_system(void) { /* Disable prefixing */ @@ -1690,5 +1720,139 @@ void s390_reset_system(void) /* Disable lowcore protection */ __ctl_clear_bit(0, 28); - diag308_reset(); + diag_dma_ops.diag308_reset(); +} + +#ifdef CONFIG_KEXEC_FILE + +int ipl_report_add_component(struct ipl_report *report, struct kexec_buf *kbuf, + unsigned char flags, unsigned short cert) +{ + struct ipl_report_component *comp; + + comp = vzalloc(sizeof(*comp)); + if (!comp) + return -ENOMEM; + list_add_tail(&comp->list, &report->components); + + comp->entry.addr = kbuf->mem; + comp->entry.len = kbuf->memsz; + comp->entry.flags = flags; + comp->entry.certificate_index = cert; + + report->size += sizeof(comp->entry); + + return 0; +} + +int ipl_report_add_certificate(struct ipl_report *report, void *key, + unsigned long addr, unsigned long len) +{ + struct ipl_report_certificate *cert; + + cert = vzalloc(sizeof(*cert)); + if (!cert) + return -ENOMEM; + list_add_tail(&cert->list, &report->certificates); + + cert->entry.addr = addr; + cert->entry.len = len; + cert->key = key; + + report->size += sizeof(cert->entry); + report->size += cert->entry.len; + + return 0; +} + +struct ipl_report *ipl_report_init(struct ipl_parameter_block *ipib) +{ + struct ipl_report *report; + + report = vzalloc(sizeof(*report)); + if (!report) + return ERR_PTR(-ENOMEM); + + report->ipib = ipib; + INIT_LIST_HEAD(&report->components); + INIT_LIST_HEAD(&report->certificates); + + report->size = ALIGN(ipib->hdr.len, 8); + report->size += sizeof(struct ipl_rl_hdr); + report->size += sizeof(struct ipl_rb_components); + report->size += sizeof(struct ipl_rb_certificates); + + return report; +} + +void *ipl_report_finish(struct ipl_report *report) +{ + struct ipl_report_certificate *cert; + struct ipl_report_component *comp; + struct ipl_rb_certificates *certs; + struct ipl_parameter_block *ipib; + struct ipl_rb_components *comps; + struct ipl_rl_hdr *rl_hdr; + void *buf, *ptr; + + buf = vzalloc(report->size); + if (!buf) + return ERR_PTR(-ENOMEM); + ptr = buf; + + memcpy(ptr, report->ipib, report->ipib->hdr.len); + ipib = ptr; + if (ipl_secure_flag) + ipib->hdr.flags |= IPL_PL_FLAG_SIPL; + ipib->hdr.flags |= IPL_PL_FLAG_IPLSR; + ptr += report->ipib->hdr.len; + ptr = PTR_ALIGN(ptr, 8); + + rl_hdr = ptr; + ptr += sizeof(*rl_hdr); + + comps = ptr; + comps->rbt = IPL_RBT_COMPONENTS; + ptr += sizeof(*comps); + list_for_each_entry(comp, &report->components, list) { + memcpy(ptr, &comp->entry, sizeof(comp->entry)); + ptr += sizeof(comp->entry); + } + comps->len = ptr - (void *)comps; + + certs = ptr; + certs->rbt = IPL_RBT_CERTIFICATES; + ptr += sizeof(*certs); + list_for_each_entry(cert, &report->certificates, list) { + memcpy(ptr, &cert->entry, sizeof(cert->entry)); + ptr += sizeof(cert->entry); + } + certs->len = ptr - (void *)certs; + rl_hdr->len = ptr - (void *)rl_hdr; + + list_for_each_entry(cert, &report->certificates, list) { + memcpy(ptr, cert->key, cert->entry.len); + ptr += cert->entry.len; + } + + BUG_ON(ptr > buf + report->size); + return buf; +} + +int ipl_report_free(struct ipl_report *report) +{ + struct ipl_report_component *comp, *ncomp; + struct ipl_report_certificate *cert, *ncert; + + list_for_each_entry_safe(comp, ncomp, &report->components, list) + vfree(comp); + + list_for_each_entry_safe(cert, ncert, &report->certificates, list) + vfree(cert); + + vfree(report); + + return 0; } + +#endif diff --git a/arch/s390/kernel/ipl_vmparm.c b/arch/s390/kernel/ipl_vmparm.c index 411838c0a0af..af43535a976d 100644 --- a/arch/s390/kernel/ipl_vmparm.c +++ b/arch/s390/kernel/ipl_vmparm.c @@ -11,11 +11,11 @@ size_t ipl_block_get_ascii_vmparm(char *dest, size_t size, char has_lowercase = 0; len = 0; - if ((ipb->ipl_info.ccw.vm_flags & DIAG308_VM_FLAGS_VP_VALID) && - (ipb->ipl_info.ccw.vm_parm_len > 0)) { + if ((ipb->ccw.vm_flags & IPL_PB0_CCW_VM_FLAG_VP) && + (ipb->ccw.vm_parm_len > 0)) { - len = min_t(size_t, size - 1, ipb->ipl_info.ccw.vm_parm_len); - memcpy(dest, ipb->ipl_info.ccw.vm_parm, len); + len = min_t(size_t, size - 1, ipb->ccw.vm_parm_len); + memcpy(dest, ipb->ccw.vm_parm, len); /* If at least one character is lowercase, we assume mixed * case; otherwise we convert everything to lowercase. */ diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 0cd5a5f96729..8371855042dc 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -26,6 +26,7 @@ #include <asm/lowcore.h> #include <asm/irq.h> #include <asm/hw_irq.h> +#include <asm/stacktrace.h> #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); @@ -73,7 +74,6 @@ static const struct irq_class irqclass_sub_desc[] = { {.irq = IRQEXT_CMC, .name = "CMC", .desc = "[EXT] CPU-Measurement: Counter"}, {.irq = IRQEXT_FTP, .name = "FTP", .desc = "[EXT] HMC FTP Service"}, {.irq = IRQIO_CIO, .name = "CIO", .desc = "[I/O] Common I/O Layer Interrupt"}, - {.irq = IRQIO_QAI, .name = "QAI", .desc = "[I/O] QDIO Adapter Interrupt"}, {.irq = IRQIO_DAS, .name = "DAS", .desc = "[I/O] DASD"}, {.irq = IRQIO_C15, .name = "C15", .desc = "[I/O] 3215"}, {.irq = IRQIO_C70, .name = "C70", .desc = "[I/O] 3270"}, @@ -81,14 +81,16 @@ static const struct irq_class irqclass_sub_desc[] = { {.irq = IRQIO_VMR, .name = "VMR", .desc = "[I/O] Unit Record Devices"}, {.irq = IRQIO_LCS, .name = "LCS", .desc = "[I/O] LCS"}, {.irq = IRQIO_CTC, .name = "CTC", .desc = "[I/O] CTC"}, - {.irq = IRQIO_APB, .name = "APB", .desc = "[I/O] AP Bus"}, {.irq = IRQIO_ADM, .name = "ADM", .desc = "[I/O] EADM Subchannel"}, {.irq = IRQIO_CSC, .name = "CSC", .desc = "[I/O] CHSC Subchannel"}, - {.irq = IRQIO_PCI, .name = "PCI", .desc = "[I/O] PCI Interrupt" }, - {.irq = IRQIO_MSI, .name = "MSI", .desc = "[I/O] MSI Interrupt" }, {.irq = IRQIO_VIR, .name = "VIR", .desc = "[I/O] Virtual I/O Devices"}, - {.irq = IRQIO_VAI, .name = "VAI", .desc = "[I/O] Virtual I/O Devices AI"}, - {.irq = IRQIO_GAL, .name = "GAL", .desc = "[I/O] GIB Alert"}, + {.irq = IRQIO_QAI, .name = "QAI", .desc = "[AIO] QDIO Adapter Interrupt"}, + {.irq = IRQIO_APB, .name = "APB", .desc = "[AIO] AP Bus"}, + {.irq = IRQIO_PCF, .name = "PCF", .desc = "[AIO] PCI Floating Interrupt"}, + {.irq = IRQIO_PCD, .name = "PCD", .desc = "[AIO] PCI Directed Interrupt"}, + {.irq = IRQIO_MSI, .name = "MSI", .desc = "[AIO] MSI Interrupt"}, + {.irq = IRQIO_VAI, .name = "VAI", .desc = "[AIO] Virtual I/O Devices AI"}, + {.irq = IRQIO_GAL, .name = "GAL", .desc = "[AIO] GIB Alert"}, {.irq = NMI_NMI, .name = "NMI", .desc = "[NMI] Machine Check"}, {.irq = CPU_RST, .name = "RST", .desc = "[CPU] CPU Restart"}, }; @@ -116,6 +118,34 @@ void do_IRQ(struct pt_regs *regs, int irq) set_irq_regs(old_regs); } +static void show_msi_interrupt(struct seq_file *p, int irq) +{ + struct irq_desc *desc; + unsigned long flags; + int cpu; + + irq_lock_sparse(); + desc = irq_to_desc(irq); + if (!desc) + goto out; + + raw_spin_lock_irqsave(&desc->lock, flags); + seq_printf(p, "%3d: ", irq); + for_each_online_cpu(cpu) + seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu)); + + if (desc->irq_data.chip) + seq_printf(p, " %8s", desc->irq_data.chip->name); + + if (desc->action) + seq_printf(p, " %s", desc->action->name); + + seq_putc(p, '\n'); + raw_spin_unlock_irqrestore(&desc->lock, flags); +out: + irq_unlock_sparse(); +} + /* * show_interrupts is needed by /proc/interrupts. */ @@ -128,7 +158,7 @@ int show_interrupts(struct seq_file *p, void *v) if (index == 0) { seq_puts(p, " "); for_each_online_cpu(cpu) - seq_printf(p, "CPU%d ", cpu); + seq_printf(p, "CPU%-8d", cpu); seq_putc(p, '\n'); } if (index < NR_IRQS_BASE) { @@ -139,9 +169,10 @@ int show_interrupts(struct seq_file *p, void *v) seq_putc(p, '\n'); goto out; } - if (index > NR_IRQS_BASE) + if (index < nr_irqs) { + show_msi_interrupt(p, index); goto out; - + } for (index = 0; index < NR_ARCH_IRQS; index++) { seq_printf(p, "%s: ", irqclass_sub_desc[index].name); irq = irqclass_sub_desc[index].irq; diff --git a/arch/s390/kernel/kexec_elf.c b/arch/s390/kernel/kexec_elf.c index 5a286b012043..6d0635ceddd0 100644 --- a/arch/s390/kernel/kexec_elf.c +++ b/arch/s390/kernel/kexec_elf.c @@ -10,19 +10,26 @@ #include <linux/errno.h> #include <linux/kernel.h> #include <linux/kexec.h> +#include <asm/ipl.h> #include <asm/setup.h> -static int kexec_file_add_elf_kernel(struct kimage *image, - struct s390_load_data *data, - char *kernel, unsigned long kernel_len) +static int kexec_file_add_kernel_elf(struct kimage *image, + struct s390_load_data *data) { struct kexec_buf buf; const Elf_Ehdr *ehdr; const Elf_Phdr *phdr; + Elf_Addr entry; + void *kernel; int i, ret; + kernel = image->kernel_buf; ehdr = (Elf_Ehdr *)kernel; buf.image = image; + if (image->type == KEXEC_TYPE_CRASH) + entry = STARTUP_KDUMP_OFFSET; + else + entry = ehdr->e_entry; phdr = (void *)ehdr + ehdr->e_phoff; for (i = 0; i < ehdr->e_phnum; i++, phdr++) { @@ -33,30 +40,27 @@ static int kexec_file_add_elf_kernel(struct kimage *image, buf.bufsz = phdr->p_filesz; buf.mem = ALIGN(phdr->p_paddr, phdr->p_align); + if (image->type == KEXEC_TYPE_CRASH) + buf.mem += crashk_res.start; buf.memsz = phdr->p_memsz; + data->memsz = ALIGN(data->memsz, phdr->p_align) + buf.memsz; - if (phdr->p_paddr == 0) { + if (entry - phdr->p_paddr < phdr->p_memsz) { data->kernel_buf = buf.buffer; - data->memsz += STARTUP_NORMAL_OFFSET; - - buf.buffer += STARTUP_NORMAL_OFFSET; - buf.bufsz -= STARTUP_NORMAL_OFFSET; - - buf.mem += STARTUP_NORMAL_OFFSET; - buf.memsz -= STARTUP_NORMAL_OFFSET; + data->kernel_mem = buf.mem; + data->parm = buf.buffer + PARMAREA; } - if (image->type == KEXEC_TYPE_CRASH) - buf.mem += crashk_res.start; - + ipl_report_add_component(data->report, &buf, + IPL_RB_COMPONENT_FLAG_SIGNED | + IPL_RB_COMPONENT_FLAG_VERIFIED, + IPL_RB_CERT_UNKNOWN); ret = kexec_add_buffer(&buf); if (ret) return ret; - - data->memsz += buf.memsz; } - return 0; + return data->memsz ? 0 : -EINVAL; } static void *s390_elf_load(struct kimage *image, @@ -64,11 +68,10 @@ static void *s390_elf_load(struct kimage *image, char *initrd, unsigned long initrd_len, char *cmdline, unsigned long cmdline_len) { - struct s390_load_data data = {0}; const Elf_Ehdr *ehdr; const Elf_Phdr *phdr; size_t size; - int i, ret; + int i; /* image->fobs->probe already checked for valid ELF magic number. */ ehdr = (Elf_Ehdr *)kernel; @@ -101,24 +104,7 @@ static void *s390_elf_load(struct kimage *image, if (size > kernel_len) return ERR_PTR(-EINVAL); - ret = kexec_file_add_elf_kernel(image, &data, kernel, kernel_len); - if (ret) - return ERR_PTR(ret); - - if (!data.memsz) - return ERR_PTR(-EINVAL); - - if (initrd) { - ret = kexec_file_add_initrd(image, &data, initrd, initrd_len); - if (ret) - return ERR_PTR(ret); - } - - ret = kexec_file_add_purgatory(image, &data); - if (ret) - return ERR_PTR(ret); - - return kexec_file_update_kernel(image, &data); + return kexec_file_add_components(image, kexec_file_add_kernel_elf); } static int s390_elf_probe(const char *buf, unsigned long len) @@ -144,4 +130,7 @@ static int s390_elf_probe(const char *buf, unsigned long len) const struct kexec_file_ops s390_kexec_elf_ops = { .probe = s390_elf_probe, .load = s390_elf_load, +#ifdef CONFIG_KEXEC_VERIFY_SIG + .verify_sig = s390_verify_sig, +#endif /* CONFIG_KEXEC_VERIFY_SIG */ }; diff --git a/arch/s390/kernel/kexec_image.c b/arch/s390/kernel/kexec_image.c index 3800852595e8..58318bf89fd9 100644 --- a/arch/s390/kernel/kexec_image.c +++ b/arch/s390/kernel/kexec_image.c @@ -10,31 +10,34 @@ #include <linux/errno.h> #include <linux/kernel.h> #include <linux/kexec.h> +#include <asm/ipl.h> #include <asm/setup.h> -static int kexec_file_add_image_kernel(struct kimage *image, - struct s390_load_data *data, - char *kernel, unsigned long kernel_len) +static int kexec_file_add_kernel_image(struct kimage *image, + struct s390_load_data *data) { struct kexec_buf buf; - int ret; buf.image = image; - buf.buffer = kernel + STARTUP_NORMAL_OFFSET; - buf.bufsz = kernel_len - STARTUP_NORMAL_OFFSET; + buf.buffer = image->kernel_buf; + buf.bufsz = image->kernel_buf_len; - buf.mem = STARTUP_NORMAL_OFFSET; + buf.mem = 0; if (image->type == KEXEC_TYPE_CRASH) buf.mem += crashk_res.start; buf.memsz = buf.bufsz; - ret = kexec_add_buffer(&buf); + data->kernel_buf = image->kernel_buf; + data->kernel_mem = buf.mem; + data->parm = image->kernel_buf + PARMAREA; + data->memsz += buf.memsz; - data->kernel_buf = kernel; - data->memsz += buf.memsz + STARTUP_NORMAL_OFFSET; - - return ret; + ipl_report_add_component(data->report, &buf, + IPL_RB_COMPONENT_FLAG_SIGNED | + IPL_RB_COMPONENT_FLAG_VERIFIED, + IPL_RB_CERT_UNKNOWN); + return kexec_add_buffer(&buf); } static void *s390_image_load(struct kimage *image, @@ -42,24 +45,7 @@ static void *s390_image_load(struct kimage *image, char *initrd, unsigned long initrd_len, char *cmdline, unsigned long cmdline_len) { - struct s390_load_data data = {0}; - int ret; - - ret = kexec_file_add_image_kernel(image, &data, kernel, kernel_len); - if (ret) - return ERR_PTR(ret); - - if (initrd) { - ret = kexec_file_add_initrd(image, &data, initrd, initrd_len); - if (ret) - return ERR_PTR(ret); - } - - ret = kexec_file_add_purgatory(image, &data); - if (ret) - return ERR_PTR(ret); - - return kexec_file_update_kernel(image, &data); + return kexec_file_add_components(image, kexec_file_add_kernel_image); } static int s390_image_probe(const char *buf, unsigned long len) @@ -73,4 +59,7 @@ static int s390_image_probe(const char *buf, unsigned long len) const struct kexec_file_ops s390_kexec_image_ops = { .probe = s390_image_probe, .load = s390_image_load, +#ifdef CONFIG_KEXEC_VERIFY_SIG + .verify_sig = s390_verify_sig, +#endif /* CONFIG_KEXEC_VERIFY_SIG */ }; diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index 7c0a095e9c5f..6f1388391620 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -27,29 +27,30 @@ DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); struct kretprobe_blackpoint kretprobe_blacklist[] = { }; -DEFINE_INSN_CACHE_OPS(dmainsn); +DEFINE_INSN_CACHE_OPS(s390_insn); -static void *alloc_dmainsn_page(void) -{ - void *page; +static int insn_page_in_use; +static char insn_page[PAGE_SIZE] __aligned(PAGE_SIZE); - page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA); - if (page) - set_memory_x((unsigned long) page, 1); - return page; +static void *alloc_s390_insn_page(void) +{ + if (xchg(&insn_page_in_use, 1) == 1) + return NULL; + set_memory_x((unsigned long) &insn_page, 1); + return &insn_page; } -static void free_dmainsn_page(void *page) +static void free_s390_insn_page(void *page) { set_memory_nx((unsigned long) page, 1); - free_page((unsigned long)page); + xchg(&insn_page_in_use, 0); } -struct kprobe_insn_cache kprobe_dmainsn_slots = { - .mutex = __MUTEX_INITIALIZER(kprobe_dmainsn_slots.mutex), - .alloc = alloc_dmainsn_page, - .free = free_dmainsn_page, - .pages = LIST_HEAD_INIT(kprobe_dmainsn_slots.pages), +struct kprobe_insn_cache kprobe_s390_insn_slots = { + .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex), + .alloc = alloc_s390_insn_page, + .free = free_s390_insn_page, + .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages), .insn_size = MAX_INSN_SIZE, }; @@ -102,7 +103,7 @@ static int s390_get_insn_slot(struct kprobe *p) */ p->ainsn.insn = NULL; if (is_kernel_addr(p->addr)) - p->ainsn.insn = get_dmainsn_slot(); + p->ainsn.insn = get_s390_insn_slot(); else if (is_module_addr(p->addr)) p->ainsn.insn = get_insn_slot(); return p->ainsn.insn ? 0 : -ENOMEM; @@ -114,7 +115,7 @@ static void s390_free_insn_slot(struct kprobe *p) if (!p->ainsn.insn) return; if (is_kernel_addr(p->addr)) - free_dmainsn_slot(p->ainsn.insn, 0); + free_s390_insn_slot(p->ainsn.insn, 0); else free_insn_slot(p->ainsn.insn, 0); p->ainsn.insn = NULL; @@ -572,7 +573,7 @@ static int kprobe_trap_handler(struct pt_regs *regs, int trapnr) * In case the user-specified fault handler returned * zero, try to fix up. */ - entry = search_exception_tables(regs->psw.addr); + entry = s390_search_extables(regs->psw.addr); if (entry) { regs->psw.addr = extable_fixup(entry); return 1; diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index cb582649aba6..8a1ae140c5e2 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -27,6 +27,7 @@ #include <asm/cacheflush.h> #include <asm/os_info.h> #include <asm/set_memory.h> +#include <asm/stacktrace.h> #include <asm/switch_to.h> #include <asm/nmi.h> @@ -95,7 +96,7 @@ static void __do_machine_kdump(void *image) start_kdump(1); /* Die if start_kdump returns */ - disabled_wait((unsigned long) __builtin_return_address(0)); + disabled_wait(); } /* @@ -253,6 +254,9 @@ void arch_crash_save_vmcoreinfo(void) VMCOREINFO_SYMBOL(high_memory); VMCOREINFO_LENGTH(lowcore_ptr, NR_CPUS); mem_assign_absolute(S390_lowcore.vmcore_info, paddr_vmcoreinfo_note()); + vmcoreinfo_append_str("SDMA=%lx\n", __sdma); + vmcoreinfo_append_str("EDMA=%lx\n", __edma); + vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); } void machine_shutdown(void) @@ -280,7 +284,7 @@ static void __do_machine_kexec(void *data) (*data_mover)(&image->head, image->start); /* Die if kexec returns */ - disabled_wait((unsigned long) __builtin_return_address(0)); + disabled_wait(); } /* diff --git a/arch/s390/kernel/machine_kexec_file.c b/arch/s390/kernel/machine_kexec_file.c index 32023b4f9dc0..fbdd3ea73667 100644 --- a/arch/s390/kernel/machine_kexec_file.c +++ b/arch/s390/kernel/machine_kexec_file.c @@ -8,7 +8,12 @@ */ #include <linux/elf.h> +#include <linux/errno.h> #include <linux/kexec.h> +#include <linux/module.h> +#include <linux/verification.h> +#include <asm/boot_data.h> +#include <asm/ipl.h> #include <asm/setup.h> const struct kexec_file_ops * const kexec_file_loaders[] = { @@ -17,38 +22,78 @@ const struct kexec_file_ops * const kexec_file_loaders[] = { NULL, }; -int *kexec_file_update_kernel(struct kimage *image, - struct s390_load_data *data) -{ - unsigned long *loc; - - if (image->cmdline_buf_len >= ARCH_COMMAND_LINE_SIZE) - return ERR_PTR(-EINVAL); - - if (image->cmdline_buf_len) - memcpy(data->kernel_buf + COMMAND_LINE_OFFSET, - image->cmdline_buf, image->cmdline_buf_len); - - if (image->type == KEXEC_TYPE_CRASH) { - loc = (unsigned long *)(data->kernel_buf + OLDMEM_BASE_OFFSET); - *loc = crashk_res.start; - - loc = (unsigned long *)(data->kernel_buf + OLDMEM_SIZE_OFFSET); - *loc = crashk_res.end - crashk_res.start + 1; - } +#ifdef CONFIG_KEXEC_VERIFY_SIG +/* + * Module signature information block. + * + * The constituents of the signature section are, in order: + * + * - Signer's name + * - Key identifier + * - Signature data + * - Information block + */ +struct module_signature { + u8 algo; /* Public-key crypto algorithm [0] */ + u8 hash; /* Digest algorithm [0] */ + u8 id_type; /* Key identifier type [PKEY_ID_PKCS7] */ + u8 signer_len; /* Length of signer's name [0] */ + u8 key_id_len; /* Length of key identifier [0] */ + u8 __pad[3]; + __be32 sig_len; /* Length of signature data */ +}; - if (image->initrd_buf) { - loc = (unsigned long *)(data->kernel_buf + INITRD_START_OFFSET); - *loc = data->initrd_load_addr; +#define PKEY_ID_PKCS7 2 - loc = (unsigned long *)(data->kernel_buf + INITRD_SIZE_OFFSET); - *loc = image->initrd_buf_len; +int s390_verify_sig(const char *kernel, unsigned long kernel_len) +{ + const unsigned long marker_len = sizeof(MODULE_SIG_STRING) - 1; + struct module_signature *ms; + unsigned long sig_len; + + /* Skip signature verification when not secure IPLed. */ + if (!ipl_secure_flag) + return 0; + + if (marker_len > kernel_len) + return -EKEYREJECTED; + + if (memcmp(kernel + kernel_len - marker_len, MODULE_SIG_STRING, + marker_len)) + return -EKEYREJECTED; + kernel_len -= marker_len; + + ms = (void *)kernel + kernel_len - sizeof(*ms); + kernel_len -= sizeof(*ms); + + sig_len = be32_to_cpu(ms->sig_len); + if (sig_len >= kernel_len) + return -EKEYREJECTED; + kernel_len -= sig_len; + + if (ms->id_type != PKEY_ID_PKCS7) + return -EKEYREJECTED; + + if (ms->algo != 0 || + ms->hash != 0 || + ms->signer_len != 0 || + ms->key_id_len != 0 || + ms->__pad[0] != 0 || + ms->__pad[1] != 0 || + ms->__pad[2] != 0) { + return -EBADMSG; } - return NULL; + return verify_pkcs7_signature(kernel, kernel_len, + kernel + kernel_len, sig_len, + VERIFY_USE_PLATFORM_KEYRING, + VERIFYING_MODULE_SIGNATURE, + NULL, NULL); } +#endif /* CONFIG_KEXEC_VERIFY_SIG */ -static int kexec_file_update_purgatory(struct kimage *image) +static int kexec_file_update_purgatory(struct kimage *image, + struct s390_load_data *data) { u64 entry, type; int ret; @@ -90,7 +135,8 @@ static int kexec_file_update_purgatory(struct kimage *image) return ret; } -int kexec_file_add_purgatory(struct kimage *image, struct s390_load_data *data) +static int kexec_file_add_purgatory(struct kimage *image, + struct s390_load_data *data) { struct kexec_buf buf; int ret; @@ -105,21 +151,21 @@ int kexec_file_add_purgatory(struct kimage *image, struct s390_load_data *data) ret = kexec_load_purgatory(image, &buf); if (ret) return ret; + data->memsz += buf.memsz; - ret = kexec_file_update_purgatory(image); - return ret; + return kexec_file_update_purgatory(image, data); } -int kexec_file_add_initrd(struct kimage *image, struct s390_load_data *data, - char *initrd, unsigned long initrd_len) +static int kexec_file_add_initrd(struct kimage *image, + struct s390_load_data *data) { struct kexec_buf buf; int ret; buf.image = image; - buf.buffer = initrd; - buf.bufsz = initrd_len; + buf.buffer = image->initrd_buf; + buf.bufsz = image->initrd_buf_len; data->memsz = ALIGN(data->memsz, PAGE_SIZE); buf.mem = data->memsz; @@ -127,11 +173,115 @@ int kexec_file_add_initrd(struct kimage *image, struct s390_load_data *data, buf.mem += crashk_res.start; buf.memsz = buf.bufsz; - data->initrd_load_addr = buf.mem; + data->parm->initrd_start = buf.mem; + data->parm->initrd_size = buf.memsz; data->memsz += buf.memsz; ret = kexec_add_buffer(&buf); - return ret; + if (ret) + return ret; + + return ipl_report_add_component(data->report, &buf, 0, 0); +} + +static int kexec_file_add_ipl_report(struct kimage *image, + struct s390_load_data *data) +{ + __u32 *lc_ipl_parmblock_ptr; + unsigned int len, ncerts; + struct kexec_buf buf; + unsigned long addr; + void *ptr, *end; + + buf.image = image; + + data->memsz = ALIGN(data->memsz, PAGE_SIZE); + buf.mem = data->memsz; + if (image->type == KEXEC_TYPE_CRASH) + buf.mem += crashk_res.start; + + ptr = (void *)ipl_cert_list_addr; + end = ptr + ipl_cert_list_size; + ncerts = 0; + while (ptr < end) { + ncerts++; + len = *(unsigned int *)ptr; + ptr += sizeof(len); + ptr += len; + } + + addr = data->memsz + data->report->size; + addr += ncerts * sizeof(struct ipl_rb_certificate_entry); + ptr = (void *)ipl_cert_list_addr; + while (ptr < end) { + len = *(unsigned int *)ptr; + ptr += sizeof(len); + ipl_report_add_certificate(data->report, ptr, addr, len); + addr += len; + ptr += len; + } + + buf.buffer = ipl_report_finish(data->report); + buf.bufsz = data->report->size; + buf.memsz = buf.bufsz; + + data->memsz += buf.memsz; + + lc_ipl_parmblock_ptr = + data->kernel_buf + offsetof(struct lowcore, ipl_parmblock_ptr); + *lc_ipl_parmblock_ptr = (__u32)buf.mem; + + return kexec_add_buffer(&buf); +} + +void *kexec_file_add_components(struct kimage *image, + int (*add_kernel)(struct kimage *image, + struct s390_load_data *data)) +{ + struct s390_load_data data = {0}; + int ret; + + data.report = ipl_report_init(&ipl_block); + if (IS_ERR(data.report)) + return data.report; + + ret = add_kernel(image, &data); + if (ret) + goto out; + + if (image->cmdline_buf_len >= ARCH_COMMAND_LINE_SIZE) { + ret = -EINVAL; + goto out; + } + memcpy(data.parm->command_line, image->cmdline_buf, + image->cmdline_buf_len); + + if (image->type == KEXEC_TYPE_CRASH) { + data.parm->oldmem_base = crashk_res.start; + data.parm->oldmem_size = crashk_res.end - crashk_res.start + 1; + } + + if (image->initrd_buf) { + ret = kexec_file_add_initrd(image, &data); + if (ret) + goto out; + } + + ret = kexec_file_add_purgatory(image, &data); + if (ret) + goto out; + + if (data.kernel_mem == 0) { + unsigned long restart_psw = 0x0008000080000000UL; + restart_psw += image->start; + memcpy(data.kernel_buf, &restart_psw, sizeof(restart_psw)); + image->start = 0; + } + + ret = kexec_file_add_ipl_report(image, &data); +out: + ipl_report_free(data.report); + return ERR_PTR(ret); } int arch_kexec_apply_relocations_add(struct purgatory_info *pi, @@ -140,7 +290,7 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, const Elf_Shdr *symtab) { Elf_Rela *relas; - int i; + int i, r_type; relas = (void *)pi->ehdr + relsec->sh_offset; @@ -174,46 +324,8 @@ int arch_kexec_apply_relocations_add(struct purgatory_info *pi, addr = section->sh_addr + relas[i].r_offset; - switch (ELF64_R_TYPE(relas[i].r_info)) { - case R_390_8: /* Direct 8 bit. */ - *(u8 *)loc = val; - break; - case R_390_12: /* Direct 12 bit. */ - *(u16 *)loc &= 0xf000; - *(u16 *)loc |= val & 0xfff; - break; - case R_390_16: /* Direct 16 bit. */ - *(u16 *)loc = val; - break; - case R_390_20: /* Direct 20 bit. */ - *(u32 *)loc &= 0xf00000ff; - *(u32 *)loc |= (val & 0xfff) << 16; /* DL */ - *(u32 *)loc |= (val & 0xff000) >> 4; /* DH */ - break; - case R_390_32: /* Direct 32 bit. */ - *(u32 *)loc = val; - break; - case R_390_64: /* Direct 64 bit. */ - *(u64 *)loc = val; - break; - case R_390_PC16: /* PC relative 16 bit. */ - *(u16 *)loc = (val - addr); - break; - case R_390_PC16DBL: /* PC relative 16 bit shifted by 1. */ - *(u16 *)loc = (val - addr) >> 1; - break; - case R_390_PC32DBL: /* PC relative 32 bit shifted by 1. */ - *(u32 *)loc = (val - addr) >> 1; - break; - case R_390_PC32: /* PC relative 32 bit. */ - *(u32 *)loc = (val - addr); - break; - case R_390_PC64: /* PC relative 64 bit. */ - *(u64 *)loc = (val - addr); - break; - default: - break; - } + r_type = ELF64_R_TYPE(relas[i].r_info); + arch_kexec_do_relocs(r_type, loc, val, addr); } return 0; } @@ -225,10 +337,8 @@ int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, * load memory in head.S will be accessed, e.g. to register the next * command line. If the next kernel were smaller the current kernel * will panic at load. - * - * 0x11000 = sizeof(head.S) */ - if (buf_len < 0x11000) + if (buf_len < HEAD_END) return -ENOEXEC; return kexec_image_probe_default(image, buf, buf_len); diff --git a/arch/s390/kernel/machine_kexec_reloc.c b/arch/s390/kernel/machine_kexec_reloc.c new file mode 100644 index 000000000000..1dded39239f8 --- /dev/null +++ b/arch/s390/kernel/machine_kexec_reloc.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/elf.h> + +int arch_kexec_do_relocs(int r_type, void *loc, unsigned long val, + unsigned long addr) +{ + switch (r_type) { + case R_390_NONE: + break; + case R_390_8: /* Direct 8 bit. */ + *(u8 *)loc = val; + break; + case R_390_12: /* Direct 12 bit. */ + *(u16 *)loc &= 0xf000; + *(u16 *)loc |= val & 0xfff; + break; + case R_390_16: /* Direct 16 bit. */ + *(u16 *)loc = val; + break; + case R_390_20: /* Direct 20 bit. */ + *(u32 *)loc &= 0xf00000ff; + *(u32 *)loc |= (val & 0xfff) << 16; /* DL */ + *(u32 *)loc |= (val & 0xff000) >> 4; /* DH */ + break; + case R_390_32: /* Direct 32 bit. */ + *(u32 *)loc = val; + break; + case R_390_64: /* Direct 64 bit. */ + *(u64 *)loc = val; + break; + case R_390_PC16: /* PC relative 16 bit. */ + *(u16 *)loc = (val - addr); + break; + case R_390_PC16DBL: /* PC relative 16 bit shifted by 1. */ + *(u16 *)loc = (val - addr) >> 1; + break; + case R_390_PC32DBL: /* PC relative 32 bit shifted by 1. */ + *(u32 *)loc = (val - addr) >> 1; + break; + case R_390_PC32: /* PC relative 32 bit. */ + *(u32 *)loc = (val - addr); + break; + case R_390_PC64: /* PC relative 64 bit. */ + *(u64 *)loc = (val - addr); + break; + case R_390_RELATIVE: + *(unsigned long *) loc = val; + break; + default: + return 1; + } + return 0; +} diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index e93fbf02490c..9e1660a6b9db 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -20,6 +20,7 @@ ENTRY(ftrace_stub) BR_EX %r14 +ENDPROC(ftrace_stub) #define STACK_FRAME_SIZE (STACK_FRAME_OVERHEAD + __PT_SIZE) #define STACK_PTREGS (STACK_FRAME_OVERHEAD) @@ -28,7 +29,7 @@ ENTRY(ftrace_stub) ENTRY(_mcount) BR_EX %r14 - +ENDPROC(_mcount) EXPORT_SYMBOL(_mcount) ENTRY(ftrace_caller) @@ -61,10 +62,11 @@ ENTRY(ftrace_caller) #ifdef CONFIG_FUNCTION_GRAPH_TRACER # The j instruction gets runtime patched to a nop instruction. # See ftrace_enable_ftrace_graph_caller. -ENTRY(ftrace_graph_caller) + .globl ftrace_graph_caller +ftrace_graph_caller: j ftrace_graph_caller_end - lg %r2,(STACK_PTREGS_GPRS+14*8)(%r15) - lg %r3,(STACK_PTREGS_PSW+8)(%r15) + lmg %r2,%r3,(STACK_PTREGS_GPRS+14*8)(%r15) + lg %r4,(STACK_PTREGS_PSW+8)(%r15) brasl %r14,prepare_ftrace_return stg %r2,(STACK_PTREGS_GPRS+14*8)(%r15) ftrace_graph_caller_end: @@ -73,6 +75,7 @@ ftrace_graph_caller_end: lg %r1,(STACK_PTREGS_PSW+8)(%r15) lmg %r2,%r15,(STACK_PTREGS_GPRS+2*8)(%r15) BR_EX %r1 +ENDPROC(ftrace_caller) #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -86,5 +89,6 @@ ENTRY(return_to_handler) lgr %r14,%r2 lmg %r2,%r5,32(%r15) BR_EX %r14 +ENDPROC(return_to_handler) #endif diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 8c867b43c8eb..0a487fae763e 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -125,7 +125,7 @@ void nmi_free_per_cpu(struct lowcore *lc) static notrace void s390_handle_damage(void) { smp_emergency_stop(); - disabled_wait((unsigned long) __builtin_return_address(0)); + disabled_wait(); while (1); } NOKPROBE_SYMBOL(s390_handle_damage); diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c index 649135cbedd5..29e511f5bf06 100644 --- a/arch/s390/kernel/nospec-branch.c +++ b/arch/s390/kernel/nospec-branch.c @@ -38,7 +38,7 @@ static int __init nospec_report(void) { if (test_facility(156)) pr_info("Spectre V2 mitigation: etokens\n"); - if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) + if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) pr_info("Spectre V2 mitigation: execute trampolines\n"); if (__test_facility(82, S390_lowcore.alt_stfle_fac_list)) pr_info("Spectre V2 mitigation: limited branch prediction\n"); @@ -64,10 +64,10 @@ void __init nospec_auto_detect(void) * The machine supports etokens. * Disable expolines and disable nobp. */ - if (IS_ENABLED(CC_USING_EXPOLINE)) + if (__is_defined(CC_USING_EXPOLINE)) nospec_disable = 1; __clear_facility(82, S390_lowcore.alt_stfle_fac_list); - } else if (IS_ENABLED(CC_USING_EXPOLINE)) { + } else if (__is_defined(CC_USING_EXPOLINE)) { /* * The kernel has been compiled with expolines. * Keep expolines enabled and disable nobp. diff --git a/arch/s390/kernel/nospec-sysfs.c b/arch/s390/kernel/nospec-sysfs.c index e30e580ae362..48f472bf9290 100644 --- a/arch/s390/kernel/nospec-sysfs.c +++ b/arch/s390/kernel/nospec-sysfs.c @@ -15,7 +15,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, { if (test_facility(156)) return sprintf(buf, "Mitigation: etokens\n"); - if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) + if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) return sprintf(buf, "Mitigation: execute trampolines\n"); if (__test_facility(82, S390_lowcore.alt_stfle_fac_list)) return sprintf(buf, "Mitigation: limited branch prediction\n"); diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index e1c54d28713a..48d48b6187c0 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -2,8 +2,8 @@ /* * Performance event support for s390x - CPU-measurement Counter Facility * - * Copyright IBM Corp. 2012, 2017 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> + * Copyright IBM Corp. 2012, 2019 + * Author(s): Hendrik Brueckner <brueckner@linux.ibm.com> */ #define KMSG_COMPONENT "cpum_cf" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt @@ -26,7 +26,7 @@ static enum cpumf_ctr_set get_counter_set(u64 event) set = CPUMF_CTR_SET_USER; else if (event < 128) set = CPUMF_CTR_SET_CRYPTO; - else if (event < 256) + else if (event < 288) set = CPUMF_CTR_SET_EXT; else if (event >= 448 && event < 496) set = CPUMF_CTR_SET_MT_DIAG; @@ -50,12 +50,19 @@ static int validate_ctr_version(const struct hw_perf_event *hwc) err = -EOPNOTSUPP; break; case CPUMF_CTR_SET_CRYPTO: + if ((cpuhw->info.csvn >= 1 && cpuhw->info.csvn <= 5 && + hwc->config > 79) || + (cpuhw->info.csvn >= 6 && hwc->config > 83)) + err = -EOPNOTSUPP; + break; case CPUMF_CTR_SET_EXT: if (cpuhw->info.csvn < 1) err = -EOPNOTSUPP; if ((cpuhw->info.csvn == 1 && hwc->config > 159) || (cpuhw->info.csvn == 2 && hwc->config > 175) || - (cpuhw->info.csvn > 2 && hwc->config > 255)) + (cpuhw->info.csvn >= 3 && cpuhw->info.csvn <= 5 + && hwc->config > 255) || + (cpuhw->info.csvn >= 6 && hwc->config > 287)) err = -EOPNOTSUPP; break; case CPUMF_CTR_SET_MT_DIAG: diff --git a/arch/s390/kernel/perf_cpum_cf_diag.c b/arch/s390/kernel/perf_cpum_cf_diag.c index b6854812d2ed..d4e031f7b9c8 100644 --- a/arch/s390/kernel/perf_cpum_cf_diag.c +++ b/arch/s390/kernel/perf_cpum_cf_diag.c @@ -306,15 +306,20 @@ static size_t cf_diag_ctrset_size(enum cpumf_ctr_set ctrset, ctrset_size = 2; break; case CPUMF_CTR_SET_CRYPTO: - ctrset_size = 16; + if (info->csvn >= 1 && info->csvn <= 5) + ctrset_size = 16; + else if (info->csvn == 6) + ctrset_size = 20; break; case CPUMF_CTR_SET_EXT: if (info->csvn == 1) ctrset_size = 32; else if (info->csvn == 2) ctrset_size = 48; - else if (info->csvn >= 3) + else if (info->csvn >= 3 && info->csvn <= 5) ctrset_size = 128; + else if (info->csvn == 6) + ctrset_size = 160; break; case CPUMF_CTR_SET_MT_DIAG: if (info->csvn > 3) diff --git a/arch/s390/kernel/perf_cpum_cf_events.c b/arch/s390/kernel/perf_cpum_cf_events.c index b45238c89728..34cc96449b30 100644 --- a/arch/s390/kernel/perf_cpum_cf_events.c +++ b/arch/s390/kernel/perf_cpum_cf_events.c @@ -31,22 +31,26 @@ CPUMF_EVENT_ATTR(cf_fvn3, PROBLEM_STATE_CPU_CYCLES, 0x0020); CPUMF_EVENT_ATTR(cf_fvn3, PROBLEM_STATE_INSTRUCTIONS, 0x0021); CPUMF_EVENT_ATTR(cf_fvn3, L1D_DIR_WRITES, 0x0004); CPUMF_EVENT_ATTR(cf_fvn3, L1D_PENALTY_CYCLES, 0x0005); -CPUMF_EVENT_ATTR(cf_svn_generic, PRNG_FUNCTIONS, 0x0040); -CPUMF_EVENT_ATTR(cf_svn_generic, PRNG_CYCLES, 0x0041); -CPUMF_EVENT_ATTR(cf_svn_generic, PRNG_BLOCKED_FUNCTIONS, 0x0042); -CPUMF_EVENT_ATTR(cf_svn_generic, PRNG_BLOCKED_CYCLES, 0x0043); -CPUMF_EVENT_ATTR(cf_svn_generic, SHA_FUNCTIONS, 0x0044); -CPUMF_EVENT_ATTR(cf_svn_generic, SHA_CYCLES, 0x0045); -CPUMF_EVENT_ATTR(cf_svn_generic, SHA_BLOCKED_FUNCTIONS, 0x0046); -CPUMF_EVENT_ATTR(cf_svn_generic, SHA_BLOCKED_CYCLES, 0x0047); -CPUMF_EVENT_ATTR(cf_svn_generic, DEA_FUNCTIONS, 0x0048); -CPUMF_EVENT_ATTR(cf_svn_generic, DEA_CYCLES, 0x0049); -CPUMF_EVENT_ATTR(cf_svn_generic, DEA_BLOCKED_FUNCTIONS, 0x004a); -CPUMF_EVENT_ATTR(cf_svn_generic, DEA_BLOCKED_CYCLES, 0x004b); -CPUMF_EVENT_ATTR(cf_svn_generic, AES_FUNCTIONS, 0x004c); -CPUMF_EVENT_ATTR(cf_svn_generic, AES_CYCLES, 0x004d); -CPUMF_EVENT_ATTR(cf_svn_generic, AES_BLOCKED_FUNCTIONS, 0x004e); -CPUMF_EVENT_ATTR(cf_svn_generic, AES_BLOCKED_CYCLES, 0x004f); +CPUMF_EVENT_ATTR(cf_svn_12345, PRNG_FUNCTIONS, 0x0040); +CPUMF_EVENT_ATTR(cf_svn_12345, PRNG_CYCLES, 0x0041); +CPUMF_EVENT_ATTR(cf_svn_12345, PRNG_BLOCKED_FUNCTIONS, 0x0042); +CPUMF_EVENT_ATTR(cf_svn_12345, PRNG_BLOCKED_CYCLES, 0x0043); +CPUMF_EVENT_ATTR(cf_svn_12345, SHA_FUNCTIONS, 0x0044); +CPUMF_EVENT_ATTR(cf_svn_12345, SHA_CYCLES, 0x0045); +CPUMF_EVENT_ATTR(cf_svn_12345, SHA_BLOCKED_FUNCTIONS, 0x0046); +CPUMF_EVENT_ATTR(cf_svn_12345, SHA_BLOCKED_CYCLES, 0x0047); +CPUMF_EVENT_ATTR(cf_svn_12345, DEA_FUNCTIONS, 0x0048); +CPUMF_EVENT_ATTR(cf_svn_12345, DEA_CYCLES, 0x0049); +CPUMF_EVENT_ATTR(cf_svn_12345, DEA_BLOCKED_FUNCTIONS, 0x004a); +CPUMF_EVENT_ATTR(cf_svn_12345, DEA_BLOCKED_CYCLES, 0x004b); +CPUMF_EVENT_ATTR(cf_svn_12345, AES_FUNCTIONS, 0x004c); +CPUMF_EVENT_ATTR(cf_svn_12345, AES_CYCLES, 0x004d); +CPUMF_EVENT_ATTR(cf_svn_12345, AES_BLOCKED_FUNCTIONS, 0x004e); +CPUMF_EVENT_ATTR(cf_svn_12345, AES_BLOCKED_CYCLES, 0x004f); +CPUMF_EVENT_ATTR(cf_svn_6, ECC_FUNCTION_COUNT, 0x0050); +CPUMF_EVENT_ATTR(cf_svn_6, ECC_CYCLES_COUNT, 0x0051); +CPUMF_EVENT_ATTR(cf_svn_6, ECC_BLOCKED_FUNCTION_COUNT, 0x0052); +CPUMF_EVENT_ATTR(cf_svn_6, ECC_BLOCKED_CYCLES_COUNT, 0x0053); CPUMF_EVENT_ATTR(cf_z10, L1I_L2_SOURCED_WRITES, 0x0080); CPUMF_EVENT_ATTR(cf_z10, L1D_L2_SOURCED_WRITES, 0x0081); CPUMF_EVENT_ATTR(cf_z10, L1I_L3_LOCAL_WRITES, 0x0082); @@ -262,23 +266,47 @@ static struct attribute *cpumcf_fvn3_pmu_event_attr[] __initdata = { NULL, }; -static struct attribute *cpumcf_svn_generic_pmu_event_attr[] __initdata = { - CPUMF_EVENT_PTR(cf_svn_generic, PRNG_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, PRNG_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, PRNG_BLOCKED_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, PRNG_BLOCKED_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, SHA_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, SHA_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, SHA_BLOCKED_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, SHA_BLOCKED_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, DEA_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, DEA_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, DEA_BLOCKED_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, DEA_BLOCKED_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, AES_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, AES_CYCLES), - CPUMF_EVENT_PTR(cf_svn_generic, AES_BLOCKED_FUNCTIONS), - CPUMF_EVENT_PTR(cf_svn_generic, AES_BLOCKED_CYCLES), +static struct attribute *cpumcf_svn_12345_pmu_event_attr[] __initdata = { + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, AES_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, AES_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, AES_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, AES_BLOCKED_CYCLES), + NULL, +}; + +static struct attribute *cpumcf_svn_6_pmu_event_attr[] __initdata = { + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, PRNG_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, SHA_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, DEA_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, AES_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, AES_CYCLES), + CPUMF_EVENT_PTR(cf_svn_12345, AES_BLOCKED_FUNCTIONS), + CPUMF_EVENT_PTR(cf_svn_12345, AES_BLOCKED_CYCLES), + CPUMF_EVENT_PTR(cf_svn_6, ECC_FUNCTION_COUNT), + CPUMF_EVENT_PTR(cf_svn_6, ECC_CYCLES_COUNT), + CPUMF_EVENT_PTR(cf_svn_6, ECC_BLOCKED_FUNCTION_COUNT), + CPUMF_EVENT_PTR(cf_svn_6, ECC_BLOCKED_CYCLES_COUNT), NULL, }; @@ -562,7 +590,18 @@ __init const struct attribute_group **cpumf_cf_event_group(void) default: cfvn = none; } - csvn = cpumcf_svn_generic_pmu_event_attr; + + /* Determine version specific crypto set */ + switch (ci.csvn) { + case 1 ... 5: + csvn = cpumcf_svn_12345_pmu_event_attr; + break; + case 6: + csvn = cpumcf_svn_6_pmu_event_attr; + break; + default: + csvn = none; + } /* Determine model-specific counter set(s) */ get_cpu_id(&cpu_id); diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c index 0d770e513abf..fcb6c2e92b07 100644 --- a/arch/s390/kernel/perf_event.c +++ b/arch/s390/kernel/perf_event.c @@ -21,6 +21,7 @@ #include <asm/lowcore.h> #include <asm/processor.h> #include <asm/sysinfo.h> +#include <asm/unwind.h> const char *perf_pmu_name(void) { @@ -219,20 +220,13 @@ static int __init service_level_perf_register(void) } arch_initcall(service_level_perf_register); -static int __perf_callchain_kernel(void *data, unsigned long address, int reliable) -{ - struct perf_callchain_entry_ctx *entry = data; - - perf_callchain_store(entry, address); - return 0; -} - void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - if (user_mode(regs)) - return; - dump_trace(__perf_callchain_kernel, entry, NULL, regs->gprs[15]); + struct unwind_state state; + + unwind_for_each_frame(&state, current, regs, 0) + perf_callchain_store(entry, state.ip); } /* Perf definitions for PMU event attributes in sysfs */ diff --git a/arch/s390/kernel/pgm_check.S b/arch/s390/kernel/pgm_check.S index 3e62aae34ea3..59dee9d3bebf 100644 --- a/arch/s390/kernel/pgm_check.S +++ b/arch/s390/kernel/pgm_check.S @@ -7,7 +7,7 @@ #include <linux/linkage.h> -#define PGM_CHECK(handler) .long handler +#define PGM_CHECK(handler) .quad handler #define PGM_CHECK_DEFAULT PGM_CHECK(default_trap_handler) /* diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 6e758bb6cd29..63873aa6693f 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -37,6 +37,7 @@ #include <asm/irq.h> #include <asm/nmi.h> #include <asm/smp.h> +#include <asm/stacktrace.h> #include <asm/switch_to.h> #include <asm/runtime_instr.h> #include "entry.h" diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index 6fe2e1875058..5de13307b703 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -109,7 +109,8 @@ static void show_cpu_summary(struct seq_file *m, void *v) { static const char *hwcap_str[] = { "esan3", "zarch", "stfle", "msa", "ldisp", "eimm", "dfp", - "edat", "etf3eh", "highgprs", "te", "vx", "vxd", "vxe", "gs" + "edat", "etf3eh", "highgprs", "te", "vx", "vxd", "vxe", "gs", + "vxe2", "vxp", "sort", "dflt" }; static const char * const int_hwcap_str[] = { "sie" diff --git a/arch/s390/kernel/reipl.S b/arch/s390/kernel/reipl.S index 7f14adf512c6..4a22163962eb 100644 --- a/arch/s390/kernel/reipl.S +++ b/arch/s390/kernel/reipl.S @@ -73,6 +73,7 @@ ENTRY(store_status) lgr %r9,%r2 lgr %r2,%r3 BR_EX %r9 +ENDPROC(store_status) .section .bss .align 8 diff --git a/arch/s390/kernel/relocate_kernel.S b/arch/s390/kernel/relocate_kernel.S index c97c2d40fe15..fe396673e8a6 100644 --- a/arch/s390/kernel/relocate_kernel.S +++ b/arch/s390/kernel/relocate_kernel.S @@ -58,11 +58,15 @@ ENTRY(relocate_kernel) j .base .done: sgr %r0,%r0 # clear register r0 + cghi %r3,0 + je .diag la %r4,load_psw-.base(%r13) # load psw-address into the register o %r3,4(%r4) # or load address into psw st %r3,4(%r4) mvc 0(8,%r0),0(%r4) # copy psw to absolute address 0 + .diag: diag %r0,%r0,0x308 +ENDPROC(relocate_kernel) .align 8 load_psw: diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 2c642af526ce..f8544d517430 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -50,6 +50,7 @@ #include <linux/compat.h> #include <linux/start_kernel.h> +#include <asm/boot_data.h> #include <asm/ipl.h> #include <asm/facility.h> #include <asm/smp.h> @@ -65,11 +66,13 @@ #include <asm/diag.h> #include <asm/os_info.h> #include <asm/sclp.h> +#include <asm/stacktrace.h> #include <asm/sysinfo.h> #include <asm/numa.h> #include <asm/alternative.h> #include <asm/nospec-branch.h> #include <asm/mem_detect.h> +#include <asm/uv.h> #include "entry.h" /* @@ -89,12 +92,25 @@ char elf_platform[ELF_PLATFORM_SIZE]; unsigned long int_hwcap = 0; +#ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST +int __bootdata_preserved(prot_virt_guest); +#endif + int __bootdata(noexec_disabled); int __bootdata(memory_end_set); unsigned long __bootdata(memory_end); unsigned long __bootdata(max_physmem_end); struct mem_detect_info __bootdata(mem_detect); +struct exception_table_entry *__bootdata_preserved(__start_dma_ex_table); +struct exception_table_entry *__bootdata_preserved(__stop_dma_ex_table); +unsigned long __bootdata_preserved(__swsusp_reset_dma); +unsigned long __bootdata_preserved(__stext_dma); +unsigned long __bootdata_preserved(__etext_dma); +unsigned long __bootdata_preserved(__sdma); +unsigned long __bootdata_preserved(__edma); +unsigned long __bootdata_preserved(__kaslr_offset); + unsigned long VMALLOC_START; EXPORT_SYMBOL(VMALLOC_START); @@ -736,6 +752,15 @@ static void __init reserve_initrd(void) #endif } +/* + * Reserve the memory area used to pass the certificate lists + */ +static void __init reserve_certificate_list(void) +{ + if (ipl_cert_list_addr) + memblock_reserve(ipl_cert_list_addr, ipl_cert_list_size); +} + static void __init reserve_mem_detect_info(void) { unsigned long start, size; @@ -814,9 +839,10 @@ static void __init reserve_kernel(void) { unsigned long start_pfn = PFN_UP(__pa(_end)); - memblock_reserve(0, PARMAREA_END); + memblock_reserve(0, HEAD_END); memblock_reserve((unsigned long)_stext, PFN_PHYS(start_pfn) - (unsigned long)_stext); + memblock_reserve(__sdma, __edma - __sdma); } static void __init setup_memory(void) @@ -914,7 +940,15 @@ static int __init setup_hwcaps(void) elf_hwcap |= HWCAP_S390_VXRS_EXT; if (test_facility(135)) elf_hwcap |= HWCAP_S390_VXRS_BCD; + if (test_facility(148)) + elf_hwcap |= HWCAP_S390_VXRS_EXT2; + if (test_facility(152)) + elf_hwcap |= HWCAP_S390_VXRS_PDE; } + if (test_facility(150)) + elf_hwcap |= HWCAP_S390_SORT; + if (test_facility(151)) + elf_hwcap |= HWCAP_S390_DFLT; /* * Guarded storage support HWCAP_S390_GS is bit 12. @@ -1023,6 +1057,38 @@ static void __init setup_control_program_code(void) } /* + * Print the component list from the IPL report + */ +static void __init log_component_list(void) +{ + struct ipl_rb_component_entry *ptr, *end; + char *str; + + if (!early_ipl_comp_list_addr) + return; + if (ipl_block.hdr.flags & IPL_PL_FLAG_IPLSR) + pr_info("Linux is running with Secure-IPL enabled\n"); + else + pr_info("Linux is running with Secure-IPL disabled\n"); + ptr = (void *) early_ipl_comp_list_addr; + end = (void *) ptr + early_ipl_comp_list_size; + pr_info("The IPL report contains the following components:\n"); + while (ptr < end) { + if (ptr->flags & IPL_RB_COMPONENT_FLAG_SIGNED) { + if (ptr->flags & IPL_RB_COMPONENT_FLAG_VERIFIED) + str = "signed, verified"; + else + str = "signed, verification failed"; + } else { + str = "not signed"; + } + pr_info("%016llx - %016llx (%s)\n", + ptr->addr, ptr->addr + ptr->len, str); + ptr++; + } +} + +/* * Setup function called from init/main.c just after the banner * was printed. */ @@ -1042,6 +1108,8 @@ void __init setup_arch(char **cmdline_p) else pr_info("Linux is running as a guest in 64-bit mode\n"); + log_component_list(); + /* Have one command line that is parsed and saved in /proc/cmdline */ /* boot_command_line has been already set up in early.c */ *cmdline_p = boot_command_line; @@ -1073,6 +1141,7 @@ void __init setup_arch(char **cmdline_p) reserve_oldmem(); reserve_kernel(); reserve_initrd(); + reserve_certificate_list(); reserve_mem_detect_info(); memblock_allow_resize(); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index bd197baf1dc3..35fafa2b91a8 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -53,6 +53,7 @@ #include <asm/sigp.h> #include <asm/idle.h> #include <asm/nmi.h> +#include <asm/stacktrace.h> #include <asm/topology.h> #include "entry.h" @@ -689,7 +690,7 @@ void __init smp_save_dump_cpus(void) smp_save_cpu_regs(sa, addr, is_boot_cpu, page); } memblock_free(page, PAGE_SIZE); - diag308_reset(); + diag_dma_ops.diag308_reset(); pcpu_set_smt(0); } #endif /* CONFIG_CRASH_DUMP */ diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c index cc9ed9787068..f6a620f854e1 100644 --- a/arch/s390/kernel/stacktrace.c +++ b/arch/s390/kernel/stacktrace.c @@ -11,59 +11,52 @@ #include <linux/stacktrace.h> #include <linux/kallsyms.h> #include <linux/export.h> - -static int __save_address(void *data, unsigned long address, int nosched) -{ - struct stack_trace *trace = data; - - if (nosched && in_sched_functions(address)) - return 0; - if (trace->skip > 0) { - trace->skip--; - return 0; - } - if (trace->nr_entries < trace->max_entries) { - trace->entries[trace->nr_entries++] = address; - return 0; - } - return 1; -} - -static int save_address(void *data, unsigned long address, int reliable) -{ - return __save_address(data, address, 0); -} - -static int save_address_nosched(void *data, unsigned long address, int reliable) -{ - return __save_address(data, address, 1); -} +#include <asm/stacktrace.h> +#include <asm/unwind.h> void save_stack_trace(struct stack_trace *trace) { - unsigned long sp; - - sp = current_stack_pointer(); - dump_trace(save_address, trace, NULL, sp); + struct unwind_state state; + + unwind_for_each_frame(&state, current, NULL, 0) { + if (trace->nr_entries >= trace->max_entries) + break; + if (trace->skip > 0) + trace->skip--; + else + trace->entries[trace->nr_entries++] = state.ip; + } } EXPORT_SYMBOL_GPL(save_stack_trace); void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) { - unsigned long sp; - - sp = tsk->thread.ksp; - if (tsk == current) - sp = current_stack_pointer(); - dump_trace(save_address_nosched, trace, tsk, sp); + struct unwind_state state; + + unwind_for_each_frame(&state, tsk, NULL, 0) { + if (trace->nr_entries >= trace->max_entries) + break; + if (in_sched_functions(state.ip)) + continue; + if (trace->skip > 0) + trace->skip--; + else + trace->entries[trace->nr_entries++] = state.ip; + } } EXPORT_SYMBOL_GPL(save_stack_trace_tsk); void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) { - unsigned long sp; - - sp = kernel_stack_pointer(regs); - dump_trace(save_address, trace, NULL, sp); + struct unwind_state state; + + unwind_for_each_frame(&state, current, regs, 0) { + if (trace->nr_entries >= trace->max_entries) + break; + if (trace->skip > 0) + trace->skip--; + else + trace->entries[trace->nr_entries++] = state.ip; + } } EXPORT_SYMBOL_GPL(save_stack_trace_regs); diff --git a/arch/s390/kernel/swsusp.S b/arch/s390/kernel/swsusp.S index 993100c31d65..19a3c427801a 100644 --- a/arch/s390/kernel/swsusp.S +++ b/arch/s390/kernel/swsusp.S @@ -108,6 +108,7 @@ ENTRY(swsusp_arch_suspend) lmg %r6,%r15,STACK_FRAME_OVERHEAD + __SF_GPRS(%r15) lghi %r2,0 BR_EX %r14 +ENDPROC(swsusp_arch_suspend) /* * Restore saved memory image to correct place and restore register context. @@ -154,20 +155,13 @@ ENTRY(swsusp_arch_resume) ptlb /* flush tlb */ /* Reset System */ - larl %r1,restart_entry - larl %r2,.Lrestart_diag308_psw - og %r1,0(%r2) - stg %r1,0(%r0) larl %r1,.Lnew_pgm_check_psw epsw %r2,%r3 stm %r2,%r3,0(%r1) mvc __LC_PGM_NEW_PSW(16,%r0),0(%r1) - lghi %r0,0 - diag %r0,%r0,0x308 -restart_entry: - lhi %r1,1 - sigp %r1,%r0,SIGP_SET_ARCHITECTURE - sam64 + larl %r1,__swsusp_reset_dma + lg %r1,0(%r1) + BASR_EX %r14,%r1 #ifdef CONFIG_SMP larl %r1,smp_cpu_mt_shift icm %r1,15,0(%r1) @@ -267,6 +261,7 @@ restore_registers: lmg %r6,%r15,STACK_FRAME_OVERHEAD + __SF_GPRS(%r15) lghi %r2,0 BR_EX %r14 +ENDPROC(swsusp_arch_resume) .section .data..nosave,"aw",@progbits .align 8 @@ -275,8 +270,6 @@ restore_registers: .Lpanic_string: .asciz "Resume not possible because suspend CPU is no longer available\n" .align 8 -.Lrestart_diag308_psw: - .long 0x00080000,0x80000000 .Lrestart_suspend_psw: .quad 0x0000000180000000,restart_suspend .Lnew_pgm_check_psw: diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 8003b38c1688..82e81a9f7112 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -49,7 +49,7 @@ void do_report_trap(struct pt_regs *regs, int si_signo, int si_code, char *str) report_user_fault(regs, si_signo, 0); } else { const struct exception_table_entry *fixup; - fixup = search_exception_tables(regs->psw.addr); + fixup = s390_search_extables(regs->psw.addr); if (fixup) regs->psw.addr = extable_fixup(fixup); else { @@ -263,5 +263,6 @@ NOKPROBE_SYMBOL(kernel_stack_overflow); void __init trap_init(void) { + sort_extable(__start_dma_ex_table, __stop_dma_ex_table); local_mcck_enable(); } diff --git a/arch/s390/kernel/unwind_bc.c b/arch/s390/kernel/unwind_bc.c new file mode 100644 index 000000000000..57fd4e902f1f --- /dev/null +++ b/arch/s390/kernel/unwind_bc.c @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/sched.h> +#include <linux/sched/task.h> +#include <linux/sched/task_stack.h> +#include <linux/interrupt.h> +#include <asm/sections.h> +#include <asm/ptrace.h> +#include <asm/bitops.h> +#include <asm/stacktrace.h> +#include <asm/unwind.h> + +unsigned long unwind_get_return_address(struct unwind_state *state) +{ + if (unwind_done(state)) + return 0; + return __kernel_text_address(state->ip) ? state->ip : 0; +} +EXPORT_SYMBOL_GPL(unwind_get_return_address); + +static bool outside_of_stack(struct unwind_state *state, unsigned long sp) +{ + return (sp <= state->sp) || + (sp + sizeof(struct stack_frame) > state->stack_info.end); +} + +static bool update_stack_info(struct unwind_state *state, unsigned long sp) +{ + struct stack_info *info = &state->stack_info; + unsigned long *mask = &state->stack_mask; + + /* New stack pointer leaves the current stack */ + if (get_stack_info(sp, state->task, info, mask) != 0 || + !on_stack(info, sp, sizeof(struct stack_frame))) + /* 'sp' does not point to a valid stack */ + return false; + return true; +} + +bool unwind_next_frame(struct unwind_state *state) +{ + struct stack_info *info = &state->stack_info; + struct stack_frame *sf; + struct pt_regs *regs; + unsigned long sp, ip; + bool reliable; + + regs = state->regs; + if (unlikely(regs)) { + sp = READ_ONCE_TASK_STACK(state->task, regs->gprs[15]); + if (unlikely(outside_of_stack(state, sp))) { + if (!update_stack_info(state, sp)) + goto out_err; + } + sf = (struct stack_frame *) sp; + ip = READ_ONCE_TASK_STACK(state->task, sf->gprs[8]); + reliable = false; + regs = NULL; + } else { + sf = (struct stack_frame *) state->sp; + sp = READ_ONCE_TASK_STACK(state->task, sf->back_chain); + if (likely(sp)) { + /* Non-zero back-chain points to the previous frame */ + if (unlikely(outside_of_stack(state, sp))) { + if (!update_stack_info(state, sp)) + goto out_err; + } + sf = (struct stack_frame *) sp; + ip = READ_ONCE_TASK_STACK(state->task, sf->gprs[8]); + reliable = true; + } else { + /* No back-chain, look for a pt_regs structure */ + sp = state->sp + STACK_FRAME_OVERHEAD; + if (!on_stack(info, sp, sizeof(struct pt_regs))) + goto out_stop; + regs = (struct pt_regs *) sp; + if (user_mode(regs)) + goto out_stop; + ip = READ_ONCE_TASK_STACK(state->task, regs->psw.addr); + reliable = true; + } + } + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + /* Decode any ftrace redirection */ + if (ip == (unsigned long) return_to_handler) + ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, + ip, (void *) sp); +#endif + + /* Update unwind state */ + state->sp = sp; + state->ip = ip; + state->regs = regs; + state->reliable = reliable; + return true; + +out_err: + state->error = true; +out_stop: + state->stack_info.type = STACK_TYPE_UNKNOWN; + return false; +} +EXPORT_SYMBOL_GPL(unwind_next_frame); + +void __unwind_start(struct unwind_state *state, struct task_struct *task, + struct pt_regs *regs, unsigned long sp) +{ + struct stack_info *info = &state->stack_info; + unsigned long *mask = &state->stack_mask; + struct stack_frame *sf; + unsigned long ip; + bool reliable; + + memset(state, 0, sizeof(*state)); + state->task = task; + state->regs = regs; + + /* Don't even attempt to start from user mode regs: */ + if (regs && user_mode(regs)) { + info->type = STACK_TYPE_UNKNOWN; + return; + } + + /* Get current stack pointer and initialize stack info */ + if (get_stack_info(sp, task, info, mask) != 0 || + !on_stack(info, sp, sizeof(struct stack_frame))) { + /* Something is wrong with the stack pointer */ + info->type = STACK_TYPE_UNKNOWN; + state->error = true; + return; + } + + /* Get the instruction pointer from pt_regs or the stack frame */ + if (regs) { + ip = READ_ONCE_TASK_STACK(state->task, regs->psw.addr); + reliable = true; + } else { + sf = (struct stack_frame *) sp; + ip = READ_ONCE_TASK_STACK(state->task, sf->gprs[8]); + reliable = false; + } + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + /* Decode any ftrace redirection */ + if (ip == (unsigned long) return_to_handler) + ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, + ip, NULL); +#endif + + /* Update unwind state */ + state->sp = sp; + state->ip = ip; + state->reliable = reliable; +} +EXPORT_SYMBOL_GPL(__unwind_start); diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index e7920a68a12e..243d8b1185bf 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -29,7 +29,7 @@ #include <asm/vdso.h> #include <asm/facility.h> -#ifdef CONFIG_COMPAT +#ifdef CONFIG_COMPAT_VDSO extern char vdso32_start, vdso32_end; static void *vdso32_kbase = &vdso32_start; static unsigned int vdso32_pages; @@ -55,7 +55,7 @@ static vm_fault_t vdso_fault(const struct vm_special_mapping *sm, vdso_pagelist = vdso64_pagelist; vdso_pages = vdso64_pages; -#ifdef CONFIG_COMPAT +#ifdef CONFIG_COMPAT_VDSO if (vma->vm_mm->context.compat_mm) { vdso_pagelist = vdso32_pagelist; vdso_pages = vdso32_pages; @@ -76,7 +76,7 @@ static int vdso_mremap(const struct vm_special_mapping *sm, unsigned long vdso_pages; vdso_pages = vdso64_pages; -#ifdef CONFIG_COMPAT +#ifdef CONFIG_COMPAT_VDSO if (vma->vm_mm->context.compat_mm) vdso_pages = vdso32_pages; #endif @@ -223,7 +223,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) return 0; vdso_pages = vdso64_pages; -#ifdef CONFIG_COMPAT +#ifdef CONFIG_COMPAT_VDSO mm->context.compat_mm = is_compat_task(); if (mm->context.compat_mm) vdso_pages = vdso32_pages; @@ -280,7 +280,7 @@ static int __init vdso_init(void) int i; vdso_init_data(vdso_data); -#ifdef CONFIG_COMPAT +#ifdef CONFIG_COMPAT_VDSO /* Calculate the size of the 32 bit vDSO */ vdso32_pages = ((&vdso32_end - &vdso32_start + PAGE_SIZE - 1) >> PAGE_SHIFT) + 1; diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile index e76309fbbcb3..aee9ffbccb54 100644 --- a/arch/s390/kernel/vdso32/Makefile +++ b/arch/s390/kernel/vdso32/Makefile @@ -19,7 +19,7 @@ KBUILD_AFLAGS_31 += -m31 -s KBUILD_CFLAGS_31 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_31 += -m31 -fPIC -shared -fno-common -fno-builtin KBUILD_CFLAGS_31 += -nostdlib -Wl,-soname=linux-vdso32.so.1 \ - $(call cc-ldoption, -Wl$(comma)--hash-style=both) + -Wl,--hash-style=both $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_31) $(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_31) diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index f849ac61c5da..bec19e7e6e1c 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -19,7 +19,7 @@ KBUILD_AFLAGS_64 += -m64 -s KBUILD_CFLAGS_64 := $(filter-out -m64,$(KBUILD_CFLAGS)) KBUILD_CFLAGS_64 += -m64 -fPIC -shared -fno-common -fno-builtin KBUILD_CFLAGS_64 += -nostdlib -Wl,-soname=linux-vdso64.so.1 \ - $(call cc-ldoption, -Wl$(comma)--hash-style=both) + -Wl,--hash-style=both $(targets:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_64) $(targets:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_64) diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 8429ab079715..49d55327de0b 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -72,6 +72,7 @@ SECTIONS __end_ro_after_init = .; RW_DATA_SECTION(0x100, PAGE_SIZE, THREAD_SIZE) + BOOT_DATA_PRESERVED _edata = .; /* End of data section */ @@ -143,6 +144,18 @@ SECTIONS INIT_DATA_SECTION(0x100) PERCPU_SECTION(0x100) + + .dynsym ALIGN(8) : { + __dynsym_start = .; + *(.dynsym) + __dynsym_end = .; + } + .rela.dyn ALIGN(8) : { + __rela_dyn_start = .; + *(.rela*) + __rela_dyn_end = .; + } + . = ALIGN(PAGE_SIZE); __init_end = .; /* freed after init ends here */ @@ -161,6 +174,12 @@ SECTIONS QUAD(__bss_stop - __bss_start) /* bss_size */ QUAD(__boot_data_start) /* bootdata_off */ QUAD(__boot_data_end - __boot_data_start) /* bootdata_size */ + QUAD(__boot_data_preserved_start) /* bootdata_preserved_off */ + QUAD(__boot_data_preserved_end - + __boot_data_preserved_start) /* bootdata_preserved_size */ + QUAD(__dynsym_start) /* dynsym_start */ + QUAD(__rela_dyn_start) /* rela_dyn_start */ + QUAD(__rela_dyn_end) /* rela_dyn_end */ } :NONE /* Debugging sections. */ diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 82162867f378..37503ae62486 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3194,7 +3194,7 @@ out: } EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister); -static void gib_alert_irq_handler(struct airq_struct *airq) +static void gib_alert_irq_handler(struct airq_struct *airq, bool floating) { inc_irq_stat(IRQIO_GAL); process_gib_alert_list(); diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S index 53008da05190..dc0874f2e203 100644 --- a/arch/s390/lib/mem.S +++ b/arch/s390/lib/mem.S @@ -178,6 +178,7 @@ ENTRY(__memset\bits) BR_EX %r14 .L__memset_mvc\bits: mvc \bytes(1,%r1),0(%r1) +ENDPROC(__memset\bits) .endm __MEMSET 16,2,sth diff --git a/arch/s390/mm/Makefile b/arch/s390/mm/Makefile index f5880bfd1b0c..3175413186b9 100644 --- a/arch/s390/mm/Makefile +++ b/arch/s390/mm/Makefile @@ -4,7 +4,7 @@ # obj-y := init.o fault.o extmem.o mmap.o vmem.o maccess.o -obj-y += page-states.o gup.o pageattr.o pgtable.o pgalloc.o +obj-y += page-states.o pageattr.o pgtable.o pgalloc.o obj-$(CONFIG_CMM) += cmm.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 11613362c4e7..c220399ae196 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -247,12 +247,24 @@ static noinline void do_sigsegv(struct pt_regs *regs, int si_code) current); } +const struct exception_table_entry *s390_search_extables(unsigned long addr) +{ + const struct exception_table_entry *fixup; + + fixup = search_extable(__start_dma_ex_table, + __stop_dma_ex_table - __start_dma_ex_table, + addr); + if (!fixup) + fixup = search_exception_tables(addr); + return fixup; +} + static noinline void do_no_context(struct pt_regs *regs) { const struct exception_table_entry *fixup; /* Are we prepared to handle this kernel fault? */ - fixup = search_exception_tables(regs->psw.addr); + fixup = s390_search_extables(regs->psw.addr); if (fixup) { regs->psw.addr = extable_fixup(fixup); return; diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c deleted file mode 100644 index 2809d11c7a28..000000000000 --- a/arch/s390/mm/gup.c +++ /dev/null @@ -1,300 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Lockless get_user_pages_fast for s390 - * - * Copyright IBM Corp. 2010 - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ -#include <linux/sched.h> -#include <linux/mm.h> -#include <linux/hugetlb.h> -#include <linux/vmstat.h> -#include <linux/pagemap.h> -#include <linux/rwsem.h> -#include <asm/pgtable.h> - -/* - * The performance critical leaf functions are made noinline otherwise gcc - * inlines everything into a single function which results in too much - * register pressure. - */ -static inline int gup_pte_range(pmd_t *pmdp, pmd_t pmd, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - struct page *head, *page; - unsigned long mask; - pte_t *ptep, pte; - - mask = (write ? _PAGE_PROTECT : 0) | _PAGE_INVALID | _PAGE_SPECIAL; - - ptep = ((pte_t *) pmd_deref(pmd)) + pte_index(addr); - do { - pte = *ptep; - barrier(); - /* Similar to the PMD case, NUMA hinting must take slow path */ - if (pte_protnone(pte)) - return 0; - if ((pte_val(pte) & mask) != 0) - return 0; - VM_BUG_ON(!pfn_valid(pte_pfn(pte))); - page = pte_page(pte); - head = compound_head(page); - if (!page_cache_get_speculative(head)) - return 0; - if (unlikely(pte_val(pte) != pte_val(*ptep))) { - put_page(head); - return 0; - } - VM_BUG_ON_PAGE(compound_head(page) != head, page); - pages[*nr] = page; - (*nr)++; - - } while (ptep++, addr += PAGE_SIZE, addr != end); - - return 1; -} - -static inline int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - struct page *head, *page; - unsigned long mask; - int refs; - - mask = (write ? _SEGMENT_ENTRY_PROTECT : 0) | _SEGMENT_ENTRY_INVALID; - if ((pmd_val(pmd) & mask) != 0) - return 0; - VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT)); - - refs = 0; - head = pmd_page(pmd); - page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - do { - VM_BUG_ON(compound_head(page) != head); - pages[*nr] = page; - (*nr)++; - page++; - refs++; - } while (addr += PAGE_SIZE, addr != end); - - if (!page_cache_add_speculative(head, refs)) { - *nr -= refs; - return 0; - } - - if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { - *nr -= refs; - while (refs--) - put_page(head); - return 0; - } - - return 1; -} - - -static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - unsigned long next; - pmd_t *pmdp, pmd; - - pmdp = (pmd_t *) pudp; - if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) - pmdp = (pmd_t *) pud_deref(pud); - pmdp += pmd_index(addr); - do { - pmd = *pmdp; - barrier(); - next = pmd_addr_end(addr, end); - if (pmd_none(pmd)) - return 0; - if (unlikely(pmd_large(pmd))) { - /* - * NUMA hinting faults need to be handled in the GUP - * slowpath for accounting purposes and so that they - * can be serialised against THP migration. - */ - if (pmd_protnone(pmd)) - return 0; - if (!gup_huge_pmd(pmdp, pmd, addr, next, - write, pages, nr)) - return 0; - } else if (!gup_pte_range(pmdp, pmd, addr, next, - write, pages, nr)) - return 0; - } while (pmdp++, addr = next, addr != end); - - return 1; -} - -static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - struct page *head, *page; - unsigned long mask; - int refs; - - mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID; - if ((pud_val(pud) & mask) != 0) - return 0; - VM_BUG_ON(!pfn_valid(pud_pfn(pud))); - - refs = 0; - head = pud_page(pud); - page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - do { - VM_BUG_ON_PAGE(compound_head(page) != head, page); - pages[*nr] = page; - (*nr)++; - page++; - refs++; - } while (addr += PAGE_SIZE, addr != end); - - if (!page_cache_add_speculative(head, refs)) { - *nr -= refs; - return 0; - } - - if (unlikely(pud_val(pud) != pud_val(*pudp))) { - *nr -= refs; - while (refs--) - put_page(head); - return 0; - } - - return 1; -} - -static inline int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - unsigned long next; - pud_t *pudp, pud; - - pudp = (pud_t *) p4dp; - if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) - pudp = (pud_t *) p4d_deref(p4d); - pudp += pud_index(addr); - do { - pud = *pudp; - barrier(); - next = pud_addr_end(addr, end); - if (pud_none(pud)) - return 0; - if (unlikely(pud_large(pud))) { - if (!gup_huge_pud(pudp, pud, addr, next, write, pages, - nr)) - return 0; - } else if (!gup_pmd_range(pudp, pud, addr, next, write, pages, - nr)) - return 0; - } while (pudp++, addr = next, addr != end); - - return 1; -} - -static inline int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, - unsigned long end, int write, struct page **pages, int *nr) -{ - unsigned long next; - p4d_t *p4dp, p4d; - - p4dp = (p4d_t *) pgdp; - if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1) - p4dp = (p4d_t *) pgd_deref(pgd); - p4dp += p4d_index(addr); - do { - p4d = *p4dp; - barrier(); - next = p4d_addr_end(addr, end); - if (p4d_none(p4d)) - return 0; - if (!gup_pud_range(p4dp, p4d, addr, next, write, pages, nr)) - return 0; - } while (p4dp++, addr = next, addr != end); - - return 1; -} - -/* - * Like get_user_pages_fast() except its IRQ-safe in that it won't fall - * back to the regular GUP. - * Note a difference with get_user_pages_fast: this always returns the - * number of pages pinned, 0 if no pages were pinned. - */ -int __get_user_pages_fast(unsigned long start, int nr_pages, int write, - struct page **pages) -{ - struct mm_struct *mm = current->mm; - unsigned long addr, len, end; - unsigned long next, flags; - pgd_t *pgdp, pgd; - int nr = 0; - - start &= PAGE_MASK; - addr = start; - len = (unsigned long) nr_pages << PAGE_SHIFT; - end = start + len; - if ((end <= start) || (end > mm->context.asce_limit)) - return 0; - /* - * local_irq_save() doesn't prevent pagetable teardown, but does - * prevent the pagetables from being freed on s390. - * - * So long as we atomically load page table pointers versus teardown, - * we can follow the address down to the the page and take a ref on it. - */ - local_irq_save(flags); - pgdp = pgd_offset(mm, addr); - do { - pgd = *pgdp; - barrier(); - next = pgd_addr_end(addr, end); - if (pgd_none(pgd)) - break; - if (!gup_p4d_range(pgdp, pgd, addr, next, write, pages, &nr)) - break; - } while (pgdp++, addr = next, addr != end); - local_irq_restore(flags); - - return nr; -} - -/** - * get_user_pages_fast() - pin user pages in memory - * @start: starting user address - * @nr_pages: number of pages from start to pin - * @write: whether pages will be written to - * @pages: array that receives pointers to the pages pinned. - * Should be at least nr_pages long. - * - * Attempt to pin user pages in memory without taking mm->mmap_sem. - * If not successful, it will fall back to taking the lock and - * calling get_user_pages(). - * - * Returns number of pages pinned. This may be fewer than the number - * requested. If nr_pages is 0 or negative, returns 0. If no pages - * were pinned, returns -errno. - */ -int get_user_pages_fast(unsigned long start, int nr_pages, int write, - struct page **pages) -{ - int nr, ret; - - might_sleep(); - start &= PAGE_MASK; - nr = __get_user_pages_fast(start, nr_pages, write, pages); - if (nr == nr_pages) - return nr; - - /* Try to get the remaining pages with get_user_pages */ - start += nr << PAGE_SHIFT; - pages += nr; - ret = get_user_pages_unlocked(start, nr_pages - nr, pages, - write ? FOLL_WRITE : 0); - /* Have to be a bit careful with return values */ - if (nr > 0) - ret = (ret < 0) ? nr : ret + nr; - return ret; -} diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 3e82f66d5c61..7cf48eefec8f 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -49,6 +49,8 @@ unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); EXPORT_SYMBOL(zero_page_mask); +bool initmem_freed; + static void __init setup_zero_pages(void) { unsigned int order; @@ -148,6 +150,7 @@ void __init mem_init(void) void free_initmem(void) { + initmem_freed = true; __set_memory((unsigned long)_sinittext, (unsigned long)(_einittext - _sinittext) >> PAGE_SHIFT, SET_MEMORY_RW | SET_MEMORY_NX); diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index 97b3ee53852b..818deeb1ebc3 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -16,6 +16,7 @@ #include <linux/cpu.h> #include <asm/ctl_reg.h> #include <asm/io.h> +#include <asm/stacktrace.h> static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) { diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 8485d6dc2754..9ebd01219812 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -410,6 +410,7 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, return old; } +#ifdef CONFIG_PGSTE static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; @@ -427,6 +428,7 @@ static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr) pmd = pmd_alloc(mm, pud, addr); return pmd; } +#endif pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t new) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 0472e27febdf..b403fa14847d 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -413,6 +413,8 @@ void __init vmem_map_init(void) __set_memory((unsigned long)_sinittext, (unsigned long)(_einittext - _sinittext) >> PAGE_SHIFT, SET_MEMORY_RO | SET_MEMORY_X); + __set_memory(__stext_dma, (__etext_dma - __stext_dma) >> PAGE_SHIFT, + SET_MEMORY_RO | SET_MEMORY_X); pr_info("Write protected kernel read-only data: %luk\n", (unsigned long)(__end_rodata - _stext) >> 10); } diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 51dd0267d014..5e7c63033159 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -455,7 +455,7 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) EMIT4(0xb9040000, REG_2, BPF_REG_0); /* Restore registers */ save_restore_regs(jit, REGS_RESTORE, stack_depth); - if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) { + if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) { jit->r14_thunk_ip = jit->prg; /* Generate __s390_indirect_jump_r14 thunk */ if (test_facility(35)) { @@ -473,7 +473,7 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth) /* br %r14 */ _EMIT2(0x07fe); - if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable && + if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable && (jit->seen & SEEN_FUNC)) { jit->r1_thunk_ip = jit->prg; /* Generate __s390_indirect_jump_r1 thunk */ @@ -999,7 +999,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i /* lg %w1,<d(imm)>(%l) */ EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L, EMIT_CONST_U64(func)); - if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) { + if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) { /* brasl %r14,__s390_indirect_jump_r1 */ EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip); } else { diff --git a/arch/s390/oprofile/init.c b/arch/s390/oprofile/init.c index 43d9525c36fc..7441857df51b 100644 --- a/arch/s390/oprofile/init.c +++ b/arch/s390/oprofile/init.c @@ -13,23 +13,17 @@ #include <linux/oprofile.h> #include <linux/init.h> #include <asm/processor.h> - -static int __s390_backtrace(void *data, unsigned long address, int reliable) -{ - unsigned int *depth = data; - - if (*depth == 0) - return 1; - (*depth)--; - oprofile_add_trace(address); - return 0; -} +#include <asm/unwind.h> static void s390_backtrace(struct pt_regs *regs, unsigned int depth) { - if (user_mode(regs)) - return; - dump_trace(__s390_backtrace, &depth, NULL, regs->gprs[15]); + struct unwind_state state; + + unwind_for_each_frame(&state, current, regs, 0) { + if (depth-- == 0) + break; + oprofile_add_trace(state.ip); + } } int __init oprofile_arch_init(struct oprofile_operations *ops) diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile index 22d0871291ee..748626a33028 100644 --- a/arch/s390/pci/Makefile +++ b/arch/s390/pci/Makefile @@ -3,5 +3,5 @@ # Makefile for the s390 PCI subsystem. # -obj-$(CONFIG_PCI) += pci.o pci_dma.o pci_clp.o pci_sysfs.o \ +obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_dma.o pci_clp.o pci_sysfs.o \ pci_event.o pci_debug.o pci_insn.o pci_mmio.o diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index dc9bc82c072c..0ebb7c405a25 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -24,11 +24,9 @@ #include <linux/err.h> #include <linux/export.h> #include <linux/delay.h> -#include <linux/irq.h> -#include <linux/kernel_stat.h> #include <linux/seq_file.h> +#include <linux/jump_label.h> #include <linux/pci.h> -#include <linux/msi.h> #include <asm/isc.h> #include <asm/airq.h> @@ -37,30 +35,13 @@ #include <asm/pci_clp.h> #include <asm/pci_dma.h> -#define DEBUG /* enable pr_debug */ - -#define SIC_IRQ_MODE_ALL 0 -#define SIC_IRQ_MODE_SINGLE 1 - -#define ZPCI_NR_DMA_SPACES 1 -#define ZPCI_NR_DEVICES CONFIG_PCI_NR_FUNCTIONS - /* list of all detected zpci devices */ static LIST_HEAD(zpci_list); static DEFINE_SPINLOCK(zpci_list_lock); -static struct irq_chip zpci_irq_chip = { - .name = "zPCI", - .irq_unmask = pci_msi_unmask_irq, - .irq_mask = pci_msi_mask_irq, -}; - static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES); static DEFINE_SPINLOCK(zpci_domain_lock); -static struct airq_iv *zpci_aisb_iv; -static struct airq_iv *zpci_aibv[ZPCI_NR_DEVICES]; - #define ZPCI_IOMAP_ENTRIES \ min(((unsigned long) ZPCI_NR_DEVICES * PCI_BAR_COUNT / 2), \ ZPCI_IOMAP_MAX_ENTRIES) @@ -70,6 +51,8 @@ static unsigned long *zpci_iomap_bitmap; struct zpci_iomap_entry *zpci_iomap_start; EXPORT_SYMBOL_GPL(zpci_iomap_start); +DEFINE_STATIC_KEY_FALSE(have_mio); + static struct kmem_cache *zdev_fmb_cache; struct zpci_dev *get_zdev_by_fid(u32 fid) @@ -123,39 +106,6 @@ int pci_proc_domain(struct pci_bus *bus) } EXPORT_SYMBOL_GPL(pci_proc_domain); -/* Modify PCI: Register adapter interruptions */ -static int zpci_set_airq(struct zpci_dev *zdev) -{ - u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); - struct zpci_fib fib = {0}; - u8 status; - - fib.isc = PCI_ISC; - fib.sum = 1; /* enable summary notifications */ - fib.noi = airq_iv_end(zdev->aibv); - fib.aibv = (unsigned long) zdev->aibv->vector; - fib.aibvo = 0; /* each zdev has its own interrupt vector */ - fib.aisb = (unsigned long) zpci_aisb_iv->vector + (zdev->aisb/64)*8; - fib.aisbo = zdev->aisb & 63; - - return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; -} - -/* Modify PCI: Unregister adapter interruptions */ -static int zpci_clear_airq(struct zpci_dev *zdev) -{ - u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT); - struct zpci_fib fib = {0}; - u8 cc, status; - - cc = zpci_mod_fc(req, &fib, &status); - if (cc == 3 || (cc == 1 && status == 24)) - /* Function already gone or IRQs already deregistered. */ - cc = 0; - - return cc ? -EIO : 0; -} - /* Modify PCI: Register I/O address translation parameters */ int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, u64 base, u64 limit, u64 iota) @@ -241,7 +191,7 @@ static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) u64 data; int rc; - rc = zpci_load(&data, req, offset); + rc = __zpci_load(&data, req, offset); if (!rc) { data = le64_to_cpu((__force __le64) data); data >>= (8 - len) * 8; @@ -259,7 +209,7 @@ static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) data <<= (8 - len) * 8; data = (__force u64) cpu_to_le64(data); - rc = zpci_store(data, req, offset); + rc = __zpci_store(data, req, offset); return rc; } @@ -276,18 +226,48 @@ void __iowrite64_copy(void __iomem *to, const void *from, size_t count) zpci_memcpy_toio(to, from, count); } +void __iomem *ioremap(unsigned long ioaddr, unsigned long size) +{ + struct vm_struct *area; + unsigned long offset; + + if (!size) + return NULL; + + if (!static_branch_unlikely(&have_mio)) + return (void __iomem *) ioaddr; + + offset = ioaddr & ~PAGE_MASK; + ioaddr &= PAGE_MASK; + size = PAGE_ALIGN(size + offset); + area = get_vm_area(size, VM_IOREMAP); + if (!area) + return NULL; + + if (ioremap_page_range((unsigned long) area->addr, + (unsigned long) area->addr + size, + ioaddr, PAGE_KERNEL)) { + vunmap(area->addr); + return NULL; + } + return (void __iomem *) ((unsigned long) area->addr + offset); +} +EXPORT_SYMBOL(ioremap); + +void iounmap(volatile void __iomem *addr) +{ + if (static_branch_likely(&have_mio)) + vunmap((__force void *) ((unsigned long) addr & PAGE_MASK)); +} +EXPORT_SYMBOL(iounmap); + /* Create a virtual mapping cookie for a PCI BAR */ -void __iomem *pci_iomap_range(struct pci_dev *pdev, - int bar, - unsigned long offset, - unsigned long max) +static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, + unsigned long offset, unsigned long max) { struct zpci_dev *zdev = to_zpci(pdev); int idx; - if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) - return NULL; - idx = zdev->bars[bar].map_idx; spin_lock(&zpci_iomap_lock); /* Detect overrun */ @@ -298,6 +278,30 @@ void __iomem *pci_iomap_range(struct pci_dev *pdev, return (void __iomem *) ZPCI_ADDR(idx) + offset; } + +static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, + unsigned long offset, + unsigned long max) +{ + unsigned long barsize = pci_resource_len(pdev, bar); + struct zpci_dev *zdev = to_zpci(pdev); + void __iomem *iova; + + iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); + return iova ? iova + offset : iova; +} + +void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, + unsigned long offset, unsigned long max) +{ + if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) + return NULL; + + if (static_branch_likely(&have_mio)) + return pci_iomap_range_mio(pdev, bar, offset, max); + else + return pci_iomap_range_fh(pdev, bar, offset, max); +} EXPORT_SYMBOL(pci_iomap_range); void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) @@ -306,7 +310,37 @@ void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) } EXPORT_SYMBOL(pci_iomap); -void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) +static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, + unsigned long offset, unsigned long max) +{ + unsigned long barsize = pci_resource_len(pdev, bar); + struct zpci_dev *zdev = to_zpci(pdev); + void __iomem *iova; + + iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); + return iova ? iova + offset : iova; +} + +void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, + unsigned long offset, unsigned long max) +{ + if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT) + return NULL; + + if (static_branch_likely(&have_mio)) + return pci_iomap_wc_range_mio(pdev, bar, offset, max); + else + return pci_iomap_range_fh(pdev, bar, offset, max); +} +EXPORT_SYMBOL(pci_iomap_wc_range); + +void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) +{ + return pci_iomap_wc_range(dev, bar, 0, maxlen); +} +EXPORT_SYMBOL(pci_iomap_wc); + +static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) { unsigned int idx = ZPCI_IDX(addr); @@ -319,6 +353,19 @@ void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) } spin_unlock(&zpci_iomap_lock); } + +static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) +{ + iounmap(addr); +} + +void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) +{ + if (static_branch_likely(&have_mio)) + pci_iounmap_mio(pdev, addr); + else + pci_iounmap_fh(pdev, addr); +} EXPORT_SYMBOL(pci_iounmap); static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, @@ -354,136 +401,6 @@ static struct pci_ops pci_root_ops = { .write = pci_write, }; -static void zpci_irq_handler(struct airq_struct *airq) -{ - unsigned long si, ai; - struct airq_iv *aibv; - int irqs_on = 0; - - inc_irq_stat(IRQIO_PCI); - for (si = 0;;) { - /* Scan adapter summary indicator bit vector */ - si = airq_iv_scan(zpci_aisb_iv, si, airq_iv_end(zpci_aisb_iv)); - if (si == -1UL) { - if (irqs_on++) - /* End of second scan with interrupts on. */ - break; - /* First scan complete, reenable interrupts. */ - if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC)) - break; - si = 0; - continue; - } - - /* Scan the adapter interrupt vector for this device. */ - aibv = zpci_aibv[si]; - for (ai = 0;;) { - ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); - if (ai == -1UL) - break; - inc_irq_stat(IRQIO_MSI); - airq_iv_lock(aibv, ai); - generic_handle_irq(airq_iv_get_data(aibv, ai)); - airq_iv_unlock(aibv, ai); - } - } -} - -int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) -{ - struct zpci_dev *zdev = to_zpci(pdev); - unsigned int hwirq, msi_vecs; - unsigned long aisb; - struct msi_desc *msi; - struct msi_msg msg; - int rc, irq; - - zdev->aisb = -1UL; - if (type == PCI_CAP_ID_MSI && nvec > 1) - return 1; - msi_vecs = min_t(unsigned int, nvec, zdev->max_msi); - - /* Allocate adapter summary indicator bit */ - aisb = airq_iv_alloc_bit(zpci_aisb_iv); - if (aisb == -1UL) - return -EIO; - zdev->aisb = aisb; - - /* Create adapter interrupt vector */ - zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK); - if (!zdev->aibv) - return -ENOMEM; - - /* Wire up shortcut pointer */ - zpci_aibv[aisb] = zdev->aibv; - - /* Request MSI interrupts */ - hwirq = 0; - for_each_pci_msi_entry(msi, pdev) { - if (hwirq >= msi_vecs) - break; - irq = irq_alloc_desc(0); /* Alloc irq on node 0 */ - if (irq < 0) - return -ENOMEM; - rc = irq_set_msi_desc(irq, msi); - if (rc) - return rc; - irq_set_chip_and_handler(irq, &zpci_irq_chip, - handle_simple_irq); - msg.data = hwirq; - msg.address_lo = zdev->msi_addr & 0xffffffff; - msg.address_hi = zdev->msi_addr >> 32; - pci_write_msi_msg(irq, &msg); - airq_iv_set_data(zdev->aibv, hwirq, irq); - hwirq++; - } - - /* Enable adapter interrupts */ - rc = zpci_set_airq(zdev); - if (rc) - return rc; - - return (msi_vecs == nvec) ? 0 : msi_vecs; -} - -void arch_teardown_msi_irqs(struct pci_dev *pdev) -{ - struct zpci_dev *zdev = to_zpci(pdev); - struct msi_desc *msi; - int rc; - - /* Disable adapter interrupts */ - rc = zpci_clear_airq(zdev); - if (rc) - return; - - /* Release MSI interrupts */ - for_each_pci_msi_entry(msi, pdev) { - if (!msi->irq) - continue; - if (msi->msi_attrib.is_msix) - __pci_msix_desc_mask_irq(msi, 1); - else - __pci_msi_desc_mask_irq(msi, 1, 1); - irq_set_msi_desc(msi->irq, NULL); - irq_free_desc(msi->irq); - msi->msg.address_lo = 0; - msi->msg.address_hi = 0; - msi->msg.data = 0; - msi->irq = 0; - } - - if (zdev->aisb != -1UL) { - zpci_aibv[zdev->aisb] = NULL; - airq_iv_free_bit(zpci_aisb_iv, zdev->aisb); - zdev->aisb = -1UL; - } - if (zdev->aibv) { - airq_iv_release(zdev->aibv); - zdev->aibv = NULL; - } -} - #ifdef CONFIG_PCI_IOV static struct resource iov_res = { .name = "PCI IOV res", @@ -495,6 +412,7 @@ static struct resource iov_res = { static void zpci_map_resources(struct pci_dev *pdev) { + struct zpci_dev *zdev = to_zpci(pdev); resource_size_t len; int i; @@ -502,8 +420,13 @@ static void zpci_map_resources(struct pci_dev *pdev) len = pci_resource_len(pdev, i); if (!len) continue; - pdev->resource[i].start = - (resource_size_t __force) pci_iomap(pdev, i, 0); + + if (static_branch_likely(&have_mio)) + pdev->resource[i].start = + (resource_size_t __force) zdev->bars[i].mio_wb; + else + pdev->resource[i].start = + (resource_size_t __force) pci_iomap(pdev, i, 0); pdev->resource[i].end = pdev->resource[i].start + len - 1; } @@ -524,6 +447,9 @@ static void zpci_unmap_resources(struct pci_dev *pdev) resource_size_t len; int i; + if (static_branch_likely(&have_mio)) + return; + for (i = 0; i < PCI_BAR_COUNT; i++) { len = pci_resource_len(pdev, i); if (!len) @@ -533,41 +459,6 @@ static void zpci_unmap_resources(struct pci_dev *pdev) } } -static struct airq_struct zpci_airq = { - .handler = zpci_irq_handler, - .isc = PCI_ISC, -}; - -static int __init zpci_irq_init(void) -{ - int rc; - - rc = register_adapter_interrupt(&zpci_airq); - if (rc) - goto out; - /* Set summary to 1 to be called every time for the ISC. */ - *zpci_airq.lsi_ptr = 1; - - rc = -ENOMEM; - zpci_aisb_iv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC); - if (!zpci_aisb_iv) - goto out_airq; - - zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, NULL, PCI_ISC); - return 0; - -out_airq: - unregister_adapter_interrupt(&zpci_airq); -out: - return rc; -} - -static void zpci_irq_exit(void) -{ - airq_iv_release(zpci_aisb_iv); - unregister_adapter_interrupt(&zpci_airq); -} - static int zpci_alloc_iomap(struct zpci_dev *zdev) { unsigned long entry; @@ -958,7 +849,9 @@ static void zpci_mem_exit(void) kmem_cache_destroy(zdev_fmb_cache); } -static unsigned int s390_pci_probe = 1; +static unsigned int s390_pci_probe __initdata = 1; +static unsigned int s390_pci_no_mio __initdata; +unsigned int s390_pci_force_floating __initdata; static unsigned int s390_pci_initialized; char * __init pcibios_setup(char *str) @@ -967,6 +860,14 @@ char * __init pcibios_setup(char *str) s390_pci_probe = 0; return NULL; } + if (!strcmp(str, "nomio")) { + s390_pci_no_mio = 1; + return NULL; + } + if (!strcmp(str, "force_floating")) { + s390_pci_force_floating = 1; + return NULL; + } return str; } @@ -985,6 +886,9 @@ static int __init pci_base_init(void) if (!test_facility(69) || !test_facility(71)) return 0; + if (test_facility(153) && !s390_pci_no_mio) + static_branch_enable(&have_mio); + rc = zpci_debug_init(); if (rc) goto out; diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index eeb7450db18c..3a36b07a5571 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -163,7 +163,14 @@ static int clp_store_query_pci_fn(struct zpci_dev *zdev, memcpy(zdev->util_str, response->util_str, sizeof(zdev->util_str)); } + zdev->mio_capable = response->mio_addr_avail; + for (i = 0; i < PCI_BAR_COUNT; i++) { + if (!(response->mio_valid & (1 << (PCI_BAR_COUNT - i - 1)))) + continue; + zdev->bars[i].mio_wb = (void __iomem *) response->addr[i].wb; + zdev->bars[i].mio_wt = (void __iomem *) response->addr[i].wt; + } return 0; } @@ -279,11 +286,18 @@ int clp_enable_fh(struct zpci_dev *zdev, u8 nr_dma_as) int rc; rc = clp_set_pci_fn(&fh, nr_dma_as, CLP_SET_ENABLE_PCI_FN); - if (!rc) - /* Success -> store enabled handle in zdev */ - zdev->fh = fh; + zpci_dbg(3, "ena fid:%x, fh:%x, rc:%d\n", zdev->fid, fh, rc); + if (rc) + goto out; - zpci_dbg(3, "ena fid:%x, fh:%x, rc:%d\n", zdev->fid, zdev->fh, rc); + zdev->fh = fh; + if (zdev->mio_capable) { + rc = clp_set_pci_fn(&fh, nr_dma_as, CLP_SET_ENABLE_MIO); + zpci_dbg(3, "ena mio fid:%x, fh:%x, rc:%d\n", zdev->fid, fh, rc); + if (rc) + clp_disable_fh(zdev); + } +out: return rc; } @@ -296,11 +310,10 @@ int clp_disable_fh(struct zpci_dev *zdev) return 0; rc = clp_set_pci_fn(&fh, 0, CLP_SET_DISABLE_PCI_FN); + zpci_dbg(3, "dis fid:%x, fh:%x, rc:%d\n", zdev->fid, fh, rc); if (!rc) - /* Success -> store disabled handle in zdev */ zdev->fh = fh; - zpci_dbg(3, "dis fid:%x, fh:%x, rc:%d\n", zdev->fid, zdev->fh, rc); return rc; } diff --git a/arch/s390/pci/pci_insn.c b/arch/s390/pci/pci_insn.c index f069929e8211..02f9505c99a8 100644 --- a/arch/s390/pci/pci_insn.c +++ b/arch/s390/pci/pci_insn.c @@ -8,9 +8,11 @@ #include <linux/export.h> #include <linux/errno.h> #include <linux/delay.h> +#include <linux/jump_label.h> #include <asm/facility.h> #include <asm/pci_insn.h> #include <asm/pci_debug.h> +#include <asm/pci_io.h> #include <asm/processor.h> #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ @@ -96,13 +98,15 @@ int zpci_refresh_trans(u64 fn, u64 addr, u64 range) } /* Set Interruption Controls */ -int zpci_set_irq_ctrl(u16 ctl, char *unused, u8 isc) +int __zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib) { if (!test_facility(72)) return -EIO; - asm volatile ( - " .insn rsy,0xeb00000000d1,%[ctl],%[isc],%[u]\n" - : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [u] "Q" (*unused)); + + asm volatile( + ".insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]\n" + : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib)); + return 0; } @@ -140,7 +144,7 @@ static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) return cc; } -int zpci_load(u64 *data, u64 req, u64 offset) +int __zpci_load(u64 *data, u64 req, u64 offset) { u8 status; int cc; @@ -156,6 +160,52 @@ int zpci_load(u64 *data, u64 req, u64 offset) return (cc > 0) ? -EIO : cc; } +EXPORT_SYMBOL_GPL(__zpci_load); + +static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr, + unsigned long len) +{ + struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; + u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); + + return __zpci_load(data, req, ZPCI_OFFSET(addr)); +} + +static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) +{ + register u64 addr asm("2") = ioaddr; + register u64 r3 asm("3") = len; + int cc = -ENXIO; + u64 __data; + + asm volatile ( + " .insn rre,0xb9d60000,%[data],%[ioaddr]\n" + "0: ipm %[cc]\n" + " srl %[cc],28\n" + "1:\n" + EX_TABLE(0b, 1b) + : [cc] "+d" (cc), [data] "=d" (__data), "+d" (r3) + : [ioaddr] "d" (addr) + : "cc"); + *status = r3 >> 24 & 0xff; + *data = __data; + return cc; +} + +int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len) +{ + u8 status; + int cc; + + if (!static_branch_unlikely(&have_mio)) + return zpci_load_fh(data, addr, len); + + cc = __pcilg_mio(data, (__force u64) addr, len, &status); + if (cc) + zpci_err_insn(cc, status, 0, (__force u64) addr); + + return (cc > 0) ? -EIO : cc; +} EXPORT_SYMBOL_GPL(zpci_load); /* PCI Store */ @@ -178,7 +228,7 @@ static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) return cc; } -int zpci_store(u64 data, u64 req, u64 offset) +int __zpci_store(u64 data, u64 req, u64 offset) { u8 status; int cc; @@ -194,6 +244,50 @@ int zpci_store(u64 data, u64 req, u64 offset) return (cc > 0) ? -EIO : cc; } +EXPORT_SYMBOL_GPL(__zpci_store); + +static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data, + unsigned long len) +{ + struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; + u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); + + return __zpci_store(data, req, ZPCI_OFFSET(addr)); +} + +static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) +{ + register u64 addr asm("2") = ioaddr; + register u64 r3 asm("3") = len; + int cc = -ENXIO; + + asm volatile ( + " .insn rre,0xb9d40000,%[data],%[ioaddr]\n" + "0: ipm %[cc]\n" + " srl %[cc],28\n" + "1:\n" + EX_TABLE(0b, 1b) + : [cc] "+d" (cc), "+d" (r3) + : [data] "d" (data), [ioaddr] "d" (addr) + : "cc"); + *status = r3 >> 24 & 0xff; + return cc; +} + +int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len) +{ + u8 status; + int cc; + + if (!static_branch_unlikely(&have_mio)) + return zpci_store_fh(addr, data, len); + + cc = __pcistg_mio(data, (__force u64) addr, len, &status); + if (cc) + zpci_err_insn(cc, status, 0, (__force u64) addr); + + return (cc > 0) ? -EIO : cc; +} EXPORT_SYMBOL_GPL(zpci_store); /* PCI Store Block */ @@ -214,7 +308,7 @@ static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) return cc; } -int zpci_store_block(const u64 *data, u64 req, u64 offset) +int __zpci_store_block(const u64 *data, u64 req, u64 offset) { u8 status; int cc; @@ -230,4 +324,63 @@ int zpci_store_block(const u64 *data, u64 req, u64 offset) return (cc > 0) ? -EIO : cc; } -EXPORT_SYMBOL_GPL(zpci_store_block); +EXPORT_SYMBOL_GPL(__zpci_store_block); + +static inline int zpci_write_block_fh(volatile void __iomem *dst, + const void *src, unsigned long len) +{ + struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)]; + u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); + u64 offset = ZPCI_OFFSET(dst); + + return __zpci_store_block(src, req, offset); +} + +static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) +{ + int cc = -ENXIO; + + asm volatile ( + " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" + "0: ipm %[cc]\n" + " srl %[cc],28\n" + "1:\n" + EX_TABLE(0b, 1b) + : [cc] "+d" (cc), [len] "+d" (len) + : [ioaddr] "d" (ioaddr), [data] "Q" (*data) + : "cc"); + *status = len >> 24 & 0xff; + return cc; +} + +int zpci_write_block(volatile void __iomem *dst, + const void *src, unsigned long len) +{ + u8 status; + int cc; + + if (!static_branch_unlikely(&have_mio)) + return zpci_write_block_fh(dst, src, len); + + cc = __pcistb_mio(src, (__force u64) dst, len, &status); + if (cc) + zpci_err_insn(cc, status, 0, (__force u64) dst); + + return (cc > 0) ? -EIO : cc; +} +EXPORT_SYMBOL_GPL(zpci_write_block); + +static inline void __pciwb_mio(void) +{ + unsigned long unused = 0; + + asm volatile (".insn rre,0xb9d50000,%[op],%[op]\n" + : [op] "+d" (unused)); +} + +void zpci_barrier(void) +{ + if (static_branch_likely(&have_mio)) + __pciwb_mio(); +} +EXPORT_SYMBOL_GPL(zpci_barrier); diff --git a/arch/s390/pci/pci_irq.c b/arch/s390/pci/pci_irq.c new file mode 100644 index 000000000000..d80616ae8dd8 --- /dev/null +++ b/arch/s390/pci/pci_irq.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-2.0 +#define KMSG_COMPONENT "zpci" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include <linux/kernel.h> +#include <linux/irq.h> +#include <linux/kernel_stat.h> +#include <linux/pci.h> +#include <linux/msi.h> +#include <linux/smp.h> + +#include <asm/isc.h> +#include <asm/airq.h> + +static enum {FLOATING, DIRECTED} irq_delivery; + +#define SIC_IRQ_MODE_ALL 0 +#define SIC_IRQ_MODE_SINGLE 1 +#define SIC_IRQ_MODE_DIRECT 4 +#define SIC_IRQ_MODE_D_ALL 16 +#define SIC_IRQ_MODE_D_SINGLE 17 +#define SIC_IRQ_MODE_SET_CPU 18 + +/* + * summary bit vector + * FLOATING - summary bit per function + * DIRECTED - summary bit per cpu (only used in fallback path) + */ +static struct airq_iv *zpci_sbv; + +/* + * interrupt bit vectors + * FLOATING - interrupt bit vector per function + * DIRECTED - interrupt bit vector per cpu + */ +static struct airq_iv **zpci_ibv; + +/* Modify PCI: Register adapter interruptions */ +static int zpci_set_airq(struct zpci_dev *zdev) +{ + u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); + struct zpci_fib fib = {0}; + u8 status; + + fib.fmt0.isc = PCI_ISC; + fib.fmt0.sum = 1; /* enable summary notifications */ + fib.fmt0.noi = airq_iv_end(zdev->aibv); + fib.fmt0.aibv = (unsigned long) zdev->aibv->vector; + fib.fmt0.aibvo = 0; /* each zdev has its own interrupt vector */ + fib.fmt0.aisb = (unsigned long) zpci_sbv->vector + (zdev->aisb/64)*8; + fib.fmt0.aisbo = zdev->aisb & 63; + + return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; +} + +/* Modify PCI: Unregister adapter interruptions */ +static int zpci_clear_airq(struct zpci_dev *zdev) +{ + u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT); + struct zpci_fib fib = {0}; + u8 cc, status; + + cc = zpci_mod_fc(req, &fib, &status); + if (cc == 3 || (cc == 1 && status == 24)) + /* Function already gone or IRQs already deregistered. */ + cc = 0; + + return cc ? -EIO : 0; +} + +/* Modify PCI: Register CPU directed interruptions */ +static int zpci_set_directed_irq(struct zpci_dev *zdev) +{ + u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT_D); + struct zpci_fib fib = {0}; + u8 status; + + fib.fmt = 1; + fib.fmt1.noi = zdev->msi_nr_irqs; + fib.fmt1.dibvo = zdev->msi_first_bit; + + return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; +} + +/* Modify PCI: Unregister CPU directed interruptions */ +static int zpci_clear_directed_irq(struct zpci_dev *zdev) +{ + u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT_D); + struct zpci_fib fib = {0}; + u8 cc, status; + + fib.fmt = 1; + cc = zpci_mod_fc(req, &fib, &status); + if (cc == 3 || (cc == 1 && status == 24)) + /* Function already gone or IRQs already deregistered. */ + cc = 0; + + return cc ? -EIO : 0; +} + +static int zpci_set_irq_affinity(struct irq_data *data, const struct cpumask *dest, + bool force) +{ + struct msi_desc *entry = irq_get_msi_desc(data->irq); + struct msi_msg msg = entry->msg; + + msg.address_lo &= 0xff0000ff; + msg.address_lo |= (cpumask_first(dest) << 8); + pci_write_msi_msg(data->irq, &msg); + + return IRQ_SET_MASK_OK; +} + +static struct irq_chip zpci_irq_chip = { + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_set_affinity = zpci_set_irq_affinity, +}; + +static void zpci_handle_cpu_local_irq(bool rescan) +{ + struct airq_iv *dibv = zpci_ibv[smp_processor_id()]; + unsigned long bit; + int irqs_on = 0; + + for (bit = 0;;) { + /* Scan the directed IRQ bit vector */ + bit = airq_iv_scan(dibv, bit, airq_iv_end(dibv)); + if (bit == -1UL) { + if (!rescan || irqs_on++) + /* End of second scan with interrupts on. */ + break; + /* First scan complete, reenable interrupts. */ + if (zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC)) + break; + bit = 0; + continue; + } + inc_irq_stat(IRQIO_MSI); + generic_handle_irq(airq_iv_get_data(dibv, bit)); + } +} + +struct cpu_irq_data { + call_single_data_t csd; + atomic_t scheduled; +}; +static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_irq_data, irq_data); + +static void zpci_handle_remote_irq(void *data) +{ + atomic_t *scheduled = data; + + do { + zpci_handle_cpu_local_irq(false); + } while (atomic_dec_return(scheduled)); +} + +static void zpci_handle_fallback_irq(void) +{ + struct cpu_irq_data *cpu_data; + unsigned long cpu; + int irqs_on = 0; + + for (cpu = 0;;) { + cpu = airq_iv_scan(zpci_sbv, cpu, airq_iv_end(zpci_sbv)); + if (cpu == -1UL) { + if (irqs_on++) + /* End of second scan with interrupts on. */ + break; + /* First scan complete, reenable interrupts. */ + if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC)) + break; + cpu = 0; + continue; + } + cpu_data = &per_cpu(irq_data, cpu); + if (atomic_inc_return(&cpu_data->scheduled) > 1) + continue; + + cpu_data->csd.func = zpci_handle_remote_irq; + cpu_data->csd.info = &cpu_data->scheduled; + cpu_data->csd.flags = 0; + smp_call_function_single_async(cpu, &cpu_data->csd); + } +} + +static void zpci_directed_irq_handler(struct airq_struct *airq, bool floating) +{ + if (floating) { + inc_irq_stat(IRQIO_PCF); + zpci_handle_fallback_irq(); + } else { + inc_irq_stat(IRQIO_PCD); + zpci_handle_cpu_local_irq(true); + } +} + +static void zpci_floating_irq_handler(struct airq_struct *airq, bool floating) +{ + unsigned long si, ai; + struct airq_iv *aibv; + int irqs_on = 0; + + inc_irq_stat(IRQIO_PCF); + for (si = 0;;) { + /* Scan adapter summary indicator bit vector */ + si = airq_iv_scan(zpci_sbv, si, airq_iv_end(zpci_sbv)); + if (si == -1UL) { + if (irqs_on++) + /* End of second scan with interrupts on. */ + break; + /* First scan complete, reenable interrupts. */ + if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC)) + break; + si = 0; + continue; + } + + /* Scan the adapter interrupt vector for this device. */ + aibv = zpci_ibv[si]; + for (ai = 0;;) { + ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); + if (ai == -1UL) + break; + inc_irq_stat(IRQIO_MSI); + airq_iv_lock(aibv, ai); + generic_handle_irq(airq_iv_get_data(aibv, ai)); + airq_iv_unlock(aibv, ai); + } + } +} + +int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + struct zpci_dev *zdev = to_zpci(pdev); + unsigned int hwirq, msi_vecs, cpu; + unsigned long bit; + struct msi_desc *msi; + struct msi_msg msg; + int rc, irq; + + zdev->aisb = -1UL; + zdev->msi_first_bit = -1U; + if (type == PCI_CAP_ID_MSI && nvec > 1) + return 1; + msi_vecs = min_t(unsigned int, nvec, zdev->max_msi); + + if (irq_delivery == DIRECTED) { + /* Allocate cpu vector bits */ + bit = airq_iv_alloc(zpci_ibv[0], msi_vecs); + if (bit == -1UL) + return -EIO; + } else { + /* Allocate adapter summary indicator bit */ + bit = airq_iv_alloc_bit(zpci_sbv); + if (bit == -1UL) + return -EIO; + zdev->aisb = bit; + + /* Create adapter interrupt vector */ + zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK); + if (!zdev->aibv) + return -ENOMEM; + + /* Wire up shortcut pointer */ + zpci_ibv[bit] = zdev->aibv; + /* Each function has its own interrupt vector */ + bit = 0; + } + + /* Request MSI interrupts */ + hwirq = bit; + for_each_pci_msi_entry(msi, pdev) { + rc = -EIO; + if (hwirq - bit >= msi_vecs) + break; + irq = __irq_alloc_descs(-1, 0, 1, 0, THIS_MODULE, msi->affinity); + if (irq < 0) + return -ENOMEM; + rc = irq_set_msi_desc(irq, msi); + if (rc) + return rc; + irq_set_chip_and_handler(irq, &zpci_irq_chip, + handle_percpu_irq); + msg.data = hwirq; + if (irq_delivery == DIRECTED) { + msg.address_lo = zdev->msi_addr & 0xff0000ff; + msg.address_lo |= msi->affinity ? + (cpumask_first(&msi->affinity->mask) << 8) : 0; + for_each_possible_cpu(cpu) { + airq_iv_set_data(zpci_ibv[cpu], hwirq, irq); + } + } else { + msg.address_lo = zdev->msi_addr & 0xffffffff; + airq_iv_set_data(zdev->aibv, hwirq, irq); + } + msg.address_hi = zdev->msi_addr >> 32; + pci_write_msi_msg(irq, &msg); + hwirq++; + } + + zdev->msi_first_bit = bit; + zdev->msi_nr_irqs = msi_vecs; + + if (irq_delivery == DIRECTED) + rc = zpci_set_directed_irq(zdev); + else + rc = zpci_set_airq(zdev); + if (rc) + return rc; + + return (msi_vecs == nvec) ? 0 : msi_vecs; +} + +void arch_teardown_msi_irqs(struct pci_dev *pdev) +{ + struct zpci_dev *zdev = to_zpci(pdev); + struct msi_desc *msi; + int rc; + + /* Disable interrupts */ + if (irq_delivery == DIRECTED) + rc = zpci_clear_directed_irq(zdev); + else + rc = zpci_clear_airq(zdev); + if (rc) + return; + + /* Release MSI interrupts */ + for_each_pci_msi_entry(msi, pdev) { + if (!msi->irq) + continue; + if (msi->msi_attrib.is_msix) + __pci_msix_desc_mask_irq(msi, 1); + else + __pci_msi_desc_mask_irq(msi, 1, 1); + irq_set_msi_desc(msi->irq, NULL); + irq_free_desc(msi->irq); + msi->msg.address_lo = 0; + msi->msg.address_hi = 0; + msi->msg.data = 0; + msi->irq = 0; + } + + if (zdev->aisb != -1UL) { + zpci_ibv[zdev->aisb] = NULL; + airq_iv_free_bit(zpci_sbv, zdev->aisb); + zdev->aisb = -1UL; + } + if (zdev->aibv) { + airq_iv_release(zdev->aibv); + zdev->aibv = NULL; + } + + if ((irq_delivery == DIRECTED) && zdev->msi_first_bit != -1U) + airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); +} + +static struct airq_struct zpci_airq = { + .handler = zpci_floating_irq_handler, + .isc = PCI_ISC, +}; + +static void __init cpu_enable_directed_irq(void *unused) +{ + union zpci_sic_iib iib = {{0}}; + + iib.cdiib.dibv_addr = (u64) zpci_ibv[smp_processor_id()]->vector; + + __zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib); + zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC); +} + +static int __init zpci_directed_irq_init(void) +{ + union zpci_sic_iib iib = {{0}}; + unsigned int cpu; + + zpci_sbv = airq_iv_create(num_possible_cpus(), 0); + if (!zpci_sbv) + return -ENOMEM; + + iib.diib.isc = PCI_ISC; + iib.diib.nr_cpus = num_possible_cpus(); + iib.diib.disb_addr = (u64) zpci_sbv->vector; + __zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); + + zpci_ibv = kcalloc(num_possible_cpus(), sizeof(*zpci_ibv), + GFP_KERNEL); + if (!zpci_ibv) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + /* + * Per CPU IRQ vectors look the same but bit-allocation + * is only done on the first vector. + */ + zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, + AIRQ_IV_DATA | + AIRQ_IV_CACHELINE | + (!cpu ? AIRQ_IV_ALLOC : 0)); + if (!zpci_ibv[cpu]) + return -ENOMEM; + } + on_each_cpu(cpu_enable_directed_irq, NULL, 1); + + zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity; + + return 0; +} + +static int __init zpci_floating_irq_init(void) +{ + zpci_ibv = kcalloc(ZPCI_NR_DEVICES, sizeof(*zpci_ibv), GFP_KERNEL); + if (!zpci_ibv) + return -ENOMEM; + + zpci_sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC); + if (!zpci_sbv) + goto out_free; + + return 0; + +out_free: + kfree(zpci_ibv); + return -ENOMEM; +} + +int __init zpci_irq_init(void) +{ + int rc; + + irq_delivery = sclp.has_dirq ? DIRECTED : FLOATING; + if (s390_pci_force_floating) + irq_delivery = FLOATING; + + if (irq_delivery == DIRECTED) + zpci_airq.handler = zpci_directed_irq_handler; + + rc = register_adapter_interrupt(&zpci_airq); + if (rc) + goto out; + /* Set summary to 1 to be called every time for the ISC. */ + *zpci_airq.lsi_ptr = 1; + + switch (irq_delivery) { + case FLOATING: + rc = zpci_floating_irq_init(); + break; + case DIRECTED: + rc = zpci_directed_irq_init(); + break; + } + + if (rc) + goto out_airq; + + /* + * Enable floating IRQs (with suppression after one IRQ). When using + * directed IRQs this enables the fallback path. + */ + zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC); + + return 0; +out_airq: + unregister_adapter_interrupt(&zpci_airq); +out: + return rc; +} + +void __init zpci_irq_exit(void) +{ + unsigned int cpu; + + if (irq_delivery == DIRECTED) { + for_each_possible_cpu(cpu) { + airq_iv_release(zpci_ibv[cpu]); + } + } + kfree(zpci_ibv); + if (zpci_sbv) + airq_iv_release(zpci_sbv); + unregister_adapter_interrupt(&zpci_airq); +} diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index ce6a3f75065b..dc1ae4ff79d7 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -4,7 +4,7 @@ OBJECT_FILES_NON_STANDARD := y purgatory-y := head.o purgatory.o string.o sha256.o mem.o -targets += $(purgatory-y) purgatory.ro kexec-purgatory.c +targets += $(purgatory-y) purgatory.lds purgatory purgatory.ro PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) $(obj)/sha256.o: $(srctree)/lib/sha256.c FORCE @@ -16,22 +16,26 @@ $(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE $(obj)/string.o: $(srctree)/arch/s390/lib/string.c FORCE $(call if_changed_rule,cc_o_c) -LDFLAGS_purgatory.ro := -e purgatory_start -r --no-undefined -nostdlib -LDFLAGS_purgatory.ro += -z nodefaultlib KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding KBUILD_CFLAGS += -c -MD -Os -m64 -msoft-float -fno-common +KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) -$(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE +LDFLAGS_purgatory := -r --no-undefined -nostdlib -z nodefaultlib -T +$(obj)/purgatory: $(obj)/purgatory.lds $(PURGATORY_OBJS) FORCE $(call if_changed,ld) -quiet_cmd_bin2c = BIN2C $@ - cmd_bin2c = $(objtree)/scripts/bin2c kexec_purgatory < $< > $@ +OBJCOPYFLAGS_purgatory.ro := -O elf64-s390 +OBJCOPYFLAGS_purgatory.ro += --remove-section='*debug*' +OBJCOPYFLAGS_purgatory.ro += --remove-section='.comment' +OBJCOPYFLAGS_purgatory.ro += --remove-section='.note.*' +$(obj)/purgatory.ro: $(obj)/purgatory FORCE + $(call if_changed,objcopy) -$(obj)/kexec-purgatory.c: $(obj)/purgatory.ro FORCE - $(call if_changed,bin2c) +$(obj)/kexec-purgatory.o: $(obj)/kexec-purgatory.S $(obj)/purgatory.ro FORCE + $(call if_changed_rule,as_o_S) obj-$(CONFIG_ARCH_HAS_KEXEC_PURGATORY) += kexec-purgatory.o diff --git a/arch/s390/purgatory/kexec-purgatory.S b/arch/s390/purgatory/kexec-purgatory.S new file mode 100644 index 000000000000..8293753100ae --- /dev/null +++ b/arch/s390/purgatory/kexec-purgatory.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + + .section .rodata, "a" + + .align 8 +kexec_purgatory: + .globl kexec_purgatory + .incbin "arch/s390/purgatory/purgatory.ro" +.Lkexec_purgatroy_end: + + .align 8 +kexec_purgatory_size: + .globl kexec_purgatory_size + .quad .Lkexec_purgatroy_end - kexec_purgatory diff --git a/arch/s390/purgatory/purgatory.lds.S b/arch/s390/purgatory/purgatory.lds.S new file mode 100644 index 000000000000..482eb4fbcef1 --- /dev/null +++ b/arch/s390/purgatory/purgatory.lds.S @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <asm-generic/vmlinux.lds.h> + +OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") +OUTPUT_ARCH(s390:64-bit) + +ENTRY(purgatory_start) + +SECTIONS +{ + . = 0; + .head.text : { + _head = . ; + HEAD_TEXT + _ehead = . ; + } + .text : { + _text = .; /* Text */ + *(.text) + *(.text.*) + _etext = . ; + } + .rodata : { + _rodata = . ; + *(.rodata) /* read-only data */ + *(.rodata.*) + _erodata = . ; + } + .data : { + _data = . ; + *(.data) + *(.data.*) + _edata = . ; + } + + . = ALIGN(256); + .bss : { + _bss = . ; + *(.bss) + *(.bss.*) + *(COMMON) + . = ALIGN(8); /* For convenience during zeroing */ + _ebss = .; + } + _end = .; + + /* Sections to be discarded */ + /DISCARD/ : { + *(.eh_frame) + *(*__ksymtab*) + *(___kcrctab*) + } +} diff --git a/arch/s390/scripts/Makefile.chkbss b/arch/s390/scripts/Makefile.chkbss index cd7e8f4419f5..884a9caff5fb 100644 --- a/arch/s390/scripts/Makefile.chkbss +++ b/arch/s390/scripts/Makefile.chkbss @@ -11,7 +11,8 @@ chkbss: $(addprefix $(obj)/, $(chkbss-files)) quiet_cmd_chkbss = CHKBSS $< cmd_chkbss = \ - if ! $(OBJDUMP) -j .bss -w -h $< | awk 'END { if ($$3) exit 1 }'; then \ + if $(OBJDUMP) -h $< | grep -q "\.bss" && \ + ! $(OBJDUMP) -j .bss -w -h $< | awk 'END { if ($$3) exit 1 }'; then \ echo "error: $< .bss section is not empty" >&2; exit 1; \ fi; \ touch $@; diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index 1cbed82cd17b..64638b764d1c 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -1,3 +1,5 @@ +0000 illegal E +0002 brkpt E 0101 pr E 0102 upt E 0104 ptff E @@ -257,6 +259,7 @@ b258 bsg RRE_RR b25a bsa RRE_RR b25d clst RRE_RR b25e srst RRE_RR +b25f chsc RRE_R0 b263 cmpsc RRE_RR b274 siga S_RD b276 xsch S_00 @@ -277,6 +280,9 @@ b29d lfpc S_RD b2a5 tre RRE_RR b2a6 cu21 RRF_U0RR b2a7 cu12 RRF_U0RR +b2ad nqap RRE_RR +b2ae dqap RRE_RR +b2af pqap RRE_RR b2b0 stfle S_RD b2b1 stfl S_RD b2b2 lpswe S_RD @@ -290,6 +296,7 @@ b2e5 epctr RRE_RR b2e8 ppa RRF_U0RR b2ec etnd RRE_R0 b2ed ecpga RRE_RR +b2f0 iucv RRE_RR b2f8 tend S_00 b2fa niai IE_UU b2fc tabort S_RD @@ -559,12 +566,15 @@ b998 alcr RRE_RR b999 slbr RRE_RR b99a epair RRE_R0 b99b esair RRE_R0 +b99c eqbs RRF_U0RR b99d esea RRE_R0 b99e pti RRE_RR b99f ssair RRE_R0 +b9a0 clp RRF_U0RR b9a1 tpei RRE_RR b9a2 ptf RRE_R0 b9aa lptea RRF_RURR2 +b9ab essa RRF_U0RR b9ac irbm RRE_RR b9ae rrbm RRE_RR b9af pfmf RRE_RR @@ -1039,6 +1049,7 @@ eb7a agsi SIY_IRD eb7e algsi SIY_IRD eb80 icmh RSY_RURD eb81 icmy RSY_RURD +eb8a sqbs RSY_RDRU eb8e mvclu RSY_RRRD eb8f clclu RSY_RRRD eb90 stmy RSY_RRRD diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index e9aa71cdfc44..d2ab3f07c008 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -45,8 +45,8 @@ static struct list_head sclp_req_queue; /* Data for read and and init requests. */ static struct sclp_req sclp_read_req; static struct sclp_req sclp_init_req; -static char sclp_read_sccb[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); -static char sclp_init_sccb[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); +static void *sclp_read_sccb; +static struct init_sccb *sclp_init_sccb; /* Suspend request */ static DECLARE_COMPLETION(sclp_request_queue_flushed); @@ -753,9 +753,8 @@ EXPORT_SYMBOL(sclp_remove_processed); static inline void __sclp_make_init_req(sccb_mask_t receive_mask, sccb_mask_t send_mask) { - struct init_sccb *sccb; + struct init_sccb *sccb = sclp_init_sccb; - sccb = (struct init_sccb *) sclp_init_sccb; clear_page(sccb); memset(&sclp_init_req, 0, sizeof(struct sclp_req)); sclp_init_req.command = SCLP_CMDW_WRITE_EVENT_MASK; @@ -782,7 +781,7 @@ static int sclp_init_mask(int calculate) { unsigned long flags; - struct init_sccb *sccb = (struct init_sccb *) sclp_init_sccb; + struct init_sccb *sccb = sclp_init_sccb; sccb_mask_t receive_mask; sccb_mask_t send_mask; int retry; @@ -1175,6 +1174,9 @@ sclp_init(void) if (sclp_init_state != sclp_init_state_uninitialized) goto fail_unlock; sclp_init_state = sclp_init_state_initializing; + sclp_read_sccb = (void *) __get_free_page(GFP_ATOMIC | GFP_DMA); + sclp_init_sccb = (void *) __get_free_page(GFP_ATOMIC | GFP_DMA); + BUG_ON(!sclp_read_sccb || !sclp_init_sccb); /* Set up variables */ INIT_LIST_HEAD(&sclp_req_queue); INIT_LIST_HEAD(&sclp_reg_list); @@ -1207,6 +1209,8 @@ fail_unregister_reboot_notifier: unregister_reboot_notifier(&sclp_reboot_notifier); fail_init_state_uninitialized: sclp_init_state = sclp_init_state_uninitialized; + free_page((unsigned long) sclp_read_sccb); + free_page((unsigned long) sclp_init_sccb); fail_unlock: spin_unlock_irqrestore(&sclp_lock, flags); return rc; diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index 367e9d384d85..196333013e54 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -197,7 +197,9 @@ struct read_info_sccb { u32 hmfai; /* 124-127 */ u8 _pad_128[134 - 128]; /* 128-133 */ u8 byte_134; /* 134 */ - u8 _pad_135[4096 - 135]; /* 135-4095 */ + u8 cpudirq; /* 135 */ + u16 cbl; /* 136-137 */ + u8 _pad_138[4096 - 138]; /* 138-4095 */ } __packed __aligned(PAGE_SIZE); struct read_storage_sccb { @@ -319,7 +321,7 @@ extern int sclp_console_drop; extern unsigned long sclp_console_full; extern bool sclp_mask_compat_mode; -extern char sclp_early_sccb[PAGE_SIZE]; +extern char *sclp_early_sccb; void sclp_early_wait_irq(void); int sclp_early_cmd(sclp_cmdw_t cmd, void *sccb); @@ -365,14 +367,14 @@ sclp_ascebc(unsigned char ch) /* translate string from EBCDIC to ASCII */ static inline void -sclp_ebcasc_str(unsigned char *str, int nr) +sclp_ebcasc_str(char *str, int nr) { (MACHINE_IS_VM) ? EBCASC(str, nr) : EBCASC_500(str, nr); } /* translate string from ASCII to EBCDIC */ static inline void -sclp_ascebc_str(unsigned char *str, int nr) +sclp_ascebc_str(char *str, int nr) { (MACHINE_IS_VM) ? ASCEBC(str, nr) : ASCEBC_500(str, nr); } diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 8332788681c4..6c90aa725f23 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -40,6 +40,8 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb) sclp.has_gisaf = !!(sccb->fac118 & 0x08); sclp.has_hvs = !!(sccb->fac119 & 0x80); sclp.has_kss = !!(sccb->fac98 & 0x01); + sclp.has_sipl = !!(sccb->cbl & 0x02); + sclp.has_sipl_g2 = !!(sccb->cbl & 0x04); if (sccb->fac85 & 0x02) S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP; if (sccb->fac91 & 0x40) @@ -93,6 +95,7 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb) sclp.mtid_prev = (sccb->fac42 & 0x80) ? (sccb->fac66 & 31) : 0; sclp.hmfai = sccb->hmfai; + sclp.has_dirq = !!(sccb->cpudirq & 0x80); } /* @@ -144,7 +147,7 @@ static void __init sclp_early_console_detect(struct init_sccb *sccb) void __init sclp_early_detect(void) { - void *sccb = &sclp_early_sccb; + void *sccb = sclp_early_sccb; sclp_early_facilities_detect(sccb); sclp_early_init_core_info(sccb); diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index 387c114ded3f..7737470f8498 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -16,7 +16,7 @@ static struct read_info_sccb __bootdata(sclp_info_sccb); static int __bootdata(sclp_info_sccb_valid); -char sclp_early_sccb[PAGE_SIZE] __aligned(PAGE_SIZE) __section(.data); +char *sclp_early_sccb = (char *) EARLY_SCCB_OFFSET; int sclp_init_state __section(.data) = sclp_init_state_uninitialized; /* * Used to keep track of the size of the event masks. Qemu until version 2.11 @@ -91,8 +91,8 @@ static void sclp_early_print_lm(const char *str, unsigned int len) struct mto *mto; struct go *go; - sccb = (struct write_sccb *) &sclp_early_sccb; - end = (unsigned char *) sccb + sizeof(sclp_early_sccb) - 1; + sccb = (struct write_sccb *) sclp_early_sccb; + end = (unsigned char *) sccb + EARLY_SCCB_SIZE - 1; memset(sccb, 0, sizeof(*sccb)); ptr = (unsigned char *) &sccb->msg.mdb.mto; offset = 0; @@ -139,9 +139,9 @@ static void sclp_early_print_vt220(const char *str, unsigned int len) { struct vt220_sccb *sccb; - sccb = (struct vt220_sccb *) &sclp_early_sccb; - if (sizeof(*sccb) + len >= sizeof(sclp_early_sccb)) - len = sizeof(sclp_early_sccb) - sizeof(*sccb); + sccb = (struct vt220_sccb *) sclp_early_sccb; + if (sizeof(*sccb) + len >= EARLY_SCCB_SIZE) + len = EARLY_SCCB_SIZE - sizeof(*sccb); memset(sccb, 0, sizeof(*sccb)); memcpy(&sccb->msg.data, str, len); sccb->header.length = sizeof(*sccb) + len; @@ -199,7 +199,7 @@ static int sclp_early_setup(int disable, int *have_linemode, int *have_vt220) BUILD_BUG_ON(sizeof(struct init_sccb) > PAGE_SIZE); *have_linemode = *have_vt220 = 0; - sccb = (struct init_sccb *) &sclp_early_sccb; + sccb = (struct init_sccb *) sclp_early_sccb; receive_mask = disable ? 0 : EVTYP_OPCMD_MASK; send_mask = disable ? 0 : EVTYP_VT220MSG_MASK | EVTYP_MSG_MASK; rc = sclp_early_set_event_mask(sccb, receive_mask, send_mask); @@ -304,7 +304,7 @@ int __init sclp_early_get_hsa_size(unsigned long *hsa_size) void __weak __init add_mem_detect_block(u64 start, u64 end) {} int __init sclp_early_read_storage_info(void) { - struct read_storage_sccb *sccb = (struct read_storage_sccb *)&sclp_early_sccb; + struct read_storage_sccb *sccb = (struct read_storage_sccb *)sclp_early_sccb; int rc, id, max_id = 0; unsigned long rn, rzm; sclp_cmdw_t command; @@ -320,8 +320,8 @@ int __init sclp_early_read_storage_info(void) rzm <<= 20; for (id = 0; id <= max_id; id++) { - memset(sclp_early_sccb, 0, sizeof(sclp_early_sccb)); - sccb->header.length = sizeof(sclp_early_sccb); + memset(sclp_early_sccb, 0, EARLY_SCCB_SIZE); + sccb->header.length = EARLY_SCCB_SIZE; command = SCLP_CMDW_READ_STORAGE_INFO | (id << 8); rc = sclp_early_cmd(command, sccb); if (rc) diff --git a/drivers/s390/char/sclp_sdias.c b/drivers/s390/char/sclp_sdias.c index 8e0b69a2f11a..13f97fd73aca 100644 --- a/drivers/s390/char/sclp_sdias.c +++ b/drivers/s390/char/sclp_sdias.c @@ -29,7 +29,7 @@ static struct sclp_register sclp_sdias_register = { .send_mask = EVTYP_SDIAS_MASK, }; -static struct sdias_sccb sccb __attribute__((aligned(4096))); +static struct sdias_sccb *sclp_sdias_sccb; static struct sdias_evbuf sdias_evbuf; static DECLARE_COMPLETION(evbuf_accepted); @@ -58,6 +58,7 @@ static void sdias_callback(struct sclp_req *request, void *data) static int sdias_sclp_send(struct sclp_req *req) { + struct sdias_sccb *sccb = sclp_sdias_sccb; int retries; int rc; @@ -78,16 +79,16 @@ static int sdias_sclp_send(struct sclp_req *req) continue; } /* if not accepted, retry */ - if (!(sccb.evbuf.hdr.flags & 0x80)) { + if (!(sccb->evbuf.hdr.flags & 0x80)) { TRACE("sclp request failed: flags=%x\n", - sccb.evbuf.hdr.flags); + sccb->evbuf.hdr.flags); continue; } /* * for the sync interface the response is in the initial sccb */ if (!sclp_sdias_register.receiver_fn) { - memcpy(&sdias_evbuf, &sccb.evbuf, sizeof(sdias_evbuf)); + memcpy(&sdias_evbuf, &sccb->evbuf, sizeof(sdias_evbuf)); TRACE("sync request done\n"); return 0; } @@ -104,23 +105,24 @@ static int sdias_sclp_send(struct sclp_req *req) */ int sclp_sdias_blk_count(void) { + struct sdias_sccb *sccb = sclp_sdias_sccb; struct sclp_req request; int rc; mutex_lock(&sdias_mutex); - memset(&sccb, 0, sizeof(sccb)); + memset(sccb, 0, sizeof(*sccb)); memset(&request, 0, sizeof(request)); - sccb.hdr.length = sizeof(sccb); - sccb.evbuf.hdr.length = sizeof(struct sdias_evbuf); - sccb.evbuf.hdr.type = EVTYP_SDIAS; - sccb.evbuf.event_qual = SDIAS_EQ_SIZE; - sccb.evbuf.data_id = SDIAS_DI_FCP_DUMP; - sccb.evbuf.event_id = 4712; - sccb.evbuf.dbs = 1; + sccb->hdr.length = sizeof(*sccb); + sccb->evbuf.hdr.length = sizeof(struct sdias_evbuf); + sccb->evbuf.hdr.type = EVTYP_SDIAS; + sccb->evbuf.event_qual = SDIAS_EQ_SIZE; + sccb->evbuf.data_id = SDIAS_DI_FCP_DUMP; + sccb->evbuf.event_id = 4712; + sccb->evbuf.dbs = 1; - request.sccb = &sccb; + request.sccb = sccb; request.command = SCLP_CMDW_WRITE_EVENT_DATA; request.status = SCLP_REQ_FILLED; request.callback = sdias_callback; @@ -130,8 +132,8 @@ int sclp_sdias_blk_count(void) pr_err("sclp_send failed for get_nr_blocks\n"); goto out; } - if (sccb.hdr.response_code != 0x0020) { - TRACE("send failed: %x\n", sccb.hdr.response_code); + if (sccb->hdr.response_code != 0x0020) { + TRACE("send failed: %x\n", sccb->hdr.response_code); rc = -EIO; goto out; } @@ -163,30 +165,31 @@ out: */ int sclp_sdias_copy(void *dest, int start_blk, int nr_blks) { + struct sdias_sccb *sccb = sclp_sdias_sccb; struct sclp_req request; int rc; mutex_lock(&sdias_mutex); - memset(&sccb, 0, sizeof(sccb)); + memset(sccb, 0, sizeof(*sccb)); memset(&request, 0, sizeof(request)); - sccb.hdr.length = sizeof(sccb); - sccb.evbuf.hdr.length = sizeof(struct sdias_evbuf); - sccb.evbuf.hdr.type = EVTYP_SDIAS; - sccb.evbuf.hdr.flags = 0; - sccb.evbuf.event_qual = SDIAS_EQ_STORE_DATA; - sccb.evbuf.data_id = SDIAS_DI_FCP_DUMP; - sccb.evbuf.event_id = 4712; - sccb.evbuf.asa_size = SDIAS_ASA_SIZE_64; - sccb.evbuf.event_status = 0; - sccb.evbuf.blk_cnt = nr_blks; - sccb.evbuf.asa = (unsigned long)dest; - sccb.evbuf.fbn = start_blk; - sccb.evbuf.lbn = 0; - sccb.evbuf.dbs = 1; - - request.sccb = &sccb; + sccb->hdr.length = sizeof(*sccb); + sccb->evbuf.hdr.length = sizeof(struct sdias_evbuf); + sccb->evbuf.hdr.type = EVTYP_SDIAS; + sccb->evbuf.hdr.flags = 0; + sccb->evbuf.event_qual = SDIAS_EQ_STORE_DATA; + sccb->evbuf.data_id = SDIAS_DI_FCP_DUMP; + sccb->evbuf.event_id = 4712; + sccb->evbuf.asa_size = SDIAS_ASA_SIZE_64; + sccb->evbuf.event_status = 0; + sccb->evbuf.blk_cnt = nr_blks; + sccb->evbuf.asa = (unsigned long)dest; + sccb->evbuf.fbn = start_blk; + sccb->evbuf.lbn = 0; + sccb->evbuf.dbs = 1; + + request.sccb = sccb; request.command = SCLP_CMDW_WRITE_EVENT_DATA; request.status = SCLP_REQ_FILLED; request.callback = sdias_callback; @@ -196,8 +199,8 @@ int sclp_sdias_copy(void *dest, int start_blk, int nr_blks) pr_err("sclp_send failed: %x\n", rc); goto out; } - if (sccb.hdr.response_code != 0x0020) { - TRACE("copy failed: %x\n", sccb.hdr.response_code); + if (sccb->hdr.response_code != 0x0020) { + TRACE("copy failed: %x\n", sccb->hdr.response_code); rc = -EIO; goto out; } @@ -256,6 +259,8 @@ int __init sclp_sdias_init(void) { if (ipl_info.type != IPL_TYPE_FCP_DUMP) return 0; + sclp_sdias_sccb = (void *) __get_free_page(GFP_KERNEL | GFP_DMA); + BUG_ON(!sclp_sdias_sccb); sdias_dbf = debug_register("dump_sdias", 4, 1, 4 * sizeof(long)); debug_register_view(sdias_dbf, &debug_sprintf_view); debug_set_level(sdias_dbf, 6); @@ -264,6 +269,7 @@ int __init sclp_sdias_init(void) if (sclp_sdias_init_async() == 0) goto out; TRACE("init failed\n"); + free_page((unsigned long) sclp_sdias_sccb); return -ENODEV; out: TRACE("init done\n"); diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 76d3c50bf078..f75d3bfb5af3 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -51,7 +51,7 @@ static struct dentry *zcore_dir; static struct dentry *zcore_memmap_file; static struct dentry *zcore_reipl_file; static struct dentry *zcore_hsa_file; -static struct ipl_parameter_block *ipl_block; +static struct ipl_parameter_block *zcore_ipl_block; static char hsa_buf[PAGE_SIZE] __aligned(PAGE_SIZE); @@ -182,8 +182,8 @@ static const struct file_operations zcore_memmap_fops = { static ssize_t zcore_reipl_write(struct file *filp, const char __user *buf, size_t count, loff_t *ppos) { - if (ipl_block) { - diag308(DIAG308_SET, ipl_block); + if (zcore_ipl_block) { + diag308(DIAG308_SET, zcore_ipl_block); diag308(DIAG308_LOAD_CLEAR, NULL); } return count; @@ -265,18 +265,20 @@ static int __init zcore_reipl_init(void) return rc; if (ipib_info.ipib == 0) return 0; - ipl_block = (void *) __get_free_page(GFP_KERNEL); - if (!ipl_block) + zcore_ipl_block = (void *) __get_free_page(GFP_KERNEL); + if (!zcore_ipl_block) return -ENOMEM; if (ipib_info.ipib < sclp.hsa_size) - rc = memcpy_hsa_kernel(ipl_block, ipib_info.ipib, PAGE_SIZE); + rc = memcpy_hsa_kernel(zcore_ipl_block, ipib_info.ipib, + PAGE_SIZE); else - rc = memcpy_real(ipl_block, (void *) ipib_info.ipib, PAGE_SIZE); - if (rc || (__force u32)csum_partial(ipl_block, ipl_block->hdr.len, 0) != + rc = memcpy_real(zcore_ipl_block, (void *) ipib_info.ipib, + PAGE_SIZE); + if (rc || (__force u32)csum_partial(zcore_ipl_block, zcore_ipl_block->hdr.len, 0) != ipib_info.checksum) { TRACE("Checksum does not match\n"); - free_page((unsigned long) ipl_block); - ipl_block = NULL; + free_page((unsigned long) zcore_ipl_block); + zcore_ipl_block = NULL; } return 0; } diff --git a/drivers/s390/cio/Makefile b/drivers/s390/cio/Makefile index f230516abb96..f6a8db04177c 100644 --- a/drivers/s390/cio/Makefile +++ b/drivers/s390/cio/Makefile @@ -20,5 +20,6 @@ obj-$(CONFIG_CCWGROUP) += ccwgroup.o qdio-objs := qdio_main.o qdio_thinint.o qdio_debug.o qdio_setup.o obj-$(CONFIG_QDIO) += qdio.o -vfio_ccw-objs += vfio_ccw_drv.o vfio_ccw_cp.o vfio_ccw_ops.o vfio_ccw_fsm.o +vfio_ccw-objs += vfio_ccw_drv.o vfio_ccw_cp.o vfio_ccw_ops.o vfio_ccw_fsm.o \ + vfio_ccw_async.o obj-$(CONFIG_VFIO_CCW) += vfio_ccw.o diff --git a/drivers/s390/cio/airq.c b/drivers/s390/cio/airq.c index a45011e4529e..4534afc63591 100644 --- a/drivers/s390/cio/airq.c +++ b/drivers/s390/cio/airq.c @@ -27,6 +27,8 @@ static DEFINE_SPINLOCK(airq_lists_lock); static struct hlist_head airq_lists[MAX_ISC+1]; +static struct kmem_cache *airq_iv_cache; + /** * register_adapter_interrupt() - register adapter interrupt handler * @airq: pointer to adapter interrupt descriptor @@ -95,7 +97,7 @@ static irqreturn_t do_airq_interrupt(int irq, void *dummy) rcu_read_lock(); hlist_for_each_entry_rcu(airq, head, list) if ((*airq->lsi_ptr & airq->lsi_mask) != 0) - airq->handler(airq); + airq->handler(airq, !tpi_info->directed_irq); rcu_read_unlock(); return IRQ_HANDLED; @@ -129,10 +131,21 @@ struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags) if (!iv) goto out; iv->bits = bits; + iv->flags = flags; size = BITS_TO_LONGS(bits) * sizeof(unsigned long); - iv->vector = kzalloc(size, GFP_KERNEL); - if (!iv->vector) - goto out_free; + + if (flags & AIRQ_IV_CACHELINE) { + if ((cache_line_size() * BITS_PER_BYTE) < bits) + goto out_free; + + iv->vector = kmem_cache_zalloc(airq_iv_cache, GFP_KERNEL); + if (!iv->vector) + goto out_free; + } else { + iv->vector = kzalloc(size, GFP_KERNEL); + if (!iv->vector) + goto out_free; + } if (flags & AIRQ_IV_ALLOC) { iv->avail = kmalloc(size, GFP_KERNEL); if (!iv->avail) @@ -165,7 +178,10 @@ out_free: kfree(iv->ptr); kfree(iv->bitlock); kfree(iv->avail); - kfree(iv->vector); + if (iv->flags & AIRQ_IV_CACHELINE) + kmem_cache_free(airq_iv_cache, iv->vector); + else + kfree(iv->vector); kfree(iv); out: return NULL; @@ -181,7 +197,10 @@ void airq_iv_release(struct airq_iv *iv) kfree(iv->data); kfree(iv->ptr); kfree(iv->bitlock); - kfree(iv->vector); + if (iv->flags & AIRQ_IV_CACHELINE) + kmem_cache_free(airq_iv_cache, iv->vector); + else + kfree(iv->vector); kfree(iv->avail); kfree(iv); } @@ -275,3 +294,13 @@ unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start, return bit; } EXPORT_SYMBOL(airq_iv_scan); + +static int __init airq_init(void) +{ + airq_iv_cache = kmem_cache_create("airq_iv_cache", cache_line_size(), + cache_line_size(), 0, NULL); + if (!airq_iv_cache) + return -ENOMEM; + return 0; +} +subsys_initcall(airq_init); diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c index de744ca158fd..18f5458f90e8 100644 --- a/drivers/s390/cio/cio.c +++ b/drivers/s390/cio/cio.c @@ -564,7 +564,7 @@ static irqreturn_t do_cio_interrupt(int irq, void *dummy) } static struct irqaction io_interrupt = { - .name = "IO", + .name = "I/O", .handler = do_cio_interrupt, }; diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h index 9811fd8a0c73..06a91743335a 100644 --- a/drivers/s390/cio/cio.h +++ b/drivers/s390/cio/cio.h @@ -51,7 +51,7 @@ struct tpi_info { struct subchannel_id schid; u32 intparm; u32 adapter_IO:1; - u32 :1; + u32 directed_irq:1; u32 isc:3; u32 :27; u32 type:3; @@ -115,7 +115,7 @@ struct subchannel { struct schib_config config; } __attribute__ ((aligned(8))); -DECLARE_PER_CPU(struct irb, cio_irb); +DECLARE_PER_CPU_ALIGNED(struct irb, cio_irb); #define to_subchannel(n) container_of(n, struct subchannel, dev) diff --git a/drivers/s390/cio/ioasm.c b/drivers/s390/cio/ioasm.c index 14d328338ce2..08eb10283b18 100644 --- a/drivers/s390/cio/ioasm.c +++ b/drivers/s390/cio/ioasm.c @@ -233,6 +233,7 @@ int hsch(struct subchannel_id schid) return ccode; } +EXPORT_SYMBOL(hsch); static inline int __xsch(struct subchannel_id schid) { diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h index a6f7c2986b94..a06944399865 100644 --- a/drivers/s390/cio/qdio.h +++ b/drivers/s390/cio/qdio.h @@ -228,9 +228,6 @@ struct qdio_q { */ int first_to_check; - /* first_to_check of the last time */ - int last_move; - /* beginning position for calling the program */ int first_to_kick; @@ -341,8 +338,7 @@ static inline int multicast_outbound(struct qdio_q *q) (q->nr == q->irq_ptr->nr_output_qs - 1); } -#define pci_out_supported(q) \ - (q->irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED) +#define pci_out_supported(irq) ((irq)->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED) #define is_qebsm(q) (q->irq_ptr->sch_token != 0) #define need_siga_in(q) (q->irq_ptr->siga_flag.input) diff --git a/drivers/s390/cio/qdio_debug.c b/drivers/s390/cio/qdio_debug.c index d2f98e5829d4..35410e6eda2e 100644 --- a/drivers/s390/cio/qdio_debug.c +++ b/drivers/s390/cio/qdio_debug.c @@ -121,15 +121,14 @@ static int qstat_show(struct seq_file *m, void *v) seq_printf(m, "Timestamp: %Lx Last AI: %Lx\n", q->timestamp, last_ai_time); - seq_printf(m, "nr_used: %d ftc: %d last_move: %d\n", - atomic_read(&q->nr_buf_used), - q->first_to_check, q->last_move); + seq_printf(m, "nr_used: %d ftc: %d\n", + atomic_read(&q->nr_buf_used), q->first_to_check); if (q->is_input_q) { seq_printf(m, "polling: %d ack start: %d ack count: %d\n", q->u.in.polling, q->u.in.ack_start, q->u.in.ack_count); - seq_printf(m, "DSCI: %d IRQs disabled: %u\n", - *(u32 *)q->irq_ptr->dsci, + seq_printf(m, "DSCI: %x IRQs disabled: %u\n", + *(u8 *)q->irq_ptr->dsci, test_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state)); } diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index 9537e656e927..cfce255521ac 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -371,7 +371,7 @@ static inline int qdio_siga_input(struct qdio_q *q) static inline void qdio_sync_queues(struct qdio_q *q) { /* PCI capable outbound queues will also be scanned so sync them too */ - if (pci_out_supported(q)) + if (pci_out_supported(q->irq_ptr)) qdio_siga_sync_all(q); else qdio_siga_sync_q(q); @@ -415,7 +415,8 @@ static inline void account_sbals(struct qdio_q *q, unsigned int count) q->q_stats.nr_sbals[pos]++; } -static void process_buffer_error(struct qdio_q *q, int count) +static void process_buffer_error(struct qdio_q *q, unsigned int start, + int count) { unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT : SLSB_P_OUTPUT_NOT_INIT; @@ -424,29 +425,29 @@ static void process_buffer_error(struct qdio_q *q, int count) /* special handling for no target buffer empty */ if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q && - q->sbal[q->first_to_check]->element[15].sflags == 0x10) { + q->sbal[start]->element[15].sflags == 0x10) { qperf_inc(q, target_full); - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", - q->first_to_check); + DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start); goto set; } DBF_ERROR("%4x BUF ERROR", SCH_NO(q)); DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr); - DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count); + DBF_ERROR("FTC:%3d C:%3d", start, count); DBF_ERROR("F14:%2x F15:%2x", - q->sbal[q->first_to_check]->element[14].sflags, - q->sbal[q->first_to_check]->element[15].sflags); + q->sbal[start]->element[14].sflags, + q->sbal[start]->element[15].sflags); set: /* * Interrupts may be avoided as long as the error is present * so change the buffer state immediately to avoid starvation. */ - set_buf_states(q, q->first_to_check, state, count); + set_buf_states(q, start, state, count); } -static inline void inbound_primed(struct qdio_q *q, int count) +static inline void inbound_primed(struct qdio_q *q, unsigned int start, + int count) { int new; @@ -457,7 +458,7 @@ static inline void inbound_primed(struct qdio_q *q, int count) if (!q->u.in.polling) { q->u.in.polling = 1; q->u.in.ack_count = count; - q->u.in.ack_start = q->first_to_check; + q->u.in.ack_start = start; return; } @@ -465,7 +466,7 @@ static inline void inbound_primed(struct qdio_q *q, int count) set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT, q->u.in.ack_count); q->u.in.ack_count = count; - q->u.in.ack_start = q->first_to_check; + q->u.in.ack_start = start; return; } @@ -473,7 +474,7 @@ static inline void inbound_primed(struct qdio_q *q, int count) * ACK the newest buffer. The ACK will be removed in qdio_stop_polling * or by the next inbound run. */ - new = add_buf(q->first_to_check, count - 1); + new = add_buf(start, count - 1); if (q->u.in.polling) { /* reset the previous ACK but first set the new one */ set_buf_state(q, new, SLSB_P_INPUT_ACK); @@ -488,10 +489,10 @@ static inline void inbound_primed(struct qdio_q *q, int count) if (!count) return; /* need to change ALL buffers to get more interrupts */ - set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count); + set_buf_states(q, start, SLSB_P_INPUT_NOT_INIT, count); } -static int get_inbound_buffer_frontier(struct qdio_q *q) +static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start) { unsigned char state = 0; int count; @@ -504,64 +505,58 @@ static int get_inbound_buffer_frontier(struct qdio_q *q) */ count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK); if (!count) - goto out; + return 0; /* * No siga sync here, as a PCI or we after a thin interrupt * already sync'ed the queues. */ - count = get_buf_states(q, q->first_to_check, &state, count, 1, 0); + count = get_buf_states(q, start, &state, count, 1, 0); if (!count) - goto out; + return 0; switch (state) { case SLSB_P_INPUT_PRIMED: - inbound_primed(q, count); - q->first_to_check = add_buf(q->first_to_check, count); + inbound_primed(q, start, count); if (atomic_sub_return(count, &q->nr_buf_used) == 0) qperf_inc(q, inbound_queue_full); if (q->irq_ptr->perf_stat_enabled) account_sbals(q, count); - break; + return count; case SLSB_P_INPUT_ERROR: - process_buffer_error(q, count); - q->first_to_check = add_buf(q->first_to_check, count); + process_buffer_error(q, start, count); if (atomic_sub_return(count, &q->nr_buf_used) == 0) qperf_inc(q, inbound_queue_full); if (q->irq_ptr->perf_stat_enabled) account_sbals_error(q, count); - break; + return count; case SLSB_CU_INPUT_EMPTY: case SLSB_P_INPUT_NOT_INIT: case SLSB_P_INPUT_ACK: if (q->irq_ptr->perf_stat_enabled) q->q_stats.nr_sbal_nop++; DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x", - q->nr, q->first_to_check); - break; + q->nr, start); + return 0; default: WARN_ON_ONCE(1); + return 0; } -out: - return q->first_to_check; } -static int qdio_inbound_q_moved(struct qdio_q *q) +static int qdio_inbound_q_moved(struct qdio_q *q, unsigned int start) { - int bufnr; + int count; - bufnr = get_inbound_buffer_frontier(q); + count = get_inbound_buffer_frontier(q, start); - if (bufnr != q->last_move) { - q->last_move = bufnr; - if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR) - q->u.in.timestamp = get_tod_clock(); - return 1; - } else - return 0; + if (count && !is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR) + q->u.in.timestamp = get_tod_clock(); + + return count; } -static inline int qdio_inbound_q_done(struct qdio_q *q) +static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) { unsigned char state = 0; @@ -570,7 +565,7 @@ static inline int qdio_inbound_q_done(struct qdio_q *q) if (need_siga_sync(q)) qdio_siga_sync_q(q); - get_buf_state(q, q->first_to_check, &state, 0); + get_buf_state(q, start, &state, 0); if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR) /* more work coming */ @@ -588,8 +583,7 @@ static inline int qdio_inbound_q_done(struct qdio_q *q) * has (probably) not moved (see qdio_inbound_processing). */ if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) { - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x", - q->first_to_check); + DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x", start); return 1; } else return 0; @@ -637,17 +631,13 @@ static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q, return phys_aob; } -static void qdio_kick_handler(struct qdio_q *q) +static void qdio_kick_handler(struct qdio_q *q, unsigned int count) { int start = q->first_to_kick; - int end = q->first_to_check; - int count; if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) return; - count = sub_buf(end, start); - if (q->is_input_q) { qperf_inc(q, inbound_handler); DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count); @@ -663,7 +653,7 @@ static void qdio_kick_handler(struct qdio_q *q) q->irq_ptr->int_parm); /* for the next time */ - q->first_to_kick = end; + q->first_to_kick = add_buf(start, count); q->qdio_error = 0; } @@ -678,14 +668,20 @@ static inline int qdio_tasklet_schedule(struct qdio_q *q) static void __qdio_inbound_processing(struct qdio_q *q) { + unsigned int start = q->first_to_check; + int count; + qperf_inc(q, tasklet_inbound); - if (!qdio_inbound_q_moved(q)) + count = qdio_inbound_q_moved(q, start); + if (count == 0) return; - qdio_kick_handler(q); + start = add_buf(start, count); + q->first_to_check = start; + qdio_kick_handler(q, count); - if (!qdio_inbound_q_done(q)) { + if (!qdio_inbound_q_done(q, start)) { /* means poll time is not yet over */ qperf_inc(q, tasklet_inbound_resched); if (!qdio_tasklet_schedule(q)) @@ -697,7 +693,7 @@ static void __qdio_inbound_processing(struct qdio_q *q) * We need to check again to not lose initiative after * resetting the ACK state. */ - if (!qdio_inbound_q_done(q)) { + if (!qdio_inbound_q_done(q, start)) { qperf_inc(q, tasklet_inbound_resched2); qdio_tasklet_schedule(q); } @@ -709,7 +705,7 @@ void qdio_inbound_processing(unsigned long data) __qdio_inbound_processing(q); } -static int get_outbound_buffer_frontier(struct qdio_q *q) +static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start) { unsigned char state = 0; int count; @@ -718,7 +714,7 @@ static int get_outbound_buffer_frontier(struct qdio_q *q) if (need_siga_sync(q)) if (((queue_type(q) != QDIO_IQDIO_QFMT) && - !pci_out_supported(q)) || + !pci_out_supported(q->irq_ptr)) || (queue_type(q) == QDIO_IQDIO_QFMT && multicast_outbound(q))) qdio_siga_sync_q(q); @@ -729,12 +725,11 @@ static int get_outbound_buffer_frontier(struct qdio_q *q) */ count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK); if (!count) - goto out; + return 0; - count = get_buf_states(q, q->first_to_check, &state, count, 0, - q->u.out.use_cq); + count = get_buf_states(q, start, &state, count, 0, q->u.out.use_cq); if (!count) - goto out; + return 0; switch (state) { case SLSB_P_OUTPUT_EMPTY: @@ -743,34 +738,29 @@ static int get_outbound_buffer_frontier(struct qdio_q *q) "out empty:%1d %02x", q->nr, count); atomic_sub(count, &q->nr_buf_used); - q->first_to_check = add_buf(q->first_to_check, count); if (q->irq_ptr->perf_stat_enabled) account_sbals(q, count); - - break; + return count; case SLSB_P_OUTPUT_ERROR: - process_buffer_error(q, count); - q->first_to_check = add_buf(q->first_to_check, count); + process_buffer_error(q, start, count); atomic_sub(count, &q->nr_buf_used); if (q->irq_ptr->perf_stat_enabled) account_sbals_error(q, count); - break; + return count; case SLSB_CU_OUTPUT_PRIMED: /* the adapter has not fetched the output yet */ if (q->irq_ptr->perf_stat_enabled) q->q_stats.nr_sbal_nop++; DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr); - break; + return 0; case SLSB_P_OUTPUT_NOT_INIT: case SLSB_P_OUTPUT_HALTED: - break; + return 0; default: WARN_ON_ONCE(1); + return 0; } - -out: - return q->first_to_check; } /* all buffers processed? */ @@ -779,18 +769,16 @@ static inline int qdio_outbound_q_done(struct qdio_q *q) return atomic_read(&q->nr_buf_used) == 0; } -static inline int qdio_outbound_q_moved(struct qdio_q *q) +static inline int qdio_outbound_q_moved(struct qdio_q *q, unsigned int start) { - int bufnr; + int count; - bufnr = get_outbound_buffer_frontier(q); + count = get_outbound_buffer_frontier(q, start); - if (bufnr != q->last_move) { - q->last_move = bufnr; + if (count) DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr); - return 1; - } else - return 0; + + return count; } static int qdio_kick_outbound_q(struct qdio_q *q, unsigned long aob) @@ -837,15 +825,21 @@ retry: static void __qdio_outbound_processing(struct qdio_q *q) { + unsigned int start = q->first_to_check; + int count; + qperf_inc(q, tasklet_outbound); WARN_ON_ONCE(atomic_read(&q->nr_buf_used) < 0); - if (qdio_outbound_q_moved(q)) - qdio_kick_handler(q); + count = qdio_outbound_q_moved(q, start); + if (count) { + q->first_to_check = add_buf(start, count); + qdio_kick_handler(q, count); + } - if (queue_type(q) == QDIO_ZFCP_QFMT) - if (!pci_out_supported(q) && !qdio_outbound_q_done(q)) - goto sched; + if (queue_type(q) == QDIO_ZFCP_QFMT && !pci_out_supported(q->irq_ptr) && + !qdio_outbound_q_done(q)) + goto sched; if (q->u.out.pci_out_enabled) return; @@ -881,37 +875,40 @@ void qdio_outbound_timer(struct timer_list *t) qdio_tasklet_schedule(q); } -static inline void qdio_check_outbound_after_thinint(struct qdio_q *q) +static inline void qdio_check_outbound_pci_queues(struct qdio_irq *irq) { struct qdio_q *out; int i; - if (!pci_out_supported(q)) + if (!pci_out_supported(irq)) return; - for_each_output_queue(q->irq_ptr, out, i) + for_each_output_queue(irq, out, i) if (!qdio_outbound_q_done(out)) qdio_tasklet_schedule(out); } static void __tiqdio_inbound_processing(struct qdio_q *q) { + unsigned int start = q->first_to_check; + int count; + qperf_inc(q, tasklet_inbound); if (need_siga_sync(q) && need_siga_sync_after_ai(q)) qdio_sync_queues(q); - /* - * The interrupt could be caused by a PCI request. Check the - * PCI capable outbound queues. - */ - qdio_check_outbound_after_thinint(q); + /* The interrupt could be caused by a PCI request: */ + qdio_check_outbound_pci_queues(q->irq_ptr); - if (!qdio_inbound_q_moved(q)) + count = qdio_inbound_q_moved(q, start); + if (count == 0) return; - qdio_kick_handler(q); + start = add_buf(start, count); + q->first_to_check = start; + qdio_kick_handler(q, count); - if (!qdio_inbound_q_done(q)) { + if (!qdio_inbound_q_done(q, start)) { qperf_inc(q, tasklet_inbound_resched); if (!qdio_tasklet_schedule(q)) return; @@ -922,7 +919,7 @@ static void __tiqdio_inbound_processing(struct qdio_q *q) * We need to check again to not lose initiative after * resetting the ACK state. */ - if (!qdio_inbound_q_done(q)) { + if (!qdio_inbound_q_done(q, start)) { qperf_inc(q, tasklet_inbound_resched2); qdio_tasklet_schedule(q); } @@ -976,7 +973,7 @@ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) } } - if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED)) + if (!pci_out_supported(irq_ptr)) return; for_each_output_queue(irq_ptr, q, i) { @@ -1642,7 +1639,7 @@ int qdio_start_irq(struct ccw_device *cdev, int nr) */ if (test_nonshared_ind(irq_ptr)) goto rescan; - if (!qdio_inbound_q_done(q)) + if (!qdio_inbound_q_done(q, q->first_to_check)) goto rescan; return 0; @@ -1672,12 +1669,14 @@ int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr, int *error) { struct qdio_q *q; - int start, end; struct qdio_irq *irq_ptr = cdev->private->qdio_data; + unsigned int start; + int count; if (!irq_ptr) return -ENODEV; q = irq_ptr->input_qs[nr]; + start = q->first_to_check; /* * Cannot rely on automatic sync after interrupt since queues may @@ -1686,25 +1685,27 @@ int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr, if (need_siga_sync(q)) qdio_sync_queues(q); - /* check the PCI capable outbound queues. */ - qdio_check_outbound_after_thinint(q); + qdio_check_outbound_pci_queues(irq_ptr); - if (!qdio_inbound_q_moved(q)) + count = qdio_inbound_q_moved(q, start); + if (count == 0) return 0; + start = add_buf(start, count); + q->first_to_check = start; + /* Note: upper-layer MUST stop processing immediately here ... */ if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) return -EIO; - start = q->first_to_kick; - end = q->first_to_check; - *bufnr = start; + *bufnr = q->first_to_kick; *error = q->qdio_error; /* for the next time */ - q->first_to_kick = end; + q->first_to_kick = add_buf(q->first_to_kick, count); q->qdio_error = 0; - return sub_buf(end, start); + + return count; } EXPORT_SYMBOL(qdio_get_next_buffers); diff --git a/drivers/s390/cio/qdio_setup.c b/drivers/s390/cio/qdio_setup.c index a59887fad13e..99d7d2566a3a 100644 --- a/drivers/s390/cio/qdio_setup.c +++ b/drivers/s390/cio/qdio_setup.c @@ -523,7 +523,7 @@ void qdio_print_subchannel_info(struct qdio_irq *irq_ptr, irq_ptr->schid.sch_no, is_thinint_irq(irq_ptr), (irq_ptr->sch_token) ? 1 : 0, - (irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED) ? 1 : 0, + pci_out_supported(irq_ptr) ? 1 : 0, css_general_characteristics.aif_tdd, (irq_ptr->siga_flag.input) ? "R" : " ", (irq_ptr->siga_flag.output) ? "W" : " ", diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c index 07dea602205b..28d59ac2204c 100644 --- a/drivers/s390/cio/qdio_thinint.c +++ b/drivers/s390/cio/qdio_thinint.c @@ -40,7 +40,7 @@ static LIST_HEAD(tiq_list); static DEFINE_MUTEX(tiq_list_lock); /* Adapter interrupt definitions */ -static void tiqdio_thinint_handler(struct airq_struct *airq); +static void tiqdio_thinint_handler(struct airq_struct *airq, bool floating); static struct airq_struct tiqdio_airq = { .handler = tiqdio_thinint_handler, @@ -179,7 +179,7 @@ static inline void tiqdio_call_inq_handlers(struct qdio_irq *irq) * tiqdio_thinint_handler - thin interrupt handler for qdio * @airq: pointer to adapter interrupt descriptor */ -static void tiqdio_thinint_handler(struct airq_struct *airq) +static void tiqdio_thinint_handler(struct airq_struct *airq, bool floating) { u32 si_used = clear_shared_ind(); struct qdio_q *q; diff --git a/drivers/s390/cio/vfio_ccw_async.c b/drivers/s390/cio/vfio_ccw_async.c new file mode 100644 index 000000000000..8c1d2357ef5b --- /dev/null +++ b/drivers/s390/cio/vfio_ccw_async.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Async I/O region for vfio_ccw + * + * Copyright Red Hat, Inc. 2019 + * + * Author(s): Cornelia Huck <cohuck@redhat.com> + */ + +#include <linux/vfio.h> +#include <linux/mdev.h> + +#include "vfio_ccw_private.h" + +static ssize_t vfio_ccw_async_region_read(struct vfio_ccw_private *private, + char __user *buf, size_t count, + loff_t *ppos) +{ + unsigned int i = VFIO_CCW_OFFSET_TO_INDEX(*ppos) - VFIO_CCW_NUM_REGIONS; + loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; + struct ccw_cmd_region *region; + int ret; + + if (pos + count > sizeof(*region)) + return -EINVAL; + + mutex_lock(&private->io_mutex); + region = private->region[i].data; + if (copy_to_user(buf, (void *)region + pos, count)) + ret = -EFAULT; + else + ret = count; + mutex_unlock(&private->io_mutex); + return ret; +} + +static ssize_t vfio_ccw_async_region_write(struct vfio_ccw_private *private, + const char __user *buf, size_t count, + loff_t *ppos) +{ + unsigned int i = VFIO_CCW_OFFSET_TO_INDEX(*ppos) - VFIO_CCW_NUM_REGIONS; + loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; + struct ccw_cmd_region *region; + int ret; + + if (pos + count > sizeof(*region)) + return -EINVAL; + + if (!mutex_trylock(&private->io_mutex)) + return -EAGAIN; + + region = private->region[i].data; + if (copy_from_user((void *)region + pos, buf, count)) { + ret = -EFAULT; + goto out_unlock; + } + + vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_ASYNC_REQ); + + ret = region->ret_code ? region->ret_code : count; + +out_unlock: + mutex_unlock(&private->io_mutex); + return ret; +} + +static void vfio_ccw_async_region_release(struct vfio_ccw_private *private, + struct vfio_ccw_region *region) +{ + +} + +const struct vfio_ccw_regops vfio_ccw_async_region_ops = { + .read = vfio_ccw_async_region_read, + .write = vfio_ccw_async_region_write, + .release = vfio_ccw_async_region_release, +}; + +int vfio_ccw_register_async_dev_regions(struct vfio_ccw_private *private) +{ + return vfio_ccw_register_dev_region(private, + VFIO_REGION_SUBTYPE_CCW_ASYNC_CMD, + &vfio_ccw_async_region_ops, + sizeof(struct ccw_cmd_region), + VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE, + private->cmd_region); +} diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 384b3987eeb4..0e79799e9a71 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -362,6 +362,7 @@ static void cp_unpin_free(struct channel_program *cp) struct ccwchain *chain, *temp; int i; + cp->initialized = false; list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) { for (i = 0; i < chain->ch_len; i++) { pfn_array_table_unpin_free(chain->ch_pat + i, @@ -732,6 +733,9 @@ int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb) */ cp->orb.cmd.c64 = 1; + if (!ret) + cp->initialized = true; + return ret; } @@ -746,7 +750,8 @@ int cp_init(struct channel_program *cp, struct device *mdev, union orb *orb) */ void cp_free(struct channel_program *cp) { - cp_unpin_free(cp); + if (cp->initialized) + cp_unpin_free(cp); } /** @@ -791,6 +796,10 @@ int cp_prefetch(struct channel_program *cp) struct ccwchain *chain; int len, idx, ret; + /* this is an error in the caller */ + if (!cp->initialized) + return -EINVAL; + list_for_each_entry(chain, &cp->ccwchain_list, next) { len = chain->ch_len; for (idx = 0; idx < len; idx++) { @@ -826,6 +835,10 @@ union orb *cp_get_orb(struct channel_program *cp, u32 intparm, u8 lpm) struct ccwchain *chain; struct ccw1 *cpa; + /* this is an error in the caller */ + if (!cp->initialized) + return NULL; + orb = &cp->orb; orb->cmd.intparm = intparm; @@ -862,6 +875,9 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw) u32 cpa = scsw->cmd.cpa; u32 ccw_head; + if (!cp->initialized) + return; + /* * LATER: * For now, only update the cmd.cpa part. We may need to deal with @@ -898,6 +914,9 @@ bool cp_iova_pinned(struct channel_program *cp, u64 iova) struct ccwchain *chain; int i; + if (!cp->initialized) + return false; + list_for_each_entry(chain, &cp->ccwchain_list, next) { for (i = 0; i < chain->ch_len; i++) if (pfn_array_table_iova_pinned(chain->ch_pat + i, diff --git a/drivers/s390/cio/vfio_ccw_cp.h b/drivers/s390/cio/vfio_ccw_cp.h index a4b74fb1aa57..3c20cd208da5 100644 --- a/drivers/s390/cio/vfio_ccw_cp.h +++ b/drivers/s390/cio/vfio_ccw_cp.h @@ -21,6 +21,7 @@ * @ccwchain_list: list head of ccwchains * @orb: orb for the currently processed ssch request * @mdev: the mediated device to perform page pinning/unpinning + * @initialized: whether this instance is actually initialized * * @ccwchain_list is the head of a ccwchain list, that contents the * translated result of the guest channel program that pointed out by @@ -30,6 +31,7 @@ struct channel_program { struct list_head ccwchain_list; union orb orb; struct device *mdev; + bool initialized; }; extern int cp_init(struct channel_program *cp, struct device *mdev, diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 0b3b9de45c60..ee8767f5845a 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -3,9 +3,11 @@ * VFIO based Physical Subchannel device driver * * Copyright IBM Corp. 2017 + * Copyright Red Hat, Inc. 2019 * * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> + * Cornelia Huck <cohuck@redhat.com> */ #include <linux/module.h> @@ -23,6 +25,7 @@ struct workqueue_struct *vfio_ccw_work_q; static struct kmem_cache *vfio_ccw_io_region; +static struct kmem_cache *vfio_ccw_cmd_region; /* * Helpers @@ -40,26 +43,30 @@ int vfio_ccw_sch_quiesce(struct subchannel *sch) if (ret != -EBUSY) goto out_unlock; + iretry = 255; do { - iretry = 255; ret = cio_cancel_halt_clear(sch, &iretry); - while (ret == -EBUSY) { - /* - * Flush all I/O and wait for - * cancel/halt/clear completion. - */ - private->completion = &completion; - spin_unlock_irq(sch->lock); - wait_for_completion_timeout(&completion, 3*HZ); + if (ret == -EIO) { + pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n", + sch->schid.ssid, sch->schid.sch_no); + break; + } + + /* + * Flush all I/O and wait for + * cancel/halt/clear completion. + */ + private->completion = &completion; + spin_unlock_irq(sch->lock); - spin_lock_irq(sch->lock); - private->completion = NULL; - flush_workqueue(vfio_ccw_work_q); - ret = cio_cancel_halt_clear(sch, &iretry); - }; + if (ret == -EBUSY) + wait_for_completion_timeout(&completion, 3*HZ); + private->completion = NULL; + flush_workqueue(vfio_ccw_work_q); + spin_lock_irq(sch->lock); ret = cio_disable_subchannel(sch); } while (ret == -EBUSY); out_unlock: @@ -84,7 +91,9 @@ static void vfio_ccw_sch_io_todo(struct work_struct *work) if (is_final) cp_free(&private->cp); } + mutex_lock(&private->io_mutex); memcpy(private->io_region->irb_area, irb, sizeof(*irb)); + mutex_unlock(&private->io_mutex); if (private->io_trigger) eventfd_signal(private->io_trigger, 1); @@ -108,7 +117,7 @@ static int vfio_ccw_sch_probe(struct subchannel *sch) { struct pmcw *pmcw = &sch->schib.pmcw; struct vfio_ccw_private *private; - int ret; + int ret = -ENOMEM; if (pmcw->qf) { dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n", @@ -122,13 +131,17 @@ static int vfio_ccw_sch_probe(struct subchannel *sch) private->io_region = kmem_cache_zalloc(vfio_ccw_io_region, GFP_KERNEL | GFP_DMA); - if (!private->io_region) { - kfree(private); - return -ENOMEM; - } + if (!private->io_region) + goto out_free; + + private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region, + GFP_KERNEL | GFP_DMA); + if (!private->cmd_region) + goto out_free; private->sch = sch; dev_set_drvdata(&sch->dev, private); + mutex_init(&private->io_mutex); spin_lock_irq(sch->lock); private->state = VFIO_CCW_STATE_NOT_OPER; @@ -152,7 +165,10 @@ out_disable: cio_disable_subchannel(sch); out_free: dev_set_drvdata(&sch->dev, NULL); - kmem_cache_free(vfio_ccw_io_region, private->io_region); + if (private->cmd_region) + kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); + if (private->io_region) + kmem_cache_free(vfio_ccw_io_region, private->io_region); kfree(private); return ret; } @@ -167,6 +183,7 @@ static int vfio_ccw_sch_remove(struct subchannel *sch) dev_set_drvdata(&sch->dev, NULL); + kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region); kmem_cache_free(vfio_ccw_io_region, private->io_region); kfree(private); @@ -241,7 +258,7 @@ static struct css_driver vfio_ccw_sch_driver = { static int __init vfio_ccw_sch_init(void) { - int ret; + int ret = -ENOMEM; vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw"); if (!vfio_ccw_work_q) @@ -251,20 +268,30 @@ static int __init vfio_ccw_sch_init(void) sizeof(struct ccw_io_region), 0, SLAB_ACCOUNT, 0, sizeof(struct ccw_io_region), NULL); - if (!vfio_ccw_io_region) { - destroy_workqueue(vfio_ccw_work_q); - return -ENOMEM; - } + if (!vfio_ccw_io_region) + goto out_err; + + vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region", + sizeof(struct ccw_cmd_region), 0, + SLAB_ACCOUNT, 0, + sizeof(struct ccw_cmd_region), NULL); + if (!vfio_ccw_cmd_region) + goto out_err; isc_register(VFIO_CCW_ISC); ret = css_driver_register(&vfio_ccw_sch_driver); if (ret) { isc_unregister(VFIO_CCW_ISC); - kmem_cache_destroy(vfio_ccw_io_region); - destroy_workqueue(vfio_ccw_work_q); + goto out_err; } return ret; + +out_err: + kmem_cache_destroy(vfio_ccw_cmd_region); + kmem_cache_destroy(vfio_ccw_io_region); + destroy_workqueue(vfio_ccw_work_q); + return ret; } static void __exit vfio_ccw_sch_exit(void) diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index cab17865aafe..49d9d3da0282 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -3,8 +3,10 @@ * Finite state machine for vfio-ccw device handling * * Copyright IBM Corp. 2017 + * Copyright Red Hat, Inc. 2019 * * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> + * Cornelia Huck <cohuck@redhat.com> */ #include <linux/vfio.h> @@ -28,9 +30,12 @@ static int fsm_io_helper(struct vfio_ccw_private *private) sch = private->sch; spin_lock_irqsave(sch->lock, flags); - private->state = VFIO_CCW_STATE_BUSY; orb = cp_get_orb(&private->cp, (u32)(addr_t)sch, sch->lpm); + if (!orb) { + ret = -EIO; + goto out; + } /* Issue "Start Subchannel" */ ccode = ssch(sch->schid, orb); @@ -42,6 +47,7 @@ static int fsm_io_helper(struct vfio_ccw_private *private) */ sch->schib.scsw.cmd.actl |= SCSW_ACTL_START_PEND; ret = 0; + private->state = VFIO_CCW_STATE_CP_PENDING; break; case 1: /* Status pending */ case 2: /* Busy */ @@ -64,6 +70,76 @@ static int fsm_io_helper(struct vfio_ccw_private *private) default: ret = ccode; } +out: + spin_unlock_irqrestore(sch->lock, flags); + return ret; +} + +static int fsm_do_halt(struct vfio_ccw_private *private) +{ + struct subchannel *sch; + unsigned long flags; + int ccode; + int ret; + + sch = private->sch; + + spin_lock_irqsave(sch->lock, flags); + + /* Issue "Halt Subchannel" */ + ccode = hsch(sch->schid); + + switch (ccode) { + case 0: + /* + * Initialize device status information + */ + sch->schib.scsw.cmd.actl |= SCSW_ACTL_HALT_PEND; + ret = 0; + break; + case 1: /* Status pending */ + case 2: /* Busy */ + ret = -EBUSY; + break; + case 3: /* Device not operational */ + ret = -ENODEV; + break; + default: + ret = ccode; + } + spin_unlock_irqrestore(sch->lock, flags); + return ret; +} + +static int fsm_do_clear(struct vfio_ccw_private *private) +{ + struct subchannel *sch; + unsigned long flags; + int ccode; + int ret; + + sch = private->sch; + + spin_lock_irqsave(sch->lock, flags); + + /* Issue "Clear Subchannel" */ + ccode = csch(sch->schid); + + switch (ccode) { + case 0: + /* + * Initialize device status information + */ + sch->schib.scsw.cmd.actl = SCSW_ACTL_CLEAR_PEND; + /* TODO: check what else we might need to clear */ + ret = 0; + break; + case 3: /* Device not operational */ + ret = -ENODEV; + break; + default: + ret = ccode; + } spin_unlock_irqrestore(sch->lock, flags); return ret; } @@ -102,6 +178,30 @@ static void fsm_io_busy(struct vfio_ccw_private *private, private->io_region->ret_code = -EBUSY; } +static void fsm_io_retry(struct vfio_ccw_private *private, + enum vfio_ccw_event event) +{ + private->io_region->ret_code = -EAGAIN; +} + +static void fsm_async_error(struct vfio_ccw_private *private, + enum vfio_ccw_event event) +{ + struct ccw_cmd_region *cmd_region = private->cmd_region; + + pr_err("vfio-ccw: FSM: %s request from state:%d\n", + cmd_region->command == VFIO_CCW_ASYNC_CMD_HSCH ? "halt" : + cmd_region->command == VFIO_CCW_ASYNC_CMD_CSCH ? "clear" : + "<unknown>", private->state); + cmd_region->ret_code = -EIO; +} + +static void fsm_async_retry(struct vfio_ccw_private *private, + enum vfio_ccw_event event) +{ + private->cmd_region->ret_code = -EAGAIN; +} + static void fsm_disabled_irq(struct vfio_ccw_private *private, enum vfio_ccw_event event) { @@ -130,8 +230,7 @@ static void fsm_io_request(struct vfio_ccw_private *private, struct mdev_device *mdev = private->mdev; char *errstr = "request"; - private->state = VFIO_CCW_STATE_BUSY; - + private->state = VFIO_CCW_STATE_CP_PROCESSING; memcpy(scsw, io_region->scsw_area, sizeof(*scsw)); if (scsw->cmd.fctl & SCSW_FCTL_START_FUNC) { @@ -166,22 +265,42 @@ static void fsm_io_request(struct vfio_ccw_private *private, } return; } else if (scsw->cmd.fctl & SCSW_FCTL_HALT_FUNC) { - /* XXX: Handle halt. */ + /* halt is handled via the async cmd region */ io_region->ret_code = -EOPNOTSUPP; goto err_out; } else if (scsw->cmd.fctl & SCSW_FCTL_CLEAR_FUNC) { - /* XXX: Handle clear. */ + /* clear is handled via the async cmd region */ io_region->ret_code = -EOPNOTSUPP; goto err_out; } err_out: - private->state = VFIO_CCW_STATE_IDLE; trace_vfio_ccw_io_fctl(scsw->cmd.fctl, get_schid(private), io_region->ret_code, errstr); } /* + * Deal with an async request from userspace. + */ +static void fsm_async_request(struct vfio_ccw_private *private, + enum vfio_ccw_event event) +{ + struct ccw_cmd_region *cmd_region = private->cmd_region; + + switch (cmd_region->command) { + case VFIO_CCW_ASYNC_CMD_HSCH: + cmd_region->ret_code = fsm_do_halt(private); + break; + case VFIO_CCW_ASYNC_CMD_CSCH: + cmd_region->ret_code = fsm_do_clear(private); + break; + default: + /* should not happen? */ + cmd_region->ret_code = -EINVAL; + } +} + +/* * Got an interrupt for a normal io (state busy). */ static void fsm_irq(struct vfio_ccw_private *private, @@ -204,21 +323,31 @@ fsm_func_t *vfio_ccw_jumptable[NR_VFIO_CCW_STATES][NR_VFIO_CCW_EVENTS] = { [VFIO_CCW_STATE_NOT_OPER] = { [VFIO_CCW_EVENT_NOT_OPER] = fsm_nop, [VFIO_CCW_EVENT_IO_REQ] = fsm_io_error, + [VFIO_CCW_EVENT_ASYNC_REQ] = fsm_async_error, [VFIO_CCW_EVENT_INTERRUPT] = fsm_disabled_irq, }, [VFIO_CCW_STATE_STANDBY] = { [VFIO_CCW_EVENT_NOT_OPER] = fsm_notoper, [VFIO_CCW_EVENT_IO_REQ] = fsm_io_error, + [VFIO_CCW_EVENT_ASYNC_REQ] = fsm_async_error, [VFIO_CCW_EVENT_INTERRUPT] = fsm_irq, }, [VFIO_CCW_STATE_IDLE] = { [VFIO_CCW_EVENT_NOT_OPER] = fsm_notoper, [VFIO_CCW_EVENT_IO_REQ] = fsm_io_request, + [VFIO_CCW_EVENT_ASYNC_REQ] = fsm_async_request, + [VFIO_CCW_EVENT_INTERRUPT] = fsm_irq, + }, + [VFIO_CCW_STATE_CP_PROCESSING] = { + [VFIO_CCW_EVENT_NOT_OPER] = fsm_notoper, + [VFIO_CCW_EVENT_IO_REQ] = fsm_io_retry, + [VFIO_CCW_EVENT_ASYNC_REQ] = fsm_async_retry, [VFIO_CCW_EVENT_INTERRUPT] = fsm_irq, }, - [VFIO_CCW_STATE_BUSY] = { + [VFIO_CCW_STATE_CP_PENDING] = { [VFIO_CCW_EVENT_NOT_OPER] = fsm_notoper, [VFIO_CCW_EVENT_IO_REQ] = fsm_io_busy, + [VFIO_CCW_EVENT_ASYNC_REQ] = fsm_async_request, [VFIO_CCW_EVENT_INTERRUPT] = fsm_irq, }, }; diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index f673e106c041..5eb61116ca6f 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -3,13 +3,17 @@ * Physical device callbacks for vfio_ccw * * Copyright IBM Corp. 2017 + * Copyright Red Hat, Inc. 2019 * * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> + * Cornelia Huck <cohuck@redhat.com> */ #include <linux/vfio.h> #include <linux/mdev.h> +#include <linux/nospec.h> +#include <linux/slab.h> #include "vfio_ccw_private.h" @@ -130,11 +134,12 @@ static int vfio_ccw_mdev_remove(struct mdev_device *mdev) if ((private->state != VFIO_CCW_STATE_NOT_OPER) && (private->state != VFIO_CCW_STATE_STANDBY)) { - if (!vfio_ccw_mdev_reset(mdev)) + if (!vfio_ccw_sch_quiesce(private->sch)) private->state = VFIO_CCW_STATE_STANDBY; /* The state will be NOT_OPER on error. */ } + cp_free(&private->cp); private->mdev = NULL; atomic_inc(&private->avail); @@ -146,20 +151,66 @@ static int vfio_ccw_mdev_open(struct mdev_device *mdev) struct vfio_ccw_private *private = dev_get_drvdata(mdev_parent_dev(mdev)); unsigned long events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; + int ret; private->nb.notifier_call = vfio_ccw_mdev_notifier; - return vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, - &events, &private->nb); + ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, + &events, &private->nb); + if (ret) + return ret; + + ret = vfio_ccw_register_async_dev_regions(private); + if (ret) + vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, + &private->nb); + return ret; } static void vfio_ccw_mdev_release(struct mdev_device *mdev) { struct vfio_ccw_private *private = dev_get_drvdata(mdev_parent_dev(mdev)); + int i; + if ((private->state != VFIO_CCW_STATE_NOT_OPER) && + (private->state != VFIO_CCW_STATE_STANDBY)) { + if (!vfio_ccw_mdev_reset(mdev)) + private->state = VFIO_CCW_STATE_STANDBY; + /* The state will be NOT_OPER on error. */ + } + + cp_free(&private->cp); vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &private->nb); + + for (i = 0; i < private->num_regions; i++) + private->region[i].ops->release(private, &private->region[i]); + + private->num_regions = 0; + kfree(private->region); + private->region = NULL; +} + +static ssize_t vfio_ccw_mdev_read_io_region(struct vfio_ccw_private *private, + char __user *buf, size_t count, + loff_t *ppos) +{ + loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; + struct ccw_io_region *region; + int ret; + + if (pos + count > sizeof(*region)) + return -EINVAL; + + mutex_lock(&private->io_mutex); + region = private->io_region; + if (copy_to_user(buf, (void *)region + pos, count)) + ret = -EFAULT; + else + ret = count; + mutex_unlock(&private->io_mutex); + return ret; } static ssize_t vfio_ccw_mdev_read(struct mdev_device *mdev, @@ -167,18 +218,54 @@ static ssize_t vfio_ccw_mdev_read(struct mdev_device *mdev, size_t count, loff_t *ppos) { + unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); struct vfio_ccw_private *private; + + private = dev_get_drvdata(mdev_parent_dev(mdev)); + + if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) + return -EINVAL; + + switch (index) { + case VFIO_CCW_CONFIG_REGION_INDEX: + return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); + default: + index -= VFIO_CCW_NUM_REGIONS; + return private->region[index].ops->read(private, buf, count, + ppos); + } + + return -EINVAL; +} + +static ssize_t vfio_ccw_mdev_write_io_region(struct vfio_ccw_private *private, + const char __user *buf, + size_t count, loff_t *ppos) +{ + loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; struct ccw_io_region *region; + int ret; - if (*ppos + count > sizeof(*region)) + if (pos + count > sizeof(*region)) return -EINVAL; - private = dev_get_drvdata(mdev_parent_dev(mdev)); + if (!mutex_trylock(&private->io_mutex)) + return -EAGAIN; + region = private->io_region; - if (copy_to_user(buf, (void *)region + *ppos, count)) - return -EFAULT; + if (copy_from_user((void *)region + pos, buf, count)) { + ret = -EFAULT; + goto out_unlock; + } - return count; + vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); + if (region->ret_code != 0) + private->state = VFIO_CCW_STATE_IDLE; + ret = (region->ret_code != 0) ? region->ret_code : count; + +out_unlock: + mutex_unlock(&private->io_mutex); + return ret; } static ssize_t vfio_ccw_mdev_write(struct mdev_device *mdev, @@ -186,42 +273,47 @@ static ssize_t vfio_ccw_mdev_write(struct mdev_device *mdev, size_t count, loff_t *ppos) { + unsigned int index = VFIO_CCW_OFFSET_TO_INDEX(*ppos); struct vfio_ccw_private *private; - struct ccw_io_region *region; - - if (*ppos + count > sizeof(*region)) - return -EINVAL; private = dev_get_drvdata(mdev_parent_dev(mdev)); - if (private->state != VFIO_CCW_STATE_IDLE) - return -EACCES; - region = private->io_region; - if (copy_from_user((void *)region + *ppos, buf, count)) - return -EFAULT; + if (index >= VFIO_CCW_NUM_REGIONS + private->num_regions) + return -EINVAL; - vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_IO_REQ); - if (region->ret_code != 0) { - private->state = VFIO_CCW_STATE_IDLE; - return region->ret_code; + switch (index) { + case VFIO_CCW_CONFIG_REGION_INDEX: + return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); + default: + index -= VFIO_CCW_NUM_REGIONS; + return private->region[index].ops->write(private, buf, count, + ppos); } - return count; + return -EINVAL; } -static int vfio_ccw_mdev_get_device_info(struct vfio_device_info *info) +static int vfio_ccw_mdev_get_device_info(struct vfio_device_info *info, + struct mdev_device *mdev) { + struct vfio_ccw_private *private; + + private = dev_get_drvdata(mdev_parent_dev(mdev)); info->flags = VFIO_DEVICE_FLAGS_CCW | VFIO_DEVICE_FLAGS_RESET; - info->num_regions = VFIO_CCW_NUM_REGIONS; + info->num_regions = VFIO_CCW_NUM_REGIONS + private->num_regions; info->num_irqs = VFIO_CCW_NUM_IRQS; return 0; } static int vfio_ccw_mdev_get_region_info(struct vfio_region_info *info, - u16 *cap_type_id, - void **cap_type) + struct mdev_device *mdev, + unsigned long arg) { + struct vfio_ccw_private *private; + int i; + + private = dev_get_drvdata(mdev_parent_dev(mdev)); switch (info->index) { case VFIO_CCW_CONFIG_REGION_INDEX: info->offset = 0; @@ -229,9 +321,55 @@ static int vfio_ccw_mdev_get_region_info(struct vfio_region_info *info, info->flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE; return 0; - default: - return -EINVAL; + default: /* all other regions are handled via capability chain */ + { + struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; + struct vfio_region_info_cap_type cap_type = { + .header.id = VFIO_REGION_INFO_CAP_TYPE, + .header.version = 1 }; + int ret; + + if (info->index >= + VFIO_CCW_NUM_REGIONS + private->num_regions) + return -EINVAL; + + info->index = array_index_nospec(info->index, + VFIO_CCW_NUM_REGIONS + + private->num_regions); + + i = info->index - VFIO_CCW_NUM_REGIONS; + + info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); + info->size = private->region[i].size; + info->flags = private->region[i].flags; + + cap_type.type = private->region[i].type; + cap_type.subtype = private->region[i].subtype; + + ret = vfio_info_add_capability(&caps, &cap_type.header, + sizeof(cap_type)); + if (ret) + return ret; + + info->flags |= VFIO_REGION_INFO_FLAG_CAPS; + if (info->argsz < sizeof(*info) + caps.size) { + info->argsz = sizeof(*info) + caps.size; + info->cap_offset = 0; + } else { + vfio_info_cap_shift(&caps, sizeof(*info)); + if (copy_to_user((void __user *)arg + sizeof(*info), + caps.buf, caps.size)) { + kfree(caps.buf); + return -EFAULT; + } + info->cap_offset = sizeof(*info); + } + + kfree(caps.buf); + } + } + return 0; } static int vfio_ccw_mdev_get_irq_info(struct vfio_irq_info *info) @@ -308,6 +446,32 @@ static int vfio_ccw_mdev_set_irqs(struct mdev_device *mdev, } } +int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, + unsigned int subtype, + const struct vfio_ccw_regops *ops, + size_t size, u32 flags, void *data) +{ + struct vfio_ccw_region *region; + + region = krealloc(private->region, + (private->num_regions + 1) * sizeof(*region), + GFP_KERNEL); + if (!region) + return -ENOMEM; + + private->region = region; + private->region[private->num_regions].type = VFIO_REGION_TYPE_CCW; + private->region[private->num_regions].subtype = subtype; + private->region[private->num_regions].ops = ops; + private->region[private->num_regions].size = size; + private->region[private->num_regions].flags = flags; + private->region[private->num_regions].data = data; + + private->num_regions++; + + return 0; +} + static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev, unsigned int cmd, unsigned long arg) @@ -328,7 +492,7 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev, if (info.argsz < minsz) return -EINVAL; - ret = vfio_ccw_mdev_get_device_info(&info); + ret = vfio_ccw_mdev_get_device_info(&info, mdev); if (ret) return ret; @@ -337,8 +501,6 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev, case VFIO_DEVICE_GET_REGION_INFO: { struct vfio_region_info info; - u16 cap_type_id = 0; - void *cap_type = NULL; minsz = offsetofend(struct vfio_region_info, offset); @@ -348,8 +510,7 @@ static ssize_t vfio_ccw_mdev_ioctl(struct mdev_device *mdev, if (info.argsz < minsz) return -EINVAL; - ret = vfio_ccw_mdev_get_region_info(&info, &cap_type_id, - &cap_type); + ret = vfio_ccw_mdev_get_region_info(&info, mdev, arg); if (ret) return ret; diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index 08e9a7dc9176..f1092c3dc1b1 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -3,9 +3,11 @@ * Private stuff for vfio_ccw driver * * Copyright IBM Corp. 2017 + * Copyright Red Hat, Inc. 2019 * * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com> * Xiao Feng Ren <renxiaof@linux.vnet.ibm.com> + * Cornelia Huck <cohuck@redhat.com> */ #ifndef _VFIO_CCW_PRIVATE_H_ @@ -19,6 +21,40 @@ #include "css.h" #include "vfio_ccw_cp.h" +#define VFIO_CCW_OFFSET_SHIFT 10 +#define VFIO_CCW_OFFSET_TO_INDEX(off) (off >> VFIO_CCW_OFFSET_SHIFT) +#define VFIO_CCW_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_CCW_OFFSET_SHIFT) +#define VFIO_CCW_OFFSET_MASK (((u64)(1) << VFIO_CCW_OFFSET_SHIFT) - 1) + +/* capability chain handling similar to vfio-pci */ +struct vfio_ccw_private; +struct vfio_ccw_region; + +struct vfio_ccw_regops { + ssize_t (*read)(struct vfio_ccw_private *private, char __user *buf, + size_t count, loff_t *ppos); + ssize_t (*write)(struct vfio_ccw_private *private, + const char __user *buf, size_t count, loff_t *ppos); + void (*release)(struct vfio_ccw_private *private, + struct vfio_ccw_region *region); +}; + +struct vfio_ccw_region { + u32 type; + u32 subtype; + const struct vfio_ccw_regops *ops; + void *data; + size_t size; + u32 flags; +}; + +int vfio_ccw_register_dev_region(struct vfio_ccw_private *private, + unsigned int subtype, + const struct vfio_ccw_regops *ops, + size_t size, u32 flags, void *data); + +int vfio_ccw_register_async_dev_regions(struct vfio_ccw_private *private); + /** * struct vfio_ccw_private * @sch: pointer to the subchannel @@ -28,6 +64,10 @@ * @mdev: pointer to the mediated device * @nb: notifier for vfio events * @io_region: MMIO region to input/output I/O arguments/results + * @io_mutex: protect against concurrent update of I/O regions + * @region: additional regions for other subchannel operations + * @cmd_region: MMIO region for asynchronous I/O commands other than START + * @num_regions: number of additional regions * @cp: channel program for the current I/O operation * @irb: irb info received from interrupt * @scsw: scsw info @@ -42,6 +82,10 @@ struct vfio_ccw_private { struct mdev_device *mdev; struct notifier_block nb; struct ccw_io_region *io_region; + struct mutex io_mutex; + struct vfio_ccw_region *region; + struct ccw_cmd_region *cmd_region; + int num_regions; struct channel_program cp; struct irb irb; @@ -63,7 +107,8 @@ enum vfio_ccw_state { VFIO_CCW_STATE_NOT_OPER, VFIO_CCW_STATE_STANDBY, VFIO_CCW_STATE_IDLE, - VFIO_CCW_STATE_BUSY, + VFIO_CCW_STATE_CP_PROCESSING, + VFIO_CCW_STATE_CP_PENDING, /* last element! */ NR_VFIO_CCW_STATES }; @@ -75,6 +120,7 @@ enum vfio_ccw_event { VFIO_CCW_EVENT_NOT_OPER, VFIO_CCW_EVENT_IO_REQ, VFIO_CCW_EVENT_INTERRUPT, + VFIO_CCW_EVENT_ASYNC_REQ, /* last element! */ NR_VFIO_CCW_EVENTS }; diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 1546389d71db..cc30e4f07fff 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -116,7 +116,7 @@ static int user_set_domain; static struct bus_type ap_bus_type; /* Adapter interrupt definitions */ -static void ap_interrupt_handler(struct airq_struct *airq); +static void ap_interrupt_handler(struct airq_struct *airq, bool floating); static int ap_airq_flag; @@ -393,7 +393,7 @@ static enum hrtimer_restart ap_poll_timeout(struct hrtimer *unused) * ap_interrupt_handler() - Schedule ap_tasklet on interrupt * @airq: pointer to adapter interrupt descriptor */ -static void ap_interrupt_handler(struct airq_struct *airq) +static void ap_interrupt_handler(struct airq_struct *airq, bool floating) { inc_irq_stat(IRQIO_APB); if (!ap_suspend_flag) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 689c2af7026a..c31b2d31cd83 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -659,6 +659,7 @@ static long zcrypt_rsa_modexpo(struct ap_perms *perms, trace_s390_zcrypt_req(mex, TP_ICARSAMODEXPO); if (mex->outputdatalength < mex->inputdatalength) { + func_code = 0; rc = -EINVAL; goto out; } @@ -742,6 +743,7 @@ static long zcrypt_rsa_crt(struct ap_perms *perms, trace_s390_zcrypt_req(crt, TP_ICARSACRT); if (crt->outputdatalength < crt->inputdatalength) { + func_code = 0; rc = -EINVAL; goto out; } @@ -951,6 +953,7 @@ static long zcrypt_send_ep11_cprb(struct ap_perms *perms, targets = kcalloc(target_num, sizeof(*targets), GFP_KERNEL); if (!targets) { + func_code = 0; rc = -ENOMEM; goto out; } @@ -958,6 +961,7 @@ static long zcrypt_send_ep11_cprb(struct ap_perms *perms, uptr = (struct ep11_target_dev __force __user *) xcrb->targets; if (copy_from_user(targets, uptr, target_num * sizeof(*targets))) { + func_code = 0; rc = -EFAULT; goto out_free; } diff --git a/drivers/s390/net/ism.h b/drivers/s390/net/ism.h index 0aab90817326..66eac2b9704d 100644 --- a/drivers/s390/net/ism.h +++ b/drivers/s390/net/ism.h @@ -6,6 +6,7 @@ #include <linux/types.h> #include <linux/pci.h> #include <net/smc.h> +#include <asm/pci_insn.h> #define UTIL_STR_LEN 16 @@ -194,8 +195,6 @@ struct ism_dev { struct pci_dev *pdev; struct smcd_dev *smcd; - void __iomem *ctl; - struct ism_sba *sba; dma_addr_t sba_dma_addr; DECLARE_BITMAP(sba_bitmap, ISM_NR_DMBS); @@ -209,13 +208,37 @@ struct ism_dev { #define ISM_CREATE_REQ(dmb, idx, sf, offset) \ ((dmb) | (idx) << 24 | (sf) << 23 | (offset)) +static inline void __ism_read_cmd(struct ism_dev *ism, void *data, + unsigned long offset, unsigned long len) +{ + struct zpci_dev *zdev = to_zpci(ism->pdev); + u64 req = ZPCI_CREATE_REQ(zdev->fh, 2, 8); + + while (len > 0) { + __zpci_load(data, req, offset); + offset += 8; + data += 8; + len -= 8; + } +} + +static inline void __ism_write_cmd(struct ism_dev *ism, void *data, + unsigned long offset, unsigned long len) +{ + struct zpci_dev *zdev = to_zpci(ism->pdev); + u64 req = ZPCI_CREATE_REQ(zdev->fh, 2, len); + + if (len) + __zpci_store_block(data, req, offset); +} + static inline int __ism_move(struct ism_dev *ism, u64 dmb_req, void *data, unsigned int size) { struct zpci_dev *zdev = to_zpci(ism->pdev); u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, size); - return zpci_write_block(req, data, dmb_req); + return __zpci_store_block(data, req, dmb_req); } #endif /* S390_ISM_H */ diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index 3e132592c1fe..4fc2056bd227 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -38,19 +38,18 @@ static int ism_cmd(struct ism_dev *ism, void *cmd) struct ism_req_hdr *req = cmd; struct ism_resp_hdr *resp = cmd; - memcpy_toio(ism->ctl + sizeof(*req), req + 1, req->len - sizeof(*req)); - memcpy_toio(ism->ctl, req, sizeof(*req)); + __ism_write_cmd(ism, req + 1, sizeof(*req), req->len - sizeof(*req)); + __ism_write_cmd(ism, req, 0, sizeof(*req)); WRITE_ONCE(resp->ret, ISM_ERROR); - memcpy_fromio(resp, ism->ctl, sizeof(*resp)); + __ism_read_cmd(ism, resp, 0, sizeof(*resp)); if (resp->ret) { debug_text_event(ism_debug_info, 0, "cmd failure"); debug_event(ism_debug_info, 0, resp, sizeof(*resp)); goto out; } - memcpy_fromio(resp + 1, ism->ctl + sizeof(*resp), - resp->len - sizeof(*resp)); + __ism_read_cmd(ism, resp + 1, sizeof(*resp), resp->len - sizeof(*resp)); out: return resp->ret; } @@ -512,13 +511,9 @@ static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto err_disable; - ism->ctl = pci_iomap(pdev, 2, 0); - if (!ism->ctl) - goto err_resource; - ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (ret) - goto err_unmap; + goto err_resource; dma_set_seg_boundary(&pdev->dev, SZ_1M - 1); dma_set_max_seg_size(&pdev->dev, SZ_1M); @@ -527,7 +522,7 @@ static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id) ism->smcd = smcd_alloc_dev(&pdev->dev, dev_name(&pdev->dev), &ism_ops, ISM_NR_DMBS); if (!ism->smcd) - goto err_unmap; + goto err_resource; ism->smcd->priv = ism; ret = ism_dev_init(ism); @@ -538,8 +533,6 @@ static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_free: smcd_free_dev(ism->smcd); -err_unmap: - pci_iounmap(pdev, ism->ctl); err_resource: pci_release_mem_regions(pdev); err_disable: @@ -568,7 +561,6 @@ static void ism_remove(struct pci_dev *pdev) ism_dev_exit(ism); smcd_free_dev(ism->smcd); - pci_iounmap(pdev, ism->ctl); pci_release_mem_regions(pdev); pci_disable_device(pdev); dev_set_drvdata(&pdev->dev, NULL); diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 74c328321889..991420caa4f2 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -182,7 +182,7 @@ static void drop_airq_indicator(struct virtqueue *vq, struct airq_info *info) write_unlock_irqrestore(&info->lock, flags); } -static void virtio_airq_handler(struct airq_struct *airq) +static void virtio_airq_handler(struct airq_struct *airq, bool floating) { struct airq_info *info = container_of(airq, struct airq_info, airq); unsigned long ai; diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h index d79abca81a52..d1779d442aa5 100644 --- a/include/asm-generic/sections.h +++ b/include/asm-generic/sections.h @@ -77,6 +77,20 @@ static inline int arch_is_kernel_data(unsigned long addr) } #endif +/* + * Check if an address is part of freed initmem. This is needed on architectures + * with virt == phys kernel mapping, for code that wants to check if an address + * is part of a static object within [_stext, _end]. After initmem is freed, + * memory can be allocated from it, and such allocations would then have + * addresses within the range [_stext, _end]. + */ +#ifndef arch_is_kernel_initmem_freed +static inline int arch_is_kernel_initmem_freed(unsigned long addr) +{ + return 0; +} +#endif + /** * memory_contains - checks if an object is contained within a memory region * @begin: virtual address of the beginning of the memory region diff --git a/include/linux/ima.h b/include/linux/ima.h index dc12fbcf484c..fd9f7cf4cdf5 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -31,7 +31,7 @@ extern void ima_post_path_mknod(struct dentry *dentry); extern void ima_add_kexec_buffer(struct kimage *image); #endif -#if defined(CONFIG_X86) && defined(CONFIG_EFI) +#if (defined(CONFIG_X86) && defined(CONFIG_EFI)) || defined(CONFIG_S390) extern bool arch_ima_get_secureboot(void); extern const char * const *arch_get_ima_policy(void); #else diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 02bb7ad6e986..8f10748dac79 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -353,6 +353,10 @@ struct vfio_region_gfx_edid { #define VFIO_DEVICE_GFX_LINK_STATE_DOWN 2 }; +#define VFIO_REGION_TYPE_CCW (2) +/* ccw sub-types */ +#define VFIO_REGION_SUBTYPE_CCW_ASYNC_CMD (1) + /* * 10de vendor sub-type * diff --git a/include/uapi/linux/vfio_ccw.h b/include/uapi/linux/vfio_ccw.h index 2ec5f367ff78..cbecbf0cd54f 100644 --- a/include/uapi/linux/vfio_ccw.h +++ b/include/uapi/linux/vfio_ccw.h @@ -12,6 +12,7 @@ #include <linux/types.h> +/* used for START SUBCHANNEL, always present */ struct ccw_io_region { #define ORB_AREA_SIZE 12 __u8 orb_area[ORB_AREA_SIZE]; @@ -22,4 +23,15 @@ struct ccw_io_region { __u32 ret_code; } __packed; +/* + * used for processing commands that trigger asynchronous actions + * Note: this is controlled by a capability + */ +#define VFIO_CCW_ASYNC_CMD_HSCH (1 << 0) +#define VFIO_CCW_ASYNC_CMD_CSCH (1 << 1) +struct ccw_cmd_region { + __u32 command; + __u32 ret_code; +} __packed; + #endif diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 27b992fe8cec..d06190fa5082 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -634,6 +634,9 @@ static int static_obj(const void *obj) end = (unsigned long) &_end, addr = (unsigned long) obj; + if (arch_is_kernel_initmem_freed(addr)) + return 0; + /* * static variable? */ diff --git a/kernel/panic.c b/kernel/panic.c index 0ae0d7332f12..c1fcaad337b7 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -318,12 +318,7 @@ void panic(const char *fmt, ...) } #endif #if defined(CONFIG_S390) - { - unsigned long caller; - - caller = (unsigned long)__builtin_return_address(0); - disabled_wait(caller); - } + disabled_wait(); #endif pr_emerg("---[ end Kernel panic - not syncing: %s ]---\n", buf); local_irq_enable(); diff --git a/security/integrity/Kconfig b/security/integrity/Kconfig index 2ea4ec9991d5..3ba1168b1756 100644 --- a/security/integrity/Kconfig +++ b/security/integrity/Kconfig @@ -55,13 +55,22 @@ config INTEGRITY_PLATFORM_KEYRING bool "Provide keyring for platform/firmware trusted keys" depends on INTEGRITY_ASYMMETRIC_KEYS depends on SYSTEM_BLACKLIST_KEYRING - depends on EFI help Provide a separate, distinct keyring for platform trusted keys, which the kernel automatically populates during initialization from values provided by the platform for verifying the kexec'ed kerned image and, possibly, the initramfs signature. +config LOAD_UEFI_KEYS + depends on INTEGRITY_PLATFORM_KEYRING + depends on EFI + def_bool y + +config LOAD_IPL_KEYS + depends on INTEGRITY_PLATFORM_KEYRING + depends on S390 + def_bool y + config INTEGRITY_AUDIT bool "Enables integrity auditing support " depends on AUDIT diff --git a/security/integrity/Makefile b/security/integrity/Makefile index 86df9aba8c0f..19faace69644 100644 --- a/security/integrity/Makefile +++ b/security/integrity/Makefile @@ -9,10 +9,10 @@ integrity-y := iint.o integrity-$(CONFIG_INTEGRITY_AUDIT) += integrity_audit.o integrity-$(CONFIG_INTEGRITY_SIGNATURE) += digsig.o integrity-$(CONFIG_INTEGRITY_ASYMMETRIC_KEYS) += digsig_asymmetric.o -integrity-$(CONFIG_INTEGRITY_PLATFORM_KEYRING) += platform_certs/platform_keyring.o \ - platform_certs/efi_parser.o \ - platform_certs/load_uefi.o -obj-$(CONFIG_LOAD_UEFI_KEYS) += platform_certs/load_uefi.o +integrity-$(CONFIG_INTEGRITY_PLATFORM_KEYRING) += platform_certs/platform_keyring.o +integrity-$(CONFIG_LOAD_UEFI_KEYS) += platform_certs/efi_parser.o \ + platform_certs/load_uefi.o +integrity-$(CONFIG_LOAD_IPL_KEYS) += platform_certs/load_ipl_s390.o $(obj)/load_uefi.o: KBUILD_CFLAGS += -fshort-wchar subdir-$(CONFIG_IMA) += ima diff --git a/security/integrity/platform_certs/load_ipl_s390.c b/security/integrity/platform_certs/load_ipl_s390.c new file mode 100644 index 000000000000..e769dcb7ea94 --- /dev/null +++ b/security/integrity/platform_certs/load_ipl_s390.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/cred.h> +#include <linux/err.h> +#include <linux/efi.h> +#include <linux/slab.h> +#include <keys/asymmetric-type.h> +#include <keys/system_keyring.h> +#include <asm/boot_data.h> +#include "../integrity.h" + +/* + * Load the certs contained in the IPL report created by the machine loader + * into the platform trusted keyring. + */ +static int __init load_ipl_certs(void) +{ + void *ptr, *end; + unsigned int len; + + if (!ipl_cert_list_addr) + return 0; + /* Copy the certificates to the system keyring */ + ptr = (void *) ipl_cert_list_addr; + end = ptr + ipl_cert_list_size; + while ((void *) ptr < end) { + len = *(unsigned int *) ptr; + ptr += sizeof(unsigned int); + add_to_platform_keyring("IPL:db", ptr, len); + ptr += len; + } + return 0; +} +late_initcall(load_ipl_certs); diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h index 1069e85258ce..0afdf7957974 100644 --- a/tools/testing/selftests/rseq/rseq-s390.h +++ b/tools/testing/selftests/rseq/rseq-s390.h @@ -1,6 +1,13 @@ /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -#define RSEQ_SIG 0x53053053 +/* + * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the + * access-register mode nor the linkage stack this instruction will always + * cause a special-operation exception (the trap-enabled bit in the DUCT + * is and will stay 0). The instruction pattern is + * b2 ff 0f ff trap4 4095(%r0) + */ +#define RSEQ_SIG 0xB2FF0FFF #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory") #define rseq_smp_rmb() rseq_smp_mb() |