diff options
Diffstat (limited to 'arch/arm/kernel')
62 files changed, 771 insertions, 4573 deletions
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 553866751e1a..afc9de7ef9a1 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -40,12 +40,12 @@ obj-y += entry-armv.o endif obj-$(CONFIG_MMU) += bugs.o +obj-$(CONFIG_OF) += cacheinfo.o obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_ISA_DMA_API) += dma.o obj-$(CONFIG_FIQ) += fiq.o fiqasm.o obj-$(CONFIG_MODULES) += armksyms.o module.o obj-$(CONFIG_ARM_MODULE_PLTS) += module-plts.o -obj-$(CONFIG_ISA_DMA) += dma-isa.o obj-$(CONFIG_PCI) += bios32.o isa.o obj-$(CONFIG_ARM_CPU_SUSPEND) += sleep.o suspend.o obj-$(CONFIG_HIBERNATION) += hibernate.o @@ -60,7 +60,8 @@ obj-$(CONFIG_FUNCTION_TRACER) += entry-ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o insn.o patch.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o insn.o patch.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o insn.o patch.o -obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o # Main staffs in KPROBES are in arch/arm/probes/ . obj-$(CONFIG_KPROBES) += patch.o insn.o obj-$(CONFIG_OABI_COMPAT) += sys_oabi-compat.o @@ -71,25 +72,20 @@ obj-$(CONFIG_HAVE_TCM) += tcm.o obj-$(CONFIG_OF) += devtree.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_SWP_EMULATE) += swp_emulate.o -CFLAGS_swp_emulate.o := -Wa,-march=armv7-a obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CPU_XSCALE) += xscale-cp0.o obj-$(CONFIG_CPU_XSC3) += xscale-cp0.o obj-$(CONFIG_CPU_MOHAWK) += xscale-cp0.o -obj-$(CONFIG_CPU_PJ4) += pj4-cp0.o -obj-$(CONFIG_CPU_PJ4B) += pj4-cp0.o obj-$(CONFIG_IWMMXT) += iwmmxt.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_xscale.o perf_event_v6.o \ - perf_event_v7.o AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o obj-$(CONFIG_VDSO) += vdso.o obj-$(CONFIG_EFI) += efi.o obj-$(CONFIG_PARAVIRT) += paravirt.o -head-y := head$(MMUEXT).o +obj-y += head$(MMUEXT).o obj-$(CONFIG_DEBUG_LL) += debug.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_ARM_PATCH_PHYS_VIRT) += phys2virt.o @@ -100,7 +96,6 @@ CFLAGS_head-inflate-data.o := $(call cc-option,-Wframe-larger-than=10240) obj-$(CONFIG_XIP_DEFLATED_DATA) += head-inflate-data.o obj-$(CONFIG_ARM_VIRT_EXT) += hyp-stub.o -AFLAGS_hyp-stub.o :=-Wa,-march=armv7-a ifeq ($(CONFIG_ARM_PSCI),y) obj-$(CONFIG_SMP) += psci_smp.o endif @@ -109,4 +104,4 @@ obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o -extra-y := $(head-y) vmlinux.lds +always-$(KBUILD_BUILTIN) := vmlinux.lds diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 2c8d76fd7c66..123f4a8ef446 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -17,14 +17,16 @@ #include <asm/glue-pf.h> #include <asm/mach/arch.h> #include <asm/thread_info.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/mpu.h> #include <asm/procinfo.h> #include <asm/suspend.h> -#include <asm/vdso_datapage.h> #include <asm/hardware/cache-l2x0.h> #include <linux/kbuild.h> #include <linux/arm-smccc.h> + +#include <vdso/datapage.h> + #include "signal.h" /* @@ -47,7 +49,6 @@ int main(void) DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); DEFINE(TI_ABI_SYSCALL, offsetof(struct thread_info, abi_syscall)); - DEFINE(TI_USED_CP, offsetof(struct thread_info, used_cp)); DEFINE(TI_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TI_FPSTATE, offsetof(struct thread_info, fpstate)); #ifdef CONFIG_VFP @@ -56,6 +57,7 @@ int main(void) DEFINE(VFP_CPU, offsetof(union vfp_state, hard.cpu)); #endif #endif + DEFINE(SOFTIRQ_DISABLE_OFFSET,SOFTIRQ_DISABLE_OFFSET); #ifdef CONFIG_ARM_THUMBEE DEFINE(TI_THUMBEE_STATE, offsetof(struct thread_info, thumbee_state)); #endif @@ -83,6 +85,7 @@ int main(void) DEFINE(S_OLD_R0, offsetof(struct pt_regs, ARM_ORIG_r0)); DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs)); DEFINE(SVC_DACR, offsetof(struct svc_pt_regs, dacr)); + DEFINE(SVC_TTBCR, offsetof(struct svc_pt_regs, ttbcr)); DEFINE(SVC_REGS_SIZE, sizeof(struct svc_pt_regs)); BLANK(); DEFINE(SIGFRAME_RC3_OFFSET, offsetof(struct sigframe, retcode[3])); @@ -150,10 +153,6 @@ int main(void) DEFINE(CACHE_WRITEBACK_ORDER, __CACHE_WRITEBACK_ORDER); DEFINE(CACHE_WRITEBACK_GRANULE, __CACHE_WRITEBACK_GRANULE); BLANK(); -#ifdef CONFIG_VDSO - DEFINE(VDSO_DATA_SIZE, sizeof(union vdso_data_store)); -#endif - BLANK(); #ifdef CONFIG_ARM_MPU DEFINE(MPU_RNG_INFO_RNGS, offsetof(struct mpu_rgn_info, rgns)); DEFINE(MPU_RNG_INFO_USED, offsetof(struct mpu_rgn_info, used)); diff --git a/arch/arm/kernel/atags_parse.c b/arch/arm/kernel/atags_parse.c index 373b61f9a4f0..4ec591bde3df 100644 --- a/arch/arm/kernel/atags_parse.c +++ b/arch/arm/kernel/atags_parse.c @@ -69,18 +69,18 @@ static int __init parse_tag_mem32(const struct tag *tag) __tagtable(ATAG_MEM, parse_tag_mem32); -#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) +#if defined(CONFIG_ARCH_FOOTBRIDGE) && defined(CONFIG_VGA_CONSOLE) static int __init parse_tag_videotext(const struct tag *tag) { - screen_info.orig_x = tag->u.videotext.x; - screen_info.orig_y = tag->u.videotext.y; - screen_info.orig_video_page = tag->u.videotext.video_page; - screen_info.orig_video_mode = tag->u.videotext.video_mode; - screen_info.orig_video_cols = tag->u.videotext.video_cols; - screen_info.orig_video_ega_bx = tag->u.videotext.video_ega_bx; - screen_info.orig_video_lines = tag->u.videotext.video_lines; - screen_info.orig_video_isVGA = tag->u.videotext.video_isvga; - screen_info.orig_video_points = tag->u.videotext.video_points; + vgacon_screen_info.orig_x = tag->u.videotext.x; + vgacon_screen_info.orig_y = tag->u.videotext.y; + vgacon_screen_info.orig_video_page = tag->u.videotext.video_page; + vgacon_screen_info.orig_video_mode = tag->u.videotext.video_mode; + vgacon_screen_info.orig_video_cols = tag->u.videotext.video_cols; + vgacon_screen_info.orig_video_ega_bx = tag->u.videotext.video_ega_bx; + vgacon_screen_info.orig_video_lines = tag->u.videotext.video_lines; + vgacon_screen_info.orig_video_isVGA = tag->u.videotext.video_isvga; + vgacon_screen_info.orig_video_points = tag->u.videotext.video_points; return 0; } @@ -127,7 +127,7 @@ static int __init parse_tag_cmdline(const struct tag *tag) #elif defined(CONFIG_CMDLINE_FORCE) pr_warn("Ignoring tag cmdline (using the default kernel command line)\n"); #else - strlcpy(default_command_line, tag->u.cmdline.cmdline, + strscpy(default_command_line, tag->u.cmdline.cmdline, COMMAND_LINE_SIZE); #endif return 0; @@ -224,7 +224,7 @@ setup_machine_tags(void *atags_vaddr, unsigned int machine_nr) } /* parse_early_param needs a boot_command_line */ - strlcpy(boot_command_line, from, COMMAND_LINE_SIZE); + strscpy(boot_command_line, from, COMMAND_LINE_SIZE); return mdesc; } diff --git a/arch/arm/kernel/atags_proc.c b/arch/arm/kernel/atags_proc.c index 3ec2afe78423..cd09f8ab93e3 100644 --- a/arch/arm/kernel/atags_proc.c +++ b/arch/arm/kernel/atags_proc.c @@ -7,7 +7,7 @@ struct buffer { size_t size; - char data[]; + char data[] __counted_by(size); }; static ssize_t atags_read(struct file *file, char __user *buf, @@ -54,7 +54,7 @@ static int __init init_atags_procfs(void) WARN_ON(tag->hdr.tag != ATAG_NONE); - b = kmalloc(sizeof(*b) + size, GFP_KERNEL); + b = kmalloc(struct_size(b, data, size), GFP_KERNEL); if (!b) goto nomem; diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index e7ef2b5bea9c..d334c7fb672b 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -142,15 +142,15 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND2, PCI_DEVICE_ID_WINBOND2_89C940F, */ static void pci_fixup_dec21285(struct pci_dev *dev) { - int i; - if (dev->devfn == 0) { + struct resource *r; + dev->class &= 0xff; dev->class |= PCI_CLASS_BRIDGE_HOST << 8; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - dev->resource[i].start = 0; - dev->resource[i].end = 0; - dev->resource[i].flags = 0; + pci_dev_for_each_resource(dev, r) { + r->start = 0; + r->end = 0; + r->flags = 0; } } } @@ -162,13 +162,11 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_21285, pci_fixup_d static void pci_fixup_ide_bases(struct pci_dev *dev) { struct resource *r; - int i; if ((dev->class >> 8) != PCI_CLASS_STORAGE_IDE) return; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - r = dev->resource + i; + pci_dev_for_each_resource(dev, r) { if ((r->start & ~0x80) == 0x374) { r->start |= 2; r->end = r->start; diff --git a/arch/arm/kernel/bugs.c b/arch/arm/kernel/bugs.c index 14c8dbbb7d2d..087bce6ec8e9 100644 --- a/arch/arm/kernel/bugs.c +++ b/arch/arm/kernel/bugs.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/init.h> +#include <linux/cpu.h> #include <asm/bugs.h> #include <asm/proc-fns.h> @@ -11,7 +12,7 @@ void check_other_bugs(void) #endif } -void __init check_bugs(void) +void __init arch_cpu_finalize_init(void) { check_writebuffer_bugs(); check_other_bugs(); diff --git a/arch/arm/kernel/cacheinfo.c b/arch/arm/kernel/cacheinfo.c new file mode 100644 index 000000000000..e1469b641780 --- /dev/null +++ b/arch/arm/kernel/cacheinfo.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ARM cacheinfo support + * + * Copyright (C) 2023 Linaro Ltd. + * Copyright (C) 2015 ARM Ltd. + * All Rights Reserved + */ + +#include <linux/bitfield.h> +#include <linux/cacheinfo.h> +#include <linux/of.h> + +#include <asm/cachetype.h> +#include <asm/cputype.h> +#include <asm/system_info.h> + +/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */ +#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1)) +#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level)) +#define CLIDR_CTYPE(clidr, level) \ + (((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level)) + +#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */ + +#define CTR_FORMAT_MASK GENMASK(31, 29) +#define CTR_FORMAT_ARMV6 0 +#define CTR_FORMAT_ARMV7 4 +#define CTR_CWG_MASK GENMASK(27, 24) +#define CTR_DSIZE_LEN_MASK GENMASK(13, 12) +#define CTR_ISIZE_LEN_MASK GENMASK(1, 0) + +/* Also valid for v7m */ +static inline int cache_line_size_cp15(void) +{ + u32 ctr = read_cpuid_cachetype(); + u32 format = FIELD_GET(CTR_FORMAT_MASK, ctr); + + if (format == CTR_FORMAT_ARMV7) { + u32 cwg = FIELD_GET(CTR_CWG_MASK, ctr); + + return cwg ? 4 << cwg : ARCH_DMA_MINALIGN; + } else if (WARN_ON_ONCE(format != CTR_FORMAT_ARMV6)) { + return ARCH_DMA_MINALIGN; + } + + return 8 << max(FIELD_GET(CTR_ISIZE_LEN_MASK, ctr), + FIELD_GET(CTR_DSIZE_LEN_MASK, ctr)); +} + +int cache_line_size(void) +{ + if (coherency_max_size != 0) + return coherency_max_size; + + /* CP15 is optional / implementation defined before ARMv6 */ + if (cpu_architecture() < CPU_ARCH_ARMv6) + return ARCH_DMA_MINALIGN; + + return cache_line_size_cp15(); +} +EXPORT_SYMBOL_GPL(cache_line_size); + +static inline enum cache_type get_cache_type(int level) +{ + u32 clidr; + + if (level > MAX_CACHE_LEVEL) + return CACHE_TYPE_NOCACHE; + + clidr = read_clidr(); + + return CLIDR_CTYPE(clidr, level); +} + +static void ci_leaf_init(struct cacheinfo *this_leaf, + enum cache_type type, unsigned int level) +{ + this_leaf->level = level; + this_leaf->type = type; +} + +static int detect_cache_level(unsigned int *level_p, unsigned int *leaves_p) +{ + unsigned int ctype, level, leaves; + u32 ctr, format; + + /* CLIDR is not present before ARMv7/v7m */ + if (cpu_architecture() < CPU_ARCH_ARMv7) + return -EOPNOTSUPP; + + /* Don't try reading CLIDR if CTR declares old format */ + ctr = read_cpuid_cachetype(); + format = FIELD_GET(CTR_FORMAT_MASK, ctr); + if (format != CTR_FORMAT_ARMV7) + return -EOPNOTSUPP; + + for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) { + ctype = get_cache_type(level); + if (ctype == CACHE_TYPE_NOCACHE) { + level--; + break; + } + /* Separate instruction and data caches */ + leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1; + } + + *level_p = level; + *leaves_p = leaves; + + return 0; +} + +int early_cache_level(unsigned int cpu) +{ + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + return detect_cache_level(&this_cpu_ci->num_levels, &this_cpu_ci->num_leaves); +} + +int init_cache_level(unsigned int cpu) +{ + unsigned int level, leaves; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + int fw_level; + int ret; + + ret = detect_cache_level(&level, &leaves); + if (ret) + return ret; + + fw_level = of_find_last_cache_level(cpu); + + if (level < fw_level) { + /* + * some external caches not specified in CLIDR_EL1 + * the information may be available in the device tree + * only unified external caches are considered here + */ + leaves += (fw_level - level); + level = fw_level; + } + + this_cpu_ci->num_levels = level; + this_cpu_ci->num_leaves = leaves; + return 0; +} + +int populate_cache_leaves(unsigned int cpu) +{ + unsigned int level, idx; + enum cache_type type; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + unsigned int arch = cpu_architecture(); + + /* CLIDR is not present before ARMv7/v7m */ + if (arch < CPU_ARCH_ARMv7) + return -EOPNOTSUPP; + + for (idx = 0, level = 1; level <= this_cpu_ci->num_levels && + idx < this_cpu_ci->num_leaves; idx++, level++) { + type = get_cache_type(level); + if (type == CACHE_TYPE_SEPARATE) { + ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); + ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); + } else { + ci_leaf_init(this_leaf++, type, level); + } + } + + return 0; +} diff --git a/arch/arm/kernel/cpuidle.c b/arch/arm/kernel/cpuidle.c index e1684623e1b2..fba1f8bb03b5 100644 --- a/arch/arm/kernel/cpuidle.c +++ b/arch/arm/kernel/cpuidle.c @@ -5,7 +5,6 @@ #include <linux/cpuidle.h> #include <linux/of.h> -#include <linux/of_device.h> #include <asm/cpuidle.h> extern struct of_cpuidle_method __cpuidle_method_of_table[]; @@ -26,8 +25,8 @@ static struct cpuidle_ops cpuidle_ops[NR_CPUS] __ro_after_init; * * Returns the index passed as parameter */ -int arm_cpuidle_simple_enter(struct cpuidle_device *dev, - struct cpuidle_driver *drv, int index) +__cpuidle int arm_cpuidle_simple_enter(struct cpuidle_device *dev, struct + cpuidle_driver *drv, int index) { cpu_do_idle(); diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c index 02839d8b6202..3b78966e750a 100644 --- a/arch/arm/kernel/devtree.c +++ b/arch/arm/kernel/devtree.c @@ -13,7 +13,6 @@ #include <linux/of.h> #include <linux/of_fdt.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> #include <linux/smp.h> #include <asm/cputype.h> @@ -194,16 +193,14 @@ const struct machine_desc * __init setup_machine_fdt(void *dt_virt) { const struct machine_desc *mdesc, *mdesc_best = NULL; -#if defined(CONFIG_ARCH_MULTIPLATFORM) || defined(CONFIG_ARM_SINGLE_ARMV7M) DT_MACHINE_START(GENERIC_DT, "Generic DT based system") .l2c_aux_val = 0x0, .l2c_aux_mask = ~0x0, MACHINE_END mdesc_best = &__mach_desc_GENERIC_DT; -#endif - if (!dt_virt || !early_init_dt_verify(dt_virt)) + if (!dt_virt || !early_init_dt_verify(dt_virt, __pa(dt_virt))) return NULL; mdesc = of_flat_dt_match_machine(mdesc_best, arch_get_next_mach); diff --git a/arch/arm/kernel/dma-isa.c b/arch/arm/kernel/dma-isa.c deleted file mode 100644 index 2d90ecce5a11..000000000000 --- a/arch/arm/kernel/dma-isa.c +++ /dev/null @@ -1,225 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * linux/arch/arm/kernel/dma-isa.c - * - * Copyright (C) 1999-2000 Russell King - * - * ISA DMA primitives - * Taken from various sources, including: - * linux/include/asm/dma.h: Defines for using and allocating dma channels. - * Written by Hennus Bergman, 1992. - * High DMA channel support & info by Hannu Savolainen and John Boyd, - * Nov. 1992. - * arch/arm/kernel/dma-ebsa285.c - * Copyright (C) 1998 Phil Blundell - */ -#include <linux/ioport.h> -#include <linux/init.h> -#include <linux/dma-mapping.h> -#include <linux/io.h> - -#include <asm/dma.h> -#include <asm/mach/dma.h> - -#define ISA_DMA_MASK 0 -#define ISA_DMA_MODE 1 -#define ISA_DMA_CLRFF 2 -#define ISA_DMA_PGHI 3 -#define ISA_DMA_PGLO 4 -#define ISA_DMA_ADDR 5 -#define ISA_DMA_COUNT 6 - -static unsigned int isa_dma_port[8][7] = { - /* MASK MODE CLRFF PAGE_HI PAGE_LO ADDR COUNT */ - { 0x0a, 0x0b, 0x0c, 0x487, 0x087, 0x00, 0x01 }, - { 0x0a, 0x0b, 0x0c, 0x483, 0x083, 0x02, 0x03 }, - { 0x0a, 0x0b, 0x0c, 0x481, 0x081, 0x04, 0x05 }, - { 0x0a, 0x0b, 0x0c, 0x482, 0x082, 0x06, 0x07 }, - { 0xd4, 0xd6, 0xd8, 0x000, 0x000, 0xc0, 0xc2 }, - { 0xd4, 0xd6, 0xd8, 0x48b, 0x08b, 0xc4, 0xc6 }, - { 0xd4, 0xd6, 0xd8, 0x489, 0x089, 0xc8, 0xca }, - { 0xd4, 0xd6, 0xd8, 0x48a, 0x08a, 0xcc, 0xce } -}; - -static int isa_get_dma_residue(unsigned int chan, dma_t *dma) -{ - unsigned int io_port = isa_dma_port[chan][ISA_DMA_COUNT]; - int count; - - count = 1 + inb(io_port); - count |= inb(io_port) << 8; - - return chan < 4 ? count : (count << 1); -} - -static struct device isa_dma_dev = { - .init_name = "fallback device", - .coherent_dma_mask = ~(dma_addr_t)0, - .dma_mask = &isa_dma_dev.coherent_dma_mask, -}; - -static void isa_enable_dma(unsigned int chan, dma_t *dma) -{ - if (dma->invalid) { - unsigned long address, length; - unsigned int mode; - enum dma_data_direction direction; - - mode = (chan & 3) | dma->dma_mode; - switch (dma->dma_mode & DMA_MODE_MASK) { - case DMA_MODE_READ: - direction = DMA_FROM_DEVICE; - break; - - case DMA_MODE_WRITE: - direction = DMA_TO_DEVICE; - break; - - case DMA_MODE_CASCADE: - direction = DMA_BIDIRECTIONAL; - break; - - default: - direction = DMA_NONE; - break; - } - - if (!dma->sg) { - /* - * Cope with ISA-style drivers which expect cache - * coherence. - */ - dma->sg = &dma->buf; - dma->sgcount = 1; - dma->buf.length = dma->count; - dma->buf.dma_address = dma_map_single(&isa_dma_dev, - dma->addr, dma->count, - direction); - } - - address = dma->buf.dma_address; - length = dma->buf.length - 1; - - outb(address >> 16, isa_dma_port[chan][ISA_DMA_PGLO]); - outb(address >> 24, isa_dma_port[chan][ISA_DMA_PGHI]); - - if (chan >= 4) { - address >>= 1; - length >>= 1; - } - - outb(0, isa_dma_port[chan][ISA_DMA_CLRFF]); - - outb(address, isa_dma_port[chan][ISA_DMA_ADDR]); - outb(address >> 8, isa_dma_port[chan][ISA_DMA_ADDR]); - - outb(length, isa_dma_port[chan][ISA_DMA_COUNT]); - outb(length >> 8, isa_dma_port[chan][ISA_DMA_COUNT]); - - outb(mode, isa_dma_port[chan][ISA_DMA_MODE]); - dma->invalid = 0; - } - outb(chan & 3, isa_dma_port[chan][ISA_DMA_MASK]); -} - -static void isa_disable_dma(unsigned int chan, dma_t *dma) -{ - outb(chan | 4, isa_dma_port[chan][ISA_DMA_MASK]); -} - -static struct dma_ops isa_dma_ops = { - .type = "ISA", - .enable = isa_enable_dma, - .disable = isa_disable_dma, - .residue = isa_get_dma_residue, -}; - -static struct resource dma_resources[] = { { - .name = "dma1", - .start = 0x0000, - .end = 0x000f -}, { - .name = "dma low page", - .start = 0x0080, - .end = 0x008f -}, { - .name = "dma2", - .start = 0x00c0, - .end = 0x00df -}, { - .name = "dma high page", - .start = 0x0480, - .end = 0x048f -} }; - -static dma_t isa_dma[8]; - -/* - * ISA DMA always starts at channel 0 - */ -void __init isa_init_dma(void) -{ - /* - * Try to autodetect presence of an ISA DMA controller. - * We do some minimal initialisation, and check that - * channel 0's DMA address registers are writeable. - */ - outb(0xff, 0x0d); - outb(0xff, 0xda); - - /* - * Write high and low address, and then read them back - * in the same order. - */ - outb(0x55, 0x00); - outb(0xaa, 0x00); - - if (inb(0) == 0x55 && inb(0) == 0xaa) { - unsigned int chan, i; - - for (chan = 0; chan < 8; chan++) { - isa_dma[chan].d_ops = &isa_dma_ops; - isa_disable_dma(chan, NULL); - } - - outb(0x40, 0x0b); - outb(0x41, 0x0b); - outb(0x42, 0x0b); - outb(0x43, 0x0b); - - outb(0xc0, 0xd6); - outb(0x41, 0xd6); - outb(0x42, 0xd6); - outb(0x43, 0xd6); - - outb(0, 0xd4); - - outb(0x10, 0x08); - outb(0x10, 0xd0); - - /* - * Is this correct? According to my documentation, it - * doesn't appear to be. It should be: - * outb(0x3f, 0x40b); outb(0x3f, 0x4d6); - */ - outb(0x30, 0x40b); - outb(0x31, 0x40b); - outb(0x32, 0x40b); - outb(0x33, 0x40b); - outb(0x31, 0x4d6); - outb(0x32, 0x4d6); - outb(0x33, 0x4d6); - - for (i = 0; i < ARRAY_SIZE(dma_resources); i++) - request_resource(&ioport_resource, dma_resources + i); - - for (chan = 0; chan < 8; chan++) { - int ret = isa_dma_add(chan, &isa_dma[chan]); - if (ret) - pr_err("ISADMA%u: unable to register: %d\n", - chan, ret); - } - - request_dma(DMA_ISA_CASCADE, "cascade"); - } -} diff --git a/arch/arm/kernel/efi.c b/arch/arm/kernel/efi.c index e57dbcc89123..6f9ec7d28a71 100644 --- a/arch/arm/kernel/efi.c +++ b/arch/arm/kernel/efi.c @@ -4,6 +4,9 @@ */ #include <linux/efi.h> +#include <linux/memblock.h> +#include <linux/screen_info.h> + #include <asm/efi.h> #include <asm/mach/map.h> #include <asm/mmu_context.h> @@ -22,7 +25,8 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data) } int __init efi_set_mapping_permissions(struct mm_struct *mm, - efi_memory_desc_t *md) + efi_memory_desc_t *md, + bool ignored) { unsigned long base, size; @@ -70,6 +74,57 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) * If stricter permissions were specified, apply them now. */ if (md->attribute & (EFI_MEMORY_RO | EFI_MEMORY_XP)) - return efi_set_mapping_permissions(mm, md); + return efi_set_mapping_permissions(mm, md, false); return 0; } + +static unsigned long __initdata cpu_state_table = EFI_INVALID_TABLE_ADDR; + +const efi_config_table_type_t efi_arch_tables[] __initconst = { + {LINUX_EFI_ARM_CPU_STATE_TABLE_GUID, &cpu_state_table}, + {} +}; + +static void __init load_cpu_state_table(void) +{ + if (cpu_state_table != EFI_INVALID_TABLE_ADDR) { + struct efi_arm_entry_state *state; + bool dump_state = true; + + state = early_memremap_ro(cpu_state_table, + sizeof(struct efi_arm_entry_state)); + if (state == NULL) { + pr_warn("Unable to map CPU entry state table.\n"); + return; + } + + if ((state->sctlr_before_ebs & 1) == 0) + pr_warn(FW_BUG "EFI stub was entered with MMU and Dcache disabled, please fix your firmware!\n"); + else if ((state->sctlr_after_ebs & 1) == 0) + pr_warn(FW_BUG "ExitBootServices() returned with MMU and Dcache disabled, please fix your firmware!\n"); + else + dump_state = false; + + if (dump_state || efi_enabled(EFI_DBG)) { + pr_info("CPSR at EFI stub entry : 0x%08x\n", + state->cpsr_before_ebs); + pr_info("SCTLR at EFI stub entry : 0x%08x\n", + state->sctlr_before_ebs); + pr_info("CPSR after ExitBootServices() : 0x%08x\n", + state->cpsr_after_ebs); + pr_info("SCTLR after ExitBootServices(): 0x%08x\n", + state->sctlr_after_ebs); + } + early_memunmap(state, sizeof(struct efi_arm_entry_state)); + } +} + +void __init arm_efi_init(void) +{ + efi_init(); + + /* ARM does not permit early mappings to persist across paging_init() */ + efi_memmap_unmap(); + + load_cpu_state_table(); +} diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index c39303e5c234..ef6a657c8d13 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -15,7 +15,7 @@ #include <linux/init.h> #include <asm/assembler.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/glue-df.h> #include <asm/glue-pf.h> #include <asm/vfpmacros.h> @@ -25,10 +25,17 @@ #include <asm/tls.h> #include <asm/system_info.h> #include <asm/uaccess-asm.h> +#include <asm/kasan_def.h> #include "entry-header.S" #include <asm/probes.h> +#ifdef CONFIG_HAVE_LD_DEAD_CODE_DATA_ELIMINATION +#define RELOC_TEXT_NONE .reloc .text, R_ARM_NONE, . +#else +#define RELOC_TEXT_NONE +#endif + /* * Interrupt handling. */ @@ -446,258 +453,26 @@ ENDPROC(__irq_usr) __und_usr: usr_entry uaccess=0 - mov r2, r4 - mov r3, r5 - - @ r2 = regs->ARM_pc, which is either 2 or 4 bytes ahead of the - @ faulting instruction depending on Thumb mode. - @ r3 = regs->ARM_cpsr - @ - @ The emulation code returns using r9 if it has emulated the - @ instruction, or the more conventional lr if we are to treat - @ this as a real undefined instruction - @ - badr r9, ret_from_exception - @ IRQs must be enabled before attempting to read the instruction from @ user space since that could cause a page/translation fault if the @ page table was modified by another CPU. enable_irq - tst r3, #PSR_T_BIT @ Thumb mode? - bne __und_usr_thumb - sub r4, r2, #4 @ ARM instr at LR - 4 -1: ldrt r0, [r4] - ARM_BE8(rev r0, r0) @ little endian instruction - - uaccess_disable ip - - @ r0 = 32-bit ARM instruction which caused the exception - @ r2 = PC value for the following instruction (:= regs->ARM_pc) - @ r4 = PC value for the faulting instruction - @ lr = 32-bit undefined instruction function - badr lr, __und_usr_fault_32 - b call_fpe - -__und_usr_thumb: - @ Thumb instruction - sub r4, r2, #2 @ First half of thumb instr at LR - 2 -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 -/* - * Thumb-2 instruction handling. Note that because pre-v6 and >= v6 platforms - * can never be supported in a single kernel, this code is not applicable at - * all when __LINUX_ARM_ARCH__ < 6. This allows simplifying assumptions to be - * made about .arch directives. - */ -#if __LINUX_ARM_ARCH__ < 7 -/* If the target CPU may not be Thumb-2-capable, a run-time check is needed: */ - ldr_va r5, cpu_architecture - cmp r5, #CPU_ARCH_ARMv7 - blo __und_usr_fault_16 @ 16bit undefined instruction -/* - * The following code won't get run unless the running CPU really is v7, so - * coding round the lack of ldrht on older arches is pointless. Temporarily - * override the assembler target arch with the minimum required instead: - */ - .arch armv6t2 + tst r5, #PSR_T_BIT @ Thumb mode? + mov r1, #2 @ set insn size to 2 for Thumb + bne 0f @ handle as Thumb undef exception +#ifdef CONFIG_FPE_NWFPE + adr r9, ret_from_exception + bl call_fpe @ returns via R9 on success #endif -2: ldrht r5, [r4] -ARM_BE8(rev16 r5, r5) @ little endian instruction - cmp r5, #0xe800 @ 32bit instruction if xx != 0 - blo __und_usr_fault_16_pan @ 16bit undefined instruction -3: ldrht r0, [r2] -ARM_BE8(rev16 r0, r0) @ little endian instruction + mov r1, #4 @ set insn size to 4 for ARM +0: mov r0, sp uaccess_disable ip - add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 - str r2, [sp, #S_PC] @ it's a 2x16bit instr, update - orr r0, r0, r5, lsl #16 - badr lr, __und_usr_fault_32 - @ r0 = the two 16-bit Thumb instructions which caused the exception - @ r2 = PC value for the following Thumb instruction (:= regs->ARM_pc) - @ r4 = PC value for the first 16-bit Thumb instruction - @ lr = 32bit undefined instruction function - -#if __LINUX_ARM_ARCH__ < 7 -/* If the target arch was overridden, change it back: */ -#ifdef CONFIG_CPU_32v6K - .arch armv6k -#else - .arch armv6 -#endif -#endif /* __LINUX_ARM_ARCH__ < 7 */ -#else /* !(CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7) */ - b __und_usr_fault_16 -#endif + bl __und_fault + b ret_from_exception UNWIND(.fnend) ENDPROC(__und_usr) -/* - * The out of line fixup for the ldrt instructions above. - */ - .pushsection .text.fixup, "ax" - .align 2 -4: str r4, [sp, #S_PC] @ retry current instruction - ret r9 - .popsection - .pushsection __ex_table,"a" - .long 1b, 4b -#if CONFIG_ARM_THUMB && __LINUX_ARM_ARCH__ >= 6 && CONFIG_CPU_V7 - .long 2b, 4b - .long 3b, 4b -#endif - .popsection - -/* - * Check whether the instruction is a co-processor instruction. - * If yes, we need to call the relevant co-processor handler. - * - * Note that we don't do a full check here for the co-processor - * instructions; all instructions with bit 27 set are well - * defined. The only instructions that should fault are the - * co-processor instructions. However, we have to watch out - * for the ARM6/ARM7 SWI bug. - * - * NEON is a special case that has to be handled here. Not all - * NEON instructions are co-processor instructions, so we have - * to make a special case of checking for them. Plus, there's - * five groups of them, so we have a table of mask/opcode pairs - * to check against, and if any match then we branch off into the - * NEON handler code. - * - * Emulators may wish to make use of the following registers: - * r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) - * r2 = PC value to resume execution after successful emulation - * r9 = normal "successful" return address - * r10 = this threads thread_info structure - * lr = unrecognised instruction return address - * IRQs enabled, FIQs enabled. - */ - @ - @ Fall-through from Thumb-2 __und_usr - @ -#ifdef CONFIG_NEON - get_thread_info r10 @ get current thread - adr r6, .LCneon_thumb_opcodes - b 2f -#endif -call_fpe: - get_thread_info r10 @ get current thread -#ifdef CONFIG_NEON - adr r6, .LCneon_arm_opcodes -2: ldr r5, [r6], #4 @ mask value - ldr r7, [r6], #4 @ opcode bits matching in mask - cmp r5, #0 @ end mask? - beq 1f - and r8, r0, r5 - cmp r8, r7 @ NEON instruction? - bne 2b - mov r7, #1 - strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used - strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used - b do_vfp @ let VFP handler handle this -1: -#endif - tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 - tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 - reteq lr - and r8, r0, #0x00000f00 @ mask out CP number - mov r7, #1 - add r6, r10, r8, lsr #8 @ add used_cp[] array offset first - strb r7, [r6, #TI_USED_CP] @ set appropriate used_cp[] -#ifdef CONFIG_IWMMXT - @ Test if we need to give access to iWMMXt coprocessors - ldr r5, [r10, #TI_FLAGS] - rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only - movscs r7, r5, lsr #(TIF_USING_IWMMXT + 1) - bcs iwmmxt_task_enable -#endif - ARM( add pc, pc, r8, lsr #6 ) - THUMB( lsr r8, r8, #6 ) - THUMB( add pc, r8 ) - nop - - ret.w lr @ CP#0 - W(b) do_fpe @ CP#1 (FPE) - W(b) do_fpe @ CP#2 (FPE) - ret.w lr @ CP#3 - ret.w lr @ CP#4 - ret.w lr @ CP#5 - ret.w lr @ CP#6 - ret.w lr @ CP#7 - ret.w lr @ CP#8 - ret.w lr @ CP#9 -#ifdef CONFIG_VFP - W(b) do_vfp @ CP#10 (VFP) - W(b) do_vfp @ CP#11 (VFP) -#else - ret.w lr @ CP#10 (VFP) - ret.w lr @ CP#11 (VFP) -#endif - ret.w lr @ CP#12 - ret.w lr @ CP#13 - ret.w lr @ CP#14 (Debug) - ret.w lr @ CP#15 (Control) - -#ifdef CONFIG_NEON - .align 6 - -.LCneon_arm_opcodes: - .word 0xfe000000 @ mask - .word 0xf2000000 @ opcode - - .word 0xff100000 @ mask - .word 0xf4000000 @ opcode - - .word 0x00000000 @ mask - .word 0x00000000 @ opcode - -.LCneon_thumb_opcodes: - .word 0xef000000 @ mask - .word 0xef000000 @ opcode - - .word 0xff100000 @ mask - .word 0xf9000000 @ opcode - - .word 0x00000000 @ mask - .word 0x00000000 @ opcode -#endif - -do_fpe: - add r10, r10, #TI_FPSTATE @ r10 = workspace - ldr_va pc, fp_enter, tmp=r4 @ Call FP module USR entry point - -/* - * The FP module is called with these registers set: - * r0 = instruction - * r2 = PC+4 - * r9 = normal "successful" return address - * r10 = FP workspace - * lr = unrecognised FP instruction return address - */ - - .pushsection .data - .align 2 -ENTRY(fp_enter) - .word no_fp - .popsection - -ENTRY(no_fp) - ret lr -ENDPROC(no_fp) - -__und_usr_fault_32: - mov r1, #4 - b 1f -__und_usr_fault_16_pan: - uaccess_disable ip -__und_usr_fault_16: - mov r1, #2 -1: mov r0, sp - badr lr, ret_from_exception - b __und_fault -ENDPROC(__und_usr_fault_32) -ENDPROC(__und_usr_fault_16) - .align 5 __pabt_usr: usr_entry @@ -787,6 +562,13 @@ ENTRY(__switch_to) @ entries covering the vmalloc region. @ ldr r2, [ip] +#ifdef CONFIG_KASAN_VMALLOC + @ Also dummy read from the KASAN shadow memory for the new stack if we + @ are using KASAN + mov_l r2, KASAN_SHADOW_OFFSET + add r2, r2, ip, lsr #KASAN_SHADOW_SCALE_SHIFT + ldr r2, [r2] +#endif #endif @ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what @@ -875,7 +657,7 @@ ENDPROC(__bad_stack) * existing ones. This mechanism should be used only for things that are * really small and justified, and not be abused freely. * - * See Documentation/arm/kernel_user_helpers.rst for formal definitions. + * See Documentation/arch/arm/kernel_user_helpers.rst for formal definitions. */ THUMB( .arm ) @@ -1297,6 +1079,7 @@ vector_addrexcptn: .globl vector_fiq .section .vectors, "ax", %progbits + RELOC_TEXT_NONE W(b) vector_rst W(b) vector_und ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_swi ) @@ -1310,6 +1093,7 @@ THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_swi ) #ifdef CONFIG_HARDEN_BRANCH_HISTORY .section .vectors.bhb.loop8, "ax", %progbits + RELOC_TEXT_NONE W(b) vector_rst W(b) vector_bhb_loop8_und ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_loop8_swi ) @@ -1322,6 +1106,7 @@ THUMB( .reloc ., R_ARM_THM_PC12, .L__vector_bhb_loop8_swi ) W(b) vector_bhb_loop8_fiq .section .vectors.bhb.bpiall, "ax", %progbits + RELOC_TEXT_NONE W(b) vector_rst W(b) vector_bhb_bpiall_und ARM( .reloc ., R_ARM_LDR_PC_G0, .L__vector_bhb_bpiall_swi ) diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 405a607b754f..88336a1292bb 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -9,22 +9,13 @@ #include <asm/unistd.h> #include <asm/ftrace.h> #include <asm/unwind.h> -#include <asm/memory.h> +#include <asm/page.h> #ifdef CONFIG_AEABI #include <asm/unistd-oabi.h> #endif .equ NR_syscalls, __NR_syscalls - .macro arch_ret_to_user, tmp -#ifdef CONFIG_ARCH_IOP32X - mrc p15, 0, \tmp, c15, c1, 0 - tst \tmp, #(1 << 6) - bicne \tmp, \tmp, #(1 << 6) - mcrne p15, 0, \tmp, c15, c1, 0 @ Disable cp6 access -#endif - .endm - #include "entry-header.S" saved_psr .req r8 @@ -55,10 +46,6 @@ __ret_fast_syscall: movs r1, r1, lsl #16 bne fast_work_pending - - /* perform architecture specific actions before user return */ - arch_ret_to_user r1 - restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) ENDPROC(ret_fast_syscall) @@ -103,6 +90,7 @@ slow_work_pending: cmp r0, #0 beq no_work_pending movlt scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE) + str scno, [tsk, #TI_ABI_SYSCALL] @ make sure tracers see update ldmia sp, {r0 - r6} @ have to reload r0 - r6 b local_restart @ ... and off we go ENDPROC(ret_fast_syscall) @@ -129,10 +117,11 @@ ENTRY(ret_to_user_from_irq) no_work_pending: asm_trace_hardirqs_on save = 0 - /* perform architecture specific actions before user return */ - arch_ret_to_user r1 ct_user_enter save = 0 +#ifdef CONFIG_KSTACK_ERASE + bl stackleak_erase_on_task_stack +#endif restore_user_regs fast = 0, offset = 0 ENDPROC(ret_to_user_from_irq) ENDPROC(ret_to_user) diff --git a/arch/arm/kernel/entry-ftrace.S b/arch/arm/kernel/entry-ftrace.S index 3e7bcaca5e07..bc598e3d8dd2 100644 --- a/arch/arm/kernel/entry-ftrace.S +++ b/arch/arm/kernel/entry-ftrace.S @@ -271,6 +271,10 @@ ENTRY(ftrace_stub) ret lr ENDPROC(ftrace_stub) +ENTRY(ftrace_stub_graph) + ret lr +ENDPROC(ftrace_stub_graph) + #ifdef CONFIG_DYNAMIC_FTRACE __INIT diff --git a/arch/arm/kernel/entry-v7m.S b/arch/arm/kernel/entry-v7m.S index de8a60363c85..52bacf07ba16 100644 --- a/arch/arm/kernel/entry-v7m.S +++ b/arch/arm/kernel/entry-v7m.S @@ -6,7 +6,7 @@ * * Low-level vector interface routines for the ARMv7-M architecture */ -#include <asm/memory.h> +#include <asm/page.h> #include <asm/glue.h> #include <asm/thread_notify.h> #include <asm/v7m.h> diff --git a/arch/arm/kernel/fiq.c b/arch/arm/kernel/fiq.c index 98ca3e3fa847..d2c8e5313539 100644 --- a/arch/arm/kernel/fiq.c +++ b/arch/arm/kernel/fiq.c @@ -45,6 +45,7 @@ #include <asm/cacheflush.h> #include <asm/cp15.h> #include <asm/fiq.h> +#include <asm/mach/irq.h> #include <asm/irq.h> #include <asm/traps.h> diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index a0b6d1e3812f..845acf9ce21e 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -23,7 +23,7 @@ #include <asm/insn.h> #include <asm/set_memory.h> #include <asm/stacktrace.h> -#include <asm/patch.h> +#include <asm/text-patching.h> /* * The compiler emitted profiling hook consists of @@ -232,11 +232,24 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, unsigned long old; if (unlikely(atomic_read(¤t->tracing_graph_pause))) +err_out: return; if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) { - /* FP points one word below parent's top of stack */ - frame_pointer += 4; + /* + * Usually, the stack frames are contiguous in memory but cases + * have been observed where the next stack frame does not live + * at 'frame_pointer + 4' as this code used to assume. + * + * Instead, dereference the field in the stack frame that + * stores the SP of the calling frame: to avoid unbounded + * recursion, this cannot involve any ftrace instrumented + * functions, so use the __get_kernel_nofault() primitive + * directly. + */ + __get_kernel_nofault(&frame_pointer, + (unsigned long *)(frame_pointer - 8), + unsigned long, err_out); } else { struct stackframe frame = { .fp = frame_pointer, diff --git a/arch/arm/kernel/head-inflate-data.c b/arch/arm/kernel/head-inflate-data.c index 89a52104d32a..225c0699a12c 100644 --- a/arch/arm/kernel/head-inflate-data.c +++ b/arch/arm/kernel/head-inflate-data.c @@ -8,16 +8,13 @@ #include <linux/init.h> #include <linux/zutil.h> +#include "head.h" /* for struct inflate_state */ #include "../../../lib/zlib_inflate/inftrees.h" #include "../../../lib/zlib_inflate/inflate.h" #include "../../../lib/zlib_inflate/infutil.h" -extern char __data_loc[]; -extern char _edata_loc[]; -extern char _sdata[]; - /* * This code is called very early during the boot process to decompress * the .data segment stored compressed in ROM. Therefore none of the global diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S index 950bef83339f..b9d6818f1ee1 100644 --- a/arch/arm/kernel/head-nommu.S +++ b/arch/arm/kernel/head-nommu.S @@ -14,12 +14,11 @@ #include <asm/assembler.h> #include <asm/ptrace.h> #include <asm/asm-offsets.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/cp15.h> #include <asm/thread_info.h> #include <asm/v7m.h> #include <asm/mpu.h> -#include <asm/page.h> /* * Kernel startup entry point. diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 500612d3da2e..f22c50d4bd41 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -17,7 +17,7 @@ #include <asm/domain.h> #include <asm/ptrace.h> #include <asm/asm-offsets.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/thread_info.h> #if defined(CONFIG_DEBUG_LL) && !defined(CONFIG_DEBUG_SEMIHOSTING) @@ -38,10 +38,10 @@ #ifdef CONFIG_ARM_LPAE /* LPAE requires an additional page for the PGD */ #define PG_DIR_SIZE 0x5000 -#define PMD_ORDER 3 +#define PMD_ENTRY_ORDER 3 /* PMD entry size is 2^PMD_ENTRY_ORDER */ #else #define PG_DIR_SIZE 0x4000 -#define PMD_ORDER 2 +#define PMD_ENTRY_ORDER 2 #endif .globl swapper_pg_dir @@ -240,7 +240,7 @@ __create_page_tables: mov r6, r6, lsr #SECTION_SHIFT 1: orr r3, r7, r5, lsl #SECTION_SHIFT @ flags + kernel base - str r3, [r4, r5, lsl #PMD_ORDER] @ identity mapping + str r3, [r4, r5, lsl #PMD_ENTRY_ORDER] @ identity mapping cmp r5, r6 addlo r5, r5, #1 @ next section blo 1b @@ -250,20 +250,25 @@ __create_page_tables: * set two variables to indicate the physical start and end of the * kernel. */ - add r0, r4, #KERNEL_OFFSET >> (SECTION_SHIFT - PMD_ORDER) + add r0, r4, #KERNEL_OFFSET >> (SECTION_SHIFT - PMD_ENTRY_ORDER) ldr r6, =(_end - 1) + + /* For XIP, kernel_sec_start/kernel_sec_end are currently in RO memory */ +#ifndef CONFIG_XIP_KERNEL adr_l r5, kernel_sec_start @ _pa(kernel_sec_start) #if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32 str r8, [r5, #4] @ Save physical start of kernel (BE) #else str r8, [r5] @ Save physical start of kernel (LE) #endif +#endif orr r3, r8, r7 @ Add the MMU flags - add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) -1: str r3, [r0], #1 << PMD_ORDER + add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER) +1: str r3, [r0], #1 << PMD_ENTRY_ORDER add r3, r3, #1 << SECTION_SHIFT cmp r0, r6 bls 1b +#ifndef CONFIG_XIP_KERNEL eor r3, r3, r7 @ Remove the MMU flags adr_l r5, kernel_sec_end @ _pa(kernel_sec_end) #if defined CONFIG_CPU_ENDIAN_BE8 || defined CONFIG_CPU_ENDIAN_BE32 @@ -271,8 +276,7 @@ __create_page_tables: #else str r3, [r5] @ Save physical end of kernel (LE) #endif - -#ifdef CONFIG_XIP_KERNEL +#else /* * Map the kernel image separately as it is not located in RAM. */ @@ -280,14 +284,14 @@ __create_page_tables: mov r3, pc mov r3, r3, lsr #SECTION_SHIFT orr r3, r7, r3, lsl #SECTION_SHIFT - add r0, r4, #(XIP_START & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) - str r3, [r0, #((XIP_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ORDER]! + add r0, r4, #(XIP_START & 0xff000000) >> (SECTION_SHIFT - PMD_ENTRY_ORDER) + str r3, [r0, #((XIP_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ENTRY_ORDER]! ldr r6, =(_edata_loc - 1) - add r0, r0, #1 << PMD_ORDER - add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) + add r0, r0, #1 << PMD_ENTRY_ORDER + add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ENTRY_ORDER) 1: cmp r0, r6 add r3, r3, #1 << SECTION_SHIFT - strls r3, [r0], #1 << PMD_ORDER + strls r3, [r0], #1 << PMD_ENTRY_ORDER bls 1b #endif @@ -297,10 +301,10 @@ __create_page_tables: */ mov r0, r2, lsr #SECTION_SHIFT cmp r2, #0 - ldrne r3, =FDT_FIXED_BASE >> (SECTION_SHIFT - PMD_ORDER) + ldrne r3, =FDT_FIXED_BASE >> (SECTION_SHIFT - PMD_ENTRY_ORDER) addne r3, r3, r4 orrne r6, r7, r0, lsl #SECTION_SHIFT - strne r6, [r3], #1 << PMD_ORDER + strne r6, [r3], #1 << PMD_ENTRY_ORDER addne r6, r6, #1 << SECTION_SHIFT strne r6, [r3] @@ -319,7 +323,7 @@ __create_page_tables: addruart r7, r3, r0 mov r3, r3, lsr #SECTION_SHIFT - mov r3, r3, lsl #PMD_ORDER + mov r3, r3, lsl #PMD_ENTRY_ORDER add r0, r4, r3 mov r3, r7, lsr #SECTION_SHIFT @@ -344,12 +348,12 @@ __create_page_tables: ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags #endif -#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS) +#if defined(CONFIG_ARCH_NETWINDER) /* * If we're using the NetWinder or CATS, we also need to map * in the 16550-type serial port for the debug messages */ - add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ORDER) + add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER) orr r3, r7, #0x7c000000 str r3, [r0] #endif @@ -359,10 +363,10 @@ __create_page_tables: * Similar reasons here - for debug. This is * only for Acorn RiscPC architectures. */ - add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ORDER) + add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER) orr r3, r7, #0x02000000 str r3, [r0] - add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER) + add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ENTRY_ORDER) str r3, [r0] #endif #endif @@ -407,7 +411,11 @@ ENTRY(secondary_startup) /* * Use the page tables supplied from __cpu_up. */ +#ifdef CONFIG_XIP_KERNEL + ldr r3, =(secondary_data + PLAT_PHYS_OFFSET - PAGE_OFFSET) +#else adr_l r3, secondary_data +#endif mov_l r12, __secondary_switched ldrd r4, r5, [r3, #0] @ get secondary_data.pgdir ARM_BE8(eor r4, r4, r5) @ Swap r5 and r4 in BE: diff --git a/arch/arm/kernel/head.h b/arch/arm/kernel/head.h new file mode 100644 index 000000000000..0eb5accf7141 --- /dev/null +++ b/arch/arm/kernel/head.h @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only + +extern char __data_loc[]; +extern char _edata_loc[]; +extern char _sdata[]; + +int __init __inflate_kernel_data(void); diff --git a/arch/arm/kernel/hibernate.c b/arch/arm/kernel/hibernate.c index 2373020af965..38a90a3d12b2 100644 --- a/arch/arm/kernel/hibernate.c +++ b/arch/arm/kernel/hibernate.c @@ -19,7 +19,7 @@ #include <asm/system_misc.h> #include <asm/idmap.h> #include <asm/suspend.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/sections.h> #include "reboot.h" diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 054e9199f30d..a12efd0f43e8 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -17,6 +17,7 @@ #include <linux/perf_event.h> #include <linux/hw_breakpoint.h> #include <linux/smp.h> +#include <linux/cfi.h> #include <linux/cpu_pm.h> #include <linux/coresight.h> @@ -903,6 +904,37 @@ unlock: watchpoint_single_step_handler(addr); } +#ifdef CONFIG_CFI_CLANG +static void hw_breakpoint_cfi_handler(struct pt_regs *regs) +{ + /* + * TODO: implementing target and type to pass to CFI using the more + * elaborate report_cfi_failure() requires compiler work. To be able + * to properly extract target information the compiler needs to + * emit a stable instructions sequence for the CFI checks so we can + * decode the instructions preceding the trap and figure out which + * registers were used. + */ + + switch (report_cfi_failure_noaddr(regs, instruction_pointer(regs))) { + case BUG_TRAP_TYPE_BUG: + die("Oops - CFI", regs, 0); + break; + case BUG_TRAP_TYPE_WARN: + /* Skip the breaking instruction */ + instruction_pointer(regs) += 4; + break; + default: + die("Unknown CFI error", regs, 0); + break; + } +} +#else +static void hw_breakpoint_cfi_handler(struct pt_regs *regs) +{ +} +#endif + /* * Called from either the Data Abort Handler [watchpoint] or the * Prefetch Abort Handler [breakpoint] with interrupts disabled. @@ -932,6 +964,9 @@ static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, case ARM_ENTRY_SYNC_WATCHPOINT: watchpoint_handler(addr, fsr, regs); break; + case ARM_ENTRY_CFI_BREAKPOINT: + hw_breakpoint_cfi_handler(regs); + break; default: ret = 1; /* Unhandled fault. */ } diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S index b699b22a4db1..3a506b9095a5 100644 --- a/arch/arm/kernel/hyp-stub.S +++ b/arch/arm/kernel/hyp-stub.S @@ -9,6 +9,8 @@ #include <asm/assembler.h> #include <asm/virt.h> +.arch armv7-a + #ifndef ZIMAGE /* * For the kernel proper, we need to find out the CPU boot mode long after diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index 5c6f8d11a3ce..e1993e28a9ec 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -32,6 +32,7 @@ #include <linux/kallsyms.h> #include <linux/proc_fs.h> #include <linux/export.h> +#include <linux/vmalloc.h> #include <asm/hardware/cache-l2x0.h> #include <asm/hardware/cache-uniphier.h> @@ -70,6 +71,7 @@ static void __init init_irq_stacks(void) } } +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK static void ____do_softirq(void *arg) { __do_softirq(); @@ -80,7 +82,7 @@ void do_softirq_own_stack(void) call_with_stack(____do_softirq, NULL, __this_cpu_read(irq_stack_ptr)); } - +#endif #endif int arch_show_interrupts(struct seq_file *p, int prec) @@ -109,7 +111,7 @@ void handle_IRQ(unsigned int irq, struct pt_regs *regs) * Some hardware gives randomly wrong interrupts. Rather * than crashing, do something sensible. */ - if (unlikely(!irq || irq >= nr_irqs)) + if (unlikely(!irq || irq >= irq_get_nr_irqs())) desc = NULL; else desc = irq_to_desc(irq); @@ -149,7 +151,6 @@ void __init init_IRQ(void) #ifdef CONFIG_SPARSE_IRQ int __init arch_probe_nr_irqs(void) { - nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS; - return nr_irqs; + return irq_set_nr_irqs(machine_desc->nr_irqs ? : NR_IRQS); } #endif diff --git a/arch/arm/kernel/isa.c b/arch/arm/kernel/isa.c index d8a509c5d5bd..db8be609fab2 100644 --- a/arch/arm/kernel/isa.c +++ b/arch/arm/kernel/isa.c @@ -16,7 +16,7 @@ static unsigned int isa_membase, isa_portbase, isa_portshift; -static struct ctl_table ctl_isa_vars[4] = { +static const struct ctl_table ctl_isa_vars[] = { { .procname = "membase", .data = &isa_membase, @@ -35,32 +35,16 @@ static struct ctl_table ctl_isa_vars[4] = { .maxlen = sizeof(isa_portshift), .mode = 0444, .proc_handler = proc_dointvec, - }, {} + }, }; static struct ctl_table_header *isa_sysctl_header; -static struct ctl_table ctl_isa[2] = { - { - .procname = "isa", - .mode = 0555, - .child = ctl_isa_vars, - }, {} -}; - -static struct ctl_table ctl_bus[2] = { - { - .procname = "bus", - .mode = 0555, - .child = ctl_isa, - }, {} -}; - void __init register_isa_ports(unsigned int membase, unsigned int portbase, unsigned int portshift) { isa_membase = membase; isa_portbase = portbase; isa_portshift = portshift; - isa_sysctl_header = register_sysctl_table(ctl_bus); + isa_sysctl_header = register_sysctl("bus/isa", ctl_isa_vars); } diff --git a/arch/arm/kernel/iwmmxt.S b/arch/arm/kernel/iwmmxt.S index d2b4ac06e4ed..4a335d3c5969 100644 --- a/arch/arm/kernel/iwmmxt.S +++ b/arch/arm/kernel/iwmmxt.S @@ -18,18 +18,6 @@ #include <asm/assembler.h> #include "iwmmxt.h" -#if defined(CONFIG_CPU_PJ4) || defined(CONFIG_CPU_PJ4B) -#define PJ4(code...) code -#define XSC(code...) -#elif defined(CONFIG_CPU_MOHAWK) || \ - defined(CONFIG_CPU_XSC3) || \ - defined(CONFIG_CPU_XSCALE) -#define PJ4(code...) -#define XSC(code...) code -#else -#error "Unsupported iWMMXt architecture" -#endif - #define MMX_WR0 (0x00) #define MMX_WR1 (0x08) #define MMX_WR2 (0x10) @@ -58,9 +46,19 @@ .text .arm +ENTRY(iwmmxt_undef_handler) + push {r9, r10, lr} + get_thread_info r10 + mov r9, pc + b iwmmxt_task_enable + mov r0, #0 + pop {r9, r10, pc} +ENDPROC(iwmmxt_undef_handler) + /* * Lazy switching of Concan coprocessor context * + * r0 = struct pt_regs pointer * r10 = struct thread_info pointer * r9 = ret_from_exception * lr = undefined instr exit @@ -71,25 +69,21 @@ ENTRY(iwmmxt_task_enable) inc_preempt_count r10, r3 - XSC(mrc p15, 0, r2, c15, c1, 0) - PJ4(mrc p15, 0, r2, c1, c0, 2) + mrc p15, 0, r2, c15, c1, 0 @ CP0 and CP1 accessible? - XSC(tst r2, #0x3) - PJ4(tst r2, #0xf) + tst r2, #0x3 bne 4f @ if so no business here @ enable access to CP0 and CP1 - XSC(orr r2, r2, #0x3) - XSC(mcr p15, 0, r2, c15, c1, 0) - PJ4(orr r2, r2, #0xf) - PJ4(mcr p15, 0, r2, c1, c0, 2) + orr r2, r2, #0x3 + mcr p15, 0, r2, c15, c1, 0 ldr r3, =concan_owner - add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area - ldr r2, [sp, #60] @ current task pc value + ldr r2, [r0, #S_PC] @ current task pc value ldr r1, [r3] @ get current Concan owner - str r0, [r3] @ this task now owns Concan regs sub r2, r2, #4 @ adjust pc back - str r2, [sp, #60] + str r2, [r0, #S_PC] + add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area + str r0, [r3] @ this task now owns Concan regs mrc p15, 0, r2, c2, c0, 0 mov r2, r2 @ cpwait @@ -208,12 +202,9 @@ ENTRY(iwmmxt_task_disable) bne 1f @ no: quit @ enable access to CP0 and CP1 - XSC(mrc p15, 0, r4, c15, c1, 0) - XSC(orr r4, r4, #0x3) - XSC(mcr p15, 0, r4, c15, c1, 0) - PJ4(mrc p15, 0, r4, c1, c0, 2) - PJ4(orr r4, r4, #0xf) - PJ4(mcr p15, 0, r4, c1, c0, 2) + mrc p15, 0, r4, c15, c1, 0 + orr r4, r4, #0x3 + mcr p15, 0, r4, c15, c1, 0 mov r0, #0 @ nothing to load str r0, [r3] @ no more current owner @@ -222,10 +213,8 @@ ENTRY(iwmmxt_task_disable) bl concan_save @ disable access to CP0 and CP1 - XSC(bic r4, r4, #0x3) - XSC(mcr p15, 0, r4, c15, c1, 0) - PJ4(bic r4, r4, #0xf) - PJ4(mcr p15, 0, r4, c1, c0, 2) + bic r4, r4, #0x3 + mcr p15, 0, r4, c15, c1, 0 mrc p15, 0, r2, c2, c0, 0 mov r2, r2 @ cpwait @@ -320,11 +309,9 @@ ENDPROC(iwmmxt_task_restore) */ ENTRY(iwmmxt_task_switch) - XSC(mrc p15, 0, r1, c15, c1, 0) - PJ4(mrc p15, 0, r1, c1, c0, 2) + mrc p15, 0, r1, c15, c1, 0 @ CP0 and CP1 accessible? - XSC(tst r1, #0x3) - PJ4(tst r1, #0xf) + tst r1, #0x3 bne 1f @ yes: block them for next task ldr r2, =concan_owner @@ -334,10 +321,8 @@ ENTRY(iwmmxt_task_switch) retne lr @ no: leave Concan disabled 1: @ flip Concan access - XSC(eor r1, r1, #0x3) - XSC(mcr p15, 0, r1, c15, c1, 0) - PJ4(eor r1, r1, #0xf) - PJ4(mcr p15, 0, r1, c1, c0, 2) + eor r1, r1, #0x3 + mcr p15, 0, r1, c15, c1, 0 mrc p15, 0, r1, c2, c0, 0 sub pc, lr, r1, lsr #32 @ cpwait and return diff --git a/arch/arm/kernel/jump_label.c b/arch/arm/kernel/jump_label.c index eb9c24b6e8e2..a06a92d0f550 100644 --- a/arch/arm/kernel/jump_label.c +++ b/arch/arm/kernel/jump_label.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/jump_label.h> -#include <asm/patch.h> +#include <asm/text-patching.h> #include <asm/insn.h> static void __arch_jump_label_transform(struct jump_entry *entry, diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c index 22f937e6f3ff..ab76c55fd610 100644 --- a/arch/arm/kernel/kgdb.c +++ b/arch/arm/kernel/kgdb.c @@ -15,7 +15,7 @@ #include <linux/kgdb.h> #include <linux/uaccess.h> -#include <asm/patch.h> +#include <asm/text-patching.h> #include <asm/traps.h> struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c index f567032a09c0..dd430477e7c1 100644 --- a/arch/arm/kernel/machine_kexec.c +++ b/arch/arm/kernel/machine_kexec.c @@ -73,10 +73,12 @@ void machine_kexec_cleanup(struct kimage *image) { } -void machine_crash_nonpanic_core(void *unused) +static void machine_crash_nonpanic_core(void *unused) { struct pt_regs regs; + local_fiq_disable(); + crash_setup_regs(®s, get_irq_regs()); printk(KERN_DEBUG "CPU %u will stop doing anything useful since another CPU has crashed\n", smp_processor_id()); @@ -92,16 +94,28 @@ void machine_crash_nonpanic_core(void *unused) } } +static DEFINE_PER_CPU(call_single_data_t, cpu_stop_csd) = + CSD_INIT(machine_crash_nonpanic_core, NULL); + void crash_smp_send_stop(void) { static int cpus_stopped; unsigned long msecs; + call_single_data_t *csd; + int cpu, this_cpu = raw_smp_processor_id(); if (cpus_stopped) return; atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); - smp_call_function(machine_crash_nonpanic_core, NULL, false); + for_each_online_cpu(cpu) { + if (cpu == this_cpu) + continue; + + csd = &per_cpu(cpu_stop_csd, cpu); + smp_call_function_single_async(cpu, csd); + } + msecs = 1000; /* Wait at most a second for the other cpus to stop */ while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) { mdelay(1); @@ -113,29 +127,6 @@ void crash_smp_send_stop(void) cpus_stopped = 1; } -static void machine_kexec_mask_interrupts(void) -{ - unsigned int i; - struct irq_desc *desc; - - for_each_irq_desc(i, desc) { - struct irq_chip *chip; - - chip = irq_desc_get_chip(desc); - if (!chip) - continue; - - if (chip->irq_eoi && irqd_irq_inprogress(&desc->irq_data)) - chip->irq_eoi(&desc->irq_data); - - if (chip->irq_mask) - chip->irq_mask(&desc->irq_data); - - if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data)) - chip->irq_disable(&desc->irq_data); - } -} - void machine_crash_shutdown(struct pt_regs *regs) { local_irq_disable(); @@ -184,10 +175,3 @@ void machine_kexec(struct kimage *image) soft_restart(reboot_entry_phys); } - -void arch_crash_save_vmcoreinfo(void) -{ -#ifdef CONFIG_ARM_LPAE - VMCOREINFO_CONFIG(ARM_LPAE); -#endif -} diff --git a/arch/arm/kernel/module-plts.c b/arch/arm/kernel/module-plts.c index 1fc309b41f94..354ce16d83cb 100644 --- a/arch/arm/kernel/module-plts.c +++ b/arch/arm/kernel/module-plts.c @@ -28,11 +28,6 @@ static const u32 fixed_plts[] = { #endif }; -static bool in_init(const struct module *mod, unsigned long loc) -{ - return loc - (u32)mod->init_layout.base < mod->init_layout.size; -} - static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) { int i; @@ -50,8 +45,8 @@ static void prealloc_fixed(struct mod_plt_sec *pltsec, struct plt_entries *plt) u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val) { - struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core : - &mod->arch.init; + struct mod_plt_sec *pltsec = !within_module_init(loc, mod) ? + &mod->arch.core : &mod->arch.init; struct plt_entries *plt; int idx; @@ -256,7 +251,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, /* sort by type and symbol index */ sort(rels, numrels, sizeof(Elf32_Rel), cmp_rel, NULL); - if (strncmp(secstrings + dstsec->sh_name, ".init", 5) != 0) + if (!module_init_layout_section(secstrings + dstsec->sh_name)) core_plts += count_plts(syms, dstsec->sh_addr, rels, numrels, s->sh_info); else @@ -284,3 +279,15 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, mod->arch.core.plt->sh_size, mod->arch.init.plt->sh_size); return 0; } + +bool in_module_plt(unsigned long loc) +{ + struct module *mod; + bool ret; + + guard(rcu)(); + mod = __module_text_address(loc); + ret = mod && (loc - (u32)mod->arch.core.plt_ent < mod->arch.core.plt_count * PLT_ENT_SIZE || + loc - (u32)mod->arch.init.plt_ent < mod->arch.init.plt_count * PLT_ENT_SIZE); + return ret; +} diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index d59c36dc0494..da488d92e7a0 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -12,48 +12,14 @@ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/elf.h> -#include <linux/vmalloc.h> #include <linux/fs.h> #include <linux/string.h> -#include <linux/gfp.h> #include <asm/sections.h> #include <asm/smp_plat.h> #include <asm/unwind.h> #include <asm/opcodes.h> -#ifdef CONFIG_XIP_KERNEL -/* - * The XIP kernel text is mapped in the module area for modules and - * some other stuff to work without any indirect relocations. - * MODULES_VADDR is redefined here and not in asm/memory.h to avoid - * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. - */ -#undef MODULES_VADDR -#define MODULES_VADDR (((unsigned long)_exiprom + ~PMD_MASK) & PMD_MASK) -#endif - -#ifdef CONFIG_MMU -void *module_alloc(unsigned long size) -{ - gfp_t gfp_mask = GFP_KERNEL; - void *p; - - /* Silence the initial allocation */ - if (IS_ENABLED(CONFIG_ARM_MODULE_PLTS)) - gfp_mask |= __GFP_NOWARN; - - p = __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - gfp_mask, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, - __builtin_return_address(0)); - if (!IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || p) - return p; - return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, - __builtin_return_address(0)); -} -#endif - bool module_init_section(const char *name) { return strstarts(name, ".init") || @@ -169,8 +135,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, offset = __mem_to_opcode_arm(*(u32 *)loc); offset = (offset & 0x00ffffff) << 2; - if (offset & 0x02000000) - offset -= 0x04000000; + offset = sign_extend32(offset, 25); offset += sym->st_value - loc; @@ -236,7 +201,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, case R_ARM_MOVT_PREL: offset = tmp = __mem_to_opcode_arm(*(u32 *)loc); offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff); - offset = (offset ^ 0x8000) - 0x8000; + offset = sign_extend32(offset, 15); offset += sym->st_value; if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_PREL || @@ -344,8 +309,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, ((~(j2 ^ sign) & 1) << 22) | ((upper & 0x03ff) << 12) | ((lower & 0x07ff) << 1); - if (offset & 0x01000000) - offset -= 0x02000000; + offset = sign_extend32(offset, 24); offset += sym->st_value - loc; /* @@ -401,7 +365,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, offset = ((upper & 0x000f) << 12) | ((upper & 0x0400) << 1) | ((lower & 0x7000) >> 4) | (lower & 0x00ff); - offset = (offset ^ 0x8000) - 0x8000; + offset = sign_extend32(offset, 15); offset += sym->st_value; if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_PREL || @@ -431,11 +395,6 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, return 0; } -struct mod_unwind_map { - const Elf_Shdr *unw_sec; - const Elf_Shdr *txt_sec; -}; - static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, const char *name) { diff --git a/arch/arm/kernel/patch.c b/arch/arm/kernel/patch.c index e9e828b6bb30..4d45e60cd46d 100644 --- a/arch/arm/kernel/patch.c +++ b/arch/arm/kernel/patch.c @@ -9,7 +9,7 @@ #include <asm/fixmap.h> #include <asm/smp_plat.h> #include <asm/opcodes.h> -#include <asm/patch.h> +#include <asm/text-patching.h> struct patch { void *addr; diff --git a/arch/arm/kernel/perf_callchain.c b/arch/arm/kernel/perf_callchain.c index bc6b246ab55e..a2601b1ef318 100644 --- a/arch/arm/kernel/perf_callchain.c +++ b/arch/arm/kernel/perf_callchain.c @@ -81,13 +81,11 @@ perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs * whist unwinding the stackframe and is like a subroutine return so we use * the PC. */ -static int -callchain_trace(struct stackframe *fr, - void *data) +static bool +callchain_trace(void *data, unsigned long pc) { struct perf_callchain_entry_ctx *entry = data; - perf_callchain_store(entry, fr->pc); - return 0; + return perf_callchain_store(entry, pc) == 0; } void @@ -98,20 +96,3 @@ perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *re arm_get_current_stackframe(regs, &fr); walk_stackframe(&fr, callchain_trace, entry); } - -unsigned long perf_instruction_pointer(struct pt_regs *regs) -{ - return instruction_pointer(regs); -} - -unsigned long perf_misc_flags(struct pt_regs *regs) -{ - int misc = 0; - - if (user_mode(regs)) - misc |= PERF_RECORD_MISC_USER; - else - misc |= PERF_RECORD_MISC_KERNEL; - - return misc; -} diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c deleted file mode 100644 index 1ae99deeec54..000000000000 --- a/arch/arm/kernel/perf_event_v6.c +++ /dev/null @@ -1,590 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ARMv6 Performance counter handling code. - * - * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles - * - * ARMv6 has 2 configurable performance counters and a single cycle counter. - * They all share a single reset bit but can be written to zero so we can use - * that for a reset. - * - * The counters can't be individually enabled or disabled so when we remove - * one event and replace it with another we could get spurious counts from the - * wrong event. However, we can take advantage of the fact that the - * performance counters can export events to the event bus, and the event bus - * itself can be monitored. This requires that we *don't* export the events to - * the event bus. The procedure for disabling a configurable counter is: - * - change the counter to count the ETMEXTOUT[0] signal (0x20). This - * effectively stops the counter from counting. - * - disable the counter's interrupt generation (each counter has it's - * own interrupt enable bit). - * Once stopped, the counter value can be written as 0 to reset. - * - * To enable a counter: - * - enable the counter's interrupt generation. - * - set the new event type. - * - * Note: the dedicated cycle counter only counts cycles and can't be - * enabled/disabled independently of the others. When we want to disable the - * cycle counter, we have to just disable the interrupt reporting and start - * ignoring that counter. When re-enabling, we have to reset the value and - * enable the interrupt. - */ - -#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) - -#include <asm/cputype.h> -#include <asm/irq_regs.h> - -#include <linux/of.h> -#include <linux/perf/arm_pmu.h> -#include <linux/platform_device.h> - -enum armv6_perf_types { - ARMV6_PERFCTR_ICACHE_MISS = 0x0, - ARMV6_PERFCTR_IBUF_STALL = 0x1, - ARMV6_PERFCTR_DDEP_STALL = 0x2, - ARMV6_PERFCTR_ITLB_MISS = 0x3, - ARMV6_PERFCTR_DTLB_MISS = 0x4, - ARMV6_PERFCTR_BR_EXEC = 0x5, - ARMV6_PERFCTR_BR_MISPREDICT = 0x6, - ARMV6_PERFCTR_INSTR_EXEC = 0x7, - ARMV6_PERFCTR_DCACHE_HIT = 0x9, - ARMV6_PERFCTR_DCACHE_ACCESS = 0xA, - ARMV6_PERFCTR_DCACHE_MISS = 0xB, - ARMV6_PERFCTR_DCACHE_WBACK = 0xC, - ARMV6_PERFCTR_SW_PC_CHANGE = 0xD, - ARMV6_PERFCTR_MAIN_TLB_MISS = 0xF, - ARMV6_PERFCTR_EXPL_D_ACCESS = 0x10, - ARMV6_PERFCTR_LSU_FULL_STALL = 0x11, - ARMV6_PERFCTR_WBUF_DRAINED = 0x12, - ARMV6_PERFCTR_CPU_CYCLES = 0xFF, - ARMV6_PERFCTR_NOP = 0x20, -}; - -enum armv6_counters { - ARMV6_CYCLE_COUNTER = 0, - ARMV6_COUNTER0, - ARMV6_COUNTER1, -}; - -/* - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV6_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6_PERFCTR_INSTR_EXEC, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6_PERFCTR_BR_MISPREDICT, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV6_PERFCTR_IBUF_STALL, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV6_PERFCTR_LSU_FULL_STALL, -}; - -static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS, - - /* - * The ARM performance counters can count micro DTLB misses, micro ITLB - * misses and main TLB misses. There isn't an event for TLB misses, so - * use the micro misses here and if users want the main TLB misses they - * can use a raw counter. - */ - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, -}; - -enum armv6mpcore_perf_types { - ARMV6MPCORE_PERFCTR_ICACHE_MISS = 0x0, - ARMV6MPCORE_PERFCTR_IBUF_STALL = 0x1, - ARMV6MPCORE_PERFCTR_DDEP_STALL = 0x2, - ARMV6MPCORE_PERFCTR_ITLB_MISS = 0x3, - ARMV6MPCORE_PERFCTR_DTLB_MISS = 0x4, - ARMV6MPCORE_PERFCTR_BR_EXEC = 0x5, - ARMV6MPCORE_PERFCTR_BR_NOTPREDICT = 0x6, - ARMV6MPCORE_PERFCTR_BR_MISPREDICT = 0x7, - ARMV6MPCORE_PERFCTR_INSTR_EXEC = 0x8, - ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS = 0xA, - ARMV6MPCORE_PERFCTR_DCACHE_RDMISS = 0xB, - ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS = 0xC, - ARMV6MPCORE_PERFCTR_DCACHE_WRMISS = 0xD, - ARMV6MPCORE_PERFCTR_DCACHE_EVICTION = 0xE, - ARMV6MPCORE_PERFCTR_SW_PC_CHANGE = 0xF, - ARMV6MPCORE_PERFCTR_MAIN_TLB_MISS = 0x10, - ARMV6MPCORE_PERFCTR_EXPL_MEM_ACCESS = 0x11, - ARMV6MPCORE_PERFCTR_LSU_FULL_STALL = 0x12, - ARMV6MPCORE_PERFCTR_WBUF_DRAINED = 0x13, - ARMV6MPCORE_PERFCTR_CPU_CYCLES = 0xFF, -}; - -/* - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned armv6mpcore_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV6MPCORE_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_INSTR_EXEC, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_BR_EXEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6MPCORE_PERFCTR_BR_MISPREDICT, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV6MPCORE_PERFCTR_IBUF_STALL, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV6MPCORE_PERFCTR_LSU_FULL_STALL, -}; - -static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DCACHE_RDMISS, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DCACHE_WRMISS, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ICACHE_MISS, - - /* - * The ARM performance counters can count micro DTLB misses, micro ITLB - * misses and main TLB misses. There isn't an event for TLB misses, so - * use the micro misses here and if users want the main TLB misses they - * can use a raw counter. - */ - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS, -}; - -static inline unsigned long -armv6_pmcr_read(void) -{ - u32 val; - asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r"(val)); - return val; -} - -static inline void -armv6_pmcr_write(unsigned long val) -{ - asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r"(val)); -} - -#define ARMV6_PMCR_ENABLE (1 << 0) -#define ARMV6_PMCR_CTR01_RESET (1 << 1) -#define ARMV6_PMCR_CCOUNT_RESET (1 << 2) -#define ARMV6_PMCR_CCOUNT_DIV (1 << 3) -#define ARMV6_PMCR_COUNT0_IEN (1 << 4) -#define ARMV6_PMCR_COUNT1_IEN (1 << 5) -#define ARMV6_PMCR_CCOUNT_IEN (1 << 6) -#define ARMV6_PMCR_COUNT0_OVERFLOW (1 << 8) -#define ARMV6_PMCR_COUNT1_OVERFLOW (1 << 9) -#define ARMV6_PMCR_CCOUNT_OVERFLOW (1 << 10) -#define ARMV6_PMCR_EVT_COUNT0_SHIFT 20 -#define ARMV6_PMCR_EVT_COUNT0_MASK (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT) -#define ARMV6_PMCR_EVT_COUNT1_SHIFT 12 -#define ARMV6_PMCR_EVT_COUNT1_MASK (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT) - -#define ARMV6_PMCR_OVERFLOWED_MASK \ - (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \ - ARMV6_PMCR_CCOUNT_OVERFLOW) - -static inline int -armv6_pmcr_has_overflowed(unsigned long pmcr) -{ - return pmcr & ARMV6_PMCR_OVERFLOWED_MASK; -} - -static inline int -armv6_pmcr_counter_has_overflowed(unsigned long pmcr, - enum armv6_counters counter) -{ - int ret = 0; - - if (ARMV6_CYCLE_COUNTER == counter) - ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW; - else if (ARMV6_COUNTER0 == counter) - ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW; - else if (ARMV6_COUNTER1 == counter) - ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW; - else - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - - return ret; -} - -static inline u64 armv6pmu_read_counter(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - unsigned long value = 0; - - if (ARMV6_CYCLE_COUNTER == counter) - asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r"(value)); - else if (ARMV6_COUNTER0 == counter) - asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r"(value)); - else if (ARMV6_COUNTER1 == counter) - asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r"(value)); - else - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - - return value; -} - -static inline void armv6pmu_write_counter(struct perf_event *event, u64 value) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == counter) - asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r"(value)); - else if (ARMV6_COUNTER0 == counter) - asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r"(value)); - else if (ARMV6_COUNTER1 == counter) - asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r"(value)); - else - WARN_ONCE(1, "invalid counter number (%d)\n", counter); -} - -static void armv6pmu_enable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == idx) { - mask = 0; - evt = ARMV6_PMCR_CCOUNT_IEN; - } else if (ARMV6_COUNTER0 == idx) { - mask = ARMV6_PMCR_EVT_COUNT0_MASK; - evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) | - ARMV6_PMCR_COUNT0_IEN; - } else if (ARMV6_COUNTER1 == idx) { - mask = ARMV6_PMCR_EVT_COUNT1_MASK; - evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) | - ARMV6_PMCR_COUNT1_IEN; - } else { - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - /* - * Mask out the current event and set the counter to count the event - * that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~mask; - val |= evt; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static irqreturn_t -armv6pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - unsigned long pmcr = armv6_pmcr_read(); - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - if (!armv6_pmcr_has_overflowed(pmcr)) - return IRQ_NONE; - - regs = get_irq_regs(); - - /* - * The interrupts are cleared by writing the overflow flags back to - * the control register. All of the other bits don't have any effect - * if they are rewritten, so write the whole value back. - */ - armv6_pmcr_write(pmcr); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - /* Ignore if we don't have an event. */ - if (!event) - continue; - - /* - * We have a single interrupt for all counters. Check that - * each counter has overflowed before we process it. - */ - if (!armv6_pmcr_counter_has_overflowed(pmcr, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - /* - * Handle the pending perf events. - * - * Note: this call *must* be run with interrupts disabled. For - * platforms that can have the PMU interrupts raised as an NMI, this - * will not work. - */ - irq_work_run(); - - return IRQ_HANDLED; -} - -static void armv6pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val |= ARMV6_PMCR_ENABLE; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv6pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~ARMV6_PMCR_ENABLE; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int -armv6pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - /* Always place a cycle counter into the cycle counter. */ - if (ARMV6_PERFCTR_CPU_CYCLES == hwc->config_base) { - if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask)) - return -EAGAIN; - - return ARMV6_CYCLE_COUNTER; - } else { - /* - * For anything other than a cycle counter, try and use - * counter0 and counter1. - */ - if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) - return ARMV6_COUNTER1; - - if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) - return ARMV6_COUNTER0; - - /* The counters are all in use. */ - return -EAGAIN; - } -} - -static void armv6pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - clear_bit(event->hw.idx, cpuc->used_mask); -} - -static void armv6pmu_disable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == idx) { - mask = ARMV6_PMCR_CCOUNT_IEN; - evt = 0; - } else if (ARMV6_COUNTER0 == idx) { - mask = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK; - evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT; - } else if (ARMV6_COUNTER1 == idx) { - mask = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK; - evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT; - } else { - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - /* - * Mask out the current event and set the counter to count the number - * of ETM bus signal assertion cycles. The external reporting should - * be disabled and so this should never increment. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~mask; - val |= evt; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv6mpcore_pmu_disable_event(struct perf_event *event) -{ - unsigned long val, mask, flags, evt = 0; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (ARMV6_CYCLE_COUNTER == idx) { - mask = ARMV6_PMCR_CCOUNT_IEN; - } else if (ARMV6_COUNTER0 == idx) { - mask = ARMV6_PMCR_COUNT0_IEN; - } else if (ARMV6_COUNTER1 == idx) { - mask = ARMV6_PMCR_COUNT1_IEN; - } else { - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - /* - * Unlike UP ARMv6, we don't have a way of stopping the counters. We - * simply disable the interrupt reporting. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = armv6_pmcr_read(); - val &= ~mask; - val |= evt; - armv6_pmcr_write(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int armv6_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv6_perf_map, - &armv6_perf_cache_map, 0xFF); -} - -static void armv6pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->handle_irq = armv6pmu_handle_irq; - cpu_pmu->enable = armv6pmu_enable_event; - cpu_pmu->disable = armv6pmu_disable_event; - cpu_pmu->read_counter = armv6pmu_read_counter; - cpu_pmu->write_counter = armv6pmu_write_counter; - cpu_pmu->get_event_idx = armv6pmu_get_event_idx; - cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx; - cpu_pmu->start = armv6pmu_start; - cpu_pmu->stop = armv6pmu_stop; - cpu_pmu->map_event = armv6_map_event; - cpu_pmu->num_events = 3; -} - -static int armv6_1136_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv6pmu_init(cpu_pmu); - cpu_pmu->name = "armv6_1136"; - return 0; -} - -static int armv6_1156_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv6pmu_init(cpu_pmu); - cpu_pmu->name = "armv6_1156"; - return 0; -} - -static int armv6_1176_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv6pmu_init(cpu_pmu); - cpu_pmu->name = "armv6_1176"; - return 0; -} - -/* - * ARMv6mpcore is almost identical to single core ARMv6 with the exception - * that some of the events have different enumerations and that there is no - * *hack* to stop the programmable counters. To stop the counters we simply - * disable the interrupt reporting and update the event. When unthrottling we - * reset the period and enable the interrupt reporting. - */ - -static int armv6mpcore_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv6mpcore_perf_map, - &armv6mpcore_perf_cache_map, 0xFF); -} - -static int armv6mpcore_pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->name = "armv6_11mpcore"; - cpu_pmu->handle_irq = armv6pmu_handle_irq; - cpu_pmu->enable = armv6pmu_enable_event; - cpu_pmu->disable = armv6mpcore_pmu_disable_event; - cpu_pmu->read_counter = armv6pmu_read_counter; - cpu_pmu->write_counter = armv6pmu_write_counter; - cpu_pmu->get_event_idx = armv6pmu_get_event_idx; - cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx; - cpu_pmu->start = armv6pmu_start; - cpu_pmu->stop = armv6pmu_stop; - cpu_pmu->map_event = armv6mpcore_map_event; - cpu_pmu->num_events = 3; - - return 0; -} - -static const struct of_device_id armv6_pmu_of_device_ids[] = { - {.compatible = "arm,arm11mpcore-pmu", .data = armv6mpcore_pmu_init}, - {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init}, - {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init}, - { /* sentinel value */ } -}; - -static const struct pmu_probe_info armv6_pmu_probe_table[] = { - ARM_PMU_PROBE(ARM_CPU_PART_ARM1136, armv6_1136_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_ARM1156, armv6_1156_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_ARM1176, armv6_1176_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_ARM11MPCORE, armv6mpcore_pmu_init), - { /* sentinel value */ } -}; - -static int armv6_pmu_device_probe(struct platform_device *pdev) -{ - return arm_pmu_device_probe(pdev, armv6_pmu_of_device_ids, - armv6_pmu_probe_table); -} - -static struct platform_driver armv6_pmu_driver = { - .driver = { - .name = "armv6-pmu", - .of_match_table = armv6_pmu_of_device_ids, - }, - .probe = armv6_pmu_device_probe, -}; - -builtin_platform_driver(armv6_pmu_driver); -#endif /* CONFIG_CPU_V6 || CONFIG_CPU_V6K */ diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c deleted file mode 100644 index eb2190477da1..000000000000 --- a/arch/arm/kernel/perf_event_v7.c +++ /dev/null @@ -1,2047 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. - * - * ARMv7 support: Jean Pihet <jpihet@mvista.com> - * 2010 (c) MontaVista Software, LLC. - * - * Copied from ARMv6 code, with the low level code inspired - * by the ARMv7 Oprofile code. - * - * Cortex-A8 has up to 4 configurable performance counters and - * a single cycle counter. - * Cortex-A9 has up to 31 configurable performance counters and - * a single cycle counter. - * - * All counters can be enabled/disabled and IRQ masked separately. The cycle - * counter and all 4 performance counters together can be reset separately. - */ - -#ifdef CONFIG_CPU_V7 - -#include <asm/cp15.h> -#include <asm/cputype.h> -#include <asm/irq_regs.h> -#include <asm/vfp.h> -#include "../vfp/vfpinstr.h" - -#include <linux/of.h> -#include <linux/perf/arm_pmu.h> -#include <linux/platform_device.h> - -/* - * Common ARMv7 event types - * - * Note: An implementation may not be able to count all of these events - * but the encodings are considered to be `reserved' in the case that - * they are not available. - */ -#define ARMV7_PERFCTR_PMNC_SW_INCR 0x00 -#define ARMV7_PERFCTR_L1_ICACHE_REFILL 0x01 -#define ARMV7_PERFCTR_ITLB_REFILL 0x02 -#define ARMV7_PERFCTR_L1_DCACHE_REFILL 0x03 -#define ARMV7_PERFCTR_L1_DCACHE_ACCESS 0x04 -#define ARMV7_PERFCTR_DTLB_REFILL 0x05 -#define ARMV7_PERFCTR_MEM_READ 0x06 -#define ARMV7_PERFCTR_MEM_WRITE 0x07 -#define ARMV7_PERFCTR_INSTR_EXECUTED 0x08 -#define ARMV7_PERFCTR_EXC_TAKEN 0x09 -#define ARMV7_PERFCTR_EXC_EXECUTED 0x0A -#define ARMV7_PERFCTR_CID_WRITE 0x0B - -/* - * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. - * It counts: - * - all (taken) branch instructions, - * - instructions that explicitly write the PC, - * - exception generating instructions. - */ -#define ARMV7_PERFCTR_PC_WRITE 0x0C -#define ARMV7_PERFCTR_PC_IMM_BRANCH 0x0D -#define ARMV7_PERFCTR_PC_PROC_RETURN 0x0E -#define ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS 0x0F -#define ARMV7_PERFCTR_PC_BRANCH_MIS_PRED 0x10 -#define ARMV7_PERFCTR_CLOCK_CYCLES 0x11 -#define ARMV7_PERFCTR_PC_BRANCH_PRED 0x12 - -/* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */ -#define ARMV7_PERFCTR_MEM_ACCESS 0x13 -#define ARMV7_PERFCTR_L1_ICACHE_ACCESS 0x14 -#define ARMV7_PERFCTR_L1_DCACHE_WB 0x15 -#define ARMV7_PERFCTR_L2_CACHE_ACCESS 0x16 -#define ARMV7_PERFCTR_L2_CACHE_REFILL 0x17 -#define ARMV7_PERFCTR_L2_CACHE_WB 0x18 -#define ARMV7_PERFCTR_BUS_ACCESS 0x19 -#define ARMV7_PERFCTR_MEM_ERROR 0x1A -#define ARMV7_PERFCTR_INSTR_SPEC 0x1B -#define ARMV7_PERFCTR_TTBR_WRITE 0x1C -#define ARMV7_PERFCTR_BUS_CYCLES 0x1D - -#define ARMV7_PERFCTR_CPU_CYCLES 0xFF - -/* ARMv7 Cortex-A8 specific event types */ -#define ARMV7_A8_PERFCTR_L2_CACHE_ACCESS 0x43 -#define ARMV7_A8_PERFCTR_L2_CACHE_REFILL 0x44 -#define ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS 0x50 -#define ARMV7_A8_PERFCTR_STALL_ISIDE 0x56 - -/* ARMv7 Cortex-A9 specific event types */ -#define ARMV7_A9_PERFCTR_INSTR_CORE_RENAME 0x68 -#define ARMV7_A9_PERFCTR_STALL_ICACHE 0x60 -#define ARMV7_A9_PERFCTR_STALL_DISPATCH 0x66 - -/* ARMv7 Cortex-A5 specific event types */ -#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL 0xc2 -#define ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP 0xc3 - -/* ARMv7 Cortex-A15 specific event types */ -#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ 0x40 -#define ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41 -#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ 0x42 -#define ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE 0x43 - -#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ 0x4C -#define ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE 0x4D - -#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ 0x50 -#define ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51 -#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ 0x52 -#define ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE 0x53 - -#define ARMV7_A15_PERFCTR_PC_WRITE_SPEC 0x76 - -/* ARMv7 Cortex-A12 specific event types */ -#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ 0x40 -#define ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE 0x41 - -#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ 0x50 -#define ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE 0x51 - -#define ARMV7_A12_PERFCTR_PC_WRITE_SPEC 0x76 - -#define ARMV7_A12_PERFCTR_PF_TLB_REFILL 0xe7 - -/* ARMv7 Krait specific event types */ -#define KRAIT_PMRESR0_GROUP0 0xcc -#define KRAIT_PMRESR1_GROUP0 0xd0 -#define KRAIT_PMRESR2_GROUP0 0xd4 -#define KRAIT_VPMRESR0_GROUP0 0xd8 - -#define KRAIT_PERFCTR_L1_ICACHE_ACCESS 0x10011 -#define KRAIT_PERFCTR_L1_ICACHE_MISS 0x10010 - -#define KRAIT_PERFCTR_L1_ITLB_ACCESS 0x12222 -#define KRAIT_PERFCTR_L1_DTLB_ACCESS 0x12210 - -/* ARMv7 Scorpion specific event types */ -#define SCORPION_LPM0_GROUP0 0x4c -#define SCORPION_LPM1_GROUP0 0x50 -#define SCORPION_LPM2_GROUP0 0x54 -#define SCORPION_L2LPM_GROUP0 0x58 -#define SCORPION_VLPM_GROUP0 0x5c - -#define SCORPION_ICACHE_ACCESS 0x10053 -#define SCORPION_ICACHE_MISS 0x10052 - -#define SCORPION_DTLB_ACCESS 0x12013 -#define SCORPION_DTLB_MISS 0x12012 - -#define SCORPION_ITLB_MISS 0x12021 - -/* - * Cortex-A8 HW events mapping - * - * The hardware events that we support. We do support cache operations but - * we have harvard caches and no way to combine instruction and data - * accesses/misses in hardware. - */ -static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE, -}; - -static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A8_PERFCTR_L2_CACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A9 HW events mapping - */ -static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV7_A9_PERFCTR_STALL_DISPATCH, -}; - -static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A5 HW events mapping - */ -static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, - [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - /* - * The prefetch counters don't differentiate between the I side and the - * D side. - */ - [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL, - [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A15 HW events mapping - */ -static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A15_PERFCTR_PC_WRITE_SPEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, -}; - -static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE, - - /* - * Not all performance counters differentiate between read and write - * accesses/misses so we're not always strictly correct, but it's the - * best we can do. Writes and reads get combined in these cases. - */ - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A7 HW events mapping - */ -static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, -}; - -static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_CACHE_ACCESS, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Cortex-A12 HW events mapping - */ -static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_A12_PERFCTR_PC_WRITE_SPEC, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_BUS_CYCLES, -}; - -static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - /* - * Not all performance counters differentiate between read and write - * accesses/misses so we're not always strictly correct, but it's the - * best we can do. Writes and reads get combined in these cases. - */ - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_ICACHE_REFILL, - - [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ, - [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE, - [C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACHE_REFILL, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, - [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV7_A12_PERFCTR_PF_TLB_REFILL, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_REFILL, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Krait HW events mapping - */ -static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, -}; - -static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, -}; - -static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = KRAIT_PERFCTR_L1_ICACHE_MISS, - - [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS, - - [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS, - - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -/* - * Scorpion HW events mapping - */ -static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, - [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, - [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, -}; - -static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - /* - * The performance counters don't differentiate between read and write - * accesses/misses so this isn't strictly correct, but it's the best we - * can do. Writes and reads get combined. - */ - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, - [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS, - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS, - /* - * Only ITLB misses and DTLB refills are supported. If users want the - * DTLB refills misses a raw counter must be used. - */ - [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, - [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, - [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, -}; - -PMU_FORMAT_ATTR(event, "config:0-7"); - -static struct attribute *armv7_pmu_format_attrs[] = { - &format_attr_event.attr, - NULL, -}; - -static struct attribute_group armv7_pmu_format_attr_group = { - .name = "format", - .attrs = armv7_pmu_format_attrs, -}; - -#define ARMV7_EVENT_ATTR_RESOLVE(m) #m -#define ARMV7_EVENT_ATTR(name, config) \ - PMU_EVENT_ATTR_STRING(name, armv7_event_attr_##name, \ - "event=" ARMV7_EVENT_ATTR_RESOLVE(config)) - -ARMV7_EVENT_ATTR(sw_incr, ARMV7_PERFCTR_PMNC_SW_INCR); -ARMV7_EVENT_ATTR(l1i_cache_refill, ARMV7_PERFCTR_L1_ICACHE_REFILL); -ARMV7_EVENT_ATTR(l1i_tlb_refill, ARMV7_PERFCTR_ITLB_REFILL); -ARMV7_EVENT_ATTR(l1d_cache_refill, ARMV7_PERFCTR_L1_DCACHE_REFILL); -ARMV7_EVENT_ATTR(l1d_cache, ARMV7_PERFCTR_L1_DCACHE_ACCESS); -ARMV7_EVENT_ATTR(l1d_tlb_refill, ARMV7_PERFCTR_DTLB_REFILL); -ARMV7_EVENT_ATTR(ld_retired, ARMV7_PERFCTR_MEM_READ); -ARMV7_EVENT_ATTR(st_retired, ARMV7_PERFCTR_MEM_WRITE); -ARMV7_EVENT_ATTR(inst_retired, ARMV7_PERFCTR_INSTR_EXECUTED); -ARMV7_EVENT_ATTR(exc_taken, ARMV7_PERFCTR_EXC_TAKEN); -ARMV7_EVENT_ATTR(exc_return, ARMV7_PERFCTR_EXC_EXECUTED); -ARMV7_EVENT_ATTR(cid_write_retired, ARMV7_PERFCTR_CID_WRITE); -ARMV7_EVENT_ATTR(pc_write_retired, ARMV7_PERFCTR_PC_WRITE); -ARMV7_EVENT_ATTR(br_immed_retired, ARMV7_PERFCTR_PC_IMM_BRANCH); -ARMV7_EVENT_ATTR(br_return_retired, ARMV7_PERFCTR_PC_PROC_RETURN); -ARMV7_EVENT_ATTR(unaligned_ldst_retired, ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS); -ARMV7_EVENT_ATTR(br_mis_pred, ARMV7_PERFCTR_PC_BRANCH_MIS_PRED); -ARMV7_EVENT_ATTR(cpu_cycles, ARMV7_PERFCTR_CLOCK_CYCLES); -ARMV7_EVENT_ATTR(br_pred, ARMV7_PERFCTR_PC_BRANCH_PRED); - -static struct attribute *armv7_pmuv1_event_attrs[] = { - &armv7_event_attr_sw_incr.attr.attr, - &armv7_event_attr_l1i_cache_refill.attr.attr, - &armv7_event_attr_l1i_tlb_refill.attr.attr, - &armv7_event_attr_l1d_cache_refill.attr.attr, - &armv7_event_attr_l1d_cache.attr.attr, - &armv7_event_attr_l1d_tlb_refill.attr.attr, - &armv7_event_attr_ld_retired.attr.attr, - &armv7_event_attr_st_retired.attr.attr, - &armv7_event_attr_inst_retired.attr.attr, - &armv7_event_attr_exc_taken.attr.attr, - &armv7_event_attr_exc_return.attr.attr, - &armv7_event_attr_cid_write_retired.attr.attr, - &armv7_event_attr_pc_write_retired.attr.attr, - &armv7_event_attr_br_immed_retired.attr.attr, - &armv7_event_attr_br_return_retired.attr.attr, - &armv7_event_attr_unaligned_ldst_retired.attr.attr, - &armv7_event_attr_br_mis_pred.attr.attr, - &armv7_event_attr_cpu_cycles.attr.attr, - &armv7_event_attr_br_pred.attr.attr, - NULL, -}; - -static struct attribute_group armv7_pmuv1_events_attr_group = { - .name = "events", - .attrs = armv7_pmuv1_event_attrs, -}; - -ARMV7_EVENT_ATTR(mem_access, ARMV7_PERFCTR_MEM_ACCESS); -ARMV7_EVENT_ATTR(l1i_cache, ARMV7_PERFCTR_L1_ICACHE_ACCESS); -ARMV7_EVENT_ATTR(l1d_cache_wb, ARMV7_PERFCTR_L1_DCACHE_WB); -ARMV7_EVENT_ATTR(l2d_cache, ARMV7_PERFCTR_L2_CACHE_ACCESS); -ARMV7_EVENT_ATTR(l2d_cache_refill, ARMV7_PERFCTR_L2_CACHE_REFILL); -ARMV7_EVENT_ATTR(l2d_cache_wb, ARMV7_PERFCTR_L2_CACHE_WB); -ARMV7_EVENT_ATTR(bus_access, ARMV7_PERFCTR_BUS_ACCESS); -ARMV7_EVENT_ATTR(memory_error, ARMV7_PERFCTR_MEM_ERROR); -ARMV7_EVENT_ATTR(inst_spec, ARMV7_PERFCTR_INSTR_SPEC); -ARMV7_EVENT_ATTR(ttbr_write_retired, ARMV7_PERFCTR_TTBR_WRITE); -ARMV7_EVENT_ATTR(bus_cycles, ARMV7_PERFCTR_BUS_CYCLES); - -static struct attribute *armv7_pmuv2_event_attrs[] = { - &armv7_event_attr_sw_incr.attr.attr, - &armv7_event_attr_l1i_cache_refill.attr.attr, - &armv7_event_attr_l1i_tlb_refill.attr.attr, - &armv7_event_attr_l1d_cache_refill.attr.attr, - &armv7_event_attr_l1d_cache.attr.attr, - &armv7_event_attr_l1d_tlb_refill.attr.attr, - &armv7_event_attr_ld_retired.attr.attr, - &armv7_event_attr_st_retired.attr.attr, - &armv7_event_attr_inst_retired.attr.attr, - &armv7_event_attr_exc_taken.attr.attr, - &armv7_event_attr_exc_return.attr.attr, - &armv7_event_attr_cid_write_retired.attr.attr, - &armv7_event_attr_pc_write_retired.attr.attr, - &armv7_event_attr_br_immed_retired.attr.attr, - &armv7_event_attr_br_return_retired.attr.attr, - &armv7_event_attr_unaligned_ldst_retired.attr.attr, - &armv7_event_attr_br_mis_pred.attr.attr, - &armv7_event_attr_cpu_cycles.attr.attr, - &armv7_event_attr_br_pred.attr.attr, - &armv7_event_attr_mem_access.attr.attr, - &armv7_event_attr_l1i_cache.attr.attr, - &armv7_event_attr_l1d_cache_wb.attr.attr, - &armv7_event_attr_l2d_cache.attr.attr, - &armv7_event_attr_l2d_cache_refill.attr.attr, - &armv7_event_attr_l2d_cache_wb.attr.attr, - &armv7_event_attr_bus_access.attr.attr, - &armv7_event_attr_memory_error.attr.attr, - &armv7_event_attr_inst_spec.attr.attr, - &armv7_event_attr_ttbr_write_retired.attr.attr, - &armv7_event_attr_bus_cycles.attr.attr, - NULL, -}; - -static struct attribute_group armv7_pmuv2_events_attr_group = { - .name = "events", - .attrs = armv7_pmuv2_event_attrs, -}; - -/* - * Perf Events' indices - */ -#define ARMV7_IDX_CYCLE_COUNTER 0 -#define ARMV7_IDX_COUNTER0 1 -#define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \ - (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1) - -#define ARMV7_MAX_COUNTERS 32 -#define ARMV7_COUNTER_MASK (ARMV7_MAX_COUNTERS - 1) - -/* - * ARMv7 low level PMNC access - */ - -/* - * Perf Event to low level counters mapping - */ -#define ARMV7_IDX_TO_COUNTER(x) \ - (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK) - -/* - * Per-CPU PMNC: config reg - */ -#define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ -#define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ -#define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ -#define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ -#define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ -#define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ -#define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ -#define ARMV7_PMNC_N_MASK 0x1f -#define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ - -/* - * FLAG: counters overflow flag status reg - */ -#define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ -#define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK - -/* - * PMXEVTYPER: Event selection reg - */ -#define ARMV7_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */ -#define ARMV7_EVTYPE_EVENT 0xff /* Mask for EVENT bits */ - -/* - * Event filters for PMUv2 - */ -#define ARMV7_EXCLUDE_PL1 BIT(31) -#define ARMV7_EXCLUDE_USER BIT(30) -#define ARMV7_INCLUDE_HYP BIT(27) - -/* - * Secure debug enable reg - */ -#define ARMV7_SDER_SUNIDEN BIT(1) /* Permit non-invasive debug */ - -static inline u32 armv7_pmnc_read(void) -{ - u32 val; - asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); - return val; -} - -static inline void armv7_pmnc_write(u32 val) -{ - val &= ARMV7_PMNC_MASK; - isb(); - asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); -} - -static inline int armv7_pmnc_has_overflowed(u32 pmnc) -{ - return pmnc & ARMV7_OVERFLOWED_MASK; -} - -static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx) -{ - return idx >= ARMV7_IDX_CYCLE_COUNTER && - idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); -} - -static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx) -{ - return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx)); -} - -static inline void armv7_pmnc_select_counter(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter)); - isb(); -} - -static inline u64 armv7pmu_read_counter(struct perf_event *event) -{ - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - u32 value = 0; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u reading wrong counter %d\n", - smp_processor_id(), idx); - } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { - asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); - } else { - armv7_pmnc_select_counter(idx); - asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value)); - } - - return value; -} - -static inline void armv7pmu_write_counter(struct perf_event *event, u64 value) -{ - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u writing wrong counter %d\n", - smp_processor_id(), idx); - } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { - asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value)); - } else { - armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value)); - } -} - -static inline void armv7_pmnc_write_evtsel(int idx, u32 val) -{ - armv7_pmnc_select_counter(idx); - val &= ARMV7_EVTYPE_MASK; - asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); -} - -static inline void armv7_pmnc_enable_counter(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter))); -} - -static inline void armv7_pmnc_disable_counter(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter))); -} - -static inline void armv7_pmnc_enable_intens(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter))); -} - -static inline void armv7_pmnc_disable_intens(int idx) -{ - u32 counter = ARMV7_IDX_TO_COUNTER(idx); - asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter))); - isb(); - /* Clear the overflow flag in case an interrupt is pending. */ - asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter))); - isb(); -} - -static inline u32 armv7_pmnc_getreset_flags(void) -{ - u32 val; - - /* Read */ - asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); - - /* Write to clear flags */ - val &= ARMV7_FLAG_MASK; - asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); - - return val; -} - -#ifdef DEBUG -static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu) -{ - u32 val; - unsigned int cnt; - - pr_info("PMNC registers dump:\n"); - - asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); - pr_info("PMNC =0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); - pr_info("CNTENS=0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); - pr_info("INTENS=0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); - pr_info("FLAGS =0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); - pr_info("SELECT=0x%08x\n", val); - - asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); - pr_info("CCNT =0x%08x\n", val); - - for (cnt = ARMV7_IDX_COUNTER0; - cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) { - armv7_pmnc_select_counter(cnt); - asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); - pr_info("CNT[%d] count =0x%08x\n", - ARMV7_IDX_TO_COUNTER(cnt), val); - asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); - pr_info("CNT[%d] evtsel=0x%08x\n", - ARMV7_IDX_TO_COUNTER(cnt), val); - } -} -#endif - -static void armv7pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n", - smp_processor_id(), idx); - return; - } - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* - * Disable counter - */ - armv7_pmnc_disable_counter(idx); - - /* - * Set event (if destined for PMNx counters) - * We only need to set the event for the cycle counter if we - * have the ability to perform event filtering. - */ - if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER) - armv7_pmnc_write_evtsel(idx, hwc->config_base); - - /* - * Enable interrupt for this counter - */ - armv7_pmnc_enable_intens(idx); - - /* - * Enable counter - */ - armv7_pmnc_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv7pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) { - pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n", - smp_processor_id(), idx); - return; - } - - /* - * Disable counter and interrupt - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* - * Disable counter - */ - armv7_pmnc_disable_counter(idx); - - /* - * Disable interrupt for this counter - */ - armv7_pmnc_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static irqreturn_t armv7pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - u32 pmnc; - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - /* - * Get and reset the IRQ flags - */ - pmnc = armv7_pmnc_getreset_flags(); - - /* - * Did an overflow occur? - */ - if (!armv7_pmnc_has_overflowed(pmnc)) - return IRQ_NONE; - - /* - * Handle the counter(s) overflow(s) - */ - regs = get_irq_regs(); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - /* Ignore if we don't have an event. */ - if (!event) - continue; - - /* - * We have a single interrupt for all counters. Check that - * each counter has overflowed before we process it. - */ - if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - /* - * Handle the pending perf events. - * - * Note: this call *must* be run with interrupts disabled. For - * platforms that can have the PMU interrupts raised as an NMI, this - * will not work. - */ - irq_work_run(); - - return IRQ_HANDLED; -} - -static void armv7pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - /* Enable all counters */ - armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void armv7pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - /* Disable all counters */ - armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT; - - /* Always place a cycle counter into the cycle counter. */ - if (evtype == ARMV7_PERFCTR_CPU_CYCLES) { - if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask)) - return -EAGAIN; - - return ARMV7_IDX_CYCLE_COUNTER; - } - - /* - * For anything other than a cycle counter, try and use - * the events counters - */ - for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) { - if (!test_and_set_bit(idx, cpuc->used_mask)) - return idx; - } - - /* The counters are all in use. */ - return -EAGAIN; -} - -static void armv7pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - clear_bit(event->hw.idx, cpuc->used_mask); -} - -/* - * Add an event filter to a given event. This will only work for PMUv2 PMUs. - */ -static int armv7pmu_set_event_filter(struct hw_perf_event *event, - struct perf_event_attr *attr) -{ - unsigned long config_base = 0; - - if (attr->exclude_idle) - return -EPERM; - if (attr->exclude_user) - config_base |= ARMV7_EXCLUDE_USER; - if (attr->exclude_kernel) - config_base |= ARMV7_EXCLUDE_PL1; - if (!attr->exclude_hv) - config_base |= ARMV7_INCLUDE_HYP; - - /* - * Install the filter into config_base as this is used to - * construct the event type. - */ - event->config_base = config_base; - - return 0; -} - -static void armv7pmu_reset(void *info) -{ - struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; - u32 idx, nb_cnt = cpu_pmu->num_events, val; - - if (cpu_pmu->secure_access) { - asm volatile("mrc p15, 0, %0, c1, c1, 1" : "=r" (val)); - val |= ARMV7_SDER_SUNIDEN; - asm volatile("mcr p15, 0, %0, c1, c1, 1" : : "r" (val)); - } - - /* The counter and interrupt enable registers are unknown at reset. */ - for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv7_pmnc_disable_counter(idx); - armv7_pmnc_disable_intens(idx); - } - - /* Initialize & Reset PMNC: C and P bits */ - armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); -} - -static int armv7_a8_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a8_perf_map, - &armv7_a8_perf_cache_map, 0xFF); -} - -static int armv7_a9_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a9_perf_map, - &armv7_a9_perf_cache_map, 0xFF); -} - -static int armv7_a5_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a5_perf_map, - &armv7_a5_perf_cache_map, 0xFF); -} - -static int armv7_a15_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a15_perf_map, - &armv7_a15_perf_cache_map, 0xFF); -} - -static int armv7_a7_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a7_perf_map, - &armv7_a7_perf_cache_map, 0xFF); -} - -static int armv7_a12_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &armv7_a12_perf_map, - &armv7_a12_perf_cache_map, 0xFF); -} - -static int krait_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &krait_perf_map, - &krait_perf_cache_map, 0xFFFFF); -} - -static int krait_map_event_no_branch(struct perf_event *event) -{ - return armpmu_map_event(event, &krait_perf_map_no_branch, - &krait_perf_cache_map, 0xFFFFF); -} - -static int scorpion_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &scorpion_perf_map, - &scorpion_perf_cache_map, 0xFFFFF); -} - -static void armv7pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->handle_irq = armv7pmu_handle_irq; - cpu_pmu->enable = armv7pmu_enable_event; - cpu_pmu->disable = armv7pmu_disable_event; - cpu_pmu->read_counter = armv7pmu_read_counter; - cpu_pmu->write_counter = armv7pmu_write_counter; - cpu_pmu->get_event_idx = armv7pmu_get_event_idx; - cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx; - cpu_pmu->start = armv7pmu_start; - cpu_pmu->stop = armv7pmu_stop; - cpu_pmu->reset = armv7pmu_reset; -}; - -static void armv7_read_num_pmnc_events(void *info) -{ - int *nb_cnt = info; - - /* Read the nb of CNTx counters supported from PMNC */ - *nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; - - /* Add the CPU cycles counter */ - *nb_cnt += 1; -} - -static int armv7_probe_num_events(struct arm_pmu *arm_pmu) -{ - return smp_call_function_any(&arm_pmu->supported_cpus, - armv7_read_num_pmnc_events, - &arm_pmu->num_events, 1); -} - -static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a8"; - cpu_pmu->map_event = armv7_a8_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv1_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a9"; - cpu_pmu->map_event = armv7_a9_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv1_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a5"; - cpu_pmu->map_event = armv7_a5_map_event; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv1_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a15"; - cpu_pmu->map_event = armv7_a15_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a7"; - cpu_pmu->map_event = armv7_a7_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a12"; - cpu_pmu->map_event = armv7_a12_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return armv7_probe_num_events(cpu_pmu); -} - -static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu) -{ - int ret = armv7_a12_pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_cortex_a17"; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = - &armv7_pmuv2_events_attr_group; - cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = - &armv7_pmu_format_attr_group; - return ret; -} - -/* - * Krait Performance Monitor Region Event Selection Register (PMRESRn) - * - * 31 30 24 16 8 0 - * +--------------------------------+ - * PMRESR0 | EN | CC | CC | CC | CC | N = 1, R = 0 - * +--------------------------------+ - * PMRESR1 | EN | CC | CC | CC | CC | N = 1, R = 1 - * +--------------------------------+ - * PMRESR2 | EN | CC | CC | CC | CC | N = 1, R = 2 - * +--------------------------------+ - * VPMRESR0 | EN | CC | CC | CC | CC | N = 2, R = ? - * +--------------------------------+ - * EN | G=3 | G=2 | G=1 | G=0 - * - * Event Encoding: - * - * hwc->config_base = 0xNRCCG - * - * N = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR) - * R = region register - * CC = class of events the group G is choosing from - * G = group or particular event - * - * Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2 - * - * A region (R) corresponds to a piece of the CPU (execution unit, instruction - * unit, etc.) while the event code (CC) corresponds to a particular class of - * events (interrupts for example). An event code is broken down into - * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for - * example). - */ - -#define KRAIT_EVENT (1 << 16) -#define VENUM_EVENT (2 << 16) -#define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) -#define PMRESRn_EN BIT(31) - -#define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */ -#define EVENT_GROUP(event) ((event) & 0xf) /* G */ -#define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */ -#define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */ -#define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */ - -static u32 krait_read_pmresrn(int n) -{ - u32 val; - - switch (n) { - case 0: - asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val)); - break; - case 1: - asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val)); - break; - case 2: - asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val)); - break; - default: - BUG(); /* Should be validated in krait_pmu_get_event_idx() */ - } - - return val; -} - -static void krait_write_pmresrn(int n, u32 val) -{ - switch (n) { - case 0: - asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val)); - break; - case 1: - asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val)); - break; - case 2: - asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val)); - break; - default: - BUG(); /* Should be validated in krait_pmu_get_event_idx() */ - } -} - -static u32 venum_read_pmresr(void) -{ - u32 val; - asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); - return val; -} - -static void venum_write_pmresr(u32 val) -{ - asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); -} - -static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val) -{ - u32 venum_new_val; - u32 fp_new_val; - - BUG_ON(preemptible()); - /* CPACR Enable CP10 and CP11 access */ - *venum_orig_val = get_copro_access(); - venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11); - set_copro_access(venum_new_val); - - /* Enable FPEXC */ - *fp_orig_val = fmrx(FPEXC); - fp_new_val = *fp_orig_val | FPEXC_EN; - fmxr(FPEXC, fp_new_val); -} - -static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val) -{ - BUG_ON(preemptible()); - /* Restore FPEXC */ - fmxr(FPEXC, fp_orig_val); - isb(); - /* Restore CPACR */ - set_copro_access(venum_orig_val); -} - -static u32 krait_get_pmresrn_event(unsigned int region) -{ - static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0, - KRAIT_PMRESR1_GROUP0, - KRAIT_PMRESR2_GROUP0 }; - return pmresrn_table[region]; -} - -static void krait_evt_setup(int idx, u32 config_base) -{ - u32 val; - u32 mask; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - unsigned int code = EVENT_CODE(config_base); - unsigned int group_shift; - bool venum_event = EVENT_VENUM(config_base); - - group_shift = group * 8; - mask = 0xff << group_shift; - - /* Configure evtsel for the region and group */ - if (venum_event) - val = KRAIT_VPMRESR0_GROUP0; - else - val = krait_get_pmresrn_event(region); - val += group; - /* Mix in mode-exclusion bits */ - val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); - armv7_pmnc_write_evtsel(idx, val); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = krait_read_pmresrn(region); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - krait_write_pmresrn(region, val); - } -} - -static u32 clear_pmresrn_group(u32 val, int group) -{ - u32 mask; - int group_shift; - - group_shift = group * 8; - mask = 0xff << group_shift; - val &= ~mask; - - /* Don't clear enable bit if entire region isn't disabled */ - if (val & ~PMRESRn_EN) - return val |= PMRESRn_EN; - - return 0; -} - -static void krait_clearpmu(u32 config_base) -{ - u32 val; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - bool venum_event = EVENT_VENUM(config_base); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val = clear_pmresrn_group(val, group); - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = krait_read_pmresrn(region); - val = clear_pmresrn_group(val, group); - krait_write_pmresrn(region, val); - } -} - -static void krait_pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* Disable counter and interrupt */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Clear pmresr code (if destined for PMNx counters) - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - krait_clearpmu(hwc->config_base); - - /* Disable interrupt for this counter */ - armv7_pmnc_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void krait_pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Set event (if destined for PMNx counters) - * We set the event for the cycle counter because we - * have the ability to perform event filtering. - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - krait_evt_setup(idx, hwc->config_base); - else - armv7_pmnc_write_evtsel(idx, hwc->config_base); - - /* Enable interrupt for this counter */ - armv7_pmnc_enable_intens(idx); - - /* Enable counter */ - armv7_pmnc_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void krait_pmu_reset(void *info) -{ - u32 vval, fval; - struct arm_pmu *cpu_pmu = info; - u32 idx, nb_cnt = cpu_pmu->num_events; - - armv7pmu_reset(info); - - /* Clear all pmresrs */ - krait_write_pmresrn(0, 0); - krait_write_pmresrn(1, 0); - krait_write_pmresrn(2, 0); - - venum_pre_pmresr(&vval, &fval); - venum_write_pmresr(0); - venum_post_pmresr(vval, fval); - - /* Reset PMxEVNCTCR to sane default */ - for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - } - -} - -static int krait_event_to_bit(struct perf_event *event, unsigned int region, - unsigned int group) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - - if (hwc->config_base & VENUM_EVENT) - bit = KRAIT_VPMRESR0_GROUP0; - else - bit = krait_get_pmresrn_event(region); - bit -= krait_get_pmresrn_event(0); - bit += group; - /* - * Lower bits are reserved for use by the counters (see - * armv7pmu_get_event_idx() for more info) - */ - bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; - - return bit; -} - -/* - * We check for column exclusion constraints here. - * Two events cant use the same group within a pmresr register. - */ -static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - int bit = -1; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int code = EVENT_CODE(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool krait_event = EVENT_CPU(hwc->config_base); - - if (venum_event || krait_event) { - /* Ignore invalid events */ - if (group > 3 || region > 2) - return -EINVAL; - if (venum_event && (code & 0xe0)) - return -EINVAL; - - bit = krait_event_to_bit(event, region, group); - if (test_and_set_bit(bit, cpuc->used_mask)) - return -EAGAIN; - } - - idx = armv7pmu_get_event_idx(cpuc, event); - if (idx < 0 && bit >= 0) - clear_bit(bit, cpuc->used_mask); - - return idx; -} - -static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool krait_event = EVENT_CPU(hwc->config_base); - - armv7pmu_clear_event_idx(cpuc, event); - if (venum_event || krait_event) { - bit = krait_event_to_bit(event, region, group); - clear_bit(bit, cpuc->used_mask); - } -} - -static int krait_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_krait"; - /* Some early versions of Krait don't support PC write events */ - if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node, - "qcom,no-pc-write")) - cpu_pmu->map_event = krait_map_event_no_branch; - else - cpu_pmu->map_event = krait_map_event; - cpu_pmu->set_event_filter = armv7pmu_set_event_filter; - cpu_pmu->reset = krait_pmu_reset; - cpu_pmu->enable = krait_pmu_enable_event; - cpu_pmu->disable = krait_pmu_disable_event; - cpu_pmu->get_event_idx = krait_pmu_get_event_idx; - cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; - return armv7_probe_num_events(cpu_pmu); -} - -/* - * Scorpion Local Performance Monitor Register (LPMn) - * - * 31 30 24 16 8 0 - * +--------------------------------+ - * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0 - * +--------------------------------+ - * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1 - * +--------------------------------+ - * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2 - * +--------------------------------+ - * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3 - * +--------------------------------+ - * VLPM | EN | CC | CC | CC | CC | N = 2, R = ? - * +--------------------------------+ - * EN | G=3 | G=2 | G=1 | G=0 - * - * - * Event Encoding: - * - * hwc->config_base = 0xNRCCG - * - * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM) - * R = region register - * CC = class of events the group G is choosing from - * G = group or particular event - * - * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2 - * - * A region (R) corresponds to a piece of the CPU (execution unit, instruction - * unit, etc.) while the event code (CC) corresponds to a particular class of - * events (interrupts for example). An event code is broken down into - * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for - * example). - */ - -static u32 scorpion_read_pmresrn(int n) -{ - u32 val; - - switch (n) { - case 0: - asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val)); - break; - case 1: - asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val)); - break; - case 2: - asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val)); - break; - case 3: - asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val)); - break; - default: - BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ - } - - return val; -} - -static void scorpion_write_pmresrn(int n, u32 val) -{ - switch (n) { - case 0: - asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val)); - break; - case 1: - asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val)); - break; - case 2: - asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val)); - break; - case 3: - asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val)); - break; - default: - BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ - } -} - -static u32 scorpion_get_pmresrn_event(unsigned int region) -{ - static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0, - SCORPION_LPM1_GROUP0, - SCORPION_LPM2_GROUP0, - SCORPION_L2LPM_GROUP0 }; - return pmresrn_table[region]; -} - -static void scorpion_evt_setup(int idx, u32 config_base) -{ - u32 val; - u32 mask; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - unsigned int code = EVENT_CODE(config_base); - unsigned int group_shift; - bool venum_event = EVENT_VENUM(config_base); - - group_shift = group * 8; - mask = 0xff << group_shift; - - /* Configure evtsel for the region and group */ - if (venum_event) - val = SCORPION_VLPM_GROUP0; - else - val = scorpion_get_pmresrn_event(region); - val += group; - /* Mix in mode-exclusion bits */ - val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); - armv7_pmnc_write_evtsel(idx, val); - - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = scorpion_read_pmresrn(region); - val &= ~mask; - val |= code << group_shift; - val |= PMRESRn_EN; - scorpion_write_pmresrn(region, val); - } -} - -static void scorpion_clearpmu(u32 config_base) -{ - u32 val; - u32 vval, fval; - unsigned int region = EVENT_REGION(config_base); - unsigned int group = EVENT_GROUP(config_base); - bool venum_event = EVENT_VENUM(config_base); - - if (venum_event) { - venum_pre_pmresr(&vval, &fval); - val = venum_read_pmresr(); - val = clear_pmresrn_group(val, group); - venum_write_pmresr(val); - venum_post_pmresr(vval, fval); - } else { - val = scorpion_read_pmresrn(region); - val = clear_pmresrn_group(val, group); - scorpion_write_pmresrn(region, val); - } -} - -static void scorpion_pmu_disable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* Disable counter and interrupt */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Clear pmresr code (if destined for PMNx counters) - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - scorpion_clearpmu(hwc->config_base); - - /* Disable interrupt for this counter */ - armv7_pmnc_disable_intens(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void scorpion_pmu_enable_event(struct perf_event *event) -{ - unsigned long flags; - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - /* - * Enable counter and interrupt, and set the counter to count - * the event that we're interested in. - */ - raw_spin_lock_irqsave(&events->pmu_lock, flags); - - /* Disable counter */ - armv7_pmnc_disable_counter(idx); - - /* - * Set event (if destined for PMNx counters) - * We don't set the event for the cycle counter because we - * don't have the ability to perform event filtering. - */ - if (hwc->config_base & KRAIT_EVENT_MASK) - scorpion_evt_setup(idx, hwc->config_base); - else if (idx != ARMV7_IDX_CYCLE_COUNTER) - armv7_pmnc_write_evtsel(idx, hwc->config_base); - - /* Enable interrupt for this counter */ - armv7_pmnc_enable_intens(idx); - - /* Enable counter */ - armv7_pmnc_enable_counter(idx); - - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void scorpion_pmu_reset(void *info) -{ - u32 vval, fval; - struct arm_pmu *cpu_pmu = info; - u32 idx, nb_cnt = cpu_pmu->num_events; - - armv7pmu_reset(info); - - /* Clear all pmresrs */ - scorpion_write_pmresrn(0, 0); - scorpion_write_pmresrn(1, 0); - scorpion_write_pmresrn(2, 0); - scorpion_write_pmresrn(3, 0); - - venum_pre_pmresr(&vval, &fval); - venum_write_pmresr(0); - venum_post_pmresr(vval, fval); - - /* Reset PMxEVNCTCR to sane default */ - for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { - armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - } -} - -static int scorpion_event_to_bit(struct perf_event *event, unsigned int region, - unsigned int group) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - - if (hwc->config_base & VENUM_EVENT) - bit = SCORPION_VLPM_GROUP0; - else - bit = scorpion_get_pmresrn_event(region); - bit -= scorpion_get_pmresrn_event(0); - bit += group; - /* - * Lower bits are reserved for use by the counters (see - * armv7pmu_get_event_idx() for more info) - */ - bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; - - return bit; -} - -/* - * We check for column exclusion constraints here. - * Two events cant use the same group within a pmresr register. - */ -static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx; - int bit = -1; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool scorpion_event = EVENT_CPU(hwc->config_base); - - if (venum_event || scorpion_event) { - /* Ignore invalid events */ - if (group > 3 || region > 3) - return -EINVAL; - - bit = scorpion_event_to_bit(event, region, group); - if (test_and_set_bit(bit, cpuc->used_mask)) - return -EAGAIN; - } - - idx = armv7pmu_get_event_idx(cpuc, event); - if (idx < 0 && bit >= 0) - clear_bit(bit, cpuc->used_mask); - - return idx; -} - -static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int bit; - struct hw_perf_event *hwc = &event->hw; - unsigned int region = EVENT_REGION(hwc->config_base); - unsigned int group = EVENT_GROUP(hwc->config_base); - bool venum_event = EVENT_VENUM(hwc->config_base); - bool scorpion_event = EVENT_CPU(hwc->config_base); - - armv7pmu_clear_event_idx(cpuc, event); - if (venum_event || scorpion_event) { - bit = scorpion_event_to_bit(event, region, group); - clear_bit(bit, cpuc->used_mask); - } -} - -static int scorpion_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_scorpion"; - cpu_pmu->map_event = scorpion_map_event; - cpu_pmu->reset = scorpion_pmu_reset; - cpu_pmu->enable = scorpion_pmu_enable_event; - cpu_pmu->disable = scorpion_pmu_disable_event; - cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; - cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; - return armv7_probe_num_events(cpu_pmu); -} - -static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) -{ - armv7pmu_init(cpu_pmu); - cpu_pmu->name = "armv7_scorpion_mp"; - cpu_pmu->map_event = scorpion_map_event; - cpu_pmu->reset = scorpion_pmu_reset; - cpu_pmu->enable = scorpion_pmu_enable_event; - cpu_pmu->disable = scorpion_pmu_disable_event; - cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; - cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; - return armv7_probe_num_events(cpu_pmu); -} - -static const struct of_device_id armv7_pmu_of_device_ids[] = { - {.compatible = "arm,cortex-a17-pmu", .data = armv7_a17_pmu_init}, - {.compatible = "arm,cortex-a15-pmu", .data = armv7_a15_pmu_init}, - {.compatible = "arm,cortex-a12-pmu", .data = armv7_a12_pmu_init}, - {.compatible = "arm,cortex-a9-pmu", .data = armv7_a9_pmu_init}, - {.compatible = "arm,cortex-a8-pmu", .data = armv7_a8_pmu_init}, - {.compatible = "arm,cortex-a7-pmu", .data = armv7_a7_pmu_init}, - {.compatible = "arm,cortex-a5-pmu", .data = armv7_a5_pmu_init}, - {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, - {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init}, - {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init}, - {}, -}; - -static const struct pmu_probe_info armv7_pmu_probe_table[] = { - ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init), - ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init), - { /* sentinel value */ } -}; - - -static int armv7_pmu_device_probe(struct platform_device *pdev) -{ - return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids, - armv7_pmu_probe_table); -} - -static struct platform_driver armv7_pmu_driver = { - .driver = { - .name = "armv7-pmu", - .of_match_table = armv7_pmu_of_device_ids, - .suppress_bind_attrs = true, - }, - .probe = armv7_pmu_device_probe, -}; - -builtin_platform_driver(armv7_pmu_driver); -#endif /* CONFIG_CPU_V7 */ diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c deleted file mode 100644 index f6cdcacfb96d..000000000000 --- a/arch/arm/kernel/perf_event_xscale.c +++ /dev/null @@ -1,776 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * ARMv5 [xscale] Performance counter handling code. - * - * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com> - * - * Based on the previous xscale OProfile code. - * - * There are two variants of the xscale PMU that we support: - * - xscale1pmu: 2 event counters and a cycle counter - * - xscale2pmu: 4 event counters and a cycle counter - * The two variants share event definitions, but have different - * PMU structures. - */ - -#ifdef CONFIG_CPU_XSCALE - -#include <asm/cputype.h> -#include <asm/irq_regs.h> - -#include <linux/of.h> -#include <linux/perf/arm_pmu.h> -#include <linux/platform_device.h> - -enum xscale_perf_types { - XSCALE_PERFCTR_ICACHE_MISS = 0x00, - XSCALE_PERFCTR_ICACHE_NO_DELIVER = 0x01, - XSCALE_PERFCTR_DATA_STALL = 0x02, - XSCALE_PERFCTR_ITLB_MISS = 0x03, - XSCALE_PERFCTR_DTLB_MISS = 0x04, - XSCALE_PERFCTR_BRANCH = 0x05, - XSCALE_PERFCTR_BRANCH_MISS = 0x06, - XSCALE_PERFCTR_INSTRUCTION = 0x07, - XSCALE_PERFCTR_DCACHE_FULL_STALL = 0x08, - XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09, - XSCALE_PERFCTR_DCACHE_ACCESS = 0x0A, - XSCALE_PERFCTR_DCACHE_MISS = 0x0B, - XSCALE_PERFCTR_DCACHE_WRITE_BACK = 0x0C, - XSCALE_PERFCTR_PC_CHANGED = 0x0D, - XSCALE_PERFCTR_BCU_REQUEST = 0x10, - XSCALE_PERFCTR_BCU_FULL = 0x11, - XSCALE_PERFCTR_BCU_DRAIN = 0x12, - XSCALE_PERFCTR_BCU_ECC_NO_ELOG = 0x14, - XSCALE_PERFCTR_BCU_1_BIT_ERR = 0x15, - XSCALE_PERFCTR_RMW = 0x16, - /* XSCALE_PERFCTR_CCNT is not hardware defined */ - XSCALE_PERFCTR_CCNT = 0xFE, - XSCALE_PERFCTR_UNUSED = 0xFF, -}; - -enum xscale_counters { - XSCALE_CYCLE_COUNTER = 0, - XSCALE_COUNTER0, - XSCALE_COUNTER1, - XSCALE_COUNTER2, - XSCALE_COUNTER3, -}; - -static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = { - PERF_MAP_ALL_UNSUPPORTED, - [PERF_COUNT_HW_CPU_CYCLES] = XSCALE_PERFCTR_CCNT, - [PERF_COUNT_HW_INSTRUCTIONS] = XSCALE_PERFCTR_INSTRUCTION, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH, - [PERF_COUNT_HW_BRANCH_MISSES] = XSCALE_PERFCTR_BRANCH_MISS, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = XSCALE_PERFCTR_ICACHE_NO_DELIVER, -}; - -static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - PERF_CACHE_MAP_ALL_UNSUPPORTED, - - [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS, - [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS, - [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS, - - [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_ICACHE_MISS, - - [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS, - [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS, - - [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS, - [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS, -}; - -#define XSCALE_PMU_ENABLE 0x001 -#define XSCALE_PMN_RESET 0x002 -#define XSCALE_CCNT_RESET 0x004 -#define XSCALE_PMU_RESET (CCNT_RESET | PMN_RESET) -#define XSCALE_PMU_CNT64 0x008 - -#define XSCALE1_OVERFLOWED_MASK 0x700 -#define XSCALE1_CCOUNT_OVERFLOW 0x400 -#define XSCALE1_COUNT0_OVERFLOW 0x100 -#define XSCALE1_COUNT1_OVERFLOW 0x200 -#define XSCALE1_CCOUNT_INT_EN 0x040 -#define XSCALE1_COUNT0_INT_EN 0x010 -#define XSCALE1_COUNT1_INT_EN 0x020 -#define XSCALE1_COUNT0_EVT_SHFT 12 -#define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT) -#define XSCALE1_COUNT1_EVT_SHFT 20 -#define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT) - -static inline u32 -xscale1pmu_read_pmnc(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val)); - return val; -} - -static inline void -xscale1pmu_write_pmnc(u32 val) -{ - /* upper 4bits and 7, 11 are write-as-0 */ - val &= 0xffff77f; - asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val)); -} - -static inline int -xscale1_pmnc_counter_has_overflowed(unsigned long pmnc, - enum xscale_counters counter) -{ - int ret = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - ret = pmnc & XSCALE1_CCOUNT_OVERFLOW; - break; - case XSCALE_COUNTER0: - ret = pmnc & XSCALE1_COUNT0_OVERFLOW; - break; - case XSCALE_COUNTER1: - ret = pmnc & XSCALE1_COUNT1_OVERFLOW; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - } - - return ret; -} - -static irqreturn_t -xscale1pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - unsigned long pmnc; - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - /* - * NOTE: there's an A stepping erratum that states if an overflow - * bit already exists and another occurs, the previous - * Overflow bit gets cleared. There's no workaround. - * Fixed in B stepping or later. - */ - pmnc = xscale1pmu_read_pmnc(); - - /* - * Write the value back to clear the overflow flags. Overflow - * flags remain in pmnc for use below. We also disable the PMU - * while we process the interrupt. - */ - xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE); - - if (!(pmnc & XSCALE1_OVERFLOWED_MASK)) - return IRQ_NONE; - - regs = get_irq_regs(); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - if (!event) - continue; - - if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - irq_work_run(); - - /* - * Re-enable the PMU. - */ - pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE; - xscale1pmu_write_pmnc(pmnc); - - return IRQ_HANDLED; -} - -static void xscale1pmu_enable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - mask = 0; - evt = XSCALE1_CCOUNT_INT_EN; - break; - case XSCALE_COUNTER0: - mask = XSCALE1_COUNT0_EVT_MASK; - evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) | - XSCALE1_COUNT0_INT_EN; - break; - case XSCALE_COUNTER1: - mask = XSCALE1_COUNT1_EVT_MASK; - evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) | - XSCALE1_COUNT1_INT_EN; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val &= ~mask; - val |= evt; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale1pmu_disable_event(struct perf_event *event) -{ - unsigned long val, mask, evt, flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - mask = XSCALE1_CCOUNT_INT_EN; - evt = 0; - break; - case XSCALE_COUNTER0: - mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK; - evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT; - break; - case XSCALE_COUNTER1: - mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK; - evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val &= ~mask; - val |= evt; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int -xscale1pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - if (XSCALE_PERFCTR_CCNT == hwc->config_base) { - if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask)) - return -EAGAIN; - - return XSCALE_CYCLE_COUNTER; - } else { - if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask)) - return XSCALE_COUNTER1; - - if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask)) - return XSCALE_COUNTER0; - - return -EAGAIN; - } -} - -static void xscalepmu_clear_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - clear_bit(event->hw.idx, cpuc->used_mask); -} - -static void xscale1pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val |= XSCALE_PMU_ENABLE; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale1pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale1pmu_read_pmnc(); - val &= ~XSCALE_PMU_ENABLE; - xscale1pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static inline u64 xscale1pmu_read_counter(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - u32 val = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val)); - break; - } - - return val; -} - -static inline void xscale1pmu_write_counter(struct perf_event *event, u64 val) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val)); - break; - } -} - -static int xscale_map_event(struct perf_event *event) -{ - return armpmu_map_event(event, &xscale_perf_map, - &xscale_perf_cache_map, 0xFF); -} - -static int xscale1pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->name = "armv5_xscale1"; - cpu_pmu->handle_irq = xscale1pmu_handle_irq; - cpu_pmu->enable = xscale1pmu_enable_event; - cpu_pmu->disable = xscale1pmu_disable_event; - cpu_pmu->read_counter = xscale1pmu_read_counter; - cpu_pmu->write_counter = xscale1pmu_write_counter; - cpu_pmu->get_event_idx = xscale1pmu_get_event_idx; - cpu_pmu->clear_event_idx = xscalepmu_clear_event_idx; - cpu_pmu->start = xscale1pmu_start; - cpu_pmu->stop = xscale1pmu_stop; - cpu_pmu->map_event = xscale_map_event; - cpu_pmu->num_events = 3; - - return 0; -} - -#define XSCALE2_OVERFLOWED_MASK 0x01f -#define XSCALE2_CCOUNT_OVERFLOW 0x001 -#define XSCALE2_COUNT0_OVERFLOW 0x002 -#define XSCALE2_COUNT1_OVERFLOW 0x004 -#define XSCALE2_COUNT2_OVERFLOW 0x008 -#define XSCALE2_COUNT3_OVERFLOW 0x010 -#define XSCALE2_CCOUNT_INT_EN 0x001 -#define XSCALE2_COUNT0_INT_EN 0x002 -#define XSCALE2_COUNT1_INT_EN 0x004 -#define XSCALE2_COUNT2_INT_EN 0x008 -#define XSCALE2_COUNT3_INT_EN 0x010 -#define XSCALE2_COUNT0_EVT_SHFT 0 -#define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT) -#define XSCALE2_COUNT1_EVT_SHFT 8 -#define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT) -#define XSCALE2_COUNT2_EVT_SHFT 16 -#define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT) -#define XSCALE2_COUNT3_EVT_SHFT 24 -#define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT) - -static inline u32 -xscale2pmu_read_pmnc(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val)); - /* bits 1-2 and 4-23 are read-unpredictable */ - return val & 0xff000009; -} - -static inline void -xscale2pmu_write_pmnc(u32 val) -{ - /* bits 4-23 are write-as-0, 24-31 are write ignored */ - val &= 0xf; - asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val)); -} - -static inline u32 -xscale2pmu_read_overflow_flags(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val)); - return val; -} - -static inline void -xscale2pmu_write_overflow_flags(u32 val) -{ - asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val)); -} - -static inline u32 -xscale2pmu_read_event_select(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val)); - return val; -} - -static inline void -xscale2pmu_write_event_select(u32 val) -{ - asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val)); -} - -static inline u32 -xscale2pmu_read_int_enable(void) -{ - u32 val; - asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val)); - return val; -} - -static void -xscale2pmu_write_int_enable(u32 val) -{ - asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val)); -} - -static inline int -xscale2_pmnc_counter_has_overflowed(unsigned long of_flags, - enum xscale_counters counter) -{ - int ret = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - ret = of_flags & XSCALE2_CCOUNT_OVERFLOW; - break; - case XSCALE_COUNTER0: - ret = of_flags & XSCALE2_COUNT0_OVERFLOW; - break; - case XSCALE_COUNTER1: - ret = of_flags & XSCALE2_COUNT1_OVERFLOW; - break; - case XSCALE_COUNTER2: - ret = of_flags & XSCALE2_COUNT2_OVERFLOW; - break; - case XSCALE_COUNTER3: - ret = of_flags & XSCALE2_COUNT3_OVERFLOW; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", counter); - } - - return ret; -} - -static irqreturn_t -xscale2pmu_handle_irq(struct arm_pmu *cpu_pmu) -{ - unsigned long pmnc, of_flags; - struct perf_sample_data data; - struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); - struct pt_regs *regs; - int idx; - - /* Disable the PMU. */ - pmnc = xscale2pmu_read_pmnc(); - xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE); - - /* Check the overflow flag register. */ - of_flags = xscale2pmu_read_overflow_flags(); - if (!(of_flags & XSCALE2_OVERFLOWED_MASK)) - return IRQ_NONE; - - /* Clear the overflow bits. */ - xscale2pmu_write_overflow_flags(of_flags); - - regs = get_irq_regs(); - - for (idx = 0; idx < cpu_pmu->num_events; ++idx) { - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc; - - if (!event) - continue; - - if (!xscale2_pmnc_counter_has_overflowed(of_flags, idx)) - continue; - - hwc = &event->hw; - armpmu_event_update(event); - perf_sample_data_init(&data, 0, hwc->last_period); - if (!armpmu_event_set_period(event)) - continue; - - if (perf_event_overflow(event, &data, regs)) - cpu_pmu->disable(event); - } - - irq_work_run(); - - /* - * Re-enable the PMU. - */ - pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE; - xscale2pmu_write_pmnc(pmnc); - - return IRQ_HANDLED; -} - -static void xscale2pmu_enable_event(struct perf_event *event) -{ - unsigned long flags, ien, evtsel; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - ien = xscale2pmu_read_int_enable(); - evtsel = xscale2pmu_read_event_select(); - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - ien |= XSCALE2_CCOUNT_INT_EN; - break; - case XSCALE_COUNTER0: - ien |= XSCALE2_COUNT0_INT_EN; - evtsel &= ~XSCALE2_COUNT0_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT; - break; - case XSCALE_COUNTER1: - ien |= XSCALE2_COUNT1_INT_EN; - evtsel &= ~XSCALE2_COUNT1_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT; - break; - case XSCALE_COUNTER2: - ien |= XSCALE2_COUNT2_INT_EN; - evtsel &= ~XSCALE2_COUNT2_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT; - break; - case XSCALE_COUNTER3: - ien |= XSCALE2_COUNT3_INT_EN; - evtsel &= ~XSCALE2_COUNT3_EVT_MASK; - evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - xscale2pmu_write_event_select(evtsel); - xscale2pmu_write_int_enable(ien); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale2pmu_disable_event(struct perf_event *event) -{ - unsigned long flags, ien, evtsel, of_flags; - struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); - struct hw_perf_event *hwc = &event->hw; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - int idx = hwc->idx; - - ien = xscale2pmu_read_int_enable(); - evtsel = xscale2pmu_read_event_select(); - - switch (idx) { - case XSCALE_CYCLE_COUNTER: - ien &= ~XSCALE2_CCOUNT_INT_EN; - of_flags = XSCALE2_CCOUNT_OVERFLOW; - break; - case XSCALE_COUNTER0: - ien &= ~XSCALE2_COUNT0_INT_EN; - evtsel &= ~XSCALE2_COUNT0_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT; - of_flags = XSCALE2_COUNT0_OVERFLOW; - break; - case XSCALE_COUNTER1: - ien &= ~XSCALE2_COUNT1_INT_EN; - evtsel &= ~XSCALE2_COUNT1_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT; - of_flags = XSCALE2_COUNT1_OVERFLOW; - break; - case XSCALE_COUNTER2: - ien &= ~XSCALE2_COUNT2_INT_EN; - evtsel &= ~XSCALE2_COUNT2_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT; - of_flags = XSCALE2_COUNT2_OVERFLOW; - break; - case XSCALE_COUNTER3: - ien &= ~XSCALE2_COUNT3_INT_EN; - evtsel &= ~XSCALE2_COUNT3_EVT_MASK; - evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT; - of_flags = XSCALE2_COUNT3_OVERFLOW; - break; - default: - WARN_ONCE(1, "invalid counter number (%d)\n", idx); - return; - } - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - xscale2pmu_write_event_select(evtsel); - xscale2pmu_write_int_enable(ien); - xscale2pmu_write_overflow_flags(of_flags); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static int -xscale2pmu_get_event_idx(struct pmu_hw_events *cpuc, - struct perf_event *event) -{ - int idx = xscale1pmu_get_event_idx(cpuc, event); - if (idx >= 0) - goto out; - - if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask)) - idx = XSCALE_COUNTER3; - else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask)) - idx = XSCALE_COUNTER2; -out: - return idx; -} - -static void xscale2pmu_start(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64; - val |= XSCALE_PMU_ENABLE; - xscale2pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static void xscale2pmu_stop(struct arm_pmu *cpu_pmu) -{ - unsigned long flags, val; - struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); - - raw_spin_lock_irqsave(&events->pmu_lock, flags); - val = xscale2pmu_read_pmnc(); - val &= ~XSCALE_PMU_ENABLE; - xscale2pmu_write_pmnc(val); - raw_spin_unlock_irqrestore(&events->pmu_lock, flags); -} - -static inline u64 xscale2pmu_read_counter(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - u32 val = 0; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val)); - break; - case XSCALE_COUNTER2: - asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val)); - break; - case XSCALE_COUNTER3: - asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val)); - break; - } - - return val; -} - -static inline void xscale2pmu_write_counter(struct perf_event *event, u64 val) -{ - struct hw_perf_event *hwc = &event->hw; - int counter = hwc->idx; - - switch (counter) { - case XSCALE_CYCLE_COUNTER: - asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val)); - break; - case XSCALE_COUNTER0: - asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val)); - break; - case XSCALE_COUNTER1: - asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val)); - break; - case XSCALE_COUNTER2: - asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val)); - break; - case XSCALE_COUNTER3: - asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val)); - break; - } -} - -static int xscale2pmu_init(struct arm_pmu *cpu_pmu) -{ - cpu_pmu->name = "armv5_xscale2"; - cpu_pmu->handle_irq = xscale2pmu_handle_irq; - cpu_pmu->enable = xscale2pmu_enable_event; - cpu_pmu->disable = xscale2pmu_disable_event; - cpu_pmu->read_counter = xscale2pmu_read_counter; - cpu_pmu->write_counter = xscale2pmu_write_counter; - cpu_pmu->get_event_idx = xscale2pmu_get_event_idx; - cpu_pmu->clear_event_idx = xscalepmu_clear_event_idx; - cpu_pmu->start = xscale2pmu_start; - cpu_pmu->stop = xscale2pmu_stop; - cpu_pmu->map_event = xscale_map_event; - cpu_pmu->num_events = 5; - - return 0; -} - -static const struct pmu_probe_info xscale_pmu_probe_table[] = { - XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V1, xscale1pmu_init), - XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V2, xscale2pmu_init), - { /* sentinel value */ } -}; - -static int xscale_pmu_device_probe(struct platform_device *pdev) -{ - return arm_pmu_device_probe(pdev, NULL, xscale_pmu_probe_table); -} - -static struct platform_driver xscale_pmu_driver = { - .driver = { - .name = "xscale-pmu", - }, - .probe = xscale_pmu_device_probe, -}; - -builtin_platform_driver(xscale_pmu_driver); -#endif /* CONFIG_CPU_XSCALE */ diff --git a/arch/arm/kernel/pj4-cp0.c b/arch/arm/kernel/pj4-cp0.c deleted file mode 100644 index 1d1fb22f44f3..000000000000 --- a/arch/arm/kernel/pj4-cp0.c +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * linux/arch/arm/kernel/pj4-cp0.c - * - * PJ4 iWMMXt coprocessor context switching and handling - * - * Copyright (c) 2010 Marvell International Inc. - */ - -#include <linux/types.h> -#include <linux/kernel.h> -#include <linux/signal.h> -#include <linux/sched.h> -#include <linux/init.h> -#include <linux/io.h> -#include <asm/thread_notify.h> -#include <asm/cputype.h> - -static int iwmmxt_do(struct notifier_block *self, unsigned long cmd, void *t) -{ - struct thread_info *thread = t; - - switch (cmd) { - case THREAD_NOTIFY_FLUSH: - /* - * flush_thread() zeroes thread->fpstate, so no need - * to do anything here. - * - * FALLTHROUGH: Ensure we don't try to overwrite our newly - * initialised state information on the first fault. - */ - - case THREAD_NOTIFY_EXIT: - iwmmxt_task_release(thread); - break; - - case THREAD_NOTIFY_SWITCH: - iwmmxt_task_switch(thread); - break; - } - - return NOTIFY_DONE; -} - -static struct notifier_block __maybe_unused iwmmxt_notifier_block = { - .notifier_call = iwmmxt_do, -}; - - -static u32 __init pj4_cp_access_read(void) -{ - u32 value; - - __asm__ __volatile__ ( - "mrc p15, 0, %0, c1, c0, 2\n\t" - : "=r" (value)); - return value; -} - -static void __init pj4_cp_access_write(u32 value) -{ - u32 temp; - - __asm__ __volatile__ ( - "mcr p15, 0, %1, c1, c0, 2\n\t" -#ifdef CONFIG_THUMB2_KERNEL - "isb\n\t" -#else - "mrc p15, 0, %0, c1, c0, 2\n\t" - "mov %0, %0\n\t" - "sub pc, pc, #4\n\t" -#endif - : "=r" (temp) : "r" (value)); -} - -static int __init pj4_get_iwmmxt_version(void) -{ - u32 cp_access, wcid; - - cp_access = pj4_cp_access_read(); - pj4_cp_access_write(cp_access | 0xf); - - /* check if coprocessor 0 and 1 are available */ - if ((pj4_cp_access_read() & 0xf) != 0xf) { - pj4_cp_access_write(cp_access); - return -ENODEV; - } - - /* read iWMMXt coprocessor id register p1, c0 */ - __asm__ __volatile__ ("mrc p1, 0, %0, c0, c0, 0\n" : "=r" (wcid)); - - pj4_cp_access_write(cp_access); - - /* iWMMXt v1 */ - if ((wcid & 0xffffff00) == 0x56051000) - return 1; - /* iWMMXt v2 */ - if ((wcid & 0xffffff00) == 0x56052000) - return 2; - - return -EINVAL; -} - -/* - * Disable CP0/CP1 on boot, and let call_fpe() and the iWMMXt lazy - * switch code handle iWMMXt context switching. - */ -static int __init pj4_cp0_init(void) -{ - u32 __maybe_unused cp_access; - int vers; - - if (!cpu_is_pj4()) - return 0; - - vers = pj4_get_iwmmxt_version(); - if (vers < 0) - return 0; - -#ifndef CONFIG_IWMMXT - pr_info("PJ4 iWMMXt coprocessor detected, but kernel support is missing.\n"); -#else - cp_access = pj4_cp_access_read() & ~0xf; - pj4_cp_access_write(cp_access); - - pr_info("PJ4 iWMMXt v%d coprocessor enabled.\n", vers); - elf_hwcap |= HWCAP_IWMMXT; - thread_register_notifier(&iwmmxt_notifier_block); -#endif - - return 0; -} - -late_initcall(pj4_cp0_init); diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 3d9cace63884..e16ed102960c 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -78,7 +78,6 @@ void arch_cpu_idle(void) arm_pm_idle(); else cpu_do_idle(); - raw_local_irq_enable(); } void arch_cpu_idle_prepare(void) @@ -201,7 +200,7 @@ void __show_regs(struct pt_regs *regs) void show_regs(struct pt_regs * regs) { __show_regs(regs); - dump_stack(); + dump_backtrace(regs, NULL, KERN_DEFAULT); } ATOMIC_NOTIFIER_HEAD(thread_notify_head); @@ -223,7 +222,6 @@ void flush_thread(void) flush_ptrace_hw_breakpoint(tsk); - memset(thread->used_cp, 0, sizeof(thread->used_cp)); memset(&tsk->thread.debug, 0, sizeof(struct debug_info)); memset(&thread->fpstate, 0, sizeof(union fp_state)); @@ -232,10 +230,6 @@ void flush_thread(void) thread_notify(THREAD_NOTIFY_FLUSH, thread); } -void release_thread(struct task_struct *dead_task) -{ -} - asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) @@ -320,7 +314,7 @@ static int __init gate_vma_init(void) gate_vma.vm_page_prot = PAGE_READONLY_EXEC; gate_vma.vm_start = 0xffff0000; gate_vma.vm_end = 0xffff0000 + PAGE_SIZE; - gate_vma.vm_flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC; + vm_flags_init(&gate_vma, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYEXEC); return 0; } arch_initcall(gate_vma_init); @@ -375,7 +369,7 @@ static unsigned long sigpage_addr(const struct mm_struct *mm, slots = ((last - first) >> PAGE_SHIFT) + 1; - offset = get_random_int() % slots; + offset = get_random_u32_below(slots); addr = first + (offset << PAGE_SHIFT); diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c index d4392e177484..3bb0c4dcfc5c 100644 --- a/arch/arm/kernel/psci_smp.c +++ b/arch/arm/kernel/psci_smp.c @@ -45,8 +45,15 @@ extern void secondary_startup(void); static int psci_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (psci_ops.cpu_on) +#ifdef CONFIG_XIP_KERNEL + return psci_ops.cpu_on(cpu_logical_map(cpu), + ((phys_addr_t)(&secondary_startup) + - XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) + + CONFIG_XIP_PHYS_ADDR)); +#else return psci_ops.cpu_on(cpu_logical_map(cpu), virt_to_idmap(&secondary_startup)); +#endif return -ENODEV; } diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index bfe88c6e60d5..7951b2c06fec 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -584,8 +584,6 @@ static int fpa_set(struct task_struct *target, { struct thread_info *thread = task_thread_info(target); - thread->used_cp[1] = thread->used_cp[2] = 1; - return user_regset_copyin(&pos, &count, &kbuf, &ubuf, &thread->fpstate, 0, sizeof(struct user_fp)); @@ -651,11 +649,9 @@ static int vfp_set(struct task_struct *target, if (ret) return ret; - ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, - user_fpregs_offset + sizeof(new_vfp.fpregs), - user_fpscr_offset); - if (ret) - return ret; + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + user_fpregs_offset + sizeof(new_vfp.fpregs), + user_fpscr_offset); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &new_vfp.fpscr, @@ -681,7 +677,7 @@ enum arm_regset { static const struct user_regset arm_regsets[] = { [REGSET_GPR] = { - .core_note_type = NT_PRSTATUS, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = ELF_NGREG, .size = sizeof(u32), .align = sizeof(u32), @@ -693,7 +689,7 @@ static const struct user_regset arm_regsets[] = { * For the FPA regs in fpstate, the real fields are a mixture * of sizes, so pretend that the registers are word-sized: */ - .core_note_type = NT_PRFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = sizeof(struct user_fp) / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), @@ -706,7 +702,7 @@ static const struct user_regset arm_regsets[] = { * Pretend that the VFP regs are word-sized, since the FPSCR is * a single word dangling at the end of struct user_vfp: */ - .core_note_type = NT_ARM_VFP, + USER_REGSET_NOTE_TYPE(ARM_VFP), .n = ARM_VFPREGS_SIZE / sizeof(u32), .size = sizeof(u32), .align = sizeof(u32), @@ -785,8 +781,9 @@ long arch_ptrace(struct task_struct *child, long request, break; case PTRACE_SET_SYSCALL: - task_thread_info(child)->abi_syscall = data & - __NR_SYSCALL_MASK; + if (data != -1) + data &= __NR_SYSCALL_MASK; + task_thread_info(child)->abi_syscall = data; ret = 0; break; diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c index 2cb943422554..3f0d5c3dae11 100644 --- a/arch/arm/kernel/reboot.c +++ b/arch/arm/kernel/reboot.c @@ -10,6 +10,7 @@ #include <asm/cacheflush.h> #include <asm/idmap.h> #include <asm/virt.h> +#include <asm/system_misc.h> #include "reboot.h" diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c index 8aac1e10b117..ac15db66df4c 100644 --- a/arch/arm/kernel/return_address.c +++ b/arch/arm/kernel/return_address.c @@ -16,17 +16,17 @@ struct return_address_data { void *addr; }; -static int save_return_addr(struct stackframe *frame, void *d) +static bool save_return_addr(void *d, unsigned long pc) { struct return_address_data *data = d; if (!data->level) { - data->addr = (void *)frame->pc; + data->addr = (void *)pc; - return 1; + return false; } else { --data->level; - return 0; + return true; } } @@ -47,6 +47,7 @@ here: frame.kr_cur = NULL; frame.tsk = current; #endif + frame.ex_frame = false; walk_stackframe(&frame, save_return_addr, &data); diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 1e8a50a97edf..0bfd66c7ada0 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -15,10 +15,10 @@ #include <linux/console.h> #include <linux/seq_file.h> #include <linux/screen_info.h> -#include <linux/of_platform.h> #include <linux/init.h> #include <linux/kexec.h> #include <linux/libfdt.h> +#include <linux/of.h> #include <linux/of_fdt.h> #include <linux/cpu.h> #include <linux/interrupt.h> @@ -76,13 +76,6 @@ static int __init fpe_setup(char *line) __setup("fpe=", fpe_setup); #endif -extern void init_default_cache_policy(unsigned long); -extern void paging_init(const struct machine_desc *desc); -extern void early_mm_init(const struct machine_desc *); -extern void adjust_lowmem_bounds(void); -extern enum reboot_mode reboot_mode; -extern void setup_dma_zone(const struct machine_desc *desc); - unsigned int processor_id; EXPORT_SYMBOL(processor_id); unsigned int __machine_arch_type __read_mostly; @@ -450,6 +443,8 @@ static void __init cpuid_init_hwcaps(void) { int block; u32 isar5; + u32 isar6; + u32 pfr2; if (cpu_architecture() < CPU_ARCH_ARMv7) return; @@ -485,6 +480,18 @@ static void __init cpuid_init_hwcaps(void) block = cpuid_feature_extract_field(isar5, 16); if (block >= 1) elf_hwcap2 |= HWCAP2_CRC32; + + /* Check for Speculation barrier instruction */ + isar6 = read_cpuid_ext(CPUID_EXT_ISAR6); + block = cpuid_feature_extract_field(isar6, 12); + if (block >= 1) + elf_hwcap2 |= HWCAP2_SB; + + /* Check for Speculative Store Bypassing control */ + pfr2 = read_cpuid_ext(CPUID_EXT_PFR2); + block = cpuid_feature_extract_field(pfr2, 4); + if (block >= 1) + elf_hwcap2 |= HWCAP2_SSBS; } static void __init elf_hwcap_fixup(void) @@ -873,10 +880,7 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) */ boot_alias_start = phys_to_idmap(start); if (arm_has_idmap_alias() && boot_alias_start != IDMAP_INVALID_ADDR) { - res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); - if (!res) - panic("%s: Failed to allocate %zu bytes\n", - __func__, sizeof(*res)); + res = memblock_alloc_or_panic(sizeof(*res), SMP_CACHE_BYTES); res->name = "System RAM (boot alias)"; res->start = boot_alias_start; res->end = phys_to_idmap(res_end); @@ -884,10 +888,7 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) request_resource(&iomem_resource, res); } - res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES); - if (!res) - panic("%s: Failed to allocate %zu bytes\n", __func__, - sizeof(*res)); + res = memblock_alloc_or_panic(sizeof(*res), SMP_CACHE_BYTES); res->name = "System RAM"; res->start = start; res->end = res_end; @@ -921,9 +922,8 @@ static void __init request_standard_resources(const struct machine_desc *mdesc) request_resource(&ioport_resource, &lp2); } -#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) || \ - defined(CONFIG_EFI) -struct screen_info screen_info = { +#if defined(CONFIG_VGA_CONSOLE) +struct screen_info vgacon_screen_info = { .orig_video_lines = 30, .orig_video_cols = 80, .orig_video_mode = 0, @@ -973,7 +973,7 @@ static int __init init_machine_late(void) } late_initcall(init_machine_late); -#ifdef CONFIG_KEXEC +#ifdef CONFIG_CRASH_RESERVE /* * The crash region must be aligned to 128MB to avoid * zImage relocating below the reserved region. @@ -1003,7 +1003,8 @@ static void __init reserve_crashkernel(void) total_mem = get_total_mem(); ret = parse_crashkernel(boot_command_line, total_mem, - &crash_size, &crash_base); + &crash_size, &crash_base, + NULL, NULL, NULL); /* invalid value specified or crashkernel=0 */ if (ret || !crash_size) return; @@ -1059,7 +1060,7 @@ static void __init reserve_crashkernel(void) } #else static inline void reserve_crashkernel(void) {} -#endif /* CONFIG_KEXEC */ +#endif /* CONFIG_CRASH_RESERVE*/ void __init hyp_mode_check(void) { @@ -1128,7 +1129,7 @@ void __init setup_arch(char **cmdline_p) setup_initial_init_mm(_text, _etext, _edata, _end); /* populate cmd_line too for later use, preserving boot_command_line */ - strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); + strscpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = cmd_line; early_fixmap_init(); @@ -1141,7 +1142,7 @@ void __init setup_arch(char **cmdline_p) #endif setup_dma_zone(mdesc); xen_early_init(); - efi_init(); + arm_efi_init(); /* * Make sure the calculation for lowmem/highmem is set appropriately * before reserving/allocating any memory @@ -1184,13 +1185,9 @@ void __init setup_arch(char **cmdline_p) reserve_crashkernel(); -#ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER - handle_arch_irq = mdesc->handle_irq; -#endif - #ifdef CONFIG_VT #if defined(CONFIG_VGA_CONSOLE) - conswitchp = &vga_con; + vgacon_register_screen(&vgacon_screen_info); #endif #endif @@ -1198,20 +1195,10 @@ void __init setup_arch(char **cmdline_p) mdesc->init_early(); } - -static int __init topology_init(void) +bool arch_cpu_is_hotpluggable(int num) { - int cpu; - - for_each_possible_cpu(cpu) { - struct cpuinfo_arm *cpuinfo = &per_cpu(cpu_data, cpu); - cpuinfo->cpu.hotpluggable = platform_can_hotplug_cpu(cpu); - register_cpu(&cpuinfo->cpu, cpu); - } - - return 0; + return platform_can_hotplug_cpu(num); } -subsys_initcall(topology_init); #ifdef CONFIG_HAVE_PROC_CPU static int __init proc_cpu_init(void) @@ -1249,6 +1236,12 @@ static const char *hwcap_str[] = { "vfpd32", "lpae", "evtstrm", + "fphp", + "asimdhp", + "asimddp", + "asimdfhm", + "asimdbf16", + "i8mm", NULL }; @@ -1258,6 +1251,8 @@ static const char *hwcap2_str[] = { "sha1", "sha2", "crc32", + "sb", + "ssbs", NULL }; diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index ea128e32e8ca..79a6730fa0eb 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -18,6 +18,7 @@ #include <asm/traps.h> #include <asm/unistd.h> #include <asm/vfp.h> +#include <asm/syscalls.h> #include "signal.h" @@ -655,7 +656,7 @@ struct page *get_signal_page(void) PAGE_SIZE / sizeof(u32)); /* Give the signal return code some randomness */ - offset = 0x200 + (get_random_int() & 0x7fc); + offset = 0x200 + (get_random_u16() & 0x7fc); signal_return_offset = offset; /* Copy signal return handlers into the page */ @@ -681,7 +682,7 @@ asmlinkage void do_rseq_syscall(struct pt_regs *regs) */ static_assert(NSIGILL == 11); static_assert(NSIGFPE == 15); -static_assert(NSIGSEGV == 9); +static_assert(NSIGSEGV == 10); static_assert(NSIGBUS == 5); static_assert(NSIGTRAP == 6); static_assert(NSIGCHLD == 6); diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index a86a1d4f3461..93afd1005b43 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -127,6 +127,10 @@ cpu_resume_after_mmu: instr_sync #endif bl cpu_init @ restore the und/abt/irq banked regs +#if defined(CONFIG_KASAN) && defined(CONFIG_KASAN_STACK) + mov r0, sp + bl kasan_unpoison_task_stack_below +#endif mov r0, #0 @ return zero on success ldmfd sp!, {r4 - r11, pc} ENDPROC(cpu_resume_after_mmu) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 73fc645fc4c7..50999886a8b5 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -48,7 +48,6 @@ #include <asm/mach/arch.h> #include <asm/mpu.h> -#define CREATE_TRACE_POINTS #include <trace/events/ipi.h> /* @@ -289,15 +288,11 @@ int __cpu_disable(void) } /* - * called on the thread which is asking for a CPU to be shutdown - - * waits until shutdown has completed, or it is timed out. + * called on the thread which is asking for a CPU to be shutdown after the + * shutdown completed. */ -void __cpu_die(unsigned int cpu) +void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) { - if (!cpu_wait_death(cpu, 5)) { - pr_err("CPU%u: cpu didn't die\n", cpu); - return; - } pr_debug("CPU%u: shutdown\n", cpu); clear_tasks_mm_cpumask(cpu); @@ -320,7 +315,7 @@ void __cpu_die(unsigned int cpu) * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void arch_cpu_idle_dead(void) +void __noreturn arch_cpu_idle_dead(void) { unsigned int cpu = smp_processor_id(); @@ -337,11 +332,11 @@ void arch_cpu_idle_dead(void) flush_cache_louis(); /* - * Tell __cpu_die() that this CPU is now safe to dispose of. Once - * this returns, power and/or clocks can be removed at any point - * from this CPU and its cache by platform_cpu_kill(). + * Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose + * of. Once this returns, power and/or clocks can be removed at + * any point from this CPU and its cache by platform_cpu_kill(). */ - (void)cpu_report_death(); + cpuhp_ap_report_dead(); /* * Ensure that the cache lines associated with that completion are @@ -382,6 +377,8 @@ void arch_cpu_idle_dead(void) : "r" (task_stack_page(current) + THREAD_SIZE - 8), "r" (current) : "r0"); + + unreachable(); } #endif /* CONFIG_HOTPLUG_CPU */ @@ -554,7 +551,8 @@ void show_ipi_list(struct seq_file *p, int prec) if (!ipi_desc[i]) continue; - seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, + prec >= 4 ? " " : ""); for_each_online_cpu(cpu) seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu)); @@ -600,6 +598,8 @@ static DEFINE_RAW_SPINLOCK(stop_lock); */ static void ipi_cpu_stop(unsigned int cpu) { + local_fiq_disable(); + if (system_state <= SYSTEM_RUNNING) { raw_spin_lock(&stop_lock); pr_crit("CPU%u: stopping\n", cpu); @@ -609,9 +609,6 @@ static void ipi_cpu_stop(unsigned int cpu) set_cpu_online(cpu, false); - local_fiq_disable(); - local_irq_disable(); - while (1) { cpu_relax(); wfe(); @@ -639,7 +636,7 @@ static void do_handle_IPI(int ipinr) unsigned int cpu = smp_processor_id(); if ((unsigned)ipinr < NR_IPI) - trace_ipi_entry_rcuidle(ipi_types[ipinr]); + trace_ipi_entry(ipi_types[ipinr]); switch (ipinr) { case IPI_WAKEUP: @@ -686,7 +683,7 @@ static void do_handle_IPI(int ipinr) } if ((unsigned)ipinr < NR_IPI) - trace_ipi_exit_rcuidle(ipi_types[ipinr]); + trace_ipi_exit(ipi_types[ipinr]); } /* Legacy version, should go away once all irqchips have been converted */ @@ -709,7 +706,7 @@ static irqreturn_t ipi_handler(int irq, void *data) static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { - trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); + trace_ipi_raise(target, ipi_types[ipinr]); __ipi_send_mask(ipi_desc[ipinr], target); } @@ -748,7 +745,7 @@ void __init set_smp_ipi_range(int ipi_base, int n) ipi_setup(smp_processor_id()); } -void smp_send_reschedule(int cpu) +void arch_smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); } @@ -778,7 +775,7 @@ void smp_send_stop(void) * kdump fails. So split out the panic_smp_self_stop() and add * set_cpu_online(smp_processor_id(), false). */ -void panic_smp_self_stop(void) +void __noreturn panic_smp_self_stop(void) { pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n", smp_processor_id()); @@ -787,14 +784,6 @@ void panic_smp_self_stop(void) cpu_relax(); } -/* - * not supported here - */ -int setup_profiling_timer(unsigned int multiplier) -{ - return -EINVAL; -} - #ifdef CONFIG_CPU_FREQ static DEFINE_PER_CPU(unsigned long, l_p_j_ref); @@ -858,7 +847,7 @@ static void raise_nmi(cpumask_t *mask) __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { - nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_nmi); + nmi_trigger_cpumask_backtrace(mask, exclude_cpu, raise_nmi); } diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index 9a14f721a2b0..42a3706e16a6 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -93,7 +93,6 @@ static void twd_timer_stop(void) { struct clock_event_device *clk = raw_cpu_ptr(twd_evt); - twd_shutdown(clk); disable_percpu_irq(clk->irq); } diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index d0fa2037460a..620aa82e3bdd 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -9,6 +9,8 @@ #include <asm/stacktrace.h> #include <asm/traps.h> +#include "reboot.h" + #if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) /* * Unwind the current stack frame and store the new register values in the @@ -39,29 +41,74 @@ * Note that with framepointer enabled, even the leaf functions have the same * prologue and epilogue, therefore we can ignore the LR value in this case. */ -int notrace unwind_frame(struct stackframe *frame) + +extern unsigned long call_with_stack_end; + +static int frame_pointer_check(struct stackframe *frame) { unsigned long high, low; unsigned long fp = frame->fp; + unsigned long pc = frame->pc; + + /* + * call_with_stack() is the only place we allow SP to jump from one + * stack to another, with FP and SP pointing to different stacks, + * skipping the FP boundary check at this point. + */ + if (pc >= (unsigned long)&call_with_stack && + pc < (unsigned long)&call_with_stack_end) + return 0; /* only go to a higher address on the stack */ low = frame->sp; high = ALIGN(low, THREAD_SIZE); -#ifdef CONFIG_CC_IS_CLANG /* check current frame pointer is within bounds */ +#ifdef CONFIG_CC_IS_CLANG if (fp < low + 4 || fp > high - 4) return -EINVAL; - - frame->sp = frame->fp; - frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); - frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4)); #else - /* check current frame pointer is within bounds */ if (fp < low + 12 || fp > high - 4) return -EINVAL; +#endif + + return 0; +} + +int notrace unwind_frame(struct stackframe *frame) +{ + unsigned long fp = frame->fp; + + if (frame_pointer_check(frame)) + return -EINVAL; + + /* + * When we unwind through an exception stack, include the saved PC + * value into the stack trace. + */ + if (frame->ex_frame) { + struct pt_regs *regs = (struct pt_regs *)frame->sp; + + /* + * We check that 'regs + sizeof(struct pt_regs)' (that is, + * ®s[1]) does not exceed the bottom of the stack to avoid + * accessing data outside the task's stack. This may happen + * when frame->ex_frame is a false positive. + */ + if ((unsigned long)®s[1] > ALIGN(frame->sp, THREAD_SIZE)) + return -EINVAL; + + frame->pc = regs->ARM_pc; + frame->ex_frame = false; + return 0; + } /* restore the registers from the stack frame */ +#ifdef CONFIG_CC_IS_CLANG + frame->sp = frame->fp; + frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); + frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4)); +#else frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 12)); frame->sp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 8)); frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 4)); @@ -72,17 +119,20 @@ int notrace unwind_frame(struct stackframe *frame) (void *)frame->fp, &frame->kr_cur); #endif + if (in_entry_text(frame->pc)) + frame->ex_frame = true; + return 0; } #endif void notrace walk_stackframe(struct stackframe *frame, - int (*fn)(struct stackframe *, void *), void *data) + bool (*fn)(void *, unsigned long), void *data) { while (1) { int ret; - if (fn(frame, data)) + if (!fn(data, frame->pc)) break; ret = unwind_frame(frame); if (ret < 0) @@ -92,55 +142,32 @@ void notrace walk_stackframe(struct stackframe *frame, EXPORT_SYMBOL(walk_stackframe); #ifdef CONFIG_STACKTRACE -struct stack_trace_data { - struct stack_trace *trace; - unsigned int no_sched_functions; - unsigned int skip; -}; - -static int save_trace(struct stackframe *frame, void *d) +static void start_stack_trace(struct stackframe *frame, struct task_struct *task, + unsigned long fp, unsigned long sp, + unsigned long lr, unsigned long pc) { - struct stack_trace_data *data = d; - struct stack_trace *trace = data->trace; - struct pt_regs *regs; - unsigned long addr = frame->pc; - - if (data->no_sched_functions && in_sched_functions(addr)) - return 0; - if (data->skip) { - data->skip--; - return 0; - } - - trace->entries[trace->nr_entries++] = addr; - - if (trace->nr_entries >= trace->max_entries) - return 1; - - if (!in_entry_text(frame->pc)) - return 0; - - regs = (struct pt_regs *)frame->sp; - if ((unsigned long)®s[1] > ALIGN(frame->sp, THREAD_SIZE)) - return 0; - - trace->entries[trace->nr_entries++] = regs->ARM_pc; - - return trace->nr_entries >= trace->max_entries; + frame->fp = fp; + frame->sp = sp; + frame->lr = lr; + frame->pc = pc; +#ifdef CONFIG_KRETPROBES + frame->kr_cur = NULL; + frame->tsk = task; +#endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + frame->ex_frame = in_entry_text(frame->pc); +#endif } -/* This must be noinline to so that our skip calculation works correctly */ -static noinline void __save_stack_trace(struct task_struct *tsk, - struct stack_trace *trace, unsigned int nosched) +void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, + struct task_struct *task, struct pt_regs *regs) { - struct stack_trace_data data; struct stackframe frame; - data.trace = trace; - data.skip = trace->skip; - data.no_sched_functions = nosched; - - if (tsk != current) { + if (regs) { + start_stack_trace(&frame, NULL, regs->ARM_fp, regs->ARM_sp, + regs->ARM_lr, regs->ARM_pc); + } else if (task != current) { #ifdef CONFIG_SMP /* * What guarantees do we have here that 'tsk' is not @@ -149,58 +176,22 @@ static noinline void __save_stack_trace(struct task_struct *tsk, */ return; #else - frame.fp = thread_saved_fp(tsk); - frame.sp = thread_saved_sp(tsk); - frame.lr = 0; /* recovered from the stack */ - frame.pc = thread_saved_pc(tsk); + start_stack_trace(&frame, task, thread_saved_fp(task), + thread_saved_sp(task), 0, + thread_saved_pc(task)); #endif } else { - /* We don't want this function nor the caller */ - data.skip += 2; - frame.fp = (unsigned long)__builtin_frame_address(0); - frame.sp = current_stack_pointer; - frame.lr = (unsigned long)__builtin_return_address(0); here: - frame.pc = (unsigned long)&&here; + start_stack_trace(&frame, task, + (unsigned long)__builtin_frame_address(0), + current_stack_pointer, + (unsigned long)__builtin_return_address(0), + (unsigned long)&&here); + /* skip this function */ + if (unwind_frame(&frame)) + return; } -#ifdef CONFIG_KRETPROBES - frame.kr_cur = NULL; - frame.tsk = tsk; -#endif - walk_stackframe(&frame, save_trace, &data); -} - -void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) -{ - struct stack_trace_data data; - struct stackframe frame; - - data.trace = trace; - data.skip = trace->skip; - data.no_sched_functions = 0; - - frame.fp = regs->ARM_fp; - frame.sp = regs->ARM_sp; - frame.lr = regs->ARM_lr; - frame.pc = regs->ARM_pc; -#ifdef CONFIG_KRETPROBES - frame.kr_cur = NULL; - frame.tsk = current; -#endif - - walk_stackframe(&frame, save_trace, &data); -} - -void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) -{ - __save_stack_trace(tsk, trace, 1); -} -EXPORT_SYMBOL(save_stack_trace_tsk); - -void save_stack_trace(struct stack_trace *trace) -{ - __save_stack_trace(current, trace, 0); + walk_stackframe(&frame, consume_entry, cookie); } -EXPORT_SYMBOL_GPL(save_stack_trace); #endif diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c index 43f0a3ebf390..58a6441b58c4 100644 --- a/arch/arm/kernel/suspend.c +++ b/arch/arm/kernel/suspend.c @@ -8,10 +8,11 @@ #include <asm/bugs.h> #include <asm/cacheflush.h> #include <asm/idmap.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/smp_plat.h> #include <asm/suspend.h> #include <asm/tlbflush.h> +#include <asm/uaccess.h> extern int __cpu_suspend(unsigned long, int (*)(unsigned long), u32 cpuid); extern void cpu_resume_mmu(void); @@ -27,6 +28,13 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) return -EINVAL; /* + * Needed for the MMU disabling/enabing code to be able to run from + * TTBR0 addresses. + */ + if (IS_ENABLED(CONFIG_CPU_TTBR0_PAN)) + uaccess_save_and_enable(); + + /* * Function graph tracer state gets incosistent when the kernel * calls functions that never return (aka suspend finishers) hence * disable graph tracing during their execution. diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index b74bfcf94fb1..fdce83c95acb 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -34,6 +34,7 @@ */ #define __user_swpX_asm(data, addr, res, temp, B) \ __asm__ __volatile__( \ + ".arch armv7-a\n" \ "0: ldrex"B" %2, [%3]\n" \ "1: strex"B" %0, %1, [%3]\n" \ " cmp %0, #0\n" \ diff --git a/arch/arm/kernel/sys_arm.c b/arch/arm/kernel/sys_arm.c index a5f183cfecb1..0141e9bb02e8 100644 --- a/arch/arm/kernel/sys_arm.c +++ b/arch/arm/kernel/sys_arm.c @@ -24,6 +24,7 @@ #include <linux/ipc.h> #include <linux/uaccess.h> #include <linux/slab.h> +#include <asm/syscalls.h> /* * Since loff_t is a 64 bit type we avoid a lot of ABI hassle diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index 68112c172025..2944721e82a2 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -10,6 +10,8 @@ * Copyright: MontaVista Software, Inc. */ +#include <asm/syscalls.h> + /* * The legacy ABI and the new ARM EABI have different rules making some * syscalls incompatible especially with structure arguments. @@ -73,6 +75,7 @@ #include <linux/syscalls.h> #include <linux/errno.h> #include <linux/fs.h> +#include <linux/filelock.h> #include <linux/cred.h> #include <linux/fcntl.h> #include <linux/eventpoll.h> @@ -232,23 +235,23 @@ asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, unsigned long arg) { void __user *argp = (void __user *)arg; - struct fd f = fdget_raw(fd); + CLASS(fd_raw, f)(fd); struct flock64 flock; - long err = -EBADF; + long err; - if (!f.file) - goto out; + if (fd_empty(f)) + return -EBADF; switch (cmd) { case F_GETLK64: case F_OFD_GETLK: - err = security_file_fcntl(f.file, cmd, arg); + err = security_file_fcntl(fd_file(f), cmd, arg); if (err) break; err = get_oabi_flock(&flock, argp); if (err) break; - err = fcntl_getlk64(f.file, cmd, &flock); + err = fcntl_getlk64(fd_file(f), cmd, &flock); if (!err) err = put_oabi_flock(&flock, argp); break; @@ -256,20 +259,18 @@ asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, case F_SETLKW64: case F_OFD_SETLK: case F_OFD_SETLKW: - err = security_file_fcntl(f.file, cmd, arg); + err = security_file_fcntl(fd_file(f), cmd, arg); if (err) break; err = get_oabi_flock(&flock, argp); if (err) break; - err = fcntl_setlk64(fd, f.file, cmd, &flock); + err = fcntl_setlk64(fd, fd_file(f), cmd, &flock); break; default: err = sys_fcntl64(fd, cmd, arg); break; } - fdput(f); -out: return err; } diff --git a/arch/arm/kernel/tcm.c b/arch/arm/kernel/tcm.c index d3a85f01b328..f59927bcfbce 100644 --- a/arch/arm/kernel/tcm.c +++ b/arch/arm/kernel/tcm.c @@ -15,7 +15,7 @@ #include <linux/string.h> /* memcpy */ #include <asm/cputype.h> #include <asm/mach/map.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/system_info.h> #include <asm/traps.h> #include <asm/tcm.h> diff --git a/arch/arm/kernel/topology.c b/arch/arm/kernel/topology.c index ef0058de432b..2336ee2aa44a 100644 --- a/arch/arm/kernel/topology.c +++ b/arch/arm/kernel/topology.c @@ -42,7 +42,7 @@ * can take this difference into account during load balance. A per cpu * structure is preferred because each CPU updates its own cpu_capacity field * during the load balance except for idle cores. One idle core is selected - * to run the rebalance_domains for all idle cores and the cpu_capacity can be + * to run the sched_balance_domains for all idle cores and the cpu_capacity can be * updated during this sequence. */ diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 9283dc65be31..afbd2ebe5c39 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -26,6 +26,7 @@ #include <linux/sched/debug.h> #include <linux/sched/task_stack.h> #include <linux/irq.h> +#include <linux/vmalloc.h> #include <linux/atomic.h> #include <asm/cacheflush.h> @@ -178,19 +179,22 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) for (i = -4; i < 1 + !!thumb; i++) { unsigned int val, bad; - if (!user_mode(regs)) { - if (thumb) { - u16 val16; - bad = get_kernel_nofault(val16, &((u16 *)addr)[i]); - val = val16; - } else { - bad = get_kernel_nofault(val, &((u32 *)addr)[i]); - } + if (thumb) { + u16 tmp; + + if (user_mode(regs)) + bad = get_user(tmp, &((u16 __user *)addr)[i]); + else + bad = get_kernel_nofault(tmp, &((u16 *)addr)[i]); + + val = __mem_to_opcode_thumb16(tmp); } else { - if (thumb) - bad = get_user(val, &((u16 *)addr)[i]); + if (user_mode(regs)) + bad = get_user(val, &((u32 __user *)addr)[i]); else - bad = get_user(val, &((u32 *)addr)[i]); + bad = get_kernel_nofault(val, &((u32 *)addr)[i]); + + val = __mem_to_opcode_arm(val); } if (!bad) @@ -205,19 +209,19 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) } #ifdef CONFIG_ARM_UNWIND -static inline void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, - const char *loglvl) +void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, + const char *loglvl) { unwind_backtrace(regs, tsk, loglvl); } #else -static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, - const char *loglvl) +void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, + const char *loglvl) { unsigned int fp, mode; int ok = 1; - printk("%sBacktrace: ", loglvl); + printk("%sCall trace: ", loglvl); if (!tsk) tsk = current; @@ -254,13 +258,6 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) barrier(); } -#ifdef CONFIG_PREEMPT -#define S_PREEMPT " PREEMPT" -#elif defined(CONFIG_PREEMPT_RT) -#define S_PREEMPT " PREEMPT_RT" -#else -#define S_PREEMPT "" -#endif #ifdef CONFIG_SMP #define S_SMP " SMP" #else @@ -278,8 +275,8 @@ static int __die(const char *str, int err, struct pt_regs *regs) static int die_counter; int ret; - pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP S_ISA "\n", - str, err, ++die_counter); + pr_emerg("Internal error: %s: %x [#%d]" S_SMP S_ISA "\n", + str, err, ++die_counter); /* trap and error numbers are mostly meaningless on ARM */ ret = notify_die(DIE_OOPS, str, regs, err, tsk->thread.trap_no, SIGSEGV); @@ -487,7 +484,7 @@ asmlinkage void do_undefinstr(struct pt_regs *regs) die_sig: #ifdef CONFIG_DEBUG_USER if (user_debug & UDBG_UNDEFINED) { - pr_info("%s (%d): undefined instruction: pc=%p\n", + pr_info("%s (%d): undefined instruction: pc=%px\n", current->comm, task_pid_nr(current), pc); __show_regs(regs); dump_instr(KERN_INFO, regs); @@ -566,6 +563,7 @@ static int bad_syscall(int n, struct pt_regs *regs) static inline int __do_cache_op(unsigned long start, unsigned long end) { + unsigned int ua_flags; int ret; do { @@ -574,7 +572,9 @@ __do_cache_op(unsigned long start, unsigned long end) if (fatal_signal_pending(current)) return 0; + ua_flags = uaccess_save_and_enable(); ret = flush_icache_user_range(start, start + chunk); + uaccess_restore(ua_flags); if (ret) return ret; @@ -753,6 +753,7 @@ void __readwrite_bug(const char *fn) } EXPORT_SYMBOL(__readwrite_bug); +#ifdef CONFIG_MMU void __pte_error(const char *file, int line, pte_t pte) { pr_err("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); @@ -767,6 +768,7 @@ void __pgd_error(const char *file, int line, pgd_t pgd) { pr_err("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); } +#endif asmlinkage void __div0(void) { @@ -920,9 +922,9 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) { unsigned long tsk_stk = (unsigned long)current->stack; #ifdef CONFIG_IRQSTACKS - unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); + unsigned long irq_stk = (unsigned long)raw_cpu_read(irq_stack_ptr); #endif - unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); + unsigned long ovf_stk = (unsigned long)raw_cpu_read(overflow_stack_ptr); console_verbose(); pr_emerg("Insufficient stack space to handle exception!"); diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index a37ea6c772cd..f60547dadc93 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -28,6 +28,7 @@ #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/list.h> +#include <linux/module.h> #include <asm/stacktrace.h> #include <asm/traps.h> @@ -307,6 +308,29 @@ static int unwind_exec_pop_subset_r0_to_r3(struct unwind_ctrl_block *ctrl, return URC_OK; } +static unsigned long unwind_decode_uleb128(struct unwind_ctrl_block *ctrl) +{ + unsigned long bytes = 0; + unsigned long insn; + unsigned long result = 0; + + /* + * unwind_get_byte() will advance `ctrl` one instruction at a time, so + * loop until we get an instruction byte where bit 7 is not set. + * + * Note: This decodes a maximum of 4 bytes to output 28 bits data where + * max is 0xfffffff: that will cover a vsp increment of 1073742336, hence + * it is sufficient for unwinding the stack. + */ + do { + insn = unwind_get_byte(ctrl); + result |= (insn & 0x7f) << (bytes * 7); + bytes++; + } while (!!(insn & 0x80) && (bytes != sizeof(result))); + + return result; +} + /* * Execute the current unwind instruction. */ @@ -360,7 +384,7 @@ static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) if (ret) goto error; } else if (insn == 0xb2) { - unsigned long uleb128 = unwind_get_byte(ctrl); + unsigned long uleb128 = unwind_decode_uleb128(ctrl); ctrl->vrs[SP] += 0x204 + (uleb128 << 2); } else { @@ -395,8 +419,18 @@ int unwind_frame(struct stackframe *frame) idx = unwind_find_idx(frame->pc); if (!idx) { - if (frame->pc && kernel_text_address(frame->pc)) + if (frame->pc && kernel_text_address(frame->pc)) { + if (in_module_plt(frame->pc) && frame->pc != frame->lr) { + /* + * Quoting Ard: Veneers only set PC using a + * PC+immediate LDR, and so they don't affect + * the state of the stack or the register file + */ + frame->pc = frame->lr; + return URC_OK; + } pr_warn("unwind: Index not found %08lx\n", frame->pc); + } return -URC_FAILURE; } @@ -490,6 +524,8 @@ void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk, { struct stackframe frame; + printk("%sCall trace: ", loglvl); + pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); if (!tsk) diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index 3408269d19c7..325448ffbba0 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -7,6 +7,7 @@ */ #include <linux/cache.h> +#include <linux/vdso_datastore.h> #include <linux/elf.h> #include <linux/err.h> #include <linux/kernel.h> @@ -14,14 +15,12 @@ #include <linux/of.h> #include <linux/printk.h> #include <linux/slab.h> -#include <linux/timekeeper_internal.h> #include <linux/vmalloc.h> #include <asm/arch_timer.h> #include <asm/barrier.h> #include <asm/cacheflush.h> #include <asm/page.h> #include <asm/vdso.h> -#include <asm/vdso_datapage.h> #include <clocksource/arm_arch_timer.h> #include <vdso/helpers.h> #include <vdso/vsyscall.h> @@ -35,18 +34,6 @@ extern char vdso_start[], vdso_end[]; /* Total number of pages needed for the data and text portions of the VDSO. */ unsigned int vdso_total_pages __ro_after_init; -/* - * The VDSO data page. - */ -static union vdso_data_store vdso_data_store __page_aligned_data; -struct vdso_data *vdso_data = vdso_data_store.data; - -static struct page *vdso_data_page __ro_after_init; -static const struct vm_special_mapping vdso_data_mapping = { - .name = "[vvar]", - .pages = &vdso_data_page, -}; - static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma) { @@ -135,7 +122,7 @@ static Elf32_Sym * __init find_symbol(struct elfinfo *lib, const char *symname) if (lib->dynsym[i].st_name == 0) continue; - strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, + strscpy(name, lib->dynstr + lib->dynsym[i].st_name, MAX_SYMNAME); c = strchr(name, '@'); if (c) @@ -197,9 +184,6 @@ static int __init vdso_init(void) if (vdso_text_pagelist == NULL) return -ENOMEM; - /* Grab the VDSO data page. */ - vdso_data_page = virt_to_page(vdso_data); - /* Grab the VDSO text pages. */ for (i = 0; i < text_pages; i++) { struct page *page; @@ -210,7 +194,7 @@ static int __init vdso_init(void) vdso_text_mapping.pages = vdso_text_pagelist; - vdso_total_pages = 1; /* for the data/vvar page */ + vdso_total_pages = VDSO_NR_PAGES; /* for the data/vvar pages */ vdso_total_pages += text_pages; cntvct_ok = cntvct_functional(); @@ -221,16 +205,7 @@ static int __init vdso_init(void) } arch_initcall(vdso_init); -static int install_vvar(struct mm_struct *mm, unsigned long addr) -{ - struct vm_area_struct *vma; - - vma = _install_special_mapping(mm, addr, PAGE_SIZE, - VM_READ | VM_MAYREAD, - &vdso_data_mapping); - - return PTR_ERR_OR_ZERO(vma); -} +static_assert(__VDSO_PAGES == VDSO_NR_PAGES); /* assumes mmap_lock is write-locked */ void arm_install_vdso(struct mm_struct *mm, unsigned long addr) @@ -243,12 +218,12 @@ void arm_install_vdso(struct mm_struct *mm, unsigned long addr) if (vdso_text_pagelist == NULL) return; - if (install_vvar(mm, addr)) + if (IS_ERR(vdso_install_vvar_mapping(mm, addr))) return; - /* Account for vvar page. */ - addr += PAGE_SIZE; - len = (vdso_total_pages - 1) << PAGE_SHIFT; + /* Account for vvar pages. */ + addr += VDSO_NR_PAGES * PAGE_SIZE; + len = (vdso_total_pages - VDSO_NR_PAGES) << PAGE_SHIFT; vma = _install_special_mapping(mm, addr, len, VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, diff --git a/arch/arm/kernel/vmcore_info.c b/arch/arm/kernel/vmcore_info.c new file mode 100644 index 000000000000..1437aba47787 --- /dev/null +++ b/arch/arm/kernel/vmcore_info.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/vmcore_info.h> + +void arch_crash_save_vmcoreinfo(void) +{ +#ifdef CONFIG_ARM_LPAE + VMCOREINFO_CONFIG(ARM_LPAE); +#endif +} diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S index 76678732c60d..f2e8d4fac068 100644 --- a/arch/arm/kernel/vmlinux-xip.lds.S +++ b/arch/arm/kernel/vmlinux-xip.lds.S @@ -12,9 +12,8 @@ #include <asm/vmlinux.lds.h> #include <asm/cache.h> #include <asm/thread_info.h> -#include <asm/memory.h> -#include <asm/mpu.h> #include <asm/page.h> +#include <asm/mpu.h> OUTPUT_ARCH(arm) ENTRY(stext) @@ -84,7 +83,7 @@ SECTIONS } .init.arch.info : { __arch_info_begin = .; - *(.arch.info.init) + KEEP(*(.arch.info.init)) __arch_info_end = .; } .init.tagtable : { diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index aa12b65a7fd6..d592a203f9c6 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -12,9 +12,8 @@ #include <asm/vmlinux.lds.h> #include <asm/cache.h> #include <asm/thread_info.h> -#include <asm/memory.h> -#include <asm/mpu.h> #include <asm/page.h> +#include <asm/mpu.h> OUTPUT_ARCH(arm) ENTRY(stext) @@ -100,7 +99,7 @@ SECTIONS } .init.arch.info : { __arch_info_begin = .; - *(.arch.info.init) + KEEP(*(.arch.info.init)) __arch_info_end = .; } .init.tagtable : { @@ -117,7 +116,7 @@ SECTIONS #endif .init.pv_table : { __pv_table_begin = .; - *(.pv_table) + KEEP(*(.pv_table)) __pv_table_end = .; } diff --git a/arch/arm/kernel/xscale-cp0.c b/arch/arm/kernel/xscale-cp0.c index ed4f6e77616d..00d00d3aae97 100644 --- a/arch/arm/kernel/xscale-cp0.c +++ b/arch/arm/kernel/xscale-cp0.c @@ -166,6 +166,7 @@ static int __init xscale_cp0_init(void) pr_info("XScale iWMMXt coprocessor detected.\n"); elf_hwcap |= HWCAP_IWMMXT; thread_register_notifier(&iwmmxt_notifier_block); + register_iwmmxt_undef_handler(); #endif } else { pr_info("XScale DSP coprocessor detected.\n"); |