diff options
Diffstat (limited to 'arch/arm/include/asm')
101 files changed, 1051 insertions, 674 deletions
diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index f82a819eb0db..311e83038bdb 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -252,5 +252,10 @@ static inline void gic_arch_enable_irqs(void) WARN_ON_ONCE(true); } +static inline bool gic_has_relaxed_pmr_sync(void) +{ + return false; +} + #endif /* !__ASSEMBLY__ */ #endif /* !__ASM_ARCH_GICV3_H */ diff --git a/arch/arm/include/asm/arm_pmuv3.h b/arch/arm/include/asm/arm_pmuv3.h new file mode 100644 index 000000000000..2ec0e5e83fc9 --- /dev/null +++ b/arch/arm/include/asm/arm_pmuv3.h @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2012 ARM Ltd. + */ + +#ifndef __ASM_PMUV3_H +#define __ASM_PMUV3_H + +#include <asm/cp15.h> +#include <asm/cputype.h> + +#define PMCCNTR __ACCESS_CP15_64(0, c9) + +#define PMCR __ACCESS_CP15(c9, 0, c12, 0) +#define PMCNTENSET __ACCESS_CP15(c9, 0, c12, 1) +#define PMCNTENCLR __ACCESS_CP15(c9, 0, c12, 2) +#define PMOVSR __ACCESS_CP15(c9, 0, c12, 3) +#define PMSELR __ACCESS_CP15(c9, 0, c12, 5) +#define PMCEID0 __ACCESS_CP15(c9, 0, c12, 6) +#define PMCEID1 __ACCESS_CP15(c9, 0, c12, 7) +#define PMXEVTYPER __ACCESS_CP15(c9, 0, c13, 1) +#define PMXEVCNTR __ACCESS_CP15(c9, 0, c13, 2) +#define PMUSERENR __ACCESS_CP15(c9, 0, c14, 0) +#define PMINTENSET __ACCESS_CP15(c9, 0, c14, 1) +#define PMINTENCLR __ACCESS_CP15(c9, 0, c14, 2) +#define PMCEID2 __ACCESS_CP15(c9, 0, c14, 4) +#define PMCEID3 __ACCESS_CP15(c9, 0, c14, 5) +#define PMMIR __ACCESS_CP15(c9, 0, c14, 6) +#define PMCCFILTR __ACCESS_CP15(c14, 0, c15, 7) + +#define PMEVCNTR0 __ACCESS_CP15(c14, 0, c8, 0) +#define PMEVCNTR1 __ACCESS_CP15(c14, 0, c8, 1) +#define PMEVCNTR2 __ACCESS_CP15(c14, 0, c8, 2) +#define PMEVCNTR3 __ACCESS_CP15(c14, 0, c8, 3) +#define PMEVCNTR4 __ACCESS_CP15(c14, 0, c8, 4) +#define PMEVCNTR5 __ACCESS_CP15(c14, 0, c8, 5) +#define PMEVCNTR6 __ACCESS_CP15(c14, 0, c8, 6) +#define PMEVCNTR7 __ACCESS_CP15(c14, 0, c8, 7) +#define PMEVCNTR8 __ACCESS_CP15(c14, 0, c9, 0) +#define PMEVCNTR9 __ACCESS_CP15(c14, 0, c9, 1) +#define PMEVCNTR10 __ACCESS_CP15(c14, 0, c9, 2) +#define PMEVCNTR11 __ACCESS_CP15(c14, 0, c9, 3) +#define PMEVCNTR12 __ACCESS_CP15(c14, 0, c9, 4) +#define PMEVCNTR13 __ACCESS_CP15(c14, 0, c9, 5) +#define PMEVCNTR14 __ACCESS_CP15(c14, 0, c9, 6) +#define PMEVCNTR15 __ACCESS_CP15(c14, 0, c9, 7) +#define PMEVCNTR16 __ACCESS_CP15(c14, 0, c10, 0) +#define PMEVCNTR17 __ACCESS_CP15(c14, 0, c10, 1) +#define PMEVCNTR18 __ACCESS_CP15(c14, 0, c10, 2) +#define PMEVCNTR19 __ACCESS_CP15(c14, 0, c10, 3) +#define PMEVCNTR20 __ACCESS_CP15(c14, 0, c10, 4) +#define PMEVCNTR21 __ACCESS_CP15(c14, 0, c10, 5) +#define PMEVCNTR22 __ACCESS_CP15(c14, 0, c10, 6) +#define PMEVCNTR23 __ACCESS_CP15(c14, 0, c10, 7) +#define PMEVCNTR24 __ACCESS_CP15(c14, 0, c11, 0) +#define PMEVCNTR25 __ACCESS_CP15(c14, 0, c11, 1) +#define PMEVCNTR26 __ACCESS_CP15(c14, 0, c11, 2) +#define PMEVCNTR27 __ACCESS_CP15(c14, 0, c11, 3) +#define PMEVCNTR28 __ACCESS_CP15(c14, 0, c11, 4) +#define PMEVCNTR29 __ACCESS_CP15(c14, 0, c11, 5) +#define PMEVCNTR30 __ACCESS_CP15(c14, 0, c11, 6) + +#define PMEVTYPER0 __ACCESS_CP15(c14, 0, c12, 0) +#define PMEVTYPER1 __ACCESS_CP15(c14, 0, c12, 1) +#define PMEVTYPER2 __ACCESS_CP15(c14, 0, c12, 2) +#define PMEVTYPER3 __ACCESS_CP15(c14, 0, c12, 3) +#define PMEVTYPER4 __ACCESS_CP15(c14, 0, c12, 4) +#define PMEVTYPER5 __ACCESS_CP15(c14, 0, c12, 5) +#define PMEVTYPER6 __ACCESS_CP15(c14, 0, c12, 6) +#define PMEVTYPER7 __ACCESS_CP15(c14, 0, c12, 7) +#define PMEVTYPER8 __ACCESS_CP15(c14, 0, c13, 0) +#define PMEVTYPER9 __ACCESS_CP15(c14, 0, c13, 1) +#define PMEVTYPER10 __ACCESS_CP15(c14, 0, c13, 2) +#define PMEVTYPER11 __ACCESS_CP15(c14, 0, c13, 3) +#define PMEVTYPER12 __ACCESS_CP15(c14, 0, c13, 4) +#define PMEVTYPER13 __ACCESS_CP15(c14, 0, c13, 5) +#define PMEVTYPER14 __ACCESS_CP15(c14, 0, c13, 6) +#define PMEVTYPER15 __ACCESS_CP15(c14, 0, c13, 7) +#define PMEVTYPER16 __ACCESS_CP15(c14, 0, c14, 0) +#define PMEVTYPER17 __ACCESS_CP15(c14, 0, c14, 1) +#define PMEVTYPER18 __ACCESS_CP15(c14, 0, c14, 2) +#define PMEVTYPER19 __ACCESS_CP15(c14, 0, c14, 3) +#define PMEVTYPER20 __ACCESS_CP15(c14, 0, c14, 4) +#define PMEVTYPER21 __ACCESS_CP15(c14, 0, c14, 5) +#define PMEVTYPER22 __ACCESS_CP15(c14, 0, c14, 6) +#define PMEVTYPER23 __ACCESS_CP15(c14, 0, c14, 7) +#define PMEVTYPER24 __ACCESS_CP15(c14, 0, c15, 0) +#define PMEVTYPER25 __ACCESS_CP15(c14, 0, c15, 1) +#define PMEVTYPER26 __ACCESS_CP15(c14, 0, c15, 2) +#define PMEVTYPER27 __ACCESS_CP15(c14, 0, c15, 3) +#define PMEVTYPER28 __ACCESS_CP15(c14, 0, c15, 4) +#define PMEVTYPER29 __ACCESS_CP15(c14, 0, c15, 5) +#define PMEVTYPER30 __ACCESS_CP15(c14, 0, c15, 6) + +#define RETURN_READ_PMEVCNTRN(n) \ + return read_sysreg(PMEVCNTR##n) +static inline unsigned long read_pmevcntrn(int n) +{ + PMEVN_SWITCH(n, RETURN_READ_PMEVCNTRN); + return 0; +} + +#define WRITE_PMEVCNTRN(n) \ + write_sysreg(val, PMEVCNTR##n) +static inline void write_pmevcntrn(int n, unsigned long val) +{ + PMEVN_SWITCH(n, WRITE_PMEVCNTRN); +} + +#define WRITE_PMEVTYPERN(n) \ + write_sysreg(val, PMEVTYPER##n) +static inline void write_pmevtypern(int n, unsigned long val) +{ + PMEVN_SWITCH(n, WRITE_PMEVTYPERN); +} + +static inline unsigned long read_pmmir(void) +{ + return read_sysreg(PMMIR); +} + +static inline u32 read_pmuver(void) +{ + /* PMUVers is not a signed field */ + u32 dfr0 = read_cpuid_ext(CPUID_EXT_DFR0); + + return (dfr0 >> 24) & 0xf; +} + +static inline bool pmuv3_has_icntr(void) +{ + /* FEAT_PMUv3_ICNTR not accessible for 32-bit */ + return false; +} + +static inline void write_pmcr(u32 val) +{ + write_sysreg(val, PMCR); +} + +static inline u32 read_pmcr(void) +{ + return read_sysreg(PMCR); +} + +static inline void write_pmselr(u32 val) +{ + write_sysreg(val, PMSELR); +} + +static inline void write_pmccntr(u64 val) +{ + write_sysreg(val, PMCCNTR); +} + +static inline u64 read_pmccntr(void) +{ + return read_sysreg(PMCCNTR); +} + +static inline void write_pmicntr(u64 val) {} + +static inline u64 read_pmicntr(void) +{ + return 0; +} + +static inline void write_pmcntenset(u32 val) +{ + write_sysreg(val, PMCNTENSET); +} + +static inline void write_pmcntenclr(u32 val) +{ + write_sysreg(val, PMCNTENCLR); +} + +static inline void write_pmintenset(u32 val) +{ + write_sysreg(val, PMINTENSET); +} + +static inline void write_pmintenclr(u32 val) +{ + write_sysreg(val, PMINTENCLR); +} + +static inline void write_pmccfiltr(u32 val) +{ + write_sysreg(val, PMCCFILTR); +} + +static inline void write_pmicfiltr(u64 val) {} + +static inline u64 read_pmicfiltr(void) +{ + return 0; +} + +static inline void write_pmovsclr(u32 val) +{ + write_sysreg(val, PMOVSR); +} + +static inline u32 read_pmovsclr(void) +{ + return read_sysreg(PMOVSR); +} + +static inline void write_pmuserenr(u32 val) +{ + write_sysreg(val, PMUSERENR); +} + +static inline void write_pmuacr(u64 val) {} + +static inline void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) {} +static inline void kvm_clr_pmu_events(u32 clr) {} +static inline bool kvm_pmu_counter_deferred(struct perf_event_attr *attr) +{ + return false; +} + +static inline bool kvm_set_pmuserenr(u64 val) +{ + return false; +} + +static inline void kvm_vcpu_pmu_resync_el0(void) {} + +/* PMU Version in DFR Register */ +#define ARMV8_PMU_DFR_VER_NI 0 +#define ARMV8_PMU_DFR_VER_V3P1 0x4 +#define ARMV8_PMU_DFR_VER_V3P4 0x5 +#define ARMV8_PMU_DFR_VER_V3P5 0x6 +#define ARMV8_PMU_DFR_VER_V3P9 0x9 +#define ARMV8_PMU_DFR_VER_IMP_DEF 0xF + +static inline bool pmuv3_implemented(int pmuver) +{ + return !(pmuver == ARMV8_PMU_DFR_VER_IMP_DEF || + pmuver == ARMV8_PMU_DFR_VER_NI); +} + +static inline bool is_pmuv3p4(int pmuver) +{ + return pmuver >= ARMV8_PMU_DFR_VER_V3P4; +} + +static inline bool is_pmuv3p5(int pmuver) +{ + return pmuver >= ARMV8_PMU_DFR_VER_V3P5; +} + +static inline bool is_pmuv3p9(int pmuver) +{ + return pmuver >= ARMV8_PMU_DFR_VER_V3P9; +} + +static inline u64 read_pmceid0(void) +{ + u64 val = read_sysreg(PMCEID0); + + if (read_pmuver() >= ARMV8_PMU_DFR_VER_V3P1) + val |= (u64)read_sysreg(PMCEID2) << 32; + + return val; +} + +static inline u64 read_pmceid1(void) +{ + u64 val = read_sysreg(PMCEID1); + + if (read_pmuver() >= ARMV8_PMU_DFR_VER_V3P1) + val |= (u64)read_sysreg(PMCEID3) << 32; + + return val; +} + +#endif diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 90fbe4a3f9c8..d33c1e24e00b 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -21,6 +21,7 @@ #include <asm/opcodes-virt.h> #include <asm/asm-offsets.h> #include <asm/page.h> +#include <asm/pgtable.h> #include <asm/thread_info.h> #include <asm/uaccess-asm.h> @@ -236,20 +237,12 @@ THUMB( fpreg .req r7 ) sub \tmp, \tmp, #1 @ decrement it str \tmp, [\ti, #TI_PREEMPT] .endm - - .macro dec_preempt_count_ti, ti, tmp - get_thread_info \ti - dec_preempt_count \ti, \tmp - .endm #else .macro inc_preempt_count, ti, tmp .endm .macro dec_preempt_count, ti, tmp .endm - - .macro dec_preempt_count_ti, ti, tmp - .endm #endif #define USERL(l, x...) \ @@ -402,6 +395,23 @@ ALT_UP_B(.L0_\@) #endif .endm +/* + * Raw SMP data memory barrier + */ + .macro __smp_dmb mode +#if __LINUX_ARM_ARCH__ >= 7 + .ifeqs "\mode","arm" + dmb ish + .else + W(dmb) ish + .endif +#elif __LINUX_ARM_ARCH__ == 6 + mcr p15, 0, r0, c7, c10, 5 @ dmb +#else + .error "Incompatible SMP platform" +#endif + .endm + #if defined(CONFIG_CPU_V7M) /* * setmode is used to assert to be in svc mode during boot. For v7-M @@ -761,6 +771,12 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) .endif .endm + .if __LINUX_ARM_ARCH__ < 6 + .set .Lrev_l_uses_tmp, 1 + .else + .set .Lrev_l_uses_tmp, 0 + .endif + /* * bl_r - branch and link to register * diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index db8512d9a918..f0e3b01afa74 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -197,6 +197,16 @@ static inline int arch_atomic_fetch_##op(int i, atomic_t *v) \ return val; \ } +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub + +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor + static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; @@ -210,8 +220,7 @@ static inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) return ret; } - -#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#define arch_atomic_cmpxchg arch_atomic_cmpxchg #endif /* __LINUX_ARM_ARCH__ */ @@ -240,8 +249,6 @@ ATOMIC_OPS(xor, ^=, eor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -#define arch_atomic_xchg(v, new) (arch_xchg(&((v)->counter), new)) - #ifndef CONFIG_GENERIC_ATOMIC64 typedef struct { s64 counter; diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index 8e94fe7ab5eb..714440fa2fc6 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -160,18 +160,20 @@ extern int _test_and_change_bit(int nr, volatile unsigned long * p); /* * Little endian assembly bitops. nr = 0 -> byte 0 bit 0. */ -extern int _find_first_zero_bit_le(const unsigned long *p, unsigned size); -extern int _find_next_zero_bit_le(const unsigned long *p, int size, int offset); -extern int _find_first_bit_le(const unsigned long *p, unsigned size); -extern int _find_next_bit_le(const unsigned long *p, int size, int offset); +unsigned long _find_first_zero_bit_le(const unsigned long *p, unsigned long size); +unsigned long _find_next_zero_bit_le(const unsigned long *p, + unsigned long size, unsigned long offset); +unsigned long _find_first_bit_le(const unsigned long *p, unsigned long size); +unsigned long _find_next_bit_le(const unsigned long *p, unsigned long size, unsigned long offset); /* * Big endian assembly bitops. nr = 0 -> byte 3 bit 0. */ -extern int _find_first_zero_bit_be(const unsigned long *p, unsigned size); -extern int _find_next_zero_bit_be(const unsigned long *p, int size, int offset); -extern int _find_first_bit_be(const unsigned long *p, unsigned size); -extern int _find_next_bit_be(const unsigned long *p, int size, int offset); +unsigned long _find_first_zero_bit_be(const unsigned long *p, unsigned long size); +unsigned long _find_next_zero_bit_be(const unsigned long *p, + unsigned long size, unsigned long offset); +unsigned long _find_first_bit_be(const unsigned long *p, unsigned long size); +unsigned long _find_next_bit_be(const unsigned long *p, unsigned long size, unsigned long offset); #ifndef CONFIG_SMP /* diff --git a/arch/arm/include/asm/bugs.h b/arch/arm/include/asm/bugs.h index 97a312ba0840..fe385551edec 100644 --- a/arch/arm/include/asm/bugs.h +++ b/arch/arm/include/asm/bugs.h @@ -1,7 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * arch/arm/include/asm/bugs.h - * * Copyright (C) 1995-2003 Russell King */ #ifndef __ASM_BUGS_H @@ -10,10 +8,8 @@ extern void check_writebuffer_bugs(void); #ifdef CONFIG_MMU -extern void check_bugs(void); extern void check_other_bugs(void); #else -#define check_bugs() do { } while (0) #define check_other_bugs() do { } while (0) #endif diff --git a/arch/arm/include/asm/cache.h b/arch/arm/include/asm/cache.h index e3ea34558ada..ecbc100d22a5 100644 --- a/arch/arm/include/asm/cache.h +++ b/arch/arm/include/asm/cache.h @@ -26,4 +26,10 @@ #define __read_mostly __section(".data..read_mostly") +#ifndef __ASSEMBLY__ +#ifdef CONFIG_ARCH_HAS_CACHE_LINE_SIZE +int cache_line_size(void); +#endif +#endif + #endif diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index a094f964c869..8ed8b9a24efe 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -231,14 +231,15 @@ vivt_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned vma->vm_flags); } -static inline void -vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) +static inline void vivt_flush_cache_pages(struct vm_area_struct *vma, + unsigned long user_addr, unsigned long pfn, unsigned int nr) { struct mm_struct *mm = vma->vm_mm; if (!mm || cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) { unsigned long addr = user_addr & PAGE_MASK; - __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags); + __cpuc_flush_user_range(addr, addr + nr * PAGE_SIZE, + vma->vm_flags); } } @@ -247,15 +248,17 @@ vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig vivt_flush_cache_mm(mm) #define flush_cache_range(vma,start,end) \ vivt_flush_cache_range(vma,start,end) -#define flush_cache_page(vma,addr,pfn) \ - vivt_flush_cache_page(vma,addr,pfn) +#define flush_cache_pages(vma, addr, pfn, nr) \ + vivt_flush_cache_pages(vma, addr, pfn, nr) #else -extern void flush_cache_mm(struct mm_struct *mm); -extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn); +void flush_cache_mm(struct mm_struct *mm); +void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +void flush_cache_pages(struct vm_area_struct *vma, unsigned long user_addr, + unsigned long pfn, unsigned int nr); #endif #define flush_cache_dup_mm(mm) flush_cache_mm(mm) +#define flush_cache_page(vma, addr, pfn) flush_cache_pages(vma, addr, pfn, 1) /* * flush_icache_user_range is used when we want to ensure that the @@ -280,7 +283,7 @@ extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr * flush_dcache_page is used when the kernel has written to the page * cache page at virtual address page->virtual. * - * If this page isn't mapped (ie, page_mapping == NULL), or it might + * If this page isn't mapped (ie, folio_mapping == NULL), or it might * have userspace mappings, then we _must_ always clean + invalidate * the dcache entries associated with the kernel mapping. * @@ -289,7 +292,9 @@ extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr * See update_mmu_cache for the user space part. */ #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *); +void flush_dcache_page(struct page *); +void flush_dcache_folio(struct folio *folio); +#define flush_dcache_folio flush_dcache_folio #define ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 1 static inline void flush_kernel_vmap_range(void *addr, int size) @@ -317,12 +322,6 @@ static inline void flush_anon_page(struct vm_area_struct *vma, #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) /* - * We don't appear to need to do anything here. In fact, if we did, we'd - * duplicate cache flushing elsewhere performed by flush_dcache_page(). - */ -#define flush_icache_page(vma,page) do { } while (0) - -/* * flush_cache_vmap() is used when creating mappings (eg, via vmap, * vmalloc, ioremap etc) in kernel space for pages. On non-VIPT * caches, since the direct-mappings of these pages may contain cached @@ -341,6 +340,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end) dsb(ishst); } +#define flush_cache_vmap_early(start, end) do { } while (0) + static inline void flush_cache_vunmap(unsigned long start, unsigned long end) { if (!cache_is_vipt_nonaliasing()) diff --git a/arch/arm/include/asm/cachetype.h b/arch/arm/include/asm/cachetype.h index e8c30430be33..b01c59076b84 100644 --- a/arch/arm/include/asm/cachetype.h +++ b/arch/arm/include/asm/cachetype.h @@ -20,6 +20,8 @@ extern unsigned int cacheid; #define icache_is_vipt_aliasing() cacheid_is(CACHEID_VIPT_I_ALIASING) #define icache_is_pipt() cacheid_is(CACHEID_PIPT) +#define cpu_dcache_is_aliasing() (cache_is_vivt() || cache_is_vipt_aliasing()) + /* * __LINUX_ARM_ARCH__ is the minimum supported CPU architecture * Mask out support which will never be present on newer CPUs. @@ -81,6 +83,14 @@ static inline unsigned int read_ccsidr(void) asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r" (val)); return val; } + +static inline unsigned int read_clidr(void) +{ + unsigned int val; + + asm volatile("mrc p15, 1, %0, c0, c0, 1" : "=r" (val)); + return val; +} #else /* CONFIG_CPU_V7M */ #include <linux/io.h> #include "asm/v7m.h" @@ -94,6 +104,11 @@ static inline unsigned int read_ccsidr(void) { return readl(BASEADDR_V7M_SCB + V7M_SCB_CCSIDR); } + +static inline unsigned int read_clidr(void) +{ + return readl(BASEADDR_V7M_SCB + V7M_SCB_CLIDR); +} #endif #endif diff --git a/arch/arm/include/asm/checksum.h b/arch/arm/include/asm/checksum.h index f0f54aef3724..d8a13959bff0 100644 --- a/arch/arm/include/asm/checksum.h +++ b/arch/arm/include/asm/checksum.h @@ -11,6 +11,7 @@ #define __ASM_ARM_CHECKSUM_H #include <linux/in6.h> +#include <linux/uaccess.h> /* * computes the checksum of a memory block at buff, length len, diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index 4dfe538dfc68..9beb64d30586 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -5,6 +5,7 @@ #include <linux/irqflags.h> #include <linux/prefetch.h> #include <asm/barrier.h> +#include <linux/cmpxchg-emu.h> #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) /* @@ -25,7 +26,8 @@ #define swp_is_buggy #endif -static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) +static inline unsigned long +__arch_xchg(unsigned long x, volatile void *ptr, int size) { extern void __bad_xchg(volatile void *, int); unsigned long ret; @@ -115,8 +117,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size } #define arch_xchg_relaxed(ptr, x) ({ \ - (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ - sizeof(*(ptr))); \ + (__typeof__(*(ptr)))__arch_xchg((unsigned long)(x), (ptr), \ + sizeof(*(ptr))); \ }) #include <asm-generic/cmpxchg-local.h> @@ -161,7 +163,11 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, prefetchw((const void *)ptr); switch (size) { -#ifndef CONFIG_CPU_V6 /* min ARCH >= ARMv6K */ +#ifdef CONFIG_CPU_V6 /* ARCH == ARMv6 */ + case 1: + oldval = cmpxchg_emu_u8((volatile u8 *)ptr, old, new); + break; +#else /* min ARCH > ARMv6 */ case 1: do { asm volatile("@ __cmpxchg1\n" diff --git a/arch/arm/include/asm/cpu.h b/arch/arm/include/asm/cpu.h index bd6fdb4b922d..9d8863537aa5 100644 --- a/arch/arm/include/asm/cpu.h +++ b/arch/arm/include/asm/cpu.h @@ -11,7 +11,6 @@ #include <linux/cpu.h> struct cpuinfo_arm { - struct cpu cpu; u32 cpuid; #ifdef CONFIG_SMP unsigned int loops_per_jiffy; diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h index 775cac3c02bb..0163c3e78a67 100644 --- a/arch/arm/include/asm/cputype.h +++ b/arch/arm/include/asm/cputype.h @@ -25,6 +25,8 @@ #define CPUID_EXT_ISAR3 0x6c #define CPUID_EXT_ISAR4 0x70 #define CPUID_EXT_ISAR5 0x74 +#define CPUID_EXT_ISAR6 0x7c +#define CPUID_EXT_PFR2 0x90 #else #define CPUID_EXT_PFR0 "c1, 0" #define CPUID_EXT_PFR1 "c1, 1" @@ -40,6 +42,8 @@ #define CPUID_EXT_ISAR3 "c2, 3" #define CPUID_EXT_ISAR4 "c2, 4" #define CPUID_EXT_ISAR5 "c2, 5" +#define CPUID_EXT_ISAR6 "c2, 7" +#define CPUID_EXT_PFR2 "c3, 4" #endif #define MPIDR_SMP_BITMASK (0x3 << 30) diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 1e1178bf176d..5225cb1c803b 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -18,18 +18,12 @@ static __always_inline __attribute_const__ struct task_struct *get_current(void) { struct task_struct *cur; -#if __has_builtin(__builtin_thread_pointer) && \ - defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && \ - !(defined(CONFIG_THUMB2_KERNEL) && \ - defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 130001) +#if __has_builtin(__builtin_thread_pointer) && defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) /* * Use the __builtin helper when available - this results in better * code, especially when using GCC in combination with the per-task * stack protector, as the compiler will recognize that it needs to * load the TLS register only once in every function. - * - * Clang < 13.0.1 gets this wrong for Thumb2 builds: - * https://github.com/ClangBuiltLinux/linux/issues/1485 */ cur = __builtin_thread_pointer(); #elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) diff --git a/arch/arm/include/asm/delay.h b/arch/arm/include/asm/delay.h index 4f80b72372b4..1d069e558d8d 100644 --- a/arch/arm/include/asm/delay.h +++ b/arch/arm/include/asm/delay.h @@ -7,7 +7,7 @@ #ifndef __ASM_ARM_DELAY_H #define __ASM_ARM_DELAY_H -#include <asm/memory.h> +#include <asm/page.h> #include <asm/param.h> /* HZ */ /* diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index be666f58bf7a..c6beb1708c64 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -6,13 +6,9 @@ #define ASMARM_DEVICE_H struct dev_archdata { -#ifdef CONFIG_DMABOUNCE - struct dmabounce_device_info *dmabounce; -#endif #ifdef CONFIG_ARM_DMA_USE_IOMMU struct dma_iommu_mapping *mapping; #endif - unsigned int dma_coherent:1; unsigned int dma_ops_setup:1; }; diff --git a/arch/arm/include/asm/div64.h b/arch/arm/include/asm/div64.h index 4b69cf850451..d3ef8e416b27 100644 --- a/arch/arm/include/asm/div64.h +++ b/arch/arm/include/asm/div64.h @@ -52,10 +52,17 @@ static inline uint32_t __div64_32(uint64_t *n, uint32_t base) #else -static inline uint64_t __arch_xprod_64(uint64_t m, uint64_t n, bool bias) +#ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE +static __always_inline +#else +static inline +#endif +uint64_t __arch_xprod_64(uint64_t m, uint64_t n, bool bias) { unsigned long long res; register unsigned int tmp asm("ip") = 0; + bool no_ovf = __builtin_constant_p(m) && + ((m >> 32) + (m & 0xffffffff) < 0x100000000); if (!bias) { asm ( "umull %Q0, %R0, %Q1, %Q2\n\t" @@ -63,7 +70,7 @@ static inline uint64_t __arch_xprod_64(uint64_t m, uint64_t n, bool bias) : "=&r" (res) : "r" (m), "r" (n) : "cc"); - } else if (!(m & ((1ULL << 63) | (1ULL << 31)))) { + } else if (no_ovf) { res = m; asm ( "umlal %Q0, %R0, %Q1, %Q2\n\t" "mov %Q0, #0" @@ -80,7 +87,7 @@ static inline uint64_t __arch_xprod_64(uint64_t m, uint64_t n, bool bias) : "cc"); } - if (!(m & ((1ULL << 63) | (1ULL << 31)))) { + if (no_ovf) { asm ( "umlal %R0, %Q0, %R1, %Q2\n\t" "umlal %R0, %Q0, %Q1, %R2\n\t" "mov %R0, #0\n\t" diff --git a/arch/arm/include/asm/dma-direct.h b/arch/arm/include/asm/dma-direct.h deleted file mode 100644 index 77fcb7ee5ec9..000000000000 --- a/arch/arm/include/asm/dma-direct.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_ARM_DMA_DIRECT_H -#define ASM_ARM_DMA_DIRECT_H 1 - -#include <asm/memory.h> - -/* - * dma_to_pfn/pfn_to_dma/virt_to_dma are architecture private - * functions used internally by the DMA-mapping API to provide DMA - * addresses. They must not be used by drivers. - */ -static inline dma_addr_t pfn_to_dma(struct device *dev, unsigned long pfn) -{ - if (dev && dev->dma_range_map) - pfn = PFN_DOWN(translate_phys_to_dma(dev, PFN_PHYS(pfn))); - return (dma_addr_t)__pfn_to_bus(pfn); -} - -static inline unsigned long dma_to_pfn(struct device *dev, dma_addr_t addr) -{ - unsigned long pfn = __bus_to_pfn(addr); - - if (dev && dev->dma_range_map) - pfn = PFN_DOWN(translate_dma_to_phys(dev, PFN_PHYS(pfn))); - return pfn; -} - -static inline dma_addr_t virt_to_dma(struct device *dev, void *addr) -{ - if (dev) - return pfn_to_dma(dev, virt_to_pfn(addr)); - - return (dma_addr_t)__virt_to_bus((unsigned long)(addr)); -} - -static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - unsigned int offset = paddr & ~PAGE_MASK; - return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset; -} - -static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) -{ - unsigned int offset = dev_addr & ~PAGE_MASK; - return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset; -} - -#endif /* ASM_ARM_DMA_DIRECT_H */ diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h index fe9ef6f79e9c..2ce4c5683e6d 100644 --- a/arch/arm/include/asm/dma-iommu.h +++ b/arch/arm/include/asm/dma-iommu.h @@ -24,7 +24,7 @@ struct dma_iommu_mapping { }; struct dma_iommu_mapping * -arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size); +arm_iommu_create_mapping(struct device *dev, dma_addr_t base, u64 size); void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping); diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h deleted file mode 100644 index 77082246a5e1..000000000000 --- a/arch/arm/include/asm/dma-mapping.h +++ /dev/null @@ -1,128 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASMARM_DMA_MAPPING_H -#define ASMARM_DMA_MAPPING_H - -#ifdef __KERNEL__ - -#include <linux/mm_types.h> -#include <linux/scatterlist.h> - -#include <xen/xen.h> -#include <asm/xen/hypervisor.h> - -extern const struct dma_map_ops arm_dma_ops; -extern const struct dma_map_ops arm_coherent_dma_ops; - -static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) -{ - if (IS_ENABLED(CONFIG_MMU) && !IS_ENABLED(CONFIG_ARM_LPAE)) - return &arm_dma_ops; - return NULL; -} - -/** - * arm_dma_alloc - allocate consistent memory for DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @size: required memory size - * @handle: bus-specific DMA address - * @attrs: optinal attributes that specific mapping properties - * - * Allocate some memory for a device for performing DMA. This function - * allocates pages, and will return the CPU-viewed address, and sets @handle - * to be the device-viewed address. - */ -extern void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, unsigned long attrs); - -/** - * arm_dma_free - free memory allocated by arm_dma_alloc - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @size: size of memory originally requested in dma_alloc_coherent - * @cpu_addr: CPU-view address returned from dma_alloc_coherent - * @handle: device-view address returned from dma_alloc_coherent - * @attrs: optinal attributes that specific mapping properties - * - * Free (and unmap) a DMA buffer previously allocated by - * arm_dma_alloc(). - * - * References to memory and mappings associated with cpu_addr/handle - * during and after this call executing are illegal. - */ -extern void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs); - -/** - * arm_dma_mmap - map a coherent DMA allocation into user space - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @vma: vm_area_struct describing requested user mapping - * @cpu_addr: kernel CPU-view address returned from dma_alloc_coherent - * @handle: device-view address returned from dma_alloc_coherent - * @size: size of memory originally requested in dma_alloc_coherent - * @attrs: optinal attributes that specific mapping properties - * - * Map a coherent DMA buffer previously allocated by dma_alloc_coherent - * into user space. The coherent DMA buffer must not be freed by the - * driver until the user space mapping has been released. - */ -extern int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs); - -/* - * For SA-1111, IXP425, and ADI systems the dma-mapping functions are "magic" - * and utilize bounce buffers as needed to work around limited DMA windows. - * - * On the SA-1111, a bug limits DMA to only certain regions of RAM. - * On the IXP425, the PCI inbound window is 64MB (256MB total RAM) - * On some ADI engineering systems, PCI inbound window is 32MB (12MB total RAM) - * - * The following are helper functions used by the dmabounce subystem - * - */ - -/** - * dmabounce_register_dev - * - * @dev: valid struct device pointer - * @small_buf_size: size of buffers to use with small buffer pool - * @large_buf_size: size of buffers to use with large buffer pool (can be 0) - * @needs_bounce_fn: called to determine whether buffer needs bouncing - * - * This function should be called by low-level platform code to register - * a device as requireing DMA buffer bouncing. The function will allocate - * appropriate DMA pools for the device. - */ -extern int dmabounce_register_dev(struct device *, unsigned long, - unsigned long, int (*)(struct device *, dma_addr_t, size_t)); - -/** - * dmabounce_unregister_dev - * - * @dev: valid struct device pointer - * - * This function should be called by low-level platform code when device - * that was previously registered with dmabounce_register_dev is removed - * from the system. - * - */ -extern void dmabounce_unregister_dev(struct device *); - - - -/* - * The scatter list versions of the above methods. - */ -extern int arm_dma_map_sg(struct device *, struct scatterlist *, int, - enum dma_data_direction, unsigned long attrs); -extern void arm_dma_unmap_sg(struct device *, struct scatterlist *, int, - enum dma_data_direction, unsigned long attrs); -extern void arm_dma_sync_sg_for_cpu(struct device *, struct scatterlist *, int, - enum dma_data_direction); -extern void arm_dma_sync_sg_for_device(struct device *, struct scatterlist *, int, - enum dma_data_direction); -extern int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs); - -#endif /* __KERNEL__ */ -#endif diff --git a/arch/arm/include/asm/dma.h b/arch/arm/include/asm/dma.h index 45180a2cc47c..e2a1916013e7 100644 --- a/arch/arm/include/asm/dma.h +++ b/arch/arm/include/asm/dma.h @@ -12,6 +12,9 @@ extern phys_addr_t arm_dma_zone_size; \ arm_dma_zone_size && arm_dma_zone_size < (0x100000000ULL - PAGE_OFFSET) ? \ (PAGE_OFFSET + arm_dma_zone_size) : 0xffffffffUL; }) + +extern phys_addr_t arm_dma_limit; +#define ARCH_LOW_ADDRESS_LIMIT arm_dma_limit #endif #ifdef CONFIG_ISA_DMA_API @@ -106,7 +109,7 @@ extern void set_dma_sg(unsigned int chan, struct scatterlist *sg, int nr_sg); */ extern void __set_dma_addr(unsigned int chan, void *addr); #define set_dma_addr(chan, addr) \ - __set_dma_addr(chan, (void *)__bus_to_virt(addr)) + __set_dma_addr(chan, (void *)isa_bus_to_virt(addr)) /* Set the DMA byte count for this channel * @@ -143,10 +146,4 @@ extern int get_dma_residue(unsigned int chan); #endif /* CONFIG_ISA_DMA_API */ -#ifdef CONFIG_PCI -extern int isa_dma_bridge_buggy; -#else -#define isa_dma_bridge_buggy (0) -#endif - #endif /* __ASM_ARM_DMA_H */ diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 41536feb4392..d48859fdf32c 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -8,8 +8,8 @@ #define __ASM_PROC_DOMAIN_H #ifndef __ASSEMBLY__ +#include <linux/thread_info.h> #include <asm/barrier.h> -#include <asm/thread_info.h> #endif /* diff --git a/arch/arm/include/asm/ecard.h b/arch/arm/include/asm/ecard.h index 4befe8d2ae19..7cbe001bf9cc 100644 --- a/arch/arm/include/asm/ecard.h +++ b/arch/arm/include/asm/ecard.h @@ -195,7 +195,7 @@ void __iomem *ecardm_iomap(struct expansion_card *ec, unsigned int res, unsigned long offset, unsigned long maxsize); #define ecardm_iounmap(__ec, __addr) devm_iounmap(&(__ec)->dev, __addr) -extern struct bus_type ecard_bus_type; +extern const struct bus_type ecard_bus_type; #define ECARD_DEV(_d) container_of((_d), struct expansion_card, dev) diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index 3088ef72704e..e408399d5f0e 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -14,16 +14,30 @@ #include <asm/mach/map.h> #include <asm/mmu_context.h> #include <asm/ptrace.h> +#include <asm/uaccess.h> #ifdef CONFIG_EFI void efi_init(void); +void arm_efi_init(void); int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); -int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); +int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md, bool); #define arch_efi_call_virt_setup() efi_virtmap_load() #define arch_efi_call_virt_teardown() efi_virtmap_unload() +#ifdef CONFIG_CPU_TTBR0_PAN +#undef arch_efi_call_virt +#define arch_efi_call_virt(p, f, args...) ({ \ + unsigned int flags = uaccess_save_and_enable(); \ + efi_status_t res = _Generic((p)->f(args), \ + efi_status_t: (p)->f(args), \ + default: ((p)->f(args), EFI_ABORTED)); \ + uaccess_restore(flags); \ + res; \ +}) +#endif + #define ARCH_EFI_IRQ_FLAGS_MASK \ (PSR_J_BIT | PSR_E_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | \ PSR_T_BIT | MODE_MASK) @@ -37,14 +51,11 @@ void efi_virtmap_load(void); void efi_virtmap_unload(void); #else -#define efi_init() +#define arm_efi_init() #endif /* CONFIG_EFI */ /* arch specific definitions used by the stub code */ -struct screen_info *alloc_screen_info(void); -void free_screen_info(struct screen_info *si); - /* * A reasonable upper bound for the uncompressed kernel size is 32 MBytes, * so we will reserve that amount of memory. We have no easy way to tell what diff --git a/arch/arm/include/asm/elf.h b/arch/arm/include/asm/elf.h index d68101655b74..9f21e170320f 100644 --- a/arch/arm/include/asm/elf.h +++ b/arch/arm/include/asm/elf.h @@ -4,7 +4,6 @@ #include <asm/auxvec.h> #include <asm/hwcap.h> -#include <asm/vdso_datapage.h> /* * ELF register definitions.. diff --git a/arch/arm/include/asm/exception.h b/arch/arm/include/asm/exception.h index 58e039a851af..3c82975d46db 100644 --- a/arch/arm/include/asm/exception.h +++ b/arch/arm/include/asm/exception.h @@ -10,10 +10,6 @@ #include <linux/interrupt.h> -#ifdef CONFIG_FUNCTION_GRAPH_TRACER #define __exception_irq_entry __irq_entry -#else -#define __exception_irq_entry -#endif #endif /* __ASM_ARM_EXCEPTION_H */ diff --git a/arch/arm/include/asm/fb.h b/arch/arm/include/asm/fb.h deleted file mode 100644 index d92e99cd8c8a..000000000000 --- a/arch/arm/include/asm/fb.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef _ASM_FB_H_ -#define _ASM_FB_H_ - -#include <linux/fb.h> -#include <linux/fs.h> -#include <asm/page.h> - -static inline void fb_pgprotect(struct file *file, struct vm_area_struct *vma, - unsigned long off) -{ - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); -} - -static inline int fb_is_primary_device(struct fb_info *info) -{ - return 0; -} - -#endif /* _ASM_FB_H_ */ diff --git a/arch/arm/include/asm/fpstate.h b/arch/arm/include/asm/fpstate.h index ca42fd9ae0b3..e29d9c7a5238 100644 --- a/arch/arm/include/asm/fpstate.h +++ b/arch/arm/include/asm/fpstate.h @@ -46,9 +46,6 @@ union vfp_state { struct vfp_hard_struct hard; }; -extern void vfp_flush_thread(union vfp_state *); -extern void vfp_release_thread(union vfp_state *); - #define FP_HARD_SIZE 35 struct fp_hard_struct { diff --git a/arch/arm/include/asm/fpu.h b/arch/arm/include/asm/fpu.h new file mode 100644 index 000000000000..2ae50bdce59b --- /dev/null +++ b/arch/arm/include/asm/fpu.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023 SiFive + */ + +#ifndef __ASM_FPU_H +#define __ASM_FPU_H + +#include <asm/neon.h> + +#define kernel_fpu_available() cpu_has_neon() +#define kernel_fpu_begin() kernel_neon_begin() +#define kernel_fpu_end() kernel_neon_end() + +#endif /* ! __ASM_FPU_H */ diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index 7e9251ca29fe..5be3ddc96a50 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -75,6 +75,10 @@ static inline bool arch_syscall_match_sym_name(const char *sym, return !strcasecmp(sym, name); } +void prepare_ftrace_return(unsigned long *parent, unsigned long self, + unsigned long frame_pointer, + unsigned long stack_pointer); + #endif /* ifndef __ASSEMBLY__ */ #endif /* _ASM_ARM_FTRACE */ diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h index 724f8dac1e5b..4186fbf7341f 100644 --- a/arch/arm/include/asm/glue-cache.h +++ b/arch/arm/include/asm/glue-cache.h @@ -118,6 +118,10 @@ # define MULTI_CACHE 1 #endif +#ifdef CONFIG_CPU_CACHE_NOP +# define MULTI_CACHE 1 +#endif + #if defined(CONFIG_CPU_V7M) # define MULTI_CACHE 1 #endif @@ -126,29 +130,15 @@ #error Unknown cache maintenance model #endif -#ifndef __ASSEMBLER__ -static inline void nop_flush_icache_all(void) { } -static inline void nop_flush_kern_cache_all(void) { } -static inline void nop_flush_kern_cache_louis(void) { } -static inline void nop_flush_user_cache_all(void) { } -static inline void nop_flush_user_cache_range(unsigned long a, - unsigned long b, unsigned int c) { } - -static inline void nop_coherent_kern_range(unsigned long a, unsigned long b) { } -static inline int nop_coherent_user_range(unsigned long a, - unsigned long b) { return 0; } -static inline void nop_flush_kern_dcache_area(void *a, size_t s) { } - -static inline void nop_dma_flush_range(const void *a, const void *b) { } - -static inline void nop_dma_map_area(const void *s, size_t l, int f) { } -static inline void nop_dma_unmap_area(const void *s, size_t l, int f) { } -#endif - #ifndef MULTI_CACHE #define __cpuc_flush_icache_all __glue(_CACHE,_flush_icache_all) #define __cpuc_flush_kern_all __glue(_CACHE,_flush_kern_cache_all) +/* This function only has a dedicated assembly callback on the v7 cache */ +#ifdef CONFIG_CPU_CACHE_V7 #define __cpuc_flush_kern_louis __glue(_CACHE,_flush_kern_cache_louis) +#else +#define __cpuc_flush_kern_louis __glue(_CACHE,_flush_kern_cache_all) +#endif #define __cpuc_flush_user_all __glue(_CACHE,_flush_user_cache_all) #define __cpuc_flush_user_range __glue(_CACHE,_flush_user_cache_range) #define __cpuc_coherent_kern_range __glue(_CACHE,_coherent_kern_range) diff --git a/arch/arm/include/asm/gpio.h b/arch/arm/include/asm/gpio.h deleted file mode 100644 index f3bb8a2bf788..000000000000 --- a/arch/arm/include/asm/gpio.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ARCH_ARM_GPIO_H -#define _ARCH_ARM_GPIO_H - -/* Note: this may rely upon the value of ARCH_NR_GPIOS set in mach/gpio.h */ -#include <asm-generic/gpio.h> - -/* The trivial gpiolib dispatchers */ -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep - -/* - * Provide a default gpio_to_irq() which should satisfy every case. - * However, some platforms want to do this differently, so allow them - * to override it. - */ -#ifndef gpio_to_irq -#define gpio_to_irq __gpio_to_irq -#endif - -#endif /* _ARCH_ARM_GPIO_H */ diff --git a/arch/arm/include/asm/hardware/cache-aurora-l2.h b/arch/arm/include/asm/hardware/cache-aurora-l2.h index 39769ffa0051..9694808ee97c 100644 --- a/arch/arm/include/asm/hardware/cache-aurora-l2.h +++ b/arch/arm/include/asm/hardware/cache-aurora-l2.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * AURORA shared L2 cache controller support * @@ -5,10 +6,6 @@ * * Yehuda Yitschak <yehuday@marvell.com> * Gregory CLEMENT <gregory.clement@free-electrons.com> - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #ifndef __ASM_ARM_HARDWARE_AURORA_L2_H diff --git a/arch/arm/include/asm/hardware/cache-feroceon-l2.h b/arch/arm/include/asm/hardware/cache-feroceon-l2.h index 12e1588dc4f1..eb2e7b7f70a8 100644 --- a/arch/arm/include/asm/hardware/cache-feroceon-l2.h +++ b/arch/arm/include/asm/hardware/cache-feroceon-l2.h @@ -1,13 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * arch/arm/include/asm/hardware/cache-feroceon-l2.h * * Copyright (C) 2008 Marvell Semiconductor - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ extern void __init feroceon_l2_init(int l2_wt_override); extern int __init feroceon_of_init(void); - diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h index a6d4ee86ba54..5a7ee70f561c 100644 --- a/arch/arm/include/asm/hardware/cache-l2x0.h +++ b/arch/arm/include/asm/hardware/cache-l2x0.h @@ -9,6 +9,8 @@ #define __ASM_ARM_HARDWARE_L2X0_H #include <linux/errno.h> +#include <linux/init.h> +#include <linux/types.h> #define L2X0_CACHE_ID 0x000 #define L2X0_CACHE_TYPE 0x004 diff --git a/arch/arm/include/asm/hardware/cache-tauros2.h b/arch/arm/include/asm/hardware/cache-tauros2.h index 295e2e40151b..4e493facaa31 100644 --- a/arch/arm/include/asm/hardware/cache-tauros2.h +++ b/arch/arm/include/asm/hardware/cache-tauros2.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * arch/arm/include/asm/hardware/cache-tauros2.h * * Copyright (C) 2008 Marvell Semiconductor - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #define CACHE_TAUROS2_PREFETCH_ON (1 << 0) diff --git a/arch/arm/include/asm/hardware/dec21285.h b/arch/arm/include/asm/hardware/dec21285.h index 3f18a56a025d..894f2a635cbb 100644 --- a/arch/arm/include/asm/hardware/dec21285.h +++ b/arch/arm/include/asm/hardware/dec21285.h @@ -22,6 +22,13 @@ #define DC21285_IO(x) (x) #endif +/* + * The footbridge is programmed to expose the system RAM at 0xe0000000. + * The requirement is that the RAM isn't placed at bus address 0, which + * would clash with VGA cards. + */ +#define BUS_OFFSET 0xe0000000 + #define CSR_PCICMD DC21285_IO(0x0004) #define CSR_CLASSREV DC21285_IO(0x0008) #define CSR_PCICACHELINESIZE DC21285_IO(0x000c) @@ -81,19 +88,6 @@ #define SA110_CNTL_XCSDIR(x) ((x)<<28) #define SA110_CNTL_PCICFN (1 << 31) -/* - * footbridge_cfn_mode() is used when we want - * to check whether we are the central function - */ -#define __footbridge_cfn_mode() (*CSR_SA110_CNTL & SA110_CNTL_PCICFN) -#if defined(CONFIG_FOOTBRIDGE_HOST) && defined(CONFIG_FOOTBRIDGE_ADDIN) -#define footbridge_cfn_mode() __footbridge_cfn_mode() -#elif defined(CONFIG_FOOTBRIDGE_HOST) -#define footbridge_cfn_mode() (1) -#else -#define footbridge_cfn_mode() (0) -#endif - #define CSR_PCIADDR_EXTN DC21285_IO(0x0140) #define CSR_PREFETCHMEMRANGE DC21285_IO(0x0144) #define CSR_XBUS_CYCLE DC21285_IO(0x0148) diff --git a/arch/arm/include/asm/hardware/locomo.h b/arch/arm/include/asm/hardware/locomo.h index 246a3de25931..3190e1e5067a 100644 --- a/arch/arm/include/asm/hardware/locomo.h +++ b/arch/arm/include/asm/hardware/locomo.h @@ -158,8 +158,6 @@ #define LOCOMO_LPT_TOH(TOH) ((TOH & 0x7) << 4) #define LOCOMO_LPT_TOL(TOL) ((TOL & 0x7)) -extern struct bus_type locomo_bus_type; - #define LOCOMO_DEVID_KEYBOARD 0 #define LOCOMO_DEVID_FRONTLIGHT 1 #define LOCOMO_DEVID_BACKLIGHT 2 @@ -191,11 +189,11 @@ struct locomo_driver { void (*remove)(struct locomo_dev *); }; -#define LOCOMO_DRV(_d) container_of((_d), struct locomo_driver, drv) +#define LOCOMO_DRV(_d) container_of_const((_d), struct locomo_driver, drv) #define LOCOMO_DRIVER_NAME(_ldev) ((_ldev)->dev.driver->name) -void locomo_lcd_power(struct locomo_dev *, int, unsigned int); +extern void locomolcd_power(int on); int locomo_driver_register(struct locomo_driver *); void locomo_driver_unregister(struct locomo_driver *); diff --git a/arch/arm/include/asm/hardware/sa1111.h b/arch/arm/include/asm/hardware/sa1111.h index d8c6f8a99dfa..a815f39b4243 100644 --- a/arch/arm/include/asm/hardware/sa1111.h +++ b/arch/arm/include/asm/hardware/sa1111.h @@ -404,7 +404,7 @@ struct sa1111_driver { void (*remove)(struct sa1111_dev *); }; -#define SA1111_DRV(_d) container_of((_d), struct sa1111_driver, drv) +#define SA1111_DRV(_d) container_of_const((_d), struct sa1111_driver, drv) #define SA1111_DRIVER_NAME(_sadev) ((_sadev)->dev.driver->name) diff --git a/arch/arm/include/asm/hugetlb-3level.h b/arch/arm/include/asm/hugetlb-3level.h index a30be5505793..87d48e2d90ad 100644 --- a/arch/arm/include/asm/hugetlb-3level.h +++ b/arch/arm/include/asm/hugetlb-3level.h @@ -13,12 +13,12 @@ /* * If our huge pte is non-zero then mark the valid bit. - * This allows pte_present(huge_ptep_get(ptep)) to return true for non-zero + * This allows pte_present(huge_ptep_get(mm,addr,ptep)) to return true for non-zero * ptes. * (The valid bit is automatically cleared by set_pte_at for PROT_NONE ptes). */ #define __HAVE_ARCH_HUGE_PTEP_GET -static inline pte_t huge_ptep_get(pte_t *ptep) +static inline pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_t retval = *ptep; if (pte_val(retval)) diff --git a/arch/arm/include/asm/hugetlb.h b/arch/arm/include/asm/hugetlb.h index d02d6ca88e92..b766c4b373f6 100644 --- a/arch/arm/include/asm/hugetlb.h +++ b/arch/arm/include/asm/hugetlb.h @@ -10,14 +10,15 @@ #ifndef _ASM_ARM_HUGETLB_H #define _ASM_ARM_HUGETLB_H +#include <asm/cacheflush.h> #include <asm/page.h> #include <asm/hugetlb-3level.h> #include <asm-generic/hugetlb.h> -static inline void arch_clear_hugepage_flags(struct page *page) +static inline void arch_clear_hugetlb_flags(struct folio *folio) { - clear_bit(PG_dcache_clean, &page->flags); + clear_bit(PG_dcache_clean, &folio->flags); } -#define arch_clear_hugepage_flags arch_clear_hugepage_flags +#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags #endif /* _ASM_ARM_HUGETLB_H */ diff --git a/arch/arm/include/asm/hw_breakpoint.h b/arch/arm/include/asm/hw_breakpoint.h index 62358d3ca0a8..e7f9961c53b2 100644 --- a/arch/arm/include/asm/hw_breakpoint.h +++ b/arch/arm/include/asm/hw_breakpoint.h @@ -84,6 +84,7 @@ static inline void decode_ctrl_reg(u32 reg, #define ARM_DSCR_MOE(x) ((x >> 2) & 0xf) #define ARM_ENTRY_BREAKPOINT 0x1 #define ARM_ENTRY_ASYNC_WATCHPOINT 0x2 +#define ARM_ENTRY_CFI_BREAKPOINT 0x3 #define ARM_ENTRY_SYNC_WATCHPOINT 0xa /* DSCR monitor/halting bits. */ diff --git a/arch/arm/include/asm/hypervisor.h b/arch/arm/include/asm/hypervisor.h index bd61502b9715..8a648e506540 100644 --- a/arch/arm/include/asm/hypervisor.h +++ b/arch/arm/include/asm/hypervisor.h @@ -7,4 +7,6 @@ void kvm_init_hyp_services(void); bool kvm_arm_hyp_service_available(u32 func_id); +static inline void kvm_arch_init_hyp_services(void) { }; + #endif diff --git a/arch/arm/include/asm/ide.h b/arch/arm/include/asm/ide.h deleted file mode 100644 index a81e0b0d6747..000000000000 --- a/arch/arm/include/asm/ide.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * arch/arm/include/asm/ide.h - * - * Copyright (C) 1994-1996 Linus Torvalds & authors - */ - -/* - * This file contains the ARM architecture specific IDE code. - */ - -#ifndef __ASMARM_IDE_H -#define __ASMARM_IDE_H - -#ifdef __KERNEL__ - -#define __ide_mm_insw(port,addr,len) readsw(port,addr,len) -#define __ide_mm_insl(port,addr,len) readsl(port,addr,len) -#define __ide_mm_outsw(port,addr,len) writesw(port,addr,len) -#define __ide_mm_outsl(port,addr,len) writesl(port,addr,len) - -#endif /* __KERNEL__ */ - -#endif /* __ASMARM_IDE_H */ diff --git a/arch/arm/include/asm/io.h b/arch/arm/include/asm/io.h index 7fcdc785366c..bae5edf348ef 100644 --- a/arch/arm/include/asm/io.h +++ b/arch/arm/include/asm/io.h @@ -23,7 +23,7 @@ #include <linux/string.h> #include <linux/types.h> #include <asm/byteorder.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm-generic/pci_iomap.h> /* @@ -381,7 +381,7 @@ void __iomem *ioremap_wc(resource_size_t res_cookie, size_t size); void iounmap(volatile void __iomem *io_addr); #define iounmap iounmap -void *arch_memremap_wb(phys_addr_t phys_addr, size_t size); +void *arch_memremap_wb(phys_addr_t phys_addr, size_t size, unsigned long flags); #define arch_memremap_wb arch_memremap_wb /* @@ -407,12 +407,6 @@ struct pci_dev; #define pci_iounmap pci_iounmap extern void pci_iounmap(struct pci_dev *dev, void __iomem *addr); -/* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - #include <asm-generic/io.h> #ifdef CONFIG_MMU diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index a7c2337b0c7d..26c1d2ced4ce 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -27,13 +27,12 @@ struct irqaction; struct pt_regs; void handle_IRQ(unsigned int, struct pt_regs *); -void init_IRQ(void); #ifdef CONFIG_SMP #include <linux/cpumask.h> extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, - bool exclude_self); + int exclude_cpu); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace #endif diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index e12d7d096fc0..a35aba7f548c 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -9,13 +9,17 @@ #define JUMP_LABEL_NOP_SIZE 4 +/* This macro is also expanded on the Rust side. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "1:\n\t" \ + WASM(nop) "\n\t" \ + ".pushsection __jump_table, \"aw\"\n\t" \ + ".word 1b, " label ", " key "\n\t" \ + ".popsection\n\t" \ + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm_volatile_goto("1:\n\t" - WASM(nop) "\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection\n\t" + asm goto(ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]") : : "i" (&((char *)key)[branch]) : : l_yes); return false; @@ -25,7 +29,7 @@ l_yes: static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { - asm_volatile_goto("1:\n\t" + asm goto("1:\n\t" WASM(b) " %l[l_yes]\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".word 1b, %l[l_yes], %c0\n\t" diff --git a/arch/arm/include/asm/kexec.h b/arch/arm/include/asm/kexec.h index e62832dcba76..a8287e7ab9d4 100644 --- a/arch/arm/include/asm/kexec.h +++ b/arch/arm/include/asm/kexec.h @@ -2,8 +2,6 @@ #ifndef _ARM_KEXEC_H #define _ARM_KEXEC_H -#ifdef CONFIG_KEXEC - /* Maximum physical address we can use pages from */ #define KEXEC_SOURCE_MEMORY_LIMIT (-1UL) /* Maximum address we can reach in physical address mode */ @@ -82,6 +80,4 @@ static inline struct page *boot_pfn_to_page(unsigned long boot_pfn) #endif /* __ASSEMBLY__ */ -#endif /* CONFIG_KEXEC */ - #endif /* _ARM_KEXEC_H */ diff --git a/arch/arm/include/asm/kprobes.h b/arch/arm/include/asm/kprobes.h index e26a278d301a..5b8dbf1b0be4 100644 --- a/arch/arm/include/asm/kprobes.h +++ b/arch/arm/include/asm/kprobes.h @@ -40,8 +40,6 @@ struct kprobe_ctlblk { void arch_remove_kprobe(struct kprobe *); int kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr); -int kprobe_exceptions_notify(struct notifier_block *self, - unsigned long val, void *data); /* optinsn template addresses */ extern __visible kprobe_opcode_t optprobe_template_entry[]; diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 9349e7a82c9c..2b18a258204d 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -56,7 +56,6 @@ struct machine_desc { void (*init_time)(void); void (*init_machine)(void); void (*init_late)(void); - void (*handle_irq)(struct pt_regs *); void (*restart)(enum reboot_mode, const char *); }; diff --git a/arch/arm/include/asm/mach/dma.h b/arch/arm/include/asm/mach/dma.h index 1506422af383..5ec11d7f0d04 100644 --- a/arch/arm/include/asm/mach/dma.h +++ b/arch/arm/include/asm/mach/dma.h @@ -44,8 +44,3 @@ struct dma_struct { * isa_dma_add - add an ISA-style DMA channel */ extern int isa_dma_add(unsigned int, dma_t *dma); - -/* - * Add the ISA DMA controller. Always takes channels 0-7. - */ -extern void isa_init_dma(void); diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index f673e13e0f94..7c2fa7dcec6d 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -5,11 +5,16 @@ * Copyright (C) 2000-2002 Russell King * modification for nommu, Hyok S. Choi, 2004 * - * Note: this file should not be included by non-asm/.h files + * Note: this file should not be included explicitly, include <asm/page.h> + * to get access to these definitions. */ #ifndef __ASM_ARM_MEMORY_H #define __ASM_ARM_MEMORY_H +#ifndef _ASMARM_PAGE_H +#error "Do not include <asm/memory.h> directly" +#endif + #include <linux/compiler.h> #include <linux/const.h> #include <linux/types.h> @@ -143,12 +148,6 @@ extern unsigned long vectors_base; #endif /* - * Convert a page to/from a physical address - */ -#define page_to_phys(page) (__pfn_to_phys(page_to_pfn(page))) -#define phys_to_page(phys) (pfn_to_page(__phys_to_pfn(phys))) - -/* * PLAT_PHYS_OFFSET is the offset (from zero) of the start of physical * memory. This is used for XIP and NoMMU kernels, and on platforms that don't * have CONFIG_ARM_PATCH_PHYS_VIRT. Assembly code must always use @@ -288,10 +287,12 @@ static inline unsigned long __phys_to_virt(phys_addr_t x) #endif -#define virt_to_pfn(kaddr) \ - ((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \ - PHYS_PFN_OFFSET) - +static inline unsigned long virt_to_pfn(const void *p) +{ + unsigned long kaddr = (unsigned long)p; + return (((kaddr - PAGE_OFFSET) >> PAGE_SHIFT) + + PHYS_PFN_OFFSET); +} #define __pa_symbol_nodebug(x) __virt_to_phys_nodebug((x)) #ifdef CONFIG_DEBUG_VIRTUAL @@ -370,19 +371,6 @@ static inline unsigned long __virt_to_idmap(unsigned long x) #define virt_to_idmap(x) __virt_to_idmap((unsigned long)(x)) /* - * Virtual <-> DMA view memory address translations - * Again, these are *only* valid on the kernel direct mapped RAM - * memory. Use of these is *deprecated* (and that doesn't mean - * use the __ prefixed forms instead.) See dma-mapping.h. - */ -#ifndef __virt_to_bus -#define __virt_to_bus __virt_to_phys -#define __bus_to_virt __phys_to_virt -#define __pfn_to_bus(x) __pfn_to_phys(x) -#define __bus_to_pfn(x) __phys_to_pfn(x) -#endif - -/* * Conversion between a struct page and a physical address. * * page_to_pfn(page) convert a struct page * to a PFN number @@ -399,6 +387,4 @@ static inline unsigned long __virt_to_idmap(unsigned long x) #endif -#include <asm-generic/memory_model.h> - #endif diff --git a/arch/arm/include/asm/mman.h b/arch/arm/include/asm/mman.h new file mode 100644 index 000000000000..2189e507c8e0 --- /dev/null +++ b/arch/arm/include/asm/mman.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMAN_H__ +#define __ASM_MMAN_H__ + +#include <asm/system_info.h> +#include <uapi/asm/mman.h> + +static inline bool arch_memory_deny_write_exec_supported(void) +{ + return cpu_architecture() >= CPU_ARCH_ARMv6; +} +#define arch_memory_deny_write_exec_supported arch_memory_deny_write_exec_supported + +#endif /* __ASM_MMAN_H__ */ diff --git a/arch/arm/include/asm/module.h b/arch/arm/include/asm/module.h index 5546c9751478..07c51a34f77d 100644 --- a/arch/arm/include/asm/module.h +++ b/arch/arm/include/asm/module.h @@ -37,6 +37,11 @@ struct mod_arch_specific { struct module; u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val); +#ifdef CONFIG_ARM_MODULE_PLTS +bool in_module_plt(unsigned long loc); +#else +static inline bool in_module_plt(unsigned long loc) { return false; } +#endif #ifdef CONFIG_THUMB2_KERNEL #define HAVE_ARCH_KALLSYMS_SYMBOL_VALUE diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index 5fcc8a600e36..ef11b721230e 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -7,10 +7,7 @@ #ifndef _ASMARM_PAGE_H #define _ASMARM_PAGE_H -/* PAGE_SHIFT determines the page size */ -#define PAGE_SHIFT 12 -#define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) -#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1)) +#include <vdso/page.h> #ifndef __ASSEMBLY__ @@ -113,6 +110,28 @@ struct cpu_user_fns { unsigned long vaddr, struct vm_area_struct *vma); }; +void fa_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void fa_clear_user_highpage(struct page *page, unsigned long vaddr); +void feroceon_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void feroceon_clear_user_highpage(struct page *page, unsigned long vaddr); +void v4_mc_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void v4_mc_clear_user_highpage(struct page *page, unsigned long vaddr); +void v4wb_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void v4wb_clear_user_highpage(struct page *page, unsigned long vaddr); +void v4wt_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void v4wt_clear_user_highpage(struct page *page, unsigned long vaddr); +void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void xsc3_mc_clear_user_highpage(struct page *page, unsigned long vaddr); +void xscale_mc_copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +void xscale_mc_clear_user_highpage(struct page *page, unsigned long vaddr); + #ifdef MULTI_USER extern struct cpu_user_fns cpu_user; @@ -158,14 +177,16 @@ typedef struct page *pgtable_t; #ifdef CONFIG_HAVE_ARCH_PFN_VALID extern int pfn_valid(unsigned long); +#define pfn_valid pfn_valid #endif -#include <asm/memory.h> - #endif /* !__ASSEMBLY__ */ +#include <asm/memory.h> + #define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_TSK_EXEC #include <asm-generic/getorder.h> +#include <asm-generic/memory_model.h> #endif diff --git a/arch/arm/include/asm/pci.h b/arch/arm/include/asm/pci.h index 68e6f25784a4..5916b88d4c94 100644 --- a/arch/arm/include/asm/pci.h +++ b/arch/arm/include/asm/pci.h @@ -22,11 +22,6 @@ static inline int pci_proc_domain(struct pci_bus *bus) #define HAVE_PCI_MMAP #define ARCH_GENERIC_PCI_MMAP_RESOURCE -static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) -{ - return channel ? 15 : 14; -} - extern void pcibios_report_status(unsigned int status_mask, int warn); #endif /* __KERNEL__ */ diff --git a/arch/arm/include/asm/perf_event.h b/arch/arm/include/asm/perf_event.h index fe87397c3d8c..c08f16f2e243 100644 --- a/arch/arm/include/asm/perf_event.h +++ b/arch/arm/include/asm/perf_event.h @@ -8,16 +8,9 @@ #ifndef __ARM_PERF_EVENT_H__ #define __ARM_PERF_EVENT_H__ -#ifdef CONFIG_PERF_EVENTS -struct pt_regs; -extern unsigned long perf_instruction_pointer(struct pt_regs *regs); -extern unsigned long perf_misc_flags(struct pt_regs *regs); -#define perf_misc_flags(regs) perf_misc_flags(regs) -#endif - #define perf_arch_fetch_caller_regs(regs, __ip) { \ (regs)->ARM_pc = (__ip); \ - (regs)->ARM_fp = (unsigned long) __builtin_frame_address(0); \ + frame_pointer((regs)) = (unsigned long) __builtin_frame_address(0); \ (regs)->ARM_sp = current_stack_pointer; \ (regs)->ARM_cpsr = SVC_MODE; \ } diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index 92abd4cd8ca2..6b5392e20f41 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -126,6 +126,9 @@ #define L_PTE_SHARED (_AT(pteval_t, 1) << 10) /* shared(v6), coherent(xsc3) */ #define L_PTE_NONE (_AT(pteval_t, 1) << 11) +/* We borrow bit 7 to store the exclusive marker in swap PTEs. */ +#define L_PTE_SWP_EXCLUSIVE L_PTE_RDONLY + /* * These are the memory types, defined to be compatible with * pre-ARMv6 CPUs cacheable and bufferable bits: n/a,n/a,C,B @@ -210,9 +213,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) #define pmd_pfn(pmd) (__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) -#define pmd_large(pmd) (pmd_val(pmd) & 2) -#define pmd_leaf(pmd) (pmd_val(pmd) & 2) -#define pmd_bad(pmd) (pmd_val(pmd) & 2) +#define pmd_leaf(pmd) (pmd_val(pmd) & PMD_TYPE_SECT) +#define pmd_bad(pmd) pmd_leaf(pmd) #define pmd_present(pmd) (pmd_val(pmd)) #define copy_pmd(pmdpd,pmdps) \ @@ -239,7 +241,6 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) * define empty stubs for use by pin_page_for_write. */ #define pmd_hugewillfault(pmd) (0) -#define pmd_thp_or_huge(pmd) (0) #endif /* __ASSEMBLY__ */ diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h index 2f35b4eddaa8..944fc9955528 100644 --- a/arch/arm/include/asm/pgtable-3level-hwdef.h +++ b/arch/arm/include/asm/pgtable-3level-hwdef.h @@ -14,6 +14,7 @@ * + Level 1/2 descriptor * - common */ +#define PUD_TABLE_BIT (_AT(pmdval_t, 1) << 1) #define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0) #define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0) #define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0) @@ -74,6 +75,7 @@ #define PHYS_MASK_SHIFT (40) #define PHYS_MASK ((1ULL << PHYS_MASK_SHIFT) - 1) +#ifndef CONFIG_CPU_TTBR0_PAN /* * TTBR0/TTBR1 split (PAGE_OFFSET): * 0x40000000: T0SZ = 2, T1SZ = 0 (not used) @@ -93,5 +95,35 @@ #endif #define TTBR1_SIZE (((PAGE_OFFSET >> 30) - 1) << 16) +#else +/* + * With CONFIG_CPU_TTBR0_PAN enabled, TTBR1 is only used during uaccess + * disabled regions when TTBR0 is disabled. + */ +#define TTBR1_OFFSET 0 /* pointing to swapper_pg_dir */ +#define TTBR1_SIZE 0 /* TTBR1 size controlled via TTBCR.T0SZ */ +#endif + +/* + * TTBCR register bits. + * + * The ORGN0 and IRGN0 bits enables different forms of caching when + * walking the translation table. Clearing these bits (which is claimed + * to be the reset default) means "normal memory, [outer|inner] + * non-cacheable" + */ +#define TTBCR_EAE (1 << 31) +#define TTBCR_IMP (1 << 30) +#define TTBCR_SH1_MASK (3 << 28) +#define TTBCR_ORGN1_MASK (3 << 26) +#define TTBCR_IRGN1_MASK (3 << 24) +#define TTBCR_EPD1 (1 << 23) +#define TTBCR_A1 (1 << 22) +#define TTBCR_T1SZ_MASK (7 << 16) +#define TTBCR_SH0_MASK (3 << 12) +#define TTBCR_ORGN0_MASK (3 << 10) +#define TTBCR_IRGN0_MASK (3 << 8) +#define TTBCR_EPD0 (1 << 7) +#define TTBCR_T0SZ_MASK (7 << 0) #endif diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index eabe72ff7381..7b71a3d414b7 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -76,6 +76,9 @@ #define L_PTE_NONE (_AT(pteval_t, 1) << 57) /* PROT_NONE */ #define L_PTE_RDONLY (_AT(pteval_t, 1) << 58) /* READ ONLY */ +/* We borrow bit 7 to store the exclusive marker in swap PTEs. */ +#define L_PTE_SWP_EXCLUSIVE (_AT(pteval_t, 1) << 7) + #define L_PMD_SECT_VALID (_AT(pmdval_t, 1) << 0) #define L_PMD_SECT_DIRTY (_AT(pmdval_t, 1) << 55) #define L_PMD_SECT_NONE (_AT(pmdval_t, 1) << 57) @@ -109,13 +112,12 @@ #ifndef __ASSEMBLY__ #define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (!(pud_val(pud) & 2)) +#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT)) #define pud_present(pud) (pud_val(pud)) #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ PMD_TYPE_TABLE) #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ PMD_TYPE_SECT) -#define pmd_large(pmd) pmd_sect(pmd) #define pmd_leaf(pmd) pmd_sect(pmd) #define pud_clear(pudp) \ @@ -135,7 +137,7 @@ static inline pmd_t *pud_pgtable(pud_t pud) return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); } -#define pmd_bad(pmd) (!(pmd_val(pmd) & 2)) +#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT)) #define copy_pmd(pmdpd,pmdps) \ do { \ @@ -188,7 +190,6 @@ static inline pte_t pte_mkspecial(pte_t pte) #define pmd_dirty(pmd) (pmd_isset((pmd), L_PMD_SECT_DIRTY)) #define pmd_hugewillfault(pmd) (!pmd_young(pmd) || !pmd_write(pmd)) -#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd)) #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define pmd_trans_huge(pmd) (pmd_val(pmd) && !pmd_table(pmd)) @@ -199,7 +200,7 @@ static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } PMD_BIT_FUNC(wrprotect, |= L_PMD_SECT_RDONLY); PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF); -PMD_BIT_FUNC(mkwrite, &= ~L_PMD_SECT_RDONLY); +PMD_BIT_FUNC(mkwrite_novma, &= ~L_PMD_SECT_RDONLY); PMD_BIT_FUNC(mkdirty, |= L_PMD_SECT_DIRTY); PMD_BIT_FUNC(mkclean, &= ~L_PMD_SECT_DIRTY); PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF); @@ -208,7 +209,6 @@ PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF); #define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT) #define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) -#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) /* No hardware dirty/accessed bits -- generic_pmdp_establish() fits */ #define pmdp_establish generic_pmdp_establish diff --git a/arch/arm/include/asm/pgtable-nommu.h b/arch/arm/include/asm/pgtable-nommu.h index d16aba48fa0a..61480d096054 100644 --- a/arch/arm/include/asm/pgtable-nommu.h +++ b/arch/arm/include/asm/pgtable-nommu.h @@ -21,8 +21,6 @@ #define pgd_none(pgd) (0) #define pgd_bad(pgd) (0) #define pgd_clear(pgdp) -#define kern_addr_valid(addr) (1) -/* FIXME */ /* * PMD_SHIFT determines the size of the area a second-level page table can map * PGDIR_SHIFT determines what a third-level page table entry can map @@ -45,12 +43,6 @@ typedef pte_t *pte_addr_t; /* - * ZERO_PAGE is a global shared page that is always zero: used - * for zero-mapped memory areas etc.. - */ -#define ZERO_PAGE(vaddr) (virt_to_page(0)) - -/* * Mark the prot value as uncacheable and unbufferable. */ #define pgprot_noncached(prot) (prot) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index cd1f84bb40ae..86378eec7757 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -10,15 +10,23 @@ #include <linux/const.h> #include <asm/proc-fns.h> -#ifndef CONFIG_MMU +#ifndef __ASSEMBLY__ +/* + * ZERO_PAGE is a global shared page that is always zero: used + * for zero-mapped memory areas etc.. + */ +extern struct page *empty_zero_page; +#define ZERO_PAGE(vaddr) (empty_zero_page) +#endif #include <asm-generic/pgtable-nopud.h> + +#ifndef CONFIG_MMU #include <asm/pgtable-nommu.h> #else -#include <asm-generic/pgtable-nopud.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/pgtable-hwdef.h> @@ -137,35 +145,13 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, * 2) If we could do execute protection, then read is implied * 3) write implies read permissions */ -#define __P000 __PAGE_NONE -#define __P001 __PAGE_READONLY -#define __P010 __PAGE_COPY -#define __P011 __PAGE_COPY -#define __P100 __PAGE_READONLY_EXEC -#define __P101 __PAGE_READONLY_EXEC -#define __P110 __PAGE_COPY_EXEC -#define __P111 __PAGE_COPY_EXEC - -#define __S000 __PAGE_NONE -#define __S001 __PAGE_READONLY -#define __S010 __PAGE_SHARED -#define __S011 __PAGE_SHARED -#define __S100 __PAGE_READONLY_EXEC -#define __S101 __PAGE_READONLY_EXEC -#define __S110 __PAGE_SHARED_EXEC -#define __S111 __PAGE_SHARED_EXEC #ifndef __ASSEMBLY__ -/* - * ZERO_PAGE is a global shared page that is always zero: used - * for zero-mapped memory areas etc.. - */ -extern struct page *empty_zero_page; -#define ZERO_PAGE(vaddr) (empty_zero_page) - extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; +#define pgdp_get(pgpd) READ_ONCE(*pgdp) + #define pud_page(pud) pmd_page(__pmd(pud_val(pud))) #define pud_write(pud) pmd_write(__pmd(pud_val(pud))) @@ -182,7 +168,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pfn_pte(pfn,prot) __pte(__pfn_to_phys(pfn) | pgprot_val(prot)) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) -#define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot) #define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0) @@ -222,8 +207,11 @@ static inline void __sync_icache_dcache(pte_t pteval) extern void __sync_icache_dcache(pte_t pteval); #endif -void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval); +#define PFN_PTE_SHIFT PAGE_SHIFT + +void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval, unsigned int nr); +#define set_ptes set_ptes static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) { @@ -242,7 +230,7 @@ static inline pte_t pte_wrprotect(pte_t pte) return set_pte_bit(pte, __pgprot(L_PTE_RDONLY)); } -static inline pte_t pte_mkwrite(pte_t pte) +static inline pte_t pte_mkwrite_novma(pte_t pte) { return clear_pte_bit(pte, __pgprot(L_PTE_RDONLY)); } @@ -286,27 +274,47 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) } /* - * Encode and decode a swap entry. Swap entries are stored in the Linux - * page tables as follows: + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs: * * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 - * <--------------- offset ------------------------> < type -> 0 0 + * <------------------- offset ------------------> E < type -> 0 0 + * + * E is the exclusive marker that is not stored in swap entries. * - * This gives us up to 31 swap files and 128GB per swap file. Note that + * This gives us up to 31 swap files and 64GB per swap file. Note that * the offset field is always non-zero. */ #define __SWP_TYPE_SHIFT 2 #define __SWP_TYPE_BITS 5 #define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) -#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) +#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT + 1) #define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) #define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT) -#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) }) +#define __swp_entry(type, offset) ((swp_entry_t) { (((type) & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT) | \ + ((offset) << __SWP_OFFSET_SHIFT) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) -#define __swp_entry_to_pte(swp) __pte((swp).val | PTE_TYPE_FAULT) +#define __swp_entry_to_pte(swp) __pte((swp).val) + +static inline bool pte_swp_exclusive(pte_t pte) +{ + return pte_isset(pte, L_PTE_SWP_EXCLUSIVE); +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(L_PTE_SWP_EXCLUSIVE)); +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(L_PTE_SWP_EXCLUSIVE)); +} /* * It is an error for the kernel to have more swap files than we can @@ -315,10 +323,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) */ #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) -/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ -/* FIXME: this is not correct */ -#define kern_addr_valid(addr) (1) - /* * We provide our own arch_get_unmapped_area to cope with VIPT caches. */ diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index c82f7a29ec4a..b4986a23d852 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -147,8 +147,6 @@ static inline void init_proc_vtable(const struct processor *p) extern void cpu_resume(void); -#include <asm/memory.h> - #ifdef CONFIG_MMU #define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) @@ -180,6 +178,18 @@ extern void cpu_resume(void); }) #endif +static inline unsigned int cpu_get_ttbcr(void) +{ + unsigned int ttbcr; + asm("mrc p15, 0, %0, c2, c0, 2" : "=r" (ttbcr)); + return ttbcr; +} + +static inline void cpu_set_ttbcr(unsigned int ttbcr) +{ + asm volatile("mcr p15, 0, %0, c2, c0, 2" : : "r" (ttbcr) : "memory"); +} + #else /*!CONFIG_MMU */ #define cpu_switch_mm(pgd,mm) { } diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index bdc35c0e8dfb..326864f79d18 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -81,9 +81,6 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset, /* Forward declaration, a strange C thing */ struct task_struct; -/* Free all resources held by a thread. */ -extern void release_thread(struct task_struct *); - unsigned long __get_wchan(struct task_struct *p); #define task_pt_regs(p) \ diff --git a/arch/arm/include/asm/ptdump.h b/arch/arm/include/asm/ptdump.h index 0c2d3d0d4cc6..46a4575146ee 100644 --- a/arch/arm/include/asm/ptdump.h +++ b/arch/arm/include/asm/ptdump.h @@ -21,6 +21,7 @@ struct ptdump_info { void ptdump_walk_pgd(struct seq_file *s, struct ptdump_info *info); #ifdef CONFIG_ARM_PTDUMP_DEBUGFS +#define EFI_RUNTIME_MAP_END SZ_1G void ptdump_debugfs_register(struct ptdump_info *info, const char *name); #else static inline void ptdump_debugfs_register(struct ptdump_info *info, @@ -31,10 +32,10 @@ void ptdump_check_wx(void); #endif /* CONFIG_ARM_PTDUMP_CORE */ -#ifdef CONFIG_DEBUG_WX -#define debug_checkwx() ptdump_check_wx() +#ifdef CONFIG_ARM_DEBUG_WX +#define arm_debug_checkwx() ptdump_check_wx() #else -#define debug_checkwx() do { } while (0) +#define arm_debug_checkwx() do { } while (0) #endif #endif /* __ASM_PTDUMP_H */ diff --git a/arch/arm/include/asm/ptrace.h b/arch/arm/include/asm/ptrace.h index 1408a6a15d0e..6eb311fb2da0 100644 --- a/arch/arm/include/asm/ptrace.h +++ b/arch/arm/include/asm/ptrace.h @@ -10,6 +10,7 @@ #include <uapi/asm/ptrace.h> #ifndef __ASSEMBLY__ +#include <linux/bitfield.h> #include <linux/types.h> struct pt_regs { @@ -19,6 +20,7 @@ struct pt_regs { struct svc_pt_regs { struct pt_regs regs; u32 dacr; + u32 ttbcr; }; #define to_svc_pt_regs(r) container_of(r, struct svc_pt_regs, regs) @@ -35,8 +37,8 @@ struct svc_pt_regs { #ifndef CONFIG_CPU_V7M #define isa_mode(regs) \ - ((((regs)->ARM_cpsr & PSR_J_BIT) >> (__ffs(PSR_J_BIT) - 1)) | \ - (((regs)->ARM_cpsr & PSR_T_BIT) >> (__ffs(PSR_T_BIT)))) + (FIELD_GET(PSR_J_BIT, (regs)->ARM_cpsr) << 1 | \ + FIELD_GET(PSR_T_BIT, (regs)->ARM_cpsr)) #else #define isa_mode(regs) 1 /* Thumb */ #endif @@ -163,6 +165,10 @@ static inline unsigned long user_stack_pointer(struct pt_regs *regs) ((current_stack_pointer | (THREAD_SIZE - 1)) - 7) - 1; \ }) +static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) +{ + regs->ARM_r0 = rc; +} /* * Update ITSTATE after normal execution of an IT block instruction. @@ -189,5 +195,8 @@ static inline unsigned long it_advance(unsigned long cpsr) return cpsr; } +int syscall_trace_enter(struct pt_regs *regs); +void syscall_trace_exit(struct pt_regs *regs); + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm/include/asm/semihost.h b/arch/arm/include/asm/semihost.h new file mode 100644 index 000000000000..f365787e7c23 --- /dev/null +++ b/arch/arm/include/asm/semihost.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2012 ARM Ltd. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * + * Adapted for ARM and earlycon: + * Copyright (C) 2014 Linaro Ltd. + * Author: Rob Herring <robh@kernel.org> + */ + +#ifndef _ARM_SEMIHOST_H_ +#define _ARM_SEMIHOST_H_ + +#ifdef CONFIG_THUMB2_KERNEL +#define SEMIHOST_SWI "0xab" +#else +#define SEMIHOST_SWI "0x123456" +#endif + +struct uart_port; + +static inline void smh_putc(struct uart_port *port, unsigned char c) +{ + asm volatile("mov r1, %0\n" + "mov r0, #3\n" + "svc " SEMIHOST_SWI "\n" + : : "r" (&c) : "r0", "r1", "memory"); +} + +#endif /* _ARM_SEMIHOST_H_ */ diff --git a/arch/arm/include/asm/setup.h b/arch/arm/include/asm/setup.h index ba0872a8dcda..cc106f946c69 100644 --- a/arch/arm/include/asm/setup.h +++ b/arch/arm/include/asm/setup.h @@ -5,12 +5,13 @@ * Copyright (C) 1997-1999 Russell King * * Structure passed to kernel to tell it about the - * hardware it's running on. See Documentation/arm/setup.rst + * hardware it's running on. See Documentation/arch/arm/setup.rst * for more info. */ #ifndef __ASMARM_SETUP_H #define __ASMARM_SETUP_H +#include <linux/screen_info.h> #include <uapi/asm/setup.h> @@ -28,4 +29,15 @@ extern void save_atags(const struct tag *tags); static inline void save_atags(const struct tag *tags) { } #endif +struct machine_desc; +void init_default_cache_policy(unsigned long); +void paging_init(const struct machine_desc *desc); +void early_mm_init(const struct machine_desc *); +void adjust_lowmem_bounds(void); +void setup_dma_zone(const struct machine_desc *desc); + +#ifdef CONFIG_VGA_CONSOLE +extern struct screen_info vgacon_screen_info; +#endif + #endif diff --git a/arch/arm/include/asm/signal.h b/arch/arm/include/asm/signal.h index 430be7774402..8b84092d1518 100644 --- a/arch/arm/include/asm/signal.h +++ b/arch/arm/include/asm/signal.h @@ -22,4 +22,9 @@ typedef struct { #define __ARCH_HAS_SA_RESTORER #include <asm/sigcontext.h> + +void do_rseq_syscall(struct pt_regs *regs); +int do_work_pending(struct pt_regs *regs, unsigned int thread_flags, + int syscall); + #endif diff --git a/arch/arm/include/asm/simd.h b/arch/arm/include/asm/simd.h new file mode 100644 index 000000000000..be08a8da046f --- /dev/null +++ b/arch/arm/include/asm/simd.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SIMD_H +#define _ASM_SIMD_H + +#include <linux/compiler_attributes.h> +#include <linux/preempt.h> +#include <linux/types.h> + +static __must_check inline bool may_use_simd(void) +{ + return IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !in_hardirq() + && !irqs_disabled(); +} + +#endif /* _ASM_SIMD_H */ diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 7c1c90d9f582..8c05a7f374d8 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -64,7 +64,7 @@ extern void secondary_startup_arm(void); extern int __cpu_disable(void); -extern void __cpu_die(unsigned int cpu); +static inline void __cpu_die(unsigned int cpu) { } extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); diff --git a/arch/arm/include/asm/sparsemem.h b/arch/arm/include/asm/sparsemem.h index d362233856a5..421e3415338a 100644 --- a/arch/arm/include/asm/sparsemem.h +++ b/arch/arm/include/asm/sparsemem.h @@ -2,7 +2,7 @@ #ifndef ASMARM_SPARSEMEM_H #define ASMARM_SPARSEMEM_H -#include <asm/memory.h> +#include <asm/page.h> /* * Two definitions are required for sparsemem: diff --git a/arch/arm/include/asm/spectre.h b/arch/arm/include/asm/spectre.h index 85f9e538fb32..d9c28b3b6b62 100644 --- a/arch/arm/include/asm/spectre.h +++ b/arch/arm/include/asm/spectre.h @@ -35,4 +35,8 @@ static inline void spectre_v2_update_state(unsigned int state, int spectre_bhb_update_vectors(unsigned int method); +void cpu_v7_ca8_ibe(void); +void cpu_v7_ca15_ibe(void); +void cpu_v7_bugs_init(void); + #endif diff --git a/arch/arm/include/asm/spinlock_types.h b/arch/arm/include/asm/spinlock_types.h index 0c14b36ef101..5404a2a96bf3 100644 --- a/arch/arm/include/asm/spinlock_types.h +++ b/arch/arm/include/asm/spinlock_types.h @@ -3,7 +3,7 @@ #define __ASM_SPINLOCK_TYPES_H #ifndef __LINUX_SPINLOCK_TYPES_RAW_H -# error "please don't include this file directly" +# error "Please do not include this file directly." #endif #define TICKET_SHIFT 16 diff --git a/arch/arm/include/asm/stackprotector.h b/arch/arm/include/asm/stackprotector.h index 088d03161be5..0bd4979759f1 100644 --- a/arch/arm/include/asm/stackprotector.h +++ b/arch/arm/include/asm/stackprotector.h @@ -15,9 +15,6 @@ #ifndef _ASM_STACKPROTECTOR_H #define _ASM_STACKPROTECTOR_H 1 -#include <linux/random.h> -#include <linux/version.h> - #include <asm/thread_info.h> extern unsigned long __stack_chk_guard; @@ -30,11 +27,7 @@ extern unsigned long __stack_chk_guard; */ static __always_inline void boot_init_stack_canary(void) { - unsigned long canary; - - /* Try to get a semi random initial value. */ - get_random_bytes(&canary, sizeof(canary)); - canary ^= LINUX_VERSION_CODE; + unsigned long canary = get_random_canary(); current->stack_canary = canary; #ifndef CONFIG_STACKPROTECTOR_PER_TASK diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index 3e78f921b8b2..f80a85b091d6 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -21,8 +21,18 @@ struct stackframe { struct llist_node *kr_cur; struct task_struct *tsk; #endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + bool ex_frame; +#endif }; +static inline bool on_thread_stack(void) +{ + unsigned long delta = current_stack_pointer ^ (unsigned long)current->stack; + + return delta < THREAD_SIZE; +} + static __always_inline void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) { @@ -34,12 +44,17 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) frame->kr_cur = NULL; frame->tsk = current; #endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + frame->ex_frame = in_entry_text(frame->pc); +#endif } extern int unwind_frame(struct stackframe *frame); extern void walk_stackframe(struct stackframe *frame, - int (*fn)(struct stackframe *, void *), void *data); + bool (*fn)(void *, unsigned long), void *data); extern void dump_mem(const char *lvl, const char *str, unsigned long bottom, unsigned long top); +extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, + const char *loglvl); #endif /* __ASM_STACKTRACE_H */ diff --git a/arch/arm/include/asm/suspend.h b/arch/arm/include/asm/suspend.h index 506314265c6f..be81b9ca2ea1 100644 --- a/arch/arm/include/asm/suspend.h +++ b/arch/arm/include/asm/suspend.h @@ -13,5 +13,6 @@ extern void cpu_resume(void); extern void cpu_resume_no_hyp(void); extern void cpu_resume_arm(void); extern int cpu_suspend(unsigned long, int (*)(unsigned long)); +extern void __cpu_suspend_save(u32 *ptr, u32 ptrsz, u32 sp, u32 *save_ptr); #endif diff --git a/arch/arm/include/asm/sync_bitops.h b/arch/arm/include/asm/sync_bitops.h index 6f5d627c44a3..f46b3c570f92 100644 --- a/arch/arm/include/asm/sync_bitops.h +++ b/arch/arm/include/asm/sync_bitops.h @@ -14,14 +14,35 @@ * ops which are SMP safe even on a UP kernel. */ +/* + * Unordered + */ + #define sync_set_bit(nr, p) _set_bit(nr, p) #define sync_clear_bit(nr, p) _clear_bit(nr, p) #define sync_change_bit(nr, p) _change_bit(nr, p) -#define sync_test_and_set_bit(nr, p) _test_and_set_bit(nr, p) -#define sync_test_and_clear_bit(nr, p) _test_and_clear_bit(nr, p) -#define sync_test_and_change_bit(nr, p) _test_and_change_bit(nr, p) #define sync_test_bit(nr, addr) test_bit(nr, addr) -#define arch_sync_cmpxchg arch_cmpxchg +/* + * Fully ordered + */ + +int _sync_test_and_set_bit(int nr, volatile unsigned long * p); +#define sync_test_and_set_bit(nr, p) _sync_test_and_set_bit(nr, p) + +int _sync_test_and_clear_bit(int nr, volatile unsigned long * p); +#define sync_test_and_clear_bit(nr, p) _sync_test_and_clear_bit(nr, p) + +int _sync_test_and_change_bit(int nr, volatile unsigned long * p); +#define sync_test_and_change_bit(nr, p) _sync_test_and_change_bit(nr, p) + +#define arch_sync_cmpxchg(ptr, old, new) \ +({ \ + __typeof__(*(ptr)) __ret; \ + __smp_mb__before_atomic(); \ + __ret = arch_cmpxchg_relaxed((ptr), (old), (new)); \ + __smp_mb__after_atomic(); \ + __ret; \ +}) #endif diff --git a/arch/arm/include/asm/syscall.h b/arch/arm/include/asm/syscall.h index dfeed440254a..18b102a30741 100644 --- a/arch/arm/include/asm/syscall.h +++ b/arch/arm/include/asm/syscall.h @@ -25,6 +25,9 @@ static inline int syscall_get_nr(struct task_struct *task, if (IS_ENABLED(CONFIG_AEABI) && !IS_ENABLED(CONFIG_OABI_COMPAT)) return task_thread_info(task)->abi_syscall; + if (task_thread_info(task)->abi_syscall == -1) + return -1; + return task_thread_info(task)->abi_syscall & __NR_SYSCALL_MASK; } @@ -65,6 +68,30 @@ static inline void syscall_set_return_value(struct task_struct *task, regs->ARM_r0 = (long) error ? error : val; } +static inline void syscall_set_nr(struct task_struct *task, + struct pt_regs *regs, + int nr) +{ + if (nr == -1) { + task_thread_info(task)->abi_syscall = -1; + /* + * When the syscall number is set to -1, the syscall will be + * skipped. In this case the syscall return value has to be + * set explicitly, otherwise the first syscall argument is + * returned as the syscall return value. + */ + syscall_set_return_value(task, regs, -ENOSYS, 0); + return; + } + if ((IS_ENABLED(CONFIG_AEABI) && !IS_ENABLED(CONFIG_OABI_COMPAT))) { + task_thread_info(task)->abi_syscall = nr; + return; + } + task_thread_info(task)->abi_syscall = + (task_thread_info(task)->abi_syscall & ~__NR_SYSCALL_MASK) | + (nr & __NR_SYSCALL_MASK); +} + #define SYSCALL_MAX_ARGS 7 static inline void syscall_get_arguments(struct task_struct *task, @@ -77,6 +104,19 @@ static inline void syscall_get_arguments(struct task_struct *task, memcpy(args, ®s->ARM_r0 + 1, 5 * sizeof(args[0])); } +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + memcpy(®s->ARM_r0, args, 6 * sizeof(args[0])); + /* + * Also copy the first argument into ARM_ORIG_r0 + * so that syscall_get_arguments() would return it + * instead of the previous value. + */ + regs->ARM_ORIG_r0 = regs->ARM_r0; +} + static inline int syscall_get_arch(struct task_struct *task) { /* ARM tasks don't change audit architectures on the fly. */ diff --git a/arch/arm/include/asm/syscalls.h b/arch/arm/include/asm/syscalls.h new file mode 100644 index 000000000000..5912e7cffa6a --- /dev/null +++ b/arch/arm/include/asm/syscalls.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_SYSCALLS_H +#define __ASM_SYSCALLS_H + +#include <linux/linkage.h> +#include <linux/types.h> + +struct pt_regs; +asmlinkage int sys_sigreturn(struct pt_regs *regs); +asmlinkage int sys_rt_sigreturn(struct pt_regs *regs); +asmlinkage long sys_arm_fadvise64_64(int fd, int advice, + loff_t offset, loff_t len); + +struct oldabi_stat64; +asmlinkage long sys_oabi_stat64(const char __user * filename, + struct oldabi_stat64 __user * statbuf); +asmlinkage long sys_oabi_lstat64(const char __user * filename, + struct oldabi_stat64 __user * statbuf); +asmlinkage long sys_oabi_fstat64(unsigned long fd, + struct oldabi_stat64 __user * statbuf); +asmlinkage long sys_oabi_fstatat64(int dfd, + const char __user *filename, + struct oldabi_stat64 __user *statbuf, + int flag); +asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, + unsigned long arg); +struct oabi_epoll_event; +asmlinkage long sys_oabi_epoll_ctl(int epfd, int op, int fd, + struct oabi_epoll_event __user *event); +struct oabi_sembuf; +struct old_timespec32; +asmlinkage long sys_oabi_semtimedop(int semid, + struct oabi_sembuf __user *tsops, + unsigned nsops, + const struct old_timespec32 __user *timeout); +asmlinkage long sys_oabi_semop(int semid, struct oabi_sembuf __user *tsops, + unsigned nsops); +asmlinkage int sys_oabi_ipc(uint call, int first, int second, int third, + void __user *ptr, long fifth); +struct sockaddr; +asmlinkage long sys_oabi_bind(int fd, struct sockaddr __user *addr, int addrlen); +asmlinkage long sys_oabi_connect(int fd, struct sockaddr __user *addr, int addrlen); +asmlinkage long sys_oabi_sendto(int fd, void __user *buff, + size_t len, unsigned flags, + struct sockaddr __user *addr, + int addrlen); +struct user_msghdr; +asmlinkage long sys_oabi_sendmsg(int fd, struct user_msghdr __user *msg, unsigned flags); +asmlinkage long sys_oabi_socketcall(int call, unsigned long __user *args); + +#endif diff --git a/arch/arm/include/asm/tcm.h b/arch/arm/include/asm/tcm.h index d8bd8a4b0ede..e1f7dca86a22 100644 --- a/arch/arm/include/asm/tcm.h +++ b/arch/arm/include/asm/tcm.h @@ -9,9 +9,7 @@ #ifndef __ASMARM_TCM_H #define __ASMARM_TCM_H -#ifndef CONFIG_HAVE_TCM -#error "You should not be including tcm.h unless you have a TCM!" -#endif +#ifdef CONFIG_HAVE_TCM #include <linux/compiler.h> @@ -29,4 +27,11 @@ void tcm_free(void *addr, size_t len); bool tcm_dtcm_present(void); bool tcm_itcm_present(void); +void __init tcm_init(void); +#else +/* No TCM support, just blank inlines to be optimized out */ +static inline void tcm_init(void) +{ +} +#endif #endif diff --git a/arch/arm/include/asm/patch.h b/arch/arm/include/asm/text-patching.h index 0b48247c4600..0b48247c4600 100644 --- a/arch/arm/include/asm/patch.h +++ b/arch/arm/include/asm/text-patching.h diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index aecc403b2880..943ffcf069d2 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -40,6 +40,7 @@ struct task_struct; DECLARE_PER_CPU(struct task_struct *, __entry_task); #include <asm/types.h> +#include <asm/traps.h> struct cpu_context_save { __u32 r4; @@ -66,7 +67,6 @@ struct thread_info { __u32 cpu_domain; /* cpu domain */ struct cpu_context_save cpu_context; /* cpu context */ __u32 abi_syscall; /* ABI type and syscall nr */ - __u8 used_cp[16]; /* thread used copro */ unsigned long tp_value[2]; /* TLS registers */ union fp_state fpstate __attribute__((aligned(8))); union vfp_state vfpstate; @@ -105,6 +105,21 @@ extern void iwmmxt_task_restore(struct thread_info *, void *); extern void iwmmxt_task_release(struct thread_info *); extern void iwmmxt_task_switch(struct thread_info *); +extern int iwmmxt_undef_handler(struct pt_regs *, u32); + +static inline void register_iwmmxt_undef_handler(void) +{ + static struct undef_hook iwmmxt_undef_hook = { + .instr_mask = 0x0c000e00, + .instr_val = 0x0c000000, + .cpsr_mask = MODE_MASK | PSR_T_BIT, + .cpsr_val = USR_MODE, + .fn = iwmmxt_undef_handler, + }; + + register_undef_hook(&iwmmxt_undef_hook); +} + extern void vfp_sync_hwstate(struct thread_info *); extern void vfp_flush_hwstate(struct thread_info *); @@ -128,15 +143,16 @@ extern int vfp_restore_user_hwstate(struct user_vfp *, #define TIF_NEED_RESCHED 1 /* rescheduling necessary */ #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ #define TIF_UPROBE 3 /* breakpointed or singlestepping */ -#define TIF_SYSCALL_TRACE 4 /* syscall trace active */ -#define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ -#define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ -#define TIF_SECCOMP 7 /* seccomp syscall filtering active */ -#define TIF_NOTIFY_SIGNAL 8 /* signal notifications exist */ +#define TIF_NOTIFY_SIGNAL 4 /* signal notifications exist */ #define TIF_USING_IWMMXT 17 #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ -#define TIF_RESTORE_SIGMASK 20 +#define TIF_RESTORE_SIGMASK 19 +#define TIF_SYSCALL_TRACE 20 /* syscall trace active */ +#define TIF_SYSCALL_AUDIT 21 /* syscall auditing active */ +#define TIF_SYSCALL_TRACEPOINT 22 /* syscall tracepoint instrumentation */ +#define TIF_SECCOMP 23 /* seccomp syscall filtering active */ + #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index b8cbe03ad260..ea4fbe7b17f6 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -26,20 +26,13 @@ #else /* !CONFIG_MMU */ -#include <linux/swap.h> #include <asm/tlbflush.h> - -static inline void __tlb_remove_table(void *_table) -{ - free_page_and_swap_cache((struct page *)_table); -} - #include <asm-generic/tlb.h> static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - pgtable_pte_page_dtor(pte); + struct ptdesc *ptdesc = page_ptdesc(pte); #ifndef CONFIG_ARM_LPAE /* @@ -50,17 +43,16 @@ __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) __tlb_adjust_range(tlb, addr - PAGE_SIZE, 2 * PAGE_SIZE); #endif - tlb_remove_table(tlb, pte); + tlb_remove_ptdesc(tlb, ptdesc); } static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) { #ifdef CONFIG_ARM_LPAE - struct page *page = virt_to_page(pmdp); + struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); - pgtable_pmd_page_dtor(page); - tlb_remove_table(tlb, page); + tlb_remove_ptdesc(tlb, ptdesc); #endif } diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index 0ccc985b90af..38c6e4a2a0b6 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -619,18 +619,22 @@ extern void flush_bp_all(void); * If PG_dcache_clean is not set for the page, we need to ensure that any * cache entries for the kernels virtual memory range are written * back to the page. On ARMv6 and later, the cache coherency is handled via - * the set_pte_at() function. + * the set_ptes() function. */ #if __LINUX_ARM_ARCH__ < 6 -extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, - pte_t *ptep); +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned int nr); #else -static inline void update_mmu_cache(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) +static inline void update_mmu_cache_range(struct vm_fault *vmf, + struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, + unsigned int nr) { } #endif +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) + #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) #endif diff --git a/arch/arm/include/asm/topology.h b/arch/arm/include/asm/topology.h index c7d2510e5a78..ad36b6570067 100644 --- a/arch/arm/include/asm/topology.h +++ b/arch/arm/include/asm/topology.h @@ -13,6 +13,7 @@ #define arch_set_freq_scale topology_set_freq_scale #define arch_scale_freq_capacity topology_get_freq_scale #define arch_scale_freq_invariant topology_scale_freq_invariant +#define arch_scale_freq_ref topology_get_freq_ref #endif /* Replace task scheduler's default cpu-invariant accounting */ @@ -21,9 +22,9 @@ /* Enable topology flag updates */ #define arch_update_cpu_topology topology_update_cpu_topology -/* Replace task scheduler's default thermal pressure API */ -#define arch_scale_thermal_pressure topology_get_thermal_pressure -#define arch_update_thermal_pressure topology_update_thermal_pressure +/* Replace task scheduler's default HW pressure API */ +#define arch_scale_hw_pressure topology_get_hw_pressure +#define arch_update_hw_pressure topology_update_hw_pressure #else diff --git a/arch/arm/include/asm/traps.h b/arch/arm/include/asm/traps.h index 987fefb0a4db..2621b9fb9b19 100644 --- a/arch/arm/include/asm/traps.h +++ b/arch/arm/include/asm/traps.h @@ -2,6 +2,7 @@ #ifndef _ASMARM_TRAP_H #define _ASMARM_TRAP_H +#include <linux/linkage.h> #include <linux/list.h> struct pt_regs; @@ -28,11 +29,20 @@ static inline int __in_irqentry_text(unsigned long ptr) ptr < (unsigned long)&__irqentry_text_end; } -extern void __init early_trap_init(void *); +extern void early_trap_init(void *); extern void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame, const char *loglvl); extern void ptrace_break(struct pt_regs *regs); extern void *vectors_page; +asmlinkage void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl); +asmlinkage void do_undefinstr(struct pt_regs *regs); +asmlinkage void handle_fiq_as_nmi(struct pt_regs *regs); +asmlinkage void bad_mode(struct pt_regs *regs, int reason); +asmlinkage int arm_syscall(int no, struct pt_regs *regs); +asmlinkage void baddataabort(int code, unsigned long instr, struct pt_regs *regs); +asmlinkage void __div0(void); +asmlinkage void handle_bad_stack(struct pt_regs *regs); + #endif diff --git a/arch/arm/include/asm/uaccess-asm.h b/arch/arm/include/asm/uaccess-asm.h index 6451a433912c..4bccd895d954 100644 --- a/arch/arm/include/asm/uaccess-asm.h +++ b/arch/arm/include/asm/uaccess-asm.h @@ -5,7 +5,7 @@ #include <asm/asm-offsets.h> #include <asm/domain.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/thread_info.h> .macro csdb @@ -39,8 +39,9 @@ #endif .endm +#if defined(CONFIG_CPU_SW_DOMAIN_PAN) + .macro uaccess_disable, tmp, isb=1 -#ifdef CONFIG_CPU_SW_DOMAIN_PAN /* * Whenever we re-enter userspace, the domains should always be * set appropriately. @@ -50,11 +51,9 @@ .if \isb instr_sync .endif -#endif .endm .macro uaccess_enable, tmp, isb=1 -#ifdef CONFIG_CPU_SW_DOMAIN_PAN /* * Whenever we re-enter userspace, the domains should always be * set appropriately. @@ -64,15 +63,61 @@ .if \isb instr_sync .endif -#endif .endm +#elif defined(CONFIG_CPU_TTBR0_PAN) + + .macro uaccess_disable, tmp, isb=1 + /* + * Disable TTBR0 page table walks (EDP0 = 1), use the reserved ASID + * from TTBR1 (A1 = 1) and enable TTBR1 page table walks for kernel + * addresses by reducing TTBR0 range to 32MB (T0SZ = 7). + */ + mrc p15, 0, \tmp, c2, c0, 2 @ read TTBCR + orr \tmp, \tmp, #TTBCR_EPD0 | TTBCR_T0SZ_MASK + orr \tmp, \tmp, #TTBCR_A1 + mcr p15, 0, \tmp, c2, c0, 2 @ write TTBCR + .if \isb + instr_sync + .endif + .endm + + .macro uaccess_enable, tmp, isb=1 + /* + * Enable TTBR0 page table walks (T0SZ = 0, EDP0 = 0) and ASID from + * TTBR0 (A1 = 0). + */ + mrc p15, 0, \tmp, c2, c0, 2 @ read TTBCR + bic \tmp, \tmp, #TTBCR_EPD0 | TTBCR_T0SZ_MASK + bic \tmp, \tmp, #TTBCR_A1 + mcr p15, 0, \tmp, c2, c0, 2 @ write TTBCR + .if \isb + instr_sync + .endif + .endm + +#else + + .macro uaccess_disable, tmp, isb=1 + .endm + + .macro uaccess_enable, tmp, isb=1 + .endm + +#endif + #if defined(CONFIG_CPU_SW_DOMAIN_PAN) || defined(CONFIG_CPU_USE_DOMAINS) #define DACR(x...) x #else #define DACR(x...) #endif +#ifdef CONFIG_CPU_TTBR0_PAN +#define PAN(x...) x +#else +#define PAN(x...) +#endif + /* * Save the address limit on entry to a privileged exception. * @@ -86,6 +131,8 @@ .macro uaccess_entry, tsk, tmp0, tmp1, tmp2, disable DACR( mrc p15, 0, \tmp0, c3, c0, 0) DACR( str \tmp0, [sp, #SVC_DACR]) + PAN( mrc p15, 0, \tmp0, c2, c0, 2) + PAN( str \tmp0, [sp, #SVC_TTBCR]) .if \disable && IS_ENABLED(CONFIG_CPU_SW_DOMAIN_PAN) /* kernel=client, user=no access */ mov \tmp2, #DACR_UACCESS_DISABLE @@ -104,8 +151,11 @@ .macro uaccess_exit, tsk, tmp0, tmp1 DACR( ldr \tmp0, [sp, #SVC_DACR]) DACR( mcr p15, 0, \tmp0, c3, c0, 0) + PAN( ldr \tmp0, [sp, #SVC_TTBCR]) + PAN( mcr p15, 0, \tmp0, c2, c0, 2) .endm #undef DACR +#undef PAN #endif /* __ASM_UACCESS_ASM_H__ */ diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 2fcbec9c306c..f90be312418e 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -8,11 +8,14 @@ /* * User space memory access functions */ +#include <linux/kernel.h> #include <linux/string.h> -#include <asm/memory.h> +#include <asm/page.h> #include <asm/domain.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include <asm/unified.h> +#include <asm/pgtable.h> +#include <asm/proc-fns.h> #include <asm/compiler.h> #include <asm/extable.h> @@ -23,9 +26,10 @@ * perform such accesses (eg, via list poison values) which could then * be exploited for priviledge escalation. */ +#if defined(CONFIG_CPU_SW_DOMAIN_PAN) + static __always_inline unsigned int uaccess_save_and_enable(void) { -#ifdef CONFIG_CPU_SW_DOMAIN_PAN unsigned int old_domain = get_domain(); /* Set the current domain access to permit user accesses */ @@ -33,19 +37,49 @@ static __always_inline unsigned int uaccess_save_and_enable(void) domain_val(DOMAIN_USER, DOMAIN_CLIENT)); return old_domain; -#else - return 0; -#endif } static __always_inline void uaccess_restore(unsigned int flags) { -#ifdef CONFIG_CPU_SW_DOMAIN_PAN /* Restore the user access mask */ set_domain(flags); -#endif } +#elif defined(CONFIG_CPU_TTBR0_PAN) + +static __always_inline unsigned int uaccess_save_and_enable(void) +{ + unsigned int old_ttbcr = cpu_get_ttbcr(); + + /* + * Enable TTBR0 page table walks (T0SZ = 0, EDP0 = 0) and ASID from + * TTBR0 (A1 = 0). + */ + cpu_set_ttbcr(old_ttbcr & ~(TTBCR_A1 | TTBCR_EPD0 | TTBCR_T0SZ_MASK)); + isb(); + + return old_ttbcr; +} + +static inline void uaccess_restore(unsigned int flags) +{ + cpu_set_ttbcr(flags); + isb(); +} + +#else + +static inline unsigned int uaccess_save_and_enable(void) +{ + return 0; +} + +static inline void uaccess_restore(unsigned int flags) +{ +} + +#endif + /* * These two are intentionally not defined anywhere - if the kernel * code generates any references to them, that's a bug. @@ -109,16 +143,6 @@ extern int __get_user_64t_1(void *); extern int __get_user_64t_2(void *); extern int __get_user_64t_4(void *); -#define __GUP_CLOBBER_1 "lr", "cc" -#ifdef CONFIG_CPU_USE_DOMAINS -#define __GUP_CLOBBER_2 "ip", "lr", "cc" -#else -#define __GUP_CLOBBER_2 "lr", "cc" -#endif -#define __GUP_CLOBBER_4 "lr", "cc" -#define __GUP_CLOBBER_32t_8 "lr", "cc" -#define __GUP_CLOBBER_8 "lr", "cc" - #define __get_user_x(__r2, __p, __e, __l, __s) \ __asm__ __volatile__ ( \ __asmeq("%0", "r0") __asmeq("%1", "r2") \ @@ -126,7 +150,7 @@ extern int __get_user_64t_4(void *); "bl __get_user_" #__s \ : "=&r" (__e), "=r" (__r2) \ : "0" (__p), "r" (__l) \ - : __GUP_CLOBBER_##__s) + : "ip", "lr", "cc") /* narrowing a double-word get into a single 32bit word register: */ #ifdef __ARMEB__ @@ -148,7 +172,7 @@ extern int __get_user_64t_4(void *); "bl __get_user_64t_" #__s \ : "=&r" (__e), "=r" (__r2) \ : "0" (__p), "r" (__l) \ - : __GUP_CLOBBER_##__s) + : "ip", "lr", "cc") #else #define __get_user_x_64t __get_user_x #endif diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index 3676e82cf95c..9fb00973c608 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -37,7 +37,6 @@ #define __ARCH_WANT_SYS_FORK #define __ARCH_WANT_SYS_VFORK #define __ARCH_WANT_SYS_CLONE -#define __ARCH_WANT_SYS_CLONE3 /* * Unimplemented (or alternatively implemented) syscalls diff --git a/arch/arm/include/asm/unwind.h b/arch/arm/include/asm/unwind.h index b51f85417f58..d60b09a5acfc 100644 --- a/arch/arm/include/asm/unwind.h +++ b/arch/arm/include/asm/unwind.h @@ -40,6 +40,10 @@ extern void unwind_table_del(struct unwind_table *tab); extern void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk, const char *loglvl); +void __aeabi_unwind_cpp_pr0(void); +void __aeabi_unwind_cpp_pr1(void); +void __aeabi_unwind_cpp_pr2(void); + #endif /* !__ASSEMBLY__ */ #ifdef CONFIG_ARM_UNWIND diff --git a/arch/arm/include/asm/vdso.h b/arch/arm/include/asm/vdso.h index 5b85889f82ee..88364a6727ff 100644 --- a/arch/arm/include/asm/vdso.h +++ b/arch/arm/include/asm/vdso.h @@ -4,6 +4,8 @@ #ifdef __KERNEL__ +#define __VDSO_PAGES 4 + #ifndef __ASSEMBLY__ struct mm_struct; diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h index 2134cbd5469f..1e9f81639c88 100644 --- a/arch/arm/include/asm/vdso/gettimeofday.h +++ b/arch/arm/include/asm/vdso/gettimeofday.h @@ -15,8 +15,6 @@ #define VDSO_HAS_CLOCK_GETRES 1 -extern struct vdso_data *__get_datapage(void); - static __always_inline int gettimeofday_fallback( struct __kernel_old_timeval *_tv, struct timezone *_tz) @@ -114,7 +112,7 @@ static inline bool arm_vdso_hres_capable(void) #define __arch_vdso_hres_capable arm_vdso_hres_capable static __always_inline u64 __arch_get_hw_counter(int clock_mode, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { #ifdef CONFIG_ARM_ARCH_TIMER u64 cycle_now; @@ -137,11 +135,6 @@ static __always_inline u64 __arch_get_hw_counter(int clock_mode, #endif } -static __always_inline const struct vdso_data *__arch_get_vdso_data(void) -{ - return __get_datapage(); -} - #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/arm/include/asm/vdso/vsyscall.h b/arch/arm/include/asm/vdso/vsyscall.h index 47e41ae8ccd0..4e7226ad02ec 100644 --- a/arch/arm/include/asm/vdso/vsyscall.h +++ b/arch/arm/include/asm/vdso/vsyscall.h @@ -4,29 +4,17 @@ #ifndef __ASSEMBLY__ -#include <linux/timekeeper_internal.h> #include <vdso/datapage.h> #include <asm/cacheflush.h> -extern struct vdso_data *vdso_data; extern bool cntvct_ok; -/* - * Update the vDSO data page to keep in sync with kernel timekeeping. - */ static __always_inline -struct vdso_data *__arm_get_k_vdso_data(void) -{ - return vdso_data; -} -#define __arch_get_k_vdso_data __arm_get_k_vdso_data - -static __always_inline -void __arm_sync_vdso_data(struct vdso_data *vdata) +void __arch_sync_vdso_time_data(struct vdso_time_data *vdata) { flush_dcache_page(virt_to_page(vdata)); } -#define __arch_sync_vdso_data __arm_sync_vdso_data +#define __arch_sync_vdso_time_data __arch_sync_vdso_time_data /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h deleted file mode 100644 index bef68f59928d..000000000000 --- a/arch/arm/include/asm/vdso_datapage.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Adapted from arm64 version. - * - * Copyright (C) 2012 ARM Limited - */ -#ifndef __ASM_VDSO_DATAPAGE_H -#define __ASM_VDSO_DATAPAGE_H - -#ifdef __KERNEL__ - -#ifndef __ASSEMBLY__ - -#include <vdso/datapage.h> -#include <asm/page.h> - -union vdso_data_store { - struct vdso_data data[CS_BASES]; - u8 page[PAGE_SIZE]; -}; - -#endif /* !__ASSEMBLY__ */ - -#endif /* __KERNEL__ */ - -#endif /* __ASM_VDSO_DATAPAGE_H */ diff --git a/arch/arm/include/asm/vfp.h b/arch/arm/include/asm/vfp.h index 19928bfb4f9c..85ccc422d4d0 100644 --- a/arch/arm/include/asm/vfp.h +++ b/arch/arm/include/asm/vfp.h @@ -9,16 +9,6 @@ #ifndef __ASM_VFP_H #define __ASM_VFP_H -#ifndef CONFIG_AS_VFP_VMRS_FPINST -#define FPSID cr0 -#define FPSCR cr1 -#define MVFR1 cr6 -#define MVFR0 cr7 -#define FPEXC cr8 -#define FPINST cr9 -#define FPINST2 cr10 -#endif - /* FPSID bits */ #define FPSID_IMPLEMENTER_BIT (24) #define FPSID_IMPLEMENTER_MASK (0xff << FPSID_IMPLEMENTER_BIT) @@ -87,6 +77,12 @@ #define MVFR0_DP_BIT (8) #define MVFR0_DP_MASK (0xf << MVFR0_DP_BIT) +/* MVFR1 bits */ +#define MVFR1_ASIMDHP_BIT (20) +#define MVFR1_ASIMDHP_MASK (0xf << MVFR1_ASIMDHP_BIT) +#define MVFR1_FPHP_BIT (24) +#define MVFR1_FPHP_MASK (0xf << MVFR1_FPHP_BIT) + /* Bit patterns for decoding the packaged operation descriptors */ #define VFPOPDESC_LENGTH_BIT (9) #define VFPOPDESC_LENGTH_MASK (0x07 << VFPOPDESC_LENGTH_BIT) diff --git a/arch/arm/include/asm/vfpmacros.h b/arch/arm/include/asm/vfpmacros.h index ba0d4cb5377e..e2e1d5a3727a 100644 --- a/arch/arm/include/asm/vfpmacros.h +++ b/arch/arm/include/asm/vfpmacros.h @@ -8,7 +8,6 @@ #include <asm/vfp.h> -#ifdef CONFIG_AS_VFP_VMRS_FPINST .macro VFPFMRX, rd, sysreg, cond vmrs\cond \rd, \sysreg .endm @@ -16,16 +15,6 @@ .macro VFPFMXR, sysreg, rd, cond vmsr\cond \sysreg, \rd .endm -#else - @ Macros to allow building with old toolkits (with no VFP support) - .macro VFPFMRX, rd, sysreg, cond - MRC\cond p10, 7, \rd, \sysreg, cr0, 0 @ FMRX \rd, \sysreg - .endm - - .macro VFPFMXR, sysreg, rd, cond - MCR\cond p10, 7, \rd, \sysreg, cr0, 0 @ FMXR \sysreg, \rd - .endm -#endif @ read all the working registers back into the VFP .macro VFPFLDMIA, base, tmp diff --git a/arch/arm/include/asm/vga.h b/arch/arm/include/asm/vga.h index 7c0bee57855a..6c430ec371df 100644 --- a/arch/arm/include/asm/vga.h +++ b/arch/arm/include/asm/vga.h @@ -5,6 +5,7 @@ #include <linux/io.h> extern unsigned long vga_base; +extern struct screen_info vgacon_screen_info; #define VGA_MAP_MEM(x,s) (vga_base + (x)) diff --git a/arch/arm/include/asm/vmlinux.lds.h b/arch/arm/include/asm/vmlinux.lds.h index fad45c884e98..0341973e30e1 100644 --- a/arch/arm/include/asm/vmlinux.lds.h +++ b/arch/arm/include/asm/vmlinux.lds.h @@ -19,7 +19,7 @@ #endif #ifdef CONFIG_MMU -#define ARM_MMU_KEEP(x) x +#define ARM_MMU_KEEP(x) KEEP(x) #define ARM_MMU_DISCARD(x) #else #define ARM_MMU_KEEP(x) @@ -34,6 +34,12 @@ #define NOCROSSREFS #endif +#ifdef CONFIG_LD_CAN_USE_KEEP_IN_OVERLAY +#define OVERLAY_KEEP(x) KEEP(x) +#else +#define OVERLAY_KEEP(x) x +#endif + /* Set start/end symbol names to the LMA for the section */ #define ARM_LMA(sym, section) \ sym##_start = LOADADDR(section); \ @@ -42,7 +48,7 @@ #define PROC_INFO \ . = ALIGN(4); \ __proc_info_begin = .; \ - *(.proc.info.init) \ + KEEP(*(.proc.info.init)) \ __proc_info_end = .; #define IDMAP_TEXT \ @@ -96,7 +102,6 @@ SOFTIRQENTRY_TEXT \ TEXT_TEXT \ SCHED_TEXT \ - CPUIDLE_TEXT \ LOCK_TEXT \ KPROBES_TEXT \ ARM_STUBS_TEXT \ @@ -126,13 +131,13 @@ __vectors_lma = .; \ OVERLAY 0xffff0000 : NOCROSSREFS AT(__vectors_lma) { \ .vectors { \ - *(.vectors) \ + OVERLAY_KEEP(*(.vectors)) \ } \ .vectors.bhb.loop8 { \ - *(.vectors.bhb.loop8) \ + OVERLAY_KEEP(*(.vectors.bhb.loop8)) \ } \ .vectors.bhb.bpiall { \ - *(.vectors.bhb.bpiall) \ + OVERLAY_KEEP(*(.vectors.bhb.bpiall)) \ } \ } \ ARM_LMA(__vectors, .vectors); \ diff --git a/arch/arm/include/asm/word-at-a-time.h b/arch/arm/include/asm/word-at-a-time.h index 352ab213520d..f9a3897b06e7 100644 --- a/arch/arm/include/asm/word-at-a-time.h +++ b/arch/arm/include/asm/word-at-a-time.h @@ -8,7 +8,8 @@ * Little-endian word-at-a-time zero byte handling. * Heavily based on the x86 algorithm. */ -#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/wordpart.h> struct word_at_a_time { const unsigned long one_bits, high_bits; diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h index 669cad5194d3..934b549905f5 100644 --- a/arch/arm/include/asm/xor.h +++ b/arch/arm/include/asm/xor.h @@ -51,7 +51,7 @@ xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1, register unsigned int a1 __asm__("r4"); register unsigned int a2 __asm__("r5"); register unsigned int a3 __asm__("r6"); - register unsigned int a4 __asm__("r7"); + register unsigned int a4 __asm__("r10"); register unsigned int b1 __asm__("r8"); register unsigned int b2 __asm__("r9"); register unsigned int b3 __asm__("ip"); @@ -73,7 +73,7 @@ xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1, register unsigned int a1 __asm__("r4"); register unsigned int a2 __asm__("r5"); register unsigned int a3 __asm__("r6"); - register unsigned int a4 __asm__("r7"); + register unsigned int a4 __asm__("r10"); register unsigned int b1 __asm__("r8"); register unsigned int b2 __asm__("r9"); register unsigned int b3 __asm__("ip"); |