diff options
Diffstat (limited to 'arch/powerpc/kernel')
50 files changed, 1223 insertions, 683 deletions
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 53d4b8d5b54d..cb7f0bb9ee71 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -69,7 +69,7 @@ obj-$(CONFIG_FA_DUMP) += fadump.o ifdef CONFIG_PPC32 obj-$(CONFIG_E500) += idle_e500.o endif -obj-$(CONFIG_6xx) += idle_6xx.o l2cr_6xx.o cpu_setup_6xx.o +obj-$(CONFIG_PPC_BOOK3S_32) += idle_6xx.o l2cr_6xx.o cpu_setup_6xx.o obj-$(CONFIG_TAU) += tau_6xx.o obj-$(CONFIG_HIBERNATION) += swsusp.o suspend.o ifdef CONFIG_FSL_BOOKE @@ -160,16 +160,6 @@ extra-$(CONFIG_ALTIVEC) += vector.o extra-$(CONFIG_PPC64) += entry_64.o extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o -extra-y += systbl_chk.i -$(obj)/systbl.o: systbl_chk - -quiet_cmd_systbl_chk = CALL $< - cmd_systbl_chk = $(CONFIG_SHELL) $< $(obj)/systbl_chk.i - -PHONY += systbl_chk -systbl_chk: $(src)/systbl_chk.sh $(obj)/systbl_chk.i - $(call cmd,systbl_chk) - ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE $(obj)/built-in.a: prom_init_check diff --git a/arch/powerpc/kernel/btext.c b/arch/powerpc/kernel/btext.c index b4241ed1456e..6dfceaa820e4 100644 --- a/arch/powerpc/kernel/btext.c +++ b/arch/powerpc/kernel/btext.c @@ -232,20 +232,12 @@ static int btext_initialize(struct device_node *np) int __init btext_find_display(int allow_nonstdout) { - const char *name; - struct device_node *np = NULL; + struct device_node *np = of_stdout; int rc = -ENODEV; - name = of_get_property(of_chosen, "linux,stdout-path", NULL); - if (name != NULL) { - np = of_find_node_by_path(name); - if (np != NULL) { - if (strcmp(np->type, "display") != 0) { - printk("boot stdout isn't a display !\n"); - of_node_put(np); - np = NULL; - } - } + if (!of_node_is_type(np, "display")) { + printk("boot stdout isn't a display !\n"); + np = NULL; } if (np) rc = btext_initialize(np); diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c index be57bd07596d..53102764fd2f 100644 --- a/arch/powerpc/kernel/cacheinfo.c +++ b/arch/powerpc/kernel/cacheinfo.c @@ -428,7 +428,7 @@ static void link_cache_lists(struct cache *smaller, struct cache *bigger) static void do_subsidiary_caches_debugcheck(struct cache *cache) { WARN_ON_ONCE(cache->level != 1); - WARN_ON_ONCE(strcmp(cache->ofnode->type, "cpu")); + WARN_ON_ONCE(!of_node_is_type(cache->ofnode, "cpu")); } static void do_subsidiary_caches(struct cache *cache) diff --git a/arch/powerpc/kernel/cpu_setup_6xx.S b/arch/powerpc/kernel/cpu_setup_6xx.S index fa3c2c91290c..8c069e96c478 100644 --- a/arch/powerpc/kernel/cpu_setup_6xx.S +++ b/arch/powerpc/kernel/cpu_setup_6xx.S @@ -326,7 +326,7 @@ _GLOBAL(__save_cpu_setup) lis r5,cpu_state_storage@h ori r5,r5,cpu_state_storage@l - /* Save HID0 (common to all CONFIG_6xx cpus) */ + /* Save HID0 (common to all CONFIG_PPC_BOOK3S_32 cpus) */ mfspr r3,SPRN_HID0 stw r3,CS_HID0(r5) diff --git a/arch/powerpc/kernel/cpu_setup_fsl_booke.S b/arch/powerpc/kernel/cpu_setup_fsl_booke.S index 8d142e5d84cd..5fbc890d1094 100644 --- a/arch/powerpc/kernel/cpu_setup_fsl_booke.S +++ b/arch/powerpc/kernel/cpu_setup_fsl_booke.S @@ -17,7 +17,7 @@ #include <asm/processor.h> #include <asm/cputable.h> #include <asm/ppc_asm.h> -#include <asm/mmu-book3e.h> +#include <asm/nohash/mmu-book3e.h> #include <asm/asm-offsets.h> #include <asm/mpc85xx.h> diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index 2da01340c84c..1eab54bc6ee9 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -1141,6 +1141,7 @@ static struct cpu_spec __initdata cpu_specs[] = { .machine_check = machine_check_generic, .platform = "ppc603", }, +#ifdef CONFIG_PPC_83xx { /* e300c1 (a 603e core, plus some) on 83xx */ .pvr_mask = 0x7fff0000, .pvr_value = 0x00830000, @@ -1151,7 +1152,7 @@ static struct cpu_spec __initdata cpu_specs[] = { .icache_bsize = 32, .dcache_bsize = 32, .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, + .machine_check = machine_check_83xx, .platform = "ppc603", }, { /* e300c2 (an e300c1 core, plus some, minus FPU) on 83xx */ @@ -1165,7 +1166,7 @@ static struct cpu_spec __initdata cpu_specs[] = { .icache_bsize = 32, .dcache_bsize = 32, .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, + .machine_check = machine_check_83xx, .platform = "ppc603", }, { /* e300c3 (e300c1, plus one IU, half cache size) on 83xx */ @@ -1179,7 +1180,7 @@ static struct cpu_spec __initdata cpu_specs[] = { .icache_bsize = 32, .dcache_bsize = 32, .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, + .machine_check = machine_check_83xx, .num_pmcs = 4, .oprofile_cpu_type = "ppc/e300", .oprofile_type = PPC_OPROFILE_FSL_EMB, @@ -1196,12 +1197,13 @@ static struct cpu_spec __initdata cpu_specs[] = { .icache_bsize = 32, .dcache_bsize = 32, .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, + .machine_check = machine_check_83xx, .num_pmcs = 4, .oprofile_cpu_type = "ppc/e300", .oprofile_type = PPC_OPROFILE_FSL_EMB, .platform = "ppc603", }, +#endif { /* default match, we assume split I/D cache & TB (non-601)... */ .pvr_mask = 0x00000000, .pvr_value = 0x00000000, diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index f9fe2080ceb9..9c9bcaae2f75 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -6,7 +6,6 @@ * busses using the iommu infrastructure */ -#include <linux/export.h> #include <asm/iommu.h> /* @@ -106,11 +105,6 @@ static u64 dma_iommu_get_required_mask(struct device *dev) return mask; } -int dma_iommu_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return dma_addr == IOMMU_MAPPING_ERROR; -} - struct dma_map_ops dma_iommu_ops = { .alloc = dma_iommu_alloc_coherent, .free = dma_iommu_free_coherent, @@ -121,6 +115,4 @@ struct dma_map_ops dma_iommu_ops = { .map_page = dma_iommu_map_page, .unmap_page = dma_iommu_unmap_page, .get_required_mask = dma_iommu_get_required_mask, - .mapping_error = dma_iommu_mapping_error, }; -EXPORT_SYMBOL(dma_iommu_ops); diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c index 5fc335f4d9cd..7d5fc9751622 100644 --- a/arch/powerpc/kernel/dma-swiotlb.c +++ b/arch/powerpc/kernel/dma-swiotlb.c @@ -50,16 +50,15 @@ const struct dma_map_ops powerpc_swiotlb_dma_ops = { .alloc = __dma_nommu_alloc_coherent, .free = __dma_nommu_free_coherent, .mmap = dma_nommu_mmap_coherent, - .map_sg = swiotlb_map_sg_attrs, - .unmap_sg = swiotlb_unmap_sg_attrs, + .map_sg = dma_direct_map_sg, + .unmap_sg = dma_direct_unmap_sg, .dma_supported = swiotlb_dma_supported, - .map_page = swiotlb_map_page, - .unmap_page = swiotlb_unmap_page, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = swiotlb_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = swiotlb_sync_sg_for_device, - .mapping_error = dma_direct_mapping_error, + .map_page = dma_direct_map_page, + .unmap_page = dma_direct_unmap_page, + .sync_single_for_cpu = dma_direct_sync_single_for_cpu, + .sync_single_for_device = dma_direct_sync_single_for_device, + .sync_sg_for_cpu = dma_direct_sync_sg_for_cpu, + .sync_sg_for_device = dma_direct_sync_sg_for_device, .get_required_mask = swiotlb_powerpc_get_required, }; @@ -108,12 +107,8 @@ int __init swiotlb_setup_bus_notifier(void) void __init swiotlb_detect_4g(void) { - if ((memblock_end_of_DRAM() - 1) > 0xffffffff) { + if ((memblock_end_of_DRAM() - 1) > 0xffffffff) ppc_swiotlb_enable = 1; -#ifdef CONFIG_ZONE_DMA32 - limit_zone_pfn(ZONE_DMA32, (1ULL << 32) >> PAGE_SHIFT); -#endif - } } static int __init check_swiotlb_enabled(void) diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c index dbfc7056d7df..b1903ebb2e9c 100644 --- a/arch/powerpc/kernel/dma.c +++ b/arch/powerpc/kernel/dma.c @@ -50,7 +50,8 @@ static int dma_nommu_dma_supported(struct device *dev, u64 mask) return 1; #ifdef CONFIG_FSL_SOC - /* Freescale gets another chance via ZONE_DMA/ZONE_DMA32, however + /* + * Freescale gets another chance via ZONE_DMA, however * that will have to be refined if/when they support iommus */ return 1; @@ -62,18 +63,12 @@ static int dma_nommu_dma_supported(struct device *dev, u64 mask) #endif } +#ifndef CONFIG_NOT_COHERENT_CACHE void *__dma_nommu_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) { void *ret; -#ifdef CONFIG_NOT_COHERENT_CACHE - ret = __dma_alloc_coherent(dev, size, dma_handle, flag); - if (ret == NULL) - return NULL; - *dma_handle += get_dma_offset(dev); - return ret; -#else struct page *page; int node = dev_to_node(dev); #ifdef CONFIG_FSL_SOC @@ -94,13 +89,10 @@ void *__dma_nommu_alloc_coherent(struct device *dev, size_t size, } switch (zone) { +#ifdef CONFIG_ZONE_DMA case ZONE_DMA: flag |= GFP_DMA; break; -#ifdef CONFIG_ZONE_DMA32 - case ZONE_DMA32: - flag |= GFP_DMA32; - break; #endif }; #endif /* CONFIG_FSL_SOC */ @@ -113,19 +105,15 @@ void *__dma_nommu_alloc_coherent(struct device *dev, size_t size, *dma_handle = __pa(ret) + get_dma_offset(dev); return ret; -#endif } void __dma_nommu_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { -#ifdef CONFIG_NOT_COHERENT_CACHE - __dma_free_coherent(size, vaddr); -#else free_pages((unsigned long)vaddr, get_order(size)); -#endif } +#endif /* !CONFIG_NOT_COHERENT_CACHE */ static void *dma_nommu_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, @@ -210,10 +198,15 @@ static int dma_nommu_map_sg(struct device *dev, struct scatterlist *sgl, return nents; } -static void dma_nommu_unmap_sg(struct device *dev, struct scatterlist *sg, +static void dma_nommu_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction direction, unsigned long attrs) { + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nents, i) + __dma_sync_page(sg_page(sg), sg->offset, sg->length, direction); } static u64 dma_nommu_get_required_mask(struct device *dev) @@ -247,6 +240,8 @@ static inline void dma_nommu_unmap_page(struct device *dev, enum dma_data_direction direction, unsigned long attrs) { + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + __dma_sync(bus_to_virt(dma_address), size, direction); } #ifdef CONFIG_NOT_COHERENT_CACHE diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 6cae6b56ffd6..3230137469ab 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1808,10 +1808,10 @@ static int eeh_freeze_dbgfs_get(void *data, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get, - eeh_enable_dbgfs_set, "0x%llx\n"); -DEFINE_SIMPLE_ATTRIBUTE(eeh_freeze_dbgfs_ops, eeh_freeze_dbgfs_get, - eeh_freeze_dbgfs_set, "0x%llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get, + eeh_enable_dbgfs_set, "0x%llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(eeh_freeze_dbgfs_ops, eeh_freeze_dbgfs_get, + eeh_freeze_dbgfs_set, "0x%llx\n"); #endif static int __init eeh_init_proc(void) @@ -1819,12 +1819,12 @@ static int __init eeh_init_proc(void) if (machine_is(pseries) || machine_is(powernv)) { proc_create_single("powerpc/eeh", 0, NULL, proc_eeh_show); #ifdef CONFIG_DEBUG_FS - debugfs_create_file("eeh_enable", 0600, - powerpc_debugfs_root, NULL, - &eeh_enable_dbgfs_ops); - debugfs_create_file("eeh_max_freezes", 0600, - powerpc_debugfs_root, NULL, - &eeh_freeze_dbgfs_ops); + debugfs_create_file_unsafe("eeh_enable", 0600, + powerpc_debugfs_root, NULL, + &eeh_enable_dbgfs_ops); + debugfs_create_file_unsafe("eeh_max_freezes", 0600, + powerpc_debugfs_root, NULL, + &eeh_freeze_dbgfs_ops); #endif } diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 9446248eb6b8..99eab7bc7edc 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -60,7 +60,7 @@ static int eeh_result_priority(enum pci_ers_result result) } }; -const char *pci_ers_result_name(enum pci_ers_result result) +static const char *pci_ers_result_name(enum pci_ers_result result) { switch (result) { case PCI_ERS_RESULT_NONE: diff --git a/arch/powerpc/kernel/eeh_event.c b/arch/powerpc/kernel/eeh_event.c index 61c9356bf9c9..227e57f980df 100644 --- a/arch/powerpc/kernel/eeh_event.c +++ b/arch/powerpc/kernel/eeh_event.c @@ -35,7 +35,7 @@ */ static DEFINE_SPINLOCK(eeh_eventlist_lock); -static struct semaphore eeh_eventlist_sem; +static DECLARE_COMPLETION(eeh_eventlist_event); static LIST_HEAD(eeh_eventlist); /** @@ -55,7 +55,7 @@ static int eeh_event_handler(void * dummy) struct eeh_pe *pe; while (!kthread_should_stop()) { - if (down_interruptible(&eeh_eventlist_sem)) + if (wait_for_completion_interruptible(&eeh_eventlist_event)) break; /* Fetch EEH event from the queue */ @@ -102,9 +102,6 @@ int eeh_event_init(void) struct task_struct *t; int ret = 0; - /* Initialize semaphore */ - sema_init(&eeh_eventlist_sem, 0); - t = kthread_run(eeh_event_handler, NULL, "eehd"); if (IS_ERR(t)) { ret = PTR_ERR(t); @@ -142,7 +139,7 @@ int eeh_send_failure_event(struct eeh_pe *pe) spin_unlock_irqrestore(&eeh_eventlist_lock, flags); /* For EEH deamon to knick in */ - up(&eeh_eventlist_sem); + complete(&eeh_eventlist_event); return 0; } diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 77decded1175..0768dfd8a64e 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -200,14 +200,14 @@ transfer_to_handler: cmplw r1,r9 /* if r1 <= ksp_limit */ ble- stack_ovf /* then the kernel stack overflowed */ 5: -#if defined(CONFIG_6xx) || defined(CONFIG_E500) +#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) CURRENT_THREAD_INFO(r9, r1) tophys(r9,r9) /* check local flags */ lwz r12,TI_LOCAL_FLAGS(r9) mtcrf 0x01,r12 bt- 31-TLF_NAPPING,4f bt- 31-TLF_SLEEPING,7f -#endif /* CONFIG_6xx || CONFIG_E500 */ +#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */ .globl transfer_to_handler_cont transfer_to_handler_cont: 3: @@ -273,7 +273,7 @@ reenable_mmu: /* re-enable mmu so we can */ RFI /* jump to handler, enable MMU */ #endif /* CONFIG_TRACE_IRQFLAGS */ -#if defined (CONFIG_6xx) || defined(CONFIG_E500) +#if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) 4: rlwinm r12,r12,0,~_TLF_NAPPING stw r12,TI_LOCAL_FLAGS(r9) b power_save_ppc32_restore @@ -612,7 +612,7 @@ ppc_swapcontext: handle_page_fault: stw r4,_DAR(r1) addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 andis. r0,r5,DSISR_DABRMATCH@h bne- handle_dabr_fault #endif @@ -629,7 +629,7 @@ handle_page_fault: bl bad_page_fault b ret_from_except_full -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 /* We have a data breakpoint exception - handle it */ handle_dabr_fault: SAVE_NVGPRS(r1) diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 7b1693adff2a..435927f549c4 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -54,6 +54,9 @@ SYS_CALL_TABLE: .tc sys_call_table[TC],sys_call_table +COMPAT_SYS_CALL_TABLE: + .tc compat_sys_call_table[TC],compat_sys_call_table + /* This value is used to mark exception frames on the stack. */ exception_marker: .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER @@ -80,6 +83,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) std r0,GPR0(r1) std r10,GPR1(r1) beq 2f /* if from kernel mode */ +#ifdef CONFIG_PPC_FSL_BOOK3E +START_BTB_FLUSH_SECTION + BTB_FLUSH(r10) +END_BTB_FLUSH_SECTION +#endif ACCOUNT_CPU_USER_ENTRY(r13, r10, r11) 2: std r2,GPR2(r1) std r3,GPR3(r1) @@ -173,7 +181,7 @@ system_call: /* label this so stack traces look sane */ ld r11,SYS_CALL_TABLE@toc(2) andis. r10,r10,_TIF_32BIT@h beq 15f - addi r11,r11,8 /* use 32-bit syscall entries */ + ld r11,COMPAT_SYS_CALL_TABLE@toc(2) clrldi r3,r3,32 clrldi r4,r4,32 clrldi r5,r5,32 @@ -181,7 +189,7 @@ system_call: /* label this so stack traces look sane */ clrldi r7,r7,32 clrldi r8,r8,32 15: - slwi r0,r0,4 + slwi r0,r0,3 barrier_nospec_asm /* @@ -286,6 +294,10 @@ BEGIN_FTR_SECTION HMT_MEDIUM_LOW END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + std r8, PACATMSCRATCH(r13) +#endif + ld r13,GPR13(r1) /* only restore r13 if returning to usermode */ ld r2,GPR2(r1) ld r1,GPR1(r1) diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 6d6e144a28ce..afb638778f44 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -296,7 +296,8 @@ ret_from_mc_except: andi. r10,r11,MSR_PR; /* save stack pointer */ \ beq 1f; /* branch around if supervisor */ \ ld r1,PACAKSAVE(r13); /* get kernel stack coming from usr */\ -1: cmpdi cr1,r1,0; /* check if SP makes sense */ \ +1: type##_BTB_FLUSH \ + cmpdi cr1,r1,0; /* check if SP makes sense */ \ bge- cr1,exc_##n##_bad_stack;/* bad stack (TODO: out of line) */ \ mfspr r10,SPRN_##type##_SRR0; /* read SRR0 before touching stack */ @@ -328,6 +329,29 @@ ret_from_mc_except: #define SPRN_MC_SRR0 SPRN_MCSRR0 #define SPRN_MC_SRR1 SPRN_MCSRR1 +#ifdef CONFIG_PPC_FSL_BOOK3E +#define GEN_BTB_FLUSH \ + START_BTB_FLUSH_SECTION \ + beq 1f; \ + BTB_FLUSH(r10) \ + 1: \ + END_BTB_FLUSH_SECTION + +#define CRIT_BTB_FLUSH \ + START_BTB_FLUSH_SECTION \ + BTB_FLUSH(r10) \ + END_BTB_FLUSH_SECTION + +#define DBG_BTB_FLUSH CRIT_BTB_FLUSH +#define MC_BTB_FLUSH CRIT_BTB_FLUSH +#define GDBELL_BTB_FLUSH GEN_BTB_FLUSH +#else +#define GEN_BTB_FLUSH +#define CRIT_BTB_FLUSH +#define DBG_BTB_FLUSH +#define GDBELL_BTB_FLUSH +#endif + #define NORMAL_EXCEPTION_PROLOG(n, intnum, addition) \ EXCEPTION_PROLOG(n, intnum, GEN, addition##_GEN(n)) diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 89d32bb79d5e..9e253ce27e08 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -995,7 +995,16 @@ EXC_COMMON_BEGIN(h_data_storage_common) bl save_nvgprs RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD +BEGIN_MMU_FTR_SECTION + ld r4,PACA_EXGEN+EX_DAR(r13) + lwz r5,PACA_EXGEN+EX_DSISR(r13) + std r4,_DAR(r1) + std r5,_DSISR(r1) + li r5,SIGSEGV + bl bad_page_fault +MMU_FTR_SECTION_ELSE bl unknown_exception +ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) b ret_from_except @@ -1031,7 +1040,7 @@ TRAMP_REAL_BEGIN(hmi_exception_early) EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN) EXCEPTION_PROLOG_COMMON_3(0xe60) addi r3,r1,STACK_FRAME_OVERHEAD - BRANCH_LINK_TO_FAR(hmi_exception_realmode) /* Function call ABI */ + BRANCH_LINK_TO_FAR(DOTSYM(hmi_exception_realmode)) /* Function call ABI */ cmpdi cr0,r3,0 /* Windup the stack. */ diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index 761b28b1427d..45a8d0be1c96 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -35,6 +35,7 @@ #include <linux/kobject.h> #include <linux/sysfs.h> #include <linux/slab.h> +#include <linux/cma.h> #include <asm/debugfs.h> #include <asm/page.h> @@ -46,6 +47,9 @@ static struct fw_dump fw_dump; static struct fadump_mem_struct fdm; static const struct fadump_mem_struct *fdm_active; +#ifdef CONFIG_CMA +static struct cma *fadump_cma; +#endif static DEFINE_MUTEX(fadump_mutex); struct fad_crash_memory_ranges *crash_memory_ranges; @@ -53,6 +57,67 @@ int crash_memory_ranges_size; int crash_mem_ranges; int max_crash_mem_ranges; +#ifdef CONFIG_CMA +/* + * fadump_cma_init() - Initialize CMA area from a fadump reserved memory + * + * This function initializes CMA area from fadump reserved memory. + * The total size of fadump reserved memory covers for boot memory size + * + cpu data size + hpte size and metadata. + * Initialize only the area equivalent to boot memory size for CMA use. + * The reamining portion of fadump reserved memory will be not given + * to CMA and pages for thoes will stay reserved. boot memory size is + * aligned per CMA requirement to satisy cma_init_reserved_mem() call. + * But for some reason even if it fails we still have the memory reservation + * with us and we can still continue doing fadump. + */ +int __init fadump_cma_init(void) +{ + unsigned long long base, size; + int rc; + + if (!fw_dump.fadump_enabled) + return 0; + + /* + * Do not use CMA if user has provided fadump=nocma kernel parameter. + * Return 1 to continue with fadump old behaviour. + */ + if (fw_dump.nocma) + return 1; + + base = fw_dump.reserve_dump_area_start; + size = fw_dump.boot_memory_size; + + if (!size) + return 0; + + rc = cma_init_reserved_mem(base, size, 0, "fadump_cma", &fadump_cma); + if (rc) { + pr_err("Failed to init cma area for firmware-assisted dump,%d\n", rc); + /* + * Though the CMA init has failed we still have memory + * reservation with us. The reserved memory will be + * blocked from production system usage. Hence return 1, + * so that we can continue with fadump. + */ + return 1; + } + + /* + * So we now have successfully initialized cma area for fadump. + */ + pr_info("Initialized 0x%lx bytes cma area at %ldMB from 0x%lx " + "bytes of memory reserved for firmware-assisted dump\n", + cma_get_size(fadump_cma), + (unsigned long)cma_get_base(fadump_cma) >> 20, + fw_dump.reserve_dump_area_size); + return 1; +} +#else +static int __init fadump_cma_init(void) { return 1; } +#endif /* CONFIG_CMA */ + /* Scan the Firmware Assisted dump configuration details. */ int __init early_init_dt_scan_fw_dump(unsigned long node, const char *uname, int depth, void *data) @@ -118,13 +183,19 @@ int __init early_init_dt_scan_fw_dump(unsigned long node, /* * If fadump is registered, check if the memory provided - * falls within boot memory area. + * falls within boot memory area and reserved memory area. */ -int is_fadump_boot_memory_area(u64 addr, ulong size) +int is_fadump_memory_area(u64 addr, ulong size) { + u64 d_start = fw_dump.reserve_dump_area_start; + u64 d_end = d_start + fw_dump.reserve_dump_area_size; + if (!fw_dump.dump_registered) return 0; + if (((addr + size) > d_start) && (addr <= d_end)) + return 1; + return (addr + size) > RMA_START && addr <= fw_dump.boot_memory_size; } @@ -172,6 +243,35 @@ static int is_boot_memory_area_contiguous(void) return ret; } +/* + * Returns true, if there are no holes in reserved memory area, + * false otherwise. + */ +static bool is_reserved_memory_area_contiguous(void) +{ + struct memblock_region *reg; + unsigned long start, end; + unsigned long d_start = fw_dump.reserve_dump_area_start; + unsigned long d_end = d_start + fw_dump.reserve_dump_area_size; + + for_each_memblock(memory, reg) { + start = max(d_start, (unsigned long)reg->base); + end = min(d_end, (unsigned long)(reg->base + reg->size)); + if (d_start < end) { + /* Memory hole from d_start to start */ + if (start > d_start) + break; + + if (end == d_end) + return true; + + d_start = end + 1; + } + } + + return false; +} + /* Print firmware assisted dump configurations for debugging purpose. */ static void fadump_show_config(void) { @@ -378,8 +478,15 @@ int __init fadump_reserve_mem(void) */ if (fdm_active) fw_dump.boot_memory_size = be64_to_cpu(fdm_active->rmr_region.source_len); - else + else { fw_dump.boot_memory_size = fadump_calculate_reserve_size(); +#ifdef CONFIG_CMA + if (!fw_dump.nocma) + fw_dump.boot_memory_size = + ALIGN(fw_dump.boot_memory_size, + FADUMP_CMA_ALIGNMENT); +#endif + } /* * Calculate the memory boundary. @@ -426,8 +533,9 @@ int __init fadump_reserve_mem(void) fw_dump.fadumphdr_addr = be64_to_cpu(fdm_active->rmr_region.destination_address) + be64_to_cpu(fdm_active->rmr_region.source_len); - pr_debug("fadumphdr_addr = %p\n", - (void *) fw_dump.fadumphdr_addr); + pr_debug("fadumphdr_addr = %pa\n", &fw_dump.fadumphdr_addr); + fw_dump.reserve_dump_area_start = base; + fw_dump.reserve_dump_area_size = size; } else { size = get_fadump_area_size(); @@ -455,10 +563,11 @@ int __init fadump_reserve_mem(void) (unsigned long)(size >> 20), (unsigned long)(base >> 20), (unsigned long)(memblock_phys_mem_size() >> 20)); - } - fw_dump.reserve_dump_area_start = base; - fw_dump.reserve_dump_area_size = size; + fw_dump.reserve_dump_area_start = base; + fw_dump.reserve_dump_area_size = size; + return fadump_cma_init(); + } return 1; } @@ -477,6 +586,10 @@ static int __init early_fadump_param(char *p) fw_dump.fadump_enabled = 1; else if (strncmp(p, "off", 3) == 0) fw_dump.fadump_enabled = 0; + else if (strncmp(p, "nocma", 5) == 0) { + fw_dump.fadump_enabled = 1; + fw_dump.nocma = 1; + } return 0; } @@ -525,8 +638,10 @@ static int register_fw_dump(struct fadump_mem_struct *fdm) break; case -3: if (!is_boot_memory_area_contiguous()) - pr_err("Can't have holes in boot memory area while " - "registering fadump\n"); + pr_err("Can't have holes in boot memory area while registering fadump\n"); + else if (!is_reserved_memory_area_contiguous()) + pr_err("Can't have holes in reserved memory area while" + " registering fadump\n"); printk(KERN_ERR "Failed to register firmware-assisted kernel" " dump. Parameter Error(%d).\n", rc); @@ -1229,7 +1344,7 @@ static int fadump_unregister_dump(struct fadump_mem_struct *fdm) return 0; } -static int fadump_invalidate_dump(struct fadump_mem_struct *fdm) +static int fadump_invalidate_dump(const struct fadump_mem_struct *fdm) { int rc = 0; unsigned int wait_time; @@ -1260,9 +1375,8 @@ void fadump_cleanup(void) { /* Invalidate the registration only if dump is active. */ if (fw_dump.dump_active) { - init_fadump_mem_struct(&fdm, - be64_to_cpu(fdm_active->cpu_state_data.destination_address)); - fadump_invalidate_dump(&fdm); + /* pass the same memory dump structure provided by platform */ + fadump_invalidate_dump(fdm_active); } else if (fw_dump.dump_registered) { /* Un-register Firmware-assisted dump if it was registered. */ fadump_unregister_dump(&fdm); @@ -1531,17 +1645,7 @@ static struct kobj_attribute fadump_register_attr = __ATTR(fadump_registered, 0644, fadump_register_show, fadump_register_store); -static int fadump_region_open(struct inode *inode, struct file *file) -{ - return single_open(file, fadump_region_show, inode->i_private); -} - -static const struct file_operations fadump_region_fops = { - .open = fadump_region_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(fadump_region); static void fadump_init_files(void) { diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S index 61ca27929355..05b08db3901d 100644 --- a/arch/powerpc/kernel/head_32.S +++ b/arch/powerpc/kernel/head_32.S @@ -176,10 +176,10 @@ __after_mmu_off: bl reloc_offset li r24,0 /* cpu# */ bl call_setup_cpu /* Call setup_cpu for this CPU */ -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 bl reloc_offset bl init_idle_6xx -#endif /* CONFIG_6xx */ +#endif /* CONFIG_PPC_BOOK3S_32 */ /* @@ -393,7 +393,9 @@ DataAccess: bne 1f /* if not, try to put a PTE */ mfspr r4,SPRN_DAR /* into the hash table */ rlwinm r3,r10,32-15,21,21 /* DSISR_STORE -> _PAGE_RW */ +BEGIN_MMU_FTR_SECTION bl hash_page +END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) 1: lwz r5,_DSISR(r11) /* get DSISR value */ mfspr r4,SPRN_DAR EXC_XFER_LITE(0x300, handle_page_fault) @@ -408,7 +410,9 @@ InstructionAccess: beq 1f /* if so, try to put a PTE */ li r3,0 /* into the hash table */ mr r4,r12 /* SRR0 is fault address */ +BEGIN_MMU_FTR_SECTION bl hash_page +END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) 1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ EXC_XFER_LITE(0x400, handle_page_fault) @@ -499,7 +503,7 @@ InstructionTLBMiss: lis r1,PAGE_OFFSET@h /* check if kernel address */ cmplw 0,r1,r3 mfspr r2,SPRN_SPRG_THREAD - li r1,_PAGE_USER|_PAGE_PRESENT /* low addresses tested as user */ + li r1,_PAGE_USER|_PAGE_PRESENT|_PAGE_EXEC /* low addresses tested as user */ lwz r2,PGDIR(r2) bge- 112f mfspr r2,SPRN_SRR1 /* and MSR_PR bit from SRR1 */ @@ -836,10 +840,10 @@ __secondary_start: lis r3,-KERNELBASE@h mr r4,r24 bl call_setup_cpu /* Call setup_cpu for this CPU */ -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 lis r3,-KERNELBASE@h bl init_idle_6xx -#endif /* CONFIG_6xx */ +#endif /* CONFIG_PPC_BOOK3S_32 */ /* get current_thread_info and current */ lis r1,secondary_ti@ha @@ -880,14 +884,14 @@ __secondary_start: /* * Those generic dummy functions are kept for CPUs not - * included in CONFIG_6xx + * included in CONFIG_PPC_BOOK3S_32 */ -#if !defined(CONFIG_6xx) +#if !defined(CONFIG_PPC_BOOK3S_32) _ENTRY(__save_cpu_setup) blr _ENTRY(__restore_cpu_setup) blr -#endif /* !defined(CONFIG_6xx) */ +#endif /* !defined(CONFIG_PPC_BOOK3S_32) */ /* diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 37e4a7cf0065..bf23c19c92d6 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -40,6 +40,7 @@ #include <asm/ptrace.h> #include <asm/synch.h> #include <asm/export.h> +#include <asm/code-patching-asm.h> #include "head_booke.h" @@ -382,10 +383,9 @@ interrupt_base: /* Increment, rollover, and store TLB index */ addi r13,r13,1 + patch_site 0f, patch__tlb_44x_hwater_D /* Compare with watermark (instruction gets patched) */ - .globl tlb_44x_patch_hwater_D -tlb_44x_patch_hwater_D: - cmpwi 0,r13,1 /* reserve entries */ +0: cmpwi 0,r13,1 /* reserve entries */ ble 5f li r13,0 5: @@ -478,10 +478,9 @@ tlb_44x_patch_hwater_D: /* Increment, rollover, and store TLB index */ addi r13,r13,1 + patch_site 0f, patch__tlb_44x_hwater_I /* Compare with watermark (instruction gets patched) */ - .globl tlb_44x_patch_hwater_I -tlb_44x_patch_hwater_I: - cmpwi 0,r13,1 /* reserve entries */ +0: cmpwi 0,r13,1 /* reserve entries */ ble 5f li r13,0 5: diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 3b67b9533c82..57deb1e9ffea 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -106,6 +106,23 @@ turn_on_mmu: mtspr SPRN_SRR0,r0 rfi /* enables MMU */ + +#ifdef CONFIG_PERF_EVENTS + .align 4 + + .globl itlb_miss_counter +itlb_miss_counter: + .space 4 + + .globl dtlb_miss_counter +dtlb_miss_counter: + .space 4 + + .globl instruction_counter +instruction_counter: + .space 4 +#endif + /* * Exception entry code. This code runs with address translation * turned off, i.e. using physical addresses. @@ -149,6 +166,9 @@ turn_on_mmu: li r10,MSR_KERNEL & ~(MSR_IR|MSR_DR); /* can take exceptions */ \ mtmsr r10; \ stw r0,GPR0(r11); \ + lis r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */ \ + addi r10, r10, STACK_FRAME_REGS_MARKER@l; \ + stw r10, 8(r11); \ SAVE_4GPRS(3, r11); \ SAVE_2GPRS(7, r11) @@ -275,7 +295,7 @@ SystemCall: . = 0x1100 /* * For the MPC8xx, this is a software tablewalk to load the instruction - * TLB. The task switch loads the M_TW register with the pointer to the first + * TLB. The task switch loads the M_TWB register with the pointer to the first * level table. * If we discover there is no second level table (value is zero) or if there * is an invalid pte, we load that into the TLB, which causes another fault @@ -285,186 +305,154 @@ SystemCall: */ #ifdef CONFIG_8xx_CPU15 -#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr) \ - addi tmp, addr, PAGE_SIZE; \ - tlbie tmp; \ - addi tmp, addr, -PAGE_SIZE; \ - tlbie tmp +#define INVALIDATE_ADJACENT_PAGES_CPU15(addr) \ + addi addr, addr, PAGE_SIZE; \ + tlbie addr; \ + addi addr, addr, -(PAGE_SIZE << 1); \ + tlbie addr; \ + addi addr, addr, PAGE_SIZE #else -#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr) +#define INVALIDATE_ADJACENT_PAGES_CPU15(addr) #endif InstructionTLBMiss: mtspr SPRN_SPRG_SCRATCH0, r10 +#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) mtspr SPRN_SPRG_SCRATCH1, r11 -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) - mtspr SPRN_SPRG_SCRATCH2, r12 #endif /* If we are faulting a kernel address, we have to use the * kernel page tables. */ mfspr r10, SPRN_SRR0 /* Get effective address of fault */ - INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10) + INVALIDATE_ADJACENT_PAGES_CPU15(r10) + mtspr SPRN_MD_EPN, r10 /* Only modules will cause ITLB Misses as we always * pin the first 8MB of kernel memory */ -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) - mfcr r12 -#endif #ifdef ITLB_MISS_KERNEL + mfcr r11 #if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT) - andis. r11, r10, 0x8000 /* Address >= 0x80000000 */ + cmpi cr0, r10, 0 /* Address >= 0x80000000 */ #else - rlwinm r11, r10, 16, 0xfff8 - cmpli cr0, r11, PAGE_OFFSET@h + rlwinm r10, r10, 16, 0xfff8 + cmpli cr0, r10, PAGE_OFFSET@h #ifndef CONFIG_PIN_TLB_TEXT /* It is assumed that kernel code fits into the first 8M page */ -0: cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h +0: cmpli cr7, r10, (PAGE_OFFSET + 0x0800000)@h patch_site 0b, patch__itlbmiss_linmem_top #endif #endif #endif - mfspr r11, SPRN_M_TW /* Get level 1 table */ + mfspr r10, SPRN_M_TWB /* Get level 1 table */ #ifdef ITLB_MISS_KERNEL #if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT) - beq+ 3f + bge+ 3f #else blt+ 3f #endif #ifndef CONFIG_PIN_TLB_TEXT blt cr7, ITLBMissLinear #endif - lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha + rlwinm r10, r10, 0, 20, 31 + oris r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha 3: #endif - /* Insert level 1 index */ - rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29 - lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */ + lwz r10, (swapper_pg_dir-PAGE_OFFSET)@l(r10) /* Get level 1 entry */ + mtspr SPRN_MI_TWC, r10 /* Set segment attributes */ - /* Extract level 2 index */ - rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29 -#ifdef CONFIG_HUGETLB_PAGE - mtcr r11 - bt- 28, 10f /* bit 28 = Large page (8M) */ - bt- 29, 20f /* bit 29 = Large page (8M or 512k) */ -#endif - rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */ + mtspr SPRN_MD_TWC, r10 + mfspr r10, SPRN_MD_TWC lwz r10, 0(r10) /* Get the pte */ -4: -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) - mtcr r12 +#ifdef ITLB_MISS_KERNEL + mtcr r11 #endif - /* Load the MI_TWC with the attributes for this "segment." */ - mtspr SPRN_MI_TWC, r11 /* Set segment attributes */ - #ifdef CONFIG_SWAP rlwinm r11, r10, 32-5, _PAGE_PRESENT and r11, r11, r10 rlwimi r10, r11, 0, _PAGE_PRESENT #endif - li r11, RPN_PATTERN | 0x200 /* The Linux PTE won't go exactly into the MMU TLB. * Software indicator bits 20 and 23 must be clear. * Software indicator bits 22, 24, 25, 26, and 27 must be * set. All other Linux PTE bits control the behavior * of the MMU. */ - rlwimi r11, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */ - rlwimi r10, r11, 0, 0x0ff0 /* Set 22, 24-27, clear 20,23 */ + rlwimi r10, r10, 0, 0x0f00 /* Clear bits 20-23 */ + rlwimi r10, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */ + ori r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */ mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ /* Restore registers */ 0: mfspr r10, SPRN_SPRG_SCRATCH0 +#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) mfspr r11, SPRN_SPRG_SCRATCH1 -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) - mfspr r12, SPRN_SPRG_SCRATCH2 #endif rfi patch_site 0b, patch__itlbmiss_exit_1 #ifdef CONFIG_PERF_EVENTS patch_site 0f, patch__itlbmiss_perf -0: lis r10, (itlb_miss_counter - PAGE_OFFSET)@ha - lwz r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10) - addi r11, r11, 1 - stw r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10) -#endif +0: lwz r10, (itlb_miss_counter - PAGE_OFFSET)@l(0) + addi r10, r10, 1 + stw r10, (itlb_miss_counter - PAGE_OFFSET)@l(0) mfspr r10, SPRN_SPRG_SCRATCH0 +#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP) mfspr r11, SPRN_SPRG_SCRATCH1 -#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) - mfspr r12, SPRN_SPRG_SCRATCH2 #endif rfi - -#ifdef CONFIG_HUGETLB_PAGE -10: /* 8M pages */ -#ifdef CONFIG_PPC_16K_PAGES - /* Extract level 2 index */ - rlwinm r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29 - /* Add level 2 base */ - rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1 -#else - /* Level 2 base */ - rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK #endif - lwz r10, 0(r10) /* Get the pte */ - b 4b -20: /* 512k pages */ - /* Extract level 2 index */ - rlwinm r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29 - /* Add level 2 base */ - rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1 - lwz r10, 0(r10) /* Get the pte */ - b 4b +#ifndef CONFIG_PIN_TLB_TEXT +ITLBMissLinear: + mtcr r11 + /* Set 8M byte page and mark it valid */ + li r11, MI_PS8MEG | MI_SVALID + mtspr SPRN_MI_TWC, r11 + rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */ + ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SH | _PAGE_DIRTY | \ + _PAGE_PRESENT + mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ + +0: mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH1 + rfi + patch_site 0b, patch__itlbmiss_exit_2 #endif . = 0x1200 DataStoreTLBMiss: mtspr SPRN_SPRG_SCRATCH0, r10 mtspr SPRN_SPRG_SCRATCH1, r11 - mtspr SPRN_SPRG_SCRATCH2, r12 - mfcr r12 + mfcr r11 /* If we are faulting a kernel address, we have to use the * kernel page tables. */ mfspr r10, SPRN_MD_EPN - rlwinm r11, r10, 16, 0xfff8 - cmpli cr0, r11, PAGE_OFFSET@h - mfspr r11, SPRN_M_TW /* Get level 1 table */ - blt+ 3f - rlwinm r11, r10, 16, 0xfff8 + rlwinm r10, r10, 16, 0xfff8 + cmpli cr0, r10, PAGE_OFFSET@h #ifndef CONFIG_PIN_TLB_IMMR - cmpli cr0, r11, VIRT_IMMR_BASE@h + cmpli cr6, r10, VIRT_IMMR_BASE@h #endif -0: cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h +0: cmpli cr7, r10, (PAGE_OFFSET + 0x1800000)@h patch_site 0b, patch__dtlbmiss_linmem_top + + mfspr r10, SPRN_M_TWB /* Get level 1 table */ + blt+ 3f #ifndef CONFIG_PIN_TLB_IMMR -0: beq- DTLBMissIMMR +0: beq- cr6, DTLBMissIMMR patch_site 0b, patch__dtlbmiss_immr_jmp #endif blt cr7, DTLBMissLinear - lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha + rlwinm r10, r10, 0, 20, 31 + oris r10, r10, (swapper_pg_dir - PAGE_OFFSET)@ha 3: - - /* Insert level 1 index */ - rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29 - lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */ - - /* We have a pte table, so load fetch the pte from the table. - */ - /* Extract level 2 index */ - rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29 -#ifdef CONFIG_HUGETLB_PAGE mtcr r11 - bt- 28, 10f /* bit 28 = Large page (8M) */ - bt- 29, 20f /* bit 29 = Large page (8M or 512k) */ -#endif - rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */ + lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r10) /* Get level 1 entry */ + + mtspr SPRN_MD_TWC, r11 + mfspr r10, SPRN_MD_TWC lwz r10, 0(r10) /* Get the pte */ -4: - mtcr r12 /* Insert the Guarded flag into the TWC from the Linux PTE. * It is bit 27 of both the Linux PTE and the TWC (at least @@ -503,44 +491,55 @@ DataStoreTLBMiss: 0: mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r12, SPRN_SPRG_SCRATCH2 rfi patch_site 0b, patch__dtlbmiss_exit_1 #ifdef CONFIG_PERF_EVENTS patch_site 0f, patch__dtlbmiss_perf -0: lis r10, (dtlb_miss_counter - PAGE_OFFSET)@ha - lwz r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10) - addi r11, r11, 1 - stw r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10) -#endif +0: lwz r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) + addi r10, r10, 1 + stw r10, (dtlb_miss_counter - PAGE_OFFSET)@l(0) mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r12, SPRN_SPRG_SCRATCH2 rfi - -#ifdef CONFIG_HUGETLB_PAGE -10: /* 8M pages */ - /* Extract level 2 index */ -#ifdef CONFIG_PPC_16K_PAGES - rlwinm r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29 - /* Add level 2 base */ - rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1 -#else - /* Level 2 base */ - rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK #endif - lwz r10, 0(r10) /* Get the pte */ - b 4b -20: /* 512k pages */ - /* Extract level 2 index */ - rlwinm r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29 - /* Add level 2 base */ - rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1 - lwz r10, 0(r10) /* Get the pte */ - b 4b -#endif +DTLBMissIMMR: + mtcr r11 + /* Set 512k byte guarded page and mark it valid */ + li r10, MD_PS512K | MD_GUARDED | MD_SVALID + mtspr SPRN_MD_TWC, r10 + mfspr r10, SPRN_IMMR /* Get current IMMR */ + rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */ + ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \ + _PAGE_PRESENT | _PAGE_NO_CACHE + mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ + + li r11, RPN_PATTERN + mtspr SPRN_DAR, r11 /* Tag DAR */ + +0: mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH1 + rfi + patch_site 0b, patch__dtlbmiss_exit_2 + +DTLBMissLinear: + mtcr r11 + /* Set 8M byte page and mark it valid */ + li r11, MD_PS8MEG | MD_SVALID + mtspr SPRN_MD_TWC, r11 + rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */ + ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \ + _PAGE_PRESENT + mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ + + li r11, RPN_PATTERN + mtspr SPRN_DAR, r11 /* Tag DAR */ + +0: mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH1 + rfi + patch_site 0b, patch__dtlbmiss_exit_3 /* This is an instruction TLB error on the MPC8xx. This could be due * to many reasons, such as executing guarded memory or illegal instruction @@ -625,16 +624,13 @@ DataBreakpoint: . = 0x1d00 InstructionBreakpoint: mtspr SPRN_SPRG_SCRATCH0, r10 - mtspr SPRN_SPRG_SCRATCH1, r11 - lis r10, (instruction_counter - PAGE_OFFSET)@ha - lwz r11, (instruction_counter - PAGE_OFFSET)@l(r10) - addi r11, r11, -1 - stw r11, (instruction_counter - PAGE_OFFSET)@l(r10) + lwz r10, (instruction_counter - PAGE_OFFSET)@l(0) + addi r10, r10, -1 + stw r10, (instruction_counter - PAGE_OFFSET)@l(0) lis r10, 0xffff ori r10, r10, 0x01 mtspr SPRN_COUNTA, r10 mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 rfi #else EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_EE) @@ -644,67 +640,6 @@ InstructionBreakpoint: . = 0x2000 -/* - * Bottom part of DataStoreTLBMiss handlers for IMMR area and linear RAM. - * not enough space in the DataStoreTLBMiss area. - */ -DTLBMissIMMR: - mtcr r12 - /* Set 512k byte guarded page and mark it valid */ - li r10, MD_PS512K | MD_GUARDED | MD_SVALID - mtspr SPRN_MD_TWC, r10 - mfspr r10, SPRN_IMMR /* Get current IMMR */ - rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */ - ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \ - _PAGE_PRESENT | _PAGE_NO_CACHE - mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ - - li r11, RPN_PATTERN - mtspr SPRN_DAR, r11 /* Tag DAR */ - -0: mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r12, SPRN_SPRG_SCRATCH2 - rfi - patch_site 0b, patch__dtlbmiss_exit_2 - -DTLBMissLinear: - mtcr r12 - /* Set 8M byte page and mark it valid */ - li r11, MD_PS8MEG | MD_SVALID - mtspr SPRN_MD_TWC, r11 - rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ - ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SH | _PAGE_DIRTY | \ - _PAGE_PRESENT - mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ - - li r11, RPN_PATTERN - mtspr SPRN_DAR, r11 /* Tag DAR */ - -0: mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r12, SPRN_SPRG_SCRATCH2 - rfi - patch_site 0b, patch__dtlbmiss_exit_3 - -#ifndef CONFIG_PIN_TLB_TEXT -ITLBMissLinear: - mtcr r12 - /* Set 8M byte page and mark it valid */ - li r11, MI_PS8MEG | MI_SVALID - mtspr SPRN_MI_TWC, r11 - rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ - ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SH | _PAGE_DIRTY | \ - _PAGE_PRESENT - mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ - -0: mfspr r10, SPRN_SPRG_SCRATCH0 - mfspr r11, SPRN_SPRG_SCRATCH1 - mfspr r12, SPRN_SPRG_SCRATCH2 - rfi - patch_site 0b, patch__itlbmiss_exit_2 -#endif - /* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions * by decoding the registers used by the dcbx instruction and adding them. * DAR is set to the calculated address. @@ -712,12 +647,13 @@ ITLBMissLinear: /* define if you don't want to use self modifying code */ #define NO_SELF_MODIFYING_CODE FixupDAR:/* Entry point for dcbx workaround. */ - mtspr SPRN_SPRG_SCRATCH2, r10 + mtspr SPRN_M_TW, r10 /* fetch instruction from memory. */ mfspr r10, SPRN_SRR0 + mtspr SPRN_MD_EPN, r10 rlwinm r11, r10, 16, 0xfff8 cmpli cr0, r11, PAGE_OFFSET@h - mfspr r11, SPRN_M_TW /* Get level 1 table */ + mfspr r11, SPRN_M_TWB /* Get level 1 table */ blt+ 3f rlwinm r11, r10, 16, 0xfff8 @@ -727,17 +663,17 @@ FixupDAR:/* Entry point for dcbx workaround. */ /* create physical page address from effective address */ tophys(r11, r10) blt- cr7, 201f - lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha - /* Insert level 1 index */ -3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29 + mfspr r11, SPRN_M_TWB /* Get level 1 table */ + rlwinm r11, r11, 0, 20, 31 + oris r11, r11, (swapper_pg_dir - PAGE_OFFSET)@ha +3: lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */ + mtspr SPRN_MD_TWC, r11 mtcr r11 + mfspr r11, SPRN_MD_TWC + lwz r11, 0(r11) /* Get the pte */ bt 28,200f /* bit 28 = Large page (8M) */ bt 29,202f /* bit 29 = Large page (8M or 512K) */ - rlwinm r11, r11,0,0,19 /* Extract page descriptor page address */ - /* Insert level 2 index */ - rlwimi r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29 - lwz r11, 0(r11) /* Get the pte */ /* concat physical page address(r11) and page offset(r10) */ rlwimi r11, r10, 0, 32 - PAGE_SHIFT, 31 201: lwz r11,0(r11) @@ -756,26 +692,15 @@ FixupDAR:/* Entry point for dcbx workaround. */ beq+ 142f cmpwi cr0, r10, 1964 /* Is icbi? */ beq+ 142f -141: mfspr r10,SPRN_SPRG_SCRATCH2 +141: mfspr r10,SPRN_M_TW b DARFixed /* Nope, go back to normal TLB processing */ - /* concat physical page address(r11) and page offset(r10) */ 200: -#ifdef CONFIG_PPC_16K_PAGES - rlwinm r11, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1 - rlwimi r11, r10, 32 - (PAGE_SHIFT_8M - 2), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29 -#else - rlwinm r11, r10, 0, ~HUGEPD_SHIFT_MASK -#endif - lwz r11, 0(r11) /* Get the pte */ /* concat physical page address(r11) and page offset(r10) */ rlwimi r11, r10, 0, 32 - PAGE_SHIFT_8M, 31 b 201b 202: - rlwinm r11, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1 - rlwimi r11, r10, 32 - (PAGE_SHIFT_512K - 2), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29 - lwz r11, 0(r11) /* Get the pte */ /* concat physical page address(r11) and page offset(r10) */ rlwimi r11, r10, 0, 32 - PAGE_SHIFT_512K, 31 b 201b @@ -802,7 +727,7 @@ modified_instr: bne+ 143f subf r10,r0,r10 /* r10=r10-r0, only if reg RA is r0 */ 143: mtdar r10 /* store faulting EA in DAR */ - mfspr r10,SPRN_SPRG_SCRATCH2 + mfspr r10,SPRN_M_TW b DARFixed /* Go back to normal TLB handling */ #else mfctr r10 @@ -856,7 +781,7 @@ modified_instr: mfdar r11 mtctr r11 /* restore ctr reg from DAR */ mtdar r10 /* save fault EA to DAR */ - mfspr r10,SPRN_SPRG_SCRATCH2 + mfspr r10,SPRN_M_TW b DARFixed /* Go back to normal TLB handling */ /* special handling for r10,r11 since these are modified already */ @@ -891,7 +816,7 @@ start_here: lis r6, swapper_pg_dir@ha tophys(r6,r6) - mtspr SPRN_M_TW, r6 + mtspr SPRN_M_TWB, r6 bl early_init /* We have to do this with MMU on */ @@ -1065,17 +990,3 @@ swapper_pg_dir: */ abatron_pteptrs: .space 8 - -#ifdef CONFIG_PERF_EVENTS - .globl itlb_miss_counter -itlb_miss_counter: - .space 4 - - .globl dtlb_miss_counter -dtlb_miss_counter: - .space 4 - - .globl instruction_counter -instruction_counter: - .space 4 -#endif diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h index d0862a100d29..15ac51072eb3 100644 --- a/arch/powerpc/kernel/head_booke.h +++ b/arch/powerpc/kernel/head_booke.h @@ -43,6 +43,9 @@ andi. r11, r11, MSR_PR; /* check whether user or kernel */\ mr r11, r1; \ beq 1f; \ +START_BTB_FLUSH_SECTION \ + BTB_FLUSH(r11) \ +END_BTB_FLUSH_SECTION \ /* if from user, start at top of this thread's kernel stack */ \ lwz r11, THREAD_INFO-THREAD(r10); \ ALLOC_STACK_FRAME(r11, THREAD_SIZE); \ @@ -128,6 +131,9 @@ stw r9,_CCR(r8); /* save CR on stack */\ mfspr r11,exc_level_srr1; /* check whether user or kernel */\ DO_KVM BOOKE_INTERRUPT_##intno exc_level_srr1; \ +START_BTB_FLUSH_SECTION \ + BTB_FLUSH(r10) \ +END_BTB_FLUSH_SECTION \ andi. r11,r11,MSR_PR; \ mfspr r11,SPRN_SPRG_THREAD; /* if from user, start at top of */\ lwz r11,THREAD_INFO-THREAD(r11); /* this thread's kernel stack */\ diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index e2750b856c8f..2386ce2a9c6e 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -453,6 +453,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) mfcr r13 stw r13, THREAD_NORMSAVE(3)(r10) DO_KVM BOOKE_INTERRUPT_DTLB_MISS SPRN_SRR1 +START_BTB_FLUSH_SECTION + mfspr r11, SPRN_SRR1 + andi. r10,r11,MSR_PR + beq 1f + BTB_FLUSH(r10) +1: +END_BTB_FLUSH_SECTION mfspr r10, SPRN_DEAR /* Get faulting address */ /* If we are faulting a kernel address, we have to use the @@ -547,6 +554,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) mfcr r13 stw r13, THREAD_NORMSAVE(3)(r10) DO_KVM BOOKE_INTERRUPT_ITLB_MISS SPRN_SRR1 +START_BTB_FLUSH_SECTION + mfspr r11, SPRN_SRR1 + andi. r10,r11,MSR_PR + beq 1f + BTB_FLUSH(r10) +1: +END_BTB_FLUSH_SECTION + mfspr r10, SPRN_SRR0 /* Get faulting address */ /* If we are faulting a kernel address, we have to use the diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index f0dc680e659a..d0625480b59e 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -47,6 +47,7 @@ #include <asm/fadump.h> #include <asm/vio.h> #include <asm/tce.h> +#include <asm/mmu_context.h> #define DBG(...) @@ -197,11 +198,11 @@ static unsigned long iommu_range_alloc(struct device *dev, if (unlikely(npages == 0)) { if (printk_ratelimit()) WARN_ON(1); - return IOMMU_MAPPING_ERROR; + return DMA_MAPPING_ERROR; } if (should_fail_iommu(dev)) - return IOMMU_MAPPING_ERROR; + return DMA_MAPPING_ERROR; /* * We don't need to disable preemption here because any CPU can @@ -277,7 +278,7 @@ again: } else { /* Give up */ spin_unlock_irqrestore(&(pool->lock), flags); - return IOMMU_MAPPING_ERROR; + return DMA_MAPPING_ERROR; } } @@ -309,13 +310,13 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl, unsigned long attrs) { unsigned long entry; - dma_addr_t ret = IOMMU_MAPPING_ERROR; + dma_addr_t ret = DMA_MAPPING_ERROR; int build_fail; entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order); - if (unlikely(entry == IOMMU_MAPPING_ERROR)) - return IOMMU_MAPPING_ERROR; + if (unlikely(entry == DMA_MAPPING_ERROR)) + return DMA_MAPPING_ERROR; entry += tbl->it_offset; /* Offset into real TCE table */ ret = entry << tbl->it_page_shift; /* Set the return dma address */ @@ -327,12 +328,12 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl, /* tbl->it_ops->set() only returns non-zero for transient errors. * Clean up the table bitmap in this case and return - * IOMMU_MAPPING_ERROR. For all other errors the functionality is + * DMA_MAPPING_ERROR. For all other errors the functionality is * not altered. */ if (unlikely(build_fail)) { __iommu_free(tbl, ret, npages); - return IOMMU_MAPPING_ERROR; + return DMA_MAPPING_ERROR; } /* Flush/invalidate TLB caches if necessary */ @@ -477,7 +478,7 @@ int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen); /* Handle failure */ - if (unlikely(entry == IOMMU_MAPPING_ERROR)) { + if (unlikely(entry == DMA_MAPPING_ERROR)) { if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) dev_info(dev, "iommu_alloc failed, tbl %p " @@ -544,7 +545,7 @@ int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, */ if (outcount < incount) { outs = sg_next(outs); - outs->dma_address = IOMMU_MAPPING_ERROR; + outs->dma_address = DMA_MAPPING_ERROR; outs->dma_length = 0; } @@ -562,7 +563,7 @@ int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, npages = iommu_num_pages(s->dma_address, s->dma_length, IOMMU_PAGE_SIZE(tbl)); __iommu_free(tbl, vaddr, npages); - s->dma_address = IOMMU_MAPPING_ERROR; + s->dma_address = DMA_MAPPING_ERROR; s->dma_length = 0; } if (s == outs) @@ -776,7 +777,7 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, unsigned long mask, enum dma_data_direction direction, unsigned long attrs) { - dma_addr_t dma_handle = IOMMU_MAPPING_ERROR; + dma_addr_t dma_handle = DMA_MAPPING_ERROR; void *vaddr; unsigned long uaddr; unsigned int npages, align; @@ -796,7 +797,7 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction, mask >> tbl->it_page_shift, align, attrs); - if (dma_handle == IOMMU_MAPPING_ERROR) { + if (dma_handle == DMA_MAPPING_ERROR) { if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) { dev_info(dev, "iommu_alloc failed, tbl %p " @@ -868,7 +869,7 @@ void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, io_order = get_iommu_order(size, tbl); mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL, mask >> tbl->it_page_shift, io_order, 0); - if (mapping == IOMMU_MAPPING_ERROR) { + if (mapping == DMA_MAPPING_ERROR) { free_pages((unsigned long)ret, order); return NULL; } @@ -993,15 +994,19 @@ int iommu_tce_check_gpa(unsigned long page_shift, unsigned long gpa) } EXPORT_SYMBOL_GPL(iommu_tce_check_gpa); -long iommu_tce_xchg(struct iommu_table *tbl, unsigned long entry, - unsigned long *hpa, enum dma_data_direction *direction) +long iommu_tce_xchg(struct mm_struct *mm, struct iommu_table *tbl, + unsigned long entry, unsigned long *hpa, + enum dma_data_direction *direction) { long ret; + unsigned long size = 0; ret = tbl->it_ops->exchange(tbl, entry, hpa, direction); if (!ret && ((*direction == DMA_FROM_DEVICE) || - (*direction == DMA_BIDIRECTIONAL))) + (*direction == DMA_BIDIRECTIONAL)) && + !mm_iommu_is_devmem(mm, *hpa, tbl->it_page_shift, + &size)) SetPageDirty(pfn_to_page(*hpa >> PAGE_SHIFT)); /* if (unlikely(ret)) @@ -1073,11 +1078,8 @@ void iommu_release_ownership(struct iommu_table *tbl) } EXPORT_SYMBOL_GPL(iommu_release_ownership); -int iommu_add_device(struct device *dev) +int iommu_add_device(struct iommu_table_group *table_group, struct device *dev) { - struct iommu_table *tbl; - struct iommu_table_group_link *tgl; - /* * The sysfs entries should be populated before * binding IOMMU group. If sysfs entries isn't @@ -1093,32 +1095,10 @@ int iommu_add_device(struct device *dev) return -EBUSY; } - tbl = get_iommu_table_base(dev); - if (!tbl) { - pr_debug("%s: Skipping device %s with no tbl\n", - __func__, dev_name(dev)); - return 0; - } - - tgl = list_first_entry_or_null(&tbl->it_group_list, - struct iommu_table_group_link, next); - if (!tgl) { - pr_debug("%s: Skipping device %s with no group\n", - __func__, dev_name(dev)); - return 0; - } pr_debug("%s: Adding %s to iommu group %d\n", - __func__, dev_name(dev), - iommu_group_id(tgl->table_group->group)); - - if (PAGE_SIZE < IOMMU_PAGE_SIZE(tbl)) { - pr_err("%s: Invalid IOMMU page size %lx (%lx) on %s\n", - __func__, IOMMU_PAGE_SIZE(tbl), - PAGE_SIZE, dev_name(dev)); - return -EINVAL; - } + __func__, dev_name(dev), iommu_group_id(table_group->group)); - return iommu_group_add_device(tgl->table_group->group, dev); + return iommu_group_add_device(table_group->group, dev); } EXPORT_SYMBOL_GPL(iommu_add_device); @@ -1138,31 +1118,4 @@ void iommu_del_device(struct device *dev) iommu_group_remove_device(dev); } EXPORT_SYMBOL_GPL(iommu_del_device); - -static int tce_iommu_bus_notifier(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct device *dev = data; - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: - return iommu_add_device(dev); - case BUS_NOTIFY_DEL_DEVICE: - if (dev->iommu_group) - iommu_del_device(dev); - return 0; - default: - return 0; - } -} - -static struct notifier_block tce_iommu_bus_nb = { - .notifier_call = tce_iommu_bus_notifier, -}; - -int __init tce_iommu_bus_notifier_init(void) -{ - bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb); - return 0; -} #endif /* CONFIG_IOMMU_API */ diff --git a/arch/powerpc/kernel/isa-bridge.c b/arch/powerpc/kernel/isa-bridge.c index fda3ae48480c..0e7099da4f25 100644 --- a/arch/powerpc/kernel/isa-bridge.c +++ b/arch/powerpc/kernel/isa-bridge.c @@ -327,8 +327,7 @@ static int isa_bridge_notify(struct notifier_block *nb, unsigned long action, /* Check if we have no ISA device, and this happens to be one, * register it as such if it has an OF device */ - if (!isa_bridge_devnode && devnode && devnode->type && - !strcmp(devnode->type, "isa")) + if (!isa_bridge_devnode && of_node_is_type(devnode, "isa")) isa_bridge_find_late(pdev, devnode); return 0; diff --git a/arch/powerpc/kernel/legacy_serial.c b/arch/powerpc/kernel/legacy_serial.c index 33b34a58fc62..7cea5978f21f 100644 --- a/arch/powerpc/kernel/legacy_serial.c +++ b/arch/powerpc/kernel/legacy_serial.c @@ -192,7 +192,7 @@ static int __init add_legacy_soc_port(struct device_node *np, /* Add port, irq will be dealt with later. We passed a translated * IO port value. It will be fixed up later along with the irq */ - if (tsi && !strcmp(tsi->type, "tsi-bridge")) + if (of_node_is_type(tsi, "tsi-bridge")) return add_legacy_port(np, -1, UPIO_TSI, addr, addr, 0, legacy_port_flags, 0); else @@ -372,6 +372,8 @@ void __init find_legacy_serial_ports(void) /* Now find out if one of these is out firmware console */ path = of_get_property(of_chosen, "linux,stdout-path", NULL); + if (path == NULL) + path = of_get_property(of_chosen, "stdout-path", NULL); if (path != NULL) { stdout = of_find_node_by_path(path); if (stdout) @@ -398,8 +400,7 @@ void __init find_legacy_serial_ports(void) /* Next, fill our array with ISA ports */ for_each_node_by_type(np, "serial") { struct device_node *isa = of_get_parent(np); - if (isa && (!strcmp(isa->name, "isa") || - !strcmp(isa->name, "lpc"))) { + if (of_node_name_eq(isa, "isa") || of_node_name_eq(isa, "lpc")) { if (of_device_is_available(np)) { index = add_legacy_isa_port(np, isa); if (index >= 0 && np == stdout) @@ -413,11 +414,12 @@ void __init find_legacy_serial_ports(void) /* Next, try to locate PCI ports */ for (np = NULL; (np = of_find_all_nodes(np));) { struct device_node *pci, *parent = of_get_parent(np); - if (parent && !strcmp(parent->name, "isa")) { + if (of_node_name_eq(parent, "isa")) { of_node_put(parent); continue; } - if (strcmp(np->name, "serial") && strcmp(np->type, "serial")) { + if (!of_node_name_eq(np, "serial") && + !of_node_is_type(np, "serial")) { of_node_put(parent); continue; } @@ -595,8 +597,10 @@ static int __init check_legacy_serial_console(void) /* We are getting a weird phandle from OF ... */ /* ... So use the full path instead */ name = of_get_property(of_chosen, "linux,stdout-path", NULL); + if (name == NULL) + name = of_get_property(of_chosen, "stdout-path", NULL); if (name == NULL) { - DBG(" no linux,stdout-path !\n"); + DBG(" no stdout-path !\n"); return -ENODEV; } prom_stdout = of_find_node_by_path(name); diff --git a/arch/powerpc/kernel/machine_kexec_file_64.c b/arch/powerpc/kernel/machine_kexec_file_64.c index c77e95e9b384..0d20c7ad40fa 100644 --- a/arch/powerpc/kernel/machine_kexec_file_64.c +++ b/arch/powerpc/kernel/machine_kexec_file_64.c @@ -24,7 +24,6 @@ #include <linux/slab.h> #include <linux/kexec.h> -#include <linux/memblock.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> #include <asm/ima.h> @@ -47,59 +46,6 @@ int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, } /** - * arch_kexec_walk_mem - call func(data) for each unreserved memory block - * @kbuf: Context info for the search. Also passed to @func. - * @func: Function to call for each memory block. - * - * This function is used by kexec_add_buffer and kexec_locate_mem_hole - * to find unreserved memory to load kexec segments into. - * - * Return: The memory walk will stop when func returns a non-zero value - * and that value will be returned. If all free regions are visited without - * func returning non-zero, then zero will be returned. - */ -int arch_kexec_walk_mem(struct kexec_buf *kbuf, - int (*func)(struct resource *, void *)) -{ - int ret = 0; - u64 i; - phys_addr_t mstart, mend; - struct resource res = { }; - - if (kbuf->top_down) { - for_each_free_mem_range_reverse(i, NUMA_NO_NODE, 0, - &mstart, &mend, NULL) { - /* - * In memblock, end points to the first byte after the - * range while in kexec, end points to the last byte - * in the range. - */ - res.start = mstart; - res.end = mend - 1; - ret = func(&res, kbuf); - if (ret) - break; - } - } else { - for_each_free_mem_range(i, NUMA_NO_NODE, 0, &mstart, &mend, - NULL) { - /* - * In memblock, end points to the first byte after the - * range while in kexec, end points to the last byte - * in the range. - */ - res.start = mstart; - res.end = mend - 1; - ret = func(&res, kbuf); - if (ret) - break; - } - } - - return ret; -} - -/** * setup_purgatory - initialize the purgatory's global variables * @image: kexec image. * @slave_code: Slave code for the purgatory. diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index 695b24a2d954..57d2ffb2d45c 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -153,7 +153,7 @@ _GLOBAL(call_setup_cpu) mtctr r5 bctr -#if defined(CONFIG_CPU_FREQ_PMAC) && defined(CONFIG_6xx) +#if defined(CONFIG_CPU_FREQ_PMAC) && defined(CONFIG_PPC_BOOK3S_32) /* This gets called by via-pmu.c to switch the PLL selection * on 750fx CPU. This function should really be moved to some @@ -223,7 +223,7 @@ _GLOBAL(low_choose_7447a_dfs) mtmsr r7 blr -#endif /* CONFIG_CPU_FREQ_PMAC && CONFIG_6xx */ +#endif /* CONFIG_CPU_FREQ_PMAC && CONFIG_PPC_BOOK3S_32 */ /* * complement mask on the msr then "or" some values on. diff --git a/arch/powerpc/kernel/msi.c b/arch/powerpc/kernel/msi.c index dab616a33b8d..f2197654be07 100644 --- a/arch/powerpc/kernel/msi.c +++ b/arch/powerpc/kernel/msi.c @@ -34,5 +34,10 @@ void arch_teardown_msi_irqs(struct pci_dev *dev) { struct pci_controller *phb = pci_bus_to_host(dev->bus); - phb->controller_ops.teardown_msi_irqs(dev); + /* + * We can be called even when arch_setup_msi_irqs() returns -ENOSYS, + * so check the pointer again. + */ + if (phb->controller_ops.teardown_msi_irqs) + phb->controller_ops.teardown_msi_irqs(dev); } diff --git a/arch/powerpc/kernel/nvram_64.c b/arch/powerpc/kernel/nvram_64.c index 22e9d281324d..38b03a330cd2 100644 --- a/arch/powerpc/kernel/nvram_64.c +++ b/arch/powerpc/kernel/nvram_64.c @@ -563,8 +563,6 @@ static int nvram_pstore_init(void) nvram_pstore_info.buf = oops_data; nvram_pstore_info.bufsize = oops_data_sz; - spin_lock_init(&nvram_pstore_info.buf_lock); - rc = pstore_register(&nvram_pstore_info); if (rc && (rc != -EPERM)) /* Print error only when pstore.backend == nvram */ @@ -809,6 +807,7 @@ static long dev_nvram_ioctl(struct file *file, unsigned int cmd, #ifdef CONFIG_PPC_PMAC case OBSOLETE_PMAC_NVRAM_GET_OFFSET: printk(KERN_WARNING "nvram: Using obsolete PMAC_NVRAM_GET_OFFSET ioctl\n"); + /* fall through */ case IOC_NVRAM_GET_OFFSET: { int part, offset; diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c index 98f04725def7..24191ea2d9a7 100644 --- a/arch/powerpc/kernel/pci_of_scan.c +++ b/arch/powerpc/kernel/pci_of_scan.c @@ -125,16 +125,13 @@ struct pci_dev *of_create_pci_dev(struct device_node *node, struct pci_bus *bus, int devfn) { struct pci_dev *dev; - const char *type; dev = pci_alloc_dev(bus); if (!dev) return NULL; - type = of_get_property(node, "device_type", NULL); - if (type == NULL) - type = ""; - pr_debug(" create device, devfn: %x, type: %s\n", devfn, type); + pr_debug(" create device, devfn: %x, type: %s\n", devfn, + of_node_get_device_type(node)); dev->dev.of_node = of_node_get(node); dev->dev.parent = bus->bridge; @@ -167,12 +164,12 @@ struct pci_dev *of_create_pci_dev(struct device_node *node, /* Early fixups, before probing the BARs */ pci_fixup_device(pci_fixup_early, dev); - if (!strcmp(type, "pci") || !strcmp(type, "pciex")) { + if (of_node_is_type(node, "pci") || of_node_is_type(node, "pciex")) { /* a PCI-PCI bridge */ dev->hdr_type = PCI_HEADER_TYPE_BRIDGE; dev->rom_base_reg = PCI_ROM_ADDRESS1; set_pcie_hotplug_bridge(dev); - } else if (!strcmp(type, "cardbus")) { + } else if (of_node_is_type(node, "cardbus")) { dev->hdr_type = PCI_HEADER_TYPE_CARDBUS; } else { dev->hdr_type = PCI_HEADER_TYPE_NORMAL; diff --git a/arch/powerpc/kernel/pmc.c b/arch/powerpc/kernel/pmc.c index 58eaa3ddf7b9..2de71faca911 100644 --- a/arch/powerpc/kernel/pmc.c +++ b/arch/powerpc/kernel/pmc.c @@ -29,7 +29,7 @@ static void dummy_perf(struct pt_regs *regs) { #if defined(CONFIG_FSL_EMB_PERFMON) mtpmr(PMRN_PMGC0, mfpmr(PMRN_PMGC0) & ~PMGC0_PMIE); -#elif defined(CONFIG_PPC64) || defined(CONFIG_6xx) +#elif defined(CONFIG_PPC64) || defined(CONFIG_PPC_BOOK3S_32) if (cur_cpu_spec->pmc_type == PPC_PMC_IBM) mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~(MMCR0_PMXE|MMCR0_PMAO)); #else diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index fe758cedb93f..4181ec715f88 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -124,12 +124,12 @@ static void __init move_device_tree(void) size = fdt_totalsize(initial_boot_params); if ((memory_limit && (start + size) > PHYSICAL_START + memory_limit) || - overlaps_crashkernel(start, size) || - overlaps_initrd(start, size)) { + !memblock_is_memory(start + size - 1) || + overlaps_crashkernel(start, size) || overlaps_initrd(start, size)) { p = __va(memblock_phys_alloc(size, PAGE_SIZE)); memcpy(p, initial_boot_params, size); initial_boot_params = p; - DBG("Moved device tree to 0x%p\n", p); + DBG("Moved device tree to 0x%px\n", p); } DBG("<- move_device_tree\n"); @@ -689,7 +689,7 @@ void __init early_init_devtree(void *params) { phys_addr_t limit; - DBG(" -> early_init_devtree(%p)\n", params); + DBG(" -> early_init_devtree(%px)\n", params); /* Too early to BUG_ON(), do it by hand */ if (!early_init_dt_verify(params)) @@ -749,7 +749,7 @@ void __init early_init_devtree(void *params) memblock_allow_resize(); memblock_dump_all(); - DBG("Phys. mem: %llx\n", memblock_phys_mem_size()); + DBG("Phys. mem: %llx\n", (unsigned long long)memblock_phys_mem_size()); /* We may need to relocate the flat tree, do it now. * FIXME .. and the initrd too? */ diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index afb819f4ca68..cdd5d1d3ae41 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -3263,27 +3263,40 @@ static inline int do_seccomp(struct pt_regs *regs) { return 0; } */ long do_syscall_trace_enter(struct pt_regs *regs) { + u32 flags; + user_exit(); - if (test_thread_flag(TIF_SYSCALL_EMU)) { - ptrace_report_syscall(regs); - /* - * Returning -1 will skip the syscall execution. We want to - * avoid clobbering any register also, thus, not 'gotoing' - * skip label. - */ - return -1; - } + flags = READ_ONCE(current_thread_info()->flags) & + (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE); - /* - * The tracer may decide to abort the syscall, if so tracehook - * will return !0. Note that the tracer may also just change - * regs->gpr[0] to an invalid syscall number, that is handled - * below on the exit path. - */ - if (test_thread_flag(TIF_SYSCALL_TRACE) && - tracehook_report_syscall_entry(regs)) - goto skip; + if (flags) { + int rc = tracehook_report_syscall_entry(regs); + + if (unlikely(flags & _TIF_SYSCALL_EMU)) { + /* + * A nonzero return code from + * tracehook_report_syscall_entry() tells us to prevent + * the syscall execution, but we are not going to + * execute it anyway. + * + * Returning -1 will skip the syscall execution. We want + * to avoid clobbering any registers, so we don't goto + * the skip label below. + */ + return -1; + } + + if (rc) { + /* + * The tracer decided to abort the syscall. Note that + * the tracer may also just change regs->gpr[0] to an + * invalid syscall number, that is handled below on the + * exit path. + */ + goto skip; + } + } /* Run seccomp after ptrace; allow it to set gpr[3]. */ if (do_seccomp(regs)) diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c index f6f469fc4073..9b8631533e02 100644 --- a/arch/powerpc/kernel/security.c +++ b/arch/powerpc/kernel/security.c @@ -4,6 +4,7 @@ // // Copyright 2018, Michael Ellerman, IBM Corporation. +#include <linux/cpu.h> #include <linux/kernel.h> #include <linux/device.h> #include <linux/seq_buf.h> @@ -22,10 +23,14 @@ enum count_cache_flush_type { COUNT_CACHE_FLUSH_SW = 0x2, COUNT_CACHE_FLUSH_HW = 0x4, }; -static enum count_cache_flush_type count_cache_flush_type; +static enum count_cache_flush_type count_cache_flush_type = COUNT_CACHE_FLUSH_NONE; bool barrier_nospec_enabled; static bool no_nospec; +static bool btb_flush_enabled; +#ifdef CONFIG_PPC_FSL_BOOK3E +static bool no_spectrev2; +#endif static void enable_barrier_nospec(bool enable) { @@ -101,6 +106,23 @@ static __init int barrier_nospec_debugfs_init(void) device_initcall(barrier_nospec_debugfs_init); #endif /* CONFIG_DEBUG_FS */ +#ifdef CONFIG_PPC_FSL_BOOK3E +static int __init handle_nospectre_v2(char *p) +{ + no_spectrev2 = true; + + return 0; +} +early_param("nospectre_v2", handle_nospectre_v2); +void setup_spectre_v2(void) +{ + if (no_spectrev2) + do_btb_flush_fixups(); + else + btb_flush_enabled = true; +} +#endif /* CONFIG_PPC_FSL_BOOK3E */ + #ifdef CONFIG_PPC_BOOK3S_64 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) { @@ -191,8 +213,11 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c if (count_cache_flush_type == COUNT_CACHE_FLUSH_HW) seq_buf_printf(&s, "(hardware accelerated)"); - } else + } else if (btb_flush_enabled) { + seq_buf_printf(&s, "Mitigation: Branch predictor state flush"); + } else { seq_buf_printf(&s, "Vulnerable"); + } seq_buf_printf(&s, "\n"); diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 93ee3703b42f..ca00fbb97cf8 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -687,7 +687,7 @@ int check_legacy_ioport(unsigned long base_port) return ret; parent = of_get_parent(np); if (parent) { - if (strcmp(parent->type, "isa") == 0) + if (of_node_is_type(parent, "isa")) ret = 0; of_node_put(parent); } @@ -800,7 +800,7 @@ static __init void print_system_info(void) #ifdef CONFIG_PPC_BOOK3S_64 pr_info("ppc64_pft_size = 0x%llx\n", ppc64_pft_size); #endif -#ifdef CONFIG_PPC_STD_MMU_32 +#ifdef CONFIG_PPC_BOOK3S_32 pr_info("Hash_size = 0x%lx\n", Hash_size); #endif pr_info("phys_mem_size = 0x%llx\n", @@ -830,7 +830,7 @@ static __init void print_system_info(void) if (htab_hash_mask) pr_info("htab_hash_mask = 0x%lx\n", htab_hash_mask); #endif -#ifdef CONFIG_PPC_STD_MMU_32 +#ifdef CONFIG_PPC_BOOK3S_32 if (Hash) pr_info("Hash = 0x%p\n", Hash); if (Hash_mask) @@ -974,6 +974,7 @@ void __init setup_arch(char **cmdline_p) ppc_md.setup_arch(); setup_barrier_nospec(); + setup_spectre_v2(); paging_init(); diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c index 81909600013a..947f904688b0 100644 --- a/arch/powerpc/kernel/setup_32.c +++ b/arch/powerpc/kernel/setup_32.c @@ -59,7 +59,6 @@ unsigned long ISA_DMA_THRESHOLD; unsigned int DMA_MODE_READ; unsigned int DMA_MODE_WRITE; -EXPORT_SYMBOL(ISA_DMA_THRESHOLD); EXPORT_SYMBOL(DMA_MODE_READ); EXPORT_SYMBOL(DMA_MODE_WRITE); @@ -101,8 +100,7 @@ notrace unsigned long __init early_init(unsigned long dt_ptr) */ notrace void __init machine_init(u64 dt_ptr) { - unsigned int *addr = (unsigned int *)((unsigned long)&patch__memset_nocache + - patch__memset_nocache); + unsigned int *addr = (unsigned int *)patch_site_addr(&patch__memset_nocache); unsigned long insn; /* Configure static keys first, now that we're relocated. */ @@ -240,7 +238,7 @@ void __init exc_lvl_early_init(void) void __init setup_power_save(void) { -#ifdef CONFIG_6xx +#ifdef CONFIG_PPC_BOOK3S_32 if (cpu_has_feature(CPU_FTR_CAN_DOZE) || cpu_has_feature(CPU_FTR_CAN_NAP)) ppc_md.power_save = ppc6xx_idle; diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 2a51e4cc8246..236c1151a3a7 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -636,6 +636,8 @@ static void *__init alloc_stack(unsigned long limit, int cpu) { unsigned long pa; + BUILD_BUG_ON(STACK_INT_FRAME_SIZE % 16); + pa = memblock_alloc_base_nid(THREAD_SIZE, THREAD_SIZE, limit, early_cpu_to_node(cpu), MEMBLOCK_NONE); if (!pa) { diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index e6474a45cef5..2d47cc79e5b3 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -470,9 +470,9 @@ static int save_user_regs(struct pt_regs *regs, struct mcontext __user *frame, return 1; if (sigret) { - /* Set up the sigreturn trampoline: li r0,sigret; sc */ - if (__put_user(0x38000000UL + sigret, &frame->tramp[0]) - || __put_user(0x44000002UL, &frame->tramp[1])) + /* Set up the sigreturn trampoline: li 0,sigret; sc */ + if (__put_user(PPC_INST_ADDI + sigret, &frame->tramp[0]) + || __put_user(PPC_INST_SC, &frame->tramp[1])) return 1; flush_icache_range((unsigned long) &frame->tramp[0], (unsigned long) &frame->tramp[2]); @@ -619,9 +619,9 @@ static int save_tm_user_regs(struct pt_regs *regs, if (__put_user(msr, &frame->mc_gregs[PT_MSR])) return 1; if (sigret) { - /* Set up the sigreturn trampoline: li r0,sigret; sc */ - if (__put_user(0x38000000UL + sigret, &frame->tramp[0]) - || __put_user(0x44000002UL, &frame->tramp[1])) + /* Set up the sigreturn trampoline: li 0,sigret; sc */ + if (__put_user(PPC_INST_ADDI + sigret, &frame->tramp[0]) + || __put_user(PPC_INST_SC, &frame->tramp[1])) return 1; flush_icache_range((unsigned long) &frame->tramp[0], (unsigned long) &frame->tramp[2]); @@ -848,7 +848,23 @@ static long restore_tm_user_regs(struct pt_regs *regs, /* If TM bits are set to the reserved value, it's an invalid context */ if (MSR_TM_RESV(msr_hi)) return 1; - /* Pull in the MSR TM bits from the user context */ + + /* + * Disabling preemption, since it is unsafe to be preempted + * with MSR[TS] set without recheckpointing. + */ + preempt_disable(); + + /* + * CAUTION: + * After regs->MSR[TS] being updated, make sure that get_user(), + * put_user() or similar functions are *not* called. These + * functions can generate page faults which will cause the process + * to be de-scheduled with MSR[TS] set but without calling + * tm_recheckpoint(). This can cause a bug. + * + * Pull in the MSR TM bits from the user context + */ regs->msr = (regs->msr & ~MSR_TS_MASK) | (msr_hi & MSR_TS_MASK); /* Now, recheckpoint. This loads up all of the checkpointed (older) * registers, including FP and V[S]Rs. After recheckpointing, the @@ -873,6 +889,8 @@ static long restore_tm_user_regs(struct pt_regs *regs, } #endif + preempt_enable(); + return 0; } #endif @@ -1140,11 +1158,11 @@ SYSCALL_DEFINE0(rt_sigreturn) { struct rt_sigframe __user *rt_sf; struct pt_regs *regs = current_pt_regs(); + int tm_restore = 0; #ifdef CONFIG_PPC_TRANSACTIONAL_MEM struct ucontext __user *uc_transact; unsigned long msr_hi; unsigned long tmp; - int tm_restore = 0; #endif /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; @@ -1192,11 +1210,19 @@ SYSCALL_DEFINE0(rt_sigreturn) goto bad; } } - if (!tm_restore) - /* Fall through, for non-TM restore */ + if (!tm_restore) { + /* + * Unset regs->msr because ucontext MSR TS is not + * set, and recheckpoint was not called. This avoid + * hitting a TM Bad thing at RFID + */ + regs->msr &= ~MSR_TS_MASK; + } + /* Fall through, for non-TM restore */ #endif - if (do_setcontext(&rt_sf->uc, regs, 1)) - goto bad; + if (!tm_restore) + if (do_setcontext(&rt_sf->uc, regs, 1)) + goto bad; /* * It's not clear whether or why it is desirable to save the diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c index 83d51bf586c7..0935fe6c282a 100644 --- a/arch/powerpc/kernel/signal_64.c +++ b/arch/powerpc/kernel/signal_64.c @@ -467,20 +467,6 @@ static long restore_tm_sigcontexts(struct task_struct *tsk, if (MSR_TM_RESV(msr)) return -EINVAL; - /* pull in MSR TS bits from user context */ - regs->msr = (regs->msr & ~MSR_TS_MASK) | (msr & MSR_TS_MASK); - - /* - * Ensure that TM is enabled in regs->msr before we leave the signal - * handler. It could be the case that (a) user disabled the TM bit - * through the manipulation of the MSR bits in uc_mcontext or (b) the - * TM bit was disabled because a sufficient number of context switches - * happened whilst in the signal handler and load_tm overflowed, - * disabling the TM bit. In either case we can end up with an illegal - * TM state leading to a TM Bad Thing when we return to userspace. - */ - regs->msr |= MSR_TM; - /* pull in MSR LE from user context */ regs->msr = (regs->msr & ~MSR_LE) | (msr & MSR_LE); @@ -572,6 +558,34 @@ static long restore_tm_sigcontexts(struct task_struct *tsk, tm_enable(); /* Make sure the transaction is marked as failed */ tsk->thread.tm_texasr |= TEXASR_FS; + + /* + * Disabling preemption, since it is unsafe to be preempted + * with MSR[TS] set without recheckpointing. + */ + preempt_disable(); + + /* pull in MSR TS bits from user context */ + regs->msr = (regs->msr & ~MSR_TS_MASK) | (msr & MSR_TS_MASK); + + /* + * Ensure that TM is enabled in regs->msr before we leave the signal + * handler. It could be the case that (a) user disabled the TM bit + * through the manipulation of the MSR bits in uc_mcontext or (b) the + * TM bit was disabled because a sufficient number of context switches + * happened whilst in the signal handler and load_tm overflowed, + * disabling the TM bit. In either case we can end up with an illegal + * TM state leading to a TM Bad Thing when we return to userspace. + * + * CAUTION: + * After regs->MSR[TS] being updated, make sure that get_user(), + * put_user() or similar functions are *not* called. These + * functions can generate page faults which will cause the process + * to be de-scheduled with MSR[TS] set but without calling + * tm_recheckpoint(). This can cause a bug. + */ + regs->msr |= MSR_TM; + /* This loads the checkpointed FP/VEC state, if used */ tm_recheckpoint(&tsk->thread); @@ -585,6 +599,8 @@ static long restore_tm_sigcontexts(struct task_struct *tsk, regs->msr |= MSR_VEC; } + preempt_enable(); + return err; } #endif @@ -598,11 +614,12 @@ static long setup_trampoline(unsigned int syscall, unsigned int __user *tramp) long err = 0; /* addi r1, r1, __SIGNAL_FRAMESIZE # Pop the dummy stackframe */ - err |= __put_user(0x38210000UL | (__SIGNAL_FRAMESIZE & 0xffff), &tramp[0]); + err |= __put_user(PPC_INST_ADDI | __PPC_RT(R1) | __PPC_RA(R1) | + (__SIGNAL_FRAMESIZE & 0xffff), &tramp[0]); /* li r0, __NR_[rt_]sigreturn| */ - err |= __put_user(0x38000000UL | (syscall & 0xffff), &tramp[1]); + err |= __put_user(PPC_INST_ADDI | (syscall & 0xffff), &tramp[1]); /* sc */ - err |= __put_user(0x44000002UL, &tramp[2]); + err |= __put_user(PPC_INST_SC, &tramp[2]); /* Minimal traceback info */ for (i=TRAMP_TRACEBACK; i < TRAMP_SIZE ;i++) @@ -740,11 +757,23 @@ SYSCALL_DEFINE0(rt_sigreturn) &uc_transact->uc_mcontext)) goto badframe; } - else - /* Fall through, for non-TM restore */ #endif - if (restore_sigcontext(current, NULL, 1, &uc->uc_mcontext)) - goto badframe; + /* Fall through, for non-TM restore */ + if (!MSR_TM_ACTIVE(msr)) { + /* + * Unset MSR[TS] on the thread regs since MSR from user + * context does not have MSR active, and recheckpoint was + * not called since restore_tm_sigcontexts() was not called + * also. + * + * If not unsetting it, the code can RFID to userspace with + * MSR[TS] set, but without CPU in the proper state, + * causing a TM bad thing. + */ + current->thread.regs->msr &= ~MSR_TS_MASK; + if (restore_sigcontext(current, NULL, 1, &uc->uc_mcontext)) + goto badframe; + } if (restore_altstack(&uc->uc_stack)) goto badframe; diff --git a/arch/powerpc/kernel/syscalls/Makefile b/arch/powerpc/kernel/syscalls/Makefile new file mode 100644 index 000000000000..27b48954808d --- /dev/null +++ b/arch/powerpc/kernel/syscalls/Makefile @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0 +kapi := arch/$(SRCARCH)/include/generated/asm +uapi := arch/$(SRCARCH)/include/generated/uapi/asm + +_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ + $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') + +syscall := $(srctree)/$(src)/syscall.tbl +syshdr := $(srctree)/$(src)/syscallhdr.sh +systbl := $(srctree)/$(src)/syscalltbl.sh + +quiet_cmd_syshdr = SYSHDR $@ + cmd_syshdr = $(CONFIG_SHELL) '$(syshdr)' '$<' '$@' \ + '$(syshdr_abis_$(basetarget))' \ + '$(syshdr_pfx_$(basetarget))' \ + '$(syshdr_offset_$(basetarget))' + +quiet_cmd_systbl = SYSTBL $@ + cmd_systbl = $(CONFIG_SHELL) '$(systbl)' '$<' '$@' \ + '$(systbl_abis_$(basetarget))' \ + '$(systbl_abi_$(basetarget))' \ + '$(systbl_offset_$(basetarget))' + +syshdr_abis_unistd_32 := common,nospu,32 +$(uapi)/unistd_32.h: $(syscall) $(syshdr) + $(call if_changed,syshdr) + +syshdr_abis_unistd_64 := common,nospu,64 +$(uapi)/unistd_64.h: $(syscall) $(syshdr) + $(call if_changed,syshdr) + +systbl_abis_syscall_table_32 := common,nospu,32 +systbl_abi_syscall_table_32 := 32 +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) + $(call if_changed,systbl) + +systbl_abis_syscall_table_64 := common,nospu,64 +systbl_abi_syscall_table_64 := 64 +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) + $(call if_changed,systbl) + +systbl_abis_syscall_table_c32 := common,nospu,32 +systbl_abi_syscall_table_c32 := c32 +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) + $(call if_changed,systbl) + +systbl_abis_syscall_table_spu := common,spu +systbl_abi_syscall_table_spu := spu +$(kapi)/syscall_table_spu.h: $(syscall) $(systbl) + $(call if_changed,systbl) + +uapisyshdr-y += unistd_32.h unistd_64.h +kapisyshdr-y += syscall_table_32.h \ + syscall_table_64.h \ + syscall_table_c32.h \ + syscall_table_spu.h + +targets += $(uapisyshdr-y) $(kapisyshdr-y) + +PHONY += all +all: $(addprefix $(uapi)/,$(uapisyshdr-y)) +all: $(addprefix $(kapi)/,$(kapisyshdr-y)) + @: diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl new file mode 100644 index 000000000000..db3bbb8744af --- /dev/null +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -0,0 +1,427 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for powerpc +# +# The format is: +# <number> <abi> <name> <entry point> <compat entry point> +# +# The <abi> can be common, spu, nospu, 64, or 32 for this file. +# +0 nospu restart_syscall sys_restart_syscall +1 nospu exit sys_exit +2 nospu fork ppc_fork +3 common read sys_read +4 common write sys_write +5 common open sys_open compat_sys_open +6 common close sys_close +7 common waitpid sys_waitpid +8 common creat sys_creat +9 common link sys_link +10 common unlink sys_unlink +11 nospu execve sys_execve compat_sys_execve +12 common chdir sys_chdir +13 common time sys_time compat_sys_time +14 common mknod sys_mknod +15 common chmod sys_chmod +16 common lchown sys_lchown +17 common break sys_ni_syscall +18 32 oldstat sys_stat sys_ni_syscall +18 64 oldstat sys_ni_syscall +18 spu oldstat sys_ni_syscall +19 common lseek sys_lseek compat_sys_lseek +20 common getpid sys_getpid +21 nospu mount sys_mount compat_sys_mount +22 32 umount sys_oldumount +22 64 umount sys_ni_syscall +22 spu umount sys_ni_syscall +23 common setuid sys_setuid +24 common getuid sys_getuid +25 common stime sys_stime compat_sys_stime +26 nospu ptrace sys_ptrace compat_sys_ptrace +27 common alarm sys_alarm +28 32 oldfstat sys_fstat sys_ni_syscall +28 64 oldfstat sys_ni_syscall +28 spu oldfstat sys_ni_syscall +29 nospu pause sys_pause +30 nospu utime sys_utime compat_sys_utime +31 common stty sys_ni_syscall +32 common gtty sys_ni_syscall +33 common access sys_access +34 common nice sys_nice +35 common ftime sys_ni_syscall +36 common sync sys_sync +37 common kill sys_kill +38 common rename sys_rename +39 common mkdir sys_mkdir +40 common rmdir sys_rmdir +41 common dup sys_dup +42 common pipe sys_pipe +43 common times sys_times compat_sys_times +44 common prof sys_ni_syscall +45 common brk sys_brk +46 common setgid sys_setgid +47 common getgid sys_getgid +48 nospu signal sys_signal +49 common geteuid sys_geteuid +50 common getegid sys_getegid +51 nospu acct sys_acct +52 nospu umount2 sys_umount +53 common lock sys_ni_syscall +54 common ioctl sys_ioctl compat_sys_ioctl +55 common fcntl sys_fcntl compat_sys_fcntl +56 common mpx sys_ni_syscall +57 common setpgid sys_setpgid +58 common ulimit sys_ni_syscall +59 32 oldolduname sys_olduname +59 64 oldolduname sys_ni_syscall +59 spu oldolduname sys_ni_syscall +60 common umask sys_umask +61 common chroot sys_chroot +62 nospu ustat sys_ustat compat_sys_ustat +63 common dup2 sys_dup2 +64 common getppid sys_getppid +65 common getpgrp sys_getpgrp +66 common setsid sys_setsid +67 32 sigaction sys_sigaction compat_sys_sigaction +67 64 sigaction sys_ni_syscall +67 spu sigaction sys_ni_syscall +68 common sgetmask sys_sgetmask +69 common ssetmask sys_ssetmask +70 common setreuid sys_setreuid +71 common setregid sys_setregid +72 32 sigsuspend sys_sigsuspend +72 64 sigsuspend sys_ni_syscall +72 spu sigsuspend sys_ni_syscall +73 32 sigpending sys_sigpending compat_sys_sigpending +73 64 sigpending sys_ni_syscall +73 spu sigpending sys_ni_syscall +74 common sethostname sys_sethostname +75 common setrlimit sys_setrlimit compat_sys_setrlimit +76 32 getrlimit sys_old_getrlimit compat_sys_old_getrlimit +76 64 getrlimit sys_ni_syscall +76 spu getrlimit sys_ni_syscall +77 common getrusage sys_getrusage compat_sys_getrusage +78 common gettimeofday sys_gettimeofday compat_sys_gettimeofday +79 common settimeofday sys_settimeofday compat_sys_settimeofday +80 common getgroups sys_getgroups +81 common setgroups sys_setgroups +82 32 select ppc_select sys_ni_syscall +82 64 select sys_ni_syscall +82 spu select sys_ni_syscall +83 common symlink sys_symlink +84 32 oldlstat sys_lstat sys_ni_syscall +84 64 oldlstat sys_ni_syscall +84 spu oldlstat sys_ni_syscall +85 common readlink sys_readlink +86 nospu uselib sys_uselib +87 nospu swapon sys_swapon +88 nospu reboot sys_reboot +89 32 readdir sys_old_readdir compat_sys_old_readdir +89 64 readdir sys_ni_syscall +89 spu readdir sys_ni_syscall +90 common mmap sys_mmap +91 common munmap sys_munmap +92 common truncate sys_truncate compat_sys_truncate +93 common ftruncate sys_ftruncate compat_sys_ftruncate +94 common fchmod sys_fchmod +95 common fchown sys_fchown +96 common getpriority sys_getpriority +97 common setpriority sys_setpriority +98 common profil sys_ni_syscall +99 nospu statfs sys_statfs compat_sys_statfs +100 nospu fstatfs sys_fstatfs compat_sys_fstatfs +101 common ioperm sys_ni_syscall +102 common socketcall sys_socketcall compat_sys_socketcall +103 common syslog sys_syslog +104 common setitimer sys_setitimer compat_sys_setitimer +105 common getitimer sys_getitimer compat_sys_getitimer +106 common stat sys_newstat compat_sys_newstat +107 common lstat sys_newlstat compat_sys_newlstat +108 common fstat sys_newfstat compat_sys_newfstat +109 32 olduname sys_uname +109 64 olduname sys_ni_syscall +109 spu olduname sys_ni_syscall +110 common iopl sys_ni_syscall +111 common vhangup sys_vhangup +112 common idle sys_ni_syscall +113 common vm86 sys_ni_syscall +114 common wait4 sys_wait4 compat_sys_wait4 +115 nospu swapoff sys_swapoff +116 common sysinfo sys_sysinfo compat_sys_sysinfo +117 nospu ipc sys_ipc compat_sys_ipc +118 common fsync sys_fsync +119 32 sigreturn sys_sigreturn compat_sys_sigreturn +119 64 sigreturn sys_ni_syscall +119 spu sigreturn sys_ni_syscall +120 nospu clone ppc_clone +121 common setdomainname sys_setdomainname +122 common uname sys_newuname +123 common modify_ldt sys_ni_syscall +124 common adjtimex sys_adjtimex compat_sys_adjtimex +125 common mprotect sys_mprotect +126 32 sigprocmask sys_sigprocmask compat_sys_sigprocmask +126 64 sigprocmask sys_ni_syscall +126 spu sigprocmask sys_ni_syscall +127 common create_module sys_ni_syscall +128 nospu init_module sys_init_module +129 nospu delete_module sys_delete_module +130 common get_kernel_syms sys_ni_syscall +131 nospu quotactl sys_quotactl +132 common getpgid sys_getpgid +133 common fchdir sys_fchdir +134 common bdflush sys_bdflush +135 common sysfs sys_sysfs +136 32 personality sys_personality ppc64_personality +136 64 personality ppc64_personality +136 spu personality ppc64_personality +137 common afs_syscall sys_ni_syscall +138 common setfsuid sys_setfsuid +139 common setfsgid sys_setfsgid +140 common _llseek sys_llseek +141 common getdents sys_getdents compat_sys_getdents +142 common _newselect sys_select compat_sys_select +143 common flock sys_flock +144 common msync sys_msync +145 common readv sys_readv compat_sys_readv +146 common writev sys_writev compat_sys_writev +147 common getsid sys_getsid +148 common fdatasync sys_fdatasync +149 nospu _sysctl sys_sysctl compat_sys_sysctl +150 common mlock sys_mlock +151 common munlock sys_munlock +152 common mlockall sys_mlockall +153 common munlockall sys_munlockall +154 common sched_setparam sys_sched_setparam +155 common sched_getparam sys_sched_getparam +156 common sched_setscheduler sys_sched_setscheduler +157 common sched_getscheduler sys_sched_getscheduler +158 common sched_yield sys_sched_yield +159 common sched_get_priority_max sys_sched_get_priority_max +160 common sched_get_priority_min sys_sched_get_priority_min +161 common sched_rr_get_interval sys_sched_rr_get_interval compat_sys_sched_rr_get_interval +162 common nanosleep sys_nanosleep compat_sys_nanosleep +163 common mremap sys_mremap +164 common setresuid sys_setresuid +165 common getresuid sys_getresuid +166 common query_module sys_ni_syscall +167 common poll sys_poll +168 common nfsservctl sys_ni_syscall +169 common setresgid sys_setresgid +170 common getresgid sys_getresgid +171 common prctl sys_prctl +172 nospu rt_sigreturn sys_rt_sigreturn compat_sys_rt_sigreturn +173 nospu rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction +174 nospu rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask +175 nospu rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending +176 nospu rt_sigtimedwait sys_rt_sigtimedwait compat_sys_rt_sigtimedwait +177 nospu rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo +178 nospu rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend +179 common pread64 sys_pread64 compat_sys_pread64 +180 common pwrite64 sys_pwrite64 compat_sys_pwrite64 +181 common chown sys_chown +182 common getcwd sys_getcwd +183 common capget sys_capget +184 common capset sys_capset +185 nospu sigaltstack sys_sigaltstack compat_sys_sigaltstack +186 32 sendfile sys_sendfile compat_sys_sendfile +186 64 sendfile sys_sendfile64 +186 spu sendfile sys_sendfile64 +187 common getpmsg sys_ni_syscall +188 common putpmsg sys_ni_syscall +189 nospu vfork ppc_vfork +190 common ugetrlimit sys_getrlimit compat_sys_getrlimit +191 common readahead sys_readahead compat_sys_readahead +192 32 mmap2 sys_mmap2 compat_sys_mmap2 +193 32 truncate64 sys_truncate64 compat_sys_truncate64 +194 32 ftruncate64 sys_ftruncate64 compat_sys_ftruncate64 +195 32 stat64 sys_stat64 +196 32 lstat64 sys_lstat64 +197 32 fstat64 sys_fstat64 +198 nospu pciconfig_read sys_pciconfig_read +199 nospu pciconfig_write sys_pciconfig_write +200 nospu pciconfig_iobase sys_pciconfig_iobase +201 common multiplexer sys_ni_syscall +202 common getdents64 sys_getdents64 +203 common pivot_root sys_pivot_root +204 32 fcntl64 sys_fcntl64 compat_sys_fcntl64 +205 common madvise sys_madvise +206 common mincore sys_mincore +207 common gettid sys_gettid +208 common tkill sys_tkill +209 common setxattr sys_setxattr +210 common lsetxattr sys_lsetxattr +211 common fsetxattr sys_fsetxattr +212 common getxattr sys_getxattr +213 common lgetxattr sys_lgetxattr +214 common fgetxattr sys_fgetxattr +215 common listxattr sys_listxattr +216 common llistxattr sys_llistxattr +217 common flistxattr sys_flistxattr +218 common removexattr sys_removexattr +219 common lremovexattr sys_lremovexattr +220 common fremovexattr sys_fremovexattr +221 common futex sys_futex compat_sys_futex +222 common sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity +223 common sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity +# 224 unused +225 common tuxcall sys_ni_syscall +226 32 sendfile64 sys_sendfile64 compat_sys_sendfile64 +227 common io_setup sys_io_setup compat_sys_io_setup +228 common io_destroy sys_io_destroy +229 common io_getevents sys_io_getevents compat_sys_io_getevents +230 common io_submit sys_io_submit compat_sys_io_submit +231 common io_cancel sys_io_cancel +232 nospu set_tid_address sys_set_tid_address +233 common fadvise64 sys_fadvise64 ppc32_fadvise64 +234 nospu exit_group sys_exit_group +235 nospu lookup_dcookie sys_lookup_dcookie compat_sys_lookup_dcookie +236 common epoll_create sys_epoll_create +237 common epoll_ctl sys_epoll_ctl +238 common epoll_wait sys_epoll_wait +239 common remap_file_pages sys_remap_file_pages +240 common timer_create sys_timer_create compat_sys_timer_create +241 common timer_settime sys_timer_settime compat_sys_timer_settime +242 common timer_gettime sys_timer_gettime compat_sys_timer_gettime +243 common timer_getoverrun sys_timer_getoverrun +244 common timer_delete sys_timer_delete +245 common clock_settime sys_clock_settime compat_sys_clock_settime +246 common clock_gettime sys_clock_gettime compat_sys_clock_gettime +247 common clock_getres sys_clock_getres compat_sys_clock_getres +248 common clock_nanosleep sys_clock_nanosleep compat_sys_clock_nanosleep +249 32 swapcontext ppc_swapcontext ppc32_swapcontext +249 64 swapcontext ppc64_swapcontext +249 spu swapcontext sys_ni_syscall +250 common tgkill sys_tgkill +251 common utimes sys_utimes compat_sys_utimes +252 common statfs64 sys_statfs64 compat_sys_statfs64 +253 common fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 +254 32 fadvise64_64 ppc_fadvise64_64 +254 spu fadvise64_64 sys_ni_syscall +255 common rtas sys_rtas +256 32 sys_debug_setcontext sys_debug_setcontext sys_ni_syscall +256 64 sys_debug_setcontext sys_ni_syscall +256 spu sys_debug_setcontext sys_ni_syscall +# 257 reserved for vserver +258 nospu migrate_pages sys_migrate_pages compat_sys_migrate_pages +259 nospu mbind sys_mbind compat_sys_mbind +260 nospu get_mempolicy sys_get_mempolicy compat_sys_get_mempolicy +261 nospu set_mempolicy sys_set_mempolicy compat_sys_set_mempolicy +262 nospu mq_open sys_mq_open compat_sys_mq_open +263 nospu mq_unlink sys_mq_unlink +264 nospu mq_timedsend sys_mq_timedsend compat_sys_mq_timedsend +265 nospu mq_timedreceive sys_mq_timedreceive compat_sys_mq_timedreceive +266 nospu mq_notify sys_mq_notify compat_sys_mq_notify +267 nospu mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr +268 nospu kexec_load sys_kexec_load compat_sys_kexec_load +269 nospu add_key sys_add_key +270 nospu request_key sys_request_key +271 nospu keyctl sys_keyctl compat_sys_keyctl +272 nospu waitid sys_waitid compat_sys_waitid +273 nospu ioprio_set sys_ioprio_set +274 nospu ioprio_get sys_ioprio_get +275 nospu inotify_init sys_inotify_init +276 nospu inotify_add_watch sys_inotify_add_watch +277 nospu inotify_rm_watch sys_inotify_rm_watch +278 nospu spu_run sys_spu_run +279 nospu spu_create sys_spu_create +280 nospu pselect6 sys_pselect6 compat_sys_pselect6 +281 nospu ppoll sys_ppoll compat_sys_ppoll +282 common unshare sys_unshare +283 common splice sys_splice +284 common tee sys_tee +285 common vmsplice sys_vmsplice compat_sys_vmsplice +286 common openat sys_openat compat_sys_openat +287 common mkdirat sys_mkdirat +288 common mknodat sys_mknodat +289 common fchownat sys_fchownat +290 common futimesat sys_futimesat compat_sys_futimesat +291 32 fstatat64 sys_fstatat64 +291 64 newfstatat sys_newfstatat +291 spu newfstatat sys_newfstatat +292 common unlinkat sys_unlinkat +293 common renameat sys_renameat +294 common linkat sys_linkat +295 common symlinkat sys_symlinkat +296 common readlinkat sys_readlinkat +297 common fchmodat sys_fchmodat +298 common faccessat sys_faccessat +299 common get_robust_list sys_get_robust_list compat_sys_get_robust_list +300 common set_robust_list sys_set_robust_list compat_sys_set_robust_list +301 common move_pages sys_move_pages compat_sys_move_pages +302 common getcpu sys_getcpu +303 nospu epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait +304 common utimensat sys_utimensat compat_sys_utimensat +305 common signalfd sys_signalfd compat_sys_signalfd +306 common timerfd_create sys_timerfd_create +307 common eventfd sys_eventfd +308 common sync_file_range2 sys_sync_file_range2 compat_sys_sync_file_range2 +309 nospu fallocate sys_fallocate compat_sys_fallocate +310 nospu subpage_prot sys_subpage_prot +311 common timerfd_settime sys_timerfd_settime compat_sys_timerfd_settime +312 common timerfd_gettime sys_timerfd_gettime compat_sys_timerfd_gettime +313 common signalfd4 sys_signalfd4 compat_sys_signalfd4 +314 common eventfd2 sys_eventfd2 +315 common epoll_create1 sys_epoll_create1 +316 common dup3 sys_dup3 +317 common pipe2 sys_pipe2 +318 nospu inotify_init1 sys_inotify_init1 +319 common perf_event_open sys_perf_event_open +320 common preadv sys_preadv compat_sys_preadv +321 common pwritev sys_pwritev compat_sys_pwritev +322 nospu rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +323 nospu fanotify_init sys_fanotify_init +324 nospu fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark +325 common prlimit64 sys_prlimit64 +326 common socket sys_socket +327 common bind sys_bind +328 common connect sys_connect +329 common listen sys_listen +330 common accept sys_accept +331 common getsockname sys_getsockname +332 common getpeername sys_getpeername +333 common socketpair sys_socketpair +334 common send sys_send +335 common sendto sys_sendto +336 common recv sys_recv compat_sys_recv +337 common recvfrom sys_recvfrom compat_sys_recvfrom +338 common shutdown sys_shutdown +339 common setsockopt sys_setsockopt compat_sys_setsockopt +340 common getsockopt sys_getsockopt compat_sys_getsockopt +341 common sendmsg sys_sendmsg compat_sys_sendmsg +342 common recvmsg sys_recvmsg compat_sys_recvmsg +343 common recvmmsg sys_recvmmsg compat_sys_recvmmsg +344 common accept4 sys_accept4 +345 common name_to_handle_at sys_name_to_handle_at +346 common open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at +347 common clock_adjtime sys_clock_adjtime compat_sys_clock_adjtime +348 common syncfs sys_syncfs +349 common sendmmsg sys_sendmmsg compat_sys_sendmmsg +350 common setns sys_setns +351 nospu process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv +352 nospu process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev +353 nospu finit_module sys_finit_module +354 nospu kcmp sys_kcmp +355 common sched_setattr sys_sched_setattr +356 common sched_getattr sys_sched_getattr +357 common renameat2 sys_renameat2 +358 common seccomp sys_seccomp +359 common getrandom sys_getrandom +360 common memfd_create sys_memfd_create +361 common bpf sys_bpf +362 nospu execveat sys_execveat compat_sys_execveat +363 32 switch_endian sys_ni_syscall +363 64 switch_endian ppc_switch_endian +363 spu switch_endian sys_ni_syscall +364 common userfaultfd sys_userfaultfd +365 common membarrier sys_membarrier +378 nospu mlock2 sys_mlock2 +379 nospu copy_file_range sys_copy_file_range +380 common preadv2 sys_preadv2 compat_sys_preadv2 +381 common pwritev2 sys_pwritev2 compat_sys_pwritev2 +382 nospu kexec_file_load sys_kexec_file_load +383 nospu statx sys_statx +384 nospu pkey_alloc sys_pkey_alloc +385 nospu pkey_free sys_pkey_free +386 nospu pkey_mprotect sys_pkey_mprotect +387 nospu rseq sys_rseq +388 nospu io_pgetevents sys_io_pgetevents compat_sys_io_pgetevents diff --git a/arch/powerpc/kernel/syscalls/syscallhdr.sh b/arch/powerpc/kernel/syscalls/syscallhdr.sh new file mode 100644 index 000000000000..c0a9a32937f1 --- /dev/null +++ b/arch/powerpc/kernel/syscalls/syscallhdr.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +in="$1" +out="$2" +my_abis=`echo "($3)" | tr ',' '|'` +prefix="$4" +offset="$5" + +fileguard=_UAPI_ASM_POWERPC_`basename "$out" | sed \ + -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/' \ + -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'` +grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( + printf "#ifndef %s\n" "${fileguard}" + printf "#define %s\n" "${fileguard}" + printf "\n" + + nxt=0 + while read nr abi name entry compat ; do + if [ -z "$offset" ]; then + printf "#define __NR_%s%s\t%s\n" \ + "${prefix}" "${name}" "${nr}" + else + printf "#define __NR_%s%s\t(%s + %s)\n" \ + "${prefix}" "${name}" "${offset}" "${nr}" + fi + nxt=$((nr+1)) + done + + printf "\n" + printf "#ifdef __KERNEL__\n" + printf "#define __NR_syscalls\t%s\n" "${nxt}" + printf "#endif\n" + printf "\n" + printf "#endif /* %s */" "${fileguard}" + printf "\n" +) > "$out" diff --git a/arch/powerpc/kernel/syscalls/syscalltbl.sh b/arch/powerpc/kernel/syscalls/syscalltbl.sh new file mode 100644 index 000000000000..fd620490a542 --- /dev/null +++ b/arch/powerpc/kernel/syscalls/syscalltbl.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +in="$1" +out="$2" +my_abis=`echo "($3)" | tr ',' '|'` +my_abi="$4" +offset="$5" + +emit() { + t_nxt="$1" + t_nr="$2" + t_entry="$3" + + while [ $t_nxt -lt $t_nr ]; do + printf "__SYSCALL(%s,sys_ni_syscall, )\n" "${t_nxt}" + t_nxt=$((t_nxt+1)) + done + printf "__SYSCALL(%s,%s, )\n" "${t_nxt}" "${t_entry}" +} + +grep -E "^[0-9A-Fa-fXx]+[[:space:]]+${my_abis}" "$in" | sort -n | ( + nxt=0 + if [ -z "$offset" ]; then + offset=0 + fi + + while read nr abi name entry compat ; do + if [ "$my_abi" = "c32" ] && [ ! -z "$compat" ]; then + emit $((nxt+offset)) $((nr+offset)) $compat + else + emit $((nxt+offset)) $((nr+offset)) $entry + fi + nxt=$((nr+1)) + done +) > "$out" diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index 755dc98a57ae..e8e93c2c7d03 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -457,7 +457,7 @@ static ssize_t __used \ #define HAS_PPC_PMC_CLASSIC 1 #define HAS_PPC_PMC_IBM 1 #define HAS_PPC_PMC_PA6T 1 -#elif defined(CONFIG_6xx) +#elif defined(CONFIG_PPC_BOOK3S_32) #define HAS_PPC_PMC_CLASSIC 1 #define HAS_PPC_PMC_IBM 1 #define HAS_PPC_PMC_G4 1 diff --git a/arch/powerpc/kernel/systbl.S b/arch/powerpc/kernel/systbl.S index 919a32746ede..23265a28740b 100644 --- a/arch/powerpc/kernel/systbl.S +++ b/arch/powerpc/kernel/systbl.S @@ -16,28 +16,6 @@ #include <asm/ppc_asm.h> -#ifdef CONFIG_PPC64 -#define SYSCALL(func) .8byte DOTSYM(sys_##func),DOTSYM(sys_##func) -#define COMPAT_SYS(func) .8byte DOTSYM(sys_##func),DOTSYM(compat_sys_##func) -#define PPC_SYS(func) .8byte DOTSYM(ppc_##func),DOTSYM(ppc_##func) -#define OLDSYS(func) .8byte DOTSYM(sys_ni_syscall),DOTSYM(sys_ni_syscall) -#define SYS32ONLY(func) .8byte DOTSYM(sys_ni_syscall),DOTSYM(compat_sys_##func) -#define PPC64ONLY(func) .8byte DOTSYM(ppc_##func),DOTSYM(sys_ni_syscall) -#define SYSX(f, f3264, f32) .8byte DOTSYM(f),DOTSYM(f3264) -#else -#define SYSCALL(func) .long sys_##func -#define COMPAT_SYS(func) .long sys_##func -#define PPC_SYS(func) .long ppc_##func -#define OLDSYS(func) .long sys_##func -#define SYS32ONLY(func) .long sys_##func -#define PPC64ONLY(func) .long sys_ni_syscall -#define SYSX(f, f3264, f32) .long f32 -#endif -#define SYSCALL_SPU(func) SYSCALL(func) -#define COMPAT_SYS_SPU(func) COMPAT_SYS(func) -#define COMPAT_SPU_NEW(func) COMPAT_SYS(func) -#define SYSX_SPU(f, f3264, f32) SYSX(f, f3264, f32) - .section .rodata,"a" #ifdef CONFIG_PPC64 @@ -46,5 +24,21 @@ .globl sys_call_table sys_call_table: +#ifdef CONFIG_PPC64 +#define __SYSCALL(nr, entry, nargs) .8byte DOTSYM(entry) +#include <asm/syscall_table_64.h> +#undef __SYSCALL +#else +#define __SYSCALL(nr, entry, nargs) .long entry +#include <asm/syscall_table_32.h> +#undef __SYSCALL +#endif -#include <asm/systbl.h> +#ifdef CONFIG_COMPAT +.globl compat_sys_call_table +compat_sys_call_table: +#define compat_sys_sigsuspend sys_sigsuspend +#define __SYSCALL(nr, entry, nargs) .8byte DOTSYM(entry) +#include <asm/syscall_table_c32.h> +#undef __SYSCALL +#endif diff --git a/arch/powerpc/kernel/systbl_chk.c b/arch/powerpc/kernel/systbl_chk.c deleted file mode 100644 index 4653258722ac..000000000000 --- a/arch/powerpc/kernel/systbl_chk.c +++ /dev/null @@ -1,60 +0,0 @@ -/* - * This file, when run through CPP produces a list of syscall numbers - * in the order of systbl.h. That way we can check for gaps and syscalls - * that are out of order. - * - * Unfortunately, we cannot check for the correct ordering of entries - * using SYSX(). - * - * Copyright © IBM Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ -#include <asm/unistd.h> - -#define SYSCALL(func) __NR_##func -#define COMPAT_SYS(func) __NR_##func -#define PPC_SYS(func) __NR_##func -#ifdef CONFIG_PPC64 -#define OLDSYS(func) -1 -#define SYS32ONLY(func) -1 -#define PPC64ONLY(func) __NR_##func -#else -#define OLDSYS(func) __NR_old##func -#define SYS32ONLY(func) __NR_##func -#define PPC64ONLY(func) -1 -#endif -#define SYSX(f, f3264, f32) -1 - -#define SYSCALL_SPU(func) SYSCALL(func) -#define COMPAT_SYS_SPU(func) COMPAT_SYS(func) -#define COMPAT_SPU_NEW(func) COMPAT_SYS(_new##func) -#define SYSX_SPU(f, f3264, f32) SYSX(f, f3264, f32) - -/* Just insert a marker for ni_syscalls */ -#define __NR_ni_syscall -1 - -/* - * These are the known exceptions. - * Hopefully, there will be no more. - */ -#define __NR_llseek __NR__llseek -#undef __NR_umount -#define __NR_umount __NR_umount2 -#define __NR_old_getrlimit __NR_getrlimit -#define __NR_newstat __NR_stat -#define __NR_newlstat __NR_lstat -#define __NR_newfstat __NR_fstat -#define __NR_newuname __NR_uname -#define __NR_sysctl __NR__sysctl -#define __NR_olddebug_setcontext __NR_sys_debug_setcontext - -/* We call sys_ugetrlimit for syscall number __NR_getrlimit */ -#define getrlimit ugetrlimit - -START_TABLE -#include <asm/systbl.h> -END_TABLE NR_syscalls diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c index 4bf051d3e21e..29746dc28df5 100644 --- a/arch/powerpc/kernel/trace/ftrace.c +++ b/arch/powerpc/kernel/trace/ftrace.c @@ -107,7 +107,7 @@ static int is_b_op(unsigned int op) static unsigned long find_bl_target(unsigned long ip, unsigned int op) { - static int offset; + int offset; offset = (op & 0x03fffffc); /* make it signed */ @@ -950,7 +950,6 @@ int ftrace_disable_ftrace_graph_caller(void) */ unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) { - struct ftrace_graph_ent trace; unsigned long return_hooker; if (unlikely(ftrace_graph_is_dead())) @@ -961,18 +960,8 @@ unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) return_hooker = ppc_function_entry(return_to_handler); - trace.func = ip; - trace.depth = current->curr_ret_stack + 1; - - /* Only trace if the calling function expects to */ - if (!ftrace_graph_entry(&trace)) - goto out; - - if (ftrace_push_return_trace(parent, ip, &trace.depth, 0, - NULL) == -EBUSY) - goto out; - - parent = return_hooker; + if (!function_graph_enter(parent, ip, 0, NULL)) + parent = return_hooker; out: return parent; } diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 9a86572db1ef..00af2c4febf4 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -1434,7 +1434,8 @@ void program_check_exception(struct pt_regs *regs) goto bail; } else { printk(KERN_EMERG "Unexpected TM Bad Thing exception " - "at %lx (msr 0x%lx)\n", regs->nip, regs->msr); + "at %lx (msr 0x%lx) tm_scratch=%llx\n", + regs->nip, regs->msr, get_paca()->tm_scratch); die("Unrecoverable exception", regs, SIGABRT); } } diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index 65b3bdb99f0b..7725a9714736 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -671,15 +671,18 @@ static void __init vdso_setup_syscall_map(void) { unsigned int i; extern unsigned long *sys_call_table; +#ifdef CONFIG_PPC64 + extern unsigned long *compat_sys_call_table; +#endif extern unsigned long sys_ni_syscall; for (i = 0; i < NR_syscalls; i++) { #ifdef CONFIG_PPC64 - if (sys_call_table[i*2] != sys_ni_syscall) + if (sys_call_table[i] != sys_ni_syscall) vdso_data->syscall_map_64[i >> 5] |= 0x80000000UL >> (i & 0x1f); - if (sys_call_table[i*2+1] != sys_ni_syscall) + if (compat_sys_call_table[i] != sys_ni_syscall) vdso_data->syscall_map_32[i >> 5] |= 0x80000000UL >> (i & 0x1f); #else /* CONFIG_PPC64 */ diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index 434581bcd5b4..ad1c77f71f54 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -170,6 +170,14 @@ SECTIONS } #endif /* CONFIG_PPC_BARRIER_NOSPEC */ +#ifdef CONFIG_PPC_FSL_BOOK3E + . = ALIGN(8); + __spec_btb_flush_fixup : AT(ADDR(__spec_btb_flush_fixup) - LOAD_OFFSET) { + __start__btb_flush_fixup = .; + *(__btb_flush_fixup) + __stop__btb_flush_fixup = .; + } +#endif EXCEPTION_TABLE(0) NOTES :kernel :notes @@ -206,12 +214,6 @@ SECTIONS .init.data : AT(ADDR(.init.data) - LOAD_OFFSET) { INIT_DATA - __vtop_table_begin = .; - KEEP(*(.vtop_fixup)); - __vtop_table_end = .; - __ptov_table_begin = .; - KEEP(*(.ptov_fixup)); - __ptov_table_end = .; } .init.setup : AT(ADDR(.init.setup) - LOAD_OFFSET) { @@ -308,6 +310,10 @@ SECTIONS #ifdef CONFIG_PPC32 .data : AT(ADDR(.data) - LOAD_OFFSET) { DATA_DATA +#ifdef CONFIG_UBSAN + *(.data..Lubsan_data*) + *(.data..Lubsan_type*) +#endif *(.data.rel*) *(SDATA_MAIN) *(.sdata2) |