From 8f65873e47784a390949f0d61e5692dbf2a8253e Mon Sep 17 00:00:00 2001 From: Graf Yang Date: Tue, 18 Nov 2008 17:48:22 +0800 Subject: Blackfin arch: SMP supporting patchset: Blackfin kernel and memory management code Blackfin dual core BF561 processor can support SMP like features. https://docs.blackfin.uclinux.org/doku.php?id=linux-kernel:smp-like In this patch, we provide SMP extend to Blackfin kernel and memory management code Singed-off-by: Graf Yang Signed-off-by: Mike Frysinger Signed-off-by: Bryan Wu --- arch/blackfin/kernel/traps.c | 56 ++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 30 deletions(-) (limited to 'arch/blackfin/kernel/traps.c') diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c index bef025b07443..af7cc43630de 100644 --- a/arch/blackfin/kernel/traps.c +++ b/arch/blackfin/kernel/traps.c @@ -75,16 +75,6 @@ void __init trap_init(void) CSYNC(); } -/* - * Used to save the RETX, SEQSTAT, I/D CPLB FAULT ADDR - * values across the transition from exception to IRQ5. - * We put these in L1, so they are going to be in a valid - * location during exception context - */ -__attribute__((l1_data)) -unsigned long saved_retx, saved_seqstat, - saved_icplb_fault_addr, saved_dcplb_fault_addr; - static void decode_address(char *buf, unsigned long address) { #ifdef CONFIG_DEBUG_VERBOSE @@ -211,18 +201,18 @@ asmlinkage void double_fault_c(struct pt_regs *fp) printk(KERN_EMERG "\n" KERN_EMERG "Double Fault\n"); #ifdef CONFIG_DEBUG_DOUBLEFAULT_PRINT if (((long)fp->seqstat & SEQSTAT_EXCAUSE) == VEC_UNCOV) { + unsigned int cpu = smp_processor_id(); char buf[150]; - decode_address(buf, saved_retx); + decode_address(buf, cpu_pda[cpu].retx); printk(KERN_EMERG "While handling exception (EXCAUSE = 0x%x) at %s:\n", - (int)saved_seqstat & SEQSTAT_EXCAUSE, buf); - decode_address(buf, saved_dcplb_fault_addr); + (unsigned int)cpu_pda[cpu].seqstat & SEQSTAT_EXCAUSE, buf); + decode_address(buf, cpu_pda[cpu].dcplb_fault_addr); printk(KERN_NOTICE " DCPLB_FAULT_ADDR: %s\n", buf); - decode_address(buf, saved_icplb_fault_addr); + decode_address(buf, cpu_pda[cpu].icplb_fault_addr); printk(KERN_NOTICE " ICPLB_FAULT_ADDR: %s\n", buf); decode_address(buf, fp->retx); - printk(KERN_NOTICE "The instruction at %s caused a double exception\n", - buf); + printk(KERN_NOTICE "The instruction at %s caused a double exception\n", buf); } else #endif { @@ -239,6 +229,9 @@ asmlinkage void trap_c(struct pt_regs *fp) { #ifdef CONFIG_DEBUG_BFIN_HWTRACE_ON int j; +#endif +#ifdef CONFIG_DEBUG_HUNT_FOR_ZERO + unsigned int cpu = smp_processor_id(); #endif int sig = 0; siginfo_t info; @@ -417,7 +410,7 @@ asmlinkage void trap_c(struct pt_regs *fp) info.si_code = ILL_CPLB_MULHIT; sig = SIGSEGV; #ifdef CONFIG_DEBUG_HUNT_FOR_ZERO - if (saved_dcplb_fault_addr < FIXED_CODE_START) + if (cpu_pda[cpu].dcplb_fault_addr < FIXED_CODE_START) verbose_printk(KERN_NOTICE "NULL pointer access\n"); else #endif @@ -471,7 +464,7 @@ asmlinkage void trap_c(struct pt_regs *fp) info.si_code = ILL_CPLB_MULHIT; sig = SIGSEGV; #ifdef CONFIG_DEBUG_HUNT_FOR_ZERO - if (saved_icplb_fault_addr < FIXED_CODE_START) + if (cpu_pda[cpu].icplb_fault_addr < FIXED_CODE_START) verbose_printk(KERN_NOTICE "Jump to NULL address\n"); else #endif @@ -960,6 +953,7 @@ void dump_bfin_process(struct pt_regs *fp) else verbose_printk(KERN_NOTICE "COMM= invalid\n"); + printk(KERN_NOTICE "CPU = %d\n", current_thread_info()->cpu); if (!((unsigned long)current->mm & 0x3) && (unsigned long)current->mm >= FIXED_CODE_START) verbose_printk(KERN_NOTICE "TEXT = 0x%p-0x%p DATA = 0x%p-0x%p\n" KERN_NOTICE " BSS = 0x%p-0x%p USER-STACK = 0x%p\n" @@ -1053,6 +1047,7 @@ void show_regs(struct pt_regs *fp) struct irqaction *action; unsigned int i; unsigned long flags; + unsigned int cpu = smp_processor_id(); verbose_printk(KERN_NOTICE "\n" KERN_NOTICE "SEQUENCER STATUS:\t\t%s\n", print_tainted()); verbose_printk(KERN_NOTICE " SEQSTAT: %08lx IPEND: %04lx SYSCFG: %04lx\n", @@ -1112,9 +1107,9 @@ unlock: if (((long)fp->seqstat & SEQSTAT_EXCAUSE) && (((long)fp->seqstat & SEQSTAT_EXCAUSE) != VEC_HWERR)) { - decode_address(buf, saved_dcplb_fault_addr); + decode_address(buf, cpu_pda[cpu].dcplb_fault_addr); verbose_printk(KERN_NOTICE "DCPLB_FAULT_ADDR: %s\n", buf); - decode_address(buf, saved_icplb_fault_addr); + decode_address(buf, cpu_pda[cpu].icplb_fault_addr); verbose_printk(KERN_NOTICE "ICPLB_FAULT_ADDR: %s\n", buf); } @@ -1153,20 +1148,21 @@ unlock: asmlinkage int sys_bfin_spinlock(int *spinlock)__attribute__((l1_text)); #endif -asmlinkage int sys_bfin_spinlock(int *spinlock) +static DEFINE_SPINLOCK(bfin_spinlock_lock); + +asmlinkage int sys_bfin_spinlock(int *p) { - int ret = 0; - int tmp = 0; + int ret, tmp = 0; - local_irq_disable(); - ret = get_user(tmp, spinlock); - if (ret == 0) { - if (tmp) + spin_lock(&bfin_spinlock_lock); /* This would also hold kernel preemption. */ + ret = get_user(tmp, p); + if (likely(ret == 0)) { + if (unlikely(tmp)) ret = 1; - tmp = 1; - put_user(tmp, spinlock); + else + put_user(1, p); } - local_irq_enable(); + spin_unlock(&bfin_spinlock_lock); return ret; } -- cgit v1.2.3-59-g8ed1b From 36f649a55aa3ad1e2196403ba95a652f9900bc50 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Tue, 18 Nov 2008 17:48:22 +0800 Subject: Blackfin arch: do not define decode_instruction if hwtrace is turned off Signed-off-by: Mike Frysinger Signed-off-by: Bryan Wu --- arch/blackfin/kernel/traps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/blackfin/kernel/traps.c') diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c index af7cc43630de..950cc822fb75 100644 --- a/arch/blackfin/kernel/traps.c +++ b/arch/blackfin/kernel/traps.c @@ -649,13 +649,13 @@ static bool get_instruction(unsigned short *val, unsigned short *address) return false; } -/* +/* * decode the instruction if we are printing out the trace, as it * makes things easier to follow, without running it through objdump * These are the normal instructions which cause change of flow, which * would be at the source of the trace buffer */ -#ifdef CONFIG_DEBUG_VERBOSE +#if defined(CONFIG_DEBUG_VERBOSE) && defined(CONFIG_DEBUG_BFIN_HWTRACE_ON) static void decode_instruction(unsigned short *address) { unsigned short opcode; -- cgit v1.2.3-59-g8ed1b From 6a01f230339321292cf065551f8cf55361052461 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Wed, 7 Jan 2009 23:14:39 +0800 Subject: Blackfin arch: merge adeos blackfin part to arch/blackfin/ [Mike Frysinger : - handle bf531/bf532/bf534/bf536 variants in ipipe.h - cleanup IPIPE logic for bfin_set_irq_handler() - cleanup ipipe asm code a bit and add missing ENDPROC() - simplify IPIPE code in trap_c - unify some of the IPIPE code and fix style - simplify DO_IRQ_L1 handling with ipipe code - revert IRQ_SW_INT# addition from ipipe merge - remove duplicate get_{c,s}clk() prototypes ] Signed-off-by: Yi Li Signed-off-by: Mike Frysinger Signed-off-by: Bryan Wu --- arch/blackfin/configs/BF518F-EZBRD_defconfig | 16 +- arch/blackfin/configs/BF526-EZBRD_defconfig | 16 +- arch/blackfin/configs/BF527-EZKIT_defconfig | 16 +- arch/blackfin/configs/BF537-STAMP_defconfig | 16 +- arch/blackfin/configs/BF538-EZKIT_defconfig | 6 +- arch/blackfin/configs/CM-BF527_defconfig | 16 +- arch/blackfin/configs/CM-BF537E_defconfig | 16 +- arch/blackfin/configs/CM-BF537U_defconfig | 16 +- arch/blackfin/configs/PNAV-10_defconfig | 16 +- arch/blackfin/configs/SRV1_defconfig | 16 +- arch/blackfin/configs/TCM-BF537_defconfig | 16 +- arch/blackfin/include/asm/atomic.h | 32 +- arch/blackfin/include/asm/bitops.h | 24 +- arch/blackfin/include/asm/entry.h | 9 + arch/blackfin/include/asm/ipipe.h | 278 +++++++++++++ arch/blackfin/include/asm/ipipe_base.h | 80 ++++ arch/blackfin/include/asm/irq.h | 223 +++++++++-- arch/blackfin/include/asm/system.h | 4 +- arch/blackfin/kernel/Makefile | 2 + arch/blackfin/kernel/bfin_gpio.c | 100 ++--- arch/blackfin/kernel/cplb-mpu/cplbmgr.c | 8 +- arch/blackfin/kernel/entry.S | 4 + arch/blackfin/kernel/ipipe.c | 428 +++++++++++++++++++++ arch/blackfin/kernel/irqchip.c | 5 +- arch/blackfin/kernel/mcount.S | 70 ++++ arch/blackfin/kernel/process.c | 7 +- arch/blackfin/kernel/time.c | 15 +- arch/blackfin/kernel/traps.c | 13 +- arch/blackfin/lib/ins.S | 163 +++++++- arch/blackfin/mach-bf518/Kconfig | 34 +- .../mach-bf518/include/mach/cdefBF51x_base.h | 8 +- arch/blackfin/mach-bf518/include/mach/irq.h | 32 +- arch/blackfin/mach-bf518/ints-priority.c | 16 +- arch/blackfin/mach-bf527/Kconfig | 34 +- .../mach-bf527/include/mach/cdefBF52x_base.h | 8 +- arch/blackfin/mach-bf527/include/mach/irq.h | 32 +- arch/blackfin/mach-bf527/ints-priority.c | 16 +- arch/blackfin/mach-bf533/Kconfig | 2 +- arch/blackfin/mach-bf533/include/mach/cdefBF532.h | 16 +- arch/blackfin/mach-bf533/include/mach/irq.h | 6 +- arch/blackfin/mach-bf537/Kconfig | 34 +- arch/blackfin/mach-bf537/include/mach/cdefBF534.h | 8 +- arch/blackfin/mach-bf537/include/mach/irq.h | 32 +- arch/blackfin/mach-bf537/ints-priority.c | 16 +- arch/blackfin/mach-bf538/Kconfig | 14 +- arch/blackfin/mach-bf538/include/mach/cdefBF538.h | 8 +- arch/blackfin/mach-bf538/include/mach/irq.h | 12 +- arch/blackfin/mach-bf538/ints-priority.c | 6 +- arch/blackfin/mach-bf548/Kconfig | 2 +- .../mach-bf548/include/mach/cdefBF54x_base.h | 8 +- arch/blackfin/mach-bf548/include/mach/irq.h | 2 +- arch/blackfin/mach-bf561/Kconfig | 2 +- arch/blackfin/mach-bf561/include/mach/cdefBF561.h | 8 +- arch/blackfin/mach-common/cpufreq.c | 4 +- arch/blackfin/mach-common/interrupt.S | 60 +++ arch/blackfin/mach-common/ints-priority.c | 235 +++++++++-- arch/blackfin/mach-common/pm.c | 10 +- arch/blackfin/mach-common/smp.c | 4 +- 58 files changed, 1881 insertions(+), 419 deletions(-) create mode 100644 arch/blackfin/include/asm/ipipe.h create mode 100644 arch/blackfin/include/asm/ipipe_base.h create mode 100644 arch/blackfin/kernel/ipipe.c create mode 100644 arch/blackfin/kernel/mcount.S (limited to 'arch/blackfin/kernel/traps.c') diff --git a/arch/blackfin/configs/BF518F-EZBRD_defconfig b/arch/blackfin/configs/BF518F-EZBRD_defconfig index 98cf7c3cfef9..699781c4a4cf 100644 --- a/arch/blackfin/configs/BF518F-EZBRD_defconfig +++ b/arch/blackfin/configs/BF518F-EZBRD_defconfig @@ -204,14 +204,14 @@ CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_PORTH_INTA=11 CONFIG_IRQ_MAC_TX=11 CONFIG_IRQ_PORTH_INTB=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTA=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 diff --git a/arch/blackfin/configs/BF526-EZBRD_defconfig b/arch/blackfin/configs/BF526-EZBRD_defconfig index 667af6bb0c46..6dfcd2705133 100644 --- a/arch/blackfin/configs/BF526-EZBRD_defconfig +++ b/arch/blackfin/configs/BF526-EZBRD_defconfig @@ -199,14 +199,14 @@ CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_PORTH_INTA=11 CONFIG_IRQ_MAC_TX=11 CONFIG_IRQ_PORTH_INTB=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTA=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 diff --git a/arch/blackfin/configs/BF527-EZKIT_defconfig b/arch/blackfin/configs/BF527-EZKIT_defconfig index 70b50d010471..2947dcba5ff3 100644 --- a/arch/blackfin/configs/BF527-EZKIT_defconfig +++ b/arch/blackfin/configs/BF527-EZKIT_defconfig @@ -188,14 +188,14 @@ CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_PORTH_INTA=11 CONFIG_IRQ_MAC_TX=11 CONFIG_IRQ_PORTH_INTB=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=8 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTA=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 diff --git a/arch/blackfin/configs/BF537-STAMP_defconfig b/arch/blackfin/configs/BF537-STAMP_defconfig index ed84c620d03b..4aa668723032 100644 --- a/arch/blackfin/configs/BF537-STAMP_defconfig +++ b/arch/blackfin/configs/BF537-STAMP_defconfig @@ -148,14 +148,14 @@ CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_MAC_TX=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=8 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 CONFIG_IRQ_MEM_DMA1=13 diff --git a/arch/blackfin/configs/BF538-EZKIT_defconfig b/arch/blackfin/configs/BF538-EZKIT_defconfig index f1d2b3669940..baf33e662096 100644 --- a/arch/blackfin/configs/BF538-EZKIT_defconfig +++ b/arch/blackfin/configs/BF538-EZKIT_defconfig @@ -163,9 +163,9 @@ CONFIG_IRQ_UART0_RX=10 CONFIG_IRQ_UART0_TX=10 CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 CONFIG_IRQ_WATCH=13 CONFIG_IRQ_PORTF_INTA=12 CONFIG_IRQ_PORTF_INTB=12 diff --git a/arch/blackfin/configs/CM-BF527_defconfig b/arch/blackfin/configs/CM-BF527_defconfig index 0043da5f4938..719538571a5c 100644 --- a/arch/blackfin/configs/CM-BF527_defconfig +++ b/arch/blackfin/configs/CM-BF527_defconfig @@ -190,14 +190,14 @@ CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_PORTH_INTA=11 CONFIG_IRQ_MAC_TX=11 CONFIG_IRQ_PORTH_INTB=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTA=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 diff --git a/arch/blackfin/configs/CM-BF537E_defconfig b/arch/blackfin/configs/CM-BF537E_defconfig index 1b50d382f6e9..cbf4803b448c 100644 --- a/arch/blackfin/configs/CM-BF537E_defconfig +++ b/arch/blackfin/configs/CM-BF537E_defconfig @@ -157,14 +157,14 @@ CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_MAC_TX=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 CONFIG_IRQ_MEM_DMA1=13 diff --git a/arch/blackfin/configs/CM-BF537U_defconfig b/arch/blackfin/configs/CM-BF537U_defconfig index 2ca768d15b56..f2ac0cc37bf1 100644 --- a/arch/blackfin/configs/CM-BF537U_defconfig +++ b/arch/blackfin/configs/CM-BF537U_defconfig @@ -157,14 +157,14 @@ CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_MAC_TX=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 CONFIG_IRQ_MEM_DMA1=13 diff --git a/arch/blackfin/configs/PNAV-10_defconfig b/arch/blackfin/configs/PNAV-10_defconfig index 648edd99a94e..834fb8c9eefb 100644 --- a/arch/blackfin/configs/PNAV-10_defconfig +++ b/arch/blackfin/configs/PNAV-10_defconfig @@ -153,14 +153,14 @@ CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_MAC_TX=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 CONFIG_IRQ_MEM_DMA1=13 diff --git a/arch/blackfin/configs/SRV1_defconfig b/arch/blackfin/configs/SRV1_defconfig index ffa2396e6121..010f0aa7e4cb 100644 --- a/arch/blackfin/configs/SRV1_defconfig +++ b/arch/blackfin/configs/SRV1_defconfig @@ -172,14 +172,14 @@ CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_MAC_TX=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 CONFIG_IRQ_MEM_DMA1=13 diff --git a/arch/blackfin/configs/TCM-BF537_defconfig b/arch/blackfin/configs/TCM-BF537_defconfig index 6431192073c6..0ba4aa892139 100644 --- a/arch/blackfin/configs/TCM-BF537_defconfig +++ b/arch/blackfin/configs/TCM-BF537_defconfig @@ -144,14 +144,14 @@ CONFIG_IRQ_UART1_RX=10 CONFIG_IRQ_UART1_TX=10 CONFIG_IRQ_MAC_RX=11 CONFIG_IRQ_MAC_TX=11 -CONFIG_IRQ_TMR0=12 -CONFIG_IRQ_TMR1=12 -CONFIG_IRQ_TMR2=12 -CONFIG_IRQ_TMR3=12 -CONFIG_IRQ_TMR4=12 -CONFIG_IRQ_TMR5=12 -CONFIG_IRQ_TMR6=12 -CONFIG_IRQ_TMR7=12 +CONFIG_IRQ_TIMER0=12 +CONFIG_IRQ_TIMER1=12 +CONFIG_IRQ_TIMER2=12 +CONFIG_IRQ_TIMER3=12 +CONFIG_IRQ_TIMER4=12 +CONFIG_IRQ_TIMER5=12 +CONFIG_IRQ_TIMER6=12 +CONFIG_IRQ_TIMER7=12 CONFIG_IRQ_PORTG_INTB=12 CONFIG_IRQ_MEM_DMA0=13 CONFIG_IRQ_MEM_DMA1=13 diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index d76275e5638c..94b2a9b19451 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -92,18 +92,18 @@ static inline void atomic_add(int i, atomic_t *v) { long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter += i; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void atomic_sub(int i, atomic_t *v) { long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter -= i; - local_irq_restore(flags); + local_irq_restore_hw(flags); } @@ -112,10 +112,10 @@ static inline int atomic_add_return(int i, atomic_t *v) int __temp = 0; long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter += i; __temp = v->counter; - local_irq_restore(flags); + local_irq_restore_hw(flags); return __temp; @@ -126,10 +126,10 @@ static inline int atomic_sub_return(int i, atomic_t *v) int __temp = 0; long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter -= i; __temp = v->counter; - local_irq_restore(flags); + local_irq_restore_hw(flags); return __temp; } @@ -138,36 +138,36 @@ static inline void atomic_inc(volatile atomic_t *v) { long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter++; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void atomic_dec(volatile atomic_t *v) { long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter--; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) { long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter &= ~mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void atomic_set_mask(unsigned int mask, atomic_t *v) { long flags; - local_irq_save(flags); + local_irq_save_hw(flags); v->counter |= mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Atomic operations are already serializing */ diff --git a/arch/blackfin/include/asm/bitops.h b/arch/blackfin/include/asm/bitops.h index 9964e17232e9..21b036eadab1 100644 --- a/arch/blackfin/include/asm/bitops.h +++ b/arch/blackfin/include/asm/bitops.h @@ -90,9 +90,9 @@ static inline void set_bit(int nr, volatile unsigned long *addr) unsigned long flags; a += nr >> 5; mask = 1 << (nr & 0x1f); - local_irq_save(flags); + local_irq_save_hw(flags); *a |= mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void clear_bit(int nr, volatile unsigned long *addr) @@ -102,9 +102,9 @@ static inline void clear_bit(int nr, volatile unsigned long *addr) unsigned long flags; a += nr >> 5; mask = 1 << (nr & 0x1f); - local_irq_save(flags); + local_irq_save_hw(flags); *a &= ~mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void change_bit(int nr, volatile unsigned long *addr) @@ -114,9 +114,9 @@ static inline void change_bit(int nr, volatile unsigned long *addr) ADDR += nr >> 5; mask = 1 << (nr & 31); - local_irq_save(flags); + local_irq_save_hw(flags); *ADDR ^= mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline int test_and_set_bit(int nr, volatile unsigned long *addr) @@ -127,10 +127,10 @@ static inline int test_and_set_bit(int nr, volatile unsigned long *addr) a += nr >> 5; mask = 1 << (nr & 0x1f); - local_irq_save(flags); + local_irq_save_hw(flags); retval = (mask & *a) != 0; *a |= mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); return retval; } @@ -143,10 +143,10 @@ static inline int test_and_clear_bit(int nr, volatile unsigned long *addr) a += nr >> 5; mask = 1 << (nr & 0x1f); - local_irq_save(flags); + local_irq_save_hw(flags); retval = (mask & *a) != 0; *a &= ~mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); return retval; } @@ -159,10 +159,10 @@ static inline int test_and_change_bit(int nr, volatile unsigned long *addr) a += nr >> 5; mask = 1 << (nr & 0x1f); - local_irq_save(flags); + local_irq_save_hw(flags); retval = (mask & *a) != 0; *a ^= mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); return retval; } diff --git a/arch/blackfin/include/asm/entry.h b/arch/blackfin/include/asm/entry.h index d94e4f5139d2..b30a2968e274 100644 --- a/arch/blackfin/include/asm/entry.h +++ b/arch/blackfin/include/asm/entry.h @@ -27,6 +27,14 @@ #define SAVE_ALL_SYS save_context_no_interrupts /* This is used for all normal interrupts. It saves a minimum of registers to the stack, loads the IRQ number, and jumps to common code. */ +#ifdef CONFIG_IPIPE +# define LOAD_IPIPE_IPEND \ + P0.l = lo(IPEND); \ + P0.h = hi(IPEND); \ + R1 = [P0]; +#else +# define LOAD_IPIPE_IPEND +#endif #define INTERRUPT_ENTRY(N) \ [--sp] = SYSCFG; \ \ @@ -34,6 +42,7 @@ [--sp] = R0; /*orig_r0*/ \ [--sp] = (R7:0,P5:0); \ R0 = (N); \ + LOAD_IPIPE_IPEND \ jump __common_int_entry; /* For timer interrupts, we need to save IPEND, since the user_mode diff --git a/arch/blackfin/include/asm/ipipe.h b/arch/blackfin/include/asm/ipipe.h new file mode 100644 index 000000000000..76f53d8b9a0d --- /dev/null +++ b/arch/blackfin/include/asm/ipipe.h @@ -0,0 +1,278 @@ +/* -*- linux-c -*- + * include/asm-blackfin/ipipe.h + * + * Copyright (C) 2002-2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ASM_BLACKFIN_IPIPE_H +#define __ASM_BLACKFIN_IPIPE_H + +#ifdef CONFIG_IPIPE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IPIPE_ARCH_STRING "1.8-00" +#define IPIPE_MAJOR_NUMBER 1 +#define IPIPE_MINOR_NUMBER 8 +#define IPIPE_PATCH_NUMBER 0 + +#ifdef CONFIG_SMP +#error "I-pipe/blackfin: SMP not implemented" +#else /* !CONFIG_SMP */ +#define ipipe_processor_id() 0 +#endif /* CONFIG_SMP */ + +#define prepare_arch_switch(next) \ +do { \ + ipipe_schedule_notify(current, next); \ + local_irq_disable_hw(); \ +} while (0) + +#define task_hijacked(p) \ + ({ \ + int __x__ = ipipe_current_domain != ipipe_root_domain; \ + /* We would need to clear the SYNC flag for the root domain */ \ + /* over the current processor in SMP mode. */ \ + local_irq_enable_hw(); __x__; \ + }) + +struct ipipe_domain; + +struct ipipe_sysinfo { + + int ncpus; /* Number of CPUs on board */ + u64 cpufreq; /* CPU frequency (in Hz) */ + + /* Arch-dependent block */ + + struct { + unsigned tmirq; /* Timer tick IRQ */ + u64 tmfreq; /* Timer frequency */ + } archdep; +}; + +#define ipipe_read_tsc(t) \ + ({ \ + unsigned long __cy2; \ + __asm__ __volatile__ ("1: %0 = CYCLES2\n" \ + "%1 = CYCLES\n" \ + "%2 = CYCLES2\n" \ + "CC = %2 == %0\n" \ + "if ! CC jump 1b\n" \ + : "=r" (((unsigned long *)&t)[1]), \ + "=r" (((unsigned long *)&t)[0]), \ + "=r" (__cy2) \ + : /*no input*/ : "CC"); \ + t; \ + }) + +#define ipipe_cpu_freq() __ipipe_core_clock +#define ipipe_tsc2ns(_t) (((unsigned long)(_t)) * __ipipe_freq_scale) +#define ipipe_tsc2us(_t) (ipipe_tsc2ns(_t) / 1000 + 1) + +/* Private interface -- Internal use only */ + +#define __ipipe_check_platform() do { } while (0) + +#define __ipipe_init_platform() do { } while (0) + +extern atomic_t __ipipe_irq_lvdepth[IVG15 + 1]; + +extern unsigned long __ipipe_irq_lvmask; + +extern struct ipipe_domain ipipe_root; + +/* enable/disable_irqdesc _must_ be used in pairs. */ + +void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, + unsigned irq); + +void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, + unsigned irq); + +#define __ipipe_enable_irq(irq) (irq_desc[irq].chip->unmask(irq)) + +#define __ipipe_disable_irq(irq) (irq_desc[irq].chip->mask(irq)) + +#define __ipipe_lock_root() \ + set_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags) + +#define __ipipe_unlock_root() \ + clear_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags) + +void __ipipe_enable_pipeline(void); + +#define __ipipe_hook_critical_ipi(ipd) do { } while (0) + +#define __ipipe_sync_pipeline(syncmask) \ + do { \ + struct ipipe_domain *ipd = ipipe_current_domain; \ + if (likely(ipd != ipipe_root_domain || !test_bit(IPIPE_ROOTLOCK_FLAG, &ipd->flags))) \ + __ipipe_sync_stage(syncmask); \ + } while (0) + +void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs); + +int __ipipe_get_irq_priority(unsigned irq); + +int __ipipe_get_irqthread_priority(unsigned irq); + +void __ipipe_stall_root_raw(void); + +void __ipipe_unstall_root_raw(void); + +void __ipipe_serial_debug(const char *fmt, ...); + +DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs); + +extern unsigned long __ipipe_core_clock; + +extern unsigned long __ipipe_freq_scale; + +extern unsigned long __ipipe_irq_tail_hook; + +static inline unsigned long __ipipe_ffnz(unsigned long ul) +{ + return ffs(ul) - 1; +} + +#define __ipipe_run_irqtail() /* Must be a macro */ \ + do { \ + asmlinkage void __ipipe_call_irqtail(void); \ + unsigned long __pending; \ + CSYNC(); \ + __pending = bfin_read_IPEND(); \ + if (__pending & 0x8000) { \ + __pending &= ~0x8010; \ + if (__pending && (__pending & (__pending - 1)) == 0) \ + __ipipe_call_irqtail(); \ + } \ + } while (0) + +#define __ipipe_run_isr(ipd, irq) \ + do { \ + if (ipd == ipipe_root_domain) { \ + /* \ + * Note: the I-pipe implements a threaded interrupt model on \ + * this arch for Linux external IRQs. The interrupt handler we \ + * call here only wakes up the associated IRQ thread. \ + */ \ + if (ipipe_virtual_irq_p(irq)) { \ + /* No irqtail here; virtual interrupts have no effect \ + on IPEND so there is no need for processing \ + deferral. */ \ + local_irq_enable_nohead(ipd); \ + ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); \ + local_irq_disable_nohead(ipd); \ + } else \ + /* \ + * No need to run the irqtail here either; \ + * we can't be preempted by hw IRQs, so \ + * non-Linux IRQs cannot stack over the short \ + * thread wakeup code. Which in turn means \ + * that no irqtail condition could be pending \ + * for domains above Linux in the pipeline. \ + */ \ + ipd->irqs[irq].handler(irq, &__raw_get_cpu_var(__ipipe_tick_regs)); \ + } else { \ + __clear_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \ + local_irq_enable_nohead(ipd); \ + ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); \ + /* Attempt to exit the outer interrupt level before \ + * starting the deferred IRQ processing. */ \ + local_irq_disable_nohead(ipd); \ + __ipipe_run_irqtail(); \ + __set_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \ + } \ + } while (0) + +#define __ipipe_syscall_watched_p(p, sc) \ + (((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= NR_syscalls) + +void ipipe_init_irq_threads(void); + +int ipipe_start_irq_thread(unsigned irq, struct irq_desc *desc); + +#define IS_SYSIRQ(irq) ((irq) > IRQ_CORETMR && (irq) <= SYS_IRQS) +#define IS_GPIOIRQ(irq) ((irq) >= GPIO_IRQ_BASE && (irq) < NR_IRQS) + +#define IRQ_SYSTMR IRQ_TIMER0 +#define IRQ_PRIOTMR CONFIG_IRQ_TIMER0 + +#if defined(CONFIG_BF531) || defined(CONFIG_BF532) || defined(CONFIG_BF533) +#define PRIO_GPIODEMUX(irq) CONFIG_PFA +#elif defined(CONFIG_BF534) || defined(CONFIG_BF536) || defined(CONFIG_BF537) +#define PRIO_GPIODEMUX(irq) CONFIG_IRQ_PROG_INTA +#elif defined(CONFIG_BF52x) +#define PRIO_GPIODEMUX(irq) ((irq) == IRQ_PORTF_INTA ? CONFIG_IRQ_PORTF_INTA : \ + (irq) == IRQ_PORTG_INTA ? CONFIG_IRQ_PORTG_INTA : \ + (irq) == IRQ_PORTH_INTA ? CONFIG_IRQ_PORTH_INTA : \ + -1) +#elif defined(CONFIG_BF561) +#define PRIO_GPIODEMUX(irq) ((irq) == IRQ_PROG0_INTA ? CONFIG_IRQ_PROG0_INTA : \ + (irq) == IRQ_PROG1_INTA ? CONFIG_IRQ_PROG1_INTA : \ + (irq) == IRQ_PROG2_INTA ? CONFIG_IRQ_PROG2_INTA : \ + -1) +#define bfin_write_TIMER_DISABLE(val) bfin_write_TMRS8_DISABLE(val) +#define bfin_write_TIMER_ENABLE(val) bfin_write_TMRS8_ENABLE(val) +#define bfin_write_TIMER_STATUS(val) bfin_write_TMRS8_STATUS(val) +#define bfin_read_TIMER_STATUS() bfin_read_TMRS8_STATUS() +#elif defined(CONFIG_BF54x) +#define PRIO_GPIODEMUX(irq) ((irq) == IRQ_PINT0 ? CONFIG_IRQ_PINT0 : \ + (irq) == IRQ_PINT1 ? CONFIG_IRQ_PINT1 : \ + (irq) == IRQ_PINT2 ? CONFIG_IRQ_PINT2 : \ + (irq) == IRQ_PINT3 ? CONFIG_IRQ_PINT3 : \ + -1) +#define bfin_write_TIMER_DISABLE(val) bfin_write_TIMER_DISABLE0(val) +#define bfin_write_TIMER_ENABLE(val) bfin_write_TIMER_ENABLE0(val) +#define bfin_write_TIMER_STATUS(val) bfin_write_TIMER_STATUS0(val) +#define bfin_read_TIMER_STATUS(val) bfin_read_TIMER_STATUS0(val) +#else +# error "no PRIO_GPIODEMUX() for this part" +#endif + +#define __ipipe_root_tick_p(regs) ((regs->ipend & 0x10) != 0) + +#else /* !CONFIG_IPIPE */ + +#define task_hijacked(p) 0 +#define ipipe_trap_notify(t, r) 0 + +#define __ipipe_stall_root_raw() do { } while (0) +#define __ipipe_unstall_root_raw() do { } while (0) + +#define ipipe_init_irq_threads() do { } while (0) +#define ipipe_start_irq_thread(irq, desc) 0 + +#define IRQ_SYSTMR IRQ_CORETMR +#define IRQ_PRIOTMR IRQ_CORETMR + +#define __ipipe_root_tick_p(regs) 1 + +#endif /* !CONFIG_IPIPE */ + +#endif /* !__ASM_BLACKFIN_IPIPE_H */ diff --git a/arch/blackfin/include/asm/ipipe_base.h b/arch/blackfin/include/asm/ipipe_base.h new file mode 100644 index 000000000000..cb1025aeabcf --- /dev/null +++ b/arch/blackfin/include/asm/ipipe_base.h @@ -0,0 +1,80 @@ +/* -*- linux-c -*- + * include/asm-blackfin/_baseipipe.h + * + * Copyright (C) 2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ASM_BLACKFIN_IPIPE_BASE_H +#define __ASM_BLACKFIN_IPIPE_BASE_H + +#ifdef CONFIG_IPIPE + +#define IPIPE_NR_XIRQS NR_IRQS +#define IPIPE_IRQ_ISHIFT 5 /* 2^5 for 32bits arch. */ + +/* Blackfin-specific, global domain flags */ +#define IPIPE_ROOTLOCK_FLAG 1 /* Lock pipeline for root */ + + /* Blackfin traps -- i.e. exception vector numbers */ +#define IPIPE_NR_FAULTS 52 /* We leave a gap after VEC_ILL_RES. */ +/* Pseudo-vectors used for kernel events */ +#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS +#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT) +#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1) +#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2) +#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3) +#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4) +#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5) +#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6) +#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP +#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1) + +#define IPIPE_TIMER_IRQ IRQ_CORETMR + +#ifndef __ASSEMBLY__ + +#include + +extern int test_bit(int nr, const void *addr); + + +extern unsigned long __ipipe_root_status; /* Alias to ipipe_root_cpudom_var(status) */ + +static inline void __ipipe_stall_root(void) +{ + volatile unsigned long *p = &__ipipe_root_status; + set_bit(0, p); +} + +static inline unsigned long __ipipe_test_and_stall_root(void) +{ + volatile unsigned long *p = &__ipipe_root_status; + return test_and_set_bit(0, p); +} + +static inline unsigned long __ipipe_test_root(void) +{ + const unsigned long *p = &__ipipe_root_status; + return test_bit(0, p); +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* CONFIG_IPIPE */ + +#endif /* !__ASM_BLACKFIN_IPIPE_BASE_H */ diff --git a/arch/blackfin/include/asm/irq.h b/arch/blackfin/include/asm/irq.h index 21e25f778a63..3d977909ce7d 100644 --- a/arch/blackfin/include/asm/irq.h +++ b/arch/blackfin/include/asm/irq.h @@ -22,11 +22,176 @@ #include #include -static __inline__ int irq_canonicalize(int irq) +#ifdef CONFIG_SMP +/* Forward decl needed due to cdef inter dependencies */ +static inline uint32_t __pure bfin_dspid(void); +# define blackfin_core_id() (bfin_dspid() & 0xff) +# define bfin_irq_flags cpu_pda[blackfin_core_id()].imask +#else +extern unsigned long bfin_irq_flags; +#endif + +#ifdef CONFIG_IPIPE + +#include + +void __ipipe_unstall_root(void); + +void __ipipe_restore_root(unsigned long flags); + +#ifdef CONFIG_DEBUG_HWERR +# define __all_masked_irq_flags 0x3f +# define __save_and_cli_hw(x) \ + __asm__ __volatile__( \ + "cli %0;" \ + "sti %1;" \ + : "=&d"(x) \ + : "d" (0x3F) \ + ) +#else +# define __all_masked_irq_flags 0x1f +# define __save_and_cli_hw(x) \ + __asm__ __volatile__( \ + "cli %0;" \ + : "=&d"(x) \ + ) +#endif + +#define irqs_enabled_from_flags_hw(x) ((x) != __all_masked_irq_flags) +#define raw_irqs_disabled_flags(flags) (!irqs_enabled_from_flags_hw(flags)) +#define local_test_iflag_hw(x) irqs_enabled_from_flags_hw(x) + +#define local_save_flags(x) \ + do { \ + (x) = __ipipe_test_root() ? \ + __all_masked_irq_flags : bfin_irq_flags; \ + } while (0) + +#define local_irq_save(x) \ + do { \ + (x) = __ipipe_test_and_stall_root(); \ + } while (0) + +#define local_irq_restore(x) __ipipe_restore_root(x) +#define local_irq_disable() __ipipe_stall_root() +#define local_irq_enable() __ipipe_unstall_root() +#define irqs_disabled() __ipipe_test_root() + +#define local_save_flags_hw(x) \ + __asm__ __volatile__( \ + "cli %0;" \ + "sti %0;" \ + : "=d"(x) \ + ) + +#define irqs_disabled_hw() \ + ({ \ + unsigned long flags; \ + local_save_flags_hw(flags); \ + !irqs_enabled_from_flags_hw(flags); \ + }) + +static inline unsigned long raw_mangle_irq_bits(int virt, unsigned long real) { - return irq; + /* Merge virtual and real interrupt mask bits into a single + 32bit word. */ + return (real & ~(1 << 31)) | ((virt != 0) << 31); } +static inline int raw_demangle_irq_bits(unsigned long *x) +{ + int virt = (*x & (1 << 31)) != 0; + *x &= ~(1L << 31); + return virt; +} + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + +#define local_irq_disable_hw() \ + do { \ + int _tmp_dummy; \ + if (!irqs_disabled_hw()) \ + ipipe_trace_begin(0x80000000); \ + __asm__ __volatile__ ("cli %0;" : "=d" (_tmp_dummy) : ); \ + } while (0) + +#define local_irq_enable_hw() \ + do { \ + if (irqs_disabled_hw()) \ + ipipe_trace_end(0x80000000); \ + __asm__ __volatile__ ("sti %0;" : : "d"(bfin_irq_flags)); \ + } while (0) + +#define local_irq_save_hw(x) \ + do { \ + __save_and_cli_hw(x); \ + if (local_test_iflag_hw(x)) \ + ipipe_trace_begin(0x80000001); \ + } while (0) + +#define local_irq_restore_hw(x) \ + do { \ + if (local_test_iflag_hw(x)) { \ + ipipe_trace_end(0x80000001); \ + local_irq_enable_hw_notrace(); \ + } \ + } while (0) + +#define local_irq_disable_hw_notrace() \ + do { \ + int _tmp_dummy; \ + __asm__ __volatile__ ("cli %0;" : "=d" (_tmp_dummy) : ); \ + } while (0) + +#define local_irq_enable_hw_notrace() \ + __asm__ __volatile__( \ + "sti %0;" \ + : \ + : "d"(bfin_irq_flags) \ + ) + +#define local_irq_save_hw_notrace(x) __save_and_cli_hw(x) + +#define local_irq_restore_hw_notrace(x) \ + do { \ + if (local_test_iflag_hw(x)) \ + local_irq_enable_hw_notrace(); \ + } while (0) + +#else /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define local_irq_enable_hw() \ + __asm__ __volatile__( \ + "sti %0;" \ + : \ + : "d"(bfin_irq_flags) \ + ) + +#define local_irq_disable_hw() \ + do { \ + int _tmp_dummy; \ + __asm__ __volatile__ ( \ + "cli %0;" \ + : "=d" (_tmp_dummy)); \ + } while (0) + +#define local_irq_restore_hw(x) \ + do { \ + if (irqs_enabled_from_flags_hw(x)) \ + local_irq_enable_hw(); \ + } while (0) + +#define local_irq_save_hw(x) __save_and_cli_hw(x) + +#define local_irq_disable_hw_notrace() local_irq_disable_hw() +#define local_irq_enable_hw_notrace() local_irq_enable_hw() +#define local_irq_save_hw_notrace(x) local_irq_save_hw(x) +#define local_irq_restore_hw_notrace(x) local_irq_restore_hw(x) + +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#else /* !CONFIG_IPIPE */ + /* * Interrupt configuring macros. */ @@ -39,21 +204,6 @@ static __inline__ int irq_canonicalize(int irq) ); \ } while (0) -#if ANOMALY_05000244 && defined(CONFIG_BFIN_ICACHE) -# define NOP_PAD_ANOMALY_05000244 "nop; nop;" -#else -# define NOP_PAD_ANOMALY_05000244 -#endif - -#ifdef CONFIG_SMP -/* Forward decl needed due to cdef inter dependencies */ -static inline uint32_t __pure bfin_dspid(void); -# define blackfin_core_id() (bfin_dspid() & 0xff) -# define bfin_irq_flags cpu_pda[blackfin_core_id()].imask -#else -extern unsigned long bfin_irq_flags; -#endif - #define local_irq_enable() \ __asm__ __volatile__( \ "sti %0;" \ @@ -61,16 +211,6 @@ extern unsigned long bfin_irq_flags; : "d" (bfin_irq_flags) \ ) -#define idle_with_irq_disabled() \ - __asm__ __volatile__( \ - NOP_PAD_ANOMALY_05000244 \ - ".align 8;" \ - "sti %0;" \ - "idle;" \ - : \ - : "d" (bfin_irq_flags) \ - ) - #ifdef CONFIG_DEBUG_HWERR # define __save_and_cli(x) \ __asm__ __volatile__( \ @@ -116,4 +256,33 @@ extern unsigned long bfin_irq_flags; !irqs_enabled_from_flags(flags); \ }) +#define local_irq_save_hw(x) local_irq_save(x) +#define local_irq_restore_hw(x) local_irq_restore(x) +#define local_irq_enable_hw() local_irq_enable() +#define local_irq_disable_hw() local_irq_disable() +#define irqs_disabled_hw() irqs_disabled() + +#endif /* !CONFIG_IPIPE */ + +#if ANOMALY_05000244 && defined(CONFIG_BFIN_ICACHE) +# define NOP_PAD_ANOMALY_05000244 "nop; nop;" +#else +# define NOP_PAD_ANOMALY_05000244 +#endif + +#define idle_with_irq_disabled() \ + __asm__ __volatile__( \ + NOP_PAD_ANOMALY_05000244 \ + ".align 8;" \ + "sti %0;" \ + "idle;" \ + : \ + : "d" (bfin_irq_flags) \ + ) + +static inline int irq_canonicalize(int irq) +{ + return irq; +} + #endif /* _BFIN_IRQ_H_ */ diff --git a/arch/blackfin/include/asm/system.h b/arch/blackfin/include/asm/system.h index 812e6e6e2cee..a4c8254bec55 100644 --- a/arch/blackfin/include/asm/system.h +++ b/arch/blackfin/include/asm/system.h @@ -141,7 +141,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, unsigned long tmp = 0; unsigned long flags = 0; - local_irq_save(flags); + local_irq_save_hw(flags); switch (size) { case 1: @@ -163,7 +163,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, : "=&d" (tmp) : "d" (x), "m" (*__xg(ptr)) : "memory"); break; } - local_irq_restore(flags); + local_irq_restore_hw(flags); return tmp; } diff --git a/arch/blackfin/kernel/Makefile b/arch/blackfin/kernel/Makefile index f0902c120dc2..38a233374f07 100644 --- a/arch/blackfin/kernel/Makefile +++ b/arch/blackfin/kernel/Makefile @@ -15,6 +15,8 @@ else obj-y += time.o endif +obj-$(CONFIG_IPIPE) += ipipe.o +obj-$(CONFIG_IPIPE_TRACE_MCOUNT) += mcount.o obj-$(CONFIG_BFIN_GPTIMERS) += gptimers.o obj-$(CONFIG_CPLB_INFO) += cplbinfo.o obj-$(CONFIG_MODULES) += module.o diff --git a/arch/blackfin/kernel/bfin_gpio.c b/arch/blackfin/kernel/bfin_gpio.c index 2c72b15b71b0..4c14331978f6 100644 --- a/arch/blackfin/kernel/bfin_gpio.c +++ b/arch/blackfin/kernel/bfin_gpio.c @@ -422,13 +422,13 @@ arch_initcall(bfin_gpio_init); void set_gpio_ ## name(unsigned gpio, unsigned short arg) \ { \ unsigned long flags; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ if (arg) \ gpio_bankb[gpio_bank(gpio)]->name |= gpio_bit(gpio); \ else \ gpio_bankb[gpio_bank(gpio)]->name &= ~gpio_bit(gpio); \ AWA_DUMMY_READ(name); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ } \ EXPORT_SYMBOL(set_gpio_ ## name); @@ -444,13 +444,13 @@ SET_GPIO(both) void set_gpio_ ## name(unsigned gpio, unsigned short arg) \ { \ unsigned long flags; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ if (arg) \ gpio_bankb[gpio_bank(gpio)]->name ## _set = gpio_bit(gpio); \ else \ gpio_bankb[gpio_bank(gpio)]->name ## _clear = gpio_bit(gpio); \ AWA_DUMMY_READ(name); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ } \ EXPORT_SYMBOL(set_gpio_ ## name); #else @@ -473,10 +473,10 @@ SET_GPIO_SC(data) void set_gpio_toggle(unsigned gpio) { unsigned long flags; - local_irq_save(flags); + local_irq_save_hw(flags); gpio_bankb[gpio_bank(gpio)]->toggle = gpio_bit(gpio); AWA_DUMMY_READ(toggle); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #else void set_gpio_toggle(unsigned gpio) @@ -494,10 +494,10 @@ EXPORT_SYMBOL(set_gpio_toggle); void set_gpiop_ ## name(unsigned gpio, unsigned short arg) \ { \ unsigned long flags; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ gpio_bankb[gpio_bank(gpio)]->name = arg; \ AWA_DUMMY_READ(name); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ } \ EXPORT_SYMBOL(set_gpiop_ ## name); #else @@ -525,10 +525,10 @@ unsigned short get_gpio_ ## name(unsigned gpio) \ { \ unsigned long flags; \ unsigned short ret; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ ret = 0x01 & (gpio_bankb[gpio_bank(gpio)]->name >> gpio_sub_n(gpio)); \ AWA_DUMMY_READ(name); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ return ret; \ } \ EXPORT_SYMBOL(get_gpio_ ## name); @@ -558,10 +558,10 @@ unsigned short get_gpiop_ ## name(unsigned gpio) \ { \ unsigned long flags; \ unsigned short ret; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ ret = (gpio_bankb[gpio_bank(gpio)]->name); \ AWA_DUMMY_READ(name); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ return ret; \ } \ EXPORT_SYMBOL(get_gpiop_ ## name); @@ -611,10 +611,10 @@ int gpio_pm_wakeup_request(unsigned gpio, unsigned char type) if ((check_gpio(gpio) < 0) || !type) return -EINVAL; - local_irq_save(flags); + local_irq_save_hw(flags); wakeup_map[gpio_bank(gpio)] |= gpio_bit(gpio); wakeup_flags_map[gpio] = type; - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } @@ -627,11 +627,11 @@ void gpio_pm_wakeup_free(unsigned gpio) if (check_gpio(gpio) < 0) return; - local_irq_save(flags); + local_irq_save_hw(flags); wakeup_map[gpio_bank(gpio)] &= ~gpio_bit(gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); } EXPORT_SYMBOL(gpio_pm_wakeup_free); @@ -882,7 +882,7 @@ int peripheral_request(unsigned short per, const char *label) if (!(per & P_DEFINED)) return -ENODEV; - local_irq_save(flags); + local_irq_save_hw(flags); /* If a pin can be muxed as either GPIO or peripheral, make * sure it is not already a GPIO pin when we request it. @@ -893,7 +893,7 @@ int peripheral_request(unsigned short per, const char *label) printk(KERN_ERR "%s: Peripheral %d is already reserved as GPIO by %s !\n", __func__, ident, get_label(ident)); - local_irq_restore(flags); + local_irq_restore_hw(flags); return -EBUSY; } @@ -923,7 +923,7 @@ int peripheral_request(unsigned short per, const char *label) printk(KERN_ERR "%s: Peripheral %d function %d is already reserved by %s !\n", __func__, ident, P_FUNCT2MUX(per), get_label(ident)); - local_irq_restore(flags); + local_irq_restore_hw(flags); return -EBUSY; } } @@ -938,7 +938,7 @@ int peripheral_request(unsigned short per, const char *label) #endif port_setup(ident, PERIPHERAL_USAGE); - local_irq_restore(flags); + local_irq_restore_hw(flags); set_label(ident, label); return 0; @@ -980,10 +980,10 @@ void peripheral_free(unsigned short per) if (check_gpio(ident) < 0) return; - local_irq_save(flags); + local_irq_save_hw(flags); if (unlikely(!(reserved_peri_map[gpio_bank(ident)] & gpio_bit(ident)))) { - local_irq_restore(flags); + local_irq_restore_hw(flags); return; } @@ -994,7 +994,7 @@ void peripheral_free(unsigned short per) set_label(ident, "free"); - local_irq_restore(flags); + local_irq_restore_hw(flags); } EXPORT_SYMBOL(peripheral_free); @@ -1028,7 +1028,7 @@ int bfin_gpio_request(unsigned gpio, const char *label) if (check_gpio(gpio) < 0) return -EINVAL; - local_irq_save(flags); + local_irq_save_hw(flags); /* * Allow that the identical GPIO can @@ -1037,7 +1037,7 @@ int bfin_gpio_request(unsigned gpio, const char *label) */ if (cmp_label(gpio, label) == 0) { - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } @@ -1045,7 +1045,7 @@ int bfin_gpio_request(unsigned gpio, const char *label) dump_stack(); printk(KERN_ERR "bfin-gpio: GPIO %d is already reserved by %s !\n", gpio, get_label(gpio)); - local_irq_restore(flags); + local_irq_restore_hw(flags); return -EBUSY; } if (unlikely(reserved_peri_map[gpio_bank(gpio)] & gpio_bit(gpio))) { @@ -1053,7 +1053,7 @@ int bfin_gpio_request(unsigned gpio, const char *label) printk(KERN_ERR "bfin-gpio: GPIO %d is already reserved as Peripheral by %s !\n", gpio, get_label(gpio)); - local_irq_restore(flags); + local_irq_restore_hw(flags); return -EBUSY; } if (unlikely(reserved_gpio_irq_map[gpio_bank(gpio)] & gpio_bit(gpio))) @@ -1063,7 +1063,7 @@ int bfin_gpio_request(unsigned gpio, const char *label) reserved_gpio_map[gpio_bank(gpio)] |= gpio_bit(gpio); set_label(gpio, label); - local_irq_restore(flags); + local_irq_restore_hw(flags); port_setup(gpio, GPIO_USAGE); @@ -1078,12 +1078,12 @@ void bfin_gpio_free(unsigned gpio) if (check_gpio(gpio) < 0) return; - local_irq_save(flags); + local_irq_save_hw(flags); if (unlikely(!(reserved_gpio_map[gpio_bank(gpio)] & gpio_bit(gpio)))) { dump_stack(); gpio_error(gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); return; } @@ -1091,7 +1091,7 @@ void bfin_gpio_free(unsigned gpio) set_label(gpio, "free"); - local_irq_restore(flags); + local_irq_restore_hw(flags); } EXPORT_SYMBOL(bfin_gpio_free); @@ -1102,14 +1102,14 @@ int bfin_gpio_irq_request(unsigned gpio, const char *label) if (check_gpio(gpio) < 0) return -EINVAL; - local_irq_save(flags); + local_irq_save_hw(flags); if (unlikely(reserved_gpio_irq_map[gpio_bank(gpio)] & gpio_bit(gpio))) { dump_stack(); printk(KERN_ERR "bfin-gpio: GPIO %d is already reserved as gpio-irq !\n", gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); return -EBUSY; } if (unlikely(reserved_peri_map[gpio_bank(gpio)] & gpio_bit(gpio))) { @@ -1117,7 +1117,7 @@ int bfin_gpio_irq_request(unsigned gpio, const char *label) printk(KERN_ERR "bfin-gpio: GPIO %d is already reserved as Peripheral by %s !\n", gpio, get_label(gpio)); - local_irq_restore(flags); + local_irq_restore_hw(flags); return -EBUSY; } if (unlikely(reserved_gpio_map[gpio_bank(gpio)] & gpio_bit(gpio))) @@ -1128,7 +1128,7 @@ int bfin_gpio_irq_request(unsigned gpio, const char *label) reserved_gpio_irq_map[gpio_bank(gpio)] |= gpio_bit(gpio); set_label(gpio, label); - local_irq_restore(flags); + local_irq_restore_hw(flags); port_setup(gpio, GPIO_USAGE); @@ -1142,12 +1142,12 @@ void bfin_gpio_irq_free(unsigned gpio) if (check_gpio(gpio) < 0) return; - local_irq_save(flags); + local_irq_save_hw(flags); if (unlikely(!(reserved_gpio_irq_map[gpio_bank(gpio)] & gpio_bit(gpio)))) { dump_stack(); gpio_error(gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); return; } @@ -1155,7 +1155,7 @@ void bfin_gpio_irq_free(unsigned gpio) set_label(gpio, "free"); - local_irq_restore(flags); + local_irq_restore_hw(flags); } @@ -1169,10 +1169,10 @@ int bfin_gpio_direction_input(unsigned gpio) return -EINVAL; } - local_irq_save(flags); + local_irq_save_hw(flags); gpio_array[gpio_bank(gpio)]->port_dir_clear = gpio_bit(gpio); gpio_array[gpio_bank(gpio)]->port_inen |= gpio_bit(gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } @@ -1187,11 +1187,11 @@ int bfin_gpio_direction_output(unsigned gpio, int value) return -EINVAL; } - local_irq_save(flags); + local_irq_save_hw(flags); gpio_array[gpio_bank(gpio)]->port_inen &= ~gpio_bit(gpio); gpio_set_value(gpio, value); gpio_array[gpio_bank(gpio)]->port_dir_set = gpio_bit(gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } @@ -1218,10 +1218,10 @@ void bfin_gpio_irq_prepare(unsigned gpio) port_setup(gpio, GPIO_USAGE); - local_irq_save(flags); + local_irq_save_hw(flags); gpio_array[gpio_bank(gpio)]->port_dir_clear = gpio_bit(gpio); gpio_array[gpio_bank(gpio)]->port_inen |= gpio_bit(gpio); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #else @@ -1232,11 +1232,11 @@ int bfin_gpio_get_value(unsigned gpio) int ret; if (unlikely(get_gpio_edge(gpio))) { - local_irq_save(flags); + local_irq_save_hw(flags); set_gpio_edge(gpio, 0); ret = get_gpio_data(gpio); set_gpio_edge(gpio, 1); - local_irq_restore(flags); + local_irq_restore_hw(flags); return ret; } else @@ -1254,11 +1254,11 @@ int bfin_gpio_direction_input(unsigned gpio) return -EINVAL; } - local_irq_save(flags); + local_irq_save_hw(flags); gpio_bankb[gpio_bank(gpio)]->dir &= ~gpio_bit(gpio); gpio_bankb[gpio_bank(gpio)]->inen |= gpio_bit(gpio); AWA_DUMMY_READ(inen); - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } @@ -1273,7 +1273,7 @@ int bfin_gpio_direction_output(unsigned gpio, int value) return -EINVAL; } - local_irq_save(flags); + local_irq_save_hw(flags); gpio_bankb[gpio_bank(gpio)]->inen &= ~gpio_bit(gpio); if (value) @@ -1283,7 +1283,7 @@ int bfin_gpio_direction_output(unsigned gpio, int value) gpio_bankb[gpio_bank(gpio)]->dir |= gpio_bit(gpio); AWA_DUMMY_READ(dir); - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } diff --git a/arch/blackfin/kernel/cplb-mpu/cplbmgr.c b/arch/blackfin/kernel/cplb-mpu/cplbmgr.c index 5ef5d1a787fc..87463ce87f5a 100644 --- a/arch/blackfin/kernel/cplb-mpu/cplbmgr.c +++ b/arch/blackfin/kernel/cplb-mpu/cplbmgr.c @@ -332,7 +332,7 @@ void flush_switched_cplbs(unsigned int cpu) nr_cplb_flush[cpu]++; - local_irq_save(flags); + local_irq_save_hw(flags); disable_icplb(); for (i = first_switched_icplb; i < MAX_CPLBS; i++) { icplb_tbl[cpu][i].data = 0; @@ -346,7 +346,7 @@ void flush_switched_cplbs(unsigned int cpu) bfin_write32(DCPLB_DATA0 + i * 4, 0); } enable_dcplb(); - local_irq_restore(flags); + local_irq_restore_hw(flags); } @@ -362,7 +362,7 @@ void set_mask_dcplbs(unsigned long *masks, unsigned int cpu) return; } - local_irq_save(flags); + local_irq_save_hw(flags); current_rwx_mask[cpu] = masks; d_data = CPLB_SUPV_WR | CPLB_VALID | CPLB_DIRTY | PAGE_SIZE_4KB; @@ -382,5 +382,5 @@ void set_mask_dcplbs(unsigned long *masks, unsigned int cpu) addr += PAGE_SIZE; } enable_dcplb(); - local_irq_restore(flags); + local_irq_restore_hw(flags); } diff --git a/arch/blackfin/kernel/entry.S b/arch/blackfin/kernel/entry.S index c0c3fe811228..a9cfba9946b5 100644 --- a/arch/blackfin/kernel/entry.S +++ b/arch/blackfin/kernel/entry.S @@ -42,6 +42,10 @@ #endif ENTRY(_ret_from_fork) +#ifdef CONFIG_IPIPE + [--sp] = reti; /* IRQs on. */ + SP += 4; +#endif /* CONFIG_IPIPE */ SP += -12; call _schedule_tail; SP += 12; diff --git a/arch/blackfin/kernel/ipipe.c b/arch/blackfin/kernel/ipipe.c new file mode 100644 index 000000000000..339be5a3ae6a --- /dev/null +++ b/arch/blackfin/kernel/ipipe.c @@ -0,0 +1,428 @@ +/* -*- linux-c -*- + * linux/arch/blackfin/kernel/ipipe.c + * + * Copyright (C) 2005-2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-dependent I-pipe support for the Blackfin. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int create_irq_threads; + +DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); + +static DEFINE_PER_CPU(unsigned long, pending_irqthread_mask); + +static DEFINE_PER_CPU(int [IVG13 + 1], pending_irq_count); + +asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs); + +static void __ipipe_no_irqtail(void); + +unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail; +EXPORT_SYMBOL(__ipipe_irq_tail_hook); + +unsigned long __ipipe_core_clock; +EXPORT_SYMBOL(__ipipe_core_clock); + +unsigned long __ipipe_freq_scale; +EXPORT_SYMBOL(__ipipe_freq_scale); + +atomic_t __ipipe_irq_lvdepth[IVG15 + 1]; + +unsigned long __ipipe_irq_lvmask = __all_masked_irq_flags; +EXPORT_SYMBOL(__ipipe_irq_lvmask); + +static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc) +{ + desc->ipipe_ack(irq, desc); +} + +/* + * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw + * interrupts are off, and secondary CPUs are still lost in space. + */ +void __ipipe_enable_pipeline(void) +{ + unsigned irq; + + __ipipe_core_clock = get_cclk(); /* Fetch this once. */ + __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock; + + for (irq = 0; irq < NR_IRQS; ++irq) + ipipe_virtualize_irq(ipipe_root_domain, + irq, + (ipipe_irq_handler_t)&asm_do_IRQ, + NULL, + &__ipipe_ack_irq, + IPIPE_HANDLE_MASK | IPIPE_PASS_MASK); +} + +/* + * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic + * interrupt protection log is maintained here for each domain. Hw + * interrupts are masked on entry. + */ +void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs) +{ + struct ipipe_domain *this_domain, *next_domain; + struct list_head *head, *pos; + int m_ack, s = -1; + + /* + * Software-triggered IRQs do not need any ack. The contents + * of the register frame should only be used when processing + * the timer interrupt, but not for handling any other + * interrupt. + */ + m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR); + + this_domain = ipipe_current_domain; + + if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))) + head = &this_domain->p_link; + else { + head = __ipipe_pipeline.next; + next_domain = list_entry(head, struct ipipe_domain, p_link); + if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) { + if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) + next_domain->irqs[irq].acknowledge(irq, irq_desc + irq); + if (test_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags)) + s = __test_and_set_bit(IPIPE_STALL_FLAG, + &ipipe_root_cpudom_var(status)); + __ipipe_dispatch_wired(next_domain, irq); + goto finalize; + return; + } + } + + /* Ack the interrupt. */ + + pos = head; + + while (pos != &__ipipe_pipeline) { + next_domain = list_entry(pos, struct ipipe_domain, p_link); + /* + * For each domain handling the incoming IRQ, mark it + * as pending in its log. + */ + if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) { + /* + * Domains that handle this IRQ are polled for + * acknowledging it by decreasing priority + * order. The interrupt must be made pending + * _first_ in the domain's status flags before + * the PIC is unlocked. + */ + __ipipe_set_irq_pending(next_domain, irq); + + if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) { + next_domain->irqs[irq].acknowledge(irq, irq_desc + irq); + m_ack = 1; + } + } + + /* + * If the domain does not want the IRQ to be passed + * down the interrupt pipe, exit the loop now. + */ + if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control)) + break; + + pos = next_domain->p_link.next; + } + + /* + * Now walk the pipeline, yielding control to the highest + * priority domain that has pending interrupt(s) or + * immediately to the current domain if the interrupt has been + * marked as 'sticky'. This search does not go beyond the + * current domain in the pipeline. We also enforce the + * additional root stage lock (blackfin-specific). */ + + if (test_bit(IPIPE_ROOTLOCK_FLAG, &ipipe_root_domain->flags)) + s = __test_and_set_bit(IPIPE_STALL_FLAG, + &ipipe_root_cpudom_var(status)); +finalize: + + __ipipe_walk_pipeline(head); + + if (!s) + __clear_bit(IPIPE_STALL_FLAG, + &ipipe_root_cpudom_var(status)); +} + +int __ipipe_check_root(void) +{ + return ipipe_root_domain_p; +} + +void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct irq_desc *desc = irq_desc + irq; + int prio = desc->ic_prio; + + desc->depth = 0; + if (ipd != &ipipe_root && + atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1) + __set_bit(prio, &__ipipe_irq_lvmask); +} +EXPORT_SYMBOL(__ipipe_enable_irqdesc); + +void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct irq_desc *desc = irq_desc + irq; + int prio = desc->ic_prio; + + if (ipd != &ipipe_root && + atomic_dec_and_test(&__ipipe_irq_lvdepth[prio])) + __clear_bit(prio, &__ipipe_irq_lvmask); +} +EXPORT_SYMBOL(__ipipe_disable_irqdesc); + +void __ipipe_stall_root_raw(void) +{ + /* + * This code is called by the ins{bwl} routines (see + * arch/blackfin/lib/ins.S), which are heavily used by the + * network stack. It masks all interrupts but those handled by + * non-root domains, so that we keep decent network transfer + * rates for Linux without inducing pathological jitter for + * the real-time domain. + */ + __asm__ __volatile__ ("sti %0;" : : "d"(__ipipe_irq_lvmask)); + + __set_bit(IPIPE_STALL_FLAG, + &ipipe_root_cpudom_var(status)); +} + +void __ipipe_unstall_root_raw(void) +{ + __clear_bit(IPIPE_STALL_FLAG, + &ipipe_root_cpudom_var(status)); + + __asm__ __volatile__ ("sti %0;" : : "d"(bfin_irq_flags)); +} + +int __ipipe_syscall_root(struct pt_regs *regs) +{ + unsigned long flags; + + /* We need to run the IRQ tail hook whenever we don't + * propagate a syscall to higher domains, because we know that + * important operations might be pending there (e.g. Xenomai + * deferred rescheduling). */ + + if (!__ipipe_syscall_watched_p(current, regs->orig_p0)) { + void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook; + hook(); + return 0; + } + + /* + * This routine either returns: + * 0 -- if the syscall is to be passed to Linux; + * 1 -- if the syscall should not be passed to Linux, and no + * tail work should be performed; + * -1 -- if the syscall should not be passed to Linux but the + * tail work has to be performed (for handling signals etc). + */ + + if (__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) && + __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs) > 0) { + if (ipipe_root_domain_p && !in_atomic()) { + /* + * Sync pending VIRQs before _TIF_NEED_RESCHED + * is tested. + */ + local_irq_save_hw(flags); + if ((ipipe_root_cpudom_var(irqpend_himask) & IPIPE_IRQMASK_VIRT) != 0) + __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); + local_irq_restore_hw(flags); + return -1; + } + return 1; + } + + return 0; +} + +unsigned long ipipe_critical_enter(void (*syncfn) (void)) +{ + unsigned long flags; + + local_irq_save_hw(flags); + + return flags; +} + +void ipipe_critical_exit(unsigned long flags) +{ + local_irq_restore_hw(flags); +} + +static void __ipipe_no_irqtail(void) +{ +} + +int ipipe_get_sysinfo(struct ipipe_sysinfo *info) +{ + info->ncpus = num_online_cpus(); + info->cpufreq = ipipe_cpu_freq(); + info->archdep.tmirq = IPIPE_TIMER_IRQ; + info->archdep.tmfreq = info->cpufreq; + + return 0; +} + +/* + * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline + * just like if it has been actually received from a hw source. Also + * works for virtual interrupts. + */ +int ipipe_trigger_irq(unsigned irq) +{ + unsigned long flags; + + if (irq >= IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) + return -EINVAL; + + local_irq_save_hw(flags); + + __ipipe_handle_irq(irq, NULL); + + local_irq_restore_hw(flags); + + return 1; +} + +/* Move Linux IRQ to threads. */ + +static int do_irqd(void *__desc) +{ + struct irq_desc *desc = __desc; + unsigned irq = desc - irq_desc; + int thrprio = desc->thr_prio; + int thrmask = 1 << thrprio; + int cpu = smp_processor_id(); + cpumask_t cpumask; + + sigfillset(¤t->blocked); + current->flags |= PF_NOFREEZE; + cpumask = cpumask_of_cpu(cpu); + set_cpus_allowed(current, cpumask); + ipipe_setscheduler_root(current, SCHED_FIFO, 50 + thrprio); + + while (!kthread_should_stop()) { + local_irq_disable(); + if (!(desc->status & IRQ_SCHEDULED)) { + set_current_state(TASK_INTERRUPTIBLE); +resched: + local_irq_enable(); + schedule(); + local_irq_disable(); + } + __set_current_state(TASK_RUNNING); + /* + * If higher priority interrupt servers are ready to + * run, reschedule immediately. We need this for the + * GPIO demux IRQ handler to unmask the interrupt line + * _last_, after all GPIO IRQs have run. + */ + if (per_cpu(pending_irqthread_mask, cpu) & ~(thrmask|(thrmask-1))) + goto resched; + if (--per_cpu(pending_irq_count[thrprio], cpu) == 0) + per_cpu(pending_irqthread_mask, cpu) &= ~thrmask; + desc->status &= ~IRQ_SCHEDULED; + desc->thr_handler(irq, &__raw_get_cpu_var(__ipipe_tick_regs)); + local_irq_enable(); + } + __set_current_state(TASK_RUNNING); + return 0; +} + +static void kick_irqd(unsigned irq, void *cookie) +{ + struct irq_desc *desc = irq_desc + irq; + int thrprio = desc->thr_prio; + int thrmask = 1 << thrprio; + int cpu = smp_processor_id(); + + if (!(desc->status & IRQ_SCHEDULED)) { + desc->status |= IRQ_SCHEDULED; + per_cpu(pending_irqthread_mask, cpu) |= thrmask; + ++per_cpu(pending_irq_count[thrprio], cpu); + wake_up_process(desc->thread); + } +} + +int ipipe_start_irq_thread(unsigned irq, struct irq_desc *desc) +{ + if (desc->thread || !create_irq_threads) + return 0; + + desc->thread = kthread_create(do_irqd, desc, "IRQ %d", irq); + if (desc->thread == NULL) { + printk(KERN_ERR "irqd: could not create IRQ thread %d!\n", irq); + return -ENOMEM; + } + + wake_up_process(desc->thread); + + desc->thr_handler = ipipe_root_domain->irqs[irq].handler; + ipipe_root_domain->irqs[irq].handler = &kick_irqd; + + return 0; +} + +void __init ipipe_init_irq_threads(void) +{ + unsigned irq; + struct irq_desc *desc; + + create_irq_threads = 1; + + for (irq = 0; irq < NR_IRQS; irq++) { + desc = irq_desc + irq; + if (desc->action != NULL || + (desc->status & IRQ_NOREQUEST) != 0) + ipipe_start_irq_thread(irq, desc); + } +} + +EXPORT_SYMBOL(show_stack); + +#ifdef CONFIG_IPIPE_TRACE_MCOUNT +void notrace _mcount(void); +EXPORT_SYMBOL(_mcount); +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ diff --git a/arch/blackfin/kernel/irqchip.c b/arch/blackfin/kernel/irqchip.c index 1624e1129681..ab8209cbbad0 100644 --- a/arch/blackfin/kernel/irqchip.c +++ b/arch/blackfin/kernel/irqchip.c @@ -108,8 +108,9 @@ asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs) { struct pt_regs *old_regs; struct irq_desc *desc = irq_desc + irq; +#ifndef CONFIG_IPIPE unsigned short pending, other_ints; - +#endif old_regs = set_irq_regs(regs); /* @@ -137,6 +138,7 @@ asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs) #endif generic_handle_irq(irq); +#ifndef CONFIG_IPIPE /* Useless and bugous over the I-pipe: IRQs are threaded. */ /* If we're the only interrupt running (ignoring IRQ15 which is for syscalls), lower our priority to IRQ14 so that softirqs run at that level. If there's another, lower-level interrupt, irq_exit @@ -146,6 +148,7 @@ asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs) other_ints = pending & (pending - 1); if (other_ints == 0) lower_to_irq14(); +#endif /* !CONFIG_IPIPE */ irq_exit(); set_irq_regs(old_regs); diff --git a/arch/blackfin/kernel/mcount.S b/arch/blackfin/kernel/mcount.S new file mode 100644 index 000000000000..edcfb3865f46 --- /dev/null +++ b/arch/blackfin/kernel/mcount.S @@ -0,0 +1,70 @@ +/* + * linux/arch/blackfin/mcount.S + * + * Copyright (C) 2006 Analog Devices Inc. + * + * 2007/04/12 Save index, length, modify and base registers. --rpm + */ + +#include +#include + +.text + +.align 4 /* just in case */ + +ENTRY(__mcount) + [--sp] = i0; + [--sp] = i1; + [--sp] = i2; + [--sp] = i3; + [--sp] = l0; + [--sp] = l1; + [--sp] = l2; + [--sp] = l3; + [--sp] = m0; + [--sp] = m1; + [--sp] = m2; + [--sp] = m3; + [--sp] = b0; + [--sp] = b1; + [--sp] = b2; + [--sp] = b3; + [--sp] = ( r7:0, p5:0 ); + [--sp] = ASTAT; + + p1.L = _ipipe_trace_enable; + p1.H = _ipipe_trace_enable; + r7 = [p1]; + CC = r7 == 0; + if CC jump out; + link 0x10; + r0 = 0x0; + [sp + 0xc] = r0; /* v */ + r0 = 0x0; /* type: IPIPE_TRACE_FN */ + r1 = rets; + p0 = [fp]; /* p0: Prior FP */ + r2 = [p0 + 4]; /* r2: Prior RETS */ + call ___ipipe_trace; + unlink; +out: + ASTAT = [sp++]; + ( r7:0, p5:0 ) = [sp++]; + b3 = [sp++]; + b2 = [sp++]; + b1 = [sp++]; + b0 = [sp++]; + m3 = [sp++]; + m2 = [sp++]; + m1 = [sp++]; + m0 = [sp++]; + l3 = [sp++]; + l2 = [sp++]; + l1 = [sp++]; + l0 = [sp++]; + i3 = [sp++]; + i2 = [sp++]; + i1 = [sp++]; + i0 = [sp++]; + rts; +ENDPROC(__mcount) diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c index 1ec0faa8c68d..33e2e8993f7f 100644 --- a/arch/blackfin/kernel/process.c +++ b/arch/blackfin/kernel/process.c @@ -82,11 +82,14 @@ void cpu_idle(void)__attribute__((l1_text)); */ static void default_idle(void) { - local_irq_disable(); +#ifdef CONFIG_IPIPE + ipipe_suspend_domain(); +#endif + local_irq_disable_hw(); if (!need_resched()) idle_with_irq_disabled(); - local_irq_enable(); + local_irq_enable_hw(); } /* diff --git a/arch/blackfin/kernel/time.c b/arch/blackfin/kernel/time.c index ec4dfa38eb0a..172b4c588467 100644 --- a/arch/blackfin/kernel/time.c +++ b/arch/blackfin/kernel/time.c @@ -31,7 +31,7 @@ static struct irqaction bfin_timer_irq = { #endif }; -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) || defined(CONFIG_IPIPE) void __init setup_system_timer0(void) { /* Power down the core timer, just to play safe. */ @@ -74,7 +74,7 @@ void __init setup_core_timer(void) static void __init time_sched_init(irqreturn_t(*timer_routine) (int, void *)) { -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) || defined(CONFIG_IPIPE) setup_system_timer0(); bfin_timer_irq.handler = timer_routine; setup_irq(IRQ_TIMER0, &bfin_timer_irq); @@ -94,7 +94,7 @@ static unsigned long gettimeoffset(void) unsigned long offset; unsigned long clocks_per_jiffy; -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) || defined(CONFIG_IPIPE) clocks_per_jiffy = bfin_read_TIMER0_PERIOD(); offset = bfin_read_TIMER0_COUNTER() / \ (((clocks_per_jiffy + 1) * HZ) / USEC_PER_SEC); @@ -133,7 +133,8 @@ irqreturn_t timer_interrupt(int irq, void *dummy) static long last_rtc_update; write_seqlock(&xtime_lock); -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) && !defined(CONFIG_IPIPE) +/* FIXME: Here TIMIL0 is not set when IPIPE enabled, why? */ if (get_gptimer_status(0) & TIMER_STATUS_TIMIL0) { #endif do_timer(1); @@ -155,13 +156,17 @@ irqreturn_t timer_interrupt(int irq, void *dummy) /* Do it again in 60s. */ last_rtc_update = xtime.tv_sec - 600; } -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) && !defined(CONFIG_IPIPE) set_gptimer_status(0, TIMER_STATUS_TIMIL0); } #endif write_sequnlock(&xtime_lock); +#ifdef CONFIG_IPIPE + update_root_process_times(get_irq_regs()); +#else update_process_times(user_mode(get_irq_regs())); +#endif profile_tick(CPU_PROFILING); return IRQ_HANDLED; diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c index 950cc822fb75..956aefb84687 100644 --- a/arch/blackfin/kernel/traps.c +++ b/arch/blackfin/kernel/traps.c @@ -577,10 +577,15 @@ asmlinkage void trap_c(struct pt_regs *fp) } } - info.si_signo = sig; - info.si_errno = 0; - info.si_addr = (void __user *)fp->pc; - force_sig_info(sig, &info, current); +#ifdef CONFIG_IPIPE + if (!ipipe_trap_notify(fp->seqstat & 0x3f, fp)) +#endif + { + info.si_signo = sig; + info.si_errno = 0; + info.si_addr = (void __user *)fp->pc; + force_sig_info(sig, &info, current); + } trace_buffer_restore(j); return; diff --git a/arch/blackfin/lib/ins.S b/arch/blackfin/lib/ins.S index d60554dce87b..1b84b21ca7d1 100644 --- a/arch/blackfin/lib/ins.S +++ b/arch/blackfin/lib/ins.S @@ -56,7 +56,16 @@ ENTRY(_insl) #ifdef CONFIG_BFIN_INS_LOWOVERHEAD P0 = R0; /* P0 = port */ +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else cli R3; +#endif P1 = R1; /* P1 = address */ P2 = R2; /* P2 = count */ SSYNC; @@ -65,7 +74,14 @@ ENTRY(_insl) [P1++] = R0; NOP; .Llong_loop_e: NOP; +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else sti R3; +#endif RTS; #else P0 = R0; /* P0 = port */ @@ -74,13 +90,28 @@ ENTRY(_insl) SSYNC; LSETUP( .Llong_loop_s, .Llong_loop_e) LC0 = P2; .Llong_loop_s: +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else CLI R3; +#endif NOP; NOP; NOP; R0 = [P0]; [P1++] = R0; .Llong_loop_e: +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else STI R3; - +#endif RTS; #endif ENDPROC(_insl) @@ -88,7 +119,16 @@ ENDPROC(_insl) ENTRY(_insw) #ifdef CONFIG_BFIN_INS_LOWOVERHEAD P0 = R0; /* P0 = port */ +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else cli R3; +#endif P1 = R1; /* P1 = address */ P2 = R2; /* P2 = count */ SSYNC; @@ -97,7 +137,14 @@ ENTRY(_insw) W[P1++] = R0; NOP; .Lword_loop_e: NOP; +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else sti R3; +#endif RTS; #else P0 = R0; /* P0 = port */ @@ -106,12 +153,28 @@ ENTRY(_insw) SSYNC; LSETUP( .Lword_loop_s, .Lword_loop_e) LC0 = P2; .Lword_loop_s: +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else CLI R3; +#endif NOP; NOP; NOP; R0 = W[P0]; W[P1++] = R0; .Lword_loop_e: +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else STI R3; +#endif RTS; #endif @@ -120,7 +183,16 @@ ENDPROC(_insw) ENTRY(_insw_8) #ifdef CONFIG_BFIN_INS_LOWOVERHEAD P0 = R0; /* P0 = port */ +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else cli R3; +#endif P1 = R1; /* P1 = address */ P2 = R2; /* P2 = count */ SSYNC; @@ -131,7 +203,14 @@ ENTRY(_insw_8) B[P1++] = R0; NOP; .Lword8_loop_e: NOP; +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else sti R3; +#endif RTS; #else P0 = R0; /* P0 = port */ @@ -140,7 +219,16 @@ ENTRY(_insw_8) SSYNC; LSETUP( .Lword8_loop_s, .Lword8_loop_e) LC0 = P2; .Lword8_loop_s: +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else CLI R3; +#endif NOP; NOP; NOP; R0 = W[P0]; B[P1++] = R0; @@ -148,8 +236,14 @@ ENTRY(_insw_8) B[P1++] = R0; NOP; .Lword8_loop_e: +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else STI R3; - +#endif RTS; #endif ENDPROC(_insw_8) @@ -157,7 +251,16 @@ ENDPROC(_insw_8) ENTRY(_insb) #ifdef CONFIG_BFIN_INS_LOWOVERHEAD P0 = R0; /* P0 = port */ +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else cli R3; +#endif P1 = R1; /* P1 = address */ P2 = R2; /* P2 = count */ SSYNC; @@ -166,7 +269,14 @@ ENTRY(_insb) B[P1++] = R0; NOP; .Lbyte_loop_e: NOP; +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else sti R3; +#endif RTS; #else P0 = R0; /* P0 = port */ @@ -175,13 +285,28 @@ ENTRY(_insb) SSYNC; LSETUP( .Lbyte_loop_s, .Lbyte_loop_e) LC0 = P2; .Lbyte_loop_s: +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else CLI R3; +#endif NOP; NOP; NOP; R0 = B[P0]; B[P1++] = R0; .Lbyte_loop_e: +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else STI R3; - +#endif RTS; #endif ENDPROC(_insb) @@ -189,7 +314,16 @@ ENDPROC(_insb) ENTRY(_insl_16) #ifdef CONFIG_BFIN_INS_LOWOVERHEAD P0 = R0; /* P0 = port */ +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else cli R3; +#endif P1 = R1; /* P1 = address */ P2 = R2; /* P2 = count */ SSYNC; @@ -200,7 +334,14 @@ ENTRY(_insl_16) W[P1++] = R0; NOP; .Llong16_loop_e: NOP; +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else sti R3; +#endif RTS; #else P0 = R0; /* P0 = port */ @@ -209,14 +350,30 @@ ENTRY(_insl_16) SSYNC; LSETUP( .Llong16_loop_s, .Llong16_loop_e) LC0 = P2; .Llong16_loop_s: +#ifdef CONFIG_IPIPE + [--sp] = rets + [--sp] = (P5:0); + sp += -12 + call ___ipipe_stall_root_raw + sp += 12 + (P5:0) = [sp++]; +#else CLI R3; +#endif NOP; NOP; NOP; R0 = [P0]; W[P1++] = R0; R0 = R0 >> 16; W[P1++] = R0; .Llong16_loop_e: +#ifdef CONFIG_IPIPE + sp += -12 + call ___ipipe_unstall_root_raw + sp += 12 + rets = [sp++] +#else STI R3; +#endif RTS; #endif ENDPROC(_insl_16) diff --git a/arch/blackfin/mach-bf518/Kconfig b/arch/blackfin/mach-bf518/Kconfig index 00f2d3700637..f397ede006bf 100644 --- a/arch/blackfin/mach-bf518/Kconfig +++ b/arch/blackfin/mach-bf518/Kconfig @@ -154,29 +154,29 @@ config IRQ_MAC_TX config IRQ_PORTH_INTB int "IRQ_PORTH_INTB" default 11 -config IRQ_TMR0 - int "IRQ_TMR0" - default 12 -config IRQ_TMR1 - int "IRQ_TMR1" +config IRQ_TIMER0 + int "IRQ_TIMER0" + default 8 +config IRQ_TIMER1 + int "IRQ_TIMER1" default 12 -config IRQ_TMR2 - int "IRQ_TMR2" +config IRQ_TIMER2 + int "IRQ_TIMER2" default 12 -config IRQ_TMR3 - int "IRQ_TMR3" +config IRQ_TIMER3 + int "IRQ_TIMER3" default 12 -config IRQ_TMR4 - int "IRQ_TMR4" +config IRQ_TIMER4 + int "IRQ_TIMER4" default 12 -config IRQ_TMR5 - int "IRQ_TMR5" +config IRQ_TIMER5 + int "IRQ_TIMER5" default 12 -config IRQ_TMR6 - int "IRQ_TMR6" +config IRQ_TIMER6 + int "IRQ_TIMER6" default 12 -config IRQ_TMR7 - int "IRQ_TMR7" +config IRQ_TIMER7 + int "IRQ_TIMER7" default 12 config IRQ_PORTG_INTA int "IRQ_PORTG_INTA" diff --git a/arch/blackfin/mach-bf518/include/mach/cdefBF51x_base.h b/arch/blackfin/mach-bf518/include/mach/cdefBF51x_base.h index 9fbcd2221986..ee3d4733369c 100644 --- a/arch/blackfin/mach-bf518/include/mach/cdefBF51x_base.h +++ b/arch/blackfin/mach-bf518/include/mach/cdefBF51x_base.h @@ -1163,7 +1163,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -1177,7 +1177,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -1188,7 +1188,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -1202,7 +1202,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* _CDEF_BF52X_H */ diff --git a/arch/blackfin/mach-bf518/include/mach/irq.h b/arch/blackfin/mach-bf518/include/mach/irq.h index e5062f107ae2..3ff0f093313d 100644 --- a/arch/blackfin/mach-bf518/include/mach/irq.h +++ b/arch/blackfin/mach-bf518/include/mach/irq.h @@ -98,14 +98,14 @@ #define IRQ_PORTH_INTA BFIN_IRQ(29) /* Port H Interrupt A */ #define IRQ_MAC_TX BFIN_IRQ(30) /* DMA2 Channel (MAC TX) */ #define IRQ_PORTH_INTB BFIN_IRQ(31) /* Port H Interrupt B */ -#define IRQ_TMR0 BFIN_IRQ(32) /* Timer 0 */ -#define IRQ_TMR1 BFIN_IRQ(33) /* Timer 1 */ -#define IRQ_TMR2 BFIN_IRQ(34) /* Timer 2 */ -#define IRQ_TMR3 BFIN_IRQ(35) /* Timer 3 */ -#define IRQ_TMR4 BFIN_IRQ(36) /* Timer 4 */ -#define IRQ_TMR5 BFIN_IRQ(37) /* Timer 5 */ -#define IRQ_TMR6 BFIN_IRQ(38) /* Timer 6 */ -#define IRQ_TMR7 BFIN_IRQ(39) /* Timer 7 */ +#define IRQ_TIMER0 BFIN_IRQ(32) /* Timer 0 */ +#define IRQ_TIMER1 BFIN_IRQ(33) /* Timer 1 */ +#define IRQ_TIMER2 BFIN_IRQ(34) /* Timer 2 */ +#define IRQ_TIMER3 BFIN_IRQ(35) /* Timer 3 */ +#define IRQ_TIMER4 BFIN_IRQ(36) /* Timer 4 */ +#define IRQ_TIMER5 BFIN_IRQ(37) /* Timer 5 */ +#define IRQ_TIMER6 BFIN_IRQ(38) /* Timer 6 */ +#define IRQ_TIMER7 BFIN_IRQ(39) /* Timer 7 */ #define IRQ_PORTG_INTA BFIN_IRQ(40) /* Port G Interrupt A */ #define IRQ_PORTG_INTB BFIN_IRQ(41) /* Port G Interrupt B */ #define IRQ_MEM_DMA0 BFIN_IRQ(42) /* MDMA Stream 0 */ @@ -230,14 +230,14 @@ #define IRQ_PORTH_INTB_POS 28 /* IAR4 BIT FIELDS */ -#define IRQ_TMR0_POS 0 -#define IRQ_TMR1_POS 4 -#define IRQ_TMR2_POS 8 -#define IRQ_TMR3_POS 12 -#define IRQ_TMR4_POS 16 -#define IRQ_TMR5_POS 20 -#define IRQ_TMR6_POS 24 -#define IRQ_TMR7_POS 28 +#define IRQ_TIMER0_POS 0 +#define IRQ_TIMER1_POS 4 +#define IRQ_TIMER2_POS 8 +#define IRQ_TIMER3_POS 12 +#define IRQ_TIMER4_POS 16 +#define IRQ_TIMER5_POS 20 +#define IRQ_TIMER6_POS 24 +#define IRQ_TIMER7_POS 28 /* IAR5 BIT FIELDS */ #define IRQ_PORTG_INTA_POS 0 diff --git a/arch/blackfin/mach-bf518/ints-priority.c b/arch/blackfin/mach-bf518/ints-priority.c index c490c79194c0..3151fd5501ca 100644 --- a/arch/blackfin/mach-bf518/ints-priority.c +++ b/arch/blackfin/mach-bf518/ints-priority.c @@ -70,14 +70,14 @@ void __init program_IAR(void) ((CONFIG_IRQ_MAC_TX - 7) << IRQ_MAC_TX_POS) | ((CONFIG_IRQ_PORTH_INTB - 7) << IRQ_PORTH_INTB_POS)); - bfin_write_SIC_IAR4(((CONFIG_IRQ_TMR0 - 7) << IRQ_TMR0_POS) | - ((CONFIG_IRQ_TMR1 - 7) << IRQ_TMR1_POS) | - ((CONFIG_IRQ_TMR2 - 7) << IRQ_TMR2_POS) | - ((CONFIG_IRQ_TMR3 - 7) << IRQ_TMR3_POS) | - ((CONFIG_IRQ_TMR4 - 7) << IRQ_TMR4_POS) | - ((CONFIG_IRQ_TMR5 - 7) << IRQ_TMR5_POS) | - ((CONFIG_IRQ_TMR6 - 7) << IRQ_TMR6_POS) | - ((CONFIG_IRQ_TMR7 - 7) << IRQ_TMR7_POS)); + bfin_write_SIC_IAR4(((CONFIG_IRQ_TIMER0 - 7) << IRQ_TIMER0_POS) | + ((CONFIG_IRQ_TIMER1 - 7) << IRQ_TIMER1_POS) | + ((CONFIG_IRQ_TIMER2 - 7) << IRQ_TIMER2_POS) | + ((CONFIG_IRQ_TIMER3 - 7) << IRQ_TIMER3_POS) | + ((CONFIG_IRQ_TIMER4 - 7) << IRQ_TIMER4_POS) | + ((CONFIG_IRQ_TIMER5 - 7) << IRQ_TIMER5_POS) | + ((CONFIG_IRQ_TIMER6 - 7) << IRQ_TIMER6_POS) | + ((CONFIG_IRQ_TIMER7 - 7) << IRQ_TIMER7_POS)); bfin_write_SIC_IAR5(((CONFIG_IRQ_PORTG_INTA - 7) << IRQ_PORTG_INTA_POS) | ((CONFIG_IRQ_PORTG_INTB - 7) << IRQ_PORTG_INTB_POS) | diff --git a/arch/blackfin/mach-bf527/Kconfig b/arch/blackfin/mach-bf527/Kconfig index 3cde4beeb214..8438ec6d6679 100644 --- a/arch/blackfin/mach-bf527/Kconfig +++ b/arch/blackfin/mach-bf527/Kconfig @@ -168,29 +168,29 @@ config IRQ_MAC_TX config IRQ_PORTH_INTB int "IRQ_PORTH_INTB" default 11 -config IRQ_TMR0 - int "IRQ_TMR0" - default 12 -config IRQ_TMR1 - int "IRQ_TMR1" +config IRQ_TIMER0 + int "IRQ_TIMER0" + default 8 +config IRQ_TIMER1 + int "IRQ_TIMER1" default 12 -config IRQ_TMR2 - int "IRQ_TMR2" +config IRQ_TIMER2 + int "IRQ_TIMER2" default 12 -config IRQ_TMR3 - int "IRQ_TMR3" +config IRQ_TIMER3 + int "IRQ_TIMER3" default 12 -config IRQ_TMR4 - int "IRQ_TMR4" +config IRQ_TIMER4 + int "IRQ_TIMER4" default 12 -config IRQ_TMR5 - int "IRQ_TMR5" +config IRQ_TIMER5 + int "IRQ_TIMER5" default 12 -config IRQ_TMR6 - int "IRQ_TMR6" +config IRQ_TIMER6 + int "IRQ_TIMER6" default 12 -config IRQ_TMR7 - int "IRQ_TMR7" +config IRQ_TIMER7 + int "IRQ_TIMER7" default 12 config IRQ_PORTG_INTA int "IRQ_PORTG_INTA" diff --git a/arch/blackfin/mach-bf527/include/mach/cdefBF52x_base.h b/arch/blackfin/mach-bf527/include/mach/cdefBF52x_base.h index 8a374c358edf..1fe76d8e0403 100644 --- a/arch/blackfin/mach-bf527/include/mach/cdefBF52x_base.h +++ b/arch/blackfin/mach-bf527/include/mach/cdefBF52x_base.h @@ -1163,7 +1163,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -1177,7 +1177,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -1188,7 +1188,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -1202,7 +1202,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* _CDEF_BF52X_H */ diff --git a/arch/blackfin/mach-bf527/include/mach/irq.h b/arch/blackfin/mach-bf527/include/mach/irq.h index 4e2b3f2020e5..8ea660d8151f 100644 --- a/arch/blackfin/mach-bf527/include/mach/irq.h +++ b/arch/blackfin/mach-bf527/include/mach/irq.h @@ -96,14 +96,14 @@ #define IRQ_MAC_TX BFIN_IRQ(30) /* DMA2 Channel (MAC TX/NAND) */ #define IRQ_NFC BFIN_IRQ(30) /* DMA2 Channel (MAC TX/NAND) */ #define IRQ_PORTH_INTB BFIN_IRQ(31) /* Port H Interrupt B */ -#define IRQ_TMR0 BFIN_IRQ(32) /* Timer 0 */ -#define IRQ_TMR1 BFIN_IRQ(33) /* Timer 1 */ -#define IRQ_TMR2 BFIN_IRQ(34) /* Timer 2 */ -#define IRQ_TMR3 BFIN_IRQ(35) /* Timer 3 */ -#define IRQ_TMR4 BFIN_IRQ(36) /* Timer 4 */ -#define IRQ_TMR5 BFIN_IRQ(37) /* Timer 5 */ -#define IRQ_TMR6 BFIN_IRQ(38) /* Timer 6 */ -#define IRQ_TMR7 BFIN_IRQ(39) /* Timer 7 */ +#define IRQ_TIMER0 BFIN_IRQ(32) /* Timer 0 */ +#define IRQ_TIMER1 BFIN_IRQ(33) /* Timer 1 */ +#define IRQ_TIMER2 BFIN_IRQ(34) /* Timer 2 */ +#define IRQ_TIMER3 BFIN_IRQ(35) /* Timer 3 */ +#define IRQ_TIMER4 BFIN_IRQ(36) /* Timer 4 */ +#define IRQ_TIMER5 BFIN_IRQ(37) /* Timer 5 */ +#define IRQ_TIMER6 BFIN_IRQ(38) /* Timer 6 */ +#define IRQ_TIMER7 BFIN_IRQ(39) /* Timer 7 */ #define IRQ_PORTG_INTA BFIN_IRQ(40) /* Port G Interrupt A */ #define IRQ_PORTG_INTB BFIN_IRQ(41) /* Port G Interrupt B */ #define IRQ_MEM_DMA0 BFIN_IRQ(42) /* MDMA Stream 0 */ @@ -227,14 +227,14 @@ #define IRQ_PORTH_INTB_POS 28 /* IAR4 BIT FIELDS */ -#define IRQ_TMR0_POS 0 -#define IRQ_TMR1_POS 4 -#define IRQ_TMR2_POS 8 -#define IRQ_TMR3_POS 12 -#define IRQ_TMR4_POS 16 -#define IRQ_TMR5_POS 20 -#define IRQ_TMR6_POS 24 -#define IRQ_TMR7_POS 28 +#define IRQ_TIMER0_POS 0 +#define IRQ_TIMER1_POS 4 +#define IRQ_TIMER2_POS 8 +#define IRQ_TIMER3_POS 12 +#define IRQ_TIMER4_POS 16 +#define IRQ_TIMER5_POS 20 +#define IRQ_TIMER6_POS 24 +#define IRQ_TIMER7_POS 28 /* IAR5 BIT FIELDS */ #define IRQ_PORTG_INTA_POS 0 diff --git a/arch/blackfin/mach-bf527/ints-priority.c b/arch/blackfin/mach-bf527/ints-priority.c index 8a2367403d2b..f8c8acd73e30 100644 --- a/arch/blackfin/mach-bf527/ints-priority.c +++ b/arch/blackfin/mach-bf527/ints-priority.c @@ -69,14 +69,14 @@ void __init program_IAR(void) ((CONFIG_IRQ_MAC_TX - 7) << IRQ_MAC_TX_POS) | ((CONFIG_IRQ_PORTH_INTB - 7) << IRQ_PORTH_INTB_POS)); - bfin_write_SIC_IAR4(((CONFIG_IRQ_TMR0 - 7) << IRQ_TMR0_POS) | - ((CONFIG_IRQ_TMR1 - 7) << IRQ_TMR1_POS) | - ((CONFIG_IRQ_TMR2 - 7) << IRQ_TMR2_POS) | - ((CONFIG_IRQ_TMR3 - 7) << IRQ_TMR3_POS) | - ((CONFIG_IRQ_TMR4 - 7) << IRQ_TMR4_POS) | - ((CONFIG_IRQ_TMR5 - 7) << IRQ_TMR5_POS) | - ((CONFIG_IRQ_TMR6 - 7) << IRQ_TMR6_POS) | - ((CONFIG_IRQ_TMR7 - 7) << IRQ_TMR7_POS)); + bfin_write_SIC_IAR4(((CONFIG_IRQ_TIMER0 - 7) << IRQ_TIMER0_POS) | + ((CONFIG_IRQ_TIMER1 - 7) << IRQ_TIMER1_POS) | + ((CONFIG_IRQ_TIMER2 - 7) << IRQ_TIMER2_POS) | + ((CONFIG_IRQ_TIMER3 - 7) << IRQ_TIMER3_POS) | + ((CONFIG_IRQ_TIMER4 - 7) << IRQ_TIMER4_POS) | + ((CONFIG_IRQ_TIMER5 - 7) << IRQ_TIMER5_POS) | + ((CONFIG_IRQ_TIMER6 - 7) << IRQ_TIMER6_POS) | + ((CONFIG_IRQ_TIMER7 - 7) << IRQ_TIMER7_POS)); bfin_write_SIC_IAR5(((CONFIG_IRQ_PORTG_INTA - 7) << IRQ_PORTG_INTA_POS) | ((CONFIG_IRQ_PORTG_INTB - 7) << IRQ_PORTG_INTB_POS) | diff --git a/arch/blackfin/mach-bf533/Kconfig b/arch/blackfin/mach-bf533/Kconfig index 76beb75f12da..14427de7d77f 100644 --- a/arch/blackfin/mach-bf533/Kconfig +++ b/arch/blackfin/mach-bf533/Kconfig @@ -59,7 +59,7 @@ config DMA7_UARTTX default 10 config TIMER0 int "TIMER0" - default 11 + default 8 config TIMER1 int "TIMER1" default 11 diff --git a/arch/blackfin/mach-bf533/include/mach/cdefBF532.h b/arch/blackfin/mach-bf533/include/mach/cdefBF532.h index d7b08f638ea4..bbc3c8386d48 100644 --- a/arch/blackfin/mach-bf533/include/mach/cdefBF532.h +++ b/arch/blackfin/mach-bf533/include/mach/cdefBF532.h @@ -684,10 +684,10 @@ static inline void bfin_write_FIO_FLAG_##name(unsigned short val) \ { \ unsigned long flags; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ bfin_write16(FIO_FLAG_##name, val); \ bfin_read_CHIPID(); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ } BFIN_WRITE_FIO_FLAG(D) BFIN_WRITE_FIO_FLAG(C) @@ -699,10 +699,10 @@ static inline u16 bfin_read_FIO_FLAG_##name(void) \ { \ unsigned long flags; \ u16 ret; \ - local_irq_save(flags); \ + local_irq_save_hw(flags); \ ret = bfin_read16(FIO_FLAG_##name); \ bfin_read_CHIPID(); \ - local_irq_restore(flags); \ + local_irq_restore_hw(flags); \ return ret; \ } BFIN_READ_FIO_FLAG(D) @@ -729,7 +729,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr = bfin_read32(SIC_IWR); /* Only allow PPL Wakeup) */ @@ -740,7 +740,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) asm("IDLE;"); bfin_write32(SIC_IWR, iwr); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -751,7 +751,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr = bfin_read32(SIC_IWR); /* Only allow PPL Wakeup) */ @@ -762,7 +762,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) asm("IDLE;"); bfin_write32(SIC_IWR, iwr); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* _CDEF_BF532_H */ diff --git a/arch/blackfin/mach-bf533/include/mach/irq.h b/arch/blackfin/mach-bf533/include/mach/irq.h index e7dd315159df..db1e346cd1aa 100644 --- a/arch/blackfin/mach-bf533/include/mach/irq.h +++ b/arch/blackfin/mach-bf533/include/mach/irq.h @@ -100,9 +100,9 @@ Core Emulation ** #define IRQ_SPI 20 /*DMA5 Interrupt (SPI) */ #define IRQ_UART0_RX 21 /*DMA6 Interrupt (UART RX) */ #define IRQ_UART0_TX 22 /*DMA7 Interrupt (UART TX) */ -#define IRQ_TMR0 23 /*Timer 0 */ -#define IRQ_TMR1 24 /*Timer 1 */ -#define IRQ_TMR2 25 /*Timer 2 */ +#define IRQ_TIMER0 23 /*Timer 0 */ +#define IRQ_TIMER1 24 /*Timer 1 */ +#define IRQ_TIMER2 25 /*Timer 2 */ #define IRQ_PROG_INTA 26 /*Programmable Flags A (8) */ #define IRQ_PROG_INTB 27 /*Programmable Flags B (8) */ #define IRQ_MEM_DMA0 28 /*DMA8/9 Interrupt (Memory DMA Stream 0) */ diff --git a/arch/blackfin/mach-bf537/Kconfig b/arch/blackfin/mach-bf537/Kconfig index 8255374c04aa..bbc08fd4f122 100644 --- a/arch/blackfin/mach-bf537/Kconfig +++ b/arch/blackfin/mach-bf537/Kconfig @@ -64,29 +64,29 @@ config IRQ_MAC_RX config IRQ_MAC_TX int "IRQ_MAC_TX" default 11 -config IRQ_TMR0 - int "IRQ_TMR0" - default 12 -config IRQ_TMR1 - int "IRQ_TMR1" +config IRQ_TIMER0 + int "IRQ_TIMER0" + default 8 +config IRQ_TIMER1 + int "IRQ_TIMER1" default 12 -config IRQ_TMR2 - int "IRQ_TMR2" +config IRQ_TIMER2 + int "IRQ_TIMER2" default 12 -config IRQ_TMR3 - int "IRQ_TMR3" +config IRQ_TIMER3 + int "IRQ_TIMER3" default 12 -config IRQ_TMR4 - int "IRQ_TMR4" +config IRQ_TIMER4 + int "IRQ_TIMER4" default 12 -config IRQ_TMR5 - int "IRQ_TMR5" +config IRQ_TIMER5 + int "IRQ_TIMER5" default 12 -config IRQ_TMR6 - int "IRQ_TMR6" +config IRQ_TIMER6 + int "IRQ_TIMER6" default 12 -config IRQ_TMR7 - int "IRQ_TMR7" +config IRQ_TIMER7 + int "IRQ_TIMER7" default 12 config IRQ_PROG_INTA int "IRQ_PROG_INTA" diff --git a/arch/blackfin/mach-bf537/include/mach/cdefBF534.h b/arch/blackfin/mach-bf537/include/mach/cdefBF534.h index f3416923be69..5f8b5f845be6 100644 --- a/arch/blackfin/mach-bf537/include/mach/cdefBF534.h +++ b/arch/blackfin/mach-bf537/include/mach/cdefBF534.h @@ -1783,7 +1783,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr = bfin_read32(SIC_IWR); /* Only allow PPL Wakeup) */ @@ -1794,7 +1794,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) asm("IDLE;"); bfin_write32(SIC_IWR, iwr); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -1805,7 +1805,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr = bfin_read32(SIC_IWR); /* Only allow PPL Wakeup) */ @@ -1816,7 +1816,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) asm("IDLE;"); bfin_write32(SIC_IWR, iwr); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* _CDEF_BF534_H */ diff --git a/arch/blackfin/mach-bf537/include/mach/irq.h b/arch/blackfin/mach-bf537/include/mach/irq.h index 2e68a8a1e730..b2a71d5d4e5f 100644 --- a/arch/blackfin/mach-bf537/include/mach/irq.h +++ b/arch/blackfin/mach-bf537/include/mach/irq.h @@ -82,14 +82,14 @@ #define IRQ_CAN_TX 23 /*CAN Transmit Interrupt */ #define IRQ_MAC_RX 24 /*DMA1 (Ethernet RX) Interrupt */ #define IRQ_MAC_TX 25 /*DMA2 (Ethernet TX) Interrupt */ -#define IRQ_TMR0 26 /*Timer 0 */ -#define IRQ_TMR1 27 /*Timer 1 */ -#define IRQ_TMR2 28 /*Timer 2 */ -#define IRQ_TMR3 29 /*Timer 3 */ -#define IRQ_TMR4 30 /*Timer 4 */ -#define IRQ_TMR5 31 /*Timer 5 */ -#define IRQ_TMR6 32 /*Timer 6 */ -#define IRQ_TMR7 33 /*Timer 7 */ +#define IRQ_TIMER0 26 /*Timer 0 */ +#define IRQ_TIMER1 27 /*Timer 1 */ +#define IRQ_TIMER2 28 /*Timer 2 */ +#define IRQ_TIMER3 29 /*Timer 3 */ +#define IRQ_TIMER4 30 /*Timer 4 */ +#define IRQ_TIMER5 31 /*Timer 5 */ +#define IRQ_TIMER6 32 /*Timer 6 */ +#define IRQ_TIMER7 33 /*Timer 7 */ #define IRQ_PROG_INTA 34 /* PF Ports F&G (PF15:0) Interrupt A */ #define IRQ_PORTG_INTB 35 /* PF Port G (PF15:0) Interrupt B */ #define IRQ_MEM_DMA0 36 /*(Memory DMA Stream 0) */ @@ -195,16 +195,16 @@ #define IRQ_CAN_TX_POS 0 #define IRQ_MAC_RX_POS 4 #define IRQ_MAC_TX_POS 8 -#define IRQ_TMR0_POS 12 -#define IRQ_TMR1_POS 16 -#define IRQ_TMR2_POS 20 -#define IRQ_TMR3_POS 24 -#define IRQ_TMR4_POS 28 +#define IRQ_TIMER0_POS 12 +#define IRQ_TIMER1_POS 16 +#define IRQ_TIMER2_POS 20 +#define IRQ_TIMER3_POS 24 +#define IRQ_TIMER4_POS 28 /* IAR3 BIT FIELDS*/ -#define IRQ_TMR5_POS 0 -#define IRQ_TMR6_POS 4 -#define IRQ_TMR7_POS 8 +#define IRQ_TIMER5_POS 0 +#define IRQ_TIMER6_POS 4 +#define IRQ_TIMER7_POS 8 #define IRQ_PROG_INTA_POS 12 #define IRQ_PORTG_INTB_POS 16 #define IRQ_MEM_DMA0_POS 20 diff --git a/arch/blackfin/mach-bf537/ints-priority.c b/arch/blackfin/mach-bf537/ints-priority.c index b1300b3f1812..51c48087e03b 100644 --- a/arch/blackfin/mach-bf537/ints-priority.c +++ b/arch/blackfin/mach-bf537/ints-priority.c @@ -55,15 +55,15 @@ void __init program_IAR(void) bfin_write_SIC_IAR2(((CONFIG_IRQ_CAN_TX - 7) << IRQ_CAN_TX_POS) | ((CONFIG_IRQ_MAC_RX - 7) << IRQ_MAC_RX_POS) | ((CONFIG_IRQ_MAC_TX - 7) << IRQ_MAC_TX_POS) | - ((CONFIG_IRQ_TMR0 - 7) << IRQ_TMR0_POS) | - ((CONFIG_IRQ_TMR1 - 7) << IRQ_TMR1_POS) | - ((CONFIG_IRQ_TMR2 - 7) << IRQ_TMR2_POS) | - ((CONFIG_IRQ_TMR3 - 7) << IRQ_TMR3_POS) | - ((CONFIG_IRQ_TMR4 - 7) << IRQ_TMR4_POS)); + ((CONFIG_IRQ_TIMER0 - 7) << IRQ_TIMER0_POS) | + ((CONFIG_IRQ_TIMER1 - 7) << IRQ_TIMER1_POS) | + ((CONFIG_IRQ_TIMER2 - 7) << IRQ_TIMER2_POS) | + ((CONFIG_IRQ_TIMER3 - 7) << IRQ_TIMER3_POS) | + ((CONFIG_IRQ_TIMER4 - 7) << IRQ_TIMER4_POS)); - bfin_write_SIC_IAR3(((CONFIG_IRQ_TMR5 - 7) << IRQ_TMR5_POS) | - ((CONFIG_IRQ_TMR6 - 7) << IRQ_TMR6_POS) | - ((CONFIG_IRQ_TMR7 - 7) << IRQ_TMR7_POS) | + bfin_write_SIC_IAR3(((CONFIG_IRQ_TIMER5 - 7) << IRQ_TIMER5_POS) | + ((CONFIG_IRQ_TIMER6 - 7) << IRQ_TIMER6_POS) | + ((CONFIG_IRQ_TIMER7 - 7) << IRQ_TIMER7_POS) | ((CONFIG_IRQ_PROG_INTA - 7) << IRQ_PROG_INTA_POS) | ((CONFIG_IRQ_PORTG_INTB - 7) << IRQ_PORTG_INTB_POS) | ((CONFIG_IRQ_MEM_DMA0 - 7) << IRQ_MEM_DMA0_POS) | diff --git a/arch/blackfin/mach-bf538/Kconfig b/arch/blackfin/mach-bf538/Kconfig index a6f3307758c0..f068c3523cdc 100644 --- a/arch/blackfin/mach-bf538/Kconfig +++ b/arch/blackfin/mach-bf538/Kconfig @@ -55,14 +55,14 @@ config IRQ_UART0_RX config IRQ_UART0_TX int "IRQ_UART0_TX" default 10 -config IRQ_TMR0 - int "IRQ_TMR0" - default 11 -config IRQ_TMR1 - int "IRQ_TMR1" +config IRQ_TIMER0 + int "IRQ_TIMER0" + default 8 +config IRQ_TIMER1 + int "IRQ_TIMER1" default 11 -config IRQ_TMR2 - int "IRQ_TMR2" +config IRQ_TIMER2 + int "IRQ_TIMER2" default 11 config IRQ_PORTF_INTA int "IRQ_PORTF_INTA" diff --git a/arch/blackfin/mach-bf538/include/mach/cdefBF538.h b/arch/blackfin/mach-bf538/include/mach/cdefBF538.h index 7e469b8d939c..241725bc6988 100644 --- a/arch/blackfin/mach-bf538/include/mach/cdefBF538.h +++ b/arch/blackfin/mach-bf538/include/mach/cdefBF538.h @@ -2063,7 +2063,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -2077,7 +2077,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -2088,7 +2088,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -2102,7 +2102,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif diff --git a/arch/blackfin/mach-bf538/include/mach/irq.h b/arch/blackfin/mach-bf538/include/mach/irq.h index 60bdac4cb6a4..fdc87fe2c174 100644 --- a/arch/blackfin/mach-bf538/include/mach/irq.h +++ b/arch/blackfin/mach-bf538/include/mach/irq.h @@ -81,9 +81,9 @@ #define IRQ_SPI0 BFIN_IRQ(13) /* DMA 5 Channel (SPI0) */ #define IRQ_UART0_RX BFIN_IRQ(14) /* DMA 6 Channel (UART0 RX) */ #define IRQ_UART0_TX BFIN_IRQ(15) /* DMA 7 Channel (UART0 TX) */ -#define IRQ_TMR0 BFIN_IRQ(16) /* Timer 0 */ -#define IRQ_TMR1 BFIN_IRQ(17) /* Timer 1 */ -#define IRQ_TMR2 BFIN_IRQ(18) /* Timer 2 */ +#define IRQ_TIMER0 BFIN_IRQ(16) /* Timer 0 */ +#define IRQ_TIMER1 BFIN_IRQ(17) /* Timer 1 */ +#define IRQ_TIMER2 BFIN_IRQ(18) /* Timer 2 */ #define IRQ_PORTF_INTA BFIN_IRQ(19) /* Port F Interrupt A */ #define IRQ_PORTF_INTB BFIN_IRQ(20) /* Port F Interrupt B */ #define IRQ_MEM0_DMA0 BFIN_IRQ(21) /* MDMA0 Stream 0 */ @@ -168,9 +168,9 @@ #define IRQ_UART0_TX_POS 28 /* IAR2 BIT FIELDS */ -#define IRQ_TMR0_POS 0 -#define IRQ_TMR1_POS 4 -#define IRQ_TMR2_POS 8 +#define IRQ_TIMER0_POS 0 +#define IRQ_TIMER1_POS 4 +#define IRQ_TIMER2_POS 8 #define IRQ_PORTF_INTA_POS 12 #define IRQ_PORTF_INTB_POS 16 #define IRQ_MEM0_DMA0_POS 20 diff --git a/arch/blackfin/mach-bf538/ints-priority.c b/arch/blackfin/mach-bf538/ints-priority.c index f81f2ac91840..70d17e550e05 100644 --- a/arch/blackfin/mach-bf538/ints-priority.c +++ b/arch/blackfin/mach-bf538/ints-priority.c @@ -53,9 +53,9 @@ void __init program_IAR(void) ((CONFIG_IRQ_UART0_RX - 7) << IRQ_UART0_RX_POS) | ((CONFIG_IRQ_UART0_TX - 7) << IRQ_UART0_TX_POS)); - bfin_write_SIC_IAR2(((CONFIG_IRQ_TMR0 - 7) << IRQ_TMR0_POS) | - ((CONFIG_IRQ_TMR1 - 7) << IRQ_TMR1_POS) | - ((CONFIG_IRQ_TMR2 - 7) << IRQ_TMR2_POS) | + bfin_write_SIC_IAR2(((CONFIG_IRQ_TIMER0 - 7) << IRQ_TIMER0_POS) | + ((CONFIG_IRQ_TIMER1 - 7) << IRQ_TIMER1_POS) | + ((CONFIG_IRQ_TIMER2 - 7) << IRQ_TIMER2_POS) | ((CONFIG_IRQ_PORTF_INTA - 7) << IRQ_PORTF_INTA_POS) | ((CONFIG_IRQ_PORTF_INTB - 7) << IRQ_PORTF_INTB_POS) | ((CONFIG_IRQ_MEM0_DMA0 - 7) << IRQ_MEM0_DMA0_POS) | diff --git a/arch/blackfin/mach-bf548/Kconfig b/arch/blackfin/mach-bf548/Kconfig index 1bfcd8f646ab..dcf657159051 100644 --- a/arch/blackfin/mach-bf548/Kconfig +++ b/arch/blackfin/mach-bf548/Kconfig @@ -250,7 +250,7 @@ config IRQ_OTPSEC default 11 config IRQ_TIMER0 int "IRQ_TIMER0" - default 11 + default 8 config IRQ_TIMER1 int "IRQ_TIMER1" default 11 diff --git a/arch/blackfin/mach-bf548/include/mach/cdefBF54x_base.h b/arch/blackfin/mach-bf548/include/mach/cdefBF54x_base.h index 0a3b210daadf..6e636c418cb0 100644 --- a/arch/blackfin/mach-bf548/include/mach/cdefBF54x_base.h +++ b/arch/blackfin/mach-bf548/include/mach/cdefBF54x_base.h @@ -2702,7 +2702,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -2719,7 +2719,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); bfin_write32(SIC_IWR2, iwr2); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -2730,7 +2730,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SIC_IWR0); iwr1 = bfin_read32(SIC_IWR1); @@ -2747,7 +2747,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) bfin_write32(SIC_IWR0, iwr0); bfin_write32(SIC_IWR1, iwr1); bfin_write32(SIC_IWR2, iwr2); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* _CDEF_BF54X_H */ diff --git a/arch/blackfin/mach-bf548/include/mach/irq.h b/arch/blackfin/mach-bf548/include/mach/irq.h index ad380d1f5872..60299a71e090 100644 --- a/arch/blackfin/mach-bf548/include/mach/irq.h +++ b/arch/blackfin/mach-bf548/include/mach/irq.h @@ -158,7 +158,7 @@ Events (highest priority) EMU 0 #define IRQ_PINT2 BFIN_IRQ(94) /* PINT2 Interrupt */ #define IRQ_PINT3 BFIN_IRQ(95) /* PINT3 Interrupt */ -#define SYS_IRQS IRQ_PINT3 +#define SYS_IRQS IRQ_PINT3 #define BFIN_PA_IRQ(x) ((x) + SYS_IRQS + 1) #define IRQ_PA0 BFIN_PA_IRQ(0) diff --git a/arch/blackfin/mach-bf561/Kconfig b/arch/blackfin/mach-bf561/Kconfig index 5d56438cad2f..638ec38ca470 100644 --- a/arch/blackfin/mach-bf561/Kconfig +++ b/arch/blackfin/mach-bf561/Kconfig @@ -138,7 +138,7 @@ config IRQ_DMA2_11 default 9 config IRQ_TIMER0 int "TIMER 0 Interrupt" - default 10 + default 8 config IRQ_TIMER1 int "TIMER 1 Interrupt" default 10 diff --git a/arch/blackfin/mach-bf561/include/mach/cdefBF561.h b/arch/blackfin/mach-bf561/include/mach/cdefBF561.h index b16875d735b3..95d609f11c97 100644 --- a/arch/blackfin/mach-bf561/include/mach/cdefBF561.h +++ b/arch/blackfin/mach-bf561/include/mach/cdefBF561.h @@ -1537,7 +1537,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) if (val == bfin_read_PLL_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SICA_IWR0); iwr1 = bfin_read32(SICA_IWR1); @@ -1551,7 +1551,7 @@ static __inline__ void bfin_write_PLL_CTL(unsigned int val) bfin_write32(SICA_IWR0, iwr0); bfin_write32(SICA_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } /* Writing to VR_CTL initiates a PLL relock sequence. */ @@ -1562,7 +1562,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) if (val == bfin_read_VR_CTL()) return; - local_irq_save(flags); + local_irq_save_hw(flags); /* Enable the PLL Wakeup bit in SIC IWR */ iwr0 = bfin_read32(SICA_IWR0); iwr1 = bfin_read32(SICA_IWR1); @@ -1576,7 +1576,7 @@ static __inline__ void bfin_write_VR_CTL(unsigned int val) bfin_write32(SICA_IWR0, iwr0); bfin_write32(SICA_IWR1, iwr1); - local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* _CDEF_BF561_H */ diff --git a/arch/blackfin/mach-common/cpufreq.c b/arch/blackfin/mach-common/cpufreq.c index 56f68d879b33..72e16605ca09 100644 --- a/arch/blackfin/mach-common/cpufreq.c +++ b/arch/blackfin/mach-common/cpufreq.c @@ -104,7 +104,7 @@ static int bfin_target(struct cpufreq_policy *policy, cclk_hz, target_freq, freqs.old); cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); - local_irq_save(flags); + local_irq_save_hw(flags); plldiv = (bfin_read_PLL_DIV() & SSEL) | dpm_state_table[index].csel; tscale = dpm_state_table[index].tscale; bfin_write_PLL_DIV(plldiv); @@ -115,7 +115,7 @@ static int bfin_target(struct cpufreq_policy *policy, cycles += 10; /* ~10 cycles we lose after get_cycles() */ __bfin_cycles_off += (cycles << __bfin_cycles_mod) - (cycles << index); __bfin_cycles_mod = index; - local_irq_restore(flags); + local_irq_restore_hw(flags); /* TODO: just test case for cycles clock source, remove later */ pr_debug("cpufreq: done\n"); cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); diff --git a/arch/blackfin/mach-common/interrupt.S b/arch/blackfin/mach-common/interrupt.S index 2604b532897c..c1bdd1edc8eb 100644 --- a/arch/blackfin/mach-common/interrupt.S +++ b/arch/blackfin/mach-common/interrupt.S @@ -129,8 +129,15 @@ __common_int_entry: #endif r1 = sp; SP += -12; +#ifdef CONFIG_IPIPE + call ___ipipe_grab_irq + SP += 12; + cc = r0 == 0; + if cc jump .Lcommon_restore_context; +#else /* CONFIG_IPIPE */ call _do_irq; SP += 12; +#endif /* CONFIG_IPIPE */ call _return_from_int; .Lcommon_restore_context: RESTORE_CONTEXT @@ -247,3 +254,56 @@ ENTRY(_evt_system_call) call _system_call; jump .Lcommon_restore_context; ENDPROC(_evt_system_call) + +#ifdef CONFIG_IPIPE +ENTRY(___ipipe_call_irqtail) + r0.l = 1f; + r0.h = 1f; + reti = r0; + rti; +1: + [--sp] = rets; + [--sp] = ( r7:4, p5:3 ); + p0.l = ___ipipe_irq_tail_hook; + p0.h = ___ipipe_irq_tail_hook; + p0 = [p0]; + sp += -12; + call (p0); + sp += 12; + ( r7:4, p5:3 ) = [sp++]; + rets = [sp++]; + + [--sp] = reti; + reti = [sp++]; /* IRQs are off. */ + r0.h = 3f; + r0.l = 3f; + p0.l = lo(EVT14); + p0.h = hi(EVT14); + [p0] = r0; + csync; + r0 = 0x401f; + sti r0; + raise 14; + [--sp] = reti; /* IRQs on. */ +2: + jump 2b; /* Likely paranoid. */ +3: + sp += 4; /* Discard saved RETI */ + r0.h = _evt14_softirq; + r0.l = _evt14_softirq; + p0.l = lo(EVT14); + p0.h = hi(EVT14); + [p0] = r0; + csync; + p0.l = _bfin_irq_flags; + p0.h = _bfin_irq_flags; + r0 = [p0]; + sti r0; +#if 0 /* FIXME: this actually raises scheduling latencies */ + /* Reenable interrupts */ + [--sp] = reti; + r0 = [sp++]; +#endif + rts; +ENDPROC(___ipipe_call_irqtail) +#endif /* CONFIG_IPIPE */ diff --git a/arch/blackfin/mach-common/ints-priority.c b/arch/blackfin/mach-common/ints-priority.c index 7c1db775751b..1bba6030dce9 100644 --- a/arch/blackfin/mach-common/ints-priority.c +++ b/arch/blackfin/mach-common/ints-priority.c @@ -34,6 +34,9 @@ #include #include #include +#ifdef CONFIG_IPIPE +#include +#endif #ifdef CONFIG_KGDB #include #endif @@ -135,8 +138,8 @@ static void bfin_ack_noop(unsigned int irq) static void bfin_core_mask_irq(unsigned int irq) { bfin_irq_flags &= ~(1 << irq); - if (!irqs_disabled()) - local_irq_enable(); + if (!irqs_disabled_hw()) + local_irq_enable_hw(); } static void bfin_core_unmask_irq(unsigned int irq) @@ -151,8 +154,8 @@ static void bfin_core_unmask_irq(unsigned int irq) * local_irq_enable just does "STI bfin_irq_flags", so it's exactly * what we need. */ - if (!irqs_disabled()) - local_irq_enable(); + if (!irqs_disabled_hw()) + local_irq_enable_hw(); return; } @@ -235,7 +238,7 @@ int bfin_internal_set_wake(unsigned int irq, unsigned int state) break; } - local_irq_save(flags); + local_irq_save_hw(flags); if (state) { bfin_sic_iwr[bank] |= (1 << bit); @@ -246,7 +249,7 @@ int bfin_internal_set_wake(unsigned int irq, unsigned int state) vr_wakeup &= ~wakeup; } - local_irq_restore(flags); + local_irq_restore_hw(flags); return 0; } @@ -272,6 +275,19 @@ static struct irq_chip bfin_internal_irqchip = { #endif }; +static void bfin_handle_irq(unsigned irq) +{ +#ifdef CONFIG_IPIPE + struct pt_regs regs; /* Contents not used. */ + ipipe_trace_irq_entry(irq); + __ipipe_handle_irq(irq, ®s); + ipipe_trace_irq_exit(irq); +#else /* !CONFIG_IPIPE */ + struct irq_desc *desc = irq_desc + irq; + desc->handle_irq(irq, desc); +#endif /* !CONFIG_IPIPE */ +} + #ifdef BF537_GENERIC_ERROR_INT_DEMUX static int error_int_mask; @@ -325,10 +341,9 @@ static void bfin_demux_error_irq(unsigned int int_err_irq, irq = IRQ_UART1_ERROR; if (irq) { - if (error_int_mask & (1L << (irq - IRQ_PPI_ERROR))) { - struct irq_desc *desc = irq_desc + irq; - desc->handle_irq(irq, desc); - } else { + if (error_int_mask & (1L << (irq - IRQ_PPI_ERROR))) + bfin_handle_irq(irq); + else { switch (irq) { case IRQ_PPI_ERROR: @@ -374,10 +389,14 @@ static void bfin_demux_error_irq(unsigned int int_err_irq, static inline void bfin_set_irq_handler(unsigned irq, irq_flow_handler_t handle) { +#ifdef CONFIG_IPIPE + _set_irq_handler(irq, handle_edge_irq); +#else struct irq_desc *desc = irq_desc + irq; /* May not call generic set_irq_handler() due to spinlock recursion. */ desc->handle_irq = handle; +#endif } static DECLARE_BITMAP(gpio_enabled, MAX_BLACKFIN_GPIOS); @@ -563,10 +582,8 @@ static void bfin_demux_gpio_irq(unsigned int inta_irq, mask = get_gpiop_data(i) & get_gpiop_maska(i); while (mask) { - if (mask & 1) { - desc = irq_desc + irq; - desc->handle_irq(irq, desc); - } + if (mask & 1) + bfin_handle_irq(irq); irq++; mask >>= 1; } @@ -576,10 +593,8 @@ static void bfin_demux_gpio_irq(unsigned int inta_irq, mask = get_gpiop_data(gpio) & get_gpiop_maska(gpio); do { - if (mask & 1) { - desc = irq_desc + irq; - desc->handle_irq(irq, desc); - } + if (mask & 1) + bfin_handle_irq(irq); irq++; mask >>= 1; } while (mask); @@ -900,8 +915,7 @@ static void bfin_demux_gpio_irq(unsigned int inta_irq, while (request) { if (request & 1) { irq = pint2irq_lut[pint_val] + SYS_IRQS; - desc = irq_desc + irq; - desc->handle_irq(irq, desc); + bfin_handle_irq(irq); } pint_val++; request >>= 1; @@ -1025,11 +1039,10 @@ int __init init_arch_irq(void) break; #ifdef BF537_GENERIC_ERROR_INT_DEMUX case IRQ_GENERIC_ERROR: - set_irq_handler(irq, bfin_demux_error_irq); - + set_irq_chained_handler(irq, bfin_demux_error_irq); break; #endif -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) || defined(CONFIG_IPIPE) case IRQ_TIMER0: set_irq_handler(irq, handle_percpu_irq); break; @@ -1041,7 +1054,17 @@ int __init init_arch_irq(void) break; #endif default: +#ifdef CONFIG_IPIPE + /* + * We want internal interrupt sources to be masked, because + * ISRs may trigger interrupts recursively (e.g. DMA), but + * interrupts are _not_ masked at CPU level. So let's handle + * them as level interrupts. + */ + set_irq_handler(irq, handle_level_irq); +#else /* !CONFIG_IPIPE */ set_irq_handler(irq, handle_simple_irq); +#endif /* !CONFIG_IPIPE */ break; } } @@ -1101,6 +1124,14 @@ int __init init_arch_irq(void) bfin_write_SIC_IWR(IWR_DISABLE_ALL); #endif +#ifdef CONFIG_IPIPE + for (irq = 0; irq < NR_IRQS; irq++) { + struct irq_desc *desc = irq_desc + irq; + desc->ic_prio = __ipipe_get_irq_priority(irq); + desc->thr_prio = __ipipe_get_irqthread_priority(irq); + } +#endif /* CONFIG_IPIPE */ + return 0; } @@ -1156,3 +1187,161 @@ void do_irq(int vec, struct pt_regs *fp) } asm_do_IRQ(vec, fp); } + +#ifdef CONFIG_IPIPE + +int __ipipe_get_irq_priority(unsigned irq) +{ + int ient, prio; + + if (irq <= IRQ_CORETMR) + return irq; + + for (ient = 0; ient < NR_PERI_INTS; ient++) { + struct ivgx *ivg = ivg_table + ient; + if (ivg->irqno == irq) { + for (prio = 0; prio <= IVG13-IVG7; prio++) { + if (ivg7_13[prio].ifirst <= ivg && + ivg7_13[prio].istop > ivg) + return IVG7 + prio; + } + } + } + + return IVG15; +} + +int __ipipe_get_irqthread_priority(unsigned irq) +{ + int ient, prio; + int demux_irq; + + /* The returned priority value is rescaled to [0..IVG13+1] + * with 0 being the lowest effective priority level. */ + + if (irq <= IRQ_CORETMR) + return IVG13 - irq + 1; + + /* GPIO IRQs are given the priority of the demux + * interrupt. */ + if (IS_GPIOIRQ(irq)) { +#if defined(CONFIG_BF54x) + u32 bank = PINT_2_BANK(irq2pint_lut[irq - SYS_IRQS]); + demux_irq = (bank == 0 ? IRQ_PINT0 : + bank == 1 ? IRQ_PINT1 : + bank == 2 ? IRQ_PINT2 : + IRQ_PINT3); +#elif defined(CONFIG_BF561) + demux_irq = (irq >= IRQ_PF32 ? IRQ_PROG2_INTA : + irq >= IRQ_PF16 ? IRQ_PROG1_INTA : + IRQ_PROG0_INTA); +#elif defined(CONFIG_BF52x) + demux_irq = (irq >= IRQ_PH0 ? IRQ_PORTH_INTA : + irq >= IRQ_PG0 ? IRQ_PORTG_INTA : + IRQ_PORTF_INTA); +#else + demux_irq = irq; +#endif + return IVG13 - PRIO_GPIODEMUX(demux_irq) + 1; + } + + /* The GPIO demux interrupt is given a lower priority + * than the GPIO IRQs, so that its threaded handler + * unmasks the interrupt line after the decoded IRQs + * have been processed. */ + prio = PRIO_GPIODEMUX(irq); + /* demux irq? */ + if (prio != -1) + return IVG13 - prio; + + for (ient = 0; ient < NR_PERI_INTS; ient++) { + struct ivgx *ivg = ivg_table + ient; + if (ivg->irqno == irq) { + for (prio = 0; prio <= IVG13-IVG7; prio++) { + if (ivg7_13[prio].ifirst <= ivg && + ivg7_13[prio].istop > ivg) + return IVG7 - prio; + } + } + } + + return 0; +} + +/* Hw interrupts are disabled on entry (check SAVE_CONTEXT). */ +#ifdef CONFIG_DO_IRQ_L1 +__attribute__((l1_text)) +#endif +asmlinkage int __ipipe_grab_irq(int vec, struct pt_regs *regs) +{ + struct ivgx *ivg_stop = ivg7_13[vec-IVG7].istop; + struct ivgx *ivg = ivg7_13[vec-IVG7].ifirst; + int irq; + + if (likely(vec == EVT_IVTMR_P)) { + irq = IRQ_CORETMR; + goto handle_irq; + } + + SSYNC(); + +#if defined(CONFIG_BF54x) || defined(CONFIG_BF52x) || defined(CONFIG_BF561) + { + unsigned long sic_status[3]; + + sic_status[0] = bfin_read_SIC_ISR0() & bfin_read_SIC_IMASK0(); + sic_status[1] = bfin_read_SIC_ISR1() & bfin_read_SIC_IMASK1(); +#ifdef CONFIG_BF54x + sic_status[2] = bfin_read_SIC_ISR2() & bfin_read_SIC_IMASK2(); +#endif + for (;; ivg++) { + if (ivg >= ivg_stop) { + atomic_inc(&num_spurious); + return 0; + } + if (sic_status[(ivg->irqno - IVG7) / 32] & ivg->isrflag) + break; + } + } +#else + { + unsigned long sic_status; + + sic_status = bfin_read_SIC_IMASK() & bfin_read_SIC_ISR(); + + for (;; ivg++) { + if (ivg >= ivg_stop) { + atomic_inc(&num_spurious); + return 0; + } else if (sic_status & ivg->isrflag) + break; + } + } +#endif + + irq = ivg->irqno; + + if (irq == IRQ_SYSTMR) { + bfin_write_TIMER_STATUS(1); /* Latch TIMIL0 */ + /* This is basically what we need from the register frame. */ + __raw_get_cpu_var(__ipipe_tick_regs).ipend = regs->ipend; + __raw_get_cpu_var(__ipipe_tick_regs).pc = regs->pc; + if (!ipipe_root_domain_p) + __raw_get_cpu_var(__ipipe_tick_regs).ipend |= 0x10; + else + __raw_get_cpu_var(__ipipe_tick_regs).ipend &= ~0x10; + } + +handle_irq: + + ipipe_trace_irq_entry(irq); + __ipipe_handle_irq(irq, regs); + ipipe_trace_irq_exit(irq); + + if (ipipe_root_domain_p) + return !test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)); + + return 0; +} + +#endif /* CONFIG_IPIPE */ diff --git a/arch/blackfin/mach-common/pm.c b/arch/blackfin/mach-common/pm.c index 96600b8cb6ad..d3d70fd67c16 100644 --- a/arch/blackfin/mach-common/pm.c +++ b/arch/blackfin/mach-common/pm.c @@ -71,7 +71,7 @@ void bfin_pm_suspend_standby_enter(void) gpio_pm_wakeup_request(CONFIG_PM_WAKEUP_GPIO_NUMBER, WAKEUP_TYPE); #endif - local_irq_save(flags); + local_irq_save_hw(flags); bfin_pm_standby_setup(); #ifdef CONFIG_PM_BFIN_SLEEP_DEEPER @@ -105,7 +105,7 @@ void bfin_pm_suspend_standby_enter(void) bfin_write_SIC_IWR(IWR_DISABLE_ALL); #endif - local_irq_restore(flags); + local_irq_restore_hw(flags); } int bf53x_suspend_l1_mem(unsigned char *memptr) @@ -249,12 +249,12 @@ int bfin_pm_suspend_mem_enter(void) wakeup |= GPWE; #endif - local_irq_save(flags); + local_irq_save_hw(flags); ret = blackfin_dma_suspend(); if (ret) { - local_irq_restore(flags); + local_irq_restore_hw(flags); kfree(memptr); return ret; } @@ -275,7 +275,7 @@ int bfin_pm_suspend_mem_enter(void) bfin_gpio_pm_hibernate_restore(); blackfin_dma_resume(); - local_irq_restore(flags); + local_irq_restore_hw(flags); kfree(memptr); return 0; diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c index 66c47a702da4..77c992847094 100644 --- a/arch/blackfin/mach-common/smp.c +++ b/arch/blackfin/mach-common/smp.c @@ -348,7 +348,7 @@ int __cpuinit __cpu_up(unsigned int cpu) static void __cpuinit setup_secondary(unsigned int cpu) { -#ifndef CONFIG_TICK_SOURCE_SYSTMR0 +#if !(defined(CONFIG_TICK_SOURCE_SYSTMR0) || defined(CONFIG_IPIPE)) struct irq_desc *timer_desc; #endif unsigned long ilat; @@ -369,7 +369,7 @@ static void __cpuinit setup_secondary(unsigned int cpu) IMASK_IVG14 | IMASK_IVG13 | IMASK_IVG12 | IMASK_IVG11 | IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW; -#ifdef CONFIG_TICK_SOURCE_SYSTMR0 +#if defined(CONFIG_TICK_SOURCE_SYSTMR0) || defined(CONFIG_IPIPE) /* Power down the core timer, just to play safe. */ bfin_write_TCNTL(0); -- cgit v1.2.3-59-g8ed1b From b339dc79b49841eff0aeecfc444cbb7b26007649 Mon Sep 17 00:00:00 2001 From: Jie Zhang Date: Wed, 7 Jan 2009 23:14:39 +0800 Subject: Blackfin arch: Print FP at level KERN_NOTICE Signed-off-by: Jie Zhang Signed-off-by: Bryan Wu --- arch/blackfin/kernel/traps.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/blackfin/kernel/traps.c') diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c index 956aefb84687..17d8e4172896 100644 --- a/arch/blackfin/kernel/traps.c +++ b/arch/blackfin/kernel/traps.c @@ -844,7 +844,7 @@ void show_stack(struct task_struct *task, unsigned long *stack) } if (fp) { frame = fp; - printk(" FP: (0x%p)\n", fp); + printk(KERN_NOTICE " FP: (0x%p)\n", fp); } else frame = 0; -- cgit v1.2.3-59-g8ed1b From 8feae13110d60cc6287afabc2887366b0eb226c2 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 8 Jan 2009 12:04:47 +0000 Subject: NOMMU: Make VMAs per MM as for MMU-mode linux Make VMAs per mm_struct as for MMU-mode linux. This solves two problems: (1) In SYSV SHM where nattch for a segment does not reflect the number of shmat's (and forks) done. (2) In mmap() where the VMA's vm_mm is set to point to the parent mm by an exec'ing process when VM_EXECUTABLE is specified, regardless of the fact that a VMA might be shared and already have its vm_mm assigned to another process or a dead process. A new struct (vm_region) is introduced to track a mapped region and to remember the circumstances under which it may be shared and the vm_list_struct structure is discarded as it's no longer required. This patch makes the following additional changes: (1) Regions are now allocated with alloc_pages() rather than kmalloc() and with no recourse to __GFP_COMP, so the pages are not composite. Instead, each page has a reference on it held by the region. Anything else that is interested in such a page will have to get a reference on it to retain it. When the pages are released due to unmapping, each page is passed to put_page() and will be freed when the page usage count reaches zero. (2) Excess pages are trimmed after an allocation as the allocation must be made as a power-of-2 quantity of pages. (3) VMAs are added to the parent MM's R/B tree and mmap lists. As an MM may end up with overlapping VMAs within the tree, the VMA struct address is appended to the sort key. (4) Non-anonymous VMAs are now added to the backing inode's prio list. (5) Holes may be punched in anonymous VMAs with munmap(), releasing parts of the backing region. The VMA and region structs will be split if necessary. (6) sys_shmdt() only releases one attachment to a SYSV IPC shared memory segment instead of all the attachments at that addresss. Multiple shmat()'s return the same address under NOMMU-mode instead of different virtual addresses as under MMU-mode. (7) Core dumping for ELF-FDPIC requires fewer exceptions for NOMMU-mode. (8) /proc/maps is now the global list of mapped regions, and may list bits that aren't actually mapped anywhere. (9) /proc/meminfo gains a line (tagged "MmapCopy") that indicates the amount of RAM currently allocated by mmap to hold mappable regions that can't be mapped directly. These are copies of the backing device or file if not anonymous. These changes make NOMMU mode more similar to MMU mode. The downside is that NOMMU mode requires some extra memory to track things over NOMMU without this patch (VMAs are no longer shared, and there are now region structs). Signed-off-by: David Howells Tested-by: Mike Frysinger Acked-by: Paul Mundt --- Documentation/nommu-mmap.txt | 18 +- arch/arm/include/asm/mmu.h | 1 - arch/blackfin/include/asm/mmu.h | 1 - arch/blackfin/kernel/ptrace.c | 6 +- arch/blackfin/kernel/traps.c | 11 +- arch/frv/kernel/ptrace.c | 11 +- arch/h8300/include/asm/mmu.h | 1 - arch/m68knommu/include/asm/mmu.h | 1 - arch/sh/include/asm/mmu.h | 1 - fs/binfmt_elf_fdpic.c | 27 +- fs/proc/internal.h | 2 - fs/proc/meminfo.c | 6 + fs/proc/nommu.c | 71 ++- fs/proc/task_nommu.c | 108 +++-- include/asm-frv/mmu.h | 1 - include/asm-m32r/mmu.h | 1 - include/linux/mm.h | 18 +- include/linux/mm_types.h | 18 +- ipc/shm.c | 12 + kernel/fork.c | 4 +- lib/Kconfig.debug | 7 + mm/mmap.c | 10 + mm/nommu.c | 960 +++++++++++++++++++++++++++------------ 23 files changed, 860 insertions(+), 436 deletions(-) (limited to 'arch/blackfin/kernel/traps.c') diff --git a/Documentation/nommu-mmap.txt b/Documentation/nommu-mmap.txt index 7714f57caad5..02b89dcf38ac 100644 --- a/Documentation/nommu-mmap.txt +++ b/Documentation/nommu-mmap.txt @@ -109,12 +109,18 @@ and it's also much more restricted in the latter case: FURTHER NOTES ON NO-MMU MMAP ============================ - (*) A request for a private mapping of less than a page in size may not return - a page-aligned buffer. This is because the kernel calls kmalloc() to - allocate the buffer, not get_free_page(). - - (*) A list of all the mappings on the system is visible through /proc/maps in - no-MMU mode. + (*) A request for a private mapping of a file may return a buffer that is not + page-aligned. This is because XIP may take place, and the data may not be + paged aligned in the backing store. + + (*) A request for an anonymous mapping will always be page aligned. If + possible the size of the request should be a power of two otherwise some + of the space may be wasted as the kernel must allocate a power-of-2 + granule but will only discard the excess if appropriately configured as + this has an effect on fragmentation. + + (*) A list of all the private copy and anonymous mappings on the system is + visible through /proc/maps in no-MMU mode. (*) A list of all the mappings in use by a process is visible through /proc//maps in no-MMU mode. diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index 53099d4ee421..b561584d04a1 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -24,7 +24,6 @@ typedef struct { * modified for 2.6 by Hyok S. Choi */ typedef struct { - struct vm_list_struct *vmlist; unsigned long end_brk; } mm_context_t; diff --git a/arch/blackfin/include/asm/mmu.h b/arch/blackfin/include/asm/mmu.h index 757e43906ed4..dbfd686360e6 100644 --- a/arch/blackfin/include/asm/mmu.h +++ b/arch/blackfin/include/asm/mmu.h @@ -10,7 +10,6 @@ struct sram_list_struct { }; typedef struct { - struct vm_list_struct *vmlist; unsigned long end_brk; unsigned long stack_start; diff --git a/arch/blackfin/kernel/ptrace.c b/arch/blackfin/kernel/ptrace.c index d2d388536630..594e325b40e4 100644 --- a/arch/blackfin/kernel/ptrace.c +++ b/arch/blackfin/kernel/ptrace.c @@ -160,15 +160,15 @@ put_reg(struct task_struct *task, int regno, unsigned long data) static inline int is_user_addr_valid(struct task_struct *child, unsigned long start, unsigned long len) { - struct vm_list_struct *vml; + struct vm_area_struct *vma; struct sram_list_struct *sraml; /* overflow */ if (start + len < start) return -EIO; - for (vml = child->mm->context.vmlist; vml; vml = vml->next) - if (start >= vml->vma->vm_start && start + len < vml->vma->vm_end) + vma = find_vma(child->mm, start); + if (vma && start >= vma->vm_start && start + len <= vma->vm_end) return 0; for (sraml = child->mm->context.sram_list; sraml; sraml = sraml->next) diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c index 17d8e4172896..5b0667da8d05 100644 --- a/arch/blackfin/kernel/traps.c +++ b/arch/blackfin/kernel/traps.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +84,7 @@ static void decode_address(char *buf, unsigned long address) struct mm_struct *mm; unsigned long flags, offset; unsigned char in_atomic = (bfin_read_IPEND() & 0x10) || in_atomic(); + struct rb_node *n; #ifdef CONFIG_KALLSYMS unsigned long symsize; @@ -128,9 +130,10 @@ static void decode_address(char *buf, unsigned long address) if (!mm) continue; - vml = mm->context.vmlist; - while (vml) { - struct vm_area_struct *vma = vml->vma; + for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) { + struct vm_area_struct *vma; + + vma = rb_entry(n, struct vm_area_struct, vm_rb); if (address >= vma->vm_start && address < vma->vm_end) { char _tmpbuf[256]; @@ -176,8 +179,6 @@ static void decode_address(char *buf, unsigned long address) goto done; } - - vml = vml->next; } if (!in_atomic) mmput(mm); diff --git a/arch/frv/kernel/ptrace.c b/arch/frv/kernel/ptrace.c index 709e9bdc6126..5e7d401d21e7 100644 --- a/arch/frv/kernel/ptrace.c +++ b/arch/frv/kernel/ptrace.c @@ -69,7 +69,8 @@ static inline int put_reg(struct task_struct *task, int regno, } /* - * check that an address falls within the bounds of the target process's memory mappings + * check that an address falls within the bounds of the target process's memory + * mappings */ static inline int is_user_addr_valid(struct task_struct *child, unsigned long start, unsigned long len) @@ -79,11 +80,11 @@ static inline int is_user_addr_valid(struct task_struct *child, return -EIO; return 0; #else - struct vm_list_struct *vml; + struct vm_area_struct *vma; - for (vml = child->mm->context.vmlist; vml; vml = vml->next) - if (start >= vml->vma->vm_start && start + len <= vml->vma->vm_end) - return 0; + vma = find_vma(child->mm, start); + if (vma && start >= vma->vm_start && start + len <= vma->vm_end) + return 0; return -EIO; #endif diff --git a/arch/h8300/include/asm/mmu.h b/arch/h8300/include/asm/mmu.h index 2ce06ea46104..31309969df70 100644 --- a/arch/h8300/include/asm/mmu.h +++ b/arch/h8300/include/asm/mmu.h @@ -4,7 +4,6 @@ /* Copyright (C) 2002, David McCullough */ typedef struct { - struct vm_list_struct *vmlist; unsigned long end_brk; } mm_context_t; diff --git a/arch/m68knommu/include/asm/mmu.h b/arch/m68knommu/include/asm/mmu.h index 5fa6b68353ba..e2da1e6f09fe 100644 --- a/arch/m68knommu/include/asm/mmu.h +++ b/arch/m68knommu/include/asm/mmu.h @@ -4,7 +4,6 @@ /* Copyright (C) 2002, David McCullough */ typedef struct { - struct vm_list_struct *vmlist; unsigned long end_brk; } mm_context_t; diff --git a/arch/sh/include/asm/mmu.h b/arch/sh/include/asm/mmu.h index fdcb93bc6d11..6c43625bb1a5 100644 --- a/arch/sh/include/asm/mmu.h +++ b/arch/sh/include/asm/mmu.h @@ -9,7 +9,6 @@ typedef struct { mm_context_id_t id; void *vdso; #else - struct vm_list_struct *vmlist; unsigned long end_brk; #endif #ifdef CONFIG_BINFMT_ELF_FDPIC diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index aa5b43205e37..22baf1b13493 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -1567,11 +1567,9 @@ end_coredump: static int elf_fdpic_dump_segments(struct file *file, size_t *size, unsigned long *limit, unsigned long mm_flags) { - struct vm_list_struct *vml; - - for (vml = current->mm->context.vmlist; vml; vml = vml->next) { - struct vm_area_struct *vma = vml->vma; + struct vm_area_struct *vma; + for (vma = current->mm->mmap; vma; vma = vma->vm_next) { if (!maydump(vma, mm_flags)) continue; @@ -1617,9 +1615,6 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs, elf_fpxregset_t *xfpu = NULL; #endif int thread_status_size = 0; -#ifndef CONFIG_MMU - struct vm_list_struct *vml; -#endif elf_addr_t *auxv; unsigned long mm_flags; @@ -1685,13 +1680,7 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs, fill_prstatus(prstatus, current, signr); elf_core_copy_regs(&prstatus->pr_reg, regs); -#ifdef CONFIG_MMU segs = current->mm->map_count; -#else - segs = 0; - for (vml = current->mm->context.vmlist; vml; vml = vml->next) - segs++; -#endif #ifdef ELF_CORE_EXTRA_PHDRS segs += ELF_CORE_EXTRA_PHDRS; #endif @@ -1766,20 +1755,10 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs, mm_flags = current->mm->flags; /* write program headers for segments dump */ - for ( -#ifdef CONFIG_MMU - vma = current->mm->mmap; vma; vma = vma->vm_next -#else - vml = current->mm->context.vmlist; vml; vml = vml->next -#endif - ) { + for (vma = current->mm->mmap; vma; vma = vma->vm_next) { struct elf_phdr phdr; size_t sz; -#ifndef CONFIG_MMU - vma = vml->vma; -#endif - sz = vma->vm_end - vma->vm_start; phdr.p_type = PT_LOAD; diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 3e8aeb8b61ce..cd53ff838498 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -41,8 +41,6 @@ do { \ (vmi)->used = 0; \ (vmi)->largest_chunk = 0; \ } while(0) - -extern int nommu_vma_show(struct seq_file *, struct vm_area_struct *); #endif extern int proc_tid_stat(struct seq_file *m, struct pid_namespace *ns, diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index b1675c4e66da..43d23948384a 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -73,6 +73,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v) "HighFree: %8lu kB\n" "LowTotal: %8lu kB\n" "LowFree: %8lu kB\n" +#endif +#ifndef CONFIG_MMU + "MmapCopy: %8lu kB\n" #endif "SwapTotal: %8lu kB\n" "SwapFree: %8lu kB\n" @@ -115,6 +118,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v) K(i.freehigh), K(i.totalram-i.totalhigh), K(i.freeram-i.freehigh), +#endif +#ifndef CONFIG_MMU + K((unsigned long) atomic_read(&mmap_pages_allocated)), #endif K(i.totalswap), K(i.freeswap), diff --git a/fs/proc/nommu.c b/fs/proc/nommu.c index 3f87d2632947..b446d7ad0b0d 100644 --- a/fs/proc/nommu.c +++ b/fs/proc/nommu.c @@ -33,33 +33,33 @@ #include "internal.h" /* - * display a single VMA to a sequenced file + * display a single region to a sequenced file */ -int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) +static int nommu_region_show(struct seq_file *m, struct vm_region *region) { unsigned long ino = 0; struct file *file; dev_t dev = 0; int flags, len; - flags = vma->vm_flags; - file = vma->vm_file; + flags = region->vm_flags; + file = region->vm_file; if (file) { - struct inode *inode = vma->vm_file->f_path.dentry->d_inode; + struct inode *inode = region->vm_file->f_path.dentry->d_inode; dev = inode->i_sb->s_dev; ino = inode->i_ino; } seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu %n", - vma->vm_start, - vma->vm_end, + region->vm_start, + region->vm_end, flags & VM_READ ? 'r' : '-', flags & VM_WRITE ? 'w' : '-', flags & VM_EXEC ? 'x' : '-', flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p', - ((loff_t)vma->vm_pgoff) << PAGE_SHIFT, + ((loff_t)region->vm_pgoff) << PAGE_SHIFT, MAJOR(dev), MINOR(dev), ino, &len); if (file) { @@ -75,61 +75,54 @@ int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) } /* - * display a list of all the VMAs the kernel knows about + * display a list of all the REGIONs the kernel knows about * - nommu kernals have a single flat list */ -static int nommu_vma_list_show(struct seq_file *m, void *v) +static int nommu_region_list_show(struct seq_file *m, void *_p) { - struct vm_area_struct *vma; + struct rb_node *p = _p; - vma = rb_entry((struct rb_node *) v, struct vm_area_struct, vm_rb); - return nommu_vma_show(m, vma); + return nommu_region_show(m, rb_entry(p, struct vm_region, vm_rb)); } -static void *nommu_vma_list_start(struct seq_file *m, loff_t *_pos) +static void *nommu_region_list_start(struct seq_file *m, loff_t *_pos) { - struct rb_node *_rb; + struct rb_node *p; loff_t pos = *_pos; - void *next = NULL; - down_read(&nommu_vma_sem); + down_read(&nommu_region_sem); - for (_rb = rb_first(&nommu_vma_tree); _rb; _rb = rb_next(_rb)) { - if (pos == 0) { - next = _rb; - break; - } - pos--; - } - - return next; + for (p = rb_first(&nommu_region_tree); p; p = rb_next(p)) + if (pos-- == 0) + return p; + return NULL; } -static void nommu_vma_list_stop(struct seq_file *m, void *v) +static void nommu_region_list_stop(struct seq_file *m, void *v) { - up_read(&nommu_vma_sem); + up_read(&nommu_region_sem); } -static void *nommu_vma_list_next(struct seq_file *m, void *v, loff_t *pos) +static void *nommu_region_list_next(struct seq_file *m, void *v, loff_t *pos) { (*pos)++; return rb_next((struct rb_node *) v); } -static const struct seq_operations proc_nommu_vma_list_seqop = { - .start = nommu_vma_list_start, - .next = nommu_vma_list_next, - .stop = nommu_vma_list_stop, - .show = nommu_vma_list_show +static struct seq_operations proc_nommu_region_list_seqop = { + .start = nommu_region_list_start, + .next = nommu_region_list_next, + .stop = nommu_region_list_stop, + .show = nommu_region_list_show }; -static int proc_nommu_vma_list_open(struct inode *inode, struct file *file) +static int proc_nommu_region_list_open(struct inode *inode, struct file *file) { - return seq_open(file, &proc_nommu_vma_list_seqop); + return seq_open(file, &proc_nommu_region_list_seqop); } -static const struct file_operations proc_nommu_vma_list_operations = { - .open = proc_nommu_vma_list_open, +static const struct file_operations proc_nommu_region_list_operations = { + .open = proc_nommu_region_list_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release, @@ -137,7 +130,7 @@ static const struct file_operations proc_nommu_vma_list_operations = { static int __init proc_nommu_init(void) { - proc_create("maps", S_IRUGO, NULL, &proc_nommu_vma_list_operations); + proc_create("maps", S_IRUGO, NULL, &proc_nommu_region_list_operations); return 0; } diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c index d4a8be32b902..ca4a48d0d311 100644 --- a/fs/proc/task_nommu.c +++ b/fs/proc/task_nommu.c @@ -15,25 +15,25 @@ */ void task_mem(struct seq_file *m, struct mm_struct *mm) { - struct vm_list_struct *vml; + struct vm_area_struct *vma; + struct rb_node *p; unsigned long bytes = 0, sbytes = 0, slack = 0; down_read(&mm->mmap_sem); - for (vml = mm->context.vmlist; vml; vml = vml->next) { - if (!vml->vma) - continue; + for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { + vma = rb_entry(p, struct vm_area_struct, vm_rb); - bytes += kobjsize(vml); + bytes += kobjsize(vma); if (atomic_read(&mm->mm_count) > 1 || - atomic_read(&vml->vma->vm_usage) > 1 - ) { - sbytes += kobjsize((void *) vml->vma->vm_start); - sbytes += kobjsize(vml->vma); + vma->vm_region || + vma->vm_flags & VM_MAYSHARE) { + sbytes += kobjsize((void *) vma->vm_start); + if (vma->vm_region) + sbytes += kobjsize(vma->vm_region); } else { - bytes += kobjsize((void *) vml->vma->vm_start); - bytes += kobjsize(vml->vma); - slack += kobjsize((void *) vml->vma->vm_start) - - (vml->vma->vm_end - vml->vma->vm_start); + bytes += kobjsize((void *) vma->vm_start); + slack += kobjsize((void *) vma->vm_start) - + (vma->vm_end - vma->vm_start); } } @@ -70,13 +70,14 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { - struct vm_list_struct *tbp; + struct vm_area_struct *vma; + struct rb_node *p; unsigned long vsize = 0; down_read(&mm->mmap_sem); - for (tbp = mm->context.vmlist; tbp; tbp = tbp->next) { - if (tbp->vma) - vsize += kobjsize((void *) tbp->vma->vm_start); + for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { + vma = rb_entry(p, struct vm_area_struct, vm_rb); + vsize += vma->vm_region->vm_end - vma->vm_region->vm_start; } up_read(&mm->mmap_sem); return vsize; @@ -85,16 +86,15 @@ unsigned long task_vsize(struct mm_struct *mm) int task_statm(struct mm_struct *mm, int *shared, int *text, int *data, int *resident) { - struct vm_list_struct *tbp; + struct vm_area_struct *vma; + struct rb_node *p; int size = kobjsize(mm); down_read(&mm->mmap_sem); - for (tbp = mm->context.vmlist; tbp; tbp = tbp->next) { - size += kobjsize(tbp); - if (tbp->vma) { - size += kobjsize(tbp->vma); - size += kobjsize((void *) tbp->vma->vm_start); - } + for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { + vma = rb_entry(p, struct vm_area_struct, vm_rb); + size += kobjsize(vma); + size += kobjsize((void *) vma->vm_start); } size += (*text = mm->end_code - mm->start_code); @@ -104,21 +104,63 @@ int task_statm(struct mm_struct *mm, int *shared, int *text, return size; } +/* + * display a single VMA to a sequenced file + */ +static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) +{ + unsigned long ino = 0; + struct file *file; + dev_t dev = 0; + int flags, len; + + flags = vma->vm_flags; + file = vma->vm_file; + + if (file) { + struct inode *inode = vma->vm_file->f_path.dentry->d_inode; + dev = inode->i_sb->s_dev; + ino = inode->i_ino; + } + + seq_printf(m, + "%08lx-%08lx %c%c%c%c %08lx %02x:%02x %lu %n", + vma->vm_start, + vma->vm_end, + flags & VM_READ ? 'r' : '-', + flags & VM_WRITE ? 'w' : '-', + flags & VM_EXEC ? 'x' : '-', + flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p', + vma->vm_pgoff << PAGE_SHIFT, + MAJOR(dev), MINOR(dev), ino, &len); + + if (file) { + len = 25 + sizeof(void *) * 6 - len; + if (len < 1) + len = 1; + seq_printf(m, "%*c", len, ' '); + seq_path(m, &file->f_path, ""); + } + + seq_putc(m, '\n'); + return 0; +} + /* * display mapping lines for a particular process's /proc/pid/maps */ -static int show_map(struct seq_file *m, void *_vml) +static int show_map(struct seq_file *m, void *_p) { - struct vm_list_struct *vml = _vml; + struct rb_node *p = _p; - return nommu_vma_show(m, vml->vma); + return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb)); } static void *m_start(struct seq_file *m, loff_t *pos) { struct proc_maps_private *priv = m->private; - struct vm_list_struct *vml; struct mm_struct *mm; + struct rb_node *p; loff_t n = *pos; /* pin the task and mm whilst we play with them */ @@ -134,9 +176,9 @@ static void *m_start(struct seq_file *m, loff_t *pos) } /* start from the Nth VMA */ - for (vml = mm->context.vmlist; vml; vml = vml->next) + for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) if (n-- == 0) - return vml; + return p; return NULL; } @@ -152,12 +194,12 @@ static void m_stop(struct seq_file *m, void *_vml) } } -static void *m_next(struct seq_file *m, void *_vml, loff_t *pos) +static void *m_next(struct seq_file *m, void *_p, loff_t *pos) { - struct vm_list_struct *vml = _vml; + struct rb_node *p = _p; (*pos)++; - return vml ? vml->next : NULL; + return p ? rb_next(p) : NULL; } static const struct seq_operations proc_pid_maps_ops = { diff --git a/include/asm-frv/mmu.h b/include/asm-frv/mmu.h index 22c03714fb14..86ca0e86e7d2 100644 --- a/include/asm-frv/mmu.h +++ b/include/asm-frv/mmu.h @@ -22,7 +22,6 @@ typedef struct { unsigned long dtlb_ptd_mapping; /* [DAMR5] PTD mapping for dtlb cached PGE */ #else - struct vm_list_struct *vmlist; unsigned long end_brk; #endif diff --git a/include/asm-m32r/mmu.h b/include/asm-m32r/mmu.h index d9bd724479cf..150cb92bb666 100644 --- a/include/asm-m32r/mmu.h +++ b/include/asm-m32r/mmu.h @@ -4,7 +4,6 @@ #if !defined(CONFIG_MMU) typedef struct { - struct vm_list_struct *vmlist; unsigned long end_brk; } mm_context_t; diff --git a/include/linux/mm.h b/include/linux/mm.h index 4a3d28c86443..b91a73fd1bcc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -56,19 +56,9 @@ extern unsigned long mmap_min_addr; extern struct kmem_cache *vm_area_cachep; -/* - * This struct defines the per-mm list of VMAs for uClinux. If CONFIG_MMU is - * disabled, then there's a single shared list of VMAs maintained by the - * system, and mm's subscribe to these individually - */ -struct vm_list_struct { - struct vm_list_struct *next; - struct vm_area_struct *vma; -}; - #ifndef CONFIG_MMU -extern struct rb_root nommu_vma_tree; -extern struct rw_semaphore nommu_vma_sem; +extern struct rb_root nommu_region_tree; +extern struct rw_semaphore nommu_region_sem; extern unsigned int kobjsize(const void *objp); #endif @@ -1061,6 +1051,7 @@ extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long, enum memmap_context); extern void setup_per_zone_pages_min(void); extern void mem_init(void); +extern void __init mmap_init(void); extern void show_mem(void); extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); @@ -1072,6 +1063,9 @@ extern void setup_per_cpu_pageset(void); static inline void setup_per_cpu_pageset(void) {} #endif +/* nommu.c */ +extern atomic_t mmap_pages_allocated; + /* prio_tree.c */ void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old); void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 9cfc9b627fdd..1c1e0d3a1714 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -96,6 +96,22 @@ struct page { #endif /* WANT_PAGE_VIRTUAL */ }; +/* + * A region containing a mapping of a non-memory backed file under NOMMU + * conditions. These are held in a global tree and are pinned by the VMAs that + * map parts of them. + */ +struct vm_region { + struct rb_node vm_rb; /* link in global region tree */ + unsigned long vm_flags; /* VMA vm_flags */ + unsigned long vm_start; /* start address of region */ + unsigned long vm_end; /* region initialised to here */ + unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ + struct file *vm_file; /* the backing file or NULL */ + + atomic_t vm_usage; /* region usage count */ +}; + /* * This struct defines a memory VMM memory area. There is one of these * per VM-area/task. A VM area is any part of the process virtual memory @@ -152,7 +168,7 @@ struct vm_area_struct { unsigned long vm_truncate_count;/* truncate_count or restart_addr */ #ifndef CONFIG_MMU - atomic_t vm_usage; /* refcount (VMAs shared if !MMU) */ + struct vm_region *vm_region; /* NOMMU mapping region */ #endif #ifdef CONFIG_NUMA struct mempolicy *vm_policy; /* NUMA policy for the VMA */ diff --git a/ipc/shm.c b/ipc/shm.c index b125b560240e..d0ab5527bf45 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -990,6 +990,7 @@ asmlinkage long sys_shmdt(char __user *shmaddr) */ vma = find_vma(mm, addr); +#ifdef CONFIG_MMU while (vma) { next = vma->vm_next; @@ -1034,6 +1035,17 @@ asmlinkage long sys_shmdt(char __user *shmaddr) vma = next; } +#else /* CONFIG_MMU */ + /* under NOMMU conditions, the exact address to be destroyed must be + * given */ + retval = -EINVAL; + if (vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) { + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start); + retval = 0; + } + +#endif + up_write(&mm->mmap_sem); return retval; } diff --git a/kernel/fork.c b/kernel/fork.c index 7b8f2a78be3d..0bce4a43bb37 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1481,12 +1481,10 @@ void __init proc_caches_init(void) fs_cachep = kmem_cache_create("fs_cache", sizeof(struct fs_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); - vm_area_cachep = kmem_cache_create("vm_area_struct", - sizeof(struct vm_area_struct), 0, - SLAB_PANIC, NULL); mm_cachep = kmem_cache_create("mm_struct", sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); + mmap_init(); } /* diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 2e75478e9c69..d0a32aab03ff 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -512,6 +512,13 @@ config DEBUG_VIRTUAL If unsure, say N. +config DEBUG_NOMMU_REGIONS + bool "Debug the global anon/private NOMMU mapping region tree" + depends on DEBUG_KERNEL && !MMU + help + This option causes the global tree of anonymous and private mapping + regions to be regularly checked for invalid topology. + config DEBUG_WRITECOUNT bool "Debug filesystem writers count" depends on DEBUG_KERNEL diff --git a/mm/mmap.c b/mm/mmap.c index a910c045cfd4..749623196cb9 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2472,3 +2472,13 @@ void mm_drop_all_locks(struct mm_struct *mm) mutex_unlock(&mm_all_locks_mutex); } + +/* + * initialise the VMA slab + */ +void __init mmap_init(void) +{ + vm_area_cachep = kmem_cache_create("vm_area_struct", + sizeof(struct vm_area_struct), 0, + SLAB_PANIC, NULL); +} diff --git a/mm/nommu.c b/mm/nommu.c index 23f355bbe262..0d363dfcf10e 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -6,7 +6,7 @@ * * See Documentation/nommu-mmap.txt * - * Copyright (c) 2004-2005 David Howells + * Copyright (c) 2004-2008 David Howells * Copyright (c) 2000-2003 David McCullough * Copyright (c) 2000-2001 D Jeff Dionne * Copyright (c) 2002 Greg Ungerer @@ -33,6 +33,28 @@ #include #include #include +#include "internal.h" + +static inline __attribute__((format(printf, 1, 2))) +void no_printk(const char *fmt, ...) +{ +} + +#if 0 +#define kenter(FMT, ...) \ + printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__) +#define kleave(FMT, ...) \ + printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__) +#define kdebug(FMT, ...) \ + printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__) +#else +#define kenter(FMT, ...) \ + no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__) +#define kleave(FMT, ...) \ + no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__) +#define kdebug(FMT, ...) \ + no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__) +#endif #include "internal.h" @@ -46,12 +68,15 @@ int sysctl_overcommit_ratio = 50; /* default is 50% */ int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; int heap_stack_gap = 0; +atomic_t mmap_pages_allocated; + EXPORT_SYMBOL(mem_map); EXPORT_SYMBOL(num_physpages); -/* list of shareable VMAs */ -struct rb_root nommu_vma_tree = RB_ROOT; -DECLARE_RWSEM(nommu_vma_sem); +/* list of mapped, potentially shareable regions */ +static struct kmem_cache *vm_region_jar; +struct rb_root nommu_region_tree = RB_ROOT; +DECLARE_RWSEM(nommu_region_sem); struct vm_operations_struct generic_file_vm_ops = { }; @@ -400,129 +425,174 @@ asmlinkage unsigned long sys_brk(unsigned long brk) return mm->brk = brk; } -#ifdef DEBUG -static void show_process_blocks(void) +/* + * initialise the VMA and region record slabs + */ +void __init mmap_init(void) { - struct vm_list_struct *vml; - - printk("Process blocks %d:", current->pid); - - for (vml = ¤t->mm->context.vmlist; vml; vml = vml->next) { - printk(" %p: %p", vml, vml->vma); - if (vml->vma) - printk(" (%d @%lx #%d)", - kobjsize((void *) vml->vma->vm_start), - vml->vma->vm_start, - atomic_read(&vml->vma->vm_usage)); - printk(vml->next ? " ->" : ".\n"); - } + vm_region_jar = kmem_cache_create("vm_region_jar", + sizeof(struct vm_region), 0, + SLAB_PANIC, NULL); + vm_area_cachep = kmem_cache_create("vm_area_struct", + sizeof(struct vm_area_struct), 0, + SLAB_PANIC, NULL); } -#endif /* DEBUG */ /* - * add a VMA into a process's mm_struct in the appropriate place in the list - * - should be called with mm->mmap_sem held writelocked + * validate the region tree + * - the caller must hold the region lock */ -static void add_vma_to_mm(struct mm_struct *mm, struct vm_list_struct *vml) +#ifdef CONFIG_DEBUG_NOMMU_REGIONS +static noinline void validate_nommu_regions(void) { - struct vm_list_struct **ppv; + struct vm_region *region, *last; + struct rb_node *p, *lastp; - for (ppv = ¤t->mm->context.vmlist; *ppv; ppv = &(*ppv)->next) - if ((*ppv)->vma->vm_start > vml->vma->vm_start) - break; + lastp = rb_first(&nommu_region_tree); + if (!lastp) + return; + + last = rb_entry(lastp, struct vm_region, vm_rb); + if (unlikely(last->vm_end <= last->vm_start)) + BUG(); + + while ((p = rb_next(lastp))) { + region = rb_entry(p, struct vm_region, vm_rb); + last = rb_entry(lastp, struct vm_region, vm_rb); + + if (unlikely(region->vm_end <= region->vm_start)) + BUG(); + if (unlikely(region->vm_start < last->vm_end)) + BUG(); - vml->next = *ppv; - *ppv = vml; + lastp = p; + } } +#else +#define validate_nommu_regions() do {} while(0) +#endif /* - * look up the first VMA in which addr resides, NULL if none - * - should be called with mm->mmap_sem at least held readlocked + * add a region into the global tree */ -struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) +static void add_nommu_region(struct vm_region *region) { - struct vm_list_struct *loop, *vml; + struct vm_region *pregion; + struct rb_node **p, *parent; - /* search the vm_start ordered list */ - vml = NULL; - for (loop = mm->context.vmlist; loop; loop = loop->next) { - if (loop->vma->vm_start > addr) - break; - vml = loop; + validate_nommu_regions(); + + BUG_ON(region->vm_start & ~PAGE_MASK); + + parent = NULL; + p = &nommu_region_tree.rb_node; + while (*p) { + parent = *p; + pregion = rb_entry(parent, struct vm_region, vm_rb); + if (region->vm_start < pregion->vm_start) + p = &(*p)->rb_left; + else if (region->vm_start > pregion->vm_start) + p = &(*p)->rb_right; + else if (pregion == region) + return; + else + BUG(); } - if (vml && vml->vma->vm_end > addr) - return vml->vma; + rb_link_node(®ion->vm_rb, parent, p); + rb_insert_color(®ion->vm_rb, &nommu_region_tree); - return NULL; + validate_nommu_regions(); } -EXPORT_SYMBOL(find_vma); /* - * find a VMA - * - we don't extend stack VMAs under NOMMU conditions + * delete a region from the global tree */ -struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) +static void delete_nommu_region(struct vm_region *region) { - return find_vma(mm, addr); -} + BUG_ON(!nommu_region_tree.rb_node); -int expand_stack(struct vm_area_struct *vma, unsigned long address) -{ - return -ENOMEM; + validate_nommu_regions(); + rb_erase(®ion->vm_rb, &nommu_region_tree); + validate_nommu_regions(); } /* - * look up the first VMA exactly that exactly matches addr - * - should be called with mm->mmap_sem at least held readlocked + * free a contiguous series of pages */ -static inline struct vm_area_struct *find_vma_exact(struct mm_struct *mm, - unsigned long addr) +static void free_page_series(unsigned long from, unsigned long to) { - struct vm_list_struct *vml; - - /* search the vm_start ordered list */ - for (vml = mm->context.vmlist; vml; vml = vml->next) { - if (vml->vma->vm_start == addr) - return vml->vma; - if (vml->vma->vm_start > addr) - break; + for (; from < to; from += PAGE_SIZE) { + struct page *page = virt_to_page(from); + + kdebug("- free %lx", from); + atomic_dec(&mmap_pages_allocated); + if (page_count(page) != 1) + kdebug("free page %p [%d]", page, page_count(page)); + put_page(page); } - - return NULL; } /* - * find a VMA in the global tree + * release a reference to a region + * - the caller must hold the region semaphore, which this releases + * - the region may not have been added to the tree yet, in which case vm_end + * will equal vm_start */ -static inline struct vm_area_struct *find_nommu_vma(unsigned long start) +static void __put_nommu_region(struct vm_region *region) + __releases(nommu_region_sem) { - struct vm_area_struct *vma; - struct rb_node *n = nommu_vma_tree.rb_node; + kenter("%p{%d}", region, atomic_read(®ion->vm_usage)); - while (n) { - vma = rb_entry(n, struct vm_area_struct, vm_rb); + BUG_ON(!nommu_region_tree.rb_node); - if (start < vma->vm_start) - n = n->rb_left; - else if (start > vma->vm_start) - n = n->rb_right; - else - return vma; + if (atomic_dec_and_test(®ion->vm_usage)) { + if (region->vm_end > region->vm_start) + delete_nommu_region(region); + up_write(&nommu_region_sem); + + if (region->vm_file) + fput(region->vm_file); + + /* IO memory and memory shared directly out of the pagecache + * from ramfs/tmpfs mustn't be released here */ + if (region->vm_flags & VM_MAPPED_COPY) { + kdebug("free series"); + free_page_series(region->vm_start, region->vm_end); + } + kmem_cache_free(vm_region_jar, region); + } else { + up_write(&nommu_region_sem); } +} - return NULL; +/* + * release a reference to a region + */ +static void put_nommu_region(struct vm_region *region) +{ + down_write(&nommu_region_sem); + __put_nommu_region(region); } /* - * add a VMA in the global tree + * add a VMA into a process's mm_struct in the appropriate place in the list + * and tree and add to the address space's page tree also if not an anonymous + * page + * - should be called with mm->mmap_sem held writelocked */ -static void add_nommu_vma(struct vm_area_struct *vma) +static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) { - struct vm_area_struct *pvma; + struct vm_area_struct *pvma, **pp; struct address_space *mapping; - struct rb_node **p = &nommu_vma_tree.rb_node; - struct rb_node *parent = NULL; + struct rb_node **p, *parent; + + kenter(",%p", vma); + + BUG_ON(!vma->vm_region); + + mm->map_count++; + vma->vm_mm = mm; /* add the VMA to the mapping */ if (vma->vm_file) { @@ -533,42 +603,62 @@ static void add_nommu_vma(struct vm_area_struct *vma) flush_dcache_mmap_unlock(mapping); } - /* add the VMA to the master list */ + /* add the VMA to the tree */ + parent = NULL; + p = &mm->mm_rb.rb_node; while (*p) { parent = *p; pvma = rb_entry(parent, struct vm_area_struct, vm_rb); - if (vma->vm_start < pvma->vm_start) { + /* sort by: start addr, end addr, VMA struct addr in that order + * (the latter is necessary as we may get identical VMAs) */ + if (vma->vm_start < pvma->vm_start) p = &(*p)->rb_left; - } - else if (vma->vm_start > pvma->vm_start) { + else if (vma->vm_start > pvma->vm_start) p = &(*p)->rb_right; - } - else { - /* mappings are at the same address - this can only - * happen for shared-mem chardevs and shared file - * mappings backed by ramfs/tmpfs */ - BUG_ON(!(pvma->vm_flags & VM_SHARED)); - - if (vma < pvma) - p = &(*p)->rb_left; - else if (vma > pvma) - p = &(*p)->rb_right; - else - BUG(); - } + else if (vma->vm_end < pvma->vm_end) + p = &(*p)->rb_left; + else if (vma->vm_end > pvma->vm_end) + p = &(*p)->rb_right; + else if (vma < pvma) + p = &(*p)->rb_left; + else if (vma > pvma) + p = &(*p)->rb_right; + else + BUG(); } rb_link_node(&vma->vm_rb, parent, p); - rb_insert_color(&vma->vm_rb, &nommu_vma_tree); + rb_insert_color(&vma->vm_rb, &mm->mm_rb); + + /* add VMA to the VMA list also */ + for (pp = &mm->mmap; (pvma = *pp); pp = &(*pp)->vm_next) { + if (pvma->vm_start > vma->vm_start) + break; + if (pvma->vm_start < vma->vm_start) + continue; + if (pvma->vm_end < vma->vm_end) + break; + } + + vma->vm_next = *pp; + *pp = vma; } /* - * delete a VMA from the global list + * delete a VMA from its owning mm_struct and address space */ -static void delete_nommu_vma(struct vm_area_struct *vma) +static void delete_vma_from_mm(struct vm_area_struct *vma) { + struct vm_area_struct **pp; struct address_space *mapping; + struct mm_struct *mm = vma->vm_mm; + + kenter("%p", vma); + + mm->map_count--; + if (mm->mmap_cache == vma) + mm->mmap_cache = NULL; /* remove the VMA from the mapping */ if (vma->vm_file) { @@ -579,8 +669,115 @@ static void delete_nommu_vma(struct vm_area_struct *vma) flush_dcache_mmap_unlock(mapping); } - /* remove from the master list */ - rb_erase(&vma->vm_rb, &nommu_vma_tree); + /* remove from the MM's tree and list */ + rb_erase(&vma->vm_rb, &mm->mm_rb); + for (pp = &mm->mmap; *pp; pp = &(*pp)->vm_next) { + if (*pp == vma) { + *pp = vma->vm_next; + break; + } + } + + vma->vm_mm = NULL; +} + +/* + * destroy a VMA record + */ +static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) +{ + kenter("%p", vma); + if (vma->vm_ops && vma->vm_ops->close) + vma->vm_ops->close(vma); + if (vma->vm_file) { + fput(vma->vm_file); + if (vma->vm_flags & VM_EXECUTABLE) + removed_exe_file_vma(mm); + } + put_nommu_region(vma->vm_region); + kmem_cache_free(vm_area_cachep, vma); +} + +/* + * look up the first VMA in which addr resides, NULL if none + * - should be called with mm->mmap_sem at least held readlocked + */ +struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + struct rb_node *n = mm->mm_rb.rb_node; + + /* check the cache first */ + vma = mm->mmap_cache; + if (vma && vma->vm_start <= addr && vma->vm_end > addr) + return vma; + + /* trawl the tree (there may be multiple mappings in which addr + * resides) */ + for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) { + vma = rb_entry(n, struct vm_area_struct, vm_rb); + if (vma->vm_start > addr) + return NULL; + if (vma->vm_end > addr) { + mm->mmap_cache = vma; + return vma; + } + } + + return NULL; +} +EXPORT_SYMBOL(find_vma); + +/* + * find a VMA + * - we don't extend stack VMAs under NOMMU conditions + */ +struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) +{ + return find_vma(mm, addr); +} + +/* + * expand a stack to a given address + * - not supported under NOMMU conditions + */ +int expand_stack(struct vm_area_struct *vma, unsigned long address) +{ + return -ENOMEM; +} + +/* + * look up the first VMA exactly that exactly matches addr + * - should be called with mm->mmap_sem at least held readlocked + */ +static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, + unsigned long addr, + unsigned long len) +{ + struct vm_area_struct *vma; + struct rb_node *n = mm->mm_rb.rb_node; + unsigned long end = addr + len; + + /* check the cache first */ + vma = mm->mmap_cache; + if (vma && vma->vm_start == addr && vma->vm_end == end) + return vma; + + /* trawl the tree (there may be multiple mappings in which addr + * resides) */ + for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) { + vma = rb_entry(n, struct vm_area_struct, vm_rb); + if (vma->vm_start < addr) + continue; + if (vma->vm_start > addr) + return NULL; + if (vma->vm_end == end) { + mm->mmap_cache = vma; + return vma; + } + } + + return NULL; } /* @@ -595,7 +792,7 @@ static int validate_mmap_request(struct file *file, unsigned long pgoff, unsigned long *_capabilities) { - unsigned long capabilities; + unsigned long capabilities, rlen; unsigned long reqprot = prot; int ret; @@ -615,12 +812,12 @@ static int validate_mmap_request(struct file *file, return -EINVAL; /* Careful about overflows.. */ - len = PAGE_ALIGN(len); - if (!len || len > TASK_SIZE) + rlen = PAGE_ALIGN(len); + if (!rlen || rlen > TASK_SIZE) return -ENOMEM; /* offset overflow? */ - if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff) return -EOVERFLOW; if (file) { @@ -794,9 +991,10 @@ static unsigned long determine_vm_flags(struct file *file, } /* - * set up a shared mapping on a file + * set up a shared mapping on a file (the driver or filesystem provides and + * pins the storage) */ -static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) +static int do_mmap_shared_file(struct vm_area_struct *vma) { int ret; @@ -814,10 +1012,14 @@ static int do_mmap_shared_file(struct vm_area_struct *vma, unsigned long len) /* * set up a private mapping or an anonymous shared mapping */ -static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) +static int do_mmap_private(struct vm_area_struct *vma, + struct vm_region *region, + unsigned long len) { + struct page *pages; + unsigned long total, point, n, rlen; void *base; - int ret; + int ret, order; /* invoke the file's mapping function so that it can keep track of * shared mappings on devices or memory @@ -836,23 +1038,46 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) * make a private copy of the data and map that instead */ } + rlen = PAGE_ALIGN(len); + /* allocate some memory to hold the mapping * - note that this may not return a page-aligned address if the object * we're allocating is smaller than a page */ - base = kmalloc(len, GFP_KERNEL|__GFP_COMP); - if (!base) + order = get_order(rlen); + kdebug("alloc order %d for %lx", order, len); + + pages = alloc_pages(GFP_KERNEL, order); + if (!pages) goto enomem; - vma->vm_start = (unsigned long) base; - vma->vm_end = vma->vm_start + len; - vma->vm_flags |= VM_MAPPED_COPY; + /* we allocated a power-of-2 sized page set, so we need to trim off the + * excess */ + total = 1 << order; + atomic_add(total, &mmap_pages_allocated); + + point = rlen >> PAGE_SHIFT; + while (total > point) { + order = ilog2(total - point); + n = 1 << order; + kdebug("shave %lu/%lu @%lu", n, total - point, total); + atomic_sub(n, &mmap_pages_allocated); + total -= n; + set_page_refcounted(pages + total); + __free_pages(pages + total, order); + } + + total = rlen >> PAGE_SHIFT; + for (point = 1; point < total; point++) + set_page_refcounted(&pages[point]); -#ifdef WARN_ON_SLACK - if (len + WARN_ON_SLACK <= kobjsize(result)) - printk("Allocation of %lu bytes from process %d has %lu bytes of slack\n", - len, current->pid, kobjsize(result) - len); -#endif + base = page_address(pages); + region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY; + region->vm_start = (unsigned long) base; + region->vm_end = region->vm_start + rlen; + + vma->vm_start = region->vm_start; + vma->vm_end = region->vm_start + len; if (vma->vm_file) { /* read the contents of a file into the copy */ @@ -864,26 +1089,27 @@ static int do_mmap_private(struct vm_area_struct *vma, unsigned long len) old_fs = get_fs(); set_fs(KERNEL_DS); - ret = vma->vm_file->f_op->read(vma->vm_file, base, len, &fpos); + ret = vma->vm_file->f_op->read(vma->vm_file, base, rlen, &fpos); set_fs(old_fs); if (ret < 0) goto error_free; /* clear the last little bit */ - if (ret < len) - memset(base + ret, 0, len - ret); + if (ret < rlen) + memset(base + ret, 0, rlen - ret); } else { /* if it's an anonymous mapping, then just clear it */ - memset(base, 0, len); + memset(base, 0, rlen); } return 0; error_free: - kfree(base); - vma->vm_start = 0; + free_page_series(region->vm_start, region->vm_end); + region->vm_start = vma->vm_start = 0; + region->vm_end = vma->vm_end = 0; return ret; enomem: @@ -903,13 +1129,14 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long flags, unsigned long pgoff) { - struct vm_list_struct *vml = NULL; - struct vm_area_struct *vma = NULL; + struct vm_area_struct *vma; + struct vm_region *region; struct rb_node *rb; - unsigned long capabilities, vm_flags; - void *result; + unsigned long capabilities, vm_flags, result; int ret; + kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff); + if (!(flags & MAP_FIXED)) addr = round_hint_to_min(addr); @@ -917,73 +1144,120 @@ unsigned long do_mmap_pgoff(struct file *file, * mapping */ ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, &capabilities); - if (ret < 0) + if (ret < 0) { + kleave(" = %d [val]", ret); return ret; + } /* we've determined that we can make the mapping, now translate what we * now know into VMA flags */ vm_flags = determine_vm_flags(file, prot, flags, capabilities); - /* we're going to need to record the mapping if it works */ - vml = kzalloc(sizeof(struct vm_list_struct), GFP_KERNEL); - if (!vml) - goto error_getting_vml; + /* we're going to need to record the mapping */ + region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); + if (!region) + goto error_getting_region; + + vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); + if (!vma) + goto error_getting_vma; + + atomic_set(®ion->vm_usage, 1); + region->vm_flags = vm_flags; + region->vm_pgoff = pgoff; - down_write(&nommu_vma_sem); + INIT_LIST_HEAD(&vma->anon_vma_node); + vma->vm_flags = vm_flags; + vma->vm_pgoff = pgoff; - /* if we want to share, we need to check for VMAs created by other + if (file) { + region->vm_file = file; + get_file(file); + vma->vm_file = file; + get_file(file); + if (vm_flags & VM_EXECUTABLE) { + added_exe_file_vma(current->mm); + vma->vm_mm = current->mm; + } + } + + down_write(&nommu_region_sem); + + /* if we want to share, we need to check for regions created by other * mmap() calls that overlap with our proposed mapping - * - we can only share with an exact match on most regular files + * - we can only share with a superset match on most regular files * - shared mappings on character devices and memory backed files are * permitted to overlap inexactly as far as we are concerned for in * these cases, sharing is handled in the driver or filesystem rather * than here */ if (vm_flags & VM_MAYSHARE) { - unsigned long pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; - unsigned long vmpglen; + struct vm_region *pregion; + unsigned long pglen, rpglen, pgend, rpgend, start; - /* suppress VMA sharing for shared regions */ - if (vm_flags & VM_SHARED && - capabilities & BDI_CAP_MAP_DIRECT) - goto dont_share_VMAs; + pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; + pgend = pgoff + pglen; - for (rb = rb_first(&nommu_vma_tree); rb; rb = rb_next(rb)) { - vma = rb_entry(rb, struct vm_area_struct, vm_rb); + for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) { + pregion = rb_entry(rb, struct vm_region, vm_rb); - if (!(vma->vm_flags & VM_MAYSHARE)) + if (!(pregion->vm_flags & VM_MAYSHARE)) continue; /* search for overlapping mappings on the same file */ - if (vma->vm_file->f_path.dentry->d_inode != file->f_path.dentry->d_inode) + if (pregion->vm_file->f_path.dentry->d_inode != + file->f_path.dentry->d_inode) continue; - if (vma->vm_pgoff >= pgoff + pglen) + if (pregion->vm_pgoff >= pgend) continue; - vmpglen = vma->vm_end - vma->vm_start + PAGE_SIZE - 1; - vmpglen >>= PAGE_SHIFT; - if (pgoff >= vma->vm_pgoff + vmpglen) + rpglen = pregion->vm_end - pregion->vm_start; + rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT; + rpgend = pregion->vm_pgoff + rpglen; + if (pgoff >= rpgend) continue; - /* handle inexactly overlapping matches between mappings */ - if (vma->vm_pgoff != pgoff || vmpglen != pglen) { + /* handle inexactly overlapping matches between + * mappings */ + if ((pregion->vm_pgoff != pgoff || rpglen != pglen) && + !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) { + /* new mapping is not a subset of the region */ if (!(capabilities & BDI_CAP_MAP_DIRECT)) goto sharing_violation; continue; } - /* we've found a VMA we can share */ - atomic_inc(&vma->vm_usage); - - vml->vma = vma; - result = (void *) vma->vm_start; - goto shared; + /* we've found a region we can share */ + atomic_inc(&pregion->vm_usage); + vma->vm_region = pregion; + start = pregion->vm_start; + start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT; + vma->vm_start = start; + vma->vm_end = start + len; + + if (pregion->vm_flags & VM_MAPPED_COPY) { + kdebug("share copy"); + vma->vm_flags |= VM_MAPPED_COPY; + } else { + kdebug("share mmap"); + ret = do_mmap_shared_file(vma); + if (ret < 0) { + vma->vm_region = NULL; + vma->vm_start = 0; + vma->vm_end = 0; + atomic_dec(&pregion->vm_usage); + pregion = NULL; + goto error_just_free; + } + } + fput(region->vm_file); + kmem_cache_free(vm_region_jar, region); + region = pregion; + result = start; + goto share; } - dont_share_VMAs: - vma = NULL; - /* obtain the address at which to make a shared mapping * - this is the hook for quasi-memory character devices to * tell us the location of a shared mapping @@ -994,102 +1268,93 @@ unsigned long do_mmap_pgoff(struct file *file, if (IS_ERR((void *) addr)) { ret = addr; if (ret != (unsigned long) -ENOSYS) - goto error; + goto error_just_free; /* the driver refused to tell us where to site * the mapping so we'll have to attempt to copy * it */ ret = (unsigned long) -ENODEV; if (!(capabilities & BDI_CAP_MAP_COPY)) - goto error; + goto error_just_free; capabilities &= ~BDI_CAP_MAP_DIRECT; + } else { + vma->vm_start = region->vm_start = addr; + vma->vm_end = region->vm_end = addr + len; } } } - /* we're going to need a VMA struct as well */ - vma = kzalloc(sizeof(struct vm_area_struct), GFP_KERNEL); - if (!vma) - goto error_getting_vma; - - INIT_LIST_HEAD(&vma->anon_vma_node); - atomic_set(&vma->vm_usage, 1); - if (file) { - get_file(file); - if (vm_flags & VM_EXECUTABLE) { - added_exe_file_vma(current->mm); - vma->vm_mm = current->mm; - } - } - vma->vm_file = file; - vma->vm_flags = vm_flags; - vma->vm_start = addr; - vma->vm_end = addr + len; - vma->vm_pgoff = pgoff; - - vml->vma = vma; + vma->vm_region = region; /* set up the mapping */ if (file && vma->vm_flags & VM_SHARED) - ret = do_mmap_shared_file(vma, len); + ret = do_mmap_shared_file(vma); else - ret = do_mmap_private(vma, len); + ret = do_mmap_private(vma, region, len); if (ret < 0) - goto error; + goto error_put_region; + + add_nommu_region(region); /* okay... we have a mapping; now we have to register it */ - result = (void *) vma->vm_start; + result = vma->vm_start; current->mm->total_vm += len >> PAGE_SHIFT; - add_nommu_vma(vma); +share: + add_vma_to_mm(current->mm, vma); - shared: - add_vma_to_mm(current->mm, vml); - - up_write(&nommu_vma_sem); + up_write(&nommu_region_sem); if (prot & PROT_EXEC) - flush_icache_range((unsigned long) result, - (unsigned long) result + len); + flush_icache_range(result, result + len); -#ifdef DEBUG - printk("do_mmap:\n"); - show_process_blocks(); -#endif + kleave(" = %lx", result); + return result; - return (unsigned long) result; - - error: - up_write(&nommu_vma_sem); - kfree(vml); +error_put_region: + __put_nommu_region(region); if (vma) { if (vma->vm_file) { fput(vma->vm_file); if (vma->vm_flags & VM_EXECUTABLE) removed_exe_file_vma(vma->vm_mm); } - kfree(vma); + kmem_cache_free(vm_area_cachep, vma); } + kleave(" = %d [pr]", ret); return ret; - sharing_violation: - up_write(&nommu_vma_sem); - printk("Attempt to share mismatched mappings\n"); - kfree(vml); - return -EINVAL; +error_just_free: + up_write(&nommu_region_sem); +error: + fput(region->vm_file); + kmem_cache_free(vm_region_jar, region); + fput(vma->vm_file); + if (vma->vm_flags & VM_EXECUTABLE) + removed_exe_file_vma(vma->vm_mm); + kmem_cache_free(vm_area_cachep, vma); + kleave(" = %d", ret); + return ret; + +sharing_violation: + up_write(&nommu_region_sem); + printk(KERN_WARNING "Attempt to share mismatched mappings\n"); + ret = -EINVAL; + goto error; - error_getting_vma: - up_write(&nommu_vma_sem); - kfree(vml); - printk("Allocation of vma for %lu byte allocation from process %d failed\n", +error_getting_vma: + kmem_cache_free(vm_region_jar, region); + printk(KERN_WARNING "Allocation of vma for %lu byte allocation" + " from process %d failed\n", len, current->pid); show_free_areas(); return -ENOMEM; - error_getting_vml: - printk("Allocation of vml for %lu byte allocation from process %d failed\n", +error_getting_region: + printk(KERN_WARNING "Allocation of vm region for %lu byte allocation" + " from process %d failed\n", len, current->pid); show_free_areas(); return -ENOMEM; @@ -1097,77 +1362,180 @@ unsigned long do_mmap_pgoff(struct file *file, EXPORT_SYMBOL(do_mmap_pgoff); /* - * handle mapping disposal for uClinux + * split a vma into two pieces at address 'addr', a new vma is allocated either + * for the first part or the tail. */ -static void put_vma(struct mm_struct *mm, struct vm_area_struct *vma) +int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, int new_below) { - if (vma) { - down_write(&nommu_vma_sem); + struct vm_area_struct *new; + struct vm_region *region; + unsigned long npages; - if (atomic_dec_and_test(&vma->vm_usage)) { - delete_nommu_vma(vma); + kenter(""); - if (vma->vm_ops && vma->vm_ops->close) - vma->vm_ops->close(vma); + /* we're only permitted to split anonymous regions that have a single + * owner */ + if (vma->vm_file || + atomic_read(&vma->vm_region->vm_usage) != 1) + return -ENOMEM; - /* IO memory and memory shared directly out of the pagecache from - * ramfs/tmpfs mustn't be released here */ - if (vma->vm_flags & VM_MAPPED_COPY) - kfree((void *) vma->vm_start); + if (mm->map_count >= sysctl_max_map_count) + return -ENOMEM; - if (vma->vm_file) { - fput(vma->vm_file); - if (vma->vm_flags & VM_EXECUTABLE) - removed_exe_file_vma(mm); - } - kfree(vma); - } + region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL); + if (!region) + return -ENOMEM; + + new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); + if (!new) { + kmem_cache_free(vm_region_jar, region); + return -ENOMEM; + } + + /* most fields are the same, copy all, and then fixup */ + *new = *vma; + *region = *vma->vm_region; + new->vm_region = region; + + npages = (addr - vma->vm_start) >> PAGE_SHIFT; + + if (new_below) { + region->vm_end = new->vm_end = addr; + } else { + region->vm_start = new->vm_start = addr; + region->vm_pgoff = new->vm_pgoff += npages; + } - up_write(&nommu_vma_sem); + if (new->vm_ops && new->vm_ops->open) + new->vm_ops->open(new); + + delete_vma_from_mm(vma); + down_write(&nommu_region_sem); + delete_nommu_region(vma->vm_region); + if (new_below) { + vma->vm_region->vm_start = vma->vm_start = addr; + vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; + } else { + vma->vm_region->vm_end = vma->vm_end = addr; } + add_nommu_region(vma->vm_region); + add_nommu_region(new->vm_region); + up_write(&nommu_region_sem); + add_vma_to_mm(mm, vma); + add_vma_to_mm(mm, new); + return 0; } /* - * release a mapping - * - under NOMMU conditions the parameters must match exactly to the mapping to - * be removed + * shrink a VMA by removing the specified chunk from either the beginning or + * the end */ -int do_munmap(struct mm_struct *mm, unsigned long addr, size_t len) +static int shrink_vma(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long from, unsigned long to) { - struct vm_list_struct *vml, **parent; - unsigned long end = addr + len; + struct vm_region *region; -#ifdef DEBUG - printk("do_munmap:\n"); -#endif + kenter(""); - for (parent = &mm->context.vmlist; *parent; parent = &(*parent)->next) { - if ((*parent)->vma->vm_start > addr) - break; - if ((*parent)->vma->vm_start == addr && - ((len == 0) || ((*parent)->vma->vm_end == end))) - goto found; - } + /* adjust the VMA's pointers, which may reposition it in the MM's tree + * and list */ + delete_vma_from_mm(vma); + if (from > vma->vm_start) + vma->vm_end = from; + else + vma->vm_start = to; + add_vma_to_mm(mm, vma); - printk("munmap of non-mmaped memory by process %d (%s): %p\n", - current->pid, current->comm, (void *) addr); - return -EINVAL; + /* cut the backing region down to size */ + region = vma->vm_region; + BUG_ON(atomic_read(®ion->vm_usage) != 1); - found: - vml = *parent; + down_write(&nommu_region_sem); + delete_nommu_region(region); + if (from > region->vm_start) + region->vm_end = from; + else + region->vm_start = to; + add_nommu_region(region); + up_write(&nommu_region_sem); - put_vma(mm, vml->vma); + free_page_series(from, to); + return 0; +} - *parent = vml->next; - kfree(vml); +/* + * release a mapping + * - under NOMMU conditions the chunk to be unmapped must be backed by a single + * VMA, though it need not cover the whole VMA + */ +int do_munmap(struct mm_struct *mm, unsigned long start, size_t len) +{ + struct vm_area_struct *vma; + struct rb_node *rb; + unsigned long end = start + len; + int ret; - update_hiwater_vm(mm); - mm->total_vm -= len >> PAGE_SHIFT; + kenter(",%lx,%zx", start, len); -#ifdef DEBUG - show_process_blocks(); -#endif + if (len == 0) + return -EINVAL; + + /* find the first potentially overlapping VMA */ + vma = find_vma(mm, start); + if (!vma) { + printk(KERN_WARNING + "munmap of memory not mmapped by process %d (%s):" + " 0x%lx-0x%lx\n", + current->pid, current->comm, start, start + len - 1); + return -EINVAL; + } + /* we're allowed to split an anonymous VMA but not a file-backed one */ + if (vma->vm_file) { + do { + if (start > vma->vm_start) { + kleave(" = -EINVAL [miss]"); + return -EINVAL; + } + if (end == vma->vm_end) + goto erase_whole_vma; + rb = rb_next(&vma->vm_rb); + vma = rb_entry(rb, struct vm_area_struct, vm_rb); + } while (rb); + kleave(" = -EINVAL [split file]"); + return -EINVAL; + } else { + /* the chunk must be a subset of the VMA found */ + if (start == vma->vm_start && end == vma->vm_end) + goto erase_whole_vma; + if (start < vma->vm_start || end > vma->vm_end) { + kleave(" = -EINVAL [superset]"); + return -EINVAL; + } + if (start & ~PAGE_MASK) { + kleave(" = -EINVAL [unaligned start]"); + return -EINVAL; + } + if (end != vma->vm_end && end & ~PAGE_MASK) { + kleave(" = -EINVAL [unaligned split]"); + return -EINVAL; + } + if (start != vma->vm_start && end != vma->vm_end) { + ret = split_vma(mm, vma, start, 1); + if (ret < 0) { + kleave(" = %d [split]", ret); + return ret; + } + } + return shrink_vma(mm, vma, start, end); + } + +erase_whole_vma: + delete_vma_from_mm(vma); + delete_vma(mm, vma); + kleave(" = 0"); return 0; } EXPORT_SYMBOL(do_munmap); @@ -1184,29 +1552,26 @@ asmlinkage long sys_munmap(unsigned long addr, size_t len) } /* - * Release all mappings + * release all the mappings made in a process's VM space */ -void exit_mmap(struct mm_struct * mm) +void exit_mmap(struct mm_struct *mm) { - struct vm_list_struct *tmp; + struct vm_area_struct *vma; - if (mm) { -#ifdef DEBUG - printk("Exit_mmap:\n"); -#endif + if (!mm) + return; - mm->total_vm = 0; + kenter(""); - while ((tmp = mm->context.vmlist)) { - mm->context.vmlist = tmp->next; - put_vma(mm, tmp->vma); - kfree(tmp); - } + mm->total_vm = 0; -#ifdef DEBUG - show_process_blocks(); -#endif + while ((vma = mm->mmap)) { + mm->mmap = vma->vm_next; + delete_vma_from_mm(vma); + delete_vma(mm, vma); } + + kleave(""); } unsigned long do_brk(unsigned long addr, unsigned long len) @@ -1219,8 +1584,8 @@ unsigned long do_brk(unsigned long addr, unsigned long len) * time (controlled by the MREMAP_MAYMOVE flag and available VM space) * * under NOMMU conditions, we only permit changing a mapping's size, and only - * as long as it stays within the hole allocated by the kmalloc() call in - * do_mmap_pgoff() and the block is not shareable + * as long as it stays within the region allocated by do_mmap_private() and the + * block is not shareable * * MREMAP_FIXED is not supported under NOMMU conditions */ @@ -1231,13 +1596,16 @@ unsigned long do_mremap(unsigned long addr, struct vm_area_struct *vma; /* insanity checks first */ - if (new_len == 0) + if (old_len == 0 || new_len == 0) return (unsigned long) -EINVAL; + if (addr & ~PAGE_MASK) + return -EINVAL; + if (flags & MREMAP_FIXED && new_addr != addr) return (unsigned long) -EINVAL; - vma = find_vma_exact(current->mm, addr); + vma = find_vma_exact(current->mm, addr, old_len); if (!vma) return (unsigned long) -EINVAL; @@ -1247,19 +1615,19 @@ unsigned long do_mremap(unsigned long addr, if (vma->vm_flags & VM_MAYSHARE) return (unsigned long) -EPERM; - if (new_len > kobjsize((void *) addr)) + if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start) return (unsigned long) -ENOMEM; /* all checks complete - do it */ vma->vm_end = vma->vm_start + new_len; - return vma->vm_start; } EXPORT_SYMBOL(do_mremap); -asmlinkage unsigned long sys_mremap(unsigned long addr, - unsigned long old_len, unsigned long new_len, - unsigned long flags, unsigned long new_addr) +asmlinkage +unsigned long sys_mremap(unsigned long addr, + unsigned long old_len, unsigned long new_len, + unsigned long flags, unsigned long new_addr) { unsigned long ret; -- cgit v1.2.3-59-g8ed1b