From 24f287e412ae90de8d281543c8b1043b6ed6c019 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 15 Oct 2007 16:41:44 -0700 Subject: [SPARC64]: Implement atomic backoff. When the cpu count is high and contention hits an atomic object, the processors can synchronize such that some cpus continually get knocked out and cannot complete the atomic update. So implement an exponential backoff when SMP. Signed-off-by: David S. Miller --- arch/sparc64/lib/atomic.S | 38 +++++++++++++++++++++++++++----------- arch/sparc64/lib/bitops.S | 30 +++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 20 deletions(-) (limited to 'arch') diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S index 9633750167d0..70ac4186f62b 100644 --- a/arch/sparc64/lib/atomic.S +++ b/arch/sparc64/lib/atomic.S @@ -1,10 +1,10 @@ -/* $Id: atomic.S,v 1.4 2001/11/18 00:12:56 davem Exp $ - * atomic.S: These things are too big to do inline. +/* atomic.S: These things are too big to do inline. * - * Copyright (C) 1999 David S. Miller (davem@redhat.com) + * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) */ #include +#include .text @@ -16,27 +16,31 @@ .globl atomic_add .type atomic_add,#function atomic_add: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 add %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_add, .-atomic_add .globl atomic_sub .type atomic_sub,#function atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 sub %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_sub, .-atomic_sub /* On SMP we need to use memory barriers to ensure @@ -60,89 +64,101 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ .globl atomic_add_ret .type atomic_add_ret,#function atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: lduw [%o1], %g1 add %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f add %g7, %o0, %g7 sra %g7, 0, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_add_ret, .-atomic_add_ret .globl atomic_sub_ret .type atomic_sub_ret,#function atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: lduw [%o1], %g1 sub %g1, %o0, %g7 cas [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %icc, 1b + bne,pn %icc, 2f sub %g7, %o0, %g7 sra %g7, 0, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic_sub_ret, .-atomic_sub_ret .globl atomic64_add .type atomic64_add,#function atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 add %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_add, .-atomic64_add .globl atomic64_sub .type atomic64_sub,#function atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 sub %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_sub, .-atomic64_sub .globl atomic64_add_ret .type atomic64_add_ret,#function atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: ldx [%o1], %g1 add %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f add %g7, %o0, %g7 mov %g7, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_add_ret, .-atomic64_add_ret .globl atomic64_sub_ret .type atomic64_sub_ret,#function atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ + BACKOFF_SETUP(%o2) ATOMIC_PRE_BARRIER 1: ldx [%o1], %g1 sub %g1, %o0, %g7 casx [%o1], %g1, %g7 cmp %g1, %g7 - bne,pn %xcc, 1b + bne,pn %xcc, 2f sub %g7, %o0, %g7 mov %g7, %o0 ATOMIC_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o2, %o3, 1b) .size atomic64_sub_ret, .-atomic64_sub_ret diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S index 892431a82131..6b015a6eefb5 100644 --- a/arch/sparc64/lib/bitops.S +++ b/arch/sparc64/lib/bitops.S @@ -1,10 +1,10 @@ -/* $Id: bitops.S,v 1.3 2001/11/18 00:12:56 davem Exp $ - * bitops.S: Sparc64 atomic bit operations. +/* bitops.S: Sparc64 atomic bit operations. * - * Copyright (C) 2000 David S. Miller (davem@redhat.com) + * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net) */ #include +#include .text @@ -29,6 +29,7 @@ .globl test_and_set_bit .type test_and_set_bit,#function test_and_set_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) BITOP_PRE_BARRIER srlx %o0, 6, %g1 mov 1, %o2 @@ -40,18 +41,20 @@ test_and_set_bit: /* %o0=nr, %o1=addr */ or %g7, %o2, %g1 casx [%o1], %g7, %g1 cmp %g7, %g1 - bne,pn %xcc, 1b + bne,pn %xcc, 2f and %g7, %o2, %g2 clr %o0 movrne %g2, 1, %o0 BITOP_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o3, %o4, 1b) .size test_and_set_bit, .-test_and_set_bit .globl test_and_clear_bit .type test_and_clear_bit,#function test_and_clear_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) BITOP_PRE_BARRIER srlx %o0, 6, %g1 mov 1, %o2 @@ -63,18 +66,20 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */ andn %g7, %o2, %g1 casx [%o1], %g7, %g1 cmp %g7, %g1 - bne,pn %xcc, 1b + bne,pn %xcc, 2f and %g7, %o2, %g2 clr %o0 movrne %g2, 1, %o0 BITOP_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o3, %o4, 1b) .size test_and_clear_bit, .-test_and_clear_bit .globl test_and_change_bit .type test_and_change_bit,#function test_and_change_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) BITOP_PRE_BARRIER srlx %o0, 6, %g1 mov 1, %o2 @@ -86,18 +91,20 @@ test_and_change_bit: /* %o0=nr, %o1=addr */ xor %g7, %o2, %g1 casx [%o1], %g7, %g1 cmp %g7, %g1 - bne,pn %xcc, 1b + bne,pn %xcc, 2f and %g7, %o2, %g2 clr %o0 movrne %g2, 1, %o0 BITOP_POST_BARRIER retl nop +2: BACKOFF_SPIN(%o3, %o4, 1b) .size test_and_change_bit, .-test_and_change_bit .globl set_bit .type set_bit,#function set_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 sllx %g1, 3, %g3 @@ -108,15 +115,17 @@ set_bit: /* %o0=nr, %o1=addr */ or %g7, %o2, %g1 casx [%o1], %g7, %g1 cmp %g7, %g1 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o3, %o4, 1b) .size set_bit, .-set_bit .globl clear_bit .type clear_bit,#function clear_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 sllx %g1, 3, %g3 @@ -127,15 +136,17 @@ clear_bit: /* %o0=nr, %o1=addr */ andn %g7, %o2, %g1 casx [%o1], %g7, %g1 cmp %g7, %g1 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o3, %o4, 1b) .size clear_bit, .-clear_bit .globl change_bit .type change_bit,#function change_bit: /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 sllx %g1, 3, %g3 @@ -146,8 +157,9 @@ change_bit: /* %o0=nr, %o1=addr */ xor %g7, %o2, %g1 casx [%o1], %g7, %g1 cmp %g7, %g1 - bne,pn %xcc, 1b + bne,pn %xcc, 2f nop retl nop +2: BACKOFF_SPIN(%o3, %o4, 1b) .size change_bit, .-change_bit -- cgit v1.2.3-59-g8ed1b From 719023fb90009855c4515a16349fc6d0a2f93a74 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 15 Oct 2007 16:43:19 -0700 Subject: [SPARC64]: Fix boot failures due to bootmem. Do not use *alloc_bootmem_low*(), because ARCH_LOW_ADDRESS_LIMIT is 4GB and this results in boot failures if all of the physical memory in the machine is above 4GB. Signed-off-by: David S. Miller --- arch/sparc64/kernel/irq.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c index f3922e5a89f6..2c3bea228159 100644 --- a/arch/sparc64/kernel/irq.c +++ b/arch/sparc64/kernel/irq.c @@ -877,7 +877,7 @@ void __cpuinit sun4v_register_mondo_queues(int this_cpu) static void __init alloc_one_mondo(unsigned long *pa_ptr, unsigned long qmask) { unsigned long size = PAGE_ALIGN(qmask + 1); - void *p = __alloc_bootmem_low(size, size, 0); + void *p = __alloc_bootmem(size, size, 0); if (!p) { prom_printf("SUN4V: Error, cannot allocate mondo queue.\n"); prom_halt(); @@ -889,7 +889,7 @@ static void __init alloc_one_mondo(unsigned long *pa_ptr, unsigned long qmask) static void __init alloc_one_kbuf(unsigned long *pa_ptr, unsigned long qmask) { unsigned long size = PAGE_ALIGN(qmask + 1); - void *p = __alloc_bootmem_low(size, size, 0); + void *p = __alloc_bootmem(size, size, 0); if (!p) { prom_printf("SUN4V: Error, cannot allocate kbuf page.\n"); @@ -906,7 +906,7 @@ static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb) BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > (PAGE_SIZE - 64)); - page = alloc_bootmem_low_pages(PAGE_SIZE); + page = alloc_bootmem_pages(PAGE_SIZE); if (!page) { prom_printf("SUN4V: Error, cannot allocate cpu mondo page.\n"); prom_halt(); @@ -953,7 +953,7 @@ void __init init_IRQ(void) kill_prom_timer(); size = sizeof(struct ino_bucket) * NUM_IVECS; - ivector_table = alloc_bootmem_low(size); + ivector_table = alloc_bootmem(size); if (!ivector_table) { prom_printf("Fatal error, cannot allocate ivector_table\n"); prom_halt(); -- cgit v1.2.3-59-g8ed1b From 4209ab098c61b1a9684bb4d3fc287352fc2438c3 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 15 Oct 2007 20:45:32 -0700 Subject: [SPARC64]: Check of_get_property() return in pci_determine_mem_io_space(). If the PCI controller lacks the 'ranges' property nothing is going to work. Noticed by Al Viro. Signed-off-by: David S. Miller --- arch/sparc64/kernel/pci_common.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch') diff --git a/arch/sparc64/kernel/pci_common.c b/arch/sparc64/kernel/pci_common.c index c76bfbb7da08..923e0bcc3bfd 100644 --- a/arch/sparc64/kernel/pci_common.c +++ b/arch/sparc64/kernel/pci_common.c @@ -396,6 +396,13 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) saw_mem = saw_io = 0; pbm_ranges = of_get_property(pbm->prom_node, "ranges", &i); + if (!pbm_ranges) { + prom_printf("PCI: Fatal error, missing PBM ranges property " + " for %s\n", + pbm->name); + prom_halt(); + } + num_pbm_ranges = i / sizeof(*pbm_ranges); for (i = 0; i < num_pbm_ranges; i++) { -- cgit v1.2.3-59-g8ed1b From 5c45708352a040f19caceb683c78bc86aad466f6 Mon Sep 17 00:00:00 2001 From: Stephen Rothwell Date: Wed, 17 Oct 2007 21:17:42 -0700 Subject: [SPARC/64]: Consolidate of_register_driver Also of_unregister_driver. These will be shortly also used by the PowerPC code. Signed-off-by: Stephen Rothwell Signed-off-by: David S. Miller --- arch/sparc/kernel/of_device.c | 20 -------------------- arch/sparc64/kernel/of_device.c | 20 -------------------- drivers/of/platform.c | 21 +++++++++++++++++++++ include/asm-sparc/of_platform.h | 5 +---- include/asm-sparc64/of_platform.h | 5 +---- include/linux/of_platform.h | 4 ++++ 6 files changed, 27 insertions(+), 48 deletions(-) (limited to 'arch') diff --git a/arch/sparc/kernel/of_device.c b/arch/sparc/kernel/of_device.c index fb2caef79cec..3ea000d15e3a 100644 --- a/arch/sparc/kernel/of_device.c +++ b/arch/sparc/kernel/of_device.c @@ -585,24 +585,6 @@ static int __init of_debug(char *str) __setup("of_debug=", of_debug); -int of_register_driver(struct of_platform_driver *drv, struct bus_type *bus) -{ - /* initialize common driver fields */ - if (!drv->driver.name) - drv->driver.name = drv->name; - if (!drv->driver.owner) - drv->driver.owner = drv->owner; - drv->driver.bus = bus; - - /* register with core */ - return driver_register(&drv->driver); -} - -void of_unregister_driver(struct of_platform_driver *drv) -{ - driver_unregister(&drv->driver); -} - struct of_device* of_platform_device_create(struct device_node *np, const char *bus_id, struct device *parent, @@ -628,6 +610,4 @@ struct of_device* of_platform_device_create(struct device_node *np, return dev; } -EXPORT_SYMBOL(of_register_driver); -EXPORT_SYMBOL(of_unregister_driver); EXPORT_SYMBOL(of_platform_device_create); diff --git a/arch/sparc64/kernel/of_device.c b/arch/sparc64/kernel/of_device.c index 42d779866fba..fc5c0cc793b8 100644 --- a/arch/sparc64/kernel/of_device.c +++ b/arch/sparc64/kernel/of_device.c @@ -869,26 +869,6 @@ static int __init of_debug(char *str) __setup("of_debug=", of_debug); -int of_register_driver(struct of_platform_driver *drv, struct bus_type *bus) -{ - /* initialize common driver fields */ - if (!drv->driver.name) - drv->driver.name = drv->name; - if (!drv->driver.owner) - drv->driver.owner = drv->owner; - drv->driver.bus = bus; - - /* register with core */ - return driver_register(&drv->driver); -} -EXPORT_SYMBOL(of_register_driver); - -void of_unregister_driver(struct of_platform_driver *drv) -{ - driver_unregister(&drv->driver); -} -EXPORT_SYMBOL(of_unregister_driver); - struct of_device* of_platform_device_create(struct device_node *np, const char *bus_id, struct device *parent, diff --git a/drivers/of/platform.c b/drivers/of/platform.c index 864f09fd9f86..b47bb2d7476a 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -12,6 +12,7 @@ * */ #include +#include #include #include #include @@ -94,3 +95,23 @@ int of_bus_type_init(struct bus_type *bus, const char *name) bus->resume = of_platform_device_resume; return bus_register(bus); } + +int of_register_driver(struct of_platform_driver *drv, struct bus_type *bus) +{ + /* initialize common driver fields */ + if (!drv->driver.name) + drv->driver.name = drv->name; + if (!drv->driver.owner) + drv->driver.owner = drv->owner; + drv->driver.bus = bus; + + /* register with core */ + return driver_register(&drv->driver); +} +EXPORT_SYMBOL(of_register_driver); + +void of_unregister_driver(struct of_platform_driver *drv) +{ + driver_unregister(&drv->driver); +} +EXPORT_SYMBOL(of_unregister_driver); diff --git a/include/asm-sparc/of_platform.h b/include/asm-sparc/of_platform.h index 64a230064ef2..d638737ff13c 100644 --- a/include/asm-sparc/of_platform.h +++ b/include/asm-sparc/of_platform.h @@ -18,12 +18,9 @@ extern struct bus_type ebus_bus_type; extern struct bus_type sbus_bus_type; -extern struct bus_type of_platform_bus_type; + #define of_bus_type of_platform_bus_type /* for compatibility */ -extern int of_register_driver(struct of_platform_driver *drv, - struct bus_type *bus); -extern void of_unregister_driver(struct of_platform_driver *drv); extern struct of_device *of_platform_device_create(struct device_node *np, const char *bus_id, struct device *parent, diff --git a/include/asm-sparc64/of_platform.h b/include/asm-sparc64/of_platform.h index f7c1f17c7d52..f15cfa723916 100644 --- a/include/asm-sparc64/of_platform.h +++ b/include/asm-sparc64/of_platform.h @@ -19,12 +19,9 @@ extern struct bus_type isa_bus_type; extern struct bus_type ebus_bus_type; extern struct bus_type sbus_bus_type; -extern struct bus_type of_platform_bus_type; + #define of_bus_type of_platform_bus_type /* for compatibility */ -extern int of_register_driver(struct of_platform_driver *drv, - struct bus_type *bus); -extern void of_unregister_driver(struct of_platform_driver *drv); extern struct of_device *of_platform_device_create(struct device_node *np, const char *bus_id, struct device *parent, diff --git a/include/linux/of_platform.h b/include/linux/of_platform.h index 448f70b30a0c..a8efcfeea732 100644 --- a/include/linux/of_platform.h +++ b/include/linux/of_platform.h @@ -48,6 +48,10 @@ struct of_platform_driver #define to_of_platform_driver(drv) \ container_of(drv,struct of_platform_driver, driver) +extern int of_register_driver(struct of_platform_driver *drv, + struct bus_type *bus); +extern void of_unregister_driver(struct of_platform_driver *drv); + #include extern struct of_device *of_find_device_by_node(struct device_node *np); -- cgit v1.2.3-59-g8ed1b