From 23198ddffb6cddb5d5824230af4dd4b46e4046a4 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Wed, 27 Sep 2017 22:38:19 -0700 Subject: sparc32: Add cmpxchg64(). This fixes the build with i40e driver enabled. Reported-by: Guenter Roeck Signed-off-by: David S. Miller --- arch/sparc/lib/atomic32.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 2c373329d5cb..ddacb5aeb424 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -172,6 +172,20 @@ unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new) } EXPORT_SYMBOL(__cmpxchg_u32); +u64 __cmpxchg_u64(u64 *ptr, u64 old, u64 new) +{ + unsigned long flags; + u64 prev; + + spin_lock_irqsave(ATOMIC_HASH(ptr), flags); + if ((prev = *ptr) == old) + *ptr = new; + spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags); + + return prev; +} +EXPORT_SYMBOL(__cmpxchg_u64); + unsigned long __xchg_u32(volatile u32 *ptr, u32 new) { unsigned long flags; -- cgit v1.2.3-59-g8ed1b From 41413a60352cf8a7f0dd4a869d6e5c3e25aa89b2 Mon Sep 17 00:00:00 2001 From: Vijay Kumar Date: Wed, 11 Oct 2017 12:50:02 -0600 Subject: sparc64: Define SPARC default fls function fls will now require a boot time patching on T4 and above. Redefining it under arch/sparc/lib. Signed-off-by: Vijay Kumar Signed-off-by: David S. Miller --- arch/sparc/include/asm/bitops_64.h | 3 +- arch/sparc/lib/Makefile | 1 + arch/sparc/lib/fls.S | 67 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 arch/sparc/lib/fls.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index 2d522402a937..30aea561176f 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -22,9 +22,10 @@ void set_bit(unsigned long nr, volatile unsigned long *addr); void clear_bit(unsigned long nr, volatile unsigned long *addr); void change_bit(unsigned long nr, volatile unsigned long *addr); +int fls(unsigned int word); + #include -#include #include #include diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index a1a2d39ec96e..3b9f5e0c1bcd 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -16,6 +16,7 @@ lib-$(CONFIG_SPARC64) += atomic_64.o lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o lib-$(CONFIG_SPARC64) += multi3.o +lib-$(CONFIG_SPARC64) += fls.o lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o lib-$(CONFIG_SPARC64) += csum_copy.o csum_copy_from_user.o csum_copy_to_user.o diff --git a/arch/sparc/lib/fls.S b/arch/sparc/lib/fls.S new file mode 100644 index 000000000000..06b8d300bcae --- /dev/null +++ b/arch/sparc/lib/fls.S @@ -0,0 +1,67 @@ +/* fls.S: SPARC default fls definition. + * + * SPARC default fls definition, which follows the same algorithm as + * in generic fls(). This function will be boot time patched on T4 + * and onward. + */ + +#include +#include + + .text + .register %g2, #scratch + .register %g3, #scratch +ENTRY(fls) + brz,pn %o0, 6f + mov 0, %o1 + sethi %hi(0xffff0000), %g3 + mov %o0, %g2 + andcc %o0, %g3, %g0 + be,pt %icc, 8f + mov 32, %o1 + sethi %hi(0xff000000), %g3 + andcc %g2, %g3, %g0 + bne,pt %icc, 3f + sethi %hi(0xf0000000), %g3 + sll %o0, 8, %o0 +1: + add %o1, -8, %o1 + sra %o0, 0, %o0 + mov %o0, %g2 +2: + sethi %hi(0xf0000000), %g3 +3: + andcc %g2, %g3, %g0 + bne,pt %icc, 4f + sethi %hi(0xc0000000), %g3 + sll %o0, 4, %o0 + add %o1, -4, %o1 + sra %o0, 0, %o0 + mov %o0, %g2 +4: + andcc %g2, %g3, %g0 + be,a,pt %icc, 7f + sll %o0, 2, %o0 +5: + xnor %g0, %o0, %o0 + srl %o0, 31, %o0 + sub %o1, %o0, %o1 +6: + jmp %o7 + 8 + sra %o1, 0, %o0 +7: + add %o1, -2, %o1 + ba,pt %xcc, 5b + sra %o0, 0, %o0 +8: + sll %o0, 16, %o0 + sethi %hi(0xff000000), %g3 + sra %o0, 0, %o0 + mov %o0, %g2 + andcc %g2, %g3, %g0 + bne,pt %icc, 2b + mov 16, %o1 + ba,pt %xcc, 1b + sll %o0, 8, %o0 +ENDPROC(fls) +EXPORT_SYMBOL(fls) -- cgit v1.2.3-59-g8ed1b From be52bbe3ea1ae2619ca64d67d56e84a9c7e358f9 Mon Sep 17 00:00:00 2001 From: Vijay Kumar Date: Wed, 11 Oct 2017 12:50:03 -0600 Subject: sparc64: Define SPARC default __fls function __fls will now require a boot time patching on T4 and above. Redefining it under arch/sparc/lib. Signed-off-by: Vijay Kumar Signed-off-by: David S. Miller --- arch/sparc/include/asm/bitops_64.h | 2 +- arch/sparc/lib/Makefile | 1 + arch/sparc/lib/fls64.S | 61 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 arch/sparc/lib/fls64.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index 30aea561176f..d7a46e2480c0 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -23,10 +23,10 @@ void clear_bit(unsigned long nr, volatile unsigned long *addr); void change_bit(unsigned long nr, volatile unsigned long *addr); int fls(unsigned int word); +int __fls(unsigned long word); #include -#include #include #ifdef __KERNEL__ diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 3b9f5e0c1bcd..5380c5985420 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -17,6 +17,7 @@ lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o lib-$(CONFIG_SPARC64) += multi3.o lib-$(CONFIG_SPARC64) += fls.o +lib-$(CONFIG_SPARC64) += fls64.o lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o lib-$(CONFIG_SPARC64) += csum_copy.o csum_copy_from_user.o csum_copy_to_user.o diff --git a/arch/sparc/lib/fls64.S b/arch/sparc/lib/fls64.S new file mode 100644 index 000000000000..c83e22ae9586 --- /dev/null +++ b/arch/sparc/lib/fls64.S @@ -0,0 +1,61 @@ +/* fls64.S: SPARC default __fls definition. + * + * SPARC default __fls definition, which follows the same algorithm as + * in generic __fls(). This function will be boot time patched on T4 + * and onward. + */ + +#include +#include + + .text + .register %g2, #scratch + .register %g3, #scratch +ENTRY(__fls) + mov -1, %g2 + sllx %g2, 32, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 1f + mov 63, %g1 + sllx %o0, 32, %o0 + mov 31, %g1 +1: + mov -1, %g2 + sllx %g2, 48, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 2f + mov -1, %g2 + sllx %o0, 16, %o0 + add %g1, -16, %g1 +2: + mov -1, %g2 + sllx %g2, 56, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 3f + mov -1, %g2 + sllx %o0, 8, %o0 + add %g1, -8, %g1 +3: + sllx %g2, 60, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 4f + mov -1, %g2 + sllx %o0, 4, %o0 + add %g1, -4, %g1 +4: + sllx %g2, 62, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 5f + mov -1, %g3 + sllx %o0, 2, %o0 + add %g1, -2, %g1 +5: + mov 0, %g2 + sllx %g3, 63, %g3 + and %o0, %g3, %o0 + movre %o0, 1, %g2 + sub %g1, %g2, %g1 + jmp %o7+8 + sra %g1, 0, %o0 +ENDPROC(__fls) +EXPORT_SYMBOL(__fls) -- cgit v1.2.3-59-g8ed1b From 70cbec0c53739e170ea9c418e740f007ab5bca52 Mon Sep 17 00:00:00 2001 From: Vijay Kumar Date: Wed, 11 Oct 2017 12:50:04 -0600 Subject: sparc64: SPARC optimized fls function Defined SPARC optimized fls using lzcnt opcode. Signed-off-by: Vijay Kumar Signed-off-by: David S. Miller --- arch/sparc/lib/Makefile | 1 + arch/sparc/lib/NG4fls.S | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 arch/sparc/lib/NG4fls.S (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 5380c5985420..2823b8e530ed 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -18,6 +18,7 @@ lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o lib-$(CONFIG_SPARC64) += multi3.o lib-$(CONFIG_SPARC64) += fls.o lib-$(CONFIG_SPARC64) += fls64.o +obj-$(CONFIG_SPARC64) += NG4fls.o lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o lib-$(CONFIG_SPARC64) += csum_copy.o csum_copy_from_user.o csum_copy_to_user.o diff --git a/arch/sparc/lib/NG4fls.S b/arch/sparc/lib/NG4fls.S new file mode 100644 index 000000000000..bc17b65f1052 --- /dev/null +++ b/arch/sparc/lib/NG4fls.S @@ -0,0 +1,20 @@ +/* NG4fls.S: SPARC optimized fls and __fls for T4 and above. + * + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + */ + +#include + +#define LZCNT_O0_G2 \ + .word 0x85b002e8 + + .text + .register %g2, #scratch + .register %g3, #scratch + +ENTRY(NG4fls) + LZCNT_O0_G2 !lzcnt %o0, %g2 + mov 64, %g3 + retl + sub %g3, %g2, %o0 +ENDPROC(NG4fls) -- cgit v1.2.3-59-g8ed1b From 2b41ce5df20b698436526fcd8478094140b91706 Mon Sep 17 00:00:00 2001 From: Vijay Kumar Date: Wed, 11 Oct 2017 12:50:05 -0600 Subject: sparc64: SPARC optimized __fls function Defined SPARC optimized __fls using lzcnt opcode. Signed-off-by: Vijay Kumar Signed-off-by: David S. Miller --- arch/sparc/lib/NG4fls.S | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'arch/sparc/lib') diff --git a/arch/sparc/lib/NG4fls.S b/arch/sparc/lib/NG4fls.S index bc17b65f1052..2d0991e5b034 100644 --- a/arch/sparc/lib/NG4fls.S +++ b/arch/sparc/lib/NG4fls.S @@ -18,3 +18,13 @@ ENTRY(NG4fls) retl sub %g3, %g2, %o0 ENDPROC(NG4fls) + +ENTRY(__NG4fls) + brz,pn %o0, 1f + LZCNT_O0_G2 !lzcnt %o0, %g2 + mov 63, %g3 + sub %g3, %g2, %o0 +1: + retl + nop +ENDPROC(__NG4fls) -- cgit v1.2.3-59-g8ed1b From 46ad8d2d22c17e2b577adec55ae87161666a3267 Mon Sep 17 00:00:00 2001 From: Vijay Kumar Date: Wed, 11 Oct 2017 12:50:06 -0600 Subject: sparc64: Use sparc optimized fls and __fls for T4 and above For T4 and above, patch fls and __fls functions at the boot time to use lzcnt instruction. Signed-off-by: Vijay Kumar Signed-off-by: David S. Miller --- arch/sparc/kernel/head_64.S | 2 ++ arch/sparc/lib/NG4patch.S | 9 +++++++++ 2 files changed, 11 insertions(+) (limited to 'arch/sparc/lib') diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S index 4de9fbd1a177..f362ecb9955d 100644 --- a/arch/sparc/kernel/head_64.S +++ b/arch/sparc/kernel/head_64.S @@ -640,6 +640,8 @@ niagara4_patch: nop call niagara4_patch_pageops nop + call niagara4_patch_fls + nop ba,a,pt %xcc, 80f nop diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S index 3cc0f8cc95df..da65a3ebb7cc 100644 --- a/arch/sparc/lib/NG4patch.S +++ b/arch/sparc/lib/NG4patch.S @@ -3,6 +3,8 @@ * Copyright (C) 2012 David S. Miller */ +#include + #define BRANCH_ALWAYS 0x10680000 #define NOP 0x01000000 #define NG_DO_PATCH(OLD, NEW) \ @@ -52,3 +54,10 @@ niagara4_patch_pageops: retl nop .size niagara4_patch_pageops,.-niagara4_patch_pageops + +ENTRY(niagara4_patch_fls) + NG_DO_PATCH(fls, NG4fls) + NG_DO_PATCH(__fls, __NG4fls) + retl + nop +ENDPROC(niagara4_patch_fls) -- cgit v1.2.3-59-g8ed1b