From b57de923d8c725ef8d55e7ad458f1f456f98adbf Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 2 Jun 2019 19:45:08 +0200 Subject: curve25519: not all linkers support bmi2 and adx --- .../zinc/curve25519/curve25519-x86_64-glue.c | 14 ++++---- src/crypto/zinc/curve25519/curve25519-x86_64.c | 40 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) (limited to 'src/crypto/zinc/curve25519') diff --git a/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c b/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c index 19c86c6..d62bd37 100644 --- a/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c +++ b/src/crypto/zinc/curve25519/curve25519-x86_64-glue.c @@ -15,8 +15,10 @@ static bool *const curve25519_nobs[] __initconst = { static void __init curve25519_fpu_init(void) { - curve25519_use_bmi2 = boot_cpu_has(X86_FEATURE_BMI2); - curve25519_use_adx = boot_cpu_has(X86_FEATURE_BMI2) && + curve25519_use_bmi2 = IS_ENABLED(CONFIG_AS_BMI2) && + boot_cpu_has(X86_FEATURE_BMI2); + curve25519_use_adx = IS_ENABLED(CONFIG_AS_ADX) && + boot_cpu_has(X86_FEATURE_BMI2) && boot_cpu_has(X86_FEATURE_ADX); } @@ -24,10 +26,10 @@ static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], const u8 basepoint[CURVE25519_KEY_SIZE]) { - if (curve25519_use_adx) { + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { curve25519_adx(mypublic, secret, basepoint); return true; - } else if (curve25519_use_bmi2) { + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { curve25519_bmi2(mypublic, secret, basepoint); return true; } @@ -37,10 +39,10 @@ static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]) { - if (curve25519_use_adx) { + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { curve25519_adx_base(pub, secret); return true; - } else if (curve25519_use_bmi2) { + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { curve25519_bmi2_base(pub, secret); return true; } diff --git a/src/crypto/zinc/curve25519/curve25519-x86_64.c b/src/crypto/zinc/curve25519/curve25519-x86_64.c index 3d1806f..e8af3e2 100644 --- a/src/crypto/zinc/curve25519/curve25519-x86_64.c +++ b/src/crypto/zinc/curve25519/curve25519-x86_64.c @@ -582,6 +582,7 @@ __aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = { 0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL }; +#ifdef CONFIG_AS_ADX /* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7] * a is two 256-bit integers: a0[0:3] and a1[4:7] * b is two 256-bit integers: b0[0:3] and b1[4:7] @@ -736,7 +737,9 @@ static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a, : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r14", "%r15"); } +#endif +#ifdef CONFIG_AS_BMI2 static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, const u64 *const b) { @@ -885,7 +888,9 @@ static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r15"); } +#endif +#ifdef CONFIG_AS_ADX static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) { asm volatile( @@ -1010,7 +1015,9 @@ static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r14", "%r15"); } +#endif +#ifdef CONFIG_AS_BMI2 static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) { asm volatile( @@ -1136,7 +1143,9 @@ static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r14", "%r15"); } +#endif +#ifdef CONFIG_AS_ADX static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) { asm volatile( @@ -1201,7 +1210,9 @@ static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11"); } +#endif +#ifdef CONFIG_AS_BMI2 static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) { asm volatile( @@ -1264,7 +1275,9 @@ static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11"); } +#endif +#ifdef CONFIG_AS_ADX static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, const u64 *const b) { @@ -1354,7 +1367,9 @@ static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, : "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r14", "%r15"); } +#endif +#ifdef CONFIG_AS_BMI2 static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, const u64 *const b) { @@ -1433,7 +1448,9 @@ static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r15"); } +#endif +#ifdef CONFIG_AS_ADX static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) { asm volatile( @@ -1499,7 +1516,9 @@ static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r14", "%r15"); } +#endif +#ifdef CONFIG_AS_BMI2 static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) { asm volatile( @@ -1566,7 +1585,9 @@ static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13", "%r14", "%r15"); } +#endif +#ifdef CONFIG_AS_ADX static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) { asm volatile( @@ -1603,7 +1624,9 @@ static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11"); } +#endif +#ifdef CONFIG_AS_BMI2 static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) { asm volatile( @@ -1639,7 +1662,9 @@ static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11"); } +#endif +#ifdef CONFIG_AS_ADX static __always_inline void add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) { @@ -1671,7 +1696,9 @@ add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) : "r"(c), "r"(a), "r"(b) : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); } +#endif +#ifdef CONFIG_AS_BMI2 static __always_inline void add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) { @@ -1702,6 +1729,7 @@ add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) : "r"(c), "r"(a), "r"(b) : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); } +#endif static __always_inline void sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b) @@ -1769,6 +1797,7 @@ mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a) "%r11"); } +#ifdef CONFIG_AS_ADX static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) { struct { @@ -1815,7 +1844,9 @@ static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) memzero_explicit(&m, sizeof(m)); } +#endif +#ifdef CONFIG_AS_BMI2 static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) { struct { @@ -1862,6 +1893,7 @@ static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) memzero_explicit(&m, sizeof(m)); } +#endif /* Given c, a 256-bit number, fred_eltfp25519_1w updates c * with a number such that 0 <= C < 2**255-19. @@ -1939,6 +1971,7 @@ static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], const u8 private_key[CURVE25519_KEY_SIZE], const u8 session_key[CURVE25519_KEY_SIZE]) { +#ifdef CONFIG_AS_ADX struct { u64 buffer[4 * NUM_WORDS_ELTFP25519]; u64 coordinates[4 * NUM_WORDS_ELTFP25519]; @@ -2034,11 +2067,13 @@ static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], fred_eltfp25519_1w((u64 *)shared); memzero_explicit(&m, sizeof(m)); +#endif } static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], const u8 private_key[CURVE25519_KEY_SIZE]) { +#ifdef CONFIG_AS_ADX struct { u64 buffer[4 * NUM_WORDS_ELTFP25519]; u64 coordinates[4 * NUM_WORDS_ELTFP25519]; @@ -2128,12 +2163,14 @@ static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], fred_eltfp25519_1w((u64 *)session_key); memzero_explicit(&m, sizeof(m)); +#endif } static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], const u8 private_key[CURVE25519_KEY_SIZE], const u8 session_key[CURVE25519_KEY_SIZE]) { +#ifdef CONFIG_AS_BMI2 struct { u64 buffer[4 * NUM_WORDS_ELTFP25519]; u64 coordinates[4 * NUM_WORDS_ELTFP25519]; @@ -2229,11 +2266,13 @@ static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], fred_eltfp25519_1w((u64 *)shared); memzero_explicit(&m, sizeof(m)); +#endif } static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], const u8 private_key[CURVE25519_KEY_SIZE]) { +#ifdef CONFIG_AS_BMI2 struct { u64 buffer[4 * NUM_WORDS_ELTFP25519]; u64 coordinates[4 * NUM_WORDS_ELTFP25519]; @@ -2323,4 +2362,5 @@ static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], fred_eltfp25519_1w((u64 *)session_key); memzero_explicit(&m, sizeof(m)); +#endif } -- cgit v1.2.3-59-g8ed1b