From b161aff379d6efcd593c75a0d17ef724e8daee63 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 19 Sep 2018 04:42:56 +0200 Subject: poly1305: account for simd being toggled off midway This is a very rare occurance, but we should account for it, so that the calculations aren't wrong. Here we convert from base 2^26 back to base 2^64. --- src/crypto/zinc/poly1305/poly1305-arm-glue.h | 56 +++++++++++++++++ src/crypto/zinc/poly1305/poly1305-x86_64-glue.h | 84 +++++++++++++++++-------- src/crypto/zinc/selftest/poly1305.h | 17 +++++ 3 files changed, 131 insertions(+), 26 deletions(-) (limited to 'src/crypto/zinc') diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h index dd3fa5a..8a3daf8 100644 --- a/src/crypto/zinc/poly1305/poly1305-arm-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h @@ -29,6 +29,58 @@ static void __init poly1305_fpu_init(void) #endif } +#if defined(CONFIG_ARM64) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1, h2; + }; + }; + u32 is_base2_26; + u64 r[2]; +}; +#elif defined(CONFIG_ARM) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1; + u32 h2; + } __packed; + }; + u32 r[4]; + u32 is_base2_26; +}; +#endif + +#if defined(ARM_USE_NEON) +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->h2 = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->h2 >> 2) + (state->h2 & ~3ULL); + state->h2 &= 3; + state->h0 += cy; + state->h1 += (cy = ULT(state->h0, cy)); + state->h2 += ULT(state->h1, cy); +#undef ULT + state->is_base2_26 = 0; +} +#endif + static inline bool poly1305_init_arch(void *ctx, const u8 key[POLY1305_KEY_SIZE]) { @@ -45,7 +97,9 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, poly1305_blocks_neon(ctx, inp, len, padbit); return true; } + convert_to_base2_64(ctx); #endif + poly1305_blocks_arm(ctx, inp, len, padbit); return true; } @@ -59,7 +113,9 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], poly1305_emit_neon(ctx, mac, nonce); return true; } + convert_to_base2_64(ctx); #endif + poly1305_emit_arm(ctx, mac, nonce); return true; } diff --git a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h index d6495cd..f821f34 100644 --- a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h +++ b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h @@ -61,13 +61,43 @@ static inline bool poly1305_init_arch(void *ctx, } struct poly1305_arch_internal { - u32 h[5]; - u32 is_base2_26; + union { + struct { + u32 h[5]; + u32 is_base2_26; + }; + u64 hs[3]; + }; u64 r[2]; u64 pad; struct { u32 r2, r1, r4, r3; } rn[9]; }; +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; +} + static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, const size_t len, const u32 padbit, simd_context_t *simd_context) @@ -77,24 +107,32 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, if (!poly1305_use_avx || (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || !simd_use(simd_context)) - poly1305_blocks_x86_64(ctx, inp, len, padbit); - else + goto scalar; + #ifdef CONFIG_AS_AVX512 - if (poly1305_use_avx512) + if (poly1305_use_avx512) { poly1305_blocks_avx512(ctx, inp, len, padbit); - else + return true; + } #endif + #ifdef CONFIG_AS_AVX2 - if (poly1305_use_avx2) + if (poly1305_use_avx2) { poly1305_blocks_avx2(ctx, inp, len, padbit); - else + return true; + } #endif + #ifdef CONFIG_AS_AVX - if (poly1305_use_avx) + if (poly1305_use_avx) { poly1305_blocks_avx(ctx, inp, len, padbit); - else + return true; + } #endif - poly1305_blocks_x86_64(ctx, inp, len, padbit); + +scalar: + convert_to_base2_64(ctx); + poly1305_blocks_x86_64(ctx, inp, len, padbit); return true; } @@ -105,23 +143,17 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], struct poly1305_arch_internal *state = ctx; if (!poly1305_use_avx || !state->is_base2_26 ||!simd_use(simd_context)) - poly1305_emit_x86_64(ctx, mac, nonce); - else -#ifdef CONFIG_AS_AVX512 - if (poly1305_use_avx512) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif -#ifdef CONFIG_AS_AVX2 - if (poly1305_use_avx2) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif + goto scalar; + #ifdef CONFIG_AS_AVX - if (poly1305_use_avx) + if (poly1305_use_avx || poly1305_use_avx2 || poly1305_use_avx512) { poly1305_emit_avx(ctx, mac, nonce); - else + return true; + } #endif - poly1305_emit_x86_64(ctx, mac, nonce); + +scalar: + convert_to_base2_64(ctx); + poly1305_emit_x86_64(ctx, mac, nonce); return true; } diff --git a/src/crypto/zinc/selftest/poly1305.h b/src/crypto/zinc/selftest/poly1305.h index 1439c98..71bee1b 100644 --- a/src/crypto/zinc/selftest/poly1305.h +++ b/src/crypto/zinc/selftest/poly1305.h @@ -862,6 +862,23 @@ static bool __init poly1305_selftest(void) i + 1, j); success = false; } + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + (simd_context_t []){ HAVE_NO_SIMD }); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_info("poly1305 self-test %zu (split %zu, mixed simd): FAIL\n", + i + 1, j); + success = false; + } simd_relax(&simd_context); } } -- cgit v1.2.3-59-g8ed1b