aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/src/crypto/zinc/poly1305/poly1305-arm-glue.h
diff options
context:
space:
mode:
authorJason A. Donenfeld <Jason@zx2c4.com>2018-09-19 04:42:56 +0200
committerJason A. Donenfeld <Jason@zx2c4.com>2018-09-20 19:41:22 +0200
commitb161aff379d6efcd593c75a0d17ef724e8daee63 (patch)
treecb24479660537007c39a66b01baba4cb7d72d332 /src/crypto/zinc/poly1305/poly1305-arm-glue.h
parentallowedips: change from BUG_ON to WARN_ON (diff)
downloadwireguard-monolithic-historical-b161aff379d6efcd593c75a0d17ef724e8daee63.tar.xz
wireguard-monolithic-historical-b161aff379d6efcd593c75a0d17ef724e8daee63.zip
poly1305: account for simd being toggled off midway
This is a very rare occurance, but we should account for it, so that the calculations aren't wrong. Here we convert from base 2^26 back to base 2^64.
Diffstat (limited to 'src/crypto/zinc/poly1305/poly1305-arm-glue.h')
-rw-r--r--src/crypto/zinc/poly1305/poly1305-arm-glue.h56
1 files changed, 56 insertions, 0 deletions
diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h
index dd3fa5a..8a3daf8 100644
--- a/src/crypto/zinc/poly1305/poly1305-arm-glue.h
+++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h
@@ -29,6 +29,58 @@ static void __init poly1305_fpu_init(void)
#endif
}
+#if defined(CONFIG_ARM64)
+struct poly1305_arch_internal {
+ union {
+ u32 h[5];
+ struct {
+ u64 h0, h1, h2;
+ };
+ };
+ u32 is_base2_26;
+ u64 r[2];
+};
+#elif defined(CONFIG_ARM)
+struct poly1305_arch_internal {
+ union {
+ u32 h[5];
+ struct {
+ u64 h0, h1;
+ u32 h2;
+ } __packed;
+ };
+ u32 r[4];
+ u32 is_base2_26;
+};
+#endif
+
+#if defined(ARM_USE_NEON)
+static void convert_to_base2_64(void *ctx)
+{
+ struct poly1305_arch_internal *state = ctx;
+ u32 cy;
+
+ if (!state->is_base2_26)
+ return;
+
+ cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
+ cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
+ cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
+ cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
+ state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
+ state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
+ state->h2 = state->h[4] >> 24;
+#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
+ cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
+ state->h2 &= 3;
+ state->h0 += cy;
+ state->h1 += (cy = ULT(state->h0, cy));
+ state->h2 += ULT(state->h1, cy);
+#undef ULT
+ state->is_base2_26 = 0;
+}
+#endif
+
static inline bool poly1305_init_arch(void *ctx,
const u8 key[POLY1305_KEY_SIZE])
{
@@ -45,7 +97,9 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
poly1305_blocks_neon(ctx, inp, len, padbit);
return true;
}
+ convert_to_base2_64(ctx);
#endif
+
poly1305_blocks_arm(ctx, inp, len, padbit);
return true;
}
@@ -59,7 +113,9 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
poly1305_emit_neon(ctx, mac, nonce);
return true;
}
+ convert_to_base2_64(ctx);
#endif
+
poly1305_emit_arm(ctx, mac, nonce);
return true;
}