diff options
author | Jason A. Donenfeld <Jason@zx2c4.com> | 2018-08-28 23:50:35 -0600 |
---|---|---|
committer | Jason A. Donenfeld <Jason@zx2c4.com> | 2018-09-03 23:52:11 -0600 |
commit | 4a0e319af86c0d38304535293f6fc32fe436ef1d (patch) | |
tree | 6fca1e89becf3ff1afdcec7b6bc725e256af5811 /src/crypto/zinc/poly1305/poly1305.c | |
parent | uapi: reformat (diff) | |
download | wireguard-monolithic-historical-4a0e319af86c0d38304535293f6fc32fe436ef1d.tar.xz wireguard-monolithic-historical-4a0e319af86c0d38304535293f6fc32fe436ef1d.zip |
crypto: import zinc
Diffstat (limited to '')
-rw-r--r-- | src/crypto/zinc/poly1305/poly1305.c (renamed from src/crypto/poly1305.c) | 192 |
1 files changed, 52 insertions, 140 deletions
diff --git a/src/crypto/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c index d35154a..4b90523 100644 --- a/src/crypto/poly1305.c +++ b/src/crypto/zinc/poly1305/poly1305.c @@ -2,82 +2,42 @@ * * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Implementation of the Poly1305 message authenticator. + * + * Information: https://cr.yp.to/mac.html */ -#include "poly1305.h" -#include "simd.h" +#include <zinc/poly1305.h> #include <asm/unaligned.h> #include <linux/kernel.h> #include <linux/string.h> -#if defined(CONFIG_X86_64) -#include <asm/fpu/api.h> -#include <asm/cpufeature.h> -#include <asm/processor.h> -#include <asm/intel-family.h> -asmlinkage void poly1305_init_x86_64(void *ctx, const u8 key[POLY1305_KEY_SIZE]); -asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4]); -#ifdef CONFIG_AS_AVX -asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4]); -asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -#endif -#ifdef CONFIG_AS_AVX2 -asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -#endif -#ifdef CONFIG_AS_AVX512 -asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -#endif - -static bool poly1305_use_avx __ro_after_init; -static bool poly1305_use_avx2 __ro_after_init; -static bool poly1305_use_avx512 __ro_after_init; - -void __init poly1305_fpu_init(void) +#ifndef HAVE_POLY1305_ARCH_IMPLEMENTATION +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) { -#ifndef CONFIG_UML - poly1305_use_avx = boot_cpu_has(X86_FEATURE_AVX) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); - poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); -#ifndef COMPAT_CANNOT_USE_AVX512 - poly1305_use_avx512 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && - boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; -#endif -#endif + return false; +} +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + const size_t len, const u32 padbit, + simd_context_t simd_context) +{ + return false; +} +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t simd_context) +{ + return false; } -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]); -asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]); -#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (!defined(__LINUX_ARM_ARCH__) || __LINUX_ARM_ARCH__ >= 7) -#define ARM_USE_NEON -#include <asm/hwcap.h> -#include <asm/neon.h> -asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); -#endif -static bool poly1305_use_neon __ro_after_init; void __init poly1305_fpu_init(void) { -#if defined(CONFIG_ARM64) - poly1305_use_neon = elf_hwcap & HWCAP_ASIMD; -#elif defined(CONFIG_ARM) - poly1305_use_neon = elf_hwcap & HWCAP_NEON; -#endif } -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) -asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); -asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); -void __init poly1305_fpu_init(void) { } -#else -void __init poly1305_fpu_init(void) { } #endif -#if !(defined(CONFIG_X86_64) || defined(CONFIG_ARM) || defined(CONFIG_ARM64) || (defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)))) struct poly1305_internal { u32 h[5]; u32 r[4]; @@ -101,9 +61,11 @@ static void poly1305_init_generic(void *ctx, const u8 key[16]) st->r[3] = get_unaligned_le32(&key[12]) & 0x0ffffffc; } -static void poly1305_blocks_generic(void *ctx, const u8 *inp, size_t len, const u32 padbit) +static void poly1305_blocks_generic(void *ctx, const u8 *inp, size_t len, + const u32 padbit) { -#define CONSTANT_TIME_CARRY(a,b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) +#define CONSTANT_TIME_CARRY(a, b) \ + ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) struct poly1305_internal *st = (struct poly1305_internal *)ctx; u32 r0, r1, r2, r3; u32 s1, s2, s3; @@ -236,94 +198,39 @@ static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) put_unaligned_le32(h2, &mac[ 8]); put_unaligned_le32(h3, &mac[12]); } -#endif -void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE], simd_context_t simd_context) +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) { ctx->nonce[0] = get_unaligned_le32(&key[16]); ctx->nonce[1] = get_unaligned_le32(&key[20]); ctx->nonce[2] = get_unaligned_le32(&key[24]); ctx->nonce[3] = get_unaligned_le32(&key[28]); -#if defined(CONFIG_X86_64) - poly1305_init_x86_64(ctx->opaque, key); -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) - poly1305_init_arm(ctx->opaque, key); -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) - poly1305_init_mips(ctx->opaque, key); -#else - poly1305_init_generic(ctx->opaque, key); -#endif + if (!poly1305_init_arch(ctx->opaque, key, simd_context)) + poly1305_init_generic(ctx->opaque, key); ctx->num = 0; } +EXPORT_SYMBOL(poly1305_init); -static inline void poly1305_blocks(void *ctx, const u8 *inp, const size_t len, const u32 padbit, simd_context_t simd_context) +static inline void poly1305_blocks(void *ctx, const u8 *inp, const size_t len, + const u32 padbit, + simd_context_t simd_context) { -#if defined(CONFIG_X86_64) -#ifdef CONFIG_AS_AVX512 - if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_avx512(ctx, inp, len, padbit); - else -#endif -#ifdef CONFIG_AS_AVX2 - if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_avx2(ctx, inp, len, padbit); - else -#endif -#ifdef CONFIG_AS_AVX - if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_avx(ctx, inp, len, padbit); - else -#endif - poly1305_blocks_x86_64(ctx, inp, len, padbit); -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -#if defined(ARM_USE_NEON) - if (poly1305_use_neon && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_neon(ctx, inp, len, padbit); - else -#endif - poly1305_blocks_arm(ctx, inp, len, padbit); -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) - poly1305_blocks_mips(ctx, inp, len, padbit); -#else - poly1305_blocks_generic(ctx, inp, len, padbit); -#endif + if (!poly1305_blocks_arch(ctx, inp, len, padbit, simd_context)) + poly1305_blocks_generic(ctx, inp, len, padbit); } -static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], const u32 nonce[4], simd_context_t simd_context) +static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], + const u32 nonce[4], + simd_context_t simd_context) { -#if defined(CONFIG_X86_64) -#ifdef CONFIG_AS_AVX512 - if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif -#ifdef CONFIG_AS_AVX2 - if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif -#ifdef CONFIG_AS_AVX - if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif - poly1305_emit_x86_64(ctx, mac, nonce); -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -#if defined(ARM_USE_NEON) - if (poly1305_use_neon && simd_context == HAVE_FULL_SIMD) - poly1305_emit_neon(ctx, mac, nonce); - else -#endif - poly1305_emit_arm(ctx, mac, nonce); -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) - poly1305_emit_mips(ctx, mac, nonce); -#else - poly1305_emit_generic(ctx, mac, nonce); -#endif + if (!poly1305_emit_arch(ctx, mac, nonce, simd_context)) + poly1305_emit_generic(ctx, mac, nonce); } -void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_context_t simd_context) +void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, + simd_context_t simd_context) { const size_t num = ctx->num % POLY1305_BLOCK_SIZE; size_t rem; @@ -332,7 +239,8 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_c rem = POLY1305_BLOCK_SIZE - num; if (len >= rem) { memcpy(ctx->data + num, inp, rem); - poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1, simd_context); + poly1305_blocks(ctx->opaque, ctx->data, + POLY1305_BLOCK_SIZE, 1, simd_context); inp += rem; len -= rem; } else { @@ -356,16 +264,19 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_c ctx->num = rem; } +EXPORT_SYMBOL(poly1305_update); -void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_context_t simd_context) +void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t simd_context) { size_t num = ctx->num % POLY1305_BLOCK_SIZE; if (num) { - ctx->data[num++] = 1; /* pad bit */ + ctx->data[num++] = 1; /* pad bit */ while (num < POLY1305_BLOCK_SIZE) ctx->data[num++] = 0; - poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, simd_context); + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, + simd_context); } poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context); @@ -373,5 +284,6 @@ void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_c /* zero out the state */ memzero_explicit(ctx, sizeof(*ctx)); } +EXPORT_SYMBOL(poly1305_finish); #include "../selftest/poly1305.h" |