diff options
Diffstat (limited to '')
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-arm-glue.h | 69 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-arm.S (renamed from src/crypto/poly1305-arm.S) | 2 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-arm64.S (renamed from src/crypto/poly1305-arm64.S) | 2 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-mips-glue.h | 40 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-mips.S (renamed from src/crypto/poly1305-mips.S) | 0 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-mips64.S (renamed from src/crypto/poly1305-mips64.S) | 0 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-x86_64-glue.h | 111 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305-x86_64.S (renamed from src/crypto/poly1305-x86_64.S) | 2 | ||||
| -rw-r--r-- | src/crypto/zinc/poly1305/poly1305.c (renamed from src/crypto/poly1305.c) | 192 |
9 files changed, 278 insertions, 140 deletions
diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h new file mode 100644 index 0000000..53f8fec --- /dev/null +++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include <zinc/poly1305.h> +#include <asm/hwcap.h> +#include <asm/neon.h> + +asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]); +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && \ + (defined(CONFIG_64BIT) || __LINUX_ARM_ARCH__ >= 7) +#define ARM_USE_NEON +asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); +#endif + +static bool poly1305_use_neon __ro_after_init; + +void __init poly1305_fpu_init(void) +{ +#if defined(CONFIG_ARM64) + poly1305_use_neon = elf_hwcap & HWCAP_ASIMD; +#elif defined(CONFIG_ARM) + poly1305_use_neon = elf_hwcap & HWCAP_NEON; +#endif +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) +{ + poly1305_init_arm(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + const size_t len, const u32 padbit, + simd_context_t simd_context) +{ +#if defined(ARM_USE_NEON) + if (simd_context == HAVE_FULL_SIMD && poly1305_use_neon) { + poly1305_blocks_neon(ctx, inp, len, padbit); + return true; + } +#endif + poly1305_blocks_arm(ctx, inp, len, padbit); + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t simd_context) +{ +#if defined(ARM_USE_NEON) + if (simd_context == HAVE_FULL_SIMD && poly1305_use_neon) { + poly1305_emit_neon(ctx, mac, nonce); + return true; + } +#endif + poly1305_emit_arm(ctx, mac, nonce); + return true; +} + +#define HAVE_POLY1305_ARCH_IMPLEMENTATION diff --git a/src/crypto/poly1305-arm.S b/src/crypto/zinc/poly1305/poly1305-arm.S index d6b9a80..277a11a 100644 --- a/src/crypto/poly1305-arm.S +++ b/src/crypto/zinc/poly1305/poly1305-arm.S @@ -2,6 +2,8 @@ * * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * This is based in part on Andy Polyakov's implementation from OpenSSL. */ #include <linux/linkage.h> diff --git a/src/crypto/poly1305-arm64.S b/src/crypto/zinc/poly1305/poly1305-arm64.S index 911b57e..706e664 100644 --- a/src/crypto/poly1305-arm64.S +++ b/src/crypto/zinc/poly1305/poly1305-arm64.S @@ -2,6 +2,8 @@ * * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * This is based in part on Andy Polyakov's implementation from OpenSSL. */ #include <linux/linkage.h> diff --git a/src/crypto/zinc/poly1305/poly1305-mips-glue.h b/src/crypto/zinc/poly1305/poly1305-mips-glue.h new file mode 100644 index 0000000..e29f859 --- /dev/null +++ b/src/crypto/zinc/poly1305/poly1305-mips-glue.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include <zinc/poly1305.h> + +asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); +void __init poly1305_fpu_init(void) +{ +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) +{ + poly1305_init_mips(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + const size_t len, const u32 padbit, + simd_context_t simd_context) +{ + poly1305_blocks_mips(ctx, inp, len, padbit); + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t simd_context) +{ + poly1305_emit_mips(ctx, mac, nonce); + return true; +} + +#define HAVE_POLY1305_ARCH_IMPLEMENTATION diff --git a/src/crypto/poly1305-mips.S b/src/crypto/zinc/poly1305/poly1305-mips.S index 32d8558..32d8558 100644 --- a/src/crypto/poly1305-mips.S +++ b/src/crypto/zinc/poly1305/poly1305-mips.S diff --git a/src/crypto/poly1305-mips64.S b/src/crypto/zinc/poly1305/poly1305-mips64.S index 1a45fbe..1a45fbe 100644 --- a/src/crypto/poly1305-mips64.S +++ b/src/crypto/zinc/poly1305/poly1305-mips64.S diff --git a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h new file mode 100644 index 0000000..e24bee7 --- /dev/null +++ b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. + */ + +#include <zinc/poly1305.h> +#include <asm/cpufeature.h> +#include <asm/processor.h> +#include <asm/intel-family.h> + +asmlinkage void poly1305_init_x86_64(void *ctx, + const u8 key[POLY1305_KEY_SIZE]); +asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +#ifdef CONFIG_AS_AVX +asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +#endif +#ifdef CONFIG_AS_AVX2 +asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +#endif +#ifdef CONFIG_AS_AVX512 +asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +#endif + +static bool poly1305_use_avx __ro_after_init; +static bool poly1305_use_avx2 __ro_after_init; +static bool poly1305_use_avx512 __ro_after_init; + +void __init poly1305_fpu_init(void) +{ + poly1305_use_avx = + boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); + poly1305_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + poly1305_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; +#endif +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) +{ + poly1305_init_x86_64(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + const size_t len, const u32 padbit, + simd_context_t simd_context) +{ +#ifdef CONFIG_AS_AVX512 + if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD) + poly1305_blocks_avx512(ctx, inp, len, padbit); + else +#endif +#ifdef CONFIG_AS_AVX2 + if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD) + poly1305_blocks_avx2(ctx, inp, len, padbit); + else +#endif +#ifdef CONFIG_AS_AVX + if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD) + poly1305_blocks_avx(ctx, inp, len, padbit); + else +#endif + poly1305_blocks_x86_64(ctx, inp, len, padbit); + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t simd_context) +{ +#ifdef CONFIG_AS_AVX512 + if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD) + poly1305_emit_avx(ctx, mac, nonce); + else +#endif +#ifdef CONFIG_AS_AVX2 + if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD) + poly1305_emit_avx(ctx, mac, nonce); + else +#endif +#ifdef CONFIG_AS_AVX + if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD) + poly1305_emit_avx(ctx, mac, nonce); + else +#endif + poly1305_emit_x86_64(ctx, mac, nonce); + return true; +} + +#define HAVE_POLY1305_ARCH_IMPLEMENTATION diff --git a/src/crypto/poly1305-x86_64.S b/src/crypto/zinc/poly1305/poly1305-x86_64.S index 7c1b736..4d51f40 100644 --- a/src/crypto/poly1305-x86_64.S +++ b/src/crypto/zinc/poly1305/poly1305-x86_64.S @@ -3,6 +3,8 @@ * Copyright (C) 2017 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved. * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * This is based in part on Andy Polyakov's implementation from OpenSSL. */ #include <linux/linkage.h> diff --git a/src/crypto/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c index d35154a..4b90523 100644 --- a/src/crypto/poly1305.c +++ b/src/crypto/zinc/poly1305/poly1305.c @@ -2,82 +2,42 @@ * * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Implementation of the Poly1305 message authenticator. + * + * Information: https://cr.yp.to/mac.html */ -#include "poly1305.h" -#include "simd.h" +#include <zinc/poly1305.h> #include <asm/unaligned.h> #include <linux/kernel.h> #include <linux/string.h> -#if defined(CONFIG_X86_64) -#include <asm/fpu/api.h> -#include <asm/cpufeature.h> -#include <asm/processor.h> -#include <asm/intel-family.h> -asmlinkage void poly1305_init_x86_64(void *ctx, const u8 key[POLY1305_KEY_SIZE]); -asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4]); -#ifdef CONFIG_AS_AVX -asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4]); -asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -#endif -#ifdef CONFIG_AS_AVX2 -asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -#endif -#ifdef CONFIG_AS_AVX512 -asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -#endif - -static bool poly1305_use_avx __ro_after_init; -static bool poly1305_use_avx2 __ro_after_init; -static bool poly1305_use_avx512 __ro_after_init; - -void __init poly1305_fpu_init(void) +#ifndef HAVE_POLY1305_ARCH_IMPLEMENTATION +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) { -#ifndef CONFIG_UML - poly1305_use_avx = boot_cpu_has(X86_FEATURE_AVX) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); - poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); -#ifndef COMPAT_CANNOT_USE_AVX512 - poly1305_use_avx512 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && - boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; -#endif -#endif + return false; +} +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + const size_t len, const u32 padbit, + simd_context_t simd_context) +{ + return false; +} +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t simd_context) +{ + return false; } -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]); -asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]); -#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (!defined(__LINUX_ARM_ARCH__) || __LINUX_ARM_ARCH__ >= 7) -#define ARM_USE_NEON -#include <asm/hwcap.h> -#include <asm/neon.h> -asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); -#endif -static bool poly1305_use_neon __ro_after_init; void __init poly1305_fpu_init(void) { -#if defined(CONFIG_ARM64) - poly1305_use_neon = elf_hwcap & HWCAP_ASIMD; -#elif defined(CONFIG_ARM) - poly1305_use_neon = elf_hwcap & HWCAP_NEON; -#endif } -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) -asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); -asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); -void __init poly1305_fpu_init(void) { } -#else -void __init poly1305_fpu_init(void) { } #endif -#if !(defined(CONFIG_X86_64) || defined(CONFIG_ARM) || defined(CONFIG_ARM64) || (defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)))) struct poly1305_internal { u32 h[5]; u32 r[4]; @@ -101,9 +61,11 @@ static void poly1305_init_generic(void *ctx, const u8 key[16]) st->r[3] = get_unaligned_le32(&key[12]) & 0x0ffffffc; } -static void poly1305_blocks_generic(void *ctx, const u8 *inp, size_t len, const u32 padbit) +static void poly1305_blocks_generic(void *ctx, const u8 *inp, size_t len, + const u32 padbit) { -#define CONSTANT_TIME_CARRY(a,b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) +#define CONSTANT_TIME_CARRY(a, b) \ + ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) struct poly1305_internal *st = (struct poly1305_internal *)ctx; u32 r0, r1, r2, r3; u32 s1, s2, s3; @@ -236,94 +198,39 @@ static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) put_unaligned_le32(h2, &mac[ 8]); put_unaligned_le32(h3, &mac[12]); } -#endif -void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE], simd_context_t simd_context) +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE], + simd_context_t simd_context) { ctx->nonce[0] = get_unaligned_le32(&key[16]); ctx->nonce[1] = get_unaligned_le32(&key[20]); ctx->nonce[2] = get_unaligned_le32(&key[24]); ctx->nonce[3] = get_unaligned_le32(&key[28]); -#if defined(CONFIG_X86_64) - poly1305_init_x86_64(ctx->opaque, key); -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) - poly1305_init_arm(ctx->opaque, key); -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) - poly1305_init_mips(ctx->opaque, key); -#else - poly1305_init_generic(ctx->opaque, key); -#endif + if (!poly1305_init_arch(ctx->opaque, key, simd_context)) + poly1305_init_generic(ctx->opaque, key); ctx->num = 0; } +EXPORT_SYMBOL(poly1305_init); -static inline void poly1305_blocks(void *ctx, const u8 *inp, const size_t len, const u32 padbit, simd_context_t simd_context) +static inline void poly1305_blocks(void *ctx, const u8 *inp, const size_t len, + const u32 padbit, + simd_context_t simd_context) { -#if defined(CONFIG_X86_64) -#ifdef CONFIG_AS_AVX512 - if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_avx512(ctx, inp, len, padbit); - else -#endif -#ifdef CONFIG_AS_AVX2 - if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_avx2(ctx, inp, len, padbit); - else -#endif -#ifdef CONFIG_AS_AVX - if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_avx(ctx, inp, len, padbit); - else -#endif - poly1305_blocks_x86_64(ctx, inp, len, padbit); -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -#if defined(ARM_USE_NEON) - if (poly1305_use_neon && simd_context == HAVE_FULL_SIMD) - poly1305_blocks_neon(ctx, inp, len, padbit); - else -#endif - poly1305_blocks_arm(ctx, inp, len, padbit); -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) - poly1305_blocks_mips(ctx, inp, len, padbit); -#else - poly1305_blocks_generic(ctx, inp, len, padbit); -#endif + if (!poly1305_blocks_arch(ctx, inp, len, padbit, simd_context)) + poly1305_blocks_generic(ctx, inp, len, padbit); } -static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], const u32 nonce[4], simd_context_t simd_context) +static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], + const u32 nonce[4], + simd_context_t simd_context) { -#if defined(CONFIG_X86_64) -#ifdef CONFIG_AS_AVX512 - if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif -#ifdef CONFIG_AS_AVX2 - if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif -#ifdef CONFIG_AS_AVX - if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD) - poly1305_emit_avx(ctx, mac, nonce); - else -#endif - poly1305_emit_x86_64(ctx, mac, nonce); -#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -#if defined(ARM_USE_NEON) - if (poly1305_use_neon && simd_context == HAVE_FULL_SIMD) - poly1305_emit_neon(ctx, mac, nonce); - else -#endif - poly1305_emit_arm(ctx, mac, nonce); -#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2)) - poly1305_emit_mips(ctx, mac, nonce); -#else - poly1305_emit_generic(ctx, mac, nonce); -#endif + if (!poly1305_emit_arch(ctx, mac, nonce, simd_context)) + poly1305_emit_generic(ctx, mac, nonce); } -void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_context_t simd_context) +void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, + simd_context_t simd_context) { const size_t num = ctx->num % POLY1305_BLOCK_SIZE; size_t rem; @@ -332,7 +239,8 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_c rem = POLY1305_BLOCK_SIZE - num; if (len >= rem) { memcpy(ctx->data + num, inp, rem); - poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1, simd_context); + poly1305_blocks(ctx->opaque, ctx->data, + POLY1305_BLOCK_SIZE, 1, simd_context); inp += rem; len -= rem; } else { @@ -356,16 +264,19 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_c ctx->num = rem; } +EXPORT_SYMBOL(poly1305_update); -void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_context_t simd_context) +void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t simd_context) { size_t num = ctx->num % POLY1305_BLOCK_SIZE; if (num) { - ctx->data[num++] = 1; /* pad bit */ + ctx->data[num++] = 1; /* pad bit */ while (num < POLY1305_BLOCK_SIZE) ctx->data[num++] = 0; - poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, simd_context); + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, + simd_context); } poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context); @@ -373,5 +284,6 @@ void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_c /* zero out the state */ memzero_explicit(ctx, sizeof(*ctx)); } +EXPORT_SYMBOL(poly1305_finish); #include "../selftest/poly1305.h" |