crypto: import zinc

9 files changed, 278 insertions, 140 deletions

diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h
new file mode 100644
index 0000000..53f8fec
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <zinc/poly1305.h>
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+
+asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len,
+				    const u32 padbit);
+asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]);
+#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&                                     \
+	(defined(CONFIG_64BIT) || __LINUX_ARM_ARCH__ >= 7)
+#define ARM_USE_NEON
+asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len,
+				     const u32 padbit);
+asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]);
+#endif
+
+static bool poly1305_use_neon __ro_after_init;
+
+void __init poly1305_fpu_init(void)
+{
+#if defined(CONFIG_ARM64)
+	poly1305_use_neon = elf_hwcap & HWCAP_ASIMD;
+#elif defined(CONFIG_ARM)
+	poly1305_use_neon = elf_hwcap & HWCAP_NEON;
+#endif
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE],
+				      simd_context_t simd_context)
+{
+	poly1305_init_arm(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					const size_t len, const u32 padbit,
+					simd_context_t simd_context)
+{
+#if defined(ARM_USE_NEON)
+	if (simd_context == HAVE_FULL_SIMD && poly1305_use_neon) {
+		poly1305_blocks_neon(ctx, inp, len, padbit);
+		return true;
+	}
+#endif
+	poly1305_blocks_arm(ctx, inp, len, padbit);
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t simd_context)
+{
+#if defined(ARM_USE_NEON)
+	if (simd_context == HAVE_FULL_SIMD && poly1305_use_neon) {
+		poly1305_emit_neon(ctx, mac, nonce);
+		return true;
+	}
+#endif
+	poly1305_emit_arm(ctx, mac, nonce);
+	return true;
+}
+
+#define HAVE_POLY1305_ARCH_IMPLEMENTATION
diff --git a/src/crypto/poly1305-arm.S b/src/crypto/zinc/poly1305/poly1305-arm.S
index d6b9a80..277a11a 100644
--- a/src/crypto/poly1305-arm.S
+++ b/src/crypto/zinc/poly1305/poly1305-arm.S
@@ -2,6 +2,8 @@
  *
  * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * This is based in part on Andy Polyakov's implementation from OpenSSL.
  */
 
 #include <linux/linkage.h>
diff --git a/src/crypto/poly1305-arm64.S b/src/crypto/zinc/poly1305/poly1305-arm64.S
index 911b57e..706e664 100644
--- a/src/crypto/poly1305-arm64.S
+++ b/src/crypto/zinc/poly1305/poly1305-arm64.S
@@ -2,6 +2,8 @@
  *
  * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * This is based in part on Andy Polyakov's implementation from OpenSSL.
  */
 
 #include <linux/linkage.h>
diff --git a/src/crypto/zinc/poly1305/poly1305-mips-glue.h b/src/crypto/zinc/poly1305/poly1305-mips-glue.h
new file mode 100644
index 0000000..e29f859
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-mips-glue.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <zinc/poly1305.h>
+
+asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]);
+asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len,
+				     const u32 padbit);
+asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]);
+void __init poly1305_fpu_init(void)
+{
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE],
+				      simd_context_t simd_context)
+{
+	poly1305_init_mips(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					const size_t len, const u32 padbit,
+					simd_context_t simd_context)
+{
+	poly1305_blocks_mips(ctx, inp, len, padbit);
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t simd_context)
+{
+	poly1305_emit_mips(ctx, mac, nonce);
+	return true;
+}
+
+#define HAVE_POLY1305_ARCH_IMPLEMENTATION
diff --git a/src/crypto/poly1305-mips.S b/src/crypto/zinc/poly1305/poly1305-mips.S
index 32d8558..32d8558 100644
--- a/src/crypto/poly1305-mips.S
+++ b/src/crypto/zinc/poly1305/poly1305-mips.S
diff --git a/src/crypto/poly1305-mips64.S b/src/crypto/zinc/poly1305/poly1305-mips64.S
index 1a45fbe..1a45fbe 100644
--- a/src/crypto/poly1305-mips64.S
+++ b/src/crypto/zinc/poly1305/poly1305-mips64.S
diff --git a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h
new file mode 100644
index 0000000..e24bee7
--- /dev/null
+++ b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h
@@ -0,0 +1,111 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
+ */
+
+#include <zinc/poly1305.h>
+#include <asm/cpufeature.h>
+#include <asm/processor.h>
+#include <asm/intel-family.h>
+
+asmlinkage void poly1305_init_x86_64(void *ctx,
+				     const u8 key[POLY1305_KEY_SIZE]);
+asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp,
+				       const size_t len, const u32 padbit);
+asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				     const u32 nonce[4]);
+#ifdef CONFIG_AS_AVX
+asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				  const u32 nonce[4]);
+asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len,
+				    const u32 padbit);
+#endif
+#ifdef CONFIG_AS_AVX2
+asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len,
+				     const u32 padbit);
+#endif
+#ifdef CONFIG_AS_AVX512
+asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp,
+				       const size_t len, const u32 padbit);
+#endif
+
+static bool poly1305_use_avx __ro_after_init;
+static bool poly1305_use_avx2 __ro_after_init;
+static bool poly1305_use_avx512 __ro_after_init;
+
+void __init poly1305_fpu_init(void)
+{
+	poly1305_use_avx =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
+	poly1305_use_avx2 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
+#ifndef COMPAT_CANNOT_USE_AVX512
+	poly1305_use_avx512 =
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX2) &&
+		boot_cpu_has(X86_FEATURE_AVX512F) &&
+		cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
+				  XFEATURE_MASK_AVX512, NULL) &&
+		/* Skylake downclocks unacceptably much when using zmm. */
+		boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X;
+#endif
+}
+
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE],
+				      simd_context_t simd_context)
+{
+	poly1305_init_x86_64(ctx, key);
+	return true;
+}
+
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					const size_t len, const u32 padbit,
+					simd_context_t simd_context)
+{
+#ifdef CONFIG_AS_AVX512
+	if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD)
+		poly1305_blocks_avx512(ctx, inp, len, padbit);
+	else
+#endif
+#ifdef CONFIG_AS_AVX2
+	if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD)
+		poly1305_blocks_avx2(ctx, inp, len, padbit);
+	else
+#endif
+#ifdef CONFIG_AS_AVX
+	if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD)
+		poly1305_blocks_avx(ctx, inp, len, padbit);
+	else
+#endif
+		poly1305_blocks_x86_64(ctx, inp, len, padbit);
+	return true;
+}
+
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t simd_context)
+{
+#ifdef CONFIG_AS_AVX512
+	if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD)
+		poly1305_emit_avx(ctx, mac, nonce);
+	else
+#endif
+#ifdef CONFIG_AS_AVX2
+	if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD)
+		poly1305_emit_avx(ctx, mac, nonce);
+	else
+#endif
+#ifdef CONFIG_AS_AVX
+	if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD)
+		poly1305_emit_avx(ctx, mac, nonce);
+	else
+#endif
+		poly1305_emit_x86_64(ctx, mac, nonce);
+	return true;
+}
+
+#define HAVE_POLY1305_ARCH_IMPLEMENTATION
diff --git a/src/crypto/poly1305-x86_64.S b/src/crypto/zinc/poly1305/poly1305-x86_64.S
index 7c1b736..4d51f40 100644
--- a/src/crypto/poly1305-x86_64.S
+++ b/src/crypto/zinc/poly1305/poly1305-x86_64.S
@@ -3,6 +3,8 @@
  * Copyright (C) 2017 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
  * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * This is based in part on Andy Polyakov's implementation from OpenSSL.
  */
 
 #include <linux/linkage.h>
diff --git a/src/crypto/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c
index d35154a..4b90523 100644
--- a/src/crypto/poly1305.c
+++ b/src/crypto/zinc/poly1305/poly1305.c
@@ -2,82 +2,42 @@
  *
  * Copyright (C) 2015-2018 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Implementation of the Poly1305 message authenticator.
+ *
+ * Information: https://cr.yp.to/mac.html
  */
 
-#include "poly1305.h"
-#include "simd.h"
+#include <zinc/poly1305.h>
 
 #include <asm/unaligned.h>
 #include <linux/kernel.h>
 #include <linux/string.h>
 
-#if defined(CONFIG_X86_64)
-#include <asm/fpu/api.h>
-#include <asm/cpufeature.h>
-#include <asm/processor.h>
-#include <asm/intel-family.h>
-asmlinkage void poly1305_init_x86_64(void *ctx, const u8 key[POLY1305_KEY_SIZE]);
-asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4]);
-#ifdef CONFIG_AS_AVX
-asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], const u32 nonce[4]);
-asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-#endif
-#ifdef CONFIG_AS_AVX2
-asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-#endif
-#ifdef CONFIG_AS_AVX512
-asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-#endif
-
-static bool poly1305_use_avx __ro_after_init;
-static bool poly1305_use_avx2 __ro_after_init;
-static bool poly1305_use_avx512 __ro_after_init;
-
-void __init poly1305_fpu_init(void)
+#ifndef HAVE_POLY1305_ARCH_IMPLEMENTATION
+static inline bool poly1305_init_arch(void *ctx,
+				      const u8 key[POLY1305_KEY_SIZE],
+				      simd_context_t simd_context)
 {
-#ifndef CONFIG_UML
-	poly1305_use_avx = boot_cpu_has(X86_FEATURE_AVX) &&
-			   cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
-	poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) &&
-			    cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
-#ifndef COMPAT_CANNOT_USE_AVX512
-	poly1305_use_avx512 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) &&
-			      cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) &&
-			      boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X;
-#endif
-#endif
+	return false;
+}
+static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
+					const size_t len, const u32 padbit,
+					simd_context_t simd_context)
+{
+	return false;
+}
+static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
+				      const u32 nonce[4],
+				      simd_context_t simd_context)
+{
+	return false;
 }
-#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]);
-asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]);
-#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (!defined(__LINUX_ARM_ARCH__) || __LINUX_ARM_ARCH__ >= 7)
-#define ARM_USE_NEON
-#include <asm/hwcap.h>
-#include <asm/neon.h>
-asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]);
-#endif
-static bool poly1305_use_neon __ro_after_init;
 void __init poly1305_fpu_init(void)
 {
-#if defined(CONFIG_ARM64)
-	poly1305_use_neon = elf_hwcap & HWCAP_ASIMD;
-#elif defined(CONFIG_ARM)
-	poly1305_use_neon = elf_hwcap & HWCAP_NEON;
-#endif
 }
-#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2))
-asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]);
-asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, const u32 padbit);
-asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]);
-void __init poly1305_fpu_init(void) { }
-#else
-void __init poly1305_fpu_init(void) { }
 #endif
 
-#if !(defined(CONFIG_X86_64) || defined(CONFIG_ARM) || defined(CONFIG_ARM64) || (defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2))))
 struct poly1305_internal {
 	u32 h[5];
 	u32 r[4];
@@ -101,9 +61,11 @@ static void poly1305_init_generic(void *ctx, const u8 key[16])
 	st->r[3] = get_unaligned_le32(&key[12]) & 0x0ffffffc;
 }
 
-static void poly1305_blocks_generic(void *ctx, const u8 *inp, size_t len, const u32 padbit)
+static void poly1305_blocks_generic(void *ctx, const u8 *inp, size_t len,
+				    const u32 padbit)
 {
-#define CONSTANT_TIME_CARRY(a,b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
+#define CONSTANT_TIME_CARRY(a, b)                                              \
+	((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
 	struct poly1305_internal *st = (struct poly1305_internal *)ctx;
 	u32 r0, r1, r2, r3;
 	u32 s1, s2, s3;
@@ -236,94 +198,39 @@ static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4])
 	put_unaligned_le32(h2, &mac[ 8]);
 	put_unaligned_le32(h3, &mac[12]);
 }
-#endif
 
-void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE], simd_context_t simd_context)
+void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE],
+		   simd_context_t simd_context)
 {
 	ctx->nonce[0] = get_unaligned_le32(&key[16]);
 	ctx->nonce[1] = get_unaligned_le32(&key[20]);
 	ctx->nonce[2] = get_unaligned_le32(&key[24]);
 	ctx->nonce[3] = get_unaligned_le32(&key[28]);
 
-#if defined(CONFIG_X86_64)
-	poly1305_init_x86_64(ctx->opaque, key);
-#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-	poly1305_init_arm(ctx->opaque, key);
-#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2))
-	poly1305_init_mips(ctx->opaque, key);
-#else
-	poly1305_init_generic(ctx->opaque, key);
-#endif
+	if (!poly1305_init_arch(ctx->opaque, key, simd_context))
+		poly1305_init_generic(ctx->opaque, key);
 	ctx->num = 0;
 }
+EXPORT_SYMBOL(poly1305_init);
 
-static inline void poly1305_blocks(void *ctx, const u8 *inp, const size_t len, const u32 padbit, simd_context_t simd_context)
+static inline void poly1305_blocks(void *ctx, const u8 *inp, const size_t len,
+				   const u32 padbit,
+				   simd_context_t simd_context)
 {
-#if defined(CONFIG_X86_64)
-#ifdef CONFIG_AS_AVX512
-	if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD)
-		poly1305_blocks_avx512(ctx, inp, len, padbit);
-	else
-#endif
-#ifdef CONFIG_AS_AVX2
-	if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD)
-		poly1305_blocks_avx2(ctx, inp, len, padbit);
-	else
-#endif
-#ifdef CONFIG_AS_AVX
-	if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD)
-		poly1305_blocks_avx(ctx, inp, len, padbit);
-	else
-#endif
-		poly1305_blocks_x86_64(ctx, inp, len, padbit);
-#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-#if defined(ARM_USE_NEON)
-	if (poly1305_use_neon && simd_context == HAVE_FULL_SIMD)
-		poly1305_blocks_neon(ctx, inp, len, padbit);
-	else
-#endif
-		poly1305_blocks_arm(ctx, inp, len, padbit);
-#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2))
-	poly1305_blocks_mips(ctx, inp, len, padbit);
-#else
-	poly1305_blocks_generic(ctx, inp, len, padbit);
-#endif
+	if (!poly1305_blocks_arch(ctx, inp, len, padbit, simd_context))
+		poly1305_blocks_generic(ctx, inp, len, padbit);
 }
 
-static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], const u32 nonce[4], simd_context_t simd_context)
+static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE],
+				 const u32 nonce[4],
+				 simd_context_t simd_context)
 {
-#if defined(CONFIG_X86_64)
-#ifdef CONFIG_AS_AVX512
-	if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD)
-		poly1305_emit_avx(ctx, mac, nonce);
-	else
-#endif
-#ifdef CONFIG_AS_AVX2
-	if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD)
-		poly1305_emit_avx(ctx, mac, nonce);
-	else
-#endif
-#ifdef CONFIG_AS_AVX
-	if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD)
-		poly1305_emit_avx(ctx, mac, nonce);
-	else
-#endif
-		poly1305_emit_x86_64(ctx, mac, nonce);
-#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64)
-#if defined(ARM_USE_NEON)
-	if (poly1305_use_neon && simd_context == HAVE_FULL_SIMD)
-		poly1305_emit_neon(ctx, mac, nonce);
-	else
-#endif
-		poly1305_emit_arm(ctx, mac, nonce);
-#elif defined(CONFIG_MIPS) && (defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2))
-	poly1305_emit_mips(ctx, mac, nonce);
-#else
-	poly1305_emit_generic(ctx, mac, nonce);
-#endif
+	if (!poly1305_emit_arch(ctx, mac, nonce, simd_context))
+		poly1305_emit_generic(ctx, mac, nonce);
 }
 
-void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_context_t simd_context)
+void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len,
+		     simd_context_t simd_context)
 {
 	const size_t num = ctx->num % POLY1305_BLOCK_SIZE;
 	size_t rem;
@@ -332,7 +239,8 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_c
 		rem = POLY1305_BLOCK_SIZE - num;
 		if (len >= rem) {
 			memcpy(ctx->data + num, inp, rem);
-			poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1, simd_context);
+			poly1305_blocks(ctx->opaque, ctx->data,
+					POLY1305_BLOCK_SIZE, 1, simd_context);
 			inp += rem;
 			len -= rem;
 		} else {
@@ -356,16 +264,19 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *inp, size_t len, simd_c
 
 	ctx->num = rem;
 }
+EXPORT_SYMBOL(poly1305_update);
 
-void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_context_t simd_context)
+void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE],
+		     simd_context_t simd_context)
 {
 	size_t num = ctx->num % POLY1305_BLOCK_SIZE;
 
 	if (num) {
-		ctx->data[num++] = 1;   /* pad bit */
+		ctx->data[num++] = 1; /* pad bit */
 		while (num < POLY1305_BLOCK_SIZE)
 			ctx->data[num++] = 0;
-		poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, simd_context);
+		poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0,
+				simd_context);
 	}
 
 	poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context);
@@ -373,5 +284,6 @@ void poly1305_finish(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], simd_c
 	/* zero out the state */
 	memzero_explicit(ctx, sizeof(*ctx));
 }
+EXPORT_SYMBOL(poly1305_finish);
 
 #include "../selftest/poly1305.h"