curve25519: add NEON versions for ARM

1 files changed, 54 insertions, 27 deletions

diff --git a/src/crypto/curve25519.c b/src/crypto/curve25519.c
index 032f275..f0e045e 100644
--- a/src/crypto/curve25519.c
+++ b/src/crypto/curve25519.c
@@ -24,7 +24,7 @@ static __always_inline void normalize_secret(u8 secret[CURVE25519_POINT_SIZE])
 }
 static const u8 null_point[CURVE25519_POINT_SIZE] = { 0 };
 
-#ifdef CONFIG_X86_64
+#if defined(CONFIG_X86_64)
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
 #include <asm/fpu/api.h>
@@ -110,6 +110,16 @@ static void curve25519_sandy2x_base(u8 pub[CURVE25519_POINT_SIZE], const u8 secr
 	memzero_explicit(x_51, sizeof(x_51));
 	memzero_explicit(z_51, sizeof(z_51));
 }
+#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && defined(CONFIG_ARM)
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+asmlinkage void curve25519_asm_neon(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE]);
+static bool curve25519_use_neon __read_mostly = false;
+void curve25519_fpu_init(void)
+{
+	curve25519_use_neon = elf_hwcap & HWCAP_NEON;
+}
 #else
 void curve25519_fpu_init(void) { }
 #endif
@@ -1326,23 +1336,32 @@ static void cmult(limb *resultx, limb *resultz, const u8 *n, const limb *q)
 
 bool curve25519(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE])
 {
-	limb bp[10], x[10], z[11], zmone[10];
-	u8 e[32];
+#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && defined(CONFIG_ARM)
+	if (curve25519_use_neon && may_use_simd()) {
+		kernel_neon_begin();
+		curve25519_asm_neon(mypublic, secret, basepoint);
+		kernel_neon_end();
+	} else
+#endif
+	{
+		limb bp[10], x[10], z[11], zmone[10];
+		u8 e[32];
 
-	memcpy(e, secret, 32);
-	normalize_secret(e);
+		memcpy(e, secret, 32);
+		normalize_secret(e);
 
-	fexpand(bp, basepoint);
-	cmult(x, z, e, bp);
-	crecip(zmone, z);
-	fmul(z, x, zmone);
-	fcontract(mypublic, z);
+		fexpand(bp, basepoint);
+		cmult(x, z, e, bp);
+		crecip(zmone, z);
+		fmul(z, x, zmone);
+		fcontract(mypublic, z);
 
-	memzero_explicit(e, sizeof(e));
-	memzero_explicit(bp, sizeof(bp));
-	memzero_explicit(x, sizeof(x));
-	memzero_explicit(z, sizeof(z));
-	memzero_explicit(zmone, sizeof(zmone));
+		memzero_explicit(e, sizeof(e));
+		memzero_explicit(bp, sizeof(bp));
+		memzero_explicit(x, sizeof(x));
+		memzero_explicit(z, sizeof(z));
+		memzero_explicit(zmone, sizeof(zmone));
+	}
 	return crypto_memneq(mypublic, null_point, CURVE25519_POINT_SIZE);
 }
 #else
@@ -1491,21 +1510,29 @@ static void cmult(struct other_stack *s, limb *resultx, limb *resultz, const u8
 
 bool curve25519(u8 mypublic[CURVE25519_POINT_SIZE], const u8 secret[CURVE25519_POINT_SIZE], const u8 basepoint[CURVE25519_POINT_SIZE])
 {
-	struct other_stack *s = kzalloc(sizeof(struct other_stack), GFP_KERNEL);
-	if (unlikely(!s))
-		return false;
-
-	memcpy(s->ee, secret, 32);
-	normalize_secret(s->ee);
+#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && defined(CONFIG_ARM)
+	if (curve25519_use_neon && may_use_simd()) {
+		kernel_neon_begin();
+		curve25519_asm_neon(mypublic, secret, basepoint);
+		kernel_neon_end();
+	} else
+#endif
+	{
+		struct other_stack *s = kzalloc(sizeof(struct other_stack), GFP_KERNEL);
+		if (unlikely(!s))
+			return false;
 
-	fexpand(s->bp, basepoint);
-	cmult(s, s->x, s->z, s->ee, s->bp);
-	crecip(s->zmone, s->z);
-	fmul(s->z, s->x, s->zmone);
-	fcontract(mypublic, s->z);
+		memcpy(s->ee, secret, 32);
+		normalize_secret(s->ee);
 
-	kzfree(s);
+		fexpand(s->bp, basepoint);
+		cmult(s, s->x, s->z, s->ee, s->bp);
+		crecip(s->zmone, s->z);
+		fmul(s->z, s->x, s->zmone);
+		fcontract(mypublic, s->z);
 
+		kzfree(s);
+	}
 	return crypto_memneq(mypublic, null_point, CURVE25519_POINT_SIZE);
 }
 #endif