Allow overriding ghash_update() with an optimized MD function. Use

this on amd64 to provide a version that uses the PCLMUL instruction
on CPUs that support it but don't have AESNI.  ok mikeb@

5 files changed, 42 insertions, 9 deletions

diff --git a/sys/arch/amd64/amd64/aesni.c b/sys/arch/amd64/amd64/aesni.c
index adf4c3948ae..5693f28ee1f 100644
--- a/sys/arch/amd64/amd64/aesni.c
+++ b/sys/arch/amd64/amd64/aesni.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: aesni.c,v 1.35 2015/08/28 19:59:36 tedu Exp $	*/
+/*	$OpenBSD: aesni.c,v 1.36 2015/11/07 01:37:26 naddy Exp $	*/
 /*-
  * Copyright (c) 2003 Jason Wright
  * Copyright (c) 2003, 2004 Theo de Raadt
@@ -120,6 +120,9 @@ int	aesni_swauth(struct cryptop *, struct cryptodesc *, struct swcr_data *,
 int	aesni_encdec(struct cryptop *, struct cryptodesc *,
 	    struct cryptodesc *, struct aesni_session *);
 
+void	pclmul_setup(void);
+void	ghash_update_pclmul(GHASH_CTX *, uint8_t *, size_t);
+
 void
 aesni_setup(void)
 {
@@ -663,3 +666,17 @@ out:
 	crypto_done(crp);
 	return (err);
 }
+
+void
+pclmul_setup(void)
+{
+	ghash_update = ghash_update_pclmul;
+}
+
+void
+ghash_update_pclmul(GHASH_CTX *ghash, uint8_t *src, size_t len)
+{
+	fpu_kernel_enter();
+	aesni_gmac_update(ghash, src, len);
+	fpu_kernel_exit();
+}
diff --git a/sys/arch/amd64/amd64/autoconf.c b/sys/arch/amd64/amd64/autoconf.c
index 26cb2f3770c..0cb241770c1 100644
--- a/sys/arch/amd64/amd64/autoconf.c
+++ b/sys/arch/amd64/amd64/autoconf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.c,v 1.43 2015/07/17 21:53:56 mlarkin Exp $	*/
+/*	$OpenBSD: autoconf.c,v 1.44 2015/11/07 01:37:26 naddy Exp $	*/
 /*	$NetBSD: autoconf.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $	*/
 
 /*-
@@ -97,6 +97,9 @@ void		rdrand(void *);
 void		viac3_crypto_setup(void);
 extern int	amd64_has_xcrypt;
 
+void		pclmul_setup(void);
+extern int	amd64_has_pclmul;
+
 void		aesni_setup(void);
 extern int	amd64_has_aesni;
 #endif
@@ -146,6 +149,9 @@ cpu_configure(void)
 	if (amd64_has_xcrypt)
 		viac3_crypto_setup();
 
+	if (amd64_has_pclmul)
+		pclmul_setup();
+
 	if (amd64_has_aesni)
 		aesni_setup();
 #endif
diff --git a/sys/arch/amd64/amd64/identcpu.c b/sys/arch/amd64/amd64/identcpu.c
index c0aa9409028..352c3f39beb 100644
--- a/sys/arch/amd64/amd64/identcpu.c
+++ b/sys/arch/amd64/amd64/identcpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: identcpu.c,v 1.64 2015/08/12 05:31:41 mlarkin Exp $	*/
+/*	$OpenBSD: identcpu.c,v 1.65 2015/11/07 01:37:26 naddy Exp $	*/
 /*	$NetBSD: identcpu.c,v 1.1 2003/04/26 18:39:28 fvdl Exp $	*/
 
 /*
@@ -52,6 +52,7 @@ int cpuspeed;
 
 int amd64_has_xcrypt;
 #ifdef CRYPTO
+int amd64_has_pclmul;
 int amd64_has_aesni;
 #endif
 int has_rdrand;
@@ -560,6 +561,9 @@ identifycpu(struct cpu_info *ci)
 			setperf_setup = est_init;
 
 #ifdef CRYPTO
+		if (cpu_ecxfeature & CPUIDECX_PCLMUL)
+			amd64_has_pclmul = 1;
+
 		if (cpu_ecxfeature & CPUIDECX_AES)
 			amd64_has_aesni = 1;
 #endif
diff --git a/sys/crypto/gmac.c b/sys/crypto/gmac.c
index 4dd2019edb2..cff97e50be0 100644
--- a/sys/crypto/gmac.c
+++ b/sys/crypto/gmac.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: gmac.c,v 1.5 2015/11/06 16:43:51 naddy Exp $	*/
+/*	$OpenBSD: gmac.c,v 1.6 2015/11/07 01:37:26 naddy Exp $	*/
 
 /*
  * Copyright (c) 2010 Mike Belopuhov <mike@vantronix.net>
@@ -29,7 +29,10 @@
 #include <crypto/gmac.h>
 
 void	ghash_gfmul(uint32_t *, uint32_t *, uint32_t *);
-void	ghash_update(GHASH_CTX *, uint8_t *, size_t);
+void	ghash_update_mi(GHASH_CTX *, uint8_t *, size_t);
+
+/* Allow overriding with optimized MD function */
+void	(*ghash_update)(GHASH_CTX *, uint8_t *, size_t) = ghash_update_mi;
 
 /* Computes a block multiplication in the GF(2^128) */
 void
@@ -70,7 +73,7 @@ ghash_gfmul(uint32_t *X, uint32_t *Y, uint32_t *product)
 }
 
 void
-ghash_update(GHASH_CTX *ctx, uint8_t *X, size_t len)
+ghash_update_mi(GHASH_CTX *ctx, uint8_t *X, size_t len)
 {
 	uint32_t	*x = (uint32_t *)X;
 	uint32_t	*s = (uint32_t *)ctx->S;
@@ -131,11 +134,12 @@ AES_GMAC_Update(AES_GMAC_CTX *ctx, const uint8_t *data, uint16_t len)
 	if (len > 0) {
 		plen = len % GMAC_BLOCK_LEN;
 		if (len >= GMAC_BLOCK_LEN)
-			ghash_update(&ctx->ghash, (uint8_t *)data, len - plen);
+			(*ghash_update)(&ctx->ghash, (uint8_t *)data,
+			    len - plen);
 		if (plen) {
 			bcopy((uint8_t *)data + (len - plen), (uint8_t *)blk,
 			    plen);
-			ghash_update(&ctx->ghash, (uint8_t *)blk,
+			(*ghash_update)(&ctx->ghash, (uint8_t *)blk,
 			    GMAC_BLOCK_LEN);
 		}
 	}
diff --git a/sys/crypto/gmac.h b/sys/crypto/gmac.h
index 94c1247bbd7..393b21d302d 100644
--- a/sys/crypto/gmac.h
+++ b/sys/crypto/gmac.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: gmac.h,v 1.2 2012/12/05 23:20:15 deraadt Exp $	*/
+/*	$OpenBSD: gmac.h,v 1.3 2015/11/07 01:37:26 naddy Exp $	*/
 
 /*
  * Copyright (c) 2010 Mike Belopuhov <mike@vantronix.net>
@@ -38,6 +38,8 @@ typedef struct _AES_GMAC_CTX {
 } AES_GMAC_CTX;
 
 __BEGIN_DECLS
+extern void (*ghash_update)(GHASH_CTX *, uint8_t *, size_t);
+
 void	AES_GMAC_Init(AES_GMAC_CTX *);
 void	AES_GMAC_Setkey(AES_GMAC_CTX *, const uint8_t *, uint16_t);
 void	AES_GMAC_Reinit(AES_GMAC_CTX *, const uint8_t *, uint16_t);