aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/compat/simd/include/linux/simd.h44
-rw-r--r--src/crypto/include/zinc/chacha20.h4
-rw-r--r--src/crypto/include/zinc/chacha20poly1305.h4
-rw-r--r--src/crypto/include/zinc/poly1305.h4
-rw-r--r--src/crypto/zinc/chacha20/chacha20-arm-glue.h6
-rw-r--r--src/crypto/zinc/chacha20/chacha20-mips-glue.h4
-rw-r--r--src/crypto/zinc/chacha20/chacha20-x86_64-glue.h8
-rw-r--r--src/crypto/zinc/chacha20/chacha20.c8
-rw-r--r--src/crypto/zinc/chacha20poly1305.c39
-rw-r--r--src/crypto/zinc/poly1305/poly1305-arm-glue.h8
-rw-r--r--src/crypto/zinc/poly1305/poly1305-mips-glue.h4
-rw-r--r--src/crypto/zinc/poly1305/poly1305-x86_64-glue.h16
-rw-r--r--src/crypto/zinc/poly1305/poly1305.c12
-rw-r--r--src/crypto/zinc/selftest/chacha20poly1305.h36
-rw-r--r--src/crypto/zinc/selftest/poly1305.h19
-rw-r--r--src/receive.c11
-rw-r--r--src/send.c11
17 files changed, 129 insertions, 109 deletions
diff --git a/src/compat/simd/include/linux/simd.h b/src/compat/simd/include/linux/simd.h
index f31059d..1a0ecae 100644
--- a/src/compat/simd/include/linux/simd.h
+++ b/src/compat/simd/include/linux/simd.h
@@ -17,49 +17,61 @@
#endif
typedef enum {
- HAVE_NO_SIMD,
- HAVE_FULL_SIMD
+ HAVE_NO_SIMD = 1 << 0,
+ HAVE_FULL_SIMD = 1 << 1,
+ HAVE_SIMD_IN_USE = 1 << 31
} simd_context_t;
-static __must_check inline simd_context_t simd_get(void)
+static inline void simd_get(simd_context_t *ctx)
{
bool have_simd = false;
#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) && !defined(CONFIG_PREEMPT_RT_BASE)
have_simd = irq_fpu_usable();
- if (have_simd)
- kernel_fpu_begin();
#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !defined(CONFIG_PREEMPT_RT_BASE)
#if defined(CONFIG_ARM64)
have_simd = true; /* ARM64 supports NEON in any context. */
#elif defined(CONFIG_ARM)
have_simd = may_use_simd(); /* ARM doesn't support NEON in interrupt context. */
#endif
- if (have_simd)
- kernel_neon_begin();
#endif
- return have_simd ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
+ *ctx = have_simd ? HAVE_FULL_SIMD : HAVE_NO_SIMD;
}
-static inline void simd_put(simd_context_t prior_context)
+static inline void simd_put(simd_context_t *ctx)
{
#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) && !defined(CONFIG_PREEMPT_RT_BASE)
- if (prior_context != HAVE_NO_SIMD)
+ if (*ctx & HAVE_SIMD_IN_USE)
kernel_fpu_end();
#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !defined(CONFIG_PREEMPT_RT_BASE)
- if (prior_context != HAVE_NO_SIMD)
+ if (*ctx & HAVE_SIMD_IN_USE)
kernel_neon_end();
#endif
+ *ctx = HAVE_NO_SIMD;
}
-static __must_check inline simd_context_t simd_relax(simd_context_t prior_context)
+static inline void simd_relax(simd_context_t *ctx)
{
#ifdef CONFIG_PREEMPT
- if (prior_context != HAVE_NO_SIMD && need_resched()) {
- simd_put(prior_context);
- return simd_get();
+ if ((*ctx & HAVE_SIMD_IN_USE) && need_resched()) {
+ simd_put(ctx);
+ simd_get(ctx);
}
#endif
- return prior_context;
+}
+
+static __must_check inline bool simd_use(simd_context_t *ctx)
+{
+ if (!(*ctx & HAVE_FULL_SIMD))
+ return false;
+ if (*ctx & HAVE_SIMD_IN_USE)
+ return true;
+#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) && !defined(CONFIG_PREEMPT_RT_BASE)
+ kernel_fpu_begin();
+#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && !defined(CONFIG_PREEMPT_RT_BASE)
+ kernel_neon_begin();
+#endif
+ *ctx |= HAVE_SIMD_IN_USE;
+ return true;
}
#endif /* _WG_SIMD_H */
diff --git a/src/crypto/include/zinc/chacha20.h b/src/crypto/include/zinc/chacha20.h
index 43e67cb..8b3cc8c 100644
--- a/src/crypto/include/zinc/chacha20.h
+++ b/src/crypto/include/zinc/chacha20.h
@@ -44,11 +44,11 @@ static inline void chacha20_init(struct chacha20_ctx *state,
state->counter[3] = nonce >> 32;
}
void chacha20(struct chacha20_ctx *state, u8 *dst, const u8 *src, u32 len,
- simd_context_t simd_context);
+ simd_context_t *simd_context);
/* Derived key should be 32-bit aligned */
void hchacha20(u8 derived_key[CHACHA20_KEY_SIZE],
const u8 nonce[HCHACHA20_NONCE_SIZE],
- const u8 key[HCHACHA20_KEY_SIZE], simd_context_t simd_context);
+ const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context);
#endif /* _ZINC_CHACHA20_H */
diff --git a/src/crypto/include/zinc/chacha20poly1305.h b/src/crypto/include/zinc/chacha20poly1305.h
index 0212685..03979b6 100644
--- a/src/crypto/include/zinc/chacha20poly1305.h
+++ b/src/crypto/include/zinc/chacha20poly1305.h
@@ -25,7 +25,7 @@ void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
bool __must_check chacha20poly1305_encrypt_sg(
struct scatterlist *dst, struct scatterlist *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
- const u8 key[CHACHA20POLY1305_KEYLEN], simd_context_t simd_context);
+ const u8 key[CHACHA20POLY1305_KEYLEN], simd_context_t *simd_context);
bool __must_check
chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
@@ -35,7 +35,7 @@ chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
bool __must_check chacha20poly1305_decrypt_sg(
struct scatterlist *dst, struct scatterlist *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
- const u8 key[CHACHA20POLY1305_KEYLEN], simd_context_t simd_context);
+ const u8 key[CHACHA20POLY1305_KEYLEN], simd_context_t *simd_context);
void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len,
diff --git a/src/crypto/include/zinc/poly1305.h b/src/crypto/include/zinc/poly1305.h
index a7f552c..1cdbadd 100644
--- a/src/crypto/include/zinc/poly1305.h
+++ b/src/crypto/include/zinc/poly1305.h
@@ -26,9 +26,9 @@ void poly1305_fpu_init(void);
void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]);
void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len,
- simd_context_t simd_context);
+ simd_context_t *simd_context);
void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE],
- simd_context_t simd_context);
+ simd_context_t *simd_context);
#ifdef DEBUG
bool poly1305_selftest(void);
diff --git a/src/crypto/zinc/chacha20/chacha20-arm-glue.h b/src/crypto/zinc/chacha20/chacha20-arm-glue.h
index e661ed2..0c8c9d5 100644
--- a/src/crypto/zinc/chacha20/chacha20-arm-glue.h
+++ b/src/crypto/zinc/chacha20/chacha20-arm-glue.h
@@ -29,10 +29,10 @@ void __init chacha20_fpu_init(void)
static inline bool chacha20_arch(u8 *dst, const u8 *src, const size_t len,
const u32 key[8], const u32 counter[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
#if defined(ARM_USE_NEON)
- if (simd_context == HAVE_FULL_SIMD && chacha20_use_neon) {
+ if (chacha20_use_neon && simd_use(simd_context)) {
chacha20_neon(dst, src, len, key, counter);
return true;
}
@@ -42,7 +42,7 @@ static inline bool chacha20_arch(u8 *dst, const u8 *src, const size_t len,
}
static inline bool hchacha20_arch(u8 *derived_key, const u8 *nonce,
- const u8 *key, simd_context_t simd_context)
+ const u8 *key, simd_context_t *simd_context)
{
return false;
}
diff --git a/src/crypto/zinc/chacha20/chacha20-mips-glue.h b/src/crypto/zinc/chacha20/chacha20-mips-glue.h
index ef8e4ab..e4185e1 100644
--- a/src/crypto/zinc/chacha20/chacha20-mips-glue.h
+++ b/src/crypto/zinc/chacha20/chacha20-mips-glue.h
@@ -13,14 +13,14 @@ void __init chacha20_fpu_init(void)
static inline bool chacha20_arch(u8 *dst, const u8 *src, const size_t len,
const u32 key[8], const u32 counter[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
chacha20_mips(dst, src, len, key, counter);
return true;
}
static inline bool hchacha20_arch(u8 *derived_key, const u8 *nonce,
- const u8 *key, simd_context_t simd_context)
+ const u8 *key, simd_context_t *simd_context)
{
return false;
}
diff --git a/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h b/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h
index 78270d7..34919c7 100644
--- a/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h
+++ b/src/crypto/zinc/chacha20/chacha20-x86_64-glue.h
@@ -59,9 +59,9 @@ void __init chacha20_fpu_init(void)
static inline bool chacha20_arch(u8 *dst, const u8 *src, const size_t len,
const u32 key[8], const u32 counter[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
- if (simd_context != HAVE_FULL_SIMD)
+ if (!simd_use(simd_context))
return false;
#ifdef CONFIG_AS_AVX512
@@ -90,10 +90,10 @@ static inline bool chacha20_arch(u8 *dst, const u8 *src, const size_t len,
}
static inline bool hchacha20_arch(u8 *derived_key, const u8 *nonce,
- const u8 *key, simd_context_t simd_context)
+ const u8 *key, simd_context_t *simd_context)
{
#if defined(CONFIG_AS_SSSE3)
- if (simd_context == HAVE_FULL_SIMD && chacha20_use_ssse3) {
+ if (chacha20_use_ssse3 && simd_use(simd_context)) {
hchacha20_ssse3(derived_key, nonce, key);
return true;
}
diff --git a/src/crypto/zinc/chacha20/chacha20.c b/src/crypto/zinc/chacha20/chacha20.c
index fdfccdc..da04d5b 100644
--- a/src/crypto/zinc/chacha20/chacha20.c
+++ b/src/crypto/zinc/chacha20/chacha20.c
@@ -18,12 +18,12 @@ void __init chacha20_fpu_init(void)
}
static inline bool chacha20_arch(u8 *out, const u8 *in, const size_t len,
const u32 key[8], const u32 counter[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
return false;
}
static inline bool hchacha20_arch(u8 *derived_key, const u8 *nonce,
- const u8 *key, simd_context_t simd_context)
+ const u8 *key, simd_context_t *simd_context)
{
return false;
}
@@ -113,7 +113,7 @@ static void chacha20_generic(u8 *out, const u8 *in, u32 len, const u32 key[8],
}
void chacha20(struct chacha20_ctx *state, u8 *dst, const u8 *src, u32 len,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
if (!chacha20_arch(dst, src, len, state->key, state->counter,
simd_context))
@@ -157,7 +157,7 @@ static void hchacha20_generic(u8 derived_key[CHACHA20_KEY_SIZE],
/* Derived key should be 32-bit aligned */
void hchacha20(u8 derived_key[CHACHA20_KEY_SIZE],
const u8 nonce[HCHACHA20_NONCE_SIZE],
- const u8 key[HCHACHA20_KEY_SIZE], simd_context_t simd_context)
+ const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context)
{
if (!hchacha20_arch(derived_key, nonce, key, simd_context))
hchacha20_generic(derived_key, nonce, key);
diff --git a/src/crypto/zinc/chacha20poly1305.c b/src/crypto/zinc/chacha20poly1305.c
index 92a5b9c..7a8e03f 100644
--- a/src/crypto/zinc/chacha20poly1305.c
+++ b/src/crypto/zinc/chacha20poly1305.c
@@ -33,7 +33,7 @@ static inline void
__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
const u8 key[CHACHA20POLY1305_KEYLEN],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
@@ -75,10 +75,10 @@ void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
{
simd_context_t simd_context;
- simd_context = simd_get();
+ simd_get(&simd_context);
__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key,
- simd_context);
- simd_put(simd_context);
+ &simd_context);
+ simd_put(&simd_context);
}
EXPORT_SYMBOL(chacha20poly1305_encrypt);
@@ -87,7 +87,7 @@ bool chacha20poly1305_encrypt_sg(struct scatterlist *dst,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEYLEN],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
@@ -155,7 +155,7 @@ static inline bool
__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 *ad, const size_t ad_len, const u64 nonce,
const u8 key[CHACHA20POLY1305_KEYLEN],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
@@ -208,10 +208,10 @@ bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
{
simd_context_t simd_context, ret;
- simd_context = simd_get();
+ simd_get(&simd_context);
ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce,
- key, simd_context);
- simd_put(simd_context);
+ key, &simd_context);
+ simd_put(&simd_context);
return ret;
}
EXPORT_SYMBOL(chacha20poly1305_decrypt);
@@ -221,7 +221,7 @@ bool chacha20poly1305_decrypt_sg(struct scatterlist *dst,
const u8 *ad, const size_t ad_len,
const u64 nonce,
const u8 key[CHACHA20POLY1305_KEYLEN],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
@@ -300,15 +300,16 @@ void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 nonce[XCHACHA20POLY1305_NONCELEN],
const u8 key[CHACHA20POLY1305_KEYLEN])
{
- simd_context_t simd_context = simd_get();
+ simd_context_t simd_context;
u8 derived_key[CHACHA20POLY1305_KEYLEN] __aligned(16);
- hchacha20(derived_key, nonce, key, simd_context);
+ simd_get(&simd_context);
+ hchacha20(derived_key, nonce, key, &simd_context);
__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
get_unaligned_le64(nonce + 16),
- derived_key, simd_context);
+ derived_key, &simd_context);
memzero_explicit(derived_key, CHACHA20POLY1305_KEYLEN);
- simd_put(simd_context);
+ simd_put(&simd_context);
}
EXPORT_SYMBOL(xchacha20poly1305_encrypt);
@@ -317,15 +318,17 @@ bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
const u8 nonce[XCHACHA20POLY1305_NONCELEN],
const u8 key[CHACHA20POLY1305_KEYLEN])
{
- bool ret, simd_context = simd_get();
+ bool ret;
+ simd_context_t simd_context;
u8 derived_key[CHACHA20POLY1305_KEYLEN] __aligned(16);
- hchacha20(derived_key, nonce, key, simd_context);
+ simd_get(&simd_context);
+ hchacha20(derived_key, nonce, key, &simd_context);
ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
get_unaligned_le64(nonce + 16),
- derived_key, simd_context);
+ derived_key, &simd_context);
memzero_explicit(derived_key, CHACHA20POLY1305_KEYLEN);
- simd_put(simd_context);
+ simd_put(&simd_context);
return ret;
}
EXPORT_SYMBOL(xchacha20poly1305_decrypt);
diff --git a/src/crypto/zinc/poly1305/poly1305-arm-glue.h b/src/crypto/zinc/poly1305/poly1305-arm-glue.h
index 8181703..6ec2fc8 100644
--- a/src/crypto/zinc/poly1305/poly1305-arm-glue.h
+++ b/src/crypto/zinc/poly1305/poly1305-arm-glue.h
@@ -39,10 +39,10 @@ static inline bool poly1305_init_arch(void *ctx,
static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
const size_t len, const u32 padbit,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
#if defined(ARM_USE_NEON)
- if (simd_context == HAVE_FULL_SIMD && poly1305_use_neon) {
+ if (poly1305_use_neon && simd_use(simd_context)) {
poly1305_blocks_neon(ctx, inp, len, padbit);
return true;
}
@@ -53,10 +53,10 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
const u32 nonce[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
#if defined(ARM_USE_NEON)
- if (simd_context == HAVE_FULL_SIMD && poly1305_use_neon) {
+ if (poly1305_use_neon && simd_use(simd_context)) {
poly1305_emit_neon(ctx, mac, nonce);
return true;
}
diff --git a/src/crypto/zinc/poly1305/poly1305-mips-glue.h b/src/crypto/zinc/poly1305/poly1305-mips-glue.h
index 960abee..0e72c8b 100644
--- a/src/crypto/zinc/poly1305/poly1305-mips-glue.h
+++ b/src/crypto/zinc/poly1305/poly1305-mips-glue.h
@@ -22,7 +22,7 @@ static inline bool poly1305_init_arch(void *ctx,
static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
const size_t len, const u32 padbit,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
poly1305_blocks_mips(ctx, inp, len, padbit);
return true;
@@ -30,7 +30,7 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
const u32 nonce[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
poly1305_emit_mips(ctx, mac, nonce);
return true;
diff --git a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h
index 7f1af44..1afd1c5 100644
--- a/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h
+++ b/src/crypto/zinc/poly1305/poly1305-x86_64-glue.h
@@ -63,20 +63,20 @@ static inline bool poly1305_init_arch(void *ctx,
static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
const size_t len, const u32 padbit,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
#ifdef CONFIG_AS_AVX512
- if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD)
+ if (poly1305_use_avx512 && simd_use(simd_context))
poly1305_blocks_avx512(ctx, inp, len, padbit);
else
#endif
#ifdef CONFIG_AS_AVX2
- if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD)
+ if (poly1305_use_avx2 && simd_use(simd_context))
poly1305_blocks_avx2(ctx, inp, len, padbit);
else
#endif
#ifdef CONFIG_AS_AVX
- if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD)
+ if (poly1305_use_avx && simd_use(simd_context))
poly1305_blocks_avx(ctx, inp, len, padbit);
else
#endif
@@ -86,20 +86,20 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
const u32 nonce[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
#ifdef CONFIG_AS_AVX512
- if (poly1305_use_avx512 && simd_context == HAVE_FULL_SIMD)
+ if (poly1305_use_avx512 && simd_use(simd_context))
poly1305_emit_avx(ctx, mac, nonce);
else
#endif
#ifdef CONFIG_AS_AVX2
- if (poly1305_use_avx2 && simd_context == HAVE_FULL_SIMD)
+ if (poly1305_use_avx2 && simd_use(simd_context))
poly1305_emit_avx(ctx, mac, nonce);
else
#endif
#ifdef CONFIG_AS_AVX
- if (poly1305_use_avx && simd_context == HAVE_FULL_SIMD)
+ if (poly1305_use_avx && simd_use(simd_context))
poly1305_emit_avx(ctx, mac, nonce);
else
#endif
diff --git a/src/crypto/zinc/poly1305/poly1305.c b/src/crypto/zinc/poly1305/poly1305.c
index d8c103f..a098b61 100644
--- a/src/crypto/zinc/poly1305/poly1305.c
+++ b/src/crypto/zinc/poly1305/poly1305.c
@@ -21,13 +21,13 @@ static inline bool poly1305_init_arch(void *ctx,
}
static inline bool poly1305_blocks_arch(void *ctx, const u8 *input,
const size_t len, const u32 padbit,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
return false;
}
static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
const u32 nonce[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
return false;
}
@@ -58,7 +58,7 @@ EXPORT_SYMBOL(poly1305_init);
static inline void poly1305_blocks(void *ctx, const u8 *input, const size_t len,
const u32 padbit,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
if (!poly1305_blocks_arch(ctx, input, len, padbit, simd_context))
poly1305_blocks_generic(ctx, input, len, padbit);
@@ -66,14 +66,14 @@ static inline void poly1305_blocks(void *ctx, const u8 *input, const size_t len,
static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE],
const u32 nonce[4],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
if (!poly1305_emit_arch(ctx, mac, nonce, simd_context))
poly1305_emit_generic(ctx, mac, nonce);
}
void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
const size_t num = ctx->num % POLY1305_BLOCK_SIZE;
size_t rem;
@@ -108,7 +108,7 @@ void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len,
EXPORT_SYMBOL(poly1305_update);
void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE],
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
size_t num = ctx->num % POLY1305_BLOCK_SIZE;
diff --git a/src/crypto/zinc/selftest/chacha20poly1305.h b/src/crypto/zinc/selftest/chacha20poly1305.h
index e5e8de5..9aae3f5 100644
--- a/src/crypto/zinc/selftest/chacha20poly1305.h
+++ b/src/crypto/zinc/selftest/chacha20poly1305.h
@@ -7635,7 +7635,7 @@ chacha20poly1305_selftest_encrypt_bignonce(u8 *dst, const u8 *src,
const u8 nonce[12],
const u8 key[CHACHA20POLY1305_KEYLEN])
{
- simd_context_t simd_context = simd_get();
+ simd_context_t simd_context;
struct poly1305_ctx poly1305_state;
struct chacha20_ctx chacha20_state;
union {
@@ -7643,26 +7643,27 @@ chacha20poly1305_selftest_encrypt_bignonce(u8 *dst, const u8 *src,
__le64 lens[2];
} b = {{ 0 }};
+ simd_get(&simd_context);
chacha20_init(&chacha20_state, key, 0);
chacha20_state.counter[1] = get_unaligned_le32(nonce + 0);
chacha20_state.counter[2] = get_unaligned_le32(nonce + 4);
chacha20_state.counter[3] = get_unaligned_le32(nonce + 8);
chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0),
- simd_context);
+ &simd_context);
poly1305_init(&poly1305_state, b.block0);
- poly1305_update(&poly1305_state, ad, ad_len, simd_context);
+ poly1305_update(&poly1305_state, ad, ad_len, &simd_context);
poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf,
- simd_context);
- chacha20(&chacha20_state, dst, src, src_len, simd_context);
- poly1305_update(&poly1305_state, dst, src_len, simd_context);
+ &simd_context);
+ chacha20(&chacha20_state, dst, src, src_len, &simd_context);
+ poly1305_update(&poly1305_state, dst, src_len, &simd_context);
poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf,
- simd_context);
+ &simd_context);
b.lens[0] = cpu_to_le64(ad_len);
b.lens[1] = cpu_to_le64(src_len);
poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens),
- simd_context);
- poly1305_final(&poly1305_state, dst + src_len, simd_context);
- simd_put(simd_context);
+ &simd_context);
+ poly1305_final(&poly1305_state, dst + src_len, &simd_context);
+ simd_put(&simd_context);
memzero_explicit(&chacha20_state, sizeof(chacha20_state));
memzero_explicit(&b, sizeof(b));
}
@@ -7698,7 +7699,8 @@ bool __init chacha20poly1305_selftest(void)
{
size_t i;
u8 computed_result[MAXIMUM_TEST_BUFFER_LEN], *heap_src, *heap_dst;
- bool success = true, ret, simd_context;
+ bool success = true, ret;
+ simd_context_t simd_context;
struct scatterlist sg_src, sg_dst;
heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL);
@@ -7729,7 +7731,7 @@ bool __init chacha20poly1305_selftest(void)
success = false;
}
}
- simd_context = simd_get();
+ simd_get(&simd_context);
for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) {
if (chacha20poly1305_enc_vectors[i].nlen != 8)
continue;
@@ -7747,7 +7749,7 @@ bool __init chacha20poly1305_selftest(void)
chacha20poly1305_enc_vectors[i].alen,
get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce),
chacha20poly1305_enc_vectors[i].key,
- simd_context);
+ &simd_context);
if (!ret || memcmp(heap_dst,
chacha20poly1305_enc_vectors[i].result,
chacha20poly1305_enc_vectors[i].ilen +
@@ -7757,7 +7759,7 @@ bool __init chacha20poly1305_selftest(void)
success = false;
}
}
- simd_put(simd_context);
+ simd_put(&simd_context);
for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
memset(computed_result, 0, sizeof(computed_result));
ret = chacha20poly1305_decrypt(computed_result,
@@ -7778,7 +7780,7 @@ bool __init chacha20poly1305_selftest(void)
success = false;
}
}
- simd_context = simd_get();
+ simd_get(&simd_context);
for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) {
memset(heap_dst, 0, MAXIMUM_TEST_BUFFER_LEN);
memcpy(heap_src, chacha20poly1305_dec_vectors[i].input,
@@ -7793,7 +7795,7 @@ bool __init chacha20poly1305_selftest(void)
chacha20poly1305_dec_vectors[i].assoc,
chacha20poly1305_dec_vectors[i].alen,
get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce),
- chacha20poly1305_dec_vectors[i].key, simd_context);
+ chacha20poly1305_dec_vectors[i].key, &simd_context);
if (!decryption_success(ret,
chacha20poly1305_dec_vectors[i].failure,
memcmp(heap_dst, chacha20poly1305_dec_vectors[i].result,
@@ -7804,7 +7806,7 @@ bool __init chacha20poly1305_selftest(void)
success = false;
}
}
- simd_put(simd_context);
+ simd_put(&simd_context);
for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) {
memset(computed_result, 0, sizeof(computed_result));
xchacha20poly1305_encrypt(computed_result,
diff --git a/src/crypto/zinc/selftest/poly1305.h b/src/crypto/zinc/selftest/poly1305.h
index d02941f..02cd4ba 100644
--- a/src/crypto/zinc/selftest/poly1305.h
+++ b/src/crypto/zinc/selftest/poly1305.h
@@ -820,10 +820,11 @@ static const struct poly1305_testvec poly1305_testvecs[] __initconst = {
bool __init poly1305_selftest(void)
{
- simd_context_t simd_context = simd_get();
+ simd_context_t simd_context;
bool success = true;
size_t i, j;
+ simd_get(&simd_context);
for (i = 0; i < ARRAY_SIZE(poly1305_testvecs); ++i) {
struct poly1305_ctx poly1305;
u8 out[POLY1305_MAC_SIZE];
@@ -832,14 +833,14 @@ bool __init poly1305_selftest(void)
memset(&poly1305, 0, sizeof(poly1305));
poly1305_init(&poly1305, poly1305_testvecs[i].key);
poly1305_update(&poly1305, poly1305_testvecs[i].input,
- poly1305_testvecs[i].ilen, simd_context);
- poly1305_final(&poly1305, out, simd_context);
+ poly1305_testvecs[i].ilen, &simd_context);
+ poly1305_final(&poly1305, out, &simd_context);
if (memcmp(out, poly1305_testvecs[i].output,
POLY1305_MAC_SIZE)) {
pr_info("poly1305 self-test %zu: FAIL\n", i + 1);
success = false;
}
- simd_context = simd_relax(simd_context);
+ simd_relax(&simd_context);
if (poly1305_testvecs[i].ilen <= 1)
continue;
@@ -849,22 +850,22 @@ bool __init poly1305_selftest(void)
memset(&poly1305, 0, sizeof(poly1305));
poly1305_init(&poly1305, poly1305_testvecs[i].key);
poly1305_update(&poly1305, poly1305_testvecs[i].input,
- j, simd_context);
+ j, &simd_context);
poly1305_update(&poly1305,
poly1305_testvecs[i].input + j,
poly1305_testvecs[i].ilen - j,
- simd_context);
- poly1305_final(&poly1305, out, simd_context);
+ &simd_context);
+ poly1305_final(&poly1305, out, &simd_context);
if (memcmp(out, poly1305_testvecs[i].output,
POLY1305_MAC_SIZE)) {
pr_info("poly1305 self-test %zu (split %zu): FAIL\n",
i + 1, j);
success = false;
}
- simd_context = simd_relax(simd_context);
+ simd_relax(&simd_context);
}
}
- simd_put(simd_context);
+ simd_put(&simd_context);
if (success)
pr_info("poly1305 self-tests: pass\n");
diff --git a/src/receive.c b/src/receive.c
index 4500a85..aa1f216 100644
--- a/src/receive.c
+++ b/src/receive.c
@@ -245,7 +245,7 @@ static void keep_key_fresh(struct wireguard_peer *peer)
}
static bool skb_decrypt(struct sk_buff *skb, struct noise_symmetric_key *key,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
struct scatterlist sg[MAX_SKB_FRAGS + 8];
struct sk_buff *trailer;
@@ -500,20 +500,21 @@ void packet_decrypt_worker(struct work_struct *work)
{
struct crypt_queue *queue =
container_of(work, struct multicore_worker, work)->ptr;
- simd_context_t simd_context = simd_get();
+ simd_context_t simd_context;
struct sk_buff *skb;
+ simd_get(&simd_context);
while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
enum packet_state state = likely(skb_decrypt(skb,
&PACKET_CB(skb)->keypair->receiving,
- simd_context)) ?
+ &simd_context)) ?
PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
queue_enqueue_per_peer_napi(&PACKET_PEER(skb)->rx_queue, skb,
state);
- simd_context = simd_relax(simd_context);
+ simd_relax(&simd_context);
}
- simd_put(simd_context);
+ simd_put(&simd_context);
}
static void packet_consume_data(struct wireguard_device *wg,
diff --git a/src/send.c b/src/send.c
index dc50b2e..a0c8dd2 100644
--- a/src/send.c
+++ b/src/send.c
@@ -157,7 +157,7 @@ static unsigned int skb_padding(struct sk_buff *skb)
}
static bool skb_encrypt(struct sk_buff *skb, struct noise_keypair *keypair,
- simd_context_t simd_context)
+ simd_context_t *simd_context)
{
unsigned int padding_len, plaintext_len, trailer_len;
struct scatterlist sg[MAX_SKB_FRAGS + 8];
@@ -296,14 +296,15 @@ void packet_encrypt_worker(struct work_struct *work)
struct crypt_queue *queue =
container_of(work, struct multicore_worker, work)->ptr;
struct sk_buff *first, *skb, *next;
- simd_context_t simd_context = simd_get();
+ simd_context_t simd_context;
+ simd_get(&simd_context);
while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
enum packet_state state = PACKET_STATE_CRYPTED;
skb_walk_null_queue_safe (first, skb, next) {
if (likely(skb_encrypt(skb, PACKET_CB(first)->keypair,
- simd_context)))
+ &simd_context)))
skb_reset(skb);
else {
state = PACKET_STATE_DEAD;
@@ -313,9 +314,9 @@ void packet_encrypt_worker(struct work_struct *work)
queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first,
state);
- simd_context = simd_relax(simd_context);
+ simd_relax(&simd_context);
}
- simd_put(simd_context);
+ simd_put(&simd_context);
}
static void packet_create_data(struct sk_buff *first)
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.