diff options
| author |   Samuel Neves <sneves@dei.uc.pt> | 2017-11-22 20:57:37 +0000 |
|---|---|---|
| committer |   Jason A. Donenfeld <Jason@zx2c4.com> | 2017-11-22 22:55:50 +0100 |
| commit | f4561dd5377487100abbfbec0a26aa7498091141 (patch) | |
| tree | eb5a95c4fbb36701c6d18795df50ef9e195f28ff /src/crypto/blake2s.c | |
| parent | poly1305-avx512: requires AVX512F+VL+BW (diff) | |
| download | wireguard-monolithic-historical-f4561dd5377487100abbfbec0a26aa7498091141.tar.xz wireguard-monolithic-historical-f4561dd5377487100abbfbec0a26aa7498091141.zip | |
blake2s: AVX512F+VL implementation
Signed-off-by: Samuel Neves <sneves@dei.uc.pt>
Diffstat (limited to 'src/crypto/blake2s.c')
| -rw-r--r-- | src/crypto/blake2s.c | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/src/crypto/blake2s.c b/src/crypto/blake2s.c index 91f154f..3e13277 100644 --- a/src/crypto/blake2s.c +++ b/src/crypto/blake2s.c @@ -114,11 +114,20 @@ void blake2s_init_key(struct blake2s_state *state, const size_t outlen, const vo #include <asm/fpu/api.h> #include <asm/simd.h> static bool blake2s_use_avx __read_mostly; +static bool blake2s_use_avx512 __read_mostly; void __init blake2s_fpu_init(void) { blake2s_use_avx = boot_cpu_has(X86_FEATURE_AVX) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + blake2s_use_avx512 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512VL) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_ZMM_Hi256, NULL); +#endif } +#ifdef CONFIG_AS_AVX asmlinkage void blake2s_compress_avx(struct blake2s_state *state, const u8 *block, size_t nblocks, u32 inc); +#endif +#ifdef CONFIG_AS_AVX512 +asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, const u8 *block, size_t nblocks, u32 inc); +#endif #else void __init blake2s_fpu_init(void) { } #endif @@ -134,6 +143,15 @@ static inline void blake2s_compress(struct blake2s_state *state, const u8 *block #endif #ifdef CONFIG_X86_64 +#ifdef CONFIG_AS_AVX512 + if (blake2s_use_avx512 && irq_fpu_usable()) { + kernel_fpu_begin(); + blake2s_compress_avx512(state, block, nblocks, inc); + kernel_fpu_end(); + return; + } +#endif +#ifdef CONFIG_AS_AVX if (blake2s_use_avx && irq_fpu_usable()) { kernel_fpu_begin(); blake2s_compress_avx(state, block, nblocks, inc); @@ -141,6 +159,7 @@ static inline void blake2s_compress(struct blake2s_state *state, const u8 *block return; } #endif +#endif while (nblocks > 0) { blake2s_increment_counter(state, inc); |