From f4561dd5377487100abbfbec0a26aa7498091141 Mon Sep 17 00:00:00 2001 From: Samuel Neves Date: Wed, 22 Nov 2017 20:57:37 +0000 Subject: blake2s: AVX512F+VL implementation Signed-off-by: Samuel Neves --- src/crypto/blake2s.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'src/crypto/blake2s.c') diff --git a/src/crypto/blake2s.c b/src/crypto/blake2s.c index 91f154f..3e13277 100644 --- a/src/crypto/blake2s.c +++ b/src/crypto/blake2s.c @@ -114,11 +114,20 @@ void blake2s_init_key(struct blake2s_state *state, const size_t outlen, const vo #include #include static bool blake2s_use_avx __read_mostly; +static bool blake2s_use_avx512 __read_mostly; void __init blake2s_fpu_init(void) { blake2s_use_avx = boot_cpu_has(X86_FEATURE_AVX) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + blake2s_use_avx512 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512VL) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_ZMM_Hi256, NULL); +#endif } +#ifdef CONFIG_AS_AVX asmlinkage void blake2s_compress_avx(struct blake2s_state *state, const u8 *block, size_t nblocks, u32 inc); +#endif +#ifdef CONFIG_AS_AVX512 +asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, const u8 *block, size_t nblocks, u32 inc); +#endif #else void __init blake2s_fpu_init(void) { } #endif @@ -134,12 +143,22 @@ static inline void blake2s_compress(struct blake2s_state *state, const u8 *block #endif #ifdef CONFIG_X86_64 +#ifdef CONFIG_AS_AVX512 + if (blake2s_use_avx512 && irq_fpu_usable()) { + kernel_fpu_begin(); + blake2s_compress_avx512(state, block, nblocks, inc); + kernel_fpu_end(); + return; + } +#endif +#ifdef CONFIG_AS_AVX if (blake2s_use_avx && irq_fpu_usable()) { kernel_fpu_begin(); blake2s_compress_avx(state, block, nblocks, inc); kernel_fpu_end(); return; } +#endif #endif while (nblocks > 0) { -- cgit v1.2.3-11-g984f