diff options
| author |   Jason A. Donenfeld <Jason@zx2c4.com> | 2017-05-21 17:51:58 +0200 |
|---|---|---|
| committer | Jason A. Donenfeld <Jason@zx2c4.com> | 2017-05-30 18:07:28 +0200 |
| commit | feffaa608b7e5e6802dabf55a2abb81662caa1c9 (patch) | |
| tree | 9411a846a84754266906d8f7109efc9ceac39754 /src/crypto/chacha20poly1305.c | |
| parent | chacha20poly1305: move constants to rodata (diff) | |
| download | wireguard-monolithic-historical-feffaa608b7e5e6802dabf55a2abb81662caa1c9.tar.xz wireguard-monolithic-historical-feffaa608b7e5e6802dabf55a2abb81662caa1c9.zip | |
chacha20poly1305: add NEON versions for ARM and ARM64
Diffstat (limited to 'src/crypto/chacha20poly1305.c')
| -rw-r--r-- | src/crypto/chacha20poly1305.c | 44 |
1 files changed, 41 insertions, 3 deletions
diff --git a/src/crypto/chacha20poly1305.c b/src/crypto/chacha20poly1305.c index 611008e..ed40ebb 100644 --- a/src/crypto/chacha20poly1305.c +++ b/src/crypto/chacha20poly1305.c @@ -12,7 +12,7 @@ #include <crypto/scatterwalk.h> #include <asm/unaligned.h> -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) #include <asm/cpufeature.h> #include <asm/processor.h> #ifdef CONFIG_AS_SSSE3 @@ -37,6 +37,20 @@ void chacha20poly1305_fpu_init(void) chacha20poly1305_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); chacha20poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2); } +#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) +#include <asm/hwcap.h> +#include <asm/neon.h> +asmlinkage void chacha20_asm_block_xor_neon(u32 *state, u8 *dst, const u8 *src); +asmlinkage void chacha20_asm_4block_xor_neon(u32 *state, u8 *dst, const u8 *src); +static bool chacha20poly1305_use_neon __read_mostly = false; +void chacha20poly1305_fpu_init(void) +{ +#if defined(CONFIG_ARM64) + chacha20poly1305_use_neon = elf_hwcap & HWCAP_ASIMD; +#elif defined(CONFIG_ARM) + chacha20poly1305_use_neon = elf_hwcap & HWCAP_NEON; +#endif +} #else void chacha20poly1305_fpu_init(void) { } #endif @@ -257,13 +271,16 @@ static void chacha20_crypt(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, uns u8 buf[CHACHA20_BLOCK_SIZE]; if (!have_simd -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) || !chacha20poly1305_use_ssse3 + +#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) + || !chacha20poly1305_use_neon #endif ) goto no_simd; -#ifdef CONFIG_X86_64 +#if defined(CONFIG_X86_64) #ifdef CONFIG_AS_AVX2 if (chacha20poly1305_use_avx2) { while (bytes >= CHACHA20_BLOCK_SIZE * 8) { @@ -297,6 +314,27 @@ static void chacha20_crypt(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, uns } return; #endif +#elif IS_ENABLED(CONFIG_KERNEL_MODE_NEON) + while (bytes >= CHACHA20_BLOCK_SIZE * 4) { + chacha20_asm_4block_xor_neon(ctx->state, dst, src); + bytes -= CHACHA20_BLOCK_SIZE * 4; + src += CHACHA20_BLOCK_SIZE * 4; + dst += CHACHA20_BLOCK_SIZE * 4; + ctx->state[12] += 4; + } + while (bytes >= CHACHA20_BLOCK_SIZE) { + chacha20_asm_block_xor_neon(ctx->state, dst, src); + bytes -= CHACHA20_BLOCK_SIZE; + src += CHACHA20_BLOCK_SIZE; + dst += CHACHA20_BLOCK_SIZE; + ctx->state[12]++; + } + if (bytes) { + memcpy(buf, src, bytes); + chacha20_asm_block_xor_neon(ctx->state, buf, buf); + memcpy(dst, buf, bytes); + } + return; #endif no_simd: |