#ifndef __KERNEL__ # include "arm_arch.h" .extern OPENSSL_armcap_P #endif .text // forward "declarations" are required for Apple .globl poly1305_blocks .globl poly1305_emit .globl poly1305_init .type poly1305_init,%function .align 5 poly1305_init: cmp x1,xzr stp xzr,xzr,[x0] // zero hash value stp xzr,xzr,[x0,#16] // [along with is_base2_26] csel x0,xzr,x0,eq b.eq .Lno_key #ifndef __KERNEL__ adrp x17,OPENSSL_armcap_P ldr w17,[x17,#:lo12:OPENSSL_armcap_P] #endif ldp x7,x8,[x1] // load key mov x9,#0xfffffffc0fffffff movk x9,#0x0fff,lsl#48 #ifdef __AARCH64EB__ rev x7,x7 // flip bytes rev x8,x8 #endif and x7,x7,x9 // &=0ffffffc0fffffff and x9,x9,#-4 and x8,x8,x9 // &=0ffffffc0ffffffc mov w9,#-1 stp x7,x8,[x0,#32] // save key value str w9,[x0,#48] // impossible key power value #ifndef __KERNEL__ tst w17,#ARMV7_NEON adr x12,.Lpoly1305_blocks adr x7,.Lpoly1305_blocks_neon adr x13,.Lpoly1305_emit csel x12,x12,x7,eq # ifdef __ILP32__ stp w12,w13,[x2] # else stp x12,x13,[x2] # endif #endif mov x0,#1 .Lno_key: ret .size poly1305_init,.-poly1305_init .type poly1305_blocks,%function .align 5 poly1305_blocks: .Lpoly1305_blocks: ands x2,x2,#-16 b.eq .Lno_data ldp x4,x5,[x0] // load hash value ldp x6,x17,[x0,#16] // [along with is_base2_26] ldp x7,x8,[x0,#32] // load key value #ifdef __AARCH64EB__ lsr x12,x4,#32 mov w13,w4 lsr x14,x5,#32 mov w15,w5 lsr x16,x6,#32 #else mov w12,w4 lsr x13,x4,#32 mov w14,w5 lsr x15,x5,#32 mov w16,w6 #endif add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64 lsr x13,x14,#12 adds x12,x12,x14,lsl#52 add x13,x13,x15,lsl#14 adc x13,x13,xzr lsr x14,x16,#24 adds x13,x13,x16,lsl#40 adc x14,x14,xzr cmp x17,#0 // is_base2_26? add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) csel x4,x4,x12,eq // choose between radixes csel x5,x5,x13,eq csel x6,x6,x14,eq .Loop: ldp x10,x11,[x1],#16 // load input sub x2,x2,#16 #ifdef __AARCH64EB__ rev x10,x10 rev x11,x11 #endif adds x4,x4,x10 // accumulate input adcs x5,x5,x11 mul x12,x4,x7 // h0*r0 adc x6,x6,x3 umulh x13,x4,x7 mul x10,x5,x9 // h1*5*r1 umulh x11,x5,x9 adds x12,x12,x10 mul x10,x4,x8 // h0*r1 adc x13,x13,x11 umulh x14,x4,x8 adds x13,x13,x10 mul x10,x5,x7 // h1*r0 adc x14,x14,xzr umulh x11,x5,x7 adds x13,x13,x10 mul x10,x6,x9 // h2*5*r1 adc x14,x14,x11 mul x11,x6,x7 // h2*r0 adds x13,x13,x10 adc x14,x14,x11 and x10,x14,#-4 // final reduction and x6,x14,#3 add x10,x10,x14,lsr#2 adds x4,x12,x10 adcs x5,x13,xzr adc x6,x6,xzr cbnz x2,.Loop stp x4,x5,[x0] // store hash value stp x6,xzr,[x0,#16] // [and clear is_base2_26] .Lno_data: ret .size poly1305_blocks,.-poly1305_blocks .type poly1305_emit,%function .align 5 poly1305_emit: .Lpoly1305_emit: ldp x4,x5,[x0] // load hash base 2^64 ldp x6,x7,[x0,#16] // [along with is_base2_26] ldp x10,x11,[x2] // load nonce #ifdef __AARCH64EB__ lsr x12,x4,#32 mov w13,w4 lsr x14,x5,#32 mov w15,w5 lsr x16,x6,#32 #else mov w12,w4 lsr x13,x4,#32 mov w14,w5 lsr x15,x5,#32 mov w16,w6 #endif add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64 lsr x13,x14,#12 adds x12,x12,x14,lsl#52 add x13,x13,x15,lsl#14 adc x13,x13,xzr lsr x14,x16,#24 adds x13,x13,x16,lsl#40 adc x14,x14,xzr cmp x7,#0 // is_base2_26? csel x4,x4,x12,eq // choose between radixes csel x5,x5,x13,eq csel x6,x6,x14,eq adds x12,x4,#5 // compare to modulus adcs x13,x5,xzr adc x14,x6,xzr tst x14,#-4 // see if it's carried/borrowed csel x4,x4,x12,eq csel x5,x5,x13,eq #ifdef __AARCH64EB__ ror x10,x10,#32 // flip nonce words ror x11,x11,#32 #endif adds x4,x4,x10 // accumulate nonce adc x5,x5,x11 #ifdef __AARCH64EB__ rev x4,x4 // flip output bytes rev x5,x5 #endif stp x4,x5,[x1] // write result ret .size poly1305_emit,.-poly1305_emit .type poly1305_mult,%function .align 5 poly1305_mult: mul x12,x4,x7 // h0*r0 umulh x13,x4,x7 mul x10,x5,x9 // h1*5*r1 umulh x11,x5,x9 adds x12,x12,x10 mul x10,x4,x8 // h0*r1 adc x13,x13,x11 umulh x14,x4,x8 adds x13,x13,x10 mul x10,x5,x7 // h1*r0 adc x14,x14,xzr umulh x11,x5,x7 adds x13,x13,x10 mul x10,x6,x9 // h2*5*r1 adc x14,x14,x11 mul x11,x6,x7 // h2*r0 adds x13,x13,x10 adc x14,x14,x11 and x10,x14,#-4 // final reduction and x6,x14,#3 add x10,x10,x14,lsr#2 adds x4,x12,x10 adcs x5,x13,xzr adc x6,x6,xzr ret .size poly1305_mult,.-poly1305_mult .type poly1305_splat,%function .align 4 poly1305_splat: and x12,x4,#0x03ffffff // base 2^64 -> base 2^26 ubfx x13,x4,#26,#26 extr x14,x5,x4,#52 and x14,x14,#0x03ffffff ubfx x15,x5,#14,#26 extr x16,x6,x5,#40 str w12,[x0,#16*0] // r0 add w12,w13,w13,lsl#2 // r1*5 str w13,[x0,#16*1] // r1 add w13,w14,w14,lsl#2 // r2*5 str w12,[x0,#16*2] // s1 str w14,[x0,#16*3] // r2 add w14,w15,w15,lsl#2 // r3*5 str w13,[x0,#16*4] // s2 str w15,[x0,#16*5] // r3 add w15,w16,w16,lsl#2 // r4*5 str w14,[x0,#16*6] // s3 str w16,[x0,#16*7] // r4 str w15,[x0,#16*8] // s4 ret .size poly1305_splat,.-poly1305_splat #ifdef __KERNEL__ .globl poly1305_blocks_neon #endif .type poly1305_blocks_neon,%function .align 5 poly1305_blocks_neon: .Lpoly1305_blocks_neon: ldr x17,[x0,#24] cmp x2,#128 b.lo .Lpoly1305_blocks .inst 0xd503233f // paciasp stp x29,x30,[sp,#-80]! add x29,sp,#0 stp d8,d9,[sp,#16] // meet ABI requirements stp d10,d11,[sp,#32] stp d12,d13,[sp,#48] stp d14,d15,[sp,#64] cbz x17,.Lbase2_64_neon ldp w10,w11,[x0] // load hash value base 2^26 ldp w12,w13,[x0,#8] ldr w14,[x0,#16] tst x2,#31 b.eq .Leven_neon ldp x7,x8,[x0,#32] // load key value add x4,x10,x11,lsl#26 // base 2^26 -> base 2^64 lsr x5,x12,#12 adds x4,x4,x12,lsl#52 add x5,x5,x13,lsl#14 adc x5,x5,xzr lsr x6,x14,#24 adds x5,x5,x14,lsl#40 adc x14,x6,xzr // can be partially reduced... ldp x12,x13,[x1],#16 // load input sub x2,x2,#16 add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) #ifdef __AARCH64EB__ rev x12,x12 rev x13,x13 #endif adds x4,x4,x12 // accumulate input adcs x5,x5,x13 adc x6,x6,x3 bl poly1305_mult and x10,x4,#0x03ffffff // base 2^64 -> base 2^26 ubfx x11,x4,#26,#26 extr x12,x5,x4,#52 and x12,x12,#0x03ffffff ubfx x13,x5,#14,#26 extr x14,x6,x5,#40 b .Leven_neon .align 4 .Lbase2_64_neon: ldp x7,x8,[x0,#32] // load key value ldp x4,x5,[x0] // load hash value base 2^64 ldr x6,[x0,#16] tst x2,#31 b.eq .Linit_neon ldp x12,x13,[x1],#16 // load input sub x2,x2,#16 add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) #ifdef __AARCH64EB__ rev x12,x12 rev x13,x13 #endif adds x4,x4,x12 // accumulate input adcs x5,x5,x13 adc x6,x6,x3 bl poly1305_mult .Linit_neon: ldr w17,[x0,#48] // first table element and x10,x4,#0x03ffffff // base 2^64 -> base 2^26 ubfx x11,x4,#26,#26 extr x12,x5,x4,#52 and x12,x12,#0x03ffffff ubfx x13,x5,#14,#26 extr x14,x6,x5,#40 cmp w17,#-1 // is value impossible? b.ne .Leven_neon fmov d24,x10 fmov d25,x11 fmov d26,x12 fmov d27,x13 fmov d28,x14 ////////////////////////////////// initialize r^n table mov x4,x7 // r^1 add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) mov x5,x8 mov x6,xzr add x0,x0,#48+12 bl poly1305_splat bl poly1305_mult // r^2 sub x0,x0,#4 bl poly1305_splat bl poly1305_mult // r^3 sub x0,x0,#4 bl poly1305_splat bl poly1305_mult // r^4 sub x0,x0,#4 bl poly1305_splat sub x0,x0,#48 // restore original x0 b .Ldo_neon .align 4 .Leven_neon: fmov d24,x10 fmov d25,x11 fmov d26,x12 fmov d27,x13 fmov d28,x14 .Ldo_neon: ldp x8,x12,[x1,#32] // inp[2:3] subs x2,x2,#64 ldp x9,x13,[x1,#48] add x16,x1,#96 adr x17,.Lzeros lsl x3,x3,#24 add x15,x0,#48 #ifdef __AARCH64EB__ rev x8,x8 rev x12,x12 rev x9,x9 rev x13,x13 #endif and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 and x5,x9,#0x03ffffff ubfx x6,x8,#26,#26 ubfx x7,x9,#26,#26 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 extr x8,x12,x8,#52 extr x9,x13,x9,#52 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 fmov d14,x4 and x8,x8,#0x03ffffff and x9,x9,#0x03ffffff ubfx x10,x12,#14,#26 ubfx x11,x13,#14,#26 add x12,x3,x12,lsr#40 add x13,x3,x13,lsr#40 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 fmov d15,x6 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 fmov d16,x8 fmov d17,x10 fmov d18,x12 ldp x8,x12,[x1],#16 // inp[0:1] ldp x9,x13,[x1],#48 ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x15],#64 ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x15],#64 ld1 {v8.4s},[x15] #ifdef __AARCH64EB__ rev x8,x8 rev x12,x12 rev x9,x9 rev x13,x13 #endif and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 and x5,x9,#0x03ffffff ubfx x6,x8,#26,#26 ubfx x7,x9,#26,#26 add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 extr x8,x12,x8,#52 extr x9,x13,x9,#52 add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 fmov d9,x4 and x8,x8,#0x03ffffff and x9,x9,#0x03ffffff ubfx x10,x12,#14,#26 ubfx x11,x13,#14,#26 add x12,x3,x12,lsr#40 add x13,x3,x13,lsr#40 add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 fmov d10,x6 add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 movi v31.2d,#-1 fmov d11,x8 fmov d12,x10 fmov d13,x12 ushr v31.2d,v31.2d,#38 b.ls .Lskip_loop .align 4 .Loop_neon: //////////////////////////////////////////////////////////////// // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r // ___________________/ // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r // ___________________/ ____________________/ // // Note that we start with inp[2:3]*r^2. This is because it // doesn't depend on reduction in previous iteration. //////////////////////////////////////////////////////////////// // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0 // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4 // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3 // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2 // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1 subs x2,x2,#64 umull v23.2d,v14.2s,v7.s[2] csel x16,x17,x16,lo umull v22.2d,v14.2s,v5.s[2] umull v21.2d,v14.2s,v3.s[2] ldp x8,x12,[x16],#16 // inp[2:3] (or zero) umull v20.2d,v14.2s,v1.s[2] ldp x9,x13,[x16],#48 umull v19.2d,v14.2s,v0.s[2] #ifdef __AARCH64EB__ rev x8,x8 rev x12,x12 rev x9,x9 rev x13,x13 #endif umlal v23.2d,v15.2s,v5.s[2] and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 umlal v22.2d,v15.2s,v3.s[2] and x5,x9,#0x03ffffff umlal v21.2d,v15.2s,v1.s[2] ubfx x6,x8,#26,#26 umlal v20.2d,v15.2s,v0.s[2] ubfx x7,x9,#26,#26 umlal v19.2d,v15.2s,v8.s[2] add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 umlal v23.2d,v16.2s,v3.s[2] extr x8,x12,x8,#52 umlal v22.2d,v16.2s,v1.s[2] extr x9,x13,x9,#52 umlal v21.2d,v16.2s,v0.s[2] add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 umlal v20.2d,v16.2s,v8.s[2] fmov d14,x4 umlal v19.2d,v16.2s,v6.s[2] and x8,x8,#0x03ffffff umlal v23.2d,v17.2s,v1.s[2] and x9,x9,#0x03ffffff umlal v22.2d,v17.2s,v0.s[2] ubfx x10,x12,#14,#26 umlal v21.2d,v17.2s,v8.s[2] ubfx x11,x13,#14,#26 umlal v20.2d,v17.2s,v6.s[2] add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 umlal v19.2d,v17.2s,v4.s[2] fmov d15,x6 add v11.2s,v11.2s,v26.2s add x12,x3,x12,lsr#40 umlal v23.2d,v18.2s,v0.s[2] add x13,x3,x13,lsr#40 umlal v22.2d,v18.2s,v8.s[2] add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 umlal v21.2d,v18.2s,v6.s[2] add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 umlal v20.2d,v18.2s,v4.s[2] fmov d16,x8 umlal v19.2d,v18.2s,v2.s[2] fmov d17,x10 //////////////////////////////////////////////////////////////// // (hash+inp[0:1])*r^4 and accumulate add v9.2s,v9.2s,v24.2s fmov d18,x12 umlal v22.2d,v11.2s,v1.s[0] ldp x8,x12,[x1],#16 // inp[0:1] umlal v19.2d,v11.2s,v6.s[0] ldp x9,x13,[x1],#48 umlal v23.2d,v11.2s,v3.s[0] umlal v20.2d,v11.2s,v8.s[0] umlal v21.2d,v11.2s,v0.s[0] #ifdef __AARCH64EB__ rev x8,x8 rev x12,x12 rev x9,x9 rev x13,x13 #endif add v10.2s,v10.2s,v25.2s umlal v22.2d,v9.2s,v5.s[0] umlal v23.2d,v9.2s,v7.s[0] and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 umlal v21.2d,v9.2s,v3.s[0] and x5,x9,#0x03ffffff umlal v19.2d,v9.2s,v0.s[0] ubfx x6,x8,#26,#26 umlal v20.2d,v9.2s,v1.s[0] ubfx x7,x9,#26,#26 add v12.2s,v12.2s,v27.2s add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 umlal v22.2d,v10.2s,v3.s[0] extr x8,x12,x8,#52 umlal v23.2d,v10.2s,v5.s[0] extr x9,x13,x9,#52 umlal v19.2d,v10.2s,v8.s[0] add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 umlal v21.2d,v10.2s,v1.s[0] fmov d9,x4 umlal v20.2d,v10.2s,v0.s[0] and x8,x8,#0x03ffffff add v13.2s,v13.2s,v28.2s and x9,x9,#0x03ffffff umlal v22.2d,v12.2s,v0.s[0] ubfx x10,x12,#14,#26 umlal v19.2d,v12.2s,v4.s[0] ubfx x11,x13,#14,#26 umlal v23.2d,v12.2s,v1.s[0] add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 umlal v20.2d,v12.2s,v6.s[0] fmov d10,x6 umlal v21.2d,v12.2s,v8.s[0] add x12,x3,x12,lsr#40 umlal v22.2d,v13.2s,v8.s[0] add x13,x3,x13,lsr#40 umlal v19.2d,v13.2s,v2.s[0] add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 umlal v23.2d,v13.2s,v0.s[0] add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 umlal v20.2d,v13.2s,v4.s[0] fmov d11,x8 umlal v21.2d,v13.2s,v6.s[0] fmov d12,x10 fmov d13,x12 ///////////////////////////////////////////////////////////////// // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein // and P. Schwabe // // [see discussion in poly1305-armv4 module] ushr v29.2d,v22.2d,#26 xtn v27.2s,v22.2d ushr v30.2d,v19.2d,#26 and v19.16b,v19.16b,v31.16b add v23.2d,v23.2d,v29.2d // h3 -> h4 bic v27.2s,#0xfc,lsl#24 // &=0x03ffffff add v20.2d,v20.2d,v30.2d // h0 -> h1 ushr v29.2d,v23.2d,#26 xtn v28.2s,v23.2d ushr v30.2d,v20.2d,#26 xtn v25.2s,v20.2d bic v28.2s,#0xfc,lsl#24 add v21.2d,v21.2d,v30.2d // h1 -> h2 add v19.2d,v19.2d,v29.2d shl v29.2d,v29.2d,#2 shrn v30.2s,v21.2d,#26 xtn v26.2s,v21.2d add v19.2d,v19.2d,v29.2d // h4 -> h0 bic v25.2s,#0xfc,lsl#24 add v27.2s,v27.2s,v30.2s // h2 -> h3 bic v26.2s,#0xfc,lsl#24 shrn v29.2s,v19.2d,#26 xtn v24.2s,v19.2d ushr v30.2s,v27.2s,#26 bic v27.2s,#0xfc,lsl#24 bic v24.2s,#0xfc,lsl#24 add v25.2s,v25.2s,v29.2s // h0 -> h1 add v28.2s,v28.2s,v30.2s // h3 -> h4 b.hi .Loop_neon .Lskip_loop: dup v16.2d,v16.d[0] add v11.2s,v11.2s,v26.2s //////////////////////////////////////////////////////////////// // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 adds x2,x2,#32 b.ne .Long_tail dup v16.2d,v11.d[0] add v14.2s,v9.2s,v24.2s add v17.2s,v12.2s,v27.2s add v15.2s,v10.2s,v25.2s add v18.2s,v13.2s,v28.2s .Long_tail: dup v14.2d,v14.d[0] umull2 v19.2d,v16.4s,v6.4s umull2 v22.2d,v16.4s,v1.4s umull2 v23.2d,v16.4s,v3.4s umull2 v21.2d,v16.4s,v0.4s umull2 v20.2d,v16.4s,v8.4s dup v15.2d,v15.d[0] umlal2 v19.2d,v14.4s,v0.4s umlal2 v21.2d,v14.4s,v3.4s umlal2 v22.2d,v14.4s,v5.4s umlal2 v23.2d,v14.4s,v7.4s umlal2 v20.2d,v14.4s,v1.4s dup v17.2d,v17.d[0] umlal2 v19.2d,v15.4s,v8.4s umlal2 v22.2d,v15.4s,v3.4s umlal2 v21.2d,v15.4s,v1.4s umlal2 v23.2d,v15.4s,v5.4s umlal2 v20.2d,v15.4s,v0.4s dup v18.2d,v18.d[0] umlal2 v22.2d,v17.4s,v0.4s umlal2 v23.2d,v17.4s,v1.4s umlal2 v19.2d,v17.4s,v4.4s umlal2 v20.2d,v17.4s,v6.4s umlal2 v21.2d,v17.4s,v8.4s umlal2 v22.2d,v18.4s,v8.4s umlal2 v19.2d,v18.4s,v2.4s umlal2 v23.2d,v18.4s,v0.4s umlal2 v20.2d,v18.4s,v4.4s umlal2 v21.2d,v18.4s,v6.4s b.eq .Lshort_tail //////////////////////////////////////////////////////////////// // (hash+inp[0:1])*r^4:r^3 and accumulate add v9.2s,v9.2s,v24.2s umlal v22.2d,v11.2s,v1.2s umlal v19.2d,v11.2s,v6.2s umlal v23.2d,v11.2s,v3.2s umlal v20.2d,v11.2s,v8.2s umlal v21.2d,v11.2s,v0.2s add v10.2s,v10.2s,v25.2s umlal v22.2d,v9.2s,v5.2s umlal v19.2d,v9.2s,v0.2s umlal v23.2d,v9.2s,v7.2s umlal v20.2d,v9.2s,v1.2s umlal v21.2d,v9.2s,v3.2s add v12.2s,v12.2s,v27.2s umlal v22.2d,v10.2s,v3.2s umlal v19.2d,v10.2s,v8.2s umlal v23.2d,v10.2s,v5.2s umlal v20.2d,v10.2s,v0.2s umlal v21.2d,v10.2s,v1.2s add v13.2s,v13.2s,v28.2s umlal v22.2d,v12.2s,v0.2s umlal v19.2d,v12.2s,v4.2s umlal v23.2d,v12.2s,v1.2s umlal v20.2d,v12.2s,v6.2s umlal v21.2d,v12.2s,v8.2s umlal v22.2d,v13.2s,v8.2s umlal v19.2d,v13.2s,v2.2s umlal v23.2d,v13.2s,v0.2s umlal v20.2d,v13.2s,v4.2s umlal v21.2d,v13.2s,v6.2s .Lshort_tail: //////////////////////////////////////////////////////////////// // horizontal add addp v22.2d,v22.2d,v22.2d ldp d8,d9,[sp,#16] // meet ABI requirements addp v19.2d,v19.2d,v19.2d ldp d10,d11,[sp,#32] addp v23.2d,v23.2d,v23.2d ldp d12,d13,[sp,#48] addp v20.2d,v20.2d,v20.2d ldp d14,d15,[sp,#64] addp v21.2d,v21.2d,v21.2d ldr x30,[sp,#8] .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing ushr v29.2d,v22.2d,#26 and v22.16b,v22.16b,v31.16b ushr v30.2d,v19.2d,#26 and v19.16b,v19.16b,v31.16b add v23.2d,v23.2d,v29.2d // h3 -> h4 add v20.2d,v20.2d,v30.2d // h0 -> h1 ushr v29.2d,v23.2d,#26 and v23.16b,v23.16b,v31.16b ushr v30.2d,v20.2d,#26 and v20.16b,v20.16b,v31.16b add v21.2d,v21.2d,v30.2d // h1 -> h2 add v19.2d,v19.2d,v29.2d shl v29.2d,v29.2d,#2 ushr v30.2d,v21.2d,#26 and v21.16b,v21.16b,v31.16b add v19.2d,v19.2d,v29.2d // h4 -> h0 add v22.2d,v22.2d,v30.2d // h2 -> h3 ushr v29.2d,v19.2d,#26 and v19.16b,v19.16b,v31.16b ushr v30.2d,v22.2d,#26 and v22.16b,v22.16b,v31.16b add v20.2d,v20.2d,v29.2d // h0 -> h1 add v23.2d,v23.2d,v30.2d // h3 -> h4 //////////////////////////////////////////////////////////////// // write the result, can be partially reduced st4 {v19.s,v20.s,v21.s,v22.s}[0],[x0],#16 mov x4,#1 st1 {v23.s}[0],[x0] str x4,[x0,#8] // set is_base2_26 ldr x29,[sp],#80 ret .size poly1305_blocks_neon,.-poly1305_blocks_neon .align 5 .Lzeros: .long 0,0,0,0,0,0,0,0 .asciz "Poly1305 for ARMv8, CRYPTOGAMS by @dot-asm" .align 2 #if !defined(__KERNEL__) && !defined(_WIN64) .comm OPENSSL_armcap_P,4,4 .hidden OPENSSL_armcap_P #endif