From ffc13a3b2c05f6e1aabf166fbeb1f046ed57e99f Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Thu, 18 Jan 2018 11:46:01 +0100 Subject: tools: import new curve25519 implementations --- src/tools/curve25519.c | 2552 +++++++++++++++++++++++++++--------------------- 1 file changed, 1423 insertions(+), 1129 deletions(-) (limited to 'src') diff --git a/src/tools/curve25519.c b/src/tools/curve25519.c index 3e770b2..ee7df84 100644 --- a/src/tools/curve25519.c +++ b/src/tools/curve25519.c @@ -1,1277 +1,1571 @@ /* SPDX-License-Identifier: GPL-2.0 * - * Copyright (C) 2008 Google Inc. All Rights Reserved. - * Copyright (C) 2015-2018 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2016-2017 INRIA and Microsoft Corporation. + * Copyright (C) 2015-2016 The fiat-crypto Authors. + * Copyright (C) 2018 Jason A. Donenfeld . All Rights Reserved. * - * Original author: Adam Langley + * This is a machine-generated formally verified implementation of curve25519 DH from: + * https://github.com/mitls/hacl-star and https://github.com/mit-plv/fiat-crypto */ #include "curve25519.h" #include #include +#include #ifndef __always_inline #define __always_inline __inline __attribute__((__always_inline__)) #endif +#ifndef __aligned +#define __aligned(x) __attribute__((aligned(x))) +#endif + #ifdef __SIZEOF_INT128__ -typedef uint64_t limb; -typedef limb felem[5]; typedef __uint128_t uint128_t; -/* Sum two numbers: output += in */ -static __always_inline void fsum(limb *output, const limb *in) +static __always_inline uint64_t uint64_t_eq_mask(uint64_t x, uint64_t y) { - output[0] += in[0]; - output[1] += in[1]; - output[2] += in[2]; - output[3] += in[3]; - output[4] += in[4]; + x = ~(x ^ y); + x &= x << 32; + x &= x << 16; + x &= x << 8; + x &= x << 4; + x &= x << 2; + x &= x << 1; + return ((int64_t)x) >> 63; } -/* Find the difference of two numbers: output = in - output - * (note the order of the arguments!) - * - * Assumes that out[i] < 2**52 - * On return, out[i] < 2**55 - */ -static __always_inline void fdifference_backwards(felem out, const felem in) +static __always_inline uint64_t uint64_t_gte_mask(uint64_t x, uint64_t y) { - /* 152 is 19 << 3 */ - static const limb two54m152 = (((limb)1) << 54) - 152; - static const limb two54m8 = (((limb)1) << 54) - 8; + uint64_t low63 = ~((uint64_t)((int64_t)((int64_t)(x & 0x7fffffffffffffffLLU) - (int64_t)(y & 0x7fffffffffffffffLLU)) >> 63)); + uint64_t high_bit = ~((uint64_t)((int64_t)((int64_t)(x & 0x8000000000000000LLU) - (int64_t)(y & 0x8000000000000000LLU)) >> 63)); + return low63 & high_bit; +} - out[0] = in[0] + two54m152 - out[0]; - out[1] = in[1] + two54m8 - out[1]; - out[2] = in[2] + two54m8 - out[2]; - out[3] = in[3] + two54m8 - out[3]; - out[4] = in[4] + two54m8 - out[4]; +static __always_inline void modulo_carry_top(uint64_t *b) +{ + uint64_t b4 = b[4]; + uint64_t b0 = b[0]; + uint64_t b4_ = b4 & 0x7ffffffffffffLLU; + uint64_t b0_ = b0 + 19 * (b4 >> 51); + b[4] = b4_; + b[0] = b0_; } -/* Multiply a number by a scalar: output = in * scalar */ -static __always_inline void fscalar_product(felem output, const felem in, const limb scalar) +static __always_inline void fproduct_copy_from_wide_(uint64_t *output, uint128_t *input) { - uint128_t a; + { + uint128_t xi = input[0]; + output[0] = ((uint64_t)(xi)); + } + { + uint128_t xi = input[1]; + output[1] = ((uint64_t)(xi)); + } + { + uint128_t xi = input[2]; + output[2] = ((uint64_t)(xi)); + } + { + uint128_t xi = input[3]; + output[3] = ((uint64_t)(xi)); + } + { + uint128_t xi = input[4]; + output[4] = ((uint64_t)(xi)); + } +} - a = ((uint128_t) in[0]) * scalar; - output[0] = ((limb)a) & 0x7ffffffffffffUL; +static __always_inline void fproduct_sum_scalar_multiplication_(uint128_t *output, uint64_t *input, uint64_t s) +{ + uint32_t i; + for (i = 0; i < 5; ++i) { + uint128_t xi = output[i]; + uint64_t yi = input[i]; + output[i] = ((xi) + (((uint128_t)(yi) * (s)))); + } +} - a = ((uint128_t) in[1]) * scalar + ((limb) (a >> 51)); - output[1] = ((limb)a) & 0x7ffffffffffffUL; +static __always_inline void fproduct_carry_wide_(uint128_t *tmp) +{ + uint32_t i; + for (i = 0; i < 4; ++i) { + uint32_t ctr = i; + uint128_t tctr = tmp[ctr]; + uint128_t tctrp1 = tmp[ctr + 1]; + uint64_t r0 = ((uint64_t)(tctr)) & 0x7ffffffffffffLLU; + uint128_t c = ((tctr) >> (51)); + tmp[ctr] = ((uint128_t)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } +} - a = ((uint128_t) in[2]) * scalar + ((limb) (a >> 51)); - output[2] = ((limb)a) & 0x7ffffffffffffUL; +static __always_inline void fmul_shift_reduce(uint64_t *output) +{ + uint64_t tmp = output[4]; + uint64_t b0; + { + uint32_t ctr = 5 - 0 - 1; + uint64_t z = output[ctr - 1]; + output[ctr] = z; + } + { + uint32_t ctr = 5 - 1 - 1; + uint64_t z = output[ctr - 1]; + output[ctr] = z; + } + { + uint32_t ctr = 5 - 2 - 1; + uint64_t z = output[ctr - 1]; + output[ctr] = z; + } + { + uint32_t ctr = 5 - 3 - 1; + uint64_t z = output[ctr - 1]; + output[ctr] = z; + } + output[0] = tmp; + b0 = output[0]; + output[0] = 19 * b0; +} - a = ((uint128_t) in[3]) * scalar + ((limb) (a >> 51)); - output[3] = ((limb)a) & 0x7ffffffffffffUL; +static __always_inline void fmul_mul_shift_reduce_(uint128_t *output, uint64_t *input, uint64_t *input21) +{ + uint32_t i; + uint64_t input2i; + { + uint64_t input2i = input21[0]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + uint64_t input2i = input21[1]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + uint64_t input2i = input21[2]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + uint64_t input2i = input21[3]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + i = 4; + input2i = input21[i]; + fproduct_sum_scalar_multiplication_(output, input, input2i); +} - a = ((uint128_t) in[4]) * scalar + ((limb) (a >> 51)); - output[4] = ((limb)a) & 0x7ffffffffffffUL; +static __always_inline void fmul_fmul(uint64_t *output, uint64_t *input, uint64_t *input21) +{ + uint64_t tmp[5]; + memcpy(tmp, input, 5 * sizeof(*input)); + { + uint128_t b4; + uint128_t b0; + uint128_t b4_; + uint128_t b0_; + uint64_t i0; + uint64_t i1; + uint64_t i0_; + uint64_t i1_; + uint128_t t[5]; + { + uint32_t _i; + for (_i = 0; _i < 5; ++_i) + t[_i] = ((uint128_t)(0)); + } + fmul_mul_shift_reduce_(t, tmp, input21); + fproduct_carry_wide_(t); + b4 = t[4]; + b0 = t[0]; + b4_ = ((b4) & (((uint128_t)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((uint128_t)(19) * (((uint64_t)(((b4) >> (51)))))))); + t[4] = b4_; + t[0] = b0_; + fproduct_copy_from_wide_(output, t); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; + } +} - output[0] += (a >> 51) * 19; +static __always_inline void fsquare_fsquare__(uint128_t *tmp, uint64_t *output) +{ + uint64_t r0 = output[0]; + uint64_t r1 = output[1]; + uint64_t r2 = output[2]; + uint64_t r3 = output[3]; + uint64_t r4 = output[4]; + uint64_t d0 = r0 * 2; + uint64_t d1 = r1 * 2; + uint64_t d2 = r2 * 2 * 19; + uint64_t d419 = r4 * 19; + uint64_t d4 = d419 * 2; + uint128_t s0 = ((((((uint128_t)(r0) * (r0))) + (((uint128_t)(d4) * (r1))))) + (((uint128_t)(d2) * (r3)))); + uint128_t s1 = ((((((uint128_t)(d0) * (r1))) + (((uint128_t)(d4) * (r2))))) + (((uint128_t)(r3 * 19) * (r3)))); + uint128_t s2 = ((((((uint128_t)(d0) * (r2))) + (((uint128_t)(r1) * (r1))))) + (((uint128_t)(d4) * (r3)))); + uint128_t s3 = ((((((uint128_t)(d0) * (r3))) + (((uint128_t)(d1) * (r2))))) + (((uint128_t)(r4) * (d419)))); + uint128_t s4 = ((((((uint128_t)(d0) * (r4))) + (((uint128_t)(d1) * (r3))))) + (((uint128_t)(r2) * (r2)))); + tmp[0] = s0; + tmp[1] = s1; + tmp[2] = s2; + tmp[3] = s3; + tmp[4] = s4; } -/* Multiply two numbers: output = in2 * in - * - * output must be distinct to both inputs. The inputs are reduced coefficient - * form, the output is not. - * - * Assumes that in[i] < 2**55 and likewise for in2. - * On return, output[i] < 2**52 - */ -static __always_inline void fmul(felem output, const felem in2, const felem in) +static __always_inline void fsquare_fsquare_(uint128_t *tmp, uint64_t *output) { - uint128_t t[5]; - limb r0, r1, r2, r3, r4, s0, s1, s2, s3, s4, c; - - r0 = in[0]; - r1 = in[1]; - r2 = in[2]; - r3 = in[3]; - r4 = in[4]; - - s0 = in2[0]; - s1 = in2[1]; - s2 = in2[2]; - s3 = in2[3]; - s4 = in2[4]; - - t[0] = ((uint128_t) r0) * s0; - t[1] = ((uint128_t) r0) * s1 + ((uint128_t) r1) * s0; - t[2] = ((uint128_t) r0) * s2 + ((uint128_t) r2) * s0 + ((uint128_t) r1) * s1; - t[3] = ((uint128_t) r0) * s3 + ((uint128_t) r3) * s0 + ((uint128_t) r1) * s2 + ((uint128_t) r2) * s1; - t[4] = ((uint128_t) r0) * s4 + ((uint128_t) r4) * s0 + ((uint128_t) r3) * s1 + ((uint128_t) r1) * s3 + ((uint128_t) r2) * s2; - - r4 *= 19; - r1 *= 19; - r2 *= 19; - r3 *= 19; - - t[0] += ((uint128_t) r4) * s1 + ((uint128_t) r1) * s4 + ((uint128_t) r2) * s3 + ((uint128_t) r3) * s2; - t[1] += ((uint128_t) r4) * s2 + ((uint128_t) r2) * s4 + ((uint128_t) r3) * s3; - t[2] += ((uint128_t) r4) * s3 + ((uint128_t) r3) * s4; - t[3] += ((uint128_t) r4) * s4; - - r0 = (limb)t[0] & 0x7ffffffffffffUL; c = (limb)(t[0] >> 51); - t[1] += c; r1 = (limb)t[1] & 0x7ffffffffffffUL; c = (limb)(t[1] >> 51); - t[2] += c; r2 = (limb)t[2] & 0x7ffffffffffffUL; c = (limb)(t[2] >> 51); - t[3] += c; r3 = (limb)t[3] & 0x7ffffffffffffUL; c = (limb)(t[3] >> 51); - t[4] += c; r4 = (limb)t[4] & 0x7ffffffffffffUL; c = (limb)(t[4] >> 51); - r0 += c * 19; c = r0 >> 51; r0 = r0 & 0x7ffffffffffffUL; - r1 += c; c = r1 >> 51; r1 = r1 & 0x7ffffffffffffUL; - r2 += c; - - output[0] = r0; - output[1] = r1; - output[2] = r2; - output[3] = r3; - output[4] = r4; -} - -static __always_inline void fsquare_times(felem output, const felem in, limb count) + uint128_t b4; + uint128_t b0; + uint128_t b4_; + uint128_t b0_; + uint64_t i0; + uint64_t i1; + uint64_t i0_; + uint64_t i1_; + fsquare_fsquare__(tmp, output); + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((uint128_t)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((uint128_t)(19) * (((uint64_t)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; +} + +static __always_inline void fsquare_fsquare_times_(uint64_t *input, uint128_t *tmp, uint32_t count1) { - uint128_t t[5]; - limb r0, r1, r2, r3, r4, c; - limb d0, d1, d2, d4, d419; - - r0 = in[0]; - r1 = in[1]; - r2 = in[2]; - r3 = in[3]; - r4 = in[4]; - - do { - d0 = r0 * 2; - d1 = r1 * 2; - d2 = r2 * 2 * 19; - d419 = r4 * 19; - d4 = d419 * 2; - - t[0] = ((uint128_t) r0) * r0 + ((uint128_t) d4) * r1 + (((uint128_t) d2) * (r3 )); - t[1] = ((uint128_t) d0) * r1 + ((uint128_t) d4) * r2 + (((uint128_t) r3) * (r3 * 19)); - t[2] = ((uint128_t) d0) * r2 + ((uint128_t) r1) * r1 + (((uint128_t) d4) * (r3 )); - t[3] = ((uint128_t) d0) * r3 + ((uint128_t) d1) * r2 + (((uint128_t) r4) * (d419 )); - t[4] = ((uint128_t) d0) * r4 + ((uint128_t) d1) * r3 + (((uint128_t) r2) * (r2 )); - - r0 = (limb)t[0] & 0x7ffffffffffffUL; c = (limb)(t[0] >> 51); - t[1] += c; r1 = (limb)t[1] & 0x7ffffffffffffUL; c = (limb)(t[1] >> 51); - t[2] += c; r2 = (limb)t[2] & 0x7ffffffffffffUL; c = (limb)(t[2] >> 51); - t[3] += c; r3 = (limb)t[3] & 0x7ffffffffffffUL; c = (limb)(t[3] >> 51); - t[4] += c; r4 = (limb)t[4] & 0x7ffffffffffffUL; c = (limb)(t[4] >> 51); - r0 += c * 19; c = r0 >> 51; r0 = r0 & 0x7ffffffffffffUL; - r1 += c; c = r1 >> 51; r1 = r1 & 0x7ffffffffffffUL; - r2 += c; - } while (--count); - - output[0] = r0; - output[1] = r1; - output[2] = r2; - output[3] = r3; - output[4] = r4; -} - -/* Load a little-endian 64-bit number */ -static limb load_limb(const uint8_t *in) -{ - return - ((limb)in[0]) | - (((limb)in[1]) << 8) | - (((limb)in[2]) << 16) | - (((limb)in[3]) << 24) | - (((limb)in[4]) << 32) | - (((limb)in[5]) << 40) | - (((limb)in[6]) << 48) | - (((limb)in[7]) << 56); -} - -static void store_limb(uint8_t *out, limb in) -{ - out[0] = in & 0xff; - out[1] = (in >> 8) & 0xff; - out[2] = (in >> 16) & 0xff; - out[3] = (in >> 24) & 0xff; - out[4] = (in >> 32) & 0xff; - out[5] = (in >> 40) & 0xff; - out[6] = (in >> 48) & 0xff; - out[7] = (in >> 56) & 0xff; -} - -/* Take a little-endian, 32-byte number and expand it into polynomial form */ -static void fexpand(limb *output, const uint8_t *in) -{ - output[0] = load_limb(in) & 0x7ffffffffffffUL; - output[1] = (load_limb(in+6) >> 3) & 0x7ffffffffffffUL; - output[2] = (load_limb(in+12) >> 6) & 0x7ffffffffffffUL; - output[3] = (load_limb(in+19) >> 1) & 0x7ffffffffffffUL; - output[4] = (load_limb(in+24) >> 12) & 0x7ffffffffffffUL; -} - -/* Take a fully reduced polynomial form number and contract it into a - * little-endian, 32-byte array - */ -static void fcontract(uint8_t *output, const felem input) + uint32_t i; + fsquare_fsquare_(tmp, input); + for (i = 1; i < count1; ++i) + fsquare_fsquare_(tmp, input); +} + +static __always_inline void fsquare_fsquare_times(uint64_t *output, uint64_t *input, uint32_t count1) { uint128_t t[5]; + { + uint32_t _i; + for (_i = 0; _i < 5; ++_i) + t[_i] = ((uint128_t)(0)); + } + memcpy(output, input, 5 * sizeof(*input)); + fsquare_fsquare_times_(output, t, count1); +} - t[0] = input[0]; - t[1] = input[1]; - t[2] = input[2]; - t[3] = input[3]; - t[4] = input[4]; - - t[1] += t[0] >> 51; t[0] &= 0x7ffffffffffffUL; - t[2] += t[1] >> 51; t[1] &= 0x7ffffffffffffUL; - t[3] += t[2] >> 51; t[2] &= 0x7ffffffffffffUL; - t[4] += t[3] >> 51; t[3] &= 0x7ffffffffffffUL; - t[0] += 19 * (t[4] >> 51); t[4] &= 0x7ffffffffffffUL; - - t[1] += t[0] >> 51; t[0] &= 0x7ffffffffffffUL; - t[2] += t[1] >> 51; t[1] &= 0x7ffffffffffffUL; - t[3] += t[2] >> 51; t[2] &= 0x7ffffffffffffUL; - t[4] += t[3] >> 51; t[3] &= 0x7ffffffffffffUL; - t[0] += 19 * (t[4] >> 51); t[4] &= 0x7ffffffffffffUL; +static __always_inline void fsquare_fsquare_times_inplace(uint64_t *output, uint32_t count1) +{ + uint128_t t[5]; + { + uint32_t _i; + for (_i = 0; _i < 5; ++_i) + t[_i] = ((uint128_t)(0)); + } + fsquare_fsquare_times_(output, t, count1); +} - /* now t is between 0 and 2^255-1, properly carried. */ - /* case 1: between 0 and 2^255-20. case 2: between 2^255-19 and 2^255-1. */ +static __always_inline void crecip_crecip(uint64_t *out, uint64_t *z) +{ + uint64_t buf[20] = { 0 }; + uint64_t *a0 = buf; + uint64_t *t00 = buf + 5; + uint64_t *b0 = buf + 10; + uint64_t *t01; + uint64_t *b1; + uint64_t *c0; + uint64_t *a; + uint64_t *t0; + uint64_t *b; + uint64_t *c; + fsquare_fsquare_times(a0, z, 1); + fsquare_fsquare_times(t00, a0, 2); + fmul_fmul(b0, t00, z); + fmul_fmul(a0, b0, a0); + fsquare_fsquare_times(t00, a0, 1); + fmul_fmul(b0, t00, b0); + fsquare_fsquare_times(t00, b0, 5); + t01 = buf + 5; + b1 = buf + 10; + c0 = buf + 15; + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 10); + fmul_fmul(c0, t01, b1); + fsquare_fsquare_times(t01, c0, 20); + fmul_fmul(t01, t01, c0); + fsquare_fsquare_times_inplace(t01, 10); + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 50); + a = buf; + t0 = buf + 5; + b = buf + 10; + c = buf + 15; + fmul_fmul(c, t0, b); + fsquare_fsquare_times(t0, c, 100); + fmul_fmul(t0, t0, c); + fsquare_fsquare_times_inplace(t0, 50); + fmul_fmul(t0, t0, b); + fsquare_fsquare_times_inplace(t0, 5); + fmul_fmul(out, t0, a); +} - t[0] += 19; +static __always_inline void fsum(uint64_t *a, uint64_t *b) +{ + uint32_t i; + for (i = 0; i < 5; ++i) { + uint64_t xi = a[i]; + uint64_t yi = b[i]; + a[i] = xi + yi; + } +} - t[1] += t[0] >> 51; t[0] &= 0x7ffffffffffffUL; - t[2] += t[1] >> 51; t[1] &= 0x7ffffffffffffUL; - t[3] += t[2] >> 51; t[2] &= 0x7ffffffffffffUL; - t[4] += t[3] >> 51; t[3] &= 0x7ffffffffffffUL; - t[0] += 19 * (t[4] >> 51); t[4] &= 0x7ffffffffffffUL; +static __always_inline void fdifference(uint64_t *a, uint64_t *b) +{ + uint64_t tmp[5] = { 0 }; + uint64_t b0; + uint64_t b1; + uint64_t b2; + uint64_t b3; + uint64_t b4; + memcpy(tmp, b, 5 * sizeof(*b)); + b0 = tmp[0]; + b1 = tmp[1]; + b2 = tmp[2]; + b3 = tmp[3]; + b4 = tmp[4]; + tmp[0] = b0 + 0x3fffffffffff68LLU; + tmp[1] = b1 + 0x3ffffffffffff8LLU; + tmp[2] = b2 + 0x3ffffffffffff8LLU; + tmp[3] = b3 + 0x3ffffffffffff8LLU; + tmp[4] = b4 + 0x3ffffffffffff8LLU; + { + uint64_t xi = a[0]; + uint64_t yi = tmp[0]; + a[0] = yi - xi; + } + { + uint64_t xi = a[1]; + uint64_t yi = tmp[1]; + a[1] = yi - xi; + } + { + uint64_t xi = a[2]; + uint64_t yi = tmp[2]; + a[2] = yi - xi; + } + { + uint64_t xi = a[3]; + uint64_t yi = tmp[3]; + a[3] = yi - xi; + } + { + uint64_t xi = a[4]; + uint64_t yi = tmp[4]; + a[4] = yi - xi; + } +} - /* now between 19 and 2^255-1 in both cases, and offset by 19. */ +static __always_inline void fscalar(uint64_t *output, uint64_t *b, uint64_t s) +{ + uint128_t tmp[5]; + uint128_t b4; + uint128_t b0; + uint128_t b4_; + uint128_t b0_; + { + uint64_t xi = b[0]; + tmp[0] = ((uint128_t)(xi) * (s)); + } + { + uint64_t xi = b[1]; + tmp[1] = ((uint128_t)(xi) * (s)); + } + { + uint64_t xi = b[2]; + tmp[2] = ((uint128_t)(xi) * (s)); + } + { + uint64_t xi = b[3]; + tmp[3] = ((uint128_t)(xi) * (s)); + } + { + uint64_t xi = b[4]; + tmp[4] = ((uint128_t)(xi) * (s)); + } + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((uint128_t)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((uint128_t)(19) * (((uint64_t)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); +} - t[0] += 0x8000000000000UL - 19; - t[1] += 0x8000000000000UL - 1; - t[2] += 0x8000000000000UL - 1; - t[3] += 0x8000000000000UL - 1; - t[4] += 0x8000000000000UL - 1; +static __always_inline void fmul(uint64_t *output, uint64_t *a, uint64_t *b) +{ + fmul_fmul(output, a, b); +} - /* now between 2^255 and 2^256-20, and offset by 2^255. */ +static __always_inline void crecip(uint64_t *output, uint64_t *input) +{ + crecip_crecip(output, input); +} - t[1] += t[0] >> 51; t[0] &= 0x7ffffffffffffUL; - t[2] += t[1] >> 51; t[1] &= 0x7ffffffffffffUL; - t[3] += t[2] >> 51; t[2] &= 0x7ffffffffffffUL; - t[4] += t[3] >> 51; t[3] &= 0x7ffffffffffffUL; - t[4] &= 0x7ffffffffffffUL; +static __always_inline void point_swap_conditional_step(uint64_t *a, uint64_t *b, uint64_t swap1, uint32_t ctr) +{ + uint32_t i = ctr - 1; + uint64_t ai = a[i]; + uint64_t bi = b[i]; + uint64_t x = swap1 & (ai ^ bi); + uint64_t ai1 = ai ^ x; + uint64_t bi1 = bi ^ x; + a[i] = ai1; + b[i] = bi1; +} - store_limb(output, t[0] | (t[1] << 51)); - store_limb(output+8, (t[1] >> 13) | (t[2] << 38)); - store_limb(output+16, (t[2] >> 26) | (t[3] << 25)); - store_limb(output+24, (t[3] >> 39) | (t[4] << 12)); +static __always_inline void point_swap_conditional_(uint64_t *a, uint64_t *b, uint64_t swap1, uint32_t ctr) +{ + uint32_t i; + for (i = ctr; i > 0; --i) + point_swap_conditional_step(a, b, swap1, i); } -/* Input: Q, Q', Q-Q' - * Output: 2Q, Q+Q' - * - * x2 z3: long form - * x3 z3: long form - * x z: short form, destroyed - * xprime zprime: short form, destroyed - * qmqp: short form, preserved - */ -static void fmonty(limb *x2, limb *z2, /* output 2Q */ - limb *x3, limb *z3, /* output Q + Q' */ - limb *x, limb *z, /* input Q */ - limb *xprime, limb *zprime, /* input Q' */ +static __always_inline void point_swap_conditional(uint64_t *a, uint64_t *b, uint64_t iswap) +{ + uint64_t swap1 = 0 - iswap; + point_swap_conditional_(a, b, swap1, 5); + point_swap_conditional_(a + 5, b + 5, swap1, 5); +} - const limb *qmqp /* input Q - Q' */) +static __always_inline void point_copy(uint64_t *output, uint64_t *input) { - limb origx[5], origxprime[5], zzz[5], xx[5], zz[5], xxprime[5], zzprime[5], zzzprime[5]; + memcpy(output, input, 5 * sizeof(*input)); + memcpy(output + 5, input + 5, 5 * sizeof(*input)); +} - memcpy(origx, x, 5 * sizeof(limb)); +static __always_inline void addanddouble_fmonty(uint64_t *pp, uint64_t *ppq, uint64_t *p, uint64_t *pq, uint64_t *qmqp) +{ + uint64_t *qx = qmqp; + uint64_t *x2 = pp; + uint64_t *z2 = pp + 5; + uint64_t *x3 = ppq; + uint64_t *z3 = ppq + 5; + uint64_t *x = p; + uint64_t *z = p + 5; + uint64_t *xprime = pq; + uint64_t *zprime = pq + 5; + uint64_t buf[40] = { 0 }; + uint64_t *origx = buf; + uint64_t *origxprime0 = buf + 5; + uint64_t *xxprime0; + uint64_t *zzprime0; + uint64_t *origxprime; + xxprime0 = buf + 25; + zzprime0 = buf + 30; + memcpy(origx, x, 5 * sizeof(*x)); fsum(x, z); - fdifference_backwards(z, origx); // does x - z - - memcpy(origxprime, xprime, sizeof(limb) * 5); + fdifference(z, origx); + memcpy(origxprime0, xprime, 5 * sizeof(*xprime)); fsum(xprime, zprime); - fdifference_backwards(zprime, origxprime); - fmul(xxprime, xprime, z); - fmul(zzprime, x, zprime); - memcpy(origxprime, xxprime, sizeof(limb) * 5); - fsum(xxprime, zzprime); - fdifference_backwards(zzprime, origxprime); - fsquare_times(x3, xxprime, 1); - fsquare_times(zzzprime, zzprime, 1); - fmul(z3, zzzprime, qmqp); - - fsquare_times(xx, x, 1); - fsquare_times(zz, z, 1); - fmul(x2, xx, zz); - fdifference_backwards(zz, xx); // does zz = xx - zz - fscalar_product(zzz, zz, 121665); - fsum(zzz, xx); - fmul(z2, zz, zzz); -} - -/* Maybe swap the contents of two limb arrays (@a and @b), each @len elements - * long. Perform the swap iff @swap is non-zero. - * - * This function performs the swap without leaking any side-channel - * information. - */ -static void swap_conditional(limb a[static 5], limb b[static 5], limb iswap) + fdifference(zprime, origxprime0); + fmul(xxprime0, xprime, z); + fmul(zzprime0, x, zprime); + origxprime = buf + 5; + { + uint64_t *xx0; + uint64_t *zz0; + uint64_t *xxprime; + uint64_t *zzprime; + uint64_t *zzzprime; + xx0 = buf + 15; + zz0 = buf + 20; + xxprime = buf + 25; + zzprime = buf + 30; + zzzprime = buf + 35; + memcpy(origxprime, xxprime, 5 * sizeof(*xxprime)); + fsum(xxprime, zzprime); + fdifference(zzprime, origxprime); + fsquare_fsquare_times(x3, xxprime, 1); + fsquare_fsquare_times(zzzprime, zzprime, 1); + fmul(z3, zzzprime, qx); + fsquare_fsquare_times(xx0, x, 1); + fsquare_fsquare_times(zz0, z, 1); + { + uint64_t *zzz; + uint64_t *xx; + uint64_t *zz; + uint64_t scalar; + zzz = buf + 10; + xx = buf + 15; + zz = buf + 20; + fmul(x2, xx, zz); + fdifference(zz, xx); + scalar = 121665; + fscalar(zzz, zz, scalar); + fsum(zzz, xx); + fmul(z2, zzz, zz); + } + } +} + +static __always_inline void ladder_smallloop_cmult_small_loop_step(uint64_t *nq, uint64_t *nqpq, uint64_t *nq2, uint64_t *nqpq2, uint64_t *q, uint8_t byt) { - unsigned int i; - const limb swap = -iswap; + uint64_t bit0 = (uint64_t)(byt >> 7); + uint64_t bit; + point_swap_conditional(nq, nqpq, bit0); + addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q); + bit = (uint64_t)(byt >> 7); + point_swap_conditional(nq2, nqpq2, bit); +} - for (i = 0; i < 5; ++i) { - const limb x = swap & (a[i] ^ b[i]); +static __always_inline void ladder_smallloop_cmult_small_loop_double_step(uint64_t *nq, uint64_t *nqpq, uint64_t *nq2, uint64_t *nqpq2, uint64_t *q, uint8_t byt) +{ + uint8_t byt1; + ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt); + byt1 = byt << 1; + ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1); +} - a[i] ^= x; - b[i] ^= x; +static __always_inline void ladder_smallloop_cmult_small_loop(uint64_t *nq, uint64_t *nqpq, uint64_t *nq2, uint64_t *nqpq2, uint64_t *q, uint8_t byt, uint32_t i) +{ + while (i--) { + ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2, nqpq2, q, byt); + byt <<= 2; } } -/* Calculates nQ where Q is the x-coordinate of a point on the curve - * - * resultx/resultz: the x coordinate of the resulting curve point (short form) - * n: a little endian, 32-byte number - * q: a point of the curve (short form) - */ -static void cmult(limb *resultx, limb *resultz, const uint8_t *n, const limb *q) -{ - limb a[5] = {0}, b[5] = {1}, c[5] = {1}, d[5] = {0}; - limb *nqpqx = a, *nqpqz = b, *nqx = c, *nqz = d, *t; - limb e[5] = {0}, f[5] = {1}, g[5] = {0}, h[5] = {1}; - limb *nqpqx2 = e, *nqpqz2 = f, *nqx2 = g, *nqz2 = h; - - unsigned int i, j; - - memcpy(nqpqx, q, sizeof(limb) * 5); - - for (i = 0; i < 32; ++i) { - uint8_t byte = n[31 - i]; - - for (j = 0; j < 8; ++j) { - const limb bit = byte >> 7; - - swap_conditional(nqx, nqpqx, bit); - swap_conditional(nqz, nqpqz, bit); - fmonty(nqx2, nqz2, - nqpqx2, nqpqz2, - nqx, nqz, - nqpqx, nqpqz, - q); - swap_conditional(nqx2, nqpqx2, bit); - swap_conditional(nqz2, nqpqz2, bit); - - t = nqx; - nqx = nqx2; - nqx2 = t; - t = nqz; - nqz = nqz2; - nqz2 = t; - t = nqpqx; - nqpqx = nqpqx2; - nqpqx2 = t; - t = nqpqz; - nqpqz = nqpqz2; - nqpqz2 = t; - - byte <<= 1; - } +static __always_inline void ladder_bigloop_cmult_big_loop(uint8_t *n1, uint64_t *nq, uint64_t *nqpq, uint64_t *nq2, uint64_t *nqpq2, uint64_t *q, uint32_t i) +{ + while (i--) { + uint8_t byte = n1[i]; + ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q, byte, 4); } - - memcpy(resultx, nqx, sizeof(limb) * 5); - memcpy(resultz, nqz, sizeof(limb) * 5); -} - -static void crecip(felem out, const felem z) -{ - felem a, t0, b, c; - - /* 2 */ fsquare_times(a, z, 1); // a = 2 - /* 8 */ fsquare_times(t0, a, 2); - /* 9 */ fmul(b, t0, z); // b = 9 - /* 11 */ fmul(a, b, a); // a = 11 - /* 22 */ fsquare_times(t0, a, 1); - /* 2^5 - 2^0 = 31 */ fmul(b, t0, b); - /* 2^10 - 2^5 */ fsquare_times(t0, b, 5); - /* 2^10 - 2^0 */ fmul(b, t0, b); - /* 2^20 - 2^10 */ fsquare_times(t0, b, 10); - /* 2^20 - 2^0 */ fmul(c, t0, b); - /* 2^40 - 2^20 */ fsquare_times(t0, c, 20); - /* 2^40 - 2^0 */ fmul(t0, t0, c); - /* 2^50 - 2^10 */ fsquare_times(t0, t0, 10); - /* 2^50 - 2^0 */ fmul(b, t0, b); - /* 2^100 - 2^50 */ fsquare_times(t0, b, 50); - /* 2^100 - 2^0 */ fmul(c, t0, b); - /* 2^200 - 2^100 */ fsquare_times(t0, c, 100); - /* 2^200 - 2^0 */ fmul(t0, t0, c); - /* 2^250 - 2^50 */ fsquare_times(t0, t0, 50); - /* 2^250 - 2^0 */ fmul(t0, t0, b); - /* 2^255 - 2^5 */ fsquare_times(t0, t0, 5); - /* 2^255 - 21 */ fmul(out, t0, a); } -void curve25519(uint8_t mypublic[static CURVE25519_POINT_SIZE], const uint8_t secret[static CURVE25519_POINT_SIZE], const uint8_t basepoint[static CURVE25519_POINT_SIZE]) +static __always_inline void ladder_cmult(uint64_t *result, uint8_t *n1, uint64_t *q) { - limb bp[5], x[5], z[5], zmone[5]; - uint8_t e[32]; + uint64_t point_buf[40] = { 0 }; + uint64_t *nq = point_buf; + uint64_t *nqpq = point_buf + 10; + uint64_t *nq2 = point_buf + 20; + uint64_t *nqpq2 = point_buf + 30; + point_copy(nqpq, q); + nq[0] = 1; + ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32); + point_copy(result, nq); +} - memcpy(e, secret, 32); - curve25519_normalize_secret(e); +static __always_inline void format_fexpand(uint64_t *output, const uint8_t *input) +{ + const uint8_t *x00 = input + 6; + const uint8_t *x01 = input + 12; + const uint8_t *x02 = input + 19; + const uint8_t *x0 = input + 24; + uint64_t i0, i1, i2, i3, i4, output0, output1, output2, output3, output4; + i0 = le64toh(*(uint64_t *)input); + i1 = le64toh(*(uint64_t *)x00); + i2 = le64toh(*(uint64_t *)x01); + i3 = le64toh(*(uint64_t *)x02); + i4 = le64toh(*(uint64_t *)x0); + output0 = i0 & 0x7ffffffffffffLLU; + output1 = i1 >> 3 & 0x7ffffffffffffLLU; + output2 = i2 >> 6 & 0x7ffffffffffffLLU; + output3 = i3 >> 1 & 0x7ffffffffffffLLU; + output4 = i4 >> 12 & 0x7ffffffffffffLLU; + output[0] = output0; + output[1] = output1; + output[2] = output2; + output[3] = output3; + output[4] = output4; +} - fexpand(bp, basepoint); - cmult(x, z, e, bp); - crecip(zmone, z); - fmul(z, x, zmone); - fcontract(mypublic, z); +static __always_inline void format_fcontract_first_carry_pass(uint64_t *input) +{ + uint64_t t0 = input[0]; + uint64_t t1 = input[1]; + uint64_t t2 = input[2]; + uint64_t t3 = input[3]; + uint64_t t4 = input[4]; + uint64_t t1_ = t1 + (t0 >> 51); + uint64_t t0_ = t0 & 0x7ffffffffffffLLU; + uint64_t t2_ = t2 + (t1_ >> 51); + uint64_t t1__ = t1_ & 0x7ffffffffffffLLU; + uint64_t t3_ = t3 + (t2_ >> 51); + uint64_t t2__ = t2_ & 0x7ffffffffffffLLU; + uint64_t t4_ = t4 + (t3_ >> 51); + uint64_t t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; } -#else -typedef int64_t limb; +static __always_inline void format_fcontract_first_carry_full(uint64_t *input) +{ + format_fcontract_first_carry_pass(input); + modulo_carry_top(input); +} -/* Field element representation: - * - * Field elements are written as an array of signed, 64-bit limbs, least - * significant first. The value of the field element is: - * x[0] + 2^26·x[1] + x^51·x[2] + 2^102·x[3] + ... - * - * i.e. the limbs are 26, 25, 26, 25, ... bits wide. - */ +static __always_inline void format_fcontract_second_carry_pass(uint64_t *input) +{ + uint64_t t0 = input[0]; + uint64_t t1 = input[1]; + uint64_t t2 = input[2]; + uint64_t t3 = input[3]; + uint64_t t4 = input[4]; + uint64_t t1_ = t1 + (t0 >> 51); + uint64_t t0_ = t0 & 0x7ffffffffffffLLU; + uint64_t t2_ = t2 + (t1_ >> 51); + uint64_t t1__ = t1_ & 0x7ffffffffffffLLU; + uint64_t t3_ = t3 + (t2_ >> 51); + uint64_t t2__ = t2_ & 0x7ffffffffffffLLU; + uint64_t t4_ = t4 + (t3_ >> 51); + uint64_t t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} -/* Sum two numbers: output += in */ -static void fsum(limb *output, const limb *in) +static __always_inline void format_fcontract_second_carry_full(uint64_t *input) { - unsigned int i; + uint64_t i0; + uint64_t i1; + uint64_t i0_; + uint64_t i1_; + format_fcontract_second_carry_pass(input); + modulo_carry_top(input); + i0 = input[0]; + i1 = input[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + input[0] = i0_; + input[1] = i1_; +} - for (i = 0; i < 10; i += 2) { - output[0 + i] = output[0 + i] + in[0 + i]; - output[1 + i] = output[1 + i] + in[1 + i]; - } +static __always_inline void format_fcontract_trim(uint64_t *input) +{ + uint64_t a0 = input[0]; + uint64_t a1 = input[1]; + uint64_t a2 = input[2]; + uint64_t a3 = input[3]; + uint64_t a4 = input[4]; + uint64_t mask0 = uint64_t_gte_mask(a0, 0x7ffffffffffedLLU); + uint64_t mask1 = uint64_t_eq_mask(a1, 0x7ffffffffffffLLU); + uint64_t mask2 = uint64_t_eq_mask(a2, 0x7ffffffffffffLLU); + uint64_t mask3 = uint64_t_eq_mask(a3, 0x7ffffffffffffLLU); + uint64_t mask4 = uint64_t_eq_mask(a4, 0x7ffffffffffffLLU); + uint64_t mask = (((mask0 & mask1) & mask2) & mask3) & mask4; + uint64_t a0_ = a0 - (0x7ffffffffffedLLU & mask); + uint64_t a1_ = a1 - (0x7ffffffffffffLLU & mask); + uint64_t a2_ = a2 - (0x7ffffffffffffLLU & mask); + uint64_t a3_ = a3 - (0x7ffffffffffffLLU & mask); + uint64_t a4_ = a4 - (0x7ffffffffffffLLU & mask); + input[0] = a0_; + input[1] = a1_; + input[2] = a2_; + input[3] = a3_; + input[4] = a4_; } -/* Find the difference of two numbers: output = in - output - * (note the order of the arguments!). - */ -static void fdifference(limb *output, const limb *in) +static __always_inline void format_fcontract_store(uint8_t *output, uint64_t *input) { - unsigned int i; + uint64_t t0 = input[0]; + uint64_t t1 = input[1]; + uint64_t t2 = input[2]; + uint64_t t3 = input[3]; + uint64_t t4 = input[4]; + uint64_t o0 = t1 << 51 | t0; + uint64_t o1 = t2 << 38 | t1 >> 13; + uint64_t o2 = t3 << 25 | t2 >> 26; + uint64_t o3 = t4 << 12 | t3 >> 39; + uint8_t *b0 = output; + uint8_t *b1 = output + 8; + uint8_t *b2 = output + 16; + uint8_t *b3 = output + 24; + *(uint64_t *)b0 = htole64(o0); + *(uint64_t *)b1 = htole64(o1); + *(uint64_t *)b2 = htole64(o2); + *(uint64_t *)b3 = htole64(o3); +} - for (i = 0; i < 10; ++i) { - output[i] = in[i] - output[i]; - } +static __always_inline void format_fcontract(uint8_t *output, uint64_t *input) +{ + format_fcontract_first_carry_full(input); + format_fcontract_second_carry_full(input); + format_fcontract_trim(input); + format_fcontract_store(output, input); } -/* Multiply a number by a scalar: output = in * scalar */ -static void fscalar_product(limb *output, const limb *in, const limb scalar) +static __always_inline void format_scalar_of_point(uint8_t *scalar, uint64_t *point) { - unsigned int i; + uint64_t *x = point; + uint64_t *z = point + 5; + uint64_t buf[10] __aligned(32) = { 0 }; + uint64_t *zmone = buf; + uint64_t *sc = buf + 5; + crecip(zmone, z); + fmul(sc, x, zmone); + format_fcontract(scalar, sc); +} - for (i = 0; i < 10; ++i) { - output[i] = in[i] * scalar; +void curve25519(uint8_t mypublic[static CURVE25519_POINT_SIZE], const uint8_t secret[static CURVE25519_POINT_SIZE], const uint8_t basepoint[static CURVE25519_POINT_SIZE]) +{ + uint64_t buf0[10] __aligned(32) = { 0 }; + uint64_t *x0 = buf0; + uint64_t *z = buf0 + 5; + uint64_t *q; + format_fexpand(x0, basepoint); + z[0] = 1; + q = buf0; + { + uint8_t e[32] __aligned(32) = { 0 }; + uint8_t *scalar; + memcpy(e, secret, 32); + curve25519_normalize_secret(e); + scalar = e; + { + uint64_t buf[15] = { 0 }; + uint64_t *nq = buf; + uint64_t *x = nq; + x[0] = 1; + ladder_cmult(nq, scalar, q); + format_scalar_of_point(mypublic, nq); + } } } - -/* Multiply two numbers: output = in2 * in - * - * output must be distinct to both inputs. The inputs are reduced coefficient - * form, the output is not. - * - * output[x] <= 14 * the largest product of the input limbs. - */ -static void fproduct(limb *output, const limb *in2, const limb *in) -{ - output[0] = ((limb) ((int32_t) in2[0])) * ((int32_t) in[0]); - output[1] = ((limb) ((int32_t) in2[0])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[0]); - output[2] = 2 * ((limb) ((int32_t) in2[1])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[0]); - output[3] = ((limb) ((int32_t) in2[1])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[0]); - output[4] = ((limb) ((int32_t) in2[2])) * ((int32_t) in[2]) + - 2 * (((limb) ((int32_t) in2[1])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[1])) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[0]); - output[5] = ((limb) ((int32_t) in2[2])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[0]); - output[6] = 2 * (((limb) ((int32_t) in2[3])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[1])) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[0]); - output[7] = ((limb) ((int32_t) in2[3])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[0]); - output[8] = ((limb) ((int32_t) in2[4])) * ((int32_t) in[4]) + - 2 * (((limb) ((int32_t) in2[3])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[1])) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[0]); - output[9] = ((limb) ((int32_t) in2[4])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in2[0])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[0]); - output[10] = 2 * (((limb) ((int32_t) in2[5])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[1])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[1])) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[2]); - output[11] = ((limb) ((int32_t) in2[5])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in2[2])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[2]); - output[12] = ((limb) ((int32_t) in2[6])) * ((int32_t) in[6]) + - 2 * (((limb) ((int32_t) in2[5])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[3])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[3])) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[4]); - output[13] = ((limb) ((int32_t) in2[6])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[7])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in2[4])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[4]); - output[14] = 2 * (((limb) ((int32_t) in2[7])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[5])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[5])) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[6]); - output[15] = ((limb) ((int32_t) in2[7])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in2[8])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in2[6])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[6]); - output[16] = ((limb) ((int32_t) in2[8])) * ((int32_t) in[8]) + - 2 * (((limb) ((int32_t) in2[7])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[7])); - output[17] = ((limb) ((int32_t) in2[8])) * ((int32_t) in[9]) + - ((limb) ((int32_t) in2[9])) * ((int32_t) in[8]); - output[18] = 2 * ((limb) ((int32_t) in2[9])) * ((int32_t) in[9]); -} - -/* Reduce a long form to a short form by taking the input mod 2^255 - 19. - * - * On entry: |output[i]| < 14*2^54 - * On exit: |output[0..8]| < 280*2^54 +#else +/* fe means field element. Here the field is \Z/(2^255-19). An element t, + * entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77 + * t[3]+2^102 t[4]+...+2^230 t[9]. + * fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc. + * Multiplication and carrying produce fe from fe_loose. */ -static void freduce_degree(limb *output) -{ - /* Each of these shifts and adds ends up multiplying the value by 19. - * - * For output[0..8], the absolute entry value is < 14*2^54 and we add, at - * most, 19*14*2^54 thus, on exit, |output[0..8]| < 280*2^54. - */ - output[8] += output[18] << 4; - output[8] += output[18] << 1; - output[8] += output[18]; - output[7] += output[17] << 4; - output[7] += output[17] << 1; - output[7] += output[17]; - output[6] += output[16] << 4; - output[6] += output[16] << 1; - output[6] += output[16]; - output[5] += output[15] << 4; - output[5] += output[15] << 1; - output[5] += output[15]; - output[4] += output[14] << 4; - output[4] += output[14] << 1; - output[4] += output[14]; - output[3] += output[13] << 4; - output[3] += output[13] << 1; - output[3] += output[13]; - output[2] += output[12] << 4; - output[2] += output[12] << 1; - output[2] += output[12]; - output[1] += output[11] << 4; - output[1] += output[11] << 1; - output[1] += output[11]; - output[0] += output[10] << 4; - output[0] += output[10] << 1; - output[0] += output[10]; -} - -#if (-1 & 3) != 3 -#error "This code only works on a two's complement system" -#endif +typedef struct fe { uint32_t v[10]; } fe; -/* return v / 2^26, using only shifts and adds. - * - * On entry: v can take any value. +/* fe_loose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc. + * Addition and subtraction produce fe_loose from (fe, fe). */ -static inline limb div_by_2_26(const limb v) +typedef struct fe_loose { uint32_t v[10]; } fe_loose; + +static __always_inline void fe_frombytes_impl(uint32_t h[10], const uint8_t *s) { - /* High word of v; no shift needed. */ - const uint32_t highword = (uint32_t) (((uint64_t) v) >> 32); - /* Set to all 1s if v was negative; else set to 0s. */ - const int32_t sign = ((int32_t) highword) >> 31; - /* Set to 0x3ffffff if v was negative; else set to 0. */ - const int32_t roundoff = ((uint32_t) sign) >> 6; - /* Should return v / (1<<26) */ - return (v + roundoff) >> 26; + /* Ignores top bit of s. */ + uint32_t a0 = le32toh(*(uint32_t *)(s)); + uint32_t a1 = le32toh(*(uint32_t *)(s+4)); + uint32_t a2 = le32toh(*(uint32_t *)(s+8)); + uint32_t a3 = le32toh(*(uint32_t *)(s+12)); + uint32_t a4 = le32toh(*(uint32_t *)(s+16)); + uint32_t a5 = le32toh(*(uint32_t *)(s+20)); + uint32_t a6 = le32toh(*(uint32_t *)(s+24)); + uint32_t a7 = le32toh(*(uint32_t *)(s+28)); + h[0] = a0&((1<<26)-1); /* 26 used, 32-26 left. 26 */ + h[1] = (a0>>26) | ((a1&((1<<19)-1))<< 6); /* (32-26) + 19 = 6+19 = 25 */ + h[2] = (a1>>19) | ((a2&((1<<13)-1))<<13); /* (32-19) + 13 = 13+13 = 26 */ + h[3] = (a2>>13) | ((a3&((1<< 6)-1))<<19); /* (32-13) + 6 = 19+ 6 = 25 */ + h[4] = (a3>> 6); /* (32- 6) = 26 */ + h[5] = a4&((1<<25)-1); /* 25 */ + h[6] = (a4>>25) | ((a5&((1<<19)-1))<< 7); /* (32-25) + 19 = 7+19 = 26 */ + h[7] = (a5>>19) | ((a6&((1<<12)-1))<<13); /* (32-19) + 12 = 13+12 = 25 */ + h[8] = (a6>>12) | ((a7&((1<< 6)-1))<<20); /* (32-12) + 6 = 20+ 6 = 26 */ + h[9] = (a7>> 6)&((1<<25)-1); /* 25 */ } -/* return v / (2^25), using only shifts and adds. - * - * On entry: v can take any value. - */ -static inline limb div_by_2_25(const limb v) +static __always_inline void fe_frombytes(fe *h, const uint8_t *s) { - /* High word of v; no shift needed*/ - const uint32_t highword = (uint32_t) (((uint64_t) v) >> 32); - /* Set to all 1s if v was negative; else set to 0s. */ - const int32_t sign = ((int32_t) highword) >> 31; - /* Set to 0x1ffffff if v was negative; else set to 0. */ - const int32_t roundoff = ((uint32_t) sign) >> 7; - /* Should return v / (1<<25) */ - return (v + roundoff) >> 25; + fe_frombytes_impl(h->v, s); } -/* Reduce all coefficients of the short form input so that |x| < 2^26. - * - * On entry: |output[i]| < 280*2^54 - */ -static void freduce_coefficients(limb *output) +static __always_inline uint8_t /*bool*/ addcarryx_u25(uint8_t /*bool*/ c, uint32_t a, uint32_t b, uint32_t *low) { - unsigned int i; - - output[10] = 0; - - for (i = 0; i < 10; i += 2) { - limb over = div_by_2_26(output[i]); - /* The entry condition (that |output[i]| < 280*2^54) means that over is, at - * most, 280*2^28 in the first iteration of this loop. This is added to the - * next limb and we can approximate the resulting bound of that limb by - * 281*2^54. - */ - output[i] -= over << 26; - output[i+1] += over; - - /* For the first iteration, |output[i+1]| < 281*2^54, thus |over| < - * 281*2^29. When this is added to the next limb, the resulting bound can - * be approximated as 281*2^54. - * - * For subsequent iterations of the loop, 281*2^54 remains a conservative - * bound and no overflow occurs. - */ - over = div_by_2_25(output[i+1]); - output[i+1] -= over << 25; - output[i+2] += over; - } - /* Now |output[10]| < 281*2^29 and all other coefficients are reduced. */ - output[0] += output[10] << 4; - output[0] += output[10] << 1; - output[0] += output[10]; - - output[10] = 0; - - /* Now output[1..9] are reduced, and |output[0]| < 2^26 + 19*281*2^29 - * So |over| will be no more than 2^16. + /* This function extracts 25 bits of result and 1 bit of carry (26 total), so + * a 32-bit intermediate is sufficient. */ - { - limb over = div_by_2_26(output[0]); - - output[0] -= over << 26; - output[1] += over; - } + uint32_t x = a + b + c; + *low = x & ((1 << 25) - 1); + return (x >> 25) & 1; +} - /* Now output[0,2..9] are reduced, and |output[1]| < 2^25 + 2^16 < 2^26. The - * bound on |output[1]| is sufficient to meet our needs. +static __always_inline uint8_t /*bool*/ addcarryx_u26(uint8_t /*bool*/ c, uint32_t a, uint32_t b, uint32_t *low) +{ + /* This function extracts 26 bits of result and 1 bit of carry (27 total), so + * a 32-bit intermediate is sufficient. */ + uint32_t x = a + b + c; + *low = x & ((1 << 26) - 1); + return (x >> 26) & 1; } -/* A helpful wrapper around fproduct: output = in * in2. - * - * On entry: |in[i]| < 2^27 and |in2[i]| < 2^27. - * - * output must be distinct to both inputs. The output is reduced degree - * (indeed, one need only provide storage for 10 limbs) and |output[i]| < 2^26. - */ -static void fmul(limb *output, const limb *in, const limb *in2) +static __always_inline uint8_t /*bool*/ subborrow_u25(uint8_t /*bool*/ c, uint32_t a, uint32_t b, uint32_t *low) { - limb t[19]; - - fproduct(t, in, in2); - /* |t[i]| < 14*2^54 */ - freduce_degree(t); - freduce_coefficients(t); - /* |t[i]| < 2^26 */ - memcpy(output, t, sizeof(limb) * 10); + /* This function extracts 25 bits of result and 1 bit of borrow (26 total), so + * a 32-bit intermediate is sufficient. + */ + uint32_t x = a - b - c; + *low = x & ((1 << 25) - 1); + return x >> 31; } -/* Square a number: output = in**2 - * - * output must be distinct from the input. The inputs are reduced coefficient - * form, the output is not. - * - * output[x] <= 14 * the largest product of the input limbs. - */ -static void fsquare_inner(limb *output, const limb *in) -{ - output[0] = ((limb) ((int32_t) in[0])) * ((int32_t) in[0]); - output[1] = 2 * ((limb) ((int32_t) in[0])) * ((int32_t) in[1]); - output[2] = 2 * (((limb) ((int32_t) in[1])) * ((int32_t) in[1]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[2])); - output[3] = 2 * (((limb) ((int32_t) in[1])) * ((int32_t) in[2]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[3])); - output[4] = ((limb) ((int32_t) in[2])) * ((int32_t) in[2]) + - 4 * ((limb) ((int32_t) in[1])) * ((int32_t) in[3]) + - 2 * ((limb) ((int32_t) in[0])) * ((int32_t) in[4]); - output[5] = 2 * (((limb) ((int32_t) in[2])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in[1])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[5])); - output[6] = 2 * (((limb) ((int32_t) in[3])) * ((int32_t) in[3]) + - ((limb) ((int32_t) in[2])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[6]) + - 2 * ((limb) ((int32_t) in[1])) * ((int32_t) in[5])); - output[7] = 2 * (((limb) ((int32_t) in[3])) * ((int32_t) in[4]) + - ((limb) ((int32_t) in[2])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in[1])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[7])); - output[8] = ((limb) ((int32_t) in[4])) * ((int32_t) in[4]) + - 2 * (((limb) ((int32_t) in[2])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[8]) + - 2 * (((limb) ((int32_t) in[1])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[3])) * ((int32_t) in[5]))); - output[9] = 2 * (((limb) ((int32_t) in[4])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in[3])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in[2])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[1])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in[0])) * ((int32_t) in[9])); - output[10] = 2 * (((limb) ((int32_t) in[5])) * ((int32_t) in[5]) + - ((limb) ((int32_t) in[4])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in[2])) * ((int32_t) in[8]) + - 2 * (((limb) ((int32_t) in[3])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[1])) * ((int32_t) in[9]))); - output[11] = 2 * (((limb) ((int32_t) in[5])) * ((int32_t) in[6]) + - ((limb) ((int32_t) in[4])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[3])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in[2])) * ((int32_t) in[9])); - output[12] = ((limb) ((int32_t) in[6])) * ((int32_t) in[6]) + - 2 * (((limb) ((int32_t) in[4])) * ((int32_t) in[8]) + - 2 * (((limb) ((int32_t) in[5])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[3])) * ((int32_t) in[9]))); - output[13] = 2 * (((limb) ((int32_t) in[6])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[5])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in[4])) * ((int32_t) in[9])); - output[14] = 2 * (((limb) ((int32_t) in[7])) * ((int32_t) in[7]) + - ((limb) ((int32_t) in[6])) * ((int32_t) in[8]) + - 2 * ((limb) ((int32_t) in[5])) * ((int32_t) in[9])); - output[15] = 2 * (((limb) ((int32_t) in[7])) * ((int32_t) in[8]) + - ((limb) ((int32_t) in[6])) * ((int32_t) in[9])); - output[16] = ((limb) ((int32_t) in[8])) * ((int32_t) in[8]) + - 4 * ((limb) ((int32_t) in[7])) * ((int32_t) in[9]); - output[17] = 2 * ((limb) ((int32_t) in[8])) * ((int32_t) in[9]); - output[18] = 2 * ((limb) ((int32_t) in[9])) * ((int32_t) in[9]); -} - -/* fsquare sets output = in^2. - * - * On entry: The |in| argument is in reduced coefficients form and |in[i]| < - * 2^27. - * - * On exit: The |output| argument is in reduced coefficients form (indeed, one - * need only provide storage for 10 limbs) and |out[i]| < 2^26. - */ -static void fsquare(limb *output, const limb *in) +static __always_inline uint8_t /*bool*/ subborrow_u26(uint8_t /*bool*/ c, uint32_t a, uint32_t b, uint32_t *low) { - limb t[19]; - - fsquare_inner(t, in); - /* |t[i]| < 14*2^54 because the largest product of two limbs will be < - * 2^(27+27) and fsquare_inner adds together, at most, 14 of those - * products. + /* This function extracts 26 bits of result and 1 bit of borrow (27 total), so + * a 32-bit intermediate is sufficient. */ - freduce_degree(t); - freduce_coefficients(t); - /* |t[i]| < 2^26 */ - memcpy(output, t, sizeof(limb) * 10); -} - -/* Take a little-endian, 32-byte number and expand it into polynomial form */ -static void fexpand(limb *output, const uint8_t *input) -{ -#define F(n, start, shift, mask) \ - output[n] = ((((limb) input[start + 0]) | \ - ((limb) input[start + 1]) << 8 | \ - ((limb) input[start + 2]) << 16 | \ - ((limb) input[start + 3]) << 24) >> shift) & mask; - F(0, 0, 0, 0x3ffffff); - F(1, 3, 2, 0x1ffffff); - F(2, 6, 3, 0x3ffffff); - F(3, 9, 5, 0x1ffffff); - F(4, 12, 6, 0x3ffffff); - F(5, 16, 0, 0x1ffffff); - F(6, 19, 1, 0x3ffffff); - F(7, 22, 3, 0x1ffffff); - F(8, 25, 4, 0x3ffffff); - F(9, 28, 6, 0x1ffffff); -#undef F -} - -#if (-32 >> 1) != -16 -#error "This code only works when >> does sign-extension on negative numbers" -#endif + uint32_t x = a - b - c; + *low = x & ((1 << 26) - 1); + return x >> 31; +} -/* int32_t_eq returns 0xffffffff iff a == b and zero otherwise. */ -static int32_t int32_t_eq(int32_t a, int32_t b) +static __always_inline uint32_t cmovznz32(uint32_t t, uint32_t z, uint32_t nz) { - a = ~(a ^ b); - a &= a << 16; - a &= a << 8; - a &= a << 4; - a &= a << 2; - a &= a << 1; - return a >> 31; + t = -!!t; /* all set if nonzero, 0 if 0 */ + return (t&nz) | ((~t)&z); } -/* int32_t_gte returns 0xffffffff if a >= b and zero otherwise, where a and b are - * both non-negative. - */ -static int32_t int32_t_gte(int32_t a, int32_t b) +static __always_inline void fe_freeze(uint32_t out[10], const uint32_t in1[10]) { - a -= b; - /* a >= 0 iff a >= b. */ - return ~(a >> 31); + { const uint32_t x17 = in1[9]; + { const uint32_t x18 = in1[8]; + { const uint32_t x16 = in1[7]; + { const uint32_t x14 = in1[6]; + { const uint32_t x12 = in1[5]; + { const uint32_t x10 = in1[4]; + { const uint32_t x8 = in1[3]; + { const uint32_t x6 = in1[2]; + { const uint32_t x4 = in1[1]; + { const uint32_t x2 = in1[0]; + { uint32_t x20; uint8_t/*bool*/ x21 = subborrow_u26(0x0, x2, 0x3ffffed, &x20); + { uint32_t x23; uint8_t/*bool*/ x24 = subborrow_u25(x21, x4, 0x1ffffff, &x23); + { uint32_t x26; uint8_t/*bool*/ x27 = subborrow_u26(x24, x6, 0x3ffffff, &x26); + { uint32_t x29; uint8_t/*bool*/ x30 = subborrow_u25(x27, x8, 0x1ffffff, &x29); + { uint32_t x32; uint8_t/*bool*/ x33 = subborrow_u26(x30, x10, 0x3ffffff, &x32); + { uint32_t x35; uint8_t/*bool*/ x36 = subborrow_u25(x33, x12, 0x1ffffff, &x35); + { uint32_t x38; uint8_t/*bool*/ x39 = subborrow_u26(x36, x14, 0x3ffffff, &x38); + { uint32_t x41; uint8_t/*bool*/ x42 = subborrow_u25(x39, x16, 0x1ffffff, &x41); + { uint32_t x44; uint8_t/*bool*/ x45 = subborrow_u26(x42, x18, 0x3ffffff, &x44); + { uint32_t x47; uint8_t/*bool*/ x48 = subborrow_u25(x45, x17, 0x1ffffff, &x47); + { uint32_t x49 = cmovznz32(x48, 0x0, 0xffffffff); + { uint32_t x50 = (x49 & 0x3ffffed); + { uint32_t x52; uint8_t/*bool*/ x53 = addcarryx_u26(0x0, x20, x50, &x52); + { uint32_t x54 = (x49 & 0x1ffffff); + { uint32_t x56; uint8_t/*bool*/ x57 = addcarryx_u25(x53, x23, x54, &x56); + { uint32_t x58 = (x49 & 0x3ffffff); + { uint32_t x60; uint8_t/*bool*/ x61 = addcarryx_u26(x57, x26, x58, &x60); + { uint32_t x62 = (x49 & 0x1ffffff); + { uint32_t x64; uint8_t/*bool*/ x65 = addcarryx_u25(x61, x29, x62, &x64); + { uint32_t x66 = (x49 & 0x3ffffff); + { uint32_t x68; uint8_t/*bool*/ x69 = addcarryx_u26(x65, x32, x66, &x68); + { uint32_t x70 = (x49 & 0x1ffffff); + { uint32_t x72; uint8_t/*bool*/ x73 = addcarryx_u25(x69, x35, x70, &x72); + { uint32_t x74 = (x49 & 0x3ffffff); + { uint32_t x76; uint8_t/*bool*/ x77 = addcarryx_u26(x73, x38, x74, &x76); + { uint32_t x78 = (x49 & 0x1ffffff); + { uint32_t x80; uint8_t/*bool*/ x81 = addcarryx_u25(x77, x41, x78, &x80); + { uint32_t x82 = (x49 & 0x3ffffff); + { uint32_t x84; uint8_t/*bool*/ x85 = addcarryx_u26(x81, x44, x82, &x84); + { uint32_t x86 = (x49 & 0x1ffffff); + { uint32_t x88; addcarryx_u25(x85, x47, x86, &x88); + out[0] = x52; + out[1] = x56; + out[2] = x60; + out[3] = x64; + out[4] = x68; + out[5] = x72; + out[6] = x76; + out[7] = x80; + out[8] = x84; + out[9] = x88; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} } -/* Take a fully reduced polynomial form number and contract it into a - * little-endian, 32-byte array. - * - * On entry: |input_limbs[i]| < 2^26 - */ -static void fcontract(uint8_t *output, limb *input_limbs) +static __always_inline void fe_tobytes(uint8_t s[32], const fe *f) { - int i; - int j; - int32_t input[10]; - int32_t mask; - - /* |input_limbs[i]| < 2^26, so it's valid to convert to an int32_t. */ - for (i = 0; i < 10; i++) { - input[i] = input_limbs[i]; - } - - for (j = 0; j < 2; ++j) { - for (i = 0; i < 9; ++i) { - if ((i & 1) == 1) { - /* This calculation is a time-invariant way to make input[i] - * non-negative by borrowing from the next-larger limb. - */ - const int32_t mask = input[i] >> 31; - const int32_t carry = -((input[i] & mask) >> 25); - - input[i] = input[i] + (carry << 25); - input[i+1] = input[i+1] - carry; - } else { - const int32_t mask = input[i] >> 31; - const int32_t carry = -((input[i] & mask) >> 26); - - input[i] = input[i] + (carry << 26); - input[i+1] = input[i+1] - carry; - } - } + uint32_t h[10]; + fe_freeze(h, f->v); + s[0] = h[0] >> 0; + s[1] = h[0] >> 8; + s[2] = h[0] >> 16; + s[3] = (h[0] >> 24) | (h[1] << 2); + s[4] = h[1] >> 6; + s[5] = h[1] >> 14; + s[6] = (h[1] >> 22) | (h[2] << 3); + s[7] = h[2] >> 5; + s[8] = h[2] >> 13; + s[9] = (h[2] >> 21) | (h[3] << 5); + s[10] = h[3] >> 3; + s[11] = h[3] >> 11; + s[12] = (h[3] >> 19) | (h[4] << 6); + s[13] = h[4] >> 2; + s[14] = h[4] >> 10; + s[15] = h[4] >> 18; + s[16] = h[5] >> 0; + s[17] = h[5] >> 8; + s[18] = h[5] >> 16; + s[19] = (h[5] >> 24) | (h[6] << 1); + s[20] = h[6] >> 7; + s[21] = h[6] >> 15; + s[22] = (h[6] >> 23) | (h[7] << 3); + s[23] = h[7] >> 5; + s[24] = h[7] >> 13; + s[25] = (h[7] >> 21) | (h[8] << 4); + s[26] = h[8] >> 4; + s[27] = h[8] >> 12; + s[28] = (h[8] >> 20) | (h[9] << 6); + s[29] = h[9] >> 2; + s[30] = h[9] >> 10; + s[31] = h[9] >> 18; +} - /* There's no greater limb for input[9] to borrow from, but we can multiply - * by 19 and borrow from input[0], which is valid mod 2^255-19. - */ - { - const int32_t mask = input[9] >> 31; - const int32_t carry = -((input[9] & mask) >> 25); +/* h = f */ +static __always_inline void fe_copy(fe *h, const fe *f) +{ + memmove(h, f, sizeof(uint32_t) * 10); +} - input[9] = input[9] + (carry << 25); - input[0] = input[0] - (carry * 19); - } +static __always_inline void fe_copy_lt(fe_loose *h, const fe *f) +{ + memmove(h, f, sizeof(uint32_t) * 10); +} - /* After the first iteration, input[1..9] are non-negative and fit within - * 25 or 26 bits, depending on position. However, input[0] may be - * negative. - */ - } +/* h = 0 */ +static __always_inline void fe_0(fe *h) +{ + memset(h, 0, sizeof(uint32_t) * 10); +} - /* The first borrow-propagation pass above ended with every limb - except (possibly) input[0] non-negative. - If input[0] was negative after the first pass, then it was because of a - carry from input[9]. On entry, input[9] < 2^26 so the carry was, at most, - one, since (2**26-1) >> 25 = 1. Thus input[0] >= -19. - In the second pass, each limb is decreased by at most one. Thus the second - borrow-propagation pass could only have wrapped around to decrease - input[0] again if the first pass left input[0] negative *and* input[1] - through input[9] were all zero. In that case, input[1] is now 2^25 - 1, - and this last borrow-propagation step will leave input[1] non-negative. */ - { - const int32_t mask = input[0] >> 31; - const int32_t carry = -((input[0] & mask) >> 26); +/* h = 1 */ +static __always_inline void fe_1(fe *h) +{ + memset(h, 0, sizeof(uint32_t) * 10); + h->v[0] = 1; +} - input[0] = input[0] + (carry << 26); - input[1] = input[1] - carry; - } +static __always_inline void fe_add_impl(uint32_t out[10], const uint32_t in1[10], const uint32_t in2[10]) +{ + { const uint32_t x20 = in1[9]; + { const uint32_t x21 = in1[8]; + { const uint32_t x19 = in1[7]; + { const uint32_t x17 = in1[6]; + { const uint32_t x15 = in1[5]; + { const uint32_t x13 = in1[4]; + { const uint32_t x11 = in1[3]; + { const uint32_t x9 = in1[2]; + { const uint32_t x7 = in1[1]; + { const uint32_t x5 = in1[0]; + { const uint32_t x38 = in2[9]; + { const uint32_t x39 = in2[8]; + { const uint32_t x37 = in2[7]; + { const uint32_t x35 = in2[6]; + { const uint32_t x33 = in2[5]; + { const uint32_t x31 = in2[4]; + { const uint32_t x29 = in2[3]; + { const uint32_t x27 = in2[2]; + { const uint32_t x25 = in2[1]; + { const uint32_t x23 = in2[0]; + out[0] = (x5 + x23); + out[1] = (x7 + x25); + out[2] = (x9 + x27); + out[3] = (x11 + x29); + out[4] = (x13 + x31); + out[5] = (x15 + x33); + out[6] = (x17 + x35); + out[7] = (x19 + x37); + out[8] = (x21 + x39); + out[9] = (x20 + x38); + }}}}}}}}}}}}}}}}}}}} +} - /* All input[i] are now non-negative. However, there might be values between - * 2^25 and 2^26 in a limb which is, nominally, 25 bits wide. - */ - for (j = 0; j < 2; j++) { - for (i = 0; i < 9; i++) { - if ((i & 1) == 1) { - const int32_t carry = input[i] >> 25; - - input[i] &= 0x1ffffff; - input[i+1] += carry; - } else { - const int32_t carry = input[i] >> 26; - - input[i] &= 0x3ffffff; - input[i+1] += carry; - } - } +/* h = f + g + * Can overlap h with f or g. + */ +static __always_inline void fe_add(fe_loose *h, const fe *f, const fe *g) +{ + fe_add_impl(h->v, f->v, g->v); +} - { - const int32_t carry = input[9] >> 25; +static __always_inline void fe_sub_impl(uint32_t out[10], const uint32_t in1[10], const uint32_t in2[10]) +{ + { const uint32_t x20 = in1[9]; + { const uint32_t x21 = in1[8]; + { const uint32_t x19 = in1[7]; + { const uint32_t x17 = in1[6]; + { const uint32_t x15 = in1[5]; + { const uint32_t x13 = in1[4]; + { const uint32_t x11 = in1[3]; + { const uint32_t x9 = in1[2]; + { const uint32_t x7 = in1[1]; + { const uint32_t x5 = in1[0]; + { const uint32_t x38 = in2[9]; + { const uint32_t x39 = in2[8]; + { const uint32_t x37 = in2[7]; + { const uint32_t x35 = in2[6]; + { const uint32_t x33 = in2[5]; + { const uint32_t x31 = in2[4]; + { const uint32_t x29 = in2[3]; + { const uint32_t x27 = in2[2]; + { const uint32_t x25 = in2[1]; + { const uint32_t x23 = in2[0]; + out[0] = ((0x7ffffda + x5) - x23); + out[1] = ((0x3fffffe + x7) - x25); + out[2] = ((0x7fffffe + x9) - x27); + out[3] = ((0x3fffffe + x11) - x29); + out[4] = ((0x7fffffe + x13) - x31); + out[5] = ((0x3fffffe + x15) - x33); + out[6] = ((0x7fffffe + x17) - x35); + out[7] = ((0x3fffffe + x19) - x37); + out[8] = ((0x7fffffe + x21) - x39); + out[9] = ((0x3fffffe + x20) - x38); + }}}}}}}}}}}}}}}}}}}} +} - input[9] &= 0x1ffffff; - input[0] += 19*carry; - } - } +/* h = f - g + * Can overlap h with f or g. + */ +static __always_inline void fe_sub(fe_loose *h, const fe *f, const fe *g) +{ + fe_sub_impl(h->v, f->v, g->v); +} - /* If the first carry-chain pass, just above, ended up with a carry from - * input[9], and that caused input[0] to be out-of-bounds, then input[0] was - * < 2^26 + 2*19, because the carry was, at most, two. - * - * If the second pass carried from input[9] again then input[0] is < 2*19 and - * the input[9] -> input[0] carry didn't push input[0] out of bounds. - */ +static __always_inline void fe_mul_impl(uint32_t out[10], const uint32_t in1[10], const uint32_t in2[10]) +{ + { const uint32_t x20 = in1[9]; + { const uint32_t x21 = in1[8]; + { const uint32_t x19 = in1[7]; + { const uint32_t x17 = in1[6]; + { const uint32_t x15 = in1[5]; + { const uint32_t x13 = in1[4]; + { const uint32_t x11 = in1[3]; + { const uint32_t x9 = in1[2]; + { const uint32_t x7 = in1[1]; + { const uint32_t x5 = in1[0]; + { const uint32_t x38 = in2[9]; + { const uint32_t x39 = in2[8]; + { const uint32_t x37 = in2[7]; + { const uint32_t x35 = in2[6]; + { const uint32_t x33 = in2[5]; + { const uint32_t x31 = in2[4]; + { const uint32_t x29 = in2[3]; + { const uint32_t x27 = in2[2]; + { const uint32_t x25 = in2[1]; + { const uint32_t x23 = in2[0]; + { uint64_t x40 = ((uint64_t)x23 * x5); + { uint64_t x41 = (((uint64_t)x23 * x7) + ((uint64_t)x25 * x5)); + { uint64_t x42 = ((((uint64_t)(0x2 * x25) * x7) + ((uint64_t)x23 * x9)) + ((uint64_t)x27 * x5)); + { uint64_t x43 = (((((uint64_t)x25 * x9) + ((uint64_t)x27 * x7)) + ((uint64_t)x23 * x11)) + ((uint64_t)x29 * x5)); + { uint64_t x44 = (((((uint64_t)x27 * x9) + (0x2 * (((uint64_t)x25 * x11) + ((uint64_t)x29 * x7)))) + ((uint64_t)x23 * x13)) + ((uint64_t)x31 * x5)); + { uint64_t x45 = (((((((uint64_t)x27 * x11) + ((uint64_t)x29 * x9)) + ((uint64_t)x25 * x13)) + ((uint64_t)x31 * x7)) + ((uint64_t)x23 * x15)) + ((uint64_t)x33 * x5)); + { uint64_t x46 = (((((0x2 * ((((uint64_t)x29 * x11) + ((uint64_t)x25 * x15)) + ((uint64_t)x33 * x7))) + ((uint64_t)x27 * x13)) + ((uint64_t)x31 * x9)) + ((uint64_t)x23 * x17)) + ((uint64_t)x35 * x5)); + { uint64_t x47 = (((((((((uint64_t)x29 * x13) + ((uint64_t)x31 * x11)) + ((uint64_t)x27 * x15)) + ((uint64_t)x33 * x9)) + ((uint64_t)x25 * x17)) + ((uint64_t)x35 * x7)) + ((uint64_t)x23 * x19)) + ((uint64_t)x37 * x5)); + { uint64_t x48 = (((((((uint64_t)x31 * x13) + (0x2 * (((((uint64_t)x29 * x15) + ((uint64_t)x33 * x11)) + ((uint64_t)x25 * x19)) + ((uint64_t)x37 * x7)))) + ((uint64_t)x27 * x17)) + ((uint64_t)x35 * x9)) + ((uint64_t)x23 * x21)) + ((uint64_t)x39 * x5)); + { uint64_t x49 = (((((((((((uint64_t)x31 * x15) + ((uint64_t)x33 * x13)) + ((uint64_t)x29 * x17)) + ((uint64_t)x35 * x11)) + ((uint64_t)x27 * x19)) + ((uint64_t)x37 * x9)) + ((uint64_t)x25 * x21)) + ((uint64_t)x39 * x7)) + ((uint64_t)x23 * x20)) + ((uint64_t)x38 * x5)); + { uint64_t x50 = (((((0x2 * ((((((uint64_t)x33 * x15) + ((uint64_t)x29 * x19)) + ((uint64_t)x37 * x11)) + ((uint64_t)x25 * x20)) + ((uint64_t)x38 * x7))) + ((uint64_t)x31 * x17)) + ((uint64_t)x35 * x13)) + ((uint64_t)x27 * x21)) + ((uint64_t)x39 * x9)); + { uint64_t x51 = (((((((((uint64_t)x33 * x17) + ((uint64_t)x35 * x15)) + ((uint64_t)x31 * x19)) + ((uint64_t)x37 * x13)) + ((uint64_t)x29 * x21)) + ((uint64_t)x39 * x11)) + ((uint64_t)x27 * x20)) + ((uint64_t)x38 * x9)); + { uint64_t x52 = (((((uint64_t)x35 * x17) + (0x2 * (((((uint64_t)x33 * x19) + ((uint64_t)x37 * x15)) + ((uint64_t)x29 * x20)) + ((uint64_t)x38 * x11)))) + ((uint64_t)x31 * x21)) + ((uint64_t)x39 * x13)); + { uint64_t x53 = (((((((uint64_t)x35 * x19) + ((uint64_t)x37 * x17)) + ((uint64_t)x33 * x21)) + ((uint64_t)x39 * x15)) + ((uint64_t)x31 * x20)) + ((uint64_t)x38 * x13)); + { uint64_t x54 = (((0x2 * ((((uint64_t)x37 * x19) + ((uint64_t)x33 * x20)) + ((uint64_t)x38 * x15))) + ((uint64_t)x35 * x21)) + ((uint64_t)x39 * x17)); + { uint64_t x55 = (((((uint64_t)x37 * x21) + ((uint64_t)x39 * x19)) + ((uint64_t)x35 * x20)) + ((uint64_t)x38 * x17)); + { uint64_t x56 = (((uint64_t)x39 * x21) + (0x2 * (((uint64_t)x37 * x20) + ((uint64_t)x38 * x19)))); + { uint64_t x57 = (((uint64_t)x39 * x20) + ((uint64_t)x38 * x21)); + { uint64_t x58 = ((uint64_t)(0x2 * x38) * x20); + { uint64_t x59 = (x48 + (x58 << 0x4)); + { uint64_t x60 = (x59 + (x58 << 0x1)); + { uint64_t x61 = (x60 + x58); + { uint64_t x62 = (x47 + (x57 << 0x4)); + { uint64_t x63 = (x62 + (x57 << 0x1)); + { uint64_t x64 = (x63 + x57); + { uint64_t x65 = (x46 + (x56 << 0x4)); + { uint64_t x66 = (x65 + (x56 << 0x1)); + { uint64_t x67 = (x66 + x56); + { uint64_t x68 = (x45 + (x55 << 0x4)); + { uint64_t x69 = (x68 + (x55 << 0x1)); + { uint64_t x70 = (x69 + x55); + { uint64_t x71 = (x44 + (x54 << 0x4)); + { uint64_t x72 = (x71 + (x54 << 0x1)); + { uint64_t x73 = (x72 + x54); + { uint64_t x74 = (x43 + (x53 << 0x4)); + { uint64_t x75 = (x74 + (x53 << 0x1)); + { uint64_t x76 = (x75 + x53); + { uint64_t x77 = (x42 + (x52 << 0x4)); + { uint64_t x78 = (x77 + (x52 << 0x1)); + { uint64_t x79 = (x78 + x52); + { uint64_t x80 = (x41 + (x51 << 0x4)); + { uint64_t x81 = (x80 + (x51 << 0x1)); + { uint64_t x82 = (x81 + x51); + { uint64_t x83 = (x40 + (x50 << 0x4)); + { uint64_t x84 = (x83 + (x50 << 0x1)); + { uint64_t x85 = (x84 + x50); + { uint64_t x86 = (x85 >> 0x1a); + { uint32_t x87 = ((uint32_t)x85 & 0x3ffffff); + { uint64_t x88 = (x86 + x82); + { uint64_t x89 = (x88 >> 0x19); + { uint32_t x90 = ((uint32_t)x88 & 0x1ffffff); + { uint64_t x91 = (x89 + x79); + { uint64_t x92 = (x91 >> 0x1a); + { uint32_t x93 = ((uint32_t)x91 & 0x3ffffff); + { uint64_t x94 = (x92 + x76); + { uint64_t x95 = (x94 >> 0x19); + { uint32_t x96 = ((uint32_t)x94 & 0x1ffffff); + { uint64_t x97 = (x95 + x73); + { uint64_t x98 = (x97 >> 0x1a); + { uint32_t x99 = ((uint32_t)x97 & 0x3ffffff); + { uint64_t x100 = (x98 + x70); + { uint64_t x101 = (x100 >> 0x19); + { uint32_t x102 = ((uint32_t)x100 & 0x1ffffff); + { uint64_t x103 = (x101 + x67); + { uint64_t x104 = (x103 >> 0x1a); + { uint32_t x105 = ((uint32_t)x103 & 0x3ffffff); + { uint64_t x106 = (x104 + x64); + { uint64_t x107 = (x106 >> 0x19); + { uint32_t x108 = ((uint32_t)x106 & 0x1ffffff); + { uint64_t x109 = (x107 + x61); + { uint64_t x110 = (x109 >> 0x1a); + { uint32_t x111 = ((uint32_t)x109 & 0x3ffffff); + { uint64_t x112 = (x110 + x49); + { uint64_t x113 = (x112 >> 0x19); + { uint32_t x114 = ((uint32_t)x112 & 0x1ffffff); + { uint64_t x115 = (x87 + (0x13 * x113)); + { uint32_t x116 = (uint32_t) (x115 >> 0x1a); + { uint32_t x117 = ((uint32_t)x115 & 0x3ffffff); + { uint32_t x118 = (x116 + x90); + { uint32_t x119 = (x118 >> 0x19); + { uint32_t x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} - /* It still remains the case that input might be between 2^255-19 and 2^255. - * In this case, input[1..9] must take their maximum value and input[0] must - * be >= (2^255-19) & 0x3ffffff, which is 0x3ffffed. - */ - mask = int32_t_gte(input[0], 0x3ffffed); - for (i = 1; i < 10; i++) { - if ((i & 1) == 1) { - mask &= int32_t_eq(input[i], 0x1ffffff); - } else { - mask &= int32_t_eq(input[i], 0x3ffffff); - } - } +static __always_inline void fe_mul_ttt(fe *h, const fe *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} - /* mask is either 0xffffffff (if input >= 2^255-19) and zero otherwise. Thus - * this conditionally subtracts 2^255-19. - */ - input[0] -= mask & 0x3ffffed; +static __always_inline void fe_mul_tlt(fe *h, const fe_loose *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} - for (i = 1; i < 10; i++) { - if ((i & 1) == 1) { - input[i] -= mask & 0x1ffffff; - } else { - input[i] -= mask & 0x3ffffff; - } - } +static __always_inline void fe_mul_tll(fe *h, const fe_loose *f, const fe_loose *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} - input[1] <<= 2; - input[2] <<= 3; - input[3] <<= 5; - input[4] <<= 6; - input[6] <<= 1; - input[7] <<= 3; - input[8] <<= 4; - input[9] <<= 6; -#define F(i, s) \ - output[s+0] |= input[i] & 0xff; \ - output[s+1] = (input[i] >> 8) & 0xff; \ - output[s+2] = (input[i] >> 16) & 0xff; \ - output[s+3] = (input[i] >> 24) & 0xff; - output[0] = 0; - output[16] = 0; - F(0, 0); - F(1, 3); - F(2, 6); - F(3, 9); - F(4, 12); - F(5, 16); - F(6, 19); - F(7, 22); - F(8, 25); - F(9, 28); -#undef F -} - -/* Input: Q, Q', Q-Q' - * Output: 2Q, Q+Q' - * - * x2 z3: long form - * x3 z3: long form - * x z: short form, destroyed - * xprime zprime: short form, destroyed - * qmqp: short form, preserved - * - * On entry and exit, the absolute value of the limbs of all inputs and outputs - * are < 2^26. - */ -static void fmonty(limb *x2, limb *z2, /* output 2Q */ - limb *x3, limb *z3, /* output Q + Q' */ - limb *x, limb *z, /* input Q */ - limb *xprime, limb *zprime, /* input Q' */ +static __always_inline void fe_sqr_impl(uint32_t out[10], const uint32_t in1[10]) +{ + { const uint32_t x17 = in1[9]; + { const uint32_t x18 = in1[8]; + { const uint32_t x16 = in1[7]; + { const uint32_t x14 = in1[6]; + { const uint32_t x12 = in1[5]; + { const uint32_t x10 = in1[4]; + { const uint32_t x8 = in1[3]; + { const uint32_t x6 = in1[2]; + { const uint32_t x4 = in1[1]; + { const uint32_t x2 = in1[0]; + { uint64_t x19 = ((uint64_t)x2 * x2); + { uint64_t x20 = ((uint64_t)(0x2 * x2) * x4); + { uint64_t x21 = (0x2 * (((uint64_t)x4 * x4) + ((uint64_t)x2 * x6))); + { uint64_t x22 = (0x2 * (((uint64_t)x4 * x6) + ((uint64_t)x2 * x8))); + { uint64_t x23 = ((((uint64_t)x6 * x6) + ((uint64_t)(0x4 * x4) * x8)) + ((uint64_t)(0x2 * x2) * x10)); + { uint64_t x24 = (0x2 * ((((uint64_t)x6 * x8) + ((uint64_t)x4 * x10)) + ((uint64_t)x2 * x12))); + { uint64_t x25 = (0x2 * (((((uint64_t)x8 * x8) + ((uint64_t)x6 * x10)) + ((uint64_t)x2 * x14)) + ((uint64_t)(0x2 * x4) * x12))); + { uint64_t x26 = (0x2 * (((((uint64_t)x8 * x10) + ((uint64_t)x6 * x12)) + ((uint64_t)x4 * x14)) + ((uint64_t)x2 * x16))); + { uint64_t x27 = (((uint64_t)x10 * x10) + (0x2 * ((((uint64_t)x6 * x14) + ((uint64_t)x2 * x18)) + (0x2 * (((uint64_t)x4 * x16) + ((uint64_t)x8 * x12)))))); + { uint64_t x28 = (0x2 * ((((((uint64_t)x10 * x12) + ((uint64_t)x8 * x14)) + ((uint64_t)x6 * x16)) + ((uint64_t)x4 * x18)) + ((uint64_t)x2 * x17))); + { uint64_t x29 = (0x2 * (((((uint64_t)x12 * x12) + ((uint64_t)x10 * x14)) + ((uint64_t)x6 * x18)) + (0x2 * (((uint64_t)x8 * x16) + ((uint64_t)x4 * x17))))); + { uint64_t x30 = (0x2 * (((((uint64_t)x12 * x14) + ((uint64_t)x10 * x16)) + ((uint64_t)x8 * x18)) + ((uint64_t)x6 * x17))); + { uint64_t x31 = (((uint64_t)x14 * x14) + (0x2 * (((uint64_t)x10 * x18) + (0x2 * (((uint64_t)x12 * x16) + ((uint64_t)x8 * x17)))))); + { uint64_t x32 = (0x2 * ((((uint64_t)x14 * x16) + ((uint64_t)x12 * x18)) + ((uint64_t)x10 * x17))); + { uint64_t x33 = (0x2 * ((((uint64_t)x16 * x16) + ((uint64_t)x14 * x18)) + ((uint64_t)(0x2 * x12) * x17))); + { uint64_t x34 = (0x2 * (((uint64_t)x16 * x18) + ((uint64_t)x14 * x17))); + { uint64_t x35 = (((uint64_t)x18 * x18) + ((uint64_t)(0x4 * x16) * x17)); + { uint64_t x36 = ((uint64_t)(0x2 * x18) * x17); + { uint64_t x37 = ((uint64_t)(0x2 * x17) * x17); + { uint64_t x38 = (x27 + (x37 << 0x4)); + { uint64_t x39 = (x38 + (x37 << 0x1)); + { uint64_t x40 = (x39 + x37); + { uint64_t x41 = (x26 + (x36 << 0x4)); + { uint64_t x42 = (x41 + (x36 << 0x1)); + { uint64_t x43 = (x42 + x36); + { uint64_t x44 = (x25 + (x35 << 0x4)); + { uint64_t x45 = (x44 + (x35 << 0x1)); + { uint64_t x46 = (x45 + x35); + { uint64_t x47 = (x24 + (x34 << 0x4)); + { uint64_t x48 = (x47 + (x34 << 0x1)); + { uint64_t x49 = (x48 + x34); + { uint64_t x50 = (x23 + (x33 << 0x4)); + { uint64_t x51 = (x50 + (x33 << 0x1)); + { uint64_t x52 = (x51 + x33); + { uint64_t x53 = (x22 + (x32 << 0x4)); + { uint64_t x54 = (x53 + (x32 << 0x1)); + { uint64_t x55 = (x54 + x32); + { uint64_t x56 = (x21 + (x31 << 0x4)); + { uint64_t x57 = (x56 + (x31 << 0x1)); + { uint64_t x58 = (x57 + x31); + { uint64_t x59 = (x20 + (x30 << 0x4)); + { uint64_t x60 = (x59 + (x30 << 0x1)); + { uint64_t x61 = (x60 + x30); + { uint64_t x62 = (x19 + (x29 << 0x4)); + { uint64_t x63 = (x62 + (x29 << 0x1)); + { uint64_t x64 = (x63 + x29); + { uint64_t x65 = (x64 >> 0x1a); + { uint32_t x66 = ((uint32_t)x64 & 0x3ffffff); + { uint64_t x67 = (x65 + x61); + { uint64_t x68 = (x67 >> 0x19); + { uint32_t x69 = ((uint32_t)x67 & 0x1ffffff); + { uint64_t x70 = (x68 + x58); + { uint64_t x71 = (x70 >> 0x1a); + { uint32_t x72 = ((uint32_t)x70 & 0x3ffffff); + { uint64_t x73 = (x71 + x55); + { uint64_t x74 = (x73 >> 0x19); + { uint32_t x75 = ((uint32_t)x73 & 0x1ffffff); + { uint64_t x76 = (x74 + x52); + { uint64_t x77 = (x76 >> 0x1a); + { uint32_t x78 = ((uint32_t)x76 & 0x3ffffff); + { uint64_t x79 = (x77 + x49); + { uint64_t x80 = (x79 >> 0x19); + { uint32_t x81 = ((uint32_t)x79 & 0x1ffffff); + { uint64_t x82 = (x80 + x46); + { uint64_t x83 = (x82 >> 0x1a); + { uint32_t x84 = ((uint32_t)x82 & 0x3ffffff); + { uint64_t x85 = (x83 + x43); + { uint64_t x86 = (x85 >> 0x19); + { uint32_t x87 = ((uint32_t)x85 & 0x1ffffff); + { uint64_t x88 = (x86 + x40); + { uint64_t x89 = (x88 >> 0x1a); + { uint32_t x90 = ((uint32_t)x88 & 0x3ffffff); + { uint64_t x91 = (x89 + x28); + { uint64_t x92 = (x91 >> 0x19); + { uint32_t x93 = ((uint32_t)x91 & 0x1ffffff); + { uint64_t x94 = (x66 + (0x13 * x92)); + { uint32_t x95 = (uint32_t) (x94 >> 0x1a); + { uint32_t x96 = ((uint32_t)x94 & 0x3ffffff); + { uint32_t x97 = (x95 + x69); + { uint32_t x98 = (x97 >> 0x19); + { uint32_t x99 = (x97 & 0x1ffffff); + out[0] = x96; + out[1] = x99; + out[2] = (x98 + x72); + out[3] = x75; + out[4] = x78; + out[5] = x81; + out[6] = x84; + out[7] = x87; + out[8] = x90; + out[9] = x93; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} - const limb *qmqp /* input Q - Q' */) +static __always_inline void fe_sq_tl(fe *h, const fe_loose *f) { - limb origx[10], origxprime[10], zzz[19], xx[19], zz[19], xxprime[19], - zzprime[19], zzzprime[19], xxxprime[19]; + fe_sqr_impl(h->v, f->v); +} - memcpy(origx, x, 10 * sizeof(limb)); - fsum(x, z); - /* |x[i]| < 2^27 */ - fdifference(z, origx); /* does x - z */ - /* |z[i]| < 2^27 */ +static __always_inline void fe_sq_tt(fe *h, const fe *f) +{ + fe_sqr_impl(h->v, f->v); +} - memcpy(origxprime, xprime, sizeof(limb) * 10); - fsum(xprime, zprime); - /* |xprime[i]| < 2^27 */ - fdifference(zprime, origxprime); - /* |zprime[i]| < 2^27 */ - fproduct(xxprime, xprime, z); - /* |xxprime[i]| < 14*2^54: the largest product of two limbs will be < - * 2^(27+27) and fproduct adds together, at most, 14 of those products. - * (Approximating that to 2^58 doesn't work out.) - */ - fproduct(zzprime, x, zprime); - /* |zzprime[i]| < 14*2^54 */ - freduce_degree(xxprime); - freduce_coefficients(xxprime); - /* |xxprime[i]| < 2^26 */ - freduce_degree(zzprime); - freduce_coefficients(zzprime); - /* |zzprime[i]| < 2^26 */ - memcpy(origxprime, xxprime, sizeof(limb) * 10); - fsum(xxprime, zzprime); - /* |xxprime[i]| < 2^27 */ - fdifference(zzprime, origxprime); - /* |zzprime[i]| < 2^27 */ - fsquare(xxxprime, xxprime); - /* |xxxprime[i]| < 2^26 */ - fsquare(zzzprime, zzprime); - /* |zzzprime[i]| < 2^26 */ - fproduct(zzprime, zzzprime, qmqp); - /* |zzprime[i]| < 14*2^52 */ - freduce_degree(zzprime); - freduce_coefficients(zzprime); - /* |zzprime[i]| < 2^26 */ - memcpy(x3, xxxprime, sizeof(limb) * 10); - memcpy(z3, zzprime, sizeof(limb) * 10); - - fsquare(xx, x); - /* |xx[i]| < 2^26 */ - fsquare(zz, z); - /* |zz[i]| < 2^26 */ - fproduct(x2, xx, zz); - /* |x2[i]| < 14*2^52 */ - freduce_degree(x2); - freduce_coefficients(x2); - /* |x2[i]| < 2^26 */ - fdifference(zz, xx); // does zz = xx - zz - /* |zz[i]| < 2^27 */ - memset(zzz + 10, 0, sizeof(limb) * 9); - fscalar_product(zzz, zz, 121665); - /* |zzz[i]| < 2^(27+17) */ - /* No need to call freduce_degree here: - fscalar_product doesn't increase the degree of its input. */ - freduce_coefficients(zzz); - /* |zzz[i]| < 2^26 */ - fsum(zzz, xx); - /* |zzz[i]| < 2^27 */ - fproduct(z2, zz, zzz); - /* |z2[i]| < 14*2^(26+27) */ - freduce_degree(z2); - freduce_coefficients(z2); - /* |z2|i| < 2^26 */ -} - -/* Conditionally swap two reduced-form limb arrays if 'iswap' is 1, but leave - * them unchanged if 'iswap' is 0. Runs in data-invariant time to avoid - * side-channel attacks. - * - * NOTE that this function requires that 'iswap' be 1 or 0; other values give - * wrong results. Also, the two limb arrays must be in reduced-coefficient, - * reduced-degree form: the values in a[10..19] or b[10..19] aren't swapped, - * and all all values in a[0..9],b[0..9] must have magnitude less than - * INT32_MAX. - */ -static void swap_conditional(limb a[static 19], limb b[static 19], limb iswap) +static __always_inline void fe_loose_invert(fe *out, const fe_loose *z) { - unsigned int i; - const int32_t swap = (int32_t) -iswap; + fe t0; + fe t1; + fe t2; + fe t3; + int i; - for (i = 0; i < 10; ++i) { - const int32_t x = swap & (((int32_t)a[i]) ^ ((int32_t)b[i])); + fe_sq_tl(&t0, z); + fe_sq_tt(&t1, &t0); + for (i = 1; i < 2; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_tlt(&t1, z, &t1); + fe_mul_ttt(&t0, &t0, &t1); + fe_sq_tt(&t2, &t0); + fe_mul_ttt(&t1, &t1, &t2); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 20; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 100; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t1, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_ttt(out, &t1, &t0); +} - a[i] = ((int32_t)a[i]) ^ x; - b[i] = ((int32_t)b[i]) ^ x; - } +static __always_inline void fe_invert(fe *out, const fe *z) +{ + fe_loose l; + fe_copy_lt(&l, z); + fe_loose_invert(out, &l); } -/* Calculates nQ where Q is the x-coordinate of a point on the curve +/* Replace (f,g) with (g,f) if b == 1; + * replace (f,g) with (f,g) if b == 0. * - * resultx/resultz: the x coordinate of the resulting curve point (short form) - * n: a little endian, 32-byte number - * q: a point of the curve (short form) + * Preconditions: b in {0,1} */ -static void cmult(limb *resultx, limb *resultz, const uint8_t *n, const limb *q) -{ - limb a[19] = {0}, b[19] = {1}, c[19] = {1}, d[19] = {0}; - limb *nqpqx = a, *nqpqz = b, *nqx = c, *nqz = d, *t; - limb e[19] = {0}, f[19] = {1}, g[19] = {0}, h[19] = {1}; - limb *nqpqx2 = e, *nqpqz2 = f, *nqx2 = g, *nqz2 = h; - - unsigned int i, j; - - memcpy(nqpqx, q, sizeof(limb) * 10); - - for (i = 0; i < 32; ++i) { - uint8_t byte = n[31 - i]; - - for (j = 0; j < 8; ++j) { - const limb bit = byte >> 7; - - swap_conditional(nqx, nqpqx, bit); - swap_conditional(nqz, nqpqz, bit); - fmonty(nqx2, nqz2, - nqpqx2, nqpqz2, - nqx, nqz, - nqpqx, nqpqz, - q); - swap_conditional(nqx2, nqpqx2, bit); - swap_conditional(nqz2, nqpqz2, bit); - - t = nqx; - nqx = nqx2; - nqx2 = t; - t = nqz; - nqz = nqz2; - nqz2 = t; - t = nqpqx; - nqpqx = nqpqx2; - nqpqx2 = t; - t = nqpqz; - nqpqz = nqpqz2; - nqpqz2 = t; - - byte <<= 1; - } +static __always_inline void fe_cswap(fe *f, fe *g, unsigned int b) +{ + unsigned i; + b = 0-b; + for (i = 0; i < 10; i++) { + uint32_t x = f->v[i] ^ g->v[i]; + x &= b; + f->v[i] ^= x; + g->v[i] ^= x; } - - memcpy(resultx, nqx, sizeof(limb) * 10); - memcpy(resultz, nqz, sizeof(limb) * 10); } -static void crecip(limb *out, const limb *z) +/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/ +static __always_inline void fe_mul_121666_impl(uint32_t out[10], const uint32_t in1[10]) { - limb z2[10]; - limb z9[10]; - limb z11[10]; - limb z2_5_0[10]; - limb z2_10_0[10]; - limb z2_20_0[10]; - limb z2_50_0[10]; - limb z2_100_0[10]; - limb t0[10]; - limb t1[10]; - int i; + { const uint32_t x20 = in1[9]; + { const uint32_t x21 = in1[8]; + { const uint32_t x19 = in1[7]; + { const uint32_t x17 = in1[6]; + { const uint32_t x15 = in1[5]; + { const uint32_t x13 = in1[4]; + { const uint32_t x11 = in1[3]; + { const uint32_t x9 = in1[2]; + { const uint32_t x7 = in1[1]; + { const uint32_t x5 = in1[0]; + { const uint32_t x38 = 0; + { const uint32_t x39 = 0; + { const uint32_t x37 = 0; + { const uint32_t x35 = 0; + { const uint32_t x33 = 0; + { const uint32_t x31 = 0; + { const uint32_t x29 = 0; + { const uint32_t x27 = 0; + { const uint32_t x25 = 0; + { const uint32_t x23 = 121666; + { uint64_t x40 = ((uint64_t)x23 * x5); + { uint64_t x41 = (((uint64_t)x23 * x7) + ((uint64_t)x25 * x5)); + { uint64_t x42 = ((((uint64_t)(0x2 * x25) * x7) + ((uint64_t)x23 * x9)) + ((uint64_t)x27 * x5)); + { uint64_t x43 = (((((uint64_t)x25 * x9) + ((uint64_t)x27 * x7)) + ((uint64_t)x23 * x11)) + ((uint64_t)x29 * x5)); + { uint64_t x44 = (((((uint64_t)x27 * x9) + (0x2 * (((uint64_t)x25 * x11) + ((uint64_t)x29 * x7)))) + ((uint64_t)x23 * x13)) + ((uint64_t)x31 * x5)); + { uint64_t x45 = (((((((uint64_t)x27 * x11) + ((uint64_t)x29 * x9)) + ((uint64_t)x25 * x13)) + ((uint64_t)x31 * x7)) + ((uint64_t)x23 * x15)) + ((uint64_t)x33 * x5)); + { uint64_t x46 = (((((0x2 * ((((uint64_t)x29 * x11) + ((uint64_t)x25 * x15)) + ((uint64_t)x33 * x7))) + ((uint64_t)x27 * x13)) + ((uint64_t)x31 * x9)) + ((uint64_t)x23 * x17)) + ((uint64_t)x35 * x5)); + { uint64_t x47 = (((((((((uint64_t)x29 * x13) + ((uint64_t)x31 * x11)) + ((uint64_t)x27 * x15)) + ((uint64_t)x33 * x9)) + ((uint64_t)x25 * x17)) + ((uint64_t)x35 * x7)) + ((uint64_t)x23 * x19)) + ((uint64_t)x37 * x5)); + { uint64_t x48 = (((((((uint64_t)x31 * x13) + (0x2 * (((((uint64_t)x29 * x15) + ((uint64_t)x33 * x11)) + ((uint64_t)x25 * x19)) + ((uint64_t)x37 * x7)))) + ((uint64_t)x27 * x17)) + ((uint64_t)x35 * x9)) + ((uint64_t)x23 * x21)) + ((uint64_t)x39 * x5)); + { uint64_t x49 = (((((((((((uint64_t)x31 * x15) + ((uint64_t)x33 * x13)) + ((uint64_t)x29 * x17)) + ((uint64_t)x35 * x11)) + ((uint64_t)x27 * x19)) + ((uint64_t)x37 * x9)) + ((uint64_t)x25 * x21)) + ((uint64_t)x39 * x7)) + ((uint64_t)x23 * x20)) + ((uint64_t)x38 * x5)); + { uint64_t x50 = (((((0x2 * ((((((uint64_t)x33 * x15) + ((uint64_t)x29 * x19)) + ((uint64_t)x37 * x11)) + ((uint64_t)x25 * x20)) + ((uint64_t)x38 * x7))) + ((uint64_t)x31 * x17)) + ((uint64_t)x35 * x13)) + ((uint64_t)x27 * x21)) + ((uint64_t)x39 * x9)); + { uint64_t x51 = (((((((((uint64_t)x33 * x17) + ((uint64_t)x35 * x15)) + ((uint64_t)x31 * x19)) + ((uint64_t)x37 * x13)) + ((uint64_t)x29 * x21)) + ((uint64_t)x39 * x11)) + ((uint64_t)x27 * x20)) + ((uint64_t)x38 * x9)); + { uint64_t x52 = (((((uint64_t)x35 * x17) + (0x2 * (((((uint64_t)x33 * x19) + ((uint64_t)x37 * x15)) + ((uint64_t)x29 * x20)) + ((uint64_t)x38 * x11)))) + ((uint64_t)x31 * x21)) + ((uint64_t)x39 * x13)); + { uint64_t x53 = (((((((uint64_t)x35 * x19) + ((uint64_t)x37 * x17)) + ((uint64_t)x33 * x21)) + ((uint64_t)x39 * x15)) + ((uint64_t)x31 * x20)) + ((uint64_t)x38 * x13)); + { uint64_t x54 = (((0x2 * ((((uint64_t)x37 * x19) + ((uint64_t)x33 * x20)) + ((uint64_t)x38 * x15))) + ((uint64_t)x35 * x21)) + ((uint64_t)x39 * x17)); + { uint64_t x55 = (((((uint64_t)x37 * x21) + ((uint64_t)x39 * x19)) + ((uint64_t)x35 * x20)) + ((uint64_t)x38 * x17)); + { uint64_t x56 = (((uint64_t)x39 * x21) + (0x2 * (((uint64_t)x37 * x20) + ((uint64_t)x38 * x19)))); + { uint64_t x57 = (((uint64_t)x39 * x20) + ((uint64_t)x38 * x21)); + { uint64_t x58 = ((uint64_t)(0x2 * x38) * x20); + { uint64_t x59 = (x48 + (x58 << 0x4)); + { uint64_t x60 = (x59 + (x58 << 0x1)); + { uint64_t x61 = (x60 + x58); + { uint64_t x62 = (x47 + (x57 << 0x4)); + { uint64_t x63 = (x62 + (x57 << 0x1)); + { uint64_t x64 = (x63 + x57); + { uint64_t x65 = (x46 + (x56 << 0x4)); + { uint64_t x66 = (x65 + (x56 << 0x1)); + { uint64_t x67 = (x66 + x56); + { uint64_t x68 = (x45 + (x55 << 0x4)); + { uint64_t x69 = (x68 + (x55 << 0x1)); + { uint64_t x70 = (x69 + x55); + { uint64_t x71 = (x44 + (x54 << 0x4)); + { uint64_t x72 = (x71 + (x54 << 0x1)); + { uint64_t x73 = (x72 + x54); + { uint64_t x74 = (x43 + (x53 << 0x4)); + { uint64_t x75 = (x74 + (x53 << 0x1)); + { uint64_t x76 = (x75 + x53); + { uint64_t x77 = (x42 + (x52 << 0x4)); + { uint64_t x78 = (x77 + (x52 << 0x1)); + { uint64_t x79 = (x78 + x52); + { uint64_t x80 = (x41 + (x51 << 0x4)); + { uint64_t x81 = (x80 + (x51 << 0x1)); + { uint64_t x82 = (x81 + x51); + { uint64_t x83 = (x40 + (x50 << 0x4)); + { uint64_t x84 = (x83 + (x50 << 0x1)); + { uint64_t x85 = (x84 + x50); + { uint64_t x86 = (x85 >> 0x1a); + { uint32_t x87 = ((uint32_t)x85 & 0x3ffffff); + { uint64_t x88 = (x86 + x82); + { uint64_t x89 = (x88 >> 0x19); + { uint32_t x90 = ((uint32_t)x88 & 0x1ffffff); + { uint64_t x91 = (x89 + x79); + { uint64_t x92 = (x91 >> 0x1a); + { uint32_t x93 = ((uint32_t)x91 & 0x3ffffff); + { uint64_t x94 = (x92 + x76); + { uint64_t x95 = (x94 >> 0x19); + { uint32_t x96 = ((uint32_t)x94 & 0x1ffffff); + { uint64_t x97 = (x95 + x73); + { uint64_t x98 = (x97 >> 0x1a); + { uint32_t x99 = ((uint32_t)x97 & 0x3ffffff); + { uint64_t x100 = (x98 + x70); + { uint64_t x101 = (x100 >> 0x19); + { uint32_t x102 = ((uint32_t)x100 & 0x1ffffff); + { uint64_t x103 = (x101 + x67); + { uint64_t x104 = (x103 >> 0x1a); + { uint32_t x105 = ((uint32_t)x103 & 0x3ffffff); + { uint64_t x106 = (x104 + x64); + { uint64_t x107 = (x106 >> 0x19); + { uint32_t x108 = ((uint32_t)x106 & 0x1ffffff); + { uint64_t x109 = (x107 + x61); + { uint64_t x110 = (x109 >> 0x1a); + { uint32_t x111 = ((uint32_t)x109 & 0x3ffffff); + { uint64_t x112 = (x110 + x49); + { uint64_t x113 = (x112 >> 0x19); + { uint32_t x114 = ((uint32_t)x112 & 0x1ffffff); + { uint64_t x115 = (x87 + (0x13 * x113)); + { uint32_t x116 = (uint32_t) (x115 >> 0x1a); + { uint32_t x117 = ((uint32_t)x115 & 0x3ffffff); + { uint32_t x118 = (x116 + x90); + { uint32_t x119 = (x118 >> 0x19); + { uint32_t x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} - /* 2 */ fsquare(z2, z); - /* 4 */ fsquare(t1, z2); - /* 8 */ fsquare(t0, t1); - /* 9 */ fmul(z9, t0, z); - /* 11 */ fmul(z11, z9, z2); - /* 22 */ fsquare(t0, z11); - /* 2^5 - 2^0 = 31 */ fmul(z2_5_0, t0, z9); - - /* 2^6 - 2^1 */ fsquare(t0, z2_5_0); - /* 2^7 - 2^2 */ fsquare(t1, t0); - /* 2^8 - 2^3 */ fsquare(t0, t1); - /* 2^9 - 2^4 */ fsquare(t1, t0); - /* 2^10 - 2^5 */ fsquare(t0, t1); - /* 2^10 - 2^0 */ fmul(z2_10_0, t0, z2_5_0); - - /* 2^11 - 2^1 */ fsquare(t0, z2_10_0); - /* 2^12 - 2^2 */ fsquare(t1, t0); - /* 2^20 - 2^10 */ for (i = 2; i < 10; i += 2) { fsquare(t0, t1); fsquare(t1, t0); } - /* 2^20 - 2^0 */ fmul(z2_20_0, t1, z2_10_0); - - /* 2^21 - 2^1 */ fsquare(t0, z2_20_0); - /* 2^22 - 2^2 */ fsquare(t1, t0); - /* 2^40 - 2^20 */ for (i = 2; i < 20; i += 2) { fsquare(t0, t1); fsquare(t1, t0); } - /* 2^40 - 2^0 */ fmul(t0, t1, z2_20_0); - - /* 2^41 - 2^1 */ fsquare(t1, t0); - /* 2^42 - 2^2 */ fsquare(t0, t1); - /* 2^50 - 2^10 */ for (i = 2; i < 10; i += 2) { fsquare(t1, t0); fsquare(t0, t1); } - /* 2^50 - 2^0 */ fmul(z2_50_0, t0, z2_10_0); - - /* 2^51 - 2^1 */ fsquare(t0, z2_50_0); - /* 2^52 - 2^2 */ fsquare(t1, t0); - /* 2^100 - 2^50 */ for (i = 2; i < 50; i += 2) { fsquare(t0, t1); fsquare(t1, t0); } - /* 2^100 - 2^0 */ fmul(z2_100_0, t1, z2_50_0); - - /* 2^101 - 2^1 */ fsquare(t1, z2_100_0); - /* 2^102 - 2^2 */ fsquare(t0, t1); - /* 2^200 - 2^100 */ for (i = 2; i < 100; i += 2) { fsquare(t1, t0); fsquare(t0, t1); } - /* 2^200 - 2^0 */ fmul(t1, t0, z2_100_0); - - /* 2^201 - 2^1 */ fsquare(t0, t1); - /* 2^202 - 2^2 */ fsquare(t1, t0); - /* 2^250 - 2^50 */ for (i = 2; i < 50; i += 2) { fsquare(t0, t1); fsquare(t1, t0); } - /* 2^250 - 2^0 */ fmul(t0, t1, z2_50_0); - - /* 2^251 - 2^1 */ fsquare(t1, t0); - /* 2^252 - 2^2 */ fsquare(t0, t1); - /* 2^253 - 2^3 */ fsquare(t1, t0); - /* 2^254 - 2^4 */ fsquare(t0, t1); - /* 2^255 - 2^5 */ fsquare(t1, t0); - /* 2^255 - 21 */ fmul(out, t1, z11); +static __always_inline void fe_mul121666(fe *h, const fe_loose *f) +{ + fe_mul_121666_impl(h->v, f->v); } -void curve25519(uint8_t mypublic[static CURVE25519_POINT_SIZE], const uint8_t secret[static CURVE25519_POINT_SIZE], const uint8_t basepoint[static CURVE25519_POINT_SIZE]) +void curve25519(uint8_t out[static CURVE25519_POINT_SIZE], const uint8_t scalar[static CURVE25519_POINT_SIZE], const uint8_t point[static CURVE25519_POINT_SIZE]) { - limb bp[10], x[10], z[11], zmone[10]; + fe x1, x2, z2, x3, z3, tmp0, tmp1; + fe_loose x2l, z2l, x3l, tmp0l, tmp1l; + unsigned swap = 0; + int pos; uint8_t e[32]; - memcpy(e, secret, 32); + memcpy(e, scalar, 32); curve25519_normalize_secret(e); - fexpand(bp, basepoint); - cmult(x, z, e, bp); - crecip(zmone, z); - fmul(z, x, zmone); - fcontract(mypublic, z); + /* The following implementation was transcribed to Coq and proven to + * correspond to unary scalar multiplication in affine coordinates given that + * x1 != 0 is the x coordinate of some point on the curve. It was also checked + * in Coq that doing a ladderstep with x1 = x3 = 0 gives z2' = z3' = 0, and z2 + * = z3 = 0 gives z2' = z3' = 0. The statement was quantified over the + * underlying field, so it applies to Curve25519 itself and the quadratic + * twist of Curve25519. It was not proven in Coq that prime-field arithmetic + * correctly simulates extension-field arithmetic on prime-field values. + * The decoding of the byte array representation of e was not considered. + * Specification of Montgomery curves in affine coordinates: + * + * Proof that these form a group that is isomorphic to a Weierstrass curve: + * + * Coq transcription and correctness proof of the loop (where scalarbits=255): + * + * + * preconditions: 0 <= e < 2^255 (not necessarily e < order), fe_invert(0) = 0 + */ + fe_frombytes(&x1, point); + fe_1(&x2); + fe_0(&z2); + fe_copy(&x3, &x1); + fe_1(&z3); + + for (pos = 254; pos >= 0; --pos) { + /* loop invariant as of right before the test, for the case where x1 != 0: + * pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3 is nonzero + * let r := e >> (pos+1) in the following equalities of projective points: + * to_xz (r*P) === if swap then (x3, z3) else (x2, z2) + * to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3) + * x1 is the nonzero x coordinate of the nonzero point (r*P-(r+1)*P) + */ + unsigned b = 1 & (e[pos / 8] >> (pos & 7)); + swap ^= b; + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + swap = b; + /* Coq transcription of ladderstep formula (called from transcribed loop): + * + * + * x1 != 0 + * x1 = 0 + */ + fe_sub(&tmp0l, &x3, &z3); + fe_sub(&tmp1l, &x2, &z2); + fe_add(&x2l, &x2, &z2); + fe_add(&z2l, &x3, &z3); + fe_mul_tll(&z3, &tmp0l, &x2l); + fe_mul_tll(&z2, &z2l, &tmp1l); + fe_sq_tl(&tmp0, &tmp1l); + fe_sq_tl(&tmp1, &x2l); + fe_add(&x3l, &z3, &z2); + fe_sub(&z2l, &z3, &z2); + fe_mul_ttt(&x2, &tmp1, &tmp0); + fe_sub(&tmp1l, &tmp1, &tmp0); + fe_sq_tl(&z2, &z2l); + fe_mul121666(&z3, &tmp1l); + fe_sq_tl(&x3, &x3l); + fe_add(&tmp0l, &tmp0, &z3); + fe_mul_ttt(&z3, &x1, &z2); + fe_mul_tll(&z2, &tmp1l, &tmp0l); + } + /* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3) else (x2, z2) */ + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + + fe_invert(&z2, &z2); + fe_mul_ttt(&x2, &x2, &z2); + fe_tobytes(out, &x2); } #endif -- cgit v1.2.3-59-g8ed1b