From 3ef8d945d0dafd272e77c01099bc4975c5297a5a Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Thu, 23 Oct 2014 16:11:23 +0300 Subject: crypto: caam - add support for gcm(aes) Add support for AES working in Galois Counter Mode. There is a limitation related to IV size, similar to the one present in SW implementation (crypto/gcm.c): The only IV size allowed is 12 bytes. It will be padded by HW to the right with 0x0000_0001 (up to 16 bytes - AES block size), according to the GCM specification. Signed-off-by: Tudor Ambarus Signed-off-by: Horia Geanta Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 342 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 331 insertions(+), 11 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index a80ea853701d..66e35efcedfa 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -74,6 +74,10 @@ #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) +#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) +#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ) +#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ) + #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 20 * CAAM_CMD_SZ) @@ -630,6 +634,236 @@ static int aead_setauthsize(struct crypto_aead *authenc, return 0; } +static int gcm_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = false; + u32 *key_jump_cmd, *zero_payload_jump_cmd, + *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; + u32 *desc; + + if (!ctx->enckeylen || !ctx->authsize) + return 0; + + /* + * AES GCM encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_enc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD | JUMP_COND_SELF); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen + cryptlen = seqinlen - ivsize */ + append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ); + + /* if cryptlen is ZERO jump to zero-payload commands */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* jump the zero-payload commands */ + append_jump(desc, JUMP_TEST_ALL | 7); + + /* zero-payload commands */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* if assoclen is ZERO, jump to IV reading - is the only input data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); + + /* jump to ICV writing */ + append_jump(desc, JUMP_TEST_ALL | 2); + + /* read IV - is the only input data */ + set_jump_tgt_here(desc, zero_assoc_jump_cmd2); + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | + FIFOLD_TYPE_LAST1); + + /* write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_dec; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | + JUMP_TEST_ALL | JUMP_COND_SHRD | + JUMP_COND_SELF); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, + ctx->authsize + tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ); + + /* read IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* jump to zero-payload command if cryptlen is zero */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + /* if asoclen is ZERO, skip reading assoc data */ + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* store encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* jump the zero-payload commands */ + append_jump(desc, JUMP_TEST_ALL | 4); + + /* zero-payload command */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* if assoclen is ZERO, jump to ICV reading */ + append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd2); + + /* read ICV */ + append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return 0; +} + +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + gcm_set_sh_desc(authenc); + + return 0; +} + static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen) { @@ -703,6 +937,36 @@ badkey: return -EINVAL; } +static int gcm_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + int ret = 0; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->key_dma)) { + dev_err(jrdev, "unable to map key i/o memory\n"); + return -ENOMEM; + } + ctx->enckeylen = keylen; + + ret = gcm_set_sh_desc(aead); + if (ret) { + dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, + DMA_TO_DEVICE); + } + + return ret; +} + static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const u8 *key, unsigned int keylen) { @@ -1088,6 +1352,7 @@ static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, u32 out_options = 0, in_options; dma_addr_t dst_dma, src_dma; int len, sec4_sg_index = 0; + bool is_gcm = false; #ifdef DEBUG debug("assoclen %d cryptlen %d authsize %d\n", @@ -1106,11 +1371,19 @@ static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, desc_bytes(sh_desc), 1); #endif + if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_AES) && + ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) + is_gcm = true; + len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (all_contig) { - src_dma = sg_dma_address(req->assoc); + if (is_gcm) + src_dma = edesc->iv_dma; + else + src_dma = sg_dma_address(req->assoc); in_options = 0; } else { src_dma = edesc->sec4_sg_dma; @@ -1292,6 +1565,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, int ivsize = crypto_aead_ivsize(aead); int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; unsigned int authsize = ctx->authsize; + bool is_gcm = false; assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); @@ -1326,15 +1600,31 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, return ERR_PTR(-ENOMEM); } - /* Check if data are contiguous */ - if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != - iv_dma || src_nents || iv_dma + ivsize != - sg_dma_address(req->src)) { - all_contig = false; + if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_AES) && + ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) + is_gcm = true; + + /* + * Check if data are contiguous. + * GCM expected input sequence: IV, AAD, text + * All other - expected input sequence: AAD, IV, text + */ + if (is_gcm) + all_contig = (!assoc_nents && + iv_dma + ivsize == sg_dma_address(req->assoc) && + !src_nents && sg_dma_address(req->assoc) + + req->assoclen == sg_dma_address(req->src)); + else + all_contig = (!assoc_nents && sg_dma_address(req->assoc) + + req->assoclen == iv_dma && !src_nents && + iv_dma + ivsize == sg_dma_address(req->src)); + if (!all_contig) { assoc_nents = assoc_nents ? : 1; src_nents = src_nents ? : 1; sec4_sg_len = assoc_nents + 1 + src_nents; } + sec4_sg_len += dst_nents; sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); @@ -1361,14 +1651,26 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_index = 0; if (!all_contig) { - sg_to_sec4_sg(req->assoc, - (assoc_nents ? : 1), - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += assoc_nents ? : 1; + if (!is_gcm) { + sg_to_sec4_sg(req->assoc, + (assoc_nents ? : 1), + edesc->sec4_sg + + sec4_sg_index, 0); + sec4_sg_index += assoc_nents ? : 1; + } + dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, iv_dma, ivsize, 0); sec4_sg_index += 1; + + if (is_gcm) { + sg_to_sec4_sg(req->assoc, + (assoc_nents ? : 1), + edesc->sec4_sg + + sec4_sg_index, 0); + sec4_sg_index += assoc_nents ? : 1; + } + sg_to_sec4_sg_last(req->src, (src_nents ? : 1), edesc->sec4_sg + @@ -2309,6 +2611,24 @@ static struct caam_alg_template driver_algs[] = { OP_ALG_AAI_HMAC_PRECOMP, .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, + /* Galois Counter Mode */ + { + .name = "gcm(aes)", + .driver_name = "gcm-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = NULL, + .geniv = "", + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, /* ablkcipher descriptor */ { .name = "cbc(aes)", -- cgit v1.2.3-59-g8ed1b From bac68f2c9a43e60a70e27c0c75aaca7128b6271d Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Thu, 23 Oct 2014 16:14:03 +0300 Subject: crypto: caam - add support for rfc4106(gcm(aes)) Add Advanced Encryption Standard (AES) in Galois/Counter Mode (GCM) as an IPsec Encapsulating Security Payload (ESP) mechanism to provide confidentiality and data origin authentication. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 429 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 414 insertions(+), 15 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 66e35efcedfa..c17154c32031 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -78,6 +78,11 @@ #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ) #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ) +#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ) +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ) +#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ) + #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 20 * CAAM_CMD_SZ) @@ -864,6 +869,287 @@ static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) return 0; } +static int rfc4106_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = false; + u32 *key_jump_cmd, *move_cmd, *write_iv_cmd; + u32 *desc; + u32 geniv; + + if (!ctx->enckeylen || !ctx->authsize) + return 0; + + /* + * RFC4106 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_enc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* assoclen + cryptlen = seqinlen - ivsize */ + append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); + + /* Read Salt */ + append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), + 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); + /* Read AES-GCM-ESP IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Will read cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_dec; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | + JUMP_TEST_ALL | JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, + ctx->authsize + tfm->ivsize); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + + /* Will write cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Read Salt */ + append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), + 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); + /* Read AES-GCM-ESP IV */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Will read cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* Store payload data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read encrypted data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* Read ICV */ + append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_RFC4106_GIVENC_LEN + DESC_JOB_IO_LEN + + ctx->split_key_pad_len + ctx->enckeylen <= + CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + /* rfc4106_givencrypt shared descriptor */ + desc = ctx->sh_desc_givenc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | + NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + move_cmd = append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Copy generated IV to OFIFO */ + write_iv_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_OUTFIFO | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* ivsize + cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen = seqinlen - (ivsize + cryptlen) */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* Will write ivsize + cryptlen */ + append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ); + + /* Read Salt and generated IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV | + FIFOLD_TYPE_FLUSH1 | IMMEDIATE | 12); + /* Append Salt */ + append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); + set_move_tgt_here(desc, move_cmd); + set_move_tgt_here(desc, write_iv_cmd); + /* Blank commands. Will be overwritten by generated IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* No need to reload iv */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Will read cryptlen */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Store generated IV and encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "rfc4106 givenc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return 0; +} + +static int rfc4106_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + rfc4106_set_sh_desc(authenc); + + return 0; +} + static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen) { @@ -967,6 +1253,45 @@ static int gcm_setkey(struct crypto_aead *aead, return ret; } +static int rfc4106_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + int ret = 0; + + if (keylen < 4) + return -EINVAL; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->enckeylen = keylen - 4; + + ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->key_dma)) { + dev_err(jrdev, "unable to map key i/o memory\n"); + return -ENOMEM; + } + + ret = rfc4106_set_sh_desc(aead); + if (ret) { + dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, + DMA_TO_DEVICE); + } + + return ret; +} + static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const u8 *key, unsigned int keylen) { @@ -1437,6 +1762,7 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, u32 out_options = 0, in_options; dma_addr_t dst_dma, src_dma; int len, sec4_sg_index = 0; + bool is_gcm = false; #ifdef DEBUG debug("assoclen %d cryptlen %d authsize %d\n", @@ -1454,11 +1780,19 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, desc_bytes(sh_desc), 1); #endif + if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_AES) && + ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) + is_gcm = true; + len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (contig & GIV_SRC_CONTIG) { - src_dma = sg_dma_address(req->assoc); + if (is_gcm) + src_dma = edesc->iv_dma; + else + src_dma = sg_dma_address(req->assoc); in_options = 0; } else { src_dma = edesc->sec4_sg_dma; @@ -1473,7 +1807,8 @@ static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, } else { if (likely(req->src == req->dst)) { dst_dma = src_dma + sizeof(struct sec4_sg_entry) * - edesc->assoc_nents; + (edesc->assoc_nents + + (is_gcm ? 1 + edesc->src_nents : 0)); out_options = LDST_SGF; } else { dst_dma = edesc->sec4_sg_dma + @@ -1792,6 +2127,7 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request int ivsize = crypto_aead_ivsize(aead); bool assoc_chained = false, src_chained = false, dst_chained = false; int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; + bool is_gcm = false; assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); src_nents = sg_count(req->src, req->cryptlen, &src_chained); @@ -1818,16 +2154,33 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request return ERR_PTR(-ENOMEM); } - /* Check if data are contiguous */ - if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != - iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) - contig &= ~GIV_SRC_CONTIG; + if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == + OP_ALG_ALGSEL_AES) && + ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) + is_gcm = true; + + /* + * Check if data are contiguous. + * GCM expected input sequence: IV, AAD, text + * All other - expected input sequence: AAD, IV, text + */ + + if (is_gcm) { + if (assoc_nents || iv_dma + ivsize != + sg_dma_address(req->assoc) || src_nents || + sg_dma_address(req->assoc) + req->assoclen != + sg_dma_address(req->src)) + contig &= ~GIV_SRC_CONTIG; + } else { + if (assoc_nents || + sg_dma_address(req->assoc) + req->assoclen != iv_dma || + src_nents || iv_dma + ivsize != sg_dma_address(req->src)) + contig &= ~GIV_SRC_CONTIG; + } + if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) contig &= ~GIV_DST_CONTIG; - if (unlikely(req->src != req->dst)) { - dst_nents = dst_nents ? : 1; - sec4_sg_len += 1; - } + if (!(contig & GIV_SRC_CONTIG)) { assoc_nents = assoc_nents ? : 1; src_nents = src_nents ? : 1; @@ -1835,7 +2188,18 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request if (likely(req->src == req->dst)) contig &= ~GIV_DST_CONTIG; } - sec4_sg_len += dst_nents; + + /* + * Add new sg entries for GCM output sequence. + * Expected output sequence: IV, encrypted text. + */ + if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) + sec4_sg_len += 1 + src_nents; + + if (unlikely(req->src != req->dst)) { + dst_nents = dst_nents ? : 1; + sec4_sg_len += 1 + dst_nents; + } sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); @@ -1861,18 +2225,36 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request sec4_sg_index = 0; if (!(contig & GIV_SRC_CONTIG)) { - sg_to_sec4_sg(req->assoc, assoc_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += assoc_nents; + if (!is_gcm) { + sg_to_sec4_sg(req->assoc, assoc_nents, + edesc->sec4_sg + sec4_sg_index, 0); + sec4_sg_index += assoc_nents; + } + dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, iv_dma, ivsize, 0); sec4_sg_index += 1; + + if (is_gcm) { + sg_to_sec4_sg(req->assoc, assoc_nents, + edesc->sec4_sg + sec4_sg_index, 0); + sec4_sg_index += assoc_nents; + } + sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg + sec4_sg_index, 0); sec4_sg_index += src_nents; } + + if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) { + dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, + iv_dma, ivsize, 0); + sec4_sg_index += 1; + sg_to_sec4_sg_last(req->src, src_nents, + edesc->sec4_sg + sec4_sg_index, 0); + } + if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, iv_dma, ivsize, 0); @@ -2611,6 +2993,23 @@ static struct caam_alg_template driver_algs[] = { OP_ALG_AAI_HMAC_PRECOMP, .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, + { + .name = "rfc4106(gcm(aes))", + .driver_name = "rfc4106-gcm-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, /* Galois Counter Mode */ { .name = "gcm(aes)", -- cgit v1.2.3-59-g8ed1b From 19167bfd8ae72ceeb41930343227bf8f5fb19a66 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Fri, 24 Oct 2014 18:13:37 +0300 Subject: crypto: caam - fix output sequence contiguity check This patch fixes the assumption that output sequence is not contiguous when input sequence is not contiguous and in-place encryption is done. Output sequence does not need to be contiguous with associated data. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index c17154c32031..36434d9eefad 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -2185,7 +2185,8 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request assoc_nents = assoc_nents ? : 1; src_nents = src_nents ? : 1; sec4_sg_len += assoc_nents + 1 + src_nents; - if (likely(req->src == req->dst)) + if (req->src == req->dst && + (src_nents || iv_dma + ivsize != sg_dma_address(req->src))) contig &= ~GIV_DST_CONTIG; } -- cgit v1.2.3-59-g8ed1b From 5d0429a30f06845af78a696de59e3e8d21252846 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Thu, 30 Oct 2014 18:55:07 +0200 Subject: crypto: caam - add support for rfc4543(gcm(aes)) Add AES-GMAC as an IPSec ESP mechanism to provide data origin authentication, but not confidentiality. This method is referred as ENCR_NULL_AUTH_AES_GMAC. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 458 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 457 insertions(+), 1 deletion(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 36434d9eefad..48df0520ed46 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -83,13 +83,18 @@ #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ) #define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ) +#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ) +#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ) +#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ) + #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 20 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 15 * CAAM_CMD_SZ) -#define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \ +#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \ CAAM_MAX_KEY_SIZE) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) @@ -1150,6 +1155,401 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc, return 0; } +static int rfc4543_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = false; + u32 *key_jump_cmd, *write_iv_cmd, *write_aad_cmd; + u32 *read_move_cmd, *write_move_cmd; + u32 *desc; + u32 geniv; + + if (!ctx->enckeylen || !ctx->authsize) + return 0; + + /* + * RFC4543 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_RFC4543_ENC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_enc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* Load AES-GMAC ESP IV into Math1 register */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 | + LDST_CLASS_DECO | tfm->ivsize); + + /* Wait the DMA transaction to finish */ + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | + (1 << JUMP_OFFSET_SHIFT)); + + /* Overwrite blank immediate AES-GMAC ESP IV data */ + write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Overwrite blank immediate AAD data */ + write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen = (seqinlen - ivsize) - cryptlen */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* Read Salt and AES-GMAC ESP IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); + /* Append Salt */ + append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); + set_move_tgt_here(desc, write_iv_cmd); + /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD); + + /* Will read cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Will write cryptlen bytes */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT)); + + /* Authenticate AES-GMAC ESP IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_AAD | tfm->ivsize); + set_move_tgt_here(desc, write_aad_cmd); + /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_AAD); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_RFC4543_DEC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_dec; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | + JUMP_TEST_ALL | JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* Load AES-GMAC ESP IV into Math1 register */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 | + LDST_CLASS_DECO | tfm->ivsize); + + /* Wait the DMA transaction to finish */ + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | + (1 << JUMP_OFFSET_SHIFT)); + + /* assoclen + cryptlen = (seqinlen - ivsize) - icvsize */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, ctx->authsize); + + /* Overwrite blank immediate AES-GMAC ESP IV data */ + write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Overwrite blank immediate AAD data */ + write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT)); + + /* Read Salt and AES-GMAC ESP IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); + /* Append Salt */ + append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); + set_move_tgt_here(desc, write_iv_cmd); + /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD); + + /* Will read cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* Will write cryptlen bytes */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* Authenticate AES-GMAC ESP IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_AAD | tfm->ivsize); + set_move_tgt_here(desc, write_aad_cmd); + /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* Store payload data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* In-snoop cryptlen data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Read ICV */ + append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_RFC4543_GIVENC_LEN + DESC_JOB_IO_LEN + + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + /* rfc4543_givencrypt shared descriptor */ + desc = ctx->sh_desc_givenc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, ctx->key_dma, ctx->enckeylen, + CLASS_1 | KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | + NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move generated IV to Math1 register */ + append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_MATH1 | + (tfm->ivsize << MOVE_LEN_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Overwrite blank immediate AES-GMAC IV data */ + write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Overwrite blank immediate AAD data */ + write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Copy generated IV to OFIFO */ + append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_OUTFIFO | + (tfm->ivsize << MOVE_LEN_SHIFT)); + + /* Class 1 operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* ivsize + cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* assoclen = seqinlen - (ivsize + cryptlen) */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* Will write ivsize + cryptlen */ + append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT)); + + /* Read Salt and AES-GMAC generated IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); + /* Append Salt */ + append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); + set_move_tgt_here(desc, write_iv_cmd); + /* Blank commands. Will be overwritten by AES-GMAC generated IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* No need to reload iv */ + append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD); + + /* Will read cryptlen */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Authenticate AES-GMAC IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_AAD | tfm->ivsize); + set_move_tgt_here(desc, write_aad_cmd); + /* Blank commands. Will be overwritten by AES-GMAC IV. */ + append_cmd(desc, 0x00000000); + append_cmd(desc, 0x00000000); + /* End of blank commands */ + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_AAD); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "rfc4543 givenc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return 0; +} + +static int rfc4543_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + rfc4543_set_sh_desc(authenc); + + return 0; +} + static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen) { @@ -1292,6 +1692,45 @@ static int rfc4106_setkey(struct crypto_aead *aead, return ret; } +static int rfc4543_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + int ret = 0; + + if (keylen < 4) + return -EINVAL; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->enckeylen = keylen - 4; + + ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->key_dma)) { + dev_err(jrdev, "unable to map key i/o memory\n"); + return -ENOMEM; + } + + ret = rfc4543_set_sh_desc(aead); + if (ret) { + dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, + DMA_TO_DEVICE); + } + + return ret; +} + static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const u8 *key, unsigned int keylen) { @@ -3011,6 +3450,23 @@ static struct caam_alg_template driver_algs[] = { }, .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, }, + { + .name = "rfc4543(gcm(aes))", + .driver_name = "rfc4543-gcm-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, /* Galois Counter Mode */ { .name = "gcm(aes)", -- cgit v1.2.3-59-g8ed1b From 2b22f6c547f90e1a41e3f39ad8d569e3efc74d42 Mon Sep 17 00:00:00 2001 From: Catalin Vasile Date: Fri, 31 Oct 2014 12:45:35 +0200 Subject: crypto: caam - add support for ctr(aes) Add support for AES working in Counter Mode Signed-off-by: Catalin Vasile Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 40 ++++++++++++++++++++++++++++++++++----- drivers/crypto/caam/desc_constr.h | 2 ++ 2 files changed, 37 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 48df0520ed46..2dc85f8304bb 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1740,11 +1740,21 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, int ret = 0; u32 *key_jump_cmd; u32 *desc; + u32 ctx1_iv_off = 0; + const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; memcpy(ctx->key, key, keylen); ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, @@ -1770,8 +1780,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, set_jump_tgt_here(desc, key_jump_cmd); /* Load iv */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | tfm->ivsize); + append_seq_load(desc, tfm->ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); /* Load operation */ append_operation(desc, ctx->class1_alg_type | @@ -1809,11 +1819,15 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, set_jump_tgt_here(desc, key_jump_cmd); /* load IV */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | tfm->ivsize); + append_seq_load(desc, tfm->ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); /* Choose operation */ - append_dec_op1(desc, ctx->class1_alg_type); + if (ctr_mode) + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); + else + append_dec_op1(desc, ctx->class1_alg_type); /* Perform operation */ ablkcipher_append_src_dst(desc); @@ -3533,6 +3547,22 @@ static struct caam_alg_template driver_algs[] = { .ivsize = DES_BLOCK_SIZE, }, .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "chainiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, } }; diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 7eec20bb3849..9f79fd7bd4d7 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -192,6 +192,8 @@ static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \ PRINT_POS; \ append_cmd(desc, CMD_##op | len | options); \ } + +APPEND_CMD_LEN(seq_load, SEQ_LOAD) APPEND_CMD_LEN(seq_store, SEQ_STORE) APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_LOAD) APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE) -- cgit v1.2.3-59-g8ed1b From a5f57cffce8af8d2c11204b4e289543021c73766 Mon Sep 17 00:00:00 2001 From: Catalin Vasile Date: Fri, 31 Oct 2014 12:45:36 +0200 Subject: crypto: caam - add support for rfc3686(ctr(aes)) Add support for Advanced Encryption Standard (AES) in Counter Mode (CTR) as provided in IPsec implementation standard RFC3686. ablkcipher shared descriptors now save context registers after job execution. This is used to load Nonce specific to RFC3686 only at first execution of shared job descriptor. Signed-off-by: Catalin Vasile Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 83 ++++++++++++++++++++++++++++++++++++++++--- drivers/crypto/caam/compat.h | 1 + 2 files changed, 79 insertions(+), 5 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 2dc85f8304bb..e9a4fd16031d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1735,14 +1735,19 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const u8 *key, unsigned int keylen) { struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); - struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; + struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); + const char *alg_name = crypto_tfm_alg_name(tfm); struct device *jrdev = ctx->jrdev; int ret = 0; u32 *key_jump_cmd; u32 *desc; + u32 *nonce; u32 ctx1_iv_off = 0; const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = (ctr_mode && + (strstr(alg_name, "rfc3686") != NULL)); #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", @@ -1756,6 +1761,16 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, if (ctr_mode) ctx1_iv_off = 16; + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + } + memcpy(ctx->key, key, keylen); ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); @@ -1767,7 +1782,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, /* ablkcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - init_sh_desc(desc, HDR_SHARE_SERIAL); + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); @@ -1777,12 +1792,32 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + /* Load nonce into CONTEXT1 reg */ + if (is_rfc3686) { + nonce = (u32 *)(key + keylen); + append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + set_jump_tgt_here(desc, key_jump_cmd); /* Load iv */ - append_seq_load(desc, tfm->ivsize, LDST_SRCDST_BYTE_CONTEXT | + append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); + /* Load counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + /* Load operation */ append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); @@ -1806,7 +1841,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, /* ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; - init_sh_desc(desc, HDR_SHARE_SERIAL); + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); @@ -1816,12 +1851,32 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + /* Load nonce into CONTEXT1 reg */ + if (is_rfc3686) { + nonce = (u32 *)(key + keylen); + append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + set_jump_tgt_here(desc, key_jump_cmd); /* load IV */ - append_seq_load(desc, tfm->ivsize, LDST_SRCDST_BYTE_CONTEXT | + append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); + /* Load counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + /* Choose operation */ if (ctr_mode) append_operation(desc, ctx->class1_alg_type | @@ -3563,6 +3618,24 @@ static struct caam_alg_template driver_algs[] = { .ivsize = AES_BLOCK_SIZE, }, .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "seqiv", + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, } }; diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index f227922cea38..acd7743e2603 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3-59-g8ed1b From daebc465858867f48ee86a88f56020c3fe0d96f6 Mon Sep 17 00:00:00 2001 From: Catalin Vasile Date: Fri, 31 Oct 2014 12:45:37 +0200 Subject: crypto: caam - add support for rfc3686 with authenc md5, sha1 and sha2 Add support for AES Counter Mode (CTR) compliant with RFC3686 to be used along with authenc algorithms (md5, sha1, sha224, sha256, sha384, sha512) as one-shot aead algorithms. Signed-off-by: Catalin Vasile Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 262 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 236 insertions(+), 26 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index e9a4fd16031d..34f84d87a4e4 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -60,6 +60,7 @@ #define CAAM_CRA_PRIORITY 3000 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ + CTR_RFC3686_NONCE_SIZE + \ SHA512_DIGEST_SIZE * 2) /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ #define CAAM_MAX_IV_LENGTH 16 @@ -70,6 +71,9 @@ #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +/* Note: Nonce is counted in enckeylen */ +#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ) + #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) @@ -142,11 +146,13 @@ static inline void aead_append_src_dst(u32 *desc, u32 msg_type) /* * For aead encrypt and decrypt, read iv for both classes */ -static inline void aead_append_ld_iv(u32 *desc, int ivsize) +static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset) { - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | ivsize); - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ivoffset << LDST_OFFSET_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | + (ivoffset << MOVE_OFFSET_SHIFT) | ivsize); } /* @@ -192,35 +198,60 @@ struct caam_ctx { }; static void append_key_aead(u32 *desc, struct caam_ctx *ctx, - int keys_fit_inline) + int keys_fit_inline, bool is_rfc3686) { + u32 *nonce; + unsigned int enckeylen = ctx->enckeylen; + + /* + * RFC3686 specific: + * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} + * | enckeylen = encryption key size + nonce size + */ + if (is_rfc3686) + enckeylen -= CTR_RFC3686_NONCE_SIZE; + if (keys_fit_inline) { append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, ctx->split_key_len, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); append_key_as_imm(desc, (void *)ctx->key + - ctx->split_key_pad_len, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + ctx->split_key_pad_len, enckeylen, + enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); } else { append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); append_key(desc, ctx->key_dma + ctx->split_key_pad_len, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + } + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) { + nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + + enckeylen); + append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); } } static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, - int keys_fit_inline) + int keys_fit_inline, bool is_rfc3686) { u32 *key_jump_cmd; - init_sh_desc(desc, HDR_SHARE_SERIAL); + /* Note: Context registers are saved. */ + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); /* Skip if already shared */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_SHRD); - append_key_aead(desc, ctx, keys_fit_inline); + append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); set_jump_tgt_here(desc, key_jump_cmd); } @@ -420,10 +451,17 @@ static int aead_set_sh_desc(struct crypto_aead *aead) { struct aead_tfm *tfm = &aead->base.crt_aead; struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct crypto_tfm *ctfm = crypto_aead_tfm(aead); + const char *alg_name = crypto_tfm_alg_name(ctfm); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline = false; + bool keys_fit_inline; u32 geniv, moveiv; + u32 ctx1_iv_off = 0; u32 *desc; + const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = (ctr_mode && + (strstr(alg_name, "rfc3686") != NULL)); if (!ctx->authsize) return 0; @@ -432,19 +470,37 @@ static int aead_set_sh_desc(struct crypto_aead *aead) if (!ctx->enckeylen) return aead_null_set_sh_desc(aead); + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ + keys_fit_inline = false; if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= + ctx->split_key_pad_len + ctx->enckeylen + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); /* Class 2 operation */ append_operation(desc, ctx->class2_alg_type | @@ -462,7 +518,15 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - aead_append_ld_iv(desc, tfm->ivsize); + aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); /* Class 1 operation */ append_operation(desc, ctx->class1_alg_type | @@ -496,14 +560,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) */ keys_fit_inline = false; if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= + ctx->split_key_pad_len + ctx->enckeylen + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; - init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); /* Class 2 operation */ append_operation(desc, ctx->class2_alg_type | @@ -520,9 +586,22 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - aead_append_ld_iv(desc, tfm->ivsize); + aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); - append_dec_op1(desc, ctx->class1_alg_type); + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Choose operation */ + if (ctr_mode) + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); + else + append_dec_op1(desc, ctx->class1_alg_type); /* Read and write cryptlen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); @@ -552,14 +631,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) */ keys_fit_inline = false; if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= + ctx->split_key_pad_len + ctx->enckeylen + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; /* aead_givencrypt shared descriptor */ desc = ctx->sh_desc_givenc; - init_sh_desc_key_aead(desc, ctx, keys_fit_inline); + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | @@ -568,13 +649,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - append_move(desc, MOVE_SRC_INFIFO | - MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT)); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (tfm->ivsize << MOVE_LEN_SHIFT)); append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); /* Copy IV to class 1 context */ - append_move(desc, MOVE_SRC_CLASS1CTX | - MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (tfm->ivsize << MOVE_LEN_SHIFT)); /* Return to encryption */ append_operation(desc, ctx->class2_alg_type | @@ -590,7 +674,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - /* Copy iv from class 1 ctx to class 2 fifo*/ + /* Copy iv from outfifo to class 2 fifo */ moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | @@ -598,6 +682,14 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + /* Class 1 operation */ append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); @@ -3502,6 +3594,124 @@ static struct caam_alg_template driver_algs[] = { OP_ALG_AAI_HMAC_PRECOMP, .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, + { + .name = "authenc(hmac(md5),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", + .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_givencrypt, + .geniv = "", + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, { .name = "rfc4106(gcm(aes))", .driver_name = "rfc4106-gcm-aes-caam", -- cgit v1.2.3-59-g8ed1b From 7222d1a3410388c8e21a5028ba2beb498938b57f Mon Sep 17 00:00:00 2001 From: Catalin Vasile Date: Fri, 31 Oct 2014 12:45:38 +0200 Subject: crypto: caam - add support for givencrypt cbc(aes) and rfc3686(ctr(aes)) Add support for one-shot givencrypt algorithms. Givencrypt algorithms will generate their IV and encrypt data within the same shared job descriptors. Current algorithms merged from ablkcipher to givencrypt are: - AES Cipher Block Chaining (CBC) - AES Counter Mode (CTR) compliant with RFC3686 Signed-off-by: Catalin Vasile Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 285 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 281 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 34f84d87a4e4..44b306b60fb8 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -1835,6 +1835,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, u32 *key_jump_cmd; u32 *desc; u32 *nonce; + u32 geniv; u32 ctx1_iv_off = 0; const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); @@ -1993,6 +1994,83 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif + /* ablkcipher_givencrypt shared descriptor */ + desc = ctx->sh_desc_givenc; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + /* Load Nonce into CONTEXT1 reg */ + if (is_rfc3686) { + nonce = (u32 *)(key + keylen); + append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + set_jump_tgt_here(desc, key_jump_cmd); + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | + NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_INFIFO | + MOVE_DEST_CLASS1CTX | + (crt->ivsize << MOVE_LEN_SHIFT) | + (ctx1_iv_off << MOVE_OFFSET_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Copy generated IV to memory */ + append_seq_store(desc, crt->ivsize, + LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_u32(desc, (u32)1, LDST_IMM | + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + if (ctx1_iv_off) + append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | + (1 << JUMP_OFFSET_SHIFT)); + + /* Load operation */ + append_operation(desc, ctx->class1_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + + ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif return ret; } @@ -2479,6 +2557,54 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options); } +/* + * Fill in ablkcipher givencrypt job descriptor + */ +static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req, + bool iv_contig) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + u32 *desc = edesc->hw_desc; + u32 out_options, in_options; + dma_addr_t dst_dma, src_dma; + int len, sec4_sg_index = 0; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + ivsize, 1); + print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), + edesc->src_nents ? 100 : req->nbytes, 1); +#endif + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + + if (!edesc->src_nents) { + src_dma = sg_dma_address(req->src); + in_options = 0; + } else { + src_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + in_options = LDST_SGF; + } + append_seq_in_ptr(desc, src_dma, req->nbytes, in_options); + + if (iv_contig) { + dst_dma = edesc->iv_dma; + out_options = 0; + } else { + dst_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + out_options = LDST_SGF; + } + append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options); +} + /* * allocate and map the aead extended descriptor */ @@ -3099,6 +3225,151 @@ static int ablkcipher_decrypt(struct ablkcipher_request *req) return ret; } +/* + * allocate and map the ablkcipher extended descriptor + * for ablkcipher givencrypt + */ +static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( + struct skcipher_givcrypt_request *greq, + int desc_bytes, + bool *iv_contig_out) +{ + struct ablkcipher_request *req = &greq->creq; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, dst_nents = 0, sec4_sg_bytes; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma = 0; + bool iv_contig = false; + int sgc; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + bool src_chained = false, dst_chained = false; + int sec4_sg_index; + + src_nents = sg_count(req->src, req->nbytes, &src_chained); + + if (unlikely(req->dst != req->src)) + dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); + + if (likely(req->src == req->dst)) { + sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL, src_chained); + } else { + sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE, src_chained); + sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE, dst_chained); + } + + /* + * Check if iv can be contiguous with source and destination. + * If so, include it. If not, create scatterlist. + */ + iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, iv_dma)) { + dev_err(jrdev, "unable to map IV\n"); + return ERR_PTR(-ENOMEM); + } + + if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) + iv_contig = true; + else + dst_nents = dst_nents ? : 1; + sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * + sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = kmalloc(sizeof(*edesc) + desc_bytes + + sec4_sg_bytes, GFP_DMA | flags); + if (!edesc) { + dev_err(jrdev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->src_chained = src_chained; + edesc->dst_nents = dst_nents; + edesc->dst_chained = dst_chained; + edesc->sec4_sg_bytes = sec4_sg_bytes; + edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + + desc_bytes; + + sec4_sg_index = 0; + if (src_nents) { + sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); + sec4_sg_index += src_nents; + } + + if (!iv_contig) { + dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, + iv_dma, ivsize, 0); + sec4_sg_index += 1; + sg_to_sec4_sg_last(req->dst, dst_nents, + edesc->sec4_sg + sec4_sg_index, 0); + } + + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return ERR_PTR(-ENOMEM); + } + edesc->iv_dma = iv_dma; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ablkcipher sec4_sg@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + sec4_sg_bytes, 1); +#endif + + *iv_contig_out = iv_contig; + return edesc; +} + +static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) +{ + struct ablkcipher_request *req = &creq->creq; + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + bool iv_contig; + u32 *desc; + int ret = 0; + + /* allocate extended descriptor */ + edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN * + CAAM_CMD_SZ, &iv_contig); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor*/ + init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma, + edesc, req, iv_contig); +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ablkcipher jobdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif + desc = edesc->hw_desc; + ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req); + + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(jrdev, edesc, req); + kfree(edesc); + } + + return ret; +} + #define template_aead template_u.aead #define template_ablkcipher template_u.ablkcipher struct caam_alg_template { @@ -3769,12 +4040,13 @@ static struct caam_alg_template driver_algs[] = { .name = "cbc(aes)", .driver_name = "cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, .template_ablkcipher = { .setkey = ablkcipher_setkey, .encrypt = ablkcipher_encrypt, .decrypt = ablkcipher_decrypt, - .geniv = "eseqiv", + .givencrypt = ablkcipher_givencrypt, + .geniv = "", .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, @@ -3833,12 +4105,13 @@ static struct caam_alg_template driver_algs[] = { .name = "rfc3686(ctr(aes))", .driver_name = "rfc3686-ctr-aes-caam", .blocksize = 1, - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, .template_ablkcipher = { .setkey = ablkcipher_setkey, .encrypt = ablkcipher_encrypt, .decrypt = ablkcipher_decrypt, - .geniv = "seqiv", + .givencrypt = ablkcipher_givencrypt, + .geniv = "", .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, .max_keysize = AES_MAX_KEY_SIZE + @@ -3946,6 +4219,10 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | template->type; switch (template->type) { + case CRYPTO_ALG_TYPE_GIVCIPHER: + alg->cra_type = &crypto_givcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; case CRYPTO_ALG_TYPE_ABLKCIPHER: alg->cra_type = &crypto_ablkcipher_type; alg->cra_ablkcipher = template->template_ablkcipher; -- cgit v1.2.3-59-g8ed1b From ff2c3a3b8e57d088dcf740ab72f4f16eb2132257 Mon Sep 17 00:00:00 2001 From: Catalin Vasile Date: Tue, 11 Nov 2014 16:18:13 +0200 Subject: crypto: caam - add support for givencrypt cbc(des) and cbc(des3_ede) Merge DES Cipher Block Chaining mode (CBC) and Triple DES Cipher Block Chaining mode (CBC) algorithms from ablkcipher to givencrypt. Signed-off-by: Catalin Vasile Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/caam/caamalg.c') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 44b306b60fb8..3187400daf31 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -4057,12 +4057,13 @@ static struct caam_alg_template driver_algs[] = { .name = "cbc(des3_ede)", .driver_name = "cbc-3des-caam", .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, .template_ablkcipher = { .setkey = ablkcipher_setkey, .encrypt = ablkcipher_encrypt, .decrypt = ablkcipher_decrypt, - .geniv = "eseqiv", + .givencrypt = ablkcipher_givencrypt, + .geniv = "", .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .ivsize = DES3_EDE_BLOCK_SIZE, @@ -4073,12 +4074,13 @@ static struct caam_alg_template driver_algs[] = { .name = "cbc(des)", .driver_name = "cbc-des-caam", .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, .template_ablkcipher = { .setkey = ablkcipher_setkey, .encrypt = ablkcipher_encrypt, .decrypt = ablkcipher_decrypt, - .geniv = "eseqiv", + .givencrypt = ablkcipher_givencrypt, + .geniv = "", .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, .ivsize = DES_BLOCK_SIZE, -- cgit v1.2.3-59-g8ed1b