aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/crypto/caam
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/caam')
-rw-r--r--drivers/crypto/caam/Kconfig1
-rw-r--r--drivers/crypto/caam/caamalg.c240
-rw-r--r--drivers/crypto/caam/caamalg_desc.c18
-rw-r--r--drivers/crypto/caam/caamalg_qi.c29
-rw-r--r--drivers/crypto/caam/caamalg_qi2.c85
-rw-r--r--drivers/crypto/caam/caamalg_qi2.h2
-rw-r--r--drivers/crypto/caam/caamhash.c444
-rw-r--r--drivers/crypto/caam/caamhash_desc.c68
-rw-r--r--drivers/crypto/caam/caamhash_desc.h8
-rw-r--r--drivers/crypto/caam/compat.h1
-rw-r--r--drivers/crypto/caam/ctrl.c25
-rw-r--r--drivers/crypto/caam/desc.h1
-rw-r--r--drivers/crypto/caam/error.c6
-rw-r--r--drivers/crypto/caam/error.h9
-rw-r--r--drivers/crypto/caam/intern.h1
-rw-r--r--drivers/crypto/caam/key_gen.c30
-rw-r--r--drivers/crypto/caam/qi.c4
17 files changed, 695 insertions, 277 deletions
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index c4b1cade55c1..577c9844b322 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -91,6 +91,7 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
select CRYPTO_AEAD
select CRYPTO_AUTHENC
select CRYPTO_BLKCIPHER
+ select CRYPTO_DES
help
Selecting this will offload crypto for users of the
scatterlist crypto API (such as the linux native IPSec
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 92e593e2069a..9eac5099098e 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -3,7 +3,7 @@
* caam - Freescale FSL CAAM support for crypto API
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
- * Copyright 2016-2018 NXP
+ * Copyright 2016-2019 NXP
*
* Based on talitos crypto API driver.
*
@@ -766,6 +766,27 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
return 0;
}
+static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
+ const u8 *key, unsigned int keylen)
+{
+ u32 tmp[DES3_EDE_EXPKEY_WORDS];
+ struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
+
+ if (keylen == DES3_EDE_KEY_SIZE &&
+ __des3_ede_setkey(tmp, &tfm->crt_flags, key, DES3_EDE_KEY_SIZE)) {
+ return -EINVAL;
+ }
+
+ if (!des_ekey(tmp, key) && (crypto_skcipher_get_flags(skcipher) &
+ CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
+ crypto_skcipher_set_flags(skcipher,
+ CRYPTO_TFM_RES_WEAK_KEY);
+ return -EINVAL;
+ }
+
+ return skcipher_setkey(skcipher, key, keylen);
+}
+
static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
unsigned int keylen)
{
@@ -802,6 +823,8 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
* aead_edesc - s/w-extended aead descriptor
* @src_nents: number of segments in input s/w scatterlist
* @dst_nents: number of segments in output s/w scatterlist
+ * @mapped_src_nents: number of segments in input h/w link table
+ * @mapped_dst_nents: number of segments in output h/w link table
* @sec4_sg_bytes: length of dma mapped sec4_sg space
* @sec4_sg_dma: bus physical mapped address of h/w link table
* @sec4_sg: pointer to h/w link table
@@ -810,6 +833,8 @@ static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
struct aead_edesc {
int src_nents;
int dst_nents;
+ int mapped_src_nents;
+ int mapped_dst_nents;
int sec4_sg_bytes;
dma_addr_t sec4_sg_dma;
struct sec4_sg_entry *sec4_sg;
@@ -820,6 +845,8 @@ struct aead_edesc {
* skcipher_edesc - s/w-extended skcipher descriptor
* @src_nents: number of segments in input s/w scatterlist
* @dst_nents: number of segments in output s/w scatterlist
+ * @mapped_src_nents: number of segments in input h/w link table
+ * @mapped_dst_nents: number of segments in output h/w link table
* @iv_dma: dma address of iv for checking continuity and link table
* @sec4_sg_bytes: length of dma mapped sec4_sg space
* @sec4_sg_dma: bus physical mapped address of h/w link table
@@ -830,6 +857,8 @@ struct aead_edesc {
struct skcipher_edesc {
int src_nents;
int dst_nents;
+ int mapped_src_nents;
+ int mapped_dst_nents;
dma_addr_t iv_dma;
int sec4_sg_bytes;
dma_addr_t sec4_sg_dma;
@@ -846,7 +875,8 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
if (dst != src) {
if (src_nents)
dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
- dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
+ if (dst_nents)
+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
} else {
dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
}
@@ -961,8 +991,9 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block. This is used e.g. by the CTS mode.
*/
- scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
- ivsize, 0);
+ if (ivsize)
+ scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
+ ivsize, ivsize, 0);
kfree(edesc);
@@ -1023,11 +1054,12 @@ static void init_aead_job(struct aead_request *req,
init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
if (all_contig) {
- src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0;
+ src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
+ 0;
in_options = 0;
} else {
src_dma = edesc->sec4_sg_dma;
- sec4_sg_index += edesc->src_nents;
+ sec4_sg_index += edesc->mapped_src_nents;
in_options = LDST_SGF;
}
@@ -1038,8 +1070,11 @@ static void init_aead_job(struct aead_request *req,
out_options = in_options;
if (unlikely(req->src != req->dst)) {
- if (edesc->dst_nents == 1) {
+ if (!edesc->mapped_dst_nents) {
+ dst_dma = 0;
+ } else if (edesc->mapped_dst_nents == 1) {
dst_dma = sg_dma_address(req->dst);
+ out_options = 0;
} else {
dst_dma = edesc->sec4_sg_dma +
sec4_sg_index *
@@ -1183,9 +1218,9 @@ static void init_skcipher_job(struct skcipher_request *req,
int ivsize = crypto_skcipher_ivsize(skcipher);
u32 *desc = edesc->hw_desc;
u32 *sh_desc;
- u32 out_options = 0;
- dma_addr_t dst_dma, ptr;
- int len;
+ u32 in_options = 0, out_options = 0;
+ dma_addr_t src_dma, dst_dma, ptr;
+ int len, sec4_sg_index = 0;
#ifdef DEBUG
print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
@@ -1203,21 +1238,27 @@ static void init_skcipher_job(struct skcipher_request *req,
len = desc_len(sh_desc);
init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
- append_seq_in_ptr(desc, edesc->sec4_sg_dma, req->cryptlen + ivsize,
- LDST_SGF);
+ if (ivsize || edesc->mapped_src_nents > 1) {
+ src_dma = edesc->sec4_sg_dma;
+ sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
+ in_options = LDST_SGF;
+ } else {
+ src_dma = sg_dma_address(req->src);
+ }
+
+ append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options);
if (likely(req->src == req->dst)) {
- dst_dma = edesc->sec4_sg_dma + sizeof(struct sec4_sg_entry);
- out_options = LDST_SGF;
+ dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
+ out_options = in_options;
+ } else if (edesc->mapped_dst_nents == 1) {
+ dst_dma = sg_dma_address(req->dst);
} else {
- if (edesc->dst_nents == 1) {
- dst_dma = sg_dma_address(req->dst);
- } else {
- dst_dma = edesc->sec4_sg_dma + (edesc->src_nents + 1) *
- sizeof(struct sec4_sg_entry);
- out_options = LDST_SGF;
- }
+ dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
+ sizeof(struct sec4_sg_entry);
+ out_options = LDST_SGF;
}
+
append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
}
@@ -1289,12 +1330,19 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
mapped_src_nents = 0;
}
- mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
- DMA_FROM_DEVICE);
- if (unlikely(!mapped_dst_nents)) {
- dev_err(jrdev, "unable to map destination\n");
- dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
- return ERR_PTR(-ENOMEM);
+ /* Cover also the case of null (zero length) output data */
+ if (dst_nents) {
+ mapped_dst_nents = dma_map_sg(jrdev, req->dst,
+ dst_nents,
+ DMA_FROM_DEVICE);
+ if (unlikely(!mapped_dst_nents)) {
+ dev_err(jrdev, "unable to map destination\n");
+ dma_unmap_sg(jrdev, req->src, src_nents,
+ DMA_TO_DEVICE);
+ return ERR_PTR(-ENOMEM);
+ }
+ } else {
+ mapped_dst_nents = 0;
}
}
@@ -1313,6 +1361,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
edesc->src_nents = src_nents;
edesc->dst_nents = dst_nents;
+ edesc->mapped_src_nents = mapped_src_nents;
+ edesc->mapped_dst_nents = mapped_dst_nents;
edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
desc_bytes;
*all_contig_ptr = !(mapped_src_nents > 1);
@@ -1586,7 +1636,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
GFP_KERNEL : GFP_ATOMIC;
int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
struct skcipher_edesc *edesc;
- dma_addr_t iv_dma;
+ dma_addr_t iv_dma = 0;
u8 *iv;
int ivsize = crypto_skcipher_ivsize(skcipher);
int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
@@ -1621,7 +1671,6 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
dev_err(jrdev, "unable to map source\n");
return ERR_PTR(-ENOMEM);
}
-
mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
DMA_FROM_DEVICE);
if (unlikely(!mapped_dst_nents)) {
@@ -1631,7 +1680,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
}
}
- sec4_sg_ents = 1 + mapped_src_nents;
+ if (!ivsize && mapped_src_nents == 1)
+ sec4_sg_ents = 0; // no need for an input hw s/g table
+ else
+ sec4_sg_ents = mapped_src_nents + !!ivsize;
dst_sg_idx = sec4_sg_ents;
sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
@@ -1650,39 +1702,48 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
edesc->src_nents = src_nents;
edesc->dst_nents = dst_nents;
+ edesc->mapped_src_nents = mapped_src_nents;
+ edesc->mapped_dst_nents = mapped_dst_nents;
edesc->sec4_sg_bytes = sec4_sg_bytes;
edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
desc_bytes);
/* Make sure IV is located in a DMAable area */
- iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
- memcpy(iv, req->iv, ivsize);
+ if (ivsize) {
+ iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
+ memcpy(iv, req->iv, ivsize);
+
+ iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, iv_dma)) {
+ dev_err(jrdev, "unable to map IV\n");
+ caam_unmap(jrdev, req->src, req->dst, src_nents,
+ dst_nents, 0, 0, 0, 0);
+ kfree(edesc);
+ return ERR_PTR(-ENOMEM);
+ }
- iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
- if (dma_mapping_error(jrdev, iv_dma)) {
- dev_err(jrdev, "unable to map IV\n");
- caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
- 0, 0, 0);
- kfree(edesc);
- return ERR_PTR(-ENOMEM);
+ dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
}
-
- dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
- sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg + 1, 0);
+ if (dst_sg_idx)
+ sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg +
+ !!ivsize, 0);
if (mapped_dst_nents > 1) {
sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
edesc->sec4_sg + dst_sg_idx, 0);
}
- edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
- sec4_sg_bytes, DMA_TO_DEVICE);
- if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
- dev_err(jrdev, "unable to map S/G table\n");
- caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
- iv_dma, ivsize, 0, 0);
- kfree(edesc);
- return ERR_PTR(-ENOMEM);
+ if (sec4_sg_bytes) {
+ edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
+ sec4_sg_bytes,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
+ dev_err(jrdev, "unable to map S/G table\n");
+ caam_unmap(jrdev, req->src, req->dst, src_nents,
+ dst_nents, iv_dma, ivsize, 0, 0);
+ kfree(edesc);
+ return ERR_PTR(-ENOMEM);
+ }
}
edesc->iv_dma = iv_dma;
@@ -1749,8 +1810,9 @@ static int skcipher_decrypt(struct skcipher_request *req)
* The crypto API expects us to set the IV (req->iv) to the last
* ciphertext block.
*/
- scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
- ivsize, 0);
+ if (ivsize)
+ scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
+ ivsize, ivsize, 0);
/* Create and submit job descriptor*/
init_skcipher_job(req, edesc, false);
@@ -1796,7 +1858,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-3des-caam",
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = des_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES3_EDE_KEY_SIZE,
@@ -1812,7 +1874,7 @@ static struct caam_skcipher_alg driver_algs[] = {
.cra_driver_name = "cbc-des-caam",
.cra_blocksize = DES_BLOCK_SIZE,
},
- .setkey = skcipher_setkey,
+ .setkey = des_skcipher_setkey,
.encrypt = skcipher_encrypt,
.decrypt = skcipher_decrypt,
.min_keysize = DES_KEY_SIZE,
@@ -1878,6 +1940,66 @@ static struct caam_skcipher_alg driver_algs[] = {
},
.caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
},
+ {
+ .skcipher = {
+ .base = {
+ .cra_name = "ecb(des)",
+ .cra_driver_name = "ecb-des-caam",
+ .cra_blocksize = DES_BLOCK_SIZE,
+ },
+ .setkey = des_skcipher_setkey,
+ .encrypt = skcipher_encrypt,
+ .decrypt = skcipher_decrypt,
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ },
+ .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
+ },
+ {
+ .skcipher = {
+ .base = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-caam",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ },
+ .setkey = skcipher_setkey,
+ .encrypt = skcipher_encrypt,
+ .decrypt = skcipher_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ },
+ .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
+ },
+ {
+ .skcipher = {
+ .base = {
+ .cra_name = "ecb(des3_ede)",
+ .cra_driver_name = "ecb-des3-caam",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ },
+ .setkey = des_skcipher_setkey,
+ .encrypt = skcipher_encrypt,
+ .decrypt = skcipher_decrypt,
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ },
+ .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
+ },
+ {
+ .skcipher = {
+ .base = {
+ .cra_name = "ecb(arc4)",
+ .cra_driver_name = "ecb-arc4-caam",
+ .cra_blocksize = ARC4_BLOCK_SIZE,
+ },
+ .setkey = skcipher_setkey,
+ .encrypt = skcipher_encrypt,
+ .decrypt = skcipher_decrypt,
+ .min_keysize = ARC4_MIN_KEY_SIZE,
+ .max_keysize = ARC4_MAX_KEY_SIZE,
+ },
+ .caam.class1_alg_type = OP_ALG_ALGSEL_ARC4 | OP_ALG_AAI_ECB,
+ },
};
static struct caam_aead_alg driver_aeads[] = {
@@ -3337,6 +3459,7 @@ static int __init caam_algapi_init(void)
struct caam_drv_private *priv;
int i = 0, err = 0;
u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
+ u32 arc4_inst;
unsigned int md_limit = SHA512_DIGEST_SIZE;
bool registered = false;
@@ -3381,6 +3504,8 @@ static int __init caam_algapi_init(void)
CHA_ID_LS_DES_SHIFT;
aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
+ arc4_inst = (cha_inst & CHA_ID_LS_ARC4_MASK) >>
+ CHA_ID_LS_ARC4_SHIFT;
ccha_inst = 0;
ptha_inst = 0;
} else {
@@ -3397,6 +3522,7 @@ static int __init caam_algapi_init(void)
md_inst = mdha & CHA_VER_NUM_MASK;
ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
+ arc4_inst = rd_reg32(&priv->ctrl->vreg.afha) & CHA_VER_NUM_MASK;
}
/* If MD is present, limit digest size based on LP256 */
@@ -3417,6 +3543,10 @@ static int __init caam_algapi_init(void)
if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
continue;
+ /* Skip ARC4 algorithms if not supported by device */
+ if (!arc4_inst && alg_sel == OP_ALG_ALGSEL_ARC4)
+ continue;
+
/*
* Check support for AES modes not available
* on LP devices.
@@ -3476,7 +3606,7 @@ static int __init caam_algapi_init(void)
* Skip algorithms requiring message digests
* if MD or MD size is not supported by device.
*/
- if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 &&
+ if (is_mdha(c2_alg_sel) &&
(!md_inst || t_alg->aead.maxauthsize > md_limit))
continue;
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 7db1640d3577..1e1a376edc2f 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -2,7 +2,7 @@
/*
* Shared descriptors for aead, skcipher algorithms
*
- * Copyright 2016-2018 NXP
+ * Copyright 2016-2019 NXP
*/
#include "compat.h"
@@ -1396,9 +1396,11 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
set_jump_tgt_here(desc, key_jump_cmd);
- /* Load iv */
- append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
- LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
+ /* Load IV, if there is one */
+ if (ivsize)
+ append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | (ctx1_iv_off <<
+ LDST_OFFSET_SHIFT));
/* Load counter into CONTEXT1 reg */
if (is_rfc3686)
@@ -1462,9 +1464,11 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
set_jump_tgt_here(desc, key_jump_cmd);
- /* load IV */
- append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
- LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
+ /* Load IV, if there is one */
+ if (ivsize)
+ append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | (ctx1_iv_off <<
+ LDST_OFFSET_SHIFT));
/* Load counter into CONTEXT1 reg */
if (is_rfc3686)
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index c0d55310aade..a15ce9213310 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -782,7 +782,7 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
cpu = smp_processor_id();
drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
- if (likely(!IS_ERR_OR_NULL(drv_ctx)))
+ if (!IS_ERR_OR_NULL(drv_ctx))
drv_ctx->op_type = type;
ctx->drv_ctx[type] = drv_ctx;
@@ -802,7 +802,8 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
if (dst != src) {
if (src_nents)
dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
- dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
+ if (dst_nents)
+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
} else {
dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
}
@@ -892,7 +893,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
struct caam_drv_ctx *drv_ctx;
drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
- if (unlikely(IS_ERR_OR_NULL(drv_ctx)))
+ if (IS_ERR_OR_NULL(drv_ctx))
return (struct aead_edesc *)drv_ctx;
/* allocate space for base edesc and hw desc commands, link tables */
@@ -955,13 +956,19 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
mapped_src_nents = 0;
}
- mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
- DMA_FROM_DEVICE);
- if (unlikely(!mapped_dst_nents)) {
- dev_err(qidev, "unable to map destination\n");
- dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
- qi_cache_free(edesc);
- return ERR_PTR(-ENOMEM);
+ if (dst_nents) {
+ mapped_dst_nents = dma_map_sg(qidev, req->dst,
+ dst_nents,
+ DMA_FROM_DEVICE);
+ if (unlikely(!mapped_dst_nents)) {
+ dev_err(qidev, "unable to map destination\n");
+ dma_unmap_sg(qidev, req->src, src_nents,
+ DMA_TO_DEVICE);
+ qi_cache_free(edesc);
+ return ERR_PTR(-ENOMEM);
+ }
+ } else {
+ mapped_dst_nents = 0;
}
}
@@ -1184,7 +1191,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
struct caam_drv_ctx *drv_ctx;
drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
- if (unlikely(IS_ERR_OR_NULL(drv_ctx)))
+ if (IS_ERR_OR_NULL(drv_ctx))
return (struct skcipher_edesc *)drv_ctx;
src_nents = sg_nents_for_len(req->src, req->cryptlen);
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index 425d5d974613..c2c1abc68f81 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -25,13 +25,6 @@
#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \
SHA512_DIGEST_SIZE * 2)
-#if !IS_ENABLED(CONFIG_CRYPTO_DEV_FSL_CAAM)
-bool caam_little_end;
-EXPORT_SYMBOL(caam_little_end);
-bool caam_imx;
-EXPORT_SYMBOL(caam_imx);
-#endif
-
/*
* This is a a cache of buffers, from which the users of CAAM QI driver
* can allocate short buffers. It's speedier than doing kmalloc on the hotpath.
@@ -151,7 +144,8 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
if (dst != src) {
if (src_nents)
dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
- dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
+ if (dst_nents)
+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
} else {
dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
}
@@ -392,13 +386,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
mapped_src_nents = 0;
}
- mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
- DMA_FROM_DEVICE);
- if (unlikely(!mapped_dst_nents)) {
- dev_err(dev, "unable to map destination\n");
- dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
- qi_cache_free(edesc);
- return ERR_PTR(-ENOMEM);
+ if (dst_nents) {
+ mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents,
+ DMA_FROM_DEVICE);
+ if (unlikely(!mapped_dst_nents)) {
+ dev_err(dev, "unable to map destination\n");
+ dma_unmap_sg(dev, req->src, src_nents,
+ DMA_TO_DEVICE);
+ qi_cache_free(edesc);
+ return ERR_PTR(-ENOMEM);
+ }
+ } else {
+ mapped_dst_nents = 0;
}
} else {
src_nents = sg_nents_for_len(req->src, req->assoclen +
@@ -4503,7 +4502,8 @@ static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv)
nctx->cb = dpaa2_caam_fqdan_cb;
/* Register notification callbacks */
- err = dpaa2_io_service_register(NULL, nctx);
+ ppriv->dpio = dpaa2_io_service_select(cpu);
+ err = dpaa2_io_service_register(ppriv->dpio, nctx, dev);
if (unlikely(err)) {
dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu);
nctx->cb = NULL;
@@ -4536,7 +4536,7 @@ err:
ppriv = per_cpu_ptr(priv->ppriv, cpu);
if (!ppriv->nctx.cb)
break;
- dpaa2_io_service_deregister(NULL, &ppriv->nctx);
+ dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev);
}
for_each_online_cpu(cpu) {
@@ -4556,7 +4556,8 @@ static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv)
for_each_online_cpu(cpu) {
ppriv = per_cpu_ptr(priv->ppriv, cpu);
- dpaa2_io_service_deregister(NULL, &ppriv->nctx);
+ dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx,
+ priv->dev);
dpaa2_io_store_destroy(ppriv->store);
if (++i == priv->num_pairs)
@@ -4654,7 +4655,7 @@ static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv)
/* Retry while portal is busy */
do {
- err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid,
+ err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid,
ppriv->store);
} while (err == -EBUSY);
@@ -4722,7 +4723,7 @@ static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget)
if (cleaned < budget) {
napi_complete_done(napi, cleaned);
- err = dpaa2_io_service_rearm(NULL, &ppriv->nctx);
+ err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx);
if (unlikely(err))
dev_err(priv->dev, "Notification rearm failed: %d\n",
err);
@@ -4863,21 +4864,31 @@ static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev)
i = 0;
for_each_online_cpu(cpu) {
- dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", i,
- priv->rx_queue_attr[i].fqid,
- priv->tx_queue_attr[i].fqid);
+ u8 j;
+
+ j = i % priv->num_pairs;
ppriv = per_cpu_ptr(priv->ppriv, cpu);
- ppriv->req_fqid = priv->tx_queue_attr[i].fqid;
- ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid;
- ppriv->prio = i;
+ ppriv->req_fqid = priv->tx_queue_attr[j].fqid;
+
+ /*
+ * Allow all cores to enqueue, while only some of them
+ * will take part in dequeuing.
+ */
+ if (++i > priv->num_pairs)
+ continue;
+
+ ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid;
+ ppriv->prio = j;
+
+ dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j,
+ priv->rx_queue_attr[j].fqid,
+ priv->tx_queue_attr[j].fqid);
ppriv->net_dev.dev = *dev;
INIT_LIST_HEAD(&ppriv->net_dev.napi_list);
netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll,
DPAA2_CAAM_NAPI_WEIGHT);
- if (++i == priv->num_pairs)
- break;
}
return 0;
@@ -5229,7 +5240,8 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
{
struct dpaa2_fd fd;
struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
- int err = 0, i, id;
+ struct dpaa2_caam_priv_per_cpu *ppriv;
+ int err = 0, i;
if (IS_ERR(req))
return PTR_ERR(req);
@@ -5259,23 +5271,18 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
dpaa2_fd_set_flc(&fd, req->flc_dma);
- /*
- * There is no guarantee that preemption is disabled here,
- * thus take action.
- */
- preempt_disable();
- id = smp_processor_id() % priv->dpseci_attr.num_tx_queues;
+ ppriv = this_cpu_ptr(priv->ppriv);
for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
- err = dpaa2_io_service_enqueue_fq(NULL,
- priv->tx_queue_attr[id].fqid,
+ err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
&fd);
if (err != -EBUSY)
break;
+
+ cpu_relax();
}
- preempt_enable();
if (unlikely(err)) {
- dev_err(dev, "Error enqueuing frame: %d\n", err);
+ dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err);
goto err_out;
}
diff --git a/drivers/crypto/caam/caamalg_qi2.h b/drivers/crypto/caam/caamalg_qi2.h
index 9823bdefd029..20890780fb82 100644
--- a/drivers/crypto/caam/caamalg_qi2.h
+++ b/drivers/crypto/caam/caamalg_qi2.h
@@ -76,6 +76,7 @@ struct dpaa2_caam_priv {
* @nctx: notification context of response FQ
* @store: where dequeued frames are stored
* @priv: backpointer to dpaa2_caam_priv
+ * @dpio: portal used for data path operations
*/
struct dpaa2_caam_priv_per_cpu {
struct napi_struct napi;
@@ -86,6 +87,7 @@ struct dpaa2_caam_priv_per_cpu {
struct dpaa2_io_notification_ctx nctx;
struct dpaa2_io_store *store;
struct dpaa2_caam_priv *priv;
+ struct dpaa2_io *dpio;
};
/*
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 81712aa5d0f2..d7483e4d0ce2 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -3,7 +3,7 @@
* caam - Freescale FSL CAAM support for ahash functions of crypto API
*
* Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
*
* Based on caamalg.c crypto API driver.
*
@@ -98,13 +98,14 @@ struct caam_hash_ctx {
u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
+ u8 key[CAAM_MAX_HASH_KEY_SIZE] ____cacheline_aligned;
dma_addr_t sh_desc_update_dma ____cacheline_aligned;
dma_addr_t sh_desc_update_first_dma;
dma_addr_t sh_desc_fin_dma;
dma_addr_t sh_desc_digest_dma;
+ dma_addr_t key_dma;
enum dma_data_direction dir;
struct device *jrdev;
- u8 key[CAAM_MAX_HASH_KEY_SIZE];
int ctx_len;
struct alginfo adata;
};
@@ -113,6 +114,7 @@ struct caam_hash_ctx {
struct caam_hash_state {
dma_addr_t buf_dma;
dma_addr_t ctx_dma;
+ int ctx_dma_len;
u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
int buflen_0;
u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
@@ -158,6 +160,11 @@ static inline int *alt_buflen(struct caam_hash_state *state)
return state->current_buf ? &state->buflen_0 : &state->buflen_1;
}
+static inline bool is_cmac_aes(u32 algtype)
+{
+ return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) ==
+ (OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC);
+}
/* Common job descriptor seq in/out ptr routines */
/* Map state->caam_ctx, and append seq_out_ptr command that points to it */
@@ -165,6 +172,7 @@ static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
struct caam_hash_state *state,
int ctx_len)
{
+ state->ctx_dma_len = ctx_len;
state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
ctx_len, DMA_FROM_DEVICE);
if (dma_mapping_error(jrdev, state->ctx_dma)) {
@@ -178,18 +186,6 @@ static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
return 0;
}
-/* Map req->result, and append seq_out_ptr command that points to it */
-static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
- u8 *result, int digestsize)
-{
- dma_addr_t dst_dma;
-
- dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
- append_seq_out_ptr(desc, dst_dma, digestsize, 0);
-
- return dst_dma;
-}
-
/* Map current buffer in state (if length > 0) and put it in link table */
static inline int buf_map_to_sec4_sg(struct device *jrdev,
struct sec4_sg_entry *sec4_sg,
@@ -218,6 +214,7 @@ static inline int ctx_map_to_sec4_sg(struct device *jrdev,
struct caam_hash_state *state, int ctx_len,
struct sec4_sg_entry *sec4_sg, u32 flag)
{
+ state->ctx_dma_len = ctx_len;
state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
if (dma_mapping_error(jrdev, state->ctx_dma)) {
dev_err(jrdev, "unable to map ctx\n");
@@ -292,14 +289,119 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
return 0;
}
+static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
+{
+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ int digestsize = crypto_ahash_digestsize(ahash);
+ struct device *jrdev = ctx->jrdev;
+ u32 *desc;
+
+ /* key is loaded from memory for UPDATE and FINALIZE states */
+ ctx->adata.key_dma = ctx->key_dma;
+
+ /* shared descriptor for ahash_update */
+ desc = ctx->sh_desc_update;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
+ ctx->ctx_len, ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
+
+ /* shared descriptor for ahash_{final,finup} */
+ desc = ctx->sh_desc_fin;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
+ digestsize, ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
+
+ /* key is immediate data for INIT and INITFINAL states */
+ ctx->adata.key_virt = ctx->key;
+
+ /* shared descriptor for first invocation of ahash_update */
+ desc = ctx->sh_desc_update_first;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
+ ctx->ctx_len, ctx->key_dma);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
+
+ /* shared descriptor for ahash_digest */
+ desc = ctx->sh_desc_digest;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
+ digestsize, ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+ 1);
+ return 0;
+}
+
+static int acmac_set_sh_desc(struct crypto_ahash *ahash)
+{
+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ int digestsize = crypto_ahash_digestsize(ahash);
+ struct device *jrdev = ctx->jrdev;
+ u32 *desc;
+
+ /* shared descriptor for ahash_update */
+ desc = ctx->sh_desc_update;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
+ ctx->ctx_len, ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+
+ /* shared descriptor for ahash_{final,finup} */
+ desc = ctx->sh_desc_fin;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
+ digestsize, ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+
+ /* shared descriptor for first invocation of ahash_update */
+ desc = ctx->sh_desc_update_first;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
+ ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+
+ /* shared descriptor for ahash_digest */
+ desc = ctx->sh_desc_digest;
+ cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
+ digestsize, ctx->ctx_len, 0);
+ dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
+ desc_bytes(desc), ctx->dir);
+ print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+
+ return 0;
+}
+
/* Digest hash size if it is too large */
-static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
- u32 *keylen, u8 *key_out, u32 digestsize)
+static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
+ u32 digestsize)
{
struct device *jrdev = ctx->jrdev;
u32 *desc;
struct split_key_result result;
- dma_addr_t src_dma, dst_dma;
+ dma_addr_t key_dma;
int ret;
desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
@@ -310,18 +412,9 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
init_job_desc(desc, 0);
- src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
- DMA_TO_DEVICE);
- if (dma_mapping_error(jrdev, src_dma)) {
- dev_err(jrdev, "unable to map key input memory\n");
- kfree(desc);
- return -ENOMEM;
- }
- dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
- DMA_FROM_DEVICE);
- if (dma_mapping_error(jrdev, dst_dma)) {
- dev_err(jrdev, "unable to map key output memory\n");
- dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
+ key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(jrdev, key_dma)) {
+ dev_err(jrdev, "unable to map key memory\n");
kfree(desc);
return -ENOMEM;
}
@@ -329,16 +422,16 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
/* Job descriptor to perform unkeyed hash on key_in */
append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
OP_ALG_AS_INITFINAL);
- append_seq_in_ptr(desc, src_dma, *keylen, 0);
+ append_seq_in_ptr(desc, key_dma, *keylen, 0);
append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
- append_seq_out_ptr(desc, dst_dma, digestsize, 0);
+ append_seq_out_ptr(desc, key_dma, digestsize, 0);
append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
LDST_SRCDST_BYTE_CONTEXT);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
+ DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -354,12 +447,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
#ifdef DEBUG
print_hex_dump(KERN_ERR,
"digested key@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key_in,
- digestsize, 1);
+ DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1);
#endif
}
- dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
- dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
+ dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
*keylen = digestsize;
@@ -383,13 +474,10 @@ static int ahash_setkey(struct crypto_ahash *ahash,
#endif
if (keylen > blocksize) {
- hashed_key = kmalloc_array(digestsize,
- sizeof(*hashed_key),
- GFP_KERNEL | GFP_DMA);
+ hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!hashed_key)
return -ENOMEM;
- ret = hash_digest_key(ctx, key, &keylen, hashed_key,
- digestsize);
+ ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
if (ret)
goto bad_free_key;
key = hashed_key;
@@ -424,9 +512,39 @@ static int ahash_setkey(struct crypto_ahash *ahash,
return -EINVAL;
}
+static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
+ unsigned int keylen)
+{
+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct device *jrdev = ctx->jrdev;
+
+ memcpy(ctx->key, key, keylen);
+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
+ ctx->adata.keylen = keylen;
+
+ print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1);
+
+ return axcbc_set_sh_desc(ahash);
+}
+
+static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
+ unsigned int keylen)
+{
+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+
+ /* key is immediate data for all cmac shared descriptors */
+ ctx->adata.key_virt = key;
+ ctx->adata.keylen = keylen;
+
+ print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+
+ return acmac_set_sh_desc(ahash);
+}
+
/*
* ahash_edesc - s/w-extended ahash descriptor
- * @dst_dma: physical mapped address of req->result
* @sec4_sg_dma: physical mapped address of h/w link table
* @src_nents: number of segments in input scatterlist
* @sec4_sg_bytes: length of dma mapped sec4_sg space
@@ -434,7 +552,6 @@ static int ahash_setkey(struct crypto_ahash *ahash,
* @sec4_sg: h/w link table
*/
struct ahash_edesc {
- dma_addr_t dst_dma;
dma_addr_t sec4_sg_dma;
int src_nents;
int sec4_sg_bytes;
@@ -450,8 +567,6 @@ static inline void ahash_unmap(struct device *dev,
if (edesc->src_nents)
dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
- if (edesc->dst_dma)
- dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
if (edesc->sec4_sg_bytes)
dma_unmap_single(dev, edesc->sec4_sg_dma,
@@ -468,12 +583,10 @@ static inline void ahash_unmap_ctx(struct device *dev,
struct ahash_edesc *edesc,
struct ahash_request *req, int dst_len, u32 flag)
{
- struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
- struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
struct caam_hash_state *state = ahash_request_ctx(req);
if (state->ctx_dma) {
- dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
+ dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
state->ctx_dma = 0;
}
ahash_unmap(dev, edesc, req, dst_len);
@@ -486,9 +599,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
struct ahash_edesc *edesc;
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
int digestsize = crypto_ahash_digestsize(ahash);
+ struct caam_hash_state *state = ahash_request_ctx(req);
#ifdef DEBUG
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
- struct caam_hash_state *state = ahash_request_ctx(req);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
@@ -497,17 +610,14 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
if (err)
caam_jr_strstatus(jrdev, err);
- ahash_unmap(jrdev, edesc, req, digestsize);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
+ memcpy(req->result, state->caam_ctx, digestsize);
kfree(edesc);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
- if (req->result)
- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->result,
- digestsize, 1);
#endif
req->base.complete(&req->base, err);
@@ -555,9 +665,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
struct ahash_edesc *edesc;
struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
int digestsize = crypto_ahash_digestsize(ahash);
+ struct caam_hash_state *state = ahash_request_ctx(req);
#ifdef DEBUG
struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
- struct caam_hash_state *state = ahash_request_ctx(req);
dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
#endif
@@ -566,17 +676,14 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
if (err)
caam_jr_strstatus(jrdev, err);
- ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
+ memcpy(req->result, state->caam_ctx, digestsize);
kfree(edesc);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
ctx->ctx_len, 1);
- if (req->result)
- print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, req->result,
- digestsize, 1);
#endif
req->base.complete(&req->base, err);
@@ -688,6 +795,7 @@ static int ahash_update_ctx(struct ahash_request *req)
u8 *buf = current_buf(state);
int *buflen = current_buflen(state);
u8 *next_buf = alt_buf(state);
+ int blocksize = crypto_ahash_blocksize(ahash);
int *next_buflen = alt_buflen(state), last_buflen;
int in_len = *buflen + req->nbytes, to_hash;
u32 *desc;
@@ -696,9 +804,20 @@ static int ahash_update_ctx(struct ahash_request *req)
int ret = 0;
last_buflen = *next_buflen;
- *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
+ *next_buflen = in_len & (blocksize - 1);
to_hash = in_len - *next_buflen;
+ /*
+ * For XCBC and CMAC, if to_hash is multiple of block size,
+ * keep last block in internal buffer
+ */
+ if ((is_xcbc_aes(ctx->adata.algtype) ||
+ is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
+ (*next_buflen == 0)) {
+ *next_buflen = blocksize;
+ to_hash -= blocksize;
+ }
+
if (to_hash) {
src_nents = sg_nents_for_len(req->src,
req->nbytes - (*next_buflen));
@@ -801,7 +920,7 @@ static int ahash_update_ctx(struct ahash_request *req)
#endif
return ret;
- unmap_ctx:
+unmap_ctx:
ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
kfree(edesc);
return ret;
@@ -837,7 +956,7 @@ static int ahash_final_ctx(struct ahash_request *req)
edesc->sec4_sg_bytes = sec4_sg_bytes;
ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
- edesc->sec4_sg, DMA_TO_DEVICE);
+ edesc->sec4_sg, DMA_BIDIRECTIONAL);
if (ret)
goto unmap_ctx;
@@ -857,14 +976,7 @@ static int ahash_final_ctx(struct ahash_request *req)
append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
LDST_SGF);
-
- edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
- digestsize);
- if (dma_mapping_error(jrdev, edesc->dst_dma)) {
- dev_err(jrdev, "unable to map dst\n");
- ret = -ENOMEM;
- goto unmap_ctx;
- }
+ append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
@@ -877,7 +989,7 @@ static int ahash_final_ctx(struct ahash_request *req)
return -EINPROGRESS;
unmap_ctx:
- ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
kfree(edesc);
return ret;
}
@@ -931,7 +1043,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
edesc->src_nents = src_nents;
ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
- edesc->sec4_sg, DMA_TO_DEVICE);
+ edesc->sec4_sg, DMA_BIDIRECTIONAL);
if (ret)
goto unmap_ctx;
@@ -945,13 +1057,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
if (ret)
goto unmap_ctx;
- edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
- digestsize);
- if (dma_mapping_error(jrdev, edesc->dst_dma)) {
- dev_err(jrdev, "unable to map dst\n");
- ret = -ENOMEM;
- goto unmap_ctx;
- }
+ append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
@@ -964,7 +1070,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
return -EINPROGRESS;
unmap_ctx:
- ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
kfree(edesc);
return ret;
}
@@ -1023,10 +1129,8 @@ static int ahash_digest(struct ahash_request *req)
desc = edesc->hw_desc;
- edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
- digestsize);
- if (dma_mapping_error(jrdev, edesc->dst_dma)) {
- dev_err(jrdev, "unable to map dst\n");
+ ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
+ if (ret) {
ahash_unmap(jrdev, edesc, req, digestsize);
kfree(edesc);
return -ENOMEM;
@@ -1041,7 +1145,7 @@ static int ahash_digest(struct ahash_request *req)
if (!ret) {
ret = -EINPROGRESS;
} else {
- ahash_unmap(jrdev, edesc, req, digestsize);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
kfree(edesc);
}
@@ -1072,20 +1176,20 @@ static int ahash_final_no_ctx(struct ahash_request *req)
desc = edesc->hw_desc;
- state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
- if (dma_mapping_error(jrdev, state->buf_dma)) {
- dev_err(jrdev, "unable to map src\n");
- goto unmap;
- }
+ if (buflen) {
+ state->buf_dma = dma_map_single(jrdev, buf, buflen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, state->buf_dma)) {
+ dev_err(jrdev, "unable to map src\n");
+ goto unmap;
+ }
- append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
+ append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
+ }
- edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
- digestsize);
- if (dma_mapping_error(jrdev, edesc->dst_dma)) {
- dev_err(jrdev, "unable to map dst\n");
+ ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
+ if (ret)
goto unmap;
- }
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
@@ -1096,7 +1200,7 @@ static int ahash_final_no_ctx(struct ahash_request *req)
if (!ret) {
ret = -EINPROGRESS;
} else {
- ahash_unmap(jrdev, edesc, req, digestsize);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
kfree(edesc);
}
@@ -1119,6 +1223,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
GFP_KERNEL : GFP_ATOMIC;
u8 *buf = current_buf(state);
int *buflen = current_buflen(state);
+ int blocksize = crypto_ahash_blocksize(ahash);
u8 *next_buf = alt_buf(state);
int *next_buflen = alt_buflen(state);
int in_len = *buflen + req->nbytes, to_hash;
@@ -1127,9 +1232,20 @@ static int ahash_update_no_ctx(struct ahash_request *req)
u32 *desc;
int ret = 0;
- *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
+ *next_buflen = in_len & (blocksize - 1);
to_hash = in_len - *next_buflen;
+ /*
+ * For XCBC and CMAC, if to_hash is multiple of block size,
+ * keep last block in internal buffer
+ */
+ if ((is_xcbc_aes(ctx->adata.algtype) ||
+ is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
+ (*next_buflen == 0)) {
+ *next_buflen = blocksize;
+ to_hash -= blocksize;
+ }
+
if (to_hash) {
src_nents = sg_nents_for_len(req->src,
req->nbytes - *next_buflen);
@@ -1295,12 +1411,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
goto unmap;
}
- edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
- digestsize);
- if (dma_mapping_error(jrdev, edesc->dst_dma)) {
- dev_err(jrdev, "unable to map dst\n");
+ ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
+ if (ret)
goto unmap;
- }
#ifdef DEBUG
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
@@ -1311,7 +1424,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
if (!ret) {
ret = -EINPROGRESS;
} else {
- ahash_unmap(jrdev, edesc, req, digestsize);
+ ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
kfree(edesc);
}
@@ -1335,15 +1448,26 @@ static int ahash_update_first(struct ahash_request *req)
u8 *next_buf = alt_buf(state);
int *next_buflen = alt_buflen(state);
int to_hash;
+ int blocksize = crypto_ahash_blocksize(ahash);
u32 *desc;
int src_nents, mapped_nents;
struct ahash_edesc *edesc;
int ret = 0;
- *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
- 1);
+ *next_buflen = req->nbytes & (blocksize - 1);
to_hash = req->nbytes - *next_buflen;
+ /*
+ * For XCBC and CMAC, if to_hash is multiple of block size,
+ * keep last block in internal buffer
+ */
+ if ((is_xcbc_aes(ctx->adata.algtype) ||
+ is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
+ (*next_buflen == 0)) {
+ *next_buflen = blocksize;
+ to_hash -= blocksize;
+ }
+
if (to_hash) {
src_nents = sg_nents_for_len(req->src,
req->nbytes - *next_buflen);
@@ -1443,6 +1567,7 @@ static int ahash_init(struct ahash_request *req)
state->final = ahash_final_no_ctx;
state->ctx_dma = 0;
+ state->ctx_dma_len = 0;
state->current_buf = 0;
state->buf_dma = 0;
state->buflen_0 = 0;
@@ -1651,6 +1776,44 @@ static struct caam_hash_template driver_hash[] = {
},
},
.alg_type = OP_ALG_ALGSEL_MD5,
+ }, {
+ .hmac_name = "xcbc(aes)",
+ .hmac_driver_name = "xcbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .template_ahash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .export = ahash_export,
+ .import = ahash_import,
+ .setkey = axcbc_setkey,
+ .halg = {
+ .digestsize = AES_BLOCK_SIZE,
+ .statesize = sizeof(struct caam_export_state),
+ },
+ },
+ .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC,
+ }, {
+ .hmac_name = "cmac(aes)",
+ .hmac_driver_name = "cmac-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .template_ahash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .export = ahash_export,
+ .import = ahash_import,
+ .setkey = acmac_setkey,
+ .halg = {
+ .digestsize = AES_BLOCK_SIZE,
+ .statesize = sizeof(struct caam_export_state),
+ },
+ },
+ .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC,
},
};
@@ -1692,14 +1855,45 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
}
priv = dev_get_drvdata(ctx->jrdev->parent);
- ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+
+ if (is_xcbc_aes(caam_hash->alg_type)) {
+ ctx->dir = DMA_TO_DEVICE;
+ ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
+ ctx->ctx_len = 48;
+
+ ctx->key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
+ ARRAY_SIZE(ctx->key),
+ DMA_BIDIRECTIONAL,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
+ dev_err(ctx->jrdev, "unable to map key\n");
+ caam_jr_free(ctx->jrdev);
+ return -ENOMEM;
+ }
+ } else if (is_cmac_aes(caam_hash->alg_type)) {
+ ctx->dir = DMA_TO_DEVICE;
+ ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
+ ctx->ctx_len = 32;
+ } else {
+ ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
+ ctx->ctx_len = runninglen[(ctx->adata.algtype &
+ OP_ALG_ALGSEL_SUBMASK) >>
+ OP_ALG_ALGSEL_SHIFT];
+ }
dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
- offsetof(struct caam_hash_ctx,
- sh_desc_update_dma),
+ offsetof(struct caam_hash_ctx, key),
ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
if (dma_mapping_error(ctx->jrdev, dma_addr)) {
dev_err(ctx->jrdev, "unable to map shared descriptors\n");
+
+ if (is_xcbc_aes(caam_hash->alg_type))
+ dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma,
+ ARRAY_SIZE(ctx->key),
+ DMA_BIDIRECTIONAL,
+ DMA_ATTR_SKIP_CPU_SYNC);
+
caam_jr_free(ctx->jrdev);
return -ENOMEM;
}
@@ -1713,16 +1907,14 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
sh_desc_digest);
- /* copy descriptor header template value */
- ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
-
- ctx->ctx_len = runninglen[(ctx->adata.algtype &
- OP_ALG_ALGSEL_SUBMASK) >>
- OP_ALG_ALGSEL_SHIFT];
-
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct caam_hash_state));
- return ahash_set_sh_desc(ahash);
+
+ /*
+ * For keyed hash algorithms shared descriptors
+ * will be created later in setkey() callback
+ */
+ return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
}
static void caam_hash_cra_exit(struct crypto_tfm *tfm)
@@ -1730,9 +1922,12 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
- offsetof(struct caam_hash_ctx,
- sh_desc_update_dma),
+ offsetof(struct caam_hash_ctx, key),
ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
+ if (is_xcbc_aes(ctx->adata.algtype))
+ dma_unmap_single_attrs(ctx->jrdev, ctx->key_dma,
+ ARRAY_SIZE(ctx->key), DMA_BIDIRECTIONAL,
+ DMA_ATTR_SKIP_CPU_SYNC);
caam_jr_free(ctx->jrdev);
}
@@ -1863,14 +2058,16 @@ static int __init caam_algapi_hash_init(void)
struct caam_hash_template *alg = driver_hash + i;
/* If MD size is not supported by device, skip registration */
- if (alg->template_ahash.halg.digestsize > md_limit)
+ if (is_mdha(alg->alg_type) &&
+ alg->template_ahash.halg.digestsize > md_limit)
continue;
/* register hmac version */
t_alg = caam_hash_alloc(alg, true);
if (IS_ERR(t_alg)) {
err = PTR_ERR(t_alg);
- pr_warn("%s alg allocation failed\n", alg->driver_name);
+ pr_warn("%s alg allocation failed\n",
+ alg->hmac_driver_name);
continue;
}
@@ -1883,6 +2080,9 @@ static int __init caam_algapi_hash_init(void)
} else
list_add_tail(&t_alg->entry, &hash_list);
+ if ((alg->alg_type & OP_ALG_ALGSEL_MASK) == OP_ALG_ALGSEL_AES)
+ continue;
+
/* register unkeyed version */
t_alg = caam_hash_alloc(alg, false);
if (IS_ERR(t_alg)) {
diff --git a/drivers/crypto/caam/caamhash_desc.c b/drivers/crypto/caam/caamhash_desc.c
index a12f7959a2c3..71d018343ee4 100644
--- a/drivers/crypto/caam/caamhash_desc.c
+++ b/drivers/crypto/caam/caamhash_desc.c
@@ -2,7 +2,7 @@
/*
* Shared descriptors for ahash algorithms
*
- * Copyright 2017 NXP
+ * Copyright 2017-2019 NXP
*/
#include "compat.h"
@@ -75,6 +75,72 @@ void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
}
EXPORT_SYMBOL(cnstr_shdsc_ahash);
+/**
+ * cnstr_shdsc_sk_hash - shared descriptor for symmetric key cipher-based
+ * hash algorithms
+ * @desc: pointer to buffer used for descriptor construction
+ * @adata: pointer to authentication transform definitions.
+ * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE}
+ * @digestsize: algorithm's digest size
+ * @ctx_len: size of Context Register
+ * @key_dma: I/O Virtual Address of the key
+ */
+void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
+ int digestsize, int ctx_len, dma_addr_t key_dma)
+{
+ u32 *skip_key_load;
+
+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
+
+ /* Skip loading of key, context if already shared */
+ skip_key_load = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
+
+ if (state == OP_ALG_AS_INIT || state == OP_ALG_AS_INITFINAL) {
+ append_key_as_imm(desc, adata->key_virt, adata->keylen,
+ adata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
+ } else { /* UPDATE, FINALIZE */
+ if (is_xcbc_aes(adata->algtype))
+ /* Load K1 */
+ append_key(desc, adata->key_dma, adata->keylen,
+ CLASS_1 | KEY_DEST_CLASS_REG | KEY_ENC);
+ else /* CMAC */
+ append_key_as_imm(desc, adata->key_virt, adata->keylen,
+ adata->keylen, CLASS_1 |
+ KEY_DEST_CLASS_REG);
+ /* Restore context */
+ append_seq_load(desc, ctx_len, LDST_CLASS_1_CCB |
+ LDST_SRCDST_BYTE_CONTEXT);
+ }
+
+ set_jump_tgt_here(desc, skip_key_load);
+
+ /* Class 1 operation */
+ append_operation(desc, adata->algtype | state | OP_ALG_ENCRYPT);
+
+ /*
+ * Load from buf and/or src and write to req->result or state->context
+ * Calculate remaining bytes to read
+ */
+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+ /* Read remaining bytes */
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_LAST1 |
+ FIFOLD_TYPE_MSG | FIFOLDST_VLF);
+
+ /*
+ * Save context:
+ * - xcbc: partial hash, keys K2 and K3
+ * - cmac: partial hash, constant L = E(K,0)
+ */
+ append_seq_store(desc, digestsize, LDST_CLASS_1_CCB |
+ LDST_SRCDST_BYTE_CONTEXT);
+ if (is_xcbc_aes(adata->algtype) && state == OP_ALG_AS_INIT)
+ /* Save K1 */
+ append_fifo_store(desc, key_dma, adata->keylen,
+ LDST_CLASS_1_CCB | FIFOST_TYPE_KEY_KEK);
+}
+EXPORT_SYMBOL(cnstr_shdsc_sk_hash);
+
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("FSL CAAM ahash descriptors support");
MODULE_AUTHOR("NXP Semiconductors");
diff --git a/drivers/crypto/caam/caamhash_desc.h b/drivers/crypto/caam/caamhash_desc.h
index 631fc1ac312c..6947ee1f200c 100644
--- a/drivers/crypto/caam/caamhash_desc.h
+++ b/drivers/crypto/caam/caamhash_desc.h
@@ -15,7 +15,15 @@
#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
+static inline bool is_xcbc_aes(u32 algtype)
+{
+ return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) ==
+ (OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC);
+}
+
void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state,
int digestsize, int ctx_len, bool import_ctx, int era);
+void cnstr_shdsc_sk_hash(u32 * const desc, struct alginfo *adata, u32 state,
+ int digestsize, int ctx_len, dma_addr_t key_dma);
#endif /* _CAAMHASH_DESC_H_ */
diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h
index 87d9efe4c7aa..8639b2df0371 100644
--- a/drivers/crypto/caam/compat.h
+++ b/drivers/crypto/caam/compat.h
@@ -43,6 +43,7 @@
#include <crypto/akcipher.h>
#include <crypto/scatterwalk.h>
#include <crypto/skcipher.h>
+#include <crypto/arc4.h>
#include <crypto/internal/skcipher.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/rsa.h>
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 16bbc72f041a..858bdc9ab4a3 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -18,12 +18,8 @@
#include "desc_constr.h"
#include "ctrl.h"
-bool caam_little_end;
-EXPORT_SYMBOL(caam_little_end);
bool caam_dpaa2;
EXPORT_SYMBOL(caam_dpaa2);
-bool caam_imx;
-EXPORT_SYMBOL(caam_imx);
#ifdef CONFIG_CAAM_QI
#include "qi.h"
@@ -863,27 +859,18 @@ static int caam_probe(struct platform_device *pdev)
/* Internal covering keys (useful in non-secure mode only) */
ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
- ctrlpriv->ctl_kek = debugfs_create_blob("kek",
- S_IRUSR |
- S_IRGRP | S_IROTH,
- ctrlpriv->ctl,
- &ctrlpriv->ctl_kek_wrap);
+ debugfs_create_blob("kek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl,
+ &ctrlpriv->ctl_kek_wrap);
ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0];
ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
- ctrlpriv->ctl_tkek = debugfs_create_blob("tkek",
- S_IRUSR |
- S_IRGRP | S_IROTH,
- ctrlpriv->ctl,
- &ctrlpriv->ctl_tkek_wrap);
+ debugfs_create_blob("tkek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl,
+ &ctrlpriv->ctl_tkek_wrap);
ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0];
ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32);
- ctrlpriv->ctl_tdsk = debugfs_create_blob("tdsk",
- S_IRUSR |
- S_IRGRP | S_IROTH,
- ctrlpriv->ctl,
- &ctrlpriv->ctl_tdsk_wrap);
+ debugfs_create_blob("tdsk", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl,
+ &ctrlpriv->ctl_tdsk_wrap);
#endif
return 0;
diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h
index ec10230178c5..4b6854bf896a 100644
--- a/drivers/crypto/caam/desc.h
+++ b/drivers/crypto/caam/desc.h
@@ -1155,6 +1155,7 @@
#define OP_ALG_ALGSEL_DES (0x20 << OP_ALG_ALGSEL_SHIFT)
#define OP_ALG_ALGSEL_3DES (0x21 << OP_ALG_ALGSEL_SHIFT)
#define OP_ALG_ALGSEL_ARC4 (0x30 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_CHA_MDHA (0x40 << OP_ALG_ALGSEL_SHIFT)
#define OP_ALG_ALGSEL_MD5 (0x40 << OP_ALG_ALGSEL_SHIFT)
#define OP_ALG_ALGSEL_SHA1 (0x41 << OP_ALG_ALGSEL_SHIFT)
#define OP_ALG_ALGSEL_SHA224 (0x42 << OP_ALG_ALGSEL_SHIFT)
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 7e8d690f2827..21a70fd32f5d 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -50,6 +50,12 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
#endif /* DEBUG */
EXPORT_SYMBOL(caam_dump_sg);
+bool caam_little_end;
+EXPORT_SYMBOL(caam_little_end);
+
+bool caam_imx;
+EXPORT_SYMBOL(caam_imx);
+
static const struct {
u8 value;
const char *error_text;
diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h
index 67ea94079837..8c6b83e02a70 100644
--- a/drivers/crypto/caam/error.h
+++ b/drivers/crypto/caam/error.h
@@ -7,6 +7,9 @@
#ifndef CAAM_ERROR_H
#define CAAM_ERROR_H
+
+#include "desc.h"
+
#define CAAM_ERROR_STR_MAX 302
void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
@@ -17,4 +20,10 @@ void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
int rowsize, int groupsize, struct scatterlist *sg,
size_t tlen, bool ascii);
+
+static inline bool is_mdha(u32 algtype)
+{
+ return (algtype & OP_ALG_ALGSEL_MASK & ~OP_ALG_ALGSEL_SUBMASK) ==
+ OP_ALG_CHA_MDHA;
+}
#endif /* CAAM_ERROR_H */
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index babc78abd155..5869ad58d497 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -106,7 +106,6 @@ struct caam_drv_private {
struct dentry *dfs_root;
struct dentry *ctl; /* controller dir */
struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap;
- struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk;
#endif
};
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 312b5f042f31..8d0713fae6ac 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -48,7 +48,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
{
u32 *desc;
struct split_key_result result;
- dma_addr_t dma_addr_in, dma_addr_out;
+ dma_addr_t dma_addr;
int ret = -ENOMEM;
adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK);
@@ -71,22 +71,17 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
return ret;
}
- dma_addr_in = dma_map_single(jrdev, (void *)key_in, keylen,
- DMA_TO_DEVICE);
- if (dma_mapping_error(jrdev, dma_addr_in)) {
- dev_err(jrdev, "unable to map key input memory\n");
- goto out_free;
- }
+ memcpy(key_out, key_in, keylen);
- dma_addr_out = dma_map_single(jrdev, key_out, adata->keylen_pad,
- DMA_FROM_DEVICE);
- if (dma_mapping_error(jrdev, dma_addr_out)) {
- dev_err(jrdev, "unable to map key output memory\n");
- goto out_unmap_in;
+ dma_addr = dma_map_single(jrdev, key_out, adata->keylen_pad,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(jrdev, dma_addr)) {
+ dev_err(jrdev, "unable to map key memory\n");
+ goto out_free;
}
init_job_desc(desc, 0);
- append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG);
+ append_key(desc, dma_addr, keylen, CLASS_2 | KEY_DEST_CLASS_REG);
/* Sets MDHA up into an HMAC-INIT */
append_operation(desc, (adata->algtype & OP_ALG_ALGSEL_MASK) |
@@ -104,12 +99,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
* FIFO_STORE with the explicit split-key content store
* (0x26 output type)
*/
- append_fifo_store(desc, dma_addr_out, adata->keylen,
+ append_fifo_store(desc, dma_addr, adata->keylen,
LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
#ifdef DEBUG
- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
- DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
#endif
@@ -129,10 +122,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
#endif
}
- dma_unmap_single(jrdev, dma_addr_out, adata->keylen_pad,
- DMA_FROM_DEVICE);
-out_unmap_in:
- dma_unmap_single(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE);
+ dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL);
out_free:
kfree(desc);
return ret;
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index b84e6c8b1e13..7cb8b1755e57 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -318,7 +318,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
/* Create a new req FQ in parked state */
new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
drv_ctx->context_a, 0);
- if (unlikely(IS_ERR_OR_NULL(new_fq))) {
+ if (IS_ERR_OR_NULL(new_fq)) {
dev_err(qidev, "FQ allocation for shdesc update failed\n");
return PTR_ERR(new_fq);
}
@@ -431,7 +431,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
/* Attach request FQ */
drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
QMAN_INITFQ_FLAG_SCHED);
- if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) {
+ if (IS_ERR_OR_NULL(drv_ctx->req_fq)) {
dev_err(qidev, "create_caam_req_fq failed\n");
dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
kfree(drv_ctx);
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.