diff options
Diffstat (limited to 'drivers/crypto/caam/caamalg.c')
| -rw-r--r-- | drivers/crypto/caam/caamalg.c | 130 |
1 files changed, 86 insertions, 44 deletions
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 54f3b375a453..2188235be02d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -108,6 +108,7 @@ struct caam_ctx { dma_addr_t sh_desc_dec_dma; dma_addr_t sh_desc_givenc_dma; dma_addr_t key_dma; + enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; @@ -118,6 +119,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - ctx->adata.keylen_pad; @@ -136,9 +138,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize); + cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -154,9 +157,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) /* aead_decrypt shared descriptor */ desc = ctx->sh_desc_dec; - cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize); + cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -168,6 +172,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 ctx1_iv_off = 0; u32 *desc, *nonce = NULL; u32 inl_mask; @@ -234,9 +239,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, - false); + false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); skip_enc: /* @@ -266,9 +271,9 @@ skip_enc: desc = ctx->sh_desc_dec; cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, alg->caam.geniv, is_rfc3686, - nonce, ctx1_iv_off, false); + nonce, ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); if (!alg->caam.geniv) goto skip_givenc; @@ -300,9 +305,9 @@ skip_enc: desc = ctx->sh_desc_enc; cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, ctx->authsize, is_rfc3686, nonce, - ctx1_iv_off, false); + ctx1_iv_off, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); skip_givenc: return 0; @@ -346,7 +351,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -363,7 +368,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -405,7 +410,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -422,7 +427,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -465,7 +470,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_enc; cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* * Job Descriptor and Shared Descriptors @@ -482,7 +487,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) desc = ctx->sh_desc_dec; cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -503,6 +508,7 @@ static int aead_setkey(struct crypto_aead *aead, { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); struct crypto_authenc_keys keys; int ret = 0; @@ -517,6 +523,27 @@ static int aead_setkey(struct crypto_aead *aead, DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, ctx->dir); + goto skip_split_key; + } + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, keys.authkeylen, CAAM_MAX_KEY_SIZE - keys.enckeylen); @@ -527,12 +554,14 @@ static int aead_setkey(struct crypto_aead *aead, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + - keys.enckeylen, DMA_TO_DEVICE); + keys.enckeylen, ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, ctx->adata.keylen_pad + keys.enckeylen, 1); #endif + +skip_split_key: ctx->cdata.keylen = keys.enckeylen; return aead_set_sh_desc(aead); badkey: @@ -552,7 +581,7 @@ static int gcm_setkey(struct crypto_aead *aead, #endif memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); ctx->cdata.keylen = keylen; return gcm_set_sh_desc(aead); @@ -580,7 +609,7 @@ static int rfc4106_setkey(struct crypto_aead *aead, */ ctx->cdata.keylen = keylen - 4; dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - DMA_TO_DEVICE); + ctx->dir); return rfc4106_set_sh_desc(aead); } @@ -606,7 +635,7 @@ static int rfc4543_setkey(struct crypto_aead *aead, */ ctx->cdata.keylen = keylen - 4; dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - DMA_TO_DEVICE); + ctx->dir); return rfc4543_set_sh_desc(aead); } @@ -625,7 +654,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const bool is_rfc3686 = (ctr_mode && (strstr(alg_name, "rfc3686") != NULL)); - memcpy(ctx->key, key, keylen); #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); @@ -648,9 +676,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, keylen -= CTR_RFC3686_NONCE_SIZE; } - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); ctx->cdata.keylen = keylen; - ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; /* ablkcipher_encrypt shared descriptor */ @@ -658,21 +685,21 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, ctx1_iv_off); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, ctx1_iv_off); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* ablkcipher_givencrypt shared descriptor */ desc = ctx->sh_desc_givenc; cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, ctx1_iv_off); dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -691,23 +718,21 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, return -EINVAL; } - memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); ctx->cdata.keylen = keylen; - ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; /* xts_ablkcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); /* xts_ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, - desc_bytes(desc), DMA_TO_DEVICE); + desc_bytes(desc), ctx->dir); return 0; } @@ -979,9 +1004,6 @@ static void init_aead_job(struct aead_request *req, append_seq_out_ptr(desc, dst_dma, req->assoclen + req->cryptlen - authsize, out_options); - - /* REG3 = assoclen */ - append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); } static void init_gcm_job(struct aead_request *req, @@ -992,10 +1014,11 @@ static void init_gcm_job(struct aead_request *req, struct caam_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc = edesc->hw_desc; - bool generic_gcm = (ivsize == 12); + bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); unsigned int last; init_aead_job(req, edesc, all_contig, encrypt); + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); /* BUG This should not be specific to generic GCM. */ last = 0; @@ -1004,7 +1027,7 @@ static void init_gcm_job(struct aead_request *req, /* Read GCM IV */ append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); /* Append Salt */ if (!generic_gcm) append_data(desc, ctx->key + ctx->cdata.keylen, 4); @@ -1022,6 +1045,7 @@ static void init_authenc_job(struct aead_request *req, struct caam_aead_alg, aead); unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; @@ -1045,6 +1069,15 @@ static void init_authenc_job(struct aead_request *req, init_aead_job(req, edesc, all_contig, encrypt); + /* + * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports + * having DPOVRD as destination. + */ + if (ctrlpriv->era < 3) + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); + else + append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); + if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | @@ -1953,7 +1986,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = rfc4106_setauthsize, .encrypt = ipsec_gcm_encrypt, .decrypt = ipsec_gcm_decrypt, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -1971,7 +2004,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = rfc4543_setauthsize, .encrypt = ipsec_gcm_encrypt, .decrypt = ipsec_gcm_decrypt, - .ivsize = 8, + .ivsize = GCM_RFC4543_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -1990,7 +2023,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = gcm_setauthsize, .encrypt = gcm_encrypt, .decrypt = gcm_decrypt, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -3228,9 +3261,11 @@ struct caam_crypto_alg { struct caam_alg_entry caam; }; -static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, + bool uses_dkp) { dma_addr_t dma_addr; + struct caam_drv_private *priv; ctx->jrdev = caam_jr_alloc(); if (IS_ERR(ctx->jrdev)) { @@ -3238,10 +3273,16 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) return PTR_ERR(ctx->jrdev); } + priv = dev_get_drvdata(ctx->jrdev->parent); + if (priv->era >= 6 && uses_dkp) + ctx->dir = DMA_BIDIRECTIONAL; + else + ctx->dir = DMA_TO_DEVICE; + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, offsetof(struct caam_ctx, sh_desc_enc_dma), - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); if (dma_mapping_error(ctx->jrdev, dma_addr)) { dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); caam_jr_free(ctx->jrdev); @@ -3269,7 +3310,7 @@ static int caam_cra_init(struct crypto_tfm *tfm) container_of(alg, struct caam_crypto_alg, crypto_alg); struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - return caam_init_common(ctx, &caam_alg->caam); + return caam_init_common(ctx, &caam_alg->caam, false); } static int caam_aead_init(struct crypto_aead *tfm) @@ -3279,14 +3320,15 @@ static int caam_aead_init(struct crypto_aead *tfm) container_of(alg, struct caam_aead_alg, aead); struct caam_ctx *ctx = crypto_aead_ctx(tfm); - return caam_init_common(ctx, &caam_alg->caam); + return caam_init_common(ctx, &caam_alg->caam, + alg->setkey == aead_setkey); } static void caam_exit_common(struct caam_ctx *ctx) { dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, offsetof(struct caam_ctx, sh_desc_enc_dma), - DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); caam_jr_free(ctx->jrdev); } |