From 7fc342d2e672630fdda689a4d3d3b90745c70451 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 17 Aug 2017 18:44:10 +0530 Subject: crypto: nx - constify vio_device_id vio_device_id are not supposed to change at runtime. All functions working with vio_device_id provided by work with const vio_device_id. So mark the non-const structs as const. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 036057abb257..3a5e31be4764 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -833,7 +833,7 @@ static void __exit nx_fini(void) vio_unregister_driver(&nx_driver.viodriver); } -static struct vio_device_id nx_crypto_driver_ids[] = { +static const struct vio_device_id nx_crypto_driver_ids[] = { { "ibm,sym-encryption-v1", "ibm,sym-encryption" }, { "", "" } }; -- cgit v1.2.3-59-g8ed1b From f096ced9e9d8cabdf9c61b5eef7188775c1daf05 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 17 Aug 2017 18:44:11 +0530 Subject: crypto: nx-842 - constify vio_device_id vio_device_id are not supposed to change at runtime. All functions working with vio_device_id provided by work with const vio_device_id. So mark the non-const structs as const. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-842-pseries.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c index cddc6d8b55d9..bf52cd1d7fca 100644 --- a/drivers/crypto/nx/nx-842-pseries.c +++ b/drivers/crypto/nx/nx-842-pseries.c @@ -1082,7 +1082,7 @@ static int nx842_remove(struct vio_dev *viodev) return 0; } -static struct vio_device_id nx842_vio_driver_ids[] = { +static const struct vio_device_id nx842_vio_driver_ids[] = { {"ibm,compression-v1", "ibm,compression"}, {"", ""}, }; -- cgit v1.2.3-59-g8ed1b From a502e1089e3f711599023de5baa3af2b911c630a Mon Sep 17 00:00:00 2001 From: "lionel.debieve@st.com" Date: Fri, 18 Aug 2017 15:54:01 +0200 Subject: crypto: stm32/hash - Remove uninitialized symbol Remove err symbol as this is not used in the thread context and the variable is not initialized. Signed-off-by: Lionel Debieve Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-hash.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index b585ce54a802..b34ee85509c2 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -1067,7 +1067,6 @@ static int stm32_hash_cra_sha256_init(struct crypto_tfm *tfm) static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id) { struct stm32_hash_dev *hdev = dev_id; - int err; if (HASH_FLAGS_CPU & hdev->flags) { if (HASH_FLAGS_OUTPUT_READY & hdev->flags) { @@ -1084,8 +1083,8 @@ static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id) return IRQ_HANDLED; finish: - /*Finish current request */ - stm32_hash_finish_req(hdev->req, err); + /* Finish current request */ + stm32_hash_finish_req(hdev->req, 0); return IRQ_HANDLED; } -- cgit v1.2.3-59-g8ed1b From e47270665b5fcdf8a9ae628e603119e998ca3715 Mon Sep 17 00:00:00 2001 From: Ryder Lee Date: Tue, 22 Aug 2017 15:53:39 +0800 Subject: crypto: mediatek - Add empty messages check in GCM mode Currently, empty messages are not supported in GCM mode, hence add a check to prevent producing incorrect results. Signed-off-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 9e845e866dec..72e4549ed36b 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -928,8 +928,13 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) { struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); struct mtk_aes_reqctx *rctx = aead_request_ctx(req); + /* Empty messages are not supported yet */ + if (!gctx->textlen && !req->assoclen) + return -EINVAL; + rctx->mode = AES_FLAGS_GCM | mode; return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT), -- cgit v1.2.3-59-g8ed1b From 7545e166535b349cc4dadea19cf9ee9494563182 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:09 +0200 Subject: crypto: caam - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 10 +++++----- drivers/crypto/caam/compat.h | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 54f3b375a453..baa8dd52472d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -992,7 +992,7 @@ static void init_gcm_job(struct aead_request *req, struct caam_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc = edesc->hw_desc; - bool generic_gcm = (ivsize == 12); + bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); unsigned int last; init_aead_job(req, edesc, all_contig, encrypt); @@ -1004,7 +1004,7 @@ static void init_gcm_job(struct aead_request *req, /* Read GCM IV */ append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); /* Append Salt */ if (!generic_gcm) append_data(desc, ctx->key + ctx->cdata.keylen, 4); @@ -1953,7 +1953,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = rfc4106_setauthsize, .encrypt = ipsec_gcm_encrypt, .decrypt = ipsec_gcm_decrypt, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -1971,7 +1971,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = rfc4543_setauthsize, .encrypt = ipsec_gcm_encrypt, .decrypt = ipsec_gcm_decrypt, - .ivsize = 8, + .ivsize = GCM_RFC4543_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -1990,7 +1990,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = gcm_setauthsize, .encrypt = gcm_encrypt, .decrypt = gcm_decrypt, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index 7149cd2492e0..5b8d930f3dd8 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3-59-g8ed1b From cf0bd0ae19f44b8d5ccf5e1017fde382a7fff425 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:10 +0200 Subject: crypto: ccp - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-aes-galois.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c index 52313524a4dd..ff02b713c6f6 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c @@ -19,13 +19,12 @@ #include #include #include +#include #include #include #include "ccp-crypto.h" -#define AES_GCM_IVSIZE 12 - static int ccp_aes_gcm_complete(struct crypto_async_request *async_req, int ret) { return ret; @@ -95,9 +94,9 @@ static int ccp_aes_gcm_crypt(struct aead_request *req, bool encrypt) */ /* Prepare the IV: 12 bytes + an integer (counter) */ - memcpy(rctx->iv, req->iv, AES_GCM_IVSIZE); + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); for (i = 0; i < 3; i++) - rctx->iv[i + AES_GCM_IVSIZE] = 0; + rctx->iv[i + GCM_AES_IV_SIZE] = 0; rctx->iv[AES_BLOCK_SIZE - 1] = 1; /* Set up a scatterlist for the IV */ @@ -160,7 +159,7 @@ static struct aead_alg ccp_aes_gcm_defaults = { .encrypt = ccp_aes_gcm_encrypt, .decrypt = ccp_aes_gcm_decrypt, .init = ccp_aes_gcm_cra_init, - .ivsize = AES_GCM_IVSIZE, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | -- cgit v1.2.3-59-g8ed1b From fdd0f3d8922dd2afd569a937affc331cb01aa344 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:11 +0200 Subject: crypto: nx - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/nx/nx-aes-gcm.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c index abd465f479c4..a810596b97c2 100644 --- a/drivers/crypto/nx/nx-aes-gcm.c +++ b/drivers/crypto/nx/nx-aes-gcm.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -433,7 +434,7 @@ static int gcm_aes_nx_encrypt(struct aead_request *req) struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; - memcpy(iv, req->iv, 12); + memcpy(iv, req->iv, GCM_AES_IV_SIZE); return gcm_aes_nx_crypt(req, 1, req->assoclen); } @@ -443,7 +444,7 @@ static int gcm_aes_nx_decrypt(struct aead_request *req) struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; - memcpy(iv, req->iv, 12); + memcpy(iv, req->iv, GCM_AES_IV_SIZE); return gcm_aes_nx_crypt(req, 0, req->assoclen); } @@ -498,7 +499,7 @@ struct aead_alg nx_gcm_aes_alg = { }, .init = nx_crypto_ctx_aes_gcm_init, .exit = nx_crypto_ctx_aead_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = gcm_aes_nx_set_key, .encrypt = gcm_aes_nx_encrypt, @@ -516,7 +517,7 @@ struct aead_alg nx_gcm4106_aes_alg = { }, .init = nx_crypto_ctx_aes_gcm_init, .exit = nx_crypto_ctx_aead_exit, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = gcm4106_aes_nx_set_key, .setauthsize = gcm4106_aes_nx_setauthsize, -- cgit v1.2.3-59-g8ed1b From 219d51c759c3ecd04372aa6b3ce74bdc4911f281 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:12 +0200 Subject: crypto: atmel - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 29e20c37f3a6..903fd43f23a5 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -1532,7 +1533,7 @@ static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) if (err) return atmel_aes_complete(dd, err); - if (likely(ivsize == 12)) { + if (likely(ivsize == GCM_AES_IV_SIZE)) { memcpy(ctx->j0, iv, ivsize); ctx->j0[3] = cpu_to_be32(1); return atmel_aes_gcm_process(dd); @@ -1820,7 +1821,7 @@ static struct aead_alg aes_gcm_alg = { .decrypt = atmel_aes_gcm_decrypt, .init = atmel_aes_gcm_init, .exit = atmel_aes_gcm_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { -- cgit v1.2.3-59-g8ed1b From a59851d2f220e6653f8878bc312ad61f108fd3c9 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:13 +0200 Subject: crypto: bcm - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 8 ++++---- drivers/crypto/bcm/cipher.h | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 8685c7e4debd..537a67483aa3 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -1367,11 +1367,11 @@ static int handle_aead_req(struct iproc_reqctx_s *rctx) * expects AAD to include just SPI and seqno. So * subtract off the IV len. */ - aead_parms.assoc_size -= GCM_ESP_IV_SIZE; + aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE; if (rctx->is_encrypt) { aead_parms.return_iv = true; - aead_parms.ret_iv_len = GCM_ESP_IV_SIZE; + aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE; aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE; } } else { @@ -3255,7 +3255,7 @@ static struct iproc_alg_s driver_algs[] = { .cra_flags = CRYPTO_ALG_NEED_FALLBACK }, .setkey = aead_gcm_esp_setkey, - .ivsize = GCM_ESP_IV_SIZE, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .cipher_info = { @@ -3301,7 +3301,7 @@ static struct iproc_alg_s driver_algs[] = { .cra_flags = CRYPTO_ALG_NEED_FALLBACK }, .setkey = rfc4543_gcm_esp_setkey, - .ivsize = GCM_ESP_IV_SIZE, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .cipher_info = { diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 57a55eb2a255..763c425c41ca 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -39,8 +40,6 @@ #define ARC4_STATE_SIZE 4 #define CCM_AES_IV_SIZE 16 -#define GCM_AES_IV_SIZE 12 -#define GCM_ESP_IV_SIZE 8 #define CCM_ESP_IV_SIZE 8 #define RFC4543_ICV_SIZE 16 -- cgit v1.2.3-59-g8ed1b From 07b83f437d9129adda47d78bdadd7813a008d092 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:14 +0200 Subject: crypto: mediatek - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 72e4549ed36b..32aa587783c3 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -13,6 +13,7 @@ */ #include +#include #include "mtk-platform.h" #define AES_QUEUE_SIZE 512 @@ -1103,7 +1104,7 @@ static struct aead_alg aes_gcm_alg = { .decrypt = mtk_aes_gcm_decrypt, .init = mtk_aes_gcm_init, .exit = mtk_aes_gcm_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { -- cgit v1.2.3-59-g8ed1b From 8f6acb7fbf30da326b58ba092949fcd3ebadb01d Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:15 +0200 Subject: crypto: chelsio - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 0e8160701833..936bdd895efa 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -2534,9 +2535,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { memcpy(reqctx->iv, aeadctx->salt, 4); - memcpy(reqctx->iv + 4, req->iv, 8); + memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE); } else { - memcpy(reqctx->iv, req->iv, 12); + memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE); } *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); @@ -3385,7 +3386,7 @@ static struct chcr_alg_template driver_algs[] = { sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_gcm_ctx), }, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = GHASH_DIGEST_SIZE, .setkey = chcr_gcm_setkey, .setauthsize = chcr_gcm_setauthsize, @@ -3405,7 +3406,7 @@ static struct chcr_alg_template driver_algs[] = { sizeof(struct chcr_gcm_ctx), }, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = GHASH_DIGEST_SIZE, .setkey = chcr_gcm_setkey, .setauthsize = chcr_4106_4309_setauthsize, -- cgit v1.2.3-59-g8ed1b From cb3f3817519ceed90d210b78bb85bfe92310d0ec Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Tue, 22 Aug 2017 10:08:16 +0200 Subject: crypto: omap - Use GCM IV size constant This patch replace GCM IV size value by their constant name. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes-gcm.c | 7 ++++--- drivers/crypto/omap-aes.c | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 7d4f8a4be6d8..9b8f1c752168 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -311,7 +312,7 @@ static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) int err, assoclen; memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); - memcpy(rctx->iv + 12, &counter, 4); + memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); if (err) @@ -339,7 +340,7 @@ int omap_aes_gcm_encrypt(struct aead_request *req) { struct omap_aes_reqctx *rctx = aead_request_ctx(req); - memcpy(rctx->iv, req->iv, 12); + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); } @@ -347,7 +348,7 @@ int omap_aes_gcm_decrypt(struct aead_request *req) { struct omap_aes_reqctx *rctx = aead_request_ctx(req); - memcpy(rctx->iv, req->iv, 12); + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); return omap_aes_gcm_crypt(req, FLAGS_GCM); } diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index c376a3ee7c2c..1f3686a1ebfa 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -767,7 +768,7 @@ static struct aead_alg algs_aead_gcm[] = { }, .init = omap_aes_gcm_cra_init, .exit = omap_aes_gcm_cra_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = omap_aes_gcm_setkey, .encrypt = omap_aes_gcm_encrypt, @@ -788,7 +789,7 @@ static struct aead_alg algs_aead_gcm[] = { .init = omap_aes_gcm_cra_init, .exit = omap_aes_gcm_cra_exit, .maxauthsize = AES_BLOCK_SIZE, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .setkey = omap_aes_4106gcm_setkey, .encrypt = omap_aes_4106gcm_encrypt, .decrypt = omap_aes_4106gcm_decrypt, -- cgit v1.2.3-59-g8ed1b From a728a196d253530f17da5c86dc7dfbe58c5f7094 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:14 +0200 Subject: crypto: crypto4xx - remove bad list_del alg entries are only added to the list, after the registration was successful. If the registration failed, it was never added to the list in the first place. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 65dc78b91dea..3118cec0d81e 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1033,12 +1033,10 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, break; } - if (rc) { - list_del(&alg->entry); + if (rc) kfree(alg); - } else { + else list_add_tail(&alg->entry, &sec_dev->alg_list); - } } return 0; -- cgit v1.2.3-59-g8ed1b From 81065f66dd99b3af58626a914b8c0fcff6b8b0ba Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:15 +0200 Subject: crypto: crypto4xx - remove unused definitions and write-only variables This patch removes several unused code and definitons (structs, variables, ...). Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 6 ------ drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/amcc/crypto4xx_core.h | 16 ---------------- 3 files changed, 1 insertion(+), 23 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 4afca3968773..6da92321173b 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -75,7 +75,6 @@ int crypto4xx_encrypt(struct ablkcipher_request *req) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); ctx->direction = DIR_OUTBOUND; - ctx->hash_final = 0; ctx->is_hash = 0; ctx->pd_ctl = 0x1; @@ -89,7 +88,6 @@ int crypto4xx_decrypt(struct ablkcipher_request *req) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); ctx->direction = DIR_INBOUND; - ctx->hash_final = 0; ctx->is_hash = 0; ctx->pd_ctl = 1; @@ -136,7 +134,6 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, } /* Setup SA */ sa = (struct dynamic_sa_ctl *) ctx->sa_in; - ctx->hash_final = 0; set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, @@ -191,7 +188,6 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, ctx->dev = my_alg->dev; ctx->is_hash = 1; - ctx->hash_final = 0; /* Create SA */ if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) @@ -256,7 +252,6 @@ int crypto4xx_hash_update(struct ahash_request *req) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); ctx->is_hash = 1; - ctx->hash_final = 0; ctx->pd_ctl = 0x11; ctx->direction = DIR_INBOUND; @@ -274,7 +269,6 @@ int crypto4xx_hash_digest(struct ahash_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - ctx->hash_final = 1; ctx->pd_ctl = 0x11; ctx->direction = DIR_INBOUND; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 3118cec0d81e..48215b8525a2 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -962,7 +962,7 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, sa->sa_command_1.bf.hash_crypto_offset = 0; pd->pd_ctl.w = ctx->pd_ctl; - pd->pd_ctl_len.w = 0x00400000 | (ctx->bypass << 24) | datalen; + pd->pd_ctl_len.w = 0x00400000 | datalen; pd_uinfo->state = PD_ENTRY_INUSE; wmb(); /* write any value to push engine to read a pd */ diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ecfdcfe3698d..8230d3f7edbe 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -72,7 +72,6 @@ struct pd_uinfo { struct crypto4xx_device { struct crypto4xx_core_device *core_dev; char *name; - u64 ce_phy_address; void __iomem *ce_base; void __iomem *trng_base; @@ -127,21 +126,9 @@ struct crypto4xx_ctx { u32 sa_len; u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */ u32 direction; - u32 next_hdr; u32 save_iv; - u32 pd_ctl_len; u32 pd_ctl; - u32 bypass; u32 is_hash; - u32 hash_final; -}; - -struct crypto4xx_req_ctx { - struct crypto4xx_device *dev; /* Device in which - operation to send to */ - void *sa; - u32 sa_dma_addr; - u16 sa_len; }; struct crypto4xx_alg_common { @@ -172,9 +159,6 @@ static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); -extern u32 crypto4xx_alloc_sa_rctx(struct crypto4xx_ctx *ctx, - struct crypto4xx_ctx *rctx); -extern void crypto4xx_free_sa_rctx(struct crypto4xx_ctx *rctx); extern void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); extern u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); extern u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx); -- cgit v1.2.3-59-g8ed1b From 1ef52a95ea53c3c54b061e3f1af85976356c7132 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:16 +0200 Subject: crypto: crypto4xx - set CRYPTO_ALG_KERN_DRIVER_ONLY flag The security offload function is performed by a cryptographic engine core attached to the 128-bit PLB (processor local bus) with builtin DMA and interrupt controllers. This, I think, satisfies the requirement for the CRYPTO_ALG_KERN_DRIVER_ONLY flag. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 48215b8525a2..249d35ff3806 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1114,7 +1114,9 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-ppc4xx", .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, -- cgit v1.2.3-59-g8ed1b From 886c251fd4ca40a27697afec7bc44c115e803d78 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:17 +0200 Subject: crypto: crypto4xx - remove extern statement before function declaration All function declarations are "extern" by default, there is no need to specify it explicitly. For C99 states in 6.2.2.5: "If the declaration of an identifier for a function has no storage-class specifier, its linkage is determined exactly as if it were declared with the storage-class specifier extern." Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.h | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 8230d3f7edbe..995f3da91cb2 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -157,28 +157,28 @@ static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( return container_of(x, struct crypto4xx_alg, alg.u.cipher); } -extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); -extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); -extern void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); -extern u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx); -extern void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, int len); -extern u32 crypto4xx_build_pd(struct crypto_async_request *req, - struct crypto4xx_ctx *ctx, - struct scatterlist *src, - struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len); -extern int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, - const u8 *key, unsigned int keylen); -extern int crypto4xx_encrypt(struct ablkcipher_request *req); -extern int crypto4xx_decrypt(struct ablkcipher_request *req); -extern int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); -extern int crypto4xx_hash_digest(struct ahash_request *req); -extern int crypto4xx_hash_final(struct ahash_request *req); -extern int crypto4xx_hash_update(struct ahash_request *req); -extern int crypto4xx_hash_init(struct ahash_request *req); +int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); +void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); +void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); +u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); +u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx); +u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx); +u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx); +void crypto4xx_memcpy_le(unsigned int *dst, + const unsigned char *buf, int len); +u32 crypto4xx_build_pd(struct crypto_async_request *req, + struct crypto4xx_ctx *ctx, + struct scatterlist *src, + struct scatterlist *dst, + unsigned int datalen, + void *iv, u32 iv_len); +int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt(struct ablkcipher_request *req); +int crypto4xx_decrypt(struct ablkcipher_request *req); +int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); +int crypto4xx_hash_digest(struct ahash_request *req); +int crypto4xx_hash_final(struct ahash_request *req); +int crypto4xx_hash_update(struct ahash_request *req); +int crypto4xx_hash_init(struct ahash_request *req); #endif -- cgit v1.2.3-59-g8ed1b From c587e65deacf8c86de2d7c51f1e81d0a4a9147a8 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:18 +0200 Subject: crypto: crypto4xx - remove double assignment of pd_uinfo->state crypto4xx_put_pd_to_pdr() already clears the flag. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 249d35ff3806..9e96af725b31 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1079,7 +1079,6 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) pd->pd_ctl.bf.pe_done = 0; crypto4xx_pd_done(core_dev->dev, tail); crypto4xx_put_pd_to_pdr(core_dev->dev, tail); - pd_uinfo->state = PD_ENTRY_FREE; } else { /* if tail not done, break */ break; -- cgit v1.2.3-59-g8ed1b From 453e3090b9c3f5da70b21648c2244e9821f0916d Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:19 +0200 Subject: crypto: crypto4xx - fix dynamic_sa_ctl's sa_contents declaration The driver had a union dynamic_sa_contents in place that described the meaning of the bits in the sa_contents variable. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 4 ++-- drivers/crypto/amcc/crypto4xx_sa.c | 12 ++++++------ drivers/crypto/amcc/crypto4xx_sa.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 6da92321173b..867a9230cd3f 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -149,7 +149,7 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, SA_NOT_COPY_HDR); crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx), key, keylen); - sa->sa_contents = SA_AES_CONTENTS | (keylen << 2); + sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); sa->sa_command_1.bf.key_len = keylen >> 3; ctx->is_hash = 0; ctx->direction = DIR_INBOUND; @@ -219,7 +219,7 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); ctx->direction = DIR_INBOUND; - sa->sa_contents = SA_HASH160_CONTENTS; + sa->sa_contents.w = SA_HASH160_CONTENTS; sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in; /* Need to zero hash digest in SA */ memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest)); diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c index 69182e2cc3ea..78a8ca192d0d 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.c +++ b/drivers/crypto/amcc/crypto4xx_sa.c @@ -40,9 +40,9 @@ u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx) union dynamic_sa_contents cts; if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; + cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents.w; else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; + cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents.w; offset = cts.bf.key_size + cts.bf.inner_size + cts.bf.outer_size @@ -66,9 +66,9 @@ u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx) union dynamic_sa_contents cts; if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; + cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents.w; else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; + cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents.w; return (cts.bf.iv0 + cts.bf.iv1 + cts.bf.iv2 + cts.bf.iv3) * 4; } @@ -77,9 +77,9 @@ u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx) union dynamic_sa_contents cts; if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; + cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents.w; else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; + cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents.w; return sizeof(struct dynamic_sa_ctl); } diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 1352d58d4e34..537d548a20b7 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -169,7 +169,7 @@ union sa_command_1 { } __attribute__((packed)); struct dynamic_sa_ctl { - u32 sa_contents; + union dynamic_sa_contents sa_contents; union sa_command_0 sa_command_0; union sa_command_1 sa_command_1; } __attribute__((packed)); -- cgit v1.2.3-59-g8ed1b From 249c8d98ea339325dca481d5dae93686cd494059 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:20 +0200 Subject: crypto: crypto4xx - move and refactor dynamic_contents helpers This patch refactors and moves the dynamic_contents helper functions into the crypto4xx_sa.h header file. * get_dynamic_sa_iv_size is no longer needed, as the cryptoapi provides the required IV size information as well. * refactor the function declarations to use the a pointer to the dynamic_sa_contents union, instead of the crypto4xx_ctx. * rename get_dynamic_sa_offset_key_field to get_dynamic_sa_key_field. It returns the pointer to the key directly. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/Makefile | 2 +- drivers/crypto/amcc/crypto4xx_alg.c | 20 ++++----- drivers/crypto/amcc/crypto4xx_core.h | 3 -- drivers/crypto/amcc/crypto4xx_sa.c | 85 ------------------------------------ drivers/crypto/amcc/crypto4xx_sa.h | 28 ++++++++++++ 5 files changed, 39 insertions(+), 99 deletions(-) delete mode 100644 drivers/crypto/amcc/crypto4xx_sa.c (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/Makefile b/drivers/crypto/amcc/Makefile index b95539928fdf..e33c185fc163 100644 --- a/drivers/crypto/amcc/Makefile +++ b/drivers/crypto/amcc/Makefile @@ -1,3 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o -crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx_sa.o +crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx-$(CONFIG_HW_RANDOM_PPC4XX) += crypto4xx_trng.o diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 867a9230cd3f..599b6326c3fb 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -29,8 +29,8 @@ #include #include #include "crypto4xx_reg_def.h" -#include "crypto4xx_sa.h" #include "crypto4xx_core.h" +#include "crypto4xx_sa.h" static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h, u32 save_iv, u32 ld_h, u32 ld_iv, @@ -79,8 +79,8 @@ int crypto4xx_encrypt(struct ablkcipher_request *req) ctx->pd_ctl = 0x1; return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - get_dynamic_sa_iv_size(ctx)); + req->nbytes, req->info, + crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req))); } int crypto4xx_decrypt(struct ablkcipher_request *req) @@ -92,8 +92,8 @@ int crypto4xx_decrypt(struct ablkcipher_request *req) ctx->pd_ctl = 1; return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - get_dynamic_sa_iv_size(ctx)); + req->nbytes, req->info, + crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req))); } /** @@ -147,15 +147,15 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx), + crypto4xx_memcpy_le(get_dynamic_sa_key_field(sa), key, keylen); sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); sa->sa_command_1.bf.key_len = keylen >> 3; ctx->is_hash = 0; ctx->direction = DIR_INBOUND; - memcpy(ctx->sa_in + get_dynamic_sa_offset_state_ptr_field(ctx), - (void *)&ctx->state_record_dma_addr, 4); - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx); + memcpy(sa + get_dynamic_sa_offset_state_ptr_field(sa), + (void *)&ctx->state_record_dma_addr, 4); + ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); sa = (struct dynamic_sa_ctl *) ctx->sa_out; @@ -225,7 +225,7 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest)); memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest)); sa_in->state_ptr = ctx->state_record_dma_addr; - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx); + ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); return 0; } diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 995f3da91cb2..e3b822907197 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -161,9 +161,6 @@ int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); -u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx); -u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx); -u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx); void crypto4xx_memcpy_le(unsigned int *dst, const unsigned char *buf, int len); u32 crypto4xx_build_pd(struct crypto_async_request *req, diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c deleted file mode 100644 index 78a8ca192d0d..000000000000 --- a/drivers/crypto/amcc/crypto4xx_sa.c +++ /dev/null @@ -1,85 +0,0 @@ -/** - * AMCC SoC PPC4xx Crypto Driver - * - * Copyright (c) 2008 Applied Micro Circuits Corporation. - * All rights reserved. James Hsiao - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * @file crypto4xx_sa.c - * - * This file implements the security context - * associate format. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "crypto4xx_reg_def.h" -#include "crypto4xx_sa.h" -#include "crypto4xx_core.h" - -u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx) -{ - u32 offset; - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents.w; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents.w; - offset = cts.bf.key_size - + cts.bf.inner_size - + cts.bf.outer_size - + cts.bf.spi - + cts.bf.seq_num0 - + cts.bf.seq_num1 - + cts.bf.seq_num_mask0 - + cts.bf.seq_num_mask1 - + cts.bf.seq_num_mask2 - + cts.bf.seq_num_mask3 - + cts.bf.iv0 - + cts.bf.iv1 - + cts.bf.iv2 - + cts.bf.iv3; - - return sizeof(struct dynamic_sa_ctl) + offset * 4; -} - -u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx) -{ - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents.w; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents.w; - return (cts.bf.iv0 + cts.bf.iv1 + cts.bf.iv2 + cts.bf.iv3) * 4; -} - -u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx) -{ - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents.w; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents.w; - - return sizeof(struct dynamic_sa_ctl); -} diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 537d548a20b7..3c3dfcaf4e0f 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -240,4 +240,32 @@ struct dynamic_sa_hash160 { #define SA_HASH160_LEN (sizeof(struct dynamic_sa_hash160)/4) #define SA_HASH160_CONTENTS 0x2000a502 +static inline u32 +get_dynamic_sa_offset_state_ptr_field(struct dynamic_sa_ctl *cts) +{ + u32 offset; + + offset = cts->sa_contents.bf.key_size + + cts->sa_contents.bf.inner_size + + cts->sa_contents.bf.outer_size + + cts->sa_contents.bf.spi + + cts->sa_contents.bf.seq_num0 + + cts->sa_contents.bf.seq_num1 + + cts->sa_contents.bf.seq_num_mask0 + + cts->sa_contents.bf.seq_num_mask1 + + cts->sa_contents.bf.seq_num_mask2 + + cts->sa_contents.bf.seq_num_mask3 + + cts->sa_contents.bf.iv0 + + cts->sa_contents.bf.iv1 + + cts->sa_contents.bf.iv2 + + cts->sa_contents.bf.iv3; + + return sizeof(struct dynamic_sa_ctl) + offset * 4; +} + +static inline u8 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) +{ + return (u8 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); +} + #endif -- cgit v1.2.3-59-g8ed1b From f2a13e7cba9e2b16f4888fbd9cf2bc25b95945be Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:21 +0200 Subject: crypto: crypto4xx - enable AES RFC3686, ECB, CFB and OFB offloads The crypto engine supports more than just aes-cbc. This patch enables the remaining AES block cipher modes that pass the testmanager's test vectors. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 66 ++++++++++++++++++++++++ drivers/crypto/amcc/crypto4xx_core.c | 98 ++++++++++++++++++++++++++++++++++++ drivers/crypto/amcc/crypto4xx_core.h | 10 ++++ drivers/crypto/amcc/crypto4xx_sa.h | 3 ++ 4 files changed, 177 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 599b6326c3fb..c9597824a515 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" #include "crypto4xx_sa.h" @@ -171,6 +172,71 @@ int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, CRYPTO_FEEDBACK_MODE_NO_FB); } +int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB, + CRYPTO_FEEDBACK_MODE_128BIT_CFB); +} + +int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB, + CRYPTO_FEEDBACK_MODE_NO_FB); +} + +int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB, + CRYPTO_FEEDBACK_MODE_64BIT_OFB); +} + +int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + int rc; + + rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE, + CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB); + if (rc) + return rc; + + memcpy(ctx->state_record, + key + keylen - CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); + + return 0; +} + +int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + __be32 iv[AES_IV_SIZE / 4] = { *(u32 *)ctx->state_record, + *(u32 *) req->info, *(u32 *) (req->info + 4), cpu_to_be32(1) }; + + ctx->direction = DIR_OUTBOUND; + ctx->pd_ctl = 1; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->nbytes, iv, AES_IV_SIZE); +} + +int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + __be32 iv[AES_IV_SIZE / 4] = { *(u32 *)ctx->state_record, + *(u32 *) req->info, *(u32 *) (req->info + 4), cpu_to_be32(1) }; + + ctx->direction = DIR_INBOUND; + ctx->pd_ctl = 1; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->nbytes, iv, AES_IV_SIZE); +} + /** * HASH SHA1 Functions */ diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 9e96af725b31..3c086cf7b052 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" @@ -1133,6 +1134,103 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { } } }}, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "cfb(aes)", + .cra_driver_name = "cfb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_alg_init, + .cra_exit = crypto4xx_alg_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_cfb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_alg_init, + .cra_exit = crypto4xx_alg_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = crypto4xx_setkey_rfc3686, + .encrypt = crypto4xx_rfc3686_encrypt, + .decrypt = crypto4xx_rfc3686_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_alg_init, + .cra_exit = crypto4xx_alg_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = crypto4xx_setkey_aes_ecb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "ofb(aes)", + .cra_driver_name = "ofb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_alg_init, + .cra_exit = crypto4xx_alg_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_cbc, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, }; /** diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index e3b822907197..f886b8bdc868 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -171,8 +171,18 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, void *iv, u32 iv_len); int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); int crypto4xx_encrypt(struct ablkcipher_request *req); int crypto4xx_decrypt(struct ablkcipher_request *req); +int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req); +int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req); int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); int crypto4xx_hash_digest(struct ahash_request *req); int crypto4xx_hash_final(struct ahash_request *req); diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 3c3dfcaf4e0f..7cc04f1ff8a0 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -112,6 +112,9 @@ union sa_command_0 { #define CRYPTO_MODE_ECB 0 #define CRYPTO_MODE_CBC 1 +#define CRYPTO_MODE_OFB 2 +#define CRYPTO_MODE_CFB 3 +#define CRYPTO_MODE_CTR 4 #define CRYPTO_FEEDBACK_MODE_NO_FB 0 #define CRYPTO_FEEDBACK_MODE_64BIT_OFB 0 -- cgit v1.2.3-59-g8ed1b From 5c727f92ea5e019fd216f73009eee2b6e0867726 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:22 +0200 Subject: crypto: crypto4xx - refactor crypto4xx_copy_pkt_to_dst() This patch refactors the crypto4xx_copy_pkt_to_dst() to use scatterwalk_map_and_copy() to copy the processed data between the crypto engine's scatter ring buffer and the destination specified by the ablkcipher_request. This also makes the crypto4xx_fill_one_page() function redundant. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 126 +++++++++-------------------------- 1 file changed, 30 insertions(+), 96 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 3c086cf7b052..f3f151820373 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" #include "crypto4xx_sa.h" @@ -481,111 +482,44 @@ static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, return (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx); } -static u32 crypto4xx_fill_one_page(struct crypto4xx_device *dev, - dma_addr_t *addr, u32 *length, - u32 *idx, u32 *offset, u32 *nbytes) -{ - u32 len; - - if (*length > dev->scatter_buffer_size) { - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, - dev->scatter_buffer_size); - *offset = 0; - *length -= dev->scatter_buffer_size; - *nbytes -= dev->scatter_buffer_size; - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - *addr = *addr + dev->scatter_buffer_size; - return 1; - } else if (*length < dev->scatter_buffer_size) { - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, *length); - if ((*offset + *length) == dev->scatter_buffer_size) { - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - *nbytes -= *length; - *offset = 0; - } else { - *nbytes -= *length; - *offset += *length; - } - - return 0; - } else { - len = (*nbytes <= dev->scatter_buffer_size) ? - (*nbytes) : dev->scatter_buffer_size; - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, - len); - *offset = 0; - *nbytes -= len; - - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - - return 0; - } -} - static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, struct ce_pd *pd, struct pd_uinfo *pd_uinfo, u32 nbytes, struct scatterlist *dst) { - dma_addr_t addr; - u32 this_sd; - u32 offset; - u32 len; - u32 i; - u32 sg_len; - struct scatterlist *sg; + unsigned int first_sd = pd_uinfo->first_sd; + unsigned int last_sd; + unsigned int overflow = 0; + unsigned int to_copy; + unsigned int dst_start = 0; + + /* + * Because the scatter buffers are all neatly organized in one + * big continuous ringbuffer; scatterwalk_map_and_copy() can + * be instructed to copy a range of buffers in one go. + */ - this_sd = pd_uinfo->first_sd; - offset = 0; - i = 0; + last_sd = (first_sd + pd_uinfo->num_sd); + if (last_sd > PPC4XX_LAST_SD) { + last_sd = PPC4XX_LAST_SD; + overflow = last_sd % PPC4XX_NUM_SD; + } while (nbytes) { - sg = &dst[i]; - sg_len = sg->length; - addr = dma_map_page(dev->core_dev->device, sg_page(sg), - sg->offset, sg->length, DMA_TO_DEVICE); - - if (offset == 0) { - len = (nbytes <= sg->length) ? nbytes : sg->length; - while (crypto4xx_fill_one_page(dev, &addr, &len, - &this_sd, &offset, &nbytes)) - ; - if (!nbytes) - return; - i++; - } else { - len = (nbytes <= (dev->scatter_buffer_size - offset)) ? - nbytes : (dev->scatter_buffer_size - offset); - len = (sg->length < len) ? sg->length : len; - while (crypto4xx_fill_one_page(dev, &addr, &len, - &this_sd, &offset, &nbytes)) - ; - if (!nbytes) - return; - sg_len -= len; - if (sg_len) { - addr += len; - while (crypto4xx_fill_one_page(dev, &addr, - &sg_len, &this_sd, &offset, &nbytes)) - ; - } - i++; + void *buf = dev->scatter_buffer_va + + first_sd * PPC4XX_SD_BUFFER_SIZE; + + to_copy = min(nbytes, PPC4XX_SD_BUFFER_SIZE * + (1 + last_sd - first_sd)); + scatterwalk_map_and_copy(buf, dst, dst_start, to_copy, 1); + nbytes -= to_copy; + + if (overflow) { + first_sd = 0; + last_sd = overflow; + dst_start += to_copy; + overflow = 0; } } } -- cgit v1.2.3-59-g8ed1b From 40e3b847bff70edc28c5290d209e531da6f9e534 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:23 +0200 Subject: crypto: crypto4xx - replace crypto4xx_dev's scatter_buffer_size with constant scatter_buffer_size is always set to PPC4XX_SD_BUFFER_SIZE. I don't think there's any point in keeping the variable around. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 7 +++---- drivers/crypto/amcc/crypto4xx_core.h | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index f3f151820373..c47c84f7492a 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -394,10 +394,9 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) if (!dev->sdr) return -ENOMEM; - dev->scatter_buffer_size = PPC4XX_SD_BUFFER_SIZE; dev->scatter_buffer_va = dma_alloc_coherent(dev->core_dev->device, - dev->scatter_buffer_size * PPC4XX_NUM_SD, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, &dev->scatter_buffer_pa, GFP_ATOMIC); if (!dev->scatter_buffer_va) { dma_free_coherent(dev->core_dev->device, @@ -410,7 +409,7 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) for (i = 0; i < PPC4XX_NUM_SD; i++) { sd_array[i].ptr = dev->scatter_buffer_pa + - dev->scatter_buffer_size * i; + PPC4XX_SD_BUFFER_SIZE * i; } return 0; @@ -425,7 +424,7 @@ static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) if (dev->scatter_buffer_va != NULL) dma_free_coherent(dev->core_dev->device, - dev->scatter_buffer_size * PPC4XX_NUM_SD, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, dev->scatter_buffer_va, dev->scatter_buffer_pa); } diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index f886b8bdc868..dc8452be1003 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -87,7 +87,6 @@ struct crypto4xx_device { program ce sdr_base_register */ void *scatter_buffer_va; dma_addr_t scatter_buffer_pa; - u32 scatter_buffer_size; void *shadow_sa_pool; /* pool of memory for sa in pd_uinfo */ dma_addr_t shadow_sa_pool_pa; -- cgit v1.2.3-59-g8ed1b From 5d59ad6eea82ef8df92b4109615a0dde9d8093e9 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:24 +0200 Subject: crypto: crypto4xx - fix crypto4xx_build_pdr, crypto4xx_build_sdr leak If one of the later memory allocations in rypto4xx_build_pdr() fails: dev->pdr (and/or) dev->pdr_uinfo wouldn't be freed. crypto4xx_build_sdr() has the same issue with dev->sdr. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index c47c84f7492a..817c44703a07 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -209,7 +209,7 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) dev->pdr_pa); return -ENOMEM; } - memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); + memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD, &dev->shadow_sa_pool_pa, @@ -242,13 +242,15 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) { - if (dev->pdr != NULL) + if (dev->pdr) dma_free_coherent(dev->core_dev->device, sizeof(struct ce_pd) * PPC4XX_NUM_PD, dev->pdr, dev->pdr_pa); + if (dev->shadow_sa_pool) dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD, dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + if (dev->shadow_sr_pool) dma_free_coherent(dev->core_dev->device, sizeof(struct sa_state_record) * PPC4XX_NUM_PD, @@ -417,12 +419,12 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) { - if (dev->sdr != NULL) + if (dev->sdr) dma_free_coherent(dev->core_dev->device, sizeof(struct ce_sd) * PPC4XX_NUM_SD, dev->sdr, dev->sdr_pa); - if (dev->scatter_buffer_va != NULL) + if (dev->scatter_buffer_va) dma_free_coherent(dev->core_dev->device, PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, dev->scatter_buffer_va, @@ -1223,7 +1225,7 @@ static int crypto4xx_probe(struct platform_device *ofdev) rc = crypto4xx_build_gdr(core_dev->dev); if (rc) - goto err_build_gdr; + goto err_build_pdr; rc = crypto4xx_build_sdr(core_dev->dev); if (rc) @@ -1266,12 +1268,11 @@ err_iomap: err_request_irq: irq_dispose_mapping(core_dev->irq); tasklet_kill(&core_dev->tasklet); - crypto4xx_destroy_sdr(core_dev->dev); err_build_sdr: + crypto4xx_destroy_sdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); -err_build_gdr: - crypto4xx_destroy_pdr(core_dev->dev); err_build_pdr: + crypto4xx_destroy_pdr(core_dev->dev); kfree(core_dev->dev); err_alloc_dev: kfree(core_dev); -- cgit v1.2.3-59-g8ed1b From 9e0a0b3a192af20193f074ed2ad9dd85a2e48d00 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 25 Aug 2017 15:47:25 +0200 Subject: crypto: crypto4xx - pointer arithmetic overhaul This patch improves the readability of various functions, by replacing various void* pointers declarations with their respective structs *. This makes it possible to go for the eye-friendly array-indexing methods. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 26 ++++++++-------- drivers/crypto/amcc/crypto4xx_core.c | 60 +++++++++++++++--------------------- drivers/crypto/amcc/crypto4xx_core.h | 41 +++++++++++++----------- 3 files changed, 59 insertions(+), 68 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index c9597824a515..d08e4c94abed 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -134,7 +134,7 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, } } /* Setup SA */ - sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa = ctx->sa_in; set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, @@ -159,7 +159,7 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); - sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa = ctx->sa_out; sa->sa_command_0.bf.dir = DIR_OUTBOUND; return 0; @@ -248,8 +248,7 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, struct crypto_alg *alg = tfm->__crt_alg; struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - struct dynamic_sa_ctl *sa; - struct dynamic_sa_hash160 *sa_in; + struct dynamic_sa_hash160 *sa; int rc; ctx->dev = my_alg->dev; @@ -273,25 +272,24 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct crypto4xx_ctx)); - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + sa = (struct dynamic_sa_hash160 *)ctx->sa_in; + set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV, SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA, SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL, SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, SA_OPCODE_HASH, DIR_INBOUND); - set_dynamic_sa_command_1(sa, 0, SA_HASH_MODE_HASH, + set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH, CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); ctx->direction = DIR_INBOUND; - sa->sa_contents.w = SA_HASH160_CONTENTS; - sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in; /* Need to zero hash digest in SA */ - memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest)); - memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest)); - sa_in->state_ptr = ctx->state_record_dma_addr; - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); + memset(sa->inner_digest, 0, sizeof(sa->inner_digest)); + memset(sa->outer_digest, 0, sizeof(sa->outer_digest)); + sa->state_ptr = ctx->state_record_dma_addr; + ctx->offset_to_sr_ptr = + get_dynamic_sa_offset_state_ptr_field(&sa->ctrl); return 0; } @@ -302,7 +300,7 @@ int crypto4xx_hash_init(struct ahash_request *req) int ds; struct dynamic_sa_ctl *sa; - sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa = ctx->sa_in; ds = crypto_ahash_digestsize( __crypto_ahash_cast(req->base.tfm)); sa->sa_command_0.bf.digest_len = ds >> 2; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 817c44703a07..773e5faebc47 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -211,7 +211,7 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) } memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, - 256 * PPC4XX_NUM_PD, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, &dev->shadow_sa_pool_pa, GFP_ATOMIC); if (!dev->shadow_sa_pool) @@ -223,16 +223,14 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) if (!dev->shadow_sr_pool) return -ENOMEM; for (i = 0; i < PPC4XX_NUM_PD; i++) { - pd_uinfo = (struct pd_uinfo *) (dev->pdr_uinfo + - sizeof(struct pd_uinfo) * i); + pd_uinfo = &dev->pdr_uinfo[i]; /* alloc 256 bytes which is enough for any kind of dynamic sa */ - pd_uinfo->sa_va = dev->shadow_sa_pool + 256 * i; + pd_uinfo->sa_va = &dev->shadow_sa_pool[i].sa; pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i; /* alloc state record */ - pd_uinfo->sr_va = dev->shadow_sr_pool + - sizeof(struct sa_state_record) * i; + pd_uinfo->sr_va = &dev->shadow_sr_pool[i]; pd_uinfo->sr_pa = dev->shadow_sr_pool_pa + sizeof(struct sa_state_record) * i; } @@ -248,8 +246,9 @@ static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) dev->pdr, dev->pdr_pa); if (dev->shadow_sa_pool) - dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD, - dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + dma_free_coherent(dev->core_dev->device, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, + dev->shadow_sa_pool, dev->shadow_sa_pool_pa); if (dev->shadow_sr_pool) dma_free_coherent(dev->core_dev->device, @@ -277,11 +276,9 @@ static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) { - struct pd_uinfo *pd_uinfo; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; unsigned long flags; - pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + - sizeof(struct pd_uinfo) * idx); spin_lock_irqsave(&dev->core_dev->lock, flags); if (dev->pdr_tail != PPC4XX_LAST_PD) dev->pdr_tail++; @@ -298,7 +295,7 @@ static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev, { *pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx; - return dev->pdr + sizeof(struct ce_pd) * idx; + return &dev->pdr[idx]; } /** @@ -376,7 +373,7 @@ static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, { *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx; - return (struct ce_gd *) (dev->gdr + sizeof(struct ce_gd) * idx); + return &dev->gdr[idx]; } /** @@ -387,7 +384,6 @@ static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) { int i; - struct ce_sd *sd_array; /* alloc memory for scatter descriptor ring */ dev->sdr = dma_alloc_coherent(dev->core_dev->device, @@ -407,10 +403,8 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) return -ENOMEM; } - sd_array = dev->sdr; - for (i = 0; i < PPC4XX_NUM_SD; i++) { - sd_array[i].ptr = dev->scatter_buffer_pa + + dev->sdr[i].ptr = dev->scatter_buffer_pa + PPC4XX_SD_BUFFER_SIZE * i; } @@ -480,7 +474,7 @@ static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, { *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx; - return (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx); + return &dev->sdr[idx]; } static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, @@ -529,11 +523,10 @@ static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo, struct crypto4xx_ctx *ctx) { struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; - struct sa_state_record *state_record = - (struct sa_state_record *) pd_uinfo->sr_va; if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { - memcpy((void *) pd_uinfo->dest_va, state_record->save_digest, + memcpy((void *) pd_uinfo->dest_va, + pd_uinfo->sr_va->save_digest, SA_HASH_ALG_SHA1_DIGEST_SIZE); } @@ -607,11 +600,9 @@ static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) { - struct ce_pd *pd; - struct pd_uinfo *pd_uinfo; + struct ce_pd *pd = &dev->pdr[idx]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; - pd = dev->pdr + sizeof(struct ce_pd)*idx; - pd_uinfo = dev->pdr_uinfo + sizeof(struct pd_uinfo)*idx; if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) == CRYPTO_ALG_TYPE_ABLKCIPHER) return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); @@ -712,7 +703,6 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, unsigned long flags; struct pd_uinfo *pd_uinfo = NULL; unsigned int nbytes = datalen, idx; - unsigned int ivlen = 0; u32 gd_idx = 0; /* figure how many gd is needed */ @@ -771,17 +761,15 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, } spin_unlock_irqrestore(&dev->core_dev->lock, flags); - pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + - sizeof(struct pd_uinfo) * pd_entry); + pd_uinfo = &dev->pdr_uinfo[pd_entry]; pd = crypto4xx_get_pdp(dev, &pd_dma, pd_entry); pd_uinfo->async_req = req; pd_uinfo->num_gd = num_gd; pd_uinfo->num_sd = num_sd; if (iv_len || ctx->is_hash) { - ivlen = iv_len; pd->sa = pd_uinfo->sa_pa; - sa = (struct dynamic_sa_ctl *) pd_uinfo->sa_va; + sa = pd_uinfo->sa_va; if (ctx->direction == DIR_INBOUND) memcpy(sa, ctx->sa_in, ctx->sa_len * 4); else @@ -791,14 +779,15 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, &pd_uinfo->sr_pa, 4); if (iv_len) - crypto4xx_memcpy_le(pd_uinfo->sr_va, iv, iv_len); + crypto4xx_memcpy_le(pd_uinfo->sr_va->save_iv, + iv, iv_len); } else { if (ctx->direction == DIR_INBOUND) { pd->sa = ctx->sa_in_dma_addr; - sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa = ctx->sa_in; } else { pd->sa = ctx->sa_out_dma_addr; - sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa = ctx->sa_out; } } pd->sa_len = ctx->sa_len; @@ -1006,9 +995,8 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) { tail = core_dev->dev->pdr_tail; - pd_uinfo = core_dev->dev->pdr_uinfo + - sizeof(struct pd_uinfo)*tail; - pd = core_dev->dev->pdr + sizeof(struct ce_pd) * tail; + pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; + pd = &core_dev->dev->pdr[tail]; if ((pd_uinfo->state == PD_ENTRY_INUSE) && pd->pd_ctl.bf.pe_done && !pd->pd_ctl.bf.host_ready) { diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index dc8452be1003..97fb8288ab30 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -23,6 +23,8 @@ #define __CRYPTO4XX_CORE_H__ #include +#include "crypto4xx_reg_def.h" +#include "crypto4xx_sa.h" #define MODULE_NAME "crypto4xx" @@ -48,6 +50,13 @@ struct crypto4xx_device; +union shadow_sa_buf { + struct dynamic_sa_ctl sa; + + /* alloc 256 bytes which is enough for any kind of dynamic sa */ + u8 buf[256]; +} __packed; + struct pd_uinfo { struct crypto4xx_device *dev; u32 state; @@ -60,9 +69,9 @@ struct pd_uinfo { used by this packet */ u32 num_sd; /* number of scatter discriptors used by this packet */ - void *sa_va; /* shadow sa, when using cp from ctx->sa */ + struct dynamic_sa_ctl *sa_va; /* shadow sa */ u32 sa_pa; - void *sr_va; /* state record for shadow sa */ + struct sa_state_record *sr_va; /* state record for shadow sa */ u32 sr_pa; struct scatterlist *dest_va; struct crypto_async_request *async_req; /* base crypto request @@ -75,22 +84,18 @@ struct crypto4xx_device { void __iomem *ce_base; void __iomem *trng_base; - void *pdr; /* base address of packet - descriptor ring */ - dma_addr_t pdr_pa; /* physical address used to - program ce pdr_base_register */ - void *gdr; /* gather descriptor ring */ - dma_addr_t gdr_pa; /* physical address used to - program ce gdr_base_register */ - void *sdr; /* scatter descriptor ring */ - dma_addr_t sdr_pa; /* physical address used to - program ce sdr_base_register */ + struct ce_pd *pdr; /* base address of packet descriptor ring */ + dma_addr_t pdr_pa; /* physical address of pdr_base_register */ + struct ce_gd *gdr; /* gather descriptor ring */ + dma_addr_t gdr_pa; /* physical address of gdr_base_register */ + struct ce_sd *sdr; /* scatter descriptor ring */ + dma_addr_t sdr_pa; /* physical address of sdr_base_register */ void *scatter_buffer_va; dma_addr_t scatter_buffer_pa; - void *shadow_sa_pool; /* pool of memory for sa in pd_uinfo */ + union shadow_sa_buf *shadow_sa_pool; dma_addr_t shadow_sa_pool_pa; - void *shadow_sr_pool; /* pool of memory for sr in pd_uinfo */ + struct sa_state_record *shadow_sr_pool; dma_addr_t shadow_sr_pool_pa; u32 pdr_tail; u32 pdr_head; @@ -98,7 +103,7 @@ struct crypto4xx_device { u32 gdr_head; u32 sdr_tail; u32 sdr_head; - void *pdr_uinfo; + struct pd_uinfo *pdr_uinfo; struct list_head alg_list; /* List of algorithm supported by this device */ }; @@ -116,11 +121,11 @@ struct crypto4xx_core_device { struct crypto4xx_ctx { struct crypto4xx_device *dev; - void *sa_in; + struct dynamic_sa_ctl *sa_in; dma_addr_t sa_in_dma_addr; - void *sa_out; + struct dynamic_sa_ctl *sa_out; dma_addr_t sa_out_dma_addr; - void *state_record; + struct sa_state_record *state_record; dma_addr_t state_record_dma_addr; u32 sa_len; u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */ -- cgit v1.2.3-59-g8ed1b From d989364535803dfcd97e3b6832d9698109add6ff Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Fri, 25 Aug 2017 23:53:42 +0530 Subject: crypto: padlock-aes - constify x86_cpu_id x86_cpu_id are not supposed to change at runtime. MODULE_DEVICE_TABLE and x86_match_cpu are working with const x86_cpu_id. So mark the non-const x86_cpu_id structs as const. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/padlock-aes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index b3869748cc6b..4b6642a25df5 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -482,7 +482,7 @@ static struct crypto_alg cbc_aes_alg = { } }; -static struct x86_cpu_id padlock_cpu_id[] = { +static const struct x86_cpu_id padlock_cpu_id[] = { X86_FEATURE_MATCH(X86_FEATURE_XCRYPT), {} }; -- cgit v1.2.3-59-g8ed1b From 16d5cee5cf65ff704e59a1e70fa82bf798b5e75e Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Fri, 25 Aug 2017 23:53:59 +0530 Subject: crypto: padlock-sha - constify x86_cpu_id x86_cpu_id are not supposed to change at runtime. MODULE_DEVICE_TABLE and x86_match_cpu are working with const x86_cpu_id. So mark the non-const x86_cpu_id structs as const. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/padlock-sha.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index bc72d20c32c3..d32c79328876 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -509,7 +509,7 @@ static struct shash_alg sha256_alg_nano = { } }; -static struct x86_cpu_id padlock_sha_ids[] = { +static const struct x86_cpu_id padlock_sha_ids[] = { X86_FEATURE_MATCH(X86_FEATURE_PHE), {} }; -- cgit v1.2.3-59-g8ed1b From b8d3de85acb4434fb7fd5e7dee12204a55ff8fc2 Mon Sep 17 00:00:00 2001 From: Himanshu Jha Date: Sun, 27 Aug 2017 02:45:29 +0530 Subject: crypto: n2 - remove null check before kfree kfree on NULL pointer is a no-op and therefore checking it is redundant. Signed-off-by: Himanshu Jha Acked-by: David S. Miller Signed-off-by: Herbert Xu --- drivers/crypto/n2_core.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index a9fd8b9e86cd..48de52cf2ecc 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -1962,10 +1962,8 @@ static struct n2_crypto *alloc_n2cp(void) static void free_n2cp(struct n2_crypto *np) { - if (np->cwq_info.ino_table) { - kfree(np->cwq_info.ino_table); - np->cwq_info.ino_table = NULL; - } + kfree(np->cwq_info.ino_table); + np->cwq_info.ino_table = NULL; kfree(np); } @@ -2079,10 +2077,8 @@ static struct n2_mau *alloc_ncp(void) static void free_ncp(struct n2_mau *mp) { - if (mp->mau_info.ino_table) { - kfree(mp->mau_info.ino_table); - mp->mau_info.ino_table = NULL; - } + kfree(mp->mau_info.ino_table); + mp->mau_info.ino_table = NULL; kfree(mp); } -- cgit v1.2.3-59-g8ed1b From 18fddf5fd2111dd670156b3dfb2dfed16d7889b4 Mon Sep 17 00:00:00 2001 From: Himanshu Jha Date: Sun, 27 Aug 2017 02:45:30 +0530 Subject: crypto: inside-secure - remove null check before kfree Kfree on NULL pointer is a no-op and therefore checking is redundant. Signed-off-by: Himanshu Jha Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_hash.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 3980f946874f..74feb6227101 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -308,10 +308,8 @@ unmap_cache: ctx->base.cache_sz = 0; } free_cache: - if (ctx->base.cache) { - kfree(ctx->base.cache); - ctx->base.cache = NULL; - } + kfree(ctx->base.cache); + ctx->base.cache = NULL; unlock: spin_unlock_bh(&priv->ring[ring].egress_lock); -- cgit v1.2.3-59-g8ed1b From 06ec1f82bfa5078928e0efaa5ed350a1dcc91631 Mon Sep 17 00:00:00 2001 From: Himanshu Jha Date: Sun, 27 Aug 2017 11:52:48 +0530 Subject: crypto: Use PTR_ERR_ZERO Use PTR_ERROR_ZERO rather than if(IS_ERR(...)) + PTR_ERR. Build successfully. Signed-off-by: Himanshu Jha Signed-off-by: Herbert Xu --- drivers/crypto/marvell/tdma.c | 5 +---- drivers/crypto/mv_cesa.c | 5 +---- drivers/crypto/qce/ablkcipher.c | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index c76375ff376d..d0ef171c18df 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -304,10 +304,7 @@ int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) struct mv_cesa_tdma_desc *tdma; tdma = mv_cesa_dma_add_desc(chain, flags); - if (IS_ERR(tdma)) - return PTR_ERR(tdma); - - return 0; + return PTR_ERR_OR_ZERO(tdma); } int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c index bf25f415eea6..c3883b49f56e 100644 --- a/drivers/crypto/mv_cesa.c +++ b/drivers/crypto/mv_cesa.c @@ -1052,10 +1052,7 @@ static int mv_cesa_get_sram(struct platform_device *pdev, return -EINVAL; cp->sram = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(cp->sram)) - return PTR_ERR(cp->sram); - - return 0; + return PTR_ERR_OR_ZERO(cp->sram); } static int mv_probe(struct platform_device *pdev) diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index b04b42f48366..ea4d96bf47e8 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -248,10 +248,7 @@ static int qce_ablkcipher_init(struct crypto_tfm *tfm) ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(tfm), 0, CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->fallback)) - return PTR_ERR(ctx->fallback); - - return 0; + return PTR_ERR_OR_ZERO(ctx->fallback); } static void qce_ablkcipher_exit(struct crypto_tfm *tfm) -- cgit v1.2.3-59-g8ed1b From 52872f5288ea38ab9e2bb352a47b71051f7aefcc Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Mon, 28 Aug 2017 22:00:07 +0800 Subject: crypto: drop unnecessary return statements Fix checkpatch.pl warnings: WARNING: void function return statements are not generally useful FILE: crypto/rmd128.c:218: FILE: crypto/rmd160.c:261: FILE: crypto/rmd256.c:233: FILE: crypto/rmd320.c:280: FILE: crypto/tcrypt.c:385: FILE: drivers/crypto/ixp4xx_crypto.c:538: FILE: drivers/crypto/marvell/cesa.c:81: FILE: drivers/crypto/ux500/cryp/cryp_core.c:1755: Signed-off-by: Geliang Tang Signed-off-by: Herbert Xu --- crypto/rmd128.c | 2 -- crypto/rmd160.c | 2 -- crypto/rmd256.c | 2 -- crypto/rmd320.c | 2 -- crypto/tcrypt.c | 1 - drivers/crypto/ixp4xx_crypto.c | 1 - drivers/crypto/marvell/cesa.c | 2 -- drivers/crypto/ux500/cryp/cryp_core.c | 1 - 8 files changed, 13 deletions(-) (limited to 'drivers/crypto') diff --git a/crypto/rmd128.c b/crypto/rmd128.c index 049486ede938..40e053b97b69 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -213,8 +213,6 @@ static void rmd128_transform(u32 *state, const __le32 *in) state[2] = state[3] + aa + bbb; state[3] = state[0] + bb + ccc; state[0] = ddd; - - return; } static int rmd128_init(struct shash_desc *desc) diff --git a/crypto/rmd160.c b/crypto/rmd160.c index de585e51d455..5f3e6ea35268 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -256,8 +256,6 @@ static void rmd160_transform(u32 *state, const __le32 *in) state[3] = state[4] + aa + bbb; state[4] = state[0] + bb + ccc; state[0] = ddd; - - return; } static int rmd160_init(struct shash_desc *desc) diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 4ec02a754e09..f50c025cc962 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -228,8 +228,6 @@ static void rmd256_transform(u32 *state, const __le32 *in) state[5] += bbb; state[6] += ccc; state[7] += ddd; - - return; } static int rmd256_init(struct shash_desc *desc) diff --git a/crypto/rmd320.c b/crypto/rmd320.c index 770f2cb369f8..e1315e4869e8 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -275,8 +275,6 @@ static void rmd320_transform(u32 *state, const __le32 *in) state[7] += ccc; state[8] += ddd; state[9] += eee; - - return; } static int rmd320_init(struct shash_desc *desc) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index a309c819be70..a371c072b62f 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -381,7 +381,6 @@ out_noaxbuf: testmgr_free_buf(xbuf); out_noxbuf: kfree(iv); - return; } static void test_hash_sg_init(struct scatterlist *sg) diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index dadc4a808df5..8705b28eb02c 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -534,7 +534,6 @@ static void release_ixp_crypto(struct device *dev) NPE_QLEN_TOTAL * sizeof( struct crypt_ctl), crypt_virt, crypt_phys); } - return; } static void reset_sa_dir(struct ix_sa_dir *dir) diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 6e7a5c77a00a..b657e7c2107e 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -76,8 +76,6 @@ static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine) ctx = crypto_tfm_ctx(req->tfm); ctx->ops->step(req); - - return; } static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 790f7cadc1ed..765f53e548ab 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -1751,7 +1751,6 @@ static void __exit ux500_cryp_mod_fini(void) { pr_debug("[%s] is called!", __func__); platform_driver_unregister(&cryp_driver); - return; } module_init(ux500_cryp_mod_init); -- cgit v1.2.3-59-g8ed1b From f0e2ce58f853634d7ad1a418a49bd5fbd556227c Mon Sep 17 00:00:00 2001 From: raveendra padasalagi Date: Tue, 29 Aug 2017 15:08:10 +0530 Subject: crypto: brcm - Explicity ACK mailbox message Add support to explicity ACK mailbox message because after sending message we can know the send status via error attribute of brcm_message. This is needed to support "txdone_ack" supported in mailbox controller driver. Fixes: 9d12ba86f818 ("crypto: brcm - Add Broadcom SPU driver") Signed-off-by: Raveendra Padasalagi Reviewed-by: Anup Patel Reviewed-by: Scott Branden Cc: stable@vger.kernel.org Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 101 ++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 55 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 537a67483aa3..1e7924b350a7 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -256,6 +256,44 @@ spu_ablkcipher_tx_sg_create(struct brcm_message *mssg, return 0; } +static int mailbox_send_message(struct brcm_message *mssg, u32 flags, + u8 chan_idx) +{ + int err; + int retry_cnt = 0; + struct device *dev = &(iproc_priv.pdev->dev); + + err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); + if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) { + while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { + /* + * Mailbox queue is full. Since MAY_SLEEP is set, assume + * not in atomic context and we can wait and try again. + */ + retry_cnt++; + usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); + err = mbox_send_message(iproc_priv.mbox[chan_idx], + mssg); + atomic_inc(&iproc_priv.mb_no_spc); + } + } + if (err < 0) { + atomic_inc(&iproc_priv.mb_send_fail); + return err; + } + + /* Check error returned by mailbox controller */ + err = mssg->error; + if (unlikely(err < 0)) { + dev_err(dev, "message error %d", err); + /* Signal txdone for mailbox channel */ + } + + /* Signal txdone for mailbox channel */ + mbox_client_txdone(iproc_priv.mbox[chan_idx], err); + return err; +} + /** * handle_ablkcipher_req() - Submit as much of a block cipher request as fits in * a single SPU request message, starting at the current position in the request @@ -293,7 +331,6 @@ static int handle_ablkcipher_req(struct iproc_reqctx_s *rctx) u32 pad_len; /* total length of all padding */ bool update_key = false; struct brcm_message *mssg; /* mailbox message */ - int retry_cnt = 0; /* number of entries in src and dst sg in mailbox message. */ u8 rx_frag_num = 2; /* response header and STATUS */ @@ -462,24 +499,9 @@ static int handle_ablkcipher_req(struct iproc_reqctx_s *rctx) if (err) return err; - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg); - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) { - while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { - /* - * Mailbox queue is full. Since MAY_SLEEP is set, assume - * not in atomic context and we can wait and try again. - */ - retry_cnt++; - usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], - mssg); - atomic_inc(&iproc_priv.mb_no_spc); - } - } - if (unlikely(err < 0)) { - atomic_inc(&iproc_priv.mb_send_fail); + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) return err; - } return -EINPROGRESS; } @@ -710,7 +732,6 @@ static int handle_ahash_req(struct iproc_reqctx_s *rctx) u32 spu_hdr_len; unsigned int digestsize; u16 rem = 0; - int retry_cnt = 0; /* * number of entries in src and dst sg. Always includes SPU msg header. @@ -904,24 +925,10 @@ static int handle_ahash_req(struct iproc_reqctx_s *rctx) if (err) return err; - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg); - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) { - while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { - /* - * Mailbox queue is full. Since MAY_SLEEP is set, assume - * not in atomic context and we can wait and try again. - */ - retry_cnt++; - usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], - mssg); - atomic_inc(&iproc_priv.mb_no_spc); - } - } - if (err < 0) { - atomic_inc(&iproc_priv.mb_send_fail); + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) return err; - } + return -EINPROGRESS; } @@ -1320,7 +1327,6 @@ static int handle_aead_req(struct iproc_reqctx_s *rctx) int assoc_nents = 0; bool incl_icv = false; unsigned int digestsize = ctx->digestsize; - int retry_cnt = 0; /* number of entries in src and dst sg. Always includes SPU msg header. */ @@ -1558,24 +1564,9 @@ static int handle_aead_req(struct iproc_reqctx_s *rctx) if (err) return err; - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg); - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) { - while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { - /* - * Mailbox queue is full. Since MAY_SLEEP is set, assume - * not in atomic context and we can wait and try again. - */ - retry_cnt++; - usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], - mssg); - atomic_inc(&iproc_priv.mb_no_spc); - } - } - if (err < 0) { - atomic_inc(&iproc_priv.mb_send_fail); + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) return err; - } return -EINPROGRESS; } @@ -4537,7 +4528,7 @@ static int spu_mb_init(struct device *dev) mcl->dev = dev; mcl->tx_block = false; mcl->tx_tout = 0; - mcl->knows_txdone = false; + mcl->knows_txdone = true; mcl->rx_callback = spu_rx_callback; mcl->tx_done = NULL; -- cgit v1.2.3-59-g8ed1b From 01c4c097bcdbf3c3ea288d22f00f469ac139b7bc Mon Sep 17 00:00:00 2001 From: amd Date: Tue, 5 Sep 2017 17:01:51 -0500 Subject: crypto: ccp - unmap pages and remove unmap objects in callback Clean up the mapped pages and the unmap object once we are done with it. This enables the final clean-up of the object once the transfer is complete. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dmaengine.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index d608043c0280..64f4b571202a 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -223,6 +223,7 @@ static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, desc->tx_desc.cookie, desc->status); dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); } desc = __ccp_next_dma_desc(chan, desc); -- cgit v1.2.3-59-g8ed1b From c07f7c29d1c6a7c62b66299d50acd6ffd171f612 Mon Sep 17 00:00:00 2001 From: amd Date: Tue, 5 Sep 2017 17:08:14 -0500 Subject: crypto:ccp - invoke the DMA callback in a standard way Use the provided mechanism in dmaengine.h to invoke the completion callback. Signed-off-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dmaengine.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 64f4b571202a..8b9da58459df 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -231,9 +231,7 @@ static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, spin_unlock_irqrestore(&chan->lock, flags); if (tx_desc) { - if (tx_desc->callback && - (tx_desc->flags & DMA_PREP_INTERRUPT)) - tx_desc->callback(tx_desc->callback_param); + dmaengine_desc_get_callback_invoke(tx_desc, NULL); dma_run_dependencies(tx_desc); } -- cgit v1.2.3-59-g8ed1b From 2fd23f2b9016dc5391f258a462ae0a334a3c9f86 Mon Sep 17 00:00:00 2001 From: Allen Date: Wed, 13 Sep 2017 13:02:11 +0530 Subject: crypto: omap - return -ENOMEM on allocation failure. Signed-off-by: Allen Pais Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes-gcm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 9b8f1c752168..9ec598a96861 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -187,7 +187,7 @@ static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL); if (!sk_req) { pr_err("skcipher: Failed to allocate request\n"); - return -1; + return -ENOMEM; } init_completion(&result.completion); -- cgit v1.2.3-59-g8ed1b From de915688385ca9df749a3d46e824a8200f05d65e Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 20 Sep 2017 20:31:40 +0200 Subject: crypto: stm32 - use of_device_get_match_data The usage of of_device_get_match_data reduce the code size a bit. Furthermore, it prevents an improbable dereference when of_match_device() return NULL. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-hash.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index b34ee85509c2..fd13838fcff2 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -1407,11 +1407,10 @@ MODULE_DEVICE_TABLE(of, stm32_hash_of_match); static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev, struct device *dev) { - const struct of_device_id *match; int err; - match = of_match_device(stm32_hash_of_match, dev); - if (!match) { + hdev->pdata = of_device_get_match_data(dev); + if (!hdev->pdata) { dev_err(dev, "no compatible OF match\n"); return -EINVAL; } @@ -1419,8 +1418,6 @@ static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev, err = of_property_read_u32(dev->of_node, "dma-maxburst", &hdev->dma_maxburst); - hdev->pdata = match->data; - return err; } -- cgit v1.2.3-59-g8ed1b From 7d556931520e1e0df4c3e545ab8ba74046410c37 Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 20 Sep 2017 20:42:48 +0200 Subject: crypto: omap - use of_device_get_match_data The usage of of_device_get_match_data reduce the code size a bit. Furthermore, it prevents an improbable dereference when of_match_device() return NULL. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 7 ++----- drivers/crypto/omap-des.c | 7 ++----- drivers/crypto/omap-sham.c | 7 ++----- 3 files changed, 6 insertions(+), 15 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 1f3686a1ebfa..fbec0a2e76dd 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -975,11 +975,10 @@ static int omap_aes_get_res_of(struct omap_aes_dev *dd, struct device *dev, struct resource *res) { struct device_node *node = dev->of_node; - const struct of_device_id *match; int err = 0; - match = of_match_device(of_match_ptr(omap_aes_of_match), dev); - if (!match) { + dd->pdata = of_device_get_match_data(dev); + if (!dd->pdata) { dev_err(dev, "no compatible OF match\n"); err = -EINVAL; goto err; @@ -992,8 +991,6 @@ static int omap_aes_get_res_of(struct omap_aes_dev *dd, goto err; } - dd->pdata = match->data; - err: return err; } diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index d37c9506c36c..ebc5c0f11f03 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -928,16 +928,13 @@ MODULE_DEVICE_TABLE(of, omap_des_of_match); static int omap_des_get_of(struct omap_des_dev *dd, struct platform_device *pdev) { - const struct of_device_id *match; - match = of_match_device(of_match_ptr(omap_des_of_match), &pdev->dev); - if (!match) { + dd->pdata = of_device_get_match_data(&pdev->dev); + if (!dd->pdata) { dev_err(&pdev->dev, "no compatible OF match\n"); return -EINVAL; } - dd->pdata = match->data; - return 0; } #else diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index c40ac30ec002..86b89ace836f 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1944,11 +1944,10 @@ static int omap_sham_get_res_of(struct omap_sham_dev *dd, struct device *dev, struct resource *res) { struct device_node *node = dev->of_node; - const struct of_device_id *match; int err = 0; - match = of_match_device(of_match_ptr(omap_sham_of_match), dev); - if (!match) { + dd->pdata = of_device_get_match_data(dev); + if (!dd->pdata) { dev_err(dev, "no compatible OF match\n"); err = -EINVAL; goto err; @@ -1968,8 +1967,6 @@ static int omap_sham_get_res_of(struct omap_sham_dev *dd, goto err; } - dd->pdata = match->data; - err: return err; } -- cgit v1.2.3-59-g8ed1b From d9fa482e5913213c2686404238f7f9cdb14b6c6c Mon Sep 17 00:00:00 2001 From: Corentin LABBE Date: Wed, 20 Sep 2017 20:47:26 +0200 Subject: crypto: bcm - use of_device_get_match_data The usage of of_device_get_match_data reduce the code size a bit. Furthermore, it prevents an improbable dereference when of_match_device() return NULL. Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- drivers/crypto/bcm/cipher.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 1e7924b350a7..ce70b44d0fb6 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -4809,7 +4809,6 @@ static int spu_dt_read(struct platform_device *pdev) struct device *dev = &pdev->dev; struct spu_hw *spu = &iproc_priv.spu; struct resource *spu_ctrl_regs; - const struct of_device_id *match; const struct spu_type_subtype *matched_spu_type; struct device_node *dn = pdev->dev.of_node; int err, i; @@ -4817,14 +4816,12 @@ static int spu_dt_read(struct platform_device *pdev) /* Count number of mailbox channels */ spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells"); - match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev); - if (!match) { + matched_spu_type = of_device_get_match_data(dev); + if (!matched_spu_type) { dev_err(&pdev->dev, "Failed to match device\n"); return -ENODEV; } - matched_spu_type = match->data; - spu->spu_type = matched_spu_type->type; spu->spu_subtype = matched_spu_type->subtype; -- cgit v1.2.3-59-g8ed1b From bfbd6254b8e716568327dc52d16a72c52b7471b6 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Mon, 25 Sep 2017 16:06:12 +0530 Subject: crypto: omap-aes - pr_err() strings should end with newlines pr_err() messages should terminated with a new-line to avoid other messages being concatenated onto the end. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes-gcm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 9ec598a96861..0cc3b65d7162 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -215,7 +215,7 @@ static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) } /* fall through */ default: - pr_err("Encryption of IV failed for GCM mode"); + pr_err("Encryption of IV failed for GCM mode\n"); break; } -- cgit v1.2.3-59-g8ed1b From e02b8b43f55a17826b5c0cdd23c82c3fcde08e4b Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Mon, 25 Sep 2017 16:06:13 +0530 Subject: crypto: virtio - pr_err() strings should end with newlines pr_err() messages should terminated with a new-line to avoid other messages being concatenated onto the end. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/virtio/virtio_crypto_algs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index 5035b0dc1e40..abe8c15450df 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -319,7 +319,7 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm, struct virtio_crypto *vcrypto = virtcrypto_get_dev_node(node); if (!vcrypto) { - pr_err("virtio_crypto: Could not find a virtio device in the system"); + pr_err("virtio_crypto: Could not find a virtio device in the system\n"); return -ENODEV; } -- cgit v1.2.3-59-g8ed1b From dce094ea6986a83d0056a26fd24c13b1b6ce00d1 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Mon, 25 Sep 2017 16:06:14 +0530 Subject: crypto: chelsio - pr_err() strings should end with newlines pr_err() messages should terminated with a new-line to avoid other messages being concatenated onto the end. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index b6dd9cbe815f..32618bf0ab3b 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -224,7 +224,7 @@ static int chcr_uld_state_change(void *handle, enum cxgb4_state state) static int __init chcr_crypto_init(void) { if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info)) - pr_err("ULD register fail: No chcr crypto support in cxgb4"); + pr_err("ULD register fail: No chcr crypto support in cxgb4\n"); return 0; } -- cgit v1.2.3-59-g8ed1b From ed713a257a5861a86f52563582c5cadc1233d097 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Mon, 25 Sep 2017 16:06:15 +0530 Subject: crypto: qat - pr_err() strings should end with newlines pr_err() messages should terminated with a new-line to avoid other messages being concatenated onto the end. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index e2454d90d949..399604c41cdd 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -567,26 +567,26 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, code_page->imp_expr_tab_offset); if (uc_var_tab->entry_num || imp_var_tab->entry_num || imp_expr_tab->entry_num) { - pr_err("QAT: UOF can't contain imported variable to be parsed"); + pr_err("QAT: UOF can't contain imported variable to be parsed\n"); return -EINVAL; } neigh_reg_tab = (struct icp_qat_uof_objtable *) (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain shared control store feature"); + pr_err("QAT: UOF can't contain shared control store feature\n"); return -EINVAL; } if (image->numpages > 1) { - pr_err("QAT: UOF can't contain multiple pages"); + pr_err("QAT: UOF can't contain multiple pages\n"); return -EINVAL; } if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use shared control store feature"); + pr_err("QAT: UOF can't use shared control store feature\n"); return -EFAULT; } if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use reloadable feature"); + pr_err("QAT: UOF can't use reloadable feature\n"); return -EFAULT; } return 0; @@ -702,7 +702,7 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) } } if (!mflag) { - pr_err("QAT: uimage uses AE not set"); + pr_err("QAT: uimage uses AE not set\n"); return -EINVAL; } return 0; -- cgit v1.2.3-59-g8ed1b From 924c9e3629441ce17adc66694dda239970e9f625 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Mon, 25 Sep 2017 16:06:16 +0530 Subject: crypto: bcm - pr_err() strings should end with newlines pr_err() messages should terminated with a new-line to avoid other messages being concatenated onto the end. Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu --- drivers/crypto/bcm/util.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c index 430c5570ea87..d543c010ccd9 100644 --- a/drivers/crypto/bcm/util.c +++ b/drivers/crypto/bcm/util.c @@ -271,7 +271,7 @@ int do_shash(unsigned char *name, unsigned char *result, hash = crypto_alloc_shash(name, 0, 0); if (IS_ERR(hash)) { rc = PTR_ERR(hash); - pr_err("%s: Crypto %s allocation error %d", __func__, name, rc); + pr_err("%s: Crypto %s allocation error %d\n", __func__, name, rc); return rc; } @@ -279,7 +279,7 @@ int do_shash(unsigned char *name, unsigned char *result, sdesc = kmalloc(size, GFP_KERNEL); if (!sdesc) { rc = -ENOMEM; - pr_err("%s: Memory allocation failure", __func__); + pr_err("%s: Memory allocation failure\n", __func__); goto do_shash_err; } sdesc->shash.tfm = hash; @@ -288,31 +288,31 @@ int do_shash(unsigned char *name, unsigned char *result, if (key_len > 0) { rc = crypto_shash_setkey(hash, key, key_len); if (rc) { - pr_err("%s: Could not setkey %s shash", __func__, name); + pr_err("%s: Could not setkey %s shash\n", __func__, name); goto do_shash_err; } } rc = crypto_shash_init(&sdesc->shash); if (rc) { - pr_err("%s: Could not init %s shash", __func__, name); + pr_err("%s: Could not init %s shash\n", __func__, name); goto do_shash_err; } rc = crypto_shash_update(&sdesc->shash, data1, data1_len); if (rc) { - pr_err("%s: Could not update1", __func__); + pr_err("%s: Could not update1\n", __func__); goto do_shash_err; } if (data2 && data2_len) { rc = crypto_shash_update(&sdesc->shash, data2, data2_len); if (rc) { - pr_err("%s: Could not update2", __func__); + pr_err("%s: Could not update2\n", __func__); goto do_shash_err; } } rc = crypto_shash_final(&sdesc->shash, result); if (rc) - pr_err("%s: Could not generate %s hash", __func__, name); + pr_err("%s: Could not generate %s hash\n", __func__, name); do_shash_err: crypto_free_shash(hash); -- cgit v1.2.3-59-g8ed1b From 5a4326d3a03f03c2518a2c255be33a7114af3230 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:05 +0200 Subject: crypto: crypto4xx - wire up hmac_mc to hmac_muting The hmac_mc parameter of set_dynamic_sa_command_1() was defined but not used. On closer inspection it turns out, it was never wired up. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index d08e4c94abed..57b1dcef4cb4 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -63,6 +63,7 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, sa->sa_command_1.bf.crypto_mode9_8 = cm & 3; sa->sa_command_1.bf.feedback_mode = cfb, sa->sa_command_1.bf.sa_rev = 1; + sa->sa_command_1.bf.hmac_muting = hmac_mc; sa->sa_command_1.bf.extended_seq_num = esn; sa->sa_command_1.bf.seq_num_mask = sn_mask; sa->sa_command_1.bf.mutable_bit_proc = mute; -- cgit v1.2.3-59-g8ed1b From e9b8e4e1129d0886094cfe013cdbaafc4ce0de76 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:06 +0200 Subject: crypto: crypto4xx - fix off-by-one AES-OFB I used aes-cbc as a template for ofb. But sadly I forgot to update set_key method to crypto4xx_setkey_aes_ofb(). this was caught by the testmgr: alg: skcipher: Test 1 failed (invalid result) on encr. for ofb-aes-ppc4xx 00000000: 76 49 ab ac 81 19 b2 46 ce e9 8e 9b 12 e9 19 7d 00000010: 50 86 cb 9b 50 72 19 ee 95 db 11 3a 91 76 78 b2 00000020: 73 be d6 b8 e3 c1 74 3b 71 16 e6 9e 22 22 95 16 00000030: 3f f1 ca a1 68 1f ac 09 12 0e ca 30 75 86 e1 a7 With the correct set_key method, the aes-ofb cipher passes the test. name : ofb(aes) driver : ofb-aes-ppc4xx module : crypto4xx priority : 300 refcnt : 1 selftest : passed internal : no type : ablkcipher async : yes blocksize : 16 min keysize : 16 max keysize : 32 ivsize : 16 geniv : Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 773e5faebc47..cb45365166ae 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1148,7 +1148,7 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, - .setkey = crypto4xx_setkey_aes_cbc, + .setkey = crypto4xx_setkey_aes_ofb, .encrypt = crypto4xx_encrypt, .decrypt = crypto4xx_decrypt, } -- cgit v1.2.3-59-g8ed1b From 333eb3edda3842f3e5dbd723cb18bbe47eb0508b Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:07 +0200 Subject: crypto: crypto4xx - fix type mismatch compiler error This patch fixes a type mismatch error that I accidentally introduced when I moved and refactored the dynamic_contents helpers. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_sa.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 7cc04f1ff8a0..8040c82dc354 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -266,9 +266,9 @@ get_dynamic_sa_offset_state_ptr_field(struct dynamic_sa_ctl *cts) return sizeof(struct dynamic_sa_ctl) + offset * 4; } -static inline u8 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) +static inline u32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) { - return (u8 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); + return (u32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); } #endif -- cgit v1.2.3-59-g8ed1b From 778f81d6cdb7d25360f082ac0384d5103f04eca5 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:08 +0200 Subject: crypto: crypto4xx - increase context and scatter ring buffer elements If crypto4xx is used in conjunction with dm-crypt, the available ring buffer elements are not enough to handle the load properly. On an aes-cbc-essiv:sha256 encrypted swap partition the read performance is abyssal: (tested with hdparm -t) /dev/mapper/swap_crypt: Timing buffered disk reads: 14 MB in 3.68 seconds = 3.81 MB/sec The patch increases both PPC4XX_NUM_SD and PPC4XX_NUM_PD to 256. This improves the performance considerably: /dev/mapper/swap_crypt: Timing buffered disk reads: 104 MB in 3.03 seconds = 34.31 MB/sec Furthermore, PPC4XX_LAST_SD, PPC4XX_LAST_GD and PPC4XX_LAST_PD can be easily calculated from their respective PPC4XX_NUM_* constant. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 97fb8288ab30..27e439c1f5bf 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -36,12 +36,12 @@ #define PPC405EX_CE_RESET 0x00000008 #define CRYPTO4XX_CRYPTO_PRIORITY 300 -#define PPC4XX_LAST_PD 63 -#define PPC4XX_NUM_PD 64 -#define PPC4XX_LAST_GD 1023 +#define PPC4XX_NUM_PD 256 +#define PPC4XX_LAST_PD (PPC4XX_NUM_PD - 1) #define PPC4XX_NUM_GD 1024 -#define PPC4XX_LAST_SD 63 -#define PPC4XX_NUM_SD 64 +#define PPC4XX_LAST_GD (PPC4XX_NUM_GD - 1) +#define PPC4XX_NUM_SD 256 +#define PPC4XX_LAST_SD (PPC4XX_NUM_SD - 1) #define PPC4XX_SD_BUFFER_SIZE 2048 #define PD_ENTRY_INUSE 1 -- cgit v1.2.3-59-g8ed1b From 8ef8d195430ca3542d0434cf25e5115484b9fa32 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:09 +0200 Subject: crypto: crypto4xx - add backlog queue support Previously, If the crypto4xx driver used all available security contexts, it would simply refuse new requests with -EAGAIN. CRYPTO_TFM_REQ_MAY_BACKLOG was ignored. in case of dm-crypt.c's crypt_convert() function this was causing the following errors to manifest, if the system was pushed hard enough: | EXT4-fs warning (dm-1): ext4_end_bio:314: I/O error -5 writing to ino .. | EXT4-fs warning (dm-1): ext4_end_bio:314: I/O error -5 writing to ino .. | EXT4-fs warning (dm-1): ext4_end_bio:314: I/O error -5 writing to ino .. | JBD2: Detected IO errors while flushing file data on dm-1-8 | Aborting journal on device dm-1-8. | EXT4-fs error : ext4_journal_check_start:56: Detected aborted journal | EXT4-fs (dm-1): Remounting filesystem read-only | EXT4-fs : ext4_writepages: jbd2_start: 2048 pages, inode 498...; err -30 (This did cause corruptions due to failed writes) To fix this mess, the crypto4xx driver needs to notifiy the user to slow down. This can be achieved by returning -EBUSY on requests, once the crypto hardware was falling behind. Note: -EBUSY has two different meanings. Setting the flag CRYPTO_TFM_REQ_MAY_BACKLOG implies that the request was successfully queued, by the crypto driver. To achieve this requirement, the implementation introduces a threshold check and adds logic to the completion routines in much the same way as AMD's Cryptographic Coprocessor (CCP) driver do. Note2: Tests showed that dm-crypt starved ipsec traffic. Under load, ipsec links dropped to 0 Kbits/s. This is because dm-crypt's callback would instantly queue the next request. In order to not starve ipsec, the driver reserves a small portion of the available crypto contexts for this purpose. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 47 ++++++++++++++++++++++++++++++------ drivers/crypto/amcc/crypto4xx_core.h | 3 ++- 2 files changed, 41 insertions(+), 9 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index cb45365166ae..abdf1db9b0eb 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -39,6 +39,7 @@ #include #include #include +#include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" #include "crypto4xx_sa.h" @@ -573,8 +574,10 @@ static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, dst->offset, dst->length, DMA_FROM_DEVICE); } crypto4xx_ret_sg_desc(dev, pd_uinfo); - if (ablk_req->base.complete != NULL) - ablk_req->base.complete(&ablk_req->base, 0); + + if (pd_uinfo->state & PD_ENTRY_BUSY) + ablkcipher_request_complete(ablk_req, -EINPROGRESS); + ablkcipher_request_complete(ablk_req, 0); return 0; } @@ -591,9 +594,10 @@ static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, crypto4xx_copy_digest_to_dst(pd_uinfo, crypto_tfm_ctx(ahash_req->base.tfm)); crypto4xx_ret_sg_desc(dev, pd_uinfo); - /* call user provided callback function x */ - if (ahash_req->base.complete != NULL) - ahash_req->base.complete(&ahash_req->base, 0); + + if (pd_uinfo->state & PD_ENTRY_BUSY) + ahash_request_complete(ahash_req, -EINPROGRESS); + ahash_request_complete(ahash_req, 0); return 0; } @@ -704,6 +708,7 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, struct pd_uinfo *pd_uinfo = NULL; unsigned int nbytes = datalen, idx; u32 gd_idx = 0; + bool is_busy; /* figure how many gd is needed */ num_gd = sg_nents_for_len(src, datalen); @@ -734,6 +739,31 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, * already got must be return the original place. */ spin_lock_irqsave(&dev->core_dev->lock, flags); + /* + * Let the caller know to slow down, once more than 13/16ths = 81% + * of the available data contexts are being used simultaneously. + * + * With PPC4XX_NUM_PD = 256, this will leave a "backlog queue" for + * 31 more contexts. Before new requests have to be rejected. + */ + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 13) / 16); + } else { + /* + * To fix contention issues between ipsec (no blacklog) and + * dm-crypto (backlog) reserve 32 entries for "no backlog" + * data contexts. + */ + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 15) / 16); + + if (is_busy) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EBUSY; + } + } + if (num_gd) { fst_gd = crypto4xx_get_n_gd(dev, num_gd); if (fst_gd == ERING_WAS_FULL) { @@ -888,11 +918,12 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, sa->sa_command_1.bf.hash_crypto_offset = 0; pd->pd_ctl.w = ctx->pd_ctl; pd->pd_ctl_len.w = 0x00400000 | datalen; - pd_uinfo->state = PD_ENTRY_INUSE; + pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); + wmb(); /* write any value to push engine to read a pd */ writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); - return -EINPROGRESS; + return is_busy ? -EBUSY : -EINPROGRESS; } /** @@ -997,7 +1028,7 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) tail = core_dev->dev->pdr_tail; pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; pd = &core_dev->dev->pdr[tail]; - if ((pd_uinfo->state == PD_ENTRY_INUSE) && + if ((pd_uinfo->state & PD_ENTRY_INUSE) && pd->pd_ctl.bf.pe_done && !pd->pd_ctl.bf.host_ready) { pd->pd_ctl.bf.pe_done = 0; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 27e439c1f5bf..dbe29043e0c5 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -44,7 +44,8 @@ #define PPC4XX_LAST_SD (PPC4XX_NUM_SD - 1) #define PPC4XX_SD_BUFFER_SIZE 2048 -#define PD_ENTRY_INUSE 1 +#define PD_ENTRY_BUSY BIT(1) +#define PD_ENTRY_INUSE BIT(0) #define PD_ENTRY_FREE 0 #define ERING_WAS_FULL 0xffffffff -- cgit v1.2.3-59-g8ed1b From 4865b122d4aff5151c88d2f7442d5a87f7e795ae Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:10 +0200 Subject: crypto: crypto4xx - use the correct LE32 format for IV and key defs The hardware expects that the keys, IVs (and inner/outer hashes) are in the le32 format. This patch changes all hardware interface declarations to use the correct LE32 data format for each field. In order to pass __CHECK_ENDIAN__ checks, crypto4xx_memcpy_le has to be honest about the endianness of its parameters. The function was split and moved to the common crypto4xx_core.h header. This allows the compiler to generate better code if the sizes/len is a constant (various *_IV_LEN). Please note that the hardware isn't consistent with the endiannes of the save_digest field in the state record struct though. The hashes produced by GHASH and CBC (for CCM) will be in LE32. Whereas md5 and sha{1/,256,...} do not need any conversion. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 4 +-- drivers/crypto/amcc/crypto4xx_core.c | 40 ++---------------------------- drivers/crypto/amcc/crypto4xx_core.h | 47 +++++++++++++++++++++++++++++++++--- drivers/crypto/amcc/crypto4xx_sa.h | 29 ++++++++++++---------- 4 files changed, 64 insertions(+), 56 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 57b1dcef4cb4..0e1d110a6405 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -149,8 +149,8 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - crypto4xx_memcpy_le(get_dynamic_sa_key_field(sa), - key, keylen); + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); sa->sa_command_1.bf.key_len = keylen >> 3; ctx->is_hash = 0; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index abdf1db9b0eb..c936d68f19ad 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -614,42 +614,6 @@ static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) return crypto4xx_ahash_done(dev, pd_uinfo); } -/** - * Note: Only use this function to copy items that is word aligned. - */ -void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, - int len) -{ - u8 *tmp; - for (; len >= 4; buf += 4, len -= 4) - *dst++ = cpu_to_le32(*(unsigned int *) buf); - - tmp = (u8 *)dst; - switch (len) { - case 3: - *tmp++ = 0; - *tmp++ = *(buf+2); - *tmp++ = *(buf+1); - *tmp++ = *buf; - break; - case 2: - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = *(buf+1); - *tmp++ = *buf; - break; - case 1: - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = *buf; - break; - default: - break; - } -} - static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) { crypto4xx_destroy_pdr(core_dev->dev); @@ -809,8 +773,8 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, &pd_uinfo->sr_pa, 4); if (iv_len) - crypto4xx_memcpy_le(pd_uinfo->sr_va->save_iv, - iv, iv_len); + crypto4xx_memcpy_to_le32(pd_uinfo->sr_va->save_iv, + iv, iv_len); } else { if (ctx->direction == DIR_INBOUND) { pd->sa = ctx->sa_in_dma_addr; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index dbe29043e0c5..2df6874edee1 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -166,9 +166,7 @@ int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); -void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, int len); -u32 crypto4xx_build_pd(struct crypto_async_request *req, +int crypto4xx_build_pd(struct crypto_async_request *req, struct crypto4xx_ctx *ctx, struct scatterlist *src, struct scatterlist *dst, @@ -193,4 +191,47 @@ int crypto4xx_hash_digest(struct ahash_request *req); int crypto4xx_hash_final(struct ahash_request *req); int crypto4xx_hash_update(struct ahash_request *req); int crypto4xx_hash_init(struct ahash_request *req); + +/** + * Note: Only use this function to copy items that is word aligned. + */ +static inline void crypto4xx_memcpy_swab32(u32 *dst, const void *buf, + size_t len) +{ + for (; len >= 4; buf += 4, len -= 4) + *dst++ = __swab32p((u32 *) buf); + + if (len) { + const u8 *tmp = (u8 *)buf; + + switch (len) { + case 3: + *dst = (tmp[2] << 16) | + (tmp[1] << 8) | + tmp[0]; + break; + case 2: + *dst = (tmp[1] << 8) | + tmp[0]; + break; + case 1: + *dst = tmp[0]; + break; + default: + break; + } + } +} + +static inline void crypto4xx_memcpy_from_le32(u32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32(dst, buf, len); +} + +static inline void crypto4xx_memcpy_to_le32(__le32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32((u32 *)dst, buf, len); +} #endif diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 8040c82dc354..b79da98dc9a9 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -181,9 +181,12 @@ struct dynamic_sa_ctl { * State Record for Security Association (SA) */ struct sa_state_record { - u32 save_iv[4]; - u32 save_hash_byte_cnt[2]; - u32 save_digest[16]; + __le32 save_iv[4]; + __le32 save_hash_byte_cnt[2]; + union { + u32 save_digest[16]; /* for MD5/SHA */ + __le32 save_digest_le32[16]; /* GHASH / CBC */ + }; } __attribute__((packed)); /** @@ -192,8 +195,8 @@ struct sa_state_record { */ struct dynamic_sa_aes128 { struct dynamic_sa_ctl ctrl; - u32 key[4]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[4]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -206,8 +209,8 @@ struct dynamic_sa_aes128 { */ struct dynamic_sa_aes192 { struct dynamic_sa_ctl ctrl; - u32 key[6]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[6]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -220,8 +223,8 @@ struct dynamic_sa_aes192 { */ struct dynamic_sa_aes256 { struct dynamic_sa_ctl ctrl; - u32 key[8]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[8]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -235,8 +238,8 @@ struct dynamic_sa_aes256 { */ struct dynamic_sa_hash160 { struct dynamic_sa_ctl ctrl; - u32 inner_digest[5]; - u32 outer_digest[5]; + __le32 inner_digest[5]; + __le32 outer_digest[5]; u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -266,9 +269,9 @@ get_dynamic_sa_offset_state_ptr_field(struct dynamic_sa_ctl *cts) return sizeof(struct dynamic_sa_ctl) + offset * 4; } -static inline u32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) +static inline __le32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) { - return (u32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); + return (__le32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); } #endif -- cgit v1.2.3-59-g8ed1b From cd4dcd6da7a2610e0562a6e130bb68cc544a8fb1 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:11 +0200 Subject: crypto: crypto4xx - overhaul crypto4xx_build_pd() This patch overhauls and fixes code related to crypto4xx_build_pd() * crypto4xx_build_pd() did not handle chained source scatterlist. This is fixed by replacing the buggy indexed-access of &src[idx] with sg_next() in the gather array setup loop. * The redundant is_hash, direction, save_iv and pd_ctl members in the crypto4xx_ctx struct have been removed. - is_hash can be derived from the crypto_async_request parameter. - direction is already part of the security association's bf.dir bitfield. - save_iv is unused. - pd_ctl always had the host_ready bit enabled anyway. (the hash_final case is rather pointless, since the ahash code has been deactivated). * make crypto4xx_build_pd()'s caller responsible for converting the IV to the LE32 format. * change crypto4xx_ahash_update() and crypto4xx_ahash_digest() to initialize a temporary destination scatterlist. This allows the removal of an ugly cast of req->result (which is a pointer to an u8-array) to a scatterlist pointer. * change crypto4xx_build_pd() return type to int. After all it returns -EINPROGRESS/-EBUSY. * fix crypto4xx_build_pd() thread-unsafe sa handling. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 87 +++++++++++------------- drivers/crypto/amcc/crypto4xx_core.c | 128 ++++++++++++++++------------------- drivers/crypto/amcc/crypto4xx_core.h | 12 ++-- 3 files changed, 103 insertions(+), 124 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 0e1d110a6405..195445310f0c 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -75,27 +75,29 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, int crypto4xx_encrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int ivlen = crypto_ablkcipher_ivsize( + crypto_ablkcipher_reqtfm(req)); + __le32 iv[ivlen]; - ctx->direction = DIR_OUTBOUND; - ctx->is_hash = 0; - ctx->pd_ctl = 0x1; + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req))); + req->nbytes, iv, ivlen, ctx->sa_out, ctx->sa_len); } int crypto4xx_decrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int ivlen = crypto_ablkcipher_ivsize( + crypto_ablkcipher_reqtfm(req)); + __le32 iv[ivlen]; - ctx->direction = DIR_INBOUND; - ctx->is_hash = 0; - ctx->pd_ctl = 1; + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req))); + req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len); } /** @@ -153,11 +155,6 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, key, keylen); sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); sa->sa_command_1.bf.key_len = keylen >> 3; - ctx->is_hash = 0; - ctx->direction = DIR_INBOUND; - memcpy(sa + get_dynamic_sa_offset_state_ptr_field(sa), - (void *)&ctx->state_record_dma_addr, 4); - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); sa = ctx->sa_out; @@ -206,7 +203,7 @@ int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, if (rc) return rc; - memcpy(ctx->state_record, + crypto4xx_memcpy_to_le32(ctx->state_record->save_iv, key + keylen - CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); return 0; @@ -215,27 +212,29 @@ int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - __be32 iv[AES_IV_SIZE / 4] = { *(u32 *)ctx->state_record, - *(u32 *) req->info, *(u32 *) (req->info + 4), cpu_to_be32(1) }; - - ctx->direction = DIR_OUTBOUND; - ctx->pd_ctl = 1; + __le32 iv[AES_IV_SIZE / 4] = { + ctx->state_record->save_iv[0], + cpu_to_le32p((u32 *) req->info), + cpu_to_le32p((u32 *) (req->info + 4)), + cpu_to_le32(1) }; return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, iv, AES_IV_SIZE); + req->nbytes, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len); } int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - __be32 iv[AES_IV_SIZE / 4] = { *(u32 *)ctx->state_record, - *(u32 *) req->info, *(u32 *) (req->info + 4), cpu_to_be32(1) }; - - ctx->direction = DIR_INBOUND; - ctx->pd_ctl = 1; + __le32 iv[AES_IV_SIZE / 4] = { + ctx->state_record->save_iv[0], + cpu_to_le32p((u32 *) req->info), + cpu_to_le32p((u32 *) (req->info + 4)), + cpu_to_le32(1) }; return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, iv, AES_IV_SIZE); + req->nbytes, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len); } /** @@ -253,7 +252,6 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, int rc; ctx->dev = my_alg->dev; - ctx->is_hash = 1; /* Create SA */ if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) @@ -284,13 +282,9 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - ctx->direction = DIR_INBOUND; /* Need to zero hash digest in SA */ memset(sa->inner_digest, 0, sizeof(sa->inner_digest)); memset(sa->outer_digest, 0, sizeof(sa->outer_digest)); - sa->state_ptr = ctx->state_record_dma_addr; - ctx->offset_to_sr_ptr = - get_dynamic_sa_offset_state_ptr_field(&sa->ctrl); return 0; } @@ -306,23 +300,22 @@ int crypto4xx_hash_init(struct ahash_request *req) __crypto_ahash_cast(req->base.tfm)); sa->sa_command_0.bf.digest_len = ds >> 2; sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA; - ctx->is_hash = 1; - ctx->direction = DIR_INBOUND; return 0; } int crypto4xx_hash_update(struct ahash_request *req) { + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct scatterlist dst; + unsigned int ds = crypto_ahash_digestsize(ahash); - ctx->is_hash = 1; - ctx->pd_ctl = 0x11; - ctx->direction = DIR_INBOUND; + sg_init_one(&dst, req->result, ds); - return crypto4xx_build_pd(&req->base, ctx, req->src, - (struct scatterlist *) req->result, - req->nbytes, NULL, 0); + return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, + req->nbytes, NULL, 0, ctx->sa_in, + ctx->sa_len); } int crypto4xx_hash_final(struct ahash_request *req) @@ -332,14 +325,16 @@ int crypto4xx_hash_final(struct ahash_request *req) int crypto4xx_hash_digest(struct ahash_request *req) { + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct scatterlist dst; + unsigned int ds = crypto_ahash_digestsize(ahash); - ctx->pd_ctl = 0x11; - ctx->direction = DIR_INBOUND; + sg_init_one(&dst, req->result, ds); - return crypto4xx_build_pd(&req->base, ctx, req->src, - (struct scatterlist *) req->result, - req->nbytes, NULL, 0); + return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, + req->nbytes, NULL, 0, ctx->sa_in, + ctx->sa_len); } /** diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index c936d68f19ad..254dc61c91a6 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -194,7 +194,6 @@ void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) { int i; - struct pd_uinfo *pd_uinfo; dev->pdr = dma_alloc_coherent(dev->core_dev->device, sizeof(struct ce_pd) * PPC4XX_NUM_PD, &dev->pdr_pa, GFP_ATOMIC); @@ -224,11 +223,14 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) if (!dev->shadow_sr_pool) return -ENOMEM; for (i = 0; i < PPC4XX_NUM_PD; i++) { - pd_uinfo = &dev->pdr_uinfo[i]; + struct ce_pd *pd = &dev->pdr[i]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[i]; + + pd->sa = dev->shadow_sa_pool_pa + + sizeof(union shadow_sa_buf) * i; /* alloc 256 bytes which is enough for any kind of dynamic sa */ pd_uinfo->sa_va = &dev->shadow_sa_pool[i].sa; - pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i; /* alloc state record */ pd_uinfo->sr_va = &dev->shadow_sr_pool[i]; @@ -291,14 +293,6 @@ static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) return 0; } -static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev, - dma_addr_t *pd_dma, u32 idx) -{ - *pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx; - - return &dev->pdr[idx]; -} - /** * alloc memory for the gather ring * no need to alloc buf for the ring @@ -520,18 +514,16 @@ static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, } } -static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo, +static void crypto4xx_copy_digest_to_dst(void *dst, + struct pd_uinfo *pd_uinfo, struct crypto4xx_ctx *ctx) { struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { - memcpy((void *) pd_uinfo->dest_va, - pd_uinfo->sr_va->save_digest, + memcpy(dst, pd_uinfo->sr_va->save_digest, SA_HASH_ALG_SHA1_DIGEST_SIZE); } - - return 0; } static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, @@ -591,7 +583,7 @@ static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, ahash_req = ahash_request_cast(pd_uinfo->async_req); ctx = crypto_tfm_ctx(ahash_req->base.tfm); - crypto4xx_copy_digest_to_dst(pd_uinfo, + crypto4xx_copy_digest_to_dst(ahash_req->result, pd_uinfo, crypto_tfm_ctx(ahash_req->base.tfm)); crypto4xx_ret_sg_desc(dev, pd_uinfo); @@ -651,17 +643,17 @@ static u32 get_next_sd(u32 current) return 0; } -u32 crypto4xx_build_pd(struct crypto_async_request *req, +int crypto4xx_build_pd(struct crypto_async_request *req, struct crypto4xx_ctx *ctx, struct scatterlist *src, struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len) + const unsigned int datalen, + const __le32 *iv, const u32 iv_len, + const struct dynamic_sa_ctl *req_sa, + const unsigned int sa_len) { struct crypto4xx_device *dev = ctx->dev; - dma_addr_t addr, pd_dma, sd_dma, gd_dma; struct dynamic_sa_ctl *sa; - struct scatterlist *sg; struct ce_gd *gd; struct ce_pd *pd; u32 num_gd, num_sd; @@ -669,8 +661,9 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, u32 fst_sd = 0xffffffff; u32 pd_entry; unsigned long flags; - struct pd_uinfo *pd_uinfo = NULL; - unsigned int nbytes = datalen, idx; + struct pd_uinfo *pd_uinfo; + unsigned int nbytes = datalen; + size_t offset_to_sr_ptr; u32 gd_idx = 0; bool is_busy; @@ -684,7 +677,7 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, num_gd = 0; /* figure how many sd is needed */ - if (sg_is_last(dst) || ctx->is_hash) { + if (sg_is_last(dst)) { num_sd = 0; } else { if (datalen > PPC4XX_SD_BUFFER_SIZE) { @@ -755,37 +748,27 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, } spin_unlock_irqrestore(&dev->core_dev->lock, flags); + pd = &dev->pdr[pd_entry]; + pd->sa_len = sa_len; + pd_uinfo = &dev->pdr_uinfo[pd_entry]; - pd = crypto4xx_get_pdp(dev, &pd_dma, pd_entry); pd_uinfo->async_req = req; pd_uinfo->num_gd = num_gd; pd_uinfo->num_sd = num_sd; - if (iv_len || ctx->is_hash) { - pd->sa = pd_uinfo->sa_pa; - sa = pd_uinfo->sa_va; - if (ctx->direction == DIR_INBOUND) - memcpy(sa, ctx->sa_in, ctx->sa_len * 4); - else - memcpy(sa, ctx->sa_out, ctx->sa_len * 4); + if (iv_len) + memcpy(pd_uinfo->sr_va->save_iv, iv, iv_len); - memcpy((void *) sa + ctx->offset_to_sr_ptr, - &pd_uinfo->sr_pa, 4); + sa = pd_uinfo->sa_va; + memcpy(sa, req_sa, sa_len * 4); + + offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); + *(u32 *)((unsigned long)sa + offset_to_sr_ptr) = pd_uinfo->sr_pa; - if (iv_len) - crypto4xx_memcpy_to_le32(pd_uinfo->sr_va->save_iv, - iv, iv_len); - } else { - if (ctx->direction == DIR_INBOUND) { - pd->sa = ctx->sa_in_dma_addr; - sa = ctx->sa_in; - } else { - pd->sa = ctx->sa_out_dma_addr; - sa = ctx->sa_out; - } - } - pd->sa_len = ctx->sa_len; if (num_gd) { + dma_addr_t gd_dma; + struct scatterlist *sg; + /* get first gd we are going to use */ gd_idx = fst_gd; pd_uinfo->first_gd = fst_gd; @@ -794,27 +777,30 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, pd->src = gd_dma; /* enable gather */ sa->sa_command_0.bf.gather = 1; - idx = 0; - src = &src[0]; /* walk the sg, and setup gather array */ + + sg = src; while (nbytes) { - sg = &src[idx]; - addr = dma_map_page(dev->core_dev->device, sg_page(sg), - sg->offset, sg->length, DMA_TO_DEVICE); - gd->ptr = addr; - gd->ctl_len.len = sg->length; + size_t len; + + len = min(sg->length, nbytes); + gd->ptr = dma_map_page(dev->core_dev->device, + sg_page(sg), sg->offset, len, DMA_TO_DEVICE); + gd->ctl_len.len = len; gd->ctl_len.done = 0; gd->ctl_len.ready = 1; - if (sg->length >= nbytes) + if (len >= nbytes) break; + nbytes -= sg->length; gd_idx = get_next_gd(gd_idx); gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); - idx++; + sg = sg_next(sg); } } else { pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src), - src->offset, src->length, DMA_TO_DEVICE); + src->offset, min(nbytes, src->length), + DMA_TO_DEVICE); /* * Disable gather in sa command */ @@ -825,25 +811,24 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, pd_uinfo->first_gd = 0xffffffff; pd_uinfo->num_gd = 0; } - if (ctx->is_hash || sg_is_last(dst)) { + if (sg_is_last(dst)) { /* * we know application give us dst a whole piece of memory * no need to use scatter ring. - * In case of is_hash, the icv is always at end of src data. */ pd_uinfo->using_sd = 0; pd_uinfo->first_sd = 0xffffffff; pd_uinfo->num_sd = 0; pd_uinfo->dest_va = dst; sa->sa_command_0.bf.scatter = 0; - if (ctx->is_hash) - pd->dest = virt_to_phys((void *)dst); - else - pd->dest = (u32)dma_map_page(dev->core_dev->device, - sg_page(dst), dst->offset, - dst->length, DMA_TO_DEVICE); + pd->dest = (u32)dma_map_page(dev->core_dev->device, + sg_page(dst), dst->offset, + min(datalen, dst->length), + DMA_TO_DEVICE); } else { + dma_addr_t sd_dma; struct ce_sd *sd = NULL; + u32 sd_idx = fst_sd; nbytes = datalen; sa->sa_command_0.bf.scatter = 1; @@ -857,7 +842,6 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, sd->ctl.done = 0; sd->ctl.rdy = 1; /* sd->ptr should be setup by sd_init routine*/ - idx = 0; if (nbytes >= PPC4XX_SD_BUFFER_SIZE) nbytes -= PPC4XX_SD_BUFFER_SIZE; else @@ -868,19 +852,23 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, /* setup scatter descriptor */ sd->ctl.done = 0; sd->ctl.rdy = 1; - if (nbytes >= PPC4XX_SD_BUFFER_SIZE) + if (nbytes >= PPC4XX_SD_BUFFER_SIZE) { nbytes -= PPC4XX_SD_BUFFER_SIZE; - else + } else { /* * SD entry can hold PPC4XX_SD_BUFFER_SIZE, * which is more than nbytes, so done. */ nbytes = 0; + } } } sa->sa_command_1.bf.hash_crypto_offset = 0; - pd->pd_ctl.w = ctx->pd_ctl; + pd->pd_ctl.w = 0; + pd->pd_ctl.bf.hash_final = + (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH); + pd->pd_ctl.bf.host_ready = 1; pd->pd_ctl_len.w = 0x00400000 | datalen; pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 2df6874edee1..b1d634b0f44f 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -71,7 +71,6 @@ struct pd_uinfo { u32 num_sd; /* number of scatter discriptors used by this packet */ struct dynamic_sa_ctl *sa_va; /* shadow sa */ - u32 sa_pa; struct sa_state_record *sr_va; /* state record for shadow sa */ u32 sr_pa; struct scatterlist *dest_va; @@ -129,11 +128,6 @@ struct crypto4xx_ctx { struct sa_state_record *state_record; dma_addr_t state_record_dma_addr; u32 sa_len; - u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */ - u32 direction; - u32 save_iv; - u32 pd_ctl; - u32 is_hash; }; struct crypto4xx_alg_common { @@ -170,8 +164,10 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct crypto4xx_ctx *ctx, struct scatterlist *src, struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len); + const unsigned int datalen, + const __le32 *iv, const u32 iv_len, + const struct dynamic_sa_ctl *sa, + const unsigned int sa_len); int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, -- cgit v1.2.3-59-g8ed1b From 64e1062b2371cb8d6126d4e970832365a1a84562 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:12 +0200 Subject: crypto: crypto4xx - fix various warnings crypto4xx_core.c:179:6: warning: symbol 'crypto4xx_free_state_record' was not declared. Should it be static? crypto4xx_core.c:331:5: warning: symbol 'crypto4xx_get_n_gd' was not declared. Should it be static? crypto4xx_core.c:652:6: warning: symbol 'crypto4xx_return_pd' was not declared. Should it be static? crypto4xx_return_pd() is not used by anything. Therefore it is removed. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 254dc61c91a6..1752ea2125db 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -176,7 +176,7 @@ u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx) return 0; } -void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) +static void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) { if (ctx->state_record != NULL) dma_free_coherent(ctx->dev->core_dev->device, @@ -322,10 +322,11 @@ static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev) * when this function is called. * preemption or interrupt must be disabled */ -u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) +static u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) { u32 retval; u32 tmp; + if (n >= PPC4XX_NUM_GD) return ERING_WAS_FULL; @@ -616,17 +617,6 @@ static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) kfree(core_dev); } -void crypto4xx_return_pd(struct crypto4xx_device *dev, - u32 pd_entry, struct ce_pd *pd, - struct pd_uinfo *pd_uinfo) -{ - /* irq should be already disabled */ - dev->pdr_head = pd_entry; - pd->pd_ctl.w = 0; - pd->pd_ctl_len.w = 0; - pd_uinfo->state = PD_ENTRY_FREE; -} - static u32 get_next_gd(u32 current) { if (current != PPC4XX_LAST_GD) -- cgit v1.2.3-59-g8ed1b From 4b5b79998af61db8b0506fba6c0f33b57ea457bd Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:13 +0200 Subject: crypto: crypto4xx - fix stalls under heavy load If the crypto4xx device is continuously loaded by dm-crypt and ipsec work, it will start to work intermittent after a few (between 20-30) seconds, hurting throughput and latency. This patch contains various stability improvements in order to fix this issue. So far, the hardware has survived more than a day without suffering any stalls under the continuous load. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 33 ++++++++++++++++++--------------- drivers/crypto/amcc/crypto4xx_reg_def.h | 3 +++ 2 files changed, 21 insertions(+), 15 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 1752ea2125db..de9044201a23 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -280,17 +280,20 @@ static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) { struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + u32 tail; unsigned long flags; spin_lock_irqsave(&dev->core_dev->lock, flags); + pd_uinfo->state = PD_ENTRY_FREE; + if (dev->pdr_tail != PPC4XX_LAST_PD) dev->pdr_tail++; else dev->pdr_tail = 0; - pd_uinfo->state = PD_ENTRY_FREE; + tail = dev->pdr_tail; spin_unlock_irqrestore(&dev->core_dev->lock, flags); - return 0; + return tail; } /** @@ -854,16 +857,16 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } } - sa->sa_command_1.bf.hash_crypto_offset = 0; - pd->pd_ctl.w = 0; - pd->pd_ctl.bf.hash_final = - (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH); - pd->pd_ctl.bf.host_ready = 1; + pd->pd_ctl.w = PD_CTL_HOST_READY | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | + (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? + PD_CTL_HASH_FINAL : 0); pd->pd_ctl_len.w = 0x00400000 | datalen; pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); wmb(); /* write any value to push engine to read a pd */ + writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); return is_busy ? -EBUSY : -EINPROGRESS; } @@ -964,23 +967,23 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); struct pd_uinfo *pd_uinfo; struct ce_pd *pd; - u32 tail; + u32 tail = core_dev->dev->pdr_tail; + u32 head = core_dev->dev->pdr_head; - while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) { - tail = core_dev->dev->pdr_tail; + do { pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; pd = &core_dev->dev->pdr[tail]; if ((pd_uinfo->state & PD_ENTRY_INUSE) && - pd->pd_ctl.bf.pe_done && - !pd->pd_ctl.bf.host_ready) { - pd->pd_ctl.bf.pe_done = 0; + ((READ_ONCE(pd->pd_ctl.w) & + (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) == + PD_CTL_PE_DONE)) { crypto4xx_pd_done(core_dev->dev, tail); - crypto4xx_put_pd_to_pdr(core_dev->dev, tail); + tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail); } else { /* if tail not done, break */ break; } - } + } while (head != tail); } /** diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 279b8725559f..0a22ec5d1a96 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -261,6 +261,9 @@ union ce_pd_ctl { } bf; u32 w; } __attribute__((packed)); +#define PD_CTL_HASH_FINAL BIT(4) +#define PD_CTL_PE_DONE BIT(1) +#define PD_CTL_HOST_READY BIT(0) union ce_pd_ctl_len { struct { -- cgit v1.2.3-59-g8ed1b From 2f77690dcb96e525bc6b57bce4a0eaecaa2878d1 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:14 +0200 Subject: crypto: crypto4xx - simplify sa and state context acquisition Thanks to the big overhaul of crypto4xx_build_pd(), the request-local sa_in, sa_out and state_record allocation can be simplified. There's no need to setup any dma coherent memory anymore and much of the support code can be removed. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 27 +++++-------------- drivers/crypto/amcc/crypto4xx_core.c | 50 ++++++------------------------------ drivers/crypto/amcc/crypto4xx_core.h | 6 +---- 3 files changed, 15 insertions(+), 68 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 195445310f0c..22875ec2b2c8 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -122,20 +122,13 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, } /* Create SA */ - if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) + if (ctx->sa_in || ctx->sa_out) crypto4xx_free_sa(ctx); rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4); if (rc) return rc; - if (ctx->state_record_dma_addr == 0) { - rc = crypto4xx_alloc_state_record(ctx); - if (rc) { - crypto4xx_free_sa(ctx); - return rc; - } - } /* Setup SA */ sa = ctx->sa_in; @@ -203,8 +196,8 @@ int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, if (rc) return rc; - crypto4xx_memcpy_to_le32(ctx->state_record->save_iv, - key + keylen - CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE); + ctx->iv_nonce = cpu_to_le32p((u32 *)&key[keylen - + CTR_RFC3686_NONCE_SIZE]); return 0; } @@ -213,7 +206,7 @@ int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); __le32 iv[AES_IV_SIZE / 4] = { - ctx->state_record->save_iv[0], + ctx->iv_nonce, cpu_to_le32p((u32 *) req->info), cpu_to_le32p((u32 *) (req->info + 4)), cpu_to_le32(1) }; @@ -227,7 +220,7 @@ int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); __le32 iv[AES_IV_SIZE / 4] = { - ctx->state_record->save_iv[0], + ctx->iv_nonce, cpu_to_le32p((u32 *) req->info), cpu_to_le32p((u32 *) (req->info + 4)), cpu_to_le32(1) }; @@ -254,21 +247,13 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, ctx->dev = my_alg->dev; /* Create SA */ - if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) + if (ctx->sa_in || ctx->sa_out) crypto4xx_free_sa(ctx); rc = crypto4xx_alloc_sa(ctx, sa_len); if (rc) return rc; - if (ctx->state_record_dma_addr == 0) { - crypto4xx_alloc_state_record(ctx); - if (!ctx->state_record_dma_addr) { - crypto4xx_free_sa(ctx); - return -ENOMEM; - } - } - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct crypto4xx_ctx)); sa = (struct dynamic_sa_hash160 *)ctx->sa_in; diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index de9044201a23..55a4dd8984c7 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -130,21 +130,17 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) { - ctx->sa_in = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, - &ctx->sa_in_dma_addr, GFP_ATOMIC); + ctx->sa_in = kzalloc(size * 4, GFP_ATOMIC); if (ctx->sa_in == NULL) return -ENOMEM; - ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, - &ctx->sa_out_dma_addr, GFP_ATOMIC); + ctx->sa_out = kzalloc(size * 4, GFP_ATOMIC); if (ctx->sa_out == NULL) { - dma_free_coherent(ctx->dev->core_dev->device, size * 4, - ctx->sa_in, ctx->sa_in_dma_addr); + kfree(ctx->sa_in); + ctx->sa_in = NULL; return -ENOMEM; } - memset(ctx->sa_in, 0, size * 4); - memset(ctx->sa_out, 0, size * 4); ctx->sa_len = size; return 0; @@ -152,40 +148,13 @@ int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) void crypto4xx_free_sa(struct crypto4xx_ctx *ctx) { - if (ctx->sa_in != NULL) - dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, - ctx->sa_in, ctx->sa_in_dma_addr); - if (ctx->sa_out != NULL) - dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, - ctx->sa_out, ctx->sa_out_dma_addr); - - ctx->sa_in_dma_addr = 0; - ctx->sa_out_dma_addr = 0; + kfree(ctx->sa_in); + ctx->sa_in = NULL; + kfree(ctx->sa_out); + ctx->sa_out = NULL; ctx->sa_len = 0; } -u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx) -{ - ctx->state_record = dma_alloc_coherent(ctx->dev->core_dev->device, - sizeof(struct sa_state_record), - &ctx->state_record_dma_addr, GFP_ATOMIC); - if (!ctx->state_record_dma_addr) - return -ENOMEM; - memset(ctx->state_record, 0, sizeof(struct sa_state_record)); - - return 0; -} - -static void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) -{ - if (ctx->state_record != NULL) - dma_free_coherent(ctx->dev->core_dev->device, - sizeof(struct sa_state_record), - ctx->state_record, - ctx->state_record_dma_addr); - ctx->state_record_dma_addr = 0; -} - /** * alloc memory for the gather ring * no need to alloc buf for the ring @@ -883,8 +852,6 @@ static int crypto4xx_alg_init(struct crypto_tfm *tfm) ctx->dev = amcc_alg->dev; ctx->sa_in = NULL; ctx->sa_out = NULL; - ctx->sa_in_dma_addr = 0; - ctx->sa_out_dma_addr = 0; ctx->sa_len = 0; switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { @@ -905,7 +872,6 @@ static void crypto4xx_alg_exit(struct crypto_tfm *tfm) struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); crypto4xx_free_sa(ctx); - crypto4xx_free_state_record(ctx); } int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index b1d634b0f44f..650aa94cc4b8 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -122,11 +122,8 @@ struct crypto4xx_core_device { struct crypto4xx_ctx { struct crypto4xx_device *dev; struct dynamic_sa_ctl *sa_in; - dma_addr_t sa_in_dma_addr; struct dynamic_sa_ctl *sa_out; - dma_addr_t sa_out_dma_addr; - struct sa_state_record *state_record; - dma_addr_t state_record_dma_addr; + __le32 iv_nonce; u32 sa_len; }; @@ -159,7 +156,6 @@ static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); -u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); int crypto4xx_build_pd(struct crypto_async_request *req, struct crypto4xx_ctx *ctx, struct scatterlist *src, -- cgit v1.2.3-59-g8ed1b From a0aae821ba3d35a49d4d0143dfb0c07eee22130e Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:15 +0200 Subject: crypto: crypto4xx - prepare for AEAD support This patch enhances existing interfaces and functions to support AEAD ciphers in the next patches. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 4 + drivers/crypto/amcc/crypto4xx_alg.c | 19 +-- drivers/crypto/amcc/crypto4xx_core.c | 217 +++++++++++++++++++++++++++-------- drivers/crypto/amcc/crypto4xx_core.h | 22 ++-- drivers/crypto/amcc/crypto4xx_sa.h | 41 +++++++ 5 files changed, 230 insertions(+), 73 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index fe33c199fc1a..de825b354fdf 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -315,6 +315,10 @@ config CRYPTO_DEV_PPC4XX tristate "Driver AMCC PPC4xx crypto accelerator" depends on PPC && 4xx select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_AES + select CRYPTO_CCM + select CRYPTO_GCM select CRYPTO_BLKCIPHER help This option allows you to have support for AMCC crypto acceleration. diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 22875ec2b2c8..dd4241a5bf56 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -83,7 +84,7 @@ int crypto4xx_encrypt(struct ablkcipher_request *req) crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, iv, ivlen, ctx->sa_out, ctx->sa_len); + req->nbytes, iv, ivlen, ctx->sa_out, ctx->sa_len, 0); } int crypto4xx_decrypt(struct ablkcipher_request *req) @@ -97,7 +98,7 @@ int crypto4xx_decrypt(struct ablkcipher_request *req) crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len); + req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len, 0); } /** @@ -213,7 +214,7 @@ int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req) return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, req->nbytes, iv, AES_IV_SIZE, - ctx->sa_out, ctx->sa_len); + ctx->sa_out, ctx->sa_len, 0); } int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) @@ -227,7 +228,7 @@ int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, req->nbytes, iv, AES_IV_SIZE, - ctx->sa_out, ctx->sa_len); + ctx->sa_out, ctx->sa_len, 0); } /** @@ -239,11 +240,13 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, unsigned char hm) { struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg); + struct crypto4xx_alg *my_alg; struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); struct dynamic_sa_hash160 *sa; int rc; + my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg, + alg.u.hash); ctx->dev = my_alg->dev; /* Create SA */ @@ -300,7 +303,7 @@ int crypto4xx_hash_update(struct ahash_request *req) return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, req->nbytes, NULL, 0, ctx->sa_in, - ctx->sa_len); + ctx->sa_len, 0); } int crypto4xx_hash_final(struct ahash_request *req) @@ -319,7 +322,7 @@ int crypto4xx_hash_digest(struct ahash_request *req) return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, req->nbytes, NULL, 0, ctx->sa_in, - ctx->sa_len); + ctx->sa_len, 0); } /** @@ -330,5 +333,3 @@ int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm) return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1, SA_HASH_MODE_HASH); } - - diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 55a4dd8984c7..b5108259f1a6 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -35,10 +35,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" @@ -518,7 +520,7 @@ static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, } } -static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, +static void crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo, struct ce_pd *pd) { @@ -543,11 +545,9 @@ static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, if (pd_uinfo->state & PD_ENTRY_BUSY) ablkcipher_request_complete(ablk_req, -EINPROGRESS); ablkcipher_request_complete(ablk_req, 0); - - return 0; } -static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, +static void crypto4xx_ahash_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo) { struct crypto4xx_ctx *ctx; @@ -563,20 +563,88 @@ static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, if (pd_uinfo->state & PD_ENTRY_BUSY) ahash_request_complete(ahash_req, -EINPROGRESS); ahash_request_complete(ahash_req, 0); +} - return 0; +static void crypto4xx_aead_done(struct crypto4xx_device *dev, + struct pd_uinfo *pd_uinfo, + struct ce_pd *pd) +{ + struct aead_request *aead_req; + struct crypto4xx_ctx *ctx; + struct scatterlist *dst = pd_uinfo->dest_va; + int err = 0; + + aead_req = container_of(pd_uinfo->async_req, struct aead_request, + base); + ctx = crypto_tfm_ctx(aead_req->base.tfm); + + if (pd_uinfo->using_sd) { + crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, + pd->pd_ctl_len.bf.pkt_len, + dst); + } else { + __dma_sync_page(sg_page(dst), dst->offset, dst->length, + DMA_FROM_DEVICE); + } + + if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) { + /* append icv at the end */ + size_t cp_len = crypto_aead_authsize( + crypto_aead_reqtfm(aead_req)); + u32 icv[cp_len]; + + crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest, + cp_len); + + scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen, + cp_len, 1); + } + + crypto4xx_ret_sg_desc(dev, pd_uinfo); + + if (pd->pd_ctl.bf.status & 0xff) { + if (pd->pd_ctl.bf.status & 0x1) { + /* authentication error */ + err = -EBADMSG; + } else { + if (!__ratelimit(&dev->aead_ratelimit)) { + if (pd->pd_ctl.bf.status & 2) + pr_err("pad fail error\n"); + if (pd->pd_ctl.bf.status & 4) + pr_err("seqnum fail\n"); + if (pd->pd_ctl.bf.status & 8) + pr_err("error _notify\n"); + pr_err("aead return err status = 0x%02x\n", + pd->pd_ctl.bf.status & 0xff); + pr_err("pd pad_ctl = 0x%08x\n", + pd->pd_ctl.bf.pd_pad_ctl); + } + err = -EINVAL; + } + } + + if (pd_uinfo->state & PD_ENTRY_BUSY) + aead_request_complete(aead_req, -EINPROGRESS); + + aead_request_complete(aead_req, err); } -static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) +static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) { struct ce_pd *pd = &dev->pdr[idx]; struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; - if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) == - CRYPTO_ALG_TYPE_ABLKCIPHER) - return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); - else - return crypto4xx_ahash_done(dev, pd_uinfo); + switch (crypto_tfm_alg_type(pd_uinfo->async_req->tfm)) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); + break; + case CRYPTO_ALG_TYPE_AEAD: + crypto4xx_aead_done(dev, pd_uinfo, pd); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto4xx_ahash_done(dev, pd_uinfo); + break; + } } static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) @@ -612,8 +680,10 @@ int crypto4xx_build_pd(struct crypto_async_request *req, const unsigned int datalen, const __le32 *iv, const u32 iv_len, const struct dynamic_sa_ctl *req_sa, - const unsigned int sa_len) + const unsigned int sa_len, + const unsigned int assoclen) { + struct scatterlist _dst[2]; struct crypto4xx_device *dev = ctx->dev; struct dynamic_sa_ctl *sa; struct ce_gd *gd; @@ -627,18 +697,25 @@ int crypto4xx_build_pd(struct crypto_async_request *req, unsigned int nbytes = datalen; size_t offset_to_sr_ptr; u32 gd_idx = 0; + int tmp; bool is_busy; - /* figure how many gd is needed */ - num_gd = sg_nents_for_len(src, datalen); - if ((int)num_gd < 0) { + /* figure how many gd are needed */ + tmp = sg_nents_for_len(src, assoclen + datalen); + if (tmp < 0) { dev_err(dev->core_dev->device, "Invalid number of src SG.\n"); - return -EINVAL; + return tmp; + } + if (tmp == 1) + tmp = 0; + num_gd = tmp; + + if (assoclen) { + nbytes += assoclen; + dst = scatterwalk_ffwd(_dst, dst, assoclen); } - if (num_gd == 1) - num_gd = 0; - /* figure how many sd is needed */ + /* figure how many sd are needed */ if (sg_is_last(dst)) { num_sd = 0; } else { @@ -724,6 +801,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, sa = pd_uinfo->sa_va; memcpy(sa, req_sa, sa_len * 4); + sa->sa_command_1.bf.hash_crypto_offset = (assoclen >> 2); offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); *(u32 *)((unsigned long)sa + offset_to_sr_ptr) = pd_uinfo->sr_pa; @@ -830,7 +908,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? PD_CTL_HASH_FINAL : 0); - pd->pd_ctl_len.w = 0x00400000 | datalen; + pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); wmb(); @@ -843,40 +921,68 @@ int crypto4xx_build_pd(struct crypto_async_request *req, /** * Algorithm Registration Functions */ -static int crypto4xx_alg_init(struct crypto_tfm *tfm) +static void crypto4xx_ctx_init(struct crypto4xx_alg *amcc_alg, + struct crypto4xx_ctx *ctx) { - struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *amcc_alg = crypto_alg_to_crypto4xx_alg(alg); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->dev = amcc_alg->dev; ctx->sa_in = NULL; ctx->sa_out = NULL; ctx->sa_len = 0; +} - switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { - default: - tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); - break; - case CRYPTO_ALG_TYPE_AHASH: - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct crypto4xx_ctx)); - break; - } +static int crypto4xx_ablk_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct crypto4xx_alg *amcc_alg; + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher); + crypto4xx_ctx_init(amcc_alg, ctx); + tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); return 0; } -static void crypto4xx_alg_exit(struct crypto_tfm *tfm) +static void crypto4xx_common_exit(struct crypto4xx_ctx *ctx) { - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - crypto4xx_free_sa(ctx); } -int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, - struct crypto4xx_alg_common *crypto_alg, - int array_size) +static void crypto4xx_ablk_exit(struct crypto_tfm *tfm) +{ + crypto4xx_common_exit(crypto_tfm_ctx(tfm)); +} + +static int crypto4xx_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto4xx_alg *amcc_alg; + + ctx->sw_cipher.aead = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->sw_cipher.aead)) + return PTR_ERR(ctx->sw_cipher.aead); + + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.aead); + crypto4xx_ctx_init(amcc_alg, ctx); + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + max(sizeof(struct crypto4xx_ctx), 32 + + crypto_aead_reqsize(ctx->sw_cipher.aead))); + return 0; +} + +static void crypto4xx_aead_exit(struct crypto_aead *tfm) +{ + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + + crypto4xx_common_exit(ctx); + crypto_free_aead(ctx->sw_cipher.aead); +} + +static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, + struct crypto4xx_alg_common *crypto_alg, + int array_size) { struct crypto4xx_alg *alg; int i; @@ -891,6 +997,10 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, alg->dev = sec_dev; switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AEAD: + rc = crypto_register_aead(&alg->alg.u.aead); + break; + case CRYPTO_ALG_TYPE_AHASH: rc = crypto_register_ahash(&alg->alg.u.hash); break; @@ -920,6 +1030,10 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) crypto_unregister_ahash(&alg->alg.u.hash); break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&alg->alg.u.aead); + break; + default: crypto_unregister_alg(&alg->alg.u.cipher); } @@ -973,7 +1087,7 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) /** * Supported Crypto Algorithms */ -struct crypto4xx_alg_common crypto4xx_alg[] = { +static struct crypto4xx_alg_common crypto4xx_alg[] = { /* Crypto AES modes */ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { .cra_name = "cbc(aes)", @@ -985,8 +1099,8 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1009,8 +1123,8 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1033,8 +1147,8 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1059,8 +1173,8 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1082,8 +1196,8 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1149,6 +1263,7 @@ static int crypto4xx_probe(struct platform_device *ofdev) core_dev->device = dev; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); + ratelimit_default_init(&core_dev->dev->aead_ratelimit); rc = crypto4xx_build_pdr(core_dev->dev); if (rc) goto err_build_pdr; diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 650aa94cc4b8..ab89c2af1e90 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -22,7 +22,9 @@ #ifndef __CRYPTO4XX_CORE_H__ #define __CRYPTO4XX_CORE_H__ +#include #include +#include #include "crypto4xx_reg_def.h" #include "crypto4xx_sa.h" @@ -106,6 +108,7 @@ struct crypto4xx_device { struct pd_uinfo *pdr_uinfo; struct list_head alg_list; /* List of algorithm supported by this device */ + struct ratelimit_state aead_ratelimit; }; struct crypto4xx_core_device { @@ -125,6 +128,9 @@ struct crypto4xx_ctx { struct dynamic_sa_ctl *sa_out; __le32 iv_nonce; u32 sa_len; + union { + struct crypto_aead *aead; + } sw_cipher; }; struct crypto4xx_alg_common { @@ -132,6 +138,7 @@ struct crypto4xx_alg_common { union { struct crypto_alg cipher; struct ahash_alg hash; + struct aead_alg aead; } u; }; @@ -141,18 +148,6 @@ struct crypto4xx_alg { struct crypto4xx_device *dev; }; -static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( - struct crypto_alg *x) -{ - switch (x->cra_flags & CRYPTO_ALG_TYPE_MASK) { - case CRYPTO_ALG_TYPE_AHASH: - return container_of(__crypto_ahash_alg(x), - struct crypto4xx_alg, alg.u.hash); - } - - return container_of(x, struct crypto4xx_alg, alg.u.cipher); -} - int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); @@ -163,7 +158,8 @@ int crypto4xx_build_pd(struct crypto_async_request *req, const unsigned int datalen, const __le32 *iv, const u32 iv_len, const struct dynamic_sa_ctl *sa, - const unsigned int sa_len); + const unsigned int sa_len, + const unsigned int assoclen); int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index b79da98dc9a9..a4d403528db5 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -55,6 +55,8 @@ union dynamic_sa_contents { #define SA_OP_GROUP_BASIC 0 #define SA_OPCODE_ENCRYPT 0 #define SA_OPCODE_DECRYPT 0 +#define SA_OPCODE_ENCRYPT_HASH 1 +#define SA_OPCODE_HASH_DECRYPT 1 #define SA_OPCODE_HASH 3 #define SA_CIPHER_ALG_DES 0 #define SA_CIPHER_ALG_3DES 1 @@ -65,6 +67,8 @@ union dynamic_sa_contents { #define SA_HASH_ALG_MD5 0 #define SA_HASH_ALG_SHA1 1 +#define SA_HASH_ALG_GHASH 12 +#define SA_HASH_ALG_CBC_MAC 14 #define SA_HASH_ALG_NULL 15 #define SA_HASH_ALG_SHA1_DIGEST_SIZE 20 @@ -233,6 +237,36 @@ struct dynamic_sa_aes256 { #define SA_AES256_CONTENTS 0x3e000082 #define SA_AES_CONTENTS 0x3e000002 +/** + * Security Association (SA) for AES128 CCM + */ +struct dynamic_sa_aes128_ccm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; +#define SA_AES128_CCM_LEN (sizeof(struct dynamic_sa_aes128_ccm)/4) +#define SA_AES128_CCM_CONTENTS 0x3e000042 +#define SA_AES_CCM_CONTENTS 0x3e000002 + +/** + * Security Association (SA) for AES128_GCM + */ +struct dynamic_sa_aes128_gcm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 inner_digest[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; + +#define SA_AES128_GCM_LEN (sizeof(struct dynamic_sa_aes128_gcm)/4) +#define SA_AES128_GCM_CONTENTS 0x3e000442 +#define SA_AES_GCM_CONTENTS 0x3e000402 + /** * Security Association (SA) for HASH160: HMAC-SHA1 */ @@ -274,4 +308,11 @@ static inline __le32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) return (__le32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); } +static inline __le32 *get_dynamic_sa_inner_digest(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + + sizeof(struct dynamic_sa_ctl) + + cts->sa_contents.bf.key_size * 4); +} + #endif -- cgit v1.2.3-59-g8ed1b From 65ea8b678fcf385ac18864743bae66c0643e6842 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:16 +0200 Subject: crypto: crypto4xx - add aes-ccm support This patch adds aes-ccm support. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 185 +++++++++++++++++++++++++++++++++++ drivers/crypto/amcc/crypto4xx_core.c | 23 +++++ drivers/crypto/amcc/crypto4xx_core.h | 8 ++ 3 files changed, 216 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index dd4241a5bf56..b1c4783feab9 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -231,6 +231,191 @@ int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) ctx->sa_out, ctx->sa_len, 0); } +static inline bool crypto4xx_aead_need_fallback(struct aead_request *req, + bool is_ccm, bool decrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + + /* authsize has to be a multiple of 4 */ + if (aead->authsize & 3) + return true; + + /* + * hardware does not handle cases where cryptlen + * is less than a block + */ + if (req->cryptlen < AES_BLOCK_SIZE) + return true; + + /* assoc len needs to be a multiple of 4 */ + if (req->assoclen & 0x3) + return true; + + /* CCM supports only counter field length of 2 and 4 bytes */ + if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3)) + return true; + + /* CCM - fix CBC MAC mismatch in special case */ + if (is_ccm && decrypt && !req->assoclen) + return true; + + return false; +} + +static int crypto4xx_aead_fallback(struct aead_request *req, + struct crypto4xx_ctx *ctx, bool do_decrypt) +{ + char aead_req_data[sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->sw_cipher.aead)] + __aligned(__alignof__(struct aead_request)); + + struct aead_request *subreq = (void *) aead_req_data; + + memset(subreq, 0, sizeof(aead_req_data)); + + aead_request_set_tfm(subreq, ctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + return do_decrypt ? crypto_aead_decrypt(subreq) : + crypto_aead_encrypt(subreq); +} + +static int crypto4xx_setup_fallback(struct crypto4xx_ctx *ctx, + struct crypto_aead *cipher, + const u8 *key, + unsigned int keylen) +{ + int rc; + + crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->sw_cipher.aead, + crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK); + rc = crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen); + crypto_aead_clear_flags(cipher, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(cipher, + crypto_aead_get_flags(ctx->sw_cipher.aead) & + CRYPTO_TFM_RES_MASK); + + return rc; +} + +/** + * AES-CCM Functions + */ + +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_CCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + /* Setup SA */ + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2); + + set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_HASH_DECRYPT, DIR_INBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), key, keylen); + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_ENCRYPT_HASH, DIR_OUTBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + return 0; +} + +static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int len = req->cryptlen; + __le32 iv[16]; + u32 tmp_sa[ctx->sa_len * 4]; + struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; + + if (crypto4xx_aead_need_fallback(req, true, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + if (decrypt) + len -= crypto_aead_authsize(aead); + + memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, sizeof(tmp_sa)); + sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2; + + if (req->iv[0] == 1) { + /* CRYPTO_MODE_AES_ICM */ + sa->sa_command_1.bf.crypto_mode9_8 = 1; + } + + iv[3] = cpu_to_le32(0); + crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + sa, ctx->sa_len, req->assoclen); +} + +int crypto4xx_encrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, false); +} + +int crypto4xx_decrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, true); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *cipher, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + + return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize); +} + /** * HASH SHA1 Functions */ diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index b5108259f1a6..85c650323c97 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1210,6 +1210,29 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { } } } }, + + /* AEAD */ + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_ccm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_ccm, + .decrypt = crypto4xx_decrypt_aes_ccm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, }; /** diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ab89c2af1e90..bdd5954c2388 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -222,4 +222,12 @@ static inline void crypto4xx_memcpy_to_le32(__le32 *dst, const void *buf, { crypto4xx_memcpy_swab32((u32 *)dst, buf, len); } + +int crypto4xx_setauthsize_aead(struct crypto_aead *ciper, + unsigned int authsize); +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_ccm(struct aead_request *req); +int crypto4xx_decrypt_aes_ccm(struct aead_request *req); + #endif -- cgit v1.2.3-59-g8ed1b From 59231368d3a959fc30c5142c406a045f49130daa Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Wed, 4 Oct 2017 01:00:17 +0200 Subject: crypto: crypto4xx - add aes-gcm support This patch adds aes-gcm support to crypto4xx. Signed-off-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_alg.c | 139 +++++++++++++++++++++++++++++++++++ drivers/crypto/amcc/crypto4xx_core.c | 22 ++++++ drivers/crypto/amcc/crypto4xx_core.h | 4 + 3 files changed, 165 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index b1c4783feab9..eeaf27859d80 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include "crypto4xx_reg_def.h" @@ -416,6 +417,144 @@ int crypto4xx_setauthsize_aead(struct crypto_aead *cipher, return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize); } +/** + * AES-GCM Functions + */ + +static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen) +{ + switch (keylen) { + case 16: + case 24: + case 32: + return 0; + default: + return -EINVAL; + } +} + +static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key, + unsigned int keylen) +{ + struct crypto_cipher *aes_tfm = NULL; + uint8_t src[16] = { 0 }; + int rc = 0; + + aes_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(aes_tfm)) { + rc = PTR_ERR(aes_tfm); + pr_warn("could not load aes cipher driver: %d\n", rc); + return rc; + } + + rc = crypto_cipher_setkey(aes_tfm, key, keylen); + if (rc) { + pr_err("setkey() failed: %d\n", rc); + goto out; + } + + crypto_cipher_encrypt_one(aes_tfm, src, src); + crypto4xx_memcpy_to_le32(hash_start, src, 16); +out: + crypto_free_cipher(aes_tfm); + return rc; +} + +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + if (crypto4xx_aes_gcm_validate_keylen(keylen) != 0) { + crypto_aead_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_GCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + + sa->sa_contents.w = SA_AES_GCM_CONTENTS | (keylen << 2); + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_GHASH, + SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO, + SA_OP_GROUP_BASIC, SA_OPCODE_HASH_DECRYPT, + DIR_INBOUND); + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_ON, SA_MC_DISABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + + rc = crypto4xx_compute_gcm_hash_key_sw(get_dynamic_sa_inner_digest(sa), + key, keylen); + if (rc) { + pr_err("GCM hash key setting failed = %d\n", rc); + goto err; + } + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa->sa_command_0.bf.dir = DIR_OUTBOUND; + sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT_HASH; + + return 0; +err: + crypto4xx_free_sa(ctx); + return rc; +} + +static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req, + bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int len = req->cryptlen; + __le32 iv[4]; + + if (crypto4xx_aead_need_fallback(req, false, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE); + iv[3] = cpu_to_le32(1); + + if (decrypt) + len -= crypto_aead_authsize(crypto_aead_reqtfm(req)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + decrypt ? ctx->sa_in : ctx->sa_out, + ctx->sa_len, req->assoclen); +} + +int crypto4xx_encrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, false); +} + +int crypto4xx_decrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, true); +} + /** * HASH SHA1 Functions */ diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 85c650323c97..c44954e274bc 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -1233,6 +1234,27 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_module = THIS_MODULE, }, } }, + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_gcm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_gcm, + .decrypt = crypto4xx_decrypt_aes_gcm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, }; /** diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index bdd5954c2388..8ac3bd37203b 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -229,5 +229,9 @@ int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key, unsigned int keylen); int crypto4xx_encrypt_aes_ccm(struct aead_request *req); int crypto4xx_decrypt_aes_ccm(struct aead_request *req); +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_gcm(struct aead_request *req); +int crypto4xx_decrypt_aes_gcm(struct aead_request *req); #endif -- cgit v1.2.3-59-g8ed1b From ec8c7d14acc0a477429d3a6fade5dab72c996c82 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:33 +0200 Subject: crypto: talitos - fix AEAD test failures AEAD tests fail when destination SG list has more than 1 element. [ 2.058752] alg: aead: Test 1 failed on encryption for authenc-hmac-sha1-cbc-aes-talitos [ 2.066965] 00000000: 53 69 6e 67 6c 65 20 62 6c 6f 63 6b 20 6d 73 67 00000010: c0 43 ff 74 c0 43 ff e0 de 83 d1 20 de 84 8e 54 00000020: de 83 d7 c4 [ 2.082138] alg: aead: Test 1 failed on encryption for authenc-hmac-sha1-cbc-aes-talitos [ 2.090435] 00000000: 53 69 6e 67 6c 65 20 62 6c 6f 63 6b 20 6d 73 67 00000010: de 84 ea 58 c0 93 1a 24 de 84 e8 59 de 84 f1 20 00000020: 00 00 00 00 [ 2.105721] alg: aead: Test 1 failed on encryption for authenc-hmac-sha1-cbc-3des-talitos [ 2.114259] 00000000: 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 73 74 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 00000030: 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 00000040: 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 00000050: 65 72 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 00000060: 72 63 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 00000070: 63 65 65 72 73 74 54 20 6f 6f 4d 20 6e 61 0a 79 00000080: c0 50 f1 ac c0 50 f3 38 c0 50 f3 94 c0 50 f5 30 00000090: c0 99 74 3c [ 2.166410] alg: aead: Test 1 failed on encryption for authenc-hmac-sha1-cbc-3des-talitos [ 2.174794] 00000000: 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 73 74 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 00000030: 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 00000040: 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 00000050: 65 72 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 00000060: 72 63 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 00000070: 63 65 65 72 73 74 54 20 6f 6f 4d 20 6e 61 0a 79 00000080: c0 50 f1 ac c0 50 f3 38 c0 50 f3 94 c0 50 f5 30 00000090: c0 99 74 3c [ 2.226486] alg: No test for authenc(hmac(sha224),cbc(aes)) (authenc-hmac-sha224-cbc-aes-talitos) [ 2.236459] alg: No test for authenc(hmac(sha224),cbc(aes)) (authenc-hmac-sha224-cbc-aes-talitos) [ 2.247196] alg: aead: Test 1 failed on encryption for authenc-hmac-sha224-cbc-3des-talitos [ 2.255555] 00000000: 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 73 74 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 00000030: 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 00000040: 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 00000050: 65 72 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 00000060: 72 63 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 00000070: 63 65 65 72 73 74 54 20 6f 6f 4d 20 6e 61 0a 79 00000080: c0 50 f1 ac c0 50 f3 38 c0 50 f3 94 c0 50 f5 30 00000090: c0 99 74 3c c0 96 e5 b8 [ 2.309004] alg: aead: Test 1 failed on encryption for authenc-hmac-sha224-cbc-3des-talitos [ 2.317562] 00000000: 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 73 74 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 00000030: 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 00000040: 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 00000050: 65 72 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 00000060: 72 63 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 00000070: 63 65 65 72 73 74 54 20 6f 6f 4d 20 6e 61 0a 79 00000080: c0 50 f1 ac c0 50 f3 38 c0 50 f3 94 c0 50 f5 30 00000090: c0 99 74 3c c0 96 e5 b8 [ 2.370710] alg: aead: Test 1 failed on encryption for authenc-hmac-sha256-cbc-aes-talitos [ 2.379177] 00000000: 53 69 6e 67 6c 65 20 62 6c 6f 63 6b 20 6d 73 67 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 [ 2.397863] alg: aead: Test 1 failed on encryption for authenc-hmac-sha256-cbc-aes-talitos [ 2.406134] 00000000: 53 69 6e 67 6c 65 20 62 6c 6f 63 6b 20 6d 73 67 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 [ 2.424789] alg: aead: Test 1 failed on encryption for authenc-hmac-sha256-cbc-3des-talitos [ 2.433491] 00000000: 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 73 74 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 00000030: 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 00000040: 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 00000050: 65 72 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 00000060: 72 63 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 00000070: 63 65 65 72 73 74 54 20 6f 6f 4d 20 6e 61 0a 79 00000080: c0 50 f1 ac c0 50 f3 38 c0 50 f3 94 c0 50 f5 30 00000090: c0 99 74 3c c0 96 e5 b8 c0 96 e9 20 c0 00 3d dc [ 2.488832] alg: aead: Test 1 failed on encryption for authenc-hmac-sha256-cbc-3des-talitos [ 2.497387] 00000000: 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 73 74 00000010: 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 74 65 00000020: 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 65 72 00000030: 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 72 63 00000040: 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 63 65 00000050: 65 72 73 74 54 20 6f 6f 4d 20 6e 61 20 79 65 53 00000060: 72 63 74 65 20 73 6f 54 20 6f 61 4d 79 6e 53 20 00000070: 63 65 65 72 73 74 54 20 6f 6f 4d 20 6e 61 0a 79 00000080: c0 50 f1 ac c0 50 f3 38 c0 50 f3 94 c0 50 f5 30 00000090: c0 99 74 3c c0 96 e5 b8 c0 96 e9 20 c0 00 3d dc This patch fixes that. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index dff88838dce7..cd8a37e60259 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1232,12 +1232,11 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, sg_link_tbl_len += authsize; } - sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, - &desc->ptr[4], sg_count, areq->assoclen, - tbl_off); + ret = talitos_sg_map(dev, areq->src, cryptlen, edesc, &desc->ptr[4], + sg_count, areq->assoclen, tbl_off); - if (sg_count > 1) { - tbl_off += sg_count; + if (ret > 1) { + tbl_off += ret; sync_needed = true; } -- cgit v1.2.3-59-g8ed1b From e04a61bebc5da1535b6f194b464295b8d558e2fc Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:35 +0200 Subject: crypto: talitos - fix memory corruption on SEC2 On SEC2, when using the old descriptors type (hmac snoop no afeu) for doing IPsec, the CICV out pointeur points out of the allocated memory. [ 2.502554] ============================================================================= [ 2.510740] BUG dma-kmalloc-256 (Not tainted): Redzone overwritten [ 2.516907] ----------------------------------------------------------------------------- [ 2.516907] [ 2.526535] Disabling lock debugging due to kernel taint [ 2.531845] INFO: 0xde858108-0xde85810b. First byte 0xf8 instead of 0xcc [ 2.538549] INFO: Allocated in 0x806181a9 age=0 cpu=0 pid=58 [ 2.544229] __kmalloc+0x374/0x564 [ 2.547649] talitos_edesc_alloc+0x17c/0x48c [ 2.551929] aead_edesc_alloc+0x80/0x154 [ 2.555863] aead_encrypt+0x30/0xe0 [ 2.559368] __test_aead+0x5a0/0x1f3c [ 2.563042] test_aead+0x2c/0x110 [ 2.566371] alg_test_aead+0x5c/0xf4 [ 2.569958] alg_test+0x1dc/0x5a0 [ 2.573305] cryptomgr_test+0x50/0x70 [ 2.576984] kthread+0xd8/0x134 [ 2.580155] ret_from_kernel_thread+0x5c/0x64 [ 2.584534] INFO: Freed in ipsec_esp_encrypt_done+0x130/0x240 age=6 cpu=0 pid=0 [ 2.591839] ipsec_esp_encrypt_done+0x130/0x240 [ 2.596395] flush_channel+0x1dc/0x488 [ 2.600161] talitos2_done_4ch+0x30/0x200 [ 2.604185] tasklet_action+0xa0/0x13c [ 2.607948] __do_softirq+0x148/0x6cc [ 2.611623] irq_exit+0xc0/0x124 [ 2.614869] call_do_irq+0x24/0x3c [ 2.618292] do_IRQ+0x78/0x108 [ 2.621369] ret_from_except+0x0/0x14 [ 2.625055] finish_task_switch+0x58/0x350 [ 2.629165] schedule+0x80/0x134 [ 2.632409] schedule_preempt_disabled+0x38/0xc8 [ 2.637042] cpu_startup_entry+0xe4/0x190 [ 2.641074] start_kernel+0x3f4/0x408 [ 2.644741] 0x3438 [ 2.646857] INFO: Slab 0xdffbdb00 objects=9 used=1 fp=0xde8581c0 flags=0x0080 [ 2.653978] INFO: Object 0xde858008 @offset=8 fp=0xca4395df [ 2.653978] [ 2.661032] Redzone de858000: cc cc cc cc cc cc cc cc ........ [ 2.669029] Object de858008: 00 00 00 02 00 00 00 02 00 6b 6b 6b 1e 83 ea 28 .........kkk...( [ 2.677628] Object de858018: 00 00 00 70 1e 85 80 64 ff 73 1d 21 6b 6b 6b 6b ...p...d.s.!kkkk [ 2.686228] Object de858028: 00 20 00 00 1e 84 17 24 00 10 00 00 1e 85 70 00 . .....$......p. [ 2.694829] Object de858038: 00 18 00 00 1e 84 17 44 00 08 00 00 1e 83 ea 28 .......D.......( [ 2.703430] Object de858048: 00 80 00 00 1e 84 f0 00 00 80 00 00 1e 85 70 10 ..............p. [ 2.712030] Object de858058: 00 20 6b 00 1e 85 80 f4 6b 6b 6b 6b 00 80 02 00 . k.....kkkk.... [ 2.720629] Object de858068: 1e 84 f0 00 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b ....kkkkkkkkkkkk [ 2.729230] Object de858078: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.737830] Object de858088: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.746429] Object de858098: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.755029] Object de8580a8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.763628] Object de8580b8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.772229] Object de8580c8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.780829] Object de8580d8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk [ 2.789430] Object de8580e8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 73 b0 ea 9f kkkkkkkkkkkks... [ 2.798030] Object de8580f8: e8 18 80 d6 56 38 44 c0 db e3 4f 71 f7 ce d1 d3 ....V8D...Oq.... [ 2.806629] Redzone de858108: f8 bd 3e 4f ..>O [ 2.814279] Padding de8581b0: 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZ [ 2.822283] CPU: 0 PID: 0 Comm: swapper Tainted: G B 4.9.50-g995be12679 #179 [ 2.831819] Call Trace: [ 2.834301] [dffefd20] [c01aa9a8] check_bytes_and_report+0x100/0x194 (unreliable) [ 2.841801] [dffefd50] [c01aac3c] check_object+0x200/0x530 [ 2.847306] [dffefd80] [c01ae584] free_debug_processing+0x290/0x690 [ 2.853585] [dffefde0] [c01aec8c] __slab_free+0x308/0x628 [ 2.859000] [dffefe80] [c05057f4] ipsec_esp_encrypt_done+0x130/0x240 [ 2.865378] [dffefeb0] [c05002c4] flush_channel+0x1dc/0x488 [ 2.870968] [dffeff10] [c05007a8] talitos2_done_4ch+0x30/0x200 [ 2.876814] [dffeff30] [c002fe38] tasklet_action+0xa0/0x13c [ 2.882399] [dffeff60] [c002f118] __do_softirq+0x148/0x6cc [ 2.887896] [dffeffd0] [c002f954] irq_exit+0xc0/0x124 [ 2.892968] [dffefff0] [c0013adc] call_do_irq+0x24/0x3c [ 2.898213] [c0d4be00] [c000757c] do_IRQ+0x78/0x108 [ 2.903113] [c0d4be30] [c0015c08] ret_from_except+0x0/0x14 [ 2.908634] --- interrupt: 501 at finish_task_switch+0x70/0x350 [ 2.908634] LR = finish_task_switch+0x58/0x350 [ 2.919327] [c0d4bf20] [c085e1d4] schedule+0x80/0x134 [ 2.924398] [c0d4bf50] [c085e2c0] schedule_preempt_disabled+0x38/0xc8 [ 2.930853] [c0d4bf60] [c007f064] cpu_startup_entry+0xe4/0x190 [ 2.936707] [c0d4bfb0] [c096c434] start_kernel+0x3f4/0x408 [ 2.942198] [c0d4bff0] [00003438] 0x3438 [ 2.946137] FIX dma-kmalloc-256: Restoring 0xde858108-0xde85810b=0xcc [ 2.946137] [ 2.954158] FIX dma-kmalloc-256: Object at 0xde858008 not freed This patch reworks the handling of the CICV out in order to properly handle all cases. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index cd8a37e60259..1e799886c57d 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1247,14 +1247,15 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); } - sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc, - &desc->ptr[5], sg_count, areq->assoclen, - tbl_off); + ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], + sg_count, areq->assoclen, tbl_off); if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); - if (sg_count > 1) { + /* ICV data */ + if (ret > 1) { + tbl_off += ret; edesc->icv_ool = true; sync_needed = true; @@ -1264,9 +1265,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, sizeof(struct talitos_ptr) + authsize; /* Add an entry to the link table for ICV data */ - tbl_ptr += sg_count - 1; - to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1); - tbl_ptr++; + to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, is_sec1); to_talitos_ptr_len(tbl_ptr, authsize, is_sec1); @@ -1274,18 +1273,33 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, /* icv data follows link tables */ to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, is_sec1); + } else { + dma_addr_t addr = edesc->dma_link_tbl; + + if (is_sec1) + addr += areq->assoclen + cryptlen; + else + addr += sizeof(struct talitos_ptr) * tbl_off; + + to_talitos_ptr(&desc->ptr[6], addr, is_sec1); + to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1); + } + } else if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + ret = talitos_sg_map(dev, areq->dst, authsize, edesc, + &desc->ptr[6], sg_count, areq->assoclen + + cryptlen, + tbl_off); + if (ret > 1) { + tbl_off += ret; + edesc->icv_ool = true; + sync_needed = true; + } else { + edesc->icv_ool = false; } } else { edesc->icv_ool = false; } - /* ICV data */ - if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { - to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1); - to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl + - areq->assoclen + cryptlen, is_sec1); - } - /* iv out */ if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, -- cgit v1.2.3-59-g8ed1b From f384cdc4faf350fdb6ad93c5f26952b9ba7c7566 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:37 +0200 Subject: crypto: talitos - fix setkey to check key weakness Crypto manager test report the following failures: [ 3.061081] alg: skcipher: setkey failed on test 5 for ecb-des-talitos: flags=100 [ 3.069342] alg: skcipher-ddst: setkey failed on test 5 for ecb-des-talitos: flags=100 [ 3.077754] alg: skcipher-ddst: setkey failed on test 5 for ecb-des-talitos: flags=100 This is due to setkey being expected to detect weak keys. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 1e799886c57d..8aa1212086f4 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1507,12 +1507,20 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + u32 tmp[DES_EXPKEY_WORDS]; if (keylen > TALITOS_MAX_KEY_SIZE) { crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } + if (unlikely(crypto_ablkcipher_get_flags(cipher) & + CRYPTO_TFM_REQ_WEAK_KEY) && + !des_ekey(tmp, key)) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY); + return -EINVAL; + } + memcpy(&ctx->key, key, keylen); ctx->keylen = keylen; -- cgit v1.2.3-59-g8ed1b From 6cda075aff67a1b9b5ba1b2818091dc939643b6c Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:39 +0200 Subject: crypto: talitos - fix AEAD for sha224 on non sha224 capable chips sha224 AEAD test fails with: [ 2.803125] talitos ff020000.crypto: DEUISR 0x00000000_00000000 [ 2.808743] talitos ff020000.crypto: MDEUISR 0x80100000_00000000 [ 2.814678] talitos ff020000.crypto: DESCBUF 0x20731f21_00000018 [ 2.820616] talitos ff020000.crypto: DESCBUF 0x0628d64c_00000010 [ 2.826554] talitos ff020000.crypto: DESCBUF 0x0631005c_00000018 [ 2.832492] talitos ff020000.crypto: DESCBUF 0x0628d664_00000008 [ 2.838430] talitos ff020000.crypto: DESCBUF 0x061b13a0_00000080 [ 2.844369] talitos ff020000.crypto: DESCBUF 0x0631006c_00000080 [ 2.850307] talitos ff020000.crypto: DESCBUF 0x0631006c_00000018 [ 2.856245] talitos ff020000.crypto: DESCBUF 0x063100ec_00000000 [ 2.884972] talitos ff020000.crypto: failed to reset channel 0 [ 2.890503] talitos ff020000.crypto: done overflow, internal time out, or rngu error: ISR 0x20000000_00020000 [ 2.900652] alg: aead: encryption failed on test 1 for authenc-hmac-sha224-cbc-3des-talitos: ret=22 This is due to SHA224 not being supported by the HW. Allthough for hash we are able to init the hash context by SW, it is not possible for AEAD. Therefore SHA224 AEAD has to be deactivated. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 8aa1212086f4..b7184f305867 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -3068,6 +3068,11 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, t_alg->algt.alg.aead.setkey = aead_setkey; t_alg->algt.alg.aead.encrypt = aead_encrypt; t_alg->algt.alg.aead.decrypt = aead_decrypt; + if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && + !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { + kfree(t_alg); + return ERR_PTR(-ENOTSUPP); + } break; case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; -- cgit v1.2.3-59-g8ed1b From fbb22137c4d9bab536958b152d096fb3f98020ea Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:41 +0200 Subject: crypto: talitos - fix use of sg_link_tbl_len sg_link_tbl_len shall be used instead of cryptlen, otherwise SECs which perform HW CICV verification will fail. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b7184f305867..cf5c9701b898 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1232,8 +1232,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, sg_link_tbl_len += authsize; } - ret = talitos_sg_map(dev, areq->src, cryptlen, edesc, &desc->ptr[4], - sg_count, areq->assoclen, tbl_off); + ret = talitos_sg_map(dev, areq->src, sg_link_tbl_len, edesc, + &desc->ptr[4], sg_count, areq->assoclen, tbl_off); if (ret > 1) { tbl_off += ret; -- cgit v1.2.3-59-g8ed1b From 70d355ccea899dad47dc22d3a4406998f55143fd Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:43 +0200 Subject: crypto: talitos - fix ctr-aes-talitos ctr-aes-talitos test fails as follows on SEC2 [ 0.837427] alg: skcipher: Test 1 failed (invalid result) on encryption for ctr-aes-talitos [ 0.845763] 00000000: 16 36 d5 ee 34 f8 06 25 d7 7f 8e 56 ca 88 43 45 [ 0.852345] 00000010: f9 3f f7 17 2a b2 12 23 30 43 09 15 82 dd e1 97 [ 0.858940] 00000020: a7 f7 32 b5 eb 25 06 13 9a ec f5 29 25 f8 4d 66 [ 0.865366] 00000030: b0 03 5b 8e aa 9a 42 b6 19 33 8a e2 9d 65 96 95 This patch fixes the descriptor type which is special for CTR AES Fixes: 5e75ae1b3cef6 ("crypto: talitos - add new crypto modes") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index cf5c9701b898..a19b5d0300a9 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -2635,7 +2635,7 @@ static struct talitos_alg_template driver_algs[] = { .ivsize = AES_BLOCK_SIZE, } }, - .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | DESC_HDR_SEL0_AESU | DESC_HDR_MODE0_AESU_CTR, }, -- cgit v1.2.3-59-g8ed1b From e4a647c49c581e877001676e658a8dab82a15967 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:45 +0200 Subject: crypto: talitos - zeroize the descriptor with memset() This patch zeroize the descriptor at allocation using memset(). This has two advantages: - It reduces the number of places where data has to be set to 0 - It avoids reading memory and loading the cache with data that will be entirely replaced. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 19 +------------------ drivers/crypto/talitos.h | 2 -- 2 files changed, 1 insertion(+), 20 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index a19b5d0300a9..266e7e626e12 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -75,7 +75,6 @@ static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len, bool is_sec1) { if (is_sec1) { - ptr->res = 0; ptr->len1 = cpu_to_be16(len); } else { ptr->len = cpu_to_be16(len); @@ -118,7 +117,6 @@ static void map_single_talitos_ptr(struct device *dev, to_talitos_ptr_len(ptr, len, is_sec1); to_talitos_ptr(ptr, dma_addr, is_sec1); - to_talitos_ptr_ext_set(ptr, 0, is_sec1); } /* @@ -287,7 +285,6 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, /* map descriptor and save caller data */ if (is_sec1) { desc->hdr1 = desc->hdr; - desc->next_desc = 0; request->dma_desc = dma_map_single(dev, &desc->hdr1, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); @@ -1125,7 +1122,6 @@ int talitos_sg_map(struct device *dev, struct scatterlist *src, bool is_sec1 = has_ftr_sec1(priv); to_talitos_ptr_len(ptr, len, is_sec1); - to_talitos_ptr_ext_set(ptr, 0, is_sec1); if (sg_count == 1) { to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1); @@ -1197,11 +1193,9 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1); to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1); } else { to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1); to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1); } /* cipher key */ @@ -1221,7 +1215,6 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, * typically 12 for ipsec */ to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1); sg_link_tbl_len = cryptlen; @@ -1406,6 +1399,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, err = ERR_PTR(-ENOMEM); goto error_sg; } + memset(&edesc->desc, 0, sizeof(edesc->desc)); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; @@ -1481,7 +1475,6 @@ static int aead_decrypt(struct aead_request *req) DESC_HDR_MODE1_MDEU_CICV; /* reset integrity check result bits */ - edesc->desc.hdr_lo = 0; return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); } @@ -1576,12 +1569,10 @@ static int common_nonsnoop(struct talitos_edesc *edesc, bool is_sec1 = has_ftr_sec1(priv); /* first DWORD empty */ - desc->ptr[0] = zero_entry; /* cipher iv */ to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, @@ -1620,7 +1611,6 @@ static int common_nonsnoop(struct talitos_edesc *edesc, DMA_FROM_DEVICE); /* last DWORD empty */ - desc->ptr[6] = zero_entry; if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, @@ -1766,7 +1756,6 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, int sg_count; /* first DWORD empty */ - desc->ptr[0] = zero_entry; /* hash context in */ if (!req_ctx->first || req_ctx->swinit) { @@ -1775,8 +1764,6 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, (char *)req_ctx->hw_context, DMA_TO_DEVICE); req_ctx->swinit = 0; - } else { - desc->ptr[1] = zero_entry; } /* Indicate next op is not the first. */ req_ctx->first = 0; @@ -1785,8 +1772,6 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (ctx->keylen) map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, (char *)&ctx->key, DMA_TO_DEVICE); - else - desc->ptr[2] = zero_entry; sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) @@ -1803,7 +1788,6 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, sync_needed = true; /* fifth DWORD empty */ - desc->ptr[4] = zero_entry; /* hash/HMAC out -or- hash context out */ if (req_ctx->last) @@ -1816,7 +1800,6 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, req_ctx->hw_context, DMA_FROM_DEVICE); /* last DWORD empty */ - desc->ptr[6] = zero_entry; if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 8dd8f40e2771..6112ff1fc334 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -52,8 +52,6 @@ struct talitos_ptr { __be32 ptr; /* address */ }; -static const struct talitos_ptr zero_entry; - /* descriptor */ struct talitos_desc { __be32 hdr; /* header high bits */ -- cgit v1.2.3-59-g8ed1b From 5b2cf268f193f5bc0c5212107185eb966e547641 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:47 +0200 Subject: crypto: talitos - declare local functions static talitos_handle_buggy_hash() and talitos_sg_map() are only used locally, make them static Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 266e7e626e12..dd6b1fc90020 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1113,7 +1113,7 @@ next: return count; } -int talitos_sg_map(struct device *dev, struct scatterlist *src, +static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, unsigned int offset, int tbl_off) @@ -1721,7 +1721,7 @@ static void ahash_done(struct device *dev, * SEC1 doesn't like hashing of 0 sized message, so we do the padding * ourself and submit a padded block */ -void talitos_handle_buggy_hash(struct talitos_ctx *ctx, +static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, struct talitos_edesc *edesc, struct talitos_ptr *ptr) { -- cgit v1.2.3-59-g8ed1b From 24b92ff276e8f6b5c20f6c14ae483299685b887d Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:49 +0200 Subject: crypto: talitos - use devm_kmalloc() Replace kmalloc() by devm_kmalloc() Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index dd6b1fc90020..2a53d0f2a869 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -2993,17 +2993,11 @@ static int talitos_remove(struct platform_device *ofdev) break; } list_del(&t_alg->entry); - kfree(t_alg); } if (hw_supports(dev, DESC_HDR_SEL0_RNG)) talitos_unregister_rng(dev); - for (i = 0; priv->chan && i < priv->num_channels; i++) - kfree(priv->chan[i].fifo); - - kfree(priv->chan); - for (i = 0; i < 2; i++) if (priv->irq[i]) { free_irq(priv->irq[i], dev); @@ -3016,8 +3010,6 @@ static int talitos_remove(struct platform_device *ofdev) iounmap(priv->reg); - kfree(priv); - return 0; } @@ -3029,7 +3021,8 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, struct talitos_crypto_alg *t_alg; struct crypto_alg *alg; - t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL); + t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), + GFP_KERNEL); if (!t_alg) return ERR_PTR(-ENOMEM); @@ -3053,7 +3046,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, t_alg->algt.alg.aead.decrypt = aead_decrypt; if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { - kfree(t_alg); + devm_kfree(dev, t_alg); return ERR_PTR(-ENOTSUPP); } break; @@ -3073,7 +3066,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, if (!(priv->features & TALITOS_FTR_HMAC_OK) && !strncmp(alg->cra_name, "hmac", 4)) { - kfree(t_alg); + devm_kfree(dev, t_alg); return ERR_PTR(-ENOTSUPP); } if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && @@ -3088,7 +3081,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, break; default: dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); - kfree(t_alg); + devm_kfree(dev, t_alg); return ERR_PTR(-EINVAL); } @@ -3169,7 +3162,7 @@ static int talitos_probe(struct platform_device *ofdev) int i, err; int stride; - priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL); + priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -3267,8 +3260,8 @@ static int talitos_probe(struct platform_device *ofdev) } } - priv->chan = kzalloc(sizeof(struct talitos_channel) * - priv->num_channels, GFP_KERNEL); + priv->chan = devm_kzalloc(dev, sizeof(struct talitos_channel) * + priv->num_channels, GFP_KERNEL); if (!priv->chan) { dev_err(dev, "failed to allocate channel management space\n"); err = -ENOMEM; @@ -3285,8 +3278,9 @@ static int talitos_probe(struct platform_device *ofdev) spin_lock_init(&priv->chan[i].head_lock); spin_lock_init(&priv->chan[i].tail_lock); - priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) * - priv->fifo_len, GFP_KERNEL); + priv->chan[i].fifo = devm_kzalloc(dev, + sizeof(struct talitos_request) * + priv->fifo_len, GFP_KERNEL); if (!priv->chan[i].fifo) { dev_err(dev, "failed to allocate request fifo %d\n", i); err = -ENOMEM; @@ -3352,7 +3346,7 @@ static int talitos_probe(struct platform_device *ofdev) if (err) { dev_err(dev, "%s alg registration failed\n", alg->cra_driver_name); - kfree(t_alg); + devm_kfree(dev, t_alg); } else list_add_tail(&t_alg->entry, &priv->alg_list); } -- cgit v1.2.3-59-g8ed1b From fa14c6cfcecb0011d53d77c5deb94b70d3738c44 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:51 +0200 Subject: crypto: talitos - use of_property_read_u32() Use of_property_read_u32() to simplify DT read Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 2a53d0f2a869..f139a0cef2e2 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -3158,7 +3158,6 @@ static int talitos_probe(struct platform_device *ofdev) struct device *dev = &ofdev->dev; struct device_node *np = ofdev->dev.of_node; struct talitos_private *priv; - const unsigned int *prop; int i, err; int stride; @@ -3182,21 +3181,11 @@ static int talitos_probe(struct platform_device *ofdev) } /* get SEC version capabilities from device tree */ - prop = of_get_property(np, "fsl,num-channels", NULL); - if (prop) - priv->num_channels = *prop; - - prop = of_get_property(np, "fsl,channel-fifo-len", NULL); - if (prop) - priv->chfifo_len = *prop; - - prop = of_get_property(np, "fsl,exec-units-mask", NULL); - if (prop) - priv->exec_units = *prop; - - prop = of_get_property(np, "fsl,descriptor-types-mask", NULL); - if (prop) - priv->desc_types = *prop; + of_property_read_u32(np, "fsl,num-channels", &priv->num_channels); + of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); + of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); + of_property_read_u32(np, "fsl,descriptor-types-mask", + &priv->desc_types); if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || !priv->exec_units || !priv->desc_types) { -- cgit v1.2.3-59-g8ed1b From fd5ea7f011932d121e89e470992aaff8547c761f Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:53 +0200 Subject: crypto: talitos - use devm_ioremap() Use devm_ioremap() Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index f139a0cef2e2..83b2a70a1ba7 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -3008,8 +3008,6 @@ static int talitos_remove(struct platform_device *ofdev) if (priv->irq[1]) tasklet_kill(&priv->done_task[1]); - iounmap(priv->reg); - return 0; } @@ -3160,6 +3158,7 @@ static int talitos_probe(struct platform_device *ofdev) struct talitos_private *priv; int i, err; int stride; + struct resource *res; priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL); if (!priv) @@ -3173,7 +3172,10 @@ static int talitos_probe(struct platform_device *ofdev) spin_lock_init(&priv->reg_lock); - priv->reg = of_iomap(np, 0); + res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); + if (!res) + return -ENXIO; + priv->reg = devm_ioremap(dev, res->start, resource_size(res)); if (!priv->reg) { dev_err(dev, "failed to of_iomap\n"); err = -ENOMEM; -- cgit v1.2.3-59-g8ed1b From 9c02e2852fcabf0e1542929e887867d01414dec6 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:55 +0200 Subject: crypto: talitos - don't check the number of channels at each interrupt The number of channels is known from the beginning, no need to test it everytime. This patch defines two additional done functions handling only channel 0. Then the probe registers the correct one based on the number of channels. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 27 +++++++++++++++------------ drivers/crypto/talitos.h | 4 ++++ 2 files changed, 19 insertions(+), 12 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 83b2a70a1ba7..e7e1bada03df 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -390,8 +390,6 @@ static void talitos1_done_##name(unsigned long data) \ \ if (ch_done_mask & 0x10000000) \ flush_channel(dev, 0, 0, 0); \ - if (priv->num_channels == 1) \ - goto out; \ if (ch_done_mask & 0x40000000) \ flush_channel(dev, 1, 0, 0); \ if (ch_done_mask & 0x00010000) \ @@ -399,7 +397,6 @@ static void talitos1_done_##name(unsigned long data) \ if (ch_done_mask & 0x00040000) \ flush_channel(dev, 3, 0, 0); \ \ -out: \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ @@ -409,6 +406,7 @@ out: \ } DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) +DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) #define DEF_TALITOS2_DONE(name, ch_done_mask) \ static void talitos2_done_##name(unsigned long data) \ @@ -419,8 +417,6 @@ static void talitos2_done_##name(unsigned long data) \ \ if (ch_done_mask & 1) \ flush_channel(dev, 0, 0, 0); \ - if (priv->num_channels == 1) \ - goto out; \ if (ch_done_mask & (1 << 2)) \ flush_channel(dev, 1, 0, 0); \ if (ch_done_mask & (1 << 4)) \ @@ -428,7 +424,6 @@ static void talitos2_done_##name(unsigned long data) \ if (ch_done_mask & (1 << 6)) \ flush_channel(dev, 3, 0, 0); \ \ -out: \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ @@ -438,6 +433,7 @@ out: \ } DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) +DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) @@ -3237,17 +3233,24 @@ static int talitos_probe(struct platform_device *ofdev) goto err_out; if (of_device_is_compatible(np, "fsl,sec1.0")) { - tasklet_init(&priv->done_task[0], talitos1_done_4ch, - (unsigned long)dev); - } else { - if (!priv->irq[1]) { - tasklet_init(&priv->done_task[0], talitos2_done_4ch, + if (priv->num_channels == 1) + tasklet_init(&priv->done_task[0], talitos1_done_ch0, (unsigned long)dev); - } else { + else + tasklet_init(&priv->done_task[0], talitos1_done_4ch, + (unsigned long)dev); + } else { + if (priv->irq[1]) { tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, (unsigned long)dev); tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, (unsigned long)dev); + } else if (priv->num_channels == 1) { + tasklet_init(&priv->done_task[0], talitos2_done_ch0, + (unsigned long)dev); + } else { + tasklet_init(&priv->done_task[0], talitos2_done_4ch, + (unsigned long)dev); } } diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 6112ff1fc334..2f04d83c3062 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -208,9 +208,13 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) #define TALITOS_ISR 0x1010 /* interrupt status register */ #define TALITOS1_ISR_4CHERR ISR1_FORMAT(0xa) /* 4 ch errors mask */ #define TALITOS1_ISR_4CHDONE ISR1_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS1_ISR_CH_0_ERR (2 << 28) /* ch 0 errors mask */ +#define TALITOS1_ISR_CH_0_DONE (1 << 28) /* ch 0 done mask */ #define TALITOS1_ISR_TEA_ERR 0x00000040 #define TALITOS2_ISR_4CHERR ISR2_FORMAT(0xa) /* 4 ch errors mask */ #define TALITOS2_ISR_4CHDONE ISR2_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS2_ISR_CH_0_ERR 2 /* ch 0 errors mask */ +#define TALITOS2_ISR_CH_0_DONE 1 /* ch 0 done mask */ #define TALITOS2_ISR_CH_0_2_ERR ISR2_FORMAT(0x2) /* ch 0, 2 err mask */ #define TALITOS2_ISR_CH_0_2_DONE ISR2_FORMAT(0x1) /* ch 0, 2 done mask */ #define TALITOS2_ISR_CH_1_3_ERR ISR2_FORMAT(0x8) /* ch 1, 3 err mask */ -- cgit v1.2.3-59-g8ed1b From da9de146ebd99e162c4724ebce46a2bf2bc06085 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:57 +0200 Subject: crypto: talitos - remove to_talitos_ptr_len() to_talitos_ptr() and to_talitos_ptr_len() are always called together in order to fully set a ptr, so lets merge them into a single helper. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 56 ++++++++++++++++++------------------------------ 1 file changed, 21 insertions(+), 35 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index e7e1bada03df..7e96db75347a 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -56,28 +56,26 @@ #include "talitos.h" static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, - bool is_sec1) + unsigned int len, bool is_sec1) { ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); - if (!is_sec1) + if (is_sec1) { + ptr->len1 = cpu_to_be16(len); + } else { + ptr->len = cpu_to_be16(len); ptr->eptr = upper_32_bits(dma_addr); + } } static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, struct talitos_ptr *src_ptr, bool is_sec1) { dst_ptr->ptr = src_ptr->ptr; - if (!is_sec1) - dst_ptr->eptr = src_ptr->eptr; -} - -static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len, - bool is_sec1) -{ if (is_sec1) { - ptr->len1 = cpu_to_be16(len); + dst_ptr->len1 = src_ptr->len1; } else { - ptr->len = cpu_to_be16(len); + dst_ptr->len = src_ptr->len; + dst_ptr->eptr = src_ptr->eptr; } } @@ -115,8 +113,7 @@ static void map_single_talitos_ptr(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - to_talitos_ptr_len(ptr, len, is_sec1); - to_talitos_ptr(ptr, dma_addr, is_sec1); + to_talitos_ptr(ptr, dma_addr, len, is_sec1); } /* @@ -1090,8 +1087,7 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, len = cryptlen; to_talitos_ptr(link_tbl_ptr + count, - sg_dma_address(sg) + offset, 0); - to_talitos_ptr_len(link_tbl_ptr + count, len, 0); + sg_dma_address(sg) + offset, len, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; @@ -1117,14 +1113,12 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - to_talitos_ptr_len(ptr, len, is_sec1); - if (sg_count == 1) { - to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1); + to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); return sg_count; } if (is_sec1) { - to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1); + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); return sg_count; } sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, @@ -1135,7 +1129,7 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, return sg_count; } to_talitos_ptr(ptr, edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr), is_sec1); + tbl_off * sizeof(struct talitos_ptr), len, is_sec1); to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); return sg_count; @@ -1186,13 +1180,10 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, } /* cipher iv */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { - to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1); - to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1); - } else { - to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1); - to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1); - } + if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, ivsize, is_sec1); + else + to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, ivsize, is_sec1); /* cipher key */ if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) @@ -1210,8 +1201,6 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, * extent is bytes of HMAC postpended to ciphertext, * typically 12 for ipsec */ - to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1); - sg_link_tbl_len = cryptlen; if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { @@ -1257,11 +1246,10 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, is_sec1); - to_talitos_ptr_len(tbl_ptr, authsize, is_sec1); /* icv data follows link tables */ to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, - is_sec1); + authsize, is_sec1); } else { dma_addr_t addr = edesc->dma_link_tbl; @@ -1270,8 +1258,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, else addr += sizeof(struct talitos_ptr) * tbl_off; - to_talitos_ptr(&desc->ptr[6], addr, is_sec1); - to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1); + to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1); } } else if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { ret = talitos_sg_map(dev, areq->dst, authsize, edesc, @@ -1567,8 +1554,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* first DWORD empty */ /* cipher iv */ - to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); - to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, -- cgit v1.2.3-59-g8ed1b From 9a655608a468535cd15eafff5ef93e2656e6e2fc Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:04:59 +0200 Subject: crypto: talitos - simplify tests in ipsec_esp() Do (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) only once. Limit number of if/else paths Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 7e96db75347a..307d534a0f2f 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -938,12 +938,15 @@ static void ipsec_esp_unmap(struct device *dev, struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); + bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; + struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; + struct talitos_ptr *ckey_ptr = &edesc->desc.ptr[is_ipsec_esp ? 3 : 2]; - if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) + if (is_ipsec_esp) unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); + unmap_single_talitos_ptr(dev, ckey_ptr, DMA_TO_DEVICE); + unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, @@ -953,7 +956,7 @@ static void ipsec_esp_unmap(struct device *dev, dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + if (!is_ipsec_esp) { unsigned int dst_nents = edesc->dst_nents ? : 1; sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, @@ -1156,6 +1159,9 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; + struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; + struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, @@ -1180,20 +1186,12 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, } /* cipher iv */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) - to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, ivsize, is_sec1); - else - to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, ivsize, is_sec1); + to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); /* cipher key */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) - map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); - else - map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); + map_single_talitos_ptr(dev, ckey_ptr, ctx->enckeylen, + (char *)&ctx->key + ctx->authkeylen, + DMA_TO_DEVICE); /* * cipher in @@ -1203,10 +1201,10 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, */ sg_link_tbl_len = cryptlen; - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + if (is_ipsec_esp) { to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1); - if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) + if (desc->hdr & DESC_HDR_MODE1_MDEU_CICV) sg_link_tbl_len += authsize; } @@ -1228,7 +1226,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], sg_count, areq->assoclen, tbl_off); - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + if (is_ipsec_esp) to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); /* ICV data */ @@ -1237,7 +1235,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, edesc->icv_ool = true; sync_needed = true; - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + if (is_ipsec_esp) { struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; int offset = (edesc->src_nents + edesc->dst_nents + 2) * sizeof(struct talitos_ptr) + authsize; @@ -1260,7 +1258,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1); } - } else if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + } else if (!is_ipsec_esp) { ret = talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], sg_count, areq->assoclen + cryptlen, @@ -1277,7 +1275,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, } /* iv out */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + if (is_ipsec_esp) map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, DMA_FROM_DEVICE); -- cgit v1.2.3-59-g8ed1b From 2e13ce0812c624f1ee08ab933ebb89dddffed32c Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:05:02 +0200 Subject: crypto: talitos - DMA map key in setkey() dma_map_single() is an heavy operation which doesn't need to be done at each request as the key doesn't change. Instead of DMA mapping the key at every request, this patch maps it once in setkey() Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 56 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 17 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 307d534a0f2f..ebfd6d982ed6 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -815,6 +815,7 @@ struct talitos_ctx { __be32 desc_hdr_template; u8 key[TALITOS_MAX_KEY_SIZE]; u8 iv[TALITOS_MAX_IV_LENGTH]; + dma_addr_t dma_key; unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; @@ -851,6 +852,7 @@ static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = ctx->dev; struct crypto_authenc_keys keys; if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) @@ -859,12 +861,17 @@ static int aead_setkey(struct crypto_aead *authenc, if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) goto badkey; + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + memcpy(ctx->key, keys.authkey, keys.authkeylen); memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); ctx->keylen = keys.authkeylen + keys.enckeylen; ctx->enckeylen = keys.enckeylen; ctx->authkeylen = keys.authkeylen; + ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, + DMA_TO_DEVICE); return 0; @@ -940,14 +947,11 @@ static void ipsec_esp_unmap(struct device *dev, unsigned int ivsize = crypto_aead_ivsize(aead); bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; - struct talitos_ptr *ckey_ptr = &edesc->desc.ptr[is_ipsec_esp ? 3 : 2]; if (is_ipsec_esp) unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); - unmap_single_talitos_ptr(dev, ckey_ptr, DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, areq->assoclen); @@ -976,6 +980,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); unsigned int authsize = crypto_aead_authsize(authenc); + unsigned int ivsize = crypto_aead_ivsize(authenc); struct talitos_edesc *edesc; struct scatterlist *sg; void *icvdata; @@ -996,6 +1001,8 @@ static void ipsec_esp_encrypt_done(struct device *dev, icvdata, authsize); } + dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); + kfree(edesc); aead_request_complete(areq, err); @@ -1164,8 +1171,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; /* hmac key */ - map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, - DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) @@ -1189,9 +1195,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); /* cipher key */ - map_single_talitos_ptr(dev, ckey_ptr, ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); + to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, + ctx->enckeylen, is_sec1); /* * cipher in @@ -1481,6 +1486,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + struct device *dev = ctx->dev; u32 tmp[DES_EXPKEY_WORDS]; if (keylen > TALITOS_MAX_KEY_SIZE) { @@ -1495,9 +1501,14 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, return -EINVAL; } + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + memcpy(&ctx->key, key, keylen); ctx->keylen = keylen; + ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); + return 0; } @@ -1508,7 +1519,6 @@ static void common_nonsnoop_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); if (edesc->dma_len) @@ -1555,8 +1565,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); /* cipher key */ - map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, - (char *)&ctx->key, DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) @@ -1666,10 +1675,6 @@ static void common_nonsnoop_hash_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); - if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1)) - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], - DMA_TO_DEVICE); - if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1750,8 +1755,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* HMAC key */ if (ctx->keylen) - map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, - (char *)&ctx->key, DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, + is_sec1); sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) @@ -2084,6 +2089,7 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct device *dev = ctx->dev; unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); @@ -2106,7 +2112,11 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, memcpy(ctx->key, hash, digestsize); } + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + ctx->keylen = keysize; + ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); return 0; } @@ -2935,6 +2945,15 @@ static int talitos_cra_init_ahash(struct crypto_tfm *tfm) return 0; } +static void talitos_cra_exit(struct crypto_tfm *tfm) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + struct device *dev = ctx->dev; + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); +} + /* * given the alg's descriptor header template, determine whether descriptor * type and primary/secondary execution units required match the hw @@ -3010,6 +3029,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, case CRYPTO_ALG_TYPE_ABLKCIPHER: alg = &t_alg->algt.alg.crypto; alg->cra_init = talitos_cra_init; + alg->cra_exit = talitos_cra_exit; alg->cra_type = &crypto_ablkcipher_type; alg->cra_ablkcipher.setkey = ablkcipher_setkey; alg->cra_ablkcipher.encrypt = ablkcipher_encrypt; @@ -3018,6 +3038,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, break; case CRYPTO_ALG_TYPE_AEAD: alg = &t_alg->algt.alg.aead.base; + alg->cra_exit = talitos_cra_exit; t_alg->algt.alg.aead.init = talitos_cra_init_aead; t_alg->algt.alg.aead.setkey = aead_setkey; t_alg->algt.alg.aead.encrypt = aead_encrypt; @@ -3031,6 +3052,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; alg->cra_init = talitos_cra_init_ahash; + alg->cra_exit = talitos_cra_exit; alg->cra_type = &crypto_ahash_type; t_alg->algt.alg.hash.init = ahash_init; t_alg->algt.alg.hash.update = ahash_update; -- cgit v1.2.3-59-g8ed1b From 49f9783b0cea9a4e38fdbfe70ca296e6f20914c4 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:05:04 +0200 Subject: crypto: talitos - do hw_context DMA mapping outside the requests At every request, we map and unmap the same hash hw_context. This patch moves the dma mapping/unmapping in functions ahash_init() and ahash_import(). Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 80 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 23 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index ebfd6d982ed6..d495649d5267 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -819,6 +819,7 @@ struct talitos_ctx { unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; + dma_addr_t dma_hw_context; }; #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE @@ -1663,18 +1664,9 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); - /* When using hashctx-in, must unmap it. */ - if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], - DMA_TO_DEVICE); - if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1744,10 +1736,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* hash context in */ if (!req_ctx->first || req_ctx->swinit) { - map_single_talitos_ptr(dev, &desc->ptr[1], - req_ctx->hw_context_size, - (char *)req_ctx->hw_context, - DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[1], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); req_ctx->swinit = 0; } /* Indicate next op is not the first. */ @@ -1780,9 +1770,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, crypto_ahash_digestsize(tfm), areq->result, DMA_FROM_DEVICE); else - map_single_talitos_ptr(dev, &desc->ptr[5], - req_ctx->hw_context_size, - req_ctx->hw_context, DMA_FROM_DEVICE); + to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); /* last DWORD empty */ @@ -1815,17 +1804,25 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, static int ahash_init(struct ahash_request *areq) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + unsigned int size; /* Initialize the context */ req_ctx->nbuf = 0; req_ctx->first = 1; /* first indicates h/w must init its context */ req_ctx->swinit = 0; /* assume h/w init of context */ - req_ctx->hw_context_size = - (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + req_ctx->hw_context_size = size; + if (ctx->dma_hw_context) + dma_unmap_single(dev, ctx->dma_hw_context, size, + DMA_BIDIRECTIONAL); + ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size, + DMA_BIDIRECTIONAL); return 0; } @@ -1836,6 +1833,9 @@ static int ahash_init(struct ahash_request *areq) static int ahash_init_sha224_swinit(struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; ahash_init(areq); req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ @@ -1853,6 +1853,9 @@ static int ahash_init_sha224_swinit(struct ahash_request *areq) req_ctx->hw_context[8] = 0; req_ctx->hw_context[9] = 0; + dma_sync_single_for_device(dev, ctx->dma_hw_context, + req_ctx->hw_context_size, DMA_TO_DEVICE); + return 0; } @@ -1990,7 +1993,12 @@ static int ahash_export(struct ahash_request *areq, void *out) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct talitos_export_state *export = out; + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(ahash); + struct device *dev = ctx->dev; + dma_sync_single_for_cpu(dev, ctx->dma_hw_context, + req_ctx->hw_context_size, DMA_FROM_DEVICE); memcpy(export->hw_context, req_ctx->hw_context, req_ctx->hw_context_size); memcpy(export->buf, req_ctx->buf, req_ctx->nbuf); @@ -2008,14 +2016,22 @@ static int ahash_import(struct ahash_request *areq, const void *in) struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); const struct talitos_export_state *export = in; + unsigned int size; + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; memset(req_ctx, 0, sizeof(*req_ctx)); - req_ctx->hw_context_size = - (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; - memcpy(req_ctx->hw_context, export->hw_context, - req_ctx->hw_context_size); + req_ctx->hw_context_size = size; + if (ctx->dma_hw_context) + dma_unmap_single(dev, ctx->dma_hw_context, size, + DMA_BIDIRECTIONAL); + + memcpy(req_ctx->hw_context, export->hw_context, size); + ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size, + DMA_BIDIRECTIONAL); memcpy(req_ctx->buf, export->buf, export->nbuf); req_ctx->swinit = export->swinit; req_ctx->first = export->first; @@ -2954,6 +2970,24 @@ static void talitos_cra_exit(struct crypto_tfm *tfm) dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); } +static void talitos_cra_exit_ahash(struct crypto_tfm *tfm) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + struct device *dev = ctx->dev; + unsigned int size; + + talitos_cra_exit(tfm); + + size = (crypto_ahash_digestsize(__crypto_ahash_cast(tfm)) <= + SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + + if (ctx->dma_hw_context) + dma_unmap_single(dev, ctx->dma_hw_context, size, + DMA_BIDIRECTIONAL); +} + /* * given the alg's descriptor header template, determine whether descriptor * type and primary/secondary execution units required match the hw @@ -3052,7 +3086,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; alg->cra_init = talitos_cra_init_ahash; - alg->cra_exit = talitos_cra_exit; + alg->cra_exit = talitos_cra_exit_ahash; alg->cra_type = &crypto_ahash_type; t_alg->algt.alg.hash.init = ahash_init; t_alg->algt.alg.hash.update = ahash_update; -- cgit v1.2.3-59-g8ed1b From 37b5e8897eb588a60dada00c33d2dca8a7fec5fc Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:05:06 +0200 Subject: crypto: talitos - chain in buffered data for ahash on SEC1 SEC1 doesn't support S/G in descriptors so for hash operations, the CPU has to build a buffer containing the buffered block and the incoming data. This generates a lot of memory copies which represents more than 50% of CPU time of a md5sum operation as shown below with a 'perf record'. |--86.24%-- kcapi_md_digest | | | |--86.18%-- _kcapi_common_vmsplice_chunk_fd | | | | | |--83.68%-- splice | | | | | | | |--83.59%-- ret_from_syscall | | | | | | | | | |--83.52%-- sys_splice | | | | | | | | | | | |--83.49%-- splice_from_pipe | | | | | | | | | | | | | |--83.04%-- __splice_from_pipe | | | | | | | | | | | | | | | |--80.67%-- pipe_to_sendpage | | | | | | | | | | | | | | | | | |--78.25%-- hash_sendpage | | | | | | | | | | | | | | | | | | | |--60.08%-- ahash_process_req | | | | | | | | | | | | | | | | | | | | | |--56.36%-- sg_copy_buffer | | | | | | | | | | | | | | | | | | | | | | | |--55.29%-- memcpy | | | | | | | | | | | | However, unlike SEC2+, SEC1 offers the possibility to chain descriptors. It is therefore possible to build a first descriptor pointing to the buffered data and a second descriptor pointing to the incoming data, hence avoiding the memory copy to a single buffer. With this patch, the time necessary for a md5sum on a 90Mbytes file is approximately 3 seconds. Without the patch it takes 6 seconds. Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 139 ++++++++++++++++++++++++++++++++++++++++++----- drivers/crypto/talitos.h | 1 + 2 files changed, 127 insertions(+), 13 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index d495649d5267..5c4499a85611 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -160,6 +160,10 @@ static int reset_channel(struct device *dev, int ch) /* set 36-bit addressing, done writeback enable and done IRQ enable */ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); + /* enable chaining descriptors */ + if (is_sec1) + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, + TALITOS_CCCR_LO_NE); /* and ICCR writeback, if available */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) @@ -333,7 +337,12 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr; + if (!is_sec1) + hdr = request->desc->hdr; + else if (request->desc->next_desc) + hdr = (request->desc + 1)->hdr1; + else + hdr = request->desc->hdr1; if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -454,7 +463,8 @@ static u32 current_desc_hdr(struct device *dev, int ch) tail = priv->chan[ch].tail; iter = tail; - while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) { + while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && + priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) { iter = (iter + 1) & (priv->fifo_len - 1); if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); @@ -462,6 +472,9 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } + if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) + return (priv->chan[ch].fifo[iter].desc + 1)->hdr; + return priv->chan[ch].fifo[iter].desc->hdr; } @@ -819,6 +832,7 @@ struct talitos_ctx { unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; + dma_addr_t dma_buf; dma_addr_t dma_hw_context; }; @@ -1380,6 +1394,10 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, alloc_len += icv_stashing ? authsize : 0; } + /* if its a ahash, add space for a second desc next to the first one */ + if (is_sec1 && !dst) + alloc_len += sizeof(struct talitos_desc); + edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { dev_err(dev, "could not allocate edescriptor\n"); @@ -1392,11 +1410,15 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; - if (dma_len) - edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], + if (dma_len) { + void *addr = &edesc->link_tbl[0]; + + if (is_sec1 && !dst) + addr += sizeof(struct talitos_desc); + edesc->dma_link_tbl = dma_map_single(dev, addr, edesc->dma_len, DMA_BIDIRECTIONAL); - + } return edesc; error_sg: if (iv_dma) @@ -1671,6 +1693,9 @@ static void common_nonsnoop_hash_unmap(struct device *dev, dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); + if (edesc->desc.next_desc) + dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); } static void ahash_done(struct device *dev, @@ -1717,6 +1742,7 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct ahash_request *areq, unsigned int length, + unsigned int offset, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1748,19 +1774,29 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); + if (is_sec1 && req_ctx->nbuf) + length -= req_ctx->nbuf; + sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); - else + sg_pcopy_to_buffer(req_ctx->psrc, sg_count, + edesc->buf + sizeof(struct talitos_desc), + length, req_ctx->nbuf); + else if (length) sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, DMA_TO_DEVICE); /* * data in */ - sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, 0, 0); - if (sg_count > 1) - sync_needed = true; + if (is_sec1 && req_ctx->nbuf) { + to_talitos_ptr(&desc->ptr[3], ctx->dma_buf, req_ctx->nbuf, + is_sec1); + } else { + sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, + &desc->ptr[3], sg_count, offset, 0); + if (sg_count > 1) + sync_needed = true; + } /* fifth DWORD empty */ @@ -1778,6 +1814,36 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); + if (is_sec1 && req_ctx->nbuf && length) { + struct talitos_desc *desc2 = desc + 1; + dma_addr_t next_desc; + + memset(desc2, 0, sizeof(*desc2)); + desc2->hdr = desc->hdr; + desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; + desc2->hdr1 = desc2->hdr; + desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; + desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; + desc->hdr &= ~DESC_HDR_DONE_NOTIFY; + + to_talitos_ptr(&desc2->ptr[1], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); + + copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); + sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, + &desc2->ptr[3], sg_count, offset, 0); + if (sg_count > 1) + sync_needed = true; + copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); + if (req_ctx->last) + to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); + + next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + desc->next_desc = cpu_to_be32(next_desc); + } + if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1796,6 +1862,11 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_private *priv = dev_get_drvdata(ctx->dev); + bool is_sec1 = has_ftr_sec1(priv); + + if (is_sec1) + nbytes -= req_ctx->nbuf; return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, nbytes, 0, 0, 0, areq->base.flags, false); @@ -1808,6 +1879,8 @@ static int ahash_init(struct ahash_request *areq) struct device *dev = ctx->dev; struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); unsigned int size; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); /* Initialize the context */ req_ctx->nbuf = 0; @@ -1823,6 +1896,13 @@ static int ahash_init(struct ahash_request *areq) DMA_BIDIRECTIONAL); ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size, DMA_BIDIRECTIONAL); + if (ctx->dma_buf) + dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf), + DMA_TO_DEVICE); + if (is_sec1) + ctx->dma_buf = dma_map_single(dev, req_ctx->buf, + sizeof(req_ctx->buf), + DMA_TO_DEVICE); return 0; } @@ -1871,6 +1951,10 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) unsigned int to_hash_later; unsigned int nsg; int nents; + struct device *dev = ctx->dev; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + int offset = 0; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ @@ -1901,13 +1985,27 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) } /* Chain in any previously buffered data */ - if (req_ctx->nbuf) { + if (!is_sec1 && req_ctx->nbuf) { nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; sg_init_table(req_ctx->bufsl, nsg); sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); if (nsg > 1) sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; + } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { + if (nbytes_to_hash > blocksize) + offset = blocksize - req_ctx->nbuf; + else + offset = nbytes_to_hash - req_ctx->nbuf; + nents = sg_nents_for_len(areq->src, offset); + if (nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return nents; + } + sg_copy_to_buffer(areq->src, nents, + req_ctx->buf + req_ctx->nbuf, offset); + req_ctx->nbuf += offset; + req_ctx->psrc = areq->src; } else req_ctx->psrc = areq->src; @@ -1940,6 +2038,9 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) /* request SEC to INIT hash. */ if (req_ctx->first && !req_ctx->swinit) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; + if (is_sec1) + dma_sync_single_for_device(dev, ctx->dma_buf, + req_ctx->nbuf, DMA_TO_DEVICE); /* When the tfm context has a keylen, it's an HMAC. * A first or last (ie. not middle) descriptor must request HMAC. @@ -1947,7 +2048,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (ctx->keylen && (req_ctx->first || req_ctx->last)) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, + return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset, ahash_done); } @@ -2019,6 +2120,8 @@ static int ahash_import(struct ahash_request *areq, const void *in) unsigned int size; struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct device *dev = ctx->dev; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); memset(req_ctx, 0, sizeof(*req_ctx)); size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) @@ -2032,7 +2135,14 @@ static int ahash_import(struct ahash_request *areq, const void *in) memcpy(req_ctx->hw_context, export->hw_context, size); ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size, DMA_BIDIRECTIONAL); + if (ctx->dma_buf) + dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf), + DMA_TO_DEVICE); memcpy(req_ctx->buf, export->buf, export->nbuf); + if (is_sec1) + ctx->dma_buf = dma_map_single(dev, req_ctx->buf, + sizeof(req_ctx->buf), + DMA_TO_DEVICE); req_ctx->swinit = export->swinit; req_ctx->first = export->first; req_ctx->last = export->last; @@ -2986,6 +3096,9 @@ static void talitos_cra_exit_ahash(struct crypto_tfm *tfm) if (ctx->dma_hw_context) dma_unmap_single(dev, ctx->dma_hw_context, size, DMA_BIDIRECTIONAL); + if (ctx->dma_buf) + dma_unmap_single(dev, ctx->dma_buf, HASH_MAX_BLOCK_SIZE, + DMA_TO_DEVICE); } /* diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 2f04d83c3062..a65a63e0d6c1 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -236,6 +236,7 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) #define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */ #define TALITOS_CCCR_LO_EAE 0x20 /* extended address enable */ #define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */ +#define TALITOS_CCCR_LO_NE 0x8 /* fetch next descriptor enab. */ #define TALITOS_CCCR_LO_NT 0x4 /* notification type */ #define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */ #define TALITOS1_CCCR_LO_RESET 0x1 /* channel reset on SEC1 */ -- cgit v1.2.3-59-g8ed1b From 3c0dd1903b17eb8e21d51195831880bd2ede3416 Mon Sep 17 00:00:00 2001 From: LEROY Christophe Date: Fri, 6 Oct 2017 15:05:08 +0200 Subject: crypto: talitos - avoid useless copy This patch avoids copy of buffered data to hash from bufnext to buf Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 5c4499a85611..5bd8191405d8 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -842,8 +842,8 @@ struct talitos_ctx { struct talitos_ahash_req_ctx { u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; unsigned int hw_context_size; - u8 buf[HASH_MAX_BLOCK_SIZE]; - u8 bufnext[HASH_MAX_BLOCK_SIZE]; + u8 buf[2][HASH_MAX_BLOCK_SIZE]; + int buf_idx; unsigned int swinit; unsigned int first; unsigned int last; @@ -1709,7 +1709,7 @@ static void ahash_done(struct device *dev, if (!req_ctx->last && req_ctx->to_hash_later) { /* Position any partial block for next update/final/finup */ - memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later); + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; req_ctx->nbuf = req_ctx->to_hash_later; } common_nonsnoop_hash_unmap(dev, edesc, areq); @@ -1789,8 +1789,10 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, * data in */ if (is_sec1 && req_ctx->nbuf) { - to_talitos_ptr(&desc->ptr[3], ctx->dma_buf, req_ctx->nbuf, - is_sec1); + dma_addr_t dma_buf = ctx->dma_buf + req_ctx->buf_idx * + HASH_MAX_BLOCK_SIZE; + + to_talitos_ptr(&desc->ptr[3], dma_buf, req_ctx->nbuf, is_sec1); } else { sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, &desc->ptr[3], sg_count, offset, 0); @@ -1883,6 +1885,7 @@ static int ahash_init(struct ahash_request *areq) bool is_sec1 = has_ftr_sec1(priv); /* Initialize the context */ + req_ctx->buf_idx = 0; req_ctx->nbuf = 0; req_ctx->first = 1; /* first indicates h/w must init its context */ req_ctx->swinit = 0; /* assume h/w init of context */ @@ -1955,6 +1958,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); int offset = 0; + u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ @@ -1964,7 +1968,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) return nents; } sg_copy_to_buffer(areq->src, nents, - req_ctx->buf + req_ctx->nbuf, nbytes); + ctx_buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; } @@ -1988,7 +1992,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (!is_sec1 && req_ctx->nbuf) { nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; sg_init_table(req_ctx->bufsl, nsg); - sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); + sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); if (nsg > 1) sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; @@ -2003,7 +2007,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) return nents; } sg_copy_to_buffer(areq->src, nents, - req_ctx->buf + req_ctx->nbuf, offset); + ctx_buf + req_ctx->nbuf, offset); req_ctx->nbuf += offset; req_ctx->psrc = areq->src; } else @@ -2016,7 +2020,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) return nents; } sg_pcopy_to_buffer(areq->src, nents, - req_ctx->bufnext, + req_ctx->buf[(req_ctx->buf_idx + 1) & 1], to_hash_later, nbytes - to_hash_later); } @@ -2038,9 +2042,13 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) /* request SEC to INIT hash. */ if (req_ctx->first && !req_ctx->swinit) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; - if (is_sec1) - dma_sync_single_for_device(dev, ctx->dma_buf, + if (is_sec1) { + dma_addr_t dma_buf = ctx->dma_buf + req_ctx->buf_idx * + HASH_MAX_BLOCK_SIZE; + + dma_sync_single_for_device(dev, dma_buf, req_ctx->nbuf, DMA_TO_DEVICE); + } /* When the tfm context has a keylen, it's an HMAC. * A first or last (ie. not middle) descriptor must request HMAC. @@ -2102,7 +2110,7 @@ static int ahash_export(struct ahash_request *areq, void *out) req_ctx->hw_context_size, DMA_FROM_DEVICE); memcpy(export->hw_context, req_ctx->hw_context, req_ctx->hw_context_size); - memcpy(export->buf, req_ctx->buf, req_ctx->nbuf); + memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); export->swinit = req_ctx->swinit; export->first = req_ctx->first; export->last = req_ctx->last; @@ -2138,7 +2146,7 @@ static int ahash_import(struct ahash_request *areq, const void *in) if (ctx->dma_buf) dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf), DMA_TO_DEVICE); - memcpy(req_ctx->buf, export->buf, export->nbuf); + memcpy(req_ctx->buf[0], export->buf, export->nbuf); if (is_sec1) ctx->dma_buf = dma_map_single(dev, req_ctx->buf, sizeof(req_ctx->buf), @@ -3097,7 +3105,7 @@ static void talitos_cra_exit_ahash(struct crypto_tfm *tfm) dma_unmap_single(dev, ctx->dma_hw_context, size, DMA_BIDIRECTIONAL); if (ctx->dma_buf) - dma_unmap_single(dev, ctx->dma_buf, HASH_MAX_BLOCK_SIZE, + dma_unmap_single(dev, ctx->dma_buf, HASH_MAX_BLOCK_SIZE * 2, DMA_TO_DEVICE); } -- cgit v1.2.3-59-g8ed1b From 570265bf9c8d6113d5745c8f01f5eb64e161ce3c Mon Sep 17 00:00:00 2001 From: Yeshaswi M R Gowda Date: Sun, 8 Oct 2017 13:37:18 +0530 Subject: crypto: chelsio - Remove unused parameter Remove unused parameter sent to latest fw. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 43 +++++++++++++++----------------------- drivers/crypto/chelsio/chcr_algo.h | 12 +++++------ 2 files changed, 23 insertions(+), 32 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 936bdd895efa..be4584ae70f0 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -578,36 +578,27 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, void *req, struct sk_buff *skb, - int kctx_len, int hash_sz, - int is_iv, + int hash_sz, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); - int iv_loc = IV_DSGL; int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0, nr_frags = 0; + unsigned int immdatalen = 0; - if (is_ofld_imm(skb)) { + if (is_ofld_imm(skb)) immdatalen = skb->data_len; - iv_loc = IV_IMMEDIATE; - } else { - nr_frags = skb_shinfo(skb)->nr_frags; - } - chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, - ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); + chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = - htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | - FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); + htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( (calc_tx_flits_ofld(skb) * 8), 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, - is_iv ? iv_loc : IV_NOP, !!lcb, - ctx->tx_qidx); + !!lcb, ctx->tx_qidx); chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); @@ -617,7 +608,7 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + sizeof(chcr_req->key_ctx) + - kctx_len + sc_len + immdatalen); + sc_len + immdatalen); } /** @@ -707,8 +698,8 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, + create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0, + sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; skb_get(skb); @@ -1418,8 +1409,8 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, if (param->sg_len != 0) write_sg_to_skb(skb, &frags, req->src, param->sg_len); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, - hash_size_in_response, 0, DUMMY_BYTES, 0); + create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response, + DUMMY_BYTES + kctx_len, 0); req_ctx->skb = skb; skb_get(skb); return skb; @@ -2081,8 +2072,8 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, req->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + create_wreq(ctx, chcr_req, &req->base, skb, size, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); reqctx->skb = skb; skb_get(skb); @@ -2397,8 +2388,8 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, skb_set_transport_header(skb, transhdr_len); frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + create_wreq(ctx, chcr_req, &req->base, skb, 0, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); reqctx->skb = skb; skb_get(skb); return skb; @@ -2555,8 +2546,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); write_sg_to_skb(skb, &frags, src, req->cryptlen); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, + create_wreq(ctx, chcr_req, &req->base, skb, size, + sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, reqctx->verify); reqctx->skb = skb; skb_get(skb); diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 583008de51a3..54851ecaa6c1 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -176,21 +176,21 @@ KEY_CONTEXT_SALT_PRESENT_V(1) | \ KEY_CONTEXT_CTX_LEN_V((ctx_len))) -#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \ +#define FILL_WR_OP_CCTX_SIZE \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \ FW_CRYPTO_LOOKASIDE_WR) | \ FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \ - FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) + FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((0)) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(0)) -#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \ +#define FILL_WR_RX_Q_ID(cid, qid, lcb, fid) \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ - FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \ + FW_CRYPTO_LOOKASIDE_WR_IV_V((IV_NOP)) | \ FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) #define FILL_ULPTX_CMD_DEST(cid, qid) \ -- cgit v1.2.3-59-g8ed1b From 40cdbe1a1bd989b608f86dbdc5e83c6ba777961b Mon Sep 17 00:00:00 2001 From: Yeshaswi M R Gowda Date: Sun, 8 Oct 2017 13:37:19 +0530 Subject: crypto: chelsio - Check error code with IS_ERR macro Check and return proper error code. Signed-off-by: Jitendra Lulla Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index be4584ae70f0..9c70266002d1 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1456,8 +1456,8 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); if (remainder) { u8 *temp; @@ -1520,8 +1520,8 @@ static int chcr_ahash_final(struct ahash_request *req) params.more = 0; } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -1571,8 +1571,8 @@ static int chcr_ahash_finup(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); @@ -1622,8 +1622,8 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); -- cgit v1.2.3-59-g8ed1b From de1a00ac7da115ccafb4415364d484834638aa7f Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Sun, 8 Oct 2017 13:37:21 +0530 Subject: crypto: chelsio - Use x8_ble gf multiplication to calculate IV. gf128mul_x8_ble() will reduce gf Multiplication iteration by 8. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 11 +++++++++-- drivers/crypto/chelsio/chcr_crypto.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 9c70266002d1..8c60fde07433 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -889,9 +889,11 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) int ret, i; u8 *key; unsigned int keylen; + int round = reqctx->last_req_len / AES_BLOCK_SIZE; + int round8 = round / 8; cipher = ablkctx->aes_generic; - memcpy(iv, req->info, AES_BLOCK_SIZE); + memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); keylen = ablkctx->enckey_len / 2; key = ablkctx->key + keylen; @@ -900,7 +902,10 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) goto out; crypto_cipher_encrypt_one(cipher, iv, iv); - for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++) + for (i = 0; i < round8; i++) + gf128mul_x8_ble((le128 *)iv, (le128 *)iv); + + for (i = 0; i < (round % 8); i++) gf128mul_x_ble((le128 *)iv, (le128 *)iv); crypto_cipher_decrypt_one(cipher, iv, iv); @@ -1041,6 +1046,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); reqctx->processed += bytes; + reqctx->last_req_len = bytes; wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; @@ -1133,6 +1139,7 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } reqctx->processed = bytes; + reqctx->last_req_len = bytes; reqctx->dst = reqctx->dstsg; reqctx->op = op_type; wrparam.qid = qid; diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 30af1ee17b87..b3722b3cbd38 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -247,6 +247,7 @@ struct chcr_blkcipher_req_ctx { struct scatterlist *dst; struct scatterlist *newdstsg; unsigned int processed; + unsigned int last_req_len; unsigned int op; short int dst_nents; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; -- cgit v1.2.3-59-g8ed1b From 2956f36c954ee6e31068a02cf2ca551efa5235f1 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Sun, 8 Oct 2017 13:37:22 +0530 Subject: crypto: chelsio - Remove allocation of sg list to implement 2K limit of dsgl header Update DMA address index instead of allocating new sg list to impose 2k size limit for each entry. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 237 +++++++++++------------------------ drivers/crypto/chelsio/chcr_algo.h | 3 +- drivers/crypto/chelsio/chcr_core.h | 2 +- drivers/crypto/chelsio/chcr_crypto.h | 6 - 4 files changed, 76 insertions(+), 172 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 8c60fde07433..33533fe9df43 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -118,6 +118,21 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } +static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen) +{ + int nents = 0; + unsigned int less; + + while (sgl && reqlen) { + less = min(reqlen, sgl->length); + nents += DIV_ROUND_UP(less, CHCR_SG_SIZE); + reqlen -= less; + sgl = sg_next(sgl); + } + + return nents; +} + static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -167,8 +182,6 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, kfree_skb(ctx_req.ctx.reqctx->skb); ctx_req.ctx.reqctx->skb = NULL; } - free_new_sg(ctx_req.ctx.reqctx->newdstsg); - ctx_req.ctx.reqctx->newdstsg = NULL; if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { chcr_verify_tag(ctx_req.req.aead_req, input, &err); @@ -389,31 +402,41 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, { struct phys_sge_pairs *to; unsigned int len = 0, left_size = sg_param->obsize; - unsigned int nents = sg_param->nents, i, j = 0; + unsigned int j = 0; + int offset, ent_len; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); + to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + + sizeof(struct cpl_rx_phys_dsgl)); + while (left_size && sg) { + len = min_t(u32, left_size, sg_dma_len(sg)); + offset = 0; + while (len) { + ent_len = min_t(u32, len, CHCR_SG_SIZE); + to->len[j % 8] = htons(ent_len); + to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset); + offset += ent_len; + len -= ent_len; + j++; + if ((j % 8) == 0) + to++; + } + left_size -= min(left_size, sg_dma_len(sg)); + sg = sg_next(sg); + } phys_cpl->pcirlxorder_to_noofsgentr = htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j)); phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); phys_cpl->rss_hdr_int.hash_val = 0; - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); - for (i = 0; nents && left_size; to++) { - for (j = 0; j < 8 && nents && left_size; j++, nents--) { - len = min(left_size, sg_dma_len(sg)); - to->len[j] = htons(len); - to->addr[j] = cpu_to_be64(sg_dma_address(sg)); - left_size -= len; - sg = sg_next(sg); - } - } + } static inline int map_writesg_phys_cpl(struct device *dev, @@ -524,31 +547,33 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, - unsigned int space, - short int *sent, - short int *dent) + unsigned int space) { int srclen = 0, dstlen = 0; int srcsg = minsg, dstsg = 0; + int offset = 0, less; - *sent = 0; - *dent = 0; while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { srclen += src->length; srcsg++; + offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - dstlen += dst->length; - dst = sg_next(dst); + less = min_t(unsigned int, dst->length - offset, + CHCR_SG_SIZE); + dstlen += less; + offset += less; + if (offset == dst->length) { + dst = sg_next(dst); + offset = 0; + } dstsg++; } src = sg_next(src); } - *sent = srcsg - minsg; - *dent = dstsg; return min(srclen, dstlen); } @@ -632,13 +657,15 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) struct phys_sge_parm sg_param; unsigned int frags = 0, transhdr_len, phys_dsgl; int error; + int nents; unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); - + reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes); + nents = dstsg_2k(reqctx->dst, wrparam->bytes); + phys_dsgl = get_space_for_phys_dsgl(nents); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -1021,8 +1048,7 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, goto complete; } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, &reqctx->dst_nents); + SPACE_LEFT(ablkctx->enckey_len)); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else @@ -1061,8 +1087,6 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, chcr_send_wr(skb); return 0; complete: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; req->base.complete(&req->base, err); return err; } @@ -1078,9 +1102,8 @@ static int process_cipher(struct ablkcipher_request *req, struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct ablk_ctx *ablkctx = ABLK_CTX(ctx); struct cipher_wr_param wrparam; - int bytes, nents, err = -EINVAL; + int bytes, err = -EINVAL; - reqctx->newdstsg = NULL; reqctx->processed = 0; if (!req->info) goto error; @@ -1092,18 +1115,9 @@ static int process_cipher(struct ablkcipher_request *req, goto error; } wrparam.srcsg = req->src; - if (is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return PTR_ERR(reqctx->newdstsg); - reqctx->dstsg = reqctx->newdstsg; - } else { reqctx->dstsg = req->dst; - } bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, - &reqctx->dst_nents); + SPACE_LEFT(ablkctx->enckey_len)); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else @@ -1153,8 +1167,6 @@ static int process_cipher(struct ablkcipher_request *req, return 0; error: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return err; } @@ -1825,63 +1837,6 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int is_newsg(struct scatterlist *sgl, unsigned int *newents) -{ - int nents = 0; - int ret = 0; - - while (sgl) { - if (sgl->length > CHCR_SG_SIZE) - ret = 1; - nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE); - sgl = sg_next(sgl); - } - *newents = nents; - return ret; -} - -static inline void free_new_sg(struct scatterlist *sgl) -{ - kfree(sgl); -} - -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents) -{ - struct scatterlist *newsg, *sg; - int i, len, processed = 0; - struct page *spage; - int offset; - - newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL); - if (!newsg) - return ERR_PTR(-ENOMEM); - sg = newsg; - sg_init_table(sg, nents); - offset = sgl->offset; - spage = sg_page(sgl); - for (i = 0; i < nents; i++) { - len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE); - sg_set_page(sg, spage, len, offset); - processed += len; - offset += len; - if (offset >= PAGE_SIZE) { - offset = offset % PAGE_SIZE; - spage++; - } - if (processed == sgl->length) { - processed = 0; - sgl = sg_next(sgl); - if (!sgl) - break; - spage = sg_page(sgl); - offset = sgl->offset; - } - sg = sg_next(sg); - } - return newsg; -} - static int chcr_copy_assoc(struct aead_request *req, struct chcr_aead_ctx *ctx) { @@ -1954,7 +1909,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - reqctx->newdstsg = NULL; dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : authsize); if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) @@ -1966,24 +1920,13 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, if (src_nent < 0) goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; @@ -1996,7 +1939,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, error = -EINVAL; goto err; } - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2005,8 +1950,6 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2089,8 +2032,6 @@ dstmap_fail: /* ivmap_fail: */ kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2308,7 +2249,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : authsize); - reqctx->newdstsg = NULL; if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) goto err; src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); @@ -2317,25 +2257,15 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, sub_type = get_aead_subtype(tfm); src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) { pr_err("AAD copy to destination buffer fails\n"); return ERR_PTR(error); } - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); @@ -2347,8 +2277,9 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, @@ -2356,8 +2287,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } @@ -2403,8 +2332,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, dstmap_fail: kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } @@ -2433,7 +2360,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, GFP_ATOMIC; struct adapter *adap = padap(ctx->dev); - reqctx->newdstsg = NULL; dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); /* validate key size */ @@ -2447,26 +2373,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); + reqctx->dst = src; if (req->src != req->dst) { error = chcr_copy_assoc(req, aeadctx); if (error) return ERR_PTR(error); + reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, + req->assoclen); } - - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + (op_type ? -authsize : authsize)); if (reqctx->dst_nents < 0) { @@ -2475,8 +2389,9 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, goto err; } - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + + (op_type ? -authsize : authsize)) : 0; + dst_size = get_space_for_phys_dsgl(nents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); @@ -2485,8 +2400,6 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(chcr_aead_fallback(req, op_type)); } skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); @@ -2564,8 +2477,6 @@ dstmap_fail: /* ivmap_fail: */ kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return ERR_PTR(error); } diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 54851ecaa6c1..d1a787b369c9 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -221,7 +221,7 @@ #define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2) +#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1) #define MIN_CIPHER_SG 1 /* IV */ #define MIN_AUTH_SG 2 /*IV + AAD*/ #define MIN_GCM_SG 2 /* IV + AAD*/ @@ -261,7 +261,6 @@ struct cipher_wr_param { struct scatterlist *srcsg; char *iv; int bytes; - short int snent; unsigned short qid; }; enum { diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index c9a19b2a1e9f..94e7412f6164 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -89,7 +89,7 @@ struct uld_ctx { struct chcr_dev *dev; }; -struct uld_ctx * assign_chcr_device(void); +struct uld_ctx *assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index b3722b3cbd38..8f436f80cb21 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -166,7 +166,6 @@ struct ablk_ctx { struct chcr_aead_reqctx { struct sk_buff *skb; struct scatterlist *dst; - struct scatterlist *newdstsg; struct scatterlist srcffwd[2]; struct scatterlist dstffwd[2]; short int dst_nents; @@ -245,7 +244,6 @@ struct chcr_blkcipher_req_ctx { struct scatterlist dstffwd[2]; struct scatterlist *dstsg; struct scatterlist *dst; - struct scatterlist *newdstsg; unsigned int processed; unsigned int last_req_len; unsigned int op; @@ -291,10 +289,6 @@ static int chcr_aead_op(struct aead_request *req_base, int size, create_wr_t create_wr_fn); static inline int get_aead_subtype(struct crypto_aead *aead); -static int is_newsg(struct scatterlist *sgl, unsigned int *newents); -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents); -static inline void free_new_sg(struct scatterlist *sgl); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); #endif /* __CHCR_CRYPTO_H__ */ -- cgit v1.2.3-59-g8ed1b From 2f47d58043110b0aeac1952494db668fc0fc7c0a Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Sun, 8 Oct 2017 13:37:23 +0530 Subject: crypto: chelsio - Move DMA un/mapping to chcr from lld cxgb4 driver Allow chcr to do DMA mapping/Unmapping instead of lld cxgb4. It moves "Copy AAD to dst buffer" requirement from driver to firmware. Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 1645 ++++++++++++++++++------------ drivers/crypto/chelsio/chcr_algo.h | 44 +- drivers/crypto/chelsio/chcr_crypto.h | 114 ++- drivers/net/ethernet/chelsio/cxgb4/sge.c | 8 +- 4 files changed, 1116 insertions(+), 695 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 33533fe9df43..9d298c681abe 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -71,6 +71,8 @@ #include "chcr_algo.h" #include "chcr_crypto.h" +#define IV AES_BLOCK_SIZE + static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) { return ctx->crypto_ctx->aeadctx; @@ -103,7 +105,7 @@ static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) static inline int is_ofld_imm(const struct sk_buff *skb) { - return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); + return (skb->len <= SGE_MAX_WR_LEN); } /* @@ -118,21 +120,92 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } -static int dstsg_2k(struct scatterlist *sgl, unsigned int reqlen) +static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, + unsigned int entlen, + unsigned int skip) { int nents = 0; unsigned int less; + unsigned int skip_len = 0; - while (sgl && reqlen) { - less = min(reqlen, sgl->length); - nents += DIV_ROUND_UP(less, CHCR_SG_SIZE); - reqlen -= less; - sgl = sg_next(sgl); + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } } + while (sg && reqlen) { + less = min(reqlen, sg_dma_len(sg) - skip_len); + nents += DIV_ROUND_UP(less, entlen); + reqlen -= less; + skip_len = 0; + sg = sg_next(sg); + } return nents; } +static inline void chcr_handle_ahash_resp(struct ahash_request *req, + unsigned char *input, + int err) +{ + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + int digestsize, updated_digestsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); + + if (input == NULL) + goto out; + reqctx = ahash_request_ctx(req); + digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + if (reqctx->is_sg_map) + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + if (reqctx->dma_addr) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr, + reqctx->dma_len, DMA_TO_DEVICE); + reqctx->dma_addr = 0; + updated_digestsize = digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + updated_digestsize = SHA256_DIGEST_SIZE; + else if (digestsize == SHA384_DIGEST_SIZE) + updated_digestsize = SHA512_DIGEST_SIZE; + if (reqctx->result == 1) { + reqctx->result = 0; + memcpy(req->result, input + sizeof(struct cpl_fw6_pld), + digestsize); + } else { + memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), + updated_digestsize); + } +out: + req->base.complete(&req->base, err); + + } + +static inline void chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); + if (reqctx->b0_dma) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, + reqctx->b0_len, DMA_BIDIRECTIONAL); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; +} + req->base.complete(&req->base, err); + +} static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -167,27 +240,11 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, { struct crypto_tfm *tfm = req->tfm; struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_req_ctx ctx_req; - unsigned int digestsize, updated_digestsize; struct adapter *adap = padap(ctx->dev); switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AEAD: - ctx_req.req.aead_req = aead_request_cast(req); - ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); - dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, - ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); - if (ctx_req.ctx.reqctx->skb) { - kfree_skb(ctx_req.ctx.reqctx->skb); - ctx_req.ctx.reqctx->skb = NULL; - } - if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { - chcr_verify_tag(ctx_req.req.aead_req, input, - &err); - ctx_req.ctx.reqctx->verify = VERIFY_HW; - } - ctx_req.req.aead_req->base.complete(req, err); + chcr_handle_aead_resp(aead_request_cast(req), input, err); break; case CRYPTO_ALG_TYPE_ABLKCIPHER: @@ -196,60 +253,13 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, break; case CRYPTO_ALG_TYPE_AHASH: - ctx_req.req.ahash_req = ahash_request_cast(req); - ctx_req.ctx.ahash_ctx = - ahash_request_ctx(ctx_req.req.ahash_req); - digestsize = - crypto_ahash_digestsize(crypto_ahash_reqtfm( - ctx_req.req.ahash_req)); - updated_digestsize = digestsize; - if (digestsize == SHA224_DIGEST_SIZE) - updated_digestsize = SHA256_DIGEST_SIZE; - else if (digestsize == SHA384_DIGEST_SIZE) - updated_digestsize = SHA512_DIGEST_SIZE; - if (ctx_req.ctx.ahash_ctx->skb) { - kfree_skb(ctx_req.ctx.ahash_ctx->skb); - ctx_req.ctx.ahash_ctx->skb = NULL; - } - if (ctx_req.ctx.ahash_ctx->result == 1) { - ctx_req.ctx.ahash_ctx->result = 0; - memcpy(ctx_req.req.ahash_req->result, input + - sizeof(struct cpl_fw6_pld), - digestsize); - } else { - memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + - sizeof(struct cpl_fw6_pld), - updated_digestsize); + chcr_handle_ahash_resp(ahash_request_cast(req), input, err); } - ctx_req.req.ahash_req->base.complete(req, err); - break; - } atomic_inc(&adap->chcr_stats.complete); return err; } -/* - * calc_tx_flits_ofld - calculate # of flits for an offload packet - * @skb: the packet - * Returns the number of flits needed for the given offload packet. - * These packets are already fully constructed and no additional headers - * will be added. - */ -static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) -{ - unsigned int flits, cnt; - - if (is_ofld_imm(skb)) - return DIV_ROUND_UP(skb->len, 8); - - flits = skb_transport_offset(skb) / 8; /* headers */ - cnt = skb_shinfo(skb)->nr_frags; - if (skb_tail_pointer(skb) != skb_transport_header(skb)) - cnt++; - return flits + sgl_len(cnt); -} - -static inline void get_aes_decrypt_key(unsigned char *dec_key, +static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength) { @@ -396,64 +406,193 @@ static inline int is_hmac(struct crypto_tfm *tfm) return 0; } -static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, - struct scatterlist *sg, - struct phys_sge_parm *sg_param) +static inline void dsgl_walk_init(struct dsgl_walk *walk, + struct cpl_rx_phys_dsgl *dsgl) { - struct phys_sge_pairs *to; - unsigned int len = 0, left_size = sg_param->obsize; - unsigned int j = 0; - int offset, ent_len; + walk->dsgl = dsgl; + walk->nents = 0; + walk->to = (struct phys_sge_pairs *)(dsgl + 1); +} + +static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid) +{ + struct cpl_rx_phys_dsgl *phys_cpl; + + phys_cpl = walk->dsgl; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); + phys_cpl->pcirlxorder_to_noofsgentr = + htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | + CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | + CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | + CPL_RX_PHYS_DSGL_DCAID_V(0) | + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); + phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; + phys_cpl->rss_hdr_int.qid = htons(qid); + phys_cpl->rss_hdr_int.hash_val = 0; +} + +static inline void dsgl_walk_add_page(struct dsgl_walk *walk, + size_t size, + dma_addr_t *addr) +{ + int j; + + if (!size) + return; + j = walk->nents; + walk->to->len[j % 8] = htons(size); + walk->to->addr[j % 8] = cpu_to_be64(*addr); + j++; + if ((j % 8) == 0) + walk->to++; + walk->nents = j; +} + +static void dsgl_walk_add_sg(struct dsgl_walk *walk, + struct scatterlist *sg, + unsigned int slen, + unsigned int skip) +{ + int skip_len = 0; + unsigned int left_size = slen, len = 0; + unsigned int j = walk->nents; + int offset, ent_len; + + if (!slen) + return; + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + while (left_size && sg) { - len = min_t(u32, left_size, sg_dma_len(sg)); + len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); offset = 0; while (len) { - ent_len = min_t(u32, len, CHCR_SG_SIZE); - to->len[j % 8] = htons(ent_len); - to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + - offset); + ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); + walk->to->len[j % 8] = htons(ent_len); + walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset + skip_len); offset += ent_len; len -= ent_len; j++; if ((j % 8) == 0) - to++; + walk->to++; } - left_size -= min(left_size, sg_dma_len(sg)); + walk->last_sg = sg; + walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - + skip_len) + skip_len; + left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); + skip_len = 0; sg = sg_next(sg); } - phys_cpl->pcirlxorder_to_noofsgentr = - htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | - CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | - CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | - CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | - CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(j)); - phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; - phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); - phys_cpl->rss_hdr_int.hash_val = 0; + walk->nents = j; +} + +static inline void ulptx_walk_init(struct ulptx_walk *walk, + struct ulptx_sgl *ulp) +{ + walk->sgl = ulp; + walk->nents = 0; + walk->pair_idx = 0; + walk->pair = ulp->sge; + walk->last_sg = NULL; + walk->last_sg_len = 0; +} + +static inline void ulptx_walk_end(struct ulptx_walk *walk) +{ + walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(walk->nents)); +} + +static inline void ulptx_walk_add_page(struct ulptx_walk *walk, + size_t size, + dma_addr_t *addr) +{ + if (!size) + return; + + if (walk->nents == 0) { + walk->sgl->len0 = cpu_to_be32(size); + walk->sgl->addr0 = cpu_to_be64(*addr); + } else { + walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr); + walk->pair->len[walk->pair_idx] = cpu_to_be32(size); + walk->pair_idx = !walk->pair_idx; + if (!walk->pair_idx) + walk->pair++; + } + walk->nents++; } -static inline int map_writesg_phys_cpl(struct device *dev, - struct cpl_rx_phys_dsgl *phys_cpl, +static void ulptx_walk_add_sg(struct ulptx_walk *walk, struct scatterlist *sg, - struct phys_sge_parm *sg_param) + unsigned int len, + unsigned int skip) { - if (!sg || !sg_param->nents) - return -EINVAL; + int small; + int skip_len = 0; + unsigned int sgmin; - sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); - if (sg_param->nents == 0) { - pr_err("CHCR : DMA mapping failed\n"); - return -EINVAL; + if (!len) + return; + + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + if (walk->nents == 0) { + small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->sgl->len0 = cpu_to_be32(sgmin); + walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->nents++; + len -= sgmin; + walk->last_sg = sg; + walk->last_sg_len = sgmin + skip_len; + skip_len += sgmin; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } + } + + while (sg && len) { + small = min(sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); + walk->pair->addr[walk->pair_idx] = + cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->pair_idx = !walk->pair_idx; + walk->nents++; + if (!walk->pair_idx) + walk->pair++; + len -= sgmin; + skip_len += sgmin; + walk->last_sg = sg; + walk->last_sg_len = skip_len; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } } - write_phys_cpl(phys_cpl, sg, sg_param); - return 0; } static inline int get_aead_subtype(struct crypto_aead *aead) @@ -473,45 +612,6 @@ static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; } -static inline void write_buffer_to_skb(struct sk_buff *skb, - unsigned int *frags, - char *bfr, - u8 bfr_len) -{ - skb->len += bfr_len; - skb->data_len += bfr_len; - skb->truesize += bfr_len; - get_page(virt_to_page(bfr)); - skb_fill_page_desc(skb, *frags, virt_to_page(bfr), - offset_in_page(bfr), bfr_len); - (*frags)++; -} - - -static inline void -write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, - struct scatterlist *sg, unsigned int count) -{ - struct page *spage; - unsigned int page_len; - - skb->len += count; - skb->data_len += count; - skb->truesize += count; - - while (count > 0) { - if (!sg || (!(sg->length))) - break; - spage = sg_page(sg); - get_page(spage); - page_len = min(sg->length, count); - skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); - (*frags)++; - count -= page_len; - sg = sg_next(sg); - } -} - static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) { struct adapter *adap = netdev2adap(dev); @@ -547,32 +647,46 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx, static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, - unsigned int space) + unsigned int space, + unsigned int srcskip, + unsigned int dstskip) { int srclen = 0, dstlen = 0; - int srcsg = minsg, dstsg = 0; + int srcsg = minsg, dstsg = minsg; int offset = 0, less; - while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && + if (sg_dma_len(src) == srcskip) { + src = sg_next(src); + srcskip = 0; + } + + if (sg_dma_len(dst) == dstskip) { + dst = sg_next(dst); + dstskip = 0; + } + + while (src && dst && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { - srclen += src->length; + srclen += (sg_dma_len(src) - srcskip); srcsg++; offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - less = min_t(unsigned int, dst->length - offset, - CHCR_SG_SIZE); + less = min_t(unsigned int, sg_dma_len(dst) - offset - + dstskip, CHCR_DST_SG_SIZE); dstlen += less; offset += less; - if (offset == dst->length) { + if (offset == sg_dma_len(dst)) { dst = sg_next(dst); offset = 0; } dstsg++; + dstskip = 0; } src = sg_next(src); + srcskip = 0; } return min(srclen, dstlen); } @@ -602,24 +716,22 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, } static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, - void *req, struct sk_buff *skb, + struct crypto_async_request *req, + unsigned int imm, int hash_sz, + unsigned int len16, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0; - if (is_ofld_imm(skb)) - immdatalen = skb->data_len; chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = - htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( - (calc_tx_flits_ofld(skb) * 8), 16))); + htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, @@ -627,13 +739,12 @@ static inline void create_wreq(struct chcr_context *ctx, chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); - chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), - 16) - ((sizeof(chcr_req->wreq)) >> 4))); + chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - + ((sizeof(chcr_req->wreq)) >> 4))); - chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); + chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + - sizeof(chcr_req->key_ctx) + - sc_len + immdatalen); + sizeof(chcr_req->key_ctx) + sc_len); } /** @@ -646,49 +757,52 @@ static inline void create_wreq(struct chcr_context *ctx, static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(wrparam->req); - struct phys_sge_parm sg_param; - unsigned int frags = 0, transhdr_len, phys_dsgl; + unsigned int temp = 0, transhdr_len, dst_size; int error; int nents; - unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; + unsigned int kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(c_ctx(tfm)->dev); - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, wrparam->bytes); - nents = dstsg_2k(reqctx->dst, wrparam->bytes); - phys_dsgl = get_space_for_phys_dsgl(nents); + nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, + reqctx->dst_ofst); + dst_size = get_space_for_phys_dsgl(nents + 1); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); - transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, + CHCR_SRC_SG_SIZE, reqctx->src_ofst); + temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16) + * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1); + FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1); - chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes); + chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); chcr_req->sec_cpl.aadstart_cipherstop_hi = - FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); + FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, ablkctx->ciph_mode, - 0, 0, ivsize >> 1); + 0, 0, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, - 0, 1, phys_dsgl); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; if ((reqctx->op == CHCR_DECRYPT_OP) && @@ -713,26 +827,18 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) } } phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = wrparam->bytes; - sg_param.qid = wrparam->qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto map_fail1; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); + chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); - skb_set_transport_header(skb, transhdr_len); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, 0, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl + kctx_len, + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + +(reqctx->imm ? (IV + wrparam->bytes) : 0); + create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, + transhdr_len, temp, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; - skb_get(skb); return skb; -map_fail1: - kfree_skb(skb); err: return ERR_PTR(error); } @@ -757,8 +863,7 @@ static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, unsigned int keylen) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); int err = 0; crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -776,8 +881,7 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -809,8 +913,7 @@ static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -841,8 +944,7 @@ static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -909,8 +1011,7 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct crypto_cipher *cipher; int ret, i; @@ -927,8 +1028,7 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) ret = crypto_cipher_setkey(cipher, key, keylen); if (ret) goto out; - - crypto_cipher_encrypt_one(cipher, iv, iv); + /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/ for (i = 0; i < round8; i++) gf128mul_x8_ble((le128 *)iv, (le128 *)iv); @@ -1006,64 +1106,60 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb; struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct cipher_wr_param wrparam; int bytes; - dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents, - DMA_FROM_DEVICE); - - if (reqctx->skb) { - kfree_skb(reqctx->skb); - reqctx->skb = NULL; - } if (err) - goto complete; - + goto unmap; if (req->nbytes == reqctx->processed) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_final_cipher_iv(req, fw6_pld, req->info); goto complete; } if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { err = -EBUSY; - goto complete; + goto unmap; } } - wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src, - reqctx->processed); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg, - reqctx->processed); - if (!wrparam.srcsg || !reqctx->dst) { - pr_err("Input sg list length less that nbytes\n"); - err = -EINVAL; - goto complete; - } - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len)); + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, + SPACE_LEFT(ablkctx->enckey_len), + reqctx->src_ofst, reqctx->dst_ofst); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); + } else { + /*CTR mode counter overfloa*/ + bytes = req->nbytes - reqctx->processed; + } + dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); + dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); if (err) - goto complete; + goto unmap; if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, - wrparam.srcsg, - reqctx->dst, - req->nbytes - reqctx->processed, - reqctx->iv, + req->src, + req->dst, + req->nbytes, + req->info, reqctx->op); goto complete; } @@ -1071,21 +1167,23 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); - reqctx->processed += bytes; - reqctx->last_req_len = bytes; - wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; + wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; skb = create_cipher_wr(&wrparam); if (IS_ERR(skb)) { pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); err = PTR_ERR(skb); - goto complete; + goto unmap; } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); + reqctx->last_req_len = bytes; + reqctx->processed += bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); complete: req->base.complete(&req->base, err); return err; @@ -1099,8 +1197,7 @@ static int process_cipher(struct ablkcipher_request *req, struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct cipher_wr_param wrparam; int bytes, err = -EINVAL; @@ -1114,16 +1211,41 @@ static int process_cipher(struct ablkcipher_request *req, ablkctx->enckey_len, req->nbytes, ivsize); goto error; } - wrparam.srcsg = req->src; - reqctx->dstsg = req->dst; - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len)); + chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); + if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + + AES_MIN_KEY_SIZE + + sizeof(struct cpl_rx_phys_dsgl) + + /*Min dsgl size*/ + 32))) { + /* Can be sent as Imm*/ + unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; + + dnents = sg_nents_xlen(req->dst, req->nbytes, + CHCR_DST_SG_SIZE, 0); + dnents += 1; // IV + phys_dsgl = get_space_for_phys_dsgl(dnents); + kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); + reqctx->imm = (transhdr_len + IV + req->nbytes) <= + SGE_MAX_WR_LEN; + bytes = IV + req->nbytes; + + } else { + reqctx->imm = 0; + } + + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(req->src, req->dst, + MIN_CIPHER_SG, + SPACE_LEFT(ablkctx->enckey_len), + 0, 0); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); - if (unlikely(bytes > req->nbytes)) + } else { bytes = req->nbytes; + } if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) { bytes = adjust_ctr_overflow(req->info, bytes); @@ -1140,9 +1262,11 @@ static int process_cipher(struct ablkcipher_request *req, } else { - memcpy(reqctx->iv, req->info, ivsize); + memcpy(reqctx->iv, req->info, IV); } if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, req->src, @@ -1152,20 +1276,25 @@ static int process_cipher(struct ablkcipher_request *req, op_type); goto error; } - reqctx->processed = bytes; - reqctx->last_req_len = bytes; - reqctx->dst = reqctx->dstsg; reqctx->op = op_type; + reqctx->srcsg = req->src; + reqctx->dstsg = req->dst; + reqctx->src_ofst = 0; + reqctx->dst_ofst = 0; wrparam.qid = qid; wrparam.req = req; wrparam.bytes = bytes; *skb = create_cipher_wr(&wrparam); if (IS_ERR(*skb)) { err = PTR_ERR(*skb); - goto error; + goto unmap; } + reqctx->processed = bytes; + reqctx->last_req_len = bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); error: return err; } @@ -1173,23 +1302,22 @@ error: static int chcr_aes_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct sk_buff *skb = NULL; int err; - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_ENCRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_ENCRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1197,23 +1325,22 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req) static int chcr_aes_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; int err; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_DECRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_DECRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1361,17 +1488,19 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); struct sk_buff *skb = NULL; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); struct chcr_wr *chcr_req; - unsigned int frags = 0, transhdr_len, iopad_alignment = 0; + struct ulptx_sgl *ulptx; + unsigned int nents = 0, transhdr_len, iopad_alignment = 0; unsigned int digestsize = crypto_ahash_digestsize(tfm); - unsigned int kctx_len = 0; + unsigned int kctx_len = 0, temp = 0; u8 hash_size_in_response = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(h_ctx(tfm)->dev); + int error = 0; iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); kctx_len = param->alg_prm.result_size + iopad_alignment; @@ -1383,15 +1512,22 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, else hash_size_in_response = param->alg_prm.result_size; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <= + SGE_MAX_WR_LEN; + nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0); + nents += param->bfr_len ? 1 : 0; + transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len + + param->sg_len), 16) * 16) : + (sgl_len(nents) * 8); + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) - return skb; - - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + return ERR_PTR(-ENOMEM); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0); + FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0); chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); chcr_req->sec_cpl.aadstart_cipherstop_hi = @@ -1420,37 +1556,52 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, ((kctx_len + sizeof(chcr_req->key_ctx)) >> 4)); chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); - - skb_set_transport_header(skb, transhdr_len); - if (param->bfr_len != 0) - write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, - param->bfr_len); - if (param->sg_len != 0) - write_sg_to_skb(skb, &frags, req->src, param->sg_len); + ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len + + DUMMY_BYTES); + if (param->bfr_len != 0) { + req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev, + req_ctx->reqbfr, param->bfr_len, + DMA_TO_DEVICE); + if (dma_mapping_error(&u_ctx->lldi.pdev->dev, + req_ctx->dma_addr)) { + error = -ENOMEM; + goto err; + } + req_ctx->dma_len = param->bfr_len; + } else { + req_ctx->dma_addr = 0; + } + chcr_add_hash_src_ent(req, ulptx, param); + /* Request upto max wr size */ + temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len + + param->bfr_len) : 0); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, hash_size_in_response, - DUMMY_BYTES + kctx_len, 0); + create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm, + hash_size_in_response, transhdr_len, + temp, 0); req_ctx->skb = skb; - skb_get(skb); return skb; +err: + kfree_skb(skb); + return ERR_PTR(error); } static int chcr_ahash_update(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; u8 remainder = 0, bs; unsigned int nbytes = req->nbytes; struct hash_wr_param params; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1464,7 +1615,9 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->reqlen += nbytes; return 0; } - + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.opad_needed = 0; params.more = 1; params.last = 0; @@ -1475,8 +1628,10 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } if (remainder) { u8 *temp; @@ -1490,10 +1645,13 @@ static int chcr_ahash_update(struct ahash_request *req) } req_ctx->reqlen = remainder; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) @@ -1510,13 +1668,12 @@ static int chcr_ahash_final(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct hash_wr_param params; struct sk_buff *skb; struct uld_ctx *u_ctx = NULL; u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (is_hmac(crypto_ahash_tfm(rtfm))) params.opad_needed = 1; else @@ -1543,7 +1700,7 @@ static int chcr_ahash_final(struct ahash_request *req) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1552,17 +1709,17 @@ static int chcr_ahash_finup(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1588,34 +1745,41 @@ static int chcr_ahash_finup(struct ahash_request *req) params.last = 1; params.more = 0; } + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_digest(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; rtfm->init(req); bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1624,6 +1788,9 @@ static int chcr_ahash_digest(struct ahash_request *req) params.opad_needed = 1; else params.opad_needed = 0; + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.last = 0; params.more = 0; @@ -1641,13 +1808,17 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (IS_ERR(skb)) - return PTR_ERR(skb); - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_export(struct ahash_request *areq, void *out) @@ -1657,6 +1828,8 @@ static int chcr_ahash_export(struct ahash_request *areq, void *out) state->reqlen = req_ctx->reqlen; state->data_len = req_ctx->data_len; + state->is_sg_map = 0; + state->result = 0; memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); memcpy(state->partial_hash, req_ctx->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1672,6 +1845,8 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) req_ctx->data_len = state->data_len; req_ctx->reqbfr = req_ctx->bfr1; req_ctx->skbfr = req_ctx->bfr2; + req_ctx->is_sg_map = 0; + req_ctx->result = 0; memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); memcpy(req_ctx->partial_hash, state->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1681,8 +1856,7 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int i, err = 0, updated_digestsize; @@ -1735,8 +1909,7 @@ out: static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int key_len) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned short context_size = 0; int err; @@ -1775,6 +1948,7 @@ static int chcr_sha_init(struct ahash_request *areq) req_ctx->skbfr = req_ctx->bfr2; req_ctx->skb = NULL; req_ctx->result = 0; + req_ctx->is_sg_map = 0; copy_hash_init_values(req_ctx->partial_hash, digestsize); return 0; } @@ -1790,8 +1964,7 @@ static int chcr_hmac_init(struct ahash_request *areq) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); unsigned int digestsize = crypto_ahash_digestsize(rtfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); @@ -1837,29 +2010,48 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int chcr_copy_assoc(struct aead_request *req, - struct chcr_aead_ctx *ctx) +static int chcr_aead_common_init(struct aead_request *req, + unsigned short op_type) { - SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - - skcipher_request_set_tfm(skreq, ctx->null); - skcipher_request_set_callback(skreq, aead_request_flags(req), - NULL, NULL); - skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, - NULL); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + int error = -EINVAL; + unsigned int dst_size; + unsigned int authsize = crypto_aead_authsize(tfm); - return crypto_skcipher_encrypt(skreq); + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + /* validate key size */ + if (aeadctx->enckey_len == 0) + goto err; + if (op_type && req->cryptlen < authsize) + goto err; + error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + if (error) { + error = -ENOMEM; + goto err; + } + reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen, + CHCR_SRC_SG_SIZE, 0); + reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen, + CHCR_SRC_SG_SIZE, req->assoclen); + return 0; +err: + return error; } -static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, + +static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, int aadmax, int wrlen, unsigned short op_type) { unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || + dst_nents > MAX_DSGL_ENT || (req->assoclen > aadmax) || - (src_nent > MAX_SKB_FRAGS) || - (wrlen > MAX_WR_SIZE)) + (wrlen > SGE_MAX_WR_LEN)) return 1; return 0; } @@ -1867,8 +2059,7 @@ static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_request *subreq = aead_request_ctx(req); aead_request_set_tfm(subreq, aeadctx->sw_cipher); @@ -1887,84 +2078,75 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; - unsigned int kctx_len = 0, nents; - unsigned short stop_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, temp; + unsigned int kctx_len = 0, dnents; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) - goto err; + if (req->cryptlen == 0) + return NULL; - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) - return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } + reqctx->b0_dma = 0; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; assoclen = 0; } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("AUTHENC:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : + authsize); + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, + req->assoclen); + dnents += MIN_AUTH_SG; // For IV + } else { + dnents = 0; } - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) < + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16) + * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* LLD is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - - /* Write WR */ chcr_req = __skb_put_zero(skb, transhdr_len); - stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; /* * Input order is AAD,IV and Payload. where IV should be included as @@ -1972,24 +2154,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, * to the hardware spec */ chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, - (ivsize ? (assoclen + 1) : 0)); - chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); + FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2, + assoclen + 1); + chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, - (stop_offset & 0x1F0) >> 4); + assoclen + IV + 1, + (temp & 0x1F0) >> 4); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( - stop_offset & 0xF, - null ? 0 : assoclen + ivsize + 1, - stop_offset, stop_offset); + temp & 0xF, + null ? 0 : assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, actx->auth_mode, aeadctx->hmac_ctrl, - ivsize >> 1); + IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; if (op_type == CHCR_ENCRYPT_OP) @@ -2002,39 +2184,312 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4), actx->h_iopad, kctx_len - (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); - + memcpy(reqctx->iv, req->iv, IV); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; - - skb_set_transport_header(skb, transhdr_len); - - if (assoclen) { - /* AAD buffer in */ - write_sg_to_skb(skb, &frags, req->src, assoclen); - - } - write_buffer_to_skb(skb, &frags, req->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, size, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + return ERR_PTR(error); } +static int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + int error; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return 0; + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} + +static void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return; + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type) +{ + struct ulptx_walk ulp_walk; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (reqctx->b0_dma) { + memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); + buf += reqctx->b0_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, assoclen, 0); + buf += assoclen; + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, req->cryptlen, req->assoclen); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (reqctx->b0_dma) + ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, + &reqctx->b0_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen, + req->assoclen); + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct dsgl_walk dsgl_walk; + unsigned int authsize = crypto_aead_authsize(tfm); + u32 temp; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + if (reqctx->b0_dma) + dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma); + dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + temp = req->cryptlen + (op_type ? -authsize : authsize); + dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen); + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam) +{ + struct ulptx_walk ulp_walk; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, wrparam->bytes, reqctx->processed); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, + reqctx->src_ofst); + reqctx->srcsg = ulp_walk.last_sg; + reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct dsgl_walk dsgl_walk; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, + reqctx->dst_ofst); + reqctx->dstsg = dsgl_walk.last_sg; + reqctx->dst_ofst = dsgl_walk.last_sg_len; + + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) +{ + struct ulptx_walk ulp_walk; + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (param->bfr_len) { + memcpy(buf, reqctx->reqbfr, param->bfr_len); + buf += param->bfr_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, param->sg_len, 0); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (param->bfr_len) + ulptx_walk_add_page(&ulp_walk, param->bfr_len, + &reqctx->dma_addr); + ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, + 0); +// reqctx->srcsg = ulp_walk.last_sg; +// reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + + +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + int error = 0; + + if (!req->nbytes) + return 0; + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + return error; + req_ctx->is_sg_map = 1; + return 0; +} + +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return; + + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + req_ctx->is_sg_map = 0; + +} + + +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req) +{ + int error; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + static int set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; @@ -2119,15 +2574,13 @@ static int ccm_format_packet(struct aead_request *req, static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int dst_size, struct aead_request *req, - unsigned short op_type, - struct chcr_context *chcrctx) + unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); - unsigned int ivsize = AES_BLOCK_SIZE; + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; - unsigned int c_id = chcrctx->dev->rx_channel_id; + unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id; unsigned int ccm_xtra; unsigned char tag_offset = 0, auth_offset = 0; unsigned int assoclen; @@ -2140,7 +2593,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); auth_offset = req->cryptlen ? - (assoclen + ivsize + 1 + ccm_xtra) : 0; + (assoclen + IV + 1 + ccm_xtra) : 0; if (op_type == CHCR_DECRYPT_OP) { if (crypto_aead_authsize(tfm) != req->cryptlen) tag_offset = crypto_aead_authsize(tfm); @@ -2150,14 +2603,13 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, - 2, (ivsize ? (assoclen + 1) : 0) + - ccm_xtra); + 2, assoclen + 1 + ccm_xtra); sec_cpl->pldlen = - htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); + htonl(assoclen + IV + req->cryptlen + ccm_xtra); /* For CCM there wil be b0 always. So AAD start will be 1 always */ sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1, assoclen + ccm_xtra, assoclen - + ivsize + 1 + ccm_xtra, 0); + + IV + 1 + ccm_xtra, 0); sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, auth_offset, tag_offset, @@ -2166,10 +2618,10 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, cipher_mode, mac_mode, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, - 1, dst_size); + 0, dst_size); } int aead_ccm_validate_input(unsigned short op_type, @@ -2189,119 +2641,83 @@ int aead_ccm_validate_input(unsigned short op_type, return -EINVAL; } } - if (aeadctx->enckey_len == 0) { - pr_err("CCM: Encryption key not set\n"); - return -EINVAL; - } return 0; } -unsigned int fill_aead_req_fields(struct sk_buff *skb, - struct aead_request *req, - struct scatterlist *src, - unsigned int ivsize, - struct chcr_aead_ctx *aeadctx) -{ - unsigned int frags = 0; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); - /* b0 and aad length(if available) */ - - write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + - (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); - if (req->assoclen) { - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) - write_sg_to_skb(skb, &frags, req->src, - req->assoclen - 8); - else - write_sg_to_skb(skb, &frags, req->src, req->assoclen); - } - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - if (req->cryptlen) - write_sg_to_skb(skb, &frags, src, req->cryptlen); - - return frags; -} - static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, unsigned short qid, int size, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents; - unsigned int sub_type; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, kctx_len, dnents, temp; + unsigned int sub_type, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : + reqctx->b0_dma = 0; + sub_type = get_aead_subtype(tfm); + if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) + assoclen -= 8; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); - sub_type = get_aead_subtype(tfm); - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) { - pr_err("AAD copy to destination buffer fails\n"); - return ERR_PTR(error); - } - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("CCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; - } + + reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_CCM_SG; // For IV and B0 + } else { + dnents = 0; + } + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, - T6_MAX_AAD_SIZE - 18, - transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen + + reqctx->b0_len) <= SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen + + reqctx->b0_len), 16) * 16) : + (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_CCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - + reqctx->b0_len, transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len); - chcr_req = __skb_put_zero(skb, transhdr_len); - - fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); + fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); @@ -2309,29 +2725,37 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 16), aeadctx->key, aeadctx->enckey_len); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); error = ccm_format_packet(req, aeadctx, sub_type, op_type); if (error) goto dstmap_fail; - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) + reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + &reqctx->scratch_pad, reqctx->b0_len, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + reqctx->b0_dma)) { + error = -ENOMEM; goto dstmap_fail; + } + + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); - skb_set_transport_header(skb, transhdr_len); - frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, 0, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen + + reqctx->b0_len) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; + return skb; dstmap_fail: kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2341,101 +2765,84 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen; - unsigned char tag_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len, dnents = 0; + unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); - - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - /* validate key size */ - if (aeadctx->enckey_len == 0) - goto err; + struct adapter *adap = padap(a_ctx(tfm)->dev); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) + assoclen = req->assoclen - 8; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); - reqctx->dst = src; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); + reqctx->b0_dma = 0; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); + error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, - req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("GCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, + req->cryptlen + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_GCM_SG; // For IV + } else { + dnents = 0; } - - nents = dst_size ? dstsg_2k(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)) : 0; - dst_size = get_space_for_phys_dsgl(nents); + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <= + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + + req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents + + reqctx->aad_nents + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* NIC driver is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - chcr_req = __skb_put_zero(skb, transhdr_len); - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) - assoclen = req->assoclen - 8; - - tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + //Offset of tag from end + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( - ctx->dev->rx_channel_id, 2, (ivsize ? - (assoclen + 1) : 0)); + a_ctx(tfm)->dev->rx_channel_id, 2, + (assoclen + 1)); chcr_req->sec_cpl.pldlen = - htonl(assoclen + ivsize + req->cryptlen); + htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, 0); + assoclen + IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1, - tag_offset, tag_offset); + FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_GCM, CHCR_SCMD_AUTH_MODE_GHASH, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * @@ -2453,30 +2860,21 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); - skb_set_transport_header(skb, transhdr_len); - write_sg_to_skb(skb, &frags, req->src, assoclen); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, size, - sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len, - reqctx->verify); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, reqctx->verify); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2484,8 +2882,7 @@ err: static int chcr_aead_cra_init(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_alg *alg = crypto_aead_alg(tfm); aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, @@ -2496,25 +2893,20 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), sizeof(struct aead_request) + crypto_aead_reqsize(aeadctx->sw_cipher))); - aeadctx->null = crypto_get_default_null_skcipher(); - if (IS_ERR(aeadctx->null)) - return PTR_ERR(aeadctx->null); - return chcr_device_init(ctx); + return chcr_device_init(a_ctx(tfm)); } static void chcr_aead_cra_exit(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); - crypto_put_default_null_skcipher(); crypto_free_aead(aeadctx->sw_cipher); } static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; aeadctx->mayverify = VERIFY_HW; @@ -2523,7 +2915,7 @@ static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, static int chcr_authenc_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); u32 maxauth = crypto_aead_maxauthsize(tfm); /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not @@ -2561,7 +2953,7 @@ static int chcr_authenc_setauthsize(struct crypto_aead *tfm, static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2601,7 +2993,7 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_8: @@ -2627,7 +3019,7 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, static int chcr_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2670,8 +3062,7 @@ static int chcr_ccm_common_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); unsigned char ck_size, mk_size; int key_ctx_size = 0; @@ -2704,8 +3095,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -2723,8 +3113,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; if (keylen < 3) { @@ -2750,8 +3139,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); struct crypto_cipher *cipher; unsigned int ck_size; @@ -2823,8 +3211,7 @@ out: static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; @@ -2944,8 +3331,7 @@ out: static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct crypto_authenc_keys keys; int err; @@ -3017,7 +3403,7 @@ static int chcr_aead_encrypt(struct aead_request *req) static int chcr_aead_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); int size; @@ -3050,30 +3436,29 @@ static int chcr_aead_op(struct aead_request *req, create_wr_t create_wr_fn) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); struct uld_ctx *u_ctx; struct sk_buff *skb; - if (!ctx->dev) { + if (!a_ctx(tfm)->dev) { pr_err("chcr : %s : No crypto device.\n", __func__); return -ENXIO; } - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(a_ctx(tfm)); if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx)) { + a_ctx(tfm)->tx_qidx)) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, op_type); if (IS_ERR(skb) || !skb) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index d1a787b369c9..96c9335ee728 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -214,27 +214,22 @@ calc_tx_flits_ofld(skb) * 8), 16))) #define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\ - ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1)) - + ULP_TX_SC_MORE_V((immdatalen))) #define MAX_NK 8 -#define CRYPTO_MAX_IMM_TX_PKT_LEN 256 -#define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 1) #define MIN_CIPHER_SG 1 /* IV */ -#define MIN_AUTH_SG 2 /*IV + AAD*/ -#define MIN_GCM_SG 2 /* IV + AAD*/ +#define MIN_AUTH_SG 1 /* IV */ +#define MIN_GCM_SG 1 /* IV */ #define MIN_DIGEST_SG 1 /*Partial Buffer*/ -#define MIN_CCM_SG 3 /*IV+AAD+B0*/ +#define MIN_CCM_SG 2 /*IV+B0*/ #define SPACE_LEFT(len) \ - ((MAX_WR_SIZE - WR_MIN_LEN - (len))) + ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len))) -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, - 48, 64, 72, 88, - 96, 112, 120, 136, - 144, 160, 168, 184, - 192}; +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376}; unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, 112, 112, 128, 128, 144, 144, 160, 160, 192, 192, 208, 208, 224, 224, 240, 240, @@ -258,7 +253,6 @@ struct hash_wr_param { struct cipher_wr_param { struct ablkcipher_request *req; - struct scatterlist *srcsg; char *iv; int bytes; unsigned short qid; @@ -298,31 +292,11 @@ enum { ICV_16 = 16 }; -struct hash_op_params { - unsigned char mk_size; - unsigned char pad_align; - unsigned char auth_mode; - char hash_name[MAX_HASH_NAME]; - unsigned short block_size; - unsigned short word_size; - unsigned short ipad_size; -}; - struct phys_sge_pairs { __be16 len[8]; __be64 addr[8]; }; -struct phys_sge_parm { - unsigned int nents; - unsigned int obsize; - unsigned short qid; -}; - -struct crypto_result { - struct completion completion; - int err; -}; static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 8f436f80cb21..94a87e3ad9bc 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -149,9 +149,23 @@ #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 -#define CHCR_SG_SIZE 2048 +#define CHCR_SRC_SG_SIZE (0x10000 - sizeof(int)) +#define CHCR_DST_SG_SIZE 2048 -/* Aligned to 128 bit boundary */ +static inline struct chcr_context *a_ctx(struct crypto_aead *tfm) +{ + return crypto_aead_ctx(tfm); +} + +static inline struct chcr_context *c_ctx(struct crypto_ablkcipher *tfm) +{ + return crypto_ablkcipher_ctx(tfm); +} + +static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm) +{ + return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); +} struct ablk_ctx { struct crypto_skcipher *sw_cipher; @@ -165,15 +179,39 @@ struct ablk_ctx { }; struct chcr_aead_reqctx { struct sk_buff *skb; - struct scatterlist *dst; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; + dma_addr_t iv_dma; + dma_addr_t b0_dma; + unsigned int b0_len; + unsigned int op; + short int aad_nents; + short int src_nents; short int dst_nents; + u16 imm; u16 verify; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE]; }; +struct ulptx_walk { + struct ulptx_sgl *sgl; + unsigned int nents; + unsigned int pair_idx; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct ulptx_sge_pair *pair; + +}; + +struct dsgl_walk { + unsigned int nents; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct cpl_rx_phys_dsgl *dsgl; + struct phys_sge_pairs *to; +}; + + + struct chcr_gcm_ctx { u8 ghash_h[AEAD_H_SIZE]; }; @@ -194,7 +232,6 @@ struct __aead_ctx { struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; - struct crypto_skcipher *null; struct crypto_aead *sw_cipher; u8 salt[MAX_SALT]; u8 key[CHCR_AES_MAX_KEY_LEN]; @@ -230,8 +267,11 @@ struct chcr_ahash_req_ctx { u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128]; u8 *reqbfr; u8 *skbfr; + dma_addr_t dma_addr; + u32 dma_len; u8 reqlen; - /* DMA the partial hash in it */ + u8 imm; + u8 is_sg_map; u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE]; u64 data_len; /* Data len till time */ /* SKB which is being sent to the hardware for processing */ @@ -240,14 +280,15 @@ struct chcr_ahash_req_ctx { struct chcr_blkcipher_req_ctx { struct sk_buff *skb; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; struct scatterlist *dstsg; - struct scatterlist *dst; unsigned int processed; unsigned int last_req_len; + struct scatterlist *srcsg; + unsigned int src_ofst; + unsigned int dst_ofst; unsigned int op; - short int dst_nents; + dma_addr_t iv_dma; + u16 imm; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; }; @@ -261,24 +302,6 @@ struct chcr_alg_template { } alg; }; -struct chcr_req_ctx { - union { - struct ahash_request *ahash_req; - struct aead_request *aead_req; - struct ablkcipher_request *ablk_req; - } req; - union { - struct chcr_ahash_req_ctx *ahash_ctx; - struct chcr_aead_reqctx *reqctx; - struct chcr_blkcipher_req_ctx *ablk_ctx; - } ctx; -}; - -struct sge_opaque_hdr { - void *dev; - dma_addr_t addr[MAX_SKB_FRAGS + 1]; -}; - typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, unsigned short qid, int size, @@ -291,4 +314,37 @@ static int chcr_aead_op(struct aead_request *req_base, static inline int get_aead_subtype(struct crypto_aead *aead); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); +static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +static int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +static void chcr_aead_dma_unmap(struct device *dev, struct aead_request + *req, unsigned short op_type); +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid); +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type); +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam); +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req); +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req); +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); +int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip); +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req); +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req); #endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 4ef68f69b58c..6a934c758fbb 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -1537,7 +1537,13 @@ int t4_mgmt_tx(struct adapter *adap, struct sk_buff *skb) */ static inline int is_ofld_imm(const struct sk_buff *skb) { - return skb->len <= MAX_IMM_TX_PKT_LEN; + struct work_request_hdr *req = (struct work_request_hdr *)skb->data; + unsigned long opcode = FW_WR_OP_G(ntohl(req->wr_hi)); + + if (opcode == FW_CRYPTO_LOOKASIDE_WR) + return skb->len <= SGE_MAX_WR_LEN; + else + return skb->len <= MAX_IMM_TX_PKT_LEN; } /** -- cgit v1.2.3-59-g8ed1b From 396d34f9537619842b081cbe0b5ce8dbbfa00a81 Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Sun, 8 Oct 2017 13:37:24 +0530 Subject: crypto: chelsio - Fix memory leak Fix memory leak when device does not support crypto. Reported-by: Dan Carpenter Signed-off-by: Harsh Jain Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_core.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index 32618bf0ab3b..f5a2624081dc 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -153,16 +153,16 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) { struct uld_ctx *u_ctx; + /* Create the device and add it in the device list */ + if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) + return ERR_PTR(-EOPNOTSUPP); + /* Create the device and add it in the device list */ u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); if (!u_ctx) { u_ctx = ERR_PTR(-ENOMEM); goto out; } - if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) { - u_ctx = ERR_PTR(-ENOMEM); - goto out; - } u_ctx->lldi = *lld; out: return u_ctx; -- cgit v1.2.3-59-g8ed1b From 7b0c3d693ce65900dd3c79766185f539fa37a29a Mon Sep 17 00:00:00 2001 From: Boris BREZILLON Date: Wed, 11 Oct 2017 15:16:17 +0200 Subject: crypto: marvell - Add a platform_device_id table Add a platform_device_id table to allow using this driver on orion platforms that have not been converted to DT. Signed-off-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index b657e7c2107e..1135cac869ac 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -597,9 +597,15 @@ static int mv_cesa_remove(struct platform_device *pdev) return 0; } +static const struct platform_device_id mv_cesa_plat_id_table[] = { + { .name = "mv_crypto" }, +}; +MODULE_DEVICE_TABLE(platform, mv_cesa_plat_id_table); + static struct platform_driver marvell_cesa = { .probe = mv_cesa_probe, .remove = mv_cesa_remove, + .id_table = mv_cesa_plat_id_table, .driver = { .name = "marvell-cesa", .of_match_table = mv_cesa_of_match_table, -- cgit v1.2.3-59-g8ed1b From 27b43fd95b144484713855c6d4fe832d22e48838 Mon Sep 17 00:00:00 2001 From: Boris BREZILLON Date: Wed, 11 Oct 2017 15:16:19 +0200 Subject: crypto: marvell - Remove the old mv_cesa driver All defconfigs selecting the old driver have been patched to select the new one instead. We can now remove the old driver along with the allhwsupports module parameter in the new driver that was used to check whether the new driver was allowed to take control of the CESA engine or not. Signed-off-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 22 +- drivers/crypto/Makefile | 1 - drivers/crypto/marvell/cesa.c | 7 - drivers/crypto/mv_cesa.c | 1213 ----------------------------------------- drivers/crypto/mv_cesa.h | 150 ----- 5 files changed, 3 insertions(+), 1390 deletions(-) delete mode 100644 drivers/crypto/mv_cesa.c delete mode 100644 drivers/crypto/mv_cesa.h (limited to 'drivers/crypto') diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index de825b354fdf..9f55be46a07d 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -199,22 +199,8 @@ config CRYPTO_CRC32_S390 It is available with IBM z13 or later. -config CRYPTO_DEV_MV_CESA - tristate "Marvell's Cryptographic Engine" - depends on PLAT_ORION - select CRYPTO_AES - select CRYPTO_BLKCIPHER - select CRYPTO_HASH - select SRAM - help - This driver allows you to utilize the Cryptographic Engines and - Security Accelerator (CESA) which can be found on the Marvell Orion - and Kirkwood SoCs, such as QNAP's TS-209. - - Currently the driver supports AES in ECB and CBC mode without DMA. - config CRYPTO_DEV_MARVELL_CESA - tristate "New Marvell's Cryptographic Engine driver" + tristate "Marvell's Cryptographic Engine driver" depends on PLAT_ORION || ARCH_MVEBU select CRYPTO_AES select CRYPTO_DES @@ -223,12 +209,10 @@ config CRYPTO_DEV_MARVELL_CESA select SRAM help This driver allows you to utilize the Cryptographic Engines and - Security Accelerator (CESA) which can be found on the Armada 370. + Security Accelerator (CESA) which can be found on MVEBU and ORION + platforms. This driver supports CPU offload through DMA transfers. - This driver is aimed at replacing the mv_cesa driver. This will only - happen once it has received proper testing. - config CRYPTO_DEV_NIAGARA2 tristate "Niagara2 Stream Processing Unit driver" select CRYPTO_DES diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 808432b44c6b..4292f254b01c 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -14,7 +14,6 @@ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o -obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell/ obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mediatek/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 1135cac869ac..d713d9697378 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -34,10 +34,6 @@ /* Limit of the crypto queue before reaching the backlog */ #define CESA_CRYPTO_DEFAULT_MAX_QLEN 128 -static int allhwsupport = !IS_ENABLED(CONFIG_CRYPTO_DEV_MV_CESA); -module_param_named(allhwsupport, allhwsupport, int, 0444); -MODULE_PARM_DESC(allhwsupport, "Enable support for all hardware (even it if overlaps with the mv_cesa driver)"); - struct mv_cesa_dev *cesa_dev; struct crypto_async_request * @@ -457,9 +453,6 @@ static int mv_cesa_probe(struct platform_device *pdev) caps = match->data; } - if ((caps == &orion_caps || caps == &kirkwood_caps) && !allhwsupport) - return -ENOTSUPP; - cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); if (!cesa) return -ENOMEM; diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c deleted file mode 100644 index c3883b49f56e..000000000000 --- a/drivers/crypto/mv_cesa.c +++ /dev/null @@ -1,1213 +0,0 @@ -/* - * Support for Marvell's crypto engine which can be found on some Orion5X - * boards. - * - * Author: Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > - * License: GPLv2 - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mv_cesa.h" - -#define MV_CESA "MV-CESA:" -#define MAX_HW_HASH_SIZE 0xFFFF -#define MV_CESA_EXPIRE 500 /* msec */ - -#define MV_CESA_DEFAULT_SRAM_SIZE 2048 - -/* - * STM: - * /---------------------------------------\ - * | | request complete - * \./ | - * IDLE -> new request -> BUSY -> done -> DEQUEUE - * /°\ | - * | | more scatter entries - * \________________/ - */ -enum engine_status { - ENGINE_IDLE, - ENGINE_BUSY, - ENGINE_W_DEQUEUE, -}; - -/** - * struct req_progress - used for every crypt request - * @src_sg_it: sg iterator for src - * @dst_sg_it: sg iterator for dst - * @sg_src_left: bytes left in src to process (scatter list) - * @src_start: offset to add to src start position (scatter list) - * @crypt_len: length of current hw crypt/hash process - * @hw_nbytes: total bytes to process in hw for this request - * @copy_back: whether to copy data back (crypt) or not (hash) - * @sg_dst_left: bytes left dst to process in this scatter list - * @dst_start: offset to add to dst start position (scatter list) - * @hw_processed_bytes: number of bytes processed by hw (request). - * - * sg helper are used to iterate over the scatterlist. Since the size of the - * SRAM may be less than the scatter size, this struct struct is used to keep - * track of progress within current scatterlist. - */ -struct req_progress { - struct sg_mapping_iter src_sg_it; - struct sg_mapping_iter dst_sg_it; - void (*complete) (void); - void (*process) (int is_first); - - /* src mostly */ - int sg_src_left; - int src_start; - int crypt_len; - int hw_nbytes; - /* dst mostly */ - int copy_back; - int sg_dst_left; - int dst_start; - int hw_processed_bytes; -}; - -struct crypto_priv { - void __iomem *reg; - void __iomem *sram; - struct gen_pool *sram_pool; - dma_addr_t sram_dma; - int irq; - struct clk *clk; - struct task_struct *queue_th; - - /* the lock protects queue and eng_st */ - spinlock_t lock; - struct crypto_queue queue; - enum engine_status eng_st; - struct timer_list completion_timer; - struct crypto_async_request *cur_req; - struct req_progress p; - int max_req_size; - int sram_size; - int has_sha1; - int has_hmac_sha1; -}; - -static struct crypto_priv *cpg; - -struct mv_ctx { - u8 aes_enc_key[AES_KEY_LEN]; - u32 aes_dec_key[8]; - int key_len; - u32 need_calc_aes_dkey; -}; - -enum crypto_op { - COP_AES_ECB, - COP_AES_CBC, -}; - -struct mv_req_ctx { - enum crypto_op op; - int decrypt; -}; - -enum hash_op { - COP_SHA1, - COP_HMAC_SHA1 -}; - -struct mv_tfm_hash_ctx { - struct crypto_shash *fallback; - struct crypto_shash *base_hash; - u32 ivs[2 * SHA1_DIGEST_SIZE / 4]; - int count_add; - enum hash_op op; -}; - -struct mv_req_hash_ctx { - u64 count; - u32 state[SHA1_DIGEST_SIZE / 4]; - u8 buffer[SHA1_BLOCK_SIZE]; - int first_hash; /* marks that we don't have previous state */ - int last_chunk; /* marks that this is the 'final' request */ - int extra_bytes; /* unprocessed bytes in buffer */ - enum hash_op op; - int count_add; -}; - -static void mv_completion_timer_callback(unsigned long unused) -{ - int active = readl(cpg->reg + SEC_ACCEL_CMD) & SEC_CMD_EN_SEC_ACCL0; - - printk(KERN_ERR MV_CESA - "completion timer expired (CESA %sactive), cleaning up.\n", - active ? "" : "in"); - - del_timer(&cpg->completion_timer); - writel(SEC_CMD_DISABLE_SEC, cpg->reg + SEC_ACCEL_CMD); - while(readl(cpg->reg + SEC_ACCEL_CMD) & SEC_CMD_DISABLE_SEC) - printk(KERN_INFO MV_CESA "%s: waiting for engine finishing\n", __func__); - cpg->eng_st = ENGINE_W_DEQUEUE; - wake_up_process(cpg->queue_th); -} - -static void mv_setup_timer(void) -{ - setup_timer(&cpg->completion_timer, &mv_completion_timer_callback, 0); - mod_timer(&cpg->completion_timer, - jiffies + msecs_to_jiffies(MV_CESA_EXPIRE)); -} - -static void compute_aes_dec_key(struct mv_ctx *ctx) -{ - struct crypto_aes_ctx gen_aes_key; - int key_pos; - - if (!ctx->need_calc_aes_dkey) - return; - - crypto_aes_expand_key(&gen_aes_key, ctx->aes_enc_key, ctx->key_len); - - key_pos = ctx->key_len + 24; - memcpy(ctx->aes_dec_key, &gen_aes_key.key_enc[key_pos], 4 * 4); - switch (ctx->key_len) { - case AES_KEYSIZE_256: - key_pos -= 2; - /* fall */ - case AES_KEYSIZE_192: - key_pos -= 2; - memcpy(&ctx->aes_dec_key[4], &gen_aes_key.key_enc[key_pos], - 4 * 4); - break; - } - ctx->need_calc_aes_dkey = 0; -} - -static int mv_setkey_aes(struct crypto_ablkcipher *cipher, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - struct mv_ctx *ctx = crypto_tfm_ctx(tfm); - - switch (len) { - case AES_KEYSIZE_128: - case AES_KEYSIZE_192: - case AES_KEYSIZE_256: - break; - default: - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - ctx->key_len = len; - ctx->need_calc_aes_dkey = 1; - - memcpy(ctx->aes_enc_key, key, AES_KEY_LEN); - return 0; -} - -static void copy_src_to_buf(struct req_progress *p, char *dbuf, int len) -{ - int ret; - void *sbuf; - int copy_len; - - while (len) { - if (!p->sg_src_left) { - ret = sg_miter_next(&p->src_sg_it); - BUG_ON(!ret); - p->sg_src_left = p->src_sg_it.length; - p->src_start = 0; - } - - sbuf = p->src_sg_it.addr + p->src_start; - - copy_len = min(p->sg_src_left, len); - memcpy(dbuf, sbuf, copy_len); - - p->src_start += copy_len; - p->sg_src_left -= copy_len; - - len -= copy_len; - dbuf += copy_len; - } -} - -static void setup_data_in(void) -{ - struct req_progress *p = &cpg->p; - int data_in_sram = - min(p->hw_nbytes - p->hw_processed_bytes, cpg->max_req_size); - copy_src_to_buf(p, cpg->sram + SRAM_DATA_IN_START + p->crypt_len, - data_in_sram - p->crypt_len); - p->crypt_len = data_in_sram; -} - -static void mv_process_current_q(int first_block) -{ - struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req); - struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - struct sec_accel_config op; - - switch (req_ctx->op) { - case COP_AES_ECB: - op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_ECB; - break; - case COP_AES_CBC: - default: - op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_CBC; - op.enc_iv = ENC_IV_POINT(SRAM_DATA_IV) | - ENC_IV_BUF_POINT(SRAM_DATA_IV_BUF); - if (first_block) - memcpy(cpg->sram + SRAM_DATA_IV, req->info, 16); - break; - } - if (req_ctx->decrypt) { - op.config |= CFG_DIR_DEC; - memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_dec_key, - AES_KEY_LEN); - } else { - op.config |= CFG_DIR_ENC; - memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_enc_key, - AES_KEY_LEN); - } - - switch (ctx->key_len) { - case AES_KEYSIZE_128: - op.config |= CFG_AES_LEN_128; - break; - case AES_KEYSIZE_192: - op.config |= CFG_AES_LEN_192; - break; - case AES_KEYSIZE_256: - op.config |= CFG_AES_LEN_256; - break; - } - op.enc_p = ENC_P_SRC(SRAM_DATA_IN_START) | - ENC_P_DST(SRAM_DATA_OUT_START); - op.enc_key_p = SRAM_DATA_KEY_P; - - setup_data_in(); - op.enc_len = cpg->p.crypt_len; - memcpy(cpg->sram + SRAM_CONFIG, &op, - sizeof(struct sec_accel_config)); - - /* GO */ - mv_setup_timer(); - writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD); -} - -static void mv_crypto_algo_completion(void) -{ - struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - sg_miter_stop(&cpg->p.src_sg_it); - sg_miter_stop(&cpg->p.dst_sg_it); - - if (req_ctx->op != COP_AES_CBC) - return ; - - memcpy(req->info, cpg->sram + SRAM_DATA_IV_BUF, 16); -} - -static void mv_process_hash_current(int first_block) -{ - struct ahash_request *req = ahash_request_cast(cpg->cur_req); - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req); - struct req_progress *p = &cpg->p; - struct sec_accel_config op = { 0 }; - int is_last; - - switch (req_ctx->op) { - case COP_SHA1: - default: - op.config = CFG_OP_MAC_ONLY | CFG_MACM_SHA1; - break; - case COP_HMAC_SHA1: - op.config = CFG_OP_MAC_ONLY | CFG_MACM_HMAC_SHA1; - memcpy(cpg->sram + SRAM_HMAC_IV_IN, - tfm_ctx->ivs, sizeof(tfm_ctx->ivs)); - break; - } - - op.mac_src_p = - MAC_SRC_DATA_P(SRAM_DATA_IN_START) | MAC_SRC_TOTAL_LEN((u32) - req_ctx-> - count); - - setup_data_in(); - - op.mac_digest = - MAC_DIGEST_P(SRAM_DIGEST_BUF) | MAC_FRAG_LEN(p->crypt_len); - op.mac_iv = - MAC_INNER_IV_P(SRAM_HMAC_IV_IN) | - MAC_OUTER_IV_P(SRAM_HMAC_IV_OUT); - - is_last = req_ctx->last_chunk - && (p->hw_processed_bytes + p->crypt_len >= p->hw_nbytes) - && (req_ctx->count <= MAX_HW_HASH_SIZE); - if (req_ctx->first_hash) { - if (is_last) - op.config |= CFG_NOT_FRAG; - else - op.config |= CFG_FIRST_FRAG; - - req_ctx->first_hash = 0; - } else { - if (is_last) - op.config |= CFG_LAST_FRAG; - else - op.config |= CFG_MID_FRAG; - - if (first_block) { - writel(req_ctx->state[0], cpg->reg + DIGEST_INITIAL_VAL_A); - writel(req_ctx->state[1], cpg->reg + DIGEST_INITIAL_VAL_B); - writel(req_ctx->state[2], cpg->reg + DIGEST_INITIAL_VAL_C); - writel(req_ctx->state[3], cpg->reg + DIGEST_INITIAL_VAL_D); - writel(req_ctx->state[4], cpg->reg + DIGEST_INITIAL_VAL_E); - } - } - - memcpy(cpg->sram + SRAM_CONFIG, &op, sizeof(struct sec_accel_config)); - - /* GO */ - mv_setup_timer(); - writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD); -} - -static inline int mv_hash_import_sha1_ctx(const struct mv_req_hash_ctx *ctx, - struct shash_desc *desc) -{ - int i; - struct sha1_state shash_state; - - shash_state.count = ctx->count + ctx->count_add; - for (i = 0; i < 5; i++) - shash_state.state[i] = ctx->state[i]; - memcpy(shash_state.buffer, ctx->buffer, sizeof(shash_state.buffer)); - return crypto_shash_import(desc, &shash_state); -} - -static int mv_hash_final_fallback(struct ahash_request *req) -{ - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req); - SHASH_DESC_ON_STACK(shash, tfm_ctx->fallback); - int rc; - - shash->tfm = tfm_ctx->fallback; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - if (unlikely(req_ctx->first_hash)) { - crypto_shash_init(shash); - crypto_shash_update(shash, req_ctx->buffer, - req_ctx->extra_bytes); - } else { - /* only SHA1 for now.... - */ - rc = mv_hash_import_sha1_ctx(req_ctx, shash); - if (rc) - goto out; - } - rc = crypto_shash_final(shash, req->result); -out: - return rc; -} - -static void mv_save_digest_state(struct mv_req_hash_ctx *ctx) -{ - ctx->state[0] = readl(cpg->reg + DIGEST_INITIAL_VAL_A); - ctx->state[1] = readl(cpg->reg + DIGEST_INITIAL_VAL_B); - ctx->state[2] = readl(cpg->reg + DIGEST_INITIAL_VAL_C); - ctx->state[3] = readl(cpg->reg + DIGEST_INITIAL_VAL_D); - ctx->state[4] = readl(cpg->reg + DIGEST_INITIAL_VAL_E); -} - -static void mv_hash_algo_completion(void) -{ - struct ahash_request *req = ahash_request_cast(cpg->cur_req); - struct mv_req_hash_ctx *ctx = ahash_request_ctx(req); - - if (ctx->extra_bytes) - copy_src_to_buf(&cpg->p, ctx->buffer, ctx->extra_bytes); - sg_miter_stop(&cpg->p.src_sg_it); - - if (likely(ctx->last_chunk)) { - if (likely(ctx->count <= MAX_HW_HASH_SIZE)) { - memcpy(req->result, cpg->sram + SRAM_DIGEST_BUF, - crypto_ahash_digestsize(crypto_ahash_reqtfm - (req))); - } else { - mv_save_digest_state(ctx); - mv_hash_final_fallback(req); - } - } else { - mv_save_digest_state(ctx); - } -} - -static void dequeue_complete_req(void) -{ - struct crypto_async_request *req = cpg->cur_req; - void *buf; - int ret; - cpg->p.hw_processed_bytes += cpg->p.crypt_len; - if (cpg->p.copy_back) { - int need_copy_len = cpg->p.crypt_len; - int sram_offset = 0; - do { - int dst_copy; - - if (!cpg->p.sg_dst_left) { - ret = sg_miter_next(&cpg->p.dst_sg_it); - BUG_ON(!ret); - cpg->p.sg_dst_left = cpg->p.dst_sg_it.length; - cpg->p.dst_start = 0; - } - - buf = cpg->p.dst_sg_it.addr; - buf += cpg->p.dst_start; - - dst_copy = min(need_copy_len, cpg->p.sg_dst_left); - - memcpy(buf, - cpg->sram + SRAM_DATA_OUT_START + sram_offset, - dst_copy); - sram_offset += dst_copy; - cpg->p.sg_dst_left -= dst_copy; - need_copy_len -= dst_copy; - cpg->p.dst_start += dst_copy; - } while (need_copy_len > 0); - } - - cpg->p.crypt_len = 0; - - BUG_ON(cpg->eng_st != ENGINE_W_DEQUEUE); - if (cpg->p.hw_processed_bytes < cpg->p.hw_nbytes) { - /* process next scatter list entry */ - cpg->eng_st = ENGINE_BUSY; - cpg->p.process(0); - } else { - cpg->p.complete(); - cpg->eng_st = ENGINE_IDLE; - local_bh_disable(); - req->complete(req, 0); - local_bh_enable(); - } -} - -static int count_sgs(struct scatterlist *sl, unsigned int total_bytes) -{ - int i = 0; - size_t cur_len; - - while (sl) { - cur_len = sl[i].length; - ++i; - if (total_bytes > cur_len) - total_bytes -= cur_len; - else - break; - } - - return i; -} - -static void mv_start_new_crypt_req(struct ablkcipher_request *req) -{ - struct req_progress *p = &cpg->p; - int num_sgs; - - cpg->cur_req = &req->base; - memset(p, 0, sizeof(struct req_progress)); - p->hw_nbytes = req->nbytes; - p->complete = mv_crypto_algo_completion; - p->process = mv_process_current_q; - p->copy_back = 1; - - num_sgs = count_sgs(req->src, req->nbytes); - sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG); - - num_sgs = count_sgs(req->dst, req->nbytes); - sg_miter_start(&p->dst_sg_it, req->dst, num_sgs, SG_MITER_TO_SG); - - mv_process_current_q(1); -} - -static void mv_start_new_hash_req(struct ahash_request *req) -{ - struct req_progress *p = &cpg->p; - struct mv_req_hash_ctx *ctx = ahash_request_ctx(req); - int num_sgs, hw_bytes, old_extra_bytes, rc; - cpg->cur_req = &req->base; - memset(p, 0, sizeof(struct req_progress)); - hw_bytes = req->nbytes + ctx->extra_bytes; - old_extra_bytes = ctx->extra_bytes; - - ctx->extra_bytes = hw_bytes % SHA1_BLOCK_SIZE; - if (ctx->extra_bytes != 0 - && (!ctx->last_chunk || ctx->count > MAX_HW_HASH_SIZE)) - hw_bytes -= ctx->extra_bytes; - else - ctx->extra_bytes = 0; - - num_sgs = count_sgs(req->src, req->nbytes); - sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG); - - if (hw_bytes) { - p->hw_nbytes = hw_bytes; - p->complete = mv_hash_algo_completion; - p->process = mv_process_hash_current; - - if (unlikely(old_extra_bytes)) { - memcpy(cpg->sram + SRAM_DATA_IN_START, ctx->buffer, - old_extra_bytes); - p->crypt_len = old_extra_bytes; - } - - mv_process_hash_current(1); - } else { - copy_src_to_buf(p, ctx->buffer + old_extra_bytes, - ctx->extra_bytes - old_extra_bytes); - sg_miter_stop(&p->src_sg_it); - if (ctx->last_chunk) - rc = mv_hash_final_fallback(req); - else - rc = 0; - cpg->eng_st = ENGINE_IDLE; - local_bh_disable(); - req->base.complete(&req->base, rc); - local_bh_enable(); - } -} - -static int queue_manag(void *data) -{ - cpg->eng_st = ENGINE_IDLE; - do { - struct crypto_async_request *async_req = NULL; - struct crypto_async_request *backlog = NULL; - - __set_current_state(TASK_INTERRUPTIBLE); - - if (cpg->eng_st == ENGINE_W_DEQUEUE) - dequeue_complete_req(); - - spin_lock_irq(&cpg->lock); - if (cpg->eng_st == ENGINE_IDLE) { - backlog = crypto_get_backlog(&cpg->queue); - async_req = crypto_dequeue_request(&cpg->queue); - if (async_req) { - BUG_ON(cpg->eng_st != ENGINE_IDLE); - cpg->eng_st = ENGINE_BUSY; - } - } - spin_unlock_irq(&cpg->lock); - - if (backlog) { - backlog->complete(backlog, -EINPROGRESS); - backlog = NULL; - } - - if (async_req) { - if (crypto_tfm_alg_type(async_req->tfm) != - CRYPTO_ALG_TYPE_AHASH) { - struct ablkcipher_request *req = - ablkcipher_request_cast(async_req); - mv_start_new_crypt_req(req); - } else { - struct ahash_request *req = - ahash_request_cast(async_req); - mv_start_new_hash_req(req); - } - async_req = NULL; - } - - schedule(); - - } while (!kthread_should_stop()); - return 0; -} - -static int mv_handle_req(struct crypto_async_request *req) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&cpg->lock, flags); - ret = crypto_enqueue_request(&cpg->queue, req); - spin_unlock_irqrestore(&cpg->lock, flags); - wake_up_process(cpg->queue_th); - return ret; -} - -static int mv_enc_aes_ecb(struct ablkcipher_request *req) -{ - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_ECB; - req_ctx->decrypt = 0; - - return mv_handle_req(&req->base); -} - -static int mv_dec_aes_ecb(struct ablkcipher_request *req) -{ - struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_ECB; - req_ctx->decrypt = 1; - - compute_aes_dec_key(ctx); - return mv_handle_req(&req->base); -} - -static int mv_enc_aes_cbc(struct ablkcipher_request *req) -{ - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_CBC; - req_ctx->decrypt = 0; - - return mv_handle_req(&req->base); -} - -static int mv_dec_aes_cbc(struct ablkcipher_request *req) -{ - struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_CBC; - req_ctx->decrypt = 1; - - compute_aes_dec_key(ctx); - return mv_handle_req(&req->base); -} - -static int mv_cra_init(struct crypto_tfm *tfm) -{ - tfm->crt_ablkcipher.reqsize = sizeof(struct mv_req_ctx); - return 0; -} - -static void mv_init_hash_req_ctx(struct mv_req_hash_ctx *ctx, int op, - int is_last, unsigned int req_len, - int count_add) -{ - memset(ctx, 0, sizeof(*ctx)); - ctx->op = op; - ctx->count = req_len; - ctx->first_hash = 1; - ctx->last_chunk = is_last; - ctx->count_add = count_add; -} - -static void mv_update_hash_req_ctx(struct mv_req_hash_ctx *ctx, int is_last, - unsigned req_len) -{ - ctx->last_chunk = is_last; - ctx->count += req_len; -} - -static int mv_hash_init(struct ahash_request *req) -{ - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 0, 0, - tfm_ctx->count_add); - return 0; -} - -static int mv_hash_update(struct ahash_request *req) -{ - if (!req->nbytes) - return 0; - - mv_update_hash_req_ctx(ahash_request_ctx(req), 0, req->nbytes); - return mv_handle_req(&req->base); -} - -static int mv_hash_final(struct ahash_request *req) -{ - struct mv_req_hash_ctx *ctx = ahash_request_ctx(req); - - ahash_request_set_crypt(req, NULL, req->result, 0); - mv_update_hash_req_ctx(ctx, 1, 0); - return mv_handle_req(&req->base); -} - -static int mv_hash_finup(struct ahash_request *req) -{ - mv_update_hash_req_ctx(ahash_request_ctx(req), 1, req->nbytes); - return mv_handle_req(&req->base); -} - -static int mv_hash_digest(struct ahash_request *req) -{ - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 1, - req->nbytes, tfm_ctx->count_add); - return mv_handle_req(&req->base); -} - -static void mv_hash_init_ivs(struct mv_tfm_hash_ctx *ctx, const void *istate, - const void *ostate) -{ - const struct sha1_state *isha1_state = istate, *osha1_state = ostate; - int i; - for (i = 0; i < 5; i++) { - ctx->ivs[i] = cpu_to_be32(isha1_state->state[i]); - ctx->ivs[i + 5] = cpu_to_be32(osha1_state->state[i]); - } -} - -static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key, - unsigned int keylen) -{ - int rc; - struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(&tfm->base); - int bs, ds, ss; - - if (!ctx->base_hash) - return 0; - - rc = crypto_shash_setkey(ctx->fallback, key, keylen); - if (rc) - return rc; - - /* Can't see a way to extract the ipad/opad from the fallback tfm - so I'm basically copying code from the hmac module */ - bs = crypto_shash_blocksize(ctx->base_hash); - ds = crypto_shash_digestsize(ctx->base_hash); - ss = crypto_shash_statesize(ctx->base_hash); - - { - SHASH_DESC_ON_STACK(shash, ctx->base_hash); - - unsigned int i; - char ipad[ss]; - char opad[ss]; - - shash->tfm = ctx->base_hash; - shash->flags = crypto_shash_get_flags(ctx->base_hash) & - CRYPTO_TFM_REQ_MAY_SLEEP; - - if (keylen > bs) { - int err; - - err = - crypto_shash_digest(shash, key, keylen, ipad); - if (err) - return err; - - keylen = ds; - } else - memcpy(ipad, key, keylen); - - memset(ipad + keylen, 0, bs - keylen); - memcpy(opad, ipad, bs); - - for (i = 0; i < bs; i++) { - ipad[i] ^= HMAC_IPAD_VALUE; - opad[i] ^= HMAC_OPAD_VALUE; - } - - rc = crypto_shash_init(shash) ? : - crypto_shash_update(shash, ipad, bs) ? : - crypto_shash_export(shash, ipad) ? : - crypto_shash_init(shash) ? : - crypto_shash_update(shash, opad, bs) ? : - crypto_shash_export(shash, opad); - - if (rc == 0) - mv_hash_init_ivs(ctx, ipad, opad); - - return rc; - } -} - -static int mv_cra_hash_init(struct crypto_tfm *tfm, const char *base_hash_name, - enum hash_op op, int count_add) -{ - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); - struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_shash *fallback_tfm = NULL; - struct crypto_shash *base_hash = NULL; - int err = -ENOMEM; - - ctx->op = op; - ctx->count_add = count_add; - - /* Allocate a fallback and abort if it failed. */ - fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback_tfm)) { - printk(KERN_WARNING MV_CESA - "Fallback driver '%s' could not be loaded!\n", - fallback_driver_name); - err = PTR_ERR(fallback_tfm); - goto out; - } - ctx->fallback = fallback_tfm; - - if (base_hash_name) { - /* Allocate a hash to compute the ipad/opad of hmac. */ - base_hash = crypto_alloc_shash(base_hash_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(base_hash)) { - printk(KERN_WARNING MV_CESA - "Base driver '%s' could not be loaded!\n", - base_hash_name); - err = PTR_ERR(base_hash); - goto err_bad_base; - } - } - ctx->base_hash = base_hash; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct mv_req_hash_ctx) + - crypto_shash_descsize(ctx->fallback)); - return 0; -err_bad_base: - crypto_free_shash(fallback_tfm); -out: - return err; -} - -static void mv_cra_hash_exit(struct crypto_tfm *tfm) -{ - struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_shash(ctx->fallback); - if (ctx->base_hash) - crypto_free_shash(ctx->base_hash); -} - -static int mv_cra_hash_sha1_init(struct crypto_tfm *tfm) -{ - return mv_cra_hash_init(tfm, NULL, COP_SHA1, 0); -} - -static int mv_cra_hash_hmac_sha1_init(struct crypto_tfm *tfm) -{ - return mv_cra_hash_init(tfm, "sha1", COP_HMAC_SHA1, SHA1_BLOCK_SIZE); -} - -static irqreturn_t crypto_int(int irq, void *priv) -{ - u32 val; - - val = readl(cpg->reg + SEC_ACCEL_INT_STATUS); - if (!(val & SEC_INT_ACCEL0_DONE)) - return IRQ_NONE; - - if (!del_timer(&cpg->completion_timer)) { - printk(KERN_WARNING MV_CESA - "got an interrupt but no pending timer?\n"); - } - val &= ~SEC_INT_ACCEL0_DONE; - writel(val, cpg->reg + FPGA_INT_STATUS); - writel(val, cpg->reg + SEC_ACCEL_INT_STATUS); - BUG_ON(cpg->eng_st != ENGINE_BUSY); - cpg->eng_st = ENGINE_W_DEQUEUE; - wake_up_process(cpg->queue_th); - return IRQ_HANDLED; -} - -static struct crypto_alg mv_aes_alg_ecb = { - .cra_name = "ecb(aes)", - .cra_driver_name = "mv-ecb-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = 16, - .cra_ctxsize = sizeof(struct mv_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mv_setkey_aes, - .encrypt = mv_enc_aes_ecb, - .decrypt = mv_dec_aes_ecb, - }, - }, -}; - -static struct crypto_alg mv_aes_alg_cbc = { - .cra_name = "cbc(aes)", - .cra_driver_name = "mv-cbc-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cra_init, - .cra_u = { - .ablkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mv_setkey_aes, - .encrypt = mv_enc_aes_cbc, - .decrypt = mv_dec_aes_cbc, - }, - }, -}; - -static struct ahash_alg mv_sha1_alg = { - .init = mv_hash_init, - .update = mv_hash_update, - .final = mv_hash_final, - .finup = mv_hash_finup, - .digest = mv_hash_digest, - .halg = { - .digestsize = SHA1_DIGEST_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "mv-sha1", - .cra_priority = 300, - .cra_flags = - CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx), - .cra_init = mv_cra_hash_sha1_init, - .cra_exit = mv_cra_hash_exit, - .cra_module = THIS_MODULE, - } - } -}; - -static struct ahash_alg mv_hmac_sha1_alg = { - .init = mv_hash_init, - .update = mv_hash_update, - .final = mv_hash_final, - .finup = mv_hash_finup, - .digest = mv_hash_digest, - .setkey = mv_hash_setkey, - .halg = { - .digestsize = SHA1_DIGEST_SIZE, - .base = { - .cra_name = "hmac(sha1)", - .cra_driver_name = "mv-hmac-sha1", - .cra_priority = 300, - .cra_flags = - CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx), - .cra_init = mv_cra_hash_hmac_sha1_init, - .cra_exit = mv_cra_hash_exit, - .cra_module = THIS_MODULE, - } - } -}; - -static int mv_cesa_get_sram(struct platform_device *pdev, - struct crypto_priv *cp) -{ - struct resource *res; - u32 sram_size = MV_CESA_DEFAULT_SRAM_SIZE; - - of_property_read_u32(pdev->dev.of_node, "marvell,crypto-sram-size", - &sram_size); - - cp->sram_size = sram_size; - cp->sram_pool = of_gen_pool_get(pdev->dev.of_node, - "marvell,crypto-srams", 0); - if (cp->sram_pool) { - cp->sram = gen_pool_dma_alloc(cp->sram_pool, sram_size, - &cp->sram_dma); - if (cp->sram) - return 0; - - return -ENOMEM; - } - - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "sram"); - if (!res || resource_size(res) < cp->sram_size) - return -EINVAL; - - cp->sram = devm_ioremap_resource(&pdev->dev, res); - return PTR_ERR_OR_ZERO(cp->sram); -} - -static int mv_probe(struct platform_device *pdev) -{ - struct crypto_priv *cp; - struct resource *res; - int irq; - int ret; - - if (cpg) { - printk(KERN_ERR MV_CESA "Second crypto dev?\n"); - return -EEXIST; - } - - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); - if (!res) - return -ENXIO; - - cp = devm_kzalloc(&pdev->dev, sizeof(*cp), GFP_KERNEL); - if (!cp) - return -ENOMEM; - - spin_lock_init(&cp->lock); - crypto_init_queue(&cp->queue, 50); - cp->reg = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(cp->reg)) { - ret = PTR_ERR(cp->reg); - goto err; - } - - ret = mv_cesa_get_sram(pdev, cp); - if (ret) - goto err; - - cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE; - - irq = platform_get_irq(pdev, 0); - if (irq < 0) { - ret = irq; - goto err; - } - cp->irq = irq; - - platform_set_drvdata(pdev, cp); - cpg = cp; - - cp->queue_th = kthread_run(queue_manag, cp, "mv_crypto"); - if (IS_ERR(cp->queue_th)) { - ret = PTR_ERR(cp->queue_th); - goto err; - } - - ret = request_irq(irq, crypto_int, 0, dev_name(&pdev->dev), - cp); - if (ret) - goto err_thread; - - /* Not all platforms can gate the clock, so it is not - an error if the clock does not exists. */ - cp->clk = clk_get(&pdev->dev, NULL); - if (!IS_ERR(cp->clk)) - clk_prepare_enable(cp->clk); - - writel(0, cpg->reg + SEC_ACCEL_INT_STATUS); - writel(SEC_INT_ACCEL0_DONE, cpg->reg + SEC_ACCEL_INT_MASK); - writel(SEC_CFG_STOP_DIG_ERR, cpg->reg + SEC_ACCEL_CFG); - writel(SRAM_CONFIG, cpg->reg + SEC_ACCEL_DESC_P0); - - ret = crypto_register_alg(&mv_aes_alg_ecb); - if (ret) { - printk(KERN_WARNING MV_CESA - "Could not register aes-ecb driver\n"); - goto err_irq; - } - - ret = crypto_register_alg(&mv_aes_alg_cbc); - if (ret) { - printk(KERN_WARNING MV_CESA - "Could not register aes-cbc driver\n"); - goto err_unreg_ecb; - } - - ret = crypto_register_ahash(&mv_sha1_alg); - if (ret == 0) - cpg->has_sha1 = 1; - else - printk(KERN_WARNING MV_CESA "Could not register sha1 driver\n"); - - ret = crypto_register_ahash(&mv_hmac_sha1_alg); - if (ret == 0) { - cpg->has_hmac_sha1 = 1; - } else { - printk(KERN_WARNING MV_CESA - "Could not register hmac-sha1 driver\n"); - } - - return 0; -err_unreg_ecb: - crypto_unregister_alg(&mv_aes_alg_ecb); -err_irq: - free_irq(irq, cp); - if (!IS_ERR(cp->clk)) { - clk_disable_unprepare(cp->clk); - clk_put(cp->clk); - } -err_thread: - kthread_stop(cp->queue_th); -err: - cpg = NULL; - return ret; -} - -static int mv_remove(struct platform_device *pdev) -{ - struct crypto_priv *cp = platform_get_drvdata(pdev); - - crypto_unregister_alg(&mv_aes_alg_ecb); - crypto_unregister_alg(&mv_aes_alg_cbc); - if (cp->has_sha1) - crypto_unregister_ahash(&mv_sha1_alg); - if (cp->has_hmac_sha1) - crypto_unregister_ahash(&mv_hmac_sha1_alg); - kthread_stop(cp->queue_th); - free_irq(cp->irq, cp); - memset(cp->sram, 0, cp->sram_size); - - if (!IS_ERR(cp->clk)) { - clk_disable_unprepare(cp->clk); - clk_put(cp->clk); - } - - cpg = NULL; - return 0; -} - -static const struct of_device_id mv_cesa_of_match_table[] = { - { .compatible = "marvell,orion-crypto", }, - { .compatible = "marvell,kirkwood-crypto", }, - { .compatible = "marvell,dove-crypto", }, - {} -}; -MODULE_DEVICE_TABLE(of, mv_cesa_of_match_table); - -static struct platform_driver marvell_crypto = { - .probe = mv_probe, - .remove = mv_remove, - .driver = { - .name = "mv_crypto", - .of_match_table = mv_cesa_of_match_table, - }, -}; -MODULE_ALIAS("platform:mv_crypto"); - -module_platform_driver(marvell_crypto); - -MODULE_AUTHOR("Sebastian Andrzej Siewior "); -MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/mv_cesa.h b/drivers/crypto/mv_cesa.h deleted file mode 100644 index 9249d3ed184b..000000000000 --- a/drivers/crypto/mv_cesa.h +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef __MV_CRYPTO_H__ -#define __MV_CRYPTO_H__ - -#define DIGEST_INITIAL_VAL_A 0xdd00 -#define DIGEST_INITIAL_VAL_B 0xdd04 -#define DIGEST_INITIAL_VAL_C 0xdd08 -#define DIGEST_INITIAL_VAL_D 0xdd0c -#define DIGEST_INITIAL_VAL_E 0xdd10 -#define DES_CMD_REG 0xdd58 - -#define SEC_ACCEL_CMD 0xde00 -#define SEC_CMD_EN_SEC_ACCL0 (1 << 0) -#define SEC_CMD_EN_SEC_ACCL1 (1 << 1) -#define SEC_CMD_DISABLE_SEC (1 << 2) - -#define SEC_ACCEL_DESC_P0 0xde04 -#define SEC_DESC_P0_PTR(x) (x) - -#define SEC_ACCEL_DESC_P1 0xde14 -#define SEC_DESC_P1_PTR(x) (x) - -#define SEC_ACCEL_CFG 0xde08 -#define SEC_CFG_STOP_DIG_ERR (1 << 0) -#define SEC_CFG_CH0_W_IDMA (1 << 7) -#define SEC_CFG_CH1_W_IDMA (1 << 8) -#define SEC_CFG_ACT_CH0_IDMA (1 << 9) -#define SEC_CFG_ACT_CH1_IDMA (1 << 10) - -#define SEC_ACCEL_STATUS 0xde0c -#define SEC_ST_ACT_0 (1 << 0) -#define SEC_ST_ACT_1 (1 << 1) - -/* - * FPGA_INT_STATUS looks like a FPGA leftover and is documented only in Errata - * 4.12. It looks like that it was part of an IRQ-controller in FPGA and - * someone forgot to remove it while switching to the core and moving to - * SEC_ACCEL_INT_STATUS. - */ -#define FPGA_INT_STATUS 0xdd68 -#define SEC_ACCEL_INT_STATUS 0xde20 -#define SEC_INT_AUTH_DONE (1 << 0) -#define SEC_INT_DES_E_DONE (1 << 1) -#define SEC_INT_AES_E_DONE (1 << 2) -#define SEC_INT_AES_D_DONE (1 << 3) -#define SEC_INT_ENC_DONE (1 << 4) -#define SEC_INT_ACCEL0_DONE (1 << 5) -#define SEC_INT_ACCEL1_DONE (1 << 6) -#define SEC_INT_ACC0_IDMA_DONE (1 << 7) -#define SEC_INT_ACC1_IDMA_DONE (1 << 8) - -#define SEC_ACCEL_INT_MASK 0xde24 - -#define AES_KEY_LEN (8 * 4) - -struct sec_accel_config { - - u32 config; -#define CFG_OP_MAC_ONLY 0 -#define CFG_OP_CRYPT_ONLY 1 -#define CFG_OP_MAC_CRYPT 2 -#define CFG_OP_CRYPT_MAC 3 -#define CFG_MACM_MD5 (4 << 4) -#define CFG_MACM_SHA1 (5 << 4) -#define CFG_MACM_HMAC_MD5 (6 << 4) -#define CFG_MACM_HMAC_SHA1 (7 << 4) -#define CFG_ENCM_DES (1 << 8) -#define CFG_ENCM_3DES (2 << 8) -#define CFG_ENCM_AES (3 << 8) -#define CFG_DIR_ENC (0 << 12) -#define CFG_DIR_DEC (1 << 12) -#define CFG_ENC_MODE_ECB (0 << 16) -#define CFG_ENC_MODE_CBC (1 << 16) -#define CFG_3DES_EEE (0 << 20) -#define CFG_3DES_EDE (1 << 20) -#define CFG_AES_LEN_128 (0 << 24) -#define CFG_AES_LEN_192 (1 << 24) -#define CFG_AES_LEN_256 (2 << 24) -#define CFG_NOT_FRAG (0 << 30) -#define CFG_FIRST_FRAG (1 << 30) -#define CFG_LAST_FRAG (2 << 30) -#define CFG_MID_FRAG (3 << 30) - - u32 enc_p; -#define ENC_P_SRC(x) (x) -#define ENC_P_DST(x) ((x) << 16) - - u32 enc_len; -#define ENC_LEN(x) (x) - - u32 enc_key_p; -#define ENC_KEY_P(x) (x) - - u32 enc_iv; -#define ENC_IV_POINT(x) ((x) << 0) -#define ENC_IV_BUF_POINT(x) ((x) << 16) - - u32 mac_src_p; -#define MAC_SRC_DATA_P(x) (x) -#define MAC_SRC_TOTAL_LEN(x) ((x) << 16) - - u32 mac_digest; -#define MAC_DIGEST_P(x) (x) -#define MAC_FRAG_LEN(x) ((x) << 16) - u32 mac_iv; -#define MAC_INNER_IV_P(x) (x) -#define MAC_OUTER_IV_P(x) ((x) << 16) -}__attribute__ ((packed)); - /* - * /-----------\ 0 - * | ACCEL CFG | 4 * 8 - * |-----------| 0x20 - * | CRYPT KEY | 8 * 4 - * |-----------| 0x40 - * | IV IN | 4 * 4 - * |-----------| 0x40 (inplace) - * | IV BUF | 4 * 4 - * |-----------| 0x80 - * | DATA IN | 16 * x (max ->max_req_size) - * |-----------| 0x80 (inplace operation) - * | DATA OUT | 16 * x (max ->max_req_size) - * \-----------/ SRAM size - */ - - /* Hashing memory map: - * /-----------\ 0 - * | ACCEL CFG | 4 * 8 - * |-----------| 0x20 - * | Inner IV | 5 * 4 - * |-----------| 0x34 - * | Outer IV | 5 * 4 - * |-----------| 0x48 - * | Output BUF| 5 * 4 - * |-----------| 0x80 - * | DATA IN | 64 * x (max ->max_req_size) - * \-----------/ SRAM size - */ -#define SRAM_CONFIG 0x00 -#define SRAM_DATA_KEY_P 0x20 -#define SRAM_DATA_IV 0x40 -#define SRAM_DATA_IV_BUF 0x40 -#define SRAM_DATA_IN_START 0x80 -#define SRAM_DATA_OUT_START 0x80 - -#define SRAM_HMAC_IV_IN 0x20 -#define SRAM_HMAC_IV_OUT 0x34 -#define SRAM_DIGEST_BUF 0x48 - -#define SRAM_CFG_SPACE 0x80 - -#endif -- cgit v1.2.3-59-g8ed1b From 2f1fedca9b97e87413db52c54745e9cc28b70169 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 12 Oct 2017 17:44:06 +0100 Subject: crypto: cavium - clean up clang warning on unread variable offset The variable offset is being assigned and not being used; it should be passed as the 2nd argument to call to function nitrox_write_csr but has been omitted. Fix this. Cleans up clang warning: Value stored to 'offset' is never read Fixes: 14fa93cdcd9b ("crypto: cavium - Add support for CNN55XX adapters.") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_hal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index f0655f82fa7d..73275ce6a668 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -126,7 +126,7 @@ void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) * size and interrupt threshold. */ offset = NPS_PKT_IN_INSTR_BADDRX(i); - nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma); + nitrox_write_csr(ndev, offset, cmdq->dma); /* configure ring size */ offset = NPS_PKT_IN_INSTR_RSIZEX(i); -- cgit v1.2.3-59-g8ed1b From 745952aeb599634025b568e1695fc6793b771682 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 12 Oct 2017 17:55:41 +0100 Subject: crypto: ccp - remove unused variable qim Variable qim is assigned but never read, it is redundant and can be removed. Cleans up clang warning: Value stored to 'qim' is never read Fixes: 4b394a232df7 ("crypto: ccp - Let a v5 CCP provide the same function as v3") Signed-off-by: Colin Ian King Acked-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-dev-v5.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index 65604fc65e8f..44a4d2779b15 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -788,13 +788,12 @@ static int ccp5_init(struct ccp_device *ccp) struct ccp_cmd_queue *cmd_q; struct dma_pool *dma_pool; char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; - unsigned int qmr, qim, i; + unsigned int qmr, i; u64 status; u32 status_lo, status_hi; int ret; /* Find available queues */ - qim = 0; qmr = ioread32(ccp->io_regs + Q_MASK_REG); for (i = 0; i < MAX_HW_QUEUES; i++) { -- cgit v1.2.3-59-g8ed1b From 9c290c507ca22ccf9ca2ab3fcb04e4e0e1422190 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Thu, 12 Oct 2017 18:04:56 +0100 Subject: crypto: qat - remove unused and redundant pointer vf_info The pointer vf_info is being assigned but never read, it is redundant and therefore can be removed. Cleans up clang warning: Value stored to 'vf_info' is never read Fixes: ed8ccaef52fa ("crypto: qat - Add support for SRIOV") Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_dev_mgr.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 8afac52677a6..2d06409bd3c4 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -228,11 +228,8 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, list_add_tail(&map->list, &vfs_table); } else if (accel_dev->is_vf && pf) { /* VF on host */ - struct adf_accel_vf_info *vf_info; struct vf_id_map *map; - vf_info = pf->pf.vf_info + adf_get_vf_id(accel_dev); - map = adf_find_vf(adf_get_vf_num(accel_dev)); if (map) { struct vf_id_map *next; -- cgit v1.2.3-59-g8ed1b From e3d062a6a27762f860b74f7723fa335115ccc880 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 12 Oct 2017 17:55:29 -0500 Subject: crypto: qat - mark expected switch fall-throughs in qat_uclo In preparation to enabling -Wimplicit-fallthrough, mark switch cases where we are expecting to fall through. Signed-off-by: Gustavo A. R. Silva Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 399604c41cdd..98d22c2096e3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -791,6 +791,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_GPA_ABS: case ICP_GPB_ABS: ctx_mask = 0; + /* fall through */ case ICP_GPA_REL: case ICP_GPB_REL: return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, @@ -800,6 +801,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_SR_RD_ABS: case ICP_DR_RD_ABS: ctx_mask = 0; + /* fall through */ case ICP_SR_REL: case ICP_DR_REL: case ICP_SR_RD_REL: @@ -809,6 +811,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_SR_WR_ABS: case ICP_DR_WR_ABS: ctx_mask = 0; + /* fall through */ case ICP_SR_WR_REL: case ICP_DR_WR_REL: return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, -- cgit v1.2.3-59-g8ed1b From e6cd5bf64855cc78054d6f8714f6c9983b7bb8bd Mon Sep 17 00:00:00 2001 From: Boris BREZILLON Date: Fri, 13 Oct 2017 15:30:32 +0200 Subject: crypto: marvell - Switch cipher algs to the skcipher interface crypto_alg is not supposed to be directly implemented by crypto engine driver. Drivers should instead implement specialized interfaces like ahash_alg or skcipher_alg. Migrate to all cipher algorithms to the skcipher_alg interface. While at it, get rid of all references to ablkcipher including in internal struct or function names. Signed-off-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.c | 10 +- drivers/crypto/marvell/cesa.h | 25 ++- drivers/crypto/marvell/cipher.c | 476 ++++++++++++++++++++-------------------- 3 files changed, 251 insertions(+), 260 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index d713d9697378..03604dff5401 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -196,7 +196,7 @@ static int mv_cesa_add_algs(struct mv_cesa_dev *cesa) int i, j; for (i = 0; i < cesa->caps->ncipher_algs; i++) { - ret = crypto_register_alg(cesa->caps->cipher_algs[i]); + ret = crypto_register_skcipher(cesa->caps->cipher_algs[i]); if (ret) goto err_unregister_crypto; } @@ -216,7 +216,7 @@ err_unregister_ahash: err_unregister_crypto: for (j = 0; j < i; j++) - crypto_unregister_alg(cesa->caps->cipher_algs[j]); + crypto_unregister_skcipher(cesa->caps->cipher_algs[j]); return ret; } @@ -229,10 +229,10 @@ static void mv_cesa_remove_algs(struct mv_cesa_dev *cesa) crypto_unregister_ahash(cesa->caps->ahash_algs[i]); for (i = 0; i < cesa->caps->ncipher_algs; i++) - crypto_unregister_alg(cesa->caps->cipher_algs[i]); + crypto_unregister_skcipher(cesa->caps->cipher_algs[i]); } -static struct crypto_alg *orion_cipher_algs[] = { +static struct skcipher_alg *orion_cipher_algs[] = { &mv_cesa_ecb_des_alg, &mv_cesa_cbc_des_alg, &mv_cesa_ecb_des3_ede_alg, @@ -248,7 +248,7 @@ static struct ahash_alg *orion_ahash_algs[] = { &mv_ahmac_sha1_alg, }; -static struct crypto_alg *armada_370_cipher_algs[] = { +static struct skcipher_alg *armada_370_cipher_algs[] = { &mv_cesa_ecb_des_alg, &mv_cesa_cbc_des_alg, &mv_cesa_ecb_des3_ede_alg, diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index b7872f62f674..acf81945eb85 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -372,7 +373,7 @@ struct mv_cesa_engine; struct mv_cesa_caps { int nengines; bool has_tdma; - struct crypto_alg **cipher_algs; + struct skcipher_alg **cipher_algs; int ncipher_algs; struct ahash_alg **ahash_algs; int nahash_algs; @@ -538,12 +539,12 @@ struct mv_cesa_sg_std_iter { }; /** - * struct mv_cesa_ablkcipher_std_req - cipher standard request + * struct mv_cesa_skcipher_std_req - cipher standard request * @op: operation context * @offset: current operation offset * @size: size of the crypto operation */ -struct mv_cesa_ablkcipher_std_req { +struct mv_cesa_skcipher_std_req { struct mv_cesa_op_ctx op; unsigned int offset; unsigned int size; @@ -551,14 +552,14 @@ struct mv_cesa_ablkcipher_std_req { }; /** - * struct mv_cesa_ablkcipher_req - cipher request + * struct mv_cesa_skcipher_req - cipher request * @req: type specific request information * @src_nents: number of entries in the src sg list * @dst_nents: number of entries in the dest sg list */ -struct mv_cesa_ablkcipher_req { +struct mv_cesa_skcipher_req { struct mv_cesa_req base; - struct mv_cesa_ablkcipher_std_req std; + struct mv_cesa_skcipher_std_req std; int src_nents; int dst_nents; }; @@ -868,11 +869,11 @@ extern struct ahash_alg mv_ahmac_md5_alg; extern struct ahash_alg mv_ahmac_sha1_alg; extern struct ahash_alg mv_ahmac_sha256_alg; -extern struct crypto_alg mv_cesa_ecb_des_alg; -extern struct crypto_alg mv_cesa_cbc_des_alg; -extern struct crypto_alg mv_cesa_ecb_des3_ede_alg; -extern struct crypto_alg mv_cesa_cbc_des3_ede_alg; -extern struct crypto_alg mv_cesa_ecb_aes_alg; -extern struct crypto_alg mv_cesa_cbc_aes_alg; +extern struct skcipher_alg mv_cesa_ecb_des_alg; +extern struct skcipher_alg mv_cesa_cbc_des_alg; +extern struct skcipher_alg mv_cesa_ecb_des3_ede_alg; +extern struct skcipher_alg mv_cesa_cbc_des3_ede_alg; +extern struct skcipher_alg mv_cesa_ecb_aes_alg; +extern struct skcipher_alg mv_cesa_cbc_aes_alg; #endif /* __MARVELL_CESA_H__ */ diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 098871a22a54..0ae84ec9e21c 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -32,23 +32,23 @@ struct mv_cesa_aes_ctx { struct crypto_aes_ctx aes; }; -struct mv_cesa_ablkcipher_dma_iter { +struct mv_cesa_skcipher_dma_iter { struct mv_cesa_dma_iter base; struct mv_cesa_sg_dma_iter src; struct mv_cesa_sg_dma_iter dst; }; static inline void -mv_cesa_ablkcipher_req_iter_init(struct mv_cesa_ablkcipher_dma_iter *iter, - struct ablkcipher_request *req) +mv_cesa_skcipher_req_iter_init(struct mv_cesa_skcipher_dma_iter *iter, + struct skcipher_request *req) { - mv_cesa_req_dma_iter_init(&iter->base, req->nbytes); + mv_cesa_req_dma_iter_init(&iter->base, req->cryptlen); mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); mv_cesa_sg_dma_iter_init(&iter->dst, req->dst, DMA_FROM_DEVICE); } static inline bool -mv_cesa_ablkcipher_req_iter_next_op(struct mv_cesa_ablkcipher_dma_iter *iter) +mv_cesa_skcipher_req_iter_next_op(struct mv_cesa_skcipher_dma_iter *iter) { iter->src.op_offset = 0; iter->dst.op_offset = 0; @@ -57,9 +57,9 @@ mv_cesa_ablkcipher_req_iter_next_op(struct mv_cesa_ablkcipher_dma_iter *iter) } static inline void -mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) +mv_cesa_skcipher_dma_cleanup(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); if (req->dst != req->src) { dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, @@ -73,20 +73,20 @@ mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) mv_cesa_dma_cleanup(&creq->base); } -static inline void mv_cesa_ablkcipher_cleanup(struct ablkcipher_request *req) +static inline void mv_cesa_skcipher_cleanup(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) - mv_cesa_ablkcipher_dma_cleanup(req); + mv_cesa_skcipher_dma_cleanup(req); } -static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) +static void mv_cesa_skcipher_std_step(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; struct mv_cesa_engine *engine = creq->base.engine; - size_t len = min_t(size_t, req->nbytes - sreq->offset, + size_t len = min_t(size_t, req->cryptlen - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); mv_cesa_adjust_op(engine, &sreq->op); @@ -114,11 +114,11 @@ static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } -static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, - u32 status) +static int mv_cesa_skcipher_std_process(struct skcipher_request *req, + u32 status) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; struct mv_cesa_engine *engine = creq->base.engine; size_t len; @@ -127,122 +127,130 @@ static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, sreq->size, sreq->offset); sreq->offset += len; - if (sreq->offset < req->nbytes) + if (sreq->offset < req->cryptlen) return -EINPROGRESS; return 0; } -static int mv_cesa_ablkcipher_process(struct crypto_async_request *req, - u32 status) +static int mv_cesa_skcipher_process(struct crypto_async_request *req, + u32 status) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); struct mv_cesa_req *basereq = &creq->base; if (mv_cesa_req_get_type(basereq) == CESA_STD_REQ) - return mv_cesa_ablkcipher_std_process(ablkreq, status); + return mv_cesa_skcipher_std_process(skreq, status); return mv_cesa_dma_process(basereq, status); } -static void mv_cesa_ablkcipher_step(struct crypto_async_request *req) +static void mv_cesa_skcipher_step(struct crypto_async_request *req) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_dma_step(&creq->base); else - mv_cesa_ablkcipher_std_step(ablkreq); + mv_cesa_skcipher_std_step(skreq); } static inline void -mv_cesa_ablkcipher_dma_prepare(struct ablkcipher_request *req) +mv_cesa_skcipher_dma_prepare(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_req *basereq = &creq->base; mv_cesa_dma_prepare(basereq, basereq->engine); } static inline void -mv_cesa_ablkcipher_std_prepare(struct ablkcipher_request *req) +mv_cesa_skcipher_std_prepare(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; sreq->size = 0; sreq->offset = 0; } -static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, - struct mv_cesa_engine *engine) +static inline void mv_cesa_skcipher_prepare(struct crypto_async_request *req, + struct mv_cesa_engine *engine) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); creq->base.engine = engine; if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) - mv_cesa_ablkcipher_dma_prepare(ablkreq); + mv_cesa_skcipher_dma_prepare(skreq); else - mv_cesa_ablkcipher_std_prepare(ablkreq); + mv_cesa_skcipher_std_prepare(skreq); } static inline void -mv_cesa_ablkcipher_req_cleanup(struct crypto_async_request *req) +mv_cesa_skcipher_req_cleanup(struct crypto_async_request *req) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct skcipher_request *skreq = skcipher_request_cast(req); - mv_cesa_ablkcipher_cleanup(ablkreq); + mv_cesa_skcipher_cleanup(skreq); } static void -mv_cesa_ablkcipher_complete(struct crypto_async_request *req) +mv_cesa_skcipher_complete(struct crypto_async_request *req) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); struct mv_cesa_engine *engine = creq->base.engine; unsigned int ivsize; - atomic_sub(ablkreq->nbytes, &engine->load); - ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq)); + atomic_sub(skreq->cryptlen, &engine->load); + ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(skreq)); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { struct mv_cesa_req *basereq; basereq = &creq->base; - memcpy(ablkreq->info, basereq->chain.last->op->ctx.blkcipher.iv, + memcpy(skreq->iv, basereq->chain.last->op->ctx.blkcipher.iv, ivsize); } else { - memcpy_fromio(ablkreq->info, + memcpy_fromio(skreq->iv, engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, ivsize); } } -static const struct mv_cesa_req_ops mv_cesa_ablkcipher_req_ops = { - .step = mv_cesa_ablkcipher_step, - .process = mv_cesa_ablkcipher_process, - .cleanup = mv_cesa_ablkcipher_req_cleanup, - .complete = mv_cesa_ablkcipher_complete, +static const struct mv_cesa_req_ops mv_cesa_skcipher_req_ops = { + .step = mv_cesa_skcipher_step, + .process = mv_cesa_skcipher_process, + .cleanup = mv_cesa_skcipher_req_cleanup, + .complete = mv_cesa_skcipher_complete, }; -static int mv_cesa_ablkcipher_cra_init(struct crypto_tfm *tfm) +static void mv_cesa_skcipher_cra_exit(struct crypto_tfm *tfm) { - struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); + void *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx, tfm->__crt_alg->cra_ctxsize); +} + +static int mv_cesa_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->base.ops = &mv_cesa_ablkcipher_req_ops; + ctx->ops = &mv_cesa_skcipher_req_ops; - tfm->crt_ablkcipher.reqsize = sizeof(struct mv_cesa_ablkcipher_req); + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct mv_cesa_skcipher_req)); return 0; } -static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, +static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); int remaining; int offset; @@ -251,7 +259,7 @@ static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, ret = crypto_aes_expand_key(&ctx->aes, key, len); if (ret) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return ret; } @@ -264,16 +272,16 @@ static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return 0; } -static int mv_cesa_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, +static int mv_cesa_des_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); u32 tmp[DES_EXPKEY_WORDS]; int ret; if (len != DES_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -288,14 +296,14 @@ static int mv_cesa_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return 0; } -static int mv_cesa_des3_ede_setkey(struct crypto_ablkcipher *cipher, +static int mv_cesa_des3_ede_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); if (len != DES3_EDE_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -304,14 +312,14 @@ static int mv_cesa_des3_ede_setkey(struct crypto_ablkcipher *cipher, return 0; } -static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, - const struct mv_cesa_op_ctx *op_templ) +static int mv_cesa_skcipher_dma_req_init(struct skcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; struct mv_cesa_req *basereq = &creq->base; - struct mv_cesa_ablkcipher_dma_iter iter; + struct mv_cesa_skcipher_dma_iter iter; bool skip_ctx = false; int ret; unsigned int ivsize; @@ -339,7 +347,7 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, } mv_cesa_tdma_desc_iter_init(&basereq->chain); - mv_cesa_ablkcipher_req_iter_init(&iter, req); + mv_cesa_skcipher_req_iter_init(&iter, req); do { struct mv_cesa_op_ctx *op; @@ -370,10 +378,10 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, if (ret) goto err_free_tdma; - } while (mv_cesa_ablkcipher_req_iter_next_op(&iter)); + } while (mv_cesa_skcipher_req_iter_next_op(&iter)); /* Add output data for IV */ - ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); + ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); ret = mv_cesa_dma_add_result_op(&basereq->chain, CESA_SA_CFG_SRAM_OFFSET, CESA_SA_DATA_SRAM_OFFSET, CESA_TDMA_SRC_IN_SRAM, flags); @@ -399,11 +407,11 @@ err_unmap_src: } static inline int -mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, - const struct mv_cesa_op_ctx *op_templ) +mv_cesa_skcipher_std_req_init(struct skcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; struct mv_cesa_req *basereq = &creq->base; sreq->op = *op_templ; @@ -414,23 +422,23 @@ mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, return 0; } -static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, - struct mv_cesa_op_ctx *tmpl) +static int mv_cesa_skcipher_req_init(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - unsigned int blksize = crypto_ablkcipher_blocksize(tfm); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + unsigned int blksize = crypto_skcipher_blocksize(tfm); int ret; - if (!IS_ALIGNED(req->nbytes, blksize)) + if (!IS_ALIGNED(req->cryptlen, blksize)) return -EINVAL; - creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + creq->src_nents = sg_nents_for_len(req->src, req->cryptlen); if (creq->src_nents < 0) { dev_err(cesa_dev->dev, "Invalid number of src SG"); return creq->src_nents; } - creq->dst_nents = sg_nents_for_len(req->dst, req->nbytes); + creq->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); if (creq->dst_nents < 0) { dev_err(cesa_dev->dev, "Invalid number of dst SG"); return creq->dst_nents; @@ -440,36 +448,36 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, CESA_SA_DESC_CFG_OP_MSK); if (cesa_dev->caps->has_tdma) - ret = mv_cesa_ablkcipher_dma_req_init(req, tmpl); + ret = mv_cesa_skcipher_dma_req_init(req, tmpl); else - ret = mv_cesa_ablkcipher_std_req_init(req, tmpl); + ret = mv_cesa_skcipher_std_req_init(req, tmpl); return ret; } -static int mv_cesa_ablkcipher_queue_req(struct ablkcipher_request *req, - struct mv_cesa_op_ctx *tmpl) +static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) { int ret; - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_engine *engine; - ret = mv_cesa_ablkcipher_req_init(req, tmpl); + ret = mv_cesa_skcipher_req_init(req, tmpl); if (ret) return ret; - engine = mv_cesa_select_engine(req->nbytes); - mv_cesa_ablkcipher_prepare(&req->base, engine); + engine = mv_cesa_select_engine(req->cryptlen); + mv_cesa_skcipher_prepare(&req->base, engine); ret = mv_cesa_queue_req(&req->base, &creq->base); if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ablkcipher_cleanup(req); + mv_cesa_skcipher_cleanup(req); return ret; } -static int mv_cesa_des_op(struct ablkcipher_request *req, +static int mv_cesa_des_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); @@ -479,10 +487,10 @@ static int mv_cesa_des_op(struct ablkcipher_request *req, memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); - return mv_cesa_ablkcipher_queue_req(req, tmpl); + return mv_cesa_skcipher_queue_req(req, tmpl); } -static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -493,7 +501,7 @@ static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) return mv_cesa_des_op(req, &tmpl); } -static int mv_cesa_ecb_des_decrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -504,41 +512,38 @@ static int mv_cesa_ecb_des_decrypt(struct ablkcipher_request *req) return mv_cesa_des_op(req, &tmpl); } -struct crypto_alg mv_cesa_ecb_des_alg = { - .cra_name = "ecb(des)", - .cra_driver_name = "mv-ecb-des", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = mv_cesa_des_setkey, - .encrypt = mv_cesa_ecb_des_encrypt, - .decrypt = mv_cesa_ecb_des_decrypt, - }, +struct skcipher_alg mv_cesa_ecb_des_alg = { + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_ecb_des_encrypt, + .decrypt = mv_cesa_ecb_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "mv-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_cbc_des_op(struct ablkcipher_request *req, +static int mv_cesa_cbc_des_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, CESA_SA_DESC_CFG_CRYPTCM_MSK); - memcpy(tmpl->ctx.blkcipher.iv, req->info, DES_BLOCK_SIZE); + memcpy(tmpl->ctx.blkcipher.iv, req->iv, DES_BLOCK_SIZE); return mv_cesa_des_op(req, tmpl); } -static int mv_cesa_cbc_des_encrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -547,7 +552,7 @@ static int mv_cesa_cbc_des_encrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des_op(req, &tmpl); } -static int mv_cesa_cbc_des_decrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -556,31 +561,28 @@ static int mv_cesa_cbc_des_decrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des_op(req, &tmpl); } -struct crypto_alg mv_cesa_cbc_des_alg = { - .cra_name = "cbc(des)", - .cra_driver_name = "mv-cbc-des", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = mv_cesa_des_setkey, - .encrypt = mv_cesa_cbc_des_encrypt, - .decrypt = mv_cesa_cbc_des_decrypt, - }, +struct skcipher_alg mv_cesa_cbc_des_alg = { + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_cbc_des_encrypt, + .decrypt = mv_cesa_cbc_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "mv-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_des3_op(struct ablkcipher_request *req, +static int mv_cesa_des3_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm); @@ -590,10 +592,10 @@ static int mv_cesa_des3_op(struct ablkcipher_request *req, memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES3_EDE_KEY_SIZE); - return mv_cesa_ablkcipher_queue_req(req, tmpl); + return mv_cesa_skcipher_queue_req(req, tmpl); } -static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des3_ede_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -605,7 +607,7 @@ static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) return mv_cesa_des3_op(req, &tmpl); } -static int mv_cesa_ecb_des3_ede_decrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des3_ede_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -617,39 +619,36 @@ static int mv_cesa_ecb_des3_ede_decrypt(struct ablkcipher_request *req) return mv_cesa_des3_op(req, &tmpl); } -struct crypto_alg mv_cesa_ecb_des3_ede_alg = { - .cra_name = "ecb(des3_ede)", - .cra_driver_name = "mv-ecb-des3-ede", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .setkey = mv_cesa_des3_ede_setkey, - .encrypt = mv_cesa_ecb_des3_ede_encrypt, - .decrypt = mv_cesa_ecb_des3_ede_decrypt, - }, +struct skcipher_alg mv_cesa_ecb_des3_ede_alg = { + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_ecb_des3_ede_encrypt, + .decrypt = mv_cesa_ecb_des3_ede_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "mv-ecb-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_cbc_des3_op(struct ablkcipher_request *req, +static int mv_cesa_cbc_des3_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { - memcpy(tmpl->ctx.blkcipher.iv, req->info, DES3_EDE_BLOCK_SIZE); + memcpy(tmpl->ctx.blkcipher.iv, req->iv, DES3_EDE_BLOCK_SIZE); return mv_cesa_des3_op(req, tmpl); } -static int mv_cesa_cbc_des3_ede_encrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des3_ede_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -661,7 +660,7 @@ static int mv_cesa_cbc_des3_ede_encrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des3_op(req, &tmpl); } -static int mv_cesa_cbc_des3_ede_decrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des3_ede_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -673,31 +672,28 @@ static int mv_cesa_cbc_des3_ede_decrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des3_op(req, &tmpl); } -struct crypto_alg mv_cesa_cbc_des3_ede_alg = { - .cra_name = "cbc(des3_ede)", - .cra_driver_name = "mv-cbc-des3-ede", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .setkey = mv_cesa_des3_ede_setkey, - .encrypt = mv_cesa_cbc_des3_ede_encrypt, - .decrypt = mv_cesa_cbc_des3_ede_decrypt, - }, +struct skcipher_alg mv_cesa_cbc_des3_ede_alg = { + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_cbc_des3_ede_encrypt, + .decrypt = mv_cesa_cbc_des3_ede_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "mv-cbc-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_aes_op(struct ablkcipher_request *req, +static int mv_cesa_aes_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm); @@ -724,10 +720,10 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req, CESA_SA_DESC_CFG_CRYPTM_MSK | CESA_SA_DESC_CFG_AES_LEN_MSK); - return mv_cesa_ablkcipher_queue_req(req, tmpl); + return mv_cesa_skcipher_queue_req(req, tmpl); } -static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_aes_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -738,7 +734,7 @@ static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) return mv_cesa_aes_op(req, &tmpl); } -static int mv_cesa_ecb_aes_decrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_aes_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -749,40 +745,37 @@ static int mv_cesa_ecb_aes_decrypt(struct ablkcipher_request *req) return mv_cesa_aes_op(req, &tmpl); } -struct crypto_alg mv_cesa_ecb_aes_alg = { - .cra_name = "ecb(aes)", - .cra_driver_name = "mv-ecb-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mv_cesa_aes_setkey, - .encrypt = mv_cesa_ecb_aes_encrypt, - .decrypt = mv_cesa_ecb_aes_decrypt, - }, +struct skcipher_alg mv_cesa_ecb_aes_alg = { + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_ecb_aes_encrypt, + .decrypt = mv_cesa_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "mv-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_cbc_aes_op(struct ablkcipher_request *req, +static int mv_cesa_cbc_aes_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, CESA_SA_DESC_CFG_CRYPTCM_MSK); - memcpy(tmpl->ctx.blkcipher.iv, req->info, AES_BLOCK_SIZE); + memcpy(tmpl->ctx.blkcipher.iv, req->iv, AES_BLOCK_SIZE); return mv_cesa_aes_op(req, tmpl); } -static int mv_cesa_cbc_aes_encrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_aes_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -791,7 +784,7 @@ static int mv_cesa_cbc_aes_encrypt(struct ablkcipher_request *req) return mv_cesa_cbc_aes_op(req, &tmpl); } -static int mv_cesa_cbc_aes_decrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_aes_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -800,26 +793,23 @@ static int mv_cesa_cbc_aes_decrypt(struct ablkcipher_request *req) return mv_cesa_cbc_aes_op(req, &tmpl); } -struct crypto_alg mv_cesa_cbc_aes_alg = { - .cra_name = "cbc(aes)", - .cra_driver_name = "mv-cbc-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mv_cesa_aes_setkey, - .encrypt = mv_cesa_cbc_aes_encrypt, - .decrypt = mv_cesa_cbc_aes_decrypt, - }, +struct skcipher_alg mv_cesa_cbc_aes_alg = { + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_cbc_aes_encrypt, + .decrypt = mv_cesa_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "mv-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -- cgit v1.2.3-59-g8ed1b From e666d4e9ceec94c0a88c94b7db31d56474da43b3 Mon Sep 17 00:00:00 2001 From: Paulo Flabiano Smorigo Date: Mon, 16 Oct 2017 20:54:19 -0200 Subject: crypto: vmx - Use skcipher for ctr fallback Signed-off-by: Paulo Flabiano Smorigo Signed-off-by: Herbert Xu --- drivers/crypto/vmx/aes_ctr.c | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 17d84217dd76..fc60d00a2e84 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -27,21 +27,23 @@ #include #include #include +#include + #include "aesp8-ppc.h" struct p8_aes_ctr_ctx { - struct crypto_blkcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; }; static int p8_aes_ctr_init(struct crypto_tfm *tfm) { const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_blkcipher *fallback; + struct crypto_skcipher *fallback; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); - fallback = - crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + fallback = crypto_alloc_skcipher(alg, 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n", @@ -49,11 +51,11 @@ static int p8_aes_ctr_init(struct crypto_tfm *tfm) return PTR_ERR(fallback); } printk(KERN_INFO "Using '%s' as fallback implementation.\n", - crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + crypto_skcipher_driver_name(fallback)); - crypto_blkcipher_set_flags( + crypto_skcipher_set_flags( fallback, - crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm)); + crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); ctx->fallback = fallback; return 0; @@ -64,7 +66,7 @@ static void p8_aes_ctr_exit(struct crypto_tfm *tfm) struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); if (ctx->fallback) { - crypto_free_blkcipher(ctx->fallback); + crypto_free_skcipher(ctx->fallback); ctx->fallback = NULL; } } @@ -83,7 +85,7 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); + ret += crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret; } @@ -117,15 +119,14 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - struct blkcipher_desc fallback_desc = { - .tfm = ctx->fallback, - .info = desc->info, - .flags = desc->flags - }; if (in_interrupt()) { - ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, - nbytes); + SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); + skcipher_request_set_tfm(req, ctx->fallback); + skcipher_request_set_callback(req, desc->flags, NULL, NULL); + skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); + ret = crypto_skcipher_encrypt(req); + skcipher_request_zero(req); } else { blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); -- cgit v1.2.3-59-g8ed1b From cfba73d25f0eca9f01f565f0243e3083f4667234 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 18 Oct 2017 08:00:34 +0100 Subject: crypto: ccp - use -ENOSPC for transient busy indication Replace -EBUSY with -ENOSPC when reporting transient busy indication in the absence of backlog. Signed-off-by: Gilad Ben-Yossef Reviewed-by: Gary R Hook Signed-off-by: Herbert Xu --- drivers/crypto/ccp/ccp-crypto-main.c | 8 +++----- drivers/crypto/ccp/ccp-dev.c | 7 +++++-- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index 35a9de7fd475..b95d19974aa6 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -222,9 +222,10 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) /* Check if the cmd can/should be queued */ if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { - ret = -EBUSY; - if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) + if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) { + ret = -ENOSPC; goto e_lock; + } } /* Look for an entry with the same tfm. If there is a cmd @@ -243,9 +244,6 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) ret = ccp_enqueue_cmd(crypto_cmd->cmd); if (!ccp_crypto_success(ret)) goto e_lock; /* Error, don't queue it */ - if ((ret == -EBUSY) && - !(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) - goto e_lock; /* Not backlogging, don't queue it */ } if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 4e029b176641..1b5035d56288 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -292,9 +292,12 @@ int ccp_enqueue_cmd(struct ccp_cmd *cmd) i = ccp->cmd_q_count; if (ccp->cmd_count >= MAX_CMD_QLEN) { - ret = -EBUSY; - if (cmd->flags & CCP_CMD_MAY_BACKLOG) + if (cmd->flags & CCP_CMD_MAY_BACKLOG) { + ret = -EBUSY; list_add_tail(&cmd->entry, &ccp->backlog); + } else { + ret = -ENOSPC; + } } else { ret = -EINPROGRESS; ccp->cmd_count++; -- cgit v1.2.3-59-g8ed1b From 3d549e325e3c7dd294b96720f22cbc39ec2e02e9 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 18 Oct 2017 08:00:37 +0100 Subject: crypto: marvell/cesa - remove redundant backlog checks on EBUSY Now that -EBUSY return code only indicates backlog queueing we can safely remove the now redundant check for the CRYPTO_TFM_REQ_MAY_BACKLOG flag when -EBUSY is returned. Signed-off-by: Gilad Ben-Yossef Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.c | 3 +-- drivers/crypto/marvell/cesa.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 03604dff5401..2c0d6bd94165 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -177,8 +177,7 @@ int mv_cesa_queue_req(struct crypto_async_request *req, spin_lock_bh(&engine->lock); ret = crypto_enqueue_request(&engine->queue, req); if ((mv_cesa_req_get_type(creq) == CESA_DMA_REQ) && - (ret == -EINPROGRESS || - (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + (ret == -EINPROGRESS || ret == -EBUSY)) mv_cesa_tdma_chain(engine, creq); spin_unlock_bh(&engine->lock); diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index acf81945eb85..18a32743f6c2 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -764,7 +764,7 @@ static inline int mv_cesa_req_needs_cleanup(struct crypto_async_request *req, * the backlog and will be processed later. There's no need to * clean it up. */ - if (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + if (ret == -EBUSY) return false; /* Request wasn't queued, we need to clean it up */ -- cgit v1.2.3-59-g8ed1b From f1c90ac3d71324ac7a6beebc33b0d493c4cbbf1a Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 18 Oct 2017 08:00:49 +0100 Subject: crypto: talitos - move to generic async completion The talitos driver starts several async crypto ops and waits for their completions. Move it over to generic code doing the same. Signed-off-by: Gilad Ben-Yossef Tested-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 5bd8191405d8..9c80e0cb1664 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -2160,22 +2160,6 @@ static int ahash_import(struct ahash_request *areq, const void *in) return 0; } -struct keyhash_result { - struct completion completion; - int err; -}; - -static void keyhash_complete(struct crypto_async_request *req, int err) -{ - struct keyhash_result *res = req->data; - - if (err == -EINPROGRESS) - return; - - res->err = err; - complete(&res->completion); -} - static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, u8 *hash) { @@ -2183,10 +2167,10 @@ static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, struct scatterlist sg[1]; struct ahash_request *req; - struct keyhash_result hresult; + struct crypto_wait wait; int ret; - init_completion(&hresult.completion); + crypto_init_wait(&wait); req = ahash_request_alloc(tfm, GFP_KERNEL); if (!req) @@ -2195,25 +2179,13 @@ static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, /* Keep tfm keylen == 0 during hash of the long key */ ctx->keylen = 0; ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - keyhash_complete, &hresult); + crypto_req_done, &wait); sg_init_one(&sg[0], key, keylen); ahash_request_set_crypt(req, sg, hash, keylen); - ret = crypto_ahash_digest(req); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - ret = wait_for_completion_interruptible( - &hresult.completion); - if (!ret) - ret = hresult.err; - break; - default: - break; - } + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + ahash_request_free(req); return ret; -- cgit v1.2.3-59-g8ed1b From c70e5f9403103c0a754c4c726e00ddbd7c2c3bf4 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 18 Oct 2017 08:00:50 +0100 Subject: crypto: qce - move to generic async completion The qce driver starts several async crypto ops and waits for their completions. Move it over to generic code doing the same. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/qce/sha.c | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 47e114ac09d0..53227d70d397 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -349,28 +349,12 @@ static int qce_ahash_digest(struct ahash_request *req) return qce->async_req_enqueue(tmpl->qce, &req->base); } -struct qce_ahash_result { - struct completion completion; - int error; -}; - -static void qce_digest_complete(struct crypto_async_request *req, int error) -{ - struct qce_ahash_result *result = req->data; - - if (error == -EINPROGRESS) - return; - - result->error = error; - complete(&result->completion); -} - static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { unsigned int digestsize = crypto_ahash_digestsize(tfm); struct qce_sha_ctx *ctx = crypto_tfm_ctx(&tfm->base); - struct qce_ahash_result result; + struct crypto_wait wait; struct ahash_request *req; struct scatterlist sg; unsigned int blocksize; @@ -405,9 +389,9 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, goto err_free_ahash; } - init_completion(&result.completion); + crypto_init_wait(&wait); ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - qce_digest_complete, &result); + crypto_req_done, &wait); crypto_ahash_clear_flags(ahash_tfm, ~0); buf = kzalloc(keylen + QCE_MAX_ALIGN_SIZE, GFP_KERNEL); @@ -420,13 +404,7 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, sg_init_one(&sg, buf, keylen); ahash_request_set_crypt(req, &sg, ctx->authkey, keylen); - ret = crypto_ahash_digest(req); - if (ret == -EINPROGRESS || ret == -EBUSY) { - ret = wait_for_completion_interruptible(&result.completion); - if (!ret) - ret = result.error; - } - + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); if (ret) crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); -- cgit v1.2.3-59-g8ed1b From 591b95af3da027412be17ecbcb4dbd33990b557c Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 18 Oct 2017 08:00:51 +0100 Subject: crypto: mediatek - move to generic async completion The mediatek driver starts several async crypto ops and waits for their completions. Move it over to generic code doing the same. Signed-off-by: Gilad Ben-Yossef Acked-by: Ryder Lee Signed-off-by: Herbert Xu --- drivers/crypto/mediatek/mtk-aes.c | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 32aa587783c3..c2058cf59f57 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -138,11 +138,6 @@ struct mtk_aes_gcm_ctx { struct crypto_skcipher *ctr; }; -struct mtk_aes_gcm_setkey_result { - int err; - struct completion completion; -}; - struct mtk_aes_drv { struct list_head dev_list; /* Device list lock */ @@ -942,17 +937,6 @@ static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) &req->base); } -static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err) -{ - struct mtk_aes_gcm_setkey_result *result = req->data; - - if (err == -EINPROGRESS) - return; - - result->err = err; - complete(&result->completion); -} - /* * Because of the hardware limitation, we need to pre-calculate key(H) * for the GHASH operation. The result of the encryption operation @@ -968,7 +952,7 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, u32 hash[4]; u8 iv[8]; - struct mtk_aes_gcm_setkey_result result; + struct crypto_wait wait; struct scatterlist sg[1]; struct skcipher_request req; @@ -1008,22 +992,17 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (!data) return -ENOMEM; - init_completion(&data->result.completion); + crypto_init_wait(&data->wait); sg_init_one(data->sg, &data->hash, AES_BLOCK_SIZE); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, - mtk_gcm_setkey_done, &data->result); + crypto_req_done, &data->wait); skcipher_request_set_crypt(&data->req, data->sg, data->sg, AES_BLOCK_SIZE, data->iv); - err = crypto_skcipher_encrypt(&data->req); - if (err == -EINPROGRESS || err == -EBUSY) { - err = wait_for_completion_interruptible( - &data->result.completion); - if (!err) - err = data->result.err; - } + err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), + &data->wait); if (err) goto out; -- cgit v1.2.3-59-g8ed1b From 3c88761e8a7bee9252c6ed1e2552eb0cfefa521d Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Mon, 23 Oct 2017 18:34:39 +0300 Subject: crypto: atmel - return appropriate error code Return -ENODEV when dma_request_slave_channel_compat() fails. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 3 +-- drivers/crypto/atmel-sha.c | 3 +-- drivers/crypto/atmel-tdes.c | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 903fd43f23a5..9659759dcba9 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -2383,7 +2383,6 @@ static int atmel_aes_dma_init(struct atmel_aes_dev *dd, struct crypto_platform_data *pdata) { struct at_dma_slave *slave; - int err = -ENOMEM; dma_cap_mask_t mask; dma_cap_zero(mask); @@ -2408,7 +2407,7 @@ err_dma_out: dma_release_channel(dd->src.chan); err_dma_in: dev_warn(dd->dev, "no DMA channel available\n"); - return err; + return -ENODEV; } static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 3e2f41b3eaf3..42c1f7479ab6 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -2628,7 +2628,6 @@ static bool atmel_sha_filter(struct dma_chan *chan, void *slave) static int atmel_sha_dma_init(struct atmel_sha_dev *dd, struct crypto_platform_data *pdata) { - int err = -ENOMEM; dma_cap_mask_t mask_in; /* Try to grab DMA channel */ @@ -2639,7 +2638,7 @@ static int atmel_sha_dma_init(struct atmel_sha_dev *dd, atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); if (!dd->dma_lch_in.chan) { dev_warn(dd->dev, "no DMA channel available\n"); - return err; + return -ENODEV; } dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index f4b335dda568..0ece4b85c9db 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -720,7 +720,6 @@ static bool atmel_tdes_filter(struct dma_chan *chan, void *slave) static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd, struct crypto_platform_data *pdata) { - int err = -ENOMEM; dma_cap_mask_t mask; dma_cap_zero(mask); @@ -765,7 +764,7 @@ err_dma_out: dma_release_channel(dd->dma_lch_in.chan); err_dma_in: dev_warn(dd->dev, "no DMA channel available\n"); - return err; + return -ENODEV; } static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) -- cgit v1.2.3-59-g8ed1b From d472e42aafd6ab13a5981defb696874cc6bc26b3 Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Mon, 23 Oct 2017 18:34:41 +0300 Subject: crypto: atmel - remove useless irq init irq would be set to -1 and then unused, if we failed to get IORESOURCE_MEM. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 2 -- drivers/crypto/atmel-sha.c | 2 -- drivers/crypto/atmel-tdes.c | 2 -- 3 files changed, 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 9659759dcba9..889830e5b651 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -2658,8 +2658,6 @@ static int atmel_aes_probe(struct platform_device *pdev) crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); - aes_dd->irq = -1; - /* Get the base address */ aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!aes_res) { diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 42c1f7479ab6..8874aa5ca0f7 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -2777,8 +2777,6 @@ static int atmel_sha_probe(struct platform_device *pdev) crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); - sha_dd->irq = -1; - /* Get the base address */ sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!sha_res) { diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 0ece4b85c9db..8587be54a74e 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -1381,8 +1381,6 @@ static int atmel_tdes_probe(struct platform_device *pdev) crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH); - tdes_dd->irq = -1; - /* Get the base address */ tdes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!tdes_res) { -- cgit v1.2.3-59-g8ed1b From cc2f8ab5334a736fa0e775cfccf06c1e268667f0 Mon Sep 17 00:00:00 2001 From: Radu Alexe Date: Tue, 24 Oct 2017 09:27:30 +0300 Subject: crypto: caam - fix incorrect define MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: 3ebfa92f49a6 ("crypto: caam - Add new macros for building extended SEC descriptors (> 64 words)") Signed-off-by: Radu Alexe Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/desc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 2e6766a1573f..2386d3e2d2c9 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -1439,7 +1439,7 @@ #define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) #define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) #define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) -#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC0_SHIFT) +#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT) #define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) #define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) #define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) -- cgit v1.2.3-59-g8ed1b From d3b5a87c4a8178b34b93a463870eca549a103aee Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Tue, 24 Oct 2017 09:27:31 +0300 Subject: crypto: caam/qi - abort algorithm setup on DPAA2 parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit caam/qi frontend (i.e. caamalg_qi) mustn't be used in case it runs on a DPAA2 part (this could happen when using a multiplatform kernel). Fixes: 297b9cebd2fc ("crypto: caam/jr - add support for DPAA2 parts") Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 2eefc4a26bc2..f9f08fce4356 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -7,7 +7,7 @@ */ #include "compat.h" - +#include "ctrl.h" #include "regs.h" #include "intern.h" #include "desc_constr.h" @@ -2312,6 +2312,11 @@ static int __init caam_qi_algapi_init(void) if (!priv || !priv->qi_present) return -ENODEV; + if (caam_dpaa2) { + dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); + return -ENODEV; + } + INIT_LIST_HEAD(&alg_list); /* -- cgit v1.2.3-59-g8ed1b From f947732aa64cf99e802486ed9d1a93a4da37118b Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 25 Oct 2017 03:18:42 -0700 Subject: crypto: drivers - Convert timers to use timer_setup() In preparation for unconditionally passing the struct timer_list pointer to all timer callbacks, switch to using the new timer_setup() and from_timer() to pass the timer pointer explicitly. Cc: Herbert Xu Cc: Jesper Nilsson Cc: Lars Persson Cc: Niklas Cassel Cc: "David S. Miller" Cc: Jamie Iles Cc: linux-arm-kernel@axis.com Cc: linux-crypto@vger.kernel.org Cc: linux-arm-kernel@lists.infradead.org Signed-off-by: Kees Cook Acked-by: Jamie Iles Acked-by: Lars Persson Signed-off-by: Herbert Xu --- drivers/crypto/axis/artpec6_crypto.c | 6 +++--- drivers/crypto/picoxcell_crypto.c | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 0f9754e07719..456278440863 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -2072,9 +2072,9 @@ static void artpec6_crypto_process_queue(struct artpec6_crypto *ac) del_timer(&ac->timer); } -static void artpec6_crypto_timeout(unsigned long data) +static void artpec6_crypto_timeout(struct timer_list *t) { - struct artpec6_crypto *ac = (struct artpec6_crypto *) data; + struct artpec6_crypto *ac = from_timer(ac, t, timer); dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); @@ -3063,7 +3063,7 @@ static int artpec6_crypto_probe(struct platform_device *pdev) spin_lock_init(&ac->queue_lock); INIT_LIST_HEAD(&ac->queue); INIT_LIST_HEAD(&ac->pending); - setup_timer(&ac->timer, artpec6_crypto_timeout, (unsigned long) ac); + timer_setup(&ac->timer, artpec6_crypto_timeout, 0); ac->base = base; diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index b6f14844702e..5a6dc53b2b9d 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1125,9 +1125,9 @@ static irqreturn_t spacc_spacc_irq(int irq, void *dev) return IRQ_HANDLED; } -static void spacc_packet_timeout(unsigned long data) +static void spacc_packet_timeout(struct timer_list *t) { - struct spacc_engine *engine = (struct spacc_engine *)data; + struct spacc_engine *engine = from_timer(engine, t, packet_timeout); spacc_process_done(engine); } @@ -1714,8 +1714,7 @@ static int spacc_probe(struct platform_device *pdev) writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN, engine->regs + SPA_IRQ_EN_REG_OFFSET); - setup_timer(&engine->packet_timeout, spacc_packet_timeout, - (unsigned long)engine); + timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); INIT_LIST_HEAD(&engine->pending); INIT_LIST_HEAD(&engine->completed); -- cgit v1.2.3-59-g8ed1b From e5e4090884cbd109dab00f1ff96b98f551e55445 Mon Sep 17 00:00:00 2001 From: Kamil Konieczny Date: Wed, 25 Oct 2017 17:27:34 +0200 Subject: crypto: s5p-sss - Change spaces to tabs Change #define lines to use tabs consistently. Acked-by: Vladimir Zapolskiy Reviewed-by: Krzysztof Kozlowski Signed-off-by: Kamil Konieczny Signed-off-by: Herbert Xu --- drivers/crypto/s5p-sss.c | 190 +++++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 95 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 7ac657f46d15..dfae1865c384 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -30,98 +30,98 @@ #include #include -#define _SBF(s, v) ((v) << (s)) +#define _SBF(s, v) ((v) << (s)) /* Feed control registers */ -#define SSS_REG_FCINTSTAT 0x0000 -#define SSS_FCINTSTAT_BRDMAINT BIT(3) -#define SSS_FCINTSTAT_BTDMAINT BIT(2) -#define SSS_FCINTSTAT_HRDMAINT BIT(1) -#define SSS_FCINTSTAT_PKDMAINT BIT(0) - -#define SSS_REG_FCINTENSET 0x0004 -#define SSS_FCINTENSET_BRDMAINTENSET BIT(3) -#define SSS_FCINTENSET_BTDMAINTENSET BIT(2) -#define SSS_FCINTENSET_HRDMAINTENSET BIT(1) -#define SSS_FCINTENSET_PKDMAINTENSET BIT(0) - -#define SSS_REG_FCINTENCLR 0x0008 -#define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) -#define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) -#define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) -#define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) - -#define SSS_REG_FCINTPEND 0x000C -#define SSS_FCINTPEND_BRDMAINTP BIT(3) -#define SSS_FCINTPEND_BTDMAINTP BIT(2) -#define SSS_FCINTPEND_HRDMAINTP BIT(1) -#define SSS_FCINTPEND_PKDMAINTP BIT(0) - -#define SSS_REG_FCFIFOSTAT 0x0010 -#define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7) -#define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6) -#define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5) -#define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4) -#define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3) -#define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2) -#define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1) -#define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0) - -#define SSS_REG_FCFIFOCTRL 0x0014 -#define SSS_FCFIFOCTRL_DESSEL BIT(2) -#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) -#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) -#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) - -#define SSS_REG_FCBRDMAS 0x0020 -#define SSS_REG_FCBRDMAL 0x0024 -#define SSS_REG_FCBRDMAC 0x0028 -#define SSS_FCBRDMAC_BYTESWAP BIT(1) -#define SSS_FCBRDMAC_FLUSH BIT(0) - -#define SSS_REG_FCBTDMAS 0x0030 -#define SSS_REG_FCBTDMAL 0x0034 -#define SSS_REG_FCBTDMAC 0x0038 -#define SSS_FCBTDMAC_BYTESWAP BIT(1) -#define SSS_FCBTDMAC_FLUSH BIT(0) - -#define SSS_REG_FCHRDMAS 0x0040 -#define SSS_REG_FCHRDMAL 0x0044 -#define SSS_REG_FCHRDMAC 0x0048 -#define SSS_FCHRDMAC_BYTESWAP BIT(1) -#define SSS_FCHRDMAC_FLUSH BIT(0) - -#define SSS_REG_FCPKDMAS 0x0050 -#define SSS_REG_FCPKDMAL 0x0054 -#define SSS_REG_FCPKDMAC 0x0058 -#define SSS_FCPKDMAC_BYTESWAP BIT(3) -#define SSS_FCPKDMAC_DESCEND BIT(2) -#define SSS_FCPKDMAC_TRANSMIT BIT(1) -#define SSS_FCPKDMAC_FLUSH BIT(0) - -#define SSS_REG_FCPKDMAO 0x005C +#define SSS_REG_FCINTSTAT 0x0000 +#define SSS_FCINTSTAT_BRDMAINT BIT(3) +#define SSS_FCINTSTAT_BTDMAINT BIT(2) +#define SSS_FCINTSTAT_HRDMAINT BIT(1) +#define SSS_FCINTSTAT_PKDMAINT BIT(0) + +#define SSS_REG_FCINTENSET 0x0004 +#define SSS_FCINTENSET_BRDMAINTENSET BIT(3) +#define SSS_FCINTENSET_BTDMAINTENSET BIT(2) +#define SSS_FCINTENSET_HRDMAINTENSET BIT(1) +#define SSS_FCINTENSET_PKDMAINTENSET BIT(0) + +#define SSS_REG_FCINTENCLR 0x0008 +#define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) +#define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) +#define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) +#define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) + +#define SSS_REG_FCINTPEND 0x000C +#define SSS_FCINTPEND_BRDMAINTP BIT(3) +#define SSS_FCINTPEND_BTDMAINTP BIT(2) +#define SSS_FCINTPEND_HRDMAINTP BIT(1) +#define SSS_FCINTPEND_PKDMAINTP BIT(0) + +#define SSS_REG_FCFIFOSTAT 0x0010 +#define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7) +#define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6) +#define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5) +#define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4) +#define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3) +#define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2) +#define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1) +#define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0) + +#define SSS_REG_FCFIFOCTRL 0x0014 +#define SSS_FCFIFOCTRL_DESSEL BIT(2) +#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) +#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) +#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) + +#define SSS_REG_FCBRDMAS 0x0020 +#define SSS_REG_FCBRDMAL 0x0024 +#define SSS_REG_FCBRDMAC 0x0028 +#define SSS_FCBRDMAC_BYTESWAP BIT(1) +#define SSS_FCBRDMAC_FLUSH BIT(0) + +#define SSS_REG_FCBTDMAS 0x0030 +#define SSS_REG_FCBTDMAL 0x0034 +#define SSS_REG_FCBTDMAC 0x0038 +#define SSS_FCBTDMAC_BYTESWAP BIT(1) +#define SSS_FCBTDMAC_FLUSH BIT(0) + +#define SSS_REG_FCHRDMAS 0x0040 +#define SSS_REG_FCHRDMAL 0x0044 +#define SSS_REG_FCHRDMAC 0x0048 +#define SSS_FCHRDMAC_BYTESWAP BIT(1) +#define SSS_FCHRDMAC_FLUSH BIT(0) + +#define SSS_REG_FCPKDMAS 0x0050 +#define SSS_REG_FCPKDMAL 0x0054 +#define SSS_REG_FCPKDMAC 0x0058 +#define SSS_FCPKDMAC_BYTESWAP BIT(3) +#define SSS_FCPKDMAC_DESCEND BIT(2) +#define SSS_FCPKDMAC_TRANSMIT BIT(1) +#define SSS_FCPKDMAC_FLUSH BIT(0) + +#define SSS_REG_FCPKDMAO 0x005C /* AES registers */ #define SSS_REG_AES_CONTROL 0x00 -#define SSS_AES_BYTESWAP_DI BIT(11) -#define SSS_AES_BYTESWAP_DO BIT(10) -#define SSS_AES_BYTESWAP_IV BIT(9) -#define SSS_AES_BYTESWAP_CNT BIT(8) -#define SSS_AES_BYTESWAP_KEY BIT(7) -#define SSS_AES_KEY_CHANGE_MODE BIT(6) -#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00) -#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01) -#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02) -#define SSS_AES_FIFO_MODE BIT(3) -#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00) -#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01) -#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) -#define SSS_AES_MODE_DECRYPT BIT(0) +#define SSS_AES_BYTESWAP_DI BIT(11) +#define SSS_AES_BYTESWAP_DO BIT(10) +#define SSS_AES_BYTESWAP_IV BIT(9) +#define SSS_AES_BYTESWAP_CNT BIT(8) +#define SSS_AES_BYTESWAP_KEY BIT(7) +#define SSS_AES_KEY_CHANGE_MODE BIT(6) +#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00) +#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01) +#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02) +#define SSS_AES_FIFO_MODE BIT(3) +#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00) +#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01) +#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) +#define SSS_AES_MODE_DECRYPT BIT(0) #define SSS_REG_AES_STATUS 0x04 -#define SSS_AES_BUSY BIT(2) -#define SSS_AES_INPUT_READY BIT(1) -#define SSS_AES_OUTPUT_READY BIT(0) +#define SSS_AES_BUSY BIT(2) +#define SSS_AES_INPUT_READY BIT(1) +#define SSS_AES_OUTPUT_READY BIT(0) #define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2)) #define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2)) @@ -129,22 +129,22 @@ #define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2)) #define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2)) -#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) -#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) -#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) +#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) +#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) +#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) -#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) +#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) #define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \ SSS_AES_REG(dev, reg)) /* HW engine modes */ -#define FLAGS_AES_DECRYPT BIT(0) -#define FLAGS_AES_MODE_MASK _SBF(1, 0x03) -#define FLAGS_AES_CBC _SBF(1, 0x01) -#define FLAGS_AES_CTR _SBF(1, 0x02) +#define FLAGS_AES_DECRYPT BIT(0) +#define FLAGS_AES_MODE_MASK _SBF(1, 0x03) +#define FLAGS_AES_CBC _SBF(1, 0x01) +#define FLAGS_AES_CTR _SBF(1, 0x02) -#define AES_KEY_LEN 16 -#define CRYPTO_QUEUE_LEN 1 +#define AES_KEY_LEN 16 +#define CRYPTO_QUEUE_LEN 1 /** * struct samsung_aes_variant - platform specific SSS driver data -- cgit v1.2.3-59-g8ed1b From c2afad6c6105783a72a2f7b69834a489120a1a7c Mon Sep 17 00:00:00 2001 From: Kamil Konieczny Date: Wed, 25 Oct 2017 17:27:35 +0200 Subject: crypto: s5p-sss - Add HASH support for Exynos Add support for MD5, SHA1, SHA256 hash algorithms for Exynos HW. It uses the crypto framework asynchronous hash api. It is based on omap-sham.c driver. S5P has some HW differencies and is not implemented. Modifications in s5p-sss: - Add hash supporting structures and functions. - Modify irq handler to handle both aes and hash signals. - Resize resource end in probe if EXYNOS_HASH is enabled in Kconfig. - Add new copyright line and new author. - Tested on Odroid-U3 with Exynos 4412 CPU, kernel 4.13-rc6 with crypto run-time self test testmgr and with tcrypt module with: modprobe tcrypt sec=1 mode=N where N=402, 403, 404 (MD5, SHA1, SHA256). Modifications in drivers/crypto/Kconfig: - Add new CRYPTO_DEV_EXYNOS_HASH, depend on !EXYNOS_RNG and CRYPTO_DEV_S5P - Select sw algorithms MD5, SHA1 and SHA256 in EXYNOS_HASH as they are needed for fallback. Acked-by: Vladimir Zapolskiy Reviewed-by: Krzysztof Kozlowski Signed-off-by: Kamil Konieczny Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 14 + drivers/crypto/s5p-sss.c | 1406 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 1410 insertions(+), 10 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 9f55be46a07d..47ec920d5b71 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -427,6 +427,20 @@ config CRYPTO_DEV_S5P Select this to offload Samsung S5PV210 or S5PC110, Exynos from AES algorithms execution. +config CRYPTO_DEV_EXYNOS_HASH + bool "Support for Samsung Exynos HASH accelerator" + depends on CRYPTO_DEV_S5P + depends on !CRYPTO_DEV_EXYNOS_RNG && CRYPTO_DEV_EXYNOS_RNG!=m + select CRYPTO_SHA1 + select CRYPTO_MD5 + select CRYPTO_SHA256 + help + Select this to offload Exynos from HASH MD5/SHA1/SHA256. + This will select software SHA1, MD5 and SHA256 as they are + needed for small and zero-size messages. + HASH algorithms will be disabled if EXYNOS_RNG + is enabled due to hw conflict. + config CRYPTO_DEV_NX bool "Support for IBM PowerPC Nest (NX) cryptographic acceleration" depends on PPC64 diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index dfae1865c384..142c6020cec7 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -1,14 +1,16 @@ /* * Cryptographic API. * - * Support for Samsung S5PV210 HW acceleration. + * Support for Samsung S5PV210 and Exynos HW acceleration. * * Copyright (C) 2011 NetUP Inc. All rights reserved. + * Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. * + * Hash part based on omap-sham.c driver. */ #include @@ -30,28 +32,41 @@ #include #include +#include +#include +#include +#include + #define _SBF(s, v) ((v) << (s)) /* Feed control registers */ #define SSS_REG_FCINTSTAT 0x0000 +#define SSS_FCINTSTAT_HPARTINT BIT(7) +#define SSS_FCINTSTAT_HDONEINT BIT(5) #define SSS_FCINTSTAT_BRDMAINT BIT(3) #define SSS_FCINTSTAT_BTDMAINT BIT(2) #define SSS_FCINTSTAT_HRDMAINT BIT(1) #define SSS_FCINTSTAT_PKDMAINT BIT(0) #define SSS_REG_FCINTENSET 0x0004 +#define SSS_FCINTENSET_HPARTINTENSET BIT(7) +#define SSS_FCINTENSET_HDONEINTENSET BIT(5) #define SSS_FCINTENSET_BRDMAINTENSET BIT(3) #define SSS_FCINTENSET_BTDMAINTENSET BIT(2) #define SSS_FCINTENSET_HRDMAINTENSET BIT(1) #define SSS_FCINTENSET_PKDMAINTENSET BIT(0) #define SSS_REG_FCINTENCLR 0x0008 +#define SSS_FCINTENCLR_HPARTINTENCLR BIT(7) +#define SSS_FCINTENCLR_HDONEINTENCLR BIT(5) #define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) #define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) #define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) #define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) #define SSS_REG_FCINTPEND 0x000C +#define SSS_FCINTPEND_HPARTINTP BIT(7) +#define SSS_FCINTPEND_HDONEINTP BIT(5) #define SSS_FCINTPEND_BRDMAINTP BIT(3) #define SSS_FCINTPEND_BTDMAINTP BIT(2) #define SSS_FCINTPEND_HRDMAINTP BIT(1) @@ -72,6 +87,7 @@ #define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) #define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) #define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) +#define SSS_HASHIN_MASK _SBF(0, 0x03) #define SSS_REG_FCBRDMAS 0x0020 #define SSS_REG_FCBRDMAL 0x0024 @@ -146,9 +162,80 @@ #define AES_KEY_LEN 16 #define CRYPTO_QUEUE_LEN 1 +/* HASH registers */ +#define SSS_REG_HASH_CTRL 0x00 + +#define SSS_HASH_USER_IV_EN BIT(5) +#define SSS_HASH_INIT_BIT BIT(4) +#define SSS_HASH_ENGINE_SHA1 _SBF(1, 0x00) +#define SSS_HASH_ENGINE_MD5 _SBF(1, 0x01) +#define SSS_HASH_ENGINE_SHA256 _SBF(1, 0x02) + +#define SSS_HASH_ENGINE_MASK _SBF(1, 0x03) + +#define SSS_REG_HASH_CTRL_PAUSE 0x04 + +#define SSS_HASH_PAUSE BIT(0) + +#define SSS_REG_HASH_CTRL_FIFO 0x08 + +#define SSS_HASH_FIFO_MODE_DMA BIT(0) +#define SSS_HASH_FIFO_MODE_CPU 0 + +#define SSS_REG_HASH_CTRL_SWAP 0x0C + +#define SSS_HASH_BYTESWAP_DI BIT(3) +#define SSS_HASH_BYTESWAP_DO BIT(2) +#define SSS_HASH_BYTESWAP_IV BIT(1) +#define SSS_HASH_BYTESWAP_KEY BIT(0) + +#define SSS_REG_HASH_STATUS 0x10 + +#define SSS_HASH_STATUS_MSG_DONE BIT(6) +#define SSS_HASH_STATUS_PARTIAL_DONE BIT(4) +#define SSS_HASH_STATUS_BUFFER_READY BIT(0) + +#define SSS_REG_HASH_MSG_SIZE_LOW 0x20 +#define SSS_REG_HASH_MSG_SIZE_HIGH 0x24 + +#define SSS_REG_HASH_PRE_MSG_SIZE_LOW 0x28 +#define SSS_REG_HASH_PRE_MSG_SIZE_HIGH 0x2C + +#define SSS_REG_HASH_IV(s) (0xB0 + ((s) << 2)) +#define SSS_REG_HASH_OUT(s) (0x100 + ((s) << 2)) + +#define HASH_BLOCK_SIZE 64 +#define HASH_REG_SIZEOF 4 +#define HASH_MD5_MAX_REG (MD5_DIGEST_SIZE / HASH_REG_SIZEOF) +#define HASH_SHA1_MAX_REG (SHA1_DIGEST_SIZE / HASH_REG_SIZEOF) +#define HASH_SHA256_MAX_REG (SHA256_DIGEST_SIZE / HASH_REG_SIZEOF) + +/* + * HASH bit numbers, used by device, setting in dev->hash_flags with + * functions set_bit(), clear_bit() or tested with test_bit() or BIT(), + * to keep HASH state BUSY or FREE, or to signal state from irq_handler + * to hash_tasklet. SGS keep track of allocated memory for scatterlist + */ +#define HASH_FLAGS_BUSY 0 +#define HASH_FLAGS_FINAL 1 +#define HASH_FLAGS_DMA_ACTIVE 2 +#define HASH_FLAGS_OUTPUT_READY 3 +#define HASH_FLAGS_DMA_READY 4 +#define HASH_FLAGS_SGS_COPIED 5 +#define HASH_FLAGS_SGS_ALLOCED 6 + +/* HASH HW constants */ +#define BUFLEN HASH_BLOCK_SIZE + +#define SSS_HASH_DMA_LEN_ALIGN 8 +#define SSS_HASH_DMA_ALIGN_MASK (SSS_HASH_DMA_LEN_ALIGN - 1) + +#define SSS_HASH_QUEUE_LENGTH 10 + /** * struct samsung_aes_variant - platform specific SSS driver data * @aes_offset: AES register offset from SSS module's base. + * @hash_offset: HASH register offset from SSS module's base. * * Specifies platform specific configuration of SSS module. * Note: A structure for driver specific platform data is used for future @@ -156,6 +243,7 @@ */ struct samsung_aes_variant { unsigned int aes_offset; + unsigned int hash_offset; }; struct s5p_aes_reqctx { @@ -195,6 +283,19 @@ struct s5p_aes_ctx { * protects against concurrent access to these fields. * @lock: Lock for protecting both access to device hardware registers * and fields related to current request (including the busy field). + * @res: Resources for hash. + * @io_hash_base: Per-variant offset for HASH block IO memory. + * @hash_lock: Lock for protecting hash_req, hash_queue and hash_flags + * variable. + * @hash_flags: Flags for current HASH op. + * @hash_queue: Async hash queue. + * @hash_tasklet: New HASH request scheduling job. + * @xmit_buf: Buffer for current HASH request transfer into SSS block. + * @hash_req: Current request sending to SSS HASH block. + * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block. + * @hash_sg_cnt: Counter for hash_sg_iter. + * + * @use_hash: true if HASH algs enabled */ struct s5p_aes_dev { struct device *dev; @@ -215,16 +316,83 @@ struct s5p_aes_dev { struct crypto_queue queue; bool busy; spinlock_t lock; + + struct resource *res; + void __iomem *io_hash_base; + + spinlock_t hash_lock; /* protect hash_ vars */ + unsigned long hash_flags; + struct crypto_queue hash_queue; + struct tasklet_struct hash_tasklet; + + u8 xmit_buf[BUFLEN]; + struct ahash_request *hash_req; + struct scatterlist *hash_sg_iter; + unsigned int hash_sg_cnt; + + bool use_hash; }; -static struct s5p_aes_dev *s5p_dev; +/** + * struct s5p_hash_reqctx - HASH request context + * @dd: Associated device + * @op_update: Current request operation (OP_UPDATE or OP_FINAL) + * @digcnt: Number of bytes processed by HW (without buffer[] ones) + * @digest: Digest message or IV for partial result + * @nregs: Number of HW registers for digest or IV read/write + * @engine: Bits for selecting type of HASH in SSS block + * @sg: sg for DMA transfer + * @sg_len: Length of sg for DMA transfer + * @sgl[]: sg for joining buffer and req->src scatterlist + * @skip: Skip offset in req->src for current op + * @total: Total number of bytes for current request + * @finup: Keep state for finup or final. + * @error: Keep track of error. + * @bufcnt: Number of bytes holded in buffer[] + * @buffer[]: For byte(s) from end of req->src in UPDATE op + */ +struct s5p_hash_reqctx { + struct s5p_aes_dev *dd; + bool op_update; + + u64 digcnt; + u8 digest[SHA256_DIGEST_SIZE]; + + unsigned int nregs; /* digest_size / sizeof(reg) */ + u32 engine; + + struct scatterlist *sg; + unsigned int sg_len; + struct scatterlist sgl[2]; + unsigned int skip; + unsigned int total; + bool finup; + bool error; + + u32 bufcnt; + u8 buffer[0]; +}; + +/** + * struct s5p_hash_ctx - HASH transformation context + * @dd: Associated device + * @flags: Bits for algorithm HASH. + * @fallback: Software transformation for zero message or size < BUFLEN. + */ +struct s5p_hash_ctx { + struct s5p_aes_dev *dd; + unsigned long flags; + struct crypto_shash *fallback; +}; static const struct samsung_aes_variant s5p_aes_data = { .aes_offset = 0x4000, + .hash_offset = 0x6000, }; static const struct samsung_aes_variant exynos_aes_data = { .aes_offset = 0x200, + .hash_offset = 0x400, }; static const struct of_device_id s5p_sss_dt_match[] = { @@ -254,6 +422,8 @@ static inline struct samsung_aes_variant *find_s5p_sss_version platform_get_device_id(pdev)->driver_data; } +static struct s5p_aes_dev *s5p_dev; + static void s5p_set_dma_indata(struct s5p_aes_dev *dev, struct scatterlist *sg) { SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg)); @@ -436,15 +606,65 @@ static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/) return ret; } +static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_hash_base + offset); +} + +static inline void s5p_hash_write(struct s5p_aes_dev *dd, + u32 offset, u32 value) +{ + __raw_writel(value, dd->io_hash_base + offset); +} + +/** + * s5p_set_dma_hashdata() - start DMA with sg + * @dev: device + * @sg: scatterlist ready to DMA transmit + */ +static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev, + struct scatterlist *sg) +{ + dev->hash_sg_cnt--; + SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg)); + SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */ +} + +/** + * s5p_hash_rx() - get next hash_sg_iter + * @dev: device + * + * Return: + * 2 if there is no more data and it is UPDATE op + * 1 if new receiving (input) data is ready and can be written to device + * 0 if there is no more data and it is FINAL op + */ +static int s5p_hash_rx(struct s5p_aes_dev *dev) +{ + if (dev->hash_sg_cnt > 0) { + dev->hash_sg_iter = sg_next(dev->hash_sg_iter); + return 1; + } + + set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags); + if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags)) + return 0; + + return 2; +} + static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) { struct platform_device *pdev = dev_id; struct s5p_aes_dev *dev = platform_get_drvdata(pdev); int err_dma_tx = 0; int err_dma_rx = 0; + int err_dma_hx = 0; bool tx_end = false; + bool hx_end = false; unsigned long flags; uint32_t status; + u32 st_bits; int err; spin_lock_irqsave(&dev->lock, flags); @@ -456,6 +676,8 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) * * If there is no more data in tx scatter list, call s5p_aes_complete() * and schedule new tasklet. + * + * Handle hx interrupt. If there is still data map next entry. */ status = SSS_READ(dev, FCINTSTAT); if (status & SSS_FCINTSTAT_BRDMAINT) @@ -467,7 +689,29 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) err_dma_tx = s5p_aes_tx(dev); } - SSS_WRITE(dev, FCINTPEND, status); + if (status & SSS_FCINTSTAT_HRDMAINT) + err_dma_hx = s5p_hash_rx(dev); + + st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT | + SSS_FCINTSTAT_HRDMAINT); + /* clear DMA bits */ + SSS_WRITE(dev, FCINTPEND, st_bits); + + /* clear HASH irq bits */ + if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) { + /* cannot have both HPART and HDONE */ + if (status & SSS_FCINTSTAT_HPARTINT) + st_bits = SSS_HASH_STATUS_PARTIAL_DONE; + + if (status & SSS_FCINTSTAT_HDONEINT) + st_bits = SSS_HASH_STATUS_MSG_DONE; + + set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags); + s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits); + hx_end = true; + /* when DONE or PART, do not handle HASH DMA */ + err_dma_hx = 0; + } if (err_dma_rx < 0) { err = err_dma_rx; @@ -480,6 +724,8 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) if (tx_end) { s5p_sg_done(dev); + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); spin_unlock_irqrestore(&dev->lock, flags); @@ -497,21 +743,1100 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) s5p_set_dma_outdata(dev, dev->sg_dst); if (err_dma_rx == 1) s5p_set_dma_indata(dev, dev->sg_src); + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); spin_unlock_irqrestore(&dev->lock, flags); } - return IRQ_HANDLED; + goto hash_irq_end; error: s5p_sg_done(dev); dev->busy = false; + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); + spin_unlock_irqrestore(&dev->lock, flags); s5p_aes_complete(dev, err); +hash_irq_end: + /* + * Note about else if: + * when hash_sg_iter reaches end and its UPDATE op, + * issue SSS_HASH_PAUSE and wait for HPART irq + */ + if (hx_end) + tasklet_schedule(&dev->hash_tasklet); + else if (err_dma_hx == 2) + s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE, + SSS_HASH_PAUSE); + return IRQ_HANDLED; } +/** + * s5p_hash_read_msg() - read message or IV from HW + * @req: AHASH request + */ +static void s5p_hash_read_msg(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + u32 *hash = (u32 *)ctx->digest; + unsigned int i; + + for (i = 0; i < ctx->nregs; i++) + hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i)); +} + +/** + * s5p_hash_write_ctx_iv() - write IV for next partial/finup op. + * @dd: device + * @ctx: request context + */ +static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd, + struct s5p_hash_reqctx *ctx) +{ + u32 *hash = (u32 *)ctx->digest; + unsigned int i; + + for (i = 0; i < ctx->nregs; i++) + s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]); +} + +/** + * s5p_hash_write_iv() - write IV for next partial/finup op. + * @req: AHASH request + */ +static void s5p_hash_write_iv(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + s5p_hash_write_ctx_iv(ctx->dd, ctx); +} + +/** + * s5p_hash_copy_result() - copy digest into req->result + * @req: AHASH request + */ +static void s5p_hash_copy_result(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + if (!req->result) + return; + + memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF); +} + +/** + * s5p_hash_dma_flush() - flush HASH DMA + * @dev: secss device + */ +static void s5p_hash_dma_flush(struct s5p_aes_dev *dev) +{ + SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH); +} + +/** + * s5p_hash_dma_enable() - enable DMA mode for HASH + * @dev: secss device + * + * enable DMA mode for HASH + */ +static void s5p_hash_dma_enable(struct s5p_aes_dev *dev) +{ + s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA); +} + +/** + * s5p_hash_irq_disable() - disable irq HASH signals + * @dev: secss device + * @flags: bitfield with irq's to be disabled + */ +static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags) +{ + SSS_WRITE(dev, FCINTENCLR, flags); +} + +/** + * s5p_hash_irq_enable() - enable irq signals + * @dev: secss device + * @flags: bitfield with irq's to be enabled + */ +static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags) +{ + SSS_WRITE(dev, FCINTENSET, flags); +} + +/** + * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH + * @dev: secss device + * @hashflow: HASH stream flow with/without crypto AES/DES + */ +static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow) +{ + unsigned long flags; + u32 flow; + + spin_lock_irqsave(&dev->lock, flags); + + flow = SSS_READ(dev, FCFIFOCTRL); + flow &= ~SSS_HASHIN_MASK; + flow |= hashflow; + SSS_WRITE(dev, FCFIFOCTRL, flow); + + spin_unlock_irqrestore(&dev->lock, flags); +} + +/** + * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS + * @dev: secss device + * @hashflow: HASH stream flow with/without AES/DES + * + * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW, + * enable HASH irq's HRDMA, HDONE, HPART + */ +static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow) +{ + s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR | + SSS_FCINTENCLR_HDONEINTENCLR | + SSS_FCINTENCLR_HPARTINTENCLR); + s5p_hash_dma_flush(dev); + + s5p_hash_dma_enable(dev); + s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK); + s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET | + SSS_FCINTENSET_HDONEINTENSET | + SSS_FCINTENSET_HPARTINTENSET); +} + +/** + * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing + * @dd: secss device + * @length: length for request + * @final: true if final op + * + * Prepare SSS HASH block for processing bytes in DMA mode. If it is called + * after previous updates, fill up IV words. For final, calculate and set + * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH + * length as 2^63 so it will be never reached and set to zero prelow and + * prehigh. + * + * This function does not start DMA transfer. + */ +static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length, + bool final) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + u32 prelow, prehigh, low, high; + u32 configflags, swapflags; + u64 tmplen; + + configflags = ctx->engine | SSS_HASH_INIT_BIT; + + if (likely(ctx->digcnt)) { + s5p_hash_write_ctx_iv(dd, ctx); + configflags |= SSS_HASH_USER_IV_EN; + } + + if (final) { + /* number of bytes for last part */ + low = length; + high = 0; + /* total number of bits prev hashed */ + tmplen = ctx->digcnt * 8; + prelow = (u32)tmplen; + prehigh = (u32)(tmplen >> 32); + } else { + prelow = 0; + prehigh = 0; + low = 0; + high = BIT(31); + } + + swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO | + SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY; + + s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low); + s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high); + s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow); + s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh); + + s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags); + s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags); +} + +/** + * s5p_hash_xmit_dma() - start DMA hash processing + * @dd: secss device + * @length: length for request + * @final: true if final op + * + * Update digcnt here, as it is needed for finup/final op. + */ +static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length, + bool final) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + unsigned int cnt; + + cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + if (!cnt) { + dev_err(dd->dev, "dma_map_sg error\n"); + ctx->error = true; + return -EINVAL; + } + + set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); + dd->hash_sg_iter = ctx->sg; + dd->hash_sg_cnt = cnt; + s5p_hash_write_ctrl(dd, length, final); + ctx->digcnt += length; + ctx->total -= length; + + /* catch last interrupt */ + if (final) + set_bit(HASH_FLAGS_FINAL, &dd->hash_flags); + + s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */ + + return -EINPROGRESS; +} + +/** + * s5p_hash_copy_sgs() - copy request's bytes into new buffer + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * + * Allocate new buffer, copy data for HASH into it. If there was xmit_buf + * filled, copy it first, then copy data from sg into it. Prepare one sgl[0] + * with allocated buffer. + * + * Set bit in dd->hash_flag so we can free it after irq ends processing. + */ +static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, unsigned int new_len) +{ + unsigned int pages, len; + void *buf; + + len = new_len + ctx->bufcnt; + pages = get_order(len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n"); + ctx->error = true; + return -ENOMEM; + } + + if (ctx->bufcnt) + memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt); + + scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip, + new_len, 0); + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, buf, len); + ctx->sg = ctx->sgl; + ctx->sg_len = 1; + ctx->bufcnt = 0; + ctx->skip = 0; + set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags); + + return 0; +} + +/** + * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * + * Allocate new scatterlist table, copy data for HASH into it. If there was + * xmit_buf filled, prepare it first, then copy page, length and offset from + * source sg into it, adjusting begin and/or end for skip offset and + * hash_later value. + * + * Resulting sg table will be assigned to ctx->sg. Set flag so we can free + * it after irq ends processing. + */ +static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, unsigned int new_len) +{ + unsigned int skip = ctx->skip, n = sg_nents(sg); + struct scatterlist *tmp; + unsigned int len; + + if (ctx->bufcnt) + n++; + + ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); + if (!ctx->sg) { + ctx->error = true; + return -ENOMEM; + } + + sg_init_table(ctx->sg, n); + + tmp = ctx->sg; + + ctx->sg_len = 0; + + if (ctx->bufcnt) { + sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt); + tmp = sg_next(tmp); + ctx->sg_len++; + } + + while (sg && skip >= sg->length) { + skip -= sg->length; + sg = sg_next(sg); + } + + while (sg && new_len) { + len = sg->length - skip; + if (new_len < len) + len = new_len; + + new_len -= len; + sg_set_page(tmp, sg_page(sg), len, sg->offset + skip); + skip = 0; + if (new_len <= 0) + sg_mark_end(tmp); + + tmp = sg_next(tmp); + ctx->sg_len++; + sg = sg_next(sg); + } + + set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags); + + return 0; +} + +/** + * s5p_hash_prepare_sgs() - prepare sg for processing + * @ctx: request context + * @sg: source scatterlist request + * @nbytes: number of bytes to process from sg + * @final: final flag + * + * Check two conditions: (1) if buffers in sg have len aligned data, and (2) + * sg table have good aligned elements (list_ok). If one of this checks fails, + * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy + * data into this buffer and prepare request in sgl, or (2) allocates new sg + * table and prepare sg elements. + * + * For digest or finup all conditions can be good, and we may not need any + * fixes. + */ +static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, + unsigned int new_len, bool final) +{ + unsigned int skip = ctx->skip, nbytes = new_len, n = 0; + bool aligned = true, list_ok = true; + struct scatterlist *sg_tmp = sg; + + if (!sg || !sg->length || !new_len) + return 0; + + if (skip || !final) + list_ok = false; + + while (nbytes > 0 && sg_tmp) { + n++; + if (skip >= sg_tmp->length) { + skip -= sg_tmp->length; + if (!sg_tmp->length) { + aligned = false; + break; + } + } else { + if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) { + aligned = false; + break; + } + + if (nbytes < sg_tmp->length - skip) { + list_ok = false; + break; + } + + nbytes -= sg_tmp->length - skip; + skip = 0; + } + + sg_tmp = sg_next(sg_tmp); + } + + if (!aligned) + return s5p_hash_copy_sgs(ctx, sg, new_len); + else if (!list_ok) + return s5p_hash_copy_sg_lists(ctx, sg, new_len); + + /* + * Have aligned data from previous operation and/or current + * Note: will enter here only if (digest or finup) and aligned + */ + if (ctx->bufcnt) { + ctx->sg_len = n; + sg_init_table(ctx->sgl, 2); + sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt); + sg_chain(ctx->sgl, 2, sg); + ctx->sg = ctx->sgl; + ctx->sg_len++; + } else { + ctx->sg = sg; + ctx->sg_len = n; + } + + return 0; +} + +/** + * s5p_hash_prepare_request() - prepare request for processing + * @req: AHASH request + * @update: true if UPDATE op + * + * Note 1: we can have update flag _and_ final flag at the same time. + * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or + * either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or + * we have final op + */ +static int s5p_hash_prepare_request(struct ahash_request *req, bool update) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + bool final = ctx->finup; + int xmit_len, hash_later, nbytes; + int ret; + + if (!req) + return 0; + + if (update) + nbytes = req->nbytes; + else + nbytes = 0; + + ctx->total = nbytes + ctx->bufcnt; + if (!ctx->total) + return 0; + + if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) { + /* bytes left from previous request, so fill up to BUFLEN */ + int len = BUFLEN - ctx->bufcnt % BUFLEN; + + if (len > nbytes) + len = nbytes; + + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, + 0, len, 0); + ctx->bufcnt += len; + nbytes -= len; + ctx->skip = len; + } else { + ctx->skip = 0; + } + + if (ctx->bufcnt) + memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt); + + xmit_len = ctx->total; + if (final) { + hash_later = 0; + } else { + if (IS_ALIGNED(xmit_len, BUFLEN)) + xmit_len -= BUFLEN; + else + xmit_len -= xmit_len & (BUFLEN - 1); + + hash_later = ctx->total - xmit_len; + /* copy hash_later bytes from end of req->src */ + /* previous bytes are in xmit_buf, so no overwrite */ + scatterwalk_map_and_copy(ctx->buffer, req->src, + req->nbytes - hash_later, + hash_later, 0); + } + + if (xmit_len > BUFLEN) { + ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later, + final); + if (ret) + return ret; + } else { + /* have buffered data only */ + if (unlikely(!ctx->bufcnt)) { + /* first update didn't fill up buffer */ + scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src, + 0, xmit_len, 0); + } + + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len); + + ctx->sg = ctx->sgl; + ctx->sg_len = 1; + } + + ctx->bufcnt = hash_later; + if (!final) + ctx->total = xmit_len; + + return 0; +} + +/** + * s5p_hash_update_dma_stop() - unmap DMA + * @dd: secss device + * + * Unmap scatterlist ctx->sg. + */ +static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + + dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); +} + +/** + * s5p_hash_finish() - copy calculated digest to crypto layer + * @req: AHASH request + */ +static void s5p_hash_finish(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + + if (ctx->digcnt) + s5p_hash_copy_result(req); + + dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt); +} + +/** + * s5p_hash_finish_req() - finish request + * @req: AHASH request + * @err: error + */ +static void s5p_hash_finish_req(struct ahash_request *req, int err) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + unsigned long flags; + + if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags)) + free_pages((unsigned long)sg_virt(ctx->sg), + get_order(ctx->sg->length)); + + if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags)) + kfree(ctx->sg); + + ctx->sg = NULL; + dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) | + BIT(HASH_FLAGS_SGS_COPIED)); + + if (!err && !ctx->error) { + s5p_hash_read_msg(req); + if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags)) + s5p_hash_finish(req); + } else { + ctx->error = true; + } + + spin_lock_irqsave(&dd->hash_lock, flags); + dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) | + BIT(HASH_FLAGS_DMA_READY) | + BIT(HASH_FLAGS_OUTPUT_READY)); + spin_unlock_irqrestore(&dd->hash_lock, flags); + + if (req->base.complete) + req->base.complete(&req->base, err); +} + +/** + * s5p_hash_handle_queue() - handle hash queue + * @dd: device s5p_aes_dev + * @req: AHASH request + * + * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the + * device then processes the first request from the dd->queue + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_handle_queue(struct s5p_aes_dev *dd, + struct ahash_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct s5p_hash_reqctx *ctx; + unsigned long flags; + int err = 0, ret = 0; + +retry: + spin_lock_irqsave(&dd->hash_lock, flags); + if (req) + ret = ahash_enqueue_request(&dd->hash_queue, req); + + if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { + spin_unlock_irqrestore(&dd->hash_lock, flags); + return ret; + } + + backlog = crypto_get_backlog(&dd->hash_queue); + async_req = crypto_dequeue_request(&dd->hash_queue); + if (async_req) + set_bit(HASH_FLAGS_BUSY, &dd->hash_flags); + + spin_unlock_irqrestore(&dd->hash_lock, flags); + + if (!async_req) + return ret; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ahash_request_cast(async_req); + dd->hash_req = req; + ctx = ahash_request_ctx(req); + + err = s5p_hash_prepare_request(req, ctx->op_update); + if (err || !ctx->total) + goto out; + + dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n", + ctx->op_update, req->nbytes); + + s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT); + if (ctx->digcnt) + s5p_hash_write_iv(req); /* restore hash IV */ + + if (ctx->op_update) { /* HASH_OP_UPDATE */ + err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup); + if (err != -EINPROGRESS && ctx->finup && !ctx->error) + /* no final() after finup() */ + err = s5p_hash_xmit_dma(dd, ctx->total, true); + } else { /* HASH_OP_FINAL */ + err = s5p_hash_xmit_dma(dd, ctx->total, true); + } +out: + if (err != -EINPROGRESS) { + /* hash_tasklet_cb will not finish it, so do it here */ + s5p_hash_finish_req(req, err); + req = NULL; + + /* + * Execute next request immediately if there is anything + * in queue. + */ + goto retry; + } + + return ret; +} + +/** + * s5p_hash_tasklet_cb() - hash tasklet + * @data: ptr to s5p_aes_dev + */ +static void s5p_hash_tasklet_cb(unsigned long data) +{ + struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data; + + if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { + s5p_hash_handle_queue(dd, NULL); + return; + } + + if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) { + if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE, + &dd->hash_flags)) { + s5p_hash_update_dma_stop(dd); + } + + if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY, + &dd->hash_flags)) { + /* hash or semi-hash ready */ + clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags); + goto finish; + } + } + + return; + +finish: + /* finish curent request */ + s5p_hash_finish_req(dd->hash_req, 0); + + /* If we are not busy, process next req */ + if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) + s5p_hash_handle_queue(dd, NULL); +} + +/** + * s5p_hash_enqueue() - enqueue request + * @req: AHASH request + * @op: operation UPDATE (true) or FINAL (false) + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_enqueue(struct ahash_request *req, bool op) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + + ctx->op_update = op; + + return s5p_hash_handle_queue(tctx->dd, req); +} + +/** + * s5p_hash_update() - process the hash input data + * @req: AHASH request + * + * If request will fit in buffer, copy it and return immediately + * else enqueue it with OP_UPDATE. + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_update(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return 0; + + if (ctx->bufcnt + req->nbytes <= BUFLEN) { + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, + 0, req->nbytes, 0); + ctx->bufcnt += req->nbytes; + return 0; + } + + return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */ +} + +/** + * s5p_hash_shash_digest() - calculate shash digest + * @tfm: crypto transformation + * @flags: tfm flags + * @data: input data + * @len: length of data + * @out: output buffer + */ +static int s5p_hash_shash_digest(struct crypto_shash *tfm, u32 flags, + const u8 *data, unsigned int len, u8 *out) +{ + SHASH_DESC_ON_STACK(shash, tfm); + + shash->tfm = tfm; + shash->flags = flags & ~CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_shash_digest(shash, data, len, out); +} + +/** + * s5p_hash_final_shash() - calculate shash digest + * @req: AHASH request + */ +static int s5p_hash_final_shash(struct ahash_request *req) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + return s5p_hash_shash_digest(tctx->fallback, req->base.flags, + ctx->buffer, ctx->bufcnt, req->result); +} + +/** + * s5p_hash_final() - close up hash and calculate digest + * @req: AHASH request + * + * Note: in final req->src do not have any data, and req->nbytes can be + * non-zero. + * + * If there were no input data processed yet and the buffered hash data is + * less than BUFLEN (64) then calculate the final hash immediately by using + * SW algorithm fallback. + * + * Otherwise enqueues the current AHASH request with OP_FINAL operation op + * and finalize hash message in HW. Note that if digcnt!=0 then there were + * previous update op, so there are always some buffered bytes in ctx->buffer, + * which means that ctx->bufcnt!=0 + * + * Returns: + * 0 if the request has been processed immediately, + * -EINPROGRESS if the operation has been queued for later execution or is set + * to processing by HW, + * -EBUSY if queue is full and request should be resubmitted later, + * other negative values denotes an error. + */ +static int s5p_hash_final(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + ctx->finup = true; + if (ctx->error) + return -EINVAL; /* uncompleted hash is not needed */ + + if (!ctx->digcnt && ctx->bufcnt < BUFLEN) + return s5p_hash_final_shash(req); + + return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */ +} + +/** + * s5p_hash_finup() - process last req->src and calculate digest + * @req: AHASH request containing the last update data + * + * Return values: see s5p_hash_final above. + */ +static int s5p_hash_finup(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + int err1, err2; + + ctx->finup = true; + + err1 = s5p_hash_update(req); + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + + /* + * final() has to be always called to cleanup resources even if + * update() failed, except EINPROGRESS or calculate digest for small + * size + */ + err2 = s5p_hash_final(req); + + return err1 ?: err2; +} + +/** + * s5p_hash_init() - initialize AHASH request contex + * @req: AHASH request + * + * Init async hash request context. + */ +static int s5p_hash_init(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); + + ctx->dd = tctx->dd; + ctx->error = false; + ctx->finup = false; + ctx->bufcnt = 0; + ctx->digcnt = 0; + ctx->total = 0; + ctx->skip = 0; + + dev_dbg(tctx->dd->dev, "init: digest size: %d\n", + crypto_ahash_digestsize(tfm)); + + switch (crypto_ahash_digestsize(tfm)) { + case MD5_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_MD5; + ctx->nregs = HASH_MD5_MAX_REG; + break; + case SHA1_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_SHA1; + ctx->nregs = HASH_SHA1_MAX_REG; + break; + case SHA256_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_SHA256; + ctx->nregs = HASH_SHA256_MAX_REG; + break; + default: + ctx->error = true; + return -EINVAL; + } + + return 0; +} + +/** + * s5p_hash_digest - calculate digest from req->src + * @req: AHASH request + * + * Return values: see s5p_hash_final above. + */ +static int s5p_hash_digest(struct ahash_request *req) +{ + return s5p_hash_init(req) ?: s5p_hash_finup(req); +} + +/** + * s5p_hash_cra_init_alg - init crypto alg transformation + * @tfm: crypto transformation + */ +static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(tfm); + + tctx->dd = s5p_dev; + /* Allocate a fallback and abort if it failed. */ + tctx->fallback = crypto_alloc_shash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(tctx->fallback)) { + pr_err("fallback alloc fails for '%s'\n", alg_name); + return PTR_ERR(tctx->fallback); + } + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct s5p_hash_reqctx) + BUFLEN); + + return 0; +} + +/** + * s5p_hash_cra_init - init crypto tfm + * @tfm: crypto transformation + */ +static int s5p_hash_cra_init(struct crypto_tfm *tfm) +{ + return s5p_hash_cra_init_alg(tfm); +} + +/** + * s5p_hash_cra_exit - exit crypto tfm + * @tfm: crypto transformation + * + * free allocated fallback + */ +static void s5p_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); + + crypto_free_shash(tctx->fallback); + tctx->fallback = NULL; +} + +/** + * s5p_hash_export - export hash state + * @req: AHASH request + * @out: buffer for exported state + */ +static int s5p_hash_export(struct ahash_request *req, void *out) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt); + + return 0; +} + +/** + * s5p_hash_import - import hash state + * @req: AHASH request + * @in: buffer with state to be imported from + */ +static int s5p_hash_import(struct ahash_request *req, const void *in) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); + const struct s5p_hash_reqctx *ctx_in = in; + + memcpy(ctx, in, sizeof(*ctx) + BUFLEN); + if (ctx_in->bufcnt > BUFLEN) { + ctx->error = true; + return -EINVAL; + } + + ctx->dd = tctx->dd; + ctx->error = false; + + return 0; +} + +static struct ahash_alg algs_sha1_md5_sha256[] = { +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "exynos-sha1", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +}, +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.base = { + .cra_name = "md5", + .cra_driver_name = "exynos-md5", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +}, +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "exynos-sha256", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +} + +}; + static void s5p_set_aes(struct s5p_aes_dev *dev, uint8_t *key, uint8_t *iv, unsigned int keylen) { @@ -829,6 +2154,7 @@ static int s5p_aes_probe(struct platform_device *pdev) struct samsung_aes_variant *variant; struct s5p_aes_dev *pdata; struct resource *res; + unsigned int hash_i; if (s5p_dev) return -EEXIST; @@ -837,12 +2163,34 @@ static int s5p_aes_probe(struct platform_device *pdev) if (!pdata) return -ENOMEM; + variant = find_s5p_sss_version(pdev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(pdata->ioaddr)) - return PTR_ERR(pdata->ioaddr); - variant = find_s5p_sss_version(pdev); + /* + * Note: HASH and PRNG uses the same registers in secss, avoid + * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG + * is enabled in config. We need larger size for HASH registers in + * secss, current describe only AES/DES + */ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) { + if (variant == &exynos_aes_data) { + res->end += 0x300; + pdata->use_hash = true; + } + } + + pdata->res = res; + pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pdata->ioaddr)) { + if (!pdata->use_hash) + return PTR_ERR(pdata->ioaddr); + /* try AES without HASH */ + res->end -= 0x300; + pdata->use_hash = false; + pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pdata->ioaddr)) + return PTR_ERR(pdata->ioaddr); + } pdata->clk = devm_clk_get(dev, "secss"); if (IS_ERR(pdata->clk)) { @@ -857,8 +2205,10 @@ static int s5p_aes_probe(struct platform_device *pdev) } spin_lock_init(&pdata->lock); + spin_lock_init(&pdata->hash_lock); pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset; + pdata->io_hash_base = pdata->ioaddr + variant->hash_offset; pdata->irq_fc = platform_get_irq(pdev, 0); if (pdata->irq_fc < 0) { @@ -888,12 +2238,40 @@ static int s5p_aes_probe(struct platform_device *pdev) goto err_algs; } + if (pdata->use_hash) { + tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb, + (unsigned long)pdata); + crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH); + + for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256); + hash_i++) { + struct ahash_alg *alg; + + alg = &algs_sha1_md5_sha256[hash_i]; + err = crypto_register_ahash(alg); + if (err) { + dev_err(dev, "can't register '%s': %d\n", + alg->halg.base.cra_driver_name, err); + goto err_hash; + } + } + } + dev_info(dev, "s5p-sss driver registered\n"); return 0; +err_hash: + for (j = hash_i - 1; j >= 0; j--) + crypto_unregister_ahash(&algs_sha1_md5_sha256[j]); + + tasklet_kill(&pdata->hash_tasklet); + res->end -= 0x300; + err_algs: - dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name, err); + if (i < ARRAY_SIZE(algs)) + dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name, + err); for (j = 0; j < i; j++) crypto_unregister_alg(&algs[j]); @@ -920,9 +2298,16 @@ static int s5p_aes_remove(struct platform_device *pdev) crypto_unregister_alg(&algs[i]); tasklet_kill(&pdata->tasklet); + if (pdata->use_hash) { + for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--) + crypto_unregister_ahash(&algs_sha1_md5_sha256[i]); - clk_disable_unprepare(pdata->clk); + pdata->res->end -= 0x300; + tasklet_kill(&pdata->hash_tasklet); + pdata->use_hash = false; + } + clk_disable_unprepare(pdata->clk); s5p_dev = NULL; return 0; @@ -942,3 +2327,4 @@ module_platform_driver(s5p_aes_crypto); MODULE_DESCRIPTION("S5PV210 AES hw acceleration support."); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Vladimir Zapolskiy "); +MODULE_AUTHOR("Kamil Konieczny "); -- cgit v1.2.3-59-g8ed1b From 91308019ecb46047b81d3c769cf5bf6b129ddf02 Mon Sep 17 00:00:00 2001 From: Romain Izard Date: Tue, 31 Oct 2017 16:25:23 +0100 Subject: crypto: atmel-aes - properly set IV after {en,de}crypt Certain cipher modes like CTS expect the IV (req->info) of ablkcipher_request (or equivalently req->iv of skcipher_request) to contain the last ciphertext block when the {en,de}crypt operation is done. Fix this issue for the Atmel AES hardware engine. The tcrypt test case for cts(cbc(aes)) is now correctly passed. In the case of in-place decryption, copy the ciphertext in an intermediate buffer before decryption. Signed-off-by: Romain Izard Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 889830e5b651..3f7b0966163c 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -111,6 +111,7 @@ struct atmel_aes_base_ctx { int keylen; u32 key[AES_KEYSIZE_256 / sizeof(u32)]; u16 block_size; + bool is_aead; }; struct atmel_aes_ctx { @@ -157,6 +158,7 @@ struct atmel_aes_authenc_ctx { struct atmel_aes_reqctx { unsigned long mode; + u32 lastc[AES_BLOCK_SIZE / sizeof(u32)]; }; #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC @@ -498,12 +500,34 @@ static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) { #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC - atmel_aes_authenc_complete(dd, err); + if (dd->ctx->is_aead) + atmel_aes_authenc_complete(dd, err); #endif clk_disable(dd->iclk); dd->flags &= ~AES_FLAGS_BUSY; + if (!dd->ctx->is_aead) { + struct ablkcipher_request *req = + ablkcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct crypto_ablkcipher *ablkcipher = + crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + if (rctx->mode & AES_FLAGS_ENCRYPT) { + scatterwalk_map_and_copy(req->info, req->dst, + req->nbytes - ivsize, ivsize, 0); + } else { + if (req->src == req->dst) { + memcpy(req->info, rctx->lastc, ivsize); + } else { + scatterwalk_map_and_copy(req->info, req->src, + req->nbytes - ivsize, ivsize, 0); + } + } + } + if (dd->is_async) dd->areq->complete(dd->areq, err); @@ -1072,11 +1096,11 @@ static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { - struct atmel_aes_base_ctx *ctx; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct atmel_aes_base_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct atmel_aes_reqctx *rctx; struct atmel_aes_dev *dd; - ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); switch (mode & AES_FLAGS_OPMODE_MASK) { case AES_FLAGS_CFB8: ctx->block_size = CFB8_BLOCK_SIZE; @@ -1098,6 +1122,7 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) ctx->block_size = AES_BLOCK_SIZE; break; } + ctx->is_aead = false; dd = atmel_aes_find_dev(ctx); if (!dd) @@ -1106,6 +1131,13 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) rctx = ablkcipher_request_ctx(req); rctx->mode = mode; + if (!(mode & AES_FLAGS_ENCRYPT) && (req->src == req->dst)) { + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + scatterwalk_map_and_copy(rctx->lastc, req->src, + (req->nbytes - ivsize), ivsize, 0); + } + return atmel_aes_handle_queue(dd, &req->base); } @@ -1740,6 +1772,7 @@ static int atmel_aes_gcm_crypt(struct aead_request *req, ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = true; dd = atmel_aes_find_dev(ctx); if (!dd) @@ -2224,6 +2257,7 @@ static int atmel_aes_authenc_crypt(struct aead_request *req, rctx->base.mode = mode; ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = true; dd = atmel_aes_find_dev(ctx); if (!dd) -- cgit v1.2.3-59-g8ed1b From 7a373fd74a8d1c4882e0236cc38345cec1393505 Mon Sep 17 00:00:00 2001 From: Romain Izard Date: Tue, 31 Oct 2017 16:25:24 +0100 Subject: crypto: atmel-aes - Reset the controller before each use When using the rfc4543(gcm(aes))) mode, the registers of the hardware engine are not empty after use. If the engine is not reset before its next use, the following results will be invalid. Always reset the hardware engine. Signed-off-by: Romain Izard Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 3f7b0966163c..0b49b6e9869a 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -77,12 +77,11 @@ AES_FLAGS_ENCRYPT | \ AES_FLAGS_GTAGEN) -#define AES_FLAGS_INIT BIT(2) #define AES_FLAGS_BUSY BIT(3) #define AES_FLAGS_DUMP_REG BIT(4) #define AES_FLAGS_OWN_SHA BIT(5) -#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY) +#define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY #define ATMEL_AES_QUEUE_LENGTH 50 @@ -451,11 +450,8 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd) if (err) return err; - if (!(dd->flags & AES_FLAGS_INIT)) { - atmel_aes_write(dd, AES_CR, AES_CR_SWRST); - atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); - dd->flags |= AES_FLAGS_INIT; - } + atmel_aes_write(dd, AES_CR, AES_CR_SWRST); + atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); return 0; } -- cgit v1.2.3-59-g8ed1b From f2ac67746534fab0cbc0bc29bfc3e507b1f58474 Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Wed, 1 Nov 2017 09:13:10 +0200 Subject: crypto: caam - remove unneeded edesc zeroization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extended descriptor allocation has been changed by commit dde20ae9d6383 ("crypto: caam - Change kmalloc to kzalloc to avoid residual data") to provide zeroized memory, meaning we no longer have to sanitize its members - edesc->src_nents and edesc->dst_dma. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamhash.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 698580b60b2f..7b1167437b88 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -871,7 +871,6 @@ static int ahash_final_ctx(struct ahash_request *req) desc = edesc->hw_desc; edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->src_nents = 0; ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); @@ -1123,7 +1122,6 @@ static int ahash_final_no_ctx(struct ahash_request *req) dev_err(jrdev, "unable to map dst\n"); goto unmap; } - edesc->src_nents = 0; #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1205,7 +1203,6 @@ static int ahash_update_no_ctx(struct ahash_request *req) edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->dst_dma = 0; ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); if (ret) @@ -1417,7 +1414,6 @@ static int ahash_update_first(struct ahash_request *req) } edesc->src_nents = src_nents; - edesc->dst_dma = 0; ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, to_hash); -- cgit v1.2.3-59-g8ed1b From dfcd8393efefb7a111f9cb6af69058ecaf4a4d72 Mon Sep 17 00:00:00 2001 From: Horia Geantă Date: Wed, 1 Nov 2017 09:13:11 +0200 Subject: crypto: caam - remove unused param of ctx_map_to_sec4_sg() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ctx_map_to_sec4_sg() function, added in commit 045e36780f115 ("crypto: caam - ahash hmac support") has never used the "desc" parameter, so let's drop it. Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamhash.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 7b1167437b88..616720a04e7a 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -218,7 +218,7 @@ static inline int buf_map_to_sec4_sg(struct device *jrdev, } /* Map state->caam_ctx, and add it to link table */ -static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, +static inline int ctx_map_to_sec4_sg(struct device *jrdev, struct caam_hash_state *state, int ctx_len, struct sec4_sg_entry *sec4_sg, u32 flag) { @@ -773,7 +773,7 @@ static int ahash_update_ctx(struct ahash_request *req) edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_BIDIRECTIONAL); if (ret) goto unmap_ctx; @@ -872,7 +872,7 @@ static int ahash_final_ctx(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); if (ret) goto unmap_ctx; @@ -966,7 +966,7 @@ static int ahash_finup_ctx(struct ahash_request *req) edesc->src_nents = src_nents; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); if (ret) goto unmap_ctx; -- cgit v1.2.3-59-g8ed1b From 747f6ec6e8f59bb946a5b0a4d4b8d1f45109b6ab Mon Sep 17 00:00:00 2001 From: Tudor-Dan Ambarus Date: Thu, 2 Nov 2017 17:12:25 +0200 Subject: crypto: atmel - remove empty functions Pointer members of an object with static storage duration, if not explicitly initialized, will be initialized to a NULL pointer. The crypto API checks if these pointers are not NULL before using them, therefore we can safely remove these empty functions. Signed-off-by: Tudor Ambarus Signed-off-by: Herbert Xu --- drivers/crypto/atmel-aes.c | 20 -------------------- drivers/crypto/atmel-tdes.c | 18 ------------------ 2 files changed, 38 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 0b49b6e9869a..691c6465b71e 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -1265,10 +1265,6 @@ static int atmel_aes_ctr_cra_init(struct crypto_tfm *tfm) return 0; } -static void atmel_aes_cra_exit(struct crypto_tfm *tfm) -{ -} - static struct crypto_alg aes_algs[] = { { .cra_name = "ecb(aes)", @@ -1281,7 +1277,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1301,7 +1296,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1322,7 +1316,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1343,7 +1336,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1364,7 +1356,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1385,7 +1376,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1406,7 +1396,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1427,7 +1416,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_ctr_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1450,7 +1438,6 @@ static struct crypto_alg aes_cfb64_alg = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1838,18 +1825,12 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm) return 0; } -static void atmel_aes_gcm_exit(struct crypto_aead *tfm) -{ - -} - static struct aead_alg aes_gcm_alg = { .setkey = atmel_aes_gcm_setkey, .setauthsize = atmel_aes_gcm_setauthsize, .encrypt = atmel_aes_gcm_encrypt, .decrypt = atmel_aes_gcm_decrypt, .init = atmel_aes_gcm_init, - .exit = atmel_aes_gcm_exit, .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, @@ -1985,7 +1966,6 @@ static struct crypto_alg aes_xts_alg = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_xts_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 8587be54a74e..592124f8382b 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -911,10 +911,6 @@ static int atmel_tdes_cra_init(struct crypto_tfm *tfm) return 0; } -static void atmel_tdes_cra_exit(struct crypto_tfm *tfm) -{ -} - static struct crypto_alg tdes_algs[] = { { .cra_name = "ecb(des)", @@ -927,7 +923,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -947,7 +942,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -968,7 +962,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -989,7 +982,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1010,7 +1002,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1031,7 +1022,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1052,7 +1042,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1073,7 +1062,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2 * DES_KEY_SIZE, .max_keysize = 3 * DES_KEY_SIZE, @@ -1093,7 +1081,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, @@ -1114,7 +1101,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1135,7 +1121,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1156,7 +1141,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1177,7 +1161,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1198,7 +1181,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, -- cgit v1.2.3-59-g8ed1b From 9c9034464860707669e91ce367c9eeb65e4175cd Mon Sep 17 00:00:00 2001 From: Boris BREZILLON Date: Fri, 3 Nov 2017 17:52:01 +0100 Subject: crypto: marvell - Add a NULL entry at the end of mv_cesa_plat_id_table[] struct platform_device_id should be NULL terminated to let the core detect where the last entry is. Fixes: 07c50a8be41a ("crypto: marvell - Add a platform_device_id table") Signed-off-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/crypto') diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 2c0d6bd94165..293832488cc9 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -591,6 +591,7 @@ static int mv_cesa_remove(struct platform_device *pdev) static const struct platform_device_id mv_cesa_plat_id_table[] = { { .name = "mv_crypto" }, + { /* sentinel */ }, }; MODULE_DEVICE_TABLE(platform, mv_cesa_plat_id_table); -- cgit v1.2.3-59-g8ed1b From abfa2b377f752a13e6caaa8c6c4add6ab9b71ba8 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Fri, 3 Nov 2017 11:58:23 -0500 Subject: crypto: chcr - Replace _manual_ swap with swap macro Make use of the swap macro and remove unnecessary variable temp. This makes the code easier to read and maintain. This code was detected with the help of Coccinelle. Signed-off-by: Gustavo A. R. Silva Signed-off-by: Herbert Xu --- drivers/crypto/chelsio/chcr_algo.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 9d298c681abe..4eed7171e2ae 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1634,11 +1634,8 @@ static int chcr_ahash_update(struct ahash_request *req) } if (remainder) { - u8 *temp; /* Swap buffers */ - temp = req_ctx->reqbfr; - req_ctx->reqbfr = req_ctx->skbfr; - req_ctx->skbfr = temp; + swap(req_ctx->reqbfr, req_ctx->skbfr); sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr, remainder, req->nbytes - remainder); -- cgit v1.2.3-59-g8ed1b From 5829cc8da94f500b0c1fe82688d89fd36a1a9a41 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 5 Nov 2017 18:30:47 -0800 Subject: crypto: qat - Clean up error handling in qat_dh_set_secret() Update the error handling in qat_dh_set_secret() to mirror dh_set_secret(). The new version is less error-prone because freeing memory and setting the pointers to NULL is now only done in one place. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 6f5dd68449c6..7655fdb499de 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -462,11 +462,8 @@ static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) } ctx->g = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); - if (!ctx->g) { - dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); - ctx->p = NULL; + if (!ctx->g) return -ENOMEM; - } memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, params->g_size); @@ -507,18 +504,22 @@ static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, ret = qat_dh_set_params(ctx, ¶ms); if (ret < 0) - return ret; + goto err_clear_ctx; ctx->xa = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_xa, GFP_KERNEL); if (!ctx->xa) { - qat_dh_clear_ctx(dev, ctx); - return -ENOMEM; + ret = -ENOMEM; + goto err_clear_ctx; } memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, params.key_size); return 0; + +err_clear_ctx: + qat_dh_clear_ctx(dev, ctx); + return ret; } static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) -- cgit v1.2.3-59-g8ed1b From ced6a58638432c9f731184f908208d539725ba82 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sun, 5 Nov 2017 18:30:48 -0800 Subject: crypto: dh - Remove pointless checks for NULL 'p' and 'g' Neither 'p' nor 'g' can be NULL, as they were unpacked using crypto_dh_decode_key(). And it makes no sense for them to be optional. So remove the NULL checks that were copy-and-pasted into both modules. Signed-off-by: Eric Biggers Reviewed-by: Tudor Ambarus Signed-off-by: Herbert Xu --- crypto/dh.c | 3 --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 3 --- 2 files changed, 6 deletions(-) (limited to 'drivers/crypto') diff --git a/crypto/dh.c b/crypto/dh.c index aadaf36fb56f..5659fe7f446d 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -53,9 +53,6 @@ static int dh_check_params_length(unsigned int p_len) static int dh_set_params(struct dh_ctx *ctx, struct dh *params) { - if (unlikely(!params->p || !params->g)) - return -EINVAL; - if (dh_check_params_length(params->p_size << 3)) return -EINVAL; diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 7655fdb499de..13c52d6bf630 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -443,9 +443,6 @@ static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - if (unlikely(!params->p || !params->g)) - return -EINVAL; - if (qat_dh_check_params_length(params->p_size << 3)) return -EINVAL; -- cgit v1.2.3-59-g8ed1b From 63c53823f00f0ffd13e8c86b05c1486614a2df85 Mon Sep 17 00:00:00 2001 From: Lionel Debieve Date: Mon, 6 Nov 2017 23:41:52 +0100 Subject: crypto: stm32/hash - Fix return issue on update When data append reached the threshold for processing, we must inform that processing is on going to wait before managing the next request. Signed-off-by: Lionel Debieve Signed-off-by: Herbert Xu --- drivers/crypto/stm32/stm32-hash.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index fd13838fcff2..764dbf362879 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -892,7 +892,6 @@ static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op) static int stm32_hash_update(struct ahash_request *req) { struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); - int ret; if (!req->nbytes || !(rctx->flags & HASH_FLAGS_CPU)) return 0; @@ -906,12 +905,7 @@ static int stm32_hash_update(struct ahash_request *req) return 0; } - ret = stm32_hash_enqueue(req, HASH_OP_UPDATE); - - if (rctx->flags & HASH_FLAGS_FINUP) - return ret; - - return 0; + return stm32_hash_enqueue(req, HASH_OP_UPDATE); } static int stm32_hash_final(struct ahash_request *req) -- cgit v1.2.3-59-g8ed1b