diff options
Diffstat (limited to 'drivers/crypto/hisilicon/qm.c')
| -rw-r--r-- | drivers/crypto/hisilicon/qm.c | 237 |
1 files changed, 198 insertions, 39 deletions
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6527c53b073f..530f23116d7c 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -180,7 +180,10 @@ #define QM_DBG_TMP_BUF_LEN 22 #define QM_PCI_COMMAND_INVALID ~0 +#define WAIT_PERIOD 20 +#define REMOVE_WAIT_DELAY 10 #define QM_SQE_ADDR_MASK GENMASK(7, 0) +#define QM_EQ_DEPTH (1024 * 2) #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ @@ -652,7 +655,7 @@ static void qm_work_process(struct work_struct *work) qp = qm_to_hisi_qp(qm, eqe); qm_poll_qp(qp, qm); - if (qm->status.eq_head == QM_Q_DEPTH - 1) { + if (qm->status.eq_head == QM_EQ_DEPTH - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; eqe = qm->eqe; qm->status.eq_head = 0; @@ -661,7 +664,7 @@ static void qm_work_process(struct work_struct *work) qm->status.eq_head++; } - if (eqe_num == QM_Q_DEPTH / 2 - 1) { + if (eqe_num == QM_EQ_DEPTH / 2 - 1) { eqe_num = 0; qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); } @@ -754,7 +757,7 @@ static void qm_init_qp_status(struct hisi_qp *qp) qp_status->sq_tail = 0; qp_status->cq_head = 0; qp_status->cqc_phase = true; - atomic_set(&qp_status->flags, 0); + atomic_set(&qp_status->used, 0); } static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, @@ -1046,17 +1049,7 @@ static int qm_regs_show(struct seq_file *s, void *unused) return 0; } -static int qm_regs_open(struct inode *inode, struct file *file) -{ - return single_open(file, qm_regs_show, inode->i_private); -} - -static const struct file_operations qm_regs_fops = { - .owner = THIS_MODULE, - .open = qm_regs_open, - .read = seq_read, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(qm_regs); static ssize_t qm_cmd_read(struct file *filp, char __user *buffer, size_t count, loff_t *pos) @@ -1370,7 +1363,13 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, return -EINVAL; ret = kstrtou32(s, 0, &xeqe_id); - if (ret || xeqe_id >= QM_Q_DEPTH) { + if (ret) + return -EINVAL; + + if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) { + dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1); + return -EINVAL; + } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) { dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1); return -EINVAL; } @@ -1420,17 +1419,18 @@ static int qm_dbg_help(struct hisi_qm *qm, char *s) static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) { struct device *dev = &qm->pdev->dev; - char *presult, *s; + char *presult, *s, *s_tmp; int ret; s = kstrdup(cmd_buf, GFP_KERNEL); if (!s) return -ENOMEM; + s_tmp = s; presult = strsep(&s, " "); if (!presult) { - kfree(s); - return -EINVAL; + ret = -EINVAL; + goto err_buffer_free; } if (!strcmp(presult, "sqc")) @@ -1459,7 +1459,8 @@ static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf) if (ret) dev_info(dev, "Please echo help\n"); - kfree(s); +err_buffer_free: + kfree(s_tmp); return ret; } @@ -1644,7 +1645,7 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH)) + if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1)) return NULL; return qp->sqe + sq_tail * qp->qm->sqe_size; @@ -1981,7 +1982,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg) if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || atomic_read(&qp->qm->status.flags) == QM_STOP || qp->is_resetting)) { - dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); + dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); return -EAGAIN; } @@ -2215,6 +2216,82 @@ static int qm_alloc_uacce(struct hisi_qm *qm) } /** + * qm_frozen() - Try to froze QM to cut continuous queue request. If + * there is user on the QM, return failure without doing anything. + * @qm: The qm needed to be fronzen. + * + * This function frozes QM, then we can do SRIOV disabling. + */ +static int qm_frozen(struct hisi_qm *qm) +{ + down_write(&qm->qps_lock); + + if (qm->is_frozen) { + up_write(&qm->qps_lock); + return 0; + } + + if (!qm->qp_in_used) { + qm->qp_in_used = qm->qp_num; + qm->is_frozen = true; + up_write(&qm->qps_lock); + return 0; + } + + up_write(&qm->qps_lock); + + return -EBUSY; +} + +static int qm_try_frozen_vfs(struct pci_dev *pdev, + struct hisi_qm_list *qm_list) +{ + struct hisi_qm *qm, *vf_qm; + struct pci_dev *dev; + int ret = 0; + + if (!qm_list || !pdev) + return -EINVAL; + + /* Try to frozen all the VFs as disable SRIOV */ + mutex_lock(&qm_list->lock); + list_for_each_entry(qm, &qm_list->list, list) { + dev = qm->pdev; + if (dev == pdev) + continue; + if (pci_physfn(dev) == pdev) { + vf_qm = pci_get_drvdata(dev); + ret = qm_frozen(vf_qm); + if (ret) + goto frozen_fail; + } + } + +frozen_fail: + mutex_unlock(&qm_list->lock); + + return ret; +} + +/** + * hisi_qm_wait_task_finish() - Wait until the task is finished + * when removing the driver. + * @qm: The qm needed to wait for the task to finish. + * @qm_list: The list of all available devices. + */ +void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + while (qm_frozen(qm) || + ((qm->fun_type == QM_HW_PF) && + qm_try_frozen_vfs(qm->pdev, qm_list))) { + msleep(WAIT_PERIOD); + } + + udelay(REMOVE_WAIT_DELAY); +} +EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); + +/** * hisi_qm_get_free_qp_num() - Get free number of qp in qm. * @qm: The qm which want to get free qp. * @@ -2282,7 +2359,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) + + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); @@ -2292,7 +2369,7 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) if (!qm->qdma.va) return -ENOMEM; - QM_INIT_BUF(qm, eqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); QM_INIT_BUF(qm, sqc, qm->qp_num); QM_INIT_BUF(qm, cqc, qm->qp_num); @@ -2338,6 +2415,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) mutex_init(&qm->mailbox_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; + qm->is_frozen = false; } /** @@ -2462,7 +2540,7 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); - eqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); @@ -2633,18 +2711,20 @@ static void qm_clear_queues(struct hisi_qm *qm) /** * hisi_qm_stop() - Stop a qm. * @qm: The qm which will be stopped. + * @r: The reason to stop qm. * * This function stops qm and its qps, then qm can not accept request. * Related resources are not released at this state, we can use hisi_qm_start * to let qm start again. */ -int hisi_qm_stop(struct hisi_qm *qm) +int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) { struct device *dev = &qm->pdev->dev; int ret = 0; down_write(&qm->qps_lock); + qm->status.stop_reason = r; if (!qm_avail_state(qm, QM_STOP)) { ret = -EPERM; goto err_unlock; @@ -3081,11 +3161,12 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); /** * hisi_qm_sriov_disable - disable virtual functions - * @pdev: the PCI device + * @pdev: the PCI device. + * @is_frozen: true when all the VFs are frozen. * - * Return failure if there are VFs assigned already. + * Return failure if there are VFs assigned already or VF is in used. */ -int hisi_qm_sriov_disable(struct pci_dev *pdev) +int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); @@ -3094,7 +3175,12 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev) return -EPERM; } - /* remove in hpre_pci_driver will be called to free VF resources */ + /* While VF is in used, SRIOV cannot be disabled. */ + if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { + pci_err(pdev, "Task is using its VF!\n"); + return -EBUSY; + } + pci_disable_sriov(pdev); return qm_clear_vft_config(qm); } @@ -3110,7 +3196,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { if (num_vfs == 0) - return hisi_qm_sriov_disable(pdev); + return hisi_qm_sriov_disable(pdev, 0); else return hisi_qm_sriov_enable(pdev, num_vfs); } @@ -3290,10 +3376,10 @@ static int qm_set_msi(struct hisi_qm *qm, bool set) return 0; } -static int qm_vf_reset_prepare(struct hisi_qm *qm) +static int qm_vf_reset_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) { struct hisi_qm_list *qm_list = qm->qm_list; - int stop_reason = qm->status.stop_reason; struct pci_dev *pdev = qm->pdev; struct pci_dev *virtfn; struct hisi_qm *vf_qm; @@ -3306,8 +3392,10 @@ static int qm_vf_reset_prepare(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { - vf_qm->status.stop_reason = stop_reason; - ret = hisi_qm_stop(vf_qm); + /* save VFs PCIE BAR configuration */ + pci_save_state(virtfn); + + ret = hisi_qm_stop(vf_qm, stop_reason); if (ret) goto stop_fail; } @@ -3346,15 +3434,14 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop VFs!\n"); return ret; } } - qm->status.stop_reason = QM_SOFT_RESET; - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); return ret; @@ -3471,6 +3558,9 @@ static int qm_vf_reset_done(struct hisi_qm *qm) continue; if (pci_physfn(virtfn) == pdev) { + /* enable VFs PCIE BAR configuration */ + pci_restore_state(virtfn); + ret = qm_restart(vf_qm); if (ret) goto restart_fail; @@ -3695,7 +3785,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm); + ret = qm_vf_reset_prepare(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to prepare reset, ret = %d.\n", ret); @@ -3703,7 +3793,7 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) } } - ret = hisi_qm_stop(qm); + ret = hisi_qm_stop(qm, QM_FLR); if (ret) { pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); return; @@ -3821,6 +3911,23 @@ err_aeq_irq: return ret; } +/** + * hisi_qm_dev_shutdown() - Shutdown device. + * @pdev: The device will be shutdown. + * + * This function will stop qm when OS shutdown or rebooting. + */ +void hisi_qm_dev_shutdown(struct pci_dev *pdev) +{ + struct hisi_qm *qm = pci_get_drvdata(pdev); + int ret; + + ret = hisi_qm_stop(qm, QM_NORMAL); + if (ret) + dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); +} +EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown); + static void hisi_qm_controller_reset(struct work_struct *rst_work) { struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); @@ -3834,6 +3941,58 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) } /** + * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list. + * @qm: The qm needs add. + * @qm_list: The qm list. + * + * This function adds qm to qm list, and will register algorithm to + * crypto when the qm list is empty. + */ +int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + int flag = 0; + int ret = 0; + + mutex_lock(&qm_list->lock); + if (list_empty(&qm_list->list)) + flag = 1; + list_add_tail(&qm->list, &qm_list->list); + mutex_unlock(&qm_list->lock); + + if (flag) { + ret = qm_list->register_to_crypto(); + if (ret) { + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_register); + +/** + * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from + * qm list. + * @qm: The qm needs delete. + * @qm_list: The qm list. + * + * This function deletes qm from qm list, and will unregister algorithm + * from crypto when the qm list is empty. + */ +void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) +{ + mutex_lock(&qm_list->lock); + list_del(&qm->list); + mutex_unlock(&qm_list->lock); + + if (list_empty(&qm_list->list)) + qm_list->unregister_from_crypto(); +} +EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); + +/** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. * |