diff options
Diffstat (limited to 'drivers/crypto/cavium')
| -rw-r--r-- | drivers/crypto/cavium/cpt/cptvf_reqmanager.c | 20 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/Makefile | 3 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_common.h | 19 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_csr.h | 111 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_debugfs.c | 115 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_dev.h | 162 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_hal.c | 71 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_hal.h | 23 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_isr.c | 337 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_isr.h | 10 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_lib.c | 98 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_main.c | 203 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 49 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_sriov.c | 151 |
14 files changed, 829 insertions, 543 deletions
diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index b0ba4331944b..ca549c5dc08e 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -308,21 +308,11 @@ void do_request_cleanup(struct cpt_vf *cptvf, } } - if (info->scatter_components) - kzfree(info->scatter_components); - - if (info->gather_components) - kzfree(info->gather_components); - - if (info->out_buffer) - kzfree(info->out_buffer); - - if (info->in_buffer) - kzfree(info->in_buffer); - - if (info->completion_addr) - kzfree((void *)info->completion_addr); - + kzfree(info->scatter_components); + kzfree(info->gather_components); + kzfree(info->out_buffer); + kzfree(info->in_buffer); + kzfree((void *)info->completion_addr); kzfree(info); } diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile index 45b7379e8e30..e12954791673 100644 --- a/drivers/crypto/cavium/nitrox/Makefile +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -7,3 +7,6 @@ n5pf-objs := nitrox_main.o \ nitrox_hal.o \ nitrox_reqmgr.o \ nitrox_algs.o + +n5pf-$(CONFIG_PCI_IOV) += nitrox_sriov.o +n5pf-$(CONFIG_DEBUG_FS) += nitrox_debugfs.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h index 312f72801af6..863143a8336b 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_common.h +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -12,32 +12,15 @@ void crypto_free_context(void *ctx); struct nitrox_device *nitrox_get_first_device(void); void nitrox_put_device(struct nitrox_device *ndev); -void nitrox_pf_cleanup_isr(struct nitrox_device *ndev); -int nitrox_pf_init_isr(struct nitrox_device *ndev); - int nitrox_common_sw_init(struct nitrox_device *ndev); void nitrox_common_sw_cleanup(struct nitrox_device *ndev); -void pkt_slc_resp_handler(unsigned long data); +void pkt_slc_resp_tasklet(unsigned long data); int nitrox_process_se_request(struct nitrox_device *ndev, struct se_crypto_request *req, completion_t cb, struct skcipher_request *skreq); void backlog_qflush_work(struct work_struct *work); -void nitrox_config_emu_unit(struct nitrox_device *ndev); -void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); -void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); -void nitrox_config_vfmode(struct nitrox_device *ndev, int mode); -void nitrox_config_nps_unit(struct nitrox_device *ndev); -void nitrox_config_pom_unit(struct nitrox_device *ndev); -void nitrox_config_rand_unit(struct nitrox_device *ndev); -void nitrox_config_efl_unit(struct nitrox_device *ndev); -void nitrox_config_bmi_unit(struct nitrox_device *ndev); -void nitrox_config_bmo_unit(struct nitrox_device *ndev); -void nitrox_config_lbc_unit(struct nitrox_device *ndev); -void invalidate_lbc(struct nitrox_device *ndev); -void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); -void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); #endif /* __NITROX_COMMON_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h index 9dcb7fdbe0a7..1ad27b1a87c5 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_csr.h +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -7,9 +7,16 @@ /* EMU clusters */ #define NR_CLUSTERS 4 +/* Maximum cores per cluster, + * varies based on partname + */ #define AE_CORES_PER_CLUSTER 20 #define SE_CORES_PER_CLUSTER 16 +#define AE_MAX_CORES (AE_CORES_PER_CLUSTER * NR_CLUSTERS) +#define SE_MAX_CORES (SE_CORES_PER_CLUSTER * NR_CLUSTERS) +#define ZIP_MAX_CORES 5 + /* BIST registers */ #define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000)) #define UCD_BIST_STATUS 0x12C0070 @@ -111,6 +118,9 @@ #define LBC_ELM_VF65_128_INT 0x120C000 #define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000 +#define RST_BOOT 0x10C1600 +#define FUS_DAT1 0x10C1408 + /* PEM registers */ #define PEM0_INT 0x1080428 @@ -1082,4 +1092,105 @@ union lbc_inval_status { } s; }; +/** + * struct rst_boot: RST Boot Register + * @jtcsrdis: when set, internal CSR access via JTAG TAP controller + * is disabled + * @jt_tst_mode: JTAG test mode + * @io_supply: I/O power supply setting based on IO_VDD_SELECT pin: + * 0x1 = 1.8V + * 0x2 = 2.5V + * 0x4 = 3.3V + * All other values are reserved + * @pnr_mul: clock multiplier + * @lboot: last boot cause mask, resets only with PLL_DC_OK + * @rboot: determines whether core 0 remains in reset after + * chip cold or warm or soft reset + * @rboot_pin: read only access to REMOTE_BOOT pin + */ +union rst_boot { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_63 : 1; + u64 jtcsrdis : 1; + u64 raz_59_61 : 3; + u64 jt_tst_mode : 1; + u64 raz_40_57 : 18; + u64 io_supply : 3; + u64 raz_30_36 : 7; + u64 pnr_mul : 6; + u64 raz_12_23 : 12; + u64 lboot : 10; + u64 rboot : 1; + u64 rboot_pin : 1; +#else + u64 rboot_pin : 1; + u64 rboot : 1; + u64 lboot : 10; + u64 raz_12_23 : 12; + u64 pnr_mul : 6; + u64 raz_30_36 : 7; + u64 io_supply : 3; + u64 raz_40_57 : 18; + u64 jt_tst_mode : 1; + u64 raz_59_61 : 3; + u64 jtcsrdis : 1; + u64 raz_63 : 1; +#endif + }; +}; + +/** + * struct fus_dat1: Fuse Data 1 Register + * @pll_mul: main clock PLL multiplier hardware limit + * @pll_half_dis: main clock PLL control + * @efus_lck: efuse lockdown + * @zip_info: ZIP information + * @bar2_sz_conf: when zero, BAR2 size conforms to + * PCIe specification + * @efus_ign: efuse ignore + * @nozip: ZIP disable + * @pll_alt_matrix: select alternate PLL matrix + * @pll_bwadj_denom: select CLKF denominator for + * BWADJ value + * @chip_id: chip ID + */ +union fus_dat1 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_57_63 : 7; + u64 pll_mul : 3; + u64 pll_half_dis : 1; + u64 raz_43_52 : 10; + u64 efus_lck : 3; + u64 raz_26_39 : 14; + u64 zip_info : 5; + u64 bar2_sz_conf : 1; + u64 efus_ign : 1; + u64 nozip : 1; + u64 raz_11_17 : 7; + u64 pll_alt_matrix : 1; + u64 pll_bwadj_denom : 2; + u64 chip_id : 8; +#else + u64 chip_id : 8; + u64 pll_bwadj_denom : 2; + u64 pll_alt_matrix : 1; + u64 raz_11_17 : 7; + u64 nozip : 1; + u64 efus_ign : 1; + u64 bar2_sz_conf : 1; + u64 zip_info : 5; + u64 raz_26_39 : 14; + u64 efus_lck : 3; + u64 raz_43_52 : 10; + u64 pll_half_dis : 1; + u64 pll_mul : 3; + u64 raz_57_63 : 7; +#endif + }; +}; + #endif /* __NITROX_CSR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c new file mode 100644 index 000000000000..5f3cd5fafe04 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/seq_file.h> +#include <linux/debugfs.h> + +#include "nitrox_csr.h" +#include "nitrox_dev.h" + +static int firmware_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "Version: %s\n", ndev->hw.fw_name); + return 0; +} + +static int firmware_open(struct inode *inode, struct file *file) +{ + return single_open(file, firmware_show, inode->i_private); +} + +static const struct file_operations firmware_fops = { + .owner = THIS_MODULE, + .open = firmware_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int device_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX [%d]\n", ndev->idx); + seq_printf(s, " Part Name: %s\n", ndev->hw.partname); + seq_printf(s, " Frequency: %d MHz\n", ndev->hw.freq); + seq_printf(s, " Device ID: 0x%0x\n", ndev->hw.device_id); + seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); + seq_printf(s, " Cores: [AE=%u SE=%u ZIP=%u]\n", + ndev->hw.ae_cores, ndev->hw.se_cores, ndev->hw.zip_cores); + + return 0; +} + +static int nitrox_open(struct inode *inode, struct file *file) +{ + return single_open(file, device_show, inode->i_private); +} + +static const struct file_operations nitrox_fops = { + .owner = THIS_MODULE, + .open = nitrox_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int stats_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX [%d] Request Statistics\n", ndev->idx); + seq_printf(s, " Posted: %llu\n", + (u64)atomic64_read(&ndev->stats.posted)); + seq_printf(s, " Completed: %llu\n", + (u64)atomic64_read(&ndev->stats.completed)); + seq_printf(s, " Dropped: %llu\n", + (u64)atomic64_read(&ndev->stats.dropped)); + + return 0; +} + +static int nitrox_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, stats_show, inode->i_private); +} + +static const struct file_operations nitrox_stats_fops = { + .owner = THIS_MODULE, + .open = nitrox_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ + debugfs_remove_recursive(ndev->debugfs_dir); + ndev->debugfs_dir = NULL; +} + +int nitrox_debugfs_init(struct nitrox_device *ndev) +{ + struct dentry *dir, *f; + + dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (!dir) + return -ENOMEM; + + ndev->debugfs_dir = dir; + f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); + if (!f) + goto err; + f = debugfs_create_file("device", 0400, dir, ndev, &nitrox_fops); + if (!f) + goto err; + f = debugfs_create_file("stats", 0400, dir, ndev, &nitrox_stats_fops); + if (!f) + goto err; + + return 0; + +err: + nitrox_debugfs_exit(ndev); + return -ENODEV; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h index af596455b420..283e252385fb 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_dev.h +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -5,92 +5,123 @@ #include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/pci.h> +#include <linux/if.h> #define VERSION_LEN 32 +/** + * struct nitrox_cmdq - NITROX command queue + * @cmd_qlock: command queue lock + * @resp_qlock: response queue lock + * @backlog_qlock: backlog queue lock + * @ndev: NITROX device + * @response_head: submitted request list + * @backlog_head: backlog queue + * @dbell_csr_addr: doorbell register address for this queue + * @compl_cnt_csr_addr: completion count register address of the slc port + * @base: command queue base address + * @dma: dma address of the base + * @pending_count: request pending at device + * @backlog_count: backlog request count + * @write_idx: next write index for the command + * @instr_size: command size + * @qno: command queue number + * @qsize: command queue size + * @unalign_base: unaligned base address + * @unalign_dma: unaligned dma address + */ struct nitrox_cmdq { - /* command queue lock */ - spinlock_t cmdq_lock; - /* response list lock */ - spinlock_t response_lock; - /* backlog list lock */ - spinlock_t backlog_lock; - - /* request submitted to chip, in progress */ + spinlock_t cmd_qlock; + spinlock_t resp_qlock; + spinlock_t backlog_qlock; + + struct nitrox_device *ndev; struct list_head response_head; - /* hw queue full, hold in backlog list */ struct list_head backlog_head; - /* doorbell address */ u8 __iomem *dbell_csr_addr; - /* base address of the queue */ - u8 *head; + u8 __iomem *compl_cnt_csr_addr; + u8 *base; + dma_addr_t dma; - struct nitrox_device *ndev; - /* flush pending backlog commands */ struct work_struct backlog_qflush; - /* requests posted waiting for completion */ atomic_t pending_count; - /* requests in backlog queues */ atomic_t backlog_count; int write_idx; - /* command size 32B/64B */ u8 instr_size; u8 qno; u32 qsize; - /* unaligned addresses */ - u8 *head_unaligned; - dma_addr_t dma_unaligned; - /* dma address of the base */ - dma_addr_t dma; + u8 *unalign_base; + dma_addr_t unalign_dma; }; +/** + * struct nitrox_hw - NITROX hardware information + * @partname: partname ex: CNN55xxx-xxx + * @fw_name: firmware version + * @freq: NITROX frequency + * @vendor_id: vendor ID + * @device_id: device ID + * @revision_id: revision ID + * @se_cores: number of symmetric cores + * @ae_cores: number of asymmetric cores + * @zip_cores: number of zip cores + */ struct nitrox_hw { - /* firmware version */ + char partname[IFNAMSIZ * 2]; char fw_name[VERSION_LEN]; + int freq; u16 vendor_id; u16 device_id; u8 revision_id; - /* CNN55XX cores */ u8 se_cores; u8 ae_cores; u8 zip_cores; }; -#define MAX_MSIX_VECTOR_NAME 20 -/** - * vectors for queues (64 AE, 64 SE and 64 ZIP) and - * error condition/mailbox. - */ -#define MAX_MSIX_VECTORS 192 - -struct nitrox_msix { - struct msix_entry *entries; - char **names; - DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS); - u32 nr_entries; +struct nitrox_stats { + atomic64_t posted; + atomic64_t completed; + atomic64_t dropped; }; -struct bh_data { - /* slc port completion count address */ - u8 __iomem *completion_cnt_csr_addr; +#define IRQ_NAMESZ 32 + +struct nitrox_q_vector { + char name[IRQ_NAMESZ]; + bool valid; + int ring; + struct tasklet_struct resp_tasklet; + union { + struct nitrox_cmdq *cmdq; + struct nitrox_device *ndev; + }; +}; - struct nitrox_cmdq *cmdq; - struct tasklet_struct resp_handler; +/* + * NITROX Device states + */ +enum ndev_state { + __NDEV_NOT_READY, + __NDEV_READY, + __NDEV_IN_RESET, }; -struct nitrox_bh { - struct bh_data *slc; +/* NITROX support modes for VF(s) */ +enum vf_mode { + __NDEV_MODE_PF, + __NDEV_MODE_VF16, + __NDEV_MODE_VF32, + __NDEV_MODE_VF64, + __NDEV_MODE_VF128, }; -/* NITROX-V driver state */ -#define NITROX_UCODE_LOADED 0 -#define NITROX_READY 1 +#define __NDEV_SRIOV_BIT 0 /* command queue size */ #define DEFAULT_CMD_QLEN 2048 @@ -98,7 +129,6 @@ struct nitrox_bh { #define CMD_TIMEOUT 2000 #define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev)) -#define PF_MODE 0 #define NITROX_CSR_ADDR(ndev, offset) \ ((ndev)->bar_addr + (offset)) @@ -108,17 +138,18 @@ struct nitrox_bh { * @list: pointer to linked list of devices * @bar_addr: iomap address * @pdev: PCI device information - * @status: NITROX status + * @state: NITROX device state + * @flags: flags to indicate device the features * @timeout: Request timeout in jiffies * @refcnt: Device usage count * @idx: device index (0..N) * @node: NUMA node id attached * @qlen: Command queue length * @nr_queues: Number of command queues + * @mode: Device mode PF/VF * @ctx_pool: DMA pool for crypto context - * @pkt_cmdqs: SE Command queues - * @msix: MSI-X information - * @bh: post processing work + * @pkt_inq: Packet input rings + * @qvec: MSI-X queue vectors information * @hw: hardware information * @debugfs_dir: debugfs directory */ @@ -128,7 +159,8 @@ struct nitrox_device { u8 __iomem *bar_addr; struct pci_dev *pdev; - unsigned long status; + atomic_t state; + unsigned long flags; unsigned long timeout; refcount_t refcnt; @@ -136,13 +168,16 @@ struct nitrox_device { int node; u16 qlen; u16 nr_queues; + int num_vfs; + enum vf_mode mode; struct dma_pool *ctx_pool; - struct nitrox_cmdq *pkt_cmdqs; + struct nitrox_cmdq *pkt_inq; - struct nitrox_msix msix; - struct nitrox_bh bh; + struct nitrox_q_vector *qvec; + int num_vecs; + struct nitrox_stats stats; struct nitrox_hw hw; #if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *debugfs_dir; @@ -173,9 +208,22 @@ static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset, writeq(value, (ndev->bar_addr + offset)); } -static inline int nitrox_ready(struct nitrox_device *ndev) +static inline bool nitrox_ready(struct nitrox_device *ndev) { - return test_bit(NITROX_READY, &ndev->status); + return atomic_read(&ndev->state) == __NDEV_READY; } +#ifdef CONFIG_DEBUG_FS +int nitrox_debugfs_init(struct nitrox_device *ndev); +void nitrox_debugfs_exit(struct nitrox_device *ndev); +#else +static inline int nitrox_debugfs_init(struct nitrox_device *ndev) +{ + return 0; +} + +static inline void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ } +#endif + #endif /* __NITROX_DEV_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index ab4ccf2f9e77..a9b82387cf53 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -4,6 +4,8 @@ #include "nitrox_dev.h" #include "nitrox_csr.h" +#define PLL_REF_CLK 50 + /** * emu_enable_cores - Enable EMU cluster cores. * @ndev: N5 device @@ -117,7 +119,7 @@ void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) int i; for (i = 0; i < ndev->nr_queues; i++) { - struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; union nps_pkt_in_instr_rsize pkt_in_rsize; u64 offset; @@ -256,7 +258,7 @@ void nitrox_config_nps_unit(struct nitrox_device *ndev) /* disable ILK interface */ core_gbl_vfcfg.value = 0; core_gbl_vfcfg.s.ilk_disable = 1; - core_gbl_vfcfg.s.cfg = PF_MODE; + core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF; nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value); /* config input and solicit ports */ nitrox_config_pkt_input_rings(ndev); @@ -400,3 +402,68 @@ void nitrox_config_lbc_unit(struct nitrox_device *ndev) offset = LBC_ELM_VF65_128_INT_ENA_W1S; nitrox_write_csr(ndev, offset, (~0ULL)); } + +void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode) +{ + union nps_core_gbl_vfcfg vfcfg; + + vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG); + vfcfg.s.cfg = mode & 0x7; + + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value); +} + +void nitrox_get_hwinfo(struct nitrox_device *ndev) +{ + union emu_fuse_map emu_fuse; + union rst_boot rst_boot; + union fus_dat1 fus_dat1; + unsigned char name[IFNAMSIZ * 2] = {}; + int i, dead_cores; + u64 offset; + + /* get core frequency */ + offset = RST_BOOT; + rst_boot.value = nitrox_read_csr(ndev, offset); + ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_FUSE_MAPX(i); + emu_fuse.value = nitrox_read_csr(ndev, offset); + if (emu_fuse.s.valid) { + dead_cores = hweight32(emu_fuse.s.ae_fuse); + ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; + dead_cores = hweight16(emu_fuse.s.se_fuse); + ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; + } + } + /* find zip hardware availability */ + offset = FUS_DAT1; + fus_dat1.value = nitrox_read_csr(ndev, offset); + if (!fus_dat1.nozip) { + dead_cores = hweight8(fus_dat1.zip_info); + ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores; + } + + /* determine the partname CNN55<cores>-<freq><pincount>-<rev>*/ + if (ndev->hw.ae_cores == AE_MAX_CORES) { + switch (ndev->hw.se_cores) { + case SE_MAX_CORES: + i = snprintf(name, sizeof(name), "CNN5560"); + break; + case 40: + i = snprintf(name, sizeof(name), "CNN5560s"); + break; + } + } else if (ndev->hw.ae_cores == (AE_MAX_CORES / 2)) { + i = snprintf(name, sizeof(name), "CNN5530"); + } else { + i = snprintf(name, sizeof(name), "CNN5560i"); + } + + snprintf(name + i, sizeof(name) - i, "-%3dBG676-1.%u", + ndev->hw.freq, ndev->hw.revision_id); + + /* copy partname */ + strncpy(ndev->hw.partname, name, sizeof(ndev->hw.partname)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.h b/drivers/crypto/cavium/nitrox/nitrox_hal.h new file mode 100644 index 000000000000..489ee64c119e --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_HAL_H +#define __NITROX_HAL_H + +#include "nitrox_dev.h" + +void nitrox_config_emu_unit(struct nitrox_device *ndev); +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); +void nitrox_config_nps_unit(struct nitrox_device *ndev); +void nitrox_config_pom_unit(struct nitrox_device *ndev); +void nitrox_config_rand_unit(struct nitrox_device *ndev); +void nitrox_config_efl_unit(struct nitrox_device *ndev); +void nitrox_config_bmi_unit(struct nitrox_device *ndev); +void nitrox_config_bmo_unit(struct nitrox_device *ndev); +void nitrox_config_lbc_unit(struct nitrox_device *ndev); +void invalidate_lbc(struct nitrox_device *ndev); +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); +void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode); +void nitrox_get_hwinfo(struct nitrox_device *ndev); + +#endif /* __NITROX_HAL_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c index ee0d70ba25d5..88a77b8fb3fb 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -6,9 +6,16 @@ #include "nitrox_dev.h" #include "nitrox_csr.h" #include "nitrox_common.h" +#include "nitrox_hal.h" +/** + * One vector for each type of ring + * - NPS packet ring, AQMQ ring and ZQMQ ring + */ #define NR_RING_VECTORS 3 -#define NPS_CORE_INT_ACTIVE_ENTRY 192 +/* base entry for packet ring/port */ +#define PKT_RING_MSIX_BASE 0 +#define NON_RING_MSIX_BASE 192 /** * nps_pkt_slc_isr - IRQ handler for NPS solicit port @@ -17,13 +24,14 @@ */ static irqreturn_t nps_pkt_slc_isr(int irq, void *data) { - struct bh_data *slc = data; - union nps_pkt_slc_cnts pkt_slc_cnts; + struct nitrox_q_vector *qvec = data; + union nps_pkt_slc_cnts slc_cnts; + struct nitrox_cmdq *cmdq = qvec->cmdq; - pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr); + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); /* New packet on SLC output port */ - if (pkt_slc_cnts.s.slc_int) - tasklet_hi_schedule(&slc->resp_handler); + if (slc_cnts.s.slc_int) + tasklet_hi_schedule(&qvec->resp_tasklet); return IRQ_HANDLED; } @@ -190,165 +198,92 @@ static void clear_bmi_err_intr(struct nitrox_device *ndev) dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); } +static void nps_core_int_tasklet(unsigned long data) +{ + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_device *ndev = qvec->ndev; + + /* if pf mode do queue recovery */ + if (ndev->mode == __NDEV_MODE_PF) { + } else { + /** + * if VF(s) enabled communicate the error information + * to VF(s) + */ + } +} + /** - * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts - * @ndev: NITROX device + * nps_core_int_isr - interrupt handler for NITROX errors and + * mailbox communication */ -static void clear_nps_core_int_active(struct nitrox_device *ndev) +static irqreturn_t nps_core_int_isr(int irq, void *data) { - union nps_core_int_active core_int_active; + struct nitrox_device *ndev = data; + union nps_core_int_active core_int; - core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); + core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); - if (core_int_active.s.nps_core) + if (core_int.s.nps_core) clear_nps_core_err_intr(ndev); - if (core_int_active.s.nps_pkt) + if (core_int.s.nps_pkt) clear_nps_pkt_err_intr(ndev); - if (core_int_active.s.pom) + if (core_int.s.pom) clear_pom_err_intr(ndev); - if (core_int_active.s.pem) + if (core_int.s.pem) clear_pem_err_intr(ndev); - if (core_int_active.s.lbc) + if (core_int.s.lbc) clear_lbc_err_intr(ndev); - if (core_int_active.s.efl) + if (core_int.s.efl) clear_efl_err_intr(ndev); - if (core_int_active.s.bmi) + if (core_int.s.bmi) clear_bmi_err_intr(ndev); /* If more work callback the ISR, set resend */ - core_int_active.s.resend = 1; - nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value); -} - -static irqreturn_t nps_core_int_isr(int irq, void *data) -{ - struct nitrox_device *ndev = data; - - clear_nps_core_int_active(ndev); + core_int.s.resend = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value); return IRQ_HANDLED; } -static int nitrox_enable_msix(struct nitrox_device *ndev) +void nitrox_unregister_interrupts(struct nitrox_device *ndev) { - struct msix_entry *entries; - char **names; - int i, nr_entries, ret; - - /* - * PF MSI-X vectors - * - * Entry 0: NPS PKT ring 0 - * Entry 1: AQMQ ring 0 - * Entry 2: ZQM ring 0 - * Entry 3: NPS PKT ring 1 - * Entry 4: AQMQ ring 1 - * Entry 5: ZQM ring 1 - * .... - * Entry 192: NPS_CORE_INT_ACTIVE - */ - nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1; - entries = kcalloc_node(nr_entries, sizeof(struct msix_entry), - GFP_KERNEL, ndev->node); - if (!entries) - return -ENOMEM; - - names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL); - if (!names) { - kfree(entries); - return -ENOMEM; - } - - /* fill entires */ - for (i = 0; i < (nr_entries - 1); i++) - entries[i].entry = i; - - entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY; - - for (i = 0; i < nr_entries; i++) { - *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL); - if (!(*(names + i))) { - ret = -ENOMEM; - goto msix_fail; - } - } - ndev->msix.entries = entries; - ndev->msix.names = names; - ndev->msix.nr_entries = nr_entries; - - ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries, - ndev->msix.nr_entries); - if (ret) { - dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n", - ret); - goto msix_fail; - } - return 0; - -msix_fail: - for (i = 0; i < nr_entries; i++) - kfree(*(names + i)); - - kfree(entries); - kfree(names); - return ret; -} - -static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev) -{ - int i; - - if (!ndev->bh.slc) - return; - - for (i = 0; i < ndev->nr_queues; i++) { - struct bh_data *bh = &ndev->bh.slc[i]; - - tasklet_disable(&bh->resp_handler); - tasklet_kill(&bh->resp_handler); - } - kfree(ndev->bh.slc); - ndev->bh.slc = NULL; -} - -static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev) -{ - u32 size; + struct pci_dev *pdev = ndev->pdev; int i; - size = ndev->nr_queues * sizeof(struct bh_data); - ndev->bh.slc = kzalloc(size, GFP_KERNEL); - if (!ndev->bh.slc) - return -ENOMEM; + for (i = 0; i < ndev->num_vecs; i++) { + struct nitrox_q_vector *qvec; + int vec; - for (i = 0; i < ndev->nr_queues; i++) { - struct bh_data *bh = &ndev->bh.slc[i]; - u64 offset; + qvec = ndev->qvec + i; + if (!qvec->valid) + continue; - offset = NPS_PKT_SLC_CNTSX(i); - /* pre calculate completion count address */ - bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); - bh->cmdq = &ndev->pkt_cmdqs[i]; + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + irq_set_affinity_hint(vec, NULL); + free_irq(vec, qvec); - tasklet_init(&bh->resp_handler, pkt_slc_resp_handler, - (unsigned long)bh); + tasklet_disable(&qvec->resp_tasklet); + tasklet_kill(&qvec->resp_tasklet); + qvec->valid = false; } - - return 0; + kfree(ndev->qvec); + pci_free_irq_vectors(pdev); } -static int nitrox_request_irqs(struct nitrox_device *ndev) +int nitrox_register_interrupts(struct nitrox_device *ndev) { struct pci_dev *pdev = ndev->pdev; - struct msix_entry *msix_ent = ndev->msix.entries; - int nr_ring_vectors, i = 0, ring, cpu, ret; - char *name; + struct nitrox_q_vector *qvec; + int nr_vecs, vec, cpu; + int ret, i; /* * PF MSI-X vectors @@ -357,112 +292,76 @@ static int nitrox_request_irqs(struct nitrox_device *ndev) * Entry 1: AQMQ ring 0 * Entry 2: ZQM ring 0 * Entry 3: NPS PKT ring 1 + * Entry 4: AQMQ ring 1 + * Entry 5: ZQM ring 1 * .... * Entry 192: NPS_CORE_INT_ACTIVE */ - nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS; - - /* request irq for pkt ring/ports only */ - while (i < nr_ring_vectors) { - name = *(ndev->msix.names + i); - ring = (i / NR_RING_VECTORS); - snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d", - ndev->idx, ring); + nr_vecs = pci_msix_vec_count(pdev); - ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0, - name, &ndev->bh.slc[ring]); - if (ret) { - dev_err(&pdev->dev, "failed to get irq %d for %s\n", - msix_ent[i].vector, name); - return ret; - } - cpu = ring % num_online_cpus(); - irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu)); - - set_bit(i, ndev->msix.irqs); - i += NR_RING_VECTORS; - } - - /* Request IRQ for NPS_CORE_INT_ACTIVE */ - name = *(ndev->msix.names + i); - snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx); - ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev); - if (ret) { - dev_err(&pdev->dev, "failed to get irq %d for %s\n", - msix_ent[i].vector, name); + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs); return ret; } - set_bit(i, ndev->msix.irqs); + ndev->num_vecs = nr_vecs; - return 0; -} - -static void nitrox_disable_msix(struct nitrox_device *ndev) -{ - struct msix_entry *msix_ent = ndev->msix.entries; - char **names = ndev->msix.names; - int i = 0, ring, nr_ring_vectors; - - nr_ring_vectors = ndev->msix.nr_entries - 1; - - /* clear pkt ring irqs */ - while (i < nr_ring_vectors) { - if (test_and_clear_bit(i, ndev->msix.irqs)) { - ring = (i / NR_RING_VECTORS); - irq_set_affinity_hint(msix_ent[i].vector, NULL); - free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]); - } - i += NR_RING_VECTORS; + ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL); + if (!ndev->qvec) { + pci_free_irq_vectors(pdev); + return -ENOMEM; } - irq_set_affinity_hint(msix_ent[i].vector, NULL); - free_irq(msix_ent[i].vector, ndev); - clear_bit(i, ndev->msix.irqs); - kfree(ndev->msix.entries); - for (i = 0; i < ndev->msix.nr_entries; i++) - kfree(*(names + i)); + /* request irqs for packet rings/ports */ + for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) { + qvec = &ndev->qvec[i]; - kfree(names); - pci_disable_msix(ndev->pdev); -} - -/** - * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ - * @ndev: NITROX device - */ -void nitrox_pf_cleanup_isr(struct nitrox_device *ndev) -{ - nitrox_disable_msix(ndev); - nitrox_cleanup_pkt_slc_bh(ndev); -} + qvec->ring = i / NR_RING_VECTORS; + if (qvec->ring >= ndev->nr_queues) + break; -/** - * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ - * @ndev: NITROX device - * - * Return: 0 on success, a negative value on failure. - */ -int nitrox_pf_init_isr(struct nitrox_device *ndev) -{ - int err; + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring); + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n", + qvec->ring); + goto irq_fail; + } + cpu = qvec->ring % num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); - err = nitrox_setup_pkt_slc_bh(ndev); - if (err) - return err; + tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet, + (unsigned long)qvec); + qvec->cmdq = &ndev->pkt_inq[qvec->ring]; + qvec->valid = true; + } - err = nitrox_enable_msix(ndev); - if (err) - goto msix_fail; + /* request irqs for non ring vectors */ + i = NON_RING_MSIX_BASE; + qvec = &ndev->qvec[i]; - err = nitrox_request_irqs(ndev); - if (err) + snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i); + /* get the vector number */ + vec = pci_irq_vector(pdev, i); + ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); + if (ret) { + dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i); goto irq_fail; + } + cpu = num_online_cpus(); + irq_set_affinity_hint(vec, get_cpu_mask(cpu)); + + tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, + (unsigned long)qvec); + qvec->ndev = ndev; + qvec->valid = true; return 0; irq_fail: - nitrox_disable_msix(ndev); -msix_fail: - nitrox_cleanup_pkt_slc_bh(ndev); - return err; + nitrox_unregister_interrupts(ndev); + return ret; } diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h new file mode 100644 index 000000000000..63418a6cc52c --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NITROX_ISR_H +#define __NITROX_ISR_H + +#include "nitrox_dev.h" + +int nitrox_register_interrupts(struct nitrox_device *ndev); +void nitrox_unregister_interrupts(struct nitrox_device *ndev); + +#endif /* __NITROX_ISR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c index 4d31df07777f..2260efa42308 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_lib.c +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -17,30 +17,27 @@ #define CRYPTO_CTX_SIZE 256 -/* command queue alignments */ -#define PKT_IN_ALIGN 16 +/* packet inuput ring alignments */ +#define PKTIN_Q_ALIGN_BYTES 16 -static int cmdq_common_init(struct nitrox_cmdq *cmdq) +static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes) { struct nitrox_device *ndev = cmdq->ndev; - u32 qsize; - - qsize = (ndev->qlen) * cmdq->instr_size; - cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev), - (qsize + PKT_IN_ALIGN), - &cmdq->dma_unaligned, - GFP_KERNEL); - if (!cmdq->head_unaligned) + + cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes; + cmdq->unalign_base = dma_zalloc_coherent(DEV(ndev), cmdq->qsize, + &cmdq->unalign_dma, + GFP_KERNEL); + if (!cmdq->unalign_base) return -ENOMEM; - cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN); - cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN); - cmdq->qsize = (qsize + PKT_IN_ALIGN); + cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes); + cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma); cmdq->write_idx = 0; - spin_lock_init(&cmdq->response_lock); - spin_lock_init(&cmdq->cmdq_lock); - spin_lock_init(&cmdq->backlog_lock); + spin_lock_init(&cmdq->cmd_qlock); + spin_lock_init(&cmdq->resp_qlock); + spin_lock_init(&cmdq->backlog_qlock); INIT_LIST_HEAD(&cmdq->response_head); INIT_LIST_HEAD(&cmdq->backlog_head); @@ -51,68 +48,83 @@ static int cmdq_common_init(struct nitrox_cmdq *cmdq) return 0; } -static void cmdq_common_cleanup(struct nitrox_cmdq *cmdq) +static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq) +{ + cmdq->write_idx = 0; + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); +} + +static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq) { struct nitrox_device *ndev = cmdq->ndev; + if (!cmdq->unalign_base) + return; + cancel_work_sync(&cmdq->backlog_qflush); dma_free_coherent(DEV(ndev), cmdq->qsize, - cmdq->head_unaligned, cmdq->dma_unaligned); - - atomic_set(&cmdq->pending_count, 0); - atomic_set(&cmdq->backlog_count, 0); + cmdq->unalign_base, cmdq->unalign_dma); + nitrox_cmdq_reset(cmdq); cmdq->dbell_csr_addr = NULL; - cmdq->head = NULL; + cmdq->compl_cnt_csr_addr = NULL; + cmdq->unalign_base = NULL; + cmdq->base = NULL; + cmdq->unalign_dma = 0; cmdq->dma = 0; cmdq->qsize = 0; cmdq->instr_size = 0; } -static void nitrox_cleanup_pkt_cmdqs(struct nitrox_device *ndev) +static void nitrox_free_pktin_queues(struct nitrox_device *ndev) { int i; for (i = 0; i < ndev->nr_queues; i++) { - struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i]; - cmdq_common_cleanup(cmdq); + nitrox_cmdq_cleanup(cmdq); } - kfree(ndev->pkt_cmdqs); - ndev->pkt_cmdqs = NULL; + kfree(ndev->pkt_inq); + ndev->pkt_inq = NULL; } -static int nitrox_init_pkt_cmdqs(struct nitrox_device *ndev) +static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev) { - int i, err, size; + int i, err; - size = ndev->nr_queues * sizeof(struct nitrox_cmdq); - ndev->pkt_cmdqs = kzalloc(size, GFP_KERNEL); - if (!ndev->pkt_cmdqs) + ndev->pkt_inq = kcalloc_node(ndev->nr_queues, + sizeof(struct nitrox_cmdq), + GFP_KERNEL, ndev->node); + if (!ndev->pkt_inq) return -ENOMEM; for (i = 0; i < ndev->nr_queues; i++) { struct nitrox_cmdq *cmdq; u64 offset; - cmdq = &ndev->pkt_cmdqs[i]; + cmdq = &ndev->pkt_inq[i]; cmdq->ndev = ndev; cmdq->qno = i; cmdq->instr_size = sizeof(struct nps_pkt_instr); + /* packet input ring doorbell address */ offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); - /* SE ring doorbell address for this queue */ cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + /* packet solicit port completion count address */ + offset = NPS_PKT_SLC_CNTSX(i); + cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); - err = cmdq_common_init(cmdq); + err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES); if (err) - goto pkt_cmdq_fail; + goto pktq_fail; } return 0; -pkt_cmdq_fail: - nitrox_cleanup_pkt_cmdqs(ndev); +pktq_fail: + nitrox_free_pktin_queues(ndev); return err; } @@ -122,7 +134,7 @@ static int create_crypto_dma_pool(struct nitrox_device *ndev) /* Crypto context pool, 16 byte aligned */ size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr); - ndev->ctx_pool = dma_pool_create("crypto-context", + ndev->ctx_pool = dma_pool_create("nitrox-context", DEV(ndev), size, 16, 0); if (!ndev->ctx_pool) return -ENOMEM; @@ -149,7 +161,7 @@ void *crypto_alloc_context(struct nitrox_device *ndev) void *vaddr; dma_addr_t dma; - vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_KERNEL | __GFP_ZERO), &dma); + vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma); if (!vaddr) return NULL; @@ -194,7 +206,7 @@ int nitrox_common_sw_init(struct nitrox_device *ndev) if (err) return err; - err = nitrox_init_pkt_cmdqs(ndev); + err = nitrox_alloc_pktin_queues(ndev); if (err) destroy_crypto_dma_pool(ndev); @@ -207,6 +219,6 @@ int nitrox_common_sw_init(struct nitrox_device *ndev) */ void nitrox_common_sw_cleanup(struct nitrox_device *ndev) { - nitrox_cleanup_pkt_cmdqs(ndev); + nitrox_free_pktin_queues(ndev); destroy_crypto_dma_pool(ndev); } diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index fee7cb2ce747..6595c95af9f1 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -11,13 +11,15 @@ #include "nitrox_dev.h" #include "nitrox_common.h" #include "nitrox_csr.h" +#include "nitrox_hal.h" +#include "nitrox_isr.h" #define CNN55XX_DEV_ID 0x12 #define MAX_PF_QUEUES 64 #define UCODE_HLEN 48 #define SE_GROUP 0 -#define DRIVER_VERSION "1.0" +#define DRIVER_VERSION "1.1" #define FW_DIR "cavium/" /* SE microcode */ #define SE_FW FW_DIR "cnn55xx_se.fw" @@ -42,6 +44,15 @@ static unsigned int qlen = DEFAULT_CMD_QLEN; module_param(qlen, uint, 0644); MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); +#ifdef CONFIG_PCI_IOV +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); +#else +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + /** * struct ucode - Firmware Header * @id: microcode ID @@ -136,9 +147,6 @@ static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name) write_to_ucd_unit(ndev, ucode); release_firmware(fw); - set_bit(NITROX_UCODE_LOADED, &ndev->status); - /* barrier to sync with other cpus */ - smp_mb__after_atomic(); return 0; } @@ -210,7 +218,7 @@ void nitrox_put_device(struct nitrox_device *ndev) smp_mb__after_atomic(); } -static int nitrox_reset_device(struct pci_dev *pdev) +static int nitrox_device_flr(struct pci_dev *pdev) { int pos = 0; @@ -220,15 +228,10 @@ static int nitrox_reset_device(struct pci_dev *pdev) return -ENOMEM; } - pos = pci_pcie_cap(pdev); - if (!pos) - return -ENOTTY; + /* check flr support */ + if (pcie_has_flr(pdev)) + pcie_flr(pdev); - if (!pci_wait_for_pending_transaction(pdev)) - dev_err(&pdev->dev, "waiting for pending transaction\n"); - - pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); - msleep(100); pci_restore_state(pdev); return 0; @@ -242,7 +245,7 @@ static int nitrox_pf_sw_init(struct nitrox_device *ndev) if (err) return err; - err = nitrox_pf_init_isr(ndev); + err = nitrox_register_interrupts(ndev); if (err) nitrox_common_sw_cleanup(ndev); @@ -251,7 +254,7 @@ static int nitrox_pf_sw_init(struct nitrox_device *ndev) static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) { - nitrox_pf_cleanup_isr(ndev); + nitrox_unregister_interrupts(ndev); nitrox_common_sw_cleanup(ndev); } @@ -284,26 +287,6 @@ static int nitrox_bist_check(struct nitrox_device *ndev) return 0; } -static void nitrox_get_hwinfo(struct nitrox_device *ndev) -{ - union emu_fuse_map emu_fuse; - u64 offset; - int i; - - for (i = 0; i < NR_CLUSTERS; i++) { - u8 dead_cores; - - offset = EMU_FUSE_MAPX(i); - emu_fuse.value = nitrox_read_csr(ndev, offset); - if (emu_fuse.s.valid) { - dead_cores = hweight32(emu_fuse.s.ae_fuse); - ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; - dead_cores = hweight16(emu_fuse.s.se_fuse); - ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; - } - } -} - static int nitrox_pf_hw_init(struct nitrox_device *ndev) { int err; @@ -336,135 +319,6 @@ static int nitrox_pf_hw_init(struct nitrox_device *ndev) return 0; } -#if IS_ENABLED(CONFIG_DEBUG_FS) -static int registers_show(struct seq_file *s, void *v) -{ - struct nitrox_device *ndev = s->private; - u64 offset; - - /* NPS DMA stats */ - offset = NPS_STATS_PKT_DMA_RD_CNT; - seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n", - nitrox_read_csr(ndev, offset)); - offset = NPS_STATS_PKT_DMA_WR_CNT; - seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n", - nitrox_read_csr(ndev, offset)); - - /* BMI/BMO stats */ - offset = BMI_NPS_PKT_CNT; - seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n", - nitrox_read_csr(ndev, offset)); - offset = BMO_NPS_SLC_PKT_CNT; - seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n", - nitrox_read_csr(ndev, offset)); - - return 0; -} - -static int registers_open(struct inode *inode, struct file *file) -{ - return single_open(file, registers_show, inode->i_private); -} - -static const struct file_operations register_fops = { - .owner = THIS_MODULE, - .open = registers_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int firmware_show(struct seq_file *s, void *v) -{ - struct nitrox_device *ndev = s->private; - - seq_printf(s, "Version: %s\n", ndev->hw.fw_name); - return 0; -} - -static int firmware_open(struct inode *inode, struct file *file) -{ - return single_open(file, firmware_show, inode->i_private); -} - -static const struct file_operations firmware_fops = { - .owner = THIS_MODULE, - .open = firmware_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int nitrox_show(struct seq_file *s, void *v) -{ - struct nitrox_device *ndev = s->private; - - seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx); - seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); - seq_printf(s, " Cores [AE: %u SE: %u]\n", - ndev->hw.ae_cores, ndev->hw.se_cores); - seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues); - seq_printf(s, " Queue length: %u\n", ndev->qlen); - seq_printf(s, " Node: %u\n", ndev->node); - - return 0; -} - -static int nitrox_open(struct inode *inode, struct file *file) -{ - return single_open(file, nitrox_show, inode->i_private); -} - -static const struct file_operations nitrox_fops = { - .owner = THIS_MODULE, - .open = nitrox_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static void nitrox_debugfs_exit(struct nitrox_device *ndev) -{ - debugfs_remove_recursive(ndev->debugfs_dir); - ndev->debugfs_dir = NULL; -} - -static int nitrox_debugfs_init(struct nitrox_device *ndev) -{ - struct dentry *dir, *f; - - dir = debugfs_create_dir(KBUILD_MODNAME, NULL); - if (!dir) - return -ENOMEM; - - ndev->debugfs_dir = dir; - f = debugfs_create_file("counters", 0400, dir, ndev, ®ister_fops); - if (!f) - goto err; - f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); - if (!f) - goto err; - f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops); - if (!f) - goto err; - - return 0; - -err: - nitrox_debugfs_exit(ndev); - return -ENODEV; -} -#else -static int nitrox_debugfs_init(struct nitrox_device *ndev) -{ - return 0; -} - -static void nitrox_debugfs_exit(struct nitrox_device *ndev) -{ -} -#endif - /** * nitrox_probe - NITROX Initialization function. * @pdev: PCI device information struct @@ -487,7 +341,7 @@ static int nitrox_probe(struct pci_dev *pdev, return err; /* do FLR */ - err = nitrox_reset_device(pdev); + err = nitrox_device_flr(pdev); if (err) { dev_err(&pdev->dev, "FLR failed\n"); pci_disable_device(pdev); @@ -555,7 +409,12 @@ static int nitrox_probe(struct pci_dev *pdev, if (err) goto pf_hw_fail; - set_bit(NITROX_READY, &ndev->status); + /* clear the statistics */ + atomic64_set(&ndev->stats.posted, 0); + atomic64_set(&ndev->stats.completed, 0); + atomic64_set(&ndev->stats.dropped, 0); + + atomic_set(&ndev->state, __NDEV_READY); /* barrier to sync with other cpus */ smp_mb__after_atomic(); @@ -567,7 +426,7 @@ static int nitrox_probe(struct pci_dev *pdev, crypto_fail: nitrox_debugfs_exit(ndev); - clear_bit(NITROX_READY, &ndev->status); + atomic_set(&ndev->state, __NDEV_NOT_READY); /* barrier to sync with other cpus */ smp_mb__after_atomic(); pf_hw_fail: @@ -602,11 +461,16 @@ static void nitrox_remove(struct pci_dev *pdev) dev_info(DEV(ndev), "Removing Device %x:%x\n", ndev->hw.vendor_id, ndev->hw.device_id); - clear_bit(NITROX_READY, &ndev->status); + atomic_set(&ndev->state, __NDEV_NOT_READY); /* barrier to sync with other cpus */ smp_mb__after_atomic(); nitrox_remove_from_devlist(ndev); + +#ifdef CONFIG_PCI_IOV + /* disable SR-IOV */ + nitrox_sriov_configure(pdev, 0); +#endif nitrox_crypto_unregister(); nitrox_debugfs_exit(ndev); nitrox_pf_sw_cleanup(ndev); @@ -632,6 +496,9 @@ static struct pci_driver nitrox_driver = { .probe = nitrox_probe, .remove = nitrox_remove, .shutdown = nitrox_shutdown, +#ifdef CONFIG_PCI_IOV + .sriov_configure = nitrox_sriov_configure, +#endif }; module_pci_driver(nitrox_driver); diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index 4a362fc22f62..3987cd84c033 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -382,11 +382,11 @@ static inline void backlog_list_add(struct nitrox_softreq *sr, { INIT_LIST_HEAD(&sr->backlog); - spin_lock_bh(&cmdq->backlog_lock); + spin_lock_bh(&cmdq->backlog_qlock); list_add_tail(&sr->backlog, &cmdq->backlog_head); atomic_inc(&cmdq->backlog_count); atomic_set(&sr->status, REQ_BACKLOG); - spin_unlock_bh(&cmdq->backlog_lock); + spin_unlock_bh(&cmdq->backlog_qlock); } static inline void response_list_add(struct nitrox_softreq *sr, @@ -394,17 +394,17 @@ static inline void response_list_add(struct nitrox_softreq *sr, { INIT_LIST_HEAD(&sr->response); - spin_lock_bh(&cmdq->response_lock); + spin_lock_bh(&cmdq->resp_qlock); list_add_tail(&sr->response, &cmdq->response_head); - spin_unlock_bh(&cmdq->response_lock); + spin_unlock_bh(&cmdq->resp_qlock); } static inline void response_list_del(struct nitrox_softreq *sr, struct nitrox_cmdq *cmdq) { - spin_lock_bh(&cmdq->response_lock); + spin_lock_bh(&cmdq->resp_qlock); list_del(&sr->response); - spin_unlock_bh(&cmdq->response_lock); + spin_unlock_bh(&cmdq->resp_qlock); } static struct nitrox_softreq * @@ -439,11 +439,11 @@ static void post_se_instr(struct nitrox_softreq *sr, int idx; u8 *ent; - spin_lock_bh(&cmdq->cmdq_lock); + spin_lock_bh(&cmdq->cmd_qlock); idx = cmdq->write_idx; /* copy the instruction */ - ent = cmdq->head + (idx * cmdq->instr_size); + ent = cmdq->base + (idx * cmdq->instr_size); memcpy(ent, &sr->instr, cmdq->instr_size); atomic_set(&sr->status, REQ_POSTED); @@ -459,7 +459,10 @@ static void post_se_instr(struct nitrox_softreq *sr, cmdq->write_idx = incr_index(idx, 1, ndev->qlen); - spin_unlock_bh(&cmdq->cmdq_lock); + spin_unlock_bh(&cmdq->cmd_qlock); + + /* increment the posted command count */ + atomic64_inc(&ndev->stats.posted); } static int post_backlog_cmds(struct nitrox_cmdq *cmdq) @@ -471,7 +474,7 @@ static int post_backlog_cmds(struct nitrox_cmdq *cmdq) if (!atomic_read(&cmdq->backlog_count)) return 0; - spin_lock_bh(&cmdq->backlog_lock); + spin_lock_bh(&cmdq->backlog_qlock); list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { struct skcipher_request *skreq; @@ -494,7 +497,7 @@ static int post_backlog_cmds(struct nitrox_cmdq *cmdq) /* backlog requests are posted, wakeup with -EINPROGRESS */ skcipher_request_complete(skreq, -EINPROGRESS); } - spin_unlock_bh(&cmdq->backlog_lock); + spin_unlock_bh(&cmdq->backlog_qlock); return ret; } @@ -508,8 +511,11 @@ static int nitrox_enqueue_request(struct nitrox_softreq *sr) post_backlog_cmds(cmdq); if (unlikely(cmdq_full(cmdq, ndev->qlen))) { - if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + /* increment drop count */ + atomic64_inc(&ndev->stats.dropped); return -ENOSPC; + } /* add to backlog list */ backlog_list_add(sr, cmdq); return -EBUSY; @@ -572,7 +578,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, /* select the queue */ qno = smp_processor_id() % ndev->nr_queues; - sr->cmdq = &ndev->pkt_cmdqs[qno]; + sr->cmdq = &ndev->pkt_inq[qno]; /* * 64-Byte Instruction Format @@ -694,6 +700,7 @@ static void process_response_list(struct nitrox_cmdq *cmdq) READ_ONCE(sr->resp.orh)); } atomic_dec(&cmdq->pending_count); + atomic64_inc(&ndev->stats.completed); /* sync with other cpus */ smp_mb__after_atomic(); /* remove from response list */ @@ -714,18 +721,18 @@ static void process_response_list(struct nitrox_cmdq *cmdq) } /** - * pkt_slc_resp_handler - post processing of SE responses + * pkt_slc_resp_tasklet - post processing of SE responses */ -void pkt_slc_resp_handler(unsigned long data) +void pkt_slc_resp_tasklet(unsigned long data) { - struct bh_data *bh = (void *)(uintptr_t)(data); - struct nitrox_cmdq *cmdq = bh->cmdq; - union nps_pkt_slc_cnts pkt_slc_cnts; + struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); + struct nitrox_cmdq *cmdq = qvec->cmdq; + union nps_pkt_slc_cnts slc_cnts; /* read completion count */ - pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr); + slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); /* resend the interrupt if more work to do */ - pkt_slc_cnts.s.resend = 1; + slc_cnts.s.resend = 1; process_response_list(cmdq); @@ -733,7 +740,7 @@ void pkt_slc_resp_handler(unsigned long data) * clear the interrupt with resend bit enabled, * MSI-X interrupt generates if Completion count > Threshold */ - writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr); + writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr); /* order the writes */ mmiowb(); diff --git a/drivers/crypto/cavium/nitrox/nitrox_sriov.c b/drivers/crypto/cavium/nitrox/nitrox_sriov.c new file mode 100644 index 000000000000..30c0aa874583 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_sriov.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/pci.h> +#include <linux/delay.h> + +#include "nitrox_dev.h" +#include "nitrox_hal.h" +#include "nitrox_common.h" +#include "nitrox_isr.h" + +static inline bool num_vfs_valid(int num_vfs) +{ + bool valid = false; + + switch (num_vfs) { + case 16: + case 32: + case 64: + case 128: + valid = true; + break; + } + + return valid; +} + +static inline enum vf_mode num_vfs_to_mode(int num_vfs) +{ + enum vf_mode mode = 0; + + switch (num_vfs) { + case 0: + mode = __NDEV_MODE_PF; + break; + case 16: + mode = __NDEV_MODE_VF16; + break; + case 32: + mode = __NDEV_MODE_VF32; + break; + case 64: + mode = __NDEV_MODE_VF64; + break; + case 128: + mode = __NDEV_MODE_VF128; + break; + } + + return mode; +} + +static void pf_sriov_cleanup(struct nitrox_device *ndev) +{ + /* PF has no queues in SR-IOV mode */ + atomic_set(&ndev->state, __NDEV_NOT_READY); + /* unregister crypto algorithms */ + nitrox_crypto_unregister(); + + /* cleanup PF resources */ + nitrox_unregister_interrupts(ndev); + nitrox_common_sw_cleanup(ndev); +} + +static int pf_sriov_init(struct nitrox_device *ndev) +{ + int err; + + /* allocate resources for PF */ + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_register_interrupts(ndev); + if (err) { + nitrox_common_sw_cleanup(ndev); + return err; + } + + /* configure the packet queues */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* set device to ready state */ + atomic_set(&ndev->state, __NDEV_READY); + + /* register crypto algorithms */ + return nitrox_crypto_register(); +} + +static int nitrox_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + int err; + + if (!num_vfs_valid(num_vfs)) { + dev_err(DEV(ndev), "Invalid num_vfs %d\n", num_vfs); + return -EINVAL; + } + + if (pci_num_vf(pdev) == num_vfs) + return num_vfs; + + err = pci_enable_sriov(pdev, num_vfs); + if (err) { + dev_err(DEV(ndev), "failed to enable PCI sriov %d\n", err); + return err; + } + dev_info(DEV(ndev), "Enabled VF(s) %d\n", num_vfs); + + ndev->num_vfs = num_vfs; + ndev->mode = num_vfs_to_mode(num_vfs); + /* set bit in flags */ + set_bit(__NDEV_SRIOV_BIT, &ndev->flags); + + /* cleanup PF resources */ + pf_sriov_cleanup(ndev); + + config_nps_core_vfcfg_mode(ndev, ndev->mode); + + return num_vfs; +} + +static int nitrox_sriov_disable(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!test_bit(__NDEV_SRIOV_BIT, &ndev->flags)) + return 0; + + if (pci_vfs_assigned(pdev)) { + dev_warn(DEV(ndev), "VFs are attached to VM. Can't disable SR-IOV\n"); + return -EPERM; + } + pci_disable_sriov(pdev); + /* clear bit in flags */ + clear_bit(__NDEV_SRIOV_BIT, &ndev->flags); + + ndev->num_vfs = 0; + ndev->mode = __NDEV_MODE_PF; + + config_nps_core_vfcfg_mode(ndev, ndev->mode); + + return pf_sriov_init(ndev); +} + +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (!num_vfs) + return nitrox_sriov_disable(pdev); + + return nitrox_sriov_enable(pdev, num_vfs); +} |