diff options
Diffstat (limited to 'drivers/net/ethernet/marvell')
61 files changed, 9772 insertions, 300 deletions
diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index b6be0552a6c1..8941f69d93e9 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -1603,12 +1603,12 @@ mv643xx_eth_set_link_ksettings(struct net_device *dev, static void mv643xx_eth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, mv643xx_eth_driver_name, + strscpy(drvinfo->driver, mv643xx_eth_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, mv643xx_eth_driver_version, + strscpy(drvinfo->version, mv643xx_eth_driver_version, sizeof(drvinfo->version)); - strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); + strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); } static int mv643xx_eth_get_coalesce(struct net_device *dev, @@ -2481,6 +2481,7 @@ out_free: for (i = 0; i < mp->rxq_count; i++) rxq_deinit(mp->rxq + i); out: + napi_disable(&mp->napi); free_irq(dev->irq, dev); return err; @@ -3183,7 +3184,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) INIT_WORK(&mp->tx_timeout_task, tx_timeout_task); - netif_napi_add(dev, &mp->napi, mv643xx_eth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &mp->napi, mv643xx_eth_poll); timer_setup(&mp->rx_oom, oom_timer_wrapper, 0); diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 0caa2df87c04..ff3e361e06e7 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -4656,11 +4656,11 @@ mvneta_ethtool_get_coalesce(struct net_device *dev, static void mvneta_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME, + strscpy(drvinfo->driver, MVNETA_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION, + strscpy(drvinfo->version, MVNETA_DRIVER_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(&dev->dev), + strscpy(drvinfo->bus_info, dev_name(&dev->dev), sizeof(drvinfo->bus_info)); } @@ -5600,14 +5600,13 @@ static int mvneta_probe(struct platform_device *pdev) * operation, so only single NAPI should be initialized. */ if (pp->neta_armada3700) { - netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &pp->napi, mvneta_poll); } else { for_each_present_cpu(cpu) { struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); - netif_napi_add(dev, &port->napi, mvneta_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &port->napi, mvneta_poll); port->pp = pp; } } diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h index ad73a488fc5f..11e603686a27 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h @@ -1530,6 +1530,7 @@ u32 mvpp2_read(struct mvpp2 *priv, u32 offset); void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name); void mvpp2_dbgfs_cleanup(struct mvpp2 *priv); +void mvpp2_dbgfs_exit(void); void mvpp23_rx_fifo_fc_en(struct mvpp2 *priv, int port, bool en); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c index 0eec05d905eb..75e83ea2a926 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c @@ -691,6 +691,13 @@ static int mvpp2_dbgfs_port_init(struct dentry *parent, return 0; } +static struct dentry *mvpp2_root; + +void mvpp2_dbgfs_exit(void) +{ + debugfs_remove(mvpp2_root); +} + void mvpp2_dbgfs_cleanup(struct mvpp2 *priv) { debugfs_remove_recursive(priv->dbgfs_dir); @@ -700,7 +707,6 @@ void mvpp2_dbgfs_cleanup(struct mvpp2 *priv) void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name) { - static struct dentry *mvpp2_root; struct dentry *mvpp2_dir; int ret, i; diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index b84128b549b4..eb0fb8128096 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5425,11 +5425,11 @@ mvpp2_ethtool_get_coalesce(struct net_device *dev, static void mvpp2_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME, + strscpy(drvinfo->driver, MVPP2_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION, + strscpy(drvinfo->version, MVPP2_DRIVER_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(&dev->dev), + strscpy(drvinfo->bus_info, dev_name(&dev->dev), sizeof(drvinfo->bus_info)); } @@ -5770,8 +5770,7 @@ static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port, v->irq = irq_of_parse_and_map(port_node, 0); if (v->irq <= 0) return -EINVAL; - netif_napi_add(port->dev, &v->napi, mvpp2_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(port->dev, &v->napi, mvpp2_poll); port->nqvecs = 1; @@ -5831,8 +5830,7 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port, goto err; } - netif_napi_add(port->dev, &v->napi, mvpp2_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(port->dev, &v->napi, mvpp2_poll); } return 0; @@ -7706,7 +7704,18 @@ static struct platform_driver mvpp2_driver = { }, }; -module_platform_driver(mvpp2_driver); +static int __init mvpp2_driver_init(void) +{ + return platform_driver_register(&mvpp2_driver); +} +module_init(mvpp2_driver_init); + +static void __exit mvpp2_driver_exit(void) +{ + platform_driver_unregister(&mvpp2_driver); + mvpp2_dbgfs_exit(); +} +module_exit(mvpp2_driver_exit); MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com"); MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>"); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c index 97f080c66dd4..9089adcb75f9 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -410,7 +410,7 @@ static void octep_napi_add(struct octep_device *oct) for (i = 0; i < oct->num_oqs; i++) { netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i); netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, - octep_napi_poll, 64); + octep_napi_poll); oct->oq[i]->napi = &oct->ioq_vector[i]->napi; } } diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index d9ae0937d17a..392d9b0da0d7 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -158,8 +158,7 @@ static int octep_setup_oq(struct octep_device *oct, int q_no) goto desc_dma_alloc_err; } - oq->buff_info = (struct octep_rx_buffer *) - vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE); + oq->buff_info = vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE); if (unlikely(!oq->buff_info)) { dev_err(&oct->pdev->dev, "Failed to allocate buffer info for OQ-%d\n", q_no); diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig index e1036b0eb6b1..6b4f640163f7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/Kconfig +++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig @@ -32,10 +32,12 @@ config OCTEONTX2_PF tristate "Marvell OcteonTX2 NIC Physical Function driver" select OCTEONTX2_MBOX select NET_DEVLINK + depends on MACSEC || !MACSEC depends on (64BIT && COMPILE_TEST) || ARM64 select DIMLIB depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + depends on MACSEC || !MACSEC help This driver supports Marvell's OcteonTX2 NIC physical function. diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile index 40203560b291..3cf4c8285c90 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile @@ -11,4 +11,4 @@ rvu_mbox-y := mbox.o rvu_trace.o rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \ rvu_reg.o rvu_npc.o rvu_debugfs.o ptp.o rvu_npc_fs.o \ rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \ - rvu_sdp.o rvu_npc_hash.o + rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index d7762577e285..8d5d5a0f68c4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -293,20 +293,74 @@ M(NIX_BANDPROF_ALLOC, 0x801d, nix_bandprof_alloc, nix_bandprof_alloc_req, \ M(NIX_BANDPROF_FREE, 0x801e, nix_bandprof_free, nix_bandprof_free_req, \ msg_rsp) \ M(NIX_BANDPROF_GET_HWINFO, 0x801f, nix_bandprof_get_hwinfo, msg_req, \ - nix_bandprof_get_hwinfo_rsp) - -/* Messages initiated by AF (range 0xC00 - 0xDFF) */ + nix_bandprof_get_hwinfo_rsp) \ +/* MCS mbox IDs (range 0xA000 - 0xBFFF) */ \ +M(MCS_ALLOC_RESOURCES, 0xa000, mcs_alloc_resources, mcs_alloc_rsrc_req, \ + mcs_alloc_rsrc_rsp) \ +M(MCS_FREE_RESOURCES, 0xa001, mcs_free_resources, mcs_free_rsrc_req, msg_rsp) \ +M(MCS_FLOWID_ENTRY_WRITE, 0xa002, mcs_flowid_entry_write, mcs_flowid_entry_write_req, \ + msg_rsp) \ +M(MCS_SECY_PLCY_WRITE, 0xa003, mcs_secy_plcy_write, mcs_secy_plcy_write_req, \ + msg_rsp) \ +M(MCS_RX_SC_CAM_WRITE, 0xa004, mcs_rx_sc_cam_write, mcs_rx_sc_cam_write_req, \ + msg_rsp) \ +M(MCS_SA_PLCY_WRITE, 0xa005, mcs_sa_plcy_write, mcs_sa_plcy_write_req, \ + msg_rsp) \ +M(MCS_TX_SC_SA_MAP_WRITE, 0xa006, mcs_tx_sc_sa_map_write, mcs_tx_sc_sa_map, \ + msg_rsp) \ +M(MCS_RX_SC_SA_MAP_WRITE, 0xa007, mcs_rx_sc_sa_map_write, mcs_rx_sc_sa_map, \ + msg_rsp) \ +M(MCS_FLOWID_ENA_ENTRY, 0xa008, mcs_flowid_ena_entry, mcs_flowid_ena_dis_entry, \ + msg_rsp) \ +M(MCS_PN_TABLE_WRITE, 0xa009, mcs_pn_table_write, mcs_pn_table_write_req, \ + msg_rsp) \ +M(MCS_SET_ACTIVE_LMAC, 0xa00a, mcs_set_active_lmac, mcs_set_active_lmac, \ + msg_rsp) \ +M(MCS_GET_HW_INFO, 0xa00b, mcs_get_hw_info, msg_req, mcs_hw_info) \ +M(MCS_GET_FLOWID_STATS, 0xa00c, mcs_get_flowid_stats, mcs_stats_req, \ + mcs_flowid_stats) \ +M(MCS_GET_SECY_STATS, 0xa00d, mcs_get_secy_stats, mcs_stats_req, \ + mcs_secy_stats) \ +M(MCS_GET_SC_STATS, 0xa00e, mcs_get_sc_stats, mcs_stats_req, mcs_sc_stats) \ +M(MCS_GET_SA_STATS, 0xa00f, mcs_get_sa_stats, mcs_stats_req, mcs_sa_stats) \ +M(MCS_GET_PORT_STATS, 0xa010, mcs_get_port_stats, mcs_stats_req, \ + mcs_port_stats) \ +M(MCS_CLEAR_STATS, 0xa011, mcs_clear_stats, mcs_clear_stats, msg_rsp) \ +M(MCS_INTR_CFG, 0xa012, mcs_intr_cfg, mcs_intr_cfg, msg_rsp) \ +M(MCS_SET_LMAC_MODE, 0xa013, mcs_set_lmac_mode, mcs_set_lmac_mode, msg_rsp) \ +M(MCS_SET_PN_THRESHOLD, 0xa014, mcs_set_pn_threshold, mcs_set_pn_threshold, \ + msg_rsp) \ +M(MCS_ALLOC_CTRL_PKT_RULE, 0xa015, mcs_alloc_ctrl_pkt_rule, \ + mcs_alloc_ctrl_pkt_rule_req, \ + mcs_alloc_ctrl_pkt_rule_rsp) \ +M(MCS_FREE_CTRL_PKT_RULE, 0xa016, mcs_free_ctrl_pkt_rule, \ + mcs_free_ctrl_pkt_rule_req, msg_rsp) \ +M(MCS_CTRL_PKT_RULE_WRITE, 0xa017, mcs_ctrl_pkt_rule_write, \ + mcs_ctrl_pkt_rule_write_req, msg_rsp) \ +M(MCS_PORT_RESET, 0xa018, mcs_port_reset, mcs_port_reset_req, msg_rsp) \ +M(MCS_PORT_CFG_SET, 0xa019, mcs_port_cfg_set, mcs_port_cfg_set_req, msg_rsp)\ +M(MCS_PORT_CFG_GET, 0xa020, mcs_port_cfg_get, mcs_port_cfg_get_req, \ + mcs_port_cfg_get_rsp) \ +M(MCS_CUSTOM_TAG_CFG_GET, 0xa021, mcs_custom_tag_cfg_get, \ + mcs_custom_tag_cfg_get_req, \ + mcs_custom_tag_cfg_get_rsp) + +/* Messages initiated by AF (range 0xC00 - 0xEFF) */ #define MBOX_UP_CGX_MESSAGES \ M(CGX_LINK_EVENT, 0xC00, cgx_link_event, cgx_link_info_msg, msg_rsp) #define MBOX_UP_CPT_MESSAGES \ M(CPT_INST_LMTST, 0xD00, cpt_inst_lmtst, cpt_inst_lmtst_req, msg_rsp) +#define MBOX_UP_MCS_MESSAGES \ +M(MCS_INTR_NOTIFY, 0xE00, mcs_intr_notify, mcs_intr_info, msg_rsp) + enum { #define M(_name, _id, _1, _2, _3) MBOX_MSG_ ## _name = _id, MBOX_MESSAGES MBOX_UP_CGX_MESSAGES MBOX_UP_CPT_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M }; @@ -1471,6 +1525,7 @@ enum ptp_op { PTP_OP_GET_CLOCK = 1, PTP_OP_GET_TSTMP = 2, PTP_OP_SET_THRESH = 3, + PTP_OP_EXTTS_ON = 4, }; struct ptp_req { @@ -1478,6 +1533,7 @@ struct ptp_req { u8 op; s64 scaled_ppm; u64 thresh; + int extts_on; }; struct ptp_rsp { @@ -1655,4 +1711,415 @@ enum cgx_af_status { LMAC_AF_ERR_EXACT_MATCH_TBL_LOOK_UP_FAILED = -1110, }; +enum mcs_direction { + MCS_RX, + MCS_TX, +}; + +enum mcs_rsrc_type { + MCS_RSRC_TYPE_FLOWID, + MCS_RSRC_TYPE_SECY, + MCS_RSRC_TYPE_SC, + MCS_RSRC_TYPE_SA, +}; + +struct mcs_alloc_rsrc_req { + struct mbox_msghdr hdr; + u8 rsrc_type; + u8 rsrc_cnt; /* Resources count */ + u8 mcs_id; /* MCS block ID */ + u8 dir; /* Macsec ingress or egress side */ + u8 all; /* Allocate all resource type one each */ + u64 rsvd; +}; + +struct mcs_alloc_rsrc_rsp { + struct mbox_msghdr hdr; + u8 flow_ids[128]; /* Index of reserved entries */ + u8 secy_ids[128]; + u8 sc_ids[128]; + u8 sa_ids[256]; + u8 rsrc_type; + u8 rsrc_cnt; /* No of entries reserved */ + u8 mcs_id; + u8 dir; + u8 all; + u8 rsvd[256]; /* reserved fields for future expansion */ +}; + +struct mcs_free_rsrc_req { + struct mbox_msghdr hdr; + u8 rsrc_id; /* Index of the entry to be freed */ + u8 rsrc_type; + u8 mcs_id; + u8 dir; + u8 all; /* Free all the cam resources */ + u64 rsvd; +}; + +struct mcs_flowid_entry_write_req { + struct mbox_msghdr hdr; + u64 data[4]; + u64 mask[4]; + u64 sci; /* CNF10K-B for tx_secy_mem_map */ + u8 flow_id; + u8 secy_id; /* secyid for which flowid is mapped */ + u8 sc_id; /* Valid if dir = MCS_TX, SC_CAM id mapped to flowid */ + u8 ena; /* Enable tcam entry */ + u8 ctrl_pkt; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_secy_plcy_write_req { + struct mbox_msghdr hdr; + u64 plcy; + u8 secy_id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +/* RX SC_CAM mapping */ +struct mcs_rx_sc_cam_write_req { + struct mbox_msghdr hdr; + u64 sci; /* SCI */ + u64 secy_id; /* secy index mapped to SC */ + u8 sc_id; /* SC CAM entry index */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_sa_plcy_write_req { + struct mbox_msghdr hdr; + u64 plcy[2][9]; /* Support 2 SA policy */ + u8 sa_index[2]; + u8 sa_cnt; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_tx_sc_sa_map { + struct mbox_msghdr hdr; + u8 sa_index0; + u8 sa_index1; + u8 rekey_ena; + u8 sa_index0_vld; + u8 sa_index1_vld; + u8 tx_sa_active; + u64 sectag_sci; + u8 sc_id; /* used as index for SA_MEM_MAP */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_rx_sc_sa_map { + struct mbox_msghdr hdr; + u8 sa_index; + u8 sa_in_use; + u8 sc_id; + u8 an; /* value range 0-3, sc_id + an used as index SA_MEM_MAP */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_flowid_ena_dis_entry { + struct mbox_msghdr hdr; + u8 flow_id; + u8 ena; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_pn_table_write_req { + struct mbox_msghdr hdr; + u64 next_pn; + u8 pn_id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_hw_info { + struct mbox_msghdr hdr; + u8 num_mcs_blks; /* Number of MCS blocks */ + u8 tcam_entries; /* RX/TX Tcam entries per mcs block */ + u8 secy_entries; /* RX/TX SECY entries per mcs block */ + u8 sc_entries; /* RX/TX SC CAM entries per mcs block */ + u8 sa_entries; /* PN table entries = SA entries */ + u64 rsvd[16]; +}; + +struct mcs_set_active_lmac { + struct mbox_msghdr hdr; + u32 lmac_bmap; /* bitmap of active lmac per mcs block */ + u8 mcs_id; + u16 chan_base; /* MCS channel base */ + u64 rsvd; +}; + +struct mcs_set_lmac_mode { + struct mbox_msghdr hdr; + u8 mode; /* 1:Bypass 0:Operational */ + u8 lmac_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_reset_req { + struct mbox_msghdr hdr; + u8 reset; + u8 mcs_id; + u8 port_id; + u64 rsvd; +}; + +struct mcs_port_cfg_set_req { + struct mbox_msghdr hdr; + u8 cstm_tag_rel_mode_sel; + u8 custom_hdr_enb; + u8 fifo_skid; + u8 port_mode; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_cfg_get_req { + struct mbox_msghdr hdr; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_cfg_get_rsp { + struct mbox_msghdr hdr; + u8 cstm_tag_rel_mode_sel; + u8 custom_hdr_enb; + u8 fifo_skid; + u8 port_mode; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_custom_tag_cfg_get_req { + struct mbox_msghdr hdr; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_custom_tag_cfg_get_rsp { + struct mbox_msghdr hdr; + u16 cstm_etype[8]; + u8 cstm_indx[8]; + u8 cstm_etype_en; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +/* MCS mailbox error codes + * Range 1201 - 1300. + */ +enum mcs_af_status { + MCS_AF_ERR_INVALID_MCSID = -1201, + MCS_AF_ERR_NOT_MAPPED = -1202, +}; + +struct mcs_set_pn_threshold { + struct mbox_msghdr hdr; + u64 threshold; + u8 xpn; /* '1' for setting xpn threshold */ + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +enum mcs_ctrl_pkt_rulew_type { + MCS_CTRL_PKT_RULE_TYPE_ETH, + MCS_CTRL_PKT_RULE_TYPE_DA, + MCS_CTRL_PKT_RULE_TYPE_RANGE, + MCS_CTRL_PKT_RULE_TYPE_COMBO, + MCS_CTRL_PKT_RULE_TYPE_MAC, +}; + +struct mcs_alloc_ctrl_pkt_rule_req { + struct mbox_msghdr hdr; + u8 rule_type; + u8 mcs_id; /* MCS block ID */ + u8 dir; /* Macsec ingress or egress side */ + u64 rsvd; +}; + +struct mcs_alloc_ctrl_pkt_rule_rsp { + struct mbox_msghdr hdr; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_free_ctrl_pkt_rule_req { + struct mbox_msghdr hdr; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u8 all; + u64 rsvd; +}; + +struct mcs_ctrl_pkt_rule_write_req { + struct mbox_msghdr hdr; + u64 data0; + u64 data1; + u64 data2; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_stats_req { + struct mbox_msghdr hdr; + u8 id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_flowid_stats { + struct mbox_msghdr hdr; + u64 tcam_hit_cnt; + u64 rsvd; +}; + +struct mcs_secy_stats { + struct mbox_msghdr hdr; + u64 ctl_pkt_bcast_cnt; + u64 ctl_pkt_mcast_cnt; + u64 ctl_pkt_ucast_cnt; + u64 ctl_octet_cnt; + u64 unctl_pkt_bcast_cnt; + u64 unctl_pkt_mcast_cnt; + u64 unctl_pkt_ucast_cnt; + u64 unctl_octet_cnt; + /* Valid only for RX */ + u64 octet_decrypted_cnt; + u64 octet_validated_cnt; + u64 pkt_port_disabled_cnt; + u64 pkt_badtag_cnt; + u64 pkt_nosa_cnt; + u64 pkt_nosaerror_cnt; + u64 pkt_tagged_ctl_cnt; + u64 pkt_untaged_cnt; + u64 pkt_ctl_cnt; /* CN10K-B */ + u64 pkt_notag_cnt; /* CNF10K-B */ + /* Valid only for TX */ + u64 octet_encrypted_cnt; + u64 octet_protected_cnt; + u64 pkt_noactivesa_cnt; + u64 pkt_toolong_cnt; + u64 pkt_untagged_cnt; + u64 rsvd[4]; +}; + +struct mcs_port_stats { + struct mbox_msghdr hdr; + u64 tcam_miss_cnt; + u64 parser_err_cnt; + u64 preempt_err_cnt; /* CNF10K-B */ + u64 sectag_insert_err_cnt; + u64 rsvd[4]; +}; + +/* Only for CN10K-B */ +struct mcs_sa_stats { + struct mbox_msghdr hdr; + /* RX */ + u64 pkt_invalid_cnt; + u64 pkt_nosaerror_cnt; + u64 pkt_notvalid_cnt; + u64 pkt_ok_cnt; + u64 pkt_nosa_cnt; + /* TX */ + u64 pkt_encrypt_cnt; + u64 pkt_protected_cnt; + u64 rsvd[4]; +}; + +struct mcs_sc_stats { + struct mbox_msghdr hdr; + /* RX */ + u64 hit_cnt; + u64 pkt_invalid_cnt; + u64 pkt_late_cnt; + u64 pkt_notvalid_cnt; + u64 pkt_unchecked_cnt; + u64 pkt_delay_cnt; /* CNF10K-B */ + u64 pkt_ok_cnt; /* CNF10K-B */ + u64 octet_decrypt_cnt; /* CN10K-B */ + u64 octet_validate_cnt; /* CN10K-B */ + /* TX */ + u64 pkt_encrypt_cnt; + u64 pkt_protected_cnt; + u64 octet_encrypt_cnt; /* CN10K-B */ + u64 octet_protected_cnt; /* CN10K-B */ + u64 rsvd[4]; +}; + +struct mcs_clear_stats { + struct mbox_msghdr hdr; +#define MCS_FLOWID_STATS 0 +#define MCS_SECY_STATS 1 +#define MCS_SC_STATS 2 +#define MCS_SA_STATS 3 +#define MCS_PORT_STATS 4 + u8 type; /* FLOWID, SECY, SC, SA, PORT */ + u8 id; /* type = PORT, If id = FF(invalid) port no is derived from pcifunc */ + u8 mcs_id; + u8 dir; + u8 all; /* All resources stats mapped to PF are cleared */ +}; + +struct mcs_intr_cfg { + struct mbox_msghdr hdr; +#define MCS_CPM_RX_SECTAG_V_EQ1_INT BIT_ULL(0) +#define MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT BIT_ULL(1) +#define MCS_CPM_RX_SECTAG_SL_GTE48_INT BIT_ULL(2) +#define MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT BIT_ULL(3) +#define MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT BIT_ULL(4) +#define MCS_CPM_RX_PACKET_XPN_EQ0_INT BIT_ULL(5) +#define MCS_CPM_RX_PN_THRESH_REACHED_INT BIT_ULL(6) +#define MCS_CPM_TX_PACKET_XPN_EQ0_INT BIT_ULL(7) +#define MCS_CPM_TX_PN_THRESH_REACHED_INT BIT_ULL(8) +#define MCS_CPM_TX_SA_NOT_VALID_INT BIT_ULL(9) +#define MCS_BBE_RX_DFIFO_OVERFLOW_INT BIT_ULL(10) +#define MCS_BBE_RX_PLFIFO_OVERFLOW_INT BIT_ULL(11) +#define MCS_BBE_TX_DFIFO_OVERFLOW_INT BIT_ULL(12) +#define MCS_BBE_TX_PLFIFO_OVERFLOW_INT BIT_ULL(13) +#define MCS_PAB_RX_CHAN_OVERFLOW_INT BIT_ULL(14) +#define MCS_PAB_TX_CHAN_OVERFLOW_INT BIT_ULL(15) + u64 intr_mask; /* Interrupt enable mask */ + u8 mcs_id; + u8 lmac_id; + u64 rsvd; +}; + +struct mcs_intr_info { + struct mbox_msghdr hdr; + u64 intr_mask; + int sa_id; + u8 mcs_id; + u8 lmac_id; + u64 rsvd; +}; + #endif /* MBOX_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c new file mode 100644 index 000000000000..4a343f853b28 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -0,0 +1,1603 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "mcs.h" +#include "mcs_reg.h" + +#define DRV_NAME "Marvell MCS Driver" + +#define PCI_CFG_REG_BAR_NUM 0 + +static const struct pci_device_id mcs_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_MCS) }, + { 0, } /* end of table */ +}; + +static LIST_HEAD(mcs_list); + +void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id) +{ + u64 reg; + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(id); + stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(id); + stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(id); + stats->ctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(id); + stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(id); + stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(id); + stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(id); + stats->unctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(id); + stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(id); + stats->octet_encrypted_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(id); + stats->octet_protected_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(id); + stats->pkt_noactivesa_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(id); + stats->pkt_toolong_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(id); + stats->pkt_untagged_cnt = mcs_reg_read(mcs, reg); +} + +void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id) +{ + u64 reg; + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(id); + stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(id); + stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(id); + stats->ctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(id); + stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(id); + stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(id); + stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(id); + stats->unctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(id); + stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(id); + stats->octet_decrypted_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(id); + stats->octet_validated_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(id); + stats->pkt_port_disabled_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(id); + stats->pkt_badtag_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(id); + stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(id); + stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(id); + stats->pkt_tagged_ctl_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(id); + stats->pkt_untaged_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(id); + stats->pkt_ctl_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(id); + stats->pkt_notag_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(id); + else + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(id); + + stats->tcam_hit_cnt = mcs_reg_read(mcs, reg); +} + +void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(id); + stats->tcam_miss_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(id); + stats->parser_err_cnt = mcs_reg_read(mcs, reg); + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(id); + stats->preempt_err_cnt = mcs_reg_read(mcs, reg); + } + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(id); + stats->tcam_miss_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(id); + stats->parser_err_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(id); + stats->sectag_insert_err_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(id); + stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(id); + stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(id); + stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(id); + stats->pkt_ok_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(id); + stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg); + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(id); + stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(id); + stats->pkt_protected_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(id); + stats->hit_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(id); + stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(id); + stats->pkt_late_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(id); + stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(id); + stats->pkt_unchecked_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(id); + stats->pkt_delay_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(id); + stats->pkt_ok_cnt = mcs_reg_read(mcs, reg); + } + if (mcs->hw->mcs_blks == 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(id); + stats->octet_decrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(id); + stats->octet_validate_cnt = mcs_reg_read(mcs, reg); + } + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(id); + stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(id); + stats->pkt_protected_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks == 1) { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(id); + stats->octet_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(id); + stats->octet_protected_cnt = mcs_reg_read(mcs, reg); + } + } +} + +void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir) +{ + struct mcs_flowid_stats flowid_st; + struct mcs_port_stats port_st; + struct mcs_secy_stats secy_st; + struct mcs_sc_stats sc_st; + struct mcs_sa_stats sa_st; + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CSE_RX_SLAVE_CTRL; + else + reg = MCSX_CSE_TX_SLAVE_CTRL; + + mcs_reg_write(mcs, reg, BIT_ULL(0)); + + switch (type) { + case MCS_FLOWID_STATS: + mcs_get_flowid_stats(mcs, &flowid_st, id, dir); + break; + case MCS_SECY_STATS: + if (dir == MCS_RX) + mcs_get_rx_secy_stats(mcs, &secy_st, id); + else + mcs_get_tx_secy_stats(mcs, &secy_st, id); + break; + case MCS_SC_STATS: + mcs_get_sc_stats(mcs, &sc_st, id, dir); + break; + case MCS_SA_STATS: + mcs_get_sa_stats(mcs, &sa_st, id, dir); + break; + case MCS_PORT_STATS: + mcs_get_port_stats(mcs, &port_st, id, dir); + break; + } + + mcs_reg_write(mcs, reg, 0x0); +} + +int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* Clear FLOWID stats */ + for (id = 0; id < map->flow_ids.max; id++) { + if (map->flowid2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_FLOWID_STATS, id, dir); + } + + /* Clear SECY stats */ + for (id = 0; id < map->secy.max; id++) { + if (map->secy2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SECY_STATS, id, dir); + } + + /* Clear SC stats */ + for (id = 0; id < map->secy.max; id++) { + if (map->sc2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SC_STATS, id, dir); + } + + /* Clear SA stats */ + for (id = 0; id < map->sa.max; id++) { + if (map->sa2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SA_STATS, id, dir); + } + return 0; +} + +void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(pn_id); + else + reg = MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(pn_id); + mcs_reg_write(mcs, reg, next_pn); +} + +void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map) +{ + u64 reg, val; + + val = (map->sa_index0 & 0xFF) | + (map->sa_index1 & 0xFF) << 9 | + (map->rekey_ena & 0x1) << 18 | + (map->sa_index0_vld & 0x1) << 19 | + (map->sa_index1_vld & 0x1) << 20 | + (map->tx_sa_active & 0x1) << 21 | + map->sectag_sci << 22; + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id); + mcs_reg_write(mcs, reg, val); + + val = map->sectag_sci >> 42; + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(map->sc_id); + mcs_reg_write(mcs, reg, val); +} + +void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map) +{ + u64 val, reg; + + val = (map->sa_index & 0xFF) | map->sa_in_use << 9; + + reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an); + mcs_reg_write(mcs, reg, val); +} + +void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa_id, int dir) +{ + int reg_id; + u64 reg; + + if (dir == MCS_RX) { + for (reg_id = 0; reg_id < 8; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id); + mcs_reg_write(mcs, reg, plcy[reg_id]); + } + } else { + for (reg_id = 0; reg_id < 9; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id); + mcs_reg_write(mcs, reg, plcy[reg_id]); + } + } +} + +void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int sc_id, int ena) +{ + u64 reg, val; + + reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(0); + if (sc_id > 63) + reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(1); + + if (ena) + val = mcs_reg_read(mcs, reg) | BIT_ULL(sc_id); + else + val = mcs_reg_read(mcs, reg) & ~BIT_ULL(sc_id); + + mcs_reg_write(mcs, reg, val); +} + +void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id) +{ + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(0, sc_id), sci); + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(1, sc_id), secy); + /* Enable SC CAM */ + mcs_ena_dis_sc_cam_entry(mcs, sc_id, true); +} + +void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int secy_id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(secy_id); + else + reg = MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(secy_id); + + mcs_reg_write(mcs, reg, plcy); + + if (mcs->hw->mcs_blks == 1 && dir == MCS_RX) + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(secy_id), 0x0ull); +} + +void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir) +{ + u64 reg, val; + + val = (map->secy & 0x7F) | (map->ctrl_pkt & 0x1) << 8; + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id); + } else { + val |= (map->sc & 0x7F) << 9; + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id); + } + + mcs_reg_write(mcs, reg, val); +} + +void mcs_ena_dis_flowid_entry(struct mcs *mcs, int flow_id, int dir, int ena) +{ + u64 reg, val; + + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0; + if (flow_id > 63) + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1; + } else { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0; + if (flow_id > 63) + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1; + } + + /* Enable/Disable the tcam entry */ + if (ena) + val = mcs_reg_read(mcs, reg) | BIT_ULL(flow_id); + else + val = mcs_reg_read(mcs, reg) & ~BIT_ULL(flow_id); + + mcs_reg_write(mcs, reg, val); +} + +void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int flow_id, int dir) +{ + int reg_id; + u64 reg; + + if (dir == MCS_RX) { + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id); + mcs_reg_write(mcs, reg, data[reg_id]); + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, mask[reg_id]); + } + } else { + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id); + mcs_reg_write(mcs, reg, data[reg_id]); + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, mask[reg_id]); + } + } +} + +int mcs_install_flowid_bypass_entry(struct mcs *mcs) +{ + int flow_id, secy_id, reg_id; + struct secy_mem_map map; + u64 reg, plcy = 0; + + /* Flow entry */ + flow_id = mcs->hw->tcam_entries - MCS_RSRC_RSVD_CNT; + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0)); + } + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0)); + } + /* secy */ + secy_id = mcs->hw->secy_entries - MCS_RSRC_RSVD_CNT; + + /* Set validate frames to NULL and enable control port */ + plcy = 0x7ull; + if (mcs->hw->mcs_blks > 1) + plcy = BIT_ULL(0) | 0x3ull << 4; + mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_RX); + + /* Enable control port and set mtu to max */ + plcy = BIT_ULL(0) | GENMASK_ULL(43, 28); + if (mcs->hw->mcs_blks > 1) + plcy = BIT_ULL(0) | GENMASK_ULL(63, 48); + mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_TX); + + /* Map flowid to secy */ + map.secy = secy_id; + map.ctrl_pkt = 0; + map.flow_id = flow_id; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_RX); + map.sc = secy_id; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_TX); + + /* Enable Flowid entry */ + mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_RX, true); + mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_TX, true); + return 0; +} + +void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir) +{ + struct mcs_rsrc_map *map; + int flow_id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* Clear secy memory to zero */ + mcs_secy_plcy_write(mcs, 0, secy_id, dir); + + /* Disable the tcam entry using this secy */ + for (flow_id = 0; flow_id < map->flow_ids.max; flow_id++) { + if (map->flowid2secy_map[flow_id] != secy_id) + continue; + mcs_ena_dis_flowid_entry(mcs, flow_id, dir, false); + } +} + +int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc) +{ + int rsrc_id; + + if (!rsrc->bmap) + return -EINVAL; + + rsrc_id = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, offset, 1, 0); + if (rsrc_id >= rsrc->max) + return -ENOSPC; + + bitmap_set(rsrc->bmap, rsrc_id, 1); + pf_map[rsrc_id] = pcifunc; + + return rsrc_id; +} + +int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + u64 dis, reg; + int id, rc; + + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE; + map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx; + + if (req->all) { + for (id = 0; id < map->ctrlpktrule.max; id++) { + if (map->ctrlpktrule2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, id, pcifunc); + dis = mcs_reg_read(mcs, reg); + dis &= ~BIT_ULL(id); + mcs_reg_write(mcs, reg, dis); + } + return 0; + } + + rc = mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, req->rule_idx, pcifunc); + dis = mcs_reg_read(mcs, reg); + dis &= ~BIT_ULL(req->rule_idx); + mcs_reg_write(mcs, reg, dis); + + return rc; +} + +int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req) +{ + u64 reg, enb; + u64 idx; + + switch (req->rule_type) { + case MCS_CTRL_PKT_RULE_TYPE_ETH: + req->data0 &= GENMASK(15, 0); + if (req->data0 != ETH_P_PAE) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_ETYPE_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(idx) : + MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(idx); + + mcs_reg_write(mcs, reg, req->data0); + break; + case MCS_CTRL_PKT_RULE_TYPE_DA: + if (!(req->data0 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_DA_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_DAX(idx) : + MCSX_PEX_TX_SLAVE_RULE_DAX(idx); + + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + break; + case MCS_CTRL_PKT_RULE_TYPE_RANGE: + if (!(req->data0 & BIT_ULL(40)) || !(req->data1 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_DA_RANGE_RULE_OFFSET; + if (req->dir == MCS_RX) { + reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + } else { + reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + } + break; + case MCS_CTRL_PKT_RULE_TYPE_COMBO: + req->data2 &= GENMASK(15, 0); + if (req->data2 != ETH_P_PAE || !(req->data0 & BIT_ULL(40)) || + !(req->data1 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_COMBO_RULE_OFFSET; + if (req->dir == MCS_RX) { + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(idx); + mcs_reg_write(mcs, reg, req->data2); + } else { + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(idx); + mcs_reg_write(mcs, reg, req->data2); + } + break; + case MCS_CTRL_PKT_RULE_TYPE_MAC: + if (!(req->data0 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_MAC_EN_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_MAC : + MCSX_PEX_TX_SLAVE_RULE_MAC; + + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + break; + } + + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE; + + enb = mcs_reg_read(mcs, reg); + enb |= BIT_ULL(req->rule_idx); + mcs_reg_write(mcs, reg, enb); + + return 0; +} + +int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc) +{ + /* Check if the rsrc_id is mapped to PF/VF */ + if (pf_map[rsrc_id] != pcifunc) + return -EINVAL; + + rvu_free_rsrc(rsrc, rsrc_id); + pf_map[rsrc_id] = 0; + return 0; +} + +/* Free all the cam resources mapped to pf */ +int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* free tcam entries */ + for (id = 0; id < map->flow_ids.max; id++) { + if (map->flowid2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, + id, pcifunc); + mcs_ena_dis_flowid_entry(mcs, id, dir, false); + } + + /* free secy entries */ + for (id = 0; id < map->secy.max; id++) { + if (map->secy2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->secy, map->secy2pf_map, + id, pcifunc); + mcs_clear_secy_plcy(mcs, id, dir); + } + + /* free sc entries */ + for (id = 0; id < map->secy.max; id++) { + if (map->sc2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->sc, map->sc2pf_map, id, pcifunc); + + /* Disable SC CAM only on RX side */ + if (dir == MCS_RX) + mcs_ena_dis_sc_cam_entry(mcs, id, false); + } + + /* free sa entries */ + for (id = 0; id < map->sa.max; id++) { + if (map->sa2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->sa, map->sa2pf_map, id, pcifunc); + } + return 0; +} + +int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc) +{ + int rsrc_id; + + rsrc_id = rvu_alloc_rsrc(rsrc); + if (rsrc_id < 0) + return -ENOMEM; + pf_map[rsrc_id] = pcifunc; + return rsrc_id; +} + +int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flow_id, u8 *secy_id, + u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *flow_id = id; + + id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *secy_id = id; + + id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sc_id = id; + + id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sa1_id = id; + + id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sa2_id = id; + + return 0; +} + +static void cn10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + + if (mcs->tx_sa_active[sc]) + /* SA_index1 was used and got expired */ + event.sa_id = (val >> 9) & 0xFF; + else + /* SA_index0 was used and got expired */ + event.sa_id = val & 0xFF; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void cn10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val, status; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT; + + /* TX SA interrupt is raised only if autorekey is enabled. + * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if + * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies + * SA in SA_index1 got expired else SA in SA_index0 got expired. + */ + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + /* Auto rekey is enable */ + if (!((val >> 18) & 0x1)) + continue; + + status = (val >> 21) & 0x1; + + /* Check if tx_sa_active status had changed */ + if (status == mcs->tx_sa_active[sc]) + continue; + /* SA_index0 is expired */ + if (status) + event.sa_id = val & 0xFF; + else + event.sa_id = (val >> 9) & 0xFF; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void mcs_rx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + int sa, reg; + u64 intr; + + /* Check expired SAs */ + for (reg = 0; reg < (mcs->hw->sa_entries / 64); reg++) { + /* Bit high in *PN_THRESH_REACHEDX implies + * corresponding SAs are expired. + */ + intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(reg)); + for (sa = 0; sa < 64; sa++) { + if (!(intr & BIT_ULL(sa))) + continue; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_RX_PN_THRESH_REACHED_INT; + event.sa_id = sa + (reg * 64); + event.pcifunc = mcs->rx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } + } +} + +static void mcs_rx_misc_intr_handler(struct mcs *mcs, u64 intr) +{ + struct mcs_intr_event event = { 0 }; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + if (intr & MCS_CPM_RX_INT_SECTAG_V_EQ1) + event.intr_mask = MCS_CPM_RX_SECTAG_V_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SL_GTE48) + event.intr_mask |= MCS_CPM_RX_SECTAG_SL_GTE48_INT; + if (intr & MCS_CPM_RX_INT_ES_EQ1_SC_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT; + if (intr & MCS_CPM_RX_INT_PACKET_XPN_EQ0) + event.intr_mask |= MCS_CPM_RX_PACKET_XPN_EQ0_INT; + + mcs_add_intr_wq_entry(mcs, &event); +} + +static void mcs_tx_misc_intr_handler(struct mcs *mcs, u64 intr) +{ + struct mcs_intr_event event = { 0 }; + + if (!(intr & MCS_CPM_TX_INT_SA_NOT_VALID)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + event.intr_mask = MCS_CPM_TX_SA_NOT_VALID_INT; + + mcs_add_intr_wq_entry(mcs, &event); +} + +static void mcs_bbe_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir) +{ + struct mcs_intr_event event = { 0 }; + int i; + + if (!(intr & MCS_BBE_INT_MASK)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + for (i = 0; i < MCS_MAX_BBE_INT; i++) { + if (!(intr & BIT_ULL(i))) + continue; + + /* Lower nibble denotes data fifo overflow interrupts and + * upper nibble indicates policy fifo overflow interrupts. + */ + if (intr & 0xFULL) + event.intr_mask = (dir == MCS_RX) ? + MCS_BBE_RX_DFIFO_OVERFLOW_INT : + MCS_BBE_TX_DFIFO_OVERFLOW_INT; + else + event.intr_mask = (dir == MCS_RX) ? + MCS_BBE_RX_PLFIFO_OVERFLOW_INT : + MCS_BBE_RX_PLFIFO_OVERFLOW_INT; + + /* Notify the lmac_id info which ran into BBE fatal error */ + event.lmac_id = i & 0x3ULL; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void mcs_pab_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir) +{ + struct mcs_intr_event event = { 0 }; + int i; + + if (!(intr & MCS_PAB_INT_MASK)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + for (i = 0; i < MCS_MAX_PAB_INT; i++) { + if (!(intr & BIT_ULL(i))) + continue; + + event.intr_mask = (dir == MCS_RX) ? MCS_PAB_RX_CHAN_OVERFLOW_INT : + MCS_PAB_TX_CHAN_OVERFLOW_INT; + + /* Notify the lmac_id info which ran into PAB fatal error */ + event.lmac_id = i; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static irqreturn_t mcs_ip_intr_handler(int irq, void *mcs_irq) +{ + struct mcs *mcs = (struct mcs *)mcs_irq; + u64 intr, cpm_intr, bbe_intr, pab_intr; + + /* Disable and clear the interrupt */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1C, BIT_ULL(0)); + mcs_reg_write(mcs, MCSX_IP_INT, BIT_ULL(0)); + + /* Check which block has interrupt*/ + intr = mcs_reg_read(mcs, MCSX_TOP_SLAVE_INT_SUM); + + /* CPM RX */ + if (intr & MCS_CPM_RX_INT_ENA) { + /* Check for PN thresh interrupt bit */ + cpm_intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_RX_INT); + + if (cpm_intr & MCS_CPM_RX_INT_PN_THRESH_REACHED) + mcs_rx_pn_thresh_reached_handler(mcs); + + if (cpm_intr & MCS_CPM_RX_INT_ALL) + mcs_rx_misc_intr_handler(mcs, cpm_intr); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT, cpm_intr); + } + + /* CPM TX */ + if (intr & MCS_CPM_TX_INT_ENA) { + cpm_intr = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_INT); + + if (cpm_intr & MCS_CPM_TX_INT_PN_THRESH_REACHED) { + if (mcs->hw->mcs_blks > 1) + cnf10kb_mcs_tx_pn_thresh_reached_handler(mcs); + else + cn10kb_mcs_tx_pn_thresh_reached_handler(mcs); + } + + if (cpm_intr & MCS_CPM_TX_INT_SA_NOT_VALID) + mcs_tx_misc_intr_handler(mcs, cpm_intr); + + if (cpm_intr & MCS_CPM_TX_INT_PACKET_XPN_EQ0) { + if (mcs->hw->mcs_blks > 1) + cnf10kb_mcs_tx_pn_wrapped_handler(mcs); + else + cn10kb_mcs_tx_pn_wrapped_handler(mcs); + } + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT, cpm_intr); + } + + /* BBE RX */ + if (intr & MCS_BBE_RX_INT_ENA) { + bbe_intr = mcs_reg_read(mcs, MCSX_BBE_RX_SLAVE_BBE_INT); + mcs_bbe_intr_handler(mcs, bbe_intr, MCS_RX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT, bbe_intr); + } + + /* BBE TX */ + if (intr & MCS_BBE_TX_INT_ENA) { + bbe_intr = mcs_reg_read(mcs, MCSX_BBE_TX_SLAVE_BBE_INT); + mcs_bbe_intr_handler(mcs, bbe_intr, MCS_TX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT, bbe_intr); + } + + /* PAB RX */ + if (intr & MCS_PAB_RX_INT_ENA) { + pab_intr = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PAB_INT); + mcs_pab_intr_handler(mcs, pab_intr, MCS_RX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT, pab_intr); + } + + /* PAB TX */ + if (intr & MCS_PAB_TX_INT_ENA) { + pab_intr = mcs_reg_read(mcs, MCSX_PAB_TX_SLAVE_PAB_INT); + mcs_pab_intr_handler(mcs, pab_intr, MCS_TX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT, pab_intr); + } + + /* Enable the interrupt */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0)); + + return IRQ_HANDLED; +} + +static void *alloc_mem(struct mcs *mcs, int n) +{ + return devm_kcalloc(mcs->dev, n, sizeof(u16), GFP_KERNEL); +} + +static int mcs_alloc_struct_mem(struct mcs *mcs, struct mcs_rsrc_map *res) +{ + struct hwinfo *hw = mcs->hw; + int err; + + res->flowid2pf_map = alloc_mem(mcs, hw->tcam_entries); + if (!res->flowid2pf_map) + return -ENOMEM; + + res->secy2pf_map = alloc_mem(mcs, hw->secy_entries); + if (!res->secy2pf_map) + return -ENOMEM; + + res->sc2pf_map = alloc_mem(mcs, hw->sc_entries); + if (!res->sc2pf_map) + return -ENOMEM; + + res->sa2pf_map = alloc_mem(mcs, hw->sa_entries); + if (!res->sa2pf_map) + return -ENOMEM; + + res->flowid2secy_map = alloc_mem(mcs, hw->tcam_entries); + if (!res->flowid2secy_map) + return -ENOMEM; + + res->ctrlpktrule2pf_map = alloc_mem(mcs, MCS_MAX_CTRLPKT_RULES); + if (!res->ctrlpktrule2pf_map) + return -ENOMEM; + + res->flow_ids.max = hw->tcam_entries - MCS_RSRC_RSVD_CNT; + err = rvu_alloc_bitmap(&res->flow_ids); + if (err) + return err; + + res->secy.max = hw->secy_entries - MCS_RSRC_RSVD_CNT; + err = rvu_alloc_bitmap(&res->secy); + if (err) + return err; + + res->sc.max = hw->sc_entries; + err = rvu_alloc_bitmap(&res->sc); + if (err) + return err; + + res->sa.max = hw->sa_entries; + err = rvu_alloc_bitmap(&res->sa); + if (err) + return err; + + res->ctrlpktrule.max = MCS_MAX_CTRLPKT_RULES; + err = rvu_alloc_bitmap(&res->ctrlpktrule); + if (err) + return err; + + return 0; +} + +static int mcs_register_interrupts(struct mcs *mcs) +{ + int ret = 0; + + mcs->num_vec = pci_msix_vec_count(mcs->pdev); + + ret = pci_alloc_irq_vectors(mcs->pdev, mcs->num_vec, + mcs->num_vec, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(mcs->dev, "MCS Request for %d msix vector failed err:%d\n", + mcs->num_vec, ret); + return ret; + } + + ret = request_irq(pci_irq_vector(mcs->pdev, MCS_INT_VEC_IP), + mcs_ip_intr_handler, 0, "MCS_IP", mcs); + if (ret) { + dev_err(mcs->dev, "MCS IP irq registration failed\n"); + goto exit; + } + + /* MCS enable IP interrupts */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0)); + + /* Enable CPM Rx/Tx interrupts */ + mcs_reg_write(mcs, MCSX_TOP_SLAVE_INT_SUM_ENB, + MCS_CPM_RX_INT_ENA | MCS_CPM_TX_INT_ENA | + MCS_BBE_RX_INT_ENA | MCS_BBE_TX_INT_ENA | + MCS_PAB_RX_INT_ENA | MCS_PAB_TX_INT_ENA); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT_ENB, 0x7ULL); + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT_ENB, 0x7FULL); + + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_ENB, 0xff); + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_ENB, 0xff); + + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_ENB, 0xff); + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_ENB, 0xff); + + mcs->tx_sa_active = alloc_mem(mcs, mcs->hw->sc_entries); + if (!mcs->tx_sa_active) { + ret = -ENOMEM; + goto exit; + } + + return ret; +exit: + pci_free_irq_vectors(mcs->pdev); + mcs->num_vec = 0; + return ret; +} + +int mcs_get_blkcnt(void) +{ + struct mcs *mcs; + int idmax = -ENODEV; + + /* Check MCS block is present in hardware */ + if (!pci_dev_present(mcs_id_table)) + return 0; + + list_for_each_entry(mcs, &mcs_list, mcs_list) + if (mcs->mcs_id > idmax) + idmax = mcs->mcs_id; + + if (idmax < 0) + return 0; + + return idmax + 1; +} + +struct mcs *mcs_get_pdata(int mcs_id) +{ + struct mcs *mcs_dev; + + list_for_each_entry(mcs_dev, &mcs_list, mcs_list) { + if (mcs_dev->mcs_id == mcs_id) + return mcs_dev; + } + return NULL; +} + +void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req) +{ + u64 val = 0; + + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id), + req->port_mode & MCS_PORT_MODE_MASK); + + req->cstm_tag_rel_mode_sel &= 0x3; + + if (mcs->hw->mcs_blks > 1) { + req->fifo_skid &= MCS_PORT_FIFO_SKID_MASK; + val = (u32)req->fifo_skid << 0x10; + val |= req->fifo_skid; + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id), val); + mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id), + req->cstm_tag_rel_mode_sel); + val = mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION); + + if (req->custom_hdr_enb) + val |= BIT_ULL(req->port_id); + else + val &= ~BIT_ULL(req->port_id); + + mcs_reg_write(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION, val); + } else { + val = mcs_reg_read(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id)); + val |= (req->cstm_tag_rel_mode_sel << 2); + mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id), val); + } +} + +void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp) +{ + u64 reg = 0; + + rsp->port_mode = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id)) & + MCS_PORT_MODE_MASK; + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id); + rsp->fifo_skid = mcs_reg_read(mcs, reg) & MCS_PORT_FIFO_SKID_MASK; + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id); + rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) & 0x3; + if (mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION) & BIT_ULL(req->port_id)) + rsp->custom_hdr_enb = 1; + } else { + reg = MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id); + rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) >> 2; + } + + rsp->port_id = req->port_id; + rsp->mcs_id = req->mcs_id; +} + +void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp) +{ + u64 reg = 0, val = 0; + u8 idx; + + for (idx = 0; idx < MCS_MAX_CUSTOM_TAGS; idx++) { + if (mcs->hw->mcs_blks > 1) + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(idx) : + MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(idx); + else + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_VLAN_CFGX(idx) : + MCSX_PEX_TX_SLAVE_VLAN_CFGX(idx); + + val = mcs_reg_read(mcs, reg); + if (mcs->hw->mcs_blks > 1) { + rsp->cstm_etype[idx] = val & GENMASK(15, 0); + rsp->cstm_indx[idx] = (val >> 0x16) & 0x3; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_ETYPE_ENABLE : + MCSX_PEX_TX_SLAVE_ETYPE_ENABLE; + rsp->cstm_etype_en = mcs_reg_read(mcs, reg) & 0xFF; + } else { + rsp->cstm_etype[idx] = (val >> 0x1) & GENMASK(15, 0); + rsp->cstm_indx[idx] = (val >> 0x11) & 0x3; + rsp->cstm_etype_en |= (val & 0x1) << idx; + } + } + + rsp->mcs_id = req->mcs_id; + rsp->dir = req->dir; +} + +void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset) +{ + u64 reg = MCSX_MCS_TOP_SLAVE_PORT_RESET(port_id); + + mcs_reg_write(mcs, reg, reset & 0x1); +} + +/* Set lmac to bypass/operational mode */ +void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode) +{ + u64 reg; + + reg = MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(lmac_id * 2); + mcs_reg_write(mcs, reg, (u64)mode); +} + +void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn) +{ + u64 reg; + + if (pn->dir == MCS_RX) + reg = pn->xpn ? MCSX_CPM_RX_SLAVE_XPN_THRESHOLD : MCSX_CPM_RX_SLAVE_PN_THRESHOLD; + else + reg = pn->xpn ? MCSX_CPM_TX_SLAVE_XPN_THRESHOLD : MCSX_CPM_TX_SLAVE_PN_THRESHOLD; + + mcs_reg_write(mcs, reg, pn->threshold); +} + +void cn10kb_mcs_parser_cfg(struct mcs *mcs) +{ + u64 reg, val; + + /* VLAN CTag */ + val = BIT_ULL(0) | (0x8100ull & 0xFFFF) << 1 | BIT_ULL(17); + /* RX */ + reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(0); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(0); + mcs_reg_write(mcs, reg, val); + + /* VLAN STag */ + val = BIT_ULL(0) | (0x88a8ull & 0xFFFF) << 1 | BIT_ULL(18); + /* RX */ + reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(1); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(1); + mcs_reg_write(mcs, reg, val); +} + +static void mcs_lmac_init(struct mcs *mcs, int lmac_id) +{ + u64 reg; + + /* Port mode 25GB */ + reg = MCSX_PAB_RX_SLAVE_PORT_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0xe000e); + return; + } + + reg = MCSX_PAB_TX_SLAVE_PORT_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0); +} + +int mcs_set_lmac_channels(int mcs_id, u16 base) +{ + struct mcs *mcs; + int lmac; + u64 cfg; + + mcs = mcs_get_pdata(mcs_id); + if (!mcs) + return -ENODEV; + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) { + cfg = mcs_reg_read(mcs, MCSX_LINK_LMACX_CFG(lmac)); + cfg &= ~(MCSX_LINK_LMAC_BASE_MASK | MCSX_LINK_LMAC_RANGE_MASK); + cfg |= FIELD_PREP(MCSX_LINK_LMAC_RANGE_MASK, ilog2(16)); + cfg |= FIELD_PREP(MCSX_LINK_LMAC_BASE_MASK, base); + mcs_reg_write(mcs, MCSX_LINK_LMACX_CFG(lmac), cfg); + base += 16; + } + return 0; +} + +static int mcs_x2p_calibration(struct mcs *mcs) +{ + unsigned long timeout = jiffies + usecs_to_jiffies(20000); + int i, err = 0; + u64 val; + + /* set X2P calibration */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + val |= BIT_ULL(5); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + + /* Wait for calibration to complete */ + while (!(mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS) & BIT_ULL(0))) { + if (time_before(jiffies, timeout)) { + usleep_range(80, 100); + continue; + } else { + err = -EBUSY; + dev_err(mcs->dev, "MCS X2P calibration failed..ignoring\n"); + return err; + } + } + + val = mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS); + for (i = 0; i < mcs->hw->mcs_x2p_intf; i++) { + if (val & BIT_ULL(1 + i)) + continue; + err = -EBUSY; + dev_err(mcs->dev, "MCS:%d didn't respond to X2P calibration\n", i); + } + /* Clear X2P calibrate */ + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, mcs_reg_read(mcs, MCSX_MIL_GLOBAL) & ~BIT_ULL(5)); + + return err; +} + +static void mcs_set_external_bypass(struct mcs *mcs, u8 bypass) +{ + u64 val; + + /* Set MCS to external bypass */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + if (bypass) + val |= BIT_ULL(6); + else + val &= ~BIT_ULL(6); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); +} + +static void mcs_global_cfg(struct mcs *mcs) +{ + /* Disable external bypass */ + mcs_set_external_bypass(mcs, false); + + /* Reset TX/RX stats memory */ + mcs_reg_write(mcs, MCSX_CSE_RX_SLAVE_STATS_CLEAR, 0x1F); + mcs_reg_write(mcs, MCSX_CSE_TX_SLAVE_STATS_CLEAR, 0x1F); + + /* Set MCS to perform standard IEEE802.1AE macsec processing */ + if (mcs->hw->mcs_blks == 1) { + mcs_reg_write(mcs, MCSX_IP_MODE, BIT_ULL(3)); + return; + } + + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_ENTRY, 0xe4); + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_LEN, 4); +} + +void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs) +{ + struct hwinfo *hw = mcs->hw; + + hw->tcam_entries = 128; /* TCAM entries */ + hw->secy_entries = 128; /* SecY entries */ + hw->sc_entries = 128; /* SC CAM entries */ + hw->sa_entries = 256; /* SA entries */ + hw->lmac_cnt = 20; /* lmacs/ports per mcs block */ + hw->mcs_x2p_intf = 5; /* x2p clabration intf */ + hw->mcs_blks = 1; /* MCS blocks */ +} + +static struct mcs_ops cn10kb_mcs_ops = { + .mcs_set_hw_capabilities = cn10kb_mcs_set_hw_capabilities, + .mcs_parser_cfg = cn10kb_mcs_parser_cfg, + .mcs_tx_sa_mem_map_write = cn10kb_mcs_tx_sa_mem_map_write, + .mcs_rx_sa_mem_map_write = cn10kb_mcs_rx_sa_mem_map_write, + .mcs_flowid_secy_map = cn10kb_mcs_flowid_secy_map, +}; + +static int mcs_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + int lmac, err = 0; + struct mcs *mcs; + + mcs = devm_kzalloc(dev, sizeof(*mcs), GFP_KERNEL); + if (!mcs) + return -ENOMEM; + + mcs->hw = devm_kzalloc(dev, sizeof(struct hwinfo), GFP_KERNEL); + if (!mcs->hw) + return -ENOMEM; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto exit; + } + + mcs->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); + if (!mcs->reg_base) { + dev_err(dev, "mcs: Cannot map CSR memory space, aborting\n"); + err = -ENOMEM; + goto exit; + } + + pci_set_drvdata(pdev, mcs); + mcs->pdev = pdev; + mcs->dev = &pdev->dev; + + if (pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B) + mcs->mcs_ops = &cn10kb_mcs_ops; + else + mcs->mcs_ops = cnf10kb_get_mac_ops(); + + /* Set hardware capabilities */ + mcs->mcs_ops->mcs_set_hw_capabilities(mcs); + + mcs_global_cfg(mcs); + + /* Perform X2P clibration */ + err = mcs_x2p_calibration(mcs); + if (err) + goto err_x2p; + + mcs->mcs_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) + & MCS_ID_MASK; + + /* Set mcs tx side resources */ + err = mcs_alloc_struct_mem(mcs, &mcs->tx); + if (err) + goto err_x2p; + + /* Set mcs rx side resources */ + err = mcs_alloc_struct_mem(mcs, &mcs->rx); + if (err) + goto err_x2p; + + /* per port config */ + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) + mcs_lmac_init(mcs, lmac); + + /* Parser configuration */ + mcs->mcs_ops->mcs_parser_cfg(mcs); + + err = mcs_register_interrupts(mcs); + if (err) + goto exit; + + list_add(&mcs->mcs_list, &mcs_list); + mutex_init(&mcs->stats_lock); + + return 0; + +err_x2p: + /* Enable external bypass */ + mcs_set_external_bypass(mcs, true); +exit: + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void mcs_remove(struct pci_dev *pdev) +{ + struct mcs *mcs = pci_get_drvdata(pdev); + + /* Set MCS to external bypass */ + mcs_set_external_bypass(mcs, true); + pci_free_irq_vectors(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +struct pci_driver mcs_driver = { + .name = DRV_NAME, + .id_table = mcs_id_table, + .probe = mcs_probe, + .remove = mcs_remove, +}; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h new file mode 100644 index 000000000000..64dc2b80e15d --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h @@ -0,0 +1,246 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell CN10K MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#ifndef MCS_H +#define MCS_H + +#include <linux/bits.h> +#include "rvu.h" + +#define PCI_DEVID_CN10K_MCS 0xA096 + +#define MCSX_LINK_LMAC_RANGE_MASK GENMASK_ULL(19, 16) +#define MCSX_LINK_LMAC_BASE_MASK GENMASK_ULL(11, 0) + +#define MCS_ID_MASK 0x7 +#define MCS_MAX_PFS 128 + +#define MCS_PORT_MODE_MASK 0x3 +#define MCS_PORT_FIFO_SKID_MASK 0x3F +#define MCS_MAX_CUSTOM_TAGS 0x8 + +#define MCS_CTRLPKT_ETYPE_RULE_MAX 8 +#define MCS_CTRLPKT_DA_RULE_MAX 8 +#define MCS_CTRLPKT_DA_RANGE_RULE_MAX 4 +#define MCS_CTRLPKT_COMBO_RULE_MAX 4 +#define MCS_CTRLPKT_MAC_RULE_MAX 1 + +#define MCS_MAX_CTRLPKT_RULES (MCS_CTRLPKT_ETYPE_RULE_MAX + \ + MCS_CTRLPKT_DA_RULE_MAX + \ + MCS_CTRLPKT_DA_RANGE_RULE_MAX + \ + MCS_CTRLPKT_COMBO_RULE_MAX + \ + MCS_CTRLPKT_MAC_RULE_MAX) + +#define MCS_CTRLPKT_ETYPE_RULE_OFFSET 0 +#define MCS_CTRLPKT_DA_RULE_OFFSET 8 +#define MCS_CTRLPKT_DA_RANGE_RULE_OFFSET 16 +#define MCS_CTRLPKT_COMBO_RULE_OFFSET 20 +#define MCS_CTRLPKT_MAC_EN_RULE_OFFSET 24 + +/* Reserved resources for default bypass entry */ +#define MCS_RSRC_RSVD_CNT 1 + +/* MCS Interrupt Vector Enumeration */ +enum mcs_int_vec_e { + MCS_INT_VEC_MIL_RX_GBL = 0x0, + MCS_INT_VEC_MIL_RX_LMACX = 0x1, + MCS_INT_VEC_MIL_TX_LMACX = 0x5, + MCS_INT_VEC_HIL_RX_GBL = 0x9, + MCS_INT_VEC_HIL_RX_LMACX = 0xa, + MCS_INT_VEC_HIL_TX_GBL = 0xe, + MCS_INT_VEC_HIL_TX_LMACX = 0xf, + MCS_INT_VEC_IP = 0x13, + MCS_INT_VEC_CNT = 0x14, +}; + +#define MCS_MAX_BBE_INT 8ULL +#define MCS_BBE_INT_MASK 0xFFULL + +#define MCS_MAX_PAB_INT 4ULL +#define MCS_PAB_INT_MASK 0xFULL + +#define MCS_BBE_RX_INT_ENA BIT_ULL(0) +#define MCS_BBE_TX_INT_ENA BIT_ULL(1) +#define MCS_CPM_RX_INT_ENA BIT_ULL(2) +#define MCS_CPM_TX_INT_ENA BIT_ULL(3) +#define MCS_PAB_RX_INT_ENA BIT_ULL(4) +#define MCS_PAB_TX_INT_ENA BIT_ULL(5) + +#define MCS_CPM_TX_INT_PACKET_XPN_EQ0 BIT_ULL(0) +#define MCS_CPM_TX_INT_PN_THRESH_REACHED BIT_ULL(1) +#define MCS_CPM_TX_INT_SA_NOT_VALID BIT_ULL(2) + +#define MCS_CPM_RX_INT_SECTAG_V_EQ1 BIT_ULL(0) +#define MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 BIT_ULL(1) +#define MCS_CPM_RX_INT_SL_GTE48 BIT_ULL(2) +#define MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 BIT_ULL(3) +#define MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 BIT_ULL(4) +#define MCS_CPM_RX_INT_PACKET_XPN_EQ0 BIT_ULL(5) +#define MCS_CPM_RX_INT_PN_THRESH_REACHED BIT_ULL(6) + +#define MCS_CPM_RX_INT_ALL (MCS_CPM_RX_INT_SECTAG_V_EQ1 | \ + MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 | \ + MCS_CPM_RX_INT_SL_GTE48 | \ + MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 | \ + MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 | \ + MCS_CPM_RX_INT_PACKET_XPN_EQ0 | \ + MCS_CPM_RX_INT_PN_THRESH_REACHED) + +struct mcs_pfvf { + u64 intr_mask; /* Enabled Interrupt mask */ +}; + +struct mcs_intr_event { + u16 pcifunc; + u64 intr_mask; + u64 sa_id; + u8 mcs_id; + u8 lmac_id; +}; + +struct mcs_intrq_entry { + struct list_head node; + struct mcs_intr_event intr_event; +}; + +struct secy_mem_map { + u8 flow_id; + u8 secy; + u8 ctrl_pkt; + u8 sc; + u64 sci; +}; + +struct mcs_rsrc_map { + u16 *flowid2pf_map; + u16 *secy2pf_map; + u16 *sc2pf_map; + u16 *sa2pf_map; + u16 *flowid2secy_map; /* bitmap flowid mapped to secy*/ + u16 *ctrlpktrule2pf_map; + struct rsrc_bmap flow_ids; + struct rsrc_bmap secy; + struct rsrc_bmap sc; + struct rsrc_bmap sa; + struct rsrc_bmap ctrlpktrule; +}; + +struct hwinfo { + u8 tcam_entries; + u8 secy_entries; + u8 sc_entries; + u16 sa_entries; + u8 mcs_x2p_intf; + u8 lmac_cnt; + u8 mcs_blks; + unsigned long lmac_bmap; /* bitmap of enabled mcs lmac */ +}; + +struct mcs { + void __iomem *reg_base; + struct pci_dev *pdev; + struct device *dev; + struct hwinfo *hw; + struct mcs_rsrc_map tx; + struct mcs_rsrc_map rx; + u16 pf_map[MCS_MAX_PFS]; /* List of PCIFUNC mapped to MCS */ + u8 mcs_id; + struct mcs_ops *mcs_ops; + struct list_head mcs_list; + /* Lock for mcs stats */ + struct mutex stats_lock; + struct mcs_pfvf *pf; + struct mcs_pfvf *vf; + u16 num_vec; + void *rvu; + u16 *tx_sa_active; +}; + +struct mcs_ops { + void (*mcs_set_hw_capabilities)(struct mcs *mcs); + void (*mcs_parser_cfg)(struct mcs *mcs); + void (*mcs_tx_sa_mem_map_write)(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); + void (*mcs_rx_sa_mem_map_write)(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); + void (*mcs_flowid_secy_map)(struct mcs *mcs, struct secy_mem_map *map, int dir); +}; + +extern struct pci_driver mcs_driver; + +static inline void mcs_reg_write(struct mcs *mcs, u64 offset, u64 val) +{ + writeq(val, mcs->reg_base + offset); +} + +static inline u64 mcs_reg_read(struct mcs *mcs, u64 offset) +{ + return readq(mcs->reg_base + offset); +} + +/* MCS APIs */ +struct mcs *mcs_get_pdata(int mcs_id); +int mcs_get_blkcnt(void); +int mcs_set_lmac_channels(int mcs_id, u16 base); +int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc); +int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc); +int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flowid, u8 *secy_id, + u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir); +int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc); +void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir); +void mcs_ena_dis_flowid_entry(struct mcs *mcs, int id, int dir, int ena); +void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int id, int ena); +void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int id, int dir); +void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int id, int dir); +void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id); +void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa, int dir); +void mcs_map_sc_to_sa(struct mcs *mcs, u64 *sa_map, int sc, int dir); +void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir); +void mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn); +int mcs_install_flowid_bypass_entry(struct mcs *mcs); +void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode); +void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset); +void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req); +void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp); +void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp); +int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc); +int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req); +int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req); + +/* CN10K-B APIs */ +void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs); +void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void cn10kb_mcs_parser_cfg(struct mcs *mcs); + +/* CNF10K-B APIs */ +struct mcs_ops *cnf10kb_get_mac_ops(void); +void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs); +void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void cnf10kb_mcs_parser_cfg(struct mcs *mcs); +void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs); +void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs); + +/* Stats APIs */ +void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, int id, int dir); +void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir); +void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, int id, int dir); +void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, int id, int dir); +void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id); +void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id); +void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir); +int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir); +int mcs_set_force_clk_en(struct mcs *mcs, bool set); + +int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event); + +#endif /* MCS_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c new file mode 100644 index 000000000000..7b6205414428 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include "mcs.h" +#include "mcs_reg.h" + +static struct mcs_ops cnf10kb_mcs_ops = { + .mcs_set_hw_capabilities = cnf10kb_mcs_set_hw_capabilities, + .mcs_parser_cfg = cnf10kb_mcs_parser_cfg, + .mcs_tx_sa_mem_map_write = cnf10kb_mcs_tx_sa_mem_map_write, + .mcs_rx_sa_mem_map_write = cnf10kb_mcs_rx_sa_mem_map_write, + .mcs_flowid_secy_map = cnf10kb_mcs_flowid_secy_map, +}; + +struct mcs_ops *cnf10kb_get_mac_ops(void) +{ + return &cnf10kb_mcs_ops; +} + +void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs) +{ + struct hwinfo *hw = mcs->hw; + + hw->tcam_entries = 64; /* TCAM entries */ + hw->secy_entries = 64; /* SecY entries */ + hw->sc_entries = 64; /* SC CAM entries */ + hw->sa_entries = 128; /* SA entries */ + hw->lmac_cnt = 4; /* lmacs/ports per mcs block */ + hw->mcs_x2p_intf = 1; /* x2p clabration intf */ + hw->mcs_blks = 7; /* MCS blocks */ +} + +void cnf10kb_mcs_parser_cfg(struct mcs *mcs) +{ + u64 reg, val; + + /* VLAN Ctag */ + val = (0x8100ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(22); + + reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(0); + mcs_reg_write(mcs, reg, val); + + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(0); + mcs_reg_write(mcs, reg, val); + + /* VLAN STag */ + val = (0x88a8ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(23); + + /* RX */ + reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(1); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(1); + mcs_reg_write(mcs, reg, val); + + /* Enable custom tage 0 and 1 and sectag */ + val = BIT_ULL(0) | BIT_ULL(1) | BIT_ULL(12); + + reg = MCSX_PEX_RX_SLAVE_ETYPE_ENABLE; + mcs_reg_write(mcs, reg, val); + + reg = MCSX_PEX_TX_SLAVE_ETYPE_ENABLE; + mcs_reg_write(mcs, reg, val); +} + +void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir) +{ + u64 reg, val; + + val = (map->secy & 0x3F) | (map->ctrl_pkt & 0x1) << 6; + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id); + } else { + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id); + mcs_reg_write(mcs, reg, map->sci); + val |= (map->sc & 0x3F) << 7; + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(map->flow_id); + } + + mcs_reg_write(mcs, reg, val); +} + +void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map) +{ + u64 reg, val; + + val = (map->sa_index0 & 0x7F) | (map->sa_index1 & 0x7F) << 7; + + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id); + mcs_reg_write(mcs, reg, val); + + reg = MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0; + val = mcs_reg_read(mcs, reg); + + if (map->rekey_ena) + val |= BIT_ULL(map->sc_id); + else + val &= ~BIT_ULL(map->sc_id); + + mcs_reg_write(mcs, reg, val); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(map->sc_id), map->sa_index0_vld); + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(map->sc_id), map->sa_index1_vld); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(map->sc_id), map->tx_sa_active); +} + +void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map) +{ + u64 val, reg; + + val = (map->sa_index & 0x7F) | (map->sa_in_use << 7); + + reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an); + mcs_reg_write(mcs, reg, val); +} + +int mcs_set_force_clk_en(struct mcs *mcs, bool set) +{ + unsigned long timeout = jiffies + usecs_to_jiffies(2000); + u64 val; + + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + + if (set) { + val |= BIT_ULL(4); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + + /* Poll till mcsx_mil_ip_gbl_status.mcs_ip_stats_ready value is 1 */ + while (!(mcs_reg_read(mcs, MCSX_MIL_IP_GBL_STATUS) & BIT_ULL(0))) { + if (time_after(jiffies, timeout)) { + dev_err(mcs->dev, "MCS set force clk enable failed\n"); + break; + } + } + } else { + val &= ~BIT_ULL(4); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + } + + return 0; +} + +/* TX SA interrupt is raised only if autorekey is enabled. + * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if + * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies + * SA in SA_index1 got expired else SA in SA_index0 got expired. + */ +void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event; + struct rsrc_bmap *sc_bmap; + unsigned long rekey_ena; + u64 val, sa_status; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT; + + rekey_ena = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0); + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + /* Auto rekey is enable */ + if (!test_bit(sc, &rekey_ena)) + continue; + sa_status = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(sc)); + /* Check if tx_sa_active status had changed */ + if (sa_status == mcs->tx_sa_active[sc]) + continue; + + /* SA_index0 is expired */ + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + if (sa_status) + event.sa_id = val & 0x7F; + else + event.sa_id = (val >> 7) & 0x7F; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + + if (mcs->tx_sa_active[sc]) + /* SA_index1 was used and got expired */ + event.sa_id = (val >> 7) & 0x7F; + else + /* SA_index0 was used and got expired */ + event.sa_id = val & 0x7F; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h new file mode 100644 index 000000000000..c95a8b8f5eaf --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h @@ -0,0 +1,1102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#ifndef MCS_REG_H +#define MCS_REG_H + +#include <linux/bits.h> + +/* Registers */ +#define MCSX_IP_MODE 0x900c8ull +#define MCSX_MCS_TOP_SLAVE_PORT_RESET(a) ({ \ + u64 offset; \ + \ + offset = 0x408ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa28ull; \ + offset += (a) * 0x8ull; \ + offset; }) + + +#define MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x808ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa68ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_MIL_GLOBAL ({ \ + u64 offset; \ + \ + offset = 0x80000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60000ull; \ + offset; }) + +#define MCSX_MIL_RX_LMACX_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x900a8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x700a8ull; \ + offset += (a) * 0x800ull; \ + offset; }) + +#define MCSX_HIL_GLOBAL ({ \ + u64 offset; \ + \ + offset = 0xc0000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa0000ull; \ + offset; }) + +#define MCSX_LINK_LMACX_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x90000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x70000ull; \ + offset += (a) * 0x800ull; \ + offset; }) + +#define MCSX_MIL_RX_GBL_STATUS ({ \ + u64 offset; \ + \ + offset = 0x800c8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x600c8ull; \ + offset; }) + +#define MCSX_MIL_IP_GBL_STATUS ({ \ + u64 offset; \ + \ + offset = 0x800d0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x600d0ull; \ + offset; }) + +/* PAB */ +#define MCSX_PAB_RX_SLAVE_PORT_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x1718ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x280ull; \ + offset += (a) * 0x40ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PORT_CFGX(a) (0x2930ull + (a) * 0x40ull) + +/* PEX registers */ +#define MCSX_PEX_RX_SLAVE_VLAN_CFGX(a) (0x3b58ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_VLAN_CFGX(a) (0x46f8ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(a) (0x788ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_PORT_CONFIG(a) (0x4738ull + (a) * 0x8ull) +#define MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x3fc0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x558ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DAX(a) ({ \ + u64 offset; \ + \ + offset = 0x4000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x598ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4048ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5e0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4080ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x648ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4088ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x650ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(a) ({ \ + u64 offset; \ + \ + offset = 0x4090ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x658ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_MAC ({ \ + u64 offset; \ + \ + offset = 0x40e0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6d8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_ENABLE ({ \ + u64 offset; \ + \ + offset = 0x40e8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6e0ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x4b60ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x7d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DAX(a) ({ \ + u64 offset; \ + \ + offset = 0x4ba0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4be0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x858ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4be8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x860ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c20ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8c8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c28ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8d0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c30ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_MAC ({ \ + u64 offset; \ + \ + offset = 0x4c80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x958ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_ENABLE ({ \ + u64 offset; \ + \ + offset = 0x4c88ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x960ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION ({ \ + u64 offset; \ + \ + offset = 0x3b50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4c0ull; \ + offset; }) + +/* CNF10K-B */ +#define MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(a) (0x4c8ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(a) (0x748ull + (a) * 0x8ull) +#define MCSX_PEX_RX_SLAVE_ETYPE_ENABLE 0x6e8ull +#define MCSX_PEX_TX_SLAVE_ETYPE_ENABLE 0x968ull + +/* BEE */ +#define MCSX_BBE_RX_SLAVE_PADDING_CTL 0xe08ull +#define MCSX_BBE_TX_SLAVE_PADDING_CTL 0x12f8ull +#define MCSX_BBE_RX_SLAVE_CAL_ENTRY 0x180ull +#define MCSX_BBE_RX_SLAVE_CAL_LEN 0x188ull +#define MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(a) (0x290ull + (a) * 0x40ull) + +#define MCSX_BBE_RX_SLAVE_BBE_INT ({ \ + u64 offset; \ + \ + offset = 0xe00ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x160ull; \ + offset; }) + +#define MCSX_BBE_RX_SLAVE_BBE_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0xe08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x168ull; \ + offset; }) + +#define MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0xe08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x178ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT ({ \ + u64 offset; \ + \ + offset = 0x1278ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1e0ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x1278ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1f8ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x1280ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1e8ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT ({ \ + u64 offset; \ + \ + offset = 0x16f0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x260ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x268ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x278ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT ({ \ + u64 offset; \ + \ + offset = 0x2908ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x380ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x2910ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x388ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x398ull; \ + offset; }) + +/* CPM registers */ +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x30740ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x3bf8ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x34740ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x43f8ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0 ({ \ + u64 offset; \ + \ + offset = 0x30700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x3bd8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SC_CAMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x38780ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4c08ull; \ + offset += (a) * 0x8ull + (b) * 0x10ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SC_CAM_ENA(a) ({ \ + u64 offset; \ + \ + offset = 0x38740ull + (a) * 0x8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4bf8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x23ee0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbd0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = (0x246e0ull + (a) * 0x10ull); \ + if (mcs->hw->mcs_blks > 1) \ + offset = (0xdd0ull + (a) * 0x8ull); \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_KEY_LOCKOUTX(a) ({ \ + u64 offset; \ + \ + offset = 0x23E90ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbb0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_MAP_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x256e0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfd0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x27700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x17d8ull; \ + offset += (a) * 0x8ull + (b) * 0x40ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x2f700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x37d8; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_XPN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x23e40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xb90ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_PN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x23e48ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xb98ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x23e50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xba0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1 0x30708ull +#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(a) (0x246e8ull + (a) * 0x10ull) + +/* TX registers */ +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x51d50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa7c0ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x55d50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xafc0ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0 ({ \ + u64 offset; \ + \ + offset = 0x51d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa7a0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = 0x3e508ull + (a) * 0x8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5550ull + (a) * 0x10ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x3ed08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5950ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_KEY_LOCKOUTX(a) ({ \ + u64 offset; \ + \ + offset = 0x3e4c0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5538ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = 0x3fd10ull + (a) * 0x10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6150ull + (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x40d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x63a0ull; \ + offset += (a) * 0x8ull + (b) * 0x80ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x50d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa3a0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_XPN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x3e4b0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5528ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_PN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x3e4b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5530ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(a) (0x3fd18ull + (a) * 0x10ull) +#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(a) (0x5558ull + (a) * 0x10ull) +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1 0x51d18ull +#define MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(a) (0x5b50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(a) (0x5d50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(a) (0x5f50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0 0x5500ull + +/* CSE */ +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x9e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc218ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x9680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc018ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x6e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x8e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbe18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x8680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xca18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x7e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x6680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x7680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc618ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x5e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xdc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(a)({ \ + u64 offset; \ + \ + offset = 0x5680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xda18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(a) ({ \ + u64 offset; \ + \ + offset = 0xd680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xce18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0x16a80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(a) ({ \ + u64 offset; \ + \ + offset = 0x16680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec38ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x16880ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0xfe80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xde18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(a) ({ \ + u64 offset; \ + \ + offset = 0x10680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xe418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(a) ({ \ + u64 offset; \ + \ + offset = 0x10e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xe218ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(a) ({ \ + u64 offset; \ + \ + offset = 0xae80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(a) ({ \ + u64 offset; \ + \ + offset = 0xc680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd618ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(a) ({ \ + u64 offset; \ + \ + offset = 0xce80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(a) ({ \ + u64 offset; \ + \ + offset = 0xbe80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xcc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_SLAVE_CTRL ({ \ + u64 offset; \ + \ + offset = 0x52a0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x9c0ull; \ + offset; }) + +#define MCSX_CSE_RX_SLAVE_STATS_CLEAR ({ \ + u64 offset; \ + \ + offset = 0x52b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x9d8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(a) (0xe680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(a) (0xde80ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(a) (0xa680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(a) (0xd218 + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDX(a) (0xd018ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(a) (0xee80ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(a) (0xb680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(a) (0xf680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(a) (0x12680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(a) (0x15680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(a) (0x13680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(a) (0x11680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(a) (0x14680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(a) (0xec58ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(a) (0xea18ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(a) (0xe618ull + (a) * 0x8ull) + +/* CSE TX */ +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCOMMONOCTETSX(a) (0x18440ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1c440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf478ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1bc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf278ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x19440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xee78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1b440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf078ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1ac40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfc78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1a440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfa78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x18c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf678ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x19c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf878ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x17c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10878ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x17440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10678ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSCTRLPORTDISABLEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1e440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfe78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0x23240ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10ed8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(a) ({ \ + u64 offset; \ + \ + offset = 0x22c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10e98ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x22e40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10e78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x20440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10c78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1fc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10a78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x23040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x110d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(a) ({ \ + u64 offset; \ + \ + offset = 0x1dc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10278ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(a) ({ \ + u64 offset; \ + \ + offset = 0x1d440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10478ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1cc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10078ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_SLAVE_CTRL ({ \ + u64 offset; \ + \ + offset = 0x54a0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa00ull; \ + offset; }) + +#define MCSX_CSE_TX_SLAVE_STATS_CLEAR ({ \ + u64 offset; \ + \ + offset = 0x54b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa18ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(a) (0x1f440ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(a) (0x1ec40ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSEARLYPREEMPTERRX(a) (0x10eb8ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(a) (0x21c40ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(a) (0x20c40ull + (a) * 0x8ull) + +#define MCSX_IP_INT ({ \ + u64 offset; \ + \ + offset = 0x80028ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60028ull; \ + offset; }) + +#define MCSX_IP_INT_ENA_W1S ({ \ + u64 offset; \ + \ + offset = 0x80040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60040ull; \ + offset; }) + +#define MCSX_IP_INT_ENA_W1C ({ \ + u64 offset; \ + \ + offset = 0x80038ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60038ull; \ + offset; }) + +#define MCSX_TOP_SLAVE_INT_SUM ({ \ + u64 offset; \ + \ + offset = 0xc20ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xab8ull; \ + offset; }) + +#define MCSX_TOP_SLAVE_INT_SUM_ENB ({ \ + u64 offset; \ + \ + offset = 0xc28ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xac0ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_RX_INT ({ \ + u64 offset; \ + \ + offset = 0x23c00ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x0ad8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_RX_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x23c08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xae0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_TX_INT ({ \ + u64 offset; \ + \ + offset = 0x3d490ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x54a0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_TX_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x3d498ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x54a8ull; \ + offset; }) + +#endif diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c new file mode 100644 index 000000000000..fa8029a94068 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell CN10K MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include <linux/types.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "mcs.h" +#include "rvu.h" +#include "lmac_common.h" + +#define M(_name, _id, _fn_name, _req_type, _rsp_type) \ +static struct _req_type __maybe_unused \ +*otx2_mbox_alloc_msg_ ## _fn_name(struct rvu *rvu, int devid) \ +{ \ + struct _req_type *req; \ + \ + req = (struct _req_type *)otx2_mbox_alloc_msg_rsp( \ + &rvu->afpf_wq_info.mbox_up, devid, sizeof(struct _req_type), \ + sizeof(struct _rsp_type)); \ + if (!req) \ + return NULL; \ + req->hdr.sig = OTX2_MBOX_REQ_SIG; \ + req->hdr.id = _id; \ + return req; \ +} + +MBOX_UP_MCS_MESSAGES +#undef M + +int rvu_mbox_handler_mcs_set_lmac_mode(struct rvu *rvu, + struct mcs_set_lmac_mode *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (BIT_ULL(req->lmac_id) & mcs->hw->lmac_bmap) + mcs_set_lmac_mode(mcs, req->lmac_id, req->mode); + + return 0; +} + +int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event) +{ + struct mcs_intrq_entry *qentry; + u16 pcifunc = event->pcifunc; + struct rvu *rvu = mcs->rvu; + struct mcs_pfvf *pfvf; + + /* Check if it is PF or VF */ + if (pcifunc & RVU_PFVF_FUNC_MASK) + pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)]; + else + pfvf = &mcs->pf[rvu_get_pf(pcifunc)]; + + event->intr_mask &= pfvf->intr_mask; + + /* Check PF/VF interrupt notification is enabled */ + if (!(pfvf->intr_mask && event->intr_mask)) + return 0; + + qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); + if (!qentry) + return -ENOMEM; + + qentry->intr_event = *event; + spin_lock(&rvu->mcs_intrq_lock); + list_add_tail(&qentry->node, &rvu->mcs_intrq_head); + spin_unlock(&rvu->mcs_intrq_lock); + queue_work(rvu->mcs_intr_wq, &rvu->mcs_intr_work); + + return 0; +} + +static int mcs_notify_pfvf(struct mcs_intr_event *event, struct rvu *rvu) +{ + struct mcs_intr_info *req; + int err, pf; + + pf = rvu_get_pf(event->pcifunc); + + req = otx2_mbox_alloc_msg_mcs_intr_notify(rvu, pf); + if (!req) + return -ENOMEM; + + req->mcs_id = event->mcs_id; + req->intr_mask = event->intr_mask; + req->sa_id = event->sa_id; + req->hdr.pcifunc = event->pcifunc; + req->lmac_id = event->lmac_id; + + otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, pf); + err = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, pf); + if (err) + dev_warn(rvu->dev, "MCS notification to pf %d failed\n", pf); + + return 0; +} + +static void mcs_intr_handler_task(struct work_struct *work) +{ + struct rvu *rvu = container_of(work, struct rvu, mcs_intr_work); + struct mcs_intrq_entry *qentry; + struct mcs_intr_event *event; + unsigned long flags; + + do { + spin_lock_irqsave(&rvu->mcs_intrq_lock, flags); + qentry = list_first_entry_or_null(&rvu->mcs_intrq_head, + struct mcs_intrq_entry, + node); + if (qentry) + list_del(&qentry->node); + + spin_unlock_irqrestore(&rvu->mcs_intrq_lock, flags); + if (!qentry) + break; /* nothing more to process */ + + event = &qentry->intr_event; + + mcs_notify_pfvf(event, rvu); + kfree(qentry); + } while (1); +} + +int rvu_mbox_handler_mcs_intr_cfg(struct rvu *rvu, + struct mcs_intr_cfg *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_pfvf *pfvf; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* Check if it is PF or VF */ + if (pcifunc & RVU_PFVF_FUNC_MASK) + pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)]; + else + pfvf = &mcs->pf[rvu_get_pf(pcifunc)]; + + mcs->pf_map[0] = pcifunc; + pfvf->intr_mask = req->intr_mask; + + return 0; +} + +int rvu_mbox_handler_mcs_get_hw_info(struct rvu *rvu, + struct msg_req *req, + struct mcs_hw_info *rsp) +{ + struct mcs *mcs; + + if (!rvu->mcs_blk_cnt) + return MCS_AF_ERR_NOT_MAPPED; + + /* MCS resources are same across all blocks */ + mcs = mcs_get_pdata(0); + rsp->num_mcs_blks = rvu->mcs_blk_cnt; + rsp->tcam_entries = mcs->hw->tcam_entries; + rsp->secy_entries = mcs->hw->secy_entries; + rsp->sc_entries = mcs->hw->sc_entries; + rsp->sa_entries = mcs->hw->sa_entries; + return 0; +} + +int rvu_mbox_handler_mcs_port_reset(struct rvu *rvu, struct mcs_port_reset_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_reset_port(mcs, req->port_id, req->reset); + + return 0; +} + +int rvu_mbox_handler_mcs_clear_stats(struct rvu *rvu, + struct mcs_clear_stats *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mutex_lock(&mcs->stats_lock); + if (req->all) + mcs_clear_all_stats(mcs, pcifunc, req->dir); + else + mcs_clear_stats(mcs, req->type, req->id, req->dir); + + mutex_unlock(&mcs->stats_lock); + return 0; +} + +int rvu_mbox_handler_mcs_get_flowid_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_flowid_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* In CNF10K-B, before reading the statistics, + * MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP needs to be set + * to get accurate statistics + */ + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_flowid_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + /* Clear MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP after reading + * the statistics + */ + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_secy_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_secy_stats *rsp) +{ struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + + if (req->dir == MCS_RX) + mcs_get_rx_secy_stats(mcs, rsp, req->id); + else + mcs_get_tx_secy_stats(mcs, rsp, req->id); + + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_sc_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_sc_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_sc_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_sa_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_sa_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_sa_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_port_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_port_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_port_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_set_active_lmac(struct rvu *rvu, + struct mcs_set_active_lmac *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + if (!mcs) + return MCS_AF_ERR_NOT_MAPPED; + + mcs->hw->lmac_bmap = req->lmac_bmap; + mcs_set_lmac_channels(req->mcs_id, req->chan_base); + return 0; +} + +int rvu_mbox_handler_mcs_port_cfg_set(struct rvu *rvu, struct mcs_port_cfg_set_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id))) + return -EINVAL; + + mcs_set_port_cfg(mcs, req); + + return 0; +} + +int rvu_mbox_handler_mcs_port_cfg_get(struct rvu *rvu, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id))) + return -EINVAL; + + mcs_get_port_cfg(mcs, req, rsp); + + return 0; +} + +int rvu_mbox_handler_mcs_custom_tag_cfg_get(struct rvu *rvu, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_get_custom_tag_cfg(mcs, req, rsp); + + return 0; +} + +int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc) +{ + struct mcs *mcs; + int mcs_id; + + /* CNF10K-B mcs0-6 are mapped to RPM2-8*/ + if (rvu->mcs_blk_cnt > 1) { + for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) { + mcs = mcs_get_pdata(mcs_id); + mcs_free_all_rsrc(mcs, MCS_RX, pcifunc); + mcs_free_all_rsrc(mcs, MCS_TX, pcifunc); + } + } else { + /* CN10K-B has only one mcs block */ + mcs = mcs_get_pdata(0); + mcs_free_all_rsrc(mcs, MCS_RX, pcifunc); + mcs_free_all_rsrc(mcs, MCS_TX, pcifunc); + } + return 0; +} + +int rvu_mbox_handler_mcs_flowid_ena_entry(struct rvu *rvu, + struct mcs_flowid_ena_dis_entry *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_ena_dis_flowid_entry(mcs, req->flow_id, req->dir, req->ena); + return 0; +} + +int rvu_mbox_handler_mcs_pn_table_write(struct rvu *rvu, + struct mcs_pn_table_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_pn_table_write(mcs, req->pn_id, req->next_pn, req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_set_pn_threshold(struct rvu *rvu, + struct mcs_set_pn_threshold *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_pn_threshold_set(mcs, req); + + return 0; +} + +int rvu_mbox_handler_mcs_rx_sc_sa_map_write(struct rvu *rvu, + struct mcs_rx_sc_sa_map *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs->mcs_ops->mcs_rx_sa_mem_map_write(mcs, req); + return 0; +} + +int rvu_mbox_handler_mcs_tx_sc_sa_map_write(struct rvu *rvu, + struct mcs_tx_sc_sa_map *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs->mcs_ops->mcs_tx_sa_mem_map_write(mcs, req); + mcs->tx_sa_active[req->sc_id] = req->tx_sa_active; + + return 0; +} + +int rvu_mbox_handler_mcs_sa_plcy_write(struct rvu *rvu, + struct mcs_sa_plcy_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int i; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + for (i = 0; i < req->sa_cnt; i++) + mcs_sa_plcy_write(mcs, &req->plcy[i][0], + req->sa_index[i], req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_rx_sc_cam_write(struct rvu *rvu, + struct mcs_rx_sc_cam_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_rx_sc_cam_write(mcs, req->sci, req->secy_id, req->sc_id); + return 0; +} + +int rvu_mbox_handler_mcs_secy_plcy_write(struct rvu *rvu, + struct mcs_secy_plcy_write_req *req, + struct msg_rsp *rsp) +{ struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_secy_plcy_write(mcs, req->plcy, + req->secy_id, req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_flowid_entry_write(struct rvu *rvu, + struct mcs_flowid_entry_write_req *req, + struct msg_rsp *rsp) +{ + struct secy_mem_map map; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* TODO validate the flowid */ + mcs_flowid_entry_write(mcs, req->data, req->mask, + req->flow_id, req->dir); + map.secy = req->secy_id; + map.sc = req->sc_id; + map.ctrl_pkt = req->ctrl_pkt; + map.flow_id = req->flow_id; + map.sci = req->sci; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, req->dir); + if (req->ena) + mcs_ena_dis_flowid_entry(mcs, req->flow_id, + req->dir, true); + return 0; +} + +int rvu_mbox_handler_mcs_free_resources(struct rvu *rvu, + struct mcs_free_rsrc_req *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (req->dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + /* Free all the cam resources mapped to PF/VF */ + if (req->all) { + rc = mcs_free_all_rsrc(mcs, req->dir, pcifunc); + goto exit; + } + + switch (req->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + rc = mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, req->rsrc_id, pcifunc); + mcs_ena_dis_flowid_entry(mcs, req->rsrc_id, req->dir, false); + break; + case MCS_RSRC_TYPE_SECY: + rc = mcs_free_rsrc(&map->secy, map->secy2pf_map, req->rsrc_id, pcifunc); + mcs_clear_secy_plcy(mcs, req->rsrc_id, req->dir); + break; + case MCS_RSRC_TYPE_SC: + rc = mcs_free_rsrc(&map->sc, map->sc2pf_map, req->rsrc_id, pcifunc); + /* Disable SC CAM only on RX side */ + if (req->dir == MCS_RX) + mcs_ena_dis_sc_cam_entry(mcs, req->rsrc_id, false); + break; + case MCS_RSRC_TYPE_SA: + rc = mcs_free_rsrc(&map->sa, map->sa2pf_map, req->rsrc_id, pcifunc); + break; + } +exit: + mutex_unlock(&rvu->rsrc_lock); + return rc; +} + +int rvu_mbox_handler_mcs_alloc_resources(struct rvu *rvu, + struct mcs_alloc_rsrc_req *req, + struct mcs_alloc_rsrc_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rsrc_id, i; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (req->dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + + if (req->all) { + rsrc_id = mcs_alloc_all_rsrc(mcs, &rsp->flow_ids[0], + &rsp->secy_ids[0], + &rsp->sc_ids[0], + &rsp->sa_ids[0], + &rsp->sa_ids[1], + pcifunc, req->dir); + goto exit; + } + + switch (req->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->flow_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SECY: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->secy_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SC: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->sc_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SA: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->sa_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + } + + rsp->rsrc_type = req->rsrc_type; + rsp->dir = req->dir; + rsp->mcs_id = req->mcs_id; + rsp->all = req->all; + +exit: + if (rsrc_id < 0) + dev_err(rvu->dev, "Failed to allocate the mcs resources for PCIFUNC:%d\n", pcifunc); + mutex_unlock(&rvu->rsrc_lock); + return 0; +} + +int rvu_mbox_handler_mcs_alloc_ctrl_pkt_rule(struct rvu *rvu, + struct mcs_alloc_ctrl_pkt_rule_req *req, + struct mcs_alloc_ctrl_pkt_rule_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rsrc_id; + u16 offset; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + + switch (req->rule_type) { + case MCS_CTRL_PKT_RULE_TYPE_ETH: + offset = MCS_CTRLPKT_ETYPE_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_DA: + offset = MCS_CTRLPKT_DA_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_RANGE: + offset = MCS_CTRLPKT_DA_RANGE_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_COMBO: + offset = MCS_CTRLPKT_COMBO_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_MAC: + offset = MCS_CTRLPKT_MAC_EN_RULE_OFFSET; + break; + } + + rsrc_id = mcs_alloc_ctrlpktrule(&map->ctrlpktrule, map->ctrlpktrule2pf_map, offset, + pcifunc); + if (rsrc_id < 0) + goto exit; + + rsp->rule_idx = rsrc_id; + rsp->rule_type = req->rule_type; + rsp->dir = req->dir; + rsp->mcs_id = req->mcs_id; + + mutex_unlock(&rvu->rsrc_lock); + return 0; +exit: + if (rsrc_id < 0) + dev_err(rvu->dev, "Failed to allocate the mcs ctrl pkt rule for PCIFUNC:%d\n", + pcifunc); + mutex_unlock(&rvu->rsrc_lock); + return rsrc_id; +} + +int rvu_mbox_handler_mcs_free_ctrl_pkt_rule(struct rvu *rvu, + struct mcs_free_ctrl_pkt_rule_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mutex_lock(&rvu->rsrc_lock); + + rc = mcs_free_ctrlpktrule(mcs, req); + + mutex_unlock(&rvu->rsrc_lock); + + return rc; +} + +int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu, + struct mcs_ctrl_pkt_rule_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + rc = mcs_ctrlpktrule_write(mcs, req); + + return rc; +} + +static void rvu_mcs_set_lmac_bmap(struct rvu *rvu) +{ + struct mcs *mcs = mcs_get_pdata(0); + unsigned long lmac_bmap; + int cgx, lmac, port; + + for (port = 0; port < mcs->hw->lmac_cnt; port++) { + cgx = port / rvu->hw->lmac_per_cgx; + lmac = port % rvu->hw->lmac_per_cgx; + if (!is_lmac_valid(rvu_cgx_pdata(cgx, rvu), lmac)) + continue; + set_bit(port, &lmac_bmap); + } + mcs->hw->lmac_bmap = lmac_bmap; +} + +int rvu_mcs_init(struct rvu *rvu) +{ + struct rvu_hwinfo *hw = rvu->hw; + int lmac, err = 0, mcs_id; + struct mcs *mcs; + + rvu->mcs_blk_cnt = mcs_get_blkcnt(); + + if (!rvu->mcs_blk_cnt) + return 0; + + /* Needed only for CN10K-B */ + if (rvu->mcs_blk_cnt == 1) { + err = mcs_set_lmac_channels(0, hw->cgx_chan_base); + if (err) + return err; + /* Set active lmacs */ + rvu_mcs_set_lmac_bmap(rvu); + } + + /* Install default tcam bypass entry and set port to operational mode */ + for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) { + mcs = mcs_get_pdata(mcs_id); + mcs_install_flowid_bypass_entry(mcs); + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) + mcs_set_lmac_mode(mcs, lmac, 0); + + mcs->rvu = rvu; + + /* Allocated memory for PFVF data */ + mcs->pf = devm_kcalloc(mcs->dev, hw->total_pfs, + sizeof(struct mcs_pfvf), GFP_KERNEL); + if (!mcs->pf) + return -ENOMEM; + + mcs->vf = devm_kcalloc(mcs->dev, hw->total_vfs, + sizeof(struct mcs_pfvf), GFP_KERNEL); + if (!mcs->vf) + return -ENOMEM; + } + + /* Initialize the wq for handling mcs interrupts */ + INIT_LIST_HEAD(&rvu->mcs_intrq_head); + INIT_WORK(&rvu->mcs_intr_work, mcs_intr_handler_task); + rvu->mcs_intr_wq = alloc_workqueue("mcs_intr_wq", 0, 0); + if (!rvu->mcs_intr_wq) { + dev_err(rvu->dev, "mcs alloc workqueue failed\n"); + return -ENOMEM; + } + + return err; +} + +void rvu_mcs_exit(struct rvu *rvu) +{ + if (!rvu->mcs_intr_wq) + return; + + flush_workqueue(rvu->mcs_intr_wq); + destroy_workqueue(rvu->mcs_intr_wq); + rvu->mcs_intr_wq = NULL; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c index 67a6821d2dff..3411e2e47d46 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c @@ -9,6 +9,8 @@ #include <linux/device.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/hrtimer.h> +#include <linux/ktime.h> #include "ptp.h" #include "mbox.h" @@ -50,12 +52,23 @@ #define PTP_CLOCK_COMP 0xF18ULL #define PTP_TIMESTAMP 0xF20ULL #define PTP_CLOCK_SEC 0xFD0ULL +#define PTP_SEC_ROLLOVER 0xFD8ULL #define CYCLE_MULT 1000 static struct ptp *first_ptp_block; static const struct pci_device_id ptp_id_table[]; +static bool is_ptp_dev_cnf10kb(struct ptp *ptp) +{ + return (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP) ? true : false; +} + +static bool is_ptp_dev_cn10k(struct ptp *ptp) +{ + return (ptp->pdev->device == PCI_DEVID_CN10K_PTP) ? true : false; +} + static bool cn10k_ptp_errata(struct ptp *ptp) { if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP || @@ -72,6 +85,43 @@ static bool is_ptp_tsfmt_sec_nsec(struct ptp *ptp) return false; } +static enum hrtimer_restart ptp_reset_thresh(struct hrtimer *hrtimer) +{ + struct ptp *ptp = container_of(hrtimer, struct ptp, hrtimer); + ktime_t curr_ts = ktime_get(); + ktime_t delta_ns, period_ns; + u64 ptp_clock_hi; + + /* calculate the elapsed time since last restart */ + delta_ns = ktime_to_ns(ktime_sub(curr_ts, ptp->last_ts)); + + /* if the ptp clock value has crossed 0.5 seconds, + * its too late to update pps threshold value, so + * update threshold after 1 second. + */ + ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI); + if (ptp_clock_hi > 500000000) { + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - ptp_clock_hi)); + } else { + writeq(500000000, ptp->reg_base + PTP_PPS_THRESH_HI); + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - delta_ns)); + } + + hrtimer_forward_now(hrtimer, period_ns); + ptp->last_ts = curr_ts; + + return HRTIMER_RESTART; +} + +static void ptp_hrtimer_start(struct ptp *ptp, ktime_t start_ns) +{ + ktime_t period_ns; + + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - start_ns)); + hrtimer_start(&ptp->hrtimer, period_ns, HRTIMER_MODE_REL); + ptp->last_ts = ktime_get(); +} + static u64 read_ptp_tstmp_sec_nsec(struct ptp *ptp) { u64 sec, sec1, nsec; @@ -246,6 +296,10 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) /* sclk is in MHz */ ptp->clock_rate = sclk * 1000000; + /* Program the seconds rollover value to 1 second */ + if (is_ptp_dev_cnf10kb(ptp)) + writeq(0x3b9aca00, ptp->reg_base + PTP_SEC_ROLLOVER); + /* Enable PTP clock */ clock_cfg = readq(ptp->reg_base + PTP_CLOCK_CFG); @@ -270,6 +324,18 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) /* Set 50% duty cycle for 1Hz output */ writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_HI_INCR); writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_LO_INCR); + if (cn10k_ptp_errata(ptp)) { + /* The ptp_clock_hi rollsover to zero once clock cycle before it + * reaches one second boundary. so, program the pps_lo_incr in + * such a way that the pps threshold value comparison at one + * second boundary will succeed and pps edge changes. After each + * one second boundary, the hrtimer handler will be invoked and + * reprograms the pps threshold value. + */ + ptp->clock_period = NSEC_PER_SEC / ptp->clock_rate; + writeq((0x1dcd6500ULL - ptp->clock_period) << 32, + ptp->reg_base + PTP_PPS_LO_INCR); + } if (cn10k_ptp_errata(ptp)) clock_comp = ptp_calc_adjusted_comp(ptp->clock_rate); @@ -282,14 +348,39 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) static int ptp_get_tstmp(struct ptp *ptp, u64 *clk) { - *clk = readq(ptp->reg_base + PTP_TIMESTAMP); + u64 timestamp; + + if (is_ptp_dev_cn10k(ptp)) { + timestamp = readq(ptp->reg_base + PTP_TIMESTAMP); + *clk = (timestamp >> 32) * NSEC_PER_SEC + (timestamp & 0xFFFFFFFF); + } else { + *clk = readq(ptp->reg_base + PTP_TIMESTAMP); + } return 0; } static int ptp_set_thresh(struct ptp *ptp, u64 thresh) { - writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI); + if (!cn10k_ptp_errata(ptp)) + writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI); + + return 0; +} + +static int ptp_extts_on(struct ptp *ptp, int on) +{ + u64 ptp_clock_hi; + + if (cn10k_ptp_errata(ptp)) { + if (on) { + ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI); + ptp_hrtimer_start(ptp, (ktime_t)ptp_clock_hi); + } else { + if (hrtimer_active(&ptp->hrtimer)) + hrtimer_cancel(&ptp->hrtimer); + } + } return 0; } @@ -329,6 +420,11 @@ static int ptp_probe(struct pci_dev *pdev, else ptp->read_ptp_tstmp = &read_ptp_tstmp_nsec; + if (cn10k_ptp_errata(ptp)) { + hrtimer_init(&ptp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + ptp->hrtimer.function = ptp_reset_thresh; + } + return 0; error_free: @@ -353,6 +449,9 @@ static void ptp_remove(struct pci_dev *pdev) struct ptp *ptp = pci_get_drvdata(pdev); u64 clock_cfg; + if (cn10k_ptp_errata(ptp) && hrtimer_active(&ptp->hrtimer)) + hrtimer_cancel(&ptp->hrtimer); + if (IS_ERR_OR_NULL(ptp)) return; @@ -420,6 +519,9 @@ int rvu_mbox_handler_ptp_op(struct rvu *rvu, struct ptp_req *req, case PTP_OP_SET_THRESH: err = ptp_set_thresh(rvu->ptp, req->thresh); break; + case PTP_OP_EXTTS_ON: + err = ptp_extts_on(rvu->ptp, req->extts_on); + break; default: err = -EINVAL; break; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h index 95a955159f40..b9d92abc3844 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h @@ -17,7 +17,10 @@ struct ptp { void __iomem *reg_base; u64 (*read_ptp_tstmp)(struct ptp *ptp); spinlock_t ptp_lock; /* lock */ + struct hrtimer hrtimer; + ktime_t last_ts; u32 clock_rate; + u32 clock_period; }; struct ptp *ptp_get(void); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c index ef59de43b11e..a70e1153fa04 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c @@ -415,11 +415,26 @@ void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable) return; cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_CFG); - if (enable) + if (enable) { cfg |= RPMX_RX_TS_PREPEND; - else + cfg |= RPMX_TX_PTP_1S_SUPPORT; + } else { cfg &= ~RPMX_RX_TS_PREPEND; + cfg &= ~RPMX_TX_PTP_1S_SUPPORT; + } + rpm_write(rpm, lmac_id, RPMX_CMRX_CFG, cfg); + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE); + + if (enable) { + cfg |= RPMX_ONESTEP_ENABLE; + cfg &= ~RPMX_TS_BINARY_MODE; + } else { + cfg &= ~RPMX_ONESTEP_ENABLE; + } + + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE, cfg); } int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 pfc_en) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h index c2bd6e54ea51..77f2ef9e1425 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h @@ -16,6 +16,7 @@ /* Registers */ #define RPMX_CMRX_CFG 0x00 #define RPMX_RX_TS_PREPEND BIT_ULL(22) +#define RPMX_TX_PTP_1S_SUPPORT BIT_ULL(17) #define RPMX_CMRX_SW_INT 0x180 #define RPMX_CMRX_SW_INT_W1S 0x188 #define RPMX_CMRX_SW_INT_ENA_W1S 0x198 @@ -72,6 +73,10 @@ #define RPMX_MTI_MAC100X_CL89_PAUSE_QUANTA 0x8108 #define RPM_DEFAULT_PAUSE_TIME 0x7FF +#define RPMX_MTI_MAC100X_XIF_MODE 0x8100 +#define RPMX_ONESTEP_ENABLE BIT_ULL(5) +#define RPMX_TS_BINARY_MODE BIT_ULL(11) + /* Function Declarations */ int rpm_get_nr_lmacs(void *rpmd); u8 rpm_get_lmac_type(void *rpmd, int lmac_id); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 7282a826d81e..3f5e09b77d4b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -16,6 +16,7 @@ #include "rvu.h" #include "rvu_reg.h" #include "ptp.h" +#include "mcs.h" #include "rvu_trace.h" #include "rvu_npc_hash.h" @@ -23,8 +24,6 @@ #define DRV_NAME "rvu_af" #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver" -static int rvu_get_hwvf(struct rvu *rvu, int pcifunc); - static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, struct rvu_block *block, int lf); static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, @@ -418,7 +417,7 @@ void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf) *hwvf = cfg & 0xFFF; } -static int rvu_get_hwvf(struct rvu *rvu, int pcifunc) +int rvu_get_hwvf(struct rvu *rvu, int pcifunc) { int pf, func; u64 cfg; @@ -1159,6 +1158,12 @@ cpt: rvu_program_channels(rvu); + err = rvu_mcs_init(rvu); + if (err) { + dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__); + goto nix_err; + } + return 0; nix_err: @@ -3293,6 +3298,7 @@ err_mbox: err_hwsetup: rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); + rvu_mcs_exit(rvu); rvu_reset_all_blocks(rvu); rvu_free_hw_resources(rvu); rvu_clear_rvum_blk_revid(rvu); @@ -3319,6 +3325,7 @@ static void rvu_remove(struct pci_dev *pdev) rvu_flr_wq_destroy(rvu); rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); + rvu_mcs_exit(rvu); rvu_mbox_destroy(&rvu->afpf_wq_info); rvu_disable_sriov(rvu); rvu_reset_all_blocks(rvu); @@ -3354,12 +3361,18 @@ static int __init rvu_init_module(void) if (err < 0) goto ptp_err; + err = pci_register_driver(&mcs_driver); + if (err < 0) + goto mcs_err; + err = pci_register_driver(&rvu_driver); if (err < 0) goto rvu_err; return 0; rvu_err: + pci_unregister_driver(&mcs_driver); +mcs_err: pci_unregister_driver(&ptp_driver); ptp_err: pci_unregister_driver(&cgx_driver); @@ -3370,6 +3383,7 @@ ptp_err: static void __exit rvu_cleanup_module(void) { pci_unregister_driver(&rvu_driver); + pci_unregister_driver(&mcs_driver); pci_unregister_driver(&ptp_driver); pci_unregister_driver(&cgx_driver); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index d15bc443335d..76474385a602 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -25,6 +25,8 @@ /* Subsystem Device ID */ #define PCI_SUBSYS_DEVID_96XX 0xB200 #define PCI_SUBSYS_DEVID_CN10K_A 0xB900 +#define PCI_SUBSYS_DEVID_CNF10K_B 0xBC00 +#define PCI_SUBSYS_DEVID_CN10K_B 0xBD00 /* PCI BAR nos */ #define PCI_AF_REG_BAR_NUM 0 @@ -62,6 +64,10 @@ struct rvu_debugfs { struct dentry *nix; struct dentry *npc; struct dentry *cpt; + struct dentry *mcs_root; + struct dentry *mcs; + struct dentry *mcs_rx; + struct dentry *mcs_tx; struct dump_ctx npa_aura_ctx; struct dump_ctx npa_pool_ctx; struct dump_ctx nix_cq_ctx; @@ -497,6 +503,8 @@ struct rvu { struct ptp *ptp; + int mcs_blk_cnt; + #ifdef CONFIG_DEBUG_FS struct rvu_debugfs rvu_dbg; #endif @@ -504,6 +512,12 @@ struct rvu { /* RVU switch implementation over NPC with DMAC rules */ struct rvu_switch rswitch; + + struct work_struct mcs_intr_work; + struct workqueue_struct *mcs_intr_wq; + struct list_head mcs_intrq_head; + /* mcs interrupt queue lock */ + spinlock_t mcs_intrq_lock; }; static inline void rvu_write64(struct rvu *rvu, u64 block, u64 offset, u64 val) @@ -868,4 +882,11 @@ void rvu_switch_update_rules(struct rvu *rvu, u16 pcifunc); int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, u64 pkind, u8 var_len_off, u8 var_len_off_mask, u8 shift_dir); +int rvu_get_hwvf(struct rvu *rvu, int pcifunc); + +/* CN10K MCS */ +int rvu_mcs_init(struct rvu *rvu); +int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc); +void rvu_mcs_exit(struct rvu *rvu); + #endif /* RVU_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index f42a09f04b25..a1970ebedf95 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -19,6 +19,7 @@ #include "lmac_common.h" #include "npc.h" #include "rvu_npc_hash.h" +#include "mcs.h" #define DEBUGFS_DIR_NAME "octeontx2" @@ -227,6 +228,350 @@ static const struct file_operations rvu_dbg_##name##_fops = { \ static void print_nix_qsize(struct seq_file *filp, struct rvu_pfvf *pfvf); +static int rvu_dbg_mcs_port_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_port_stats stats; + int lmac; + + seq_puts(filp, "\n port stats\n"); + mutex_lock(&mcs->stats_lock); + for_each_set_bit(lmac, &mcs->hw->lmac_bmap, mcs->hw->lmac_cnt) { + mcs_get_port_stats(mcs, &stats, lmac, dir); + seq_printf(filp, "port%d: Tcam Miss: %lld\n", lmac, stats.tcam_miss_cnt); + seq_printf(filp, "port%d: Parser errors: %lld\n", lmac, stats.parser_err_cnt); + + if (dir == MCS_RX && mcs->hw->mcs_blks > 1) + seq_printf(filp, "port%d: Preempt error: %lld\n", lmac, + stats.preempt_err_cnt); + if (dir == MCS_TX) + seq_printf(filp, "port%d: Sectag insert error: %lld\n", lmac, + stats.sectag_insert_err_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_rx_port_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_port_stats, mcs_rx_port_stats_display, NULL); + +static int rvu_dbg_mcs_tx_port_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_port_stats, mcs_tx_port_stats_display, NULL); + +static int rvu_dbg_mcs_sa_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_sa_stats stats; + struct rsrc_bmap *map; + int sa_id; + + if (dir == MCS_TX) { + map = &mcs->tx.sa; + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) { + seq_puts(filp, "\n TX SA stats\n"); + mcs_get_sa_stats(mcs, &stats, sa_id, MCS_TX); + seq_printf(filp, "sa%d: Pkts encrypted: %lld\n", sa_id, + stats.pkt_encrypt_cnt); + + seq_printf(filp, "sa%d: Pkts protected: %lld\n", sa_id, + stats.pkt_protected_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; + } + + /* RX stats */ + map = &mcs->rx.sa; + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) { + seq_puts(filp, "\n RX SA stats\n"); + mcs_get_sa_stats(mcs, &stats, sa_id, MCS_RX); + seq_printf(filp, "sa%d: Invalid pkts: %lld\n", sa_id, stats.pkt_invalid_cnt); + seq_printf(filp, "sa%d: Pkts no sa error: %lld\n", sa_id, stats.pkt_nosaerror_cnt); + seq_printf(filp, "sa%d: Pkts not valid: %lld\n", sa_id, stats.pkt_notvalid_cnt); + seq_printf(filp, "sa%d: Pkts ok: %lld\n", sa_id, stats.pkt_ok_cnt); + seq_printf(filp, "sa%d: Pkts no sa: %lld\n", sa_id, stats.pkt_nosa_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_rx_sa_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_sa_stats, mcs_rx_sa_stats_display, NULL); + +static int rvu_dbg_mcs_tx_sa_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_sa_stats, mcs_tx_sa_stats_display, NULL); + +static int rvu_dbg_mcs_tx_sc_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_sc_stats stats; + struct rsrc_bmap *map; + int sc_id; + + map = &mcs->tx.sc; + seq_puts(filp, "\n SC stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) { + mcs_get_sc_stats(mcs, &stats, sc_id, MCS_TX); + seq_printf(filp, "\n=======sc%d======\n\n", sc_id); + seq_printf(filp, "sc%d: Pkts encrypted: %lld\n", sc_id, stats.pkt_encrypt_cnt); + seq_printf(filp, "sc%d: Pkts protected: %lld\n", sc_id, stats.pkt_protected_cnt); + + if (mcs->hw->mcs_blks == 1) { + seq_printf(filp, "sc%d: Octets encrypted: %lld\n", sc_id, + stats.octet_encrypt_cnt); + seq_printf(filp, "sc%d: Octets protected: %lld\n", sc_id, + stats.octet_protected_cnt); + } + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_sc_stats, mcs_tx_sc_stats_display, NULL); + +static int rvu_dbg_mcs_rx_sc_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_sc_stats stats; + struct rsrc_bmap *map; + int sc_id; + + map = &mcs->rx.sc; + seq_puts(filp, "\n SC stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) { + mcs_get_sc_stats(mcs, &stats, sc_id, MCS_RX); + seq_printf(filp, "\n=======sc%d======\n\n", sc_id); + seq_printf(filp, "sc%d: Cam hits: %lld\n", sc_id, stats.hit_cnt); + seq_printf(filp, "sc%d: Invalid pkts: %lld\n", sc_id, stats.pkt_invalid_cnt); + seq_printf(filp, "sc%d: Late pkts: %lld\n", sc_id, stats.pkt_late_cnt); + seq_printf(filp, "sc%d: Notvalid pkts: %lld\n", sc_id, stats.pkt_notvalid_cnt); + seq_printf(filp, "sc%d: Unchecked pkts: %lld\n", sc_id, stats.pkt_unchecked_cnt); + + if (mcs->hw->mcs_blks > 1) { + seq_printf(filp, "sc%d: Delay pkts: %lld\n", sc_id, stats.pkt_delay_cnt); + seq_printf(filp, "sc%d: Pkts ok: %lld\n", sc_id, stats.pkt_ok_cnt); + } + if (mcs->hw->mcs_blks == 1) { + seq_printf(filp, "sc%d: Octets decrypted: %lld\n", sc_id, + stats.octet_decrypt_cnt); + seq_printf(filp, "sc%d: Octets validated: %lld\n", sc_id, + stats.octet_validate_cnt); + } + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_sc_stats, mcs_rx_sc_stats_display, NULL); + +static int rvu_dbg_mcs_flowid_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_flowid_stats stats; + struct rsrc_bmap *map; + int flow_id; + + seq_puts(filp, "\n Flowid stats\n"); + + if (dir == MCS_RX) + map = &mcs->rx.flow_ids; + else + map = &mcs->tx.flow_ids; + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(flow_id, map->bmap, mcs->hw->tcam_entries) { + mcs_get_flowid_stats(mcs, &stats, flow_id, dir); + seq_printf(filp, "Flowid%d: Hit:%lld\n", flow_id, stats.tcam_hit_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_tx_flowid_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_flowid_stats, mcs_tx_flowid_stats_display, NULL); + +static int rvu_dbg_mcs_rx_flowid_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_flowid_stats, mcs_rx_flowid_stats_display, NULL); + +static int rvu_dbg_mcs_tx_secy_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_secy_stats stats; + struct rsrc_bmap *map; + int secy_id; + + map = &mcs->tx.secy; + seq_puts(filp, "\n MCS TX secy stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) { + mcs_get_tx_secy_stats(mcs, &stats, secy_id); + seq_printf(filp, "\n=======Secy%d======\n\n", secy_id); + seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id, + stats.ctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id, + stats.ctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id, + stats.ctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt); + seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id, + stats.unctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id, + stats.unctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id, + stats.unctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt); + seq_printf(filp, "secy%d: Octet encrypted: %lld\n", secy_id, + stats.octet_encrypted_cnt); + seq_printf(filp, "secy%d: octet protected: %lld\n", secy_id, + stats.octet_protected_cnt); + seq_printf(filp, "secy%d: Pkts on active sa: %lld\n", secy_id, + stats.pkt_noactivesa_cnt); + seq_printf(filp, "secy%d: Pkts too long: %lld\n", secy_id, stats.pkt_toolong_cnt); + seq_printf(filp, "secy%d: Pkts untagged: %lld\n", secy_id, stats.pkt_untagged_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_secy_stats, mcs_tx_secy_stats_display, NULL); + +static int rvu_dbg_mcs_rx_secy_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_secy_stats stats; + struct rsrc_bmap *map; + int secy_id; + + map = &mcs->rx.secy; + seq_puts(filp, "\n MCS secy stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) { + mcs_get_rx_secy_stats(mcs, &stats, secy_id); + seq_printf(filp, "\n=======Secy%d======\n\n", secy_id); + seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id, + stats.ctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id, + stats.ctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id, + stats.ctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt); + seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id, + stats.unctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id, + stats.unctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id, + stats.unctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt); + seq_printf(filp, "secy%d: Octet decrypted: %lld\n", secy_id, + stats.octet_decrypted_cnt); + seq_printf(filp, "secy%d: octet validated: %lld\n", secy_id, + stats.octet_validated_cnt); + seq_printf(filp, "secy%d: Pkts on disable port: %lld\n", secy_id, + stats.pkt_port_disabled_cnt); + seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_badtag_cnt); + seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_nosa_cnt); + seq_printf(filp, "secy%d: Pkts with nosaerror: %lld\n", secy_id, + stats.pkt_nosaerror_cnt); + seq_printf(filp, "secy%d: Tagged ctrl pkts: %lld\n", secy_id, + stats.pkt_tagged_ctl_cnt); + seq_printf(filp, "secy%d: Untaged pkts: %lld\n", secy_id, stats.pkt_untaged_cnt); + seq_printf(filp, "secy%d: Ctrl pkts: %lld\n", secy_id, stats.pkt_ctl_cnt); + if (mcs->hw->mcs_blks > 1) + seq_printf(filp, "secy%d: pkts notag: %lld\n", secy_id, + stats.pkt_notag_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_secy_stats, mcs_rx_secy_stats_display, NULL); + +static void rvu_dbg_mcs_init(struct rvu *rvu) +{ + struct mcs *mcs; + char dname[10]; + int i; + + if (!rvu->mcs_blk_cnt) + return; + + rvu->rvu_dbg.mcs_root = debugfs_create_dir("mcs", rvu->rvu_dbg.root); + + for (i = 0; i < rvu->mcs_blk_cnt; i++) { + mcs = mcs_get_pdata(i); + + sprintf(dname, "mcs%d", i); + rvu->rvu_dbg.mcs = debugfs_create_dir(dname, + rvu->rvu_dbg.mcs_root); + + rvu->rvu_dbg.mcs_rx = debugfs_create_dir("rx_stats", rvu->rvu_dbg.mcs); + + debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_flowid_stats_fops); + + debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_secy_stats_fops); + + debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_sc_stats_fops); + + debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_sa_stats_fops); + + debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_port_stats_fops); + + rvu->rvu_dbg.mcs_tx = debugfs_create_dir("tx_stats", rvu->rvu_dbg.mcs); + + debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_flowid_stats_fops); + + debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_secy_stats_fops); + + debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_sc_stats_fops); + + debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_sa_stats_fops); + + debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_port_stats_fops); + } +} + #define LMT_MAPTBL_ENTRY_SIZE 16 /* Dump LMTST map table */ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, @@ -3053,6 +3398,7 @@ create: rvu_dbg_npc_init(rvu); rvu_dbg_cpt_init(rvu, BLKADDR_CPT0); rvu_dbg_cpt_init(rvu, BLKADDR_CPT1); + rvu_dbg_mcs_init(rvu); } void rvu_dbg_exit(struct rvu *rvu) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 0879a48411f3..7646bb2ec89b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -4296,8 +4296,14 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) /* Restore CINT timer delay to HW reset values */ rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL); + cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG); + /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */ - rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, 0x1ULL); + cfg |= 1ULL; + if (!is_rvu_otx2(rvu)) + cfg |= NIX_PTP_1STEP_EN; + + rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg); if (is_block_implemented(hw, blkaddr)) { err = nix_setup_txschq(rvu, nix_hw, blkaddr); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h index 77a9ade91f3e..0e0d536645ac 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h @@ -266,6 +266,7 @@ #define NIX_AF_TX_NPC_CAPTURE_CONFIG (0x0660) #define NIX_AF_TX_NPC_CAPTURE_INFO (0x0670) #define NIX_AF_SEB_CFG (0x05F0) +#define NIX_PTP_1STEP_EN BIT_ULL(2) #define NIX_AF_DEBUG_NPC_RESP_DATAX(a) (0x680 | (a) << 3) #define NIX_AF_SMQX_CFG(a) (0x700 | (a) << 16) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile index d463dc72d80a..73fdb8798614 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile @@ -13,5 +13,6 @@ rvu_nicvf-y := otx2_vf.o otx2_devlink.o rvu_nicpf-$(CONFIG_DCB) += otx2_dcbnl.o rvu_nicvf-$(CONFIG_DCB) += otx2_dcbnl.o +rvu_nicpf-$(CONFIG_MACSEC) += cn10k_macsec.o ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c index fd4f083c699e..826f691de259 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -86,8 +86,7 @@ int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ aq->sq.cq_ena = 1; aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); aq->sq.default_chan = pfvf->hw.tx_chan_base; aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c new file mode 100644 index 000000000000..9ec5f38d38a8 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -0,0 +1,1671 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MACSEC hardware offload driver + * + * Copyright (C) 2022 Marvell. + */ + +#include <linux/rtnetlink.h> +#include <linux/bitfield.h> +#include <net/macsec.h> +#include "otx2_common.h" + +#define MCS_TCAM0_MAC_SA_MASK GENMASK_ULL(63, 48) +#define MCS_TCAM1_MAC_SA_MASK GENMASK_ULL(31, 0) +#define MCS_TCAM1_ETYPE_MASK GENMASK_ULL(47, 32) + +#define MCS_SA_MAP_MEM_SA_USE BIT_ULL(9) + +#define MCS_RX_SECY_PLCY_RW_MASK GENMASK_ULL(49, 18) +#define MCS_RX_SECY_PLCY_RP BIT_ULL(17) +#define MCS_RX_SECY_PLCY_AUTH_ENA BIT_ULL(16) +#define MCS_RX_SECY_PLCY_CIP GENMASK_ULL(8, 5) +#define MCS_RX_SECY_PLCY_VAL GENMASK_ULL(2, 1) +#define MCS_RX_SECY_PLCY_ENA BIT_ULL(0) + +#define MCS_TX_SECY_PLCY_MTU GENMASK_ULL(43, 28) +#define MCS_TX_SECY_PLCY_ST_TCI GENMASK_ULL(27, 22) +#define MCS_TX_SECY_PLCY_ST_OFFSET GENMASK_ULL(21, 15) +#define MCS_TX_SECY_PLCY_INS_MODE BIT_ULL(14) +#define MCS_TX_SECY_PLCY_AUTH_ENA BIT_ULL(13) +#define MCS_TX_SECY_PLCY_CIP GENMASK_ULL(5, 2) +#define MCS_TX_SECY_PLCY_PROTECT BIT_ULL(1) +#define MCS_TX_SECY_PLCY_ENA BIT_ULL(0) + +#define MCS_GCM_AES_128 0 +#define MCS_GCM_AES_256 1 +#define MCS_GCM_AES_XPN_128 2 +#define MCS_GCM_AES_XPN_256 3 + +#define MCS_TCI_ES 0x40 /* end station */ +#define MCS_TCI_SC 0x20 /* SCI present */ +#define MCS_TCI_SCB 0x10 /* epon */ +#define MCS_TCI_E 0x08 /* encryption */ +#define MCS_TCI_C 0x04 /* changed text */ + +static struct cn10k_mcs_txsc *cn10k_mcs_get_txsc(struct cn10k_mcs_cfg *cfg, + struct macsec_secy *secy) +{ + struct cn10k_mcs_txsc *txsc; + + list_for_each_entry(txsc, &cfg->txsc_list, entry) { + if (txsc->sw_secy == secy) + return txsc; + } + + return NULL; +} + +static struct cn10k_mcs_rxsc *cn10k_mcs_get_rxsc(struct cn10k_mcs_cfg *cfg, + struct macsec_secy *secy, + struct macsec_rx_sc *rx_sc) +{ + struct cn10k_mcs_rxsc *rxsc; + + list_for_each_entry(rxsc, &cfg->rxsc_list, entry) { + if (rxsc->sw_rxsc == rx_sc && rxsc->sw_secy == secy) + return rxsc; + } + + return NULL; +} + +static const char *rsrc_name(enum mcs_rsrc_type rsrc_type) +{ + switch (rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + return "FLOW"; + case MCS_RSRC_TYPE_SC: + return "SC"; + case MCS_RSRC_TYPE_SECY: + return "SECY"; + case MCS_RSRC_TYPE_SA: + return "SA"; + default: + return "Unknown"; + }; + + return "Unknown"; +} + +static int cn10k_mcs_alloc_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, + enum mcs_rsrc_type type, u16 *rsrc_id) +{ + struct mbox *mbox = &pfvf->mbox; + struct mcs_alloc_rsrc_req *req; + struct mcs_alloc_rsrc_rsp *rsp; + int ret = -ENOMEM; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_alloc_resources(mbox); + if (!req) + goto fail; + + req->rsrc_type = type; + req->rsrc_cnt = 1; + req->dir = dir; + + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_alloc_rsrc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp) || req->rsrc_cnt != rsp->rsrc_cnt || + req->rsrc_type != rsp->rsrc_type || req->dir != rsp->dir) { + ret = -EINVAL; + goto fail; + } + + switch (rsp->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + *rsrc_id = rsp->flow_ids[0]; + break; + case MCS_RSRC_TYPE_SC: + *rsrc_id = rsp->sc_ids[0]; + break; + case MCS_RSRC_TYPE_SECY: + *rsrc_id = rsp->secy_ids[0]; + break; + case MCS_RSRC_TYPE_SA: + *rsrc_id = rsp->sa_ids[0]; + break; + default: + ret = -EINVAL; + goto fail; + } + + mutex_unlock(&mbox->lock); + + return 0; +fail: + dev_err(pfvf->dev, "Failed to allocate %s %s resource\n", + dir == MCS_TX ? "TX" : "RX", rsrc_name(type)); + mutex_unlock(&mbox->lock); + return ret; +} + +static void cn10k_mcs_free_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, + enum mcs_rsrc_type type, u16 hw_rsrc_id, + bool all) +{ + struct mbox *mbox = &pfvf->mbox; + struct mcs_free_rsrc_req *req; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_free_resources(mbox); + if (!req) + goto fail; + + req->rsrc_id = hw_rsrc_id; + req->rsrc_type = type; + req->dir = dir; + if (all) + req->all = 1; + + if (otx2_sync_mbox_msg(&pfvf->mbox)) + goto fail; + + mutex_unlock(&mbox->lock); + + return; +fail: + dev_err(pfvf->dev, "Failed to free %s %s resource\n", + dir == MCS_TX ? "TX" : "RX", rsrc_name(type)); + mutex_unlock(&mbox->lock); +} + +static int cn10k_mcs_alloc_txsa(struct otx2_nic *pfvf, u16 *hw_sa_id) +{ + return cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id); +} + +static int cn10k_mcs_alloc_rxsa(struct otx2_nic *pfvf, u16 *hw_sa_id) +{ + return cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id); +} + +static void cn10k_mcs_free_txsa(struct otx2_nic *pfvf, u16 hw_sa_id) +{ + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id, false); +} + +static void cn10k_mcs_free_rxsa(struct otx2_nic *pfvf, u16 hw_sa_id) +{ + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id, false); +} + +static int cn10k_mcs_write_rx_secy(struct otx2_nic *pfvf, + struct macsec_secy *secy, u8 hw_secy_id) +{ + struct mcs_secy_plcy_write_req *req; + struct mbox *mbox = &pfvf->mbox; + u64 policy; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + policy = FIELD_PREP(MCS_RX_SECY_PLCY_RW_MASK, secy->replay_window); + if (secy->replay_protect) + policy |= MCS_RX_SECY_PLCY_RP; + + policy |= MCS_RX_SECY_PLCY_AUTH_ENA; + policy |= FIELD_PREP(MCS_RX_SECY_PLCY_CIP, MCS_GCM_AES_128); + policy |= FIELD_PREP(MCS_RX_SECY_PLCY_VAL, secy->validate_frames); + + policy |= MCS_RX_SECY_PLCY_ENA; + + req->plcy = policy; + req->secy_id = hw_secy_id; + req->dir = MCS_RX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_flowid(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id) +{ + struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc; + struct mcs_flowid_entry_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->data[1] = FIELD_PREP(MCS_TCAM1_ETYPE_MASK, ETH_P_MACSEC); + req->mask[1] = ~0ULL; + req->mask[1] &= ~MCS_TCAM1_ETYPE_MASK; + + req->mask[0] = ~0ULL; + req->mask[2] = ~0ULL; + req->mask[3] = ~0ULL; + + req->flow_id = rxsc->hw_flow_id; + req->secy_id = hw_secy_id; + req->sc_id = rxsc->hw_sc_id; + req->dir = MCS_RX; + + if (sw_rx_sc->active) + req->ena = 1; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_sc_cam(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id) +{ + struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc; + struct mcs_rx_sc_cam_write_req *sc_req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + sc_req = otx2_mbox_alloc_msg_mcs_rx_sc_cam_write(mbox); + if (!sc_req) { + ret = -ENOMEM; + goto fail; + } + + sc_req->sci = (__force u64)cpu_to_be64((__force u64)sw_rx_sc->sci); + sc_req->sc_id = rxsc->hw_sc_id; + sc_req->secy_id = hw_secy_id; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_sa_plcy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_rxsc *rxsc, + u8 assoc_num, bool sa_in_use) +{ + unsigned char *src = rxsc->sa_key[assoc_num]; + struct mcs_sa_plcy_write_req *plcy_req; + struct mcs_rx_sc_sa_map *map_req; + struct mbox *mbox = &pfvf->mbox; + u8 reg, key_len; + int ret; + + mutex_lock(&mbox->lock); + + plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox); + if (!plcy_req) { + ret = -ENOMEM; + goto fail; + } + + map_req = otx2_mbox_alloc_msg_mcs_rx_sc_sa_map_write(mbox); + if (!map_req) { + otx2_mbox_reset(&mbox->mbox, 0); + ret = -ENOMEM; + goto fail; + } + + for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) { + memcpy((u8 *)&plcy_req->plcy[0][reg], + (src + reg * 8), 8); + reg++; + } + + plcy_req->sa_index[0] = rxsc->hw_sa_id[assoc_num]; + plcy_req->sa_cnt = 1; + plcy_req->dir = MCS_RX; + + map_req->sa_index = rxsc->hw_sa_id[assoc_num]; + map_req->sa_in_use = sa_in_use; + map_req->sc_id = rxsc->hw_sc_id; + map_req->an = assoc_num; + + /* Send two messages together */ + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_sa_pn(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, + u8 assoc_num, u64 next_pn) +{ + struct mcs_pn_table_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->pn_id = rxsc->hw_sa_id[assoc_num]; + req->next_pn = next_pn; + req->dir = MCS_RX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_secy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct mcs_secy_plcy_write_req *req; + struct mbox *mbox = &pfvf->mbox; + struct macsec_tx_sc *sw_tx_sc; + /* Insert SecTag after 12 bytes (DA+SA)*/ + u8 tag_offset = 12; + u8 sectag_tci = 0; + u64 policy; + int ret; + + sw_tx_sc = &secy->tx_sc; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + if (sw_tx_sc->send_sci) { + sectag_tci |= MCS_TCI_SC; + } else { + if (sw_tx_sc->end_station) + sectag_tci |= MCS_TCI_ES; + if (sw_tx_sc->scb) + sectag_tci |= MCS_TCI_SCB; + } + + if (sw_tx_sc->encrypt) + sectag_tci |= (MCS_TCI_E | MCS_TCI_C); + + policy = FIELD_PREP(MCS_TX_SECY_PLCY_MTU, secy->netdev->mtu); + /* Write SecTag excluding AN bits(1..0) */ + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_TCI, sectag_tci >> 2); + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_OFFSET, tag_offset); + policy |= MCS_TX_SECY_PLCY_INS_MODE; + policy |= MCS_TX_SECY_PLCY_AUTH_ENA; + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_CIP, MCS_GCM_AES_128); + + if (secy->protect_frames) + policy |= MCS_TX_SECY_PLCY_PROTECT; + + /* If the encodingsa does not exist/active and protect is + * not set then frames can be sent out as it is. Hence enable + * the policy irrespective of secy operational when !protect. + */ + if (!secy->protect_frames || secy->operational) + policy |= MCS_TX_SECY_PLCY_ENA; + + req->plcy = policy; + req->secy_id = txsc->hw_secy_id_tx; + req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_flowid(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct mcs_flowid_entry_write_req *req; + struct mbox *mbox = &pfvf->mbox; + u64 mac_sa; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + mac_sa = ether_addr_to_u64(secy->netdev->dev_addr); + + req->data[0] = FIELD_PREP(MCS_TCAM0_MAC_SA_MASK, mac_sa); + req->data[1] = FIELD_PREP(MCS_TCAM1_MAC_SA_MASK, mac_sa >> 16); + + req->mask[0] = ~0ULL; + req->mask[0] &= ~MCS_TCAM0_MAC_SA_MASK; + + req->mask[1] = ~0ULL; + req->mask[1] &= ~MCS_TCAM1_MAC_SA_MASK; + + req->mask[2] = ~0ULL; + req->mask[3] = ~0ULL; + + req->flow_id = txsc->hw_flow_id; + req->secy_id = txsc->hw_secy_id_tx; + req->sc_id = txsc->hw_sc_id; + req->sci = (__force u64)cpu_to_be64((__force u64)secy->sci); + req->dir = MCS_TX; + /* This can be enabled since stack xmits packets only when interface is up */ + req->ena = 1; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_link_tx_sa2sc(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + u8 sa_num, bool sa_active) +{ + struct mcs_tx_sc_sa_map *map_req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + /* Link the encoding_sa only to SC out of all SAs */ + if (txsc->encoding_sa != sa_num) + return 0; + + mutex_lock(&mbox->lock); + + map_req = otx2_mbox_alloc_msg_mcs_tx_sc_sa_map_write(mbox); + if (!map_req) { + otx2_mbox_reset(&mbox->mbox, 0); + ret = -ENOMEM; + goto fail; + } + + map_req->sa_index0 = txsc->hw_sa_id[sa_num]; + map_req->sa_index0_vld = sa_active; + map_req->sectag_sci = (__force u64)cpu_to_be64((__force u64)secy->sci); + map_req->sc_id = txsc->hw_sc_id; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_sa_plcy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + u8 assoc_num) +{ + unsigned char *src = txsc->sa_key[assoc_num]; + struct mcs_sa_plcy_write_req *plcy_req; + struct mbox *mbox = &pfvf->mbox; + u8 reg, key_len; + int ret; + + mutex_lock(&mbox->lock); + + plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox); + if (!plcy_req) { + ret = -ENOMEM; + goto fail; + } + + for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) { + memcpy((u8 *)&plcy_req->plcy[0][reg], (src + reg * 8), 8); + reg++; + } + + plcy_req->plcy[0][8] = assoc_num; + plcy_req->sa_index[0] = txsc->hw_sa_id[assoc_num]; + plcy_req->sa_cnt = 1; + plcy_req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_write_tx_sa_pn(struct otx2_nic *pfvf, + struct cn10k_mcs_txsc *txsc, + u8 assoc_num, u64 next_pn) +{ + struct mcs_pn_table_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->pn_id = txsc->hw_sa_id[assoc_num]; + req->next_pn = next_pn; + req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_ena_dis_flowid(struct otx2_nic *pfvf, u16 hw_flow_id, + bool enable, enum mcs_direction dir) +{ + struct mcs_flowid_ena_dis_entry *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_ena_entry(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->flow_id = hw_flow_id; + req->ena = enable; + req->dir = dir; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_sa_stats(struct otx2_nic *pfvf, u8 hw_sa_id, + struct mcs_sa_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_stats_req *req; + struct mcs_sa_stats *rsp; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_sa_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_sa_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_sa_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SA; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_sa_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_sc_stats(struct otx2_nic *pfvf, u8 hw_sc_id, + struct mcs_sc_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_stats_req *req; + struct mcs_sc_stats *rsp; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_sc_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_sc_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_sc_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SC; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_sc_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_secy_stats(struct otx2_nic *pfvf, u8 hw_secy_id, + struct mcs_secy_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_secy_stats *rsp; + struct mcs_stats_req *req; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_secy_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_secy_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_secy_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SECY; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_secy_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static struct cn10k_mcs_txsc *cn10k_mcs_create_txsc(struct otx2_nic *pfvf) +{ + struct cn10k_mcs_txsc *txsc; + int ret; + + txsc = kzalloc(sizeof(*txsc), GFP_KERNEL); + if (!txsc) + return ERR_PTR(-ENOMEM); + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + &txsc->hw_flow_id); + if (ret) + goto fail; + + /* For a SecY, one TX secy and one RX secy HW resources are needed */ + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + &txsc->hw_secy_id_tx); + if (ret) + goto free_flowid; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + &txsc->hw_secy_id_rx); + if (ret) + goto free_tx_secy; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC, + &txsc->hw_sc_id); + if (ret) + goto free_rx_secy; + + return txsc; +free_rx_secy: + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_rx, false); +free_tx_secy: + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_tx, false); +free_flowid: + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + txsc->hw_flow_id, false); +fail: + kfree(txsc); + return ERR_PTR(ret); +} + +/* Free Tx SC and its SAs(if any) resources to AF + */ +static void cn10k_mcs_delete_txsc(struct otx2_nic *pfvf, + struct cn10k_mcs_txsc *txsc) +{ + u8 sa_bmap = txsc->sa_bmap; + u8 sa_num = 0; + + while (sa_bmap) { + if (sa_bmap & 1) { + cn10k_mcs_write_tx_sa_plcy(pfvf, txsc->sw_secy, + txsc, sa_num); + cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]); + } + sa_num++; + sa_bmap >>= 1; + } + + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC, + txsc->hw_sc_id, false); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_rx, false); + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_tx, false); + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + txsc->hw_flow_id, false); +} + +static struct cn10k_mcs_rxsc *cn10k_mcs_create_rxsc(struct otx2_nic *pfvf) +{ + struct cn10k_mcs_rxsc *rxsc; + int ret; + + rxsc = kzalloc(sizeof(*rxsc), GFP_KERNEL); + if (!rxsc) + return ERR_PTR(-ENOMEM); + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + &rxsc->hw_flow_id); + if (ret) + goto fail; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC, + &rxsc->hw_sc_id); + if (ret) + goto free_flowid; + + return rxsc; +free_flowid: + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + rxsc->hw_flow_id, false); +fail: + kfree(rxsc); + return ERR_PTR(ret); +} + +/* Free Rx SC and its SAs(if any) resources to AF + */ +static void cn10k_mcs_delete_rxsc(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc) +{ + u8 sa_bmap = rxsc->sa_bmap; + u8 sa_num = 0; + + while (sa_bmap) { + if (sa_bmap & 1) { + cn10k_mcs_write_rx_sa_plcy(pfvf, rxsc->sw_secy, rxsc, + sa_num, false); + cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]); + } + sa_num++; + sa_bmap >>= 1; + } + + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC, + rxsc->hw_sc_id, false); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + rxsc->hw_flow_id, false); +} + +static int cn10k_mcs_secy_tx_cfg(struct otx2_nic *pfvf, struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + struct macsec_tx_sa *sw_tx_sa, u8 sa_num) +{ + if (sw_tx_sa) { + cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num); + cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, sa_num, + sw_tx_sa->active); + } + + cn10k_mcs_write_tx_secy(pfvf, secy, txsc); + cn10k_mcs_write_tx_flowid(pfvf, secy, txsc); + /* When updating secy, change RX secy also */ + cn10k_mcs_write_rx_secy(pfvf, secy, txsc->hw_secy_id_rx); + + return 0; +} + +static int cn10k_mcs_secy_rx_cfg(struct otx2_nic *pfvf, + struct macsec_secy *secy, u8 hw_secy_id) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *mcs_rx_sc; + struct macsec_rx_sc *sw_rx_sc; + struct macsec_rx_sa *sw_rx_sa; + u8 sa_num; + + for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active; + sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) { + mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (unlikely(!mcs_rx_sc)) + continue; + + for (sa_num = 0; sa_num < CN10K_MCS_SA_PER_SC; sa_num++) { + sw_rx_sa = rcu_dereference_bh(sw_rx_sc->sa[sa_num]); + if (!sw_rx_sa) + continue; + + cn10k_mcs_write_rx_sa_plcy(pfvf, secy, mcs_rx_sc, + sa_num, sw_rx_sa->active); + cn10k_mcs_write_rx_sa_pn(pfvf, mcs_rx_sc, sa_num, + sw_rx_sa->next_pn_halves.lower); + } + + cn10k_mcs_write_rx_flowid(pfvf, mcs_rx_sc, hw_secy_id); + cn10k_mcs_write_sc_cam(pfvf, mcs_rx_sc, hw_secy_id); + } + + return 0; +} + +static int cn10k_mcs_disable_rxscs(struct otx2_nic *pfvf, + struct macsec_secy *secy, + bool delete) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *mcs_rx_sc; + struct macsec_rx_sc *sw_rx_sc; + int ret; + + for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active; + sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) { + mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (unlikely(!mcs_rx_sc)) + continue; + + ret = cn10k_mcs_ena_dis_flowid(pfvf, mcs_rx_sc->hw_flow_id, + false, MCS_RX); + if (ret) + dev_err(pfvf->dev, "Failed to disable TCAM for SC %d\n", + mcs_rx_sc->hw_sc_id); + if (delete) { + cn10k_mcs_delete_rxsc(pfvf, mcs_rx_sc); + list_del(&mcs_rx_sc->entry); + kfree(mcs_rx_sc); + } + } + + return 0; +} + +static void cn10k_mcs_sync_stats(struct otx2_nic *pfvf, struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_secy_stats rx_rsp = { 0 }; + struct mcs_sc_stats sc_rsp = { 0 }; + struct cn10k_mcs_rxsc *rxsc; + + /* Because of shared counters for some stats in the hardware, when + * updating secy policy take a snapshot of current stats and reset them. + * Below are the effected stats because of shared counters. + */ + + /* Check if sync is really needed */ + if (secy->validate_frames == txsc->last_validate_frames && + secy->protect_frames == txsc->last_protect_frames) + return; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true); + + txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt; + txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt; + txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt; + if (txsc->last_validate_frames == MACSEC_VALIDATE_STRICT) + txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt; + else + txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt; + + list_for_each_entry(rxsc, &cfg->rxsc_list, entry) { + cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &sc_rsp, MCS_RX, true); + + rxsc->stats.InOctetsValidated += sc_rsp.octet_validate_cnt; + rxsc->stats.InOctetsDecrypted += sc_rsp.octet_decrypt_cnt; + + rxsc->stats.InPktsInvalid += sc_rsp.pkt_invalid_cnt; + rxsc->stats.InPktsNotValid += sc_rsp.pkt_notvalid_cnt; + + if (txsc->last_protect_frames) + rxsc->stats.InPktsLate += sc_rsp.pkt_late_cnt; + else + rxsc->stats.InPktsDelayed += sc_rsp.pkt_late_cnt; + + if (txsc->last_validate_frames == MACSEC_VALIDATE_CHECK) + rxsc->stats.InPktsUnchecked += sc_rsp.pkt_unchecked_cnt; + else + rxsc->stats.InPktsOK += sc_rsp.pkt_unchecked_cnt; + } + + txsc->last_validate_frames = secy->validate_frames; + txsc->last_protect_frames = secy->protect_frames; +} + +static int cn10k_mdo_open(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct macsec_tx_sa *sw_tx_sa; + struct cn10k_mcs_txsc *txsc; + u8 sa_num; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + sa_num = txsc->encoding_sa; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]); + + err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num); + if (err) + return err; + + return cn10k_mcs_secy_rx_cfg(pfvf, secy, txsc->hw_secy_id_rx); +} + +static int cn10k_mdo_stop(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + err = cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX); + if (err) + return err; + + return cn10k_mcs_disable_rxscs(pfvf, ctx->secy, false); +} + +static int cn10k_mdo_add_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_txsc *txsc; + + if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) + return -EOPNOTSUPP; + + /* Stick to 16 bytes key len until XPN support is added */ + if (secy->key_len != 16) + return -EOPNOTSUPP; + + if (secy->xpn) + return -EOPNOTSUPP; + + txsc = cn10k_mcs_create_txsc(pfvf); + if (IS_ERR(txsc)) + return -ENOSPC; + + txsc->sw_secy = secy; + txsc->encoding_sa = secy->tx_sc.encoding_sa; + txsc->last_validate_frames = secy->validate_frames; + txsc->last_protect_frames = secy->protect_frames; + + list_add(&txsc->entry, &cfg->txsc_list); + + if (netif_running(secy->netdev)) + return cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, NULL, 0); + + return 0; +} + +static int cn10k_mdo_upd_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct macsec_tx_sa *sw_tx_sa; + struct cn10k_mcs_txsc *txsc; + u8 sa_num; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + txsc->encoding_sa = secy->tx_sc.encoding_sa; + + sa_num = txsc->encoding_sa; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]); + + if (netif_running(secy->netdev)) { + cn10k_mcs_sync_stats(pfvf, secy, txsc); + + err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX); + cn10k_mcs_disable_rxscs(pfvf, ctx->secy, true); + cn10k_mcs_delete_txsc(pfvf, txsc); + list_del(&txsc->entry); + kfree(txsc); + + return 0; +} + +static int cn10k_mdo_add_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa; + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (cn10k_mcs_alloc_txsa(pfvf, &txsc->hw_sa_id[sa_num])) + return -ENOSPC; + + memcpy(&txsc->sa_key[sa_num], ctx->sa.key, secy->key_len); + txsc->sa_bmap |= 1 << sa_num; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num); + if (err) + return err; + + err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + if (err) + return err; + + err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, + sa_num, sw_tx_sa->active); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa; + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (netif_running(secy->netdev)) { + /* Keys cannot be changed after creation */ + err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + if (err) + return err; + + err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, + sa_num, sw_tx_sa->active); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]); + txsc->sa_bmap &= ~(1 << sa_num); + + return 0; +} + +static int cn10k_mdo_add_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_rxsc *rxsc; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + rxsc = cn10k_mcs_create_rxsc(pfvf); + if (IS_ERR(rxsc)) + return -ENOSPC; + + rxsc->sw_secy = ctx->secy; + rxsc->sw_rxsc = ctx->rx_sc; + list_add(&rxsc->entry, &cfg->rxsc_list); + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_flowid(pfvf, rxsc, txsc->hw_secy_id_rx); + if (err) + return err; + + err = cn10k_mcs_write_sc_cam(pfvf, rxsc, txsc->hw_secy_id_rx); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + bool enable = ctx->rx_sc->active; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + if (netif_running(secy->netdev)) + return cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, + enable, MCS_RX); + + return 0; +} + +static int cn10k_mdo_del_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, false, MCS_RX); + cn10k_mcs_delete_rxsc(pfvf, rxsc); + list_del(&rxsc->entry); + kfree(rxsc); + + return 0; +} + +static int cn10k_mdo_add_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa; + u64 next_pn = rx_sa->next_pn_halves.lower; + struct macsec_secy *secy = ctx->secy; + bool sa_in_use = rx_sa->active; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + int err; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (cn10k_mcs_alloc_rxsa(pfvf, &rxsc->hw_sa_id[sa_num])) + return -ENOSPC; + + memcpy(&rxsc->sa_key[sa_num], ctx->sa.key, ctx->secy->key_len); + rxsc->sa_bmap |= 1 << sa_num; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, + sa_num, sa_in_use); + if (err) + return err; + + err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa; + u64 next_pn = rx_sa->next_pn_halves.lower; + struct macsec_secy *secy = ctx->secy; + bool sa_in_use = rx_sa->active; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + int err; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, sa_num, sa_in_use); + if (err) + return err; + + err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_write_rx_sa_plcy(pfvf, ctx->secy, rxsc, sa_num, false); + cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]); + + rxsc->sa_bmap &= ~(1 << sa_num); + + return 0; +} + +static int cn10k_mdo_get_dev_stats(struct macsec_context *ctx) +{ + struct mcs_secy_stats tx_rsp = { 0 }, rx_rsp = { 0 }; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_tx, &tx_rsp, MCS_TX, false); + ctx->stats.dev_stats->OutPktsUntagged = tx_rsp.pkt_untagged_cnt; + ctx->stats.dev_stats->OutPktsTooLong = tx_rsp.pkt_toolong_cnt; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true); + txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt; + txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt; + txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt; + if (secy->validate_frames == MACSEC_VALIDATE_STRICT) + txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt; + else + txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt; + txsc->stats.InPktsOverrun = 0; + + ctx->stats.dev_stats->InPktsNoTag = txsc->stats.InPktsNoTag; + ctx->stats.dev_stats->InPktsUntagged = txsc->stats.InPktsUntagged; + ctx->stats.dev_stats->InPktsBadTag = txsc->stats.InPktsBadTag; + ctx->stats.dev_stats->InPktsUnknownSCI = txsc->stats.InPktsUnknownSCI; + ctx->stats.dev_stats->InPktsNoSCI = txsc->stats.InPktsNoSCI; + ctx->stats.dev_stats->InPktsOverrun = txsc->stats.InPktsOverrun; + + return 0; +} + +static int cn10k_mdo_get_tx_sc_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sc_stats rsp = { 0 }; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_sc_stats(pfvf, txsc->hw_sc_id, &rsp, MCS_TX, false); + + ctx->stats.tx_sc_stats->OutPktsProtected = rsp.pkt_protected_cnt; + ctx->stats.tx_sc_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt; + ctx->stats.tx_sc_stats->OutOctetsProtected = rsp.octet_protected_cnt; + ctx->stats.tx_sc_stats->OutOctetsEncrypted = rsp.octet_encrypt_cnt; + + return 0; +} + +static int cn10k_mdo_get_tx_sa_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sa_stats rsp = { 0 }; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_sa_stats(pfvf, txsc->hw_sa_id[sa_num], &rsp, MCS_TX, false); + + ctx->stats.tx_sa_stats->OutPktsProtected = rsp.pkt_protected_cnt; + ctx->stats.tx_sa_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt; + + return 0; +} + +static int cn10k_mdo_get_rx_sc_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct mcs_sc_stats rsp = { 0 }; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &rsp, MCS_RX, true); + + rxsc->stats.InOctetsValidated += rsp.octet_validate_cnt; + rxsc->stats.InOctetsDecrypted += rsp.octet_decrypt_cnt; + + rxsc->stats.InPktsInvalid += rsp.pkt_invalid_cnt; + rxsc->stats.InPktsNotValid += rsp.pkt_notvalid_cnt; + + if (secy->protect_frames) + rxsc->stats.InPktsLate += rsp.pkt_late_cnt; + else + rxsc->stats.InPktsDelayed += rsp.pkt_late_cnt; + + if (secy->validate_frames == MACSEC_VALIDATE_CHECK) + rxsc->stats.InPktsUnchecked += rsp.pkt_unchecked_cnt; + else + rxsc->stats.InPktsOK += rsp.pkt_unchecked_cnt; + + ctx->stats.rx_sc_stats->InOctetsValidated = rxsc->stats.InOctetsValidated; + ctx->stats.rx_sc_stats->InOctetsDecrypted = rxsc->stats.InOctetsDecrypted; + ctx->stats.rx_sc_stats->InPktsInvalid = rxsc->stats.InPktsInvalid; + ctx->stats.rx_sc_stats->InPktsNotValid = rxsc->stats.InPktsNotValid; + ctx->stats.rx_sc_stats->InPktsLate = rxsc->stats.InPktsLate; + ctx->stats.rx_sc_stats->InPktsDelayed = rxsc->stats.InPktsDelayed; + ctx->stats.rx_sc_stats->InPktsUnchecked = rxsc->stats.InPktsUnchecked; + ctx->stats.rx_sc_stats->InPktsOK = rxsc->stats.InPktsOK; + + return 0; +} + +static int cn10k_mdo_get_rx_sa_stats(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sa_stats rsp = { 0 }; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_sa_stats(pfvf, rxsc->hw_sa_id[sa_num], &rsp, MCS_RX, false); + + ctx->stats.rx_sa_stats->InPktsOK = rsp.pkt_ok_cnt; + ctx->stats.rx_sa_stats->InPktsInvalid = rsp.pkt_invalid_cnt; + ctx->stats.rx_sa_stats->InPktsNotValid = rsp.pkt_notvalid_cnt; + ctx->stats.rx_sa_stats->InPktsNotUsingSA = rsp.pkt_nosaerror_cnt; + ctx->stats.rx_sa_stats->InPktsUnusedSA = rsp.pkt_nosa_cnt; + + return 0; +} + +static const struct macsec_ops cn10k_mcs_ops = { + .mdo_dev_open = cn10k_mdo_open, + .mdo_dev_stop = cn10k_mdo_stop, + .mdo_add_secy = cn10k_mdo_add_secy, + .mdo_upd_secy = cn10k_mdo_upd_secy, + .mdo_del_secy = cn10k_mdo_del_secy, + .mdo_add_rxsc = cn10k_mdo_add_rxsc, + .mdo_upd_rxsc = cn10k_mdo_upd_rxsc, + .mdo_del_rxsc = cn10k_mdo_del_rxsc, + .mdo_add_rxsa = cn10k_mdo_add_rxsa, + .mdo_upd_rxsa = cn10k_mdo_upd_rxsa, + .mdo_del_rxsa = cn10k_mdo_del_rxsa, + .mdo_add_txsa = cn10k_mdo_add_txsa, + .mdo_upd_txsa = cn10k_mdo_upd_txsa, + .mdo_del_txsa = cn10k_mdo_del_txsa, + .mdo_get_dev_stats = cn10k_mdo_get_dev_stats, + .mdo_get_tx_sc_stats = cn10k_mdo_get_tx_sc_stats, + .mdo_get_tx_sa_stats = cn10k_mdo_get_tx_sa_stats, + .mdo_get_rx_sc_stats = cn10k_mdo_get_rx_sc_stats, + .mdo_get_rx_sa_stats = cn10k_mdo_get_rx_sa_stats, +}; + +void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_tx_sa *sw_tx_sa = NULL; + struct macsec_secy *secy = NULL; + struct cn10k_mcs_txsc *txsc; + u8 an; + + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return; + + if (!(event->intr_mask & MCS_CPM_TX_PACKET_XPN_EQ0_INT)) + return; + + /* Find the SecY to which the expired hardware SA is mapped */ + list_for_each_entry(txsc, &cfg->txsc_list, entry) { + for (an = 0; an < CN10K_MCS_SA_PER_SC; an++) + if (txsc->hw_sa_id[an] == event->sa_id) { + secy = txsc->sw_secy; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[an]); + } + } + + if (secy && sw_tx_sa) + macsec_pn_wrapped(secy, sw_tx_sa); +} + +int cn10k_mcs_init(struct otx2_nic *pfvf) +{ + struct mbox *mbox = &pfvf->mbox; + struct cn10k_mcs_cfg *cfg; + struct mcs_intr_cfg *req; + + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return 0; + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return -ENOMEM; + + INIT_LIST_HEAD(&cfg->txsc_list); + INIT_LIST_HEAD(&cfg->rxsc_list); + pfvf->macsec_cfg = cfg; + + pfvf->netdev->features |= NETIF_F_HW_MACSEC; + pfvf->netdev->macsec_ops = &cn10k_mcs_ops; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_intr_cfg(mbox); + if (!req) + goto fail; + + req->intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + if (otx2_sync_mbox_msg(mbox)) + goto fail; + + mutex_unlock(&mbox->lock); + + return 0; +fail: + dev_err(pfvf->dev, "Cannot notify PN wrapped event\n"); + mutex_unlock(&mbox->lock); + return 0; +} + +void cn10k_mcs_free(struct otx2_nic *pfvf) +{ + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return; + + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, 0, true); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, 0, true); + kfree(pfvf->macsec_cfg); + pfvf->macsec_cfg = NULL; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index d686c7b6252f..9e10e7471b88 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -586,8 +586,9 @@ void otx2_get_mac_from_af(struct net_device *netdev) } EXPORT_SYMBOL(otx2_get_mac_from_af); -int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) +int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc) { + u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC]; struct otx2_hw *hw = &pfvf->hw; struct nix_txschq_config *req; u64 schq, parent; @@ -602,7 +603,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->lvl = lvl; req->num_regs = 1; - schq = hw->txschq_list[lvl][0]; + schq_list = hw->txschq_list; +#ifdef CONFIG_DCB + if (txschq_for_pfc) + schq_list = pfvf->pfc_schq_list; +#endif + + schq = schq_list[lvl][prio]; /* Set topology e.t.c configuration */ if (lvl == NIX_TXSCH_LVL_SMQ) { req->reg[0] = NIX_AF_SMQX_CFG(schq); @@ -611,7 +618,7 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) (0x2ULL << 36); req->num_regs++; /* MDQ config */ - parent = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; + parent = schq_list[NIX_TXSCH_LVL_TL4][prio]; req->reg[1] = NIX_AF_MDQX_PARENT(schq); req->regval[1] = parent << 16; req->num_regs++; @@ -619,14 +626,14 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); req->regval[2] = dwrr_val; } else if (lvl == NIX_TXSCH_LVL_TL4) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL3][0]; + parent = schq_list[NIX_TXSCH_LVL_TL3][prio]; req->reg[0] = NIX_AF_TL4X_PARENT(schq); req->regval[0] = parent << 16; req->num_regs++; req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); req->regval[1] = dwrr_val; } else if (lvl == NIX_TXSCH_LVL_TL3) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL2][0]; + parent = schq_list[NIX_TXSCH_LVL_TL2][prio]; req->reg[0] = NIX_AF_TL3X_PARENT(schq); req->regval[0] = parent << 16; req->num_regs++; @@ -635,11 +642,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) if (lvl == hw->txschq_link_cfg_lvl) { req->num_regs++; req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); - /* Enable this queue and backpressure */ - req->regval[2] = BIT_ULL(13) | BIT_ULL(12); + /* Enable this queue and backpressure + * and set relative channel + */ + req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; } } else if (lvl == NIX_TXSCH_LVL_TL2) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL1][0]; + parent = schq_list[NIX_TXSCH_LVL_TL1][prio]; req->reg[0] = NIX_AF_TL2X_PARENT(schq); req->regval[0] = parent << 16; @@ -650,8 +659,10 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) if (lvl == hw->txschq_link_cfg_lvl) { req->num_regs++; req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); - /* Enable this queue and backpressure */ - req->regval[2] = BIT_ULL(13) | BIT_ULL(12); + /* Enable this queue and backpressure + * and set relative channel + */ + req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; } } else if (lvl == NIX_TXSCH_LVL_TL1) { /* Default config for TL1. @@ -676,6 +687,31 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) return otx2_sync_mbox_msg(&pfvf->mbox); } +EXPORT_SYMBOL(otx2_txschq_config); + +int otx2_smq_flush(struct otx2_nic *pfvf, int smq) +{ + struct nix_txschq_config *req; + int rc; + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + req->lvl = NIX_TXSCH_LVL_SMQ; + req->reg[0] = NIX_AF_SMQX_CFG(smq); + req->regval[0] |= BIT_ULL(49); + req->num_regs++; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + mutex_unlock(&pfvf->mbox.lock); + return rc; +} +EXPORT_SYMBOL(otx2_smq_flush); int otx2_txsch_alloc(struct otx2_nic *pfvf) { @@ -806,8 +842,7 @@ int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ aq->sq.cq_ena = 1; aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); aq->sq.default_chan = pfvf->hw.tx_chan_base; aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ @@ -863,6 +898,7 @@ static int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) } sq->head = 0; + sq->cons_head = 0; sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; /* Set SQE threshold to 10% of total SQEs */ @@ -1792,4 +1828,5 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ } \ EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index b28029cc4316..282db6fe3b08 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -19,6 +19,7 @@ #include <net/devlink.h> #include <linux/time64.h> #include <linux/dim.h> +#include <uapi/linux/if_macsec.h> #include <mbox.h> #include <npc.h> @@ -33,6 +34,7 @@ #define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8 #define PCI_SUBSYS_DEVID_96XX_RVU_PFVF 0xB200 +#define PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF 0xBD00 /* PCI BAR nos */ #define PCI_CFG_REG_BAR_NUM 2 @@ -40,6 +42,11 @@ #define NAME_SIZE 32 +#ifdef CONFIG_DCB +/* Max priority supported for PFC */ +#define NIX_PF_PFC_PRIO_MAX 8 +#endif + enum arua_mapped_qtypes { AURA_NIX_RQ, AURA_NIX_SQ, @@ -196,7 +203,7 @@ struct otx2_hw { /* NIX */ u8 txschq_link_cfg_lvl; - u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; u16 matchall_ipolicer; u32 dwrr_mtu; @@ -238,6 +245,8 @@ struct otx2_hw { #define CN10K_MBOX 1 #define CN10K_LMTST 2 #define CN10K_RPM 3 +#define CN10K_PTP_ONESTEP 4 +#define CN10K_HW_MACSEC 5 unsigned long cap_flag; #define LMT_LINE_SIZE 128 @@ -271,6 +280,13 @@ struct refill_work { struct otx2_nic *pf; }; +/* PTPv2 originTimestamp structure */ +struct ptpv2_tstamp { + __be16 seconds_msb; /* 16 bits + */ + __be32 seconds_lsb; /* 32 bits = 48 bits*/ + __be32 nanoseconds; +} __packed; + struct otx2_ptp { struct ptp_clock_info ptp_info; struct ptp_clock *ptp_clock; @@ -286,6 +302,9 @@ struct otx2_ptp { struct ptp_pin_desc extts_config; u64 (*convert_rx_ptp_tstmp)(u64 timestamp); u64 (*convert_tx_ptp_tstmp)(u64 timestamp); + struct delayed_work synctstamp_work; + u64 tstamp; + u32 base_ns; }; #define OTX2_HW_TIMESTAMP_LEN 8 @@ -335,6 +354,66 @@ struct dev_hw_ops { void (*aura_freeptr)(void *dev, int aura, u64 buf); }; +#define CN10K_MCS_SA_PER_SC 4 + +/* Stats which need to be accumulated in software because + * of shared counters in hardware. + */ +struct cn10k_txsc_stats { + u64 InPktsUntagged; + u64 InPktsNoTag; + u64 InPktsBadTag; + u64 InPktsUnknownSCI; + u64 InPktsNoSCI; + u64 InPktsOverrun; +}; + +struct cn10k_rxsc_stats { + u64 InOctetsValidated; + u64 InOctetsDecrypted; + u64 InPktsUnchecked; + u64 InPktsDelayed; + u64 InPktsOK; + u64 InPktsInvalid; + u64 InPktsLate; + u64 InPktsNotValid; + u64 InPktsNotUsingSA; + u64 InPktsUnusedSA; +}; + +struct cn10k_mcs_txsc { + struct macsec_secy *sw_secy; + struct cn10k_txsc_stats stats; + struct list_head entry; + enum macsec_validation_type last_validate_frames; + bool last_protect_frames; + u16 hw_secy_id_tx; + u16 hw_secy_id_rx; + u16 hw_flow_id; + u16 hw_sc_id; + u16 hw_sa_id[CN10K_MCS_SA_PER_SC]; + u8 sa_bmap; + u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN]; + u8 encoding_sa; +}; + +struct cn10k_mcs_rxsc { + struct macsec_secy *sw_secy; + struct macsec_rx_sc *sw_rxsc; + struct cn10k_rxsc_stats stats; + struct list_head entry; + u16 hw_flow_id; + u16 hw_sc_id; + u16 hw_sa_id[CN10K_MCS_SA_PER_SC]; + u8 sa_bmap; + u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN]; +}; + +struct cn10k_mcs_cfg { + struct list_head txsc_list; + struct list_head rxsc_list; +}; + struct otx2_nic { void __iomem *reg_base; struct net_device *netdev; @@ -358,6 +437,7 @@ struct otx2_nic { #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12) #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13) #define OTX2_FLAG_DMACFLTR_SUPPORT BIT_ULL(14) +#define OTX2_FLAG_PTP_ONESTEP_SYNC BIT_ULL(15) #define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16) u64 flags; u64 *cq_op_addr; @@ -415,10 +495,16 @@ struct otx2_nic { /* PFC */ u8 pfc_en; u8 *queue_to_pfc_map; + u16 pfc_schq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + bool pfc_alloc_status[NIX_PF_PFC_PRIO_MAX]; #endif /* napi event count. It is needed for adaptive irq coalescing. */ u32 napi_events; + +#if IS_ENABLED(CONFIG_MACSEC) + struct cn10k_mcs_cfg *macsec_cfg; +#endif }; static inline bool is_otx2_lbkvf(struct pci_dev *pdev) @@ -458,6 +544,11 @@ static inline bool is_dev_otx2(struct pci_dev *pdev) midr == PCI_REVISION_ID_95XXMM || midr == PCI_REVISION_ID_95XXO); } +static inline bool is_dev_cn10kb(struct pci_dev *pdev) +{ + return pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF; +} + static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf) { struct otx2_hw *hw = &pfvf->hw; @@ -487,7 +578,11 @@ static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf) __set_bit(CN10K_MBOX, &hw->cap_flag); __set_bit(CN10K_LMTST, &hw->cap_flag); __set_bit(CN10K_RPM, &hw->cap_flag); + __set_bit(CN10K_PTP_ONESTEP, &hw->cap_flag); } + + if (is_dev_cn10kb(pfvf->pdev)) + __set_bit(CN10K_HW_MACSEC, &hw->cap_flag); } /* Register read/write APIs */ @@ -743,6 +838,7 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ struct _rsp_type *rsp); \ MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M /* Time to wait before watchdog kicks off */ @@ -785,6 +881,16 @@ static inline void otx2_dma_unmap_page(struct otx2_nic *pfvf, dir, DMA_ATTR_SKIP_CPU_SYNC); } +static inline u16 otx2_get_smq_idx(struct otx2_nic *pfvf, u16 qidx) +{ +#ifdef CONFIG_DCB + if (pfvf->pfc_alloc_status[qidx]) + return pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][qidx]; +#endif + + return pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; +} + /* MSI-X APIs */ void otx2_free_cints(struct otx2_nic *pfvf, int n); void otx2_set_cints_affinity(struct otx2_nic *pfvf); @@ -807,7 +913,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type); void otx2_sq_free_sqbs(struct otx2_nic *pfvf); int otx2_config_nix(struct otx2_nic *pfvf); int otx2_config_nix_queues(struct otx2_nic *pfvf); -int otx2_txschq_config(struct otx2_nic *pfvf, int lvl); +int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool pfc_en); int otx2_txsch_alloc(struct otx2_nic *pfvf); int otx2_txschq_stop(struct otx2_nic *pfvf); void otx2_sqb_flush(struct otx2_nic *pfvf); @@ -888,6 +994,8 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx); u16 otx2_get_max_mtu(struct otx2_nic *pfvf); int otx2_handle_ntuple_tc_features(struct net_device *netdev, netdev_features_t features); +int otx2_smq_flush(struct otx2_nic *pfvf, int smq); + /* tc support */ int otx2_init_tc(struct otx2_nic *nic); void otx2_shutdown_tc(struct otx2_nic *nic); @@ -907,5 +1015,24 @@ void otx2_dmacflt_update_pfmac_flow(struct otx2_nic *pfvf); void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx, bool pfc_enable); int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf); int otx2_dcbnl_set_ops(struct net_device *dev); +/* PFC support */ +int otx2_pfc_txschq_config(struct otx2_nic *pfvf); +int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf); +int otx2_pfc_txschq_update(struct otx2_nic *pfvf); +int otx2_pfc_txschq_stop(struct otx2_nic *pfvf); #endif + +#if IS_ENABLED(CONFIG_MACSEC) +/* MACSEC offload support */ +int cn10k_mcs_init(struct otx2_nic *pfvf); +void cn10k_mcs_free(struct otx2_nic *pfvf); +void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event); +#else +static inline int cn10k_mcs_init(struct otx2_nic *pfvf) { return 0; } +static inline void cn10k_mcs_free(struct otx2_nic *pfvf) {} +static inline void cn10k_handle_mcs_event(struct otx2_nic *pfvf, + struct mcs_intr_info *event) +{} +#endif /* CONFIG_MACSEC */ + #endif /* OTX2_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c index 723d2506d309..ccaf97bb1ce0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c @@ -7,6 +7,289 @@ #include "otx2_common.h" +static int otx2_check_pfc_config(struct otx2_nic *pfvf) +{ + u8 tx_queues = pfvf->hw.tx_queues, prio; + u8 pfc_en = pfvf->pfc_en; + + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + if ((pfc_en & (1 << prio)) && + prio > tx_queues - 1) { + dev_warn(pfvf->dev, + "Increase number of tx queues from %d to %d to support PFC.\n", + tx_queues, prio + 1); + return -EINVAL; + } + } + + return 0; +} + +int otx2_pfc_txschq_config(struct otx2_nic *pfvf) +{ + u8 pfc_en, pfc_bit_set; + int prio, lvl, err; + + pfc_en = pfvf->pfc_en; + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + /* Either PFC bit is not set + * or tx scheduler is not allocated for the priority + */ + if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio]) + continue; + + /* configure the scheduler for the tls*/ + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { + err = otx2_txschq_config(pfvf, lvl, prio, true); + if (err) { + dev_err(pfvf->dev, + "%s configure PFC tx schq for lvl:%d, prio:%d failed!\n", + __func__, lvl, prio); + return err; + } + } + } + + return 0; +} + +static int otx2_pfc_txschq_alloc_one(struct otx2_nic *pfvf, u8 prio) +{ + struct nix_txsch_alloc_req *req; + struct nix_txsch_alloc_rsp *rsp; + int lvl, rc; + + /* Get memory to put this msg */ + req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox); + if (!req) + return -ENOMEM; + + /* Request one schq per level upto max level as configured + * link config level. These rest of the scheduler can be + * same as hw.txschq_list. + */ + for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) + req->schq[lvl] = 1; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + if (rc) + return rc; + + rsp = (struct nix_txsch_alloc_rsp *) + otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); + if (IS_ERR(rsp)) + return PTR_ERR(rsp); + + /* Setup transmit scheduler list */ + for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) { + if (!rsp->schq[lvl]) + return -ENOSPC; + + pfvf->pfc_schq_list[lvl][prio] = rsp->schq_list[lvl][0]; + } + + /* Set the Tx schedulers for rest of the levels same as + * hw.txschq_list as those will be common for all. + */ + for (; lvl < NIX_TXSCH_LVL_CNT; lvl++) + pfvf->pfc_schq_list[lvl][prio] = pfvf->hw.txschq_list[lvl][0]; + + pfvf->pfc_alloc_status[prio] = true; + return 0; +} + +int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf) +{ + u8 pfc_en = pfvf->pfc_en; + u8 pfc_bit_set; + int err, prio; + + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + if (!pfc_bit_set || pfvf->pfc_alloc_status[prio]) + continue; + + /* Add new scheduler to the priority */ + err = otx2_pfc_txschq_alloc_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, "%s failed to allocate PFC TX schedulers\n", __func__); + return err; + } + } + + return 0; +} + +static int otx2_pfc_txschq_stop_one(struct otx2_nic *pfvf, u8 prio) +{ + struct nix_txsch_free_req *free_req; + + mutex_lock(&pfvf->mbox.lock); + /* free PFC TLx nodes */ + free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox); + if (!free_req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + free_req->flags = TXSCHQ_FREE_ALL; + otx2_sync_mbox_msg(&pfvf->mbox); + mutex_unlock(&pfvf->mbox.lock); + + pfvf->pfc_alloc_status[prio] = false; + return 0; +} + +static int otx2_pfc_update_sq_smq_mapping(struct otx2_nic *pfvf, int prio) +{ + struct nix_cn10k_aq_enq_req *cn10k_sq_aq; + struct net_device *dev = pfvf->netdev; + bool if_up = netif_running(dev); + struct nix_aq_enq_req *sq_aq; + + if (if_up) { + if (pfvf->pfc_alloc_status[prio]) + netif_tx_stop_all_queues(pfvf->netdev); + else + netif_tx_stop_queue(netdev_get_tx_queue(dev, prio)); + } + + if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { + cn10k_sq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!cn10k_sq_aq) + return -ENOMEM; + + /* Fill AQ info */ + cn10k_sq_aq->qidx = prio; + cn10k_sq_aq->ctype = NIX_AQ_CTYPE_SQ; + cn10k_sq_aq->op = NIX_AQ_INSTOP_WRITE; + + /* Fill fields to update */ + cn10k_sq_aq->sq.ena = 1; + cn10k_sq_aq->sq_mask.ena = 1; + cn10k_sq_aq->sq_mask.smq = GENMASK(9, 0); + cn10k_sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio); + } else { + sq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); + if (!sq_aq) + return -ENOMEM; + + /* Fill AQ info */ + sq_aq->qidx = prio; + sq_aq->ctype = NIX_AQ_CTYPE_SQ; + sq_aq->op = NIX_AQ_INSTOP_WRITE; + + /* Fill fields to update */ + sq_aq->sq.ena = 1; + sq_aq->sq_mask.ena = 1; + sq_aq->sq_mask.smq = GENMASK(8, 0); + sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio); + } + + otx2_sync_mbox_msg(&pfvf->mbox); + + if (if_up) { + if (pfvf->pfc_alloc_status[prio]) + netif_tx_start_all_queues(pfvf->netdev); + else + netif_tx_start_queue(netdev_get_tx_queue(dev, prio)); + } + + return 0; +} + +int otx2_pfc_txschq_update(struct otx2_nic *pfvf) +{ + bool if_up = netif_running(pfvf->netdev); + u8 pfc_en = pfvf->pfc_en, pfc_bit_set; + struct mbox *mbox = &pfvf->mbox; + int err, prio; + + mutex_lock(&mbox->lock); + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + /* tx scheduler was created but user wants to disable now */ + if (!pfc_bit_set && pfvf->pfc_alloc_status[prio]) { + mutex_unlock(&mbox->lock); + if (if_up) + netif_tx_stop_all_queues(pfvf->netdev); + + otx2_smq_flush(pfvf, pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][prio]); + if (if_up) + netif_tx_start_all_queues(pfvf->netdev); + + /* delete the schq */ + err = otx2_pfc_txschq_stop_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, + "%s failed to stop PFC tx schedulers for priority: %d\n", + __func__, prio); + return err; + } + + mutex_lock(&mbox->lock); + goto update_sq_smq_map; + } + + /* Either PFC bit is not set + * or Tx scheduler is already mapped for the priority + */ + if (!pfc_bit_set || pfvf->pfc_alloc_status[prio]) + continue; + + /* Add new scheduler to the priority */ + err = otx2_pfc_txschq_alloc_one(pfvf, prio); + if (err) { + mutex_unlock(&mbox->lock); + dev_err(pfvf->dev, + "%s failed to allocate PFC tx schedulers for priority: %d\n", + __func__, prio); + return err; + } + +update_sq_smq_map: + err = otx2_pfc_update_sq_smq_mapping(pfvf, prio); + if (err) { + mutex_unlock(&mbox->lock); + dev_err(pfvf->dev, "%s failed PFC Tx schq sq:%d mapping", __func__, prio); + return err; + } + } + + err = otx2_pfc_txschq_config(pfvf); + mutex_unlock(&mbox->lock); + if (err) + return err; + + return 0; +} + +int otx2_pfc_txschq_stop(struct otx2_nic *pfvf) +{ + u8 pfc_en, pfc_bit_set; + int prio, err; + + pfc_en = pfvf->pfc_en; + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio]) + continue; + + /* Delete the existing scheduler */ + err = otx2_pfc_txschq_stop_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, "%s failed to stop PFC TX schedulers\n", __func__); + return err; + } + } + + return 0; +} + int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf) { struct cgx_pfc_cfg *req; @@ -128,6 +411,17 @@ static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc) /* Save PFC configuration to interface */ pfvf->pfc_en = pfc->pfc_en; + if (pfvf->hw.tx_queues >= NIX_PF_PFC_PRIO_MAX) + goto process_pfc; + + /* Check if the PFC configuration can be + * supported by the tx queue configuration + */ + err = otx2_check_pfc_config(pfvf); + if (err) + return err; + +process_pfc: err = otx2_config_priority_flow_ctrl(pfvf); if (err) return err; @@ -136,6 +430,12 @@ static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc) if (pfc->pfc_en) otx2_nix_config_bp(pfvf, true); + err = otx2_pfc_txschq_update(pfvf); + if (err) { + dev_err(pfvf->dev, "%s failed to update TX schedulers\n", __func__); + return err; + } + return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 3f60a80e34c8..0eb74e8c553d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -76,8 +76,8 @@ static void otx2_get_drvinfo(struct net_device *netdev, { struct otx2_nic *pfvf = netdev_priv(netdev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info)); } static void otx2_get_qset_strings(struct otx2_nic *pfvf, u8 **data, int qset) @@ -963,10 +963,12 @@ static int otx2_get_ts_info(struct net_device *netdev, info->phc_index = otx2_ptp_clock_index(pfvf); - info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + if (test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) + info->tx_types |= BIT(HWTSTAMP_TX_ONESTEP_SYNC); - info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | - (1 << HWTSTAMP_FILTER_ALL); + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_ALL); return 0; } @@ -1313,8 +1315,8 @@ static void otx2vf_get_drvinfo(struct net_device *netdev, { struct otx2_nic *vf = netdev_priv(netdev); - strlcpy(info->driver, DRV_VF_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_VF_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info)); } static void otx2vf_get_strings(struct net_device *netdev, u32 sset, u8 *data) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 9376d0e62914..303930499a4c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -15,6 +15,7 @@ #include <net/ip.h> #include <linux/bpf.h> #include <linux/bpf_trace.h> +#include <linux/bitfield.h> #include "otx2_reg.h" #include "otx2_common.h" @@ -858,6 +859,15 @@ static void otx2_handle_link_event(struct otx2_nic *pf) } } +int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf, + struct mcs_intr_info *event, + struct msg_rsp *rsp) +{ + cn10k_handle_mcs_event(pf, event); + + return 0; +} + int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf, struct cgx_link_info_msg *msg, struct msg_rsp *rsp) @@ -917,6 +927,7 @@ static int otx2_process_mbox_msg_up(struct otx2_nic *pf, return err; \ } MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M break; default: @@ -1161,6 +1172,59 @@ int otx2_set_real_num_queues(struct net_device *netdev, } EXPORT_SYMBOL(otx2_set_real_num_queues); +static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = { + "NIX_SQOPERR_OOR", + "NIX_SQOPERR_CTX_FAULT", + "NIX_SQOPERR_CTX_POISON", + "NIX_SQOPERR_DISABLED", + "NIX_SQOPERR_SIZE_ERR", + "NIX_SQOPERR_OFLOW", + "NIX_SQOPERR_SQB_NULL", + "NIX_SQOPERR_SQB_FAULT", + "NIX_SQOPERR_SQE_SZ_ZERO", +}; + +static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = { + "NIX_MNQERR_SQ_CTX_FAULT", + "NIX_MNQERR_SQ_CTX_POISON", + "NIX_MNQERR_SQB_FAULT", + "NIX_MNQERR_SQB_POISON", + "NIX_MNQERR_TOTAL_ERR", + "NIX_MNQERR_LSO_ERR", + "NIX_MNQERR_CQ_QUERY_ERR", + "NIX_MNQERR_MAX_SQE_SIZE_ERR", + "NIX_MNQERR_MAXLEN_ERR", + "NIX_MNQERR_SQE_SIZEM1_ZERO", +}; + +static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = { + "NIX_SND_STATUS_GOOD", + "NIX_SND_STATUS_SQ_CTX_FAULT", + "NIX_SND_STATUS_SQ_CTX_POISON", + "NIX_SND_STATUS_SQB_FAULT", + "NIX_SND_STATUS_SQB_POISON", + "NIX_SND_STATUS_HDR_ERR", + "NIX_SND_STATUS_EXT_ERR", + "NIX_SND_STATUS_JUMP_FAULT", + "NIX_SND_STATUS_JUMP_POISON", + "NIX_SND_STATUS_CRC_ERR", + "NIX_SND_STATUS_IMM_ERR", + "NIX_SND_STATUS_SG_ERR", + "NIX_SND_STATUS_MEM_ERR", + "NIX_SND_STATUS_INVALID_SUBDC", + "NIX_SND_STATUS_SUBDC_ORDER_ERR", + "NIX_SND_STATUS_DATA_FAULT", + "NIX_SND_STATUS_DATA_POISON", + "NIX_SND_STATUS_NPC_DROP_ACTION", + "NIX_SND_STATUS_LOCK_VIOL", + "NIX_SND_STATUS_NPC_UCAST_CHAN_ERR", + "NIX_SND_STATUS_NPC_MCAST_CHAN_ERR", + "NIX_SND_STATUS_NPC_MCAST_ABORT", + "NIX_SND_STATUS_NPC_VTAG_PTR_ERR", + "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR", + "NIX_SND_STATUS_SEND_STATS_ERR", +}; + static irqreturn_t otx2_q_intr_handler(int irq, void *data) { struct otx2_nic *pf = data; @@ -1194,46 +1258,67 @@ static irqreturn_t otx2_q_intr_handler(int irq, void *data) /* SQ */ for (qidx = 0; qidx < pf->hw.tot_tx_queues; qidx++) { + u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg; + u8 sq_op_err_code, mnq_err_code, snd_err_code; + + /* Below debug registers captures first errors corresponding to + * those registers. We don't have to check against SQ qid as + * these are fatal errors. + */ + ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT); val = otx2_atomic64_add((qidx << 44), ptr); otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) | (val & NIX_SQINT_BITS)); - if (!(val & (NIX_SQINT_BITS | BIT_ULL(42)))) - continue; - if (val & BIT_ULL(42)) { netdev_err(pf->netdev, "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n", qidx, otx2_read64(pf, NIX_LF_ERR_INT)); - } else { - if (val & BIT_ULL(NIX_SQINT_LMT_ERR)) { - netdev_err(pf->netdev, "SQ%lld: LMT store error NIX_LF_SQ_OP_ERR_DBG:0x%llx", - qidx, - otx2_read64(pf, - NIX_LF_SQ_OP_ERR_DBG)); - otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_MNQ_ERR)) { - netdev_err(pf->netdev, "SQ%lld: Meta-descriptor enqueue error NIX_LF_MNQ_ERR_DGB:0x%llx\n", - qidx, - otx2_read64(pf, NIX_LF_MNQ_ERR_DBG)); - otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_SEND_ERR)) { - netdev_err(pf->netdev, "SQ%lld: Send error, NIX_LF_SEND_ERR_DBG 0x%llx", - qidx, - otx2_read64(pf, - NIX_LF_SEND_ERR_DBG)); - otx2_write64(pf, NIX_LF_SEND_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) - netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", - qidx); + goto done; + } + + sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG); + if (!(sq_op_err_dbg & BIT(44))) + goto chk_mnq_err_dbg; + + sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(%llx) err=%s\n", + qidx, sq_op_err_dbg, nix_sqoperr_e_str[sq_op_err_code]); + + otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44)); + + if (sq_op_err_code == NIX_SQOPERR_SQB_NULL) + goto chk_mnq_err_dbg; + + /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure. + * TODO: But we are in irq context. How to call mbox functions which does sleep + */ + +chk_mnq_err_dbg: + mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG); + if (!(mnq_err_dbg & BIT(44))) + goto chk_snd_err_dbg; + + mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_MNQ_ERR_DBG(%llx) err=%s\n", + qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code]); + otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44)); + +chk_snd_err_dbg: + snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG); + if (snd_err_dbg & BIT(44)) { + snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s\n", + qidx, snd_err_dbg, nix_snd_status_e_str[snd_err_code]); + otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44)); } +done: + /* Print values and reset */ + if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) + netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", + qidx); + schedule_work(&pf->reset_task); } @@ -1389,18 +1474,40 @@ static int otx2_init_hw_resources(struct otx2_nic *pf) goto err_free_sq_ptrs; } +#ifdef CONFIG_DCB + if (pf->pfc_en) { + err = otx2_pfc_txschq_alloc(pf); + if (err) { + mutex_unlock(&mbox->lock); + goto err_free_sq_ptrs; + } + } +#endif + err = otx2_config_nix_queues(pf); if (err) { mutex_unlock(&mbox->lock); goto err_free_txsch; } + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { - err = otx2_txschq_config(pf, lvl); + err = otx2_txschq_config(pf, lvl, 0, false); if (err) { mutex_unlock(&mbox->lock); goto err_free_nix_queues; } } + +#ifdef CONFIG_DCB + if (pf->pfc_en) { + err = otx2_pfc_txschq_config(pf); + if (err) { + mutex_unlock(&mbox->lock); + goto err_free_nix_queues; + } + } +#endif + mutex_unlock(&mbox->lock); return err; @@ -1455,6 +1562,11 @@ static void otx2_free_hw_resources(struct otx2_nic *pf) if (err) dev_err(pf->dev, "RVUPF: Failed to stop/free TX schedulers\n"); +#ifdef CONFIG_DCB + if (pf->pfc_en) + otx2_pfc_txschq_stop(pf); +#endif + mutex_lock(&mbox->lock); /* Disable backpressure */ if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK)) @@ -1634,8 +1746,7 @@ int otx2_open(struct net_device *netdev) cq_poll->dev = (void *)pf; cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; INIT_WORK(&cq_poll->dim.work, otx2_dim_work); - netif_napi_add(netdev, &cq_poll->napi, - otx2_napi_handler, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler); napi_enable(&cq_poll->napi); } @@ -1853,6 +1964,30 @@ static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +static u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb, + struct net_device *sb_dev) +{ +#ifdef CONFIG_DCB + struct otx2_nic *pf = netdev_priv(netdev); + u8 vlan_prio; +#endif + +#ifdef CONFIG_DCB + if (!skb->vlan_present) + goto pick_tx; + + vlan_prio = skb->vlan_tci >> 13; + if ((vlan_prio > pf->hw.tx_queues - 1) || + !pf->pfc_alloc_status[vlan_prio]) + goto pick_tx; + + return vlan_prio; + +pick_tx: +#endif + return netdev_pick_tx(netdev, skb, NULL); +} + static netdev_features_t otx2_fix_features(struct net_device *dev, netdev_features_t features) { @@ -1987,8 +2122,19 @@ int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr) switch (config.tx_type) { case HWTSTAMP_TX_OFF: + if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC) + pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC; + + cancel_delayed_work(&pfvf->ptp->synctstamp_work); otx2_config_hw_tx_tstamp(pfvf, false); break; + case HWTSTAMP_TX_ONESTEP_SYNC: + if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) + return -ERANGE; + pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC; + schedule_delayed_work(&pfvf->ptp->synctstamp_work, + msecs_to_jiffies(500)); + fallthrough; case HWTSTAMP_TX_ON: otx2_config_hw_tx_tstamp(pfvf, true); break; @@ -2447,6 +2593,7 @@ static const struct net_device_ops otx2_netdev_ops = { .ndo_open = otx2_open, .ndo_stop = otx2_stop, .ndo_start_xmit = otx2_xmit, + .ndo_select_queue = otx2_select_queue, .ndo_fix_features = otx2_fix_features, .ndo_set_mac_address = otx2_set_mac_address, .ndo_change_mtu = otx2_change_mtu, @@ -2702,6 +2849,10 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_ptp_destroy; + err = cn10k_mcs_init(pf); + if (err) + goto err_del_mcam_entries; + if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT) netdev->hw_features |= NETIF_F_NTUPLE; @@ -2734,7 +2885,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_del_mcam_entries; + goto err_mcs_free; } err = otx2_wq_init(pf); @@ -2773,6 +2924,8 @@ err_mcam_flow_del: otx2_mcam_flow_del(pf); err_unreg_netdev: unregister_netdev(netdev); +err_mcs_free: + cn10k_mcs_free(pf); err_del_mcam_entries: otx2_mcam_flow_del(pf); err_ptp_destroy: @@ -2916,6 +3069,8 @@ static void otx2_remove(struct pci_dev *pdev) otx2_config_pause_frm(pf); } + cn10k_mcs_free(pf); + #ifdef CONFIG_DCB /* Disable PFC config */ if (pf->pfc_en) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c index fdc2c9315b91..896b2f9bac34 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c @@ -10,6 +10,33 @@ #include "otx2_common.h" #include "otx2_ptp.h" +static u64 otx2_ptp_get_clock(struct otx2_ptp *ptp) +{ + struct ptp_req *req; + struct ptp_rsp *rsp; + int err; + + if (!ptp->nic) + return 0; + + req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox); + if (!req) + return 0; + + req->op = PTP_OP_GET_CLOCK; + + err = otx2_sync_mbox_msg(&ptp->nic->mbox); + if (err) + return 0; + + rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0, + &req->hdr); + if (IS_ERR(rsp)) + return 0; + + return rsp->clk; +} + static int otx2_ptp_adjfine(struct ptp_clock_info *ptp_info, long scaled_ppm) { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, @@ -46,32 +73,28 @@ static int ptp_set_thresh(struct otx2_ptp *ptp, u64 thresh) return otx2_sync_mbox_msg(&ptp->nic->mbox); } -static u64 ptp_cc_read(const struct cyclecounter *cc) +static int ptp_extts_on(struct otx2_ptp *ptp, int on) { - struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter); struct ptp_req *req; - struct ptp_rsp *rsp; - int err; if (!ptp->nic) - return 0; + return -ENODEV; req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox); if (!req) - return 0; + return -ENOMEM; - req->op = PTP_OP_GET_CLOCK; + req->op = PTP_OP_EXTTS_ON; + req->extts_on = on; - err = otx2_sync_mbox_msg(&ptp->nic->mbox); - if (err) - return 0; + return otx2_sync_mbox_msg(&ptp->nic->mbox); +} - rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0, - &req->hdr); - if (IS_ERR(rsp)) - return 0; +static u64 ptp_cc_read(const struct cyclecounter *cc) +{ + struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter); - return rsp->clk; + return otx2_ptp_get_clock(ptp); } static u64 ptp_tstmp_read(struct otx2_ptp *ptp) @@ -101,6 +124,15 @@ static u64 ptp_tstmp_read(struct otx2_ptp *ptp) return rsp->clk; } +static void otx2_get_ptpclock(struct otx2_ptp *ptp, u64 *tstamp) +{ + struct otx2_nic *pfvf = ptp->nic; + + mutex_lock(&pfvf->mbox.lock); + *tstamp = timecounter_read(&ptp->time_counter); + mutex_unlock(&pfvf->mbox.lock); +} + static int otx2_ptp_adjtime(struct ptp_clock_info *ptp_info, s64 delta) { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, @@ -119,14 +151,10 @@ static int otx2_ptp_gettime(struct ptp_clock_info *ptp_info, { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, ptp_info); - struct otx2_nic *pfvf = ptp->nic; - u64 nsec; + u64 tstamp; - mutex_lock(&pfvf->mbox.lock); - nsec = timecounter_read(&ptp->time_counter); - mutex_unlock(&pfvf->mbox.lock); - - *ts = ns_to_timespec64(nsec); + otx2_get_ptpclock(ptp, &tstamp); + *ts = ns_to_timespec64(tstamp); return 0; } @@ -178,8 +206,6 @@ static void otx2_ptp_extts_check(struct work_struct *work) event.index = 0; event.timestamp = timecounter_cyc2time(&ptp->time_counter, tstmp); ptp_clock_event(ptp->ptp_clock, &event); - ptp->last_extts = tstmp; - new_thresh = tstmp % 500000000; if (ptp->thresh != new_thresh) { mutex_lock(&ptp->nic->mbox.lock); @@ -187,10 +213,28 @@ static void otx2_ptp_extts_check(struct work_struct *work) mutex_unlock(&ptp->nic->mbox.lock); ptp->thresh = new_thresh; } + ptp->last_extts = tstmp; } schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200)); } +static void otx2_sync_tstamp(struct work_struct *work) +{ + struct otx2_ptp *ptp = container_of(work, struct otx2_ptp, + synctstamp_work.work); + struct otx2_nic *pfvf = ptp->nic; + u64 tstamp; + + mutex_lock(&pfvf->mbox.lock); + tstamp = otx2_ptp_get_clock(ptp); + mutex_unlock(&pfvf->mbox.lock); + + ptp->tstamp = timecounter_cyc2time(&pfvf->ptp->time_counter, tstamp); + ptp->base_ns = tstamp % NSEC_PER_SEC; + + schedule_delayed_work(&ptp->synctstamp_work, msecs_to_jiffies(250)); +} + static int otx2_ptp_enable(struct ptp_clock_info *ptp_info, struct ptp_clock_request *rq, int on) { @@ -207,10 +251,13 @@ static int otx2_ptp_enable(struct ptp_clock_info *ptp_info, rq->extts.index); if (pin < 0) return -EBUSY; - if (on) + if (on) { + ptp_extts_on(ptp, on); schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200)); - else + } else { + ptp_extts_on(ptp, on); cancel_delayed_work_sync(&ptp->extts_work); + } return 0; default: break; @@ -302,6 +349,8 @@ int otx2_ptp_init(struct otx2_nic *pfvf) ptp_ptr->convert_tx_ptp_tstmp = &cn10k_ptp_convert_timestamp; } + INIT_DELAYED_WORK(&ptp_ptr->synctstamp_work, otx2_sync_tstamp); + pfvf->ptp = ptp_ptr; error: @@ -316,6 +365,8 @@ void otx2_ptp_destroy(struct otx2_nic *pfvf) if (!ptp) return; + cancel_delayed_work(&pfvf->ptp->synctstamp_work); + ptp_clock_unregister(ptp->ptp_clock); kfree(ptp); pfvf->ptp = NULL; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h index 4bbd12ff26e6..fa37b9f312ca 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h @@ -236,8 +236,15 @@ struct nix_sqe_sg_s { /* NIX send memory subdescriptor structure */ struct nix_sqe_mem_s { - u64 offset : 16; /* W0 */ - u64 rsvd_51_16 : 36; + u64 start_offset : 8; + u64 rsvd_11_8 : 4; + u64 rsvd_12 : 1; + u64 udp_csum_crt : 1; + u64 update64 : 1; + u64 rsvd_15_16 : 1; + u64 base_ns : 32; + u64 step_type : 1; + u64 rsvd_51_49 : 3; u64 per_lso_seg : 1; u64 wmem : 1; u64 dsz : 2; @@ -274,4 +281,61 @@ enum nix_sqint_e { BIT_ULL(NIX_SQINT_SEND_ERR) | \ BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) +enum nix_sqoperr_e { + NIX_SQOPERR_OOR = 0, + NIX_SQOPERR_CTX_FAULT = 1, + NIX_SQOPERR_CTX_POISON = 2, + NIX_SQOPERR_DISABLED = 3, + NIX_SQOPERR_SIZE_ERR = 4, + NIX_SQOPERR_OFLOW = 5, + NIX_SQOPERR_SQB_NULL = 6, + NIX_SQOPERR_SQB_FAULT = 7, + NIX_SQOPERR_SQE_SZ_ZERO = 8, + NIX_SQOPERR_MAX, +}; + +enum nix_mnqerr_e { + NIX_MNQERR_SQ_CTX_FAULT = 0, + NIX_MNQERR_SQ_CTX_POISON = 1, + NIX_MNQERR_SQB_FAULT = 2, + NIX_MNQERR_SQB_POISON = 3, + NIX_MNQERR_TOTAL_ERR = 4, + NIX_MNQERR_LSO_ERR = 5, + NIX_MNQERR_CQ_QUERY_ERR = 6, + NIX_MNQERR_MAX_SQE_SIZE_ERR = 7, + NIX_MNQERR_MAXLEN_ERR = 8, + NIX_MNQERR_SQE_SIZEM1_ZERO = 9, + NIX_MNQERR_MAX, +}; + +enum nix_snd_status_e { + NIX_SND_STATUS_GOOD = 0x0, + NIX_SND_STATUS_SQ_CTX_FAULT = 0x1, + NIX_SND_STATUS_SQ_CTX_POISON = 0x2, + NIX_SND_STATUS_SQB_FAULT = 0x3, + NIX_SND_STATUS_SQB_POISON = 0x4, + NIX_SND_STATUS_HDR_ERR = 0x5, + NIX_SND_STATUS_EXT_ERR = 0x6, + NIX_SND_STATUS_JUMP_FAULT = 0x7, + NIX_SND_STATUS_JUMP_POISON = 0x8, + NIX_SND_STATUS_CRC_ERR = 0x9, + NIX_SND_STATUS_IMM_ERR = 0x10, + NIX_SND_STATUS_SG_ERR = 0x11, + NIX_SND_STATUS_MEM_ERR = 0x12, + NIX_SND_STATUS_INVALID_SUBDC = 0x13, + NIX_SND_STATUS_SUBDC_ORDER_ERR = 0x14, + NIX_SND_STATUS_DATA_FAULT = 0x15, + NIX_SND_STATUS_DATA_POISON = 0x16, + NIX_SND_STATUS_NPC_DROP_ACTION = 0x17, + NIX_SND_STATUS_LOCK_VIOL = 0x18, + NIX_SND_STATUS_NPC_UCAST_CHAN_ERR = 0x19, + NIX_SND_STATUS_NPC_MCAST_CHAN_ERR = 0x20, + NIX_SND_STATUS_NPC_MCAST_ABORT = 0x21, + NIX_SND_STATUS_NPC_VTAG_PTR_ERR = 0x22, + NIX_SND_STATUS_NPC_VTAG_SIZE_ERR = 0x23, + NIX_SND_STATUS_SEND_MEM_FAULT = 0x24, + NIX_SND_STATUS_SEND_STATS_ERR = 0x25, + NIX_SND_STATUS_MAX, +}; + #endif /* OTX2_STRUCT_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index a18e8efd0f1e..ef10aef3cda0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -19,6 +19,12 @@ #include "cn10k.h" #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx))) +#define PTP_PORT 0x13F +/* PTPv2 header Original Timestamp starts at byte offset 34 and + * contains 6 byte seconds field and 4 byte nano seconds field. + */ +#define PTP_SYNC_SEC_OFFSET 34 + static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, struct bpf_prog *prog, struct nix_cqe_rx_s *cqe, @@ -435,6 +441,7 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int budget) { int tx_pkts = 0, tx_bytes = 0, qidx; + struct otx2_snd_queue *sq; struct nix_cqe_tx_s *cqe; int processed_cqe = 0; @@ -445,6 +452,9 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, return 0; process_cqe: + qidx = cq->cq_idx - pfvf->hw.rx_queues; + sq = &pfvf->qset.sq[qidx]; + while (likely(processed_cqe < budget) && cq->pend_cqe) { cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq); if (unlikely(!cqe)) { @@ -452,18 +462,20 @@ process_cqe: return 0; break; } + if (cq->cq_type == CQ_XDP) { - qidx = cq->cq_idx - pfvf->hw.rx_queues; - otx2_xdp_snd_pkt_handler(pfvf, &pfvf->qset.sq[qidx], - cqe); + otx2_xdp_snd_pkt_handler(pfvf, sq, cqe); } else { - otx2_snd_pkt_handler(pfvf, cq, - &pfvf->qset.sq[cq->cint_idx], - cqe, budget, &tx_pkts, &tx_bytes); + otx2_snd_pkt_handler(pfvf, cq, sq, cqe, budget, + &tx_pkts, &tx_bytes); } + cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID; processed_cqe++; cq->pend_cqe--; + + sq->cons_head++; + sq->cons_head &= (sq->sqe_cnt - 1); } /* Free CQEs to HW */ @@ -686,7 +698,8 @@ static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, } static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, - int alg, u64 iova) + int alg, u64 iova, int ptp_offset, + u64 base_ns, int udp_csum) { struct nix_sqe_mem_s *mem; @@ -696,6 +709,13 @@ static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, mem->wmem = 1; /* wait for the memory operation */ mem->addr = iova; + if (ptp_offset) { + mem->start_offset = ptp_offset; + mem->udp_csum_crt = udp_csum; + mem->base_ns = base_ns; + mem->step_type = 1; + } + *offset += sizeof(*mem); } @@ -952,16 +972,102 @@ static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb) return skb_shinfo(skb)->gso_segs; } +static bool otx2_validate_network_transport(struct sk_buff *skb) +{ + if ((ip_hdr(skb)->protocol == IPPROTO_UDP) || + (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) { + struct udphdr *udph = udp_hdr(skb); + + if (udph->source == htons(PTP_PORT) && + udph->dest == htons(PTP_PORT)) + return true; + } + + return false; +} + +static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, int *udp_csum) +{ + struct ethhdr *eth = (struct ethhdr *)(skb->data); + u16 nix_offload_hlen = 0, inner_vhlen = 0; + u8 *data = skb->data, *msgtype; + __be16 proto = eth->h_proto; + int network_depth = 0; + + /* NIX is programmed to offload outer VLAN header + * in case of single vlan protocol field holds Network header ETH_IP/V6 + * in case of stacked vlan protocol field holds Inner vlan (8100) + */ + if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX && + skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) { + if (skb->vlan_proto == htons(ETH_P_8021AD)) { + /* Get vlan protocol */ + proto = __vlan_get_protocol(skb, eth->h_proto, NULL); + /* SKB APIs like skb_transport_offset does not include + * offloaded vlan header length. Need to explicitly add + * the length + */ + nix_offload_hlen = VLAN_HLEN; + inner_vhlen = VLAN_HLEN; + } else if (skb->vlan_proto == htons(ETH_P_8021Q)) { + nix_offload_hlen = VLAN_HLEN; + } + } else if (eth_type_vlan(eth->h_proto)) { + proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth); + } + + switch (ntohs(proto)) { + case ETH_P_1588: + if (network_depth) + *offset = network_depth; + else + *offset = ETH_HLEN + nix_offload_hlen + + inner_vhlen; + break; + case ETH_P_IP: + case ETH_P_IPV6: + if (!otx2_validate_network_transport(skb)) + return false; + + *udp_csum = 1; + *offset = nix_offload_hlen + skb_transport_offset(skb) + + sizeof(struct udphdr); + } + + msgtype = data + *offset; + + /* Check PTP messageId is SYNC or not */ + return (*msgtype & 0xf) == 0; +} + static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb, struct otx2_snd_queue *sq, int *offset) { + struct ptpv2_tstamp *origin_tstamp; + int ptp_offset = 0, udp_csum = 0; + struct timespec64 ts; u64 iova; - if (!skb_shinfo(skb)->gso_size && - skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + if (unlikely(!skb_shinfo(skb)->gso_size && + (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) { + if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)) { + if (otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum)) { + origin_tstamp = (struct ptpv2_tstamp *) + ((u8 *)skb->data + ptp_offset + + PTP_SYNC_SEC_OFFSET); + ts = ns_to_timespec64(pfvf->ptp->tstamp); + origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff); + origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff); + origin_tstamp->nanoseconds = htonl(ts.tv_nsec); + /* Point to correction field in PTP packet */ + ptp_offset += 8; + } + } else { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + } iova = sq->timestamps->iova + (sq->head * sizeof(u64)); - otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova); + otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova, + ptp_offset, pfvf->ptp->base_ns, udp_csum); } else { skb_tx_timestamp(skb); } @@ -972,17 +1078,17 @@ bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq, { struct netdev_queue *txq = netdev_get_tx_queue(netdev, qidx); struct otx2_nic *pfvf = netdev_priv(netdev); - int offset, num_segs, free_sqe; + int offset, num_segs, free_desc; struct nix_sqe_hdr_s *sqe_hdr; - /* Check if there is room for new SQE. - * 'Num of SQBs freed to SQ's pool - SQ's Aura count' - * will give free SQE count. + /* Check if there is enough room between producer + * and consumer index. */ - free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb; + free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1); + if (free_desc < sq->sqe_thresh) + return false; - if (free_sqe < sq->sqe_thresh || - free_sqe < otx2_get_sqe_count(pfvf, skb)) + if (free_desc < otx2_get_sqe_count(pfvf, skb)) return false; num_segs = skb_shinfo(skb)->nr_frags + 1; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h index fbe62bbfb789..93cac2c2664c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h @@ -79,6 +79,7 @@ struct sg_list { struct otx2_snd_queue { u8 aura_id; u16 head; + u16 cons_head; u16 sqe_size; u32 sqe_cnt; u16 num_sqbs; diff --git a/drivers/net/ethernet/marvell/prestera/Makefile b/drivers/net/ethernet/marvell/prestera/Makefile index d395f4131648..df14cee80153 100644 --- a/drivers/net/ethernet/marvell/prestera/Makefile +++ b/drivers/net/ethernet/marvell/prestera/Makefile @@ -4,6 +4,6 @@ prestera-objs := prestera_main.o prestera_hw.o prestera_dsa.o \ prestera_rxtx.o prestera_devlink.o prestera_ethtool.o \ prestera_switchdev.o prestera_acl.o prestera_flow.o \ prestera_flower.o prestera_span.o prestera_counter.o \ - prestera_router.o prestera_router_hw.o + prestera_router.o prestera_router_hw.o prestera_matchall.o obj-$(CONFIG_PRESTERA_PCI) += prestera_pci.o diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h index 2f84d0fb4094..35554ee805cd 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera.h +++ b/drivers/net/ethernet/marvell/prestera/prestera.h @@ -306,17 +306,27 @@ struct prestera_switch { struct prestera_counter *counter; u8 lag_member_max; u8 lag_max; + u32 size_tbl_router_nexthop; }; struct prestera_router { struct prestera_switch *sw; struct list_head vr_list; struct list_head rif_entry_list; + struct rhashtable nh_neigh_ht; + struct rhashtable nexthop_group_ht; struct rhashtable fib_ht; + struct rhashtable kern_neigh_cache_ht; struct rhashtable kern_fib_cache_ht; struct notifier_block inetaddr_nb; struct notifier_block inetaddr_valid_nb; struct notifier_block fib_nb; + struct notifier_block netevent_nb; + u8 *nhgrp_hw_state_cache; /* Bitmap cached hw state of nhs */ + unsigned long nhgrp_hw_cache_kick; /* jiffies */ + struct { + struct delayed_work dw; + } neighs_update; }; struct prestera_rxtx_params { @@ -362,11 +372,15 @@ int prestera_port_cfg_mac_write(struct prestera_port *port, struct prestera_port *prestera_port_dev_lower_find(struct net_device *dev); void prestera_queue_work(struct work_struct *work); +void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay); +void prestera_queue_drain(void); int prestera_port_learning_set(struct prestera_port *port, bool learn_enable); int prestera_port_uc_flood_set(struct prestera_port *port, bool flood); int prestera_port_mc_flood_set(struct prestera_port *port, bool flood); +int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked); + int prestera_port_pvid_set(struct prestera_port *port, u16 vid); bool prestera_netdev_check(const struct net_device *dev); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c index 3d4b85f2d541..cba89fda504b 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c @@ -54,6 +54,10 @@ struct prestera_acl_ruleset { struct prestera_acl_ruleset_ht_key ht_key; struct rhashtable rule_ht; struct prestera_acl *acl; + struct { + u32 min; + u32 max; + } prio; unsigned long rule_count; refcount_t refcount; void *keymask; @@ -162,6 +166,9 @@ prestera_acl_ruleset_create(struct prestera_acl *acl, ruleset->pcl_id = PRESTERA_ACL_PCL_ID_MAKE((u8)uid, chain_index); ruleset->index = uid; + ruleset->prio.min = UINT_MAX; + ruleset->prio.max = 0; + err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node, prestera_acl_ruleset_ht_params); if (err) @@ -178,10 +185,14 @@ err_rhashtable_init: return ERR_PTR(err); } -void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, - void *keymask) +int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, + void *keymask) { ruleset->keymask = kmemdup(keymask, ACL_KEYMASK_SIZE, GFP_KERNEL); + if (!ruleset->keymask) + return -ENOMEM; + + return 0; } int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset) @@ -365,6 +376,26 @@ prestera_acl_ruleset_block_unbind(struct prestera_acl_ruleset *ruleset, block->ruleset_zero = NULL; } +static void +prestera_acl_ruleset_prio_refresh(struct prestera_acl *acl, + struct prestera_acl_ruleset *ruleset) +{ + struct prestera_acl_rule *rule; + + ruleset->prio.min = UINT_MAX; + ruleset->prio.max = 0; + + list_for_each_entry(rule, &acl->rules, list) { + if (ruleset->ingress != rule->ruleset->ingress) + continue; + if (ruleset->ht_key.chain_index != rule->chain_index) + continue; + + ruleset->prio.min = min(ruleset->prio.min, rule->priority); + ruleset->prio.max = max(ruleset->prio.max, rule->priority); + } +} + void prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id) { @@ -389,6 +420,13 @@ u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset) return ruleset->index; } +void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset, + u32 *prio_min, u32 *prio_max) +{ + *prio_min = ruleset->prio.min; + *prio_max = ruleset->prio.max; +} + bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset) { return ruleset->offload; @@ -429,6 +467,13 @@ void prestera_acl_rule_destroy(struct prestera_acl_rule *rule) kfree(rule); } +static void prestera_acl_ruleset_prio_update(struct prestera_acl_ruleset *ruleset, + u32 prio) +{ + ruleset->prio.min = min(ruleset->prio.min, prio); + ruleset->prio.max = max(ruleset->prio.max, prio); +} + int prestera_acl_rule_add(struct prestera_switch *sw, struct prestera_acl_rule *rule) { @@ -468,6 +513,7 @@ int prestera_acl_rule_add(struct prestera_switch *sw, list_add_tail(&rule->list, &sw->acl->rules); ruleset->rule_count++; + prestera_acl_ruleset_prio_update(ruleset, rule->priority); return 0; err_acl_block_bind: @@ -492,6 +538,7 @@ void prestera_acl_rule_del(struct prestera_switch *sw, list_del(&rule->list); prestera_acl_rule_entry_destroy(sw->acl, rule->re); + prestera_acl_ruleset_prio_refresh(sw->acl, ruleset); /* unbind block (all ports) */ if (!ruleset->ht_key.chain_index && !ruleset->rule_count) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.h b/drivers/net/ethernet/marvell/prestera/prestera_acl.h index 03fc5b9dc925..a35cc0609a1d 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.h @@ -185,8 +185,8 @@ struct prestera_acl_ruleset * prestera_acl_ruleset_lookup(struct prestera_acl *acl, struct prestera_flow_block *block, u32 chain_index); -void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, - void *keymask); +int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, + void *keymask); bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset); int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset); void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset); @@ -195,6 +195,8 @@ int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset, int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset, struct prestera_port *port); u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset); +void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset, + u32 *prio_min, u32 *prio_max); void prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c index 1da7ff889417..2f52daba58e6 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c @@ -300,8 +300,8 @@ static void prestera_ethtool_get_drvinfo(struct net_device *dev, struct prestera_port *port = netdev_priv(dev); struct prestera_switch *sw = port->sw; - strlcpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, dev_name(prestera_dev(sw)), + strscpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, dev_name(prestera_dev(sw)), sizeof(drvinfo->bus_info)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.c b/drivers/net/ethernet/marvell/prestera/prestera_flow.c index 2262693bd5cf..9f4267f326b0 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.c @@ -7,8 +7,9 @@ #include "prestera.h" #include "prestera_acl.h" #include "prestera_flow.h" -#include "prestera_span.h" #include "prestera_flower.h" +#include "prestera_matchall.h" +#include "prestera_span.h" static LIST_HEAD(prestera_block_cb_list); @@ -17,9 +18,9 @@ static int prestera_flow_block_mall_cb(struct prestera_flow_block *block, { switch (f->command) { case TC_CLSMATCHALL_REPLACE: - return prestera_span_replace(block, f); + return prestera_mall_replace(block, f); case TC_CLSMATCHALL_DESTROY: - prestera_span_destroy(block); + prestera_mall_destroy(block); return 0; default: return -EOPNOTSUPP; @@ -89,6 +90,9 @@ prestera_flow_block_create(struct prestera_switch *sw, INIT_LIST_HEAD(&block->template_list); block->net = net; block->sw = sw; + block->mall.prio_min = UINT_MAX; + block->mall.prio_max = 0; + block->mall.bound = false; block->ingress = ingress; return block; @@ -263,7 +267,7 @@ static void prestera_setup_flow_block_unbind(struct prestera_port *port, block = flow_block_cb_priv(block_cb); - prestera_span_destroy(block); + prestera_mall_destroy(block); err = prestera_flow_block_unbind(block, port); if (err) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.h b/drivers/net/ethernet/marvell/prestera/prestera_flow.h index 0c9e13263261..a85a3eb40279 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.h @@ -22,6 +22,11 @@ struct prestera_flow_block { struct prestera_acl_ruleset *ruleset_zero; struct flow_block_cb *block_cb; struct list_head template_list; + struct { + u32 prio_min; + u32 prio_max; + bool bound; + } mall; unsigned int rule_count; bool ingress; }; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index 19d3b55c578e..91a478b75cbf 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -5,6 +5,7 @@ #include "prestera_acl.h" #include "prestera_flow.h" #include "prestera_flower.h" +#include "prestera_matchall.h" struct prestera_flower_template { struct prestera_acl_ruleset *ruleset; @@ -360,6 +361,49 @@ static int prestera_flower_parse(struct prestera_flow_block *block, f->common.extack); } +static int prestera_flower_prio_check(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + u32 mall_prio_min; + u32 mall_prio_max; + int err; + + err = prestera_mall_prio_get(block, &mall_prio_min, &mall_prio_max); + if (err == -ENOENT) + /* No matchall filters installed on this chain. */ + return 0; + + if (err) { + NL_SET_ERR_MSG(f->common.extack, "Failed to get matchall priorities"); + return err; + } + + if (f->common.prio <= mall_prio_max && block->ingress) { + NL_SET_ERR_MSG(f->common.extack, + "Failed to add in front of existing matchall rules"); + return -EOPNOTSUPP; + } + if (f->common.prio >= mall_prio_min && !block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing matchall rules"); + return -EOPNOTSUPP; + } + + return 0; +} + +int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index, + u32 *prio_min, u32 *prio_max) +{ + struct prestera_acl_ruleset *ruleset; + + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block, chain_index); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + prestera_acl_ruleset_prio_get(ruleset, prio_min, prio_max); + return 0; +} + int prestera_flower_replace(struct prestera_flow_block *block, struct flow_cls_offload *f) { @@ -368,6 +412,10 @@ int prestera_flower_replace(struct prestera_flow_block *block, struct prestera_acl_rule *rule; int err; + err = prestera_flower_prio_check(block, f); + if (err) + return err; + ruleset = prestera_acl_ruleset_get(acl, block, f->common.chain_index); if (IS_ERR(ruleset)) return PTR_ERR(ruleset); @@ -452,7 +500,9 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block, } /* preserve keymask/template to this ruleset */ - prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask); + err = prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask); + if (err) + goto err_ruleset_keymask_set; /* skip error, as it is not possible to reject template operation, * so, keep the reference to the ruleset for rules to be added @@ -468,6 +518,8 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block, list_add_rcu(&template->list, &block->template_list); return 0; +err_ruleset_keymask_set: + prestera_acl_ruleset_put(ruleset); err_ruleset_get: kfree(template); err_malloc: diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.h b/drivers/net/ethernet/marvell/prestera/prestera_flower.h index 495f151e6fa9..1181115fe6fa 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.h @@ -19,5 +19,7 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block, void prestera_flower_tmplt_destroy(struct prestera_flow_block *block, struct flow_cls_offload *f); void prestera_flower_template_cleanup(struct prestera_flow_block *block); +int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index, + u32 *prio_min, u32 *prio_max); #endif /* _PRESTERA_FLOWER_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_hw.c index 962d7e0c0cb5..fc6f7d2746e8 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.c @@ -10,11 +10,14 @@ #include "prestera_hw.h" #include "prestera_acl.h" #include "prestera_counter.h" +#include "prestera_router_hw.h" #define PRESTERA_SWITCH_INIT_TIMEOUT_MS (30 * 1000) #define PRESTERA_MIN_MTU 64 +#define PRESTERA_MSG_CHUNK_SIZE 1024 + enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_SWITCH_INIT = 0x1, PRESTERA_CMD_TYPE_SWITCH_ATTR_SET = 0x2, @@ -57,6 +60,10 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_ROUTER_RIF_DELETE = 0x601, PRESTERA_CMD_TYPE_ROUTER_LPM_ADD = 0x610, PRESTERA_CMD_TYPE_ROUTER_LPM_DELETE = 0x611, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET = 0x622, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET = 0x645, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD = 0x623, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE = 0x624, PRESTERA_CMD_TYPE_ROUTER_VR_CREATE = 0x630, PRESTERA_CMD_TYPE_ROUTER_VR_DELETE = 0x631, @@ -78,9 +85,11 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_STP_PORT_SET = 0x1000, PRESTERA_CMD_TYPE_SPAN_GET = 0x1100, - PRESTERA_CMD_TYPE_SPAN_BIND = 0x1101, - PRESTERA_CMD_TYPE_SPAN_UNBIND = 0x1102, + PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND = 0x1101, + PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND = 0x1102, PRESTERA_CMD_TYPE_SPAN_RELEASE = 0x1103, + PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND = 0x1104, + PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND = 0x1105, PRESTERA_CMD_TYPE_POLICER_CREATE = 0x1500, PRESTERA_CMD_TYPE_POLICER_RELEASE = 0x1501, @@ -101,6 +110,7 @@ enum { PRESTERA_CMD_PORT_ATTR_LEARNING = 7, PRESTERA_CMD_PORT_ATTR_FLOOD = 8, PRESTERA_CMD_PORT_ATTR_CAPABILITY = 9, + PRESTERA_CMD_PORT_ATTR_LOCKED = 10, PRESTERA_CMD_PORT_ATTR_PHY_MODE = 12, PRESTERA_CMD_PORT_ATTR_TYPE = 13, PRESTERA_CMD_PORT_ATTR_STATS = 17, @@ -285,6 +295,7 @@ union prestera_msg_port_param { u8 duplex; u8 fec; u8 fc; + u8 br_locked; union { struct { u8 admin; @@ -538,6 +549,14 @@ struct prestera_msg_ip_addr { u8 __pad[3]; }; +struct prestera_msg_nh { + struct prestera_msg_iface oif; + __le32 hw_id; + u8 mac[ETH_ALEN]; + u8 is_active; + u8 pad; +}; + struct prestera_msg_rif_req { struct prestera_msg_cmd cmd; struct prestera_msg_iface iif; @@ -563,6 +582,34 @@ struct prestera_msg_lpm_req { u8 __pad[2]; }; +struct prestera_msg_nh_req { + struct prestera_msg_cmd cmd; + struct prestera_msg_nh nh[PRESTERA_NHGR_SIZE_MAX]; + __le32 size; + __le32 grp_id; +}; + +struct prestera_msg_nh_chunk_req { + struct prestera_msg_cmd cmd; + __le32 offset; +}; + +struct prestera_msg_nh_chunk_resp { + struct prestera_msg_ret ret; + u8 hw_state[PRESTERA_MSG_CHUNK_SIZE]; +}; + +struct prestera_msg_nh_grp_req { + struct prestera_msg_cmd cmd; + __le32 grp_id; + __le32 size; +}; + +struct prestera_msg_nh_grp_resp { + struct prestera_msg_ret ret; + __le32 grp_id; +}; + struct prestera_msg_vr_req { struct prestera_msg_cmd cmd; __le16 vr_id; @@ -725,11 +772,15 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_ports_reset_req) != 8); BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_create_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_destroy_req) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_req) != 124); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_req) != 12); /* structure that are part of req/resp fw messages */ BUILD_BUG_ON(sizeof(struct prestera_msg_iface) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_ip_addr) != 20); BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_port) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh) != 28); /* check responses */ BUILD_BUG_ON(sizeof(struct prestera_msg_common_resp) != 8); @@ -745,6 +796,9 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_rif_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_vr_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_policer_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_create_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_resp) != 1032); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_resp) != 12); /* check events */ BUILD_BUG_ON(sizeof(struct prestera_msg_event_port) != 20); @@ -1022,6 +1076,8 @@ int prestera_hw_switch_init(struct prestera_switch *sw) sw->id = resp.switch_id; sw->lag_member_max = resp.lag_member_max; sw->lag_max = resp.lag_max; + sw->size_tbl_router_nexthop = + __le32_to_cpu(resp.size_tbl_router_nexthop); return 0; } @@ -1431,27 +1487,39 @@ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id) return 0; } -int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id) +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id, + bool ingress) { struct prestera_msg_span_req req = { .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), .id = span_id, }; + enum prestera_cmd_type_t cmd_type; + + if (ingress) + cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND; + else + cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND; + + return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req)); - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_BIND, - &req.cmd, sizeof(req)); } -int prestera_hw_span_unbind(const struct prestera_port *port) +int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress) { struct prestera_msg_span_req req = { .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), }; + enum prestera_cmd_type_t cmd_type; - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_UNBIND, - &req.cmd, sizeof(req)); + if (ingress) + cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND; + else + cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND; + + return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req)); } int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id) @@ -1639,6 +1707,22 @@ int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood) &req.cmd, sizeof(req)); } +int prestera_hw_port_br_locked_set(const struct prestera_port *port, + bool br_locked) +{ + struct prestera_msg_port_attr_req req = { + .attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_LOCKED), + .port = __cpu_to_le32(port->hw_id), + .dev = __cpu_to_le32(port->dev_id), + .param = { + .br_locked = br_locked, + } + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_PORT_ATTR_SET, + &req.cmd, sizeof(req)); +} + int prestera_hw_vlan_create(struct prestera_switch *sw, u16 vid) { struct prestera_msg_vlan_req req = { @@ -2004,6 +2088,85 @@ int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id, sizeof(req)); } +int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count, + struct prestera_neigh_info *nhs, u32 grp_id) +{ + struct prestera_msg_nh_req req = { .size = __cpu_to_le32((u32)count), + .grp_id = __cpu_to_le32(grp_id) }; + int i, err; + + for (i = 0; i < count; i++) { + req.nh[i].is_active = nhs[i].connected; + memcpy(&req.nh[i].mac, nhs[i].ha, ETH_ALEN); + err = prestera_iface_to_msg(&nhs[i].iface, &req.nh[i].oif); + if (err) + return err; + } + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET, &req.cmd, + sizeof(req)); +} + +int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw, + u8 *hw_state, u32 buf_size /* Buffer in bytes */) +{ + static struct prestera_msg_nh_chunk_resp resp; + struct prestera_msg_nh_chunk_req req; + u32 buf_offset; + int err; + + memset(&hw_state[0], 0, buf_size); + buf_offset = 0; + while (1) { + if (buf_offset >= buf_size) + break; + + memset(&req, 0, sizeof(req)); + req.offset = __cpu_to_le32(buf_offset * 8); /* 8 bits in u8 */ + err = prestera_cmd_ret(sw, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET, + &req.cmd, sizeof(req), &resp.ret, + sizeof(resp)); + if (err) + return err; + + memcpy(&hw_state[buf_offset], &resp.hw_state[0], + buf_offset + PRESTERA_MSG_CHUNK_SIZE > buf_size ? + buf_size - buf_offset : PRESTERA_MSG_CHUNK_SIZE); + buf_offset += PRESTERA_MSG_CHUNK_SIZE; + } + + return 0; +} + +int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count, + u32 *grp_id) +{ + struct prestera_msg_nh_grp_req req = { .size = __cpu_to_le32((u32)nh_count) }; + struct prestera_msg_nh_grp_resp resp; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *grp_id = __le32_to_cpu(resp.grp_id); + return err; +} + +int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count, + u32 grp_id) +{ + struct prestera_msg_nh_grp_req req = { + .grp_id = __cpu_to_le32(grp_id), + .size = __cpu_to_le32(nh_count) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE, + &req.cmd, sizeof(req)); +} + int prestera_hw_rxtx_init(struct prestera_switch *sw, struct prestera_rxtx_params *params) { diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_hw.h index 56e043146dd2..0a929279e1ce 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.h @@ -146,6 +146,7 @@ struct prestera_counter_stats; struct prestera_iface; struct prestera_flood_domain; struct prestera_mdb_entry; +struct prestera_neigh_info; /* Switch API */ int prestera_hw_switch_init(struct prestera_switch *sw); @@ -183,6 +184,8 @@ int prestera_hw_port_speed_get(const struct prestera_port *port, u32 *speed); int prestera_hw_port_learning_set(struct prestera_port *port, bool enable); int prestera_hw_port_uc_flood_set(const struct prestera_port *port, bool flood); int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood); +int prestera_hw_port_br_locked_set(const struct prestera_port *port, + bool br_locked); int prestera_hw_port_accept_frm_type(struct prestera_port *port, enum prestera_accept_frm_type type); /* Vlan API */ @@ -243,8 +246,9 @@ int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id, /* SPAN API */ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id); -int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id); -int prestera_hw_span_unbind(const struct prestera_port *port); +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id, + bool ingress); +int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress); int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id); /* Router API */ @@ -263,6 +267,16 @@ int prestera_hw_lpm_add(struct prestera_switch *sw, u16 vr_id, int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id, __be32 dst, u32 dst_len); +/* NH API */ +int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count, + struct prestera_neigh_info *nhs, u32 grp_id); +int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw, + u8 *hw_state, u32 buf_size /* Buffer in bytes */); +int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count, + u32 *grp_id); +int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count, + u32 grp_id); + /* Event handlers */ int prestera_hw_event_handler_register(struct prestera_switch *sw, enum prestera_event_type type, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index ede3e53b9790..24f9d6024745 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -36,6 +36,17 @@ void prestera_queue_work(struct work_struct *work) queue_work(prestera_owq, work); } +void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay) +{ + queue_delayed_work(prestera_wq, work, delay); +} + +void prestera_queue_drain(void) +{ + drain_workqueue(prestera_wq); + drain_workqueue(prestera_owq); +} + int prestera_port_learning_set(struct prestera_port *port, bool learn) { return prestera_hw_port_learning_set(port, learn); @@ -51,6 +62,11 @@ int prestera_port_mc_flood_set(struct prestera_port *port, bool flood) return prestera_hw_port_mc_flood_set(port, flood); } +int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked) +{ + return prestera_hw_port_br_locked_set(port, br_locked); +} + int prestera_port_pvid_set(struct prestera_port *port, u16 vid) { enum prestera_accept_frm_type frm_type; @@ -368,6 +384,7 @@ static int prestera_port_sfp_bind(struct prestera_port *port) if (!sw->np) return 0; + of_node_get(sw->np); ports = of_find_node_by_name(sw->np, "ports"); for_each_child_of_node(ports, node) { @@ -417,6 +434,7 @@ static int prestera_port_sfp_bind(struct prestera_port *port) } out: + of_node_put(node); of_node_put(ports); return err; } @@ -797,32 +815,30 @@ static void prestera_port_handle_event(struct prestera_switch *sw, caching_dw = &port->cached_hw_stats.caching_dw; - if (port->phy_link) { - memset(&smac, 0, sizeof(smac)); - smac.valid = true; - smac.oper = pevt->data.mac.oper; - if (smac.oper) { - smac.mode = pevt->data.mac.mode; - smac.speed = pevt->data.mac.speed; - smac.duplex = pevt->data.mac.duplex; - smac.fc = pevt->data.mac.fc; - smac.fec = pevt->data.mac.fec; - phylink_mac_change(port->phy_link, true); - } else { - phylink_mac_change(port->phy_link, false); - } - prestera_port_mac_state_cache_write(port, &smac); + memset(&smac, 0, sizeof(smac)); + smac.valid = true; + smac.oper = pevt->data.mac.oper; + if (smac.oper) { + smac.mode = pevt->data.mac.mode; + smac.speed = pevt->data.mac.speed; + smac.duplex = pevt->data.mac.duplex; + smac.fc = pevt->data.mac.fc; + smac.fec = pevt->data.mac.fec; } + prestera_port_mac_state_cache_write(port, &smac); if (port->state_mac.oper) { - if (!port->phy_link) + if (port->phy_link) + phylink_mac_change(port->phy_link, true); + else netif_carrier_on(port->dev); if (!delayed_work_pending(caching_dw)) queue_delayed_work(prestera_wq, caching_dw, 0); - } else if (netif_running(port->dev) && - netif_carrier_ok(port->dev)) { - if (!port->phy_link) + } else { + if (port->phy_link) + phylink_mac_change(port->phy_link, false); + else if (netif_running(port->dev) && netif_carrier_ok(port->dev)) netif_carrier_off(port->dev); if (delayed_work_pending(caching_dw)) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.c b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c new file mode 100644 index 000000000000..1da9c1bc1ee9 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2019-2022 Marvell International Ltd. All rights reserved */ + +#include <linux/kernel.h> +#include <linux/list.h> + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_flow.h" +#include "prestera_flower.h" +#include "prestera_matchall.h" +#include "prestera_span.h" + +static int prestera_mall_prio_check(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + u32 flower_prio_min; + u32 flower_prio_max; + int err; + + err = prestera_flower_prio_get(block, f->common.chain_index, + &flower_prio_min, &flower_prio_max); + if (err == -ENOENT) + /* No flower filters installed on this chain. */ + return 0; + + if (err) { + NL_SET_ERR_MSG(f->common.extack, "Failed to get flower priorities"); + return err; + } + + if (f->common.prio <= flower_prio_max && !block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add in front of existing flower rules"); + return -EOPNOTSUPP; + } + if (f->common.prio >= flower_prio_min && block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing flower rules"); + return -EOPNOTSUPP; + } + + return 0; +} + +int prestera_mall_prio_get(struct prestera_flow_block *block, + u32 *prio_min, u32 *prio_max) +{ + if (!block->mall.bound) + return -ENOENT; + + *prio_min = block->mall.prio_min; + *prio_max = block->mall.prio_max; + return 0; +} + +static void prestera_mall_prio_update(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + block->mall.prio_min = min(block->mall.prio_min, f->common.prio); + block->mall.prio_max = max(block->mall.prio_max, f->common.prio); +} + +int prestera_mall_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + struct prestera_flow_block_binding *binding; + __be16 protocol = f->common.protocol; + struct flow_action_entry *act; + struct prestera_port *port; + int err; + + if (!flow_offload_has_one_action(&f->rule->action)) { + NL_SET_ERR_MSG(f->common.extack, + "Only singular actions are supported"); + return -EOPNOTSUPP; + } + + act = &f->rule->action.entries[0]; + + if (!prestera_netdev_check(act->dev)) { + NL_SET_ERR_MSG(f->common.extack, + "Only Marvell Prestera port is supported"); + return -EINVAL; + } + if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) + return -EOPNOTSUPP; + if (act->id != FLOW_ACTION_MIRRED) + return -EOPNOTSUPP; + if (protocol != htons(ETH_P_ALL)) + return -EOPNOTSUPP; + + err = prestera_mall_prio_check(block, f); + if (err) + return err; + + port = netdev_priv(act->dev); + + list_for_each_entry(binding, &block->binding_list, list) { + err = prestera_span_rule_add(binding, port, block->ingress); + if (err == -EEXIST) + return err; + if (err) + goto rollback; + } + + prestera_mall_prio_update(block, f); + + block->mall.bound = true; + return 0; + +rollback: + list_for_each_entry_continue_reverse(binding, + &block->binding_list, list) + prestera_span_rule_del(binding, block->ingress); + return err; +} + +void prestera_mall_destroy(struct prestera_flow_block *block) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + prestera_span_rule_del(binding, block->ingress); + + block->mall.prio_min = UINT_MAX; + block->mall.prio_max = 0; + block->mall.bound = false; +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.h b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h new file mode 100644 index 000000000000..fed08be80257 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2022 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_MATCHALL_H_ +#define _PRESTERA_MATCHALL_H_ + +#include <net/pkt_cls.h> + +struct prestera_flow_block; + +int prestera_mall_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f); +void prestera_mall_destroy(struct prestera_flow_block *block); +int prestera_mall_prio_get(struct prestera_flow_block *block, + u32 *prio_min, u32 *prio_max); + +#endif /* _PRESTERA_MATCHALL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_pci.c b/drivers/net/ethernet/marvell/prestera/prestera_pci.c index f538a749ebd4..59470d99f522 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_pci.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_pci.c @@ -872,6 +872,7 @@ static void prestera_pci_remove(struct pci_dev *pdev) static const struct pci_device_id prestera_pci_devices[] = { { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xC804) }, { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xC80C) }, + { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xCC1E) }, { } }; MODULE_DEVICE_TABLE(pci, prestera_pci_devices); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router.c b/drivers/net/ethernet/marvell/prestera/prestera_router.c index 58f4e44d5ad7..4046be0e86ff 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_router.c @@ -7,10 +7,35 @@ #include <net/inet_dscp.h> #include <net/switchdev.h> #include <linux/rhashtable.h> +#include <net/nexthop.h> +#include <net/arp.h> +#include <linux/if_vlan.h> +#include <linux/if_macvlan.h> +#include <net/netevent.h> #include "prestera.h" #include "prestera_router_hw.h" +#define PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH +#define PRESTERA_NH_PROBE_INTERVAL 5000 /* ms */ + +struct prestera_kern_neigh_cache_key { + struct prestera_ip_addr addr; + struct net_device *dev; +}; + +struct prestera_kern_neigh_cache { + struct prestera_kern_neigh_cache_key key; + struct rhash_head ht_node; + struct list_head kern_fib_cache_list; + /* Hold prepared nh_neigh info if is in_kernel */ + struct prestera_neigh_info nh_neigh_info; + /* Indicate if neighbour is reachable by direct route */ + bool reachable; + /* Lock cache if neigh is present in kernel */ + bool in_kernel; +}; + struct prestera_kern_fib_cache_key { struct prestera_ip_addr addr; u32 prefix_len; @@ -23,15 +48,29 @@ struct prestera_kern_fib_cache { struct { struct prestera_fib_key fib_key; enum prestera_fib_type fib_type; + struct prestera_nexthop_group_key nh_grp_key; } lpm_info; /* hold prepared lpm info */ /* Indicate if route is not overlapped by another table */ struct rhash_head ht_node; /* node of prestera_router */ - struct fib_info *fi; - dscp_t kern_dscp; - u8 kern_type; + struct prestera_kern_neigh_cache_head { + struct prestera_kern_fib_cache *this; + struct list_head head; + struct prestera_kern_neigh_cache *n_cache; + } kern_neigh_cache_head[PRESTERA_NHGR_SIZE_MAX]; + union { + struct fib_notifier_info info; /* point to any of 4/6 */ + struct fib_entry_notifier_info fen4_info; + }; bool reachable; }; +static const struct rhashtable_params __prestera_kern_neigh_cache_ht_params = { + .key_offset = offsetof(struct prestera_kern_neigh_cache, key), + .head_offset = offsetof(struct prestera_kern_neigh_cache, ht_node), + .key_len = sizeof(struct prestera_kern_neigh_cache_key), + .automatic_shrinking = true, +}; + static const struct rhashtable_params __prestera_kern_fib_cache_ht_params = { .key_offset = offsetof(struct prestera_kern_fib_cache, key), .head_offset = offsetof(struct prestera_kern_fib_cache, ht_node), @@ -51,15 +90,450 @@ static u32 prestera_fix_tb_id(u32 tb_id) } static void -prestera_util_fen_info2fib_cache_key(struct fib_entry_notifier_info *fen_info, +prestera_util_fen_info2fib_cache_key(struct fib_notifier_info *info, struct prestera_kern_fib_cache_key *key) { + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + memset(key, 0, sizeof(*key)); + key->addr.v = PRESTERA_IPV4; key->addr.u.ipv4 = cpu_to_be32(fen_info->dst); key->prefix_len = fen_info->dst_len; key->kern_tb_id = fen_info->tb_id; } +static int prestera_util_nhc2nc_key(struct prestera_switch *sw, + struct fib_nh_common *nhc, + struct prestera_kern_neigh_cache_key *nk) +{ + memset(nk, 0, sizeof(*nk)); + if (nhc->nhc_gw_family == AF_INET) { + nk->addr.v = PRESTERA_IPV4; + nk->addr.u.ipv4 = nhc->nhc_gw.ipv4; + } else { + nk->addr.v = PRESTERA_IPV6; + nk->addr.u.ipv6 = nhc->nhc_gw.ipv6; + } + + nk->dev = nhc->nhc_dev; + return 0; +} + +static void +prestera_util_nc_key2nh_key(struct prestera_kern_neigh_cache_key *ck, + struct prestera_nh_neigh_key *nk) +{ + memset(nk, 0, sizeof(*nk)); + nk->addr = ck->addr; + nk->rif = (void *)ck->dev; +} + +static bool +prestera_util_nhc_eq_n_cache_key(struct prestera_switch *sw, + struct fib_nh_common *nhc, + struct prestera_kern_neigh_cache_key *nk) +{ + struct prestera_kern_neigh_cache_key tk; + int err; + + err = prestera_util_nhc2nc_key(sw, nhc, &tk); + if (err) + return false; + + if (memcmp(&tk, nk, sizeof(tk))) + return false; + + return true; +} + +static int +prestera_util_neigh2nc_key(struct prestera_switch *sw, struct neighbour *n, + struct prestera_kern_neigh_cache_key *key) +{ + memset(key, 0, sizeof(*key)); + if (n->tbl->family == AF_INET) { + key->addr.v = PRESTERA_IPV4; + key->addr.u.ipv4 = *(__be32 *)n->primary_key; + } else { + return -ENOENT; + } + + key->dev = n->dev; + + return 0; +} + +static bool __prestera_fi_is_direct(struct fib_info *fi) +{ + struct fib_nh *fib_nh; + + if (fib_info_num_path(fi) == 1) { + fib_nh = fib_info_nh(fi, 0); + if (fib_nh->fib_nh_gw_family == AF_UNSPEC) + return true; + } + + return false; +} + +static bool prestera_fi_is_direct(struct fib_info *fi) +{ + if (fi->fib_type != RTN_UNICAST) + return false; + + return __prestera_fi_is_direct(fi); +} + +static bool prestera_fi_is_nh(struct fib_info *fi) +{ + if (fi->fib_type != RTN_UNICAST) + return false; + + return !__prestera_fi_is_direct(fi); +} + +static bool __prestera_fi6_is_direct(struct fib6_info *fi) +{ + if (!fi->fib6_nh->nh_common.nhc_gw_family) + return true; + + return false; +} + +static bool prestera_fi6_is_direct(struct fib6_info *fi) +{ + if (fi->fib6_type != RTN_UNICAST) + return false; + + return __prestera_fi6_is_direct(fi); +} + +static bool prestera_fi6_is_nh(struct fib6_info *fi) +{ + if (fi->fib6_type != RTN_UNICAST) + return false; + + return !__prestera_fi6_is_direct(fi); +} + +static bool prestera_fib_info_is_direct(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info = + container_of(info, struct fib6_entry_notifier_info, info); + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + if (info->family == AF_INET) + return prestera_fi_is_direct(fen_info->fi); + else + return prestera_fi6_is_direct(fen6_info->rt); +} + +static bool prestera_fib_info_is_nh(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info = + container_of(info, struct fib6_entry_notifier_info, info); + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + if (info->family == AF_INET) + return prestera_fi_is_nh(fen_info->fi); + else + return prestera_fi6_is_nh(fen6_info->rt); +} + +/* must be called with rcu_read_lock() */ +static int prestera_util_kern_get_route(struct fib_result *res, u32 tb_id, + __be32 *addr) +{ + struct flowi4 fl4; + + /* TODO: walkthrough appropriate tables in kernel + * to know if the same prefix exists in several tables + */ + memset(&fl4, 0, sizeof(fl4)); + fl4.daddr = *addr; + return fib_lookup(&init_net, &fl4, res, 0 /* FIB_LOOKUP_NOREF */); +} + +static bool +__prestera_util_kern_n_is_reachable_v4(u32 tb_id, __be32 *addr, + struct net_device *dev) +{ + struct fib_nh *fib_nh; + struct fib_result res; + bool reachable; + + reachable = false; + + if (!prestera_util_kern_get_route(&res, tb_id, addr)) + if (prestera_fi_is_direct(res.fi)) { + fib_nh = fib_info_nh(res.fi, 0); + if (dev == fib_nh->fib_nh_dev) + reachable = true; + } + + return reachable; +} + +/* Check if neigh route is reachable */ +static bool +prestera_util_kern_n_is_reachable(u32 tb_id, + struct prestera_ip_addr *addr, + struct net_device *dev) +{ + if (addr->v == PRESTERA_IPV4) + return __prestera_util_kern_n_is_reachable_v4(tb_id, + &addr->u.ipv4, + dev); + else + return false; +} + +static void prestera_util_kern_set_neigh_offload(struct neighbour *n, + bool offloaded) +{ + if (offloaded) + n->flags |= NTF_OFFLOADED; + else + n->flags &= ~NTF_OFFLOADED; +} + +static void +prestera_util_kern_set_nh_offload(struct fib_nh_common *nhc, bool offloaded, bool trap) +{ + if (offloaded) + nhc->nhc_flags |= RTNH_F_OFFLOAD; + else + nhc->nhc_flags &= ~RTNH_F_OFFLOAD; + + if (trap) + nhc->nhc_flags |= RTNH_F_TRAP; + else + nhc->nhc_flags &= ~RTNH_F_TRAP; +} + +static struct fib_nh_common * +prestera_kern_fib_info_nhc(struct fib_notifier_info *info, int n) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + struct fib6_info *iter; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return &fib_info_nh(fen4_info->fi, n)->nh_common; + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + if (!n) + return &fen6_info->rt->fib6_nh->nh_common; + + list_for_each_entry(iter, &fen6_info->rt->fib6_siblings, + fib6_siblings) { + if (!--n) + return &iter->fib6_nh->nh_common; + } + } + + /* if family is incorrect - than upper functions has BUG */ + /* if doesn't find requested index - there is alsi bug, because + * valid index must be produced by nhs, which checks list length + */ + WARN(1, "Invalid parameters passed to %s n=%d i=%p", + __func__, n, info); + return NULL; +} + +static int prestera_kern_fib_info_nhs(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return fib_info_num_path(fen4_info->fi); + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + return fen6_info->rt->fib6_nsiblings + 1; + } + + return 0; +} + +static unsigned char +prestera_kern_fib_info_type(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return fen4_info->fi->fib_type; + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + /* TODO: ECMP in ipv6 is several routes. + * Every route has single nh. + */ + return fen6_info->rt->fib6_type; + } + + return RTN_UNSPEC; +} + +/* Decided, that uc_nh route with key==nh is obviously neighbour route */ +static bool +prestera_fib_node_util_is_neighbour(struct prestera_fib_node *fib_node) +{ + if (fib_node->info.type != PRESTERA_FIB_TYPE_UC_NH) + return false; + + if (fib_node->info.nh_grp->nh_neigh_head[1].neigh) + return false; + + if (!fib_node->info.nh_grp->nh_neigh_head[0].neigh) + return false; + + if (memcmp(&fib_node->info.nh_grp->nh_neigh_head[0].neigh->key.addr, + &fib_node->key.addr, sizeof(struct prestera_ip_addr))) + return false; + + return true; +} + +static int prestera_dev_if_type(const struct net_device *dev) +{ + struct macvlan_dev *vlan; + + if (is_vlan_dev(dev) && + netif_is_bridge_master(vlan_dev_real_dev(dev))) { + return PRESTERA_IF_VID_E; + } else if (netif_is_bridge_master(dev)) { + return PRESTERA_IF_VID_E; + } else if (netif_is_lag_master(dev)) { + return PRESTERA_IF_LAG_E; + } else if (netif_is_macvlan(dev)) { + vlan = netdev_priv(dev); + return prestera_dev_if_type(vlan->lowerdev); + } else { + return PRESTERA_IF_PORT_E; + } +} + +static int +prestera_neigh_iface_init(struct prestera_switch *sw, + struct prestera_iface *iface, + struct neighbour *n) +{ + struct prestera_port *port; + + iface->vlan_id = 0; /* TODO: vlan egress */ + iface->type = prestera_dev_if_type(n->dev); + if (iface->type != PRESTERA_IF_PORT_E) + return -EINVAL; + + if (!prestera_netdev_check(n->dev)) + return -EINVAL; + + port = netdev_priv(n->dev); + iface->dev_port.hw_dev_num = port->dev_id; + iface->dev_port.port_num = port->hw_id; + + return 0; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_find(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + + n_cache = + rhashtable_lookup_fast(&sw->router->kern_neigh_cache_ht, key, + __prestera_kern_neigh_cache_ht_params); + return IS_ERR(n_cache) ? NULL : n_cache; +} + +static void +__prestera_kern_neigh_cache_destruct(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + dev_put(n_cache->key.dev); +} + +static void +__prestera_kern_neigh_cache_destroy(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + rhashtable_remove_fast(&sw->router->kern_neigh_cache_ht, + &n_cache->ht_node, + __prestera_kern_neigh_cache_ht_params); + __prestera_kern_neigh_cache_destruct(sw, n_cache); + kfree(n_cache); +} + +static struct prestera_kern_neigh_cache * +__prestera_kern_neigh_cache_create(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + int err; + + n_cache = kzalloc(sizeof(*n_cache), GFP_KERNEL); + if (!n_cache) + goto err_kzalloc; + + memcpy(&n_cache->key, key, sizeof(*key)); + dev_hold(n_cache->key.dev); + + INIT_LIST_HEAD(&n_cache->kern_fib_cache_list); + err = rhashtable_insert_fast(&sw->router->kern_neigh_cache_ht, + &n_cache->ht_node, + __prestera_kern_neigh_cache_ht_params); + if (err) + goto err_ht_insert; + + return n_cache; + +err_ht_insert: + dev_put(n_cache->key.dev); + kfree(n_cache); +err_kzalloc: + return NULL; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_get(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + + n_cache = prestera_kern_neigh_cache_find(sw, key); + if (!n_cache) + n_cache = __prestera_kern_neigh_cache_create(sw, key); + + return n_cache; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_put(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + if (!n_cache->in_kernel && + list_empty(&n_cache->kern_fib_cache_list)) { + __prestera_kern_neigh_cache_destroy(sw, n_cache); + return NULL; + } + + return n_cache; +} + static struct prestera_kern_fib_cache * prestera_kern_fib_cache_find(struct prestera_switch *sw, struct prestera_kern_fib_cache_key *key) @@ -73,24 +547,79 @@ prestera_kern_fib_cache_find(struct prestera_switch *sw, } static void +__prestera_kern_fib_cache_destruct(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fib_cache) +{ + struct prestera_kern_neigh_cache *n_cache; + int i; + + for (i = 0; i < PRESTERA_NHGR_SIZE_MAX; i++) { + n_cache = fib_cache->kern_neigh_cache_head[i].n_cache; + if (n_cache) { + list_del(&fib_cache->kern_neigh_cache_head[i].head); + prestera_kern_neigh_cache_put(sw, n_cache); + } + } + + fib_info_put(fib_cache->fen4_info.fi); +} + +static void prestera_kern_fib_cache_destroy(struct prestera_switch *sw, struct prestera_kern_fib_cache *fib_cache) { - fib_info_put(fib_cache->fi); rhashtable_remove_fast(&sw->router->kern_fib_cache_ht, &fib_cache->ht_node, __prestera_kern_fib_cache_ht_params); + __prestera_kern_fib_cache_destruct(sw, fib_cache); kfree(fib_cache); } +static int +__prestera_kern_fib_cache_create_nhs(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + struct prestera_kern_neigh_cache_key nc_key; + struct prestera_kern_neigh_cache *n_cache; + struct fib_nh_common *nhc; + int i, nhs, err; + + if (!prestera_fib_info_is_nh(&fc->info)) + return 0; + + nhs = prestera_kern_fib_info_nhs(&fc->info); + if (nhs > PRESTERA_NHGR_SIZE_MAX) + return 0; + + for (i = 0; i < nhs; i++) { + nhc = prestera_kern_fib_info_nhc(&fc->fen4_info.info, i); + err = prestera_util_nhc2nc_key(sw, nhc, &nc_key); + if (err) + return 0; + + n_cache = prestera_kern_neigh_cache_get(sw, &nc_key); + if (!n_cache) + return 0; + + fc->kern_neigh_cache_head[i].this = fc; + fc->kern_neigh_cache_head[i].n_cache = n_cache; + list_add(&fc->kern_neigh_cache_head[i].head, + &n_cache->kern_fib_cache_list); + } + + return 0; +} + /* Operations on fi (offload, etc) must be wrapped in utils. * This function just create storage. */ static struct prestera_kern_fib_cache * prestera_kern_fib_cache_create(struct prestera_switch *sw, struct prestera_kern_fib_cache_key *key, - struct fib_info *fi, dscp_t dscp, u8 type) + struct fib_notifier_info *info) { + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); struct prestera_kern_fib_cache *fib_cache; int err; @@ -99,10 +628,8 @@ prestera_kern_fib_cache_create(struct prestera_switch *sw, goto err_kzalloc; memcpy(&fib_cache->key, key, sizeof(*key)); - fib_info_hold(fi); - fib_cache->fi = fi; - fib_cache->kern_dscp = dscp; - fib_cache->kern_type = type; + fib_info_hold(fen_info->fi); + memcpy(&fib_cache->fen4_info, fen_info, sizeof(*fen_info)); err = rhashtable_insert_fast(&sw->router->kern_fib_cache_ht, &fib_cache->ht_node, @@ -110,48 +637,270 @@ prestera_kern_fib_cache_create(struct prestera_switch *sw, if (err) goto err_ht_insert; + /* Handle nexthops */ + err = __prestera_kern_fib_cache_create_nhs(sw, fib_cache); + if (err) + goto out; /* Not critical */ + +out: return fib_cache; err_ht_insert: - fib_info_put(fi); + fib_info_put(fen_info->fi); kfree(fib_cache); err_kzalloc: return NULL; } static void +__prestera_k_arb_fib_nh_offload_set(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fibc, + struct prestera_kern_neigh_cache *nc, + bool offloaded, bool trap) +{ + struct fib_nh_common *nhc; + int i, nhs; + + nhs = prestera_kern_fib_info_nhs(&fibc->info); + for (i = 0; i < nhs; i++) { + nhc = prestera_kern_fib_info_nhc(&fibc->info, i); + if (!nc) { + prestera_util_kern_set_nh_offload(nhc, offloaded, trap); + continue; + } + + if (prestera_util_nhc_eq_n_cache_key(sw, nhc, &nc->key)) { + prestera_util_kern_set_nh_offload(nhc, offloaded, trap); + break; + } + } +} + +static void +__prestera_k_arb_n_offload_set(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc, + bool offloaded) +{ + struct neighbour *n; + + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + if (!n) + return; + + prestera_util_kern_set_neigh_offload(n, offloaded); + neigh_release(n); +} + +static void __prestera_k_arb_fib_lpm_offload_set(struct prestera_switch *sw, struct prestera_kern_fib_cache *fc, bool fail, bool offload, bool trap) { struct fib_rt_info fri; - if (fc->key.addr.v != PRESTERA_IPV4) + switch (fc->key.addr.v) { + case PRESTERA_IPV4: + fri.fi = fc->fen4_info.fi; + fri.tb_id = fc->key.kern_tb_id; + fri.dst = fc->key.addr.u.ipv4; + fri.dst_len = fc->key.prefix_len; + fri.dscp = fc->fen4_info.dscp; + fri.type = fc->fen4_info.type; + /* flags begin */ + fri.offload = offload; + fri.trap = trap; + fri.offload_failed = fail; + /* flags end */ + fib_alias_hw_flags_set(&init_net, &fri); + return; + case PRESTERA_IPV6: + /* TODO */ return; + } +} + +static void +__prestera_k_arb_n_lpm_set(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache, + bool enabled) +{ + struct prestera_nexthop_group_key nh_grp_key; + struct prestera_kern_fib_cache_key fc_key; + struct prestera_kern_fib_cache *fib_cache; + struct prestera_fib_node *fib_node; + struct prestera_fib_key fib_key; + + /* Exception for fc with prefix 32: LPM entry is already used by fib */ + memset(&fc_key, 0, sizeof(fc_key)); + fc_key.addr = n_cache->key.addr; + fc_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v); + /* But better to use tb_id of route, which pointed to this neighbour. */ + /* We take it from rif, because rif inconsistent. + * Must be separated in_rif and out_rif. + * Also note: for each fib pointed to this neigh should be separated + * neigh lpm entry (for each ingress vr) + */ + fc_key.kern_tb_id = l3mdev_fib_table(n_cache->key.dev); + fib_cache = prestera_kern_fib_cache_find(sw, &fc_key); + memset(&fib_key, 0, sizeof(fib_key)); + fib_key.addr = n_cache->key.addr; + fib_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v); + fib_key.tb_id = prestera_fix_tb_id(fc_key.kern_tb_id); + fib_node = prestera_fib_node_find(sw, &fib_key); + if (!fib_cache || !fib_cache->reachable) { + if (!enabled && fib_node) { + if (prestera_fib_node_util_is_neighbour(fib_node)) + prestera_fib_node_destroy(sw, fib_node); + return; + } + } + + if (enabled && !fib_node) { + memset(&nh_grp_key, 0, sizeof(nh_grp_key)); + prestera_util_nc_key2nh_key(&n_cache->key, + &nh_grp_key.neigh[0]); + fib_node = prestera_fib_node_create(sw, &fib_key, + PRESTERA_FIB_TYPE_UC_NH, + &nh_grp_key); + if (!fib_node) + pr_err("%s failed ip=%pI4n", "prestera_fib_node_create", + &fib_key.addr.u.ipv4); + return; + } +} + +static void +__prestera_k_arb_nc_kern_fib_fetch(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + if (prestera_util_kern_n_is_reachable(l3mdev_fib_table(nc->key.dev), + &nc->key.addr, nc->key.dev)) + nc->reachable = true; + else + nc->reachable = false; +} + +/* Kernel neighbour -> neigh_cache info */ +static void +__prestera_k_arb_nc_kern_n_fetch(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct neighbour *n; + int err; + + memset(&nc->nh_neigh_info, 0, sizeof(nc->nh_neigh_info)); + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, nc->key.dev); + if (!n) + goto out; + + read_lock_bh(&n->lock); + if (n->nud_state & NUD_VALID && !n->dead) { + err = prestera_neigh_iface_init(sw, &nc->nh_neigh_info.iface, + n); + if (err) + goto n_read_out; - fri.fi = fc->fi; - fri.tb_id = fc->key.kern_tb_id; - fri.dst = fc->key.addr.u.ipv4; - fri.dst_len = fc->key.prefix_len; - fri.dscp = fc->kern_dscp; - fri.type = fc->kern_type; - /* flags begin */ - fri.offload = offload; - fri.trap = trap; - fri.offload_failed = fail; - /* flags end */ - fib_alias_hw_flags_set(&init_net, &fri); + memcpy(&nc->nh_neigh_info.ha[0], &n->ha[0], ETH_ALEN); + nc->nh_neigh_info.connected = true; + } +n_read_out: + read_unlock_bh(&n->lock); +out: + nc->in_kernel = nc->nh_neigh_info.connected; + if (n) + neigh_release(n); +} + +/* neigh_cache info -> lpm update */ +static void +__prestera_k_arb_nc_apply(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct prestera_kern_neigh_cache_head *nhead; + struct prestera_nh_neigh_key nh_key; + struct prestera_nh_neigh *nh_neigh; + int err; + + __prestera_k_arb_n_lpm_set(sw, nc, nc->reachable && nc->in_kernel); + __prestera_k_arb_n_offload_set(sw, nc, nc->reachable && nc->in_kernel); + + prestera_util_nc_key2nh_key(&nc->key, &nh_key); + nh_neigh = prestera_nh_neigh_find(sw, &nh_key); + if (!nh_neigh) + goto out; + + /* Do hw update only if something changed to prevent nh flap */ + if (memcmp(&nc->nh_neigh_info, &nh_neigh->info, + sizeof(nh_neigh->info))) { + memcpy(&nh_neigh->info, &nc->nh_neigh_info, + sizeof(nh_neigh->info)); + err = prestera_nh_neigh_set(sw, nh_neigh); + if (err) { + pr_err("%s failed with err=%d ip=%pI4n mac=%pM", + "prestera_nh_neigh_set", err, + &nh_neigh->key.addr.u.ipv4, + &nh_neigh->info.ha[0]); + goto out; + } + } + +out: + list_for_each_entry(nhead, &nc->kern_fib_cache_list, head) { + __prestera_k_arb_fib_nh_offload_set(sw, nhead->this, nc, + nc->in_kernel, + !nc->in_kernel); + } } static int __prestera_pr_k_arb_fc_lpm_info_calc(struct prestera_switch *sw, struct prestera_kern_fib_cache *fc) { + struct fib_nh_common *nhc; + int nh_cnt; + memset(&fc->lpm_info, 0, sizeof(fc->lpm_info)); - switch (fc->fi->fib_type) { + switch (prestera_kern_fib_info_type(&fc->info)) { case RTN_UNICAST: - fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_TRAP; + if (prestera_fib_info_is_direct(&fc->info) && + fc->key.prefix_len == + PRESTERA_IP_ADDR_PLEN(fc->key.addr.v)) { + /* This is special case. + * When prefix is 32. Than we will have conflict in lpm + * for direct route - once TRAP added, there is no + * place for neighbour entry. So represent direct route + * with prefix 32, as NH. So neighbour will be resolved + * as nexthop of this route. + */ + nhc = prestera_kern_fib_info_nhc(&fc->info, 0); + fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_UC_NH; + fc->lpm_info.nh_grp_key.neigh[0].addr = + fc->key.addr; + fc->lpm_info.nh_grp_key.neigh[0].rif = + nhc->nhc_dev; + + break; + } + + /* We can also get nh_grp_key from fi. This will be correct to + * because cache not always represent, what actually written to + * lpm. But we use nh cache, as well for now (for this case). + */ + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + if (!fc->kern_neigh_cache_head[nh_cnt].n_cache) + break; + + fc->lpm_info.nh_grp_key.neigh[nh_cnt].addr = + fc->kern_neigh_cache_head[nh_cnt].n_cache->key.addr; + fc->lpm_info.nh_grp_key.neigh[nh_cnt].rif = + fc->kern_neigh_cache_head[nh_cnt].n_cache->key.dev; + } + + fc->lpm_info.fib_type = nh_cnt ? + PRESTERA_FIB_TYPE_UC_NH : + PRESTERA_FIB_TYPE_TRAP; break; /* Unsupported. Leave it for kernel: */ case RTN_BROADCAST: @@ -191,7 +940,8 @@ static int __prestera_k_arb_f_lpm_set(struct prestera_switch *sw, return 0; fib_node = prestera_fib_node_create(sw, &fc->lpm_info.fib_key, - fc->lpm_info.fib_type); + fc->lpm_info.fib_type, + &fc->lpm_info.nh_grp_key); if (!fib_node) { dev_err(sw->dev->dev, "fib_node=NULL %pI4n/%d kern_tb_id = %d", @@ -220,6 +970,10 @@ static int __prestera_k_arb_fc_apply(struct prestera_switch *sw, } switch (fc->lpm_info.fib_type) { + case PRESTERA_FIB_TYPE_UC_NH: + __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, + fc->reachable, false); + break; case PRESTERA_FIB_TYPE_TRAP: __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, false, fc->reachable); @@ -271,17 +1025,140 @@ __prestera_k_arb_util_fib_overlapped(struct prestera_switch *sw, return rfc; } +static void __prestera_k_arb_hw_state_upd(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct prestera_nh_neigh_key nh_key; + struct prestera_nh_neigh *nh_neigh; + struct neighbour *n; + bool hw_active; + + prestera_util_nc_key2nh_key(&nc->key, &nh_key); + nh_neigh = prestera_nh_neigh_find(sw, &nh_key); + if (!nh_neigh) { + pr_err("Cannot find nh_neigh for cached %pI4n", + &nc->key.addr.u.ipv4); + return; + } + + hw_active = prestera_nh_neigh_util_hw_state(sw, nh_neigh); + +#ifdef PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH + if (!hw_active && nc->in_kernel) + goto out; +#else /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */ + if (!hw_active) + goto out; +#endif /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */ + + if (nc->key.addr.v == PRESTERA_IPV4) { + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + if (!n) + n = neigh_create(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + } else { + n = NULL; + } + + if (!IS_ERR(n) && n) { + neigh_event_send(n, NULL); + neigh_release(n); + } else { + pr_err("Cannot create neighbour %pI4n", &nc->key.addr.u.ipv4); + } + +out: + return; +} + +/* Propagate hw state to kernel */ +static void prestera_k_arb_hw_evt(struct prestera_switch *sw) +{ + struct prestera_kern_neigh_cache *n_cache; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter); + rhashtable_walk_start(&iter); + while (1) { + n_cache = rhashtable_walk_next(&iter); + + if (!n_cache) + break; + + if (IS_ERR(n_cache)) + continue; + + rhashtable_walk_stop(&iter); + __prestera_k_arb_hw_state_upd(sw, n_cache); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + +/* Propagate kernel event to hw */ +static void prestera_k_arb_n_evt(struct prestera_switch *sw, + struct neighbour *n) +{ + struct prestera_kern_neigh_cache_key n_key; + struct prestera_kern_neigh_cache *n_cache; + int err; + + err = prestera_util_neigh2nc_key(sw, n, &n_key); + if (err) + return; + + n_cache = prestera_kern_neigh_cache_find(sw, &n_key); + if (!n_cache) { + n_cache = prestera_kern_neigh_cache_get(sw, &n_key); + if (!n_cache) + return; + __prestera_k_arb_nc_kern_fib_fetch(sw, n_cache); + } + + __prestera_k_arb_nc_kern_n_fetch(sw, n_cache); + __prestera_k_arb_nc_apply(sw, n_cache); + + prestera_kern_neigh_cache_put(sw, n_cache); +} + +static void __prestera_k_arb_fib_evt2nc(struct prestera_switch *sw) +{ + struct prestera_kern_neigh_cache *n_cache; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter); + rhashtable_walk_start(&iter); + while (1) { + n_cache = rhashtable_walk_next(&iter); + + if (!n_cache) + break; + + if (IS_ERR(n_cache)) + continue; + + rhashtable_walk_stop(&iter); + __prestera_k_arb_nc_kern_fib_fetch(sw, n_cache); + __prestera_k_arb_nc_apply(sw, n_cache); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + static int prestera_k_arb_fib_evt(struct prestera_switch *sw, bool replace, /* replace or del */ - struct fib_entry_notifier_info *fen_info) + struct fib_notifier_info *info) { struct prestera_kern_fib_cache *tfib_cache, *bfib_cache; /* top/btm */ struct prestera_kern_fib_cache_key fc_key; struct prestera_kern_fib_cache *fib_cache; int err; - prestera_util_fen_info2fib_cache_key(fen_info, &fc_key); + prestera_util_fen_info2fib_cache_key(info, &fc_key); fib_cache = prestera_kern_fib_cache_find(sw, &fc_key); if (fib_cache) { fib_cache->reachable = false; @@ -304,10 +1181,7 @@ prestera_k_arb_fib_evt(struct prestera_switch *sw, } if (replace) { - fib_cache = prestera_kern_fib_cache_create(sw, &fc_key, - fen_info->fi, - fen_info->dscp, - fen_info->type); + fib_cache = prestera_kern_fib_cache_create(sw, &fc_key, info); if (!fib_cache) { dev_err(sw->dev->dev, "fib_cache == NULL"); return -ENOENT; @@ -331,9 +1205,65 @@ prestera_k_arb_fib_evt(struct prestera_switch *sw, dev_err(sw->dev->dev, "Applying fib_cache failed"); } + /* Update all neighs to resolve overlapped and apply related */ + __prestera_k_arb_fib_evt2nc(sw); + return 0; } +static void __prestera_k_arb_abort_neigh_ht_cb(void *ptr, void *arg) +{ + struct prestera_kern_neigh_cache *n_cache = ptr; + struct prestera_switch *sw = arg; + + if (!list_empty(&n_cache->kern_fib_cache_list)) { + WARN_ON(1); /* BUG */ + return; + } + __prestera_k_arb_n_offload_set(sw, n_cache, false); + n_cache->in_kernel = false; + /* No need to destroy lpm. + * It will be aborted by destroy_ht + */ + __prestera_kern_neigh_cache_destruct(sw, n_cache); + kfree(n_cache); +} + +static void __prestera_k_arb_abort_fib_ht_cb(void *ptr, void *arg) +{ + struct prestera_kern_fib_cache *fib_cache = ptr; + struct prestera_switch *sw = arg; + + __prestera_k_arb_fib_lpm_offload_set(sw, fib_cache, + false, false, + false); + __prestera_k_arb_fib_nh_offload_set(sw, fib_cache, NULL, + false, false); + /* No need to destroy lpm. + * It will be aborted by destroy_ht + */ + __prestera_kern_fib_cache_destruct(sw, fib_cache); + kfree(fib_cache); +} + +static void prestera_k_arb_abort(struct prestera_switch *sw) +{ + /* Function to remove all arbiter entries and related hw objects. */ + /* Sequence: + * 1) Clear arbiter tables, but don't touch hw + * 2) Clear hw + * We use such approach, because arbiter object is not directly mapped + * to hw. So deletion of one arbiter object may even lead to creation of + * hw object (e.g. in case of overlapped routes). + */ + rhashtable_free_and_destroy(&sw->router->kern_fib_cache_ht, + __prestera_k_arb_abort_fib_ht_cb, + sw); + rhashtable_free_and_destroy(&sw->router->kern_neigh_cache_ht, + __prestera_k_arb_abort_neigh_ht_cb, + sw); +} + static int __prestera_inetaddr_port_event(struct net_device *port_dev, unsigned long event, struct netlink_ext_ack *extack) @@ -469,13 +1399,15 @@ static void __prestera_router_fib_event_work(struct work_struct *work) switch (fib_work->event) { case FIB_EVENT_ENTRY_REPLACE: - err = prestera_k_arb_fib_evt(sw, true, &fib_work->fen_info); + err = prestera_k_arb_fib_evt(sw, true, + &fib_work->fen_info.info); if (err) goto err_out; break; case FIB_EVENT_ENTRY_DEL: - err = prestera_k_arb_fib_evt(sw, false, &fib_work->fen_info); + err = prestera_k_arb_fib_evt(sw, false, + &fib_work->fen_info.info); if (err) goto err_out; @@ -534,10 +1466,89 @@ static int __prestera_router_fib_event(struct notifier_block *nb, return NOTIFY_DONE; } +struct prestera_netevent_work { + struct work_struct work; + struct prestera_switch *sw; + struct neighbour *n; +}; + +static void prestera_router_neigh_event_work(struct work_struct *work) +{ + struct prestera_netevent_work *net_work = + container_of(work, struct prestera_netevent_work, work); + struct prestera_switch *sw = net_work->sw; + struct neighbour *n = net_work->n; + + /* neigh - its not hw related object. It stored only in kernel. So... */ + rtnl_lock(); + + prestera_k_arb_n_evt(sw, n); + + neigh_release(n); + rtnl_unlock(); + kfree(net_work); +} + +static int prestera_router_netevent_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct prestera_netevent_work *net_work; + struct prestera_router *router; + struct neighbour *n = ptr; + + router = container_of(nb, struct prestera_router, netevent_nb); + + switch (event) { + case NETEVENT_NEIGH_UPDATE: + if (n->tbl->family != AF_INET) + return NOTIFY_DONE; + + net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); + if (WARN_ON(!net_work)) + return NOTIFY_BAD; + + neigh_clone(n); + net_work->n = n; + net_work->sw = router->sw; + INIT_WORK(&net_work->work, prestera_router_neigh_event_work); + prestera_queue_work(&net_work->work); + } + + return NOTIFY_DONE; +} + +static void prestera_router_update_neighs_work(struct work_struct *work) +{ + struct prestera_router *router; + + router = container_of(work, struct prestera_router, + neighs_update.dw.work); + rtnl_lock(); + + prestera_k_arb_hw_evt(router->sw); + + rtnl_unlock(); + prestera_queue_delayed_work(&router->neighs_update.dw, + msecs_to_jiffies(PRESTERA_NH_PROBE_INTERVAL)); +} + +static int prestera_neigh_work_init(struct prestera_switch *sw) +{ + INIT_DELAYED_WORK(&sw->router->neighs_update.dw, + prestera_router_update_neighs_work); + prestera_queue_delayed_work(&sw->router->neighs_update.dw, 0); + return 0; +} + +static void prestera_neigh_work_fini(struct prestera_switch *sw) +{ + cancel_delayed_work_sync(&sw->router->neighs_update.dw); +} + int prestera_router_init(struct prestera_switch *sw) { struct prestera_router *router; - int err; + int err, nhgrp_cache_bytes; router = kzalloc(sizeof(*sw->router), GFP_KERNEL); if (!router) @@ -555,6 +1566,22 @@ int prestera_router_init(struct prestera_switch *sw) if (err) goto err_kern_fib_cache_ht_init; + err = rhashtable_init(&router->kern_neigh_cache_ht, + &__prestera_kern_neigh_cache_ht_params); + if (err) + goto err_kern_neigh_cache_ht_init; + + nhgrp_cache_bytes = sw->size_tbl_router_nexthop / 8 + 1; + router->nhgrp_hw_state_cache = kzalloc(nhgrp_cache_bytes, GFP_KERNEL); + if (!router->nhgrp_hw_state_cache) { + err = -ENOMEM; + goto err_nh_state_cache_alloc; + } + + err = prestera_neigh_work_init(sw); + if (err) + goto err_neigh_work_init; + router->inetaddr_valid_nb.notifier_call = __prestera_inetaddr_valid_cb; err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); if (err) @@ -565,6 +1592,11 @@ int prestera_router_init(struct prestera_switch *sw) if (err) goto err_register_inetaddr_notifier; + router->netevent_nb.notifier_call = prestera_router_netevent_event; + err = register_netevent_notifier(&router->netevent_nb); + if (err) + goto err_register_netevent_notifier; + router->fib_nb.notifier_call = __prestera_router_fib_event; err = register_fib_notifier(&init_net, &router->fib_nb, /* TODO: flush fib entries */ NULL, NULL); @@ -574,10 +1606,18 @@ int prestera_router_init(struct prestera_switch *sw) return 0; err_register_fib_notifier: + unregister_netevent_notifier(&router->netevent_nb); +err_register_netevent_notifier: unregister_inetaddr_notifier(&router->inetaddr_nb); err_register_inetaddr_notifier: unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); err_register_inetaddr_validator_notifier: + prestera_neigh_work_fini(sw); +err_neigh_work_init: + kfree(router->nhgrp_hw_state_cache); +err_nh_state_cache_alloc: + rhashtable_destroy(&router->kern_neigh_cache_ht); +err_kern_neigh_cache_ht_init: rhashtable_destroy(&router->kern_fib_cache_ht); err_kern_fib_cache_ht_init: prestera_router_hw_fini(sw); @@ -589,8 +1629,15 @@ err_router_lib_init: void prestera_router_fini(struct prestera_switch *sw) { unregister_fib_notifier(&init_net, &sw->router->fib_nb); + unregister_netevent_notifier(&sw->router->netevent_nb); unregister_inetaddr_notifier(&sw->router->inetaddr_nb); unregister_inetaddr_validator_notifier(&sw->router->inetaddr_valid_nb); + prestera_neigh_work_fini(sw); + prestera_queue_drain(); + + prestera_k_arb_abort(sw); + + kfree(sw->router->nhgrp_hw_state_cache); rhashtable_destroy(&sw->router->kern_fib_cache_ht); prestera_router_hw_fini(sw); kfree(sw->router); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c index 5b0cf3be9a9e..aa080dc57ff0 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c @@ -8,10 +8,16 @@ #include "prestera_router_hw.h" #include "prestera_acl.h" -/* +--+ - * +------->|vr|<-+ - * | +--+ | - * | | +/* Nexthop is pointed + * to port (not rif) + * +-------+ + * +>|nexthop| + * | +-------+ + * | + * +--+ +-----++ + * +------->|vr|<-+ +>|nh_grp| + * | +--+ | | +------+ + * | | | * +-+-------+ +--+---+-+ * |rif_entry| |fib_node| * +---------+ +--------+ @@ -23,6 +29,8 @@ #define PRESTERA_NHGR_UNUSED (0) #define PRESTERA_NHGR_DROP (0xFFFFFFFF) +/* Need to merge it with router_manager */ +#define PRESTERA_NH_ACTIVE_JIFFER_FILTER 3000 /* ms */ static const struct rhashtable_params __prestera_fib_ht_params = { .key_offset = offsetof(struct prestera_fib_node, key), @@ -31,10 +39,45 @@ static const struct rhashtable_params __prestera_fib_ht_params = { .automatic_shrinking = true, }; +static const struct rhashtable_params __prestera_nh_neigh_ht_params = { + .key_offset = offsetof(struct prestera_nh_neigh, key), + .key_len = sizeof(struct prestera_nh_neigh_key), + .head_offset = offsetof(struct prestera_nh_neigh, ht_node), +}; + +static const struct rhashtable_params __prestera_nexthop_group_ht_params = { + .key_offset = offsetof(struct prestera_nexthop_group, key), + .key_len = sizeof(struct prestera_nexthop_group_key), + .head_offset = offsetof(struct prestera_nexthop_group, ht_node), +}; + +static int prestera_nexthop_group_set(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp); +static bool +prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp); +static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg); + +/* TODO: move to router.h as macros */ +static bool prestera_nh_neigh_key_is_valid(struct prestera_nh_neigh_key *key) +{ + return memchr_inv(key, 0, sizeof(*key)) ? true : false; +} + int prestera_router_hw_init(struct prestera_switch *sw) { int err; + err = rhashtable_init(&sw->router->nh_neigh_ht, + &__prestera_nh_neigh_ht_params); + if (err) + goto err_nh_neigh_ht_init; + + err = rhashtable_init(&sw->router->nexthop_group_ht, + &__prestera_nexthop_group_ht_params); + if (err) + goto err_nexthop_grp_ht_init; + err = rhashtable_init(&sw->router->fib_ht, &__prestera_fib_ht_params); if (err) @@ -43,15 +86,25 @@ int prestera_router_hw_init(struct prestera_switch *sw) INIT_LIST_HEAD(&sw->router->vr_list); INIT_LIST_HEAD(&sw->router->rif_entry_list); + return 0; + err_fib_ht_init: + rhashtable_destroy(&sw->router->nexthop_group_ht); +err_nexthop_grp_ht_init: + rhashtable_destroy(&sw->router->nh_neigh_ht); +err_nh_neigh_ht_init: return 0; } void prestera_router_hw_fini(struct prestera_switch *sw) { + rhashtable_free_and_destroy(&sw->router->fib_ht, + prestera_fib_node_destroy_ht_cb, sw); WARN_ON(!list_empty(&sw->router->vr_list)); WARN_ON(!list_empty(&sw->router->rif_entry_list)); rhashtable_destroy(&sw->router->fib_ht); + rhashtable_destroy(&sw->router->nexthop_group_ht); + rhashtable_destroy(&sw->router->nh_neigh_ht); } static struct prestera_vr *__prestera_vr_find(struct prestera_switch *sw, @@ -232,6 +285,286 @@ err_kzalloc: return NULL; } +static void __prestera_nh_neigh_destroy(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + rhashtable_remove_fast(&sw->router->nh_neigh_ht, + &neigh->ht_node, + __prestera_nh_neigh_ht_params); + kfree(neigh); +} + +static struct prestera_nh_neigh * +__prestera_nh_neigh_create(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *neigh; + int err; + + neigh = kzalloc(sizeof(*neigh), GFP_KERNEL); + if (!neigh) + goto err_kzalloc; + + memcpy(&neigh->key, key, sizeof(*key)); + neigh->info.connected = false; + INIT_LIST_HEAD(&neigh->nexthop_group_list); + err = rhashtable_insert_fast(&sw->router->nh_neigh_ht, + &neigh->ht_node, + __prestera_nh_neigh_ht_params); + if (err) + goto err_rhashtable_insert; + + return neigh; + +err_rhashtable_insert: + kfree(neigh); +err_kzalloc: + return NULL; +} + +struct prestera_nh_neigh * +prestera_nh_neigh_find(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *nh_neigh; + + nh_neigh = rhashtable_lookup_fast(&sw->router->nh_neigh_ht, + key, __prestera_nh_neigh_ht_params); + return IS_ERR(nh_neigh) ? NULL : nh_neigh; +} + +struct prestera_nh_neigh * +prestera_nh_neigh_get(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *neigh; + + neigh = prestera_nh_neigh_find(sw, key); + if (!neigh) + return __prestera_nh_neigh_create(sw, key); + + return neigh; +} + +void prestera_nh_neigh_put(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + if (list_empty(&neigh->nexthop_group_list)) + __prestera_nh_neigh_destroy(sw, neigh); +} + +/* Updates new prestera_neigh_info */ +int prestera_nh_neigh_set(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + struct prestera_nh_neigh_head *nh_head; + struct prestera_nexthop_group *nh_grp; + int err; + + list_for_each_entry(nh_head, &neigh->nexthop_group_list, head) { + nh_grp = nh_head->this; + err = prestera_nexthop_group_set(sw, nh_grp); + if (err) + return err; + } + + return 0; +} + +bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, + struct prestera_nh_neigh *nh_neigh) +{ + bool state; + struct prestera_nh_neigh_head *nh_head, *tmp; + + state = false; + list_for_each_entry_safe(nh_head, tmp, + &nh_neigh->nexthop_group_list, head) { + state = prestera_nexthop_group_util_hw_state(sw, nh_head->this); + if (state) + goto out; + } + +out: + return state; +} + +static struct prestera_nexthop_group * +__prestera_nexthop_group_create(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + struct prestera_nh_neigh *nh_neigh; + int nh_cnt, err, gid; + + nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); + if (!nh_grp) + goto err_kzalloc; + + memcpy(&nh_grp->key, key, sizeof(*key)); + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + if (!prestera_nh_neigh_key_is_valid(&nh_grp->key.neigh[nh_cnt])) + break; + + nh_neigh = prestera_nh_neigh_get(sw, + &nh_grp->key.neigh[nh_cnt]); + if (!nh_neigh) + goto err_nh_neigh_get; + + nh_grp->nh_neigh_head[nh_cnt].neigh = nh_neigh; + nh_grp->nh_neigh_head[nh_cnt].this = nh_grp; + list_add(&nh_grp->nh_neigh_head[nh_cnt].head, + &nh_neigh->nexthop_group_list); + } + + err = prestera_hw_nh_group_create(sw, nh_cnt, &nh_grp->grp_id); + if (err) + goto err_nh_group_create; + + err = prestera_nexthop_group_set(sw, nh_grp); + if (err) + goto err_nexthop_group_set; + + err = rhashtable_insert_fast(&sw->router->nexthop_group_ht, + &nh_grp->ht_node, + __prestera_nexthop_group_ht_params); + if (err) + goto err_ht_insert; + + /* reset cache for created group */ + gid = nh_grp->grp_id; + sw->router->nhgrp_hw_state_cache[gid / 8] &= ~BIT(gid % 8); + + return nh_grp; + +err_ht_insert: +err_nexthop_group_set: + prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); +err_nh_group_create: +err_nh_neigh_get: + for (nh_cnt--; nh_cnt >= 0; nh_cnt--) { + list_del(&nh_grp->nh_neigh_head[nh_cnt].head); + prestera_nh_neigh_put(sw, nh_grp->nh_neigh_head[nh_cnt].neigh); + } + + kfree(nh_grp); +err_kzalloc: + return NULL; +} + +static void +__prestera_nexthop_group_destroy(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + struct prestera_nh_neigh *nh_neigh; + int nh_cnt; + + rhashtable_remove_fast(&sw->router->nexthop_group_ht, + &nh_grp->ht_node, + __prestera_nexthop_group_ht_params); + + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + nh_neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; + if (!nh_neigh) + break; + + list_del(&nh_grp->nh_neigh_head[nh_cnt].head); + prestera_nh_neigh_put(sw, nh_neigh); + } + + prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); + kfree(nh_grp); +} + +static struct prestera_nexthop_group * +__prestera_nexthop_group_find(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + + nh_grp = rhashtable_lookup_fast(&sw->router->nexthop_group_ht, + key, __prestera_nexthop_group_ht_params); + return IS_ERR(nh_grp) ? NULL : nh_grp; +} + +static struct prestera_nexthop_group * +prestera_nexthop_group_get(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + + nh_grp = __prestera_nexthop_group_find(sw, key); + if (nh_grp) { + refcount_inc(&nh_grp->refcount); + } else { + nh_grp = __prestera_nexthop_group_create(sw, key); + if (!nh_grp) + return ERR_PTR(-ENOMEM); + + refcount_set(&nh_grp->refcount, 1); + } + + return nh_grp; +} + +static void prestera_nexthop_group_put(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + if (refcount_dec_and_test(&nh_grp->refcount)) + __prestera_nexthop_group_destroy(sw, nh_grp); +} + +/* Updates with new nh_neigh's info */ +static int prestera_nexthop_group_set(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + struct prestera_neigh_info info[PRESTERA_NHGR_SIZE_MAX]; + struct prestera_nh_neigh *neigh; + int nh_cnt; + + memset(&info[0], 0, sizeof(info)); + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; + if (!neigh) + break; + + memcpy(&info[nh_cnt], &neigh->info, sizeof(neigh->info)); + } + + return prestera_hw_nh_entries_set(sw, nh_cnt, &info[0], nh_grp->grp_id); +} + +static bool +prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + int err; + u32 buf_size = sw->size_tbl_router_nexthop / 8 + 1; + u32 gid = nh_grp->grp_id; + u8 *cache = sw->router->nhgrp_hw_state_cache; + + /* Antijitter + * Prevent situation, when we read state of nh_grp twice in short time, + * and state bit is still cleared on second call. So just stuck active + * state for PRESTERA_NH_ACTIVE_JIFFER_FILTER, after last occurred. + */ + if (!time_before(jiffies, sw->router->nhgrp_hw_cache_kick + + msecs_to_jiffies(PRESTERA_NH_ACTIVE_JIFFER_FILTER))) { + err = prestera_hw_nhgrp_blk_get(sw, cache, buf_size); + if (err) { + pr_err("Failed to get hw state nh_grp's"); + return false; + } + + sw->router->nhgrp_hw_cache_kick = jiffies; + } + + if (cache[gid / 8] & BIT(gid % 8)) + return true; + + return false; +} + struct prestera_fib_node * prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key) { @@ -251,6 +584,9 @@ static void __prestera_fib_node_destruct(struct prestera_switch *sw, prestera_hw_lpm_del(sw, vr->hw_vr_id, fib_node->key.addr.u.ipv4, fib_node->key.prefix_len); switch (fib_node->info.type) { + case PRESTERA_FIB_TYPE_UC_NH: + prestera_nexthop_group_put(sw, fib_node->info.nh_grp); + break; case PRESTERA_FIB_TYPE_TRAP: break; case PRESTERA_FIB_TYPE_DROP: @@ -272,10 +608,20 @@ void prestera_fib_node_destroy(struct prestera_switch *sw, kfree(fib_node); } +static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg) +{ + struct prestera_fib_node *node = ptr; + struct prestera_switch *sw = arg; + + __prestera_fib_node_destruct(sw, node); + kfree(node); +} + struct prestera_fib_node * prestera_fib_node_create(struct prestera_switch *sw, struct prestera_fib_key *key, - enum prestera_fib_type fib_type) + enum prestera_fib_type fib_type, + struct prestera_nexthop_group_key *nh_grp_key) { struct prestera_fib_node *fib_node; u32 grp_id; @@ -302,6 +648,14 @@ prestera_fib_node_create(struct prestera_switch *sw, case PRESTERA_FIB_TYPE_DROP: grp_id = PRESTERA_NHGR_DROP; break; + case PRESTERA_FIB_TYPE_UC_NH: + fib_node->info.nh_grp = prestera_nexthop_group_get(sw, + nh_grp_key); + if (IS_ERR(fib_node->info.nh_grp)) + goto err_nh_grp_get; + + grp_id = fib_node->info.nh_grp->grp_id; + break; default: pr_err("Unsupported fib_type %d", fib_type); goto err_nh_grp_get; @@ -323,6 +677,8 @@ err_ht_insert: prestera_hw_lpm_del(sw, vr->hw_vr_id, key->addr.u.ipv4, key->prefix_len); err_lpm_add: + if (fib_type == PRESTERA_FIB_TYPE_UC_NH) + prestera_nexthop_group_put(sw, fib_node->info.nh_grp); err_nh_grp_get: prestera_vr_put(sw, vr); err_vr_get: diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h index 67dbb49c8bd4..9ca97919c863 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h @@ -31,6 +31,63 @@ struct prestera_ip_addr { PRESTERA_IPV4 = 0, PRESTERA_IPV6 } v; +#define PRESTERA_IP_ADDR_PLEN(V) ((V) == PRESTERA_IPV4 ? 32 : \ + /* (V) == PRESTERA_IPV6 ? */ 128 /* : 0 */) +}; + +struct prestera_nh_neigh_key { + struct prestera_ip_addr addr; + /* Seems like rif is obsolete, because there is iface in info ? + * Key can contain functional fields, or fields, which is used to + * filter duplicate objects on logical level (before you pass it to + * HW)... also key can be used to cover hardware restrictions. + * In our case rif - is logical interface (even can be VLAN), which + * is used in combination with IP address (which is also not related to + * hardware nexthop) to provide logical compression of created nexthops. + * You even can imagine, that rif+IPaddr is just cookie. + */ + /* struct prestera_rif *rif; */ + /* Use just as cookie, to divide ARP domains (in order with addr) */ + void *rif; +}; + +/* Used for hw call */ +struct prestera_neigh_info { + struct prestera_iface iface; + unsigned char ha[ETH_ALEN]; + u8 connected; /* bool. indicate, if mac/oif valid */ + u8 __pad[1]; +}; + +/* Used to notify nh about neigh change */ +struct prestera_nh_neigh { + struct prestera_nh_neigh_key key; + struct prestera_neigh_info info; + struct rhash_head ht_node; /* node of prestera_vr */ + struct list_head nexthop_group_list; +}; + +#define PRESTERA_NHGR_SIZE_MAX 4 + +struct prestera_nexthop_group { + struct prestera_nexthop_group_key { + struct prestera_nh_neigh_key neigh[PRESTERA_NHGR_SIZE_MAX]; + } key; + /* Store intermediate object here. + * This prevent overhead kzalloc call. + */ + /* nh_neigh is used only to notify nexthop_group */ + struct prestera_nh_neigh_head { + struct prestera_nexthop_group *this; + struct list_head head; + /* ptr to neigh is not necessary. + * It used to prevent lookup of nh_neigh by key (n) on destroy + */ + struct prestera_nh_neigh *neigh; + } nh_neigh_head[PRESTERA_NHGR_SIZE_MAX]; + struct rhash_head ht_node; /* node of prestera_vr */ + refcount_t refcount; + u32 grp_id; /* hw */ }; struct prestera_fib_key { @@ -44,12 +101,16 @@ struct prestera_fib_info { struct list_head vr_node; enum prestera_fib_type { PRESTERA_FIB_TYPE_INVALID = 0, + /* must be pointer to nh_grp id */ + PRESTERA_FIB_TYPE_UC_NH, /* It can be connected route * and will be overlapped with neighbours */ PRESTERA_FIB_TYPE_TRAP, PRESTERA_FIB_TYPE_DROP } type; + /* Valid only if type = UC_NH*/ + struct prestera_nexthop_group *nh_grp; }; struct prestera_fib_node { @@ -67,6 +128,18 @@ struct prestera_rif_entry * prestera_rif_entry_create(struct prestera_switch *sw, struct prestera_rif_entry_key *k, u32 tb_id, const unsigned char *addr); +struct prestera_nh_neigh * +prestera_nh_neigh_find(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key); +struct prestera_nh_neigh * +prestera_nh_neigh_get(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key); +void prestera_nh_neigh_put(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh); +int prestera_nh_neigh_set(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh); +bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, + struct prestera_nh_neigh *nh_neigh); struct prestera_fib_node *prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key); void prestera_fib_node_destroy(struct prestera_switch *sw, @@ -74,7 +147,8 @@ void prestera_fib_node_destroy(struct prestera_switch *sw, struct prestera_fib_node * prestera_fib_node_create(struct prestera_switch *sw, struct prestera_fib_key *key, - enum prestera_fib_type fib_type); + enum prestera_fib_type fib_type, + struct prestera_nexthop_group_key *nh_grp_key); int prestera_router_hw_init(struct prestera_switch *sw); void prestera_router_hw_fini(struct prestera_switch *sw); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c index dc3e3ddc60bf..9277a8fd1339 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c @@ -659,7 +659,7 @@ static int prestera_sdma_switch_init(struct prestera_switch *sw) init_dummy_netdev(&sdma->napi_dev); - netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll, 64); + netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll); napi_enable(&sdma->rx_napi); return 0; @@ -776,6 +776,7 @@ tx_done: int prestera_rxtx_switch_init(struct prestera_switch *sw) { struct prestera_rxtx *rxtx; + int err; rxtx = kzalloc(sizeof(*rxtx), GFP_KERNEL); if (!rxtx) @@ -783,7 +784,11 @@ int prestera_rxtx_switch_init(struct prestera_switch *sw) sw->rxtx = rxtx; - return prestera_sdma_switch_init(sw); + err = prestera_sdma_switch_init(sw); + if (err) + kfree(rxtx); + + return err; } void prestera_rxtx_switch_fini(struct prestera_switch *sw) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.c b/drivers/net/ethernet/marvell/prestera/prestera_span.c index 845e9d8c8cc7..1005182ce3bc 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.c @@ -107,7 +107,7 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id) entry = prestera_span_entry_find_by_id(sw->span, span_id); if (!entry) - return false; + return -ENOENT; if (!refcount_dec_and_test(&entry->ref_count)) return 0; @@ -120,8 +120,9 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id) return 0; } -static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, - struct prestera_port *to_port) +int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port, + bool ingress) { struct prestera_switch *sw = binding->port->sw; u8 span_id; @@ -135,7 +136,7 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, if (err) return err; - err = prestera_hw_span_bind(binding->port, span_id); + err = prestera_hw_span_bind(binding->port, span_id, ingress); if (err) { prestera_span_put(sw, span_id); return err; @@ -145,11 +146,15 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, return 0; } -static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) +int prestera_span_rule_del(struct prestera_flow_block_binding *binding, + bool ingress) { int err; - err = prestera_hw_span_unbind(binding->port); + if (binding->span_id == PRESTERA_SPAN_INVALID_ID) + return -ENOENT; + + err = prestera_hw_span_unbind(binding->port, ingress); if (err) return err; @@ -161,60 +166,6 @@ static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) return 0; } -int prestera_span_replace(struct prestera_flow_block *block, - struct tc_cls_matchall_offload *f) -{ - struct prestera_flow_block_binding *binding; - __be16 protocol = f->common.protocol; - struct flow_action_entry *act; - struct prestera_port *port; - int err; - - if (!flow_offload_has_one_action(&f->rule->action)) { - NL_SET_ERR_MSG(f->common.extack, - "Only singular actions are supported"); - return -EOPNOTSUPP; - } - - act = &f->rule->action.entries[0]; - - if (!prestera_netdev_check(act->dev)) { - NL_SET_ERR_MSG(f->common.extack, - "Only Marvell Prestera port is supported"); - return -EINVAL; - } - if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) - return -EOPNOTSUPP; - if (act->id != FLOW_ACTION_MIRRED) - return -EOPNOTSUPP; - if (protocol != htons(ETH_P_ALL)) - return -EOPNOTSUPP; - - port = netdev_priv(act->dev); - - list_for_each_entry(binding, &block->binding_list, list) { - err = prestera_span_rule_add(binding, port); - if (err) - goto rollback; - } - - return 0; - -rollback: - list_for_each_entry_continue_reverse(binding, - &block->binding_list, list) - prestera_span_rule_del(binding); - return err; -} - -void prestera_span_destroy(struct prestera_flow_block *block) -{ - struct prestera_flow_block_binding *binding; - - list_for_each_entry(binding, &block->binding_list, list) - prestera_span_rule_del(binding); -} - int prestera_span_init(struct prestera_switch *sw) { struct prestera_span *span; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.h b/drivers/net/ethernet/marvell/prestera/prestera_span.h index f0644521f78a..493b68524bcb 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.h @@ -8,13 +8,17 @@ #define PRESTERA_SPAN_INVALID_ID -1 +struct prestera_port; struct prestera_switch; -struct prestera_flow_block; +struct prestera_flow_block_binding; int prestera_span_init(struct prestera_switch *sw); void prestera_span_fini(struct prestera_switch *sw); -int prestera_span_replace(struct prestera_flow_block *block, - struct tc_cls_matchall_offload *f); -void prestera_span_destroy(struct prestera_flow_block *block); + +int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port, + bool ingress); +int prestera_span_rule_del(struct prestera_flow_block_binding *binding, + bool ingress); #endif /* _PRESTERA_SPAN_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c index 71cde97d85c8..e548cd32582e 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c @@ -143,6 +143,7 @@ prestera_br_port_flags_reset(struct prestera_bridge_port *br_port, prestera_port_uc_flood_set(port, false); prestera_port_mc_flood_set(port, false); prestera_port_learning_set(port, false); + prestera_port_br_locked_set(port, false); } static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port, @@ -162,6 +163,11 @@ static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port, if (err) goto err_out; + err = prestera_port_br_locked_set(port, + br_port->flags & BR_PORT_LOCKED); + if (err) + goto err_out; + return 0; err_out: @@ -1163,7 +1169,7 @@ static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx, break; case SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS: if (attr->u.brport_flags.mask & - ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD)) + ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD | BR_PORT_LOCKED)) err = -EINVAL; break; case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS: diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c index 349b8a94e939..cf456d62677f 100644 --- a/drivers/net/ethernet/marvell/pxa168_eth.c +++ b/drivers/net/ethernet/marvell/pxa168_eth.c @@ -1354,10 +1354,10 @@ static void pxa168_eth_netpoll(struct net_device *dev) static void pxa168_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); - strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); - strlcpy(info->bus_info, "N/A", sizeof(info->bus_info)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); + strscpy(info->bus_info, "N/A", sizeof(info->bus_info)); } static const struct ethtool_ops pxa168_ethtool_ops = { diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index c1e985416c0e..1b43704baceb 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -394,9 +394,9 @@ static void skge_get_drvinfo(struct net_device *dev, { struct skge_port *skge = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(skge->hw->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(skge->hw->pdev), sizeof(info->bus_info)); } @@ -3832,7 +3832,7 @@ static struct net_device *skge_devinit(struct skge_hw *hw, int port, dev->features |= NETIF_F_HIGHDMA; skge = netdev_priv(dev); - netif_napi_add(dev, &skge->napi, skge_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &skge->napi, skge_poll); skge->netdev = dev; skge->hw = hw; skge->msg_enable = netif_msg_init(debug, default_msg); diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index bbea5458000b..ab33ba1c3023 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -3687,9 +3687,9 @@ static void sky2_get_drvinfo(struct net_device *dev, { struct sky2_port *sky2 = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(sky2->hw->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(sky2->hw->pdev), sizeof(info->bus_info)); } @@ -4937,7 +4937,7 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - netif_napi_add(dev, &hw->napi, sky2_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &hw->napi, sky2_poll); err = register_netdev(dev); if (err) { |