diff options
Diffstat (limited to 'drivers/net/ethernet/marvell')
100 files changed, 25626 insertions, 2001 deletions
diff --git a/drivers/net/ethernet/marvell/Kconfig b/drivers/net/ethernet/marvell/Kconfig index fe0989c0fc25..f58a1c0144ba 100644 --- a/drivers/net/ethernet/marvell/Kconfig +++ b/drivers/net/ethernet/marvell/Kconfig @@ -62,6 +62,7 @@ config MVNETA select MVMDIO select PHYLINK select PAGE_POOL + select PAGE_POOL_STATS help This driver supports the network interface units in the Marvell ARMADA XP, ARMADA 370, ARMADA 38x and @@ -177,6 +178,7 @@ config SKY2_DEBUG source "drivers/net/ethernet/marvell/octeontx2/Kconfig" +source "drivers/net/ethernet/marvell/octeon_ep/Kconfig" source "drivers/net/ethernet/marvell/prestera/Kconfig" endif # NET_VENDOR_MARVELL diff --git a/drivers/net/ethernet/marvell/Makefile b/drivers/net/ethernet/marvell/Makefile index 9f88fe822555..ceba4aa4f026 100644 --- a/drivers/net/ethernet/marvell/Makefile +++ b/drivers/net/ethernet/marvell/Makefile @@ -11,5 +11,6 @@ obj-$(CONFIG_MVPP2) += mvpp2/ obj-$(CONFIG_PXA168_ETH) += pxa168_eth.o obj-$(CONFIG_SKGE) += skge.o obj-$(CONFIG_SKY2) += sky2.o +obj-y += octeon_ep/ obj-y += octeontx2/ obj-y += prestera/ diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index bb14fa2241a3..8941f69d93e9 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -775,7 +775,7 @@ txq_put_hdr_tso(struct sk_buff *skb, struct tx_queue *txq, int length, u32 *first_cmd_sts, bool first_desc) { struct mv643xx_eth_private *mp = txq_to_mp(txq); - int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + int hdr_len = skb_tcp_all_headers(skb); int tx_index; struct tx_desc *desc; int ret; @@ -1603,12 +1603,12 @@ mv643xx_eth_set_link_ksettings(struct net_device *dev, static void mv643xx_eth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, mv643xx_eth_driver_name, + strscpy(drvinfo->driver, mv643xx_eth_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, mv643xx_eth_driver_version, + strscpy(drvinfo->version, mv643xx_eth_driver_version, sizeof(drvinfo->version)); - strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); + strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); } static int mv643xx_eth_get_coalesce(struct net_device *dev, @@ -1638,7 +1638,9 @@ static int mv643xx_eth_set_coalesce(struct net_device *dev, } static void -mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er) +mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er, + struct kernel_ethtool_ringparam *kernel_er, + struct netlink_ext_ack *extack) { struct mv643xx_eth_private *mp = netdev_priv(dev); @@ -1650,14 +1652,16 @@ mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er) } static int -mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er) +mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er, + struct kernel_ethtool_ringparam *kernel_er, + struct netlink_ext_ack *extack) { struct mv643xx_eth_private *mp = netdev_priv(dev); if (er->rx_mini_pending || er->rx_jumbo_pending) return -EINVAL; - mp->rx_ring_size = er->rx_pending < 4096 ? er->rx_pending : 4096; + mp->rx_ring_size = min(er->rx_pending, 4096U); mp->tx_ring_size = clamp_t(unsigned int, er->tx_pending, MV643XX_MAX_SKB_DESCS * 2, 4096); if (mp->tx_ring_size != er->tx_pending) @@ -2477,6 +2481,7 @@ out_free: for (i = 0; i < mp->rxq_count; i++) rxq_deinit(mp->rxq + i); out: + napi_disable(&mp->napi); free_irq(dev->irq, dev); return err; @@ -2700,6 +2705,16 @@ MODULE_DEVICE_TABLE(of, mv643xx_eth_shared_ids); static struct platform_device *port_platdev[3]; +static void mv643xx_eth_shared_of_remove(void) +{ + int n; + + for (n = 0; n < 3; n++) { + platform_device_del(port_platdev[n]); + port_platdev[n] = NULL; + } +} + static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev, struct device_node *pnp) { @@ -2736,7 +2751,9 @@ static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev, return -EINVAL; } - of_get_mac_address(pnp, ppd.mac_addr); + ret = of_get_mac_address(pnp, ppd.mac_addr); + if (ret == -EPROBE_DEFER) + return ret; mv643xx_eth_property(pnp, "tx-queue-size", ppd.tx_queue_size); mv643xx_eth_property(pnp, "tx-sram-addr", ppd.tx_sram_addr); @@ -2800,21 +2817,13 @@ static int mv643xx_eth_shared_of_probe(struct platform_device *pdev) ret = mv643xx_eth_shared_of_add_port(pdev, pnp); if (ret) { of_node_put(pnp); + mv643xx_eth_shared_of_remove(); return ret; } } return 0; } -static void mv643xx_eth_shared_of_remove(void) -{ - int n; - - for (n = 0; n < 3; n++) { - platform_device_del(port_platdev[n]); - port_platdev[n] = NULL; - } -} #else static inline int mv643xx_eth_shared_of_probe(struct platform_device *pdev) { @@ -3084,8 +3093,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) struct mv643xx_eth_private *mp; struct net_device *dev; struct phy_device *phydev = NULL; - struct resource *res; - int err; + int err, irq; pd = dev_get_platdata(&pdev->dev); if (pd == NULL) { @@ -3176,14 +3184,17 @@ static int mv643xx_eth_probe(struct platform_device *pdev) INIT_WORK(&mp->tx_timeout_task, tx_timeout_task); - netif_napi_add(dev, &mp->napi, mv643xx_eth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &mp->napi, mv643xx_eth_poll); timer_setup(&mp->rx_oom, oom_timer_wrapper, 0); - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - BUG_ON(!res); - dev->irq = res->start; + irq = platform_get_irq(pdev, 0); + if (WARN_ON(irq < 0)) { + err = irq; + goto out; + } + dev->irq = irq; dev->netdev_ops = &mv643xx_eth_netdev_ops; @@ -3197,7 +3208,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) dev->hw_features = dev->features; dev->priv_flags |= IFF_UNICAST_FLT; - dev->gso_max_segs = MV643XX_MAX_TSO_SEGS; + netif_set_tso_max_segs(dev, MV643XX_MAX_TSO_SEGS); /* MTU range: 64 - 9500 */ dev->min_mtu = 64; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 5a7bdca22a63..ff3e361e06e7 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -38,6 +38,7 @@ #include <net/ipv6.h> #include <net/tso.h> #include <net/page_pool.h> +#include <net/pkt_cls.h> #include <linux/bpf_trace.h> /* Registers */ @@ -75,6 +76,8 @@ #define MVNETA_WIN_SIZE(w) (0x2204 + ((w) << 3)) #define MVNETA_WIN_REMAP(w) (0x2280 + ((w) << 2)) #define MVNETA_BASE_ADDR_ENABLE 0x2290 +#define MVNETA_AC5_CNM_DDR_TARGET 0x2 +#define MVNETA_AC5_CNM_DDR_ATTR 0xb #define MVNETA_ACCESS_PROTECT_ENABLE 0x2294 #define MVNETA_PORT_CONFIG 0x2400 #define MVNETA_UNI_PROMISC_MODE BIT(0) @@ -247,12 +250,39 @@ #define MVNETA_TXQ_SENT_DESC_MASK 0x3fff0000 #define MVNETA_PORT_TX_RESET 0x3cf0 #define MVNETA_PORT_TX_DMA_RESET BIT(0) +#define MVNETA_TXQ_CMD1_REG 0x3e00 +#define MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 BIT(3) +#define MVNETA_TXQ_CMD1_BW_LIM_EN BIT(0) +#define MVNETA_REFILL_NUM_CLK_REG 0x3e08 +#define MVNETA_REFILL_MAX_NUM_CLK 0x0000ffff #define MVNETA_TX_MTU 0x3e0c #define MVNETA_TX_TOKEN_SIZE 0x3e14 #define MVNETA_TX_TOKEN_SIZE_MAX 0xffffffff +#define MVNETA_TXQ_BUCKET_REFILL_REG(q) (0x3e20 + ((q) << 2)) +#define MVNETA_TXQ_BUCKET_REFILL_PERIOD_MASK 0x3ff00000 +#define MVNETA_TXQ_BUCKET_REFILL_PERIOD_SHIFT 20 +#define MVNETA_TXQ_BUCKET_REFILL_VALUE_MAX 0x0007ffff #define MVNETA_TXQ_TOKEN_SIZE_REG(q) (0x3e40 + ((q) << 2)) #define MVNETA_TXQ_TOKEN_SIZE_MAX 0x7fffffff +/* The values of the bucket refill base period and refill period are taken from + * the reference manual, and adds up to a base resolution of 10Kbps. This allows + * to cover all rate-limit values from 10Kbps up to 5Gbps + */ + +/* Base period for the rate limit algorithm */ +#define MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS 100 + +/* Number of Base Period to wait between each bucket refill */ +#define MVNETA_TXQ_BUCKET_REFILL_PERIOD 1000 + +/* The base resolution for rate limiting, in bps. Any max_rate value should be + * a multiple of that value. + */ +#define MVNETA_TXQ_RATE_LIMIT_RESOLUTION (NSEC_PER_SEC / \ + (MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS * \ + MVNETA_TXQ_BUCKET_REFILL_PERIOD)) + #define MVNETA_LPI_CTRL_0 0x2cc0 #define MVNETA_LPI_CTRL_1 0x2cc4 #define MVNETA_LPI_REQUEST_ENABLE BIT(0) @@ -492,13 +522,13 @@ struct mvneta_port { u8 mcast_count[256]; u16 tx_ring_size; u16 rx_ring_size; - u8 prio_tc_map[8]; phy_interface_t phy_interface; struct device_node *dn; unsigned int tx_csum_limit; struct phylink *phylink; struct phylink_config phylink_config; + struct phylink_pcs phylink_pcs; struct phy *comphy; struct mvneta_bm *bm_priv; @@ -516,6 +546,7 @@ struct mvneta_port { /* Flags for special SoC configurations */ bool neta_armada3700; + bool neta_ac5; u16 rx_offset_correction; const struct mbus_dram_target_info *dram_target_info; }; @@ -1856,8 +1887,8 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp, bytes_compl += buf->skb->len; pkts_compl++; dev_kfree_skb_any(buf->skb); - } else if (buf->type == MVNETA_TYPE_XDP_TX || - buf->type == MVNETA_TYPE_XDP_NDO) { + } else if ((buf->type == MVNETA_TYPE_XDP_TX || + buf->type == MVNETA_TYPE_XDP_NDO) && buf->xdpf) { if (napi && buf->type == MVNETA_TYPE_XDP_TX) xdp_return_frame_rx_napi(buf->xdpf); else @@ -2032,61 +2063,104 @@ int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) static void mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, - struct xdp_buff *xdp, struct skb_shared_info *sinfo, - int sync_len) + struct xdp_buff *xdp, int sync_len) { + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); int i; + if (likely(!xdp_buff_has_frags(xdp))) + goto out; + for (i = 0; i < sinfo->nr_frags; i++) page_pool_put_full_page(rxq->page_pool, skb_frag_page(&sinfo->frags[i]), true); + +out: page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data), sync_len, true); } static int mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq, - struct xdp_frame *xdpf, bool dma_map) + struct xdp_frame *xdpf, int *nxmit_byte, bool dma_map) { + struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); + struct device *dev = pp->dev->dev.parent; struct mvneta_tx_desc *tx_desc; - struct mvneta_tx_buf *buf; - dma_addr_t dma_addr; + int i, num_frames = 1; + struct page *page; - if (txq->count >= txq->tx_stop_threshold) + if (unlikely(xdp_frame_has_frags(xdpf))) + num_frames += sinfo->nr_frags; + + if (txq->count + num_frames >= txq->size) return MVNETA_XDP_DROPPED; - tx_desc = mvneta_txq_next_desc_get(txq); + for (i = 0; i < num_frames; i++) { + struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index]; + skb_frag_t *frag = NULL; + int len = xdpf->len; + dma_addr_t dma_addr; - buf = &txq->buf[txq->txq_put_index]; - if (dma_map) { - /* ndo_xdp_xmit */ - dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data, - xdpf->len, DMA_TO_DEVICE); - if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) { - mvneta_txq_desc_put(txq); - return MVNETA_XDP_DROPPED; + if (unlikely(i)) { /* paged area */ + frag = &sinfo->frags[i - 1]; + len = skb_frag_size(frag); } - buf->type = MVNETA_TYPE_XDP_NDO; - } else { - struct page *page = virt_to_page(xdpf->data); - dma_addr = page_pool_get_dma_addr(page) + - sizeof(*xdpf) + xdpf->headroom; - dma_sync_single_for_device(pp->dev->dev.parent, dma_addr, - xdpf->len, DMA_BIDIRECTIONAL); - buf->type = MVNETA_TYPE_XDP_TX; - } - buf->xdpf = xdpf; + tx_desc = mvneta_txq_next_desc_get(txq); + if (dma_map) { + /* ndo_xdp_xmit */ + void *data; + + data = unlikely(frag) ? skb_frag_address(frag) + : xdpf->data; + dma_addr = dma_map_single(dev, data, len, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_addr)) { + mvneta_txq_desc_put(txq); + goto unmap; + } - tx_desc->command = MVNETA_TXD_FLZ_DESC; - tx_desc->buf_phys_addr = dma_addr; - tx_desc->data_size = xdpf->len; + buf->type = MVNETA_TYPE_XDP_NDO; + } else { + page = unlikely(frag) ? skb_frag_page(frag) + : virt_to_page(xdpf->data); + dma_addr = page_pool_get_dma_addr(page); + if (unlikely(frag)) + dma_addr += skb_frag_off(frag); + else + dma_addr += sizeof(*xdpf) + xdpf->headroom; + dma_sync_single_for_device(dev, dma_addr, len, + DMA_BIDIRECTIONAL); + buf->type = MVNETA_TYPE_XDP_TX; + } + buf->xdpf = unlikely(i) ? NULL : xdpf; - mvneta_txq_inc_put(txq); - txq->pending++; - txq->count++; + tx_desc->command = unlikely(i) ? 0 : MVNETA_TXD_F_DESC; + tx_desc->buf_phys_addr = dma_addr; + tx_desc->data_size = len; + *nxmit_byte += len; + + mvneta_txq_inc_put(txq); + } + /*last descriptor */ + tx_desc->command |= MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD; + + txq->pending += num_frames; + txq->count += num_frames; return MVNETA_XDP_TX; + +unmap: + for (i--; i >= 0; i--) { + mvneta_txq_desc_put(txq); + tx_desc = txq->descs + txq->next_desc_to_proc; + dma_unmap_single(dev, tx_desc->buf_phys_addr, + tx_desc->data_size, + DMA_TO_DEVICE); + } + + return MVNETA_XDP_DROPPED; } static int @@ -2095,8 +2169,8 @@ mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp) struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); struct mvneta_tx_queue *txq; struct netdev_queue *nq; + int cpu, nxmit_byte = 0; struct xdp_frame *xdpf; - int cpu; u32 ret; xdpf = xdp_convert_buff_to_frame(xdp); @@ -2108,10 +2182,10 @@ mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp) nq = netdev_get_tx_queue(pp->dev, txq->id); __netif_tx_lock(nq, cpu); - ret = mvneta_xdp_submit_frame(pp, txq, xdpf, false); + ret = mvneta_xdp_submit_frame(pp, txq, xdpf, &nxmit_byte, false); if (ret == MVNETA_XDP_TX) { u64_stats_update_begin(&stats->syncp); - stats->es.ps.tx_bytes += xdpf->len; + stats->es.ps.tx_bytes += nxmit_byte; stats->es.ps.tx_packets++; stats->es.ps.xdp_tx++; u64_stats_update_end(&stats->syncp); @@ -2150,11 +2224,11 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame, __netif_tx_lock(nq, cpu); for (i = 0; i < num_frame; i++) { - ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true); + ret = mvneta_xdp_submit_frame(pp, txq, frames[i], &nxmit_byte, + true); if (ret != MVNETA_XDP_TX) break; - nxmit_byte += frames[i]->len; nxmit++; } @@ -2177,7 +2251,6 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, struct bpf_prog *prog, struct xdp_buff *xdp, u32 frame_sz, struct mvneta_stats *stats) { - struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); unsigned int len, data_len, sync; u32 ret, act; @@ -2198,7 +2271,7 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, err = xdp_do_redirect(pp->dev, xdp, prog); if (unlikely(err)) { - mvneta_xdp_put_buff(pp, rxq, xdp, sinfo, sync); + mvneta_xdp_put_buff(pp, rxq, xdp, sync); ret = MVNETA_XDP_DROPPED; } else { ret = MVNETA_XDP_REDIR; @@ -2209,16 +2282,16 @@ mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, case XDP_TX: ret = mvneta_xdp_xmit_back(pp, xdp); if (ret != MVNETA_XDP_TX) - mvneta_xdp_put_buff(pp, rxq, xdp, sinfo, sync); + mvneta_xdp_put_buff(pp, rxq, xdp, sync); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(pp->dev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(pp->dev, prog, act); fallthrough; case XDP_DROP: - mvneta_xdp_put_buff(pp, rxq, xdp, sinfo, sync); + mvneta_xdp_put_buff(pp, rxq, xdp, sync); ret = MVNETA_XDP_DROPPED; stats->xdp_drop++; break; @@ -2241,7 +2314,6 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, int data_len = -MVNETA_MH_SIZE, len; struct net_device *dev = pp->dev; enum dma_data_direction dma_dir; - struct skb_shared_info *sinfo; if (*size > MVNETA_MAX_RX_BUF_SIZE) { len = MVNETA_MAX_RX_BUF_SIZE; @@ -2261,11 +2333,9 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, /* Prefetch header */ prefetch(data); + xdp_buff_clear_frags_flag(xdp); xdp_prepare_buff(xdp, data, pp->rx_offset_correction + MVNETA_MH_SIZE, data_len, false); - - sinfo = xdp_get_shared_info_from_buff(xdp); - sinfo->nr_frags = 0; } static void @@ -2273,9 +2343,9 @@ mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, struct mvneta_rx_desc *rx_desc, struct mvneta_rx_queue *rxq, struct xdp_buff *xdp, int *size, - struct skb_shared_info *xdp_sinfo, struct page *page) { + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); struct net_device *dev = pp->dev; enum dma_data_direction dma_dir; int data_len, len; @@ -2293,25 +2363,25 @@ mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, len, dma_dir); rx_desc->buf_phys_addr = 0; - if (data_len > 0 && xdp_sinfo->nr_frags < MAX_SKB_FRAGS) { - skb_frag_t *frag = &xdp_sinfo->frags[xdp_sinfo->nr_frags++]; + if (!xdp_buff_has_frags(xdp)) + sinfo->nr_frags = 0; + + if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) { + skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags++]; skb_frag_off_set(frag, pp->rx_offset_correction); skb_frag_size_set(frag, data_len); __skb_frag_set_page(frag, page); + + if (!xdp_buff_has_frags(xdp)) { + sinfo->xdp_frags_size = *size; + xdp_buff_set_frags_flag(xdp); + } + if (page_is_pfmemalloc(page)) + xdp_buff_set_frag_pfmemalloc(xdp); } else { page_pool_put_full_page(rxq->page_pool, page, true); } - - /* last fragment */ - if (len == *size) { - struct skb_shared_info *sinfo; - - sinfo = xdp_get_shared_info_from_buff(xdp); - sinfo->nr_frags = xdp_sinfo->nr_frags; - memcpy(sinfo->frags, xdp_sinfo->frags, - sinfo->nr_frags * sizeof(skb_frag_t)); - } *size -= len; } @@ -2320,8 +2390,11 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct page_pool *pool, struct xdp_buff *xdp, u32 desc_status) { struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); - int i, num_frags = sinfo->nr_frags; struct sk_buff *skb; + u8 num_frags; + + if (unlikely(xdp_buff_has_frags(xdp))) + num_frags = sinfo->nr_frags; skb = build_skb(xdp->data_hard_start, PAGE_SIZE); if (!skb) @@ -2333,13 +2406,11 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct page_pool *pool, skb_put(skb, xdp->data_end - xdp->data); skb->ip_summed = mvneta_rx_csum(pp, desc_status); - for (i = 0; i < num_frags; i++) { - skb_frag_t *frag = &sinfo->frags[i]; - - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - skb_frag_page(frag), skb_frag_off(frag), - skb_frag_size(frag), PAGE_SIZE); - } + if (unlikely(xdp_buff_has_frags(xdp))) + xdp_update_skb_shared_info(skb, num_frags, + sinfo->xdp_frags_size, + num_frags * xdp->frame_sz, + xdp_buff_is_frag_pfmemalloc(xdp)); return skb; } @@ -2351,7 +2422,6 @@ static int mvneta_rx_swbm(struct napi_struct *napi, { int rx_proc = 0, rx_todo, refill, size = 0; struct net_device *dev = pp->dev; - struct skb_shared_info sinfo; struct mvneta_stats ps = {}; struct bpf_prog *xdp_prog; u32 desc_status, frame_sz; @@ -2360,8 +2430,6 @@ static int mvneta_rx_swbm(struct napi_struct *napi, xdp_init_buff(&xdp_buf, PAGE_SIZE, &rxq->xdp_rxq); xdp_buf.data_hard_start = NULL; - sinfo.nr_frags = 0; - /* Get number of received packets */ rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); @@ -2403,7 +2471,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, } mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf, - &size, &sinfo, page); + &size, page); } /* Middle or Last descriptor */ if (!(rx_status & MVNETA_RXD_LAST_DESC)) @@ -2411,7 +2479,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, continue; if (size) { - mvneta_xdp_put_buff(pp, rxq, &xdp_buf, &sinfo, -1); + mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1); goto next; } @@ -2423,7 +2491,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, if (IS_ERR(skb)) { struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); - mvneta_xdp_put_buff(pp, rxq, &xdp_buf, &sinfo, -1); + mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1); u64_stats_update_begin(&stats->syncp); stats->es.skb_alloc_error++; @@ -2440,11 +2508,10 @@ static int mvneta_rx_swbm(struct napi_struct *napi, napi_gro_receive(napi, skb); next: xdp_buf.data_hard_start = NULL; - sinfo.nr_frags = 0; } if (xdp_buf.data_hard_start) - mvneta_xdp_put_buff(pp, rxq, &xdp_buf, &sinfo, -1); + mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1); if (ps.xdp_redirect) xdp_do_flush_map(); @@ -2597,8 +2664,8 @@ err_drop_frame: static inline void mvneta_tso_put_hdr(struct sk_buff *skb, struct mvneta_tx_queue *txq) { - int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index]; + int hdr_len = skb_tcp_all_headers(skb); struct mvneta_tx_desc *tx_desc; tx_desc = mvneta_txq_next_desc_get(txq); @@ -2660,7 +2727,7 @@ static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev, if ((txq->count + tso_count_descs(skb)) >= txq->size) return 0; - if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) { + if (skb_headlen(skb) < skb_tcp_all_headers(skb)) { pr_info("*** Is this even possible?\n"); return 0; } @@ -3232,7 +3299,8 @@ static int mvneta_create_page_pool(struct mvneta_port *pp, return err; } - err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0); + err = __xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0, + PAGE_SIZE); if (err < 0) goto err_free_pp; @@ -3712,6 +3780,7 @@ static void mvneta_percpu_disable(void *arg) static int mvneta_change_mtu(struct net_device *dev, int mtu) { struct mvneta_port *pp = netdev_priv(dev); + struct bpf_prog *prog = pp->xdp_prog; int ret; if (!IS_ALIGNED(MVNETA_RX_PKT_SIZE(mtu), 8)) { @@ -3720,8 +3789,11 @@ static int mvneta_change_mtu(struct net_device *dev, int mtu) mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8); } - if (pp->xdp_prog && mtu > MVNETA_MAX_RX_BUF_SIZE) { - netdev_info(dev, "Illegal MTU value %d for XDP mode\n", mtu); + if (prog && !prog->aux->xdp_has_frags && + mtu > MVNETA_MAX_RX_BUF_SIZE) { + netdev_info(dev, "Illegal MTU %d for XDP prog without frags\n", + mtu); + return -EINVAL; } @@ -3819,58 +3891,31 @@ static int mvneta_set_mac_addr(struct net_device *dev, void *addr) return 0; } -static void mvneta_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) +static struct mvneta_port *mvneta_pcs_to_port(struct phylink_pcs *pcs) { - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; + return container_of(pcs, struct mvneta_port, phylink_pcs); +} +static int mvneta_pcs_validate(struct phylink_pcs *pcs, + unsigned long *supported, + const struct phylink_link_state *state) +{ /* We only support QSGMII, SGMII, 802.3z and RGMII modes. * When in 802.3z mode, we must have AN enabled: * "Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... * When <PortType> = 1 (1000BASE-X) this field must be set to 1." */ if (phy_interface_mode_is_8023z(state->interface) && - !phylink_test(state->advertising, Autoneg)) { - linkmode_zero(supported); - return; - } - - /* Allow all the expected bits */ - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - - /* Asymmetric pause is unsupported */ - phylink_set(mask, Pause); - - /* Half-duplex at speeds higher than 100Mbit is unsupported */ - if (state->interface != PHY_INTERFACE_MODE_2500BASEX) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - } - - if (state->interface == PHY_INTERFACE_MODE_2500BASEX) { - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - } - - if (!phy_interface_mode_is_8023z(state->interface)) { - /* 10M and 100M are only supported in non-802.3z mode */ - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - } + !phylink_test(state->advertising, Autoneg)) + return -EINVAL; - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); + return 0; } -static void mvneta_mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state) +static void mvneta_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) { - struct net_device *ndev = to_net_dev(config->dev); - struct mvneta_port *pp = netdev_priv(ndev); + struct mvneta_port *pp = mvneta_pcs_to_port(pcs); u32 gmac_stat; gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); @@ -3888,17 +3933,71 @@ static void mvneta_mac_pcs_get_state(struct phylink_config *config, state->link = !!(gmac_stat & MVNETA_GMAC_LINK_UP); state->duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX); - state->pause = 0; if (gmac_stat & MVNETA_GMAC_RX_FLOW_CTRL_ENABLE) state->pause |= MLO_PAUSE_RX; if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE) state->pause |= MLO_PAUSE_TX; } -static void mvneta_mac_an_restart(struct phylink_config *config) +static int mvneta_pcs_config(struct phylink_pcs *pcs, + unsigned int mode, phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) { - struct net_device *ndev = to_net_dev(config->dev); - struct mvneta_port *pp = netdev_priv(ndev); + struct mvneta_port *pp = mvneta_pcs_to_port(pcs); + u32 mask, val, an, old_an, changed; + + mask = MVNETA_GMAC_INBAND_AN_ENABLE | + MVNETA_GMAC_INBAND_RESTART_AN | + MVNETA_GMAC_AN_SPEED_EN | + MVNETA_GMAC_AN_FLOW_CTRL_EN | + MVNETA_GMAC_AN_DUPLEX_EN; + + if (phylink_autoneg_inband(mode)) { + mask |= MVNETA_GMAC_CONFIG_MII_SPEED | + MVNETA_GMAC_CONFIG_GMII_SPEED | + MVNETA_GMAC_CONFIG_FULL_DUPLEX; + val = MVNETA_GMAC_INBAND_AN_ENABLE; + + if (interface == PHY_INTERFACE_MODE_SGMII) { + /* SGMII mode receives the speed and duplex from PHY */ + val |= MVNETA_GMAC_AN_SPEED_EN | + MVNETA_GMAC_AN_DUPLEX_EN; + } else { + /* 802.3z mode has fixed speed and duplex */ + val |= MVNETA_GMAC_CONFIG_GMII_SPEED | + MVNETA_GMAC_CONFIG_FULL_DUPLEX; + + /* The FLOW_CTRL_EN bit selects either the hardware + * automatically or the CONFIG_FLOW_CTRL manually + * controls the GMAC pause mode. + */ + if (permit_pause_to_mac) + val |= MVNETA_GMAC_AN_FLOW_CTRL_EN; + + /* Update the advertisement bits */ + mask |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL; + if (phylink_test(advertising, Pause)) + val |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL; + } + } else { + /* Phy or fixed speed - disable in-band AN modes */ + val = 0; + } + + old_an = an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); + an = (an & ~mask) | val; + changed = old_an ^ an; + if (changed) + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, an); + + /* We are only interested in the advertisement bits changing */ + return !!(changed & MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL); +} + +static void mvneta_pcs_an_restart(struct phylink_pcs *pcs) +{ + struct mvneta_port *pp = mvneta_pcs_to_port(pcs); u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, @@ -3907,6 +4006,56 @@ static void mvneta_mac_an_restart(struct phylink_config *config) gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN); } +static const struct phylink_pcs_ops mvneta_phylink_pcs_ops = { + .pcs_validate = mvneta_pcs_validate, + .pcs_get_state = mvneta_pcs_get_state, + .pcs_config = mvneta_pcs_config, + .pcs_an_restart = mvneta_pcs_an_restart, +}; + +static struct phylink_pcs *mvneta_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct mvneta_port *pp = netdev_priv(ndev); + + return &pp->phylink_pcs; +} + +static int mvneta_mac_prepare(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct mvneta_port *pp = netdev_priv(ndev); + u32 val; + + if (pp->phy_interface != interface || + phylink_autoneg_inband(mode)) { + /* Force the link down when changing the interface or if in + * in-band mode. According to Armada 370 documentation, we + * can only change the port mode and in-band enable when the + * link is down. + */ + val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); + val &= ~MVNETA_GMAC_FORCE_LINK_PASS; + val |= MVNETA_GMAC_FORCE_LINK_DOWN; + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); + } + + if (pp->phy_interface != interface) + WARN_ON(phy_power_off(pp->comphy)); + + /* Enable the 1ms clock */ + if (phylink_autoneg_inband(mode)) { + unsigned long rate = clk_get_rate(pp->clk); + + mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, + MVNETA_GMAC_1MS_CLOCK_ENABLE | (rate / 1000)); + } + + return 0; +} + static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { @@ -3915,20 +4064,11 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0); u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2); u32 new_ctrl4, gmac_ctrl4 = mvreg_read(pp, MVNETA_GMAC_CTRL_4); - u32 new_clk, gmac_clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); - u32 new_an, gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X; new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE | MVNETA_GMAC2_PORT_RESET); new_ctrl4 = gmac_ctrl4 & ~(MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE); - new_clk = gmac_clk & ~MVNETA_GMAC_1MS_CLOCK_ENABLE; - new_an = gmac_an & ~(MVNETA_GMAC_INBAND_AN_ENABLE | - MVNETA_GMAC_INBAND_RESTART_AN | - MVNETA_GMAC_AN_SPEED_EN | - MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL | - MVNETA_GMAC_AN_FLOW_CTRL_EN | - MVNETA_GMAC_AN_DUPLEX_EN); /* Even though it might look weird, when we're configured in * SGMII or QSGMII mode, the RGMII bit needs to be set. @@ -3940,9 +4080,6 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, phy_interface_mode_is_8023z(state->interface)) new_ctrl2 |= MVNETA_GMAC2_PCS_ENABLE; - if (phylink_test(state->advertising, Pause)) - new_an |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL; - if (!phylink_autoneg_inband(mode)) { /* Phy or fixed speed - nothing to do, leave the * configured speed, duplex and flow control as-is. @@ -3950,66 +4087,23 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { /* SGMII mode receives the state from the PHY */ new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE; - new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; - new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | - MVNETA_GMAC_FORCE_LINK_PASS | - MVNETA_GMAC_CONFIG_MII_SPEED | - MVNETA_GMAC_CONFIG_GMII_SPEED | - MVNETA_GMAC_CONFIG_FULL_DUPLEX)) | - MVNETA_GMAC_INBAND_AN_ENABLE | - MVNETA_GMAC_AN_SPEED_EN | - MVNETA_GMAC_AN_DUPLEX_EN; } else { /* 802.3z negotiation - only 1000base-X */ new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X; - new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; - new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | - MVNETA_GMAC_FORCE_LINK_PASS | - MVNETA_GMAC_CONFIG_MII_SPEED)) | - MVNETA_GMAC_INBAND_AN_ENABLE | - MVNETA_GMAC_CONFIG_GMII_SPEED | - /* The MAC only supports FD mode */ - MVNETA_GMAC_CONFIG_FULL_DUPLEX; - - if (state->pause & MLO_PAUSE_AN && state->an_enabled) - new_an |= MVNETA_GMAC_AN_FLOW_CTRL_EN; } - /* Armada 370 documentation says we can only change the port mode - * and in-band enable when the link is down, so force it down - * while making these changes. We also do this for GMAC_CTRL2 - */ - if ((new_ctrl0 ^ gmac_ctrl0) & MVNETA_GMAC0_PORT_1000BASE_X || - (new_ctrl2 ^ gmac_ctrl2) & MVNETA_GMAC2_INBAND_AN_ENABLE || - (new_an ^ gmac_an) & MVNETA_GMAC_INBAND_AN_ENABLE) { - mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, - (gmac_an & ~MVNETA_GMAC_FORCE_LINK_PASS) | - MVNETA_GMAC_FORCE_LINK_DOWN); - } - - /* When at 2.5G, the link partner can send frames with shortened * preambles. */ if (state->interface == PHY_INTERFACE_MODE_2500BASEX) new_ctrl4 |= MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE; - if (pp->phy_interface != state->interface) { - if (pp->comphy) - WARN_ON(phy_power_off(pp->comphy)); - WARN_ON(mvneta_config_interface(pp, state->interface)); - } - if (new_ctrl0 != gmac_ctrl0) mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0); if (new_ctrl2 != gmac_ctrl2) mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2); if (new_ctrl4 != gmac_ctrl4) mvreg_write(pp, MVNETA_GMAC_CTRL_4, new_ctrl4); - if (new_clk != gmac_clk) - mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, new_clk); - if (new_an != gmac_an) - mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, new_an); if (gmac_ctrl2 & MVNETA_GMAC2_PORT_RESET) { while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) & @@ -4018,6 +4112,36 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, } } +static int mvneta_mac_finish(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct mvneta_port *pp = netdev_priv(ndev); + u32 val, clk; + + /* Disable 1ms clock if not in in-band mode */ + if (!phylink_autoneg_inband(mode)) { + clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); + clk &= ~MVNETA_GMAC_1MS_CLOCK_ENABLE; + mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, clk); + } + + if (pp->phy_interface != interface) + /* Enable the Serdes PHY */ + WARN_ON(mvneta_config_interface(pp, interface)); + + /* Allow the link to come up if in in-band mode, otherwise the + * link is forced via mac_link_down()/mac_link_up() + */ + if (phylink_autoneg_inband(mode)) { + val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); + val &= ~MVNETA_GMAC_FORCE_LINK_DOWN; + mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); + } + + return 0; +} + static void mvneta_set_eee(struct mvneta_port *pp, bool enable) { u32 lpi_ctl1; @@ -4098,16 +4222,17 @@ static void mvneta_mac_link_up(struct phylink_config *config, mvneta_port_up(pp); if (phy && pp->eee_enabled) { - pp->eee_active = phy_init_eee(phy, 0) >= 0; + pp->eee_active = phy_init_eee(phy, false) >= 0; mvneta_set_eee(pp, pp->eee_active && pp->tx_lpi_enabled); } } static const struct phylink_mac_ops mvneta_phylink_ops = { - .validate = mvneta_validate, - .mac_pcs_get_state = mvneta_mac_pcs_get_state, - .mac_an_restart = mvneta_mac_an_restart, + .validate = phylink_generic_validate, + .mac_select_pcs = mvneta_mac_select_pcs, + .mac_prepare = mvneta_mac_prepare, .mac_config = mvneta_mac_config, + .mac_finish = mvneta_mac_finish, .mac_link_down = mvneta_mac_link_down, .mac_link_up = mvneta_mac_link_up, }; @@ -4419,8 +4544,9 @@ static int mvneta_xdp_setup(struct net_device *dev, struct bpf_prog *prog, struct mvneta_port *pp = netdev_priv(dev); struct bpf_prog *old_prog; - if (prog && dev->mtu > MVNETA_MAX_RX_BUF_SIZE) { - NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP"); + if (prog && !prog->aux->xdp_has_frags && + dev->mtu > MVNETA_MAX_RX_BUF_SIZE) { + NL_SET_ERR_MSG_MOD(extack, "prog does not support XDP frags"); return -EOPNOTSUPP; } @@ -4530,17 +4656,20 @@ mvneta_ethtool_get_coalesce(struct net_device *dev, static void mvneta_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME, + strscpy(drvinfo->driver, MVNETA_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION, + strscpy(drvinfo->version, MVNETA_DRIVER_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(&dev->dev), + strscpy(drvinfo->bus_info, dev_name(&dev->dev), sizeof(drvinfo->bus_info)); } -static void mvneta_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void +mvneta_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(netdev); @@ -4550,8 +4679,11 @@ static void mvneta_ethtool_get_ringparam(struct net_device *netdev, ring->tx_pending = pp->tx_ring_size; } -static int mvneta_ethtool_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static int +mvneta_ethtool_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(dev); @@ -4603,6 +4735,9 @@ static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset, for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) memcpy(data + i * ETH_GSTRING_LEN, mvneta_statistics[i].name, ETH_GSTRING_LEN); + + data += ETH_GSTRING_LEN * ARRAY_SIZE(mvneta_statistics); + page_pool_ethtool_stats_get_strings(data); } } @@ -4715,6 +4850,17 @@ static void mvneta_ethtool_update_stats(struct mvneta_port *pp) } } +static void mvneta_ethtool_pp_stats(struct mvneta_port *pp, u64 *data) +{ + struct page_pool_stats stats = {}; + int i; + + for (i = 0; i < rxq_number; i++) + page_pool_get_stats(pp->rxqs[i].page_pool, &stats); + + page_pool_ethtool_stats_get(data, &stats); +} + static void mvneta_ethtool_get_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { @@ -4725,12 +4871,16 @@ static void mvneta_ethtool_get_stats(struct net_device *dev, for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) *data++ = pp->ethtool_stats[i]; + + mvneta_ethtool_pp_stats(pp, data); } static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset) { if (sset == ETH_SS_STATS) - return ARRAY_SIZE(mvneta_statistics); + return ARRAY_SIZE(mvneta_statistics) + + page_pool_ethtool_stats_get_count(); + return -EOPNOTSUPP; } @@ -4919,43 +5069,144 @@ static void mvneta_clear_rx_prio_map(struct mvneta_port *pp) mvreg_write(pp, MVNETA_VLAN_PRIO_TO_RXQ, 0); } -static void mvneta_setup_rx_prio_map(struct mvneta_port *pp) +static void mvneta_map_vlan_prio_to_rxq(struct mvneta_port *pp, u8 pri, u8 rxq) { - u32 val = 0; - int i; + u32 val = mvreg_read(pp, MVNETA_VLAN_PRIO_TO_RXQ); - for (i = 0; i < rxq_number; i++) - val |= MVNETA_VLAN_PRIO_RXQ_MAP(i, pp->prio_tc_map[i]); + val &= ~MVNETA_VLAN_PRIO_RXQ_MAP(pri, 0x7); + val |= MVNETA_VLAN_PRIO_RXQ_MAP(pri, rxq); mvreg_write(pp, MVNETA_VLAN_PRIO_TO_RXQ, val); } +static int mvneta_enable_per_queue_rate_limit(struct mvneta_port *pp) +{ + unsigned long core_clk_rate; + u32 refill_cycles; + u32 val; + + core_clk_rate = clk_get_rate(pp->clk); + if (!core_clk_rate) + return -EINVAL; + + refill_cycles = MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS / + (NSEC_PER_SEC / core_clk_rate); + + if (refill_cycles > MVNETA_REFILL_MAX_NUM_CLK) + return -EINVAL; + + /* Enable bw limit algorithm version 3 */ + val = mvreg_read(pp, MVNETA_TXQ_CMD1_REG); + val &= ~(MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 | MVNETA_TXQ_CMD1_BW_LIM_EN); + mvreg_write(pp, MVNETA_TXQ_CMD1_REG, val); + + /* Set the base refill rate */ + mvreg_write(pp, MVNETA_REFILL_NUM_CLK_REG, refill_cycles); + + return 0; +} + +static void mvneta_disable_per_queue_rate_limit(struct mvneta_port *pp) +{ + u32 val = mvreg_read(pp, MVNETA_TXQ_CMD1_REG); + + val |= (MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 | MVNETA_TXQ_CMD1_BW_LIM_EN); + mvreg_write(pp, MVNETA_TXQ_CMD1_REG, val); +} + +static int mvneta_setup_queue_rates(struct mvneta_port *pp, int queue, + u64 min_rate, u64 max_rate) +{ + u32 refill_val, rem; + u32 val = 0; + + /* Convert to from Bps to bps */ + max_rate *= 8; + + if (min_rate) + return -EINVAL; + + refill_val = div_u64_rem(max_rate, MVNETA_TXQ_RATE_LIMIT_RESOLUTION, + &rem); + + if (rem || !refill_val || + refill_val > MVNETA_TXQ_BUCKET_REFILL_VALUE_MAX) + return -EINVAL; + + val = refill_val; + val |= (MVNETA_TXQ_BUCKET_REFILL_PERIOD << + MVNETA_TXQ_BUCKET_REFILL_PERIOD_SHIFT); + + mvreg_write(pp, MVNETA_TXQ_BUCKET_REFILL_REG(queue), val); + + return 0; +} + static int mvneta_setup_mqprio(struct net_device *dev, - struct tc_mqprio_qopt *qopt) + struct tc_mqprio_qopt_offload *mqprio) { struct mvneta_port *pp = netdev_priv(dev); + int rxq, txq, tc, ret; u8 num_tc; - int i; - qopt->hw = TC_MQPRIO_HW_OFFLOAD_TCS; - num_tc = qopt->num_tc; + if (mqprio->qopt.hw != TC_MQPRIO_HW_OFFLOAD_TCS) + return 0; + + num_tc = mqprio->qopt.num_tc; if (num_tc > rxq_number) return -EINVAL; + mvneta_clear_rx_prio_map(pp); + if (!num_tc) { - mvneta_clear_rx_prio_map(pp); + mvneta_disable_per_queue_rate_limit(pp); netdev_reset_tc(dev); return 0; } - memcpy(pp->prio_tc_map, qopt->prio_tc_map, sizeof(pp->prio_tc_map)); + netdev_set_num_tc(dev, mqprio->qopt.num_tc); + + for (tc = 0; tc < mqprio->qopt.num_tc; tc++) { + netdev_set_tc_queue(dev, tc, mqprio->qopt.count[tc], + mqprio->qopt.offset[tc]); + + for (rxq = mqprio->qopt.offset[tc]; + rxq < mqprio->qopt.count[tc] + mqprio->qopt.offset[tc]; + rxq++) { + if (rxq >= rxq_number) + return -EINVAL; + + mvneta_map_vlan_prio_to_rxq(pp, tc, rxq); + } + } - mvneta_setup_rx_prio_map(pp); + if (mqprio->shaper != TC_MQPRIO_SHAPER_BW_RATE) { + mvneta_disable_per_queue_rate_limit(pp); + return 0; + } - netdev_set_num_tc(dev, qopt->num_tc); - for (i = 0; i < qopt->num_tc; i++) - netdev_set_tc_queue(dev, i, qopt->count[i], qopt->offset[i]); + if (mqprio->qopt.num_tc > txq_number) + return -EINVAL; + + ret = mvneta_enable_per_queue_rate_limit(pp); + if (ret) + return ret; + + for (tc = 0; tc < mqprio->qopt.num_tc; tc++) { + for (txq = mqprio->qopt.offset[tc]; + txq < mqprio->qopt.count[tc] + mqprio->qopt.offset[tc]; + txq++) { + if (txq >= txq_number) + return -EINVAL; + + ret = mvneta_setup_queue_rates(pp, txq, + mqprio->min_rate[tc], + mqprio->max_rate[tc]); + if (ret) + return ret; + } + } return 0; } @@ -5094,6 +5345,10 @@ static void mvneta_conf_mbus_windows(struct mvneta_port *pp, win_protect |= 3 << (2 * i); } } else { + if (pp->neta_ac5) + mvreg_write(pp, MVNETA_WIN_BASE(0), + (MVNETA_AC5_CNM_DDR_ATTR << 8) | + MVNETA_AC5_CNM_DDR_TARGET); /* For Armada3700 open default 4GB Mbus window, leaving * arbitration of target/attribute to a different layer * of configuration. @@ -5143,29 +5398,72 @@ static int mvneta_probe(struct platform_device *pdev) if (!dev) return -ENOMEM; - dev->irq = irq_of_parse_and_map(dn, 0); - if (dev->irq == 0) - return -EINVAL; + dev->tx_queue_len = MVNETA_MAX_TXD; + dev->watchdog_timeo = 5 * HZ; + dev->netdev_ops = &mvneta_netdev_ops; + dev->ethtool_ops = &mvneta_eth_tool_ops; + + pp = netdev_priv(dev); + spin_lock_init(&pp->lock); + pp->dn = dn; + + pp->rxq_def = rxq_def; + pp->indir[0] = rxq_def; err = of_get_phy_mode(dn, &phy_mode); if (err) { dev_err(&pdev->dev, "incorrect phy-mode\n"); - goto err_free_irq; + return err; } + pp->phy_interface = phy_mode; + comphy = devm_of_phy_get(&pdev->dev, dn, NULL); - if (comphy == ERR_PTR(-EPROBE_DEFER)) { - err = -EPROBE_DEFER; - goto err_free_irq; - } else if (IS_ERR(comphy)) { + if (comphy == ERR_PTR(-EPROBE_DEFER)) + return -EPROBE_DEFER; + + if (IS_ERR(comphy)) comphy = NULL; + + pp->comphy = comphy; + + pp->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pp->base)) + return PTR_ERR(pp->base); + + /* Get special SoC configurations */ + if (of_device_is_compatible(dn, "marvell,armada-3700-neta")) + pp->neta_armada3700 = true; + if (of_device_is_compatible(dn, "marvell,armada-ac5-neta")) { + pp->neta_armada3700 = true; + pp->neta_ac5 = true; } - pp = netdev_priv(dev); - spin_lock_init(&pp->lock); + dev->irq = irq_of_parse_and_map(dn, 0); + if (dev->irq == 0) + return -EINVAL; + + pp->clk = devm_clk_get(&pdev->dev, "core"); + if (IS_ERR(pp->clk)) + pp->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(pp->clk)) { + err = PTR_ERR(pp->clk); + goto err_free_irq; + } + + clk_prepare_enable(pp->clk); + + pp->clk_bus = devm_clk_get(&pdev->dev, "bus"); + if (!IS_ERR(pp->clk_bus)) + clk_prepare_enable(pp->clk_bus); + + pp->phylink_pcs.ops = &mvneta_phylink_pcs_ops; pp->phylink_config.dev = &dev->dev; pp->phylink_config.type = PHYLINK_NETDEV; + pp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | + MAC_100 | MAC_1000FD | MAC_2500FD; + phy_interface_set_rgmii(pp->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_QSGMII, pp->phylink_config.supported_interfaces); @@ -5196,52 +5494,16 @@ static int mvneta_probe(struct platform_device *pdev) phy_mode, &mvneta_phylink_ops); if (IS_ERR(phylink)) { err = PTR_ERR(phylink); - goto err_free_irq; + goto err_clk; } - dev->tx_queue_len = MVNETA_MAX_TXD; - dev->watchdog_timeo = 5 * HZ; - dev->netdev_ops = &mvneta_netdev_ops; - - dev->ethtool_ops = &mvneta_eth_tool_ops; - pp->phylink = phylink; - pp->comphy = comphy; - pp->phy_interface = phy_mode; - pp->dn = dn; - - pp->rxq_def = rxq_def; - pp->indir[0] = rxq_def; - - /* Get special SoC configurations */ - if (of_device_is_compatible(dn, "marvell,armada-3700-neta")) - pp->neta_armada3700 = true; - - pp->clk = devm_clk_get(&pdev->dev, "core"); - if (IS_ERR(pp->clk)) - pp->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pp->clk)) { - err = PTR_ERR(pp->clk); - goto err_free_phylink; - } - - clk_prepare_enable(pp->clk); - - pp->clk_bus = devm_clk_get(&pdev->dev, "bus"); - if (!IS_ERR(pp->clk_bus)) - clk_prepare_enable(pp->clk_bus); - - pp->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(pp->base)) { - err = PTR_ERR(pp->base); - goto err_clk; - } /* Alloc per-cpu port structure */ pp->ports = alloc_percpu(struct mvneta_pcpu_port); if (!pp->ports) { err = -ENOMEM; - goto err_clk; + goto err_free_phylink; } /* Alloc per-cpu stats */ @@ -5338,14 +5600,13 @@ static int mvneta_probe(struct platform_device *pdev) * operation, so only single NAPI should be initialized. */ if (pp->neta_armada3700) { - netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &pp->napi, mvneta_poll); } else { for_each_present_cpu(cpu) { struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); - netif_napi_add(dev, &port->napi, mvneta_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &port->napi, mvneta_poll); port->pp = pp; } } @@ -5355,7 +5616,7 @@ static int mvneta_probe(struct platform_device *pdev) dev->hw_features |= dev->features; dev->vlan_features |= dev->features; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; - dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; + netif_set_tso_max_segs(dev, MVNETA_MAX_TSO_SEGS); /* MTU range: 68 - 9676 */ dev->min_mtu = ETH_MIN_MTU; @@ -5385,12 +5646,12 @@ err_netdev: free_percpu(pp->stats); err_free_ports: free_percpu(pp->ports); -err_clk: - clk_disable_unprepare(pp->clk_bus); - clk_disable_unprepare(pp->clk); err_free_phylink: if (pp->phylink) phylink_destroy(pp->phylink); +err_clk: + clk_disable_unprepare(pp->clk_bus); + clk_disable_unprepare(pp->clk); err_free_irq: irq_dispose_mapping(dev->irq); return err; @@ -5536,6 +5797,7 @@ static const struct of_device_id mvneta_match[] = { { .compatible = "marvell,armada-370-neta" }, { .compatible = "marvell,armada-xp-neta" }, { .compatible = "marvell,armada-3700-neta" }, + { .compatible = "marvell,armada-ac5-neta" }, { } }; MODULE_DEVICE_TABLE(of, mvneta_match); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h index cf8acabb90ac..11e603686a27 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h @@ -1239,7 +1239,8 @@ struct mvpp2_port { phy_interface_t phy_interface; struct phylink *phylink; struct phylink_config phylink_config; - struct phylink_pcs phylink_pcs; + struct phylink_pcs pcs_gmac; + struct phylink_pcs pcs_xlg; struct phy *comphy; struct mvpp2_bm_pool *pool_long; @@ -1529,6 +1530,7 @@ u32 mvpp2_read(struct mvpp2 *priv, u32 offset); void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name); void mvpp2_dbgfs_cleanup(struct mvpp2 *priv); +void mvpp2_dbgfs_exit(void); void mvpp23_rx_fifo_fc_en(struct mvpp2 *priv, int port, bool en); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c index 4a3baa7e0142..75e83ea2a926 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c @@ -691,6 +691,13 @@ static int mvpp2_dbgfs_port_init(struct dentry *parent, return 0; } +static struct dentry *mvpp2_root; + +void mvpp2_dbgfs_exit(void) +{ + debugfs_remove(mvpp2_root); +} + void mvpp2_dbgfs_cleanup(struct mvpp2 *priv) { debugfs_remove_recursive(priv->dbgfs_dir); @@ -700,10 +707,9 @@ void mvpp2_dbgfs_cleanup(struct mvpp2 *priv) void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name) { - struct dentry *mvpp2_dir, *mvpp2_root; + struct dentry *mvpp2_dir; int ret, i; - mvpp2_root = debugfs_lookup(MVPP2_DRIVER_NAME, NULL); if (!mvpp2_root) mvpp2_root = debugfs_create_dir(MVPP2_DRIVER_NAME, NULL); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 6da8a595026b..eb0fb8128096 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -1488,6 +1488,7 @@ static bool mvpp2_port_supports_rgmii(struct mvpp2_port *port) static bool mvpp2_is_xlg(phy_interface_t interface) { return interface == PHY_INTERFACE_MODE_10GBASER || + interface == PHY_INTERFACE_MODE_5GBASER || interface == PHY_INTERFACE_MODE_XAUI; } @@ -1627,6 +1628,7 @@ static int mvpp22_gop_init(struct mvpp2_port *port, phy_interface_t interface) case PHY_INTERFACE_MODE_2500BASEX: mvpp22_gop_init_sgmii(port); break; + case PHY_INTERFACE_MODE_5GBASER: case PHY_INTERFACE_MODE_10GBASER: if (!mvpp2_port_supports_xlg(port)) goto invalid_conf; @@ -1867,7 +1869,7 @@ static u32 mvpp2_read_index(struct mvpp2 *priv, u32 index, u32 reg) * design, incremented at different moments in the chain of packet processing, * it is very likely that incoming packets could have been dropped after being * counted by hardware but before reaching software statistics (most probably - * multicast packets), and in the oppposite way, during transmission, FCS bytes + * multicast packets), and in the opposite way, during transmission, FCS bytes * are added in between as well as TSO skb will be split and header bytes added. * Hence, statistics gathered from userspace with ifconfig (software) and * ethtool (hardware) cannot be compared. @@ -2186,6 +2188,7 @@ static void mvpp22_pcs_reset_deassert(struct mvpp2_port *port, xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); switch (interface) { + case PHY_INTERFACE_MODE_5GBASER: case PHY_INTERFACE_MODE_10GBASER: val = readl(mpcs + MVPP22_MPCS_CLK_RESET); val |= MAC_CLK_RESET_MAC | MAC_CLK_RESET_SD_RX | @@ -3820,7 +3823,7 @@ mvpp2_run_xdp(struct mvpp2_port *port, struct bpf_prog *prog, } break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(port->dev, prog, act); fallthrough; case XDP_ABORTED: trace_xdp_exception(port->dev, prog, act); @@ -5139,9 +5142,6 @@ static int mvpp2_set_ts_config(struct mvpp2_port *port, struct ifreq *ifr) if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) return -EFAULT; - if (config.flags) - return -EINVAL; - if (config.tx_type != HWTSTAMP_TX_OFF && config.tx_type != HWTSTAMP_TX_ON) return -ERANGE; @@ -5425,16 +5425,19 @@ mvpp2_ethtool_get_coalesce(struct net_device *dev, static void mvpp2_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME, + strscpy(drvinfo->driver, MVPP2_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION, + strscpy(drvinfo->version, MVPP2_DRIVER_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(&dev->dev), + strscpy(drvinfo->bus_info, dev_name(&dev->dev), sizeof(drvinfo->bus_info)); } -static void mvpp2_ethtool_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static void +mvpp2_ethtool_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); @@ -5444,8 +5447,11 @@ static void mvpp2_ethtool_get_ringparam(struct net_device *dev, ring->tx_pending = port->tx_ring_size; } -static int mvpp2_ethtool_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static int +mvpp2_ethtool_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); u16 prev_rx_ring_size = port->rx_ring_size; @@ -5764,8 +5770,7 @@ static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port, v->irq = irq_of_parse_and_map(port_node, 0); if (v->irq <= 0) return -EINVAL; - netif_napi_add(port->dev, &v->napi, mvpp2_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(port->dev, &v->napi, mvpp2_poll); port->nqvecs = 1; @@ -5825,8 +5830,7 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port, goto err; } - netif_napi_add(port->dev, &v->napi, mvpp2_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(port->dev, &v->napi, mvpp2_poll); } return 0; @@ -6109,18 +6113,26 @@ static struct mvpp2_port *mvpp2_phylink_to_port(struct phylink_config *config) return container_of(config, struct mvpp2_port, phylink_config); } -static struct mvpp2_port *mvpp2_pcs_to_port(struct phylink_pcs *pcs) +static struct mvpp2_port *mvpp2_pcs_xlg_to_port(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct mvpp2_port, pcs_xlg); +} + +static struct mvpp2_port *mvpp2_pcs_gmac_to_port(struct phylink_pcs *pcs) { - return container_of(pcs, struct mvpp2_port, phylink_pcs); + return container_of(pcs, struct mvpp2_port, pcs_gmac); } static void mvpp2_xlg_pcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state) { - struct mvpp2_port *port = mvpp2_pcs_to_port(pcs); + struct mvpp2_port *port = mvpp2_pcs_xlg_to_port(pcs); u32 val; - state->speed = SPEED_10000; + if (port->phy_interface == PHY_INTERFACE_MODE_5GBASER) + state->speed = SPEED_5000; + else + state->speed = SPEED_10000; state->duplex = 1; state->an_complete = 1; @@ -6149,10 +6161,25 @@ static const struct phylink_pcs_ops mvpp2_phylink_xlg_pcs_ops = { .pcs_config = mvpp2_xlg_pcs_config, }; +static int mvpp2_gmac_pcs_validate(struct phylink_pcs *pcs, + unsigned long *supported, + const struct phylink_link_state *state) +{ + /* When in 802.3z mode, we must have AN enabled: + * Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... + * When <PortType> = 1 (1000BASE-X) this field must be set to 1. + */ + if (phy_interface_mode_is_8023z(state->interface) && + !phylink_test(state->advertising, Autoneg)) + return -EINVAL; + + return 0; +} + static void mvpp2_gmac_pcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state) { - struct mvpp2_port *port = mvpp2_pcs_to_port(pcs); + struct mvpp2_port *port = mvpp2_pcs_gmac_to_port(pcs); u32 val; val = readl(port->base + MVPP2_GMAC_STATUS0); @@ -6189,7 +6216,7 @@ static int mvpp2_gmac_pcs_config(struct phylink_pcs *pcs, unsigned int mode, const unsigned long *advertising, bool permit_pause_to_mac) { - struct mvpp2_port *port = mvpp2_pcs_to_port(pcs); + struct mvpp2_port *port = mvpp2_pcs_gmac_to_port(pcs); u32 mask, val, an, old_an, changed; mask = MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS | @@ -6243,7 +6270,7 @@ static int mvpp2_gmac_pcs_config(struct phylink_pcs *pcs, unsigned int mode, static void mvpp2_gmac_pcs_an_restart(struct phylink_pcs *pcs) { - struct mvpp2_port *port = mvpp2_pcs_to_port(pcs); + struct mvpp2_port *port = mvpp2_pcs_gmac_to_port(pcs); u32 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); writel(val | MVPP2_GMAC_IN_BAND_RESTART_AN, @@ -6253,78 +6280,12 @@ static void mvpp2_gmac_pcs_an_restart(struct phylink_pcs *pcs) } static const struct phylink_pcs_ops mvpp2_phylink_gmac_pcs_ops = { + .pcs_validate = mvpp2_gmac_pcs_validate, .pcs_get_state = mvpp2_gmac_pcs_get_state, .pcs_config = mvpp2_gmac_pcs_config, .pcs_an_restart = mvpp2_gmac_pcs_an_restart, }; -static void mvpp2_phylink_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct mvpp2_port *port = mvpp2_phylink_to_port(config); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - /* When in 802.3z mode, we must have AN enabled: - * Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... - * When <PortType> = 1 (1000BASE-X) this field must be set to 1. - */ - if (phy_interface_mode_is_8023z(state->interface) && - !phylink_test(state->advertising, Autoneg)) - goto empty_set; - - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - - if (port->priv->global_tx_fc) { - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - } - - switch (state->interface) { - case PHY_INTERFACE_MODE_10GBASER: - case PHY_INTERFACE_MODE_XAUI: - if (mvpp2_port_supports_xlg(port)) { - phylink_set_10g_modes(mask); - phylink_set(mask, 10000baseKR_Full); - } - break; - - case PHY_INTERFACE_MODE_RGMII: - case PHY_INTERFACE_MODE_RGMII_ID: - case PHY_INTERFACE_MODE_RGMII_RXID: - case PHY_INTERFACE_MODE_RGMII_TXID: - case PHY_INTERFACE_MODE_SGMII: - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - break; - - case PHY_INTERFACE_MODE_1000BASEX: - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - break; - - case PHY_INTERFACE_MODE_2500BASEX: - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - break; - - default: - goto empty_set; - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); - return; - -empty_set: - linkmode_zero(supported); -} - static void mvpp2_xlg_config(struct mvpp2_port *port, unsigned int mode, const struct phylink_link_state *state) { @@ -6404,8 +6365,23 @@ static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode, writel(ctrl4, port->base + MVPP22_GMAC_CTRL_4_REG); } -static int mvpp2__mac_prepare(struct phylink_config *config, unsigned int mode, - phy_interface_t interface) +static struct phylink_pcs *mvpp2_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct mvpp2_port *port = mvpp2_phylink_to_port(config); + + /* Select the appropriate PCS operations depending on the + * configured interface mode. We will only switch to a mode + * that the validate() checks have already passed. + */ + if (mvpp2_is_xlg(interface)) + return &port->pcs_xlg; + else + return &port->pcs_gmac; +} + +static int mvpp2_mac_prepare(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) { struct mvpp2_port *port = mvpp2_phylink_to_port(config); @@ -6454,31 +6430,9 @@ static int mvpp2__mac_prepare(struct phylink_config *config, unsigned int mode, } } - /* Select the appropriate PCS operations depending on the - * configured interface mode. We will only switch to a mode - * that the validate() checks have already passed. - */ - if (mvpp2_is_xlg(interface)) - port->phylink_pcs.ops = &mvpp2_phylink_xlg_pcs_ops; - else - port->phylink_pcs.ops = &mvpp2_phylink_gmac_pcs_ops; - return 0; } -static int mvpp2_mac_prepare(struct phylink_config *config, unsigned int mode, - phy_interface_t interface) -{ - struct mvpp2_port *port = mvpp2_phylink_to_port(config); - int ret; - - ret = mvpp2__mac_prepare(config, mode, interface); - if (ret == 0) - phylink_set_pcs(port->phylink, &port->phylink_pcs); - - return ret; -} - static void mvpp2_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { @@ -6649,7 +6603,8 @@ static void mvpp2_mac_link_down(struct phylink_config *config, } static const struct phylink_mac_ops mvpp2_phylink_ops = { - .validate = mvpp2_phylink_validate, + .validate = phylink_generic_validate, + .mac_select_pcs = mvpp2_select_pcs, .mac_prepare = mvpp2_mac_prepare, .mac_config = mvpp2_mac_config, .mac_finish = mvpp2_mac_finish, @@ -6667,12 +6622,15 @@ static void mvpp2_acpi_start(struct mvpp2_port *port) struct phylink_link_state state = { .interface = port->phy_interface, }; - mvpp2__mac_prepare(&port->phylink_config, MLO_AN_INBAND, - port->phy_interface); + struct phylink_pcs *pcs; + + pcs = mvpp2_select_pcs(&port->phylink_config, port->phy_interface); + + mvpp2_mac_prepare(&port->phylink_config, MLO_AN_INBAND, + port->phy_interface); mvpp2_mac_config(&port->phylink_config, MLO_AN_INBAND, &state); - port->phylink_pcs.ops->pcs_config(&port->phylink_pcs, MLO_AN_INBAND, - port->phy_interface, - state.advertising, false); + pcs->ops->pcs_config(pcs, MLO_AN_INBAND, port->phy_interface, + state.advertising, false); mvpp2_mac_finish(&port->phylink_config, MLO_AN_INBAND, port->phy_interface); mvpp2_mac_link_up(&port->phylink_config, NULL, @@ -6901,7 +6859,7 @@ static int mvpp2_port_probe(struct platform_device *pdev, mvpp2_set_hw_csum(port, port->pool_long->id); dev->vlan_features |= features; - dev->gso_max_segs = MVPP2_MAX_TSO_SEGS; + netif_set_tso_max_segs(dev, MVPP2_MAX_TSO_SEGS); dev->priv_flags |= IFF_UNICAST_FLT; /* MTU range: 68 - 9704 */ @@ -6910,15 +6868,50 @@ static int mvpp2_port_probe(struct platform_device *pdev, dev->max_mtu = MVPP2_BM_JUMBO_PKT_SIZE; dev->dev.of_node = port_node; + port->pcs_gmac.ops = &mvpp2_phylink_gmac_pcs_ops; + port->pcs_xlg.ops = &mvpp2_phylink_xlg_pcs_ops; + if (!mvpp2_use_acpi_compat_mode(port_fwnode)) { port->phylink_config.dev = &dev->dev; port->phylink_config.type = PHYLINK_NETDEV; + port->phylink_config.mac_capabilities = + MAC_2500FD | MAC_1000FD | MAC_100 | MAC_10; + + if (port->priv->global_tx_fc) + port->phylink_config.mac_capabilities |= + MAC_SYM_PAUSE | MAC_ASYM_PAUSE; if (mvpp2_port_supports_xlg(port)) { - __set_bit(PHY_INTERFACE_MODE_10GBASER, - port->phylink_config.supported_interfaces); - __set_bit(PHY_INTERFACE_MODE_XAUI, - port->phylink_config.supported_interfaces); + /* If a COMPHY is present, we can support any of + * the serdes modes and switch between them. + */ + if (comphy) { + __set_bit(PHY_INTERFACE_MODE_5GBASER, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_10GBASER, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_XAUI, + port->phylink_config.supported_interfaces); + } else if (phy_mode == PHY_INTERFACE_MODE_5GBASER) { + __set_bit(PHY_INTERFACE_MODE_5GBASER, + port->phylink_config.supported_interfaces); + } else if (phy_mode == PHY_INTERFACE_MODE_10GBASER) { + __set_bit(PHY_INTERFACE_MODE_10GBASER, + port->phylink_config.supported_interfaces); + } else if (phy_mode == PHY_INTERFACE_MODE_XAUI) { + __set_bit(PHY_INTERFACE_MODE_XAUI, + port->phylink_config.supported_interfaces); + } + + if (comphy) + port->phylink_config.mac_capabilities |= + MAC_10000FD | MAC_5000FD; + else if (phy_mode == PHY_INTERFACE_MODE_5GBASER) + port->phylink_config.mac_capabilities |= + MAC_5000FD; + else + port->phylink_config.mac_capabilities |= + MAC_10000FD; } if (mvpp2_port_supports_rgmii(port)) @@ -7711,7 +7704,18 @@ static struct platform_driver mvpp2_driver = { }, }; -module_platform_driver(mvpp2_driver); +static int __init mvpp2_driver_init(void) +{ + return platform_driver_register(&mvpp2_driver); +} +module_init(mvpp2_driver_init); + +static void __exit mvpp2_driver_exit(void) +{ + platform_driver_unregister(&mvpp2_driver); + mvpp2_dbgfs_exit(); +} +module_exit(mvpp2_driver_exit); MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com"); MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>"); diff --git a/drivers/net/ethernet/marvell/octeon_ep/Kconfig b/drivers/net/ethernet/marvell/octeon_ep/Kconfig new file mode 100644 index 000000000000..0d7db815340e --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/Kconfig @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Marvell's Octeon PCI Endpoint NIC Driver Configuration +# + +config OCTEON_EP + tristate "Marvell Octeon PCI Endpoint NIC Driver" + depends on 64BIT + depends on PCI + depends on PTP_1588_CLOCK_OPTIONAL + help + This driver supports networking functionality of Marvell's + Octeon PCI Endpoint NIC. + + To know the list of devices supported by this driver, refer + documentation in + <file:Documentation/networking/device_drivers/ethernet/marvell/octeon_ep.rst>. + + To compile this drivers as a module, choose M here. Name of the + module is octeon_ep. diff --git a/drivers/net/ethernet/marvell/octeon_ep/Makefile b/drivers/net/ethernet/marvell/octeon_ep/Makefile new file mode 100644 index 000000000000..2026c8118158 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Network driver for Marvell's Octeon PCI Endpoint NIC +# + +obj-$(CONFIG_OCTEON_EP) += octeon_ep.o + +octeon_ep-y := octep_main.o octep_cn9k_pf.o octep_tx.o octep_rx.o \ + octep_ethtool.o octep_ctrl_mbox.o octep_ctrl_net.o diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c new file mode 100644 index 000000000000..6ad88d0fe43f --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cn9k_pf.c @@ -0,0 +1,737 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> + +#include "octep_config.h" +#include "octep_main.h" +#include "octep_regs_cn9k_pf.h" + +/* Names of Hardware non-queue generic interrupts */ +static char *cn93_non_ioq_msix_names[] = { + "epf_ire_rint", + "epf_ore_rint", + "epf_vfire_rint0", + "epf_vfire_rint1", + "epf_vfore_rint0", + "epf_vfore_rint1", + "epf_mbox_rint0", + "epf_mbox_rint1", + "epf_oei_rint", + "epf_dma_rint", + "epf_dma_vf_rint0", + "epf_dma_vf_rint1", + "epf_pp_vf_rint0", + "epf_pp_vf_rint1", + "epf_misc_rint", + "epf_rsvd", +}; + +/* Dump useful hardware CSRs for debug purpose */ +static void cn93_dump_regs(struct octep_device *oct, int qno) +{ + struct device *dev = &oct->pdev->dev; + + dev_info(dev, "IQ-%d register dump\n", qno); + dev_info(dev, "R[%d]_IN_INSTR_DBELL[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_INSTR_DBELL(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(qno))); + dev_info(dev, "R[%d]_IN_CONTROL[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_CONTROL(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_CONTROL(qno))); + dev_info(dev, "R[%d]_IN_ENABLE[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_ENABLE(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_ENABLE(qno))); + dev_info(dev, "R[%d]_IN_INSTR_BADDR[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_INSTR_BADDR(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_BADDR(qno))); + dev_info(dev, "R[%d]_IN_INSTR_RSIZE[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_INSTR_RSIZE(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_RSIZE(qno))); + dev_info(dev, "R[%d]_IN_CNTS[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_CNTS(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_CNTS(qno))); + dev_info(dev, "R[%d]_IN_INT_LEVELS[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_INT_LEVELS(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(qno))); + dev_info(dev, "R[%d]_IN_PKT_CNT[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_PKT_CNT(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_PKT_CNT(qno))); + dev_info(dev, "R[%d]_IN_BYTE_CNT[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_IN_BYTE_CNT(qno), + octep_read_csr64(oct, CN93_SDP_R_IN_BYTE_CNT(qno))); + + dev_info(dev, "OQ-%d register dump\n", qno); + dev_info(dev, "R[%d]_OUT_SLIST_DBELL[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_SLIST_DBELL(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_SLIST_DBELL(qno))); + dev_info(dev, "R[%d]_OUT_CONTROL[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_CONTROL(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(qno))); + dev_info(dev, "R[%d]_OUT_ENABLE[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_ENABLE(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_ENABLE(qno))); + dev_info(dev, "R[%d]_OUT_SLIST_BADDR[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_SLIST_BADDR(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_SLIST_BADDR(qno))); + dev_info(dev, "R[%d]_OUT_SLIST_RSIZE[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_SLIST_RSIZE(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_SLIST_RSIZE(qno))); + dev_info(dev, "R[%d]_OUT_CNTS[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_CNTS(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_CNTS(qno))); + dev_info(dev, "R[%d]_OUT_INT_LEVELS[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_INT_LEVELS(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(qno))); + dev_info(dev, "R[%d]_OUT_PKT_CNT[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_PKT_CNT(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_PKT_CNT(qno))); + dev_info(dev, "R[%d]_OUT_BYTE_CNT[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_OUT_BYTE_CNT(qno), + octep_read_csr64(oct, CN93_SDP_R_OUT_BYTE_CNT(qno))); + dev_info(dev, "R[%d]_ERR_TYPE[0x%llx]: 0x%016llx\n", + qno, CN93_SDP_R_ERR_TYPE(qno), + octep_read_csr64(oct, CN93_SDP_R_ERR_TYPE(qno))); +} + +/* Reset Hardware Tx queue */ +static int cn93_reset_iq(struct octep_device *oct, int q_no) +{ + struct octep_config *conf = oct->conf; + u64 val = 0ULL; + + dev_dbg(&oct->pdev->dev, "Reset PF IQ-%d\n", q_no); + + /* Get absolute queue number */ + q_no += conf->pf_ring_cfg.srn; + + /* Disable the Tx/Instruction Ring */ + octep_write_csr64(oct, CN93_SDP_R_IN_ENABLE(q_no), val); + + /* clear the Instruction Ring packet/byte counts and doorbell CSRs */ + octep_write_csr64(oct, CN93_SDP_R_IN_CNTS(q_no), val); + octep_write_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(q_no), val); + octep_write_csr64(oct, CN93_SDP_R_IN_PKT_CNT(q_no), val); + octep_write_csr64(oct, CN93_SDP_R_IN_BYTE_CNT(q_no), val); + octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_BADDR(q_no), val); + octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_RSIZE(q_no), val); + + val = 0xFFFFFFFF; + octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(q_no), val); + + return 0; +} + +/* Reset Hardware Rx queue */ +static void cn93_reset_oq(struct octep_device *oct, int q_no) +{ + u64 val = 0ULL; + + q_no += CFG_GET_PORTS_PF_SRN(oct->conf); + + /* Disable Output (Rx) Ring */ + octep_write_csr64(oct, CN93_SDP_R_OUT_ENABLE(q_no), val); + + /* Clear count CSRs */ + val = octep_read_csr(oct, CN93_SDP_R_OUT_CNTS(q_no)); + octep_write_csr(oct, CN93_SDP_R_OUT_CNTS(q_no), val); + + octep_write_csr64(oct, CN93_SDP_R_OUT_PKT_CNT(q_no), 0xFFFFFFFFFULL); + octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_DBELL(q_no), 0xFFFFFFFF); +} + +/* Reset all hardware Tx/Rx queues */ +static void octep_reset_io_queues_cn93_pf(struct octep_device *oct) +{ + struct pci_dev *pdev = oct->pdev; + int q; + + dev_dbg(&pdev->dev, "Reset OCTEP_CN93 PF IO Queues\n"); + + for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) { + cn93_reset_iq(oct, q); + cn93_reset_oq(oct, q); + } +} + +/* Initialize windowed addresses to access some hardware registers */ +static void octep_setup_pci_window_regs_cn93_pf(struct octep_device *oct) +{ + u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr; + + oct->pci_win_regs.pci_win_wr_addr = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_WR_ADDR64); + oct->pci_win_regs.pci_win_rd_addr = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_RD_ADDR64); + oct->pci_win_regs.pci_win_wr_data = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_WR_DATA64); + oct->pci_win_regs.pci_win_rd_data = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_RD_DATA64); +} + +/* Configure Hardware mapping: inform hardware which rings belong to PF. */ +static void octep_configure_ring_mapping_cn93_pf(struct octep_device *oct) +{ + struct octep_config *conf = oct->conf; + struct pci_dev *pdev = oct->pdev; + u64 pf_srn = CFG_GET_PORTS_PF_SRN(oct->conf); + int q; + + for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(conf); q++) { + u64 regval = 0; + + if (oct->pcie_port) + regval = 8 << CN93_SDP_FUNC_SEL_EPF_BIT_POS; + + octep_write_csr64(oct, CN93_SDP_EPVF_RING(pf_srn + q), regval); + + regval = octep_read_csr64(oct, CN93_SDP_EPVF_RING(pf_srn + q)); + dev_dbg(&pdev->dev, "Write SDP_EPVF_RING[0x%llx] = 0x%llx\n", + CN93_SDP_EPVF_RING(pf_srn + q), regval); + } +} + +/* Initialize configuration limits and initial active config 93xx PF. */ +static void octep_init_config_cn93_pf(struct octep_device *oct) +{ + struct octep_config *conf = oct->conf; + struct pci_dev *pdev = oct->pdev; + u64 val; + + /* Read ring configuration: + * PF ring count, number of VFs and rings per VF supported + */ + val = octep_read_csr64(oct, CN93_SDP_EPF_RINFO); + conf->sriov_cfg.max_rings_per_vf = CN93_SDP_EPF_RINFO_RPVF(val); + conf->sriov_cfg.active_rings_per_vf = conf->sriov_cfg.max_rings_per_vf; + conf->sriov_cfg.max_vfs = CN93_SDP_EPF_RINFO_NVFS(val); + conf->sriov_cfg.active_vfs = conf->sriov_cfg.max_vfs; + conf->sriov_cfg.vf_srn = CN93_SDP_EPF_RINFO_SRN(val); + + val = octep_read_csr64(oct, CN93_SDP_MAC_PF_RING_CTL(oct->pcie_port)); + conf->pf_ring_cfg.srn = CN93_SDP_MAC_PF_RING_CTL_SRN(val); + conf->pf_ring_cfg.max_io_rings = CN93_SDP_MAC_PF_RING_CTL_RPPF(val); + conf->pf_ring_cfg.active_io_rings = conf->pf_ring_cfg.max_io_rings; + dev_info(&pdev->dev, "pf_srn=%u rpvf=%u nvfs=%u rppf=%u\n", + conf->pf_ring_cfg.srn, conf->sriov_cfg.active_rings_per_vf, + conf->sriov_cfg.active_vfs, conf->pf_ring_cfg.active_io_rings); + + conf->iq.num_descs = OCTEP_IQ_MAX_DESCRIPTORS; + conf->iq.instr_type = OCTEP_64BYTE_INSTR; + conf->iq.pkind = 0; + conf->iq.db_min = OCTEP_DB_MIN; + conf->iq.intr_threshold = OCTEP_IQ_INTR_THRESHOLD; + + conf->oq.num_descs = OCTEP_OQ_MAX_DESCRIPTORS; + conf->oq.buf_size = OCTEP_OQ_BUF_SIZE; + conf->oq.refill_threshold = OCTEP_OQ_REFILL_THRESHOLD; + conf->oq.oq_intr_pkt = OCTEP_OQ_INTR_PKT_THRESHOLD; + conf->oq.oq_intr_time = OCTEP_OQ_INTR_TIME_THRESHOLD; + + conf->msix_cfg.non_ioq_msix = CN93_NUM_NON_IOQ_INTR; + conf->msix_cfg.ioq_msix = conf->pf_ring_cfg.active_io_rings; + conf->msix_cfg.non_ioq_msix_names = cn93_non_ioq_msix_names; + + conf->ctrl_mbox_cfg.barmem_addr = (void __iomem *)oct->mmio[2].hw_addr + (0x400000ull * 7); +} + +/* Setup registers for a hardware Tx Queue */ +static void octep_setup_iq_regs_cn93_pf(struct octep_device *oct, int iq_no) +{ + struct octep_iq *iq = oct->iq[iq_no]; + u32 reset_instr_cnt; + u64 reg_val; + + iq_no += CFG_GET_PORTS_PF_SRN(oct->conf); + reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_CONTROL(iq_no)); + + /* wait for IDLE to set to 1 */ + if (!(reg_val & CN93_R_IN_CTL_IDLE)) { + do { + reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_CONTROL(iq_no)); + } while (!(reg_val & CN93_R_IN_CTL_IDLE)); + } + + reg_val |= CN93_R_IN_CTL_RDSIZE; + reg_val |= CN93_R_IN_CTL_IS_64B; + reg_val |= CN93_R_IN_CTL_ESR; + octep_write_csr64(oct, CN93_SDP_R_IN_CONTROL(iq_no), reg_val); + + /* Write the start of the input queue's ring and its size */ + octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_BADDR(iq_no), + iq->desc_ring_dma); + octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_RSIZE(iq_no), + iq->max_count); + + /* Remember the doorbell & instruction count register addr + * for this queue + */ + iq->doorbell_reg = oct->mmio[0].hw_addr + + CN93_SDP_R_IN_INSTR_DBELL(iq_no); + iq->inst_cnt_reg = oct->mmio[0].hw_addr + + CN93_SDP_R_IN_CNTS(iq_no); + iq->intr_lvl_reg = oct->mmio[0].hw_addr + + CN93_SDP_R_IN_INT_LEVELS(iq_no); + + /* Store the current instruction counter (used in flush_iq calculation) */ + reset_instr_cnt = readl(iq->inst_cnt_reg); + writel(reset_instr_cnt, iq->inst_cnt_reg); + + /* INTR_THRESHOLD is set to max(FFFFFFFF) to disable the INTR */ + reg_val = CFG_GET_IQ_INTR_THRESHOLD(oct->conf) & 0xffffffff; + octep_write_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(iq_no), reg_val); +} + +/* Setup registers for a hardware Rx Queue */ +static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no) +{ + u64 reg_val; + u64 oq_ctl = 0ULL; + u32 time_threshold = 0; + struct octep_oq *oq = oct->oq[oq_no]; + + oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); + reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no)); + + /* wait for IDLE to set to 1 */ + if (!(reg_val & CN93_R_OUT_CTL_IDLE)) { + do { + reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no)); + } while (!(reg_val & CN93_R_OUT_CTL_IDLE)); + } + + reg_val &= ~(CN93_R_OUT_CTL_IMODE); + reg_val &= ~(CN93_R_OUT_CTL_ROR_P); + reg_val &= ~(CN93_R_OUT_CTL_NSR_P); + reg_val &= ~(CN93_R_OUT_CTL_ROR_I); + reg_val &= ~(CN93_R_OUT_CTL_NSR_I); + reg_val &= ~(CN93_R_OUT_CTL_ES_I); + reg_val &= ~(CN93_R_OUT_CTL_ROR_D); + reg_val &= ~(CN93_R_OUT_CTL_NSR_D); + reg_val &= ~(CN93_R_OUT_CTL_ES_D); + reg_val |= (CN93_R_OUT_CTL_ES_P); + + octep_write_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no), reg_val); + octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_BADDR(oq_no), + oq->desc_ring_dma); + octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_RSIZE(oq_no), + oq->max_count); + + oq_ctl = octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no)); + oq_ctl &= ~0x7fffffULL; //clear the ISIZE and BSIZE (22-0) + oq_ctl |= (oq->buffer_size & 0xffff); //populate the BSIZE (15-0) + octep_write_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no), oq_ctl); + + /* Get the mapped address of the pkt_sent and pkts_credit regs */ + oq->pkts_sent_reg = oct->mmio[0].hw_addr + CN93_SDP_R_OUT_CNTS(oq_no); + oq->pkts_credit_reg = oct->mmio[0].hw_addr + + CN93_SDP_R_OUT_SLIST_DBELL(oq_no); + + time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf); + reg_val = ((u64)time_threshold << 32) | + CFG_GET_OQ_INTR_PKT(oct->conf); + octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); +} + +/* Setup registers for a PF mailbox */ +static void octep_setup_mbox_regs_cn93_pf(struct octep_device *oct, int q_no) +{ + struct octep_mbox *mbox = oct->mbox[q_no]; + + mbox->q_no = q_no; + + /* PF mbox interrupt reg */ + mbox->mbox_int_reg = oct->mmio[0].hw_addr + CN93_SDP_EPF_MBOX_RINT(0); + + /* PF to VF DATA reg. PF writes into this reg */ + mbox->mbox_write_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_PF_VF_DATA(q_no); + + /* VF to PF DATA reg. PF reads from this reg */ + mbox->mbox_read_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_VF_PF_DATA(q_no); +} + +/* Mailbox Interrupt handler */ +static void cn93_handle_pf_mbox_intr(struct octep_device *oct) +{ + u64 mbox_int_val = 0ULL, val = 0ULL, qno = 0ULL; + + mbox_int_val = readq(oct->mbox[0]->mbox_int_reg); + for (qno = 0; qno < OCTEP_MAX_VF; qno++) { + val = readq(oct->mbox[qno]->mbox_read_reg); + dev_dbg(&oct->pdev->dev, + "PF MBOX READ: val:%llx from VF:%llx\n", val, qno); + } + + writeq(mbox_int_val, oct->mbox[0]->mbox_int_reg); +} + +/* Interrupts handler for all non-queue generic interrupts. */ +static irqreturn_t octep_non_ioq_intr_handler_cn93_pf(void *dev) +{ + struct octep_device *oct = (struct octep_device *)dev; + struct pci_dev *pdev = oct->pdev; + u64 reg_val = 0; + int i = 0; + + /* Check for IRERR INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_IRERR_RINT); + if (reg_val) { + dev_info(&pdev->dev, + "received IRERR_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT, reg_val); + + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { + reg_val = octep_read_csr64(oct, + CN93_SDP_R_ERR_TYPE(i)); + if (reg_val) { + dev_info(&pdev->dev, + "Received err type on IQ-%d: 0x%llx\n", + i, reg_val); + octep_write_csr64(oct, CN93_SDP_R_ERR_TYPE(i), + reg_val); + } + } + goto irq_handled; + } + + /* Check for ORERR INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_ORERR_RINT); + if (reg_val) { + dev_info(&pdev->dev, + "Received ORERR_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT, reg_val); + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { + reg_val = octep_read_csr64(oct, CN93_SDP_R_ERR_TYPE(i)); + if (reg_val) { + dev_info(&pdev->dev, + "Received err type on OQ-%d: 0x%llx\n", + i, reg_val); + octep_write_csr64(oct, CN93_SDP_R_ERR_TYPE(i), + reg_val); + } + } + + goto irq_handled; + } + + /* Check for VFIRE INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_VFIRE_RINT(0)); + if (reg_val) { + dev_info(&pdev->dev, + "Received VFIRE_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_VFIRE_RINT(0), reg_val); + goto irq_handled; + } + + /* Check for VFORE INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_VFORE_RINT(0)); + if (reg_val) { + dev_info(&pdev->dev, + "Received VFORE_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_VFORE_RINT(0), reg_val); + goto irq_handled; + } + + /* Check for MBOX INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_MBOX_RINT(0)); + if (reg_val) { + dev_info(&pdev->dev, + "Received MBOX_RINT intr: 0x%llx\n", reg_val); + cn93_handle_pf_mbox_intr(oct); + goto irq_handled; + } + + /* Check for OEI INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_OEI_RINT); + if (reg_val) { + dev_info(&pdev->dev, + "Received OEI_EINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT, reg_val); + queue_work(octep_wq, &oct->ctrl_mbox_task); + goto irq_handled; + } + + /* Check for DMA INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_DMA_RINT); + if (reg_val) { + octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT, reg_val); + goto irq_handled; + } + + /* Check for DMA VF INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_DMA_VF_RINT(0)); + if (reg_val) { + dev_info(&pdev->dev, + "Received DMA_VF_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_DMA_VF_RINT(0), reg_val); + goto irq_handled; + } + + /* Check for PPVF INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_PP_VF_RINT(0)); + if (reg_val) { + dev_info(&pdev->dev, + "Received PP_VF_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_PP_VF_RINT(0), reg_val); + goto irq_handled; + } + + /* Check for MISC INTR */ + reg_val = octep_read_csr64(oct, CN93_SDP_EPF_MISC_RINT); + if (reg_val) { + dev_info(&pdev->dev, + "Received MISC_RINT intr: 0x%llx\n", reg_val); + octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT, reg_val); + goto irq_handled; + } + + dev_info(&pdev->dev, "Reserved interrupts raised; Ignore\n"); +irq_handled: + return IRQ_HANDLED; +} + +/* Tx/Rx queue interrupt handler */ +static irqreturn_t octep_ioq_intr_handler_cn93_pf(void *data) +{ + struct octep_ioq_vector *vector = (struct octep_ioq_vector *)data; + struct octep_oq *oq = vector->oq; + + napi_schedule_irqoff(oq->napi); + return IRQ_HANDLED; +} + +/* soft reset of 93xx */ +static int octep_soft_reset_cn93_pf(struct octep_device *oct) +{ + dev_info(&oct->pdev->dev, "CN93XX: Doing soft reset\n"); + + octep_write_csr64(oct, CN93_SDP_WIN_WR_MASK_REG, 0xFF); + + /* Set core domain reset bit */ + OCTEP_PCI_WIN_WRITE(oct, CN93_RST_CORE_DOMAIN_W1S, 1); + /* Wait for 100ms as Octeon resets. */ + mdelay(100); + /* clear core domain reset bit */ + OCTEP_PCI_WIN_WRITE(oct, CN93_RST_CORE_DOMAIN_W1C, 1); + + return 0; +} + +/* Re-initialize Octeon hardware registers */ +static void octep_reinit_regs_cn93_pf(struct octep_device *oct) +{ + u32 i; + + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) + oct->hw_ops.setup_iq_regs(oct, i); + + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) + oct->hw_ops.setup_oq_regs(oct, i); + + oct->hw_ops.enable_interrupts(oct); + oct->hw_ops.enable_io_queues(oct); + + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) + writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg); +} + +/* Enable all interrupts */ +static void octep_enable_interrupts_cn93_pf(struct octep_device *oct) +{ + u64 intr_mask = 0ULL; + int srn, num_rings, i; + + srn = CFG_GET_PORTS_PF_SRN(oct->conf); + num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); + + for (i = 0; i < num_rings; i++) + intr_mask |= (0x1ULL << (srn + i)); + + octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1S, intr_mask); + octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1S, intr_mask); + octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT_ENA_W1S, -1ULL); + octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT_ENA_W1S, intr_mask); + octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT_ENA_W1S, intr_mask); +} + +/* Disable all interrupts */ +static void octep_disable_interrupts_cn93_pf(struct octep_device *oct) +{ + u64 intr_mask = 0ULL; + int srn, num_rings, i; + + srn = CFG_GET_PORTS_PF_SRN(oct->conf); + num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); + + for (i = 0; i < num_rings; i++) + intr_mask |= (0x1ULL << (srn + i)); + + octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask); + octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask); + octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT_ENA_W1C, -1ULL); + octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT_ENA_W1C, intr_mask); + octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT_ENA_W1C, intr_mask); +} + +/* Get new Octeon Read Index: index of descriptor that Octeon reads next. */ +static u32 octep_update_iq_read_index_cn93_pf(struct octep_iq *iq) +{ + u32 pkt_in_done = readl(iq->inst_cnt_reg); + u32 last_done, new_idx; + + last_done = pkt_in_done - iq->pkt_in_done; + iq->pkt_in_done = pkt_in_done; + + new_idx = (iq->octep_read_index + last_done) % iq->max_count; + + return new_idx; +} + +/* Enable a hardware Tx Queue */ +static void octep_enable_iq_cn93_pf(struct octep_device *oct, int iq_no) +{ + u64 loop = HZ; + u64 reg_val; + + iq_no += CFG_GET_PORTS_PF_SRN(oct->conf); + + octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(iq_no), 0xFFFFFFFF); + + while (octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(iq_no)) && + loop--) { + schedule_timeout_interruptible(1); + } + + reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(iq_no)); + reg_val |= (0x1ULL << 62); + octep_write_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(iq_no), reg_val); + + reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no)); + reg_val |= 0x1ULL; + octep_write_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no), reg_val); +} + +/* Enable a hardware Rx Queue */ +static void octep_enable_oq_cn93_pf(struct octep_device *oct, int oq_no) +{ + u64 reg_val = 0ULL; + + oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); + + reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no)); + reg_val |= (0x1ULL << 62); + octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val); + + octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_DBELL(oq_no), 0xFFFFFFFF); + + reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no)); + reg_val |= 0x1ULL; + octep_write_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no), reg_val); +} + +/* Enable all hardware Tx/Rx Queues assined to PF */ +static void octep_enable_io_queues_cn93_pf(struct octep_device *oct) +{ + u8 q; + + for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) { + octep_enable_iq_cn93_pf(oct, q); + octep_enable_oq_cn93_pf(oct, q); + } +} + +/* Disable a hardware Tx Queue assined to PF */ +static void octep_disable_iq_cn93_pf(struct octep_device *oct, int iq_no) +{ + u64 reg_val = 0ULL; + + iq_no += CFG_GET_PORTS_PF_SRN(oct->conf); + + reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no)); + reg_val &= ~0x1ULL; + octep_write_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no), reg_val); +} + +/* Disable a hardware Rx Queue assined to PF */ +static void octep_disable_oq_cn93_pf(struct octep_device *oct, int oq_no) +{ + u64 reg_val = 0ULL; + + oq_no += CFG_GET_PORTS_PF_SRN(oct->conf); + reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no)); + reg_val &= ~0x1ULL; + octep_write_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no), reg_val); +} + +/* Disable all hardware Tx/Rx Queues assined to PF */ +static void octep_disable_io_queues_cn93_pf(struct octep_device *oct) +{ + int q = 0; + + for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) { + octep_disable_iq_cn93_pf(oct, q); + octep_disable_oq_cn93_pf(oct, q); + } +} + +/* Dump hardware registers (including Tx/Rx queues) for debugging. */ +static void octep_dump_registers_cn93_pf(struct octep_device *oct) +{ + u8 srn, num_rings, q; + + srn = CFG_GET_PORTS_PF_SRN(oct->conf); + num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); + + for (q = srn; q < srn + num_rings; q++) + cn93_dump_regs(oct, q); +} + +/** + * octep_device_setup_cn93_pf() - Setup Octeon device. + * + * @oct: Octeon device private data structure. + * + * - initialize hardware operations. + * - get target side pcie port number for the device. + * - setup window access to hardware registers. + * - set initial configuration and max limits. + * - setup hardware mapping of rings to the PF device. + */ +void octep_device_setup_cn93_pf(struct octep_device *oct) +{ + oct->hw_ops.setup_iq_regs = octep_setup_iq_regs_cn93_pf; + oct->hw_ops.setup_oq_regs = octep_setup_oq_regs_cn93_pf; + oct->hw_ops.setup_mbox_regs = octep_setup_mbox_regs_cn93_pf; + + oct->hw_ops.non_ioq_intr_handler = octep_non_ioq_intr_handler_cn93_pf; + oct->hw_ops.ioq_intr_handler = octep_ioq_intr_handler_cn93_pf; + oct->hw_ops.soft_reset = octep_soft_reset_cn93_pf; + oct->hw_ops.reinit_regs = octep_reinit_regs_cn93_pf; + + oct->hw_ops.enable_interrupts = octep_enable_interrupts_cn93_pf; + oct->hw_ops.disable_interrupts = octep_disable_interrupts_cn93_pf; + + oct->hw_ops.update_iq_read_idx = octep_update_iq_read_index_cn93_pf; + + oct->hw_ops.enable_iq = octep_enable_iq_cn93_pf; + oct->hw_ops.enable_oq = octep_enable_oq_cn93_pf; + oct->hw_ops.enable_io_queues = octep_enable_io_queues_cn93_pf; + + oct->hw_ops.disable_iq = octep_disable_iq_cn93_pf; + oct->hw_ops.disable_oq = octep_disable_oq_cn93_pf; + oct->hw_ops.disable_io_queues = octep_disable_io_queues_cn93_pf; + oct->hw_ops.reset_io_queues = octep_reset_io_queues_cn93_pf; + + oct->hw_ops.dump_registers = octep_dump_registers_cn93_pf; + + octep_setup_pci_window_regs_cn93_pf(oct); + + oct->pcie_port = octep_read_csr64(oct, CN93_SDP_MAC_NUMBER) & 0xff; + dev_info(&oct->pdev->dev, + "Octeon device using PCIE Port %d\n", oct->pcie_port); + + octep_init_config_cn93_pf(oct); + octep_configure_ring_mapping_cn93_pf(oct); +} diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_config.h b/drivers/net/ethernet/marvell/octeon_ep/octep_config.h new file mode 100644 index 000000000000..f208f3f9a447 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_config.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#ifndef _OCTEP_CONFIG_H_ +#define _OCTEP_CONFIG_H_ + +/* Tx instruction types by length */ +#define OCTEP_32BYTE_INSTR 32 +#define OCTEP_64BYTE_INSTR 64 + +/* Tx Queue: maximum descriptors per ring */ +#define OCTEP_IQ_MAX_DESCRIPTORS 1024 +/* Minimum input (Tx) requests to be enqueued to ring doorbell */ +#define OCTEP_DB_MIN 1 +/* Packet threshold for Tx queue interrupt */ +#define OCTEP_IQ_INTR_THRESHOLD 0x0 + +/* Rx Queue: maximum descriptors per ring */ +#define OCTEP_OQ_MAX_DESCRIPTORS 1024 + +/* Rx buffer size: Use page size buffers. + * Build skb from allocated page buffer once the packet is received. + * When a gathered packet is received, make head page as skb head and + * page buffers in consecutive Rx descriptors as fragments. + */ +#define OCTEP_OQ_BUF_SIZE (SKB_WITH_OVERHEAD(PAGE_SIZE)) +#define OCTEP_OQ_PKTS_PER_INTR 128 +#define OCTEP_OQ_REFILL_THRESHOLD (OCTEP_OQ_MAX_DESCRIPTORS / 4) + +#define OCTEP_OQ_INTR_PKT_THRESHOLD 1 +#define OCTEP_OQ_INTR_TIME_THRESHOLD 10 + +#define OCTEP_MSIX_NAME_SIZE (IFNAMSIZ + 32) + +/* Tx Queue wake threshold + * wakeup a stopped Tx queue if minimum 2 descriptors are available. + * Even a skb with fragments consume only one Tx queue descriptor entry. + */ +#define OCTEP_WAKE_QUEUE_THRESHOLD 2 + +/* Minimum MTU supported by Octeon network interface */ +#define OCTEP_MIN_MTU ETH_MIN_MTU +/* Maximum MTU supported by Octeon interface*/ +#define OCTEP_MAX_MTU (10000 - (ETH_HLEN + ETH_FCS_LEN)) +/* Default MTU */ +#define OCTEP_DEFAULT_MTU 1500 + +/* Macros to get octeon config params */ +#define CFG_GET_IQ_CFG(cfg) ((cfg)->iq) +#define CFG_GET_IQ_NUM_DESC(cfg) ((cfg)->iq.num_descs) +#define CFG_GET_IQ_INSTR_TYPE(cfg) ((cfg)->iq.instr_type) +#define CFG_GET_IQ_PKIND(cfg) ((cfg)->iq.pkind) +#define CFG_GET_IQ_INSTR_SIZE(cfg) (64) +#define CFG_GET_IQ_DB_MIN(cfg) ((cfg)->iq.db_min) +#define CFG_GET_IQ_INTR_THRESHOLD(cfg) ((cfg)->iq.intr_threshold) + +#define CFG_GET_OQ_NUM_DESC(cfg) ((cfg)->oq.num_descs) +#define CFG_GET_OQ_BUF_SIZE(cfg) ((cfg)->oq.buf_size) +#define CFG_GET_OQ_REFILL_THRESHOLD(cfg) ((cfg)->oq.refill_threshold) +#define CFG_GET_OQ_INTR_PKT(cfg) ((cfg)->oq.oq_intr_pkt) +#define CFG_GET_OQ_INTR_TIME(cfg) ((cfg)->oq.oq_intr_time) + +#define CFG_GET_PORTS_MAX_IO_RINGS(cfg) ((cfg)->pf_ring_cfg.max_io_rings) +#define CFG_GET_PORTS_ACTIVE_IO_RINGS(cfg) ((cfg)->pf_ring_cfg.active_io_rings) +#define CFG_GET_PORTS_PF_SRN(cfg) ((cfg)->pf_ring_cfg.srn) + +#define CFG_GET_DPI_PKIND(cfg) ((cfg)->core_cfg.dpi_pkind) +#define CFG_GET_CORE_TICS_PER_US(cfg) ((cfg)->core_cfg.core_tics_per_us) +#define CFG_GET_COPROC_TICS_PER_US(cfg) ((cfg)->core_cfg.coproc_tics_per_us) + +#define CFG_GET_MAX_VFS(cfg) ((cfg)->sriov_cfg.max_vfs) +#define CFG_GET_ACTIVE_VFS(cfg) ((cfg)->sriov_cfg.active_vfs) +#define CFG_GET_MAX_RPVF(cfg) ((cfg)->sriov_cfg.max_rings_per_vf) +#define CFG_GET_ACTIVE_RPVF(cfg) ((cfg)->sriov_cfg.active_rings_per_vf) +#define CFG_GET_VF_SRN(cfg) ((cfg)->sriov_cfg.vf_srn) + +#define CFG_GET_IOQ_MSIX(cfg) ((cfg)->msix_cfg.ioq_msix) +#define CFG_GET_NON_IOQ_MSIX(cfg) ((cfg)->msix_cfg.non_ioq_msix) +#define CFG_GET_NON_IOQ_MSIX_NAMES(cfg) ((cfg)->msix_cfg.non_ioq_msix_names) + +#define CFG_GET_CTRL_MBOX_MEM_ADDR(cfg) ((cfg)->ctrl_mbox_cfg.barmem_addr) + +/* Hardware Tx Queue configuration. */ +struct octep_iq_config { + /* Size of the Input queue (number of commands) */ + u16 num_descs; + + /* Command size - 32 or 64 bytes */ + u16 instr_type; + + /* pkind for packets sent to Octeon */ + u16 pkind; + + /* Minimum number of commands pending to be posted to Octeon before driver + * hits the Input queue doorbell. + */ + u16 db_min; + + /* Trigger the IQ interrupt when processed cmd count reaches + * this level. + */ + u32 intr_threshold; +}; + +/* Hardware Rx Queue configuration. */ +struct octep_oq_config { + /* Size of Output queue (number of descriptors) */ + u16 num_descs; + + /* Size of buffer in this Output queue. */ + u16 buf_size; + + /* The number of buffers that were consumed during packet processing + * by the driver on this Output queue before the driver attempts to + * replenish the descriptor ring with new buffers. + */ + u16 refill_threshold; + + /* Interrupt Coalescing (Packet Count). Octeon will interrupt the host + * only if it sent as many packets as specified by this field. + * The driver usually does not use packet count interrupt coalescing. + */ + u32 oq_intr_pkt; + + /* Interrupt Coalescing (Time Interval). Octeon will interrupt the host + * if at least one packet was sent in the time interval specified by + * this field. The driver uses time interval interrupt coalescing by + * default. The time is specified in microseconds. + */ + u32 oq_intr_time; +}; + +/* Tx/Rx configuration */ +struct octep_pf_ring_config { + /* Max number of IOQs */ + u16 max_io_rings; + + /* Number of active IOQs */ + u16 active_io_rings; + + /* Starting IOQ number: this changes based on which PEM is used */ + u16 srn; +}; + +/* Octeon Hardware SRIOV config */ +struct octep_sriov_config { + /* Max number of VF devices supported */ + u16 max_vfs; + + /* Number of VF devices enabled */ + u16 active_vfs; + + /* Max number of rings assigned to VF */ + u8 max_rings_per_vf; + + /* Number of rings enabled per VF */ + u8 active_rings_per_vf; + + /* starting ring number of VF's: ring-0 of VF-0 of the PF */ + u16 vf_srn; +}; + +/* Octeon MSI-x config. */ +struct octep_msix_config { + /* Number of IOQ interrupts */ + u16 ioq_msix; + + /* Number of Non IOQ interrupts */ + u16 non_ioq_msix; + + /* Names of Non IOQ interrupts */ + char **non_ioq_msix_names; +}; + +struct octep_ctrl_mbox_config { + /* Barmem address for control mbox */ + void __iomem *barmem_addr; +}; + +/* Data Structure to hold configuration limits and active config */ +struct octep_config { + /* Input Queue attributes. */ + struct octep_iq_config iq; + + /* Output Queue attributes. */ + struct octep_oq_config oq; + + /* NIC Port Configuration */ + struct octep_pf_ring_config pf_ring_cfg; + + /* SRIOV configuration of the PF */ + struct octep_sriov_config sriov_cfg; + + /* MSI-X interrupt config */ + struct octep_msix_config msix_cfg; + + /* ctrl mbox config */ + struct octep_ctrl_mbox_config ctrl_mbox_cfg; +}; +#endif /* _OCTEP_CONFIG_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c new file mode 100644 index 000000000000..39322e4dd100 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/mutex.h> +#include <linux/jiffies.h> +#include <linux/sched.h> +#include <linux/sched/signal.h> +#include <linux/io.h> +#include <linux/pci.h> +#include <linux/etherdevice.h> + +#include "octep_ctrl_mbox.h" +#include "octep_config.h" +#include "octep_main.h" + +/* Timeout in msecs for message response */ +#define OCTEP_CTRL_MBOX_MSG_TIMEOUT_MS 100 +/* Time in msecs to wait for message response */ +#define OCTEP_CTRL_MBOX_MSG_WAIT_MS 10 + +#define OCTEP_CTRL_MBOX_INFO_MAGIC_NUM_OFFSET(m) (m) +#define OCTEP_CTRL_MBOX_INFO_BARMEM_SZ_OFFSET(m) ((m) + 8) +#define OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(m) ((m) + 24) +#define OCTEP_CTRL_MBOX_INFO_FW_STATUS_OFFSET(m) ((m) + 144) + +#define OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m) ((m) + OCTEP_CTRL_MBOX_INFO_SZ) +#define OCTEP_CTRL_MBOX_H2FQ_PROD_OFFSET(m) (OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) +#define OCTEP_CTRL_MBOX_H2FQ_CONS_OFFSET(m) ((OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) + 4) +#define OCTEP_CTRL_MBOX_H2FQ_ELEM_SZ_OFFSET(m) ((OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) + 8) +#define OCTEP_CTRL_MBOX_H2FQ_ELEM_CNT_OFFSET(m) ((OCTEP_CTRL_MBOX_H2FQ_INFO_OFFSET(m)) + 12) + +#define OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m) ((m) + \ + OCTEP_CTRL_MBOX_INFO_SZ + \ + OCTEP_CTRL_MBOX_H2FQ_INFO_SZ) +#define OCTEP_CTRL_MBOX_F2HQ_PROD_OFFSET(m) (OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) +#define OCTEP_CTRL_MBOX_F2HQ_CONS_OFFSET(m) ((OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) + 4) +#define OCTEP_CTRL_MBOX_F2HQ_ELEM_SZ_OFFSET(m) ((OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) + 8) +#define OCTEP_CTRL_MBOX_F2HQ_ELEM_CNT_OFFSET(m) ((OCTEP_CTRL_MBOX_F2HQ_INFO_OFFSET(m)) + 12) + +#define OCTEP_CTRL_MBOX_Q_OFFSET(m, i) ((m) + \ + (sizeof(struct octep_ctrl_mbox_msg) * (i))) + +static u32 octep_ctrl_mbox_circq_inc(u32 index, u32 mask) +{ + return (index + 1) & mask; +} + +static u32 octep_ctrl_mbox_circq_space(u32 pi, u32 ci, u32 mask) +{ + return mask - ((pi - ci) & mask); +} + +static u32 octep_ctrl_mbox_circq_depth(u32 pi, u32 ci, u32 mask) +{ + return ((pi - ci) & mask); +} + +int octep_ctrl_mbox_init(struct octep_ctrl_mbox *mbox) +{ + u64 magic_num, status; + + if (!mbox) + return -EINVAL; + + if (!mbox->barmem) { + pr_info("octep_ctrl_mbox : Invalid barmem %p\n", mbox->barmem); + return -EINVAL; + } + + magic_num = readq(OCTEP_CTRL_MBOX_INFO_MAGIC_NUM_OFFSET(mbox->barmem)); + if (magic_num != OCTEP_CTRL_MBOX_MAGIC_NUMBER) { + pr_info("octep_ctrl_mbox : Invalid magic number %llx\n", magic_num); + return -EINVAL; + } + + status = readq(OCTEP_CTRL_MBOX_INFO_FW_STATUS_OFFSET(mbox->barmem)); + if (status != OCTEP_CTRL_MBOX_STATUS_READY) { + pr_info("octep_ctrl_mbox : Firmware is not ready.\n"); + return -EINVAL; + } + + mbox->barmem_sz = readl(OCTEP_CTRL_MBOX_INFO_BARMEM_SZ_OFFSET(mbox->barmem)); + + writeq(OCTEP_CTRL_MBOX_STATUS_INIT, OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem)); + + mbox->h2fq.elem_cnt = readl(OCTEP_CTRL_MBOX_H2FQ_ELEM_CNT_OFFSET(mbox->barmem)); + mbox->h2fq.elem_sz = readl(OCTEP_CTRL_MBOX_H2FQ_ELEM_SZ_OFFSET(mbox->barmem)); + mbox->h2fq.mask = (mbox->h2fq.elem_cnt - 1); + mutex_init(&mbox->h2fq_lock); + + mbox->f2hq.elem_cnt = readl(OCTEP_CTRL_MBOX_F2HQ_ELEM_CNT_OFFSET(mbox->barmem)); + mbox->f2hq.elem_sz = readl(OCTEP_CTRL_MBOX_F2HQ_ELEM_SZ_OFFSET(mbox->barmem)); + mbox->f2hq.mask = (mbox->f2hq.elem_cnt - 1); + mutex_init(&mbox->f2hq_lock); + + mbox->h2fq.hw_prod = OCTEP_CTRL_MBOX_H2FQ_PROD_OFFSET(mbox->barmem); + mbox->h2fq.hw_cons = OCTEP_CTRL_MBOX_H2FQ_CONS_OFFSET(mbox->barmem); + mbox->h2fq.hw_q = mbox->barmem + + OCTEP_CTRL_MBOX_INFO_SZ + + OCTEP_CTRL_MBOX_H2FQ_INFO_SZ + + OCTEP_CTRL_MBOX_F2HQ_INFO_SZ; + + mbox->f2hq.hw_prod = OCTEP_CTRL_MBOX_F2HQ_PROD_OFFSET(mbox->barmem); + mbox->f2hq.hw_cons = OCTEP_CTRL_MBOX_F2HQ_CONS_OFFSET(mbox->barmem); + mbox->f2hq.hw_q = mbox->h2fq.hw_q + + ((mbox->h2fq.elem_sz + sizeof(union octep_ctrl_mbox_msg_hdr)) * + mbox->h2fq.elem_cnt); + + /* ensure ready state is seen after everything is initialized */ + wmb(); + writeq(OCTEP_CTRL_MBOX_STATUS_READY, OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem)); + + pr_info("Octep ctrl mbox : Init successful.\n"); + + return 0; +} + +int octep_ctrl_mbox_send(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_msg *msg) +{ + unsigned long timeout = msecs_to_jiffies(OCTEP_CTRL_MBOX_MSG_TIMEOUT_MS); + unsigned long period = msecs_to_jiffies(OCTEP_CTRL_MBOX_MSG_WAIT_MS); + struct octep_ctrl_mbox_q *q; + unsigned long expire; + u64 *mbuf, *word0; + u8 __iomem *qidx; + u16 pi, ci; + int i; + + if (!mbox || !msg) + return -EINVAL; + + q = &mbox->h2fq; + pi = readl(q->hw_prod); + ci = readl(q->hw_cons); + + if (!octep_ctrl_mbox_circq_space(pi, ci, q->mask)) + return -ENOMEM; + + qidx = OCTEP_CTRL_MBOX_Q_OFFSET(q->hw_q, pi); + mbuf = (u64 *)msg->msg; + word0 = &msg->hdr.word0; + + mutex_lock(&mbox->h2fq_lock); + for (i = 1; i <= msg->hdr.sizew; i++) + writeq(*mbuf++, (qidx + (i * 8))); + + writeq(*word0, qidx); + + pi = octep_ctrl_mbox_circq_inc(pi, q->mask); + writel(pi, q->hw_prod); + mutex_unlock(&mbox->h2fq_lock); + + /* don't check for notification response */ + if (msg->hdr.flags & OCTEP_CTRL_MBOX_MSG_HDR_FLAG_NOTIFY) + return 0; + + expire = jiffies + timeout; + while (true) { + *word0 = readq(qidx); + if (msg->hdr.flags == OCTEP_CTRL_MBOX_MSG_HDR_FLAG_RESP) + break; + schedule_timeout_interruptible(period); + if (signal_pending(current) || time_after(jiffies, expire)) { + pr_info("octep_ctrl_mbox: Timed out\n"); + return -EBUSY; + } + } + mbuf = (u64 *)msg->msg; + for (i = 1; i <= msg->hdr.sizew; i++) + *mbuf++ = readq(qidx + (i * 8)); + + return 0; +} + +int octep_ctrl_mbox_recv(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_msg *msg) +{ + struct octep_ctrl_mbox_q *q; + u32 count, pi, ci; + u8 __iomem *qidx; + u64 *mbuf; + int i; + + if (!mbox || !msg) + return -EINVAL; + + q = &mbox->f2hq; + pi = readl(q->hw_prod); + ci = readl(q->hw_cons); + count = octep_ctrl_mbox_circq_depth(pi, ci, q->mask); + if (!count) + return -EAGAIN; + + qidx = OCTEP_CTRL_MBOX_Q_OFFSET(q->hw_q, ci); + mbuf = (u64 *)msg->msg; + + mutex_lock(&mbox->f2hq_lock); + + msg->hdr.word0 = readq(qidx); + for (i = 1; i <= msg->hdr.sizew; i++) + *mbuf++ = readq(qidx + (i * 8)); + + ci = octep_ctrl_mbox_circq_inc(ci, q->mask); + writel(ci, q->hw_cons); + + mutex_unlock(&mbox->f2hq_lock); + + if (msg->hdr.flags != OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ || !mbox->process_req) + return 0; + + mbox->process_req(mbox->user_ctx, msg); + mbuf = (u64 *)msg->msg; + for (i = 1; i <= msg->hdr.sizew; i++) + writeq(*mbuf++, (qidx + (i * 8))); + + writeq(msg->hdr.word0, qidx); + + return 0; +} + +int octep_ctrl_mbox_uninit(struct octep_ctrl_mbox *mbox) +{ + if (!mbox) + return -EINVAL; + + writeq(OCTEP_CTRL_MBOX_STATUS_UNINIT, + OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem)); + /* ensure uninit state is written before uninitialization */ + wmb(); + + mutex_destroy(&mbox->h2fq_lock); + mutex_destroy(&mbox->f2hq_lock); + + writeq(OCTEP_CTRL_MBOX_STATUS_INVALID, + OCTEP_CTRL_MBOX_INFO_HOST_STATUS_OFFSET(mbox->barmem)); + + pr_info("Octep ctrl mbox : Uninit successful.\n"); + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h new file mode 100644 index 000000000000..2dc5753cfec6 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_mbox.h @@ -0,0 +1,170 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + #ifndef __OCTEP_CTRL_MBOX_H__ +#define __OCTEP_CTRL_MBOX_H__ + +/* barmem structure + * |===========================================| + * |Info (16 + 120 + 120 = 256 bytes) | + * |-------------------------------------------| + * |magic number (8 bytes) | + * |bar memory size (4 bytes) | + * |reserved (4 bytes) | + * |-------------------------------------------| + * |host version (8 bytes) | + * |host status (8 bytes) | + * |host reserved (104 bytes) | + * |-------------------------------------------| + * |fw version (8 bytes) | + * |fw status (8 bytes) | + * |fw reserved (104 bytes) | + * |===========================================| + * |Host to Fw Queue info (16 bytes) | + * |-------------------------------------------| + * |producer index (4 bytes) | + * |consumer index (4 bytes) | + * |element size (4 bytes) | + * |element count (4 bytes) | + * |===========================================| + * |Fw to Host Queue info (16 bytes) | + * |-------------------------------------------| + * |producer index (4 bytes) | + * |consumer index (4 bytes) | + * |element size (4 bytes) | + * |element count (4 bytes) | + * |===========================================| + * |Host to Fw Queue | + * |-------------------------------------------| + * |((elem_sz + hdr(8 bytes)) * elem_cnt) bytes| + * |===========================================| + * |===========================================| + * |Fw to Host Queue | + * |-------------------------------------------| + * |((elem_sz + hdr(8 bytes)) * elem_cnt) bytes| + * |===========================================| + */ + +#define OCTEP_CTRL_MBOX_MAGIC_NUMBER 0xdeaddeadbeefbeefull + +/* Size of mbox info in bytes */ +#define OCTEP_CTRL_MBOX_INFO_SZ 256 +/* Size of mbox host to target queue info in bytes */ +#define OCTEP_CTRL_MBOX_H2FQ_INFO_SZ 16 +/* Size of mbox target to host queue info in bytes */ +#define OCTEP_CTRL_MBOX_F2HQ_INFO_SZ 16 +/* Size of mbox queue in bytes */ +#define OCTEP_CTRL_MBOX_Q_SZ(sz, cnt) (((sz) + 8) * (cnt)) +/* Size of mbox in bytes */ +#define OCTEP_CTRL_MBOX_SZ(hsz, hcnt, fsz, fcnt) (OCTEP_CTRL_MBOX_INFO_SZ + \ + OCTEP_CTRL_MBOX_H2FQ_INFO_SZ + \ + OCTEP_CTRL_MBOX_F2HQ_INFO_SZ + \ + OCTEP_CTRL_MBOX_Q_SZ(hsz, hcnt) + \ + OCTEP_CTRL_MBOX_Q_SZ(fsz, fcnt)) + +/* Valid request message */ +#define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ BIT(0) +/* Valid response message */ +#define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_RESP BIT(1) +/* Valid notification, no response required */ +#define OCTEP_CTRL_MBOX_MSG_HDR_FLAG_NOTIFY BIT(2) + +enum octep_ctrl_mbox_status { + OCTEP_CTRL_MBOX_STATUS_INVALID = 0, + OCTEP_CTRL_MBOX_STATUS_INIT, + OCTEP_CTRL_MBOX_STATUS_READY, + OCTEP_CTRL_MBOX_STATUS_UNINIT +}; + +/* mbox message */ +union octep_ctrl_mbox_msg_hdr { + u64 word0; + struct { + /* OCTEP_CTRL_MBOX_MSG_HDR_FLAG_* */ + u32 flags; + /* size of message in words excluding header */ + u32 sizew; + }; +}; + +/* mbox message */ +struct octep_ctrl_mbox_msg { + /* mbox transaction header */ + union octep_ctrl_mbox_msg_hdr hdr; + /* pointer to message buffer */ + void *msg; +}; + +/* Mbox queue */ +struct octep_ctrl_mbox_q { + /* q element size, should be aligned to unsigned long */ + u16 elem_sz; + /* q element count, should be power of 2 */ + u16 elem_cnt; + /* q mask */ + u16 mask; + /* producer address in bar mem */ + u8 __iomem *hw_prod; + /* consumer address in bar mem */ + u8 __iomem *hw_cons; + /* q base address in bar mem */ + u8 __iomem *hw_q; +}; + +struct octep_ctrl_mbox { + /* host driver version */ + u64 version; + /* size of bar memory */ + u32 barmem_sz; + /* pointer to BAR memory */ + u8 __iomem *barmem; + /* user context for callback, can be null */ + void *user_ctx; + /* callback handler for processing request, called from octep_ctrl_mbox_recv */ + int (*process_req)(void *user_ctx, struct octep_ctrl_mbox_msg *msg); + /* host-to-fw queue */ + struct octep_ctrl_mbox_q h2fq; + /* fw-to-host queue */ + struct octep_ctrl_mbox_q f2hq; + /* lock for h2fq */ + struct mutex h2fq_lock; + /* lock for f2hq */ + struct mutex f2hq_lock; +}; + +/* Initialize control mbox. + * + * @param mbox: non-null pointer to struct octep_ctrl_mbox. + * + * return value: 0 on success, -errno on failure. + */ +int octep_ctrl_mbox_init(struct octep_ctrl_mbox *mbox); + +/* Send mbox message. + * + * @param mbox: non-null pointer to struct octep_ctrl_mbox. + * + * return value: 0 on success, -errno on failure. + */ +int octep_ctrl_mbox_send(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_msg *msg); + +/* Retrieve mbox message. + * + * @param mbox: non-null pointer to struct octep_ctrl_mbox. + * + * return value: 0 on success, -errno on failure. + */ +int octep_ctrl_mbox_recv(struct octep_ctrl_mbox *mbox, struct octep_ctrl_mbox_msg *msg); + +/* Uninitialize control mbox. + * + * @param ep: non-null pointer to struct octep_ctrl_mbox. + * + * return value: 0 on success, -errno on failure. + */ +int octep_ctrl_mbox_uninit(struct octep_ctrl_mbox *mbox); + +#endif /* __OCTEP_CTRL_MBOX_H__ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c new file mode 100644 index 000000000000..7c00c896ab98 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ +#include <linux/string.h> +#include <linux/types.h> +#include <linux/etherdevice.h> +#include <linux/pci.h> + +#include "octep_config.h" +#include "octep_main.h" +#include "octep_ctrl_net.h" + +int octep_get_link_status(struct octep_device *oct) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_net_h2f_resp *resp; + struct octep_ctrl_mbox_msg msg = {}; + int err; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS; + req.link.cmd = OCTEP_CTRL_NET_CMD_GET; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_STATE_REQ_SZW; + msg.msg = &req; + err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); + if (err) + return err; + + resp = (struct octep_ctrl_net_h2f_resp *)&req; + return resp->link.state; +} + +void octep_set_link_status(struct octep_device *oct, bool up) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_mbox_msg msg = {}; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS; + req.link.cmd = OCTEP_CTRL_NET_CMD_SET; + req.link.state = (up) ? OCTEP_CTRL_NET_STATE_UP : OCTEP_CTRL_NET_STATE_DOWN; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_STATE_REQ_SZW; + msg.msg = &req; + octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); +} + +void octep_set_rx_state(struct octep_device *oct, bool up) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_mbox_msg msg = {}; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_RX_STATE; + req.link.cmd = OCTEP_CTRL_NET_CMD_SET; + req.link.state = (up) ? OCTEP_CTRL_NET_STATE_UP : OCTEP_CTRL_NET_STATE_DOWN; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_STATE_REQ_SZW; + msg.msg = &req; + octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); +} + +int octep_get_mac_addr(struct octep_device *oct, u8 *addr) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_net_h2f_resp *resp; + struct octep_ctrl_mbox_msg msg = {}; + int err; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_MAC; + req.link.cmd = OCTEP_CTRL_NET_CMD_GET; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_MAC_REQ_SZW; + msg.msg = &req; + err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); + if (err) + return err; + + resp = (struct octep_ctrl_net_h2f_resp *)&req; + memcpy(addr, resp->mac.addr, ETH_ALEN); + + return err; +} + +int octep_set_mac_addr(struct octep_device *oct, u8 *addr) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_mbox_msg msg = {}; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_MAC; + req.mac.cmd = OCTEP_CTRL_NET_CMD_SET; + memcpy(&req.mac.addr, addr, ETH_ALEN); + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_MAC_REQ_SZW; + msg.msg = &req; + + return octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); +} + +int octep_set_mtu(struct octep_device *oct, int mtu) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_mbox_msg msg = {}; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_MTU; + req.mtu.cmd = OCTEP_CTRL_NET_CMD_SET; + req.mtu.val = mtu; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_MTU_REQ_SZW; + msg.msg = &req; + + return octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); +} + +int octep_get_if_stats(struct octep_device *oct) +{ + void __iomem *iface_rx_stats; + void __iomem *iface_tx_stats; + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_mbox_msg msg = {}; + int err; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_GET_IF_STATS; + req.mac.cmd = OCTEP_CTRL_NET_CMD_GET; + req.get_stats.offset = oct->ctrl_mbox_ifstats_offset; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_GET_STATS_REQ_SZW; + msg.msg = &req; + err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); + if (err) + return err; + + iface_rx_stats = oct->ctrl_mbox.barmem + oct->ctrl_mbox_ifstats_offset; + iface_tx_stats = oct->ctrl_mbox.barmem + oct->ctrl_mbox_ifstats_offset + + sizeof(struct octep_iface_rx_stats); + memcpy_fromio(&oct->iface_rx_stats, iface_rx_stats, sizeof(struct octep_iface_rx_stats)); + memcpy_fromio(&oct->iface_tx_stats, iface_tx_stats, sizeof(struct octep_iface_tx_stats)); + + return err; +} + +int octep_get_link_info(struct octep_device *oct) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_net_h2f_resp *resp; + struct octep_ctrl_mbox_msg msg = {}; + int err; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_INFO; + req.mac.cmd = OCTEP_CTRL_NET_CMD_GET; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_LINK_INFO_REQ_SZW; + msg.msg = &req; + err = octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); + if (err) + return err; + + resp = (struct octep_ctrl_net_h2f_resp *)&req; + oct->link_info.supported_modes = resp->link_info.supported_modes; + oct->link_info.advertised_modes = resp->link_info.advertised_modes; + oct->link_info.autoneg = resp->link_info.autoneg; + oct->link_info.pause = resp->link_info.pause; + oct->link_info.speed = resp->link_info.speed; + + return err; +} + +int octep_set_link_info(struct octep_device *oct, struct octep_iface_link_info *link_info) +{ + struct octep_ctrl_net_h2f_req req = {}; + struct octep_ctrl_mbox_msg msg = {}; + + req.hdr.cmd = OCTEP_CTRL_NET_H2F_CMD_LINK_INFO; + req.link_info.cmd = OCTEP_CTRL_NET_CMD_SET; + req.link_info.info.advertised_modes = link_info->advertised_modes; + req.link_info.info.autoneg = link_info->autoneg; + req.link_info.info.pause = link_info->pause; + req.link_info.info.speed = link_info->speed; + + msg.hdr.flags = OCTEP_CTRL_MBOX_MSG_HDR_FLAG_REQ; + msg.hdr.sizew = OCTEP_CTRL_NET_H2F_LINK_INFO_REQ_SZW; + msg.msg = &req; + + return octep_ctrl_mbox_send(&oct->ctrl_mbox, &msg); +} diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h new file mode 100644 index 000000000000..f23b58381322 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ctrl_net.h @@ -0,0 +1,299 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ +#ifndef __OCTEP_CTRL_NET_H__ +#define __OCTEP_CTRL_NET_H__ + +/* Supported commands */ +enum octep_ctrl_net_cmd { + OCTEP_CTRL_NET_CMD_GET = 0, + OCTEP_CTRL_NET_CMD_SET, +}; + +/* Supported states */ +enum octep_ctrl_net_state { + OCTEP_CTRL_NET_STATE_DOWN = 0, + OCTEP_CTRL_NET_STATE_UP, +}; + +/* Supported replies */ +enum octep_ctrl_net_reply { + OCTEP_CTRL_NET_REPLY_OK = 0, + OCTEP_CTRL_NET_REPLY_GENERIC_FAIL, + OCTEP_CTRL_NET_REPLY_INVALID_PARAM, +}; + +/* Supported host to fw commands */ +enum octep_ctrl_net_h2f_cmd { + OCTEP_CTRL_NET_H2F_CMD_INVALID = 0, + OCTEP_CTRL_NET_H2F_CMD_MTU, + OCTEP_CTRL_NET_H2F_CMD_MAC, + OCTEP_CTRL_NET_H2F_CMD_GET_IF_STATS, + OCTEP_CTRL_NET_H2F_CMD_GET_XSTATS, + OCTEP_CTRL_NET_H2F_CMD_GET_Q_STATS, + OCTEP_CTRL_NET_H2F_CMD_LINK_STATUS, + OCTEP_CTRL_NET_H2F_CMD_RX_STATE, + OCTEP_CTRL_NET_H2F_CMD_LINK_INFO, +}; + +/* Supported fw to host commands */ +enum octep_ctrl_net_f2h_cmd { + OCTEP_CTRL_NET_F2H_CMD_INVALID = 0, + OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS, +}; + +struct octep_ctrl_net_req_hdr { + /* sender id */ + u16 sender; + /* receiver id */ + u16 receiver; + /* octep_ctrl_net_h2t_cmd */ + u16 cmd; + /* reserved */ + u16 rsvd0; +}; + +/* get/set mtu request */ +struct octep_ctrl_net_h2f_req_cmd_mtu { + /* enum octep_ctrl_net_cmd */ + u16 cmd; + /* 0-65535 */ + u16 val; +}; + +/* get/set mac request */ +struct octep_ctrl_net_h2f_req_cmd_mac { + /* enum octep_ctrl_net_cmd */ + u16 cmd; + /* xx:xx:xx:xx:xx:xx */ + u8 addr[ETH_ALEN]; +}; + +/* get if_stats, xstats, q_stats request */ +struct octep_ctrl_net_h2f_req_cmd_get_stats { + /* offset into barmem where fw should copy over stats */ + u32 offset; +}; + +/* get/set link state, rx state */ +struct octep_ctrl_net_h2f_req_cmd_state { + /* enum octep_ctrl_net_cmd */ + u16 cmd; + /* enum octep_ctrl_net_state */ + u16 state; +}; + +/* link info */ +struct octep_ctrl_net_link_info { + /* Bitmap of Supported link speeds/modes */ + u64 supported_modes; + /* Bitmap of Advertised link speeds/modes */ + u64 advertised_modes; + /* Autonegotation state; bit 0=disabled; bit 1=enabled */ + u8 autoneg; + /* Pause frames setting. bit 0=disabled; bit 1=enabled */ + u8 pause; + /* Negotiated link speed in Mbps */ + u32 speed; +}; + +/* get/set link info */ +struct octep_ctrl_net_h2f_req_cmd_link_info { + /* enum octep_ctrl_net_cmd */ + u16 cmd; + /* struct octep_ctrl_net_link_info */ + struct octep_ctrl_net_link_info info; +}; + +/* Host to fw request data */ +struct octep_ctrl_net_h2f_req { + struct octep_ctrl_net_req_hdr hdr; + union { + struct octep_ctrl_net_h2f_req_cmd_mtu mtu; + struct octep_ctrl_net_h2f_req_cmd_mac mac; + struct octep_ctrl_net_h2f_req_cmd_get_stats get_stats; + struct octep_ctrl_net_h2f_req_cmd_state link; + struct octep_ctrl_net_h2f_req_cmd_state rx; + struct octep_ctrl_net_h2f_req_cmd_link_info link_info; + }; +} __packed; + +struct octep_ctrl_net_resp_hdr { + /* sender id */ + u16 sender; + /* receiver id */ + u16 receiver; + /* octep_ctrl_net_h2t_cmd */ + u16 cmd; + /* octep_ctrl_net_reply */ + u16 reply; +}; + +/* get mtu response */ +struct octep_ctrl_net_h2f_resp_cmd_mtu { + /* 0-65535 */ + u16 val; +}; + +/* get mac response */ +struct octep_ctrl_net_h2f_resp_cmd_mac { + /* xx:xx:xx:xx:xx:xx */ + u8 addr[ETH_ALEN]; +}; + +/* get link state, rx state response */ +struct octep_ctrl_net_h2f_resp_cmd_state { + /* enum octep_ctrl_net_state */ + u16 state; +}; + +/* Host to fw response data */ +struct octep_ctrl_net_h2f_resp { + struct octep_ctrl_net_resp_hdr hdr; + union { + struct octep_ctrl_net_h2f_resp_cmd_mtu mtu; + struct octep_ctrl_net_h2f_resp_cmd_mac mac; + struct octep_ctrl_net_h2f_resp_cmd_state link; + struct octep_ctrl_net_h2f_resp_cmd_state rx; + struct octep_ctrl_net_link_info link_info; + }; +} __packed; + +/* link state notofication */ +struct octep_ctrl_net_f2h_req_cmd_state { + /* enum octep_ctrl_net_state */ + u16 state; +}; + +/* Fw to host request data */ +struct octep_ctrl_net_f2h_req { + struct octep_ctrl_net_req_hdr hdr; + union { + struct octep_ctrl_net_f2h_req_cmd_state link; + }; +}; + +/* Fw to host response data */ +struct octep_ctrl_net_f2h_resp { + struct octep_ctrl_net_resp_hdr hdr; +}; + +/* Size of host to fw octep_ctrl_mbox queue element */ +union octep_ctrl_net_h2f_data_sz { + struct octep_ctrl_net_h2f_req h2f_req; + struct octep_ctrl_net_h2f_resp h2f_resp; +}; + +/* Size of fw to host octep_ctrl_mbox queue element */ +union octep_ctrl_net_f2h_data_sz { + struct octep_ctrl_net_f2h_req f2h_req; + struct octep_ctrl_net_f2h_resp f2h_resp; +}; + +/* size of host to fw data in words */ +#define OCTEP_CTRL_NET_H2F_DATA_SZW ((sizeof(union octep_ctrl_net_h2f_data_sz)) / \ + (sizeof(unsigned long))) + +/* size of fw to host data in words */ +#define OCTEP_CTRL_NET_F2H_DATA_SZW ((sizeof(union octep_ctrl_net_f2h_data_sz)) / \ + (sizeof(unsigned long))) + +/* size in words of get/set mtu request */ +#define OCTEP_CTRL_NET_H2F_MTU_REQ_SZW 2 +/* size in words of get/set mac request */ +#define OCTEP_CTRL_NET_H2F_MAC_REQ_SZW 2 +/* size in words of get stats request */ +#define OCTEP_CTRL_NET_H2F_GET_STATS_REQ_SZW 2 +/* size in words of get/set state request */ +#define OCTEP_CTRL_NET_H2F_STATE_REQ_SZW 2 +/* size in words of get/set link info request */ +#define OCTEP_CTRL_NET_H2F_LINK_INFO_REQ_SZW 4 + +/* size in words of get mtu response */ +#define OCTEP_CTRL_NET_H2F_GET_MTU_RESP_SZW 2 +/* size in words of set mtu response */ +#define OCTEP_CTRL_NET_H2F_SET_MTU_RESP_SZW 1 +/* size in words of get mac response */ +#define OCTEP_CTRL_NET_H2F_GET_MAC_RESP_SZW 2 +/* size in words of set mac response */ +#define OCTEP_CTRL_NET_H2F_SET_MAC_RESP_SZW 1 +/* size in words of get state request */ +#define OCTEP_CTRL_NET_H2F_GET_STATE_RESP_SZW 2 +/* size in words of set state request */ +#define OCTEP_CTRL_NET_H2F_SET_STATE_RESP_SZW 1 +/* size in words of get link info request */ +#define OCTEP_CTRL_NET_H2F_GET_LINK_INFO_RESP_SZW 4 +/* size in words of set link info request */ +#define OCTEP_CTRL_NET_H2F_SET_LINK_INFO_RESP_SZW 1 + +/** Get link status from firmware. + * + * @param oct: non-null pointer to struct octep_device. + * + * return value: link status 0=down, 1=up. + */ +int octep_get_link_status(struct octep_device *oct); + +/** Set link status in firmware. + * + * @param oct: non-null pointer to struct octep_device. + * @param up: boolean status. + */ +void octep_set_link_status(struct octep_device *oct, bool up); + +/** Set rx state in firmware. + * + * @param oct: non-null pointer to struct octep_device. + * @param up: boolean status. + */ +void octep_set_rx_state(struct octep_device *oct, bool up); + +/** Get mac address from firmware. + * + * @param oct: non-null pointer to struct octep_device. + * @param addr: non-null pointer to mac address. + * + * return value: 0 on success, -errno on failure. + */ +int octep_get_mac_addr(struct octep_device *oct, u8 *addr); + +/** Set mac address in firmware. + * + * @param oct: non-null pointer to struct octep_device. + * @param addr: non-null pointer to mac address. + */ +int octep_set_mac_addr(struct octep_device *oct, u8 *addr); + +/** Set mtu in firmware. + * + * @param oct: non-null pointer to struct octep_device. + * @param mtu: mtu. + */ +int octep_set_mtu(struct octep_device *oct, int mtu); + +/** Get interface statistics from firmware. + * + * @param oct: non-null pointer to struct octep_device. + * + * return value: 0 on success, -errno on failure. + */ +int octep_get_if_stats(struct octep_device *oct); + +/** Get link info from firmware. + * + * @param oct: non-null pointer to struct octep_device. + * + * return value: 0 on success, -errno on failure. + */ +int octep_get_link_info(struct octep_device *oct); + +/** Set link info in firmware. + * + * @param oct: non-null pointer to struct octep_device. + */ +int octep_set_link_info(struct octep_device *oct, struct octep_iface_link_info *link_info); + +#endif /* __OCTEP_CTRL_NET_H__ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c b/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c new file mode 100644 index 000000000000..87ef129b269a --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_ethtool.c @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/ethtool.h> + +#include "octep_config.h" +#include "octep_main.h" +#include "octep_ctrl_net.h" + +static const char octep_gstrings_global_stats[][ETH_GSTRING_LEN] = { + "rx_packets", + "tx_packets", + "rx_bytes", + "tx_bytes", + "rx_alloc_errors", + "tx_busy_errors", + "rx_dropped", + "tx_dropped", + "tx_hw_pkts", + "tx_hw_octs", + "tx_hw_bcast", + "tx_hw_mcast", + "tx_hw_underflow", + "tx_hw_control", + "tx_less_than_64", + "tx_equal_64", + "tx_equal_65_to_127", + "tx_equal_128_to_255", + "tx_equal_256_to_511", + "tx_equal_512_to_1023", + "tx_equal_1024_to_1518", + "tx_greater_than_1518", + "rx_hw_pkts", + "rx_hw_bytes", + "rx_hw_bcast", + "rx_hw_mcast", + "rx_pause_pkts", + "rx_pause_bytes", + "rx_dropped_pkts_fifo_full", + "rx_dropped_bytes_fifo_full", + "rx_err_pkts", +}; + +#define OCTEP_GLOBAL_STATS_CNT (sizeof(octep_gstrings_global_stats) / ETH_GSTRING_LEN) + +static const char octep_gstrings_tx_q_stats[][ETH_GSTRING_LEN] = { + "tx_packets_posted[Q-%u]", + "tx_packets_completed[Q-%u]", + "tx_bytes[Q-%u]", + "tx_busy[Q-%u]", +}; + +#define OCTEP_TX_Q_STATS_CNT (sizeof(octep_gstrings_tx_q_stats) / ETH_GSTRING_LEN) + +static const char octep_gstrings_rx_q_stats[][ETH_GSTRING_LEN] = { + "rx_packets[Q-%u]", + "rx_bytes[Q-%u]", + "rx_alloc_errors[Q-%u]", +}; + +#define OCTEP_RX_Q_STATS_CNT (sizeof(octep_gstrings_rx_q_stats) / ETH_GSTRING_LEN) + +static void octep_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *info) +{ + struct octep_device *oct = netdev_priv(netdev); + + strscpy(info->driver, OCTEP_DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(oct->pdev), sizeof(info->bus_info)); +} + +static void octep_get_strings(struct net_device *netdev, + u32 stringset, u8 *data) +{ + struct octep_device *oct = netdev_priv(netdev); + u16 num_queues = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); + char *strings = (char *)data; + int i, j; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < OCTEP_GLOBAL_STATS_CNT; i++) { + snprintf(strings, ETH_GSTRING_LEN, + octep_gstrings_global_stats[i]); + strings += ETH_GSTRING_LEN; + } + + for (i = 0; i < num_queues; i++) { + for (j = 0; j < OCTEP_TX_Q_STATS_CNT; j++) { + snprintf(strings, ETH_GSTRING_LEN, + octep_gstrings_tx_q_stats[j], i); + strings += ETH_GSTRING_LEN; + } + } + + for (i = 0; i < num_queues; i++) { + for (j = 0; j < OCTEP_RX_Q_STATS_CNT; j++) { + snprintf(strings, ETH_GSTRING_LEN, + octep_gstrings_rx_q_stats[j], i); + strings += ETH_GSTRING_LEN; + } + } + break; + default: + break; + } +} + +static int octep_get_sset_count(struct net_device *netdev, int sset) +{ + struct octep_device *oct = netdev_priv(netdev); + u16 num_queues = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); + + switch (sset) { + case ETH_SS_STATS: + return OCTEP_GLOBAL_STATS_CNT + (num_queues * + (OCTEP_TX_Q_STATS_CNT + OCTEP_RX_Q_STATS_CNT)); + break; + default: + return -EOPNOTSUPP; + } +} + +static void +octep_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + struct octep_device *oct = netdev_priv(netdev); + struct octep_iface_tx_stats *iface_tx_stats; + struct octep_iface_rx_stats *iface_rx_stats; + u64 rx_packets, rx_bytes; + u64 tx_packets, tx_bytes; + u64 rx_alloc_errors, tx_busy_errors; + int q, i; + + rx_packets = 0; + rx_bytes = 0; + tx_packets = 0; + tx_bytes = 0; + rx_alloc_errors = 0; + tx_busy_errors = 0; + tx_packets = 0; + tx_bytes = 0; + rx_packets = 0; + rx_bytes = 0; + + octep_get_if_stats(oct); + iface_tx_stats = &oct->iface_tx_stats; + iface_rx_stats = &oct->iface_rx_stats; + + for (q = 0; q < oct->num_oqs; q++) { + struct octep_iq *iq = oct->iq[q]; + struct octep_oq *oq = oct->oq[q]; + + tx_packets += iq->stats.instr_completed; + tx_bytes += iq->stats.bytes_sent; + tx_busy_errors += iq->stats.tx_busy; + + rx_packets += oq->stats.packets; + rx_bytes += oq->stats.bytes; + rx_alloc_errors += oq->stats.alloc_failures; + } + i = 0; + data[i++] = rx_packets; + data[i++] = tx_packets; + data[i++] = rx_bytes; + data[i++] = tx_bytes; + data[i++] = rx_alloc_errors; + data[i++] = tx_busy_errors; + data[i++] = iface_rx_stats->dropped_pkts_fifo_full + + iface_rx_stats->err_pkts; + data[i++] = iface_tx_stats->xscol + + iface_tx_stats->xsdef; + data[i++] = iface_tx_stats->pkts; + data[i++] = iface_tx_stats->octs; + data[i++] = iface_tx_stats->bcst; + data[i++] = iface_tx_stats->mcst; + data[i++] = iface_tx_stats->undflw; + data[i++] = iface_tx_stats->ctl; + data[i++] = iface_tx_stats->hist_lt64; + data[i++] = iface_tx_stats->hist_eq64; + data[i++] = iface_tx_stats->hist_65to127; + data[i++] = iface_tx_stats->hist_128to255; + data[i++] = iface_tx_stats->hist_256to511; + data[i++] = iface_tx_stats->hist_512to1023; + data[i++] = iface_tx_stats->hist_1024to1518; + data[i++] = iface_tx_stats->hist_gt1518; + data[i++] = iface_rx_stats->pkts; + data[i++] = iface_rx_stats->octets; + data[i++] = iface_rx_stats->mcast_pkts; + data[i++] = iface_rx_stats->bcast_pkts; + data[i++] = iface_rx_stats->pause_pkts; + data[i++] = iface_rx_stats->pause_octets; + data[i++] = iface_rx_stats->dropped_pkts_fifo_full; + data[i++] = iface_rx_stats->dropped_octets_fifo_full; + data[i++] = iface_rx_stats->err_pkts; + + /* Per Tx Queue stats */ + for (q = 0; q < oct->num_iqs; q++) { + struct octep_iq *iq = oct->iq[q]; + + data[i++] = iq->stats.instr_posted; + data[i++] = iq->stats.instr_completed; + data[i++] = iq->stats.bytes_sent; + data[i++] = iq->stats.tx_busy; + } + + /* Per Rx Queue stats */ + for (q = 0; q < oct->num_oqs; q++) { + struct octep_oq *oq = oct->oq[q]; + + data[i++] = oq->stats.packets; + data[i++] = oq->stats.bytes; + data[i++] = oq->stats.alloc_failures; + } +} + +#define OCTEP_SET_ETHTOOL_LINK_MODES_BITMAP(octep_speeds, ksettings, name) \ +{ \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_10GBASE_T)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 10000baseT_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_10GBASE_R)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 10000baseR_FEC); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_10GBASE_CR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 10000baseCR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_10GBASE_KR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 10000baseKR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_10GBASE_LR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 10000baseLR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_10GBASE_SR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 10000baseSR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_25GBASE_CR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 25000baseCR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_25GBASE_KR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 25000baseKR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_25GBASE_SR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 25000baseSR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_40GBASE_CR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 40000baseCR4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_40GBASE_KR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 40000baseKR4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_40GBASE_LR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 40000baseLR4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_40GBASE_SR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 40000baseSR4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_CR2)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseCR2_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_KR2)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseKR2_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_SR2)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseSR2_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_CR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseCR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_KR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseKR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_LR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseLR_ER_FR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_50GBASE_SR)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 50000baseSR_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_100GBASE_CR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 100000baseCR4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_100GBASE_KR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 100000baseKR4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_100GBASE_LR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 100000baseLR4_ER4_Full); \ + if ((octep_speeds) & BIT(OCTEP_LINK_MODE_100GBASE_SR4)) \ + ethtool_link_ksettings_add_link_mode(ksettings, name, 100000baseSR4_Full); \ +} + +static int octep_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) +{ + struct octep_device *oct = netdev_priv(netdev); + struct octep_iface_link_info *link_info; + u32 advertised_modes, supported_modes; + + ethtool_link_ksettings_zero_link_mode(cmd, supported); + ethtool_link_ksettings_zero_link_mode(cmd, advertising); + + octep_get_link_info(oct); + + advertised_modes = oct->link_info.advertised_modes; + supported_modes = oct->link_info.supported_modes; + link_info = &oct->link_info; + + OCTEP_SET_ETHTOOL_LINK_MODES_BITMAP(supported_modes, cmd, supported); + OCTEP_SET_ETHTOOL_LINK_MODES_BITMAP(advertised_modes, cmd, advertising); + + if (link_info->autoneg) { + if (link_info->autoneg & OCTEP_LINK_MODE_AUTONEG_SUPPORTED) + ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg); + if (link_info->autoneg & OCTEP_LINK_MODE_AUTONEG_ADVERTISED) { + ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg); + cmd->base.autoneg = AUTONEG_ENABLE; + } else { + cmd->base.autoneg = AUTONEG_DISABLE; + } + } else { + cmd->base.autoneg = AUTONEG_DISABLE; + } + + if (link_info->pause) { + if (link_info->pause & OCTEP_LINK_MODE_PAUSE_SUPPORTED) + ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); + if (link_info->pause & OCTEP_LINK_MODE_PAUSE_ADVERTISED) + ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause); + } + + cmd->base.port = PORT_FIBRE; + ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); + ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); + + if (netif_carrier_ok(netdev)) { + cmd->base.speed = link_info->speed; + cmd->base.duplex = DUPLEX_FULL; + } else { + cmd->base.speed = SPEED_UNKNOWN; + cmd->base.duplex = DUPLEX_UNKNOWN; + } + return 0; +} + +static int octep_set_link_ksettings(struct net_device *netdev, + const struct ethtool_link_ksettings *cmd) +{ + struct octep_device *oct = netdev_priv(netdev); + struct octep_iface_link_info link_info_new; + struct octep_iface_link_info *link_info; + u64 advertised = 0; + u8 autoneg = 0; + int err; + + link_info = &oct->link_info; + memcpy(&link_info_new, link_info, sizeof(struct octep_iface_link_info)); + + /* Only Full duplex is supported; + * Assume full duplex when duplex is unknown. + */ + if (cmd->base.duplex != DUPLEX_FULL && + cmd->base.duplex != DUPLEX_UNKNOWN) + return -EOPNOTSUPP; + + if (cmd->base.autoneg == AUTONEG_ENABLE) { + if (!(link_info->autoneg & OCTEP_LINK_MODE_AUTONEG_SUPPORTED)) + return -EOPNOTSUPP; + autoneg = 1; + } + + if (!bitmap_subset(cmd->link_modes.advertising, + cmd->link_modes.supported, + __ETHTOOL_LINK_MODE_MASK_NBITS)) + return -EINVAL; + + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 10000baseT_Full)) + advertised |= BIT(OCTEP_LINK_MODE_10GBASE_T); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 10000baseR_FEC)) + advertised |= BIT(OCTEP_LINK_MODE_10GBASE_R); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 10000baseCR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_10GBASE_CR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 10000baseKR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_10GBASE_KR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 10000baseLR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_10GBASE_LR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 10000baseSR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_10GBASE_SR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 25000baseCR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_25GBASE_CR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 25000baseKR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_25GBASE_KR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 25000baseSR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_25GBASE_SR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 40000baseCR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_40GBASE_CR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 40000baseKR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_40GBASE_KR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 40000baseLR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_40GBASE_LR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 40000baseSR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_40GBASE_SR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseCR2_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_CR2); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseKR2_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_KR2); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseSR2_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_SR2); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseCR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_CR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseKR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_KR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseLR_ER_FR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_LR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 50000baseSR_Full)) + advertised |= BIT(OCTEP_LINK_MODE_50GBASE_SR); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 100000baseCR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_100GBASE_CR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 100000baseKR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_100GBASE_KR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 100000baseLR4_ER4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_100GBASE_LR4); + if (ethtool_link_ksettings_test_link_mode(cmd, advertising, + 100000baseSR4_Full)) + advertised |= BIT(OCTEP_LINK_MODE_100GBASE_SR4); + + if (advertised == link_info->advertised_modes && + cmd->base.speed == link_info->speed && + cmd->base.autoneg == link_info->autoneg) + return 0; + + link_info_new.advertised_modes = advertised; + link_info_new.speed = cmd->base.speed; + link_info_new.autoneg = autoneg; + + err = octep_set_link_info(oct, &link_info_new); + if (err) + return err; + + memcpy(link_info, &link_info_new, sizeof(struct octep_iface_link_info)); + return 0; +} + +static const struct ethtool_ops octep_ethtool_ops = { + .get_drvinfo = octep_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_strings = octep_get_strings, + .get_sset_count = octep_get_sset_count, + .get_ethtool_stats = octep_get_ethtool_stats, + .get_link_ksettings = octep_get_link_ksettings, + .set_link_ksettings = octep_set_link_ksettings, +}; + +void octep_set_ethtool_ops(struct net_device *netdev) +{ + netdev->ethtool_ops = &octep_ethtool_ops; +} diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c new file mode 100644 index 000000000000..9089adcb75f9 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -0,0 +1,1181 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/aer.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/rtnetlink.h> +#include <linux/vmalloc.h> + +#include "octep_config.h" +#include "octep_main.h" +#include "octep_ctrl_net.h" + +struct workqueue_struct *octep_wq; + +/* Supported Devices */ +static const struct pci_device_id octep_pci_id_tbl[] = { + {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)}, + {0, }, +}; +MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl); + +MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>"); +MODULE_DESCRIPTION(OCTEP_DRV_STRING); +MODULE_LICENSE("GPL"); + +/** + * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info. + * + * @oct: Octeon device private data structure. + * + * Allocate resources to hold per Tx/Rx queue interrupt info. + * This is the information passed to interrupt handler, from which napi poll + * is scheduled and includes quick access to private data of Tx/Rx queue + * corresponding to the interrupt being handled. + * + * Return: 0, on successful allocation of resources for all queue interrupts. + * -1, if failed to allocate any resource. + */ +static int octep_alloc_ioq_vectors(struct octep_device *oct) +{ + int i; + struct octep_ioq_vector *ioq_vector; + + for (i = 0; i < oct->num_oqs; i++) { + oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i])); + if (!oct->ioq_vector[i]) + goto free_ioq_vector; + + ioq_vector = oct->ioq_vector[i]; + ioq_vector->iq = oct->iq[i]; + ioq_vector->oq = oct->oq[i]; + ioq_vector->octep_dev = oct; + } + + dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs); + return 0; + +free_ioq_vector: + while (i) { + i--; + vfree(oct->ioq_vector[i]); + oct->ioq_vector[i] = NULL; + } + return -1; +} + +/** + * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info. + * + * @oct: Octeon device private data structure. + */ +static void octep_free_ioq_vectors(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_oqs; i++) { + if (oct->ioq_vector[i]) { + vfree(oct->ioq_vector[i]); + oct->ioq_vector[i] = NULL; + } + } + netdev_info(oct->netdev, "Freed IOQ Vectors\n"); +} + +/** + * octep_enable_msix_range() - enable MSI-x interrupts. + * + * @oct: Octeon device private data structure. + * + * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts) + * for the Octeon device. + * + * Return: 0, on successfully enabling all MSI-x interrupts. + * -1, if failed to enable any MSI-x interrupt. + */ +static int octep_enable_msix_range(struct octep_device *oct) +{ + int num_msix, msix_allocated; + int i; + + /* Generic interrupts apart from input/output queues */ + num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf); + oct->msix_entries = kcalloc(num_msix, + sizeof(struct msix_entry), GFP_KERNEL); + if (!oct->msix_entries) + goto msix_alloc_err; + + for (i = 0; i < num_msix; i++) + oct->msix_entries[i].entry = i; + + msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries, + num_msix, num_msix); + if (msix_allocated != num_msix) { + dev_err(&oct->pdev->dev, + "Failed to enable %d msix irqs; got only %d\n", + num_msix, msix_allocated); + goto enable_msix_err; + } + oct->num_irqs = msix_allocated; + dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n"); + + return 0; + +enable_msix_err: + if (msix_allocated > 0) + pci_disable_msix(oct->pdev); + kfree(oct->msix_entries); + oct->msix_entries = NULL; +msix_alloc_err: + return -1; +} + +/** + * octep_disable_msix() - disable MSI-x interrupts. + * + * @oct: Octeon device private data structure. + * + * Disable MSI-x on the Octeon device. + */ +static void octep_disable_msix(struct octep_device *oct) +{ + pci_disable_msix(oct->pdev); + kfree(oct->msix_entries); + oct->msix_entries = NULL; + dev_info(&oct->pdev->dev, "Disabled MSI-X\n"); +} + +/** + * octep_non_ioq_intr_handler() - common handler for all generic interrupts. + * + * @irq: Interrupt number. + * @data: interrupt data. + * + * this is common handler for all non-queue (generic) interrupts. + */ +static irqreturn_t octep_non_ioq_intr_handler(int irq, void *data) +{ + struct octep_device *oct = data; + + return oct->hw_ops.non_ioq_intr_handler(oct); +} + +/** + * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts. + * + * @irq: Interrupt number. + * @data: interrupt data contains pointers to Tx/Rx queue private data + * and correspong NAPI context. + * + * this is common handler for all non-queue (generic) interrupts. + */ +static irqreturn_t octep_ioq_intr_handler(int irq, void *data) +{ + struct octep_ioq_vector *ioq_vector = data; + struct octep_device *oct = ioq_vector->octep_dev; + + return oct->hw_ops.ioq_intr_handler(ioq_vector); +} + +/** + * octep_request_irqs() - Register interrupt handlers. + * + * @oct: Octeon device private data structure. + * + * Register handlers for all queue and non-queue interrupts. + * + * Return: 0, on successful registration of all interrupt handlers. + * -1, on any error. + */ +static int octep_request_irqs(struct octep_device *oct) +{ + struct net_device *netdev = oct->netdev; + struct octep_ioq_vector *ioq_vector; + struct msix_entry *msix_entry; + char **non_ioq_msix_names; + int num_non_ioq_msix; + int ret, i, j; + + num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf); + non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf); + + oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix, + OCTEP_MSIX_NAME_SIZE, GFP_KERNEL); + if (!oct->non_ioq_irq_names) + goto alloc_err; + + /* First few MSI-X interrupts are non-queue interrupts */ + for (i = 0; i < num_non_ioq_msix; i++) { + char *irq_name; + + irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE]; + msix_entry = &oct->msix_entries[i]; + + snprintf(irq_name, OCTEP_MSIX_NAME_SIZE, + "%s-%s", netdev->name, non_ioq_msix_names[i]); + ret = request_irq(msix_entry->vector, + octep_non_ioq_intr_handler, 0, + irq_name, oct); + if (ret) { + netdev_err(netdev, + "request_irq failed for %s; err=%d", + irq_name, ret); + goto non_ioq_irq_err; + } + } + + /* Request IRQs for Tx/Rx queues */ + for (j = 0; j < oct->num_oqs; j++) { + ioq_vector = oct->ioq_vector[j]; + msix_entry = &oct->msix_entries[j + num_non_ioq_msix]; + + snprintf(ioq_vector->name, sizeof(ioq_vector->name), + "%s-q%d", netdev->name, j); + ret = request_irq(msix_entry->vector, + octep_ioq_intr_handler, 0, + ioq_vector->name, ioq_vector); + if (ret) { + netdev_err(netdev, + "request_irq failed for Q-%d; err=%d", + j, ret); + goto ioq_irq_err; + } + + cpumask_set_cpu(j % num_online_cpus(), + &ioq_vector->affinity_mask); + irq_set_affinity_hint(msix_entry->vector, + &ioq_vector->affinity_mask); + } + + return 0; +ioq_irq_err: + while (j) { + --j; + ioq_vector = oct->ioq_vector[j]; + msix_entry = &oct->msix_entries[j + num_non_ioq_msix]; + + irq_set_affinity_hint(msix_entry->vector, NULL); + free_irq(msix_entry->vector, ioq_vector); + } +non_ioq_irq_err: + while (i) { + --i; + free_irq(oct->msix_entries[i].vector, oct); + } + kfree(oct->non_ioq_irq_names); + oct->non_ioq_irq_names = NULL; +alloc_err: + return -1; +} + +/** + * octep_free_irqs() - free all registered interrupts. + * + * @oct: Octeon device private data structure. + * + * Free all queue and non-queue interrupts of the Octeon device. + */ +static void octep_free_irqs(struct octep_device *oct) +{ + int i; + + /* First few MSI-X interrupts are non queue interrupts; free them */ + for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++) + free_irq(oct->msix_entries[i].vector, oct); + kfree(oct->non_ioq_irq_names); + + /* Free IRQs for Input/Output (Tx/Rx) queues */ + for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) { + irq_set_affinity_hint(oct->msix_entries[i].vector, NULL); + free_irq(oct->msix_entries[i].vector, + oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]); + } + netdev_info(oct->netdev, "IRQs freed\n"); +} + +/** + * octep_setup_irqs() - setup interrupts for the Octeon device. + * + * @oct: Octeon device private data structure. + * + * Allocate data structures to hold per interrupt information, allocate/enable + * MSI-x interrupt and register interrupt handlers. + * + * Return: 0, on successful allocation and registration of all interrupts. + * -1, on any error. + */ +static int octep_setup_irqs(struct octep_device *oct) +{ + if (octep_alloc_ioq_vectors(oct)) + goto ioq_vector_err; + + if (octep_enable_msix_range(oct)) + goto enable_msix_err; + + if (octep_request_irqs(oct)) + goto request_irq_err; + + return 0; + +request_irq_err: + octep_disable_msix(oct); +enable_msix_err: + octep_free_ioq_vectors(oct); +ioq_vector_err: + return -1; +} + +/** + * octep_clean_irqs() - free all interrupts and its resources. + * + * @oct: Octeon device private data structure. + */ +static void octep_clean_irqs(struct octep_device *oct) +{ + octep_free_irqs(oct); + octep_disable_msix(oct); + octep_free_ioq_vectors(oct); +} + +/** + * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue. + * + * @iq: Octeon Tx queue data structure. + * @oq: Octeon Rx queue data structure. + */ +static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq) +{ + u32 pkts_pend = oq->pkts_pending; + + netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no); + if (iq->pkts_processed) { + writel(iq->pkts_processed, iq->inst_cnt_reg); + iq->pkt_in_done -= iq->pkts_processed; + iq->pkts_processed = 0; + } + if (oq->last_pkt_count - pkts_pend) { + writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg); + oq->last_pkt_count = pkts_pend; + } + + /* Flush the previous wrties before writing to RESEND bit */ + wmb(); + writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg); + writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg); +} + +/** + * octep_napi_poll() - NAPI poll function for Tx/Rx. + * + * @napi: pointer to napi context. + * @budget: max number of packets to be processed in single invocation. + */ +static int octep_napi_poll(struct napi_struct *napi, int budget) +{ + struct octep_ioq_vector *ioq_vector = + container_of(napi, struct octep_ioq_vector, napi); + u32 tx_pending, rx_done; + + tx_pending = octep_iq_process_completions(ioq_vector->iq, budget); + rx_done = octep_oq_process_rx(ioq_vector->oq, budget); + + /* need more polling if tx completion processing is still pending or + * processed at least 'budget' number of rx packets. + */ + if (tx_pending || rx_done >= budget) + return budget; + + napi_complete(napi); + octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq); + return rx_done; +} + +/** + * octep_napi_add() - Add NAPI poll for all Tx/Rx queues. + * + * @oct: Octeon device private data structure. + */ +static void octep_napi_add(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_oqs; i++) { + netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i); + netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, + octep_napi_poll); + oct->oq[i]->napi = &oct->ioq_vector[i]->napi; + } +} + +/** + * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues. + * + * @oct: Octeon device private data structure. + */ +static void octep_napi_delete(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_oqs; i++) { + netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i); + netif_napi_del(&oct->ioq_vector[i]->napi); + oct->oq[i]->napi = NULL; + } +} + +/** + * octep_napi_enable() - enable NAPI for all Tx/Rx queues. + * + * @oct: Octeon device private data structure. + */ +static void octep_napi_enable(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_oqs; i++) { + netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i); + napi_enable(&oct->ioq_vector[i]->napi); + } +} + +/** + * octep_napi_disable() - disable NAPI for all Tx/Rx queues. + * + * @oct: Octeon device private data structure. + */ +static void octep_napi_disable(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_oqs; i++) { + netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i); + napi_disable(&oct->ioq_vector[i]->napi); + } +} + +static void octep_link_up(struct net_device *netdev) +{ + netif_carrier_on(netdev); + netif_tx_start_all_queues(netdev); +} + +/** + * octep_open() - start the octeon network device. + * + * @netdev: pointer to kernel network device. + * + * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues + * and interrupts.. + * + * Return: 0, on successfully setting up device and bring it up. + * -1, on any error. + */ +static int octep_open(struct net_device *netdev) +{ + struct octep_device *oct = netdev_priv(netdev); + int err, ret; + + netdev_info(netdev, "Starting netdev ...\n"); + netif_carrier_off(netdev); + + oct->hw_ops.reset_io_queues(oct); + + if (octep_setup_iqs(oct)) + goto setup_iq_err; + if (octep_setup_oqs(oct)) + goto setup_oq_err; + if (octep_setup_irqs(oct)) + goto setup_irq_err; + + err = netif_set_real_num_tx_queues(netdev, oct->num_oqs); + if (err) + goto set_queues_err; + err = netif_set_real_num_rx_queues(netdev, oct->num_iqs); + if (err) + goto set_queues_err; + + octep_napi_add(oct); + octep_napi_enable(oct); + + oct->link_info.admin_up = 1; + octep_set_rx_state(oct, true); + + ret = octep_get_link_status(oct); + if (!ret) + octep_set_link_status(oct, true); + + /* Enable the input and output queues for this Octeon device */ + oct->hw_ops.enable_io_queues(oct); + + /* Enable Octeon device interrupts */ + oct->hw_ops.enable_interrupts(oct); + + octep_oq_dbell_init(oct); + + ret = octep_get_link_status(oct); + if (ret) + octep_link_up(netdev); + + return 0; + +set_queues_err: + octep_napi_disable(oct); + octep_napi_delete(oct); + octep_clean_irqs(oct); +setup_irq_err: + octep_free_oqs(oct); +setup_oq_err: + octep_free_iqs(oct); +setup_iq_err: + return -1; +} + +/** + * octep_stop() - stop the octeon network device. + * + * @netdev: pointer to kernel network device. + * + * stop the device Tx/Rx operations, bring down the link and + * free up all resources allocated for Tx/Rx queues and interrupts. + */ +static int octep_stop(struct net_device *netdev) +{ + struct octep_device *oct = netdev_priv(netdev); + + netdev_info(netdev, "Stopping the device ...\n"); + + /* Stop Tx from stack */ + netif_tx_stop_all_queues(netdev); + netif_carrier_off(netdev); + netif_tx_disable(netdev); + + octep_set_link_status(oct, false); + octep_set_rx_state(oct, false); + + oct->link_info.admin_up = 0; + oct->link_info.oper_up = 0; + + oct->hw_ops.disable_interrupts(oct); + octep_napi_disable(oct); + octep_napi_delete(oct); + + octep_clean_irqs(oct); + octep_clean_iqs(oct); + + oct->hw_ops.disable_io_queues(oct); + oct->hw_ops.reset_io_queues(oct); + octep_free_oqs(oct); + octep_free_iqs(oct); + netdev_info(netdev, "Device stopped !!\n"); + return 0; +} + +/** + * octep_iq_full_check() - check if a Tx queue is full. + * + * @iq: Octeon Tx queue data structure. + * + * Return: 0, if the Tx queue is not full. + * 1, if the Tx queue is full. + */ +static inline int octep_iq_full_check(struct octep_iq *iq) +{ + if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >= + OCTEP_WAKE_QUEUE_THRESHOLD)) + return 0; + + /* Stop the queue if unable to send */ + netif_stop_subqueue(iq->netdev, iq->q_no); + + /* check again and restart the queue, in case NAPI has just freed + * enough Tx ring entries. + */ + if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >= + OCTEP_WAKE_QUEUE_THRESHOLD)) { + netif_start_subqueue(iq->netdev, iq->q_no); + iq->stats.restart_cnt++; + return 0; + } + + return 1; +} + +/** + * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue. + * + * @skb: packet skbuff pointer. + * @netdev: kernel network device. + * + * Return: NETDEV_TX_BUSY, if Tx Queue is full. + * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue. + */ +static netdev_tx_t octep_start_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + struct octep_device *oct = netdev_priv(netdev); + struct octep_tx_sglist_desc *sglist; + struct octep_tx_buffer *tx_buffer; + struct octep_tx_desc_hw *hw_desc; + struct skb_shared_info *shinfo; + struct octep_instr_hdr *ih; + struct octep_iq *iq; + skb_frag_t *frag; + u16 nr_frags, si; + u16 q_no, wi; + + q_no = skb_get_queue_mapping(skb); + if (q_no >= oct->num_iqs) { + netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no); + q_no = q_no % oct->num_iqs; + } + + iq = oct->iq[q_no]; + if (octep_iq_full_check(iq)) { + iq->stats.tx_busy++; + return NETDEV_TX_BUSY; + } + + shinfo = skb_shinfo(skb); + nr_frags = shinfo->nr_frags; + + wi = iq->host_write_index; + hw_desc = &iq->desc_ring[wi]; + hw_desc->ih64 = 0; + + tx_buffer = iq->buff_info + wi; + tx_buffer->skb = skb; + + ih = &hw_desc->ih; + ih->tlen = skb->len; + ih->pkind = oct->pkind; + + if (!nr_frags) { + tx_buffer->gather = 0; + tx_buffer->dma = dma_map_single(iq->dev, skb->data, + skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(iq->dev, tx_buffer->dma)) + goto dma_map_err; + hw_desc->dptr = tx_buffer->dma; + } else { + /* Scatter/Gather */ + dma_addr_t dma; + u16 len; + + sglist = tx_buffer->sglist; + + ih->gsz = nr_frags + 1; + ih->gather = 1; + tx_buffer->gather = 1; + + len = skb_headlen(skb); + dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE); + if (dma_mapping_error(iq->dev, dma)) + goto dma_map_err; + + dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma, + OCTEP_SGLIST_SIZE_PER_PKT, + DMA_TO_DEVICE); + memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT); + sglist[0].len[3] = len; + sglist[0].dma_ptr[0] = dma; + + si = 1; /* entry 0 is main skb, mapped above */ + frag = &shinfo->frags[0]; + while (nr_frags--) { + len = skb_frag_size(frag); + dma = skb_frag_dma_map(iq->dev, frag, 0, + len, DMA_TO_DEVICE); + if (dma_mapping_error(iq->dev, dma)) + goto dma_map_sg_err; + + sglist[si >> 2].len[3 - (si & 3)] = len; + sglist[si >> 2].dma_ptr[si & 3] = dma; + + frag++; + si++; + } + dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma, + OCTEP_SGLIST_SIZE_PER_PKT, + DMA_TO_DEVICE); + + hw_desc->dptr = tx_buffer->sglist_dma; + } + + /* Flush the hw descriptor before writing to doorbell */ + wmb(); + + /* Ring Doorbell to notify the NIC there is a new packet */ + writel(1, iq->doorbell_reg); + atomic_inc(&iq->instr_pending); + wi++; + if (wi == iq->max_count) + wi = 0; + iq->host_write_index = wi; + + netdev_tx_sent_queue(iq->netdev_q, skb->len); + iq->stats.instr_posted++; + skb_tx_timestamp(skb); + return NETDEV_TX_OK; + +dma_map_sg_err: + if (si > 0) { + dma_unmap_single(iq->dev, sglist[0].dma_ptr[0], + sglist[0].len[0], DMA_TO_DEVICE); + sglist[0].len[0] = 0; + } + while (si > 1) { + dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3], + sglist[si >> 2].len[si & 3], DMA_TO_DEVICE); + sglist[si >> 2].len[si & 3] = 0; + si--; + } + tx_buffer->gather = 0; +dma_map_err: + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +/** + * octep_get_stats64() - Get Octeon network device statistics. + * + * @netdev: kernel network device. + * @stats: pointer to stats structure to be filled in. + */ +static void octep_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) +{ + u64 tx_packets, tx_bytes, rx_packets, rx_bytes; + struct octep_device *oct = netdev_priv(netdev); + int q; + + octep_get_if_stats(oct); + tx_packets = 0; + tx_bytes = 0; + rx_packets = 0; + rx_bytes = 0; + for (q = 0; q < oct->num_oqs; q++) { + struct octep_iq *iq = oct->iq[q]; + struct octep_oq *oq = oct->oq[q]; + + tx_packets += iq->stats.instr_completed; + tx_bytes += iq->stats.bytes_sent; + rx_packets += oq->stats.packets; + rx_bytes += oq->stats.bytes; + } + stats->tx_packets = tx_packets; + stats->tx_bytes = tx_bytes; + stats->rx_packets = rx_packets; + stats->rx_bytes = rx_bytes; + stats->multicast = oct->iface_rx_stats.mcast_pkts; + stats->rx_errors = oct->iface_rx_stats.err_pkts; + stats->collisions = oct->iface_tx_stats.xscol; + stats->tx_fifo_errors = oct->iface_tx_stats.undflw; +} + +/** + * octep_tx_timeout_task - work queue task to Handle Tx queue timeout. + * + * @work: pointer to Tx queue timeout work_struct + * + * Stop and start the device so that it frees up all queue resources + * and restarts the queues, that potentially clears a Tx queue timeout + * condition. + **/ +static void octep_tx_timeout_task(struct work_struct *work) +{ + struct octep_device *oct = container_of(work, struct octep_device, + tx_timeout_task); + struct net_device *netdev = oct->netdev; + + rtnl_lock(); + if (netif_running(netdev)) { + octep_stop(netdev); + octep_open(netdev); + } + rtnl_unlock(); +} + +/** + * octep_tx_timeout() - Handle Tx Queue timeout. + * + * @netdev: pointer to kernel network device. + * @txqueue: Timed out Tx queue number. + * + * Schedule a work to handle Tx queue timeout. + */ +static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue) +{ + struct octep_device *oct = netdev_priv(netdev); + + queue_work(octep_wq, &oct->tx_timeout_task); +} + +static int octep_set_mac(struct net_device *netdev, void *p) +{ + struct octep_device *oct = netdev_priv(netdev); + struct sockaddr *addr = (struct sockaddr *)p; + int err; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + err = octep_set_mac_addr(oct, addr->sa_data); + if (err) + return err; + + memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN); + eth_hw_addr_set(netdev, addr->sa_data); + + return 0; +} + +static int octep_change_mtu(struct net_device *netdev, int new_mtu) +{ + struct octep_device *oct = netdev_priv(netdev); + struct octep_iface_link_info *link_info; + int err = 0; + + link_info = &oct->link_info; + if (link_info->mtu == new_mtu) + return 0; + + err = octep_set_mtu(oct, new_mtu); + if (!err) { + oct->link_info.mtu = new_mtu; + netdev->mtu = new_mtu; + } + + return err; +} + +static const struct net_device_ops octep_netdev_ops = { + .ndo_open = octep_open, + .ndo_stop = octep_stop, + .ndo_start_xmit = octep_start_xmit, + .ndo_get_stats64 = octep_get_stats64, + .ndo_tx_timeout = octep_tx_timeout, + .ndo_set_mac_address = octep_set_mac, + .ndo_change_mtu = octep_change_mtu, +}; + +/** + * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages. + * + * @work: pointer to ctrl mbox work_struct + * + * Poll ctrl mbox message queue and handle control messages from firmware. + **/ +static void octep_ctrl_mbox_task(struct work_struct *work) +{ + struct octep_device *oct = container_of(work, struct octep_device, + ctrl_mbox_task); + struct net_device *netdev = oct->netdev; + struct octep_ctrl_net_f2h_req req = {}; + struct octep_ctrl_mbox_msg msg; + int ret = 0; + + msg.msg = &req; + while (true) { + ret = octep_ctrl_mbox_recv(&oct->ctrl_mbox, &msg); + if (ret) + break; + + switch (req.hdr.cmd) { + case OCTEP_CTRL_NET_F2H_CMD_LINK_STATUS: + if (netif_running(netdev)) { + if (req.link.state) { + dev_info(&oct->pdev->dev, "netif_carrier_on\n"); + netif_carrier_on(netdev); + } else { + dev_info(&oct->pdev->dev, "netif_carrier_off\n"); + netif_carrier_off(netdev); + } + } + break; + default: + pr_info("Unknown mbox req : %u\n", req.hdr.cmd); + break; + } + } +} + +/** + * octep_device_setup() - Setup Octeon Device. + * + * @oct: Octeon device private data structure. + * + * Setup Octeon device hardware operations, configuration, etc ... + */ +int octep_device_setup(struct octep_device *oct) +{ + struct octep_ctrl_mbox *ctrl_mbox; + struct pci_dev *pdev = oct->pdev; + int i, ret; + + /* allocate memory for oct->conf */ + oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL); + if (!oct->conf) + return -ENOMEM; + + /* Map BAR regions */ + for (i = 0; i < OCTEP_MMIO_REGIONS; i++) { + oct->mmio[i].hw_addr = + ioremap(pci_resource_start(oct->pdev, i * 2), + pci_resource_len(oct->pdev, i * 2)); + oct->mmio[i].mapped = 1; + } + + oct->chip_id = pdev->device; + oct->rev_id = pdev->revision; + dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device); + + switch (oct->chip_id) { + case OCTEP_PCI_DEVICE_ID_CN93_PF: + dev_info(&pdev->dev, + "Setting up OCTEON CN93XX PF PASS%d.%d\n", + OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct)); + octep_device_setup_cn93_pf(oct); + break; + default: + dev_err(&pdev->dev, + "%s: unsupported device\n", __func__); + goto unsupported_dev; + } + + oct->pkind = CFG_GET_IQ_PKIND(oct->conf); + + /* Initialize control mbox */ + ctrl_mbox = &oct->ctrl_mbox; + ctrl_mbox->barmem = CFG_GET_CTRL_MBOX_MEM_ADDR(oct->conf); + ret = octep_ctrl_mbox_init(ctrl_mbox); + if (ret) { + dev_err(&pdev->dev, "Failed to initialize control mbox\n"); + return -1; + } + oct->ctrl_mbox_ifstats_offset = OCTEP_CTRL_MBOX_SZ(ctrl_mbox->h2fq.elem_sz, + ctrl_mbox->h2fq.elem_cnt, + ctrl_mbox->f2hq.elem_sz, + ctrl_mbox->f2hq.elem_cnt); + + return 0; + +unsupported_dev: + return -1; +} + +/** + * octep_device_cleanup() - Cleanup Octeon Device. + * + * @oct: Octeon device private data structure. + * + * Cleanup Octeon device allocated resources. + */ +static void octep_device_cleanup(struct octep_device *oct) +{ + int i; + + dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n"); + + for (i = 0; i < OCTEP_MAX_VF; i++) { + vfree(oct->mbox[i]); + oct->mbox[i] = NULL; + } + + octep_ctrl_mbox_uninit(&oct->ctrl_mbox); + + oct->hw_ops.soft_reset(oct); + for (i = 0; i < OCTEP_MMIO_REGIONS; i++) { + if (oct->mmio[i].mapped) + iounmap(oct->mmio[i].hw_addr); + } + + kfree(oct->conf); + oct->conf = NULL; +} + +/** + * octep_probe() - Octeon PCI device probe handler. + * + * @pdev: PCI device structure. + * @ent: entry in Octeon PCI device ID table. + * + * Initializes and enables the Octeon PCI device for network operations. + * Initializes Octeon private data structure and registers a network device. + */ +static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct octep_device *octep_dev = NULL; + struct net_device *netdev; + int err; + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "Failed to enable PCI device\n"); + return err; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "Failed to set DMA mask !!\n"); + goto err_dma_mask; + } + + err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME); + if (err) { + dev_err(&pdev->dev, "Failed to map PCI memory regions\n"); + goto err_pci_regions; + } + + pci_enable_pcie_error_reporting(pdev); + pci_set_master(pdev); + + netdev = alloc_etherdev_mq(sizeof(struct octep_device), + OCTEP_MAX_QUEUES); + if (!netdev) { + dev_err(&pdev->dev, "Failed to allocate netdev\n"); + err = -ENOMEM; + goto err_alloc_netdev; + } + SET_NETDEV_DEV(netdev, &pdev->dev); + + octep_dev = netdev_priv(netdev); + octep_dev->netdev = netdev; + octep_dev->pdev = pdev; + octep_dev->dev = &pdev->dev; + pci_set_drvdata(pdev, octep_dev); + + err = octep_device_setup(octep_dev); + if (err) { + dev_err(&pdev->dev, "Device setup failed\n"); + goto err_octep_config; + } + INIT_WORK(&octep_dev->tx_timeout_task, octep_tx_timeout_task); + INIT_WORK(&octep_dev->ctrl_mbox_task, octep_ctrl_mbox_task); + + netdev->netdev_ops = &octep_netdev_ops; + octep_set_ethtool_ops(netdev); + netif_carrier_off(netdev); + + netdev->hw_features = NETIF_F_SG; + netdev->features |= netdev->hw_features; + netdev->min_mtu = OCTEP_MIN_MTU; + netdev->max_mtu = OCTEP_MAX_MTU; + netdev->mtu = OCTEP_DEFAULT_MTU; + + octep_get_mac_addr(octep_dev, octep_dev->mac_addr); + eth_hw_addr_set(netdev, octep_dev->mac_addr); + + err = register_netdev(netdev); + if (err) { + dev_err(&pdev->dev, "Failed to register netdev\n"); + goto register_dev_err; + } + dev_info(&pdev->dev, "Device probe successful\n"); + return 0; + +register_dev_err: + octep_device_cleanup(octep_dev); +err_octep_config: + free_netdev(netdev); +err_alloc_netdev: + pci_disable_pcie_error_reporting(pdev); + pci_release_mem_regions(pdev); +err_pci_regions: +err_dma_mask: + pci_disable_device(pdev); + return err; +} + +/** + * octep_remove() - Remove Octeon PCI device from driver control. + * + * @pdev: PCI device structure of the Octeon device. + * + * Cleanup all resources allocated for the Octeon device. + * Unregister from network device and disable the PCI device. + */ +static void octep_remove(struct pci_dev *pdev) +{ + struct octep_device *oct = pci_get_drvdata(pdev); + struct net_device *netdev; + + if (!oct) + return; + + cancel_work_sync(&oct->tx_timeout_task); + cancel_work_sync(&oct->ctrl_mbox_task); + netdev = oct->netdev; + if (netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(netdev); + + octep_device_cleanup(oct); + pci_release_mem_regions(pdev); + free_netdev(netdev); + pci_disable_pcie_error_reporting(pdev); + pci_disable_device(pdev); +} + +static struct pci_driver octep_driver = { + .name = OCTEP_DRV_NAME, + .id_table = octep_pci_id_tbl, + .probe = octep_probe, + .remove = octep_remove, +}; + +/** + * octep_init_module() - Module initialiation. + * + * create common resource for the driver and register PCI driver. + */ +static int __init octep_init_module(void) +{ + int ret; + + pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING); + + /* work queue for all deferred tasks */ + octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME); + if (!octep_wq) { + pr_err("%s: Failed to create common workqueue\n", + OCTEP_DRV_NAME); + return -ENOMEM; + } + + ret = pci_register_driver(&octep_driver); + if (ret < 0) { + pr_err("%s: Failed to register PCI driver; err=%d\n", + OCTEP_DRV_NAME, ret); + destroy_workqueue(octep_wq); + return ret; + } + + pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME); + + return ret; +} + +/** + * octep_exit_module() - Module exit routine. + * + * unregister the driver with PCI subsystem and cleanup common resources. + */ +static void __exit octep_exit_module(void) +{ + pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME); + + pci_unregister_driver(&octep_driver); + destroy_workqueue(octep_wq); + + pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME); +} + +module_init(octep_init_module); +module_exit(octep_exit_module); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h new file mode 100644 index 000000000000..025626a61383 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h @@ -0,0 +1,357 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#ifndef _OCTEP_MAIN_H_ +#define _OCTEP_MAIN_H_ + +#include "octep_tx.h" +#include "octep_rx.h" +#include "octep_ctrl_mbox.h" + +#define OCTEP_DRV_NAME "octeon_ep" +#define OCTEP_DRV_STRING "Marvell Octeon EndPoint NIC Driver" + +#define OCTEP_PCIID_CN93_PF 0xB200177d +#define OCTEP_PCIID_CN93_VF 0xB203177d + +#define OCTEP_PCI_DEVICE_ID_CN93_PF 0xB200 +#define OCTEP_PCI_DEVICE_ID_CN93_VF 0xB203 + +#define OCTEP_MAX_QUEUES 63 +#define OCTEP_MAX_IQ OCTEP_MAX_QUEUES +#define OCTEP_MAX_OQ OCTEP_MAX_QUEUES +#define OCTEP_MAX_VF 64 + +#define OCTEP_MAX_MSIX_VECTORS OCTEP_MAX_OQ + +/* Flags to disable and enable Interrupts */ +#define OCTEP_INPUT_INTR (1) +#define OCTEP_OUTPUT_INTR (2) +#define OCTEP_MBOX_INTR (4) +#define OCTEP_ALL_INTR 0xff + +#define OCTEP_IQ_INTR_RESEND_BIT 59 +#define OCTEP_OQ_INTR_RESEND_BIT 59 + +#define OCTEP_MMIO_REGIONS 3 +/* PCI address space mapping information. + * Each of the 3 address spaces given by BAR0, BAR2 and BAR4 of + * Octeon gets mapped to different physical address spaces in + * the kernel. + */ +struct octep_mmio { + /* The physical address to which the PCI address space is mapped. */ + u8 __iomem *hw_addr; + + /* Flag indicating the mapping was successful. */ + int mapped; +}; + +struct octep_pci_win_regs { + u8 __iomem *pci_win_wr_addr; + u8 __iomem *pci_win_rd_addr; + u8 __iomem *pci_win_wr_data; + u8 __iomem *pci_win_rd_data; +}; + +struct octep_hw_ops { + void (*setup_iq_regs)(struct octep_device *oct, int q); + void (*setup_oq_regs)(struct octep_device *oct, int q); + void (*setup_mbox_regs)(struct octep_device *oct, int mbox); + + irqreturn_t (*non_ioq_intr_handler)(void *ioq_vector); + irqreturn_t (*ioq_intr_handler)(void *ioq_vector); + int (*soft_reset)(struct octep_device *oct); + void (*reinit_regs)(struct octep_device *oct); + u32 (*update_iq_read_idx)(struct octep_iq *iq); + + void (*enable_interrupts)(struct octep_device *oct); + void (*disable_interrupts)(struct octep_device *oct); + + void (*enable_io_queues)(struct octep_device *oct); + void (*disable_io_queues)(struct octep_device *oct); + void (*enable_iq)(struct octep_device *oct, int q); + void (*disable_iq)(struct octep_device *oct, int q); + void (*enable_oq)(struct octep_device *oct, int q); + void (*disable_oq)(struct octep_device *oct, int q); + void (*reset_io_queues)(struct octep_device *oct); + void (*dump_registers)(struct octep_device *oct); +}; + +/* Octeon mailbox data */ +struct octep_mbox_data { + u32 cmd; + u32 total_len; + u32 recv_len; + u32 rsvd; + u64 *data; +}; + +/* Octeon device mailbox */ +struct octep_mbox { + /* A spinlock to protect access to this q_mbox. */ + spinlock_t lock; + + u32 q_no; + u32 state; + + /* SLI_MAC_PF_MBOX_INT for PF, SLI_PKT_MBOX_INT for VF. */ + u8 __iomem *mbox_int_reg; + + /* SLI_PKT_PF_VF_MBOX_SIG(0) for PF, + * SLI_PKT_PF_VF_MBOX_SIG(1) for VF. + */ + u8 __iomem *mbox_write_reg; + + /* SLI_PKT_PF_VF_MBOX_SIG(1) for PF, + * SLI_PKT_PF_VF_MBOX_SIG(0) for VF. + */ + u8 __iomem *mbox_read_reg; + + struct octep_mbox_data mbox_data; +}; + +/* Tx/Rx queue vector per interrupt. */ +struct octep_ioq_vector { + char name[OCTEP_MSIX_NAME_SIZE]; + struct napi_struct napi; + struct octep_device *octep_dev; + struct octep_iq *iq; + struct octep_oq *oq; + cpumask_t affinity_mask; +}; + +/* Octeon hardware/firmware offload capability flags. */ +#define OCTEP_CAP_TX_CHECKSUM BIT(0) +#define OCTEP_CAP_RX_CHECKSUM BIT(1) +#define OCTEP_CAP_TSO BIT(2) + +/* Link modes */ +enum octep_link_mode_bit_indices { + OCTEP_LINK_MODE_10GBASE_T = 0, + OCTEP_LINK_MODE_10GBASE_R, + OCTEP_LINK_MODE_10GBASE_CR, + OCTEP_LINK_MODE_10GBASE_KR, + OCTEP_LINK_MODE_10GBASE_LR, + OCTEP_LINK_MODE_10GBASE_SR, + OCTEP_LINK_MODE_25GBASE_CR, + OCTEP_LINK_MODE_25GBASE_KR, + OCTEP_LINK_MODE_25GBASE_SR, + OCTEP_LINK_MODE_40GBASE_CR4, + OCTEP_LINK_MODE_40GBASE_KR4, + OCTEP_LINK_MODE_40GBASE_LR4, + OCTEP_LINK_MODE_40GBASE_SR4, + OCTEP_LINK_MODE_50GBASE_CR2, + OCTEP_LINK_MODE_50GBASE_KR2, + OCTEP_LINK_MODE_50GBASE_SR2, + OCTEP_LINK_MODE_50GBASE_CR, + OCTEP_LINK_MODE_50GBASE_KR, + OCTEP_LINK_MODE_50GBASE_LR, + OCTEP_LINK_MODE_50GBASE_SR, + OCTEP_LINK_MODE_100GBASE_CR4, + OCTEP_LINK_MODE_100GBASE_KR4, + OCTEP_LINK_MODE_100GBASE_LR4, + OCTEP_LINK_MODE_100GBASE_SR4, + OCTEP_LINK_MODE_NBITS +}; + +/* Hardware interface link state information. */ +struct octep_iface_link_info { + /* Bitmap of Supported link speeds/modes. */ + u64 supported_modes; + + /* Bitmap of Advertised link speeds/modes. */ + u64 advertised_modes; + + /* Negotiated link speed in Mbps. */ + u32 speed; + + /* MTU */ + u16 mtu; + + /* Autonegotation state. */ +#define OCTEP_LINK_MODE_AUTONEG_SUPPORTED BIT(0) +#define OCTEP_LINK_MODE_AUTONEG_ADVERTISED BIT(1) + u8 autoneg; + + /* Pause frames setting. */ +#define OCTEP_LINK_MODE_PAUSE_SUPPORTED BIT(0) +#define OCTEP_LINK_MODE_PAUSE_ADVERTISED BIT(1) + u8 pause; + + /* Admin state of the link (ifconfig <iface> up/down */ + u8 admin_up; + + /* Operational state of the link: physical link is up down */ + u8 oper_up; +}; + +/* The Octeon device specific private data structure. + * Each Octeon device has this structure to represent all its components. + */ +struct octep_device { + struct octep_config *conf; + + /* Octeon Chip type. */ + u16 chip_id; + u16 rev_id; + + /* Device capabilities enabled */ + u64 caps_enabled; + /* Device capabilities supported */ + u64 caps_supported; + + /* Pointer to basic Linux device */ + struct device *dev; + /* Linux PCI device pointer */ + struct pci_dev *pdev; + /* Netdev corresponding to the Octeon device */ + struct net_device *netdev; + + /* memory mapped io range */ + struct octep_mmio mmio[OCTEP_MMIO_REGIONS]; + + /* MAC address */ + u8 mac_addr[ETH_ALEN]; + + /* Tx queues (IQ: Instruction Queue) */ + u16 num_iqs; + /* pkind value to be used in every Tx hardware descriptor */ + u8 pkind; + /* Pointers to Octeon Tx queues */ + struct octep_iq *iq[OCTEP_MAX_IQ]; + + /* Rx queues (OQ: Output Queue) */ + u16 num_oqs; + /* Pointers to Octeon Rx queues */ + struct octep_oq *oq[OCTEP_MAX_OQ]; + + /* Hardware port number of the PCIe interface */ + u16 pcie_port; + + /* PCI Window registers to access some hardware CSRs */ + struct octep_pci_win_regs pci_win_regs; + /* Hardware operations */ + struct octep_hw_ops hw_ops; + + /* IRQ info */ + u16 num_irqs; + u16 num_non_ioq_irqs; + char *non_ioq_irq_names; + struct msix_entry *msix_entries; + /* IOq information of it's corresponding MSI-X interrupt. */ + struct octep_ioq_vector *ioq_vector[OCTEP_MAX_QUEUES]; + + /* Hardware Interface Tx statistics */ + struct octep_iface_tx_stats iface_tx_stats; + /* Hardware Interface Rx statistics */ + struct octep_iface_rx_stats iface_rx_stats; + + /* Hardware Interface Link info like supported modes, aneg support */ + struct octep_iface_link_info link_info; + + /* Mailbox to talk to VFs */ + struct octep_mbox *mbox[OCTEP_MAX_VF]; + + /* Work entry to handle Tx timeout */ + struct work_struct tx_timeout_task; + + /* control mbox over pf */ + struct octep_ctrl_mbox ctrl_mbox; + + /* offset for iface stats */ + u32 ctrl_mbox_ifstats_offset; + + /* Work entry to handle ctrl mbox interrupt */ + struct work_struct ctrl_mbox_task; + +}; + +static inline u16 OCTEP_MAJOR_REV(struct octep_device *oct) +{ + u16 rev = (oct->rev_id & 0xC) >> 2; + + return (rev == 0) ? 1 : rev; +} + +static inline u16 OCTEP_MINOR_REV(struct octep_device *oct) +{ + return (oct->rev_id & 0x3); +} + +/* Octeon CSR read/write access APIs */ +#define octep_write_csr(octep_dev, reg_off, value) \ + writel(value, (octep_dev)->mmio[0].hw_addr + (reg_off)) + +#define octep_write_csr64(octep_dev, reg_off, val64) \ + writeq(val64, (octep_dev)->mmio[0].hw_addr + (reg_off)) + +#define octep_read_csr(octep_dev, reg_off) \ + readl((octep_dev)->mmio[0].hw_addr + (reg_off)) + +#define octep_read_csr64(octep_dev, reg_off) \ + readq((octep_dev)->mmio[0].hw_addr + (reg_off)) + +/* Read windowed register. + * @param oct - pointer to the Octeon device. + * @param addr - Address of the register to read. + * + * This routine is called to read from the indirectly accessed + * Octeon registers that are visible through a PCI BAR0 mapped window + * register. + * @return - 64 bit value read from the register. + */ +static inline u64 +OCTEP_PCI_WIN_READ(struct octep_device *oct, u64 addr) +{ + u64 val64; + + addr |= 1ull << 53; /* read 8 bytes */ + writeq(addr, oct->pci_win_regs.pci_win_rd_addr); + val64 = readq(oct->pci_win_regs.pci_win_rd_data); + + dev_dbg(&oct->pdev->dev, + "%s: reg: 0x%016llx val: 0x%016llx\n", __func__, addr, val64); + + return val64; +} + +/* Write windowed register. + * @param oct - pointer to the Octeon device. + * @param addr - Address of the register to write + * @param val - Value to write + * + * This routine is called to write to the indirectly accessed + * Octeon registers that are visible through a PCI BAR0 mapped window + * register. + * @return Nothing. + */ +static inline void +OCTEP_PCI_WIN_WRITE(struct octep_device *oct, u64 addr, u64 val) +{ + writeq(addr, oct->pci_win_regs.pci_win_wr_addr); + writeq(val, oct->pci_win_regs.pci_win_wr_data); + + dev_dbg(&oct->pdev->dev, + "%s: reg: 0x%016llx val: 0x%016llx\n", __func__, addr, val); +} + +extern struct workqueue_struct *octep_wq; + +int octep_device_setup(struct octep_device *oct); +int octep_setup_iqs(struct octep_device *oct); +void octep_free_iqs(struct octep_device *oct); +void octep_clean_iqs(struct octep_device *oct); +int octep_setup_oqs(struct octep_device *oct); +void octep_free_oqs(struct octep_device *oct); +void octep_oq_dbell_init(struct octep_device *oct); +void octep_device_setup_cn93_pf(struct octep_device *oct); +int octep_iq_process_completions(struct octep_iq *iq, u16 budget); +int octep_oq_process_rx(struct octep_oq *oq, int budget); +void octep_set_ethtool_ops(struct net_device *netdev); + +#endif /* _OCTEP_MAIN_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h new file mode 100644 index 000000000000..3d5d39a52fe6 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cn9k_pf.h @@ -0,0 +1,367 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#ifndef _OCTEP_REGS_CN9K_PF_H_ +#define _OCTEP_REGS_CN9K_PF_H_ + +/* ############################ RST ######################### */ +#define CN93_RST_BOOT 0x000087E006001600ULL +#define CN93_RST_CORE_DOMAIN_W1S 0x000087E006001820ULL +#define CN93_RST_CORE_DOMAIN_W1C 0x000087E006001828ULL + +#define CN93_CONFIG_XPANSION_BAR 0x38 +#define CN93_CONFIG_PCIE_CAP 0x70 +#define CN93_CONFIG_PCIE_DEVCAP 0x74 +#define CN93_CONFIG_PCIE_DEVCTL 0x78 +#define CN93_CONFIG_PCIE_LINKCAP 0x7C +#define CN93_CONFIG_PCIE_LINKCTL 0x80 +#define CN93_CONFIG_PCIE_SLOTCAP 0x84 +#define CN93_CONFIG_PCIE_SLOTCTL 0x88 + +#define CN93_PCIE_SRIOV_FDL 0x188 /* 0x98 */ +#define CN93_PCIE_SRIOV_FDL_BIT_POS 0x10 +#define CN93_PCIE_SRIOV_FDL_MASK 0xFF + +#define CN93_CONFIG_PCIE_FLTMSK 0x720 + +/* ################# Offsets of RING, EPF, MAC ######################### */ +#define CN93_RING_OFFSET (0x1ULL << 17) +#define CN93_EPF_OFFSET (0x1ULL << 25) +#define CN93_MAC_OFFSET (0x1ULL << 4) +#define CN93_BIT_ARRAY_OFFSET (0x1ULL << 4) +#define CN93_EPVF_RING_OFFSET (0x1ULL << 4) + +/* ################# Scratch Registers ######################### */ +#define CN93_SDP_EPF_SCRATCH 0x205E0 + +/* ################# Window Registers ######################### */ +#define CN93_SDP_WIN_WR_ADDR64 0x20000 +#define CN93_SDP_WIN_RD_ADDR64 0x20010 +#define CN93_SDP_WIN_WR_DATA64 0x20020 +#define CN93_SDP_WIN_WR_MASK_REG 0x20030 +#define CN93_SDP_WIN_RD_DATA64 0x20040 + +#define CN93_SDP_MAC_NUMBER 0x2C100 + +/* ################# Global Previliged registers ######################### */ +#define CN93_SDP_EPF_RINFO 0x205F0 + +#define CN93_SDP_EPF_RINFO_SRN(val) ((val) & 0xFF) +#define CN93_SDP_EPF_RINFO_RPVF(val) (((val) >> 32) & 0xF) +#define CN93_SDP_EPF_RINFO_NVFS(val) (((val) >> 48) & 0xFF) + +/* SDP Function select */ +#define CN93_SDP_FUNC_SEL_EPF_BIT_POS 8 +#define CN93_SDP_FUNC_SEL_FUNC_BIT_POS 0 + +/* ##### RING IN (Into device from PCI: Tx Ring) REGISTERS #### */ +#define CN93_SDP_R_IN_CONTROL_START 0x10000 +#define CN93_SDP_R_IN_ENABLE_START 0x10010 +#define CN93_SDP_R_IN_INSTR_BADDR_START 0x10020 +#define CN93_SDP_R_IN_INSTR_RSIZE_START 0x10030 +#define CN93_SDP_R_IN_INSTR_DBELL_START 0x10040 +#define CN93_SDP_R_IN_CNTS_START 0x10050 +#define CN93_SDP_R_IN_INT_LEVELS_START 0x10060 +#define CN93_SDP_R_IN_PKT_CNT_START 0x10080 +#define CN93_SDP_R_IN_BYTE_CNT_START 0x10090 + +#define CN93_SDP_R_IN_CONTROL(ring) \ + (CN93_SDP_R_IN_CONTROL_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_ENABLE(ring) \ + (CN93_SDP_R_IN_ENABLE_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_INSTR_BADDR(ring) \ + (CN93_SDP_R_IN_INSTR_BADDR_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_INSTR_RSIZE(ring) \ + (CN93_SDP_R_IN_INSTR_RSIZE_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_INSTR_DBELL(ring) \ + (CN93_SDP_R_IN_INSTR_DBELL_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_CNTS(ring) \ + (CN93_SDP_R_IN_CNTS_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_INT_LEVELS(ring) \ + (CN93_SDP_R_IN_INT_LEVELS_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_PKT_CNT(ring) \ + (CN93_SDP_R_IN_PKT_CNT_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_BYTE_CNT(ring) \ + (CN93_SDP_R_IN_BYTE_CNT_START + ((ring) * CN93_RING_OFFSET)) + +/* Rings per Virtual Function */ +#define CN93_R_IN_CTL_RPVF_MASK (0xF) +#define CN93_R_IN_CTL_RPVF_POS (48) + +/* Number of instructions to be read in one MAC read request. + * setting to Max value(4) + */ +#define CN93_R_IN_CTL_IDLE (0x1ULL << 28) +#define CN93_R_IN_CTL_RDSIZE (0x3ULL << 25) +#define CN93_R_IN_CTL_IS_64B (0x1ULL << 24) +#define CN93_R_IN_CTL_D_NSR (0x1ULL << 8) +#define CN93_R_IN_CTL_D_ESR (0x1ULL << 6) +#define CN93_R_IN_CTL_D_ROR (0x1ULL << 5) +#define CN93_R_IN_CTL_NSR (0x1ULL << 3) +#define CN93_R_IN_CTL_ESR (0x1ULL << 1) +#define CN93_R_IN_CTL_ROR (0x1ULL << 0) + +#define CN93_R_IN_CTL_MASK (CN93_R_IN_CTL_RDSIZE | CN93_R_IN_CTL_IS_64B) + +/* ##### RING OUT (out from device to PCI host: Rx Ring) REGISTERS #### */ +#define CN93_SDP_R_OUT_CNTS_START 0x10100 +#define CN93_SDP_R_OUT_INT_LEVELS_START 0x10110 +#define CN93_SDP_R_OUT_SLIST_BADDR_START 0x10120 +#define CN93_SDP_R_OUT_SLIST_RSIZE_START 0x10130 +#define CN93_SDP_R_OUT_SLIST_DBELL_START 0x10140 +#define CN93_SDP_R_OUT_CONTROL_START 0x10150 +#define CN93_SDP_R_OUT_ENABLE_START 0x10160 +#define CN93_SDP_R_OUT_PKT_CNT_START 0x10180 +#define CN93_SDP_R_OUT_BYTE_CNT_START 0x10190 + +#define CN93_SDP_R_OUT_CONTROL(ring) \ + (CN93_SDP_R_OUT_CONTROL_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_ENABLE(ring) \ + (CN93_SDP_R_OUT_ENABLE_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_SLIST_BADDR(ring) \ + (CN93_SDP_R_OUT_SLIST_BADDR_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_SLIST_RSIZE(ring) \ + (CN93_SDP_R_OUT_SLIST_RSIZE_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_SLIST_DBELL(ring) \ + (CN93_SDP_R_OUT_SLIST_DBELL_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_CNTS(ring) \ + (CN93_SDP_R_OUT_CNTS_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_INT_LEVELS(ring) \ + (CN93_SDP_R_OUT_INT_LEVELS_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_PKT_CNT(ring) \ + (CN93_SDP_R_OUT_PKT_CNT_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_BYTE_CNT(ring) \ + (CN93_SDP_R_OUT_BYTE_CNT_START + ((ring) * CN93_RING_OFFSET)) + +/*------------------ R_OUT Masks ----------------*/ +#define CN93_R_OUT_INT_LEVELS_BMODE BIT_ULL(63) +#define CN93_R_OUT_INT_LEVELS_TIMET (32) + +#define CN93_R_OUT_CTL_IDLE BIT_ULL(40) +#define CN93_R_OUT_CTL_ES_I BIT_ULL(34) +#define CN93_R_OUT_CTL_NSR_I BIT_ULL(33) +#define CN93_R_OUT_CTL_ROR_I BIT_ULL(32) +#define CN93_R_OUT_CTL_ES_D BIT_ULL(30) +#define CN93_R_OUT_CTL_NSR_D BIT_ULL(29) +#define CN93_R_OUT_CTL_ROR_D BIT_ULL(28) +#define CN93_R_OUT_CTL_ES_P BIT_ULL(26) +#define CN93_R_OUT_CTL_NSR_P BIT_ULL(25) +#define CN93_R_OUT_CTL_ROR_P BIT_ULL(24) +#define CN93_R_OUT_CTL_IMODE BIT_ULL(23) + +/* ############### Interrupt Moderation Registers ############### */ +#define CN93_SDP_R_IN_INT_MDRT_CTL0_START 0x10280 +#define CN93_SDP_R_IN_INT_MDRT_CTL1_START 0x102A0 +#define CN93_SDP_R_IN_INT_MDRT_DBG_START 0x102C0 + +#define CN93_SDP_R_OUT_INT_MDRT_CTL0_START 0x10380 +#define CN93_SDP_R_OUT_INT_MDRT_CTL1_START 0x103A0 +#define CN93_SDP_R_OUT_INT_MDRT_DBG_START 0x103C0 + +#define CN93_SDP_R_IN_INT_MDRT_CTL0(ring) \ + (CN93_SDP_R_IN_INT_MDRT_CTL0_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_INT_MDRT_CTL1(ring) \ + (CN93_SDP_R_IN_INT_MDRT_CTL1_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_INT_MDRT_DBG(ring) \ + (CN93_SDP_R_IN_INT_MDRT_DBG_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_INT_MDRT_CTL0(ring) \ + (CN93_SDP_R_OUT_INT_MDRT_CTL0_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_INT_MDRT_CTL1(ring) \ + (CN93_SDP_R_OUT_INT_MDRT_CTL1_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_INT_MDRT_DBG(ring) \ + (CN93_SDP_R_OUT_INT_MDRT_DBG_START + ((ring) * CN93_RING_OFFSET)) + +/* ##################### Mail Box Registers ########################## */ +/* INT register for VF. when a MBOX write from PF happed to a VF, + * corresponding bit will be set in this register as well as in + * PF_VF_INT register. + * + * This is a RO register, the int can be cleared by writing 1 to PF_VF_INT + */ +/* Basically first 3 are from PF to VF. The last one is data from VF to PF */ +#define CN93_SDP_R_MBOX_PF_VF_DATA_START 0x10210 +#define CN93_SDP_R_MBOX_PF_VF_INT_START 0x10220 +#define CN93_SDP_R_MBOX_VF_PF_DATA_START 0x10230 + +#define CN93_SDP_R_MBOX_PF_VF_DATA(ring) \ + (CN93_SDP_R_MBOX_PF_VF_DATA_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_MBOX_PF_VF_INT(ring) \ + (CN93_SDP_R_MBOX_PF_VF_INT_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_MBOX_VF_PF_DATA(ring) \ + (CN93_SDP_R_MBOX_VF_PF_DATA_START + ((ring) * CN93_RING_OFFSET)) + +/* ##################### Interrupt Registers ########################## */ +#define CN93_SDP_R_ERR_TYPE_START 0x10400 + +#define CN93_SDP_R_ERR_TYPE(ring) \ + (CN93_SDP_R_ERR_TYPE_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_MBOX_ISM_START 0x10500 +#define CN93_SDP_R_OUT_CNTS_ISM_START 0x10510 +#define CN93_SDP_R_IN_CNTS_ISM_START 0x10520 + +#define CN93_SDP_R_MBOX_ISM(ring) \ + (CN93_SDP_R_MBOX_ISM_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_OUT_CNTS_ISM(ring) \ + (CN93_SDP_R_OUT_CNTS_ISM_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_R_IN_CNTS_ISM(ring) \ + (CN93_SDP_R_IN_CNTS_ISM_START + ((ring) * CN93_RING_OFFSET)) + +#define CN93_SDP_EPF_MBOX_RINT_START 0x20100 +#define CN93_SDP_EPF_MBOX_RINT_W1S_START 0x20120 +#define CN93_SDP_EPF_MBOX_RINT_ENA_W1C_START 0x20140 +#define CN93_SDP_EPF_MBOX_RINT_ENA_W1S_START 0x20160 + +#define CN93_SDP_EPF_VFIRE_RINT_START 0x20180 +#define CN93_SDP_EPF_VFIRE_RINT_W1S_START 0x201A0 +#define CN93_SDP_EPF_VFIRE_RINT_ENA_W1C_START 0x201C0 +#define CN93_SDP_EPF_VFIRE_RINT_ENA_W1S_START 0x201E0 + +#define CN93_SDP_EPF_IRERR_RINT 0x20200 +#define CN93_SDP_EPF_IRERR_RINT_W1S 0x20210 +#define CN93_SDP_EPF_IRERR_RINT_ENA_W1C 0x20220 +#define CN93_SDP_EPF_IRERR_RINT_ENA_W1S 0x20230 + +#define CN93_SDP_EPF_VFORE_RINT_START 0x20240 +#define CN93_SDP_EPF_VFORE_RINT_W1S_START 0x20260 +#define CN93_SDP_EPF_VFORE_RINT_ENA_W1C_START 0x20280 +#define CN93_SDP_EPF_VFORE_RINT_ENA_W1S_START 0x202A0 + +#define CN93_SDP_EPF_ORERR_RINT 0x20320 +#define CN93_SDP_EPF_ORERR_RINT_W1S 0x20330 +#define CN93_SDP_EPF_ORERR_RINT_ENA_W1C 0x20340 +#define CN93_SDP_EPF_ORERR_RINT_ENA_W1S 0x20350 + +#define CN93_SDP_EPF_OEI_RINT 0x20360 +#define CN93_SDP_EPF_OEI_RINT_W1S 0x20370 +#define CN93_SDP_EPF_OEI_RINT_ENA_W1C 0x20380 +#define CN93_SDP_EPF_OEI_RINT_ENA_W1S 0x20390 + +#define CN93_SDP_EPF_DMA_RINT 0x20400 +#define CN93_SDP_EPF_DMA_RINT_W1S 0x20410 +#define CN93_SDP_EPF_DMA_RINT_ENA_W1C 0x20420 +#define CN93_SDP_EPF_DMA_RINT_ENA_W1S 0x20430 + +#define CN93_SDP_EPF_DMA_INT_LEVEL_START 0x20440 +#define CN93_SDP_EPF_DMA_CNT_START 0x20460 +#define CN93_SDP_EPF_DMA_TIM_START 0x20480 + +#define CN93_SDP_EPF_MISC_RINT 0x204A0 +#define CN93_SDP_EPF_MISC_RINT_W1S 0x204B0 +#define CN93_SDP_EPF_MISC_RINT_ENA_W1C 0x204C0 +#define CN93_SDP_EPF_MISC_RINT_ENA_W1S 0x204D0 + +#define CN93_SDP_EPF_DMA_VF_RINT_START 0x204E0 +#define CN93_SDP_EPF_DMA_VF_RINT_W1S_START 0x20500 +#define CN93_SDP_EPF_DMA_VF_RINT_ENA_W1C_START 0x20520 +#define CN93_SDP_EPF_DMA_VF_RINT_ENA_W1S_START 0x20540 + +#define CN93_SDP_EPF_PP_VF_RINT_START 0x20560 +#define CN93_SDP_EPF_PP_VF_RINT_W1S_START 0x20580 +#define CN93_SDP_EPF_PP_VF_RINT_ENA_W1C_START 0x205A0 +#define CN93_SDP_EPF_PP_VF_RINT_ENA_W1S_START 0x205C0 + +#define CN93_SDP_EPF_MBOX_RINT(index) \ + (CN93_SDP_EPF_MBOX_RINT_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_MBOX_RINT_W1S(index) \ + (CN93_SDP_EPF_MBOX_RINT_W1S_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_MBOX_RINT_ENA_W1C(index) \ + (CN93_SDP_EPF_MBOX_RINT_ENA_W1C_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_MBOX_RINT_ENA_W1S(index) \ + (CN93_SDP_EPF_MBOX_RINT_ENA_W1S_START + ((index) * CN93_BIT_ARRAY_OFFSET)) + +#define CN93_SDP_EPF_VFIRE_RINT(index) \ + (CN93_SDP_EPF_VFIRE_RINT_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_VFIRE_RINT_W1S(index) \ + (CN93_SDP_EPF_VFIRE_RINT_W1S_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_VFIRE_RINT_ENA_W1C(index) \ + (CN93_SDP_EPF_VFIRE_RINT_ENA_W1C_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_VFIRE_RINT_ENA_W1S(index) \ + (CN93_SDP_EPF_VFIRE_RINT_ENA_W1S_START + ((index) * CN93_BIT_ARRAY_OFFSET)) + +#define CN93_SDP_EPF_VFORE_RINT(index) \ + (CN93_SDP_EPF_VFORE_RINT_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_VFORE_RINT_W1S(index) \ + (CN93_SDP_EPF_VFORE_RINT_W1S_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_VFORE_RINT_ENA_W1C(index) \ + (CN93_SDP_EPF_VFORE_RINT_ENA_W1C_START + ((index) * CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_VFORE_RINT_ENA_W1S(index) \ + (CN93_SDP_EPF_VFORE_RINT_ENA_W1S_START + ((index) * CN93_BIT_ARRAY_OFFSET)) + +#define CN93_SDP_EPF_DMA_VF_RINT(index) \ + (CN93_SDP_EPF_DMA_VF_RINT_START + ((index) + CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_DMA_VF_RINT_W1S(index) \ + (CN93_SDP_EPF_DMA_VF_RINT_W1S_START + ((index) + CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_DMA_VF_RINT_ENA_W1C(index) \ + (CN93_SDP_EPF_DMA_VF_RINT_ENA_W1C_START + ((index) + CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_DMA_VF_RINT_ENA_W1S(index) \ + (CN93_SDP_EPF_DMA_VF_RINT_ENA_W1S_START + ((index) + CN93_BIT_ARRAY_OFFSET)) + +#define CN93_SDP_EPF_PP_VF_RINT(index) \ + (CN93_SDP_EPF_PP_VF_RINT_START + ((index) + CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_PP_VF_RINT_W1S(index) \ + (CN93_SDP_EPF_PP_VF_RINT_W1S_START + ((index) + CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_PP_VF_RINT_ENA_W1C(index) \ + (CN93_SDP_EPF_PP_VF_RINT_ENA_W1C_START + ((index) + CN93_BIT_ARRAY_OFFSET)) +#define CN93_SDP_EPF_PP_VF_RINT_ENA_W1S(index) \ + (CN93_SDP_EPF_PP_VF_RINT_ENA_W1S_START + ((index) + CN93_BIT_ARRAY_OFFSET)) + +/*------------------ Interrupt Masks ----------------*/ +#define CN93_INTR_R_SEND_ISM BIT_ULL(63) +#define CN93_INTR_R_OUT_INT BIT_ULL(62) +#define CN93_INTR_R_IN_INT BIT_ULL(61) +#define CN93_INTR_R_MBOX_INT BIT_ULL(60) +#define CN93_INTR_R_RESEND BIT_ULL(59) +#define CN93_INTR_R_CLR_TIM BIT_ULL(58) + +/* ####################### Ring Mapping Registers ################################## */ +#define CN93_SDP_EPVF_RING_START 0x26000 +#define CN93_SDP_IN_RING_TB_MAP_START 0x28000 +#define CN93_SDP_IN_RATE_LIMIT_START 0x2A000 +#define CN93_SDP_MAC_PF_RING_CTL_START 0x2C000 + +#define CN93_SDP_EPVF_RING(ring) \ + (CN93_SDP_EPVF_RING_START + ((ring) * CN93_EPVF_RING_OFFSET)) +#define CN93_SDP_IN_RING_TB_MAP(ring) \ + (CN93_SDP_N_RING_TB_MAP_START + ((ring) * CN93_EPVF_RING_OFFSET)) +#define CN93_SDP_IN_RATE_LIMIT(ring) \ + (CN93_SDP_IN_RATE_LIMIT_START + ((ring) * CN93_EPVF_RING_OFFSET)) +#define CN93_SDP_MAC_PF_RING_CTL(mac) \ + (CN93_SDP_MAC_PF_RING_CTL_START + ((mac) * CN93_MAC_OFFSET)) + +#define CN93_SDP_MAC_PF_RING_CTL_NPFS(val) ((val) & 0xF) +#define CN93_SDP_MAC_PF_RING_CTL_SRN(val) (((val) >> 8) & 0xFF) +#define CN93_SDP_MAC_PF_RING_CTL_RPPF(val) (((val) >> 16) & 0x3F) + +/* Number of non-queue interrupts in CN93xx */ +#define CN93_NUM_NON_IOQ_INTR 16 +#endif /* _OCTEP_REGS_CN9K_PF_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c new file mode 100644 index 000000000000..392d9b0da0d7 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#include <linux/pci.h> +#include <linux/etherdevice.h> +#include <linux/vmalloc.h> + +#include "octep_config.h" +#include "octep_main.h" + +static void octep_oq_reset_indices(struct octep_oq *oq) +{ + oq->host_read_idx = 0; + oq->host_refill_idx = 0; + oq->refill_count = 0; + oq->last_pkt_count = 0; + oq->pkts_pending = 0; +} + +/** + * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring. + * + * @oq: Octeon Rx queue data structure. + * + * Return: 0, if successfully filled receive buffers for all descriptors. + * -1, if failed to allocate a buffer or failed to map for DMA. + */ +static int octep_oq_fill_ring_buffers(struct octep_oq *oq) +{ + struct octep_oq_desc_hw *desc_ring = oq->desc_ring; + struct page *page; + u32 i; + + for (i = 0; i < oq->max_count; i++) { + page = dev_alloc_page(); + if (unlikely(!page)) { + dev_err(oq->dev, "Rx buffer alloc failed\n"); + goto rx_buf_alloc_err; + } + desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0, + PAGE_SIZE, + DMA_FROM_DEVICE); + if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) { + dev_err(oq->dev, + "OQ-%d buffer alloc: DMA mapping error!\n", + oq->q_no); + put_page(page); + goto dma_map_err; + } + oq->buff_info[i].page = page; + } + + return 0; + +dma_map_err: +rx_buf_alloc_err: + while (i) { + i--; + dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE); + put_page(oq->buff_info[i].page); + oq->buff_info[i].page = NULL; + } + + return -1; +} + +/** + * octep_oq_refill() - refill buffers for used Rx ring descriptors. + * + * @oct: Octeon device private data structure. + * @oq: Octeon Rx queue data structure. + * + * Return: number of descriptors successfully refilled with receive buffers. + */ +static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq) +{ + struct octep_oq_desc_hw *desc_ring = oq->desc_ring; + struct page *page; + u32 refill_idx, i; + + refill_idx = oq->host_refill_idx; + for (i = 0; i < oq->refill_count; i++) { + page = dev_alloc_page(); + if (unlikely(!page)) { + dev_err(oq->dev, "refill: rx buffer alloc failed\n"); + oq->stats.alloc_failures++; + break; + } + + desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0, + PAGE_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) { + dev_err(oq->dev, + "OQ-%d buffer refill: DMA mapping error!\n", + oq->q_no); + put_page(page); + oq->stats.alloc_failures++; + break; + } + oq->buff_info[refill_idx].page = page; + refill_idx++; + if (refill_idx == oq->max_count) + refill_idx = 0; + } + oq->host_refill_idx = refill_idx; + oq->refill_count -= i; + + return i; +} + +/** + * octep_setup_oq() - Setup a Rx queue. + * + * @oct: Octeon device private data structure. + * @q_no: Rx queue number to be setup. + * + * Allocate resources for a Rx queue. + */ +static int octep_setup_oq(struct octep_device *oct, int q_no) +{ + struct octep_oq *oq; + u32 desc_ring_size; + + oq = vzalloc(sizeof(*oq)); + if (!oq) + goto create_oq_fail; + oct->oq[q_no] = oq; + + oq->octep_dev = oct; + oq->netdev = oct->netdev; + oq->dev = &oct->pdev->dev; + oq->q_no = q_no; + oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf); + oq->ring_size_mask = oq->max_count - 1; + oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf); + oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE; + + /* When the hardware/firmware supports additional capabilities, + * additional header is filled-in by Octeon after length field in + * Rx packets. this header contains additional packet information. + */ + if (oct->caps_enabled) + oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE; + + oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf); + + desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE; + oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size, + &oq->desc_ring_dma, GFP_KERNEL); + + if (unlikely(!oq->desc_ring)) { + dev_err(oq->dev, + "Failed to allocate DMA memory for OQ-%d !!\n", q_no); + goto desc_dma_alloc_err; + } + + oq->buff_info = vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE); + if (unlikely(!oq->buff_info)) { + dev_err(&oct->pdev->dev, + "Failed to allocate buffer info for OQ-%d\n", q_no); + goto buf_list_err; + } + + if (octep_oq_fill_ring_buffers(oq)) + goto oq_fill_buff_err; + + octep_oq_reset_indices(oq); + oct->hw_ops.setup_oq_regs(oct, q_no); + oct->num_oqs++; + + return 0; + +oq_fill_buff_err: + vfree(oq->buff_info); + oq->buff_info = NULL; +buf_list_err: + dma_free_coherent(oq->dev, desc_ring_size, + oq->desc_ring, oq->desc_ring_dma); + oq->desc_ring = NULL; +desc_dma_alloc_err: + vfree(oq); + oct->oq[q_no] = NULL; +create_oq_fail: + return -1; +} + +/** + * octep_oq_free_ring_buffers() - Free ring buffers. + * + * @oq: Octeon Rx queue data structure. + * + * Free receive buffers in unused Rx queue descriptors. + */ +static void octep_oq_free_ring_buffers(struct octep_oq *oq) +{ + struct octep_oq_desc_hw *desc_ring = oq->desc_ring; + int i; + + if (!oq->desc_ring || !oq->buff_info) + return; + + for (i = 0; i < oq->max_count; i++) { + if (oq->buff_info[i].page) { + dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, + PAGE_SIZE, DMA_FROM_DEVICE); + put_page(oq->buff_info[i].page); + oq->buff_info[i].page = NULL; + desc_ring[i].buffer_ptr = 0; + } + } + octep_oq_reset_indices(oq); +} + +/** + * octep_free_oq() - Free Rx queue resources. + * + * @oq: Octeon Rx queue data structure. + * + * Free all resources of a Rx queue. + */ +static int octep_free_oq(struct octep_oq *oq) +{ + struct octep_device *oct = oq->octep_dev; + int q_no = oq->q_no; + + octep_oq_free_ring_buffers(oq); + + vfree(oq->buff_info); + + if (oq->desc_ring) + dma_free_coherent(oq->dev, + oq->max_count * OCTEP_OQ_DESC_SIZE, + oq->desc_ring, oq->desc_ring_dma); + + vfree(oq); + oct->oq[q_no] = NULL; + oct->num_oqs--; + return 0; +} + +/** + * octep_setup_oqs() - setup resources for all Rx queues. + * + * @oct: Octeon device private data structure. + */ +int octep_setup_oqs(struct octep_device *oct) +{ + int i, retval = 0; + + oct->num_oqs = 0; + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { + retval = octep_setup_oq(oct, i); + if (retval) { + dev_err(&oct->pdev->dev, + "Failed to setup OQ(RxQ)-%d.\n", i); + goto oq_setup_err; + } + dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i); + } + + return 0; + +oq_setup_err: + while (i) { + i--; + octep_free_oq(oct->oq[i]); + } + return -1; +} + +/** + * octep_oq_dbell_init() - Initialize Rx queue doorbell. + * + * @oct: Octeon device private data structure. + * + * Write number of descriptors to Rx queue doorbell register. + */ +void octep_oq_dbell_init(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_oqs; i++) + writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg); +} + +/** + * octep_free_oqs() - Free resources of all Rx queues. + * + * @oct: Octeon device private data structure. + */ +void octep_free_oqs(struct octep_device *oct) +{ + int i; + + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { + if (!oct->oq[i]) + continue; + octep_free_oq(oct->oq[i]); + dev_dbg(&oct->pdev->dev, + "Successfully freed OQ(RxQ)-%d.\n", i); + } +} + +/** + * octep_oq_check_hw_for_pkts() - Check for new Rx packets. + * + * @oct: Octeon device private data structure. + * @oq: Octeon Rx queue data structure. + * + * Return: packets received after previous check. + */ +static int octep_oq_check_hw_for_pkts(struct octep_device *oct, + struct octep_oq *oq) +{ + u32 pkt_count, new_pkts; + + pkt_count = readl(oq->pkts_sent_reg); + new_pkts = pkt_count - oq->last_pkt_count; + + /* Clear the hardware packets counter register if the rx queue is + * being processed continuously with-in a single interrupt and + * reached half its max value. + * this counter is not cleared every time read, to save write cycles. + */ + if (unlikely(pkt_count > 0xF0000000U)) { + writel(pkt_count, oq->pkts_sent_reg); + pkt_count = readl(oq->pkts_sent_reg); + new_pkts += pkt_count; + } + oq->last_pkt_count = pkt_count; + oq->pkts_pending += new_pkts; + return new_pkts; +} + +/** + * __octep_oq_process_rx() - Process hardware Rx queue and push to stack. + * + * @oct: Octeon device private data structure. + * @oq: Octeon Rx queue data structure. + * @pkts_to_process: number of packets to be processed. + * + * Process the new packets in Rx queue. + * Packets larger than single Rx buffer arrive in consecutive descriptors. + * But, count returned by the API only accounts full packets, not fragments. + * + * Return: number of packets processed and pushed to stack. + */ +static int __octep_oq_process_rx(struct octep_device *oct, + struct octep_oq *oq, u16 pkts_to_process) +{ + struct octep_oq_resp_hw_ext *resp_hw_ext = NULL; + struct octep_rx_buffer *buff_info; + struct octep_oq_resp_hw *resp_hw; + u32 pkt, rx_bytes, desc_used; + struct sk_buff *skb; + u16 data_offset; + u32 read_idx; + + read_idx = oq->host_read_idx; + rx_bytes = 0; + desc_used = 0; + for (pkt = 0; pkt < pkts_to_process; pkt++) { + buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx]; + dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, + PAGE_SIZE, DMA_FROM_DEVICE); + resp_hw = page_address(buff_info->page); + buff_info->page = NULL; + + /* Swap the length field that is in Big-Endian to CPU */ + buff_info->len = be64_to_cpu(resp_hw->length); + if (oct->caps_enabled & OCTEP_CAP_RX_CHECKSUM) { + /* Extended response header is immediately after + * response header (resp_hw) + */ + resp_hw_ext = (struct octep_oq_resp_hw_ext *) + (resp_hw + 1); + buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE; + /* Packet Data is immediately after + * extended response header. + */ + data_offset = OCTEP_OQ_RESP_HW_SIZE + + OCTEP_OQ_RESP_HW_EXT_SIZE; + } else { + /* Data is immediately after + * Hardware Rx response header. + */ + data_offset = OCTEP_OQ_RESP_HW_SIZE; + } + rx_bytes += buff_info->len; + + if (buff_info->len <= oq->max_single_buffer_size) { + skb = build_skb((void *)resp_hw, PAGE_SIZE); + skb_reserve(skb, data_offset); + skb_put(skb, buff_info->len); + read_idx++; + desc_used++; + if (read_idx == oq->max_count) + read_idx = 0; + } else { + struct skb_shared_info *shinfo; + u16 data_len; + + skb = build_skb((void *)resp_hw, PAGE_SIZE); + skb_reserve(skb, data_offset); + /* Head fragment includes response header(s); + * subsequent fragments contains only data. + */ + skb_put(skb, oq->max_single_buffer_size); + read_idx++; + desc_used++; + if (read_idx == oq->max_count) + read_idx = 0; + + shinfo = skb_shinfo(skb); + data_len = buff_info->len - oq->max_single_buffer_size; + while (data_len) { + dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, + PAGE_SIZE, DMA_FROM_DEVICE); + buff_info = (struct octep_rx_buffer *) + &oq->buff_info[read_idx]; + if (data_len < oq->buffer_size) { + buff_info->len = data_len; + data_len = 0; + } else { + buff_info->len = oq->buffer_size; + data_len -= oq->buffer_size; + } + + skb_add_rx_frag(skb, shinfo->nr_frags, + buff_info->page, 0, + buff_info->len, + buff_info->len); + buff_info->page = NULL; + read_idx++; + desc_used++; + if (read_idx == oq->max_count) + read_idx = 0; + } + } + + skb->dev = oq->netdev; + skb->protocol = eth_type_trans(skb, skb->dev); + if (resp_hw_ext && + resp_hw_ext->csum_verified == OCTEP_CSUM_VERIFIED) + skb->ip_summed = CHECKSUM_UNNECESSARY; + else + skb->ip_summed = CHECKSUM_NONE; + napi_gro_receive(oq->napi, skb); + } + + oq->host_read_idx = read_idx; + oq->refill_count += desc_used; + oq->stats.packets += pkt; + oq->stats.bytes += rx_bytes; + + return pkt; +} + +/** + * octep_oq_process_rx() - Process Rx queue. + * + * @oq: Octeon Rx queue data structure. + * @budget: max number of packets can be processed in one invocation. + * + * Check for newly received packets and process them. + * Keeps checking for new packets until budget is used or no new packets seen. + * + * Return: number of packets processed. + */ +int octep_oq_process_rx(struct octep_oq *oq, int budget) +{ + u32 pkts_available, pkts_processed, total_pkts_processed; + struct octep_device *oct = oq->octep_dev; + + pkts_available = 0; + pkts_processed = 0; + total_pkts_processed = 0; + while (total_pkts_processed < budget) { + /* update pending count only when current one exhausted */ + if (oq->pkts_pending == 0) + octep_oq_check_hw_for_pkts(oct, oq); + pkts_available = min(budget - total_pkts_processed, + oq->pkts_pending); + if (!pkts_available) + break; + + pkts_processed = __octep_oq_process_rx(oct, oq, + pkts_available); + oq->pkts_pending -= pkts_processed; + total_pkts_processed += pkts_processed; + } + + if (oq->refill_count >= oq->refill_threshold) { + u32 desc_refilled = octep_oq_refill(oct, oq); + + /* flush pending writes before updating credits */ + wmb(); + writel(desc_refilled, oq->pkts_credit_reg); + } + + return total_pkts_processed; +} diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.h b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.h new file mode 100644 index 000000000000..782a24f27f3e --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#ifndef _OCTEP_RX_H_ +#define _OCTEP_RX_H_ + +/* struct octep_oq_desc_hw - Octeon Hardware OQ descriptor format. + * + * The descriptor ring is made of descriptors which have 2 64-bit values: + * + * @buffer_ptr: DMA address of the skb->data + * @info_ptr: DMA address of host memory, used to update pkt count by hw. + * This is currently unused to save pci writes. + */ +struct octep_oq_desc_hw { + dma_addr_t buffer_ptr; + u64 info_ptr; +}; + +#define OCTEP_OQ_DESC_SIZE (sizeof(struct octep_oq_desc_hw)) + +#define OCTEP_CSUM_L4_VERIFIED 0x1 +#define OCTEP_CSUM_IP_VERIFIED 0x2 +#define OCTEP_CSUM_VERIFIED (OCTEP_CSUM_L4_VERIFIED | OCTEP_CSUM_IP_VERIFIED) + +/* Extended Response Header in packet data received from Hardware. + * Includes metadata like checksum status. + * this is valid only if hardware/firmware published support for this. + * This is at offset 0 of packet data (skb->data). + */ +struct octep_oq_resp_hw_ext { + /* Reserved. */ + u64 reserved:62; + + /* checksum verified. */ + u64 csum_verified:2; +}; + +#define OCTEP_OQ_RESP_HW_EXT_SIZE (sizeof(struct octep_oq_resp_hw_ext)) + +/* Length of Rx packet DMA'ed by Octeon to Host. + * this is in bigendian; so need to be converted to cpu endian. + * Octeon writes this at the beginning of Rx buffer (skb->data). + */ +struct octep_oq_resp_hw { + /* The Length of the packet. */ + __be64 length; +}; + +#define OCTEP_OQ_RESP_HW_SIZE (sizeof(struct octep_oq_resp_hw)) + +/* Pointer to data buffer. + * Driver keeps a pointer to the data buffer that it made available to + * the Octeon device. Since the descriptor ring keeps physical (bus) + * addresses, this field is required for the driver to keep track of + * the virtual address pointers. The fields are operated by + * OS-dependent routines. + */ +struct octep_rx_buffer { + struct page *page; + + /* length from rx hardware descriptor after converting to cpu endian */ + u64 len; +}; + +#define OCTEP_OQ_RECVBUF_SIZE (sizeof(struct octep_rx_buffer)) + +/* Output Queue statistics. Each output queue has four stats fields. */ +struct octep_oq_stats { + /* Number of packets received from the Device. */ + u64 packets; + + /* Number of bytes received from the Device. */ + u64 bytes; + + /* Number of times failed to allocate buffers. */ + u64 alloc_failures; +}; + +#define OCTEP_OQ_STATS_SIZE (sizeof(struct octep_oq_stats)) + +/* Hardware interface Rx statistics */ +struct octep_iface_rx_stats { + /* Received packets */ + u64 pkts; + + /* Octets of received packets */ + u64 octets; + + /* Received PAUSE and Control packets */ + u64 pause_pkts; + + /* Received PAUSE and Control octets */ + u64 pause_octets; + + /* Filtered DMAC0 packets */ + u64 dmac0_pkts; + + /* Filtered DMAC0 octets */ + u64 dmac0_octets; + + /* Packets dropped due to RX FIFO full */ + u64 dropped_pkts_fifo_full; + + /* Octets dropped due to RX FIFO full */ + u64 dropped_octets_fifo_full; + + /* Error packets */ + u64 err_pkts; + + /* Filtered DMAC1 packets */ + u64 dmac1_pkts; + + /* Filtered DMAC1 octets */ + u64 dmac1_octets; + + /* NCSI-bound packets dropped */ + u64 ncsi_dropped_pkts; + + /* NCSI-bound octets dropped */ + u64 ncsi_dropped_octets; + + /* Multicast packets received. */ + u64 mcast_pkts; + + /* Broadcast packets received. */ + u64 bcast_pkts; + +}; + +/* The Descriptor Ring Output Queue structure. + * This structure has all the information required to implement a + * Octeon OQ. + */ +struct octep_oq { + u32 q_no; + + struct octep_device *octep_dev; + struct net_device *netdev; + struct device *dev; + + struct napi_struct *napi; + + /* The receive buffer list. This list has the virtual addresses + * of the buffers. + */ + struct octep_rx_buffer *buff_info; + + /* Pointer to the mapped packet credit register. + * Host writes number of info/buffer ptrs available to this register + */ + u8 __iomem *pkts_credit_reg; + + /* Pointer to the mapped packet sent register. + * Octeon writes the number of packets DMA'ed to host memory + * in this register. + */ + u8 __iomem *pkts_sent_reg; + + /* Statistics for this OQ. */ + struct octep_oq_stats stats; + + /* Packets pending to be processed */ + u32 pkts_pending; + u32 last_pkt_count; + + /* Index in the ring where the driver should read the next packet */ + u32 host_read_idx; + + /* Number of descriptors in this ring. */ + u32 max_count; + u32 ring_size_mask; + + /* The number of descriptors pending refill. */ + u32 refill_count; + + /* Index in the ring where the driver will refill the + * descriptor's buffer + */ + u32 host_refill_idx; + u32 refill_threshold; + + /* The size of each buffer pointed by the buffer pointer. */ + u32 buffer_size; + u32 max_single_buffer_size; + + /* The 8B aligned descriptor ring starts at this address. */ + struct octep_oq_desc_hw *desc_ring; + + /* DMA mapped address of the OQ descriptor ring. */ + dma_addr_t desc_ring_dma; +}; + +#define OCTEP_OQ_SIZE (sizeof(struct octep_oq)) +#endif /* _OCTEP_RX_H_ */ diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c new file mode 100644 index 000000000000..5a520d37bea0 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#include <linux/pci.h> +#include <linux/etherdevice.h> +#include <linux/vmalloc.h> + +#include "octep_config.h" +#include "octep_main.h" + +/* Reset various index of Tx queue data structure. */ +static void octep_iq_reset_indices(struct octep_iq *iq) +{ + iq->fill_cnt = 0; + iq->host_write_index = 0; + iq->octep_read_index = 0; + iq->flush_index = 0; + iq->pkts_processed = 0; + iq->pkt_in_done = 0; + atomic_set(&iq->instr_pending, 0); +} + +/** + * octep_iq_process_completions() - Process Tx queue completions. + * + * @iq: Octeon Tx queue data structure. + * @budget: max number of completions to be processed in one invocation. + */ +int octep_iq_process_completions(struct octep_iq *iq, u16 budget) +{ + u32 compl_pkts, compl_bytes, compl_sg; + struct octep_device *oct = iq->octep_dev; + struct octep_tx_buffer *tx_buffer; + struct skb_shared_info *shinfo; + u32 fi = iq->flush_index; + struct sk_buff *skb; + u8 frags, i; + + compl_pkts = 0; + compl_sg = 0; + compl_bytes = 0; + iq->octep_read_index = oct->hw_ops.update_iq_read_idx(iq); + + while (likely(budget && (fi != iq->octep_read_index))) { + tx_buffer = iq->buff_info + fi; + skb = tx_buffer->skb; + + fi++; + if (unlikely(fi == iq->max_count)) + fi = 0; + compl_bytes += skb->len; + compl_pkts++; + budget--; + + if (!tx_buffer->gather) { + dma_unmap_single(iq->dev, tx_buffer->dma, + tx_buffer->skb->len, DMA_TO_DEVICE); + dev_kfree_skb_any(skb); + continue; + } + + /* Scatter/Gather */ + shinfo = skb_shinfo(skb); + frags = shinfo->nr_frags; + compl_sg++; + + dma_unmap_single(iq->dev, tx_buffer->sglist[0].dma_ptr[0], + tx_buffer->sglist[0].len[0], DMA_TO_DEVICE); + + i = 1; /* entry 0 is main skb, unmapped above */ + while (frags--) { + dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], + tx_buffer->sglist[i >> 2].len[i & 3], DMA_TO_DEVICE); + i++; + } + + dev_kfree_skb_any(skb); + } + + iq->pkts_processed += compl_pkts; + atomic_sub(compl_pkts, &iq->instr_pending); + iq->stats.instr_completed += compl_pkts; + iq->stats.bytes_sent += compl_bytes; + iq->stats.sgentry_sent += compl_sg; + iq->flush_index = fi; + + netdev_tx_completed_queue(iq->netdev_q, compl_pkts, compl_bytes); + + if (unlikely(__netif_subqueue_stopped(iq->netdev, iq->q_no)) && + ((iq->max_count - atomic_read(&iq->instr_pending)) > + OCTEP_WAKE_QUEUE_THRESHOLD)) + netif_wake_subqueue(iq->netdev, iq->q_no); + return !budget; +} + +/** + * octep_iq_free_pending() - Free Tx buffers for pending completions. + * + * @iq: Octeon Tx queue data structure. + */ +static void octep_iq_free_pending(struct octep_iq *iq) +{ + struct octep_tx_buffer *tx_buffer; + struct skb_shared_info *shinfo; + u32 fi = iq->flush_index; + struct sk_buff *skb; + u8 frags, i; + + while (fi != iq->host_write_index) { + tx_buffer = iq->buff_info + fi; + skb = tx_buffer->skb; + + fi++; + if (unlikely(fi == iq->max_count)) + fi = 0; + + if (!tx_buffer->gather) { + dma_unmap_single(iq->dev, tx_buffer->dma, + tx_buffer->skb->len, DMA_TO_DEVICE); + dev_kfree_skb_any(skb); + continue; + } + + /* Scatter/Gather */ + shinfo = skb_shinfo(skb); + frags = shinfo->nr_frags; + + dma_unmap_single(iq->dev, + tx_buffer->sglist[0].dma_ptr[0], + tx_buffer->sglist[0].len[0], + DMA_TO_DEVICE); + + i = 1; /* entry 0 is main skb, unmapped above */ + while (frags--) { + dma_unmap_page(iq->dev, tx_buffer->sglist[i >> 2].dma_ptr[i & 3], + tx_buffer->sglist[i >> 2].len[i & 3], DMA_TO_DEVICE); + i++; + } + + dev_kfree_skb_any(skb); + } + + atomic_set(&iq->instr_pending, 0); + iq->flush_index = fi; + netdev_tx_reset_queue(netdev_get_tx_queue(iq->netdev, iq->q_no)); +} + +/** + * octep_clean_iqs() - Clean Tx queues to shutdown the device. + * + * @oct: Octeon device private data structure. + * + * Free the buffers in Tx queue descriptors pending completion and + * reset queue indices + */ +void octep_clean_iqs(struct octep_device *oct) +{ + int i; + + for (i = 0; i < oct->num_iqs; i++) { + octep_iq_free_pending(oct->iq[i]); + octep_iq_reset_indices(oct->iq[i]); + } +} + +/** + * octep_setup_iq() - Setup a Tx queue. + * + * @oct: Octeon device private data structure. + * @q_no: Tx queue number to be setup. + * + * Allocate resources for a Tx queue. + */ +static int octep_setup_iq(struct octep_device *oct, int q_no) +{ + u32 desc_ring_size, buff_info_size, sglist_size; + struct octep_iq *iq; + int i; + + iq = vzalloc(sizeof(*iq)); + if (!iq) + goto iq_alloc_err; + oct->iq[q_no] = iq; + + iq->octep_dev = oct; + iq->netdev = oct->netdev; + iq->dev = &oct->pdev->dev; + iq->q_no = q_no; + iq->max_count = CFG_GET_IQ_NUM_DESC(oct->conf); + iq->ring_size_mask = iq->max_count - 1; + iq->fill_threshold = CFG_GET_IQ_DB_MIN(oct->conf); + iq->netdev_q = netdev_get_tx_queue(iq->netdev, q_no); + + /* Allocate memory for hardware queue descriptors */ + desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); + iq->desc_ring = dma_alloc_coherent(iq->dev, desc_ring_size, + &iq->desc_ring_dma, GFP_KERNEL); + if (unlikely(!iq->desc_ring)) { + dev_err(iq->dev, + "Failed to allocate DMA memory for IQ-%d\n", q_no); + goto desc_dma_alloc_err; + } + + /* Allocate memory for hardware SGLIST descriptors */ + sglist_size = OCTEP_SGLIST_SIZE_PER_PKT * + CFG_GET_IQ_NUM_DESC(oct->conf); + iq->sglist = dma_alloc_coherent(iq->dev, sglist_size, + &iq->sglist_dma, GFP_KERNEL); + if (unlikely(!iq->sglist)) { + dev_err(iq->dev, + "Failed to allocate DMA memory for IQ-%d SGLIST\n", + q_no); + goto sglist_alloc_err; + } + + /* allocate memory to manage Tx packets pending completion */ + buff_info_size = OCTEP_IQ_TXBUFF_INFO_SIZE * iq->max_count; + iq->buff_info = vzalloc(buff_info_size); + if (!iq->buff_info) { + dev_err(iq->dev, + "Failed to allocate buff info for IQ-%d\n", q_no); + goto buff_info_err; + } + + /* Setup sglist addresses in tx_buffer entries */ + for (i = 0; i < CFG_GET_IQ_NUM_DESC(oct->conf); i++) { + struct octep_tx_buffer *tx_buffer; + + tx_buffer = &iq->buff_info[i]; + tx_buffer->sglist = + &iq->sglist[i * OCTEP_SGLIST_ENTRIES_PER_PKT]; + tx_buffer->sglist_dma = + iq->sglist_dma + (i * OCTEP_SGLIST_SIZE_PER_PKT); + } + + octep_iq_reset_indices(iq); + oct->hw_ops.setup_iq_regs(oct, q_no); + + oct->num_iqs++; + return 0; + +buff_info_err: + dma_free_coherent(iq->dev, sglist_size, iq->sglist, iq->sglist_dma); +sglist_alloc_err: + dma_free_coherent(iq->dev, desc_ring_size, + iq->desc_ring, iq->desc_ring_dma); +desc_dma_alloc_err: + vfree(iq); + oct->iq[q_no] = NULL; +iq_alloc_err: + return -1; +} + +/** + * octep_free_iq() - Free Tx queue resources. + * + * @iq: Octeon Tx queue data structure. + * + * Free all the resources allocated for a Tx queue. + */ +static void octep_free_iq(struct octep_iq *iq) +{ + struct octep_device *oct = iq->octep_dev; + u64 desc_ring_size, sglist_size; + int q_no = iq->q_no; + + desc_ring_size = OCTEP_IQ_DESC_SIZE * CFG_GET_IQ_NUM_DESC(oct->conf); + + vfree(iq->buff_info); + + if (iq->desc_ring) + dma_free_coherent(iq->dev, desc_ring_size, + iq->desc_ring, iq->desc_ring_dma); + + sglist_size = OCTEP_SGLIST_SIZE_PER_PKT * + CFG_GET_IQ_NUM_DESC(oct->conf); + if (iq->sglist) + dma_free_coherent(iq->dev, sglist_size, + iq->sglist, iq->sglist_dma); + + vfree(iq); + oct->iq[q_no] = NULL; + oct->num_iqs--; +} + +/** + * octep_setup_iqs() - setup resources for all Tx queues. + * + * @oct: Octeon device private data structure. + */ +int octep_setup_iqs(struct octep_device *oct) +{ + int i; + + oct->num_iqs = 0; + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { + if (octep_setup_iq(oct, i)) { + dev_err(&oct->pdev->dev, + "Failed to setup IQ(TxQ)-%d.\n", i); + goto iq_setup_err; + } + dev_dbg(&oct->pdev->dev, "Successfully setup IQ(TxQ)-%d.\n", i); + } + + return 0; + +iq_setup_err: + while (i) { + i--; + octep_free_iq(oct->iq[i]); + } + return -1; +} + +/** + * octep_free_iqs() - Free resources of all Tx queues. + * + * @oct: Octeon device private data structure. + */ +void octep_free_iqs(struct octep_device *oct) +{ + int i; + + for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) { + octep_free_iq(oct->iq[i]); + dev_dbg(&oct->pdev->dev, + "Successfully destroyed IQ(TxQ)-%d.\n", i); + } + oct->num_iqs = 0; +} diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h new file mode 100644 index 000000000000..2ef57980eb47 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell Octeon EP (EndPoint) Ethernet Driver + * + * Copyright (C) 2020 Marvell. + * + */ + +#ifndef _OCTEP_TX_H_ +#define _OCTEP_TX_H_ + +#define IQ_SEND_OK 0 +#define IQ_SEND_STOP 1 +#define IQ_SEND_FAILED -1 + +#define TX_BUFTYPE_NONE 0 +#define TX_BUFTYPE_NET 1 +#define TX_BUFTYPE_NET_SG 2 +#define NUM_TX_BUFTYPES 3 + +/* Hardware format for Scatter/Gather list */ +struct octep_tx_sglist_desc { + u16 len[4]; + dma_addr_t dma_ptr[4]; +}; + +/* Each Scatter/Gather entry sent to hardwar hold four pointers. + * So, number of entries required is (MAX_SKB_FRAGS + 1)/4, where '+1' + * is for main skb which also goes as a gather buffer to Octeon hardware. + * To allocate sufficient SGLIST entries for a packet with max fragments, + * align by adding 3 before calcuating max SGLIST entries per packet. + */ +#define OCTEP_SGLIST_ENTRIES_PER_PKT ((MAX_SKB_FRAGS + 1 + 3) / 4) +#define OCTEP_SGLIST_SIZE_PER_PKT \ + (OCTEP_SGLIST_ENTRIES_PER_PKT * sizeof(struct octep_tx_sglist_desc)) + +struct octep_tx_buffer { + struct sk_buff *skb; + dma_addr_t dma; + struct octep_tx_sglist_desc *sglist; + dma_addr_t sglist_dma; + u8 gather; +}; + +#define OCTEP_IQ_TXBUFF_INFO_SIZE (sizeof(struct octep_tx_buffer)) + +/* Hardware interface Tx statistics */ +struct octep_iface_tx_stats { + /* Packets dropped due to excessive collisions */ + u64 xscol; + + /* Packets dropped due to excessive deferral */ + u64 xsdef; + + /* Packets sent that experienced multiple collisions before successful + * transmission + */ + u64 mcol; + + /* Packets sent that experienced a single collision before successful + * transmission + */ + u64 scol; + + /* Total octets sent on the interface */ + u64 octs; + + /* Total frames sent on the interface */ + u64 pkts; + + /* Packets sent with an octet count < 64 */ + u64 hist_lt64; + + /* Packets sent with an octet count == 64 */ + u64 hist_eq64; + + /* Packets sent with an octet count of 65–127 */ + u64 hist_65to127; + + /* Packets sent with an octet count of 128–255 */ + u64 hist_128to255; + + /* Packets sent with an octet count of 256–511 */ + u64 hist_256to511; + + /* Packets sent with an octet count of 512–1023 */ + u64 hist_512to1023; + + /* Packets sent with an octet count of 1024-1518 */ + u64 hist_1024to1518; + + /* Packets sent with an octet count of > 1518 */ + u64 hist_gt1518; + + /* Packets sent to a broadcast DMAC */ + u64 bcst; + + /* Packets sent to the multicast DMAC */ + u64 mcst; + + /* Packets sent that experienced a transmit underflow and were + * truncated + */ + u64 undflw; + + /* Control/PAUSE packets sent */ + u64 ctl; +}; + +/* Input Queue statistics. Each input queue has four stats fields. */ +struct octep_iq_stats { + /* Instructions posted to this queue. */ + u64 instr_posted; + + /* Instructions copied by hardware for processing. */ + u64 instr_completed; + + /* Instructions that could not be processed. */ + u64 instr_dropped; + + /* Bytes sent through this queue. */ + u64 bytes_sent; + + /* Gather entries sent through this queue. */ + u64 sgentry_sent; + + /* Number of transmit failures due to TX_BUSY */ + u64 tx_busy; + + /* Number of times the queue is restarted */ + u64 restart_cnt; +}; + +/* The instruction (input) queue. + * The input queue is used to post raw (instruction) mode data or packet + * data to Octeon device from the host. Each input queue (up to 4) for + * a Octeon device has one such structure to represent it. + */ +struct octep_iq { + u32 q_no; + + struct octep_device *octep_dev; + struct net_device *netdev; + struct device *dev; + struct netdev_queue *netdev_q; + + /* Index in input ring where driver should write the next packet */ + u16 host_write_index; + + /* Index in input ring where Octeon is expected to read next packet */ + u16 octep_read_index; + + /* This index aids in finding the window in the queue where Octeon + * has read the commands. + */ + u16 flush_index; + + /* Statistics for this input queue. */ + struct octep_iq_stats stats; + + /* This field keeps track of the instructions pending in this queue. */ + atomic_t instr_pending; + + /* Pointer to the Virtual Base addr of the input ring. */ + struct octep_tx_desc_hw *desc_ring; + + /* DMA mapped base address of the input descriptor ring. */ + dma_addr_t desc_ring_dma; + + /* Info of Tx buffers pending completion. */ + struct octep_tx_buffer *buff_info; + + /* Base pointer to Scatter/Gather lists for all ring descriptors. */ + struct octep_tx_sglist_desc *sglist; + + /* DMA mapped addr of Scatter Gather Lists */ + dma_addr_t sglist_dma; + + /* Octeon doorbell register for the ring. */ + u8 __iomem *doorbell_reg; + + /* Octeon instruction count register for this ring. */ + u8 __iomem *inst_cnt_reg; + + /* interrupt level register for this ring */ + u8 __iomem *intr_lvl_reg; + + /* Maximum no. of instructions in this queue. */ + u32 max_count; + u32 ring_size_mask; + + u32 pkt_in_done; + u32 pkts_processed; + + u32 status; + + /* Number of instructions pending to be posted to Octeon. */ + u32 fill_cnt; + + /* The max. number of instructions that can be held pending by the + * driver before ringing doorbell. + */ + u32 fill_threshold; +}; + +/* Hardware Tx Instruction Header */ +struct octep_instr_hdr { + /* Data Len */ + u64 tlen:16; + + /* Reserved */ + u64 rsvd:20; + + /* PKIND for SDP */ + u64 pkind:6; + + /* Front Data size */ + u64 fsz:6; + + /* No. of entries in gather list */ + u64 gsz:14; + + /* Gather indicator 1=gather*/ + u64 gather:1; + + /* Reserved3 */ + u64 reserved3:1; +}; + +/* Hardware Tx completion response header */ +struct octep_instr_resp_hdr { + /* Request ID */ + u64 rid:16; + + /* PCIe port to use for response */ + u64 pcie_port:3; + + /* Scatter indicator 1=scatter */ + u64 scatter:1; + + /* Size of Expected result OR no. of entries in scatter list */ + u64 rlenssz:14; + + /* Desired destination port for result */ + u64 dport:6; + + /* Opcode Specific parameters */ + u64 param:8; + + /* Opcode for the return packet */ + u64 opcode:16; +}; + +/* 64-byte Tx instruction format. + * Format of instruction for a 64-byte mode input queue. + * + * only first 16-bytes (dptr and ih) are mandatory; rest are optional + * and filled by the driver based on firmware/hardware capabilities. + * These optional headers together called Front Data and its size is + * described by ih->fsz. + */ +struct octep_tx_desc_hw { + /* Pointer where the input data is available. */ + u64 dptr; + + /* Instruction Header. */ + union { + struct octep_instr_hdr ih; + u64 ih64; + }; + + /* Pointer where the response for a RAW mode packet will be written + * by Octeon. + */ + u64 rptr; + + /* Input Instruction Response Header. */ + struct octep_instr_resp_hdr irh; + + /* Additional headers available in a 64-byte instruction. */ + u64 exhdr[4]; +}; + +#define OCTEP_IQ_DESC_SIZE (sizeof(struct octep_tx_desc_hw)) +#endif /* _OCTEP_TX_H_ */ diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig index 639893d87055..6b4f640163f7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/Kconfig +++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig @@ -32,9 +32,12 @@ config OCTEONTX2_PF tristate "Marvell OcteonTX2 NIC Physical Function driver" select OCTEONTX2_MBOX select NET_DEVLINK + depends on MACSEC || !MACSEC depends on (64BIT && COMPILE_TEST) || ARM64 + select DIMLIB depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + depends on MACSEC || !MACSEC help This driver supports Marvell's OcteonTX2 NIC physical function. diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile index 7f4a4ca9af78..3cf4c8285c90 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile @@ -11,4 +11,4 @@ rvu_mbox-y := mbox.o rvu_trace.o rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \ rvu_reg.o rvu_npc.o rvu_debugfs.o ptp.o rvu_npc_fs.o \ rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \ - rvu_sdp.o + rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index 186d00a9ab35..c8724bfa86b0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -498,6 +498,32 @@ static u8 cgx_get_lmac_type(void *cgxd, int lmac_id) return (cfg >> CGX_LMAC_TYPE_SHIFT) & CGX_LMAC_TYPE_MASK; } +static u32 cgx_get_lmac_fifo_len(void *cgxd, int lmac_id) +{ + struct cgx *cgx = cgxd; + u8 num_lmacs; + u32 fifo_len; + + fifo_len = cgx->mac_ops->fifo_len; + num_lmacs = cgx->mac_ops->get_nr_lmacs(cgx); + + switch (num_lmacs) { + case 1: + return fifo_len; + case 2: + return fifo_len / 2; + case 3: + /* LMAC0 gets half of the FIFO, reset 1/4th */ + if (lmac_id == 0) + return fifo_len / 2; + return fifo_len / 4; + case 4: + default: + return fifo_len / 4; + } + return 0; +} + /* Configure CGX LMAC in internal loopback mode */ int cgx_lmac_internal_loopback(void *cgxd, int lmac_id, bool enable) { @@ -578,31 +604,78 @@ void cgx_lmac_promisc_config(int cgx_id, int lmac_id, bool enable) } } +static int cgx_lmac_get_pause_frm_status(void *cgxd, int lmac_id, + u8 *tx_pause, u8 *rx_pause) +{ + struct cgx *cgx = cgxd; + u64 cfg; + + if (is_dev_rpm(cgx)) + return 0; + + if (!is_lmac_valid(cgx, lmac_id)) + return -ENODEV; + + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); + *rx_pause = !!(cfg & CGX_SMUX_RX_FRM_CTL_CTL_BCK); + + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL); + *tx_pause = !!(cfg & CGX_SMUX_TX_CTL_L2P_BP_CONV); + return 0; +} + /* Enable or disable forwarding received pause frames to Tx block */ void cgx_lmac_enadis_rx_pause_fwding(void *cgxd, int lmac_id, bool enable) { struct cgx *cgx = cgxd; + u8 rx_pause, tx_pause; + bool is_pfc_enabled; + struct lmac *lmac; u64 cfg; if (!cgx) return; - if (enable) { - cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); - cfg |= CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); + lmac = lmac_pdata(lmac_id, cgx); + if (!lmac) + return; - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); - cfg |= CGX_SMUX_RX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); + /* Pause frames are not enabled just return */ + if (!bitmap_weight(lmac->rx_fc_pfvf_bmap.bmap, lmac->rx_fc_pfvf_bmap.max)) + return; + + cgx_lmac_get_pause_frm_status(cgx, lmac_id, &rx_pause, &tx_pause); + is_pfc_enabled = rx_pause ? false : true; + + if (enable) { + if (!is_pfc_enabled) { + cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); + cfg |= CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; + cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); + + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); + cfg |= CGX_SMUX_RX_FRM_CTL_CTL_BCK; + cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); + } else { + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_CBFC_CTL); + cfg |= CGXX_SMUX_CBFC_CTL_BCK_EN; + cgx_write(cgx, lmac_id, CGXX_SMUX_CBFC_CTL, cfg); + } } else { - cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); - cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); - cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); + if (!is_pfc_enabled) { + cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); + cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; + cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); + + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); + cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK; + cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); + } else { + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_CBFC_CTL); + cfg &= ~CGXX_SMUX_CBFC_CTL_BCK_EN; + cgx_write(cgx, lmac_id, CGXX_SMUX_CBFC_CTL, cfg); + } } } @@ -722,26 +795,6 @@ int cgx_lmac_tx_enable(void *cgxd, int lmac_id, bool enable) return !!(last & DATA_PKT_TX_EN); } -static int cgx_lmac_get_pause_frm_status(void *cgxd, int lmac_id, - u8 *tx_pause, u8 *rx_pause) -{ - struct cgx *cgx = cgxd; - u64 cfg; - - if (is_dev_rpm(cgx)) - return 0; - - if (!is_lmac_valid(cgx, lmac_id)) - return -ENODEV; - - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); - *rx_pause = !!(cfg & CGX_SMUX_RX_FRM_CTL_CTL_BCK); - - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL); - *tx_pause = !!(cfg & CGX_SMUX_TX_CTL_L2P_BP_CONV); - return 0; -} - static int cgx_lmac_enadis_pause_frm(void *cgxd, int lmac_id, u8 tx_pause, u8 rx_pause) { @@ -782,21 +835,8 @@ static void cgx_lmac_pause_frm_config(void *cgxd, int lmac_id, bool enable) if (!is_lmac_valid(cgx, lmac_id)) return; - if (enable) { - /* Enable receive pause frames */ - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); - cfg |= CGX_SMUX_RX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); - - cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); - cfg |= CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); - - /* Enable pause frames transmission */ - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL); - cfg |= CGX_SMUX_TX_CTL_L2P_BP_CONV; - cgx_write(cgx, lmac_id, CGXX_SMUX_TX_CTL, cfg); + if (enable) { /* Set pause time and interval */ cgx_write(cgx, lmac_id, CGXX_SMUX_TX_PAUSE_PKT_TIME, DEFAULT_PAUSE_TIME); @@ -813,21 +853,127 @@ static void cgx_lmac_pause_frm_config(void *cgxd, int lmac_id, bool enable) cfg &= ~0xFFFFULL; cgx_write(cgx, lmac_id, CGXX_GMP_GMI_TX_PAUSE_PKT_INTERVAL, cfg | (DEFAULT_PAUSE_TIME / 2)); - } else { - /* ALL pause frames received are completely ignored */ - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); - cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); + } - cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); - cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; - cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); + /* ALL pause frames received are completely ignored */ + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL); + cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK; + cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg); + + cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL); + cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK; + cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg); - /* Disable pause frames transmission */ - cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL); - cfg &= ~CGX_SMUX_TX_CTL_L2P_BP_CONV; - cgx_write(cgx, lmac_id, CGXX_SMUX_TX_CTL, cfg); + /* Disable pause frames transmission */ + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL); + cfg &= ~CGX_SMUX_TX_CTL_L2P_BP_CONV; + cgx_write(cgx, lmac_id, CGXX_SMUX_TX_CTL, cfg); + + cfg = cgx_read(cgx, 0, CGXX_CMR_RX_OVR_BP); + cfg |= CGX_CMR_RX_OVR_BP_EN(lmac_id); + cfg &= ~CGX_CMR_RX_OVR_BP_BP(lmac_id); + cgx_write(cgx, 0, CGXX_CMR_RX_OVR_BP, cfg); + + /* Disable all PFC classes by default */ + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_CBFC_CTL); + cfg = FIELD_SET(CGX_PFC_CLASS_MASK, 0, cfg); + cgx_write(cgx, lmac_id, CGXX_SMUX_CBFC_CTL, cfg); +} + +int verify_lmac_fc_cfg(void *cgxd, int lmac_id, u8 tx_pause, u8 rx_pause, + int pfvf_idx) +{ + struct cgx *cgx = cgxd; + struct lmac *lmac; + + lmac = lmac_pdata(lmac_id, cgx); + if (!lmac) + return -ENODEV; + + if (!rx_pause) + clear_bit(pfvf_idx, lmac->rx_fc_pfvf_bmap.bmap); + else + set_bit(pfvf_idx, lmac->rx_fc_pfvf_bmap.bmap); + + if (!tx_pause) + clear_bit(pfvf_idx, lmac->tx_fc_pfvf_bmap.bmap); + else + set_bit(pfvf_idx, lmac->tx_fc_pfvf_bmap.bmap); + + /* check if other pfvfs are using flow control */ + if (!rx_pause && bitmap_weight(lmac->rx_fc_pfvf_bmap.bmap, lmac->rx_fc_pfvf_bmap.max)) { + dev_warn(&cgx->pdev->dev, + "Receive Flow control disable not permitted as its used by other PFVFs\n"); + return -EPERM; } + + if (!tx_pause && bitmap_weight(lmac->tx_fc_pfvf_bmap.bmap, lmac->tx_fc_pfvf_bmap.max)) { + dev_warn(&cgx->pdev->dev, + "Transmit Flow control disable not permitted as its used by other PFVFs\n"); + return -EPERM; + } + + return 0; +} + +int cgx_lmac_pfc_config(void *cgxd, int lmac_id, u8 tx_pause, + u8 rx_pause, u16 pfc_en) +{ + struct cgx *cgx = cgxd; + u64 cfg; + + if (!is_lmac_valid(cgx, lmac_id)) + return -ENODEV; + + /* Return as no traffic classes are requested */ + if (tx_pause && !pfc_en) + return 0; + + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_CBFC_CTL); + pfc_en |= FIELD_GET(CGX_PFC_CLASS_MASK, cfg); + + if (rx_pause) { + cfg |= (CGXX_SMUX_CBFC_CTL_RX_EN | + CGXX_SMUX_CBFC_CTL_BCK_EN | + CGXX_SMUX_CBFC_CTL_DRP_EN); + } else { + cfg &= ~(CGXX_SMUX_CBFC_CTL_RX_EN | + CGXX_SMUX_CBFC_CTL_BCK_EN | + CGXX_SMUX_CBFC_CTL_DRP_EN); + } + + if (tx_pause) { + cfg |= CGXX_SMUX_CBFC_CTL_TX_EN; + cfg = FIELD_SET(CGX_PFC_CLASS_MASK, pfc_en, cfg); + } else { + cfg &= ~CGXX_SMUX_CBFC_CTL_TX_EN; + cfg = FIELD_SET(CGX_PFC_CLASS_MASK, 0, cfg); + } + + cgx_write(cgx, lmac_id, CGXX_SMUX_CBFC_CTL, cfg); + + /* Write source MAC address which will be filled into PFC packet */ + cfg = cgx_lmac_addr_get(cgx->cgx_id, lmac_id); + cgx_write(cgx, lmac_id, CGXX_SMUX_SMAC, cfg); + + return 0; +} + +int cgx_lmac_get_pfc_frm_cfg(void *cgxd, int lmac_id, u8 *tx_pause, + u8 *rx_pause) +{ + struct cgx *cgx = cgxd; + u64 cfg; + + if (!is_lmac_valid(cgx, lmac_id)) + return -ENODEV; + + cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_CBFC_CTL); + + *rx_pause = !!(cfg & CGXX_SMUX_CBFC_CTL_RX_EN); + *tx_pause = !!(cfg & CGXX_SMUX_CBFC_CTL_TX_EN); + + return 0; } void cgx_lmac_ptp_config(void *cgxd, int lmac_id, bool enable) @@ -892,9 +1038,9 @@ int cgx_fwi_cmd_send(u64 req, u64 *resp, struct lmac *lmac) if (!wait_event_timeout(lmac->wq_cmd_cmplt, !lmac->cmd_pend, msecs_to_jiffies(CGX_CMD_TIMEOUT))) { dev = &cgx->pdev->dev; - dev_err(dev, "cgx port %d:%d cmd timeout\n", - cgx->cgx_id, lmac->lmac_id); - err = -EIO; + dev_err(dev, "cgx port %d:%d cmd %lld timeout\n", + cgx->cgx_id, lmac->lmac_id, FIELD_GET(CMDREG_ID, req)); + err = LMAC_AF_ERR_CMD_TIMEOUT; goto unlock; } @@ -1320,11 +1466,19 @@ static int cgx_fwi_link_change(struct cgx *cgx, int lmac_id, bool enable) u64 req = 0; u64 resp; - if (enable) + if (enable) { req = FIELD_SET(CMDREG_ID, CGX_CMD_LINK_BRING_UP, req); - else - req = FIELD_SET(CMDREG_ID, CGX_CMD_LINK_BRING_DOWN, req); + /* On CN10K firmware offloads link bring up/down operations to ECP + * On Octeontx2 link operations are handled by firmware itself + * which can cause mbox errors so configure maximum time firmware + * poll for Link as 1000 ms + */ + if (!is_dev_rpm(cgx)) + req = FIELD_SET(LINKCFG_TIMEOUT, 1000, req); + } else { + req = FIELD_SET(CMDREG_ID, CGX_CMD_LINK_BRING_DOWN, req); + } return cgx_fwi_cmd_generic(req, &resp, cgx, lmac_id); } @@ -1489,6 +1643,16 @@ static int cgx_lmac_init(struct cgx *cgx) /* Reserve first entry for default MAC address */ set_bit(0, lmac->mac_to_index_bmap.bmap); + lmac->rx_fc_pfvf_bmap.max = 128; + err = rvu_alloc_bitmap(&lmac->rx_fc_pfvf_bmap); + if (err) + goto err_dmac_bmap_free; + + lmac->tx_fc_pfvf_bmap.max = 128; + err = rvu_alloc_bitmap(&lmac->tx_fc_pfvf_bmap); + if (err) + goto err_rx_fc_bmap_free; + init_waitqueue_head(&lmac->wq_cmd_cmplt); mutex_init(&lmac->cmd_lock); spin_lock_init(&lmac->event_cb_lock); @@ -1505,6 +1669,10 @@ static int cgx_lmac_init(struct cgx *cgx) return cgx_lmac_verify_fwi_version(cgx); err_bitmap_free: + rvu_free_bitmap(&lmac->tx_fc_pfvf_bmap); +err_rx_fc_bmap_free: + rvu_free_bitmap(&lmac->rx_fc_pfvf_bmap); +err_dmac_bmap_free: rvu_free_bitmap(&lmac->mac_to_index_bmap); err_name_free: kfree(lmac->name); @@ -1562,6 +1730,7 @@ static struct mac_ops cgx_mac_ops = { .tx_stats_cnt = 18, .get_nr_lmacs = cgx_get_nr_lmacs, .get_lmac_type = cgx_get_lmac_type, + .lmac_fifo_len = cgx_get_lmac_fifo_len, .mac_lmac_intl_lbk = cgx_lmac_internal_loopback, .mac_get_rx_stats = cgx_get_rx_stats, .mac_get_tx_stats = cgx_get_tx_stats, @@ -1570,6 +1739,10 @@ static struct mac_ops cgx_mac_ops = { .mac_enadis_pause_frm = cgx_lmac_enadis_pause_frm, .mac_pause_frm_config = cgx_lmac_pause_frm_config, .mac_enadis_ptp_config = cgx_lmac_ptp_config, + .mac_rx_tx_enable = cgx_lmac_rx_tx_enable, + .mac_tx_enable = cgx_lmac_tx_enable, + .pfc_config = cgx_lmac_pfc_config, + .mac_get_pfc_frm_cfg = cgx_lmac_get_pfc_frm_cfg, }; static int cgx_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -1612,6 +1785,13 @@ static int cgx_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_release_regions; } + cgx->lmac_count = cgx->mac_ops->get_nr_lmacs(cgx); + if (!cgx->lmac_count) { + dev_notice(dev, "CGX %d LMAC count is zero, skipping probe\n", cgx->cgx_id); + err = -EOPNOTSUPP; + goto err_release_regions; + } + nvec = pci_msix_vec_count(cgx->pdev); err = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX); if (err < 0 || err != nvec) { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.h b/drivers/net/ethernet/marvell/octeontx2/af/cgx.h index ab1e4abdea38..0b06788b8d80 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.h @@ -76,6 +76,13 @@ #define CGXX_SMUX_TX_CTL 0x20178 #define CGXX_SMUX_TX_PAUSE_PKT_TIME 0x20110 #define CGXX_SMUX_TX_PAUSE_PKT_INTERVAL 0x20120 +#define CGXX_SMUX_SMAC 0x20108 +#define CGXX_SMUX_CBFC_CTL 0x20218 +#define CGXX_SMUX_CBFC_CTL_RX_EN BIT_ULL(0) +#define CGXX_SMUX_CBFC_CTL_TX_EN BIT_ULL(1) +#define CGXX_SMUX_CBFC_CTL_DRP_EN BIT_ULL(2) +#define CGXX_SMUX_CBFC_CTL_BCK_EN BIT_ULL(3) +#define CGX_PFC_CLASS_MASK GENMASK_ULL(47, 32) #define CGXX_GMP_GMI_TX_PAUSE_PKT_TIME 0x38230 #define CGXX_GMP_GMI_TX_PAUSE_PKT_INTERVAL 0x38248 #define CGX_SMUX_TX_CTL_L2P_BP_CONV BIT_ULL(7) @@ -85,7 +92,7 @@ #define CGX_COMMAND_REG CGXX_SCRATCH1_REG #define CGX_EVENT_REG CGXX_SCRATCH0_REG -#define CGX_CMD_TIMEOUT 2200 /* msecs */ +#define CGX_CMD_TIMEOUT 5000 /* msecs */ #define DEFAULT_PAUSE_TIME 0x7FF #define CGX_LMAC_FWI 0 @@ -172,4 +179,10 @@ u64 cgx_lmac_read(int cgx_id, int lmac_id, u64 offset); int cgx_lmac_addr_update(u8 cgx_id, u8 lmac_id, u8 *mac_addr, u8 index); u64 cgx_read_dmac_ctrl(void *cgxd, int lmac_id); u64 cgx_read_dmac_entry(void *cgxd, int index); +int cgx_lmac_pfc_config(void *cgxd, int lmac_id, u8 tx_pause, u8 rx_pause, + u16 pfc_en); +int cgx_lmac_get_pfc_frm_cfg(void *cgxd, int lmac_id, u8 *tx_pause, + u8 *rx_pause); +int verify_lmac_fc_cfg(void *cgxd, int lmac_id, u8 tx_pause, u8 rx_pause, + int pfvf_idx); #endif /* CGX_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx_fw_if.h b/drivers/net/ethernet/marvell/octeontx2/af/cgx_fw_if.h index f72ec0e2506f..d4a27c882a5b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx_fw_if.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx_fw_if.h @@ -261,4 +261,6 @@ struct cgx_lnk_sts { #define CMDMODECHANGE_PORT GENMASK_ULL(21, 14) #define CMDMODECHANGE_FLAGS GENMASK_ULL(63, 22) +/* LINK_BRING_UP command timeout */ +#define LINKCFG_TIMEOUT GENMASK_ULL(21, 8) #endif /* __CGX_FW_INTF_H__ */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h index fc6e7423cbd8..52b6016789fa 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/lmac_common.h @@ -17,6 +17,8 @@ * @resp: command response * @link_info: link related information * @mac_to_index_bmap: Mac address to CGX table index mapping + * @rx_fc_pfvf_bmap: Receive flow control enabled netdev mapping + * @tx_fc_pfvf_bmap: Transmit flow control enabled netdev mapping * @event_cb: callback for linkchange events * @event_cb_lock: lock for serializing callback with unregister * @cgx: parent cgx port @@ -33,6 +35,8 @@ struct lmac { u64 resp; struct cgx_link_user_info link_info; struct rsrc_bmap mac_to_index_bmap; + struct rsrc_bmap rx_fc_pfvf_bmap; + struct rsrc_bmap tx_fc_pfvf_bmap; struct cgx_event_cb event_cb; /* lock for serializing callback with unregister */ spinlock_t event_cb_lock; @@ -76,6 +80,7 @@ struct mac_ops { */ int (*get_nr_lmacs)(void *cgx); u8 (*get_lmac_type)(void *cgx, int lmac_id); + u32 (*lmac_fifo_len)(void *cgx, int lmac_id); int (*mac_lmac_intl_lbk)(void *cgx, int lmac_id, bool enable); /* Register Stats related functions */ @@ -107,6 +112,15 @@ struct mac_ops { void (*mac_enadis_ptp_config)(void *cgxd, int lmac_id, bool enable); + + int (*mac_rx_tx_enable)(void *cgxd, int lmac_id, bool enable); + int (*mac_tx_enable)(void *cgxd, int lmac_id, bool enable); + int (*pfc_config)(void *cgxd, int lmac_id, + u8 tx_pause, u8 rx_pause, u16 pfc_en); + + int (*mac_get_pfc_frm_cfg)(void *cgxd, int lmac_id, + u8 *tx_pause, u8 *rx_pause); + }; struct cgx { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index 4e79e918a161..8d5d5a0f68c4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -33,7 +33,7 @@ #define INTR_MASK(pfvfs) ((pfvfs < 64) ? (BIT_ULL(pfvfs) - 1) : (~0ull)) -#define MBOX_RSP_TIMEOUT 3000 /* Time(ms) to wait for mbox response */ +#define MBOX_RSP_TIMEOUT 6000 /* Time(ms) to wait for mbox response */ #define MBOX_MSG_ALIGN 16 /* Align mbox msg start to 16bytes */ @@ -169,9 +169,12 @@ M(CGX_GET_PHY_FEC_STATS, 0x219, cgx_get_phy_fec_stats, msg_req, msg_rsp) \ M(CGX_FEATURES_GET, 0x21B, cgx_features_get, msg_req, \ cgx_features_info_msg) \ M(RPM_STATS, 0x21C, rpm_stats, msg_req, rpm_stats_rsp) \ -M(CGX_MAC_ADDR_RESET, 0x21D, cgx_mac_addr_reset, msg_req, msg_rsp) \ +M(CGX_MAC_ADDR_RESET, 0x21D, cgx_mac_addr_reset, cgx_mac_addr_reset_req, \ + msg_rsp) \ M(CGX_MAC_ADDR_UPDATE, 0x21E, cgx_mac_addr_update, cgx_mac_addr_update_req, \ - msg_rsp) \ + cgx_mac_addr_update_rsp) \ +M(CGX_PRIO_FLOW_CTRL_CFG, 0x21F, cgx_prio_flow_ctrl_cfg, cgx_pfc_cfg, \ + cgx_pfc_rsp) \ /* NPA mbox IDs (range 0x400 - 0x5FF) */ \ M(NPA_LF_ALLOC, 0x400, npa_lf_alloc, \ npa_lf_alloc_req, npa_lf_alloc_rsp) \ @@ -239,6 +242,9 @@ M(NPC_MCAM_READ_BASE_RULE, 0x6011, npc_read_base_steer_rule, \ M(NPC_MCAM_GET_STATS, 0x6012, npc_mcam_entry_stats, \ npc_mcam_get_stats_req, \ npc_mcam_get_stats_rsp) \ +M(NPC_GET_SECRET_KEY, 0x6013, npc_get_secret_key, \ + npc_get_secret_key_req, \ + npc_get_secret_key_rsp) \ /* NIX mbox IDs (range 0x8000 - 0xFFFF) */ \ M(NIX_LF_ALLOC, 0x8000, nix_lf_alloc, \ nix_lf_alloc_req, nix_lf_alloc_rsp) \ @@ -287,20 +293,74 @@ M(NIX_BANDPROF_ALLOC, 0x801d, nix_bandprof_alloc, nix_bandprof_alloc_req, \ M(NIX_BANDPROF_FREE, 0x801e, nix_bandprof_free, nix_bandprof_free_req, \ msg_rsp) \ M(NIX_BANDPROF_GET_HWINFO, 0x801f, nix_bandprof_get_hwinfo, msg_req, \ - nix_bandprof_get_hwinfo_rsp) - -/* Messages initiated by AF (range 0xC00 - 0xDFF) */ + nix_bandprof_get_hwinfo_rsp) \ +/* MCS mbox IDs (range 0xA000 - 0xBFFF) */ \ +M(MCS_ALLOC_RESOURCES, 0xa000, mcs_alloc_resources, mcs_alloc_rsrc_req, \ + mcs_alloc_rsrc_rsp) \ +M(MCS_FREE_RESOURCES, 0xa001, mcs_free_resources, mcs_free_rsrc_req, msg_rsp) \ +M(MCS_FLOWID_ENTRY_WRITE, 0xa002, mcs_flowid_entry_write, mcs_flowid_entry_write_req, \ + msg_rsp) \ +M(MCS_SECY_PLCY_WRITE, 0xa003, mcs_secy_plcy_write, mcs_secy_plcy_write_req, \ + msg_rsp) \ +M(MCS_RX_SC_CAM_WRITE, 0xa004, mcs_rx_sc_cam_write, mcs_rx_sc_cam_write_req, \ + msg_rsp) \ +M(MCS_SA_PLCY_WRITE, 0xa005, mcs_sa_plcy_write, mcs_sa_plcy_write_req, \ + msg_rsp) \ +M(MCS_TX_SC_SA_MAP_WRITE, 0xa006, mcs_tx_sc_sa_map_write, mcs_tx_sc_sa_map, \ + msg_rsp) \ +M(MCS_RX_SC_SA_MAP_WRITE, 0xa007, mcs_rx_sc_sa_map_write, mcs_rx_sc_sa_map, \ + msg_rsp) \ +M(MCS_FLOWID_ENA_ENTRY, 0xa008, mcs_flowid_ena_entry, mcs_flowid_ena_dis_entry, \ + msg_rsp) \ +M(MCS_PN_TABLE_WRITE, 0xa009, mcs_pn_table_write, mcs_pn_table_write_req, \ + msg_rsp) \ +M(MCS_SET_ACTIVE_LMAC, 0xa00a, mcs_set_active_lmac, mcs_set_active_lmac, \ + msg_rsp) \ +M(MCS_GET_HW_INFO, 0xa00b, mcs_get_hw_info, msg_req, mcs_hw_info) \ +M(MCS_GET_FLOWID_STATS, 0xa00c, mcs_get_flowid_stats, mcs_stats_req, \ + mcs_flowid_stats) \ +M(MCS_GET_SECY_STATS, 0xa00d, mcs_get_secy_stats, mcs_stats_req, \ + mcs_secy_stats) \ +M(MCS_GET_SC_STATS, 0xa00e, mcs_get_sc_stats, mcs_stats_req, mcs_sc_stats) \ +M(MCS_GET_SA_STATS, 0xa00f, mcs_get_sa_stats, mcs_stats_req, mcs_sa_stats) \ +M(MCS_GET_PORT_STATS, 0xa010, mcs_get_port_stats, mcs_stats_req, \ + mcs_port_stats) \ +M(MCS_CLEAR_STATS, 0xa011, mcs_clear_stats, mcs_clear_stats, msg_rsp) \ +M(MCS_INTR_CFG, 0xa012, mcs_intr_cfg, mcs_intr_cfg, msg_rsp) \ +M(MCS_SET_LMAC_MODE, 0xa013, mcs_set_lmac_mode, mcs_set_lmac_mode, msg_rsp) \ +M(MCS_SET_PN_THRESHOLD, 0xa014, mcs_set_pn_threshold, mcs_set_pn_threshold, \ + msg_rsp) \ +M(MCS_ALLOC_CTRL_PKT_RULE, 0xa015, mcs_alloc_ctrl_pkt_rule, \ + mcs_alloc_ctrl_pkt_rule_req, \ + mcs_alloc_ctrl_pkt_rule_rsp) \ +M(MCS_FREE_CTRL_PKT_RULE, 0xa016, mcs_free_ctrl_pkt_rule, \ + mcs_free_ctrl_pkt_rule_req, msg_rsp) \ +M(MCS_CTRL_PKT_RULE_WRITE, 0xa017, mcs_ctrl_pkt_rule_write, \ + mcs_ctrl_pkt_rule_write_req, msg_rsp) \ +M(MCS_PORT_RESET, 0xa018, mcs_port_reset, mcs_port_reset_req, msg_rsp) \ +M(MCS_PORT_CFG_SET, 0xa019, mcs_port_cfg_set, mcs_port_cfg_set_req, msg_rsp)\ +M(MCS_PORT_CFG_GET, 0xa020, mcs_port_cfg_get, mcs_port_cfg_get_req, \ + mcs_port_cfg_get_rsp) \ +M(MCS_CUSTOM_TAG_CFG_GET, 0xa021, mcs_custom_tag_cfg_get, \ + mcs_custom_tag_cfg_get_req, \ + mcs_custom_tag_cfg_get_rsp) + +/* Messages initiated by AF (range 0xC00 - 0xEFF) */ #define MBOX_UP_CGX_MESSAGES \ M(CGX_LINK_EVENT, 0xC00, cgx_link_event, cgx_link_info_msg, msg_rsp) #define MBOX_UP_CPT_MESSAGES \ M(CPT_INST_LMTST, 0xD00, cpt_inst_lmtst, cpt_inst_lmtst_req, msg_rsp) +#define MBOX_UP_MCS_MESSAGES \ +M(MCS_INTR_NOTIFY, 0xE00, mcs_intr_notify, mcs_intr_info, msg_rsp) + enum { #define M(_name, _id, _1, _2, _3) MBOX_MSG_ ## _name = _id, MBOX_MESSAGES MBOX_UP_CGX_MESSAGES MBOX_UP_CPT_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M }; @@ -426,6 +486,7 @@ struct get_hw_cap_rsp { struct mbox_msghdr hdr; u8 nix_fixed_txschq_mapping; /* Schq mapping fixed or flexible */ u8 nix_shaping; /* Is shaping and coloring supported */ + u8 npc_hash_extract; /* Is hash extract supported */ }; /* CGX mbox message formats */ @@ -449,6 +510,7 @@ struct cgx_fec_stats_rsp { struct cgx_mac_addr_set_or_get { struct mbox_msghdr hdr; u8 mac_addr[ETH_ALEN]; + u32 index; }; /* Structure for requesting the operation to @@ -464,7 +526,7 @@ struct cgx_mac_addr_add_req { */ struct cgx_mac_addr_add_rsp { struct mbox_msghdr hdr; - u8 index; + u32 index; }; /* Structure for requesting the operation to @@ -472,7 +534,7 @@ struct cgx_mac_addr_add_rsp { */ struct cgx_mac_addr_del_req { struct mbox_msghdr hdr; - u8 index; + u32 index; }; /* Structure for response against the operation to @@ -480,7 +542,7 @@ struct cgx_mac_addr_del_req { */ struct cgx_max_dmac_entries_get_rsp { struct mbox_msghdr hdr; - u8 max_dmac_filters; + u32 max_dmac_filters; }; struct cgx_link_user_info { @@ -581,10 +643,20 @@ struct cgx_set_link_mode_rsp { int status; }; +struct cgx_mac_addr_reset_req { + struct mbox_msghdr hdr; + u32 index; +}; + struct cgx_mac_addr_update_req { struct mbox_msghdr hdr; u8 mac_addr[ETH_ALEN]; - u8 index; + u32 index; +}; + +struct cgx_mac_addr_update_rsp { + struct mbox_msghdr hdr; + u32 index; }; #define RVU_LMAC_FEAT_FC BIT_ULL(0) /* pause frames */ @@ -609,6 +681,21 @@ struct rpm_stats_rsp { u64 tx_stats[RPM_TX_STATS_COUNT]; }; +struct cgx_pfc_cfg { + struct mbox_msghdr hdr; + u8 rx_pause; + u8 tx_pause; + u16 pfc_en; /* bitmap indicating pfc enabled traffic classes */ +}; + +struct cgx_pfc_rsp { + struct mbox_msghdr hdr; + u8 rx_pause; + u8 tx_pause; +}; + + /* NPA mbox message formats */ + struct npc_set_pkind { struct mbox_msghdr hdr; #define OTX2_PRIV_FLAGS_DEFAULT BIT_ULL(0) @@ -732,6 +819,7 @@ enum nix_af_status { NIX_AF_ERR_BANDPROF_INVAL_REQ = -428, NIX_AF_ERR_CQ_CTX_WRITE_ERR = -429, NIX_AF_ERR_AQ_CTX_RETRY_WRITE = -430, + NIX_AF_ERR_LINK_CREDITS = -431, }; /* For NIX RX vtag action */ @@ -1422,11 +1510,22 @@ struct npc_mcam_get_stats_rsp { u8 stat_ena; /* enabled */ }; +struct npc_get_secret_key_req { + struct mbox_msghdr hdr; + u8 intf; +}; + +struct npc_get_secret_key_rsp { + struct mbox_msghdr hdr; + u64 secret_key[3]; +}; + enum ptp_op { PTP_OP_ADJFINE = 0, PTP_OP_GET_CLOCK = 1, PTP_OP_GET_TSTMP = 2, PTP_OP_SET_THRESH = 3, + PTP_OP_EXTTS_ON = 4, }; struct ptp_req { @@ -1434,6 +1533,7 @@ struct ptp_req { u8 op; s64 scaled_ppm; u64 thresh; + int extts_on; }; struct ptp_rsp { @@ -1602,6 +1702,424 @@ enum cgx_af_status { LMAC_AF_ERR_INVALID_PARAM = -1101, LMAC_AF_ERR_PF_NOT_MAPPED = -1102, LMAC_AF_ERR_PERM_DENIED = -1103, + LMAC_AF_ERR_PFC_ENADIS_PERM_DENIED = -1104, + LMAC_AF_ERR_8023PAUSE_ENADIS_PERM_DENIED = -1105, + LMAC_AF_ERR_CMD_TIMEOUT = -1106, + LMAC_AF_ERR_FIRMWARE_DATA_NOT_MAPPED = -1107, + LMAC_AF_ERR_EXACT_MATCH_TBL_ADD_FAILED = -1108, + LMAC_AF_ERR_EXACT_MATCH_TBL_DEL_FAILED = -1109, + LMAC_AF_ERR_EXACT_MATCH_TBL_LOOK_UP_FAILED = -1110, +}; + +enum mcs_direction { + MCS_RX, + MCS_TX, +}; + +enum mcs_rsrc_type { + MCS_RSRC_TYPE_FLOWID, + MCS_RSRC_TYPE_SECY, + MCS_RSRC_TYPE_SC, + MCS_RSRC_TYPE_SA, +}; + +struct mcs_alloc_rsrc_req { + struct mbox_msghdr hdr; + u8 rsrc_type; + u8 rsrc_cnt; /* Resources count */ + u8 mcs_id; /* MCS block ID */ + u8 dir; /* Macsec ingress or egress side */ + u8 all; /* Allocate all resource type one each */ + u64 rsvd; +}; + +struct mcs_alloc_rsrc_rsp { + struct mbox_msghdr hdr; + u8 flow_ids[128]; /* Index of reserved entries */ + u8 secy_ids[128]; + u8 sc_ids[128]; + u8 sa_ids[256]; + u8 rsrc_type; + u8 rsrc_cnt; /* No of entries reserved */ + u8 mcs_id; + u8 dir; + u8 all; + u8 rsvd[256]; /* reserved fields for future expansion */ +}; + +struct mcs_free_rsrc_req { + struct mbox_msghdr hdr; + u8 rsrc_id; /* Index of the entry to be freed */ + u8 rsrc_type; + u8 mcs_id; + u8 dir; + u8 all; /* Free all the cam resources */ + u64 rsvd; +}; + +struct mcs_flowid_entry_write_req { + struct mbox_msghdr hdr; + u64 data[4]; + u64 mask[4]; + u64 sci; /* CNF10K-B for tx_secy_mem_map */ + u8 flow_id; + u8 secy_id; /* secyid for which flowid is mapped */ + u8 sc_id; /* Valid if dir = MCS_TX, SC_CAM id mapped to flowid */ + u8 ena; /* Enable tcam entry */ + u8 ctrl_pkt; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_secy_plcy_write_req { + struct mbox_msghdr hdr; + u64 plcy; + u8 secy_id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +/* RX SC_CAM mapping */ +struct mcs_rx_sc_cam_write_req { + struct mbox_msghdr hdr; + u64 sci; /* SCI */ + u64 secy_id; /* secy index mapped to SC */ + u8 sc_id; /* SC CAM entry index */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_sa_plcy_write_req { + struct mbox_msghdr hdr; + u64 plcy[2][9]; /* Support 2 SA policy */ + u8 sa_index[2]; + u8 sa_cnt; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_tx_sc_sa_map { + struct mbox_msghdr hdr; + u8 sa_index0; + u8 sa_index1; + u8 rekey_ena; + u8 sa_index0_vld; + u8 sa_index1_vld; + u8 tx_sa_active; + u64 sectag_sci; + u8 sc_id; /* used as index for SA_MEM_MAP */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_rx_sc_sa_map { + struct mbox_msghdr hdr; + u8 sa_index; + u8 sa_in_use; + u8 sc_id; + u8 an; /* value range 0-3, sc_id + an used as index SA_MEM_MAP */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_flowid_ena_dis_entry { + struct mbox_msghdr hdr; + u8 flow_id; + u8 ena; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_pn_table_write_req { + struct mbox_msghdr hdr; + u64 next_pn; + u8 pn_id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_hw_info { + struct mbox_msghdr hdr; + u8 num_mcs_blks; /* Number of MCS blocks */ + u8 tcam_entries; /* RX/TX Tcam entries per mcs block */ + u8 secy_entries; /* RX/TX SECY entries per mcs block */ + u8 sc_entries; /* RX/TX SC CAM entries per mcs block */ + u8 sa_entries; /* PN table entries = SA entries */ + u64 rsvd[16]; +}; + +struct mcs_set_active_lmac { + struct mbox_msghdr hdr; + u32 lmac_bmap; /* bitmap of active lmac per mcs block */ + u8 mcs_id; + u16 chan_base; /* MCS channel base */ + u64 rsvd; +}; + +struct mcs_set_lmac_mode { + struct mbox_msghdr hdr; + u8 mode; /* 1:Bypass 0:Operational */ + u8 lmac_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_reset_req { + struct mbox_msghdr hdr; + u8 reset; + u8 mcs_id; + u8 port_id; + u64 rsvd; +}; + +struct mcs_port_cfg_set_req { + struct mbox_msghdr hdr; + u8 cstm_tag_rel_mode_sel; + u8 custom_hdr_enb; + u8 fifo_skid; + u8 port_mode; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_cfg_get_req { + struct mbox_msghdr hdr; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_cfg_get_rsp { + struct mbox_msghdr hdr; + u8 cstm_tag_rel_mode_sel; + u8 custom_hdr_enb; + u8 fifo_skid; + u8 port_mode; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_custom_tag_cfg_get_req { + struct mbox_msghdr hdr; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_custom_tag_cfg_get_rsp { + struct mbox_msghdr hdr; + u16 cstm_etype[8]; + u8 cstm_indx[8]; + u8 cstm_etype_en; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +/* MCS mailbox error codes + * Range 1201 - 1300. + */ +enum mcs_af_status { + MCS_AF_ERR_INVALID_MCSID = -1201, + MCS_AF_ERR_NOT_MAPPED = -1202, +}; + +struct mcs_set_pn_threshold { + struct mbox_msghdr hdr; + u64 threshold; + u8 xpn; /* '1' for setting xpn threshold */ + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +enum mcs_ctrl_pkt_rulew_type { + MCS_CTRL_PKT_RULE_TYPE_ETH, + MCS_CTRL_PKT_RULE_TYPE_DA, + MCS_CTRL_PKT_RULE_TYPE_RANGE, + MCS_CTRL_PKT_RULE_TYPE_COMBO, + MCS_CTRL_PKT_RULE_TYPE_MAC, +}; + +struct mcs_alloc_ctrl_pkt_rule_req { + struct mbox_msghdr hdr; + u8 rule_type; + u8 mcs_id; /* MCS block ID */ + u8 dir; /* Macsec ingress or egress side */ + u64 rsvd; +}; + +struct mcs_alloc_ctrl_pkt_rule_rsp { + struct mbox_msghdr hdr; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_free_ctrl_pkt_rule_req { + struct mbox_msghdr hdr; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u8 all; + u64 rsvd; +}; + +struct mcs_ctrl_pkt_rule_write_req { + struct mbox_msghdr hdr; + u64 data0; + u64 data1; + u64 data2; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_stats_req { + struct mbox_msghdr hdr; + u8 id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_flowid_stats { + struct mbox_msghdr hdr; + u64 tcam_hit_cnt; + u64 rsvd; +}; + +struct mcs_secy_stats { + struct mbox_msghdr hdr; + u64 ctl_pkt_bcast_cnt; + u64 ctl_pkt_mcast_cnt; + u64 ctl_pkt_ucast_cnt; + u64 ctl_octet_cnt; + u64 unctl_pkt_bcast_cnt; + u64 unctl_pkt_mcast_cnt; + u64 unctl_pkt_ucast_cnt; + u64 unctl_octet_cnt; + /* Valid only for RX */ + u64 octet_decrypted_cnt; + u64 octet_validated_cnt; + u64 pkt_port_disabled_cnt; + u64 pkt_badtag_cnt; + u64 pkt_nosa_cnt; + u64 pkt_nosaerror_cnt; + u64 pkt_tagged_ctl_cnt; + u64 pkt_untaged_cnt; + u64 pkt_ctl_cnt; /* CN10K-B */ + u64 pkt_notag_cnt; /* CNF10K-B */ + /* Valid only for TX */ + u64 octet_encrypted_cnt; + u64 octet_protected_cnt; + u64 pkt_noactivesa_cnt; + u64 pkt_toolong_cnt; + u64 pkt_untagged_cnt; + u64 rsvd[4]; +}; + +struct mcs_port_stats { + struct mbox_msghdr hdr; + u64 tcam_miss_cnt; + u64 parser_err_cnt; + u64 preempt_err_cnt; /* CNF10K-B */ + u64 sectag_insert_err_cnt; + u64 rsvd[4]; +}; + +/* Only for CN10K-B */ +struct mcs_sa_stats { + struct mbox_msghdr hdr; + /* RX */ + u64 pkt_invalid_cnt; + u64 pkt_nosaerror_cnt; + u64 pkt_notvalid_cnt; + u64 pkt_ok_cnt; + u64 pkt_nosa_cnt; + /* TX */ + u64 pkt_encrypt_cnt; + u64 pkt_protected_cnt; + u64 rsvd[4]; +}; + +struct mcs_sc_stats { + struct mbox_msghdr hdr; + /* RX */ + u64 hit_cnt; + u64 pkt_invalid_cnt; + u64 pkt_late_cnt; + u64 pkt_notvalid_cnt; + u64 pkt_unchecked_cnt; + u64 pkt_delay_cnt; /* CNF10K-B */ + u64 pkt_ok_cnt; /* CNF10K-B */ + u64 octet_decrypt_cnt; /* CN10K-B */ + u64 octet_validate_cnt; /* CN10K-B */ + /* TX */ + u64 pkt_encrypt_cnt; + u64 pkt_protected_cnt; + u64 octet_encrypt_cnt; /* CN10K-B */ + u64 octet_protected_cnt; /* CN10K-B */ + u64 rsvd[4]; +}; + +struct mcs_clear_stats { + struct mbox_msghdr hdr; +#define MCS_FLOWID_STATS 0 +#define MCS_SECY_STATS 1 +#define MCS_SC_STATS 2 +#define MCS_SA_STATS 3 +#define MCS_PORT_STATS 4 + u8 type; /* FLOWID, SECY, SC, SA, PORT */ + u8 id; /* type = PORT, If id = FF(invalid) port no is derived from pcifunc */ + u8 mcs_id; + u8 dir; + u8 all; /* All resources stats mapped to PF are cleared */ +}; + +struct mcs_intr_cfg { + struct mbox_msghdr hdr; +#define MCS_CPM_RX_SECTAG_V_EQ1_INT BIT_ULL(0) +#define MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT BIT_ULL(1) +#define MCS_CPM_RX_SECTAG_SL_GTE48_INT BIT_ULL(2) +#define MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT BIT_ULL(3) +#define MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT BIT_ULL(4) +#define MCS_CPM_RX_PACKET_XPN_EQ0_INT BIT_ULL(5) +#define MCS_CPM_RX_PN_THRESH_REACHED_INT BIT_ULL(6) +#define MCS_CPM_TX_PACKET_XPN_EQ0_INT BIT_ULL(7) +#define MCS_CPM_TX_PN_THRESH_REACHED_INT BIT_ULL(8) +#define MCS_CPM_TX_SA_NOT_VALID_INT BIT_ULL(9) +#define MCS_BBE_RX_DFIFO_OVERFLOW_INT BIT_ULL(10) +#define MCS_BBE_RX_PLFIFO_OVERFLOW_INT BIT_ULL(11) +#define MCS_BBE_TX_DFIFO_OVERFLOW_INT BIT_ULL(12) +#define MCS_BBE_TX_PLFIFO_OVERFLOW_INT BIT_ULL(13) +#define MCS_PAB_RX_CHAN_OVERFLOW_INT BIT_ULL(14) +#define MCS_PAB_TX_CHAN_OVERFLOW_INT BIT_ULL(15) + u64 intr_mask; /* Interrupt enable mask */ + u8 mcs_id; + u8 lmac_id; + u64 rsvd; +}; + +struct mcs_intr_info { + struct mbox_msghdr hdr; + u64 intr_mask; + int sa_id; + u8 mcs_id; + u8 lmac_id; + u64 rsvd; }; #endif /* MBOX_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c new file mode 100644 index 000000000000..4a343f853b28 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -0,0 +1,1603 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "mcs.h" +#include "mcs_reg.h" + +#define DRV_NAME "Marvell MCS Driver" + +#define PCI_CFG_REG_BAR_NUM 0 + +static const struct pci_device_id mcs_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_MCS) }, + { 0, } /* end of table */ +}; + +static LIST_HEAD(mcs_list); + +void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id) +{ + u64 reg; + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(id); + stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(id); + stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(id); + stats->ctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(id); + stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(id); + stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(id); + stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(id); + stats->unctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(id); + stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(id); + stats->octet_encrypted_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(id); + stats->octet_protected_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(id); + stats->pkt_noactivesa_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(id); + stats->pkt_toolong_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(id); + stats->pkt_untagged_cnt = mcs_reg_read(mcs, reg); +} + +void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id) +{ + u64 reg; + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(id); + stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(id); + stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(id); + stats->ctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(id); + stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(id); + stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(id); + stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(id); + stats->unctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(id); + stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(id); + stats->octet_decrypted_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(id); + stats->octet_validated_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(id); + stats->pkt_port_disabled_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(id); + stats->pkt_badtag_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(id); + stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(id); + stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(id); + stats->pkt_tagged_ctl_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(id); + stats->pkt_untaged_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(id); + stats->pkt_ctl_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(id); + stats->pkt_notag_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(id); + else + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(id); + + stats->tcam_hit_cnt = mcs_reg_read(mcs, reg); +} + +void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(id); + stats->tcam_miss_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(id); + stats->parser_err_cnt = mcs_reg_read(mcs, reg); + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(id); + stats->preempt_err_cnt = mcs_reg_read(mcs, reg); + } + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(id); + stats->tcam_miss_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(id); + stats->parser_err_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(id); + stats->sectag_insert_err_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(id); + stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(id); + stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(id); + stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(id); + stats->pkt_ok_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(id); + stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg); + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(id); + stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(id); + stats->pkt_protected_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(id); + stats->hit_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(id); + stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(id); + stats->pkt_late_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(id); + stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(id); + stats->pkt_unchecked_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(id); + stats->pkt_delay_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(id); + stats->pkt_ok_cnt = mcs_reg_read(mcs, reg); + } + if (mcs->hw->mcs_blks == 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(id); + stats->octet_decrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(id); + stats->octet_validate_cnt = mcs_reg_read(mcs, reg); + } + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(id); + stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(id); + stats->pkt_protected_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks == 1) { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(id); + stats->octet_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(id); + stats->octet_protected_cnt = mcs_reg_read(mcs, reg); + } + } +} + +void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir) +{ + struct mcs_flowid_stats flowid_st; + struct mcs_port_stats port_st; + struct mcs_secy_stats secy_st; + struct mcs_sc_stats sc_st; + struct mcs_sa_stats sa_st; + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CSE_RX_SLAVE_CTRL; + else + reg = MCSX_CSE_TX_SLAVE_CTRL; + + mcs_reg_write(mcs, reg, BIT_ULL(0)); + + switch (type) { + case MCS_FLOWID_STATS: + mcs_get_flowid_stats(mcs, &flowid_st, id, dir); + break; + case MCS_SECY_STATS: + if (dir == MCS_RX) + mcs_get_rx_secy_stats(mcs, &secy_st, id); + else + mcs_get_tx_secy_stats(mcs, &secy_st, id); + break; + case MCS_SC_STATS: + mcs_get_sc_stats(mcs, &sc_st, id, dir); + break; + case MCS_SA_STATS: + mcs_get_sa_stats(mcs, &sa_st, id, dir); + break; + case MCS_PORT_STATS: + mcs_get_port_stats(mcs, &port_st, id, dir); + break; + } + + mcs_reg_write(mcs, reg, 0x0); +} + +int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* Clear FLOWID stats */ + for (id = 0; id < map->flow_ids.max; id++) { + if (map->flowid2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_FLOWID_STATS, id, dir); + } + + /* Clear SECY stats */ + for (id = 0; id < map->secy.max; id++) { + if (map->secy2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SECY_STATS, id, dir); + } + + /* Clear SC stats */ + for (id = 0; id < map->secy.max; id++) { + if (map->sc2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SC_STATS, id, dir); + } + + /* Clear SA stats */ + for (id = 0; id < map->sa.max; id++) { + if (map->sa2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SA_STATS, id, dir); + } + return 0; +} + +void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(pn_id); + else + reg = MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(pn_id); + mcs_reg_write(mcs, reg, next_pn); +} + +void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map) +{ + u64 reg, val; + + val = (map->sa_index0 & 0xFF) | + (map->sa_index1 & 0xFF) << 9 | + (map->rekey_ena & 0x1) << 18 | + (map->sa_index0_vld & 0x1) << 19 | + (map->sa_index1_vld & 0x1) << 20 | + (map->tx_sa_active & 0x1) << 21 | + map->sectag_sci << 22; + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id); + mcs_reg_write(mcs, reg, val); + + val = map->sectag_sci >> 42; + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(map->sc_id); + mcs_reg_write(mcs, reg, val); +} + +void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map) +{ + u64 val, reg; + + val = (map->sa_index & 0xFF) | map->sa_in_use << 9; + + reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an); + mcs_reg_write(mcs, reg, val); +} + +void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa_id, int dir) +{ + int reg_id; + u64 reg; + + if (dir == MCS_RX) { + for (reg_id = 0; reg_id < 8; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id); + mcs_reg_write(mcs, reg, plcy[reg_id]); + } + } else { + for (reg_id = 0; reg_id < 9; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id); + mcs_reg_write(mcs, reg, plcy[reg_id]); + } + } +} + +void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int sc_id, int ena) +{ + u64 reg, val; + + reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(0); + if (sc_id > 63) + reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(1); + + if (ena) + val = mcs_reg_read(mcs, reg) | BIT_ULL(sc_id); + else + val = mcs_reg_read(mcs, reg) & ~BIT_ULL(sc_id); + + mcs_reg_write(mcs, reg, val); +} + +void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id) +{ + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(0, sc_id), sci); + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(1, sc_id), secy); + /* Enable SC CAM */ + mcs_ena_dis_sc_cam_entry(mcs, sc_id, true); +} + +void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int secy_id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(secy_id); + else + reg = MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(secy_id); + + mcs_reg_write(mcs, reg, plcy); + + if (mcs->hw->mcs_blks == 1 && dir == MCS_RX) + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(secy_id), 0x0ull); +} + +void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir) +{ + u64 reg, val; + + val = (map->secy & 0x7F) | (map->ctrl_pkt & 0x1) << 8; + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id); + } else { + val |= (map->sc & 0x7F) << 9; + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id); + } + + mcs_reg_write(mcs, reg, val); +} + +void mcs_ena_dis_flowid_entry(struct mcs *mcs, int flow_id, int dir, int ena) +{ + u64 reg, val; + + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0; + if (flow_id > 63) + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1; + } else { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0; + if (flow_id > 63) + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1; + } + + /* Enable/Disable the tcam entry */ + if (ena) + val = mcs_reg_read(mcs, reg) | BIT_ULL(flow_id); + else + val = mcs_reg_read(mcs, reg) & ~BIT_ULL(flow_id); + + mcs_reg_write(mcs, reg, val); +} + +void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int flow_id, int dir) +{ + int reg_id; + u64 reg; + + if (dir == MCS_RX) { + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id); + mcs_reg_write(mcs, reg, data[reg_id]); + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, mask[reg_id]); + } + } else { + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id); + mcs_reg_write(mcs, reg, data[reg_id]); + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, mask[reg_id]); + } + } +} + +int mcs_install_flowid_bypass_entry(struct mcs *mcs) +{ + int flow_id, secy_id, reg_id; + struct secy_mem_map map; + u64 reg, plcy = 0; + + /* Flow entry */ + flow_id = mcs->hw->tcam_entries - MCS_RSRC_RSVD_CNT; + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0)); + } + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0)); + } + /* secy */ + secy_id = mcs->hw->secy_entries - MCS_RSRC_RSVD_CNT; + + /* Set validate frames to NULL and enable control port */ + plcy = 0x7ull; + if (mcs->hw->mcs_blks > 1) + plcy = BIT_ULL(0) | 0x3ull << 4; + mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_RX); + + /* Enable control port and set mtu to max */ + plcy = BIT_ULL(0) | GENMASK_ULL(43, 28); + if (mcs->hw->mcs_blks > 1) + plcy = BIT_ULL(0) | GENMASK_ULL(63, 48); + mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_TX); + + /* Map flowid to secy */ + map.secy = secy_id; + map.ctrl_pkt = 0; + map.flow_id = flow_id; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_RX); + map.sc = secy_id; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_TX); + + /* Enable Flowid entry */ + mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_RX, true); + mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_TX, true); + return 0; +} + +void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir) +{ + struct mcs_rsrc_map *map; + int flow_id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* Clear secy memory to zero */ + mcs_secy_plcy_write(mcs, 0, secy_id, dir); + + /* Disable the tcam entry using this secy */ + for (flow_id = 0; flow_id < map->flow_ids.max; flow_id++) { + if (map->flowid2secy_map[flow_id] != secy_id) + continue; + mcs_ena_dis_flowid_entry(mcs, flow_id, dir, false); + } +} + +int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc) +{ + int rsrc_id; + + if (!rsrc->bmap) + return -EINVAL; + + rsrc_id = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, offset, 1, 0); + if (rsrc_id >= rsrc->max) + return -ENOSPC; + + bitmap_set(rsrc->bmap, rsrc_id, 1); + pf_map[rsrc_id] = pcifunc; + + return rsrc_id; +} + +int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + u64 dis, reg; + int id, rc; + + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE; + map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx; + + if (req->all) { + for (id = 0; id < map->ctrlpktrule.max; id++) { + if (map->ctrlpktrule2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, id, pcifunc); + dis = mcs_reg_read(mcs, reg); + dis &= ~BIT_ULL(id); + mcs_reg_write(mcs, reg, dis); + } + return 0; + } + + rc = mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, req->rule_idx, pcifunc); + dis = mcs_reg_read(mcs, reg); + dis &= ~BIT_ULL(req->rule_idx); + mcs_reg_write(mcs, reg, dis); + + return rc; +} + +int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req) +{ + u64 reg, enb; + u64 idx; + + switch (req->rule_type) { + case MCS_CTRL_PKT_RULE_TYPE_ETH: + req->data0 &= GENMASK(15, 0); + if (req->data0 != ETH_P_PAE) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_ETYPE_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(idx) : + MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(idx); + + mcs_reg_write(mcs, reg, req->data0); + break; + case MCS_CTRL_PKT_RULE_TYPE_DA: + if (!(req->data0 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_DA_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_DAX(idx) : + MCSX_PEX_TX_SLAVE_RULE_DAX(idx); + + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + break; + case MCS_CTRL_PKT_RULE_TYPE_RANGE: + if (!(req->data0 & BIT_ULL(40)) || !(req->data1 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_DA_RANGE_RULE_OFFSET; + if (req->dir == MCS_RX) { + reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + } else { + reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + } + break; + case MCS_CTRL_PKT_RULE_TYPE_COMBO: + req->data2 &= GENMASK(15, 0); + if (req->data2 != ETH_P_PAE || !(req->data0 & BIT_ULL(40)) || + !(req->data1 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_COMBO_RULE_OFFSET; + if (req->dir == MCS_RX) { + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(idx); + mcs_reg_write(mcs, reg, req->data2); + } else { + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(idx); + mcs_reg_write(mcs, reg, req->data2); + } + break; + case MCS_CTRL_PKT_RULE_TYPE_MAC: + if (!(req->data0 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_MAC_EN_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_MAC : + MCSX_PEX_TX_SLAVE_RULE_MAC; + + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + break; + } + + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE; + + enb = mcs_reg_read(mcs, reg); + enb |= BIT_ULL(req->rule_idx); + mcs_reg_write(mcs, reg, enb); + + return 0; +} + +int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc) +{ + /* Check if the rsrc_id is mapped to PF/VF */ + if (pf_map[rsrc_id] != pcifunc) + return -EINVAL; + + rvu_free_rsrc(rsrc, rsrc_id); + pf_map[rsrc_id] = 0; + return 0; +} + +/* Free all the cam resources mapped to pf */ +int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* free tcam entries */ + for (id = 0; id < map->flow_ids.max; id++) { + if (map->flowid2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, + id, pcifunc); + mcs_ena_dis_flowid_entry(mcs, id, dir, false); + } + + /* free secy entries */ + for (id = 0; id < map->secy.max; id++) { + if (map->secy2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->secy, map->secy2pf_map, + id, pcifunc); + mcs_clear_secy_plcy(mcs, id, dir); + } + + /* free sc entries */ + for (id = 0; id < map->secy.max; id++) { + if (map->sc2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->sc, map->sc2pf_map, id, pcifunc); + + /* Disable SC CAM only on RX side */ + if (dir == MCS_RX) + mcs_ena_dis_sc_cam_entry(mcs, id, false); + } + + /* free sa entries */ + for (id = 0; id < map->sa.max; id++) { + if (map->sa2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->sa, map->sa2pf_map, id, pcifunc); + } + return 0; +} + +int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc) +{ + int rsrc_id; + + rsrc_id = rvu_alloc_rsrc(rsrc); + if (rsrc_id < 0) + return -ENOMEM; + pf_map[rsrc_id] = pcifunc; + return rsrc_id; +} + +int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flow_id, u8 *secy_id, + u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *flow_id = id; + + id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *secy_id = id; + + id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sc_id = id; + + id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sa1_id = id; + + id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sa2_id = id; + + return 0; +} + +static void cn10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + + if (mcs->tx_sa_active[sc]) + /* SA_index1 was used and got expired */ + event.sa_id = (val >> 9) & 0xFF; + else + /* SA_index0 was used and got expired */ + event.sa_id = val & 0xFF; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void cn10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val, status; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT; + + /* TX SA interrupt is raised only if autorekey is enabled. + * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if + * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies + * SA in SA_index1 got expired else SA in SA_index0 got expired. + */ + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + /* Auto rekey is enable */ + if (!((val >> 18) & 0x1)) + continue; + + status = (val >> 21) & 0x1; + + /* Check if tx_sa_active status had changed */ + if (status == mcs->tx_sa_active[sc]) + continue; + /* SA_index0 is expired */ + if (status) + event.sa_id = val & 0xFF; + else + event.sa_id = (val >> 9) & 0xFF; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void mcs_rx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + int sa, reg; + u64 intr; + + /* Check expired SAs */ + for (reg = 0; reg < (mcs->hw->sa_entries / 64); reg++) { + /* Bit high in *PN_THRESH_REACHEDX implies + * corresponding SAs are expired. + */ + intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(reg)); + for (sa = 0; sa < 64; sa++) { + if (!(intr & BIT_ULL(sa))) + continue; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_RX_PN_THRESH_REACHED_INT; + event.sa_id = sa + (reg * 64); + event.pcifunc = mcs->rx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } + } +} + +static void mcs_rx_misc_intr_handler(struct mcs *mcs, u64 intr) +{ + struct mcs_intr_event event = { 0 }; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + if (intr & MCS_CPM_RX_INT_SECTAG_V_EQ1) + event.intr_mask = MCS_CPM_RX_SECTAG_V_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SL_GTE48) + event.intr_mask |= MCS_CPM_RX_SECTAG_SL_GTE48_INT; + if (intr & MCS_CPM_RX_INT_ES_EQ1_SC_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT; + if (intr & MCS_CPM_RX_INT_PACKET_XPN_EQ0) + event.intr_mask |= MCS_CPM_RX_PACKET_XPN_EQ0_INT; + + mcs_add_intr_wq_entry(mcs, &event); +} + +static void mcs_tx_misc_intr_handler(struct mcs *mcs, u64 intr) +{ + struct mcs_intr_event event = { 0 }; + + if (!(intr & MCS_CPM_TX_INT_SA_NOT_VALID)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + event.intr_mask = MCS_CPM_TX_SA_NOT_VALID_INT; + + mcs_add_intr_wq_entry(mcs, &event); +} + +static void mcs_bbe_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir) +{ + struct mcs_intr_event event = { 0 }; + int i; + + if (!(intr & MCS_BBE_INT_MASK)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + for (i = 0; i < MCS_MAX_BBE_INT; i++) { + if (!(intr & BIT_ULL(i))) + continue; + + /* Lower nibble denotes data fifo overflow interrupts and + * upper nibble indicates policy fifo overflow interrupts. + */ + if (intr & 0xFULL) + event.intr_mask = (dir == MCS_RX) ? + MCS_BBE_RX_DFIFO_OVERFLOW_INT : + MCS_BBE_TX_DFIFO_OVERFLOW_INT; + else + event.intr_mask = (dir == MCS_RX) ? + MCS_BBE_RX_PLFIFO_OVERFLOW_INT : + MCS_BBE_RX_PLFIFO_OVERFLOW_INT; + + /* Notify the lmac_id info which ran into BBE fatal error */ + event.lmac_id = i & 0x3ULL; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void mcs_pab_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir) +{ + struct mcs_intr_event event = { 0 }; + int i; + + if (!(intr & MCS_PAB_INT_MASK)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + for (i = 0; i < MCS_MAX_PAB_INT; i++) { + if (!(intr & BIT_ULL(i))) + continue; + + event.intr_mask = (dir == MCS_RX) ? MCS_PAB_RX_CHAN_OVERFLOW_INT : + MCS_PAB_TX_CHAN_OVERFLOW_INT; + + /* Notify the lmac_id info which ran into PAB fatal error */ + event.lmac_id = i; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static irqreturn_t mcs_ip_intr_handler(int irq, void *mcs_irq) +{ + struct mcs *mcs = (struct mcs *)mcs_irq; + u64 intr, cpm_intr, bbe_intr, pab_intr; + + /* Disable and clear the interrupt */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1C, BIT_ULL(0)); + mcs_reg_write(mcs, MCSX_IP_INT, BIT_ULL(0)); + + /* Check which block has interrupt*/ + intr = mcs_reg_read(mcs, MCSX_TOP_SLAVE_INT_SUM); + + /* CPM RX */ + if (intr & MCS_CPM_RX_INT_ENA) { + /* Check for PN thresh interrupt bit */ + cpm_intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_RX_INT); + + if (cpm_intr & MCS_CPM_RX_INT_PN_THRESH_REACHED) + mcs_rx_pn_thresh_reached_handler(mcs); + + if (cpm_intr & MCS_CPM_RX_INT_ALL) + mcs_rx_misc_intr_handler(mcs, cpm_intr); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT, cpm_intr); + } + + /* CPM TX */ + if (intr & MCS_CPM_TX_INT_ENA) { + cpm_intr = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_INT); + + if (cpm_intr & MCS_CPM_TX_INT_PN_THRESH_REACHED) { + if (mcs->hw->mcs_blks > 1) + cnf10kb_mcs_tx_pn_thresh_reached_handler(mcs); + else + cn10kb_mcs_tx_pn_thresh_reached_handler(mcs); + } + + if (cpm_intr & MCS_CPM_TX_INT_SA_NOT_VALID) + mcs_tx_misc_intr_handler(mcs, cpm_intr); + + if (cpm_intr & MCS_CPM_TX_INT_PACKET_XPN_EQ0) { + if (mcs->hw->mcs_blks > 1) + cnf10kb_mcs_tx_pn_wrapped_handler(mcs); + else + cn10kb_mcs_tx_pn_wrapped_handler(mcs); + } + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT, cpm_intr); + } + + /* BBE RX */ + if (intr & MCS_BBE_RX_INT_ENA) { + bbe_intr = mcs_reg_read(mcs, MCSX_BBE_RX_SLAVE_BBE_INT); + mcs_bbe_intr_handler(mcs, bbe_intr, MCS_RX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT, bbe_intr); + } + + /* BBE TX */ + if (intr & MCS_BBE_TX_INT_ENA) { + bbe_intr = mcs_reg_read(mcs, MCSX_BBE_TX_SLAVE_BBE_INT); + mcs_bbe_intr_handler(mcs, bbe_intr, MCS_TX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT, bbe_intr); + } + + /* PAB RX */ + if (intr & MCS_PAB_RX_INT_ENA) { + pab_intr = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PAB_INT); + mcs_pab_intr_handler(mcs, pab_intr, MCS_RX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT, pab_intr); + } + + /* PAB TX */ + if (intr & MCS_PAB_TX_INT_ENA) { + pab_intr = mcs_reg_read(mcs, MCSX_PAB_TX_SLAVE_PAB_INT); + mcs_pab_intr_handler(mcs, pab_intr, MCS_TX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT, pab_intr); + } + + /* Enable the interrupt */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0)); + + return IRQ_HANDLED; +} + +static void *alloc_mem(struct mcs *mcs, int n) +{ + return devm_kcalloc(mcs->dev, n, sizeof(u16), GFP_KERNEL); +} + +static int mcs_alloc_struct_mem(struct mcs *mcs, struct mcs_rsrc_map *res) +{ + struct hwinfo *hw = mcs->hw; + int err; + + res->flowid2pf_map = alloc_mem(mcs, hw->tcam_entries); + if (!res->flowid2pf_map) + return -ENOMEM; + + res->secy2pf_map = alloc_mem(mcs, hw->secy_entries); + if (!res->secy2pf_map) + return -ENOMEM; + + res->sc2pf_map = alloc_mem(mcs, hw->sc_entries); + if (!res->sc2pf_map) + return -ENOMEM; + + res->sa2pf_map = alloc_mem(mcs, hw->sa_entries); + if (!res->sa2pf_map) + return -ENOMEM; + + res->flowid2secy_map = alloc_mem(mcs, hw->tcam_entries); + if (!res->flowid2secy_map) + return -ENOMEM; + + res->ctrlpktrule2pf_map = alloc_mem(mcs, MCS_MAX_CTRLPKT_RULES); + if (!res->ctrlpktrule2pf_map) + return -ENOMEM; + + res->flow_ids.max = hw->tcam_entries - MCS_RSRC_RSVD_CNT; + err = rvu_alloc_bitmap(&res->flow_ids); + if (err) + return err; + + res->secy.max = hw->secy_entries - MCS_RSRC_RSVD_CNT; + err = rvu_alloc_bitmap(&res->secy); + if (err) + return err; + + res->sc.max = hw->sc_entries; + err = rvu_alloc_bitmap(&res->sc); + if (err) + return err; + + res->sa.max = hw->sa_entries; + err = rvu_alloc_bitmap(&res->sa); + if (err) + return err; + + res->ctrlpktrule.max = MCS_MAX_CTRLPKT_RULES; + err = rvu_alloc_bitmap(&res->ctrlpktrule); + if (err) + return err; + + return 0; +} + +static int mcs_register_interrupts(struct mcs *mcs) +{ + int ret = 0; + + mcs->num_vec = pci_msix_vec_count(mcs->pdev); + + ret = pci_alloc_irq_vectors(mcs->pdev, mcs->num_vec, + mcs->num_vec, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(mcs->dev, "MCS Request for %d msix vector failed err:%d\n", + mcs->num_vec, ret); + return ret; + } + + ret = request_irq(pci_irq_vector(mcs->pdev, MCS_INT_VEC_IP), + mcs_ip_intr_handler, 0, "MCS_IP", mcs); + if (ret) { + dev_err(mcs->dev, "MCS IP irq registration failed\n"); + goto exit; + } + + /* MCS enable IP interrupts */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0)); + + /* Enable CPM Rx/Tx interrupts */ + mcs_reg_write(mcs, MCSX_TOP_SLAVE_INT_SUM_ENB, + MCS_CPM_RX_INT_ENA | MCS_CPM_TX_INT_ENA | + MCS_BBE_RX_INT_ENA | MCS_BBE_TX_INT_ENA | + MCS_PAB_RX_INT_ENA | MCS_PAB_TX_INT_ENA); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT_ENB, 0x7ULL); + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT_ENB, 0x7FULL); + + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_ENB, 0xff); + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_ENB, 0xff); + + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_ENB, 0xff); + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_ENB, 0xff); + + mcs->tx_sa_active = alloc_mem(mcs, mcs->hw->sc_entries); + if (!mcs->tx_sa_active) { + ret = -ENOMEM; + goto exit; + } + + return ret; +exit: + pci_free_irq_vectors(mcs->pdev); + mcs->num_vec = 0; + return ret; +} + +int mcs_get_blkcnt(void) +{ + struct mcs *mcs; + int idmax = -ENODEV; + + /* Check MCS block is present in hardware */ + if (!pci_dev_present(mcs_id_table)) + return 0; + + list_for_each_entry(mcs, &mcs_list, mcs_list) + if (mcs->mcs_id > idmax) + idmax = mcs->mcs_id; + + if (idmax < 0) + return 0; + + return idmax + 1; +} + +struct mcs *mcs_get_pdata(int mcs_id) +{ + struct mcs *mcs_dev; + + list_for_each_entry(mcs_dev, &mcs_list, mcs_list) { + if (mcs_dev->mcs_id == mcs_id) + return mcs_dev; + } + return NULL; +} + +void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req) +{ + u64 val = 0; + + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id), + req->port_mode & MCS_PORT_MODE_MASK); + + req->cstm_tag_rel_mode_sel &= 0x3; + + if (mcs->hw->mcs_blks > 1) { + req->fifo_skid &= MCS_PORT_FIFO_SKID_MASK; + val = (u32)req->fifo_skid << 0x10; + val |= req->fifo_skid; + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id), val); + mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id), + req->cstm_tag_rel_mode_sel); + val = mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION); + + if (req->custom_hdr_enb) + val |= BIT_ULL(req->port_id); + else + val &= ~BIT_ULL(req->port_id); + + mcs_reg_write(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION, val); + } else { + val = mcs_reg_read(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id)); + val |= (req->cstm_tag_rel_mode_sel << 2); + mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id), val); + } +} + +void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp) +{ + u64 reg = 0; + + rsp->port_mode = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id)) & + MCS_PORT_MODE_MASK; + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id); + rsp->fifo_skid = mcs_reg_read(mcs, reg) & MCS_PORT_FIFO_SKID_MASK; + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id); + rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) & 0x3; + if (mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION) & BIT_ULL(req->port_id)) + rsp->custom_hdr_enb = 1; + } else { + reg = MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id); + rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) >> 2; + } + + rsp->port_id = req->port_id; + rsp->mcs_id = req->mcs_id; +} + +void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp) +{ + u64 reg = 0, val = 0; + u8 idx; + + for (idx = 0; idx < MCS_MAX_CUSTOM_TAGS; idx++) { + if (mcs->hw->mcs_blks > 1) + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(idx) : + MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(idx); + else + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_VLAN_CFGX(idx) : + MCSX_PEX_TX_SLAVE_VLAN_CFGX(idx); + + val = mcs_reg_read(mcs, reg); + if (mcs->hw->mcs_blks > 1) { + rsp->cstm_etype[idx] = val & GENMASK(15, 0); + rsp->cstm_indx[idx] = (val >> 0x16) & 0x3; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_ETYPE_ENABLE : + MCSX_PEX_TX_SLAVE_ETYPE_ENABLE; + rsp->cstm_etype_en = mcs_reg_read(mcs, reg) & 0xFF; + } else { + rsp->cstm_etype[idx] = (val >> 0x1) & GENMASK(15, 0); + rsp->cstm_indx[idx] = (val >> 0x11) & 0x3; + rsp->cstm_etype_en |= (val & 0x1) << idx; + } + } + + rsp->mcs_id = req->mcs_id; + rsp->dir = req->dir; +} + +void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset) +{ + u64 reg = MCSX_MCS_TOP_SLAVE_PORT_RESET(port_id); + + mcs_reg_write(mcs, reg, reset & 0x1); +} + +/* Set lmac to bypass/operational mode */ +void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode) +{ + u64 reg; + + reg = MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(lmac_id * 2); + mcs_reg_write(mcs, reg, (u64)mode); +} + +void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn) +{ + u64 reg; + + if (pn->dir == MCS_RX) + reg = pn->xpn ? MCSX_CPM_RX_SLAVE_XPN_THRESHOLD : MCSX_CPM_RX_SLAVE_PN_THRESHOLD; + else + reg = pn->xpn ? MCSX_CPM_TX_SLAVE_XPN_THRESHOLD : MCSX_CPM_TX_SLAVE_PN_THRESHOLD; + + mcs_reg_write(mcs, reg, pn->threshold); +} + +void cn10kb_mcs_parser_cfg(struct mcs *mcs) +{ + u64 reg, val; + + /* VLAN CTag */ + val = BIT_ULL(0) | (0x8100ull & 0xFFFF) << 1 | BIT_ULL(17); + /* RX */ + reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(0); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(0); + mcs_reg_write(mcs, reg, val); + + /* VLAN STag */ + val = BIT_ULL(0) | (0x88a8ull & 0xFFFF) << 1 | BIT_ULL(18); + /* RX */ + reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(1); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(1); + mcs_reg_write(mcs, reg, val); +} + +static void mcs_lmac_init(struct mcs *mcs, int lmac_id) +{ + u64 reg; + + /* Port mode 25GB */ + reg = MCSX_PAB_RX_SLAVE_PORT_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0xe000e); + return; + } + + reg = MCSX_PAB_TX_SLAVE_PORT_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0); +} + +int mcs_set_lmac_channels(int mcs_id, u16 base) +{ + struct mcs *mcs; + int lmac; + u64 cfg; + + mcs = mcs_get_pdata(mcs_id); + if (!mcs) + return -ENODEV; + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) { + cfg = mcs_reg_read(mcs, MCSX_LINK_LMACX_CFG(lmac)); + cfg &= ~(MCSX_LINK_LMAC_BASE_MASK | MCSX_LINK_LMAC_RANGE_MASK); + cfg |= FIELD_PREP(MCSX_LINK_LMAC_RANGE_MASK, ilog2(16)); + cfg |= FIELD_PREP(MCSX_LINK_LMAC_BASE_MASK, base); + mcs_reg_write(mcs, MCSX_LINK_LMACX_CFG(lmac), cfg); + base += 16; + } + return 0; +} + +static int mcs_x2p_calibration(struct mcs *mcs) +{ + unsigned long timeout = jiffies + usecs_to_jiffies(20000); + int i, err = 0; + u64 val; + + /* set X2P calibration */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + val |= BIT_ULL(5); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + + /* Wait for calibration to complete */ + while (!(mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS) & BIT_ULL(0))) { + if (time_before(jiffies, timeout)) { + usleep_range(80, 100); + continue; + } else { + err = -EBUSY; + dev_err(mcs->dev, "MCS X2P calibration failed..ignoring\n"); + return err; + } + } + + val = mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS); + for (i = 0; i < mcs->hw->mcs_x2p_intf; i++) { + if (val & BIT_ULL(1 + i)) + continue; + err = -EBUSY; + dev_err(mcs->dev, "MCS:%d didn't respond to X2P calibration\n", i); + } + /* Clear X2P calibrate */ + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, mcs_reg_read(mcs, MCSX_MIL_GLOBAL) & ~BIT_ULL(5)); + + return err; +} + +static void mcs_set_external_bypass(struct mcs *mcs, u8 bypass) +{ + u64 val; + + /* Set MCS to external bypass */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + if (bypass) + val |= BIT_ULL(6); + else + val &= ~BIT_ULL(6); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); +} + +static void mcs_global_cfg(struct mcs *mcs) +{ + /* Disable external bypass */ + mcs_set_external_bypass(mcs, false); + + /* Reset TX/RX stats memory */ + mcs_reg_write(mcs, MCSX_CSE_RX_SLAVE_STATS_CLEAR, 0x1F); + mcs_reg_write(mcs, MCSX_CSE_TX_SLAVE_STATS_CLEAR, 0x1F); + + /* Set MCS to perform standard IEEE802.1AE macsec processing */ + if (mcs->hw->mcs_blks == 1) { + mcs_reg_write(mcs, MCSX_IP_MODE, BIT_ULL(3)); + return; + } + + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_ENTRY, 0xe4); + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_LEN, 4); +} + +void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs) +{ + struct hwinfo *hw = mcs->hw; + + hw->tcam_entries = 128; /* TCAM entries */ + hw->secy_entries = 128; /* SecY entries */ + hw->sc_entries = 128; /* SC CAM entries */ + hw->sa_entries = 256; /* SA entries */ + hw->lmac_cnt = 20; /* lmacs/ports per mcs block */ + hw->mcs_x2p_intf = 5; /* x2p clabration intf */ + hw->mcs_blks = 1; /* MCS blocks */ +} + +static struct mcs_ops cn10kb_mcs_ops = { + .mcs_set_hw_capabilities = cn10kb_mcs_set_hw_capabilities, + .mcs_parser_cfg = cn10kb_mcs_parser_cfg, + .mcs_tx_sa_mem_map_write = cn10kb_mcs_tx_sa_mem_map_write, + .mcs_rx_sa_mem_map_write = cn10kb_mcs_rx_sa_mem_map_write, + .mcs_flowid_secy_map = cn10kb_mcs_flowid_secy_map, +}; + +static int mcs_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + int lmac, err = 0; + struct mcs *mcs; + + mcs = devm_kzalloc(dev, sizeof(*mcs), GFP_KERNEL); + if (!mcs) + return -ENOMEM; + + mcs->hw = devm_kzalloc(dev, sizeof(struct hwinfo), GFP_KERNEL); + if (!mcs->hw) + return -ENOMEM; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto exit; + } + + mcs->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); + if (!mcs->reg_base) { + dev_err(dev, "mcs: Cannot map CSR memory space, aborting\n"); + err = -ENOMEM; + goto exit; + } + + pci_set_drvdata(pdev, mcs); + mcs->pdev = pdev; + mcs->dev = &pdev->dev; + + if (pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B) + mcs->mcs_ops = &cn10kb_mcs_ops; + else + mcs->mcs_ops = cnf10kb_get_mac_ops(); + + /* Set hardware capabilities */ + mcs->mcs_ops->mcs_set_hw_capabilities(mcs); + + mcs_global_cfg(mcs); + + /* Perform X2P clibration */ + err = mcs_x2p_calibration(mcs); + if (err) + goto err_x2p; + + mcs->mcs_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) + & MCS_ID_MASK; + + /* Set mcs tx side resources */ + err = mcs_alloc_struct_mem(mcs, &mcs->tx); + if (err) + goto err_x2p; + + /* Set mcs rx side resources */ + err = mcs_alloc_struct_mem(mcs, &mcs->rx); + if (err) + goto err_x2p; + + /* per port config */ + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) + mcs_lmac_init(mcs, lmac); + + /* Parser configuration */ + mcs->mcs_ops->mcs_parser_cfg(mcs); + + err = mcs_register_interrupts(mcs); + if (err) + goto exit; + + list_add(&mcs->mcs_list, &mcs_list); + mutex_init(&mcs->stats_lock); + + return 0; + +err_x2p: + /* Enable external bypass */ + mcs_set_external_bypass(mcs, true); +exit: + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void mcs_remove(struct pci_dev *pdev) +{ + struct mcs *mcs = pci_get_drvdata(pdev); + + /* Set MCS to external bypass */ + mcs_set_external_bypass(mcs, true); + pci_free_irq_vectors(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +struct pci_driver mcs_driver = { + .name = DRV_NAME, + .id_table = mcs_id_table, + .probe = mcs_probe, + .remove = mcs_remove, +}; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h new file mode 100644 index 000000000000..64dc2b80e15d --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h @@ -0,0 +1,246 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell CN10K MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#ifndef MCS_H +#define MCS_H + +#include <linux/bits.h> +#include "rvu.h" + +#define PCI_DEVID_CN10K_MCS 0xA096 + +#define MCSX_LINK_LMAC_RANGE_MASK GENMASK_ULL(19, 16) +#define MCSX_LINK_LMAC_BASE_MASK GENMASK_ULL(11, 0) + +#define MCS_ID_MASK 0x7 +#define MCS_MAX_PFS 128 + +#define MCS_PORT_MODE_MASK 0x3 +#define MCS_PORT_FIFO_SKID_MASK 0x3F +#define MCS_MAX_CUSTOM_TAGS 0x8 + +#define MCS_CTRLPKT_ETYPE_RULE_MAX 8 +#define MCS_CTRLPKT_DA_RULE_MAX 8 +#define MCS_CTRLPKT_DA_RANGE_RULE_MAX 4 +#define MCS_CTRLPKT_COMBO_RULE_MAX 4 +#define MCS_CTRLPKT_MAC_RULE_MAX 1 + +#define MCS_MAX_CTRLPKT_RULES (MCS_CTRLPKT_ETYPE_RULE_MAX + \ + MCS_CTRLPKT_DA_RULE_MAX + \ + MCS_CTRLPKT_DA_RANGE_RULE_MAX + \ + MCS_CTRLPKT_COMBO_RULE_MAX + \ + MCS_CTRLPKT_MAC_RULE_MAX) + +#define MCS_CTRLPKT_ETYPE_RULE_OFFSET 0 +#define MCS_CTRLPKT_DA_RULE_OFFSET 8 +#define MCS_CTRLPKT_DA_RANGE_RULE_OFFSET 16 +#define MCS_CTRLPKT_COMBO_RULE_OFFSET 20 +#define MCS_CTRLPKT_MAC_EN_RULE_OFFSET 24 + +/* Reserved resources for default bypass entry */ +#define MCS_RSRC_RSVD_CNT 1 + +/* MCS Interrupt Vector Enumeration */ +enum mcs_int_vec_e { + MCS_INT_VEC_MIL_RX_GBL = 0x0, + MCS_INT_VEC_MIL_RX_LMACX = 0x1, + MCS_INT_VEC_MIL_TX_LMACX = 0x5, + MCS_INT_VEC_HIL_RX_GBL = 0x9, + MCS_INT_VEC_HIL_RX_LMACX = 0xa, + MCS_INT_VEC_HIL_TX_GBL = 0xe, + MCS_INT_VEC_HIL_TX_LMACX = 0xf, + MCS_INT_VEC_IP = 0x13, + MCS_INT_VEC_CNT = 0x14, +}; + +#define MCS_MAX_BBE_INT 8ULL +#define MCS_BBE_INT_MASK 0xFFULL + +#define MCS_MAX_PAB_INT 4ULL +#define MCS_PAB_INT_MASK 0xFULL + +#define MCS_BBE_RX_INT_ENA BIT_ULL(0) +#define MCS_BBE_TX_INT_ENA BIT_ULL(1) +#define MCS_CPM_RX_INT_ENA BIT_ULL(2) +#define MCS_CPM_TX_INT_ENA BIT_ULL(3) +#define MCS_PAB_RX_INT_ENA BIT_ULL(4) +#define MCS_PAB_TX_INT_ENA BIT_ULL(5) + +#define MCS_CPM_TX_INT_PACKET_XPN_EQ0 BIT_ULL(0) +#define MCS_CPM_TX_INT_PN_THRESH_REACHED BIT_ULL(1) +#define MCS_CPM_TX_INT_SA_NOT_VALID BIT_ULL(2) + +#define MCS_CPM_RX_INT_SECTAG_V_EQ1 BIT_ULL(0) +#define MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 BIT_ULL(1) +#define MCS_CPM_RX_INT_SL_GTE48 BIT_ULL(2) +#define MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 BIT_ULL(3) +#define MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 BIT_ULL(4) +#define MCS_CPM_RX_INT_PACKET_XPN_EQ0 BIT_ULL(5) +#define MCS_CPM_RX_INT_PN_THRESH_REACHED BIT_ULL(6) + +#define MCS_CPM_RX_INT_ALL (MCS_CPM_RX_INT_SECTAG_V_EQ1 | \ + MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 | \ + MCS_CPM_RX_INT_SL_GTE48 | \ + MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 | \ + MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 | \ + MCS_CPM_RX_INT_PACKET_XPN_EQ0 | \ + MCS_CPM_RX_INT_PN_THRESH_REACHED) + +struct mcs_pfvf { + u64 intr_mask; /* Enabled Interrupt mask */ +}; + +struct mcs_intr_event { + u16 pcifunc; + u64 intr_mask; + u64 sa_id; + u8 mcs_id; + u8 lmac_id; +}; + +struct mcs_intrq_entry { + struct list_head node; + struct mcs_intr_event intr_event; +}; + +struct secy_mem_map { + u8 flow_id; + u8 secy; + u8 ctrl_pkt; + u8 sc; + u64 sci; +}; + +struct mcs_rsrc_map { + u16 *flowid2pf_map; + u16 *secy2pf_map; + u16 *sc2pf_map; + u16 *sa2pf_map; + u16 *flowid2secy_map; /* bitmap flowid mapped to secy*/ + u16 *ctrlpktrule2pf_map; + struct rsrc_bmap flow_ids; + struct rsrc_bmap secy; + struct rsrc_bmap sc; + struct rsrc_bmap sa; + struct rsrc_bmap ctrlpktrule; +}; + +struct hwinfo { + u8 tcam_entries; + u8 secy_entries; + u8 sc_entries; + u16 sa_entries; + u8 mcs_x2p_intf; + u8 lmac_cnt; + u8 mcs_blks; + unsigned long lmac_bmap; /* bitmap of enabled mcs lmac */ +}; + +struct mcs { + void __iomem *reg_base; + struct pci_dev *pdev; + struct device *dev; + struct hwinfo *hw; + struct mcs_rsrc_map tx; + struct mcs_rsrc_map rx; + u16 pf_map[MCS_MAX_PFS]; /* List of PCIFUNC mapped to MCS */ + u8 mcs_id; + struct mcs_ops *mcs_ops; + struct list_head mcs_list; + /* Lock for mcs stats */ + struct mutex stats_lock; + struct mcs_pfvf *pf; + struct mcs_pfvf *vf; + u16 num_vec; + void *rvu; + u16 *tx_sa_active; +}; + +struct mcs_ops { + void (*mcs_set_hw_capabilities)(struct mcs *mcs); + void (*mcs_parser_cfg)(struct mcs *mcs); + void (*mcs_tx_sa_mem_map_write)(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); + void (*mcs_rx_sa_mem_map_write)(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); + void (*mcs_flowid_secy_map)(struct mcs *mcs, struct secy_mem_map *map, int dir); +}; + +extern struct pci_driver mcs_driver; + +static inline void mcs_reg_write(struct mcs *mcs, u64 offset, u64 val) +{ + writeq(val, mcs->reg_base + offset); +} + +static inline u64 mcs_reg_read(struct mcs *mcs, u64 offset) +{ + return readq(mcs->reg_base + offset); +} + +/* MCS APIs */ +struct mcs *mcs_get_pdata(int mcs_id); +int mcs_get_blkcnt(void); +int mcs_set_lmac_channels(int mcs_id, u16 base); +int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc); +int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc); +int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flowid, u8 *secy_id, + u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir); +int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc); +void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir); +void mcs_ena_dis_flowid_entry(struct mcs *mcs, int id, int dir, int ena); +void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int id, int ena); +void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int id, int dir); +void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int id, int dir); +void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id); +void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa, int dir); +void mcs_map_sc_to_sa(struct mcs *mcs, u64 *sa_map, int sc, int dir); +void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir); +void mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn); +int mcs_install_flowid_bypass_entry(struct mcs *mcs); +void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode); +void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset); +void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req); +void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp); +void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp); +int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc); +int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req); +int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req); + +/* CN10K-B APIs */ +void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs); +void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void cn10kb_mcs_parser_cfg(struct mcs *mcs); + +/* CNF10K-B APIs */ +struct mcs_ops *cnf10kb_get_mac_ops(void); +void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs); +void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void cnf10kb_mcs_parser_cfg(struct mcs *mcs); +void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs); +void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs); + +/* Stats APIs */ +void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, int id, int dir); +void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir); +void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, int id, int dir); +void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, int id, int dir); +void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id); +void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id); +void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir); +int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir); +int mcs_set_force_clk_en(struct mcs *mcs, bool set); + +int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event); + +#endif /* MCS_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c new file mode 100644 index 000000000000..7b6205414428 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include "mcs.h" +#include "mcs_reg.h" + +static struct mcs_ops cnf10kb_mcs_ops = { + .mcs_set_hw_capabilities = cnf10kb_mcs_set_hw_capabilities, + .mcs_parser_cfg = cnf10kb_mcs_parser_cfg, + .mcs_tx_sa_mem_map_write = cnf10kb_mcs_tx_sa_mem_map_write, + .mcs_rx_sa_mem_map_write = cnf10kb_mcs_rx_sa_mem_map_write, + .mcs_flowid_secy_map = cnf10kb_mcs_flowid_secy_map, +}; + +struct mcs_ops *cnf10kb_get_mac_ops(void) +{ + return &cnf10kb_mcs_ops; +} + +void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs) +{ + struct hwinfo *hw = mcs->hw; + + hw->tcam_entries = 64; /* TCAM entries */ + hw->secy_entries = 64; /* SecY entries */ + hw->sc_entries = 64; /* SC CAM entries */ + hw->sa_entries = 128; /* SA entries */ + hw->lmac_cnt = 4; /* lmacs/ports per mcs block */ + hw->mcs_x2p_intf = 1; /* x2p clabration intf */ + hw->mcs_blks = 7; /* MCS blocks */ +} + +void cnf10kb_mcs_parser_cfg(struct mcs *mcs) +{ + u64 reg, val; + + /* VLAN Ctag */ + val = (0x8100ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(22); + + reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(0); + mcs_reg_write(mcs, reg, val); + + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(0); + mcs_reg_write(mcs, reg, val); + + /* VLAN STag */ + val = (0x88a8ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(23); + + /* RX */ + reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(1); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(1); + mcs_reg_write(mcs, reg, val); + + /* Enable custom tage 0 and 1 and sectag */ + val = BIT_ULL(0) | BIT_ULL(1) | BIT_ULL(12); + + reg = MCSX_PEX_RX_SLAVE_ETYPE_ENABLE; + mcs_reg_write(mcs, reg, val); + + reg = MCSX_PEX_TX_SLAVE_ETYPE_ENABLE; + mcs_reg_write(mcs, reg, val); +} + +void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir) +{ + u64 reg, val; + + val = (map->secy & 0x3F) | (map->ctrl_pkt & 0x1) << 6; + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id); + } else { + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id); + mcs_reg_write(mcs, reg, map->sci); + val |= (map->sc & 0x3F) << 7; + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(map->flow_id); + } + + mcs_reg_write(mcs, reg, val); +} + +void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map) +{ + u64 reg, val; + + val = (map->sa_index0 & 0x7F) | (map->sa_index1 & 0x7F) << 7; + + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id); + mcs_reg_write(mcs, reg, val); + + reg = MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0; + val = mcs_reg_read(mcs, reg); + + if (map->rekey_ena) + val |= BIT_ULL(map->sc_id); + else + val &= ~BIT_ULL(map->sc_id); + + mcs_reg_write(mcs, reg, val); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(map->sc_id), map->sa_index0_vld); + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(map->sc_id), map->sa_index1_vld); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(map->sc_id), map->tx_sa_active); +} + +void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map) +{ + u64 val, reg; + + val = (map->sa_index & 0x7F) | (map->sa_in_use << 7); + + reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an); + mcs_reg_write(mcs, reg, val); +} + +int mcs_set_force_clk_en(struct mcs *mcs, bool set) +{ + unsigned long timeout = jiffies + usecs_to_jiffies(2000); + u64 val; + + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + + if (set) { + val |= BIT_ULL(4); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + + /* Poll till mcsx_mil_ip_gbl_status.mcs_ip_stats_ready value is 1 */ + while (!(mcs_reg_read(mcs, MCSX_MIL_IP_GBL_STATUS) & BIT_ULL(0))) { + if (time_after(jiffies, timeout)) { + dev_err(mcs->dev, "MCS set force clk enable failed\n"); + break; + } + } + } else { + val &= ~BIT_ULL(4); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + } + + return 0; +} + +/* TX SA interrupt is raised only if autorekey is enabled. + * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if + * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies + * SA in SA_index1 got expired else SA in SA_index0 got expired. + */ +void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event; + struct rsrc_bmap *sc_bmap; + unsigned long rekey_ena; + u64 val, sa_status; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT; + + rekey_ena = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0); + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + /* Auto rekey is enable */ + if (!test_bit(sc, &rekey_ena)) + continue; + sa_status = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(sc)); + /* Check if tx_sa_active status had changed */ + if (sa_status == mcs->tx_sa_active[sc]) + continue; + + /* SA_index0 is expired */ + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + if (sa_status) + event.sa_id = val & 0x7F; + else + event.sa_id = (val >> 7) & 0x7F; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + + if (mcs->tx_sa_active[sc]) + /* SA_index1 was used and got expired */ + event.sa_id = (val >> 7) & 0x7F; + else + /* SA_index0 was used and got expired */ + event.sa_id = val & 0x7F; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h new file mode 100644 index 000000000000..c95a8b8f5eaf --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h @@ -0,0 +1,1102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#ifndef MCS_REG_H +#define MCS_REG_H + +#include <linux/bits.h> + +/* Registers */ +#define MCSX_IP_MODE 0x900c8ull +#define MCSX_MCS_TOP_SLAVE_PORT_RESET(a) ({ \ + u64 offset; \ + \ + offset = 0x408ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa28ull; \ + offset += (a) * 0x8ull; \ + offset; }) + + +#define MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x808ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa68ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_MIL_GLOBAL ({ \ + u64 offset; \ + \ + offset = 0x80000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60000ull; \ + offset; }) + +#define MCSX_MIL_RX_LMACX_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x900a8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x700a8ull; \ + offset += (a) * 0x800ull; \ + offset; }) + +#define MCSX_HIL_GLOBAL ({ \ + u64 offset; \ + \ + offset = 0xc0000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa0000ull; \ + offset; }) + +#define MCSX_LINK_LMACX_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x90000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x70000ull; \ + offset += (a) * 0x800ull; \ + offset; }) + +#define MCSX_MIL_RX_GBL_STATUS ({ \ + u64 offset; \ + \ + offset = 0x800c8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x600c8ull; \ + offset; }) + +#define MCSX_MIL_IP_GBL_STATUS ({ \ + u64 offset; \ + \ + offset = 0x800d0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x600d0ull; \ + offset; }) + +/* PAB */ +#define MCSX_PAB_RX_SLAVE_PORT_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x1718ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x280ull; \ + offset += (a) * 0x40ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PORT_CFGX(a) (0x2930ull + (a) * 0x40ull) + +/* PEX registers */ +#define MCSX_PEX_RX_SLAVE_VLAN_CFGX(a) (0x3b58ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_VLAN_CFGX(a) (0x46f8ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(a) (0x788ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_PORT_CONFIG(a) (0x4738ull + (a) * 0x8ull) +#define MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x3fc0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x558ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DAX(a) ({ \ + u64 offset; \ + \ + offset = 0x4000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x598ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4048ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5e0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4080ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x648ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4088ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x650ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(a) ({ \ + u64 offset; \ + \ + offset = 0x4090ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x658ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_MAC ({ \ + u64 offset; \ + \ + offset = 0x40e0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6d8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_ENABLE ({ \ + u64 offset; \ + \ + offset = 0x40e8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6e0ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x4b60ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x7d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DAX(a) ({ \ + u64 offset; \ + \ + offset = 0x4ba0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4be0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x858ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4be8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x860ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c20ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8c8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c28ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8d0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c30ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_MAC ({ \ + u64 offset; \ + \ + offset = 0x4c80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x958ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_ENABLE ({ \ + u64 offset; \ + \ + offset = 0x4c88ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x960ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION ({ \ + u64 offset; \ + \ + offset = 0x3b50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4c0ull; \ + offset; }) + +/* CNF10K-B */ +#define MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(a) (0x4c8ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(a) (0x748ull + (a) * 0x8ull) +#define MCSX_PEX_RX_SLAVE_ETYPE_ENABLE 0x6e8ull +#define MCSX_PEX_TX_SLAVE_ETYPE_ENABLE 0x968ull + +/* BEE */ +#define MCSX_BBE_RX_SLAVE_PADDING_CTL 0xe08ull +#define MCSX_BBE_TX_SLAVE_PADDING_CTL 0x12f8ull +#define MCSX_BBE_RX_SLAVE_CAL_ENTRY 0x180ull +#define MCSX_BBE_RX_SLAVE_CAL_LEN 0x188ull +#define MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(a) (0x290ull + (a) * 0x40ull) + +#define MCSX_BBE_RX_SLAVE_BBE_INT ({ \ + u64 offset; \ + \ + offset = 0xe00ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x160ull; \ + offset; }) + +#define MCSX_BBE_RX_SLAVE_BBE_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0xe08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x168ull; \ + offset; }) + +#define MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0xe08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x178ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT ({ \ + u64 offset; \ + \ + offset = 0x1278ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1e0ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x1278ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1f8ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x1280ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1e8ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT ({ \ + u64 offset; \ + \ + offset = 0x16f0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x260ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x268ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x278ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT ({ \ + u64 offset; \ + \ + offset = 0x2908ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x380ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x2910ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x388ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x398ull; \ + offset; }) + +/* CPM registers */ +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x30740ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x3bf8ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x34740ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x43f8ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0 ({ \ + u64 offset; \ + \ + offset = 0x30700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x3bd8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SC_CAMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x38780ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4c08ull; \ + offset += (a) * 0x8ull + (b) * 0x10ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SC_CAM_ENA(a) ({ \ + u64 offset; \ + \ + offset = 0x38740ull + (a) * 0x8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4bf8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x23ee0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbd0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = (0x246e0ull + (a) * 0x10ull); \ + if (mcs->hw->mcs_blks > 1) \ + offset = (0xdd0ull + (a) * 0x8ull); \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_KEY_LOCKOUTX(a) ({ \ + u64 offset; \ + \ + offset = 0x23E90ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbb0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_MAP_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x256e0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfd0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x27700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x17d8ull; \ + offset += (a) * 0x8ull + (b) * 0x40ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x2f700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x37d8; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_XPN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x23e40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xb90ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_PN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x23e48ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xb98ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x23e50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xba0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1 0x30708ull +#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(a) (0x246e8ull + (a) * 0x10ull) + +/* TX registers */ +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x51d50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa7c0ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x55d50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xafc0ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0 ({ \ + u64 offset; \ + \ + offset = 0x51d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa7a0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = 0x3e508ull + (a) * 0x8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5550ull + (a) * 0x10ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x3ed08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5950ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_KEY_LOCKOUTX(a) ({ \ + u64 offset; \ + \ + offset = 0x3e4c0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5538ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = 0x3fd10ull + (a) * 0x10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6150ull + (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x40d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x63a0ull; \ + offset += (a) * 0x8ull + (b) * 0x80ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x50d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa3a0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_XPN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x3e4b0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5528ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_PN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x3e4b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5530ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(a) (0x3fd18ull + (a) * 0x10ull) +#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(a) (0x5558ull + (a) * 0x10ull) +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1 0x51d18ull +#define MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(a) (0x5b50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(a) (0x5d50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(a) (0x5f50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0 0x5500ull + +/* CSE */ +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x9e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc218ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x9680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc018ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x6e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x8e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbe18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x8680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xca18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x7e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x6680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x7680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc618ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x5e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xdc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(a)({ \ + u64 offset; \ + \ + offset = 0x5680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xda18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(a) ({ \ + u64 offset; \ + \ + offset = 0xd680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xce18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0x16a80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(a) ({ \ + u64 offset; \ + \ + offset = 0x16680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec38ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x16880ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0xfe80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xde18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(a) ({ \ + u64 offset; \ + \ + offset = 0x10680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xe418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(a) ({ \ + u64 offset; \ + \ + offset = 0x10e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xe218ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(a) ({ \ + u64 offset; \ + \ + offset = 0xae80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(a) ({ \ + u64 offset; \ + \ + offset = 0xc680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd618ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(a) ({ \ + u64 offset; \ + \ + offset = 0xce80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(a) ({ \ + u64 offset; \ + \ + offset = 0xbe80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xcc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_SLAVE_CTRL ({ \ + u64 offset; \ + \ + offset = 0x52a0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x9c0ull; \ + offset; }) + +#define MCSX_CSE_RX_SLAVE_STATS_CLEAR ({ \ + u64 offset; \ + \ + offset = 0x52b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x9d8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(a) (0xe680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(a) (0xde80ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(a) (0xa680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(a) (0xd218 + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDX(a) (0xd018ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(a) (0xee80ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(a) (0xb680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(a) (0xf680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(a) (0x12680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(a) (0x15680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(a) (0x13680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(a) (0x11680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(a) (0x14680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(a) (0xec58ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(a) (0xea18ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(a) (0xe618ull + (a) * 0x8ull) + +/* CSE TX */ +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCOMMONOCTETSX(a) (0x18440ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1c440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf478ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1bc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf278ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x19440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xee78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1b440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf078ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1ac40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfc78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1a440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfa78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x18c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf678ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x19c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf878ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x17c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10878ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x17440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10678ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSCTRLPORTDISABLEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1e440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfe78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0x23240ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10ed8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(a) ({ \ + u64 offset; \ + \ + offset = 0x22c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10e98ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x22e40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10e78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x20440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10c78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1fc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10a78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x23040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x110d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(a) ({ \ + u64 offset; \ + \ + offset = 0x1dc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10278ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(a) ({ \ + u64 offset; \ + \ + offset = 0x1d440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10478ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1cc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10078ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_SLAVE_CTRL ({ \ + u64 offset; \ + \ + offset = 0x54a0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa00ull; \ + offset; }) + +#define MCSX_CSE_TX_SLAVE_STATS_CLEAR ({ \ + u64 offset; \ + \ + offset = 0x54b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa18ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(a) (0x1f440ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(a) (0x1ec40ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSEARLYPREEMPTERRX(a) (0x10eb8ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(a) (0x21c40ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(a) (0x20c40ull + (a) * 0x8ull) + +#define MCSX_IP_INT ({ \ + u64 offset; \ + \ + offset = 0x80028ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60028ull; \ + offset; }) + +#define MCSX_IP_INT_ENA_W1S ({ \ + u64 offset; \ + \ + offset = 0x80040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60040ull; \ + offset; }) + +#define MCSX_IP_INT_ENA_W1C ({ \ + u64 offset; \ + \ + offset = 0x80038ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60038ull; \ + offset; }) + +#define MCSX_TOP_SLAVE_INT_SUM ({ \ + u64 offset; \ + \ + offset = 0xc20ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xab8ull; \ + offset; }) + +#define MCSX_TOP_SLAVE_INT_SUM_ENB ({ \ + u64 offset; \ + \ + offset = 0xc28ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xac0ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_RX_INT ({ \ + u64 offset; \ + \ + offset = 0x23c00ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x0ad8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_RX_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x23c08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xae0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_TX_INT ({ \ + u64 offset; \ + \ + offset = 0x3d490ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x54a0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_TX_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x3d498ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x54a8ull; \ + offset; }) + +#endif diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c new file mode 100644 index 000000000000..fa8029a94068 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell CN10K MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include <linux/types.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "mcs.h" +#include "rvu.h" +#include "lmac_common.h" + +#define M(_name, _id, _fn_name, _req_type, _rsp_type) \ +static struct _req_type __maybe_unused \ +*otx2_mbox_alloc_msg_ ## _fn_name(struct rvu *rvu, int devid) \ +{ \ + struct _req_type *req; \ + \ + req = (struct _req_type *)otx2_mbox_alloc_msg_rsp( \ + &rvu->afpf_wq_info.mbox_up, devid, sizeof(struct _req_type), \ + sizeof(struct _rsp_type)); \ + if (!req) \ + return NULL; \ + req->hdr.sig = OTX2_MBOX_REQ_SIG; \ + req->hdr.id = _id; \ + return req; \ +} + +MBOX_UP_MCS_MESSAGES +#undef M + +int rvu_mbox_handler_mcs_set_lmac_mode(struct rvu *rvu, + struct mcs_set_lmac_mode *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (BIT_ULL(req->lmac_id) & mcs->hw->lmac_bmap) + mcs_set_lmac_mode(mcs, req->lmac_id, req->mode); + + return 0; +} + +int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event) +{ + struct mcs_intrq_entry *qentry; + u16 pcifunc = event->pcifunc; + struct rvu *rvu = mcs->rvu; + struct mcs_pfvf *pfvf; + + /* Check if it is PF or VF */ + if (pcifunc & RVU_PFVF_FUNC_MASK) + pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)]; + else + pfvf = &mcs->pf[rvu_get_pf(pcifunc)]; + + event->intr_mask &= pfvf->intr_mask; + + /* Check PF/VF interrupt notification is enabled */ + if (!(pfvf->intr_mask && event->intr_mask)) + return 0; + + qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); + if (!qentry) + return -ENOMEM; + + qentry->intr_event = *event; + spin_lock(&rvu->mcs_intrq_lock); + list_add_tail(&qentry->node, &rvu->mcs_intrq_head); + spin_unlock(&rvu->mcs_intrq_lock); + queue_work(rvu->mcs_intr_wq, &rvu->mcs_intr_work); + + return 0; +} + +static int mcs_notify_pfvf(struct mcs_intr_event *event, struct rvu *rvu) +{ + struct mcs_intr_info *req; + int err, pf; + + pf = rvu_get_pf(event->pcifunc); + + req = otx2_mbox_alloc_msg_mcs_intr_notify(rvu, pf); + if (!req) + return -ENOMEM; + + req->mcs_id = event->mcs_id; + req->intr_mask = event->intr_mask; + req->sa_id = event->sa_id; + req->hdr.pcifunc = event->pcifunc; + req->lmac_id = event->lmac_id; + + otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, pf); + err = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, pf); + if (err) + dev_warn(rvu->dev, "MCS notification to pf %d failed\n", pf); + + return 0; +} + +static void mcs_intr_handler_task(struct work_struct *work) +{ + struct rvu *rvu = container_of(work, struct rvu, mcs_intr_work); + struct mcs_intrq_entry *qentry; + struct mcs_intr_event *event; + unsigned long flags; + + do { + spin_lock_irqsave(&rvu->mcs_intrq_lock, flags); + qentry = list_first_entry_or_null(&rvu->mcs_intrq_head, + struct mcs_intrq_entry, + node); + if (qentry) + list_del(&qentry->node); + + spin_unlock_irqrestore(&rvu->mcs_intrq_lock, flags); + if (!qentry) + break; /* nothing more to process */ + + event = &qentry->intr_event; + + mcs_notify_pfvf(event, rvu); + kfree(qentry); + } while (1); +} + +int rvu_mbox_handler_mcs_intr_cfg(struct rvu *rvu, + struct mcs_intr_cfg *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_pfvf *pfvf; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* Check if it is PF or VF */ + if (pcifunc & RVU_PFVF_FUNC_MASK) + pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)]; + else + pfvf = &mcs->pf[rvu_get_pf(pcifunc)]; + + mcs->pf_map[0] = pcifunc; + pfvf->intr_mask = req->intr_mask; + + return 0; +} + +int rvu_mbox_handler_mcs_get_hw_info(struct rvu *rvu, + struct msg_req *req, + struct mcs_hw_info *rsp) +{ + struct mcs *mcs; + + if (!rvu->mcs_blk_cnt) + return MCS_AF_ERR_NOT_MAPPED; + + /* MCS resources are same across all blocks */ + mcs = mcs_get_pdata(0); + rsp->num_mcs_blks = rvu->mcs_blk_cnt; + rsp->tcam_entries = mcs->hw->tcam_entries; + rsp->secy_entries = mcs->hw->secy_entries; + rsp->sc_entries = mcs->hw->sc_entries; + rsp->sa_entries = mcs->hw->sa_entries; + return 0; +} + +int rvu_mbox_handler_mcs_port_reset(struct rvu *rvu, struct mcs_port_reset_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_reset_port(mcs, req->port_id, req->reset); + + return 0; +} + +int rvu_mbox_handler_mcs_clear_stats(struct rvu *rvu, + struct mcs_clear_stats *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mutex_lock(&mcs->stats_lock); + if (req->all) + mcs_clear_all_stats(mcs, pcifunc, req->dir); + else + mcs_clear_stats(mcs, req->type, req->id, req->dir); + + mutex_unlock(&mcs->stats_lock); + return 0; +} + +int rvu_mbox_handler_mcs_get_flowid_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_flowid_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* In CNF10K-B, before reading the statistics, + * MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP needs to be set + * to get accurate statistics + */ + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_flowid_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + /* Clear MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP after reading + * the statistics + */ + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_secy_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_secy_stats *rsp) +{ struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + + if (req->dir == MCS_RX) + mcs_get_rx_secy_stats(mcs, rsp, req->id); + else + mcs_get_tx_secy_stats(mcs, rsp, req->id); + + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_sc_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_sc_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_sc_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_sa_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_sa_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_sa_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_port_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_port_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_port_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_set_active_lmac(struct rvu *rvu, + struct mcs_set_active_lmac *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + if (!mcs) + return MCS_AF_ERR_NOT_MAPPED; + + mcs->hw->lmac_bmap = req->lmac_bmap; + mcs_set_lmac_channels(req->mcs_id, req->chan_base); + return 0; +} + +int rvu_mbox_handler_mcs_port_cfg_set(struct rvu *rvu, struct mcs_port_cfg_set_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id))) + return -EINVAL; + + mcs_set_port_cfg(mcs, req); + + return 0; +} + +int rvu_mbox_handler_mcs_port_cfg_get(struct rvu *rvu, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id))) + return -EINVAL; + + mcs_get_port_cfg(mcs, req, rsp); + + return 0; +} + +int rvu_mbox_handler_mcs_custom_tag_cfg_get(struct rvu *rvu, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_get_custom_tag_cfg(mcs, req, rsp); + + return 0; +} + +int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc) +{ + struct mcs *mcs; + int mcs_id; + + /* CNF10K-B mcs0-6 are mapped to RPM2-8*/ + if (rvu->mcs_blk_cnt > 1) { + for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) { + mcs = mcs_get_pdata(mcs_id); + mcs_free_all_rsrc(mcs, MCS_RX, pcifunc); + mcs_free_all_rsrc(mcs, MCS_TX, pcifunc); + } + } else { + /* CN10K-B has only one mcs block */ + mcs = mcs_get_pdata(0); + mcs_free_all_rsrc(mcs, MCS_RX, pcifunc); + mcs_free_all_rsrc(mcs, MCS_TX, pcifunc); + } + return 0; +} + +int rvu_mbox_handler_mcs_flowid_ena_entry(struct rvu *rvu, + struct mcs_flowid_ena_dis_entry *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_ena_dis_flowid_entry(mcs, req->flow_id, req->dir, req->ena); + return 0; +} + +int rvu_mbox_handler_mcs_pn_table_write(struct rvu *rvu, + struct mcs_pn_table_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_pn_table_write(mcs, req->pn_id, req->next_pn, req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_set_pn_threshold(struct rvu *rvu, + struct mcs_set_pn_threshold *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_pn_threshold_set(mcs, req); + + return 0; +} + +int rvu_mbox_handler_mcs_rx_sc_sa_map_write(struct rvu *rvu, + struct mcs_rx_sc_sa_map *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs->mcs_ops->mcs_rx_sa_mem_map_write(mcs, req); + return 0; +} + +int rvu_mbox_handler_mcs_tx_sc_sa_map_write(struct rvu *rvu, + struct mcs_tx_sc_sa_map *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs->mcs_ops->mcs_tx_sa_mem_map_write(mcs, req); + mcs->tx_sa_active[req->sc_id] = req->tx_sa_active; + + return 0; +} + +int rvu_mbox_handler_mcs_sa_plcy_write(struct rvu *rvu, + struct mcs_sa_plcy_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int i; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + for (i = 0; i < req->sa_cnt; i++) + mcs_sa_plcy_write(mcs, &req->plcy[i][0], + req->sa_index[i], req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_rx_sc_cam_write(struct rvu *rvu, + struct mcs_rx_sc_cam_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_rx_sc_cam_write(mcs, req->sci, req->secy_id, req->sc_id); + return 0; +} + +int rvu_mbox_handler_mcs_secy_plcy_write(struct rvu *rvu, + struct mcs_secy_plcy_write_req *req, + struct msg_rsp *rsp) +{ struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_secy_plcy_write(mcs, req->plcy, + req->secy_id, req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_flowid_entry_write(struct rvu *rvu, + struct mcs_flowid_entry_write_req *req, + struct msg_rsp *rsp) +{ + struct secy_mem_map map; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* TODO validate the flowid */ + mcs_flowid_entry_write(mcs, req->data, req->mask, + req->flow_id, req->dir); + map.secy = req->secy_id; + map.sc = req->sc_id; + map.ctrl_pkt = req->ctrl_pkt; + map.flow_id = req->flow_id; + map.sci = req->sci; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, req->dir); + if (req->ena) + mcs_ena_dis_flowid_entry(mcs, req->flow_id, + req->dir, true); + return 0; +} + +int rvu_mbox_handler_mcs_free_resources(struct rvu *rvu, + struct mcs_free_rsrc_req *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (req->dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + /* Free all the cam resources mapped to PF/VF */ + if (req->all) { + rc = mcs_free_all_rsrc(mcs, req->dir, pcifunc); + goto exit; + } + + switch (req->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + rc = mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, req->rsrc_id, pcifunc); + mcs_ena_dis_flowid_entry(mcs, req->rsrc_id, req->dir, false); + break; + case MCS_RSRC_TYPE_SECY: + rc = mcs_free_rsrc(&map->secy, map->secy2pf_map, req->rsrc_id, pcifunc); + mcs_clear_secy_plcy(mcs, req->rsrc_id, req->dir); + break; + case MCS_RSRC_TYPE_SC: + rc = mcs_free_rsrc(&map->sc, map->sc2pf_map, req->rsrc_id, pcifunc); + /* Disable SC CAM only on RX side */ + if (req->dir == MCS_RX) + mcs_ena_dis_sc_cam_entry(mcs, req->rsrc_id, false); + break; + case MCS_RSRC_TYPE_SA: + rc = mcs_free_rsrc(&map->sa, map->sa2pf_map, req->rsrc_id, pcifunc); + break; + } +exit: + mutex_unlock(&rvu->rsrc_lock); + return rc; +} + +int rvu_mbox_handler_mcs_alloc_resources(struct rvu *rvu, + struct mcs_alloc_rsrc_req *req, + struct mcs_alloc_rsrc_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rsrc_id, i; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (req->dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + + if (req->all) { + rsrc_id = mcs_alloc_all_rsrc(mcs, &rsp->flow_ids[0], + &rsp->secy_ids[0], + &rsp->sc_ids[0], + &rsp->sa_ids[0], + &rsp->sa_ids[1], + pcifunc, req->dir); + goto exit; + } + + switch (req->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->flow_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SECY: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->secy_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SC: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->sc_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SA: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->sa_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + } + + rsp->rsrc_type = req->rsrc_type; + rsp->dir = req->dir; + rsp->mcs_id = req->mcs_id; + rsp->all = req->all; + +exit: + if (rsrc_id < 0) + dev_err(rvu->dev, "Failed to allocate the mcs resources for PCIFUNC:%d\n", pcifunc); + mutex_unlock(&rvu->rsrc_lock); + return 0; +} + +int rvu_mbox_handler_mcs_alloc_ctrl_pkt_rule(struct rvu *rvu, + struct mcs_alloc_ctrl_pkt_rule_req *req, + struct mcs_alloc_ctrl_pkt_rule_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rsrc_id; + u16 offset; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + + switch (req->rule_type) { + case MCS_CTRL_PKT_RULE_TYPE_ETH: + offset = MCS_CTRLPKT_ETYPE_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_DA: + offset = MCS_CTRLPKT_DA_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_RANGE: + offset = MCS_CTRLPKT_DA_RANGE_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_COMBO: + offset = MCS_CTRLPKT_COMBO_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_MAC: + offset = MCS_CTRLPKT_MAC_EN_RULE_OFFSET; + break; + } + + rsrc_id = mcs_alloc_ctrlpktrule(&map->ctrlpktrule, map->ctrlpktrule2pf_map, offset, + pcifunc); + if (rsrc_id < 0) + goto exit; + + rsp->rule_idx = rsrc_id; + rsp->rule_type = req->rule_type; + rsp->dir = req->dir; + rsp->mcs_id = req->mcs_id; + + mutex_unlock(&rvu->rsrc_lock); + return 0; +exit: + if (rsrc_id < 0) + dev_err(rvu->dev, "Failed to allocate the mcs ctrl pkt rule for PCIFUNC:%d\n", + pcifunc); + mutex_unlock(&rvu->rsrc_lock); + return rsrc_id; +} + +int rvu_mbox_handler_mcs_free_ctrl_pkt_rule(struct rvu *rvu, + struct mcs_free_ctrl_pkt_rule_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mutex_lock(&rvu->rsrc_lock); + + rc = mcs_free_ctrlpktrule(mcs, req); + + mutex_unlock(&rvu->rsrc_lock); + + return rc; +} + +int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu, + struct mcs_ctrl_pkt_rule_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + rc = mcs_ctrlpktrule_write(mcs, req); + + return rc; +} + +static void rvu_mcs_set_lmac_bmap(struct rvu *rvu) +{ + struct mcs *mcs = mcs_get_pdata(0); + unsigned long lmac_bmap; + int cgx, lmac, port; + + for (port = 0; port < mcs->hw->lmac_cnt; port++) { + cgx = port / rvu->hw->lmac_per_cgx; + lmac = port % rvu->hw->lmac_per_cgx; + if (!is_lmac_valid(rvu_cgx_pdata(cgx, rvu), lmac)) + continue; + set_bit(port, &lmac_bmap); + } + mcs->hw->lmac_bmap = lmac_bmap; +} + +int rvu_mcs_init(struct rvu *rvu) +{ + struct rvu_hwinfo *hw = rvu->hw; + int lmac, err = 0, mcs_id; + struct mcs *mcs; + + rvu->mcs_blk_cnt = mcs_get_blkcnt(); + + if (!rvu->mcs_blk_cnt) + return 0; + + /* Needed only for CN10K-B */ + if (rvu->mcs_blk_cnt == 1) { + err = mcs_set_lmac_channels(0, hw->cgx_chan_base); + if (err) + return err; + /* Set active lmacs */ + rvu_mcs_set_lmac_bmap(rvu); + } + + /* Install default tcam bypass entry and set port to operational mode */ + for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) { + mcs = mcs_get_pdata(mcs_id); + mcs_install_flowid_bypass_entry(mcs); + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) + mcs_set_lmac_mode(mcs, lmac, 0); + + mcs->rvu = rvu; + + /* Allocated memory for PFVF data */ + mcs->pf = devm_kcalloc(mcs->dev, hw->total_pfs, + sizeof(struct mcs_pfvf), GFP_KERNEL); + if (!mcs->pf) + return -ENOMEM; + + mcs->vf = devm_kcalloc(mcs->dev, hw->total_vfs, + sizeof(struct mcs_pfvf), GFP_KERNEL); + if (!mcs->vf) + return -ENOMEM; + } + + /* Initialize the wq for handling mcs interrupts */ + INIT_LIST_HEAD(&rvu->mcs_intrq_head); + INIT_WORK(&rvu->mcs_intr_work, mcs_intr_handler_task); + rvu->mcs_intr_wq = alloc_workqueue("mcs_intr_wq", 0, 0); + if (!rvu->mcs_intr_wq) { + dev_err(rvu->dev, "mcs alloc workqueue failed\n"); + return -ENOMEM; + } + + return err; +} + +void rvu_mcs_exit(struct rvu *rvu) +{ + if (!rvu->mcs_intr_wq) + return; + + flush_workqueue(rvu->mcs_intr_wq); + destroy_workqueue(rvu->mcs_intr_wq); + rvu->mcs_intr_wq = NULL; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h index 77fd39e2c8db..f187293e3e08 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h @@ -10,6 +10,14 @@ #define NPC_KEX_CHAN_MASK 0xFFFULL +#define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ + rvu_write64(rvu, blkaddr, \ + NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) + +#define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ + rvu_write64(rvu, blkaddr, \ + NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) + enum NPC_LID_E { NPC_LID_LA = 0, NPC_LID_LB, @@ -200,6 +208,7 @@ enum key_fields { NPC_ERRLEV, NPC_ERRCODE, NPC_LXMB, + NPC_EXACT_RESULT, NPC_LA, NPC_LB, NPC_LC, @@ -381,6 +390,22 @@ struct nix_rx_action { }; /* NPC_AF_INTFX_KEX_CFG field masks */ +#define NPC_EXACT_NIBBLE_START 40 +#define NPC_EXACT_NIBBLE_END 43 +#define NPC_EXACT_NIBBLE GENMASK_ULL(43, 40) + +/* NPC_EXACT_KEX_S nibble definitions for each field */ +#define NPC_EXACT_NIBBLE_HIT BIT_ULL(40) +#define NPC_EXACT_NIBBLE_OPC BIT_ULL(40) +#define NPC_EXACT_NIBBLE_WAY BIT_ULL(40) +#define NPC_EXACT_NIBBLE_INDEX GENMASK_ULL(43, 41) + +#define NPC_EXACT_RESULT_HIT BIT_ULL(0) +#define NPC_EXACT_RESULT_OPC GENMASK_ULL(2, 1) +#define NPC_EXACT_RESULT_WAY GENMASK_ULL(4, 3) +#define NPC_EXACT_RESULT_IDX GENMASK_ULL(15, 5) + +/* NPC_AF_INTFX_KEX_CFG field masks */ #define NPC_PARSE_NIBBLE GENMASK_ULL(30, 0) /* NPC_PARSE_KEX_S nibble definitions for each field */ @@ -455,7 +480,7 @@ struct npc_coalesced_kpu_prfl { u8 name[NPC_NAME_LEN]; /* KPU Profile name */ u64 version; /* KPU firmware/profile version */ u8 num_prfl; /* No of NPC profiles. */ - u16 prfl_sz[0]; + u16 prfl_sz[]; }; struct npc_mcam_kex { @@ -482,7 +507,7 @@ struct npc_kpu_fwdata { * struct npc_kpu_profile_cam[entries]; * struct npc_kpu_profile_action[entries]; */ - u8 data[0]; + u8 data[]; } __packed; struct npc_lt_def { @@ -572,7 +597,7 @@ struct npc_kpu_profile_fwdata { * Custom KPU CAM and ACTION configuration entries. * struct npc_kpu_fwdata kpu[kpus]; */ - u8 data[0]; + u8 data[]; } __packed; struct rvu_npc_mcam_rule { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h index 0fe7ad35e36f..a820bad3abb2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h @@ -155,7 +155,7 @@ /* Rx parse key extract nibble enable */ #define NPC_PARSE_NIBBLE_INTF_RX (NPC_PARSE_NIBBLE_CHAN | \ - NPC_PARSE_NIBBLE_ERRCODE | \ + NPC_PARSE_NIBBLE_L2L3_BCAST | \ NPC_PARSE_NIBBLE_LA_LTYPE | \ NPC_PARSE_NIBBLE_LB_LTYPE | \ NPC_PARSE_NIBBLE_LC_LTYPE | \ @@ -185,7 +185,6 @@ enum npc_kpu_parser_state { NPC_S_KPU2_QINQ, NPC_S_KPU2_ETAG, NPC_S_KPU2_EXDSA, - NPC_S_KPU2_NGIO, NPC_S_KPU2_CPT_CTAG, NPC_S_KPU2_CPT_QINQ, NPC_S_KPU3_CTAG, @@ -212,6 +211,7 @@ enum npc_kpu_parser_state { NPC_S_KPU5_NSH, NPC_S_KPU5_CPT_IP, NPC_S_KPU5_CPT_IP6, + NPC_S_KPU5_NGIO, NPC_S_KPU6_IP6_EXT, NPC_S_KPU6_IP6_HOP_DEST, NPC_S_KPU6_IP6_ROUT, @@ -1124,15 +1124,6 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = { NPC_S_KPU1_ETHER, 0xff, NPC_ETYPE_CTAG, 0xffff, - NPC_ETYPE_NGIO, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU1_ETHER, 0xff, - NPC_ETYPE_CTAG, - 0xffff, NPC_ETYPE_CTAG, 0xffff, 0x0000, @@ -1968,6 +1959,15 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = { }, { NPC_S_KPU2_CTAG, 0xff, + NPC_ETYPE_NGIO, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU2_CTAG, 0xff, NPC_ETYPE_PPPOE, 0xffff, 0x0000, @@ -2750,15 +2750,6 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = { 0x0000, }, { - NPC_S_KPU2_NGIO, 0xff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { NPC_S_KPU2_CPT_CTAG, 0xff, NPC_ETYPE_IP, 0xffff, @@ -5090,6 +5081,15 @@ static struct npc_kpu_profile_cam kpu5_cam_entries[] = { 0x0000, }, { + NPC_S_KPU5_NGIO, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -8425,14 +8425,6 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 12, 0, 0, 0, - NPC_S_KPU2_NGIO, 12, 1, - NPC_LID_LA, NPC_LT_LA_ETHER, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 12, 0, 0, 0, NPC_S_KPU2_CTAG2, 12, 1, NPC_LID_LA, NPC_LT_LA_ETHER, NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, @@ -9196,6 +9188,14 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU5_NGIO, 6, 1, + NPC_LID_LB, NPC_LT_LB_CTAG, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 2, 0, NPC_S_KPU5_IP, 14, 1, NPC_LID_LB, NPC_LT_LB_PPPOE, @@ -9892,14 +9892,6 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 1, - NPC_LID_LC, NPC_LT_LC_NGIO, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 2, 0, NPC_S_KPU5_CPT_IP, 6, 1, NPC_LID_LB, NPC_LT_LB_CTAG, @@ -11974,6 +11966,14 @@ static struct npc_kpu_profile_action kpu5_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_NGIO, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LC, NPC_EC_UNK, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -15123,7 +15123,8 @@ static struct npc_mcam_kex npc_mkex_default = { .kpu_version = NPC_KPU_PROFILE_VER, .keyx_cfg = { /* nibble: LA..LE (ltype only) + Error code + Channel */ - [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_X2 << 32) | NPC_PARSE_NIBBLE_INTF_RX, + [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_X2 << 32) | NPC_PARSE_NIBBLE_INTF_RX | + (u64)NPC_EXACT_NIBBLE_HIT, /* nibble: LA..LE (ltype only) */ [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | NPC_PARSE_NIBBLE_INTF_TX, }, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c index d6321de3cc17..3411e2e47d46 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c @@ -9,6 +9,8 @@ #include <linux/device.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/hrtimer.h> +#include <linux/ktime.h> #include "ptp.h" #include "mbox.h" @@ -25,6 +27,9 @@ #define PCI_SUBSYS_DEVID_OCTX2_95XXO_PTP 0xB600 #define PCI_DEVID_OCTEONTX2_RST 0xA085 #define PCI_DEVID_CN10K_PTP 0xA09E +#define PCI_SUBSYS_DEVID_CN10K_A_PTP 0xB900 +#define PCI_SUBSYS_DEVID_CNF10K_A_PTP 0xBA00 +#define PCI_SUBSYS_DEVID_CNF10K_B_PTP 0xBC00 #define PCI_PTP_BAR_NO 0 @@ -46,10 +51,153 @@ #define PTP_CLOCK_HI 0xF10ULL #define PTP_CLOCK_COMP 0xF18ULL #define PTP_TIMESTAMP 0xF20ULL +#define PTP_CLOCK_SEC 0xFD0ULL +#define PTP_SEC_ROLLOVER 0xFD8ULL + +#define CYCLE_MULT 1000 static struct ptp *first_ptp_block; static const struct pci_device_id ptp_id_table[]; +static bool is_ptp_dev_cnf10kb(struct ptp *ptp) +{ + return (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP) ? true : false; +} + +static bool is_ptp_dev_cn10k(struct ptp *ptp) +{ + return (ptp->pdev->device == PCI_DEVID_CN10K_PTP) ? true : false; +} + +static bool cn10k_ptp_errata(struct ptp *ptp) +{ + if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP || + ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_A_PTP) + return true; + return false; +} + +static bool is_ptp_tsfmt_sec_nsec(struct ptp *ptp) +{ + if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP || + ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_A_PTP) + return true; + return false; +} + +static enum hrtimer_restart ptp_reset_thresh(struct hrtimer *hrtimer) +{ + struct ptp *ptp = container_of(hrtimer, struct ptp, hrtimer); + ktime_t curr_ts = ktime_get(); + ktime_t delta_ns, period_ns; + u64 ptp_clock_hi; + + /* calculate the elapsed time since last restart */ + delta_ns = ktime_to_ns(ktime_sub(curr_ts, ptp->last_ts)); + + /* if the ptp clock value has crossed 0.5 seconds, + * its too late to update pps threshold value, so + * update threshold after 1 second. + */ + ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI); + if (ptp_clock_hi > 500000000) { + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - ptp_clock_hi)); + } else { + writeq(500000000, ptp->reg_base + PTP_PPS_THRESH_HI); + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - delta_ns)); + } + + hrtimer_forward_now(hrtimer, period_ns); + ptp->last_ts = curr_ts; + + return HRTIMER_RESTART; +} + +static void ptp_hrtimer_start(struct ptp *ptp, ktime_t start_ns) +{ + ktime_t period_ns; + + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - start_ns)); + hrtimer_start(&ptp->hrtimer, period_ns, HRTIMER_MODE_REL); + ptp->last_ts = ktime_get(); +} + +static u64 read_ptp_tstmp_sec_nsec(struct ptp *ptp) +{ + u64 sec, sec1, nsec; + unsigned long flags; + + spin_lock_irqsave(&ptp->ptp_lock, flags); + sec = readq(ptp->reg_base + PTP_CLOCK_SEC) & 0xFFFFFFFFUL; + nsec = readq(ptp->reg_base + PTP_CLOCK_HI); + sec1 = readq(ptp->reg_base + PTP_CLOCK_SEC) & 0xFFFFFFFFUL; + /* check nsec rollover */ + if (sec1 > sec) { + nsec = readq(ptp->reg_base + PTP_CLOCK_HI); + sec = sec1; + } + spin_unlock_irqrestore(&ptp->ptp_lock, flags); + + return sec * NSEC_PER_SEC + nsec; +} + +static u64 read_ptp_tstmp_nsec(struct ptp *ptp) +{ + return readq(ptp->reg_base + PTP_CLOCK_HI); +} + +static u64 ptp_calc_adjusted_comp(u64 ptp_clock_freq) +{ + u64 comp, adj = 0, cycles_per_sec, ns_drift = 0; + u32 ptp_clock_nsec, cycle_time; + int cycle; + + /* Errata: + * Issue #1: At the time of 1 sec rollover of the nano-second counter, + * the nano-second counter is set to 0. However, it should be set to + * (existing counter_value - 10^9). + * + * Issue #2: The nano-second counter rolls over at 0x3B9A_C9FF. + * It should roll over at 0x3B9A_CA00. + */ + + /* calculate ptp_clock_comp value */ + comp = ((u64)1000000000ULL << 32) / ptp_clock_freq; + /* use CYCLE_MULT to avoid accuracy loss due to integer arithmetic */ + cycle_time = NSEC_PER_SEC * CYCLE_MULT / ptp_clock_freq; + /* cycles per sec */ + cycles_per_sec = ptp_clock_freq; + + /* check whether ptp nanosecond counter rolls over early */ + cycle = cycles_per_sec - 1; + ptp_clock_nsec = (cycle * comp) >> 32; + while (ptp_clock_nsec < NSEC_PER_SEC) { + if (ptp_clock_nsec == 0x3B9AC9FF) + goto calc_adj_comp; + cycle++; + ptp_clock_nsec = (cycle * comp) >> 32; + } + /* compute nanoseconds lost per second when nsec counter rolls over */ + ns_drift = ptp_clock_nsec - NSEC_PER_SEC; + /* calculate ptp_clock_comp adjustment */ + if (ns_drift > 0) { + adj = comp * ns_drift; + adj = adj / 1000000000ULL; + } + /* speed up the ptp clock to account for nanoseconds lost */ + comp += adj; + return comp; + +calc_adj_comp: + /* slow down the ptp clock to not rollover early */ + adj = comp * cycle_time; + adj = adj / 1000000000ULL; + adj = adj / CYCLE_MULT; + comp -= adj; + + return comp; +} + struct ptp *ptp_get(void) { struct ptp *ptp = first_ptp_block; @@ -60,6 +208,8 @@ struct ptp *ptp_get(void) /* Check driver is bound to PTP block */ if (!ptp) ptp = ERR_PTR(-EPROBE_DEFER); + else + pci_dev_get(ptp->pdev); return ptp; } @@ -75,8 +225,8 @@ void ptp_put(struct ptp *ptp) static int ptp_adjfine(struct ptp *ptp, long scaled_ppm) { bool neg_adj = false; - u64 comp; - u64 adj; + u32 freq, freq_adj; + u64 comp, adj; s64 ppb; if (scaled_ppm < 0) { @@ -98,15 +248,22 @@ static int ptp_adjfine(struct ptp *ptp, long scaled_ppm) * where tbase is the basic compensation value calculated * initialy in the probe function. */ - comp = ((u64)1000000000ull << 32) / ptp->clock_rate; /* convert scaled_ppm to ppb */ ppb = 1 + scaled_ppm; ppb *= 125; ppb >>= 13; - adj = comp * ppb; - adj = div_u64(adj, 1000000000ull); - comp = neg_adj ? comp - adj : comp + adj; + if (cn10k_ptp_errata(ptp)) { + /* calculate the new frequency based on ppb */ + freq_adj = (ptp->clock_rate * ppb) / 1000000000ULL; + freq = neg_adj ? ptp->clock_rate + freq_adj : ptp->clock_rate - freq_adj; + comp = ptp_calc_adjusted_comp(freq); + } else { + comp = ((u64)1000000000ull << 32) / ptp->clock_rate; + adj = comp * ppb; + adj = div_u64(adj, 1000000000ull); + comp = neg_adj ? comp - adj : comp + adj; + } writeq(comp, ptp->reg_base + PTP_CLOCK_COMP); return 0; @@ -115,7 +272,7 @@ static int ptp_adjfine(struct ptp *ptp, long scaled_ppm) static int ptp_get_clock(struct ptp *ptp, u64 *clk) { /* Return the current PTP clock */ - *clk = readq(ptp->reg_base + PTP_CLOCK_HI); + *clk = ptp->read_ptp_tstmp(ptp); return 0; } @@ -139,6 +296,10 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) /* sclk is in MHz */ ptp->clock_rate = sclk * 1000000; + /* Program the seconds rollover value to 1 second */ + if (is_ptp_dev_cnf10kb(ptp)) + writeq(0x3b9aca00, ptp->reg_base + PTP_SEC_ROLLOVER); + /* Enable PTP clock */ clock_cfg = readq(ptp->reg_base + PTP_CLOCK_CFG); @@ -163,22 +324,63 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) /* Set 50% duty cycle for 1Hz output */ writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_HI_INCR); writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_LO_INCR); + if (cn10k_ptp_errata(ptp)) { + /* The ptp_clock_hi rollsover to zero once clock cycle before it + * reaches one second boundary. so, program the pps_lo_incr in + * such a way that the pps threshold value comparison at one + * second boundary will succeed and pps edge changes. After each + * one second boundary, the hrtimer handler will be invoked and + * reprograms the pps threshold value. + */ + ptp->clock_period = NSEC_PER_SEC / ptp->clock_rate; + writeq((0x1dcd6500ULL - ptp->clock_period) << 32, + ptp->reg_base + PTP_PPS_LO_INCR); + } + + if (cn10k_ptp_errata(ptp)) + clock_comp = ptp_calc_adjusted_comp(ptp->clock_rate); + else + clock_comp = ((u64)1000000000ull << 32) / ptp->clock_rate; - clock_comp = ((u64)1000000000ull << 32) / ptp->clock_rate; /* Initial compensation value to start the nanosecs counter */ writeq(clock_comp, ptp->reg_base + PTP_CLOCK_COMP); } static int ptp_get_tstmp(struct ptp *ptp, u64 *clk) { - *clk = readq(ptp->reg_base + PTP_TIMESTAMP); + u64 timestamp; + + if (is_ptp_dev_cn10k(ptp)) { + timestamp = readq(ptp->reg_base + PTP_TIMESTAMP); + *clk = (timestamp >> 32) * NSEC_PER_SEC + (timestamp & 0xFFFFFFFF); + } else { + *clk = readq(ptp->reg_base + PTP_TIMESTAMP); + } return 0; } static int ptp_set_thresh(struct ptp *ptp, u64 thresh) { - writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI); + if (!cn10k_ptp_errata(ptp)) + writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI); + + return 0; +} + +static int ptp_extts_on(struct ptp *ptp, int on) +{ + u64 ptp_clock_hi; + + if (cn10k_ptp_errata(ptp)) { + if (on) { + ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI); + ptp_hrtimer_start(ptp, (ktime_t)ptp_clock_hi); + } else { + if (hrtimer_active(&ptp->hrtimer)) + hrtimer_cancel(&ptp->hrtimer); + } + } return 0; } @@ -212,6 +414,17 @@ static int ptp_probe(struct pci_dev *pdev, if (!first_ptp_block) first_ptp_block = ptp; + spin_lock_init(&ptp->ptp_lock); + if (is_ptp_tsfmt_sec_nsec(ptp)) + ptp->read_ptp_tstmp = &read_ptp_tstmp_sec_nsec; + else + ptp->read_ptp_tstmp = &read_ptp_tstmp_nsec; + + if (cn10k_ptp_errata(ptp)) { + hrtimer_init(&ptp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + ptp->hrtimer.function = ptp_reset_thresh; + } + return 0; error_free: @@ -236,6 +449,9 @@ static void ptp_remove(struct pci_dev *pdev) struct ptp *ptp = pci_get_drvdata(pdev); u64 clock_cfg; + if (cn10k_ptp_errata(ptp) && hrtimer_active(&ptp->hrtimer)) + hrtimer_cancel(&ptp->hrtimer); + if (IS_ERR_OR_NULL(ptp)) return; @@ -303,6 +519,9 @@ int rvu_mbox_handler_ptp_op(struct rvu *rvu, struct ptp_req *req, case PTP_OP_SET_THRESH: err = ptp_set_thresh(rvu->ptp, req->thresh); break; + case PTP_OP_EXTTS_ON: + err = ptp_extts_on(rvu->ptp, req->extts_on); + break; default: err = -EINVAL; break; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h index 1b81a0493cd3..b9d92abc3844 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h @@ -15,7 +15,12 @@ struct ptp { struct pci_dev *pdev; void __iomem *reg_base; + u64 (*read_ptp_tstmp)(struct ptp *ptp); + spinlock_t ptp_lock; /* lock */ + struct hrtimer hrtimer; + ktime_t last_ts; u32 clock_rate; + u32 clock_period; }; struct ptp *ptp_get(void); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c index e695fa0e82a9..a70e1153fa04 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c @@ -22,6 +22,7 @@ static struct mac_ops rpm_mac_ops = { .tx_stats_cnt = 34, .get_nr_lmacs = rpm_get_nr_lmacs, .get_lmac_type = rpm_get_lmac_type, + .lmac_fifo_len = rpm_get_lmac_fifo_len, .mac_lmac_intl_lbk = rpm_lmac_internal_loopback, .mac_get_rx_stats = rpm_get_rx_stats, .mac_get_tx_stats = rpm_get_tx_stats, @@ -30,6 +31,10 @@ static struct mac_ops rpm_mac_ops = { .mac_enadis_pause_frm = rpm_lmac_enadis_pause_frm, .mac_pause_frm_config = rpm_lmac_pause_frm_config, .mac_enadis_ptp_config = rpm_lmac_ptp_config, + .mac_rx_tx_enable = rpm_lmac_rx_tx_enable, + .mac_tx_enable = rpm_lmac_tx_enable, + .pfc_config = rpm_lmac_pfc_config, + .mac_get_pfc_frm_cfg = rpm_lmac_get_pfc_frm_cfg, }; struct mac_ops *rpm_get_mac_ops(void) @@ -54,14 +59,60 @@ int rpm_get_nr_lmacs(void *rpmd) return hweight8(rpm_read(rpm, 0, CGXX_CMRX_RX_LMACS) & 0xFULL); } +int rpm_lmac_tx_enable(void *rpmd, int lmac_id, bool enable) +{ + rpm_t *rpm = rpmd; + u64 cfg, last; + + if (!is_lmac_valid(rpm, lmac_id)) + return -ENODEV; + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + last = cfg; + if (enable) + cfg |= RPM_TX_EN; + else + cfg &= ~(RPM_TX_EN); + + if (cfg != last) + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + return !!(last & RPM_TX_EN); +} + +int rpm_lmac_rx_tx_enable(void *rpmd, int lmac_id, bool enable) +{ + rpm_t *rpm = rpmd; + u64 cfg; + + if (!is_lmac_valid(rpm, lmac_id)) + return -ENODEV; + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + if (enable) + cfg |= RPM_RX_EN | RPM_TX_EN; + else + cfg &= ~(RPM_RX_EN | RPM_TX_EN); + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + return 0; +} + void rpm_lmac_enadis_rx_pause_fwding(void *rpmd, int lmac_id, bool enable) { rpm_t *rpm = rpmd; + struct lmac *lmac; u64 cfg; if (!rpm) return; + lmac = lmac_pdata(lmac_id, rpm); + if (!lmac) + return; + + /* Pause frames are not enabled just return */ + if (!bitmap_weight(lmac->rx_fc_pfvf_bmap.bmap, lmac->rx_fc_pfvf_bmap.max)) + return; + if (enable) { cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE; @@ -83,13 +134,94 @@ int rpm_lmac_get_pause_frm_status(void *rpmd, int lmac_id, return -ENODEV; cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - *rx_pause = !(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE); + if (!(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_PFC_MODE)) { + *rx_pause = !(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE); + *tx_pause = !(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE); + } - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - *tx_pause = !(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE); return 0; } +static void rpm_cfg_pfc_quanta_thresh(rpm_t *rpm, int lmac_id, + unsigned long pfc_en, + bool enable) +{ + u64 quanta_offset = 0, quanta_thresh = 0, cfg; + int i, shift; + + /* Set pause time and interval */ + for_each_set_bit(i, &pfc_en, 16) { + switch (i) { + case 0: + case 1: + quanta_offset = RPMX_MTI_MAC100X_CL01_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL01_QUANTA_THRESH; + break; + case 2: + case 3: + quanta_offset = RPMX_MTI_MAC100X_CL23_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL23_QUANTA_THRESH; + break; + case 4: + case 5: + quanta_offset = RPMX_MTI_MAC100X_CL45_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL45_QUANTA_THRESH; + break; + case 6: + case 7: + quanta_offset = RPMX_MTI_MAC100X_CL67_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL67_QUANTA_THRESH; + break; + case 8: + case 9: + quanta_offset = RPMX_MTI_MAC100X_CL89_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL89_QUANTA_THRESH; + break; + case 10: + case 11: + quanta_offset = RPMX_MTI_MAC100X_CL1011_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL1011_QUANTA_THRESH; + break; + case 12: + case 13: + quanta_offset = RPMX_MTI_MAC100X_CL1213_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL1213_QUANTA_THRESH; + break; + case 14: + case 15: + quanta_offset = RPMX_MTI_MAC100X_CL1415_PAUSE_QUANTA; + quanta_thresh = RPMX_MTI_MAC100X_CL1415_QUANTA_THRESH; + break; + } + + if (!quanta_offset || !quanta_thresh) + continue; + + shift = (i % 2) ? 1 : 0; + cfg = rpm_read(rpm, lmac_id, quanta_offset); + if (enable) { + cfg |= ((u64)RPM_DEFAULT_PAUSE_TIME << shift * 16); + } else { + if (!shift) + cfg &= ~GENMASK_ULL(15, 0); + else + cfg &= ~GENMASK_ULL(31, 16); + } + rpm_write(rpm, lmac_id, quanta_offset, cfg); + + cfg = rpm_read(rpm, lmac_id, quanta_thresh); + if (enable) { + cfg |= ((u64)(RPM_DEFAULT_PAUSE_TIME / 2) << shift * 16); + } else { + if (!shift) + cfg &= ~GENMASK_ULL(15, 0); + else + cfg &= ~GENMASK_ULL(31, 16); + } + rpm_write(rpm, lmac_id, quanta_thresh, cfg); + } +} + int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause) { @@ -113,8 +245,12 @@ int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause, cfg = rpm_read(rpm, 0, RPMX_CMR_RX_OVR_BP); if (tx_pause) { + /* Configure CL0 Pause Quanta & threshold for 802.3X frames */ + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 1, true); cfg &= ~RPMX_CMR_RX_OVR_BP_EN(lmac_id); } else { + /* Disable all Pause Quanta & threshold values */ + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 0xffff, false); cfg |= RPMX_CMR_RX_OVR_BP_EN(lmac_id); cfg &= ~RPMX_CMR_RX_OVR_BP_BP(lmac_id); } @@ -127,56 +263,28 @@ void rpm_lmac_pause_frm_config(void *rpmd, int lmac_id, bool enable) rpm_t *rpm = rpmd; u64 cfg; - if (enable) { - /* Enable 802.3 pause frame mode */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_PFC_MODE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - - /* Enable receive pause frames */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - - /* Enable forward pause to TX block */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - - /* Enable pause frames transmission */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + /* ALL pause frames received are completely ignored */ + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE; + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - /* Set pause time and interval */ - cfg = rpm_read(rpm, lmac_id, - RPMX_MTI_MAC100X_CL01_PAUSE_QUANTA); - cfg &= ~0xFFFFULL; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_CL01_PAUSE_QUANTA, - cfg | RPM_DEFAULT_PAUSE_TIME); - /* Set pause interval as the hardware default is too short */ - cfg = rpm_read(rpm, lmac_id, - RPMX_MTI_MAC100X_CL01_QUANTA_THRESH); - cfg &= ~0xFFFFULL; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_CL01_QUANTA_THRESH, - cfg | (RPM_DEFAULT_PAUSE_TIME / 2)); + /* Disable forward pause to TX block */ + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE; + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - } else { - /* ALL pause frames received are completely ignored */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + /* Disable pause frames transmission */ + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - /* Disable forward pause to TX block */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + /* Disable all PFC classes */ + cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL); + cfg = FIELD_SET(RPM_PFC_CLASS_MASK, 0, cfg); + rpm_write(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL, cfg); - /* Disable pause frames transmission */ - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); - cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; - rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); - } + /* Enable channel mask for all LMACS */ + rpm_write(rpm, 0, RPMX_CMR_CHAN_MSK_OR, ~0ULL); } int rpm_get_rx_stats(void *rpmd, int lmac_id, int idx, u64 *rx_stat) @@ -243,6 +351,35 @@ u8 rpm_get_lmac_type(void *rpmd, int lmac_id) return err; } +u32 rpm_get_lmac_fifo_len(void *rpmd, int lmac_id) +{ + rpm_t *rpm = rpmd; + u64 hi_perf_lmac; + u8 num_lmacs; + u32 fifo_len; + + fifo_len = rpm->mac_ops->fifo_len; + num_lmacs = rpm->mac_ops->get_nr_lmacs(rpm); + + switch (num_lmacs) { + case 1: + return fifo_len; + case 2: + return fifo_len / 2; + case 3: + /* LMAC marked as hi_perf gets half of the FIFO and rest 1/4th */ + hi_perf_lmac = rpm_read(rpm, 0, CGXX_CMRX_RX_LMACS); + hi_perf_lmac = (hi_perf_lmac >> 4) & 0x3ULL; + if (lmac_id == hi_perf_lmac) + return fifo_len / 2; + return fifo_len / 4; + case 4: + default: + return fifo_len / 4; + } + return 0; +} + int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable) { rpm_t *rpm = rpmd; @@ -252,23 +389,20 @@ int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable) if (!rpm || lmac_id >= rpm->lmac_count) return -ENODEV; lmac_type = rpm->mac_ops->get_lmac_type(rpm, lmac_id); - if (lmac_type == LMAC_MODE_100G_R) { - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1); - - if (enable) - cfg |= RPMX_MTI_PCS_LBK; - else - cfg &= ~RPMX_MTI_PCS_LBK; - rpm_write(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1, cfg); - } else { - cfg = rpm_read(rpm, lmac_id, RPMX_MTI_LPCSX_CONTROL1); - if (enable) - cfg |= RPMX_MTI_PCS_LBK; - else - cfg &= ~RPMX_MTI_PCS_LBK; - rpm_write(rpm, lmac_id, RPMX_MTI_LPCSX_CONTROL1, cfg); + + if (lmac_type == LMAC_MODE_QSGMII || lmac_type == LMAC_MODE_SGMII) { + dev_err(&rpm->pdev->dev, "loopback not supported for LPC mode\n"); + return 0; } + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1); + + if (enable) + cfg |= RPMX_MTI_PCS_LBK; + else + cfg &= ~RPMX_MTI_PCS_LBK; + rpm_write(rpm, lmac_id, RPMX_MTI_PCS100X_CONTROL1, cfg); + return 0; } @@ -281,9 +415,85 @@ void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable) return; cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_CFG); - if (enable) + if (enable) { cfg |= RPMX_RX_TS_PREPEND; - else + cfg |= RPMX_TX_PTP_1S_SUPPORT; + } else { cfg &= ~RPMX_RX_TS_PREPEND; + cfg &= ~RPMX_TX_PTP_1S_SUPPORT; + } + rpm_write(rpm, lmac_id, RPMX_CMRX_CFG, cfg); + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE); + + if (enable) { + cfg |= RPMX_ONESTEP_ENABLE; + cfg &= ~RPMX_TS_BINARY_MODE; + } else { + cfg &= ~RPMX_ONESTEP_ENABLE; + } + + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE, cfg); +} + +int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 pfc_en) +{ + rpm_t *rpm = rpmd; + u64 cfg, class_en; + + if (!is_lmac_valid(rpm, lmac_id)) + return -ENODEV; + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + class_en = rpm_read(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL); + pfc_en |= FIELD_GET(RPM_PFC_CLASS_MASK, class_en); + + if (rx_pause) { + cfg &= ~(RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE | + RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE | + RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_FWD); + } else { + cfg |= (RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE | + RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE | + RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_FWD); + } + + if (tx_pause) { + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, pfc_en, true); + cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; + class_en = FIELD_SET(RPM_PFC_CLASS_MASK, pfc_en, class_en); + } else { + rpm_cfg_pfc_quanta_thresh(rpm, lmac_id, 0xfff, false); + cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE; + class_en = FIELD_SET(RPM_PFC_CLASS_MASK, 0, class_en); + } + + if (!rx_pause && !tx_pause) + cfg &= ~RPMX_MTI_MAC100X_COMMAND_CONFIG_PFC_MODE; + else + cfg |= RPMX_MTI_MAC100X_COMMAND_CONFIG_PFC_MODE; + + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG, cfg); + + rpm_write(rpm, lmac_id, RPMX_CMRX_PRT_CBFC_CTL, class_en); + + return 0; +} + +int rpm_lmac_get_pfc_frm_cfg(void *rpmd, int lmac_id, u8 *tx_pause, u8 *rx_pause) +{ + rpm_t *rpm = rpmd; + u64 cfg; + + if (!is_lmac_valid(rpm, lmac_id)) + return -ENODEV; + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_COMMAND_CONFIG); + if (cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_PFC_MODE) { + *rx_pause = !(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_RX_P_DISABLE); + *tx_pause = !(cfg & RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_P_DISABLE); + } + + return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h index 57c8a687b488..77f2ef9e1425 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h @@ -16,6 +16,7 @@ /* Registers */ #define RPMX_CMRX_CFG 0x00 #define RPMX_RX_TS_PREPEND BIT_ULL(22) +#define RPMX_TX_PTP_1S_SUPPORT BIT_ULL(17) #define RPMX_CMRX_SW_INT 0x180 #define RPMX_CMRX_SW_INT_W1S 0x188 #define RPMX_CMRX_SW_INT_ENA_W1S 0x198 @@ -33,20 +34,53 @@ #define RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE BIT_ULL(8) #define RPMX_MTI_MAC100X_COMMAND_CONFIG_PFC_MODE BIT_ULL(19) #define RPMX_MTI_MAC100X_CL01_PAUSE_QUANTA 0x80A8 +#define RPMX_MTI_MAC100X_CL23_PAUSE_QUANTA 0x80B0 +#define RPMX_MTI_MAC100X_CL45_PAUSE_QUANTA 0x80B8 +#define RPMX_MTI_MAC100X_CL67_PAUSE_QUANTA 0x80C0 #define RPMX_MTI_MAC100X_CL01_QUANTA_THRESH 0x80C8 -#define RPM_DEFAULT_PAUSE_TIME 0xFFFF +#define RPMX_MTI_MAC100X_CL23_QUANTA_THRESH 0x80D0 +#define RPMX_MTI_MAC100X_CL45_QUANTA_THRESH 0x80D8 +#define RPMX_MTI_MAC100X_CL67_QUANTA_THRESH 0x80E0 +#define RPMX_MTI_MAC100X_CL89_PAUSE_QUANTA 0x8108 +#define RPMX_MTI_MAC100X_CL1011_PAUSE_QUANTA 0x8110 +#define RPMX_MTI_MAC100X_CL1213_PAUSE_QUANTA 0x8118 +#define RPMX_MTI_MAC100X_CL1415_PAUSE_QUANTA 0x8120 +#define RPMX_MTI_MAC100X_CL89_QUANTA_THRESH 0x8128 +#define RPMX_MTI_MAC100X_CL1011_QUANTA_THRESH 0x8130 +#define RPMX_MTI_MAC100X_CL1213_QUANTA_THRESH 0x8138 +#define RPMX_MTI_MAC100X_CL1415_QUANTA_THRESH 0x8140 #define RPMX_CMR_RX_OVR_BP 0x4120 #define RPMX_CMR_RX_OVR_BP_EN(x) BIT_ULL((x) + 8) #define RPMX_CMR_RX_OVR_BP_BP(x) BIT_ULL((x) + 4) +#define RPMX_CMR_CHAN_MSK_OR 0x4118 #define RPMX_MTI_STAT_RX_STAT_PAGES_COUNTERX 0x12000 #define RPMX_MTI_STAT_TX_STAT_PAGES_COUNTERX 0x13000 #define RPMX_MTI_STAT_DATA_HI_CDC 0x10038 #define RPM_LMAC_FWI 0xa +#define RPM_TX_EN BIT_ULL(0) +#define RPM_RX_EN BIT_ULL(1) +#define RPMX_CMRX_PRT_CBFC_CTL 0x5B08 +#define RPMX_CMRX_PRT_CBFC_CTL_LOGL_EN_RX_SHIFT 33 +#define RPMX_CMRX_PRT_CBFC_CTL_PHYS_BP_SHIFT 16 +#define RPMX_CMRX_PRT_CBFC_CTL_LOGL_EN_TX_SHIFT 0 +#define RPM_PFC_CLASS_MASK GENMASK_ULL(48, 33) +#define RPMX_MTI_MAC100X_CL89_QUANTA_THRESH 0x8128 +#define RPMX_MTI_MAC100X_COMMAND_CONFIG_TX_PAD_EN BIT_ULL(11) +#define RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_IGNORE BIT_ULL(8) +#define RPMX_MTI_MAC100X_COMMAND_CONFIG_PAUSE_FWD BIT_ULL(7) +#define RPMX_MTI_MAC100X_CL01_PAUSE_QUANTA 0x80A8 +#define RPMX_MTI_MAC100X_CL89_PAUSE_QUANTA 0x8108 +#define RPM_DEFAULT_PAUSE_TIME 0x7FF + +#define RPMX_MTI_MAC100X_XIF_MODE 0x8100 +#define RPMX_ONESTEP_ENABLE BIT_ULL(5) +#define RPMX_TS_BINARY_MODE BIT_ULL(11) /* Function Declarations */ int rpm_get_nr_lmacs(void *rpmd); u8 rpm_get_lmac_type(void *rpmd, int lmac_id); +u32 rpm_get_lmac_fifo_len(void *rpmd, int lmac_id); int rpm_lmac_internal_loopback(void *rpmd, int lmac_id, bool enable); void rpm_lmac_enadis_rx_pause_fwding(void *rpmd, int lmac_id, bool enable); int rpm_lmac_get_pause_frm_status(void *cgxd, int lmac_id, u8 *tx_pause, @@ -57,4 +91,10 @@ int rpm_lmac_enadis_pause_frm(void *rpmd, int lmac_id, u8 tx_pause, int rpm_get_tx_stats(void *rpmd, int lmac_id, int idx, u64 *tx_stat); int rpm_get_rx_stats(void *rpmd, int lmac_id, int idx, u64 *rx_stat); void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable); +int rpm_lmac_rx_tx_enable(void *rpmd, int lmac_id, bool enable); +int rpm_lmac_tx_enable(void *rpmd, int lmac_id, bool enable); +int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, + u16 pfc_en); +int rpm_lmac_get_pfc_frm_cfg(void *rpmd, int lmac_id, u8 *tx_pause, + u8 *rx_pause); #endif /* RPM_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 3ca6b942ebe2..3f5e09b77d4b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -16,14 +16,14 @@ #include "rvu.h" #include "rvu_reg.h" #include "ptp.h" +#include "mcs.h" #include "rvu_trace.h" +#include "rvu_npc_hash.h" #define DRV_NAME "rvu_af" #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver" -static int rvu_get_hwvf(struct rvu *rvu, int pcifunc); - static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, struct rvu_block *block, int lf); static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, @@ -68,6 +68,8 @@ static void rvu_setup_hw_capabilities(struct rvu *rvu) hw->cap.nix_tx_link_bp = true; hw->cap.nix_rx_multicast = true; hw->cap.nix_shaper_toggle_wait = false; + hw->cap.npc_hash_extract = false; + hw->cap.npc_exact_match_enabled = false; hw->rvu = rvu; if (is_rvu_pre_96xx_C0(rvu)) { @@ -85,6 +87,9 @@ static void rvu_setup_hw_capabilities(struct rvu *rvu) if (!is_rvu_otx2(rvu)) hw->cap.per_pf_mbox_regs = true; + + if (is_rvu_npc_hash_extract_en(rvu)) + hw->cap.npc_hash_extract = true; } /* Poll a RVU block's register 'offset', for a 'zero' @@ -412,7 +417,7 @@ void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf) *hwvf = cfg & 0xFFF; } -static int rvu_get_hwvf(struct rvu *rvu, int pcifunc) +int rvu_get_hwvf(struct rvu *rvu, int pcifunc) { int pf, func; u64 cfg; @@ -520,8 +525,11 @@ static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg) rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0)); err = rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true); - if (err) - dev_err(rvu->dev, "HW block:%d reset failed\n", blkaddr); + if (err) { + dev_err(rvu->dev, "HW block:%d reset timeout retrying again\n", blkaddr); + while (rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true) == -EBUSY) + ; + } } static void rvu_reset_all_blocks(struct rvu *rvu) @@ -1119,6 +1127,12 @@ cpt: goto cgx_err; } + err = rvu_npc_exact_init(rvu); + if (err) { + dev_err(rvu->dev, "failed to initialize exact match table\n"); + return err; + } + /* Assign MACs for CGX mapped functions */ rvu_setup_pfvf_macaddress(rvu); @@ -1144,6 +1158,12 @@ cpt: rvu_program_channels(rvu); + err = rvu_mcs_init(rvu); + if (err) { + dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__); + goto nix_err; + } + return 0; nix_err: @@ -1988,6 +2008,7 @@ int rvu_mbox_handler_get_hw_cap(struct rvu *rvu, struct msg_req *req, rsp->nix_fixed_txschq_mapping = hw->cap.nix_fixed_txschq_mapping; rsp->nix_shaping = hw->cap.nix_shaping; + rsp->npc_hash_extract = hw->cap.npc_hash_extract; return 0; } @@ -2545,6 +2566,9 @@ static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr) static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc) { + if (rvu_npc_exact_has_match_table(rvu)) + rvu_npc_exact_reset(rvu, pcifunc); + mutex_lock(&rvu->flr_lock); /* Reset order should reflect inter-block dependencies: * 1. Reset any packet/work sources (NIX, CPT, TIM) @@ -2561,6 +2585,12 @@ static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc) rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA); rvu_reset_lmt_map_tbl(rvu, pcifunc); rvu_detach_rsrcs(rvu, NULL, pcifunc); + /* In scenarios where PF/VF drivers detach NIXLF without freeing MCAM + * entries, check and free the MCAM entries explicitly to avoid leak. + * Since LF is detached use LF number as -1. + */ + rvu_npc_free_mcam_entries(rvu, pcifunc, -1); + mutex_unlock(&rvu->flr_lock); } @@ -3268,6 +3298,7 @@ err_mbox: err_hwsetup: rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); + rvu_mcs_exit(rvu); rvu_reset_all_blocks(rvu); rvu_free_hw_resources(rvu); rvu_clear_rvum_blk_revid(rvu); @@ -3294,6 +3325,7 @@ static void rvu_remove(struct pci_dev *pdev) rvu_flr_wq_destroy(rvu); rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); + rvu_mcs_exit(rvu); rvu_mbox_destroy(&rvu->afpf_wq_info); rvu_disable_sriov(rvu); rvu_reset_all_blocks(rvu); @@ -3329,12 +3361,18 @@ static int __init rvu_init_module(void) if (err < 0) goto ptp_err; + err = pci_register_driver(&mcs_driver); + if (err < 0) + goto mcs_err; + err = pci_register_driver(&rvu_driver); if (err < 0) goto rvu_err; return 0; rvu_err: + pci_unregister_driver(&mcs_driver); +mcs_err: pci_unregister_driver(&ptp_driver); ptp_err: pci_unregister_driver(&cgx_driver); @@ -3345,6 +3383,7 @@ ptp_err: static void __exit rvu_cleanup_module(void) { pci_unregister_driver(&rvu_driver); + pci_unregister_driver(&mcs_driver); pci_unregister_driver(&ptp_driver); pci_unregister_driver(&cgx_driver); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index 66e45d733824..76474385a602 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -25,6 +25,8 @@ /* Subsystem Device ID */ #define PCI_SUBSYS_DEVID_96XX 0xB200 #define PCI_SUBSYS_DEVID_CN10K_A 0xB900 +#define PCI_SUBSYS_DEVID_CNF10K_B 0xBC00 +#define PCI_SUBSYS_DEVID_CN10K_B 0xBD00 /* PCI BAR nos */ #define PCI_AF_REG_BAR_NUM 0 @@ -62,6 +64,10 @@ struct rvu_debugfs { struct dentry *nix; struct dentry *npc; struct dentry *cpt; + struct dentry *mcs_root; + struct dentry *mcs; + struct dentry *mcs_rx; + struct dentry *mcs_tx; struct dump_ctx npa_aura_ctx; struct dump_ctx npa_pool_ctx; struct dump_ctx nix_cq_ctx; @@ -338,6 +344,8 @@ struct hw_cap { bool per_pf_mbox_regs; /* PF mbox specified in per PF registers ? */ bool programmable_chans; /* Channels programmable ? */ bool ipolicer; + bool npc_hash_extract; /* Hash extract enabled ? */ + bool npc_exact_match_enabled; /* Exact match supported ? */ }; struct rvu_hwinfo { @@ -369,6 +377,7 @@ struct rvu_hwinfo { struct rvu *rvu; struct npc_pkind pkind; struct npc_mcam mcam; + struct npc_exact_table *table; }; struct mbox_wq_info { @@ -419,6 +428,7 @@ struct npc_kpu_profile_adapter { const struct npc_kpu_profile_action *ikpu; /* array[pkinds] */ const struct npc_kpu_profile *kpu; /* array[kpus] */ struct npc_mcam_kex *mkex; + struct npc_mcam_kex_hash *mkex_hash; bool custom; size_t pkinds; size_t kpus; @@ -493,6 +503,8 @@ struct rvu { struct ptp *ptp; + int mcs_blk_cnt; + #ifdef CONFIG_DEBUG_FS struct rvu_debugfs rvu_dbg; #endif @@ -500,6 +512,12 @@ struct rvu { /* RVU switch implementation over NPC with DMAC rules */ struct rvu_switch rswitch; + + struct work_struct mcs_intr_work; + struct workqueue_struct *mcs_intr_wq; + struct list_head mcs_intrq_head; + /* mcs interrupt queue lock */ + spinlock_t mcs_intrq_lock; }; static inline void rvu_write64(struct rvu *rvu, u64 block, u64 offset, u64 val) @@ -575,6 +593,17 @@ static inline bool is_rvu_otx2(struct rvu *rvu) midr == PCI_REVISION_ID_95XXMM || midr == PCI_REVISION_ID_95XXO); } +static inline bool is_rvu_npc_hash_extract_en(struct rvu *rvu) +{ + u64 npc_const3; + + npc_const3 = rvu_read64(rvu, BLKADDR_NPC, NPC_AF_CONST3); + if (!(npc_const3 & BIT_ULL(62))) + return false; + + return true; +} + static inline u16 rvu_nix_chan_cgx(struct rvu *rvu, u8 cgxid, u8 lmacid, u8 chan) { @@ -754,7 +783,6 @@ u32 convert_dwrr_mtu_to_bytes(u8 dwrr_mtu); u32 convert_bytes_to_dwrr_mtu(u32 bytes); /* NPC APIs */ -int rvu_npc_init(struct rvu *rvu); void rvu_npc_freemem(struct rvu *rvu); int rvu_npc_get_pkind(struct rvu *rvu, u16 pf); void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf); @@ -773,14 +801,17 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, u64 chan); void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, bool enable); + void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, int nixlf, int type, bool enable); void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf); +bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable); void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf); void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf); void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf); void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, int group, int alg_idx, int mcam_index); + void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, int blkaddr, int *alloc_cnt, int *enable_cnt); @@ -806,11 +837,20 @@ bool is_mac_feature_supported(struct rvu *rvu, int pf, int feature); u32 rvu_cgx_get_fifolen(struct rvu *rvu); void *rvu_first_cgx_pdata(struct rvu *rvu); int cgxlmac_to_pf(struct rvu *rvu, int cgx_id, int lmac_id); - +int rvu_cgx_config_tx(void *cgxd, int lmac_id, bool enable); +int rvu_cgx_prio_flow_ctrl_cfg(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause, + u16 pfc_en); +int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause); +u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac); int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, u16 pcifunc, int nixlf, int type); bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index); +int rvu_npc_init(struct rvu *rvu); +int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, + u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask, + u64 bcast_mcast_val, u64 bcast_mcast_mask); +void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx); /* CPT APIs */ int rvu_cpt_register_interrupts(struct rvu *rvu); @@ -842,4 +882,11 @@ void rvu_switch_update_rules(struct rvu *rvu, u16 pcifunc); int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, u64 pkind, u8 var_len_off, u8 var_len_off_mask, u8 shift_dir); +int rvu_get_hwvf(struct rvu *rvu, int pcifunc); + +/* CN10K MCS */ +int rvu_mcs_init(struct rvu *rvu); +int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc); +void rvu_mcs_exit(struct rvu *rvu); + #endif /* RVU_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c index 2ca182a4ce82..addc69f4b65c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cgx.c @@ -14,6 +14,7 @@ #include "lmac_common.h" #include "rvu_reg.h" #include "rvu_trace.h" +#include "rvu_npc_hash.h" struct cgx_evq_entry { struct list_head evq_node; @@ -441,16 +442,26 @@ void rvu_cgx_enadis_rx_bp(struct rvu *rvu, int pf, bool enable) int rvu_cgx_config_rxtx(struct rvu *rvu, u16 pcifunc, bool start) { int pf = rvu_get_pf(pcifunc); + struct mac_ops *mac_ops; u8 cgx_id, lmac_id; + void *cgxd; if (!is_cgx_config_permitted(rvu, pcifunc)) return LMAC_AF_ERR_PERM_DENIED; rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + cgxd = rvu_cgx_pdata(cgx_id, rvu); + mac_ops = get_mac_ops(cgxd); - cgx_lmac_rx_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, start); + return mac_ops->mac_rx_tx_enable(cgxd, lmac_id, start); +} - return 0; +int rvu_cgx_config_tx(void *cgxd, int lmac_id, bool enable) +{ + struct mac_ops *mac_ops; + + mac_ops = get_mac_ops(cgxd); + return mac_ops->mac_tx_enable(cgxd, lmac_id, enable); } void rvu_cgx_disable_dmac_entries(struct rvu *rvu, u16 pcifunc) @@ -464,6 +475,11 @@ void rvu_cgx_disable_dmac_entries(struct rvu *rvu, u16 pcifunc) if (!is_cgx_config_permitted(rvu, pcifunc)) return; + if (rvu_npc_exact_has_match_table(rvu)) { + rvu_npc_exact_reset(rvu, pcifunc); + return; + } + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); cgx_dev = cgx_get_pdata(cgx_id); lmac_count = cgx_get_lmac_cnt(cgx_dev); @@ -574,6 +590,9 @@ int rvu_mbox_handler_cgx_mac_addr_set(struct rvu *rvu, if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return -EPERM; + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_mac_addr_set(rvu, req, rsp); + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); cgx_lmac_addr_set(cgx_id, lmac_id, req->mac_addr); @@ -592,6 +611,9 @@ int rvu_mbox_handler_cgx_mac_addr_add(struct rvu *rvu, if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return -EPERM; + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_mac_addr_add(rvu, req, rsp); + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); rc = cgx_lmac_addr_add(cgx_id, lmac_id, req->mac_addr); if (rc >= 0) { @@ -612,6 +634,9 @@ int rvu_mbox_handler_cgx_mac_addr_del(struct rvu *rvu, if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return -EPERM; + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_mac_addr_del(rvu, req, rsp); + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); return cgx_lmac_addr_del(cgx_id, lmac_id, req->index); } @@ -633,6 +658,11 @@ int rvu_mbox_handler_cgx_mac_max_entries_get(struct rvu *rvu, return 0; } + if (rvu_npc_exact_has_match_table(rvu)) { + rsp->max_dmac_filters = rvu_npc_exact_get_max_entries(rvu); + return 0; + } + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); rsp->max_dmac_filters = cgx_lmac_addr_max_entries_get(cgx_id, lmac_id); return 0; @@ -670,6 +700,10 @@ int rvu_mbox_handler_cgx_promisc_enable(struct rvu *rvu, struct msg_req *req, if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return -EPERM; + /* Disable drop on non hit rule */ + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_promisc_enable(rvu, req->hdr.pcifunc); + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); cgx_lmac_promisc_config(cgx_id, lmac_id, true); @@ -685,6 +719,10 @@ int rvu_mbox_handler_cgx_promisc_disable(struct rvu *rvu, struct msg_req *req, if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return -EPERM; + /* Disable drop on non hit rule */ + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_promisc_disable(rvu, req->hdr.pcifunc); + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); cgx_lmac_promisc_config(cgx_id, lmac_id, false); @@ -823,6 +861,22 @@ u32 rvu_cgx_get_fifolen(struct rvu *rvu) return fifo_len; } +u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac) +{ + struct mac_ops *mac_ops; + void *cgxd; + + cgxd = rvu_cgx_pdata(cgx, rvu); + if (!cgxd) + return 0; + + mac_ops = get_mac_ops(cgxd); + if (!mac_ops->lmac_fifo_len) + return 0; + + return mac_ops->lmac_fifo_len(cgxd, lmac); +} + static int rvu_cgx_config_intlbk(struct rvu *rvu, u16 pcifunc, bool en) { int pf = rvu_get_pf(pcifunc); @@ -853,6 +907,45 @@ int rvu_mbox_handler_cgx_intlbk_disable(struct rvu *rvu, struct msg_req *req, return 0; } +int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause) +{ + int pf = rvu_get_pf(pcifunc); + u8 rx_pfc = 0, tx_pfc = 0; + struct mac_ops *mac_ops; + u8 cgx_id, lmac_id; + void *cgxd; + + if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_FC)) + return 0; + + /* This msg is expected only from PF/VFs that are mapped to CGX LMACs, + * if received from other PF/VF simply ACK, nothing to do. + */ + if (!is_pf_cgxmapped(rvu, pf)) + return LMAC_AF_ERR_PF_NOT_MAPPED; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + cgxd = rvu_cgx_pdata(cgx_id, rvu); + mac_ops = get_mac_ops(cgxd); + + mac_ops->mac_get_pfc_frm_cfg(cgxd, lmac_id, &tx_pfc, &rx_pfc); + if (tx_pfc || rx_pfc) { + dev_warn(rvu->dev, + "Can not configure 802.3X flow control as PFC frames are enabled"); + return LMAC_AF_ERR_8023PAUSE_ENADIS_PERM_DENIED; + } + + mutex_lock(&rvu->rsrc_lock); + if (verify_lmac_fc_cfg(cgxd, lmac_id, tx_pause, rx_pause, + pcifunc & RVU_PFVF_FUNC_MASK)) { + mutex_unlock(&rvu->rsrc_lock); + return LMAC_AF_ERR_PERM_DENIED; + } + mutex_unlock(&rvu->rsrc_lock); + + return mac_ops->mac_enadis_pause_frm(cgxd, lmac_id, tx_pause, rx_pause); +} + int rvu_mbox_handler_cgx_cfg_pause_frm(struct rvu *rvu, struct cgx_pause_frm_cfg *req, struct cgx_pause_frm_cfg *rsp) @@ -860,11 +953,9 @@ int rvu_mbox_handler_cgx_cfg_pause_frm(struct rvu *rvu, int pf = rvu_get_pf(req->hdr.pcifunc); struct mac_ops *mac_ops; u8 cgx_id, lmac_id; + int err = 0; void *cgxd; - if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_FC)) - return 0; - /* This msg is expected only from PF/VFs that are mapped to CGX LMACs, * if received from other PF/VF simply ACK, nothing to do. */ @@ -876,13 +967,11 @@ int rvu_mbox_handler_cgx_cfg_pause_frm(struct rvu *rvu, mac_ops = get_mac_ops(cgxd); if (req->set) - mac_ops->mac_enadis_pause_frm(cgxd, lmac_id, - req->tx_pause, req->rx_pause); + err = rvu_cgx_cfg_pause_frm(rvu, req->hdr.pcifunc, req->tx_pause, req->rx_pause); else - mac_ops->mac_get_pause_frm_status(cgxd, lmac_id, - &rsp->tx_pause, - &rsp->rx_pause); - return 0; + mac_ops->mac_get_pause_frm_status(cgxd, lmac_id, &rsp->tx_pause, &rsp->rx_pause); + + return err; } int rvu_mbox_handler_cgx_get_phy_fec_stats(struct rvu *rvu, struct msg_req *req, @@ -1014,7 +1103,7 @@ int rvu_mbox_handler_cgx_get_aux_link_info(struct rvu *rvu, struct msg_req *req, u8 cgx_id, lmac_id; if (!rvu->fwdata) - return -ENXIO; + return LMAC_AF_ERR_FIRMWARE_DATA_NOT_MAPPED; if (!is_pf_cgxmapped(rvu, pf)) return -EPERM; @@ -1043,7 +1132,7 @@ int rvu_mbox_handler_cgx_set_link_mode(struct rvu *rvu, return 0; } -int rvu_mbox_handler_cgx_mac_addr_reset(struct rvu *rvu, struct msg_req *req, +int rvu_mbox_handler_cgx_mac_addr_reset(struct rvu *rvu, struct cgx_mac_addr_reset_req *req, struct msg_rsp *rsp) { int pf = rvu_get_pf(req->hdr.pcifunc); @@ -1053,12 +1142,16 @@ int rvu_mbox_handler_cgx_mac_addr_reset(struct rvu *rvu, struct msg_req *req, return LMAC_AF_ERR_PERM_DENIED; rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_mac_addr_reset(rvu, req, rsp); + return cgx_lmac_addr_reset(cgx_id, lmac_id); } int rvu_mbox_handler_cgx_mac_addr_update(struct rvu *rvu, struct cgx_mac_addr_update_req *req, - struct msg_rsp *rsp) + struct cgx_mac_addr_update_rsp *rsp) { int pf = rvu_get_pf(req->hdr.pcifunc); u8 cgx_id, lmac_id; @@ -1066,6 +1159,73 @@ int rvu_mbox_handler_cgx_mac_addr_update(struct rvu *rvu, if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) return LMAC_AF_ERR_PERM_DENIED; + if (rvu_npc_exact_has_match_table(rvu)) + return rvu_npc_exact_mac_addr_update(rvu, req, rsp); + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); return cgx_lmac_addr_update(cgx_id, lmac_id, req->mac_addr, req->index); } + +int rvu_cgx_prio_flow_ctrl_cfg(struct rvu *rvu, u16 pcifunc, u8 tx_pause, + u8 rx_pause, u16 pfc_en) +{ + int pf = rvu_get_pf(pcifunc); + u8 rx_8023 = 0, tx_8023 = 0; + struct mac_ops *mac_ops; + u8 cgx_id, lmac_id; + void *cgxd; + + /* This msg is expected only from PF/VFs that are mapped to CGX LMACs, + * if received from other PF/VF simply ACK, nothing to do. + */ + if (!is_pf_cgxmapped(rvu, pf)) + return -ENODEV; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + cgxd = rvu_cgx_pdata(cgx_id, rvu); + mac_ops = get_mac_ops(cgxd); + + mac_ops->mac_get_pause_frm_status(cgxd, lmac_id, &tx_8023, &rx_8023); + if (tx_8023 || rx_8023) { + dev_warn(rvu->dev, + "Can not configure PFC as 802.3X pause frames are enabled"); + return LMAC_AF_ERR_PFC_ENADIS_PERM_DENIED; + } + + mutex_lock(&rvu->rsrc_lock); + if (verify_lmac_fc_cfg(cgxd, lmac_id, tx_pause, rx_pause, + pcifunc & RVU_PFVF_FUNC_MASK)) { + mutex_unlock(&rvu->rsrc_lock); + return LMAC_AF_ERR_PERM_DENIED; + } + mutex_unlock(&rvu->rsrc_lock); + + return mac_ops->pfc_config(cgxd, lmac_id, tx_pause, rx_pause, pfc_en); +} + +int rvu_mbox_handler_cgx_prio_flow_ctrl_cfg(struct rvu *rvu, + struct cgx_pfc_cfg *req, + struct cgx_pfc_rsp *rsp) +{ + int pf = rvu_get_pf(req->hdr.pcifunc); + struct mac_ops *mac_ops; + u8 cgx_id, lmac_id; + void *cgxd; + int err; + + /* This msg is expected only from PF/VFs that are mapped to CGX LMACs, + * if received from other PF/VF simply ACK, nothing to do. + */ + if (!is_pf_cgxmapped(rvu, pf)) + return -ENODEV; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + cgxd = rvu_cgx_pdata(cgx_id, rvu); + mac_ops = get_mac_ops(cgxd); + + err = rvu_cgx_prio_flow_ctrl_cfg(rvu, req->hdr.pcifunc, req->tx_pause, + req->rx_pause, req->pfc_en); + + mac_ops->mac_get_pfc_frm_cfg(cgxd, lmac_id, &rsp->tx_pause, &rsp->rx_pause); + return err; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cpt.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cpt.c index 45357deecabb..38bbae5d9ae0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cpt.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cpt.c @@ -17,7 +17,7 @@ #define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2 /* Length of initial context fetch in 128 byte words */ -#define CPT_CTX_ILEN 2 +#define CPT_CTX_ILEN 2ULL #define cpt_get_eng_sts(e_min, e_max, rsp, etype) \ ({ \ @@ -172,14 +172,13 @@ static int cpt_10k_register_interrupts(struct rvu_block *block, int off) { struct rvu *rvu = block->rvu; int blkaddr = block->addr; - char irq_name[16]; int i, ret; for (i = CPT_10K_AF_INT_VEC_FLT0; i < CPT_10K_AF_INT_VEC_RVU; i++) { - snprintf(irq_name, sizeof(irq_name), "CPTAF FLT%d", i); + sprintf(&rvu->irq_name[(off + i) * NAME_SIZE], "CPTAF FLT%d", i); ret = rvu_cpt_do_register_interrupt(block, off + i, rvu_cpt_af_flt_intr_handler, - irq_name); + &rvu->irq_name[(off + i) * NAME_SIZE]); if (ret) goto err; rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), 0x1); @@ -481,7 +480,7 @@ static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf, */ if (!is_rvu_otx2(rvu)) { val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16); - val |= rvu->hw->cpt_chan_base; + val |= (u64)rvu->hw->cpt_chan_base; rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val); rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val); @@ -580,7 +579,7 @@ static bool is_valid_offset(struct rvu *rvu, struct cpt_rd_wr_reg_msg *req) blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); if (blkaddr < 0) - return blkaddr; + return false; /* Registers that can be accessed from PF/VF */ if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) || @@ -606,6 +605,7 @@ static bool is_valid_offset(struct rvu *rvu, struct cpt_rd_wr_reg_msg *req) } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) { /* Registers that can be accessed from PF */ switch (offset) { + case CPT_AF_DIAG: case CPT_AF_CTL: case CPT_AF_PF_FUNC: case CPT_AF_BLK_RST: diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index a09a507369ac..a1970ebedf95 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -18,6 +18,8 @@ #include "cgx.h" #include "lmac_common.h" #include "npc.h" +#include "rvu_npc_hash.h" +#include "mcs.h" #define DEBUGFS_DIR_NAME "octeontx2" @@ -226,6 +228,350 @@ static const struct file_operations rvu_dbg_##name##_fops = { \ static void print_nix_qsize(struct seq_file *filp, struct rvu_pfvf *pfvf); +static int rvu_dbg_mcs_port_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_port_stats stats; + int lmac; + + seq_puts(filp, "\n port stats\n"); + mutex_lock(&mcs->stats_lock); + for_each_set_bit(lmac, &mcs->hw->lmac_bmap, mcs->hw->lmac_cnt) { + mcs_get_port_stats(mcs, &stats, lmac, dir); + seq_printf(filp, "port%d: Tcam Miss: %lld\n", lmac, stats.tcam_miss_cnt); + seq_printf(filp, "port%d: Parser errors: %lld\n", lmac, stats.parser_err_cnt); + + if (dir == MCS_RX && mcs->hw->mcs_blks > 1) + seq_printf(filp, "port%d: Preempt error: %lld\n", lmac, + stats.preempt_err_cnt); + if (dir == MCS_TX) + seq_printf(filp, "port%d: Sectag insert error: %lld\n", lmac, + stats.sectag_insert_err_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_rx_port_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_port_stats, mcs_rx_port_stats_display, NULL); + +static int rvu_dbg_mcs_tx_port_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_port_stats, mcs_tx_port_stats_display, NULL); + +static int rvu_dbg_mcs_sa_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_sa_stats stats; + struct rsrc_bmap *map; + int sa_id; + + if (dir == MCS_TX) { + map = &mcs->tx.sa; + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) { + seq_puts(filp, "\n TX SA stats\n"); + mcs_get_sa_stats(mcs, &stats, sa_id, MCS_TX); + seq_printf(filp, "sa%d: Pkts encrypted: %lld\n", sa_id, + stats.pkt_encrypt_cnt); + + seq_printf(filp, "sa%d: Pkts protected: %lld\n", sa_id, + stats.pkt_protected_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; + } + + /* RX stats */ + map = &mcs->rx.sa; + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) { + seq_puts(filp, "\n RX SA stats\n"); + mcs_get_sa_stats(mcs, &stats, sa_id, MCS_RX); + seq_printf(filp, "sa%d: Invalid pkts: %lld\n", sa_id, stats.pkt_invalid_cnt); + seq_printf(filp, "sa%d: Pkts no sa error: %lld\n", sa_id, stats.pkt_nosaerror_cnt); + seq_printf(filp, "sa%d: Pkts not valid: %lld\n", sa_id, stats.pkt_notvalid_cnt); + seq_printf(filp, "sa%d: Pkts ok: %lld\n", sa_id, stats.pkt_ok_cnt); + seq_printf(filp, "sa%d: Pkts no sa: %lld\n", sa_id, stats.pkt_nosa_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_rx_sa_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_sa_stats, mcs_rx_sa_stats_display, NULL); + +static int rvu_dbg_mcs_tx_sa_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_sa_stats, mcs_tx_sa_stats_display, NULL); + +static int rvu_dbg_mcs_tx_sc_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_sc_stats stats; + struct rsrc_bmap *map; + int sc_id; + + map = &mcs->tx.sc; + seq_puts(filp, "\n SC stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) { + mcs_get_sc_stats(mcs, &stats, sc_id, MCS_TX); + seq_printf(filp, "\n=======sc%d======\n\n", sc_id); + seq_printf(filp, "sc%d: Pkts encrypted: %lld\n", sc_id, stats.pkt_encrypt_cnt); + seq_printf(filp, "sc%d: Pkts protected: %lld\n", sc_id, stats.pkt_protected_cnt); + + if (mcs->hw->mcs_blks == 1) { + seq_printf(filp, "sc%d: Octets encrypted: %lld\n", sc_id, + stats.octet_encrypt_cnt); + seq_printf(filp, "sc%d: Octets protected: %lld\n", sc_id, + stats.octet_protected_cnt); + } + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_sc_stats, mcs_tx_sc_stats_display, NULL); + +static int rvu_dbg_mcs_rx_sc_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_sc_stats stats; + struct rsrc_bmap *map; + int sc_id; + + map = &mcs->rx.sc; + seq_puts(filp, "\n SC stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) { + mcs_get_sc_stats(mcs, &stats, sc_id, MCS_RX); + seq_printf(filp, "\n=======sc%d======\n\n", sc_id); + seq_printf(filp, "sc%d: Cam hits: %lld\n", sc_id, stats.hit_cnt); + seq_printf(filp, "sc%d: Invalid pkts: %lld\n", sc_id, stats.pkt_invalid_cnt); + seq_printf(filp, "sc%d: Late pkts: %lld\n", sc_id, stats.pkt_late_cnt); + seq_printf(filp, "sc%d: Notvalid pkts: %lld\n", sc_id, stats.pkt_notvalid_cnt); + seq_printf(filp, "sc%d: Unchecked pkts: %lld\n", sc_id, stats.pkt_unchecked_cnt); + + if (mcs->hw->mcs_blks > 1) { + seq_printf(filp, "sc%d: Delay pkts: %lld\n", sc_id, stats.pkt_delay_cnt); + seq_printf(filp, "sc%d: Pkts ok: %lld\n", sc_id, stats.pkt_ok_cnt); + } + if (mcs->hw->mcs_blks == 1) { + seq_printf(filp, "sc%d: Octets decrypted: %lld\n", sc_id, + stats.octet_decrypt_cnt); + seq_printf(filp, "sc%d: Octets validated: %lld\n", sc_id, + stats.octet_validate_cnt); + } + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_sc_stats, mcs_rx_sc_stats_display, NULL); + +static int rvu_dbg_mcs_flowid_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_flowid_stats stats; + struct rsrc_bmap *map; + int flow_id; + + seq_puts(filp, "\n Flowid stats\n"); + + if (dir == MCS_RX) + map = &mcs->rx.flow_ids; + else + map = &mcs->tx.flow_ids; + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(flow_id, map->bmap, mcs->hw->tcam_entries) { + mcs_get_flowid_stats(mcs, &stats, flow_id, dir); + seq_printf(filp, "Flowid%d: Hit:%lld\n", flow_id, stats.tcam_hit_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_tx_flowid_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_flowid_stats, mcs_tx_flowid_stats_display, NULL); + +static int rvu_dbg_mcs_rx_flowid_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_flowid_stats, mcs_rx_flowid_stats_display, NULL); + +static int rvu_dbg_mcs_tx_secy_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_secy_stats stats; + struct rsrc_bmap *map; + int secy_id; + + map = &mcs->tx.secy; + seq_puts(filp, "\n MCS TX secy stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) { + mcs_get_tx_secy_stats(mcs, &stats, secy_id); + seq_printf(filp, "\n=======Secy%d======\n\n", secy_id); + seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id, + stats.ctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id, + stats.ctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id, + stats.ctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt); + seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id, + stats.unctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id, + stats.unctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id, + stats.unctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt); + seq_printf(filp, "secy%d: Octet encrypted: %lld\n", secy_id, + stats.octet_encrypted_cnt); + seq_printf(filp, "secy%d: octet protected: %lld\n", secy_id, + stats.octet_protected_cnt); + seq_printf(filp, "secy%d: Pkts on active sa: %lld\n", secy_id, + stats.pkt_noactivesa_cnt); + seq_printf(filp, "secy%d: Pkts too long: %lld\n", secy_id, stats.pkt_toolong_cnt); + seq_printf(filp, "secy%d: Pkts untagged: %lld\n", secy_id, stats.pkt_untagged_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_secy_stats, mcs_tx_secy_stats_display, NULL); + +static int rvu_dbg_mcs_rx_secy_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_secy_stats stats; + struct rsrc_bmap *map; + int secy_id; + + map = &mcs->rx.secy; + seq_puts(filp, "\n MCS secy stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) { + mcs_get_rx_secy_stats(mcs, &stats, secy_id); + seq_printf(filp, "\n=======Secy%d======\n\n", secy_id); + seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id, + stats.ctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id, + stats.ctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id, + stats.ctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt); + seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id, + stats.unctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id, + stats.unctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id, + stats.unctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt); + seq_printf(filp, "secy%d: Octet decrypted: %lld\n", secy_id, + stats.octet_decrypted_cnt); + seq_printf(filp, "secy%d: octet validated: %lld\n", secy_id, + stats.octet_validated_cnt); + seq_printf(filp, "secy%d: Pkts on disable port: %lld\n", secy_id, + stats.pkt_port_disabled_cnt); + seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_badtag_cnt); + seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_nosa_cnt); + seq_printf(filp, "secy%d: Pkts with nosaerror: %lld\n", secy_id, + stats.pkt_nosaerror_cnt); + seq_printf(filp, "secy%d: Tagged ctrl pkts: %lld\n", secy_id, + stats.pkt_tagged_ctl_cnt); + seq_printf(filp, "secy%d: Untaged pkts: %lld\n", secy_id, stats.pkt_untaged_cnt); + seq_printf(filp, "secy%d: Ctrl pkts: %lld\n", secy_id, stats.pkt_ctl_cnt); + if (mcs->hw->mcs_blks > 1) + seq_printf(filp, "secy%d: pkts notag: %lld\n", secy_id, + stats.pkt_notag_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_secy_stats, mcs_rx_secy_stats_display, NULL); + +static void rvu_dbg_mcs_init(struct rvu *rvu) +{ + struct mcs *mcs; + char dname[10]; + int i; + + if (!rvu->mcs_blk_cnt) + return; + + rvu->rvu_dbg.mcs_root = debugfs_create_dir("mcs", rvu->rvu_dbg.root); + + for (i = 0; i < rvu->mcs_blk_cnt; i++) { + mcs = mcs_get_pdata(i); + + sprintf(dname, "mcs%d", i); + rvu->rvu_dbg.mcs = debugfs_create_dir(dname, + rvu->rvu_dbg.mcs_root); + + rvu->rvu_dbg.mcs_rx = debugfs_create_dir("rx_stats", rvu->rvu_dbg.mcs); + + debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_flowid_stats_fops); + + debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_secy_stats_fops); + + debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_sc_stats_fops); + + debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_sa_stats_fops); + + debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_port_stats_fops); + + rvu->rvu_dbg.mcs_tx = debugfs_create_dir("tx_stats", rvu->rvu_dbg.mcs); + + debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_flowid_stats_fops); + + debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_secy_stats_fops); + + debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_sc_stats_fops); + + debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_sa_stats_fops); + + debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_port_stats_fops); + } +} + #define LMT_MAPTBL_ENTRY_SIZE 16 /* Dump LMTST map table */ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, @@ -248,7 +594,7 @@ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, buf = kzalloc(buf_size, GFP_KERNEL); if (!buf) - return -ENOSPC; + return -ENOMEM; tbl_base = rvu_read64(rvu, BLKADDR_APR, APR_AF_LMT_MAP_BASE); @@ -407,7 +753,7 @@ static ssize_t rvu_dbg_rsrc_attach_status(struct file *filp, buf = kzalloc(buf_size, GFP_KERNEL); if (!buf) - return -ENOSPC; + return -ENOMEM; /* Get the maximum width of a column */ lf_str_size = get_max_column_width(rvu); @@ -1224,6 +1570,8 @@ static void print_nix_cn10k_sq_ctx(struct seq_file *m, seq_printf(m, "W3: head_offset\t\t\t%d\nW3: smenq_next_sqb_vld\t\t%d\n\n", sq_ctx->head_offset, sq_ctx->smenq_next_sqb_vld); + seq_printf(m, "W3: smq_next_sq_vld\t\t%d\nW3: smq_pend\t\t\t%d\n", + sq_ctx->smq_next_sq_vld, sq_ctx->smq_pend); seq_printf(m, "W4: next_sqb \t\t\t%llx\n\n", sq_ctx->next_sqb); seq_printf(m, "W5: tail_sqb \t\t\t%llx\n\n", sq_ctx->tail_sqb); seq_printf(m, "W6: smenq_sqb \t\t\t%llx\n\n", sq_ctx->smenq_sqb); @@ -2598,6 +2946,170 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) RVU_DEBUG_SEQ_FOPS(npc_mcam_rules, npc_mcam_show_rules, NULL); +static int rvu_dbg_npc_exact_show_entries(struct seq_file *s, void *unused) +{ + struct npc_exact_table_entry *mem_entry[NPC_EXACT_TBL_MAX_WAYS] = { 0 }; + struct npc_exact_table_entry *cam_entry; + struct npc_exact_table *table; + struct rvu *rvu = s->private; + int i, j; + + u8 bitmap = 0; + + table = rvu->hw->table; + + mutex_lock(&table->lock); + + /* Check if there is at least one entry in mem table */ + if (!table->mem_tbl_entry_cnt) + goto dump_cam_table; + + /* Print table headers */ + seq_puts(s, "\n\tExact Match MEM Table\n"); + seq_puts(s, "Index\t"); + + for (i = 0; i < table->mem_table.ways; i++) { + mem_entry[i] = list_first_entry_or_null(&table->lhead_mem_tbl_entry[i], + struct npc_exact_table_entry, list); + + seq_printf(s, "Way-%d\t\t\t\t\t", i); + } + + seq_puts(s, "\n"); + for (i = 0; i < table->mem_table.ways; i++) + seq_puts(s, "\tChan MAC \t"); + + seq_puts(s, "\n\n"); + + /* Print mem table entries */ + for (i = 0; i < table->mem_table.depth; i++) { + bitmap = 0; + for (j = 0; j < table->mem_table.ways; j++) { + if (!mem_entry[j]) + continue; + + if (mem_entry[j]->index != i) + continue; + + bitmap |= BIT(j); + } + + /* No valid entries */ + if (!bitmap) + continue; + + seq_printf(s, "%d\t", i); + for (j = 0; j < table->mem_table.ways; j++) { + if (!(bitmap & BIT(j))) { + seq_puts(s, "nil\t\t\t\t\t"); + continue; + } + + seq_printf(s, "0x%x %pM\t\t\t", mem_entry[j]->chan, + mem_entry[j]->mac); + mem_entry[j] = list_next_entry(mem_entry[j], list); + } + seq_puts(s, "\n"); + } + +dump_cam_table: + + if (!table->cam_tbl_entry_cnt) + goto done; + + seq_puts(s, "\n\tExact Match CAM Table\n"); + seq_puts(s, "index\tchan\tMAC\n"); + + /* Traverse cam table entries */ + list_for_each_entry(cam_entry, &table->lhead_cam_tbl_entry, list) { + seq_printf(s, "%d\t0x%x\t%pM\n", cam_entry->index, cam_entry->chan, + cam_entry->mac); + } + +done: + mutex_unlock(&table->lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(npc_exact_entries, npc_exact_show_entries, NULL); + +static int rvu_dbg_npc_exact_show_info(struct seq_file *s, void *unused) +{ + struct npc_exact_table *table; + struct rvu *rvu = s->private; + int i; + + table = rvu->hw->table; + + seq_puts(s, "\n\tExact Table Info\n"); + seq_printf(s, "Exact Match Feature : %s\n", + rvu->hw->cap.npc_exact_match_enabled ? "enabled" : "disable"); + if (!rvu->hw->cap.npc_exact_match_enabled) + return 0; + + seq_puts(s, "\nMCAM Index\tMAC Filter Rules Count\n"); + for (i = 0; i < table->num_drop_rules; i++) + seq_printf(s, "%d\t\t%d\n", i, table->cnt_cmd_rules[i]); + + seq_puts(s, "\nMcam Index\tPromisc Mode Status\n"); + for (i = 0; i < table->num_drop_rules; i++) + seq_printf(s, "%d\t\t%s\n", i, table->promisc_mode[i] ? "on" : "off"); + + seq_puts(s, "\n\tMEM Table Info\n"); + seq_printf(s, "Ways : %d\n", table->mem_table.ways); + seq_printf(s, "Depth : %d\n", table->mem_table.depth); + seq_printf(s, "Mask : 0x%llx\n", table->mem_table.mask); + seq_printf(s, "Hash Mask : 0x%x\n", table->mem_table.hash_mask); + seq_printf(s, "Hash Offset : 0x%x\n", table->mem_table.hash_offset); + + seq_puts(s, "\n\tCAM Table Info\n"); + seq_printf(s, "Depth : %d\n", table->cam_table.depth); + + return 0; +} + +RVU_DEBUG_SEQ_FOPS(npc_exact_info, npc_exact_show_info, NULL); + +static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused) +{ + struct npc_exact_table *table; + struct rvu *rvu = s->private; + struct npc_key_field *field; + u16 chan, pcifunc; + int blkaddr, i; + u64 cfg, cam1; + char *str; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + table = rvu->hw->table; + + field = &rvu->hw->mcam.rx_key_fields[NPC_CHAN]; + + seq_puts(s, "\n\t Exact Hit on drop status\n"); + seq_puts(s, "\npcifunc\tmcam_idx\tHits\tchan\tstatus\n"); + + for (i = 0; i < table->num_drop_rules; i++) { + pcifunc = rvu_npc_exact_drop_rule_to_pcifunc(rvu, i); + cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(i, 0)); + + /* channel will be always in keyword 0 */ + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_MCAMEX_BANKX_CAMX_W0(i, 0, 1)); + chan = field->kw_mask[0] & cam1; + + str = (cfg & 1) ? "enabled" : "disabled"; + + seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, i, + rvu_read64(rvu, blkaddr, + NPC_AF_MATCH_STATX(table->counter_idx[i])), + chan, str); + } + + return 0; +} + +RVU_DEBUG_SEQ_FOPS(npc_exact_drop_cnt, npc_exact_drop_cnt, NULL); + static void rvu_dbg_npc_init(struct rvu *rvu) { rvu->rvu_dbg.npc = debugfs_create_dir("npc", rvu->rvu_dbg.root); @@ -2606,8 +3118,22 @@ static void rvu_dbg_npc_init(struct rvu *rvu) &rvu_dbg_npc_mcam_info_fops); debugfs_create_file("mcam_rules", 0444, rvu->rvu_dbg.npc, rvu, &rvu_dbg_npc_mcam_rules_fops); + debugfs_create_file("rx_miss_act_stats", 0444, rvu->rvu_dbg.npc, rvu, &rvu_dbg_npc_rx_miss_act_fops); + + if (!rvu->hw->cap.npc_exact_match_enabled) + return; + + debugfs_create_file("exact_entries", 0444, rvu->rvu_dbg.npc, rvu, + &rvu_dbg_npc_exact_entries_fops); + + debugfs_create_file("exact_info", 0444, rvu->rvu_dbg.npc, rvu, + &rvu_dbg_npc_exact_info_fops); + + debugfs_create_file("exact_drop_cnt", 0444, rvu->rvu_dbg.npc, rvu, + &rvu_dbg_npc_exact_drop_cnt_fops); + } static int cpt_eng_sts_display(struct seq_file *filp, u8 eng_type) @@ -2872,6 +3398,7 @@ create: rvu_dbg_npc_init(rvu); rvu_dbg_cpt_init(rvu, BLKADDR_CPT0); rvu_dbg_cpt_init(rvu, BLKADDR_CPT1); + rvu_dbg_mcs_init(rvu); } void rvu_dbg_exit(struct rvu *rvu) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index 70bacd38a6d9..88dee589cb21 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -10,6 +10,7 @@ #include "rvu.h" #include "rvu_reg.h" #include "rvu_struct.h" +#include "rvu_npc_hash.h" #define DRV_NAME "octeontx2-af" @@ -41,7 +42,7 @@ static bool rvu_common_request_irq(struct rvu *rvu, int offset, struct rvu_devlink *rvu_dl = rvu->rvu_dl; int rc; - sprintf(&rvu->irq_name[offset * NAME_SIZE], name); + sprintf(&rvu->irq_name[offset * NAME_SIZE], "%s", name); rc = request_irq(pci_irq_vector(rvu->pdev, offset), fn, 0, &rvu->irq_name[offset * NAME_SIZE], rvu_dl); if (rc) @@ -1436,14 +1437,75 @@ static int rvu_af_dl_dwrr_mtu_get(struct devlink *devlink, u32 id, enum rvu_af_dl_param_id { RVU_AF_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, RVU_AF_DEVLINK_PARAM_ID_DWRR_MTU, + RVU_AF_DEVLINK_PARAM_ID_NPC_EXACT_FEATURE_DISABLE, }; +static int rvu_af_npc_exact_feature_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + bool enabled; + + enabled = rvu_npc_exact_has_match_table(rvu); + + snprintf(ctx->val.vstr, sizeof(ctx->val.vstr), "%s", + enabled ? "enabled" : "disabled"); + + return 0; +} + +static int rvu_af_npc_exact_feature_disable(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + + rvu_npc_exact_disable_feature(rvu); + + return 0; +} + +static int rvu_af_npc_exact_feature_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + u64 enable; + + if (kstrtoull(val.vstr, 10, &enable)) { + NL_SET_ERR_MSG_MOD(extack, + "Only 1 value is supported"); + return -EINVAL; + } + + if (enable != 1) { + NL_SET_ERR_MSG_MOD(extack, + "Only disabling exact match feature is supported"); + return -EINVAL; + } + + if (rvu_npc_exact_can_disable_feature(rvu)) + return 0; + + NL_SET_ERR_MSG_MOD(extack, + "Can't disable exact match feature; Please try before any configuration"); + return -EFAULT; +} + static const struct devlink_param rvu_af_dl_params[] = { DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_DWRR_MTU, "dwrr_mtu", DEVLINK_PARAM_TYPE_U32, BIT(DEVLINK_PARAM_CMODE_RUNTIME), rvu_af_dl_dwrr_mtu_get, rvu_af_dl_dwrr_mtu_set, rvu_af_dl_dwrr_mtu_validate), + DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_NPC_EXACT_FEATURE_DISABLE, + "npc_exact_feature_disable", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + rvu_af_npc_exact_feature_get, + rvu_af_npc_exact_feature_disable, + rvu_af_npc_exact_feature_validate), }; /* Devlink switch mode */ @@ -1501,6 +1563,7 @@ int rvu_register_dl(struct rvu *rvu) { struct rvu_devlink *rvu_dl; struct devlink *dl; + size_t size; int err; dl = devlink_alloc(&rvu_devlink_ops, sizeof(struct rvu_devlink), @@ -1522,8 +1585,12 @@ int rvu_register_dl(struct rvu *rvu) goto err_dl_health; } - err = devlink_params_register(dl, rvu_af_dl_params, - ARRAY_SIZE(rvu_af_dl_params)); + /* Register exact match devlink only for CN10K-B */ + size = ARRAY_SIZE(rvu_af_dl_params); + if (!rvu_npc_exact_has_match_table(rvu)) + size -= 1; + + err = devlink_params_register(dl, rvu_af_dl_params, size); if (err) { dev_err(rvu->dev, "devlink params register failed with error %d", err); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index d8b1948aaa0a..7646bb2ec89b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -14,6 +14,7 @@ #include "npc.h" #include "cgx.h" #include "lmac_common.h" +#include "rvu_npc_hash.h" static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc); static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, @@ -296,7 +297,6 @@ static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf, struct rvu_hwinfo *hw = rvu->hw; struct sdp_node_info *sdp_info; int pkind, pf, vf, lbkid, vfid; - struct mac_ops *mac_ops; u8 cgx_id, lmac_id; bool from_vf; int err; @@ -326,13 +326,6 @@ static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf, cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind); rvu_npc_set_pkind(rvu, pkind, pfvf); - mac_ops = get_mac_ops(rvu_cgx_pdata(cgx_id, rvu)); - - /* By default we enable pause frames */ - if ((pcifunc & RVU_PFVF_FUNC_MASK) == 0) - mac_ops->mac_enadis_pause_frm(rvu_cgx_pdata(cgx_id, - rvu), - lmac_id, true, true); break; case NIX_INTF_TYPE_LBK: vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; @@ -512,11 +505,11 @@ static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST); lmac_chan_cnt = cfg & 0xFF; - cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); - sdp_chan_cnt = cfg & 0xFFF; - cgx_bpid_cnt = hw->cgx_links * lmac_chan_cnt; lbk_bpid_cnt = hw->lbk_links * ((cfg >> 16) & 0xFF); + + cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); + sdp_chan_cnt = cfg & 0xFFF; sdp_bpid_cnt = hw->sdp_links * sdp_chan_cnt; pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); @@ -533,7 +526,7 @@ static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, */ switch (type) { case NIX_INTF_TYPE_CGX: - if ((req->chan_base + req->chan_cnt) > 15) + if ((req->chan_base + req->chan_cnt) > 16) return -EINVAL; rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); /* Assign bpid based on cgx, lmac and chan id */ @@ -2068,8 +2061,8 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, /* enable cgx tx if disabled */ if (is_pf_cgxmapped(rvu, pf)) { rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); - restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), - lmac_id, true); + restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), + lmac_id, true); } cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); @@ -2092,7 +2085,7 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, rvu_cgx_enadis_rx_bp(rvu, pf, true); /* restore cgx tx state */ if (restore_tx_en) - cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); + rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); return err; } @@ -3800,9 +3793,15 @@ int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, pfvf->rx_chan_base, pfvf->rx_chan_cnt); + + if (rvu_npc_exact_has_match_table(rvu)) + rvu_npc_exact_promisc_enable(rvu, pcifunc); } else { if (!nix_rx_multicast) rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false); + + if (rvu_npc_exact_has_match_table(rvu)) + rvu_npc_exact_promisc_disable(rvu, pcifunc); } return 0; @@ -3878,7 +3877,7 @@ nix_config_link_credits(struct rvu *rvu, int blkaddr, int link, /* Enable cgx tx if disabled for credits to be back */ if (is_pf_cgxmapped(rvu, pf)) { rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); - restore_tx_en = !cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), + restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, true); } @@ -3891,8 +3890,8 @@ nix_config_link_credits(struct rvu *rvu, int blkaddr, int link, NIX_AF_TL1X_SW_XOFF(schq), BIT_ULL(0)); } - rc = -EBUSY; - poll_tmo = jiffies + usecs_to_jiffies(10000); + rc = NIX_AF_ERR_LINK_CREDITS; + poll_tmo = jiffies + usecs_to_jiffies(200000); /* Wait for credits to return */ do { if (time_after(jiffies, poll_tmo)) @@ -3918,7 +3917,7 @@ exit: /* Restore state of cgx tx */ if (restore_tx_en) - cgx_lmac_tx_enable(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); + rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); mutex_unlock(&rvu->rsrc_lock); return rc; @@ -4011,9 +4010,13 @@ linkcfg: return 0; /* Update transmit credits for CGX links */ - lmac_fifo_len = - rvu_cgx_get_fifolen(rvu) / - cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu)); + lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, lmac); + if (!lmac_fifo_len) { + dev_err(rvu->dev, + "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", + __func__, cgx, lmac); + return 0; + } return nix_config_link_credits(rvu, blkaddr, link, pcifunc, (lmac_fifo_len - req->maxlen) / 16); } @@ -4065,7 +4068,10 @@ static void nix_link_config(struct rvu *rvu, int blkaddr, struct rvu_hwinfo *hw = rvu->hw; int cgx, lmac_cnt, slink, link; u16 lbk_max_frs, lmac_max_frs; + unsigned long lmac_bmap; u64 tx_credits, cfg; + u64 lmac_fifo_len; + int iter; rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs); rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs); @@ -4099,12 +4105,23 @@ static void nix_link_config(struct rvu *rvu, int blkaddr, /* Skip when cgx is not available or lmac cnt is zero */ if (lmac_cnt <= 0) continue; - tx_credits = ((rvu_cgx_get_fifolen(rvu) / lmac_cnt) - - lmac_max_frs) / 16; - /* Enable credits and set credit pkt count to max allowed */ - cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); slink = cgx * hw->lmac_per_cgx; - for (link = slink; link < (slink + lmac_cnt); link++) { + + /* Get LMAC id's from bitmap */ + lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); + for_each_set_bit(iter, &lmac_bmap, MAX_LMAC_PER_CGX) { + lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter); + if (!lmac_fifo_len) { + dev_err(rvu->dev, + "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", + __func__, cgx, iter); + continue; + } + tx_credits = (lmac_fifo_len - lmac_max_frs) / 16; + /* Enable credits and set credit pkt count to max allowed */ + cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); + + link = iter + slink; nix_hw->tx_credits[link] = tx_credits; rvu_write64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg); @@ -4279,8 +4296,14 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) /* Restore CINT timer delay to HW reset values */ rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL); + cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG); + /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */ - rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, 0x1ULL); + cfg |= 1ULL; + if (!is_rvu_otx2(rvu)) + cfg |= NIX_PTP_1STEP_EN; + + rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg); if (is_block_implemented(hw, blkaddr)) { err = nix_setup_txschq(rvu, nix_hw, blkaddr); @@ -4578,6 +4601,12 @@ void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf) pfvf->hw_rx_tstamp_en = false; } + /* reset priority flow control config */ + rvu_cgx_prio_flow_ctrl_cfg(rvu, pcifunc, 0, 0, 0); + + /* reset 802.3x flow control config */ + rvu_cgx_cfg_pause_frm(rvu, pcifunc, 0, 0); + nix_ctx_free(rvu, pfvf); nix_free_all_bandprof(rvu, pcifunc); @@ -5314,6 +5343,7 @@ int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc, aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; aq_req.op = NIX_AQ_INSTOP_WRITE; memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s)); + memset((char *)&aq_req.prof_mask, 0xff, sizeof(struct nix_bandprof_s)); /* Clear higher layer enable bit in the mid profile, just in case */ aq_req.prof.hl_en = 0; aq_req.prof_mask.hl_en = 1; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index c0005a1feee6..1e348fd0d930 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -15,6 +15,7 @@ #include "npc.h" #include "cgx.h" #include "npc_profile.h" +#include "rvu_npc_hash.h" #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ @@ -402,6 +403,7 @@ static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index, struct mcam_entry *entry, bool *enable) { + struct rvu_npc_mcam_rule *rule; u16 owner, target_func; struct rvu_pfvf *pfvf; u64 rx_action; @@ -423,6 +425,12 @@ static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, test_bit(NIXLF_INITIALIZED, &pfvf->flags))) *enable = false; + /* fix up not needed for the rules added by user(ntuple filters) */ + list_for_each_entry(rule, &mcam->mcam_rules, list) { + if (rule->entry == index) + return; + } + /* copy VF default entry action to the VF mcam entry */ rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, target_func); @@ -489,8 +497,8 @@ static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, } /* PF installing VF rule */ - if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries) - npc_fixup_vf_rule(rvu, mcam, blkaddr, index, entry, &enable); + if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) + npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); /* Set 'action' */ rvu_write64(rvu, blkaddr, @@ -598,7 +606,7 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, struct npc_install_flow_req req = { 0 }; struct npc_install_flow_rsp rsp = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; - struct nix_rx_action action; + struct nix_rx_action action = { 0 }; int blkaddr, index; /* AF's and SDP VFs work in promiscuous mode */ @@ -619,7 +627,6 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, *(u64 *)&action = npc_get_mcam_action(rvu, mcam, blkaddr, index); } else { - *(u64 *)&action = 0x00; action.op = NIX_RX_ACTIONOP_UCAST; action.pf_func = pcifunc; } @@ -650,7 +657,7 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; int blkaddr, ucast_idx, index; - struct nix_rx_action action; + struct nix_rx_action action = { 0 }; u64 relaxed_mask; if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) @@ -678,14 +685,14 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, blkaddr, ucast_idx); if (action.op != NIX_RX_ACTIONOP_RSS) { - *(u64 *)&action = 0x00; + *(u64 *)&action = 0; action.op = NIX_RX_ACTIONOP_UCAST; } /* RX_ACTION set to MCAST for CGX PF's */ if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { - *(u64 *)&action = 0x00; + *(u64 *)&action = 0; action.op = NIX_RX_ACTIONOP_MCAST; pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); action.index = pfvf->promisc_mce_idx; @@ -825,7 +832,7 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, struct rvu_hwinfo *hw = rvu->hw; int blkaddr, ucast_idx, index; u8 mac_addr[ETH_ALEN] = { 0 }; - struct nix_rx_action action; + struct nix_rx_action action = { 0 }; struct rvu_pfvf *pfvf; u16 vf_func; @@ -854,14 +861,14 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, blkaddr, ucast_idx); if (action.op != NIX_RX_ACTIONOP_RSS) { - *(u64 *)&action = 0x00; + *(u64 *)&action = 0; action.op = NIX_RX_ACTIONOP_UCAST; action.pf_func = pcifunc; } /* RX_ACTION set to MCAST for CGX PF's */ if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { - *(u64 *)&action = 0x00; + *(u64 *)&action = 0; action.op = NIX_RX_ACTIONOP_MCAST; action.index = pfvf->mcast_mce_idx; } @@ -916,7 +923,8 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, u16 pcifunc, u64 rx_action) { int actindex, index, bank, entry; - bool enable; + struct rvu_npc_mcam_rule *rule; + bool enable, update; if (!(pcifunc & RVU_PFVF_FUNC_MASK)) return; @@ -924,6 +932,14 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, mutex_lock(&mcam->lock); for (index = 0; index < mcam->bmap_entries; index++) { if (mcam->entry2target_pffunc[index] == pcifunc) { + update = true; + /* update not needed for the rules added via ntuple filters */ + list_for_each_entry(rule, &mcam->mcam_rules, list) { + if (rule->entry == index) + update = false; + } + if (!update) + continue; bank = npc_get_bank(mcam, index); actindex = index; entry = index & (mcam->banksize - 1); @@ -1081,6 +1097,9 @@ static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) { + if (nixlf < 0) + return; + npc_enadis_default_entries(rvu, pcifunc, nixlf, false); /* Delete multicast and promisc MCAM entries */ @@ -1090,8 +1109,39 @@ void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) NIXLF_PROMISC_ENTRY, false); } +bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable) +{ + int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_npc_mcam_rule *rule, *tmp; + + mutex_lock(&mcam->lock); + + list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { + if (rule->intf != intf) + continue; + + if (rule->entry != entry) + continue; + + rule->enable = enable; + mutex_unlock(&mcam->lock); + + npc_enable_mcam_entry(rvu, mcam, blkaddr, + entry, enable); + + return true; + } + + mutex_unlock(&mcam->lock); + return false; +} + void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) { + if (nixlf < 0) + return; + /* Enables only broadcast match entry. Promisc/Allmulti are enabled * in set_rx_mode mbox handler. */ @@ -1166,14 +1216,6 @@ void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); } -#define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ - rvu_write64(rvu, blkaddr, \ - NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) - -#define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ - rvu_write64(rvu, blkaddr, \ - NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) - static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, struct npc_mcam_kex *mkex, u8 intf) { @@ -1247,6 +1289,9 @@ static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, npc_program_mkex_rx(rvu, blkaddr, mkex, intf); npc_program_mkex_tx(rvu, blkaddr, mkex, intf); } + + /* Programme mkex hash profile */ + npc_program_mkex_hash(rvu, blkaddr); } static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, @@ -1448,6 +1493,7 @@ static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) profile->kpus = ARRAY_SIZE(npc_kpu_profiles); profile->lt_def = &npc_lt_defaults; profile->mkex = &npc_mkex_default; + profile->mkex_hash = &npc_mkex_hash_default; return 0; } @@ -1635,7 +1681,7 @@ static void npc_load_kpu_profile(struct rvu *rvu) * Firmware database method. * Default KPU profile. */ - if (!request_firmware(&fw, kpu_profile, rvu->dev)) { + if (!request_firmware_direct(&fw, kpu_profile, rvu->dev)) { dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", kpu_profile); rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); @@ -1804,7 +1850,6 @@ static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) mcam->hprio_count = mcam->lprio_count; mcam->hprio_end = mcam->hprio_count; - /* Allocate bitmap for managing MCAM counters and memory * for saving counter to RVU PFFUNC allocation mapping. */ @@ -1900,6 +1945,7 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) { + struct npc_mcam_kex *mkex = rvu->kpu.mkex; struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; u64 nibble_ena, rx_kex, tx_kex; @@ -1912,15 +1958,15 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) mcam->counters.max--; mcam->rx_miss_act_cntr = mcam->counters.max; - rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; - tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; + rx_kex = mkex->keyx_cfg[NIX_INTF_RX]; + tx_kex = mkex->keyx_cfg[NIX_INTF_TX]; nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); if (nibble_ena) { tx_kex &= ~NPC_PARSE_NIBBLE; tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); - npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; + mkex->keyx_cfg[NIX_INTF_TX] = tx_kex; } /* Configure RX interfaces */ @@ -2032,6 +2078,7 @@ int rvu_npc_init(struct rvu *rvu) rvu_npc_setup_interfaces(rvu, blkaddr); + npc_config_secret_key(rvu, blkaddr); /* Configure MKEX profile */ npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); @@ -2519,7 +2566,7 @@ alloc: /* Copy MCAM entry indices into mbox response entry_list. * Requester always expects indices in ascending order, so - * so reverse the list if reverse bitmap is used for allocation. + * reverse the list if reverse bitmap is used for allocation. */ if (!req->contig && rsp->count) { index = 0; @@ -2547,6 +2594,14 @@ alloc: return 0; } +/* Marks bitmaps to reserved the mcam slot */ +void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + + npc_mcam_set_bit(mcam, entry_idx); +} + int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, struct npc_mcam_alloc_entry_req *req, struct npc_mcam_alloc_entry_rsp *rsp) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index ff2b21999f36..7c4e1acd0f77 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -10,6 +10,8 @@ #include "rvu_reg.h" #include "rvu.h" #include "npc.h" +#include "rvu_npc_fs.h" +#include "rvu_npc_hash.h" #define NPC_BYTESM GENMASK_ULL(19, 16) #define NPC_HDR_OFFSET GENMASK_ULL(15, 8) @@ -227,6 +229,25 @@ static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, return true; } +static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, + u8 key_nibble, u8 intf) +{ + u8 offset = (key_nibble * 4) % 64; /* offset within key word */ + u8 kwi = (key_nibble * 4) / 64; /* which word in key */ + u8 nr_bits = 4; /* bits in a nibble */ + u8 type; + + switch (bit_number) { + case 40 ... 43: + type = NPC_EXACT_RESULT; + break; + + default: + return; + } + npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); +} + static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, u8 key_nibble, u8 intf) { @@ -276,6 +297,7 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, default: return; } + npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); } @@ -445,7 +467,8 @@ do { \ NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2); NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2); NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6); - NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6); + /* SMAC follows the DMAC(which is 6 bytes) */ + NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start + 6, 6); /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */ NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2); } @@ -509,8 +532,8 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) { struct npc_mcam *mcam = &rvu->hw->mcam; u8 lid, lt, ld, bitnr; + u64 cfg, masked_cfg; u8 key_nibble = 0; - u64 cfg; /* Scan and note how parse result is going to be in key. * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from @@ -518,12 +541,24 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) * will be concatenated in key. */ cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); - cfg &= NPC_PARSE_NIBBLE; - for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) { + masked_cfg = cfg & NPC_PARSE_NIBBLE; + for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, 31) { npc_scan_parse_result(mcam, bitnr, key_nibble, intf); key_nibble++; } + /* Ignore exact match bits for mcam entries except the first rule + * which is drop on hit. This first rule is configured explitcitly by + * exact match code. + */ + masked_cfg = cfg & NPC_EXACT_NIBBLE; + bitnr = NPC_EXACT_NIBBLE_START; + for_each_set_bit_from(bitnr, (unsigned long *)&masked_cfg, + NPC_EXACT_NIBBLE_START) { + npc_scan_exact_result(mcam, bitnr, key_nibble, intf); + key_nibble++; + } + /* Scan and note how layer data is going to be in key */ for (lid = 0; lid < NPC_MAX_LID; lid++) { for (lt = 0; lt < NPC_MAX_LT; lt++) { @@ -624,9 +659,9 @@ static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) * If any bits in mask are 0 then corresponding bits in value are * dont care. */ -static void npc_update_entry(struct rvu *rvu, enum key_fields type, - struct mcam_entry *entry, u64 val_lo, - u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) +void npc_update_entry(struct rvu *rvu, enum key_fields type, + struct mcam_entry *entry, u64 val_lo, + u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) { struct npc_mcam *mcam = &rvu->hw->mcam; struct mcam_entry dummy = { {0} }; @@ -705,8 +740,6 @@ static void npc_update_entry(struct rvu *rvu, enum key_fields type, } } -#define IPV6_WORDS 4 - static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, u64 features, struct flow_msg *pkt, struct flow_msg *mask, @@ -779,7 +812,8 @@ static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, u64 features, struct flow_msg *pkt, struct flow_msg *mask, - struct rvu_npc_mcam_rule *output, u8 intf) + struct rvu_npc_mcam_rule *output, u8 intf, + int blkaddr) { u64 dmac_mask = ether_addr_to_u64(mask->dmac); u64 smac_mask = ether_addr_to_u64(mask->smac); @@ -828,6 +862,7 @@ do { \ } while (0) NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0); + NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0); NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0, ntohs(mask->etype), 0); @@ -854,10 +889,12 @@ do { \ npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); npc_update_vlan_features(rvu, entry, features, intf); + + npc_update_field_hash(rvu, intf, entry, blkaddr, features, + pkt, mask, opkt, omask); } -static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam, - u16 entry) +static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam, u16 entry) { struct rvu_npc_mcam_rule *iter; @@ -1023,8 +1060,9 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, u16 owner = req->hdr.pcifunc; struct msg_rsp write_rsp; struct mcam_entry *entry; - int entry_index, err; bool new = false; + u16 entry_index; + int err; installed_features = req->features; features = req->features; @@ -1032,7 +1070,7 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, entry_index = req->entry; npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, - req->intf); + req->intf, blkaddr); if (is_npc_intf_rx(req->intf)) npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); @@ -1057,7 +1095,8 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, npc_update_flow(rvu, entry, missing_features, &def_ucast_rule->packet, &def_ucast_rule->mask, - &dummy, req->intf); + &dummy, req->intf, + blkaddr); installed_features = req->features | missing_features; } @@ -1098,14 +1137,6 @@ find_rule: write_req.cntr = rule->cntr; } - err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, - &write_rsp); - if (err) { - rvu_mcam_remove_counter_from_rule(rvu, owner, rule); - if (new) - kfree(rule); - return err; - } /* update rule */ memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); @@ -1132,6 +1163,18 @@ find_rule: if (req->default_rule) pfvf->def_ucast_rule = rule; + /* write to mcam entry registers */ + err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, + &write_rsp); + if (err) { + rvu_mcam_remove_counter_from_rule(rvu, owner, rule); + if (new) { + list_del(&rule->list); + kfree(rule); + } + return err; + } + /* VF's MAC address is being changed via PF */ if (pf_set_vfs_mac) { ether_addr_copy(pfvf->default_mac, req->packet.dmac); @@ -1420,3 +1463,98 @@ void npc_mcam_disable_flows(struct rvu *rvu, u16 target) } mutex_unlock(&mcam->lock); } + +/* single drop on non hit rule starting from 0th index. This an extension + * to RPM mac filter to support more rules. + */ +int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, + u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask, + u64 bcast_mcast_val, u64 bcast_mcast_mask) +{ + struct npc_mcam_alloc_counter_req cntr_req = { 0 }; + struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; + struct npc_mcam_write_entry_req req = { 0 }; + struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_npc_mcam_rule *rule; + struct msg_rsp rsp; + bool enabled; + int blkaddr; + int err; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return -ENODEV; + } + + /* Bail out if no exact match support */ + if (!rvu_npc_exact_has_match_table(rvu)) { + dev_info(rvu->dev, "%s: No support for exact match feature\n", __func__); + return -EINVAL; + } + + /* If 0th entry is already used, return err */ + enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_idx); + if (enabled) { + dev_err(rvu->dev, "%s: failed to add single drop on non hit rule at %d th index\n", + __func__, mcam_idx); + return -EINVAL; + } + + /* Add this entry to mcam rules list */ + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return -ENOMEM; + + /* Disable rule by default. Enable rule when first dmac filter is + * installed + */ + rule->enable = false; + rule->chan = chan_val; + rule->chan_mask = chan_mask; + rule->entry = mcam_idx; + rvu_mcam_add_rule(mcam, rule); + + /* Reserve slot 0 */ + npc_mcam_rsrcs_reserve(rvu, blkaddr, mcam_idx); + + /* Allocate counter for this single drop on non hit rule */ + cntr_req.hdr.pcifunc = 0; /* AF request */ + cntr_req.contig = true; + cntr_req.count = 1; + err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); + if (err) { + dev_err(rvu->dev, "%s: Err to allocate cntr for drop rule (err=%d)\n", + __func__, err); + return -EFAULT; + } + *counter_idx = cntr_rsp.cntr; + + /* Fill in fields for this mcam entry */ + npc_update_entry(rvu, NPC_EXACT_RESULT, &req.entry_data, exact_val, 0, + exact_mask, 0, NIX_INTF_RX); + npc_update_entry(rvu, NPC_CHAN, &req.entry_data, chan_val, 0, + chan_mask, 0, NIX_INTF_RX); + npc_update_entry(rvu, NPC_LXMB, &req.entry_data, bcast_mcast_val, 0, + bcast_mcast_mask, 0, NIX_INTF_RX); + + req.intf = NIX_INTF_RX; + req.set_cntr = true; + req.cntr = cntr_rsp.cntr; + req.entry = mcam_idx; + + err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &req, &rsp); + if (err) { + dev_err(rvu->dev, "%s: Installation of single drop on non hit rule at %d failed\n", + __func__, mcam_idx); + return err; + } + + dev_err(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n", + __func__, mcam_idx, req.cntr); + + /* disable entry at Bank 0, index 0 */ + npc_enable_mcam_entry(rvu, mcam, blkaddr, mcam_idx, false); + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h new file mode 100644 index 000000000000..bdd65ce56a32 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2022 Marvell. + * + */ + +#ifndef __RVU_NPC_FS_H +#define __RVU_NPC_FS_H + +#define IPV6_WORDS 4 + +void npc_update_entry(struct rvu *rvu, enum key_fields type, + struct mcam_entry *entry, u64 val_lo, + u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf); + +#endif /* RVU_NPC_FS_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c new file mode 100644 index 000000000000..594029007f85 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c @@ -0,0 +1,2009 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2022 Marvell. + * + */ + +#include <linux/bitfield.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/firmware.h> +#include <linux/stddef.h> +#include <linux/debugfs.h> + +#include "rvu_struct.h" +#include "rvu_reg.h" +#include "rvu.h" +#include "npc.h" +#include "cgx.h" +#include "rvu_npc_fs.h" +#include "rvu_npc_hash.h" + +static u64 rvu_npc_wide_extract(const u64 input[], size_t start_bit, + size_t width_bits) +{ + const u64 mask = ~(u64)((~(__uint128_t)0) << width_bits); + const size_t msb = start_bit + width_bits - 1; + const size_t lword = start_bit >> 6; + const size_t uword = msb >> 6; + size_t lbits; + u64 hi, lo; + + if (lword == uword) + return (input[lword] >> (start_bit & 63)) & mask; + + lbits = 64 - (start_bit & 63); + hi = input[uword]; + lo = (input[lword] >> (start_bit & 63)); + return ((hi << lbits) | lo) & mask; +} + +static void rvu_npc_lshift_key(u64 *key, size_t key_bit_len) +{ + u64 prev_orig_word = 0; + u64 cur_orig_word = 0; + size_t extra = key_bit_len % 64; + size_t max_idx = key_bit_len / 64; + size_t i; + + if (extra) + max_idx++; + + for (i = 0; i < max_idx; i++) { + cur_orig_word = key[i]; + key[i] = key[i] << 1; + key[i] |= ((prev_orig_word >> 63) & 0x1); + prev_orig_word = cur_orig_word; + } +} + +static u32 rvu_npc_toeplitz_hash(const u64 *data, u64 *key, size_t data_bit_len, + size_t key_bit_len) +{ + u32 hash_out = 0; + u64 temp_data = 0; + int i; + + for (i = data_bit_len - 1; i >= 0; i--) { + temp_data = (data[i / 64]); + temp_data = temp_data >> (i % 64); + temp_data &= 0x1; + if (temp_data) + hash_out ^= (u32)(rvu_npc_wide_extract(key, key_bit_len - 32, 32)); + + rvu_npc_lshift_key(key, key_bit_len); + } + + return hash_out; +} + +u32 npc_field_hash_calc(u64 *ldata, struct npc_mcam_kex_hash *mkex_hash, + u64 *secret_key, u8 intf, u8 hash_idx) +{ + u64 hash_key[3]; + u64 data_padded[2]; + u32 field_hash; + + hash_key[0] = secret_key[1] << 31; + hash_key[0] |= secret_key[2]; + hash_key[1] = secret_key[1] >> 33; + hash_key[1] |= secret_key[0] << 31; + hash_key[2] = secret_key[0] >> 33; + + data_padded[0] = mkex_hash->hash_mask[intf][hash_idx][0] & ldata[0]; + data_padded[1] = mkex_hash->hash_mask[intf][hash_idx][1] & ldata[1]; + field_hash = rvu_npc_toeplitz_hash(data_padded, hash_key, 128, 159); + + field_hash &= mkex_hash->hash_ctrl[intf][hash_idx] >> 32; + field_hash |= mkex_hash->hash_ctrl[intf][hash_idx]; + return field_hash; +} + +static u64 npc_update_use_hash(int lt, int ld) +{ + u64 cfg = 0; + + switch (lt) { + case NPC_LT_LC_IP6: + /* Update use_hash(bit-20) and bytesm1 (bit-16:19) + * in KEX_LD_CFG + */ + cfg = KEX_LD_CFG_USE_HASH(0x1, 0x03, + ld ? 0x8 : 0x18, + 0x1, 0x0, 0x10); + break; + } + + return cfg; +} + +static void npc_program_mkex_hash_rx(struct rvu *rvu, int blkaddr, + u8 intf) +{ + struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; + int lid, lt, ld, hash_cnt = 0; + + if (is_npc_intf_tx(intf)) + return; + + /* Program HASH_CFG */ + for (lid = 0; lid < NPC_MAX_LID; lid++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + for (ld = 0; ld < NPC_MAX_LD; ld++) { + if (mkex_hash->lid_lt_ld_hash_en[intf][lid][lt][ld]) { + u64 cfg = npc_update_use_hash(lt, ld); + + hash_cnt++; + if (hash_cnt == NPC_MAX_HASH) + return; + + /* Set updated KEX configuration */ + SET_KEX_LD(intf, lid, lt, ld, cfg); + /* Set HASH configuration */ + SET_KEX_LD_HASH(intf, ld, + mkex_hash->hash[intf][ld]); + SET_KEX_LD_HASH_MASK(intf, ld, 0, + mkex_hash->hash_mask[intf][ld][0]); + SET_KEX_LD_HASH_MASK(intf, ld, 1, + mkex_hash->hash_mask[intf][ld][1]); + SET_KEX_LD_HASH_CTRL(intf, ld, + mkex_hash->hash_ctrl[intf][ld]); + } + } + } + } +} + +static void npc_program_mkex_hash_tx(struct rvu *rvu, int blkaddr, + u8 intf) +{ + struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; + int lid, lt, ld, hash_cnt = 0; + + if (is_npc_intf_rx(intf)) + return; + + /* Program HASH_CFG */ + for (lid = 0; lid < NPC_MAX_LID; lid++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + for (ld = 0; ld < NPC_MAX_LD; ld++) + if (mkex_hash->lid_lt_ld_hash_en[intf][lid][lt][ld]) { + u64 cfg = npc_update_use_hash(lt, ld); + + hash_cnt++; + if (hash_cnt == NPC_MAX_HASH) + return; + + /* Set updated KEX configuration */ + SET_KEX_LD(intf, lid, lt, ld, cfg); + /* Set HASH configuration */ + SET_KEX_LD_HASH(intf, ld, + mkex_hash->hash[intf][ld]); + SET_KEX_LD_HASH_MASK(intf, ld, 0, + mkex_hash->hash_mask[intf][ld][0]); + SET_KEX_LD_HASH_MASK(intf, ld, 1, + mkex_hash->hash_mask[intf][ld][1]); + SET_KEX_LD_HASH_CTRL(intf, ld, + mkex_hash->hash_ctrl[intf][ld]); + hash_cnt++; + if (hash_cnt == NPC_MAX_HASH) + return; + } + } + } +} + +void npc_config_secret_key(struct rvu *rvu, int blkaddr) +{ + struct hw_cap *hwcap = &rvu->hw->cap; + struct rvu_hwinfo *hw = rvu->hw; + u8 intf; + + if (!hwcap->npc_hash_extract) { + dev_info(rvu->dev, "HW does not support secret key configuration\n"); + return; + } + + for (intf = 0; intf < hw->npc_intfs; intf++) { + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_SECRET_KEY0(intf), + RVU_NPC_HASH_SECRET_KEY0); + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_SECRET_KEY1(intf), + RVU_NPC_HASH_SECRET_KEY1); + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_SECRET_KEY2(intf), + RVU_NPC_HASH_SECRET_KEY2); + } +} + +void npc_program_mkex_hash(struct rvu *rvu, int blkaddr) +{ + struct hw_cap *hwcap = &rvu->hw->cap; + struct rvu_hwinfo *hw = rvu->hw; + u8 intf; + + if (!hwcap->npc_hash_extract) { + dev_dbg(rvu->dev, "Field hash extract feature is not supported\n"); + return; + } + + for (intf = 0; intf < hw->npc_intfs; intf++) { + npc_program_mkex_hash_rx(rvu, blkaddr, intf); + npc_program_mkex_hash_tx(rvu, blkaddr, intf); + } +} + +void npc_update_field_hash(struct rvu *rvu, u8 intf, + struct mcam_entry *entry, + int blkaddr, + u64 features, + struct flow_msg *pkt, + struct flow_msg *mask, + struct flow_msg *opkt, + struct flow_msg *omask) +{ + struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; + struct npc_get_secret_key_req req; + struct npc_get_secret_key_rsp rsp; + u64 ldata[2], cfg; + u32 field_hash; + u8 hash_idx; + + if (!rvu->hw->cap.npc_hash_extract) { + dev_dbg(rvu->dev, "%s: Field hash extract feature is not supported\n", __func__); + return; + } + + req.intf = intf; + rvu_mbox_handler_npc_get_secret_key(rvu, &req, &rsp); + + for (hash_idx = 0; hash_idx < NPC_MAX_HASH; hash_idx++) { + cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_HASHX_CFG(intf, hash_idx)); + if ((cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12))) { + u8 lid = (cfg & GENMASK_ULL(10, 8)) >> 8; + u8 ltype = (cfg & GENMASK_ULL(7, 4)) >> 4; + u8 ltype_mask = cfg & GENMASK_ULL(3, 0); + + if (mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx]) { + switch (ltype & ltype_mask) { + /* If hash extract enabled is supported for IPv6 then + * 128 bit IPv6 source and destination addressed + * is hashed to 32 bit value. + */ + case NPC_LT_LC_IP6: + if (features & BIT_ULL(NPC_SIP_IPV6)) { + u32 src_ip[IPV6_WORDS]; + + be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); + ldata[0] = (u64)src_ip[0] << 32 | src_ip[1]; + ldata[1] = (u64)src_ip[2] << 32 | src_ip[3]; + field_hash = npc_field_hash_calc(ldata, + mkex_hash, + rsp.secret_key, + intf, + hash_idx); + npc_update_entry(rvu, NPC_SIP_IPV6, entry, + field_hash, 0, 32, 0, intf); + memcpy(&opkt->ip6src, &pkt->ip6src, + sizeof(pkt->ip6src)); + memcpy(&omask->ip6src, &mask->ip6src, + sizeof(mask->ip6src)); + break; + } + + if (features & BIT_ULL(NPC_DIP_IPV6)) { + u32 dst_ip[IPV6_WORDS]; + + be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); + ldata[0] = (u64)dst_ip[0] << 32 | dst_ip[1]; + ldata[1] = (u64)dst_ip[2] << 32 | dst_ip[3]; + field_hash = npc_field_hash_calc(ldata, + mkex_hash, + rsp.secret_key, + intf, + hash_idx); + npc_update_entry(rvu, NPC_DIP_IPV6, entry, + field_hash, 0, 32, 0, intf); + memcpy(&opkt->ip6dst, &pkt->ip6dst, + sizeof(pkt->ip6dst)); + memcpy(&omask->ip6dst, &mask->ip6dst, + sizeof(mask->ip6dst)); + } + break; + } + } + } + } +} + +int rvu_mbox_handler_npc_get_secret_key(struct rvu *rvu, + struct npc_get_secret_key_req *req, + struct npc_get_secret_key_rsp *rsp) +{ + u64 *secret_key = rsp->secret_key; + u8 intf = req->intf; + int blkaddr; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return -EINVAL; + } + + secret_key[0] = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_SECRET_KEY0(intf)); + secret_key[1] = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_SECRET_KEY1(intf)); + secret_key[2] = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_SECRET_KEY2(intf)); + + return 0; +} + +/** + * rvu_npc_exact_mac2u64 - utility function to convert mac address to u64. + * @mac_addr: MAC address. + * Return: mdata for exact match table. + */ +static u64 rvu_npc_exact_mac2u64(u8 *mac_addr) +{ + u64 mac = 0; + int index; + + for (index = ETH_ALEN - 1; index >= 0; index--) + mac |= ((u64)*mac_addr++) << (8 * index); + + return mac; +} + +/** + * rvu_exact_prepare_mdata - Make mdata for mcam entry + * @mac: MAC address + * @chan: Channel number. + * @ctype: Channel Type. + * @mask: LDATA mask. + * Return: Meta data + */ +static u64 rvu_exact_prepare_mdata(u8 *mac, u16 chan, u16 ctype, u64 mask) +{ + u64 ldata = rvu_npc_exact_mac2u64(mac); + + /* Please note that mask is 48bit which excludes chan and ctype. + * Increase mask bits if we need to include them as well. + */ + ldata |= ((u64)chan << 48); + ldata |= ((u64)ctype << 60); + ldata &= mask; + ldata = ldata << 2; + + return ldata; +} + +/** + * rvu_exact_calculate_hash - calculate hash index to mem table. + * @rvu: resource virtualization unit. + * @chan: Channel number + * @ctype: Channel type. + * @mac: MAC address + * @mask: HASH mask. + * @table_depth: Depth of table. + * Return: Hash value + */ +static u32 rvu_exact_calculate_hash(struct rvu *rvu, u16 chan, u16 ctype, u8 *mac, + u64 mask, u32 table_depth) +{ + struct npc_exact_table *table = rvu->hw->table; + u64 hash_key[2]; + u64 key_in[2]; + u64 ldata; + u32 hash; + + key_in[0] = RVU_NPC_HASH_SECRET_KEY0; + key_in[1] = RVU_NPC_HASH_SECRET_KEY2; + + hash_key[0] = key_in[0] << 31; + hash_key[0] |= key_in[1]; + hash_key[1] = key_in[0] >> 33; + + ldata = rvu_exact_prepare_mdata(mac, chan, ctype, mask); + + dev_dbg(rvu->dev, "%s: ldata=0x%llx hash_key0=0x%llx hash_key2=0x%llx\n", __func__, + ldata, hash_key[1], hash_key[0]); + hash = rvu_npc_toeplitz_hash(&ldata, (u64 *)hash_key, 64, 95); + + hash &= table->mem_table.hash_mask; + hash += table->mem_table.hash_offset; + dev_dbg(rvu->dev, "%s: hash=%x\n", __func__, hash); + + return hash; +} + +/** + * rvu_npc_exact_alloc_mem_table_entry - find free entry in 4 way table. + * @rvu: resource virtualization unit. + * @way: Indicate way to table. + * @index: Hash index to 4 way table. + * @hash: Hash value. + * + * Searches 4 way table using hash index. Returns 0 on success. + * Return: 0 upon success. + */ +static int rvu_npc_exact_alloc_mem_table_entry(struct rvu *rvu, u8 *way, + u32 *index, unsigned int hash) +{ + struct npc_exact_table *table; + int depth, i; + + table = rvu->hw->table; + depth = table->mem_table.depth; + + /* Check all the 4 ways for a free slot. */ + mutex_lock(&table->lock); + for (i = 0; i < table->mem_table.ways; i++) { + if (test_bit(hash + i * depth, table->mem_table.bmap)) + continue; + + set_bit(hash + i * depth, table->mem_table.bmap); + mutex_unlock(&table->lock); + + dev_dbg(rvu->dev, "%s: mem table entry alloc success (way=%d index=%d)\n", + __func__, i, hash); + + *way = i; + *index = hash; + return 0; + } + mutex_unlock(&table->lock); + + dev_dbg(rvu->dev, "%s: No space in 4 way exact way, weight=%u\n", __func__, + bitmap_weight(table->mem_table.bmap, table->mem_table.depth)); + return -ENOSPC; +} + +/** + * rvu_npc_exact_free_id - Free seq id from bitmat. + * @rvu: Resource virtualization unit. + * @seq_id: Sequence identifier to be freed. + */ +static void rvu_npc_exact_free_id(struct rvu *rvu, u32 seq_id) +{ + struct npc_exact_table *table; + + table = rvu->hw->table; + mutex_lock(&table->lock); + clear_bit(seq_id, table->id_bmap); + mutex_unlock(&table->lock); + dev_dbg(rvu->dev, "%s: freed id %d\n", __func__, seq_id); +} + +/** + * rvu_npc_exact_alloc_id - Alloc seq id from bitmap. + * @rvu: Resource virtualization unit. + * @seq_id: Sequence identifier. + * Return: True or false. + */ +static bool rvu_npc_exact_alloc_id(struct rvu *rvu, u32 *seq_id) +{ + struct npc_exact_table *table; + u32 idx; + + table = rvu->hw->table; + + mutex_lock(&table->lock); + idx = find_first_zero_bit(table->id_bmap, table->tot_ids); + if (idx == table->tot_ids) { + mutex_unlock(&table->lock); + dev_err(rvu->dev, "%s: No space in id bitmap (%d)\n", + __func__, bitmap_weight(table->id_bmap, table->tot_ids)); + + return false; + } + + /* Mark bit map to indicate that slot is used.*/ + set_bit(idx, table->id_bmap); + mutex_unlock(&table->lock); + + *seq_id = idx; + dev_dbg(rvu->dev, "%s: Allocated id (%d)\n", __func__, *seq_id); + + return true; +} + +/** + * rvu_npc_exact_alloc_cam_table_entry - find free slot in fully associative table. + * @rvu: resource virtualization unit. + * @index: Index to exact CAM table. + * Return: 0 upon success; else error number. + */ +static int rvu_npc_exact_alloc_cam_table_entry(struct rvu *rvu, int *index) +{ + struct npc_exact_table *table; + u32 idx; + + table = rvu->hw->table; + + mutex_lock(&table->lock); + idx = find_first_zero_bit(table->cam_table.bmap, table->cam_table.depth); + if (idx == table->cam_table.depth) { + mutex_unlock(&table->lock); + dev_info(rvu->dev, "%s: No space in exact cam table, weight=%u\n", __func__, + bitmap_weight(table->cam_table.bmap, table->cam_table.depth)); + return -ENOSPC; + } + + /* Mark bit map to indicate that slot is used.*/ + set_bit(idx, table->cam_table.bmap); + mutex_unlock(&table->lock); + + *index = idx; + dev_dbg(rvu->dev, "%s: cam table entry alloc success (index=%d)\n", + __func__, idx); + return 0; +} + +/** + * rvu_exact_prepare_table_entry - Data for exact match table entry. + * @rvu: Resource virtualization unit. + * @enable: Enable/Disable entry + * @ctype: Software defined channel type. Currently set as 0. + * @chan: Channel number. + * @mac_addr: Destination mac address. + * Return: mdata for exact match table. + */ +static u64 rvu_exact_prepare_table_entry(struct rvu *rvu, bool enable, + u8 ctype, u16 chan, u8 *mac_addr) + +{ + u64 ldata = rvu_npc_exact_mac2u64(mac_addr); + + /* Enable or disable */ + u64 mdata = FIELD_PREP(GENMASK_ULL(63, 63), enable ? 1 : 0); + + /* Set Ctype */ + mdata |= FIELD_PREP(GENMASK_ULL(61, 60), ctype); + + /* Set chan */ + mdata |= FIELD_PREP(GENMASK_ULL(59, 48), chan); + + /* MAC address */ + mdata |= FIELD_PREP(GENMASK_ULL(47, 0), ldata); + + return mdata; +} + +/** + * rvu_exact_config_secret_key - Configure secret key. + * @rvu: Resource virtualization unit. + */ +static void rvu_exact_config_secret_key(struct rvu *rvu) +{ + int blkaddr; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_EXACT_SECRET0(NIX_INTF_RX), + RVU_NPC_HASH_SECRET_KEY0); + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_EXACT_SECRET1(NIX_INTF_RX), + RVU_NPC_HASH_SECRET_KEY1); + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_EXACT_SECRET2(NIX_INTF_RX), + RVU_NPC_HASH_SECRET_KEY2); +} + +/** + * rvu_exact_config_search_key - Configure search key + * @rvu: Resource virtualization unit. + */ +static void rvu_exact_config_search_key(struct rvu *rvu) +{ + int blkaddr; + u64 reg_val; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + + /* HDR offset */ + reg_val = FIELD_PREP(GENMASK_ULL(39, 32), 0); + + /* BYTESM1, number of bytes - 1 */ + reg_val |= FIELD_PREP(GENMASK_ULL(18, 16), ETH_ALEN - 1); + + /* Enable LID and set LID to NPC_LID_LA */ + reg_val |= FIELD_PREP(GENMASK_ULL(11, 11), 1); + reg_val |= FIELD_PREP(GENMASK_ULL(10, 8), NPC_LID_LA); + + /* Clear layer type based extraction */ + + /* Disable LT_EN */ + reg_val |= FIELD_PREP(GENMASK_ULL(12, 12), 0); + + /* Set LTYPE_MATCH to 0 */ + reg_val |= FIELD_PREP(GENMASK_ULL(7, 4), 0); + + /* Set LTYPE_MASK to 0 */ + reg_val |= FIELD_PREP(GENMASK_ULL(3, 0), 0); + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_EXACT_CFG(NIX_INTF_RX), reg_val); +} + +/** + * rvu_exact_config_result_ctrl - Set exact table hash control + * @rvu: Resource virtualization unit. + * @depth: Depth of Exact match table. + * + * Sets mask and offset for hash for mem table. + */ +static void rvu_exact_config_result_ctrl(struct rvu *rvu, uint32_t depth) +{ + int blkaddr; + u64 reg = 0; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + + /* Set mask. Note that depth is a power of 2 */ + rvu->hw->table->mem_table.hash_mask = (depth - 1); + reg |= FIELD_PREP(GENMASK_ULL(42, 32), (depth - 1)); + + /* Set offset as 0 */ + rvu->hw->table->mem_table.hash_offset = 0; + reg |= FIELD_PREP(GENMASK_ULL(10, 0), 0); + + /* Set reg for RX */ + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_EXACT_RESULT_CTL(NIX_INTF_RX), reg); + /* Store hash mask and offset for s/w algorithm */ +} + +/** + * rvu_exact_config_table_mask - Set exact table mask. + * @rvu: Resource virtualization unit. + */ +static void rvu_exact_config_table_mask(struct rvu *rvu) +{ + int blkaddr; + u64 mask = 0; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + + /* Don't use Ctype */ + mask |= FIELD_PREP(GENMASK_ULL(61, 60), 0); + + /* Set chan */ + mask |= GENMASK_ULL(59, 48); + + /* Full ldata */ + mask |= GENMASK_ULL(47, 0); + + /* Store mask for s/w hash calcualtion */ + rvu->hw->table->mem_table.mask = mask; + + /* Set mask for RX.*/ + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_EXACT_MASK(NIX_INTF_RX), mask); +} + +/** + * rvu_npc_exact_get_max_entries - Get total number of entries in table. + * @rvu: resource virtualization unit. + * Return: Maximum table entries possible. + */ +u32 rvu_npc_exact_get_max_entries(struct rvu *rvu) +{ + struct npc_exact_table *table; + + table = rvu->hw->table; + return table->tot_ids; +} + +/** + * rvu_npc_exact_has_match_table - Checks support for exact match. + * @rvu: resource virtualization unit. + * Return: True if exact match table is supported/enabled. + */ +bool rvu_npc_exact_has_match_table(struct rvu *rvu) +{ + return rvu->hw->cap.npc_exact_match_enabled; +} + +/** + * __rvu_npc_exact_find_entry_by_seq_id - find entry by id + * @rvu: resource virtualization unit. + * @seq_id: Sequence identifier. + * + * Caller should acquire the lock. + * Return: Pointer to table entry. + */ +static struct npc_exact_table_entry * +__rvu_npc_exact_find_entry_by_seq_id(struct rvu *rvu, u32 seq_id) +{ + struct npc_exact_table *table = rvu->hw->table; + struct npc_exact_table_entry *entry = NULL; + struct list_head *lhead; + + lhead = &table->lhead_gbl; + + /* traverse to find the matching entry */ + list_for_each_entry(entry, lhead, glist) { + if (entry->seq_id != seq_id) + continue; + + return entry; + } + + return NULL; +} + +/** + * rvu_npc_exact_add_to_list - Add entry to list + * @rvu: resource virtualization unit. + * @opc_type: OPCODE to select MEM/CAM table. + * @ways: MEM table ways. + * @index: Index in MEM/CAM table. + * @cgx_id: CGX identifier. + * @lmac_id: LMAC identifier. + * @mac_addr: MAC address. + * @chan: Channel number. + * @ctype: Channel Type. + * @seq_id: Sequence identifier + * @cmd: True if function is called by ethtool cmd + * @mcam_idx: NPC mcam index of DMAC entry in NPC mcam. + * @pcifunc: pci function + * Return: 0 upon success. + */ +static int rvu_npc_exact_add_to_list(struct rvu *rvu, enum npc_exact_opc_type opc_type, u8 ways, + u32 index, u8 cgx_id, u8 lmac_id, u8 *mac_addr, u16 chan, + u8 ctype, u32 *seq_id, bool cmd, u32 mcam_idx, u16 pcifunc) +{ + struct npc_exact_table_entry *entry, *tmp, *iter; + struct npc_exact_table *table = rvu->hw->table; + struct list_head *lhead, *pprev; + + WARN_ON(ways >= NPC_EXACT_TBL_MAX_WAYS); + + if (!rvu_npc_exact_alloc_id(rvu, seq_id)) { + dev_err(rvu->dev, "%s: Generate seq id failed\n", __func__); + return -EFAULT; + } + + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) { + rvu_npc_exact_free_id(rvu, *seq_id); + dev_err(rvu->dev, "%s: Memory allocation failed\n", __func__); + return -ENOMEM; + } + + mutex_lock(&table->lock); + switch (opc_type) { + case NPC_EXACT_OPC_CAM: + lhead = &table->lhead_cam_tbl_entry; + table->cam_tbl_entry_cnt++; + break; + + case NPC_EXACT_OPC_MEM: + lhead = &table->lhead_mem_tbl_entry[ways]; + table->mem_tbl_entry_cnt++; + break; + + default: + mutex_unlock(&table->lock); + kfree(entry); + rvu_npc_exact_free_id(rvu, *seq_id); + + dev_err(rvu->dev, "%s: Unknown opc type%d\n", __func__, opc_type); + return -EINVAL; + } + + /* Add to global list */ + INIT_LIST_HEAD(&entry->glist); + list_add_tail(&entry->glist, &table->lhead_gbl); + INIT_LIST_HEAD(&entry->list); + entry->index = index; + entry->ways = ways; + entry->opc_type = opc_type; + + entry->pcifunc = pcifunc; + + ether_addr_copy(entry->mac, mac_addr); + entry->chan = chan; + entry->ctype = ctype; + entry->cgx_id = cgx_id; + entry->lmac_id = lmac_id; + + entry->seq_id = *seq_id; + + entry->mcam_idx = mcam_idx; + entry->cmd = cmd; + + pprev = lhead; + + /* Insert entry in ascending order of index */ + list_for_each_entry_safe(iter, tmp, lhead, list) { + if (index < iter->index) + break; + + pprev = &iter->list; + } + + /* Add to each table list */ + list_add(&entry->list, pprev); + mutex_unlock(&table->lock); + return 0; +} + +/** + * rvu_npc_exact_mem_table_write - Wrapper for register write + * @rvu: resource virtualization unit. + * @blkaddr: Block address + * @ways: ways for MEM table. + * @index: Index in MEM + * @mdata: Meta data to be written to register. + */ +static void rvu_npc_exact_mem_table_write(struct rvu *rvu, int blkaddr, u8 ways, + u32 index, u64 mdata) +{ + rvu_write64(rvu, blkaddr, NPC_AF_EXACT_MEM_ENTRY(ways, index), mdata); +} + +/** + * rvu_npc_exact_cam_table_write - Wrapper for register write + * @rvu: resource virtualization unit. + * @blkaddr: Block address + * @index: Index in MEM + * @mdata: Meta data to be written to register. + */ +static void rvu_npc_exact_cam_table_write(struct rvu *rvu, int blkaddr, + u32 index, u64 mdata) +{ + rvu_write64(rvu, blkaddr, NPC_AF_EXACT_CAM_ENTRY(index), mdata); +} + +/** + * rvu_npc_exact_dealloc_table_entry - dealloc table entry + * @rvu: resource virtualization unit. + * @opc_type: OPCODE for selection of table(MEM or CAM) + * @ways: ways if opc_type is MEM table. + * @index: Index of MEM or CAM table. + * Return: 0 upon success. + */ +static int rvu_npc_exact_dealloc_table_entry(struct rvu *rvu, enum npc_exact_opc_type opc_type, + u8 ways, u32 index) +{ + int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + struct npc_exact_table *table; + u8 null_dmac[6] = { 0 }; + int depth; + + /* Prepare entry with all fields set to zero */ + u64 null_mdata = rvu_exact_prepare_table_entry(rvu, false, 0, 0, null_dmac); + + table = rvu->hw->table; + depth = table->mem_table.depth; + + mutex_lock(&table->lock); + + switch (opc_type) { + case NPC_EXACT_OPC_CAM: + + /* Check whether entry is used already */ + if (!test_bit(index, table->cam_table.bmap)) { + mutex_unlock(&table->lock); + dev_err(rvu->dev, "%s: Trying to free an unused entry ways=%d index=%d\n", + __func__, ways, index); + return -EINVAL; + } + + rvu_npc_exact_cam_table_write(rvu, blkaddr, index, null_mdata); + clear_bit(index, table->cam_table.bmap); + break; + + case NPC_EXACT_OPC_MEM: + + /* Check whether entry is used already */ + if (!test_bit(index + ways * depth, table->mem_table.bmap)) { + mutex_unlock(&table->lock); + dev_err(rvu->dev, "%s: Trying to free an unused entry index=%d\n", + __func__, index); + return -EINVAL; + } + + rvu_npc_exact_mem_table_write(rvu, blkaddr, ways, index, null_mdata); + clear_bit(index + ways * depth, table->mem_table.bmap); + break; + + default: + mutex_unlock(&table->lock); + dev_err(rvu->dev, "%s: invalid opc type %d", __func__, opc_type); + return -ENOSPC; + } + + mutex_unlock(&table->lock); + + dev_dbg(rvu->dev, "%s: Successfully deleted entry (index=%d, ways=%d opc_type=%d\n", + __func__, index, ways, opc_type); + + return 0; +} + +/** + * rvu_npc_exact_alloc_table_entry - Allociate an entry + * @rvu: resource virtualization unit. + * @mac: MAC address. + * @chan: Channel number. + * @ctype: Channel Type. + * @index: Index of MEM table or CAM table. + * @ways: Ways. Only valid for MEM table. + * @opc_type: OPCODE to select table (MEM or CAM) + * + * Try allocating a slot from MEM table. If all 4 ways + * slot are full for a hash index, check availability in + * 32-entry CAM table for allocation. + * Return: 0 upon success. + */ +static int rvu_npc_exact_alloc_table_entry(struct rvu *rvu, char *mac, u16 chan, u8 ctype, + u32 *index, u8 *ways, enum npc_exact_opc_type *opc_type) +{ + struct npc_exact_table *table; + unsigned int hash; + int err; + + table = rvu->hw->table; + + /* Check in 4-ways mem entry for free slote */ + hash = rvu_exact_calculate_hash(rvu, chan, ctype, mac, table->mem_table.mask, + table->mem_table.depth); + err = rvu_npc_exact_alloc_mem_table_entry(rvu, ways, index, hash); + if (!err) { + *opc_type = NPC_EXACT_OPC_MEM; + dev_dbg(rvu->dev, "%s: inserted in 4 ways hash table ways=%d, index=%d\n", + __func__, *ways, *index); + return 0; + } + + dev_dbg(rvu->dev, "%s: failed to insert in 4 ways hash table\n", __func__); + + /* wayss is 0 for cam table */ + *ways = 0; + err = rvu_npc_exact_alloc_cam_table_entry(rvu, index); + if (!err) { + *opc_type = NPC_EXACT_OPC_CAM; + dev_dbg(rvu->dev, "%s: inserted in fully associative hash table index=%u\n", + __func__, *index); + return 0; + } + + dev_err(rvu->dev, "%s: failed to insert in fully associative hash table\n", __func__); + return -ENOSPC; +} + +/** + * rvu_npc_exact_save_drop_rule_chan_and_mask - Save drop rules info in data base. + * @rvu: resource virtualization unit. + * @drop_mcam_idx: Drop rule index in NPC mcam. + * @chan_val: Channel value. + * @chan_mask: Channel Mask. + * @pcifunc: pcifunc of interface. + * Return: True upon success. + */ +static bool rvu_npc_exact_save_drop_rule_chan_and_mask(struct rvu *rvu, int drop_mcam_idx, + u64 chan_val, u64 chan_mask, u16 pcifunc) +{ + struct npc_exact_table *table; + int i; + + table = rvu->hw->table; + + for (i = 0; i < NPC_MCAM_DROP_RULE_MAX; i++) { + if (!table->drop_rule_map[i].valid) + break; + + if (table->drop_rule_map[i].chan_val != (u16)chan_val) + continue; + + if (table->drop_rule_map[i].chan_mask != (u16)chan_mask) + continue; + + return false; + } + + if (i == NPC_MCAM_DROP_RULE_MAX) + return false; + + table->drop_rule_map[i].drop_rule_idx = drop_mcam_idx; + table->drop_rule_map[i].chan_val = (u16)chan_val; + table->drop_rule_map[i].chan_mask = (u16)chan_mask; + table->drop_rule_map[i].pcifunc = pcifunc; + table->drop_rule_map[i].valid = true; + return true; +} + +/** + * rvu_npc_exact_calc_drop_rule_chan_and_mask - Calculate Channel number and mask. + * @rvu: resource virtualization unit. + * @intf_type: Interface type (SDK, LBK or CGX) + * @cgx_id: CGX identifier. + * @lmac_id: LAMC identifier. + * @val: Channel number. + * @mask: Channel mask. + * Return: True upon success. + */ +static bool rvu_npc_exact_calc_drop_rule_chan_and_mask(struct rvu *rvu, u8 intf_type, + u8 cgx_id, u8 lmac_id, + u64 *val, u64 *mask) +{ + u16 chan_val, chan_mask; + + /* No support for SDP and LBK */ + if (intf_type != NIX_INTF_TYPE_CGX) + return false; + + chan_val = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0); + chan_mask = 0xfff; + + if (val) + *val = chan_val; + + if (mask) + *mask = chan_mask; + + return true; +} + +/** + * rvu_npc_exact_drop_rule_to_pcifunc - Retrieve pcifunc + * @rvu: resource virtualization unit. + * @drop_rule_idx: Drop rule index in NPC mcam. + * + * Debugfs (exact_drop_cnt) entry displays pcifunc for interface + * by retrieving the pcifunc value from data base. + * Return: Drop rule index. + */ +u16 rvu_npc_exact_drop_rule_to_pcifunc(struct rvu *rvu, u32 drop_rule_idx) +{ + struct npc_exact_table *table; + int i; + + table = rvu->hw->table; + + for (i = 0; i < NPC_MCAM_DROP_RULE_MAX; i++) { + if (!table->drop_rule_map[i].valid) + break; + + if (table->drop_rule_map[i].drop_rule_idx != drop_rule_idx) + continue; + + return table->drop_rule_map[i].pcifunc; + } + + dev_err(rvu->dev, "%s: drop mcam rule index (%d) >= NPC_MCAM_DROP_RULE_MAX\n", + __func__, drop_rule_idx); + return -1; +} + +/** + * rvu_npc_exact_get_drop_rule_info - Get drop rule information. + * @rvu: resource virtualization unit. + * @intf_type: Interface type (CGX, SDP or LBK) + * @cgx_id: CGX identifier. + * @lmac_id: LMAC identifier. + * @drop_mcam_idx: NPC mcam drop rule index. + * @val: Channel value. + * @mask: Channel mask. + * @pcifunc: pcifunc of interface corresponding to the drop rule. + * Return: True upon success. + */ +static bool rvu_npc_exact_get_drop_rule_info(struct rvu *rvu, u8 intf_type, u8 cgx_id, + u8 lmac_id, u32 *drop_mcam_idx, u64 *val, + u64 *mask, u16 *pcifunc) +{ + struct npc_exact_table *table; + u64 chan_val, chan_mask; + bool rc; + int i; + + table = rvu->hw->table; + + if (intf_type != NIX_INTF_TYPE_CGX) { + dev_err(rvu->dev, "%s: No drop rule for LBK/SDP mode\n", __func__); + return false; + } + + rc = rvu_npc_exact_calc_drop_rule_chan_and_mask(rvu, intf_type, cgx_id, + lmac_id, &chan_val, &chan_mask); + if (!rc) + return false; + + for (i = 0; i < NPC_MCAM_DROP_RULE_MAX; i++) { + if (!table->drop_rule_map[i].valid) + break; + + if (table->drop_rule_map[i].chan_val != (u16)chan_val) + continue; + + if (val) + *val = table->drop_rule_map[i].chan_val; + if (mask) + *mask = table->drop_rule_map[i].chan_mask; + if (pcifunc) + *pcifunc = table->drop_rule_map[i].pcifunc; + + *drop_mcam_idx = i; + return true; + } + + if (i == NPC_MCAM_DROP_RULE_MAX) { + dev_err(rvu->dev, "%s: drop mcam rule index (%d) >= NPC_MCAM_DROP_RULE_MAX\n", + __func__, *drop_mcam_idx); + return false; + } + + dev_err(rvu->dev, "%s: Could not retrieve for cgx=%d, lmac=%d\n", + __func__, cgx_id, lmac_id); + return false; +} + +/** + * __rvu_npc_exact_cmd_rules_cnt_update - Update number dmac rules against a drop rule. + * @rvu: resource virtualization unit. + * @drop_mcam_idx: NPC mcam drop rule index. + * @val: +1 or -1. + * @enable_or_disable_cam: If no exact match rules against a drop rule, disable it. + * + * when first exact match entry against a drop rule is added, enable_or_disable_cam + * is set to true. When last exact match entry against a drop rule is deleted, + * enable_or_disable_cam is set to true. + * Return: Number of rules + */ +static u16 __rvu_npc_exact_cmd_rules_cnt_update(struct rvu *rvu, int drop_mcam_idx, + int val, bool *enable_or_disable_cam) +{ + struct npc_exact_table *table; + u16 *cnt, old_cnt; + bool promisc; + + table = rvu->hw->table; + promisc = table->promisc_mode[drop_mcam_idx]; + + cnt = &table->cnt_cmd_rules[drop_mcam_idx]; + old_cnt = *cnt; + + *cnt += val; + + if (!enable_or_disable_cam) + goto done; + + *enable_or_disable_cam = false; + + if (promisc) + goto done; + + /* If all rules are deleted and not already in promisc mode; disable cam */ + if (!*cnt && val < 0) { + *enable_or_disable_cam = true; + goto done; + } + + /* If rule got added and not already in promisc mode; enable cam */ + if (!old_cnt && val > 0) { + *enable_or_disable_cam = true; + goto done; + } + +done: + return *cnt; +} + +/** + * rvu_npc_exact_del_table_entry_by_id - Delete and free table entry. + * @rvu: resource virtualization unit. + * @seq_id: Sequence identifier of the entry. + * + * Deletes entry from linked lists and free up slot in HW MEM or CAM + * table. + * Return: 0 upon success. + */ +static int rvu_npc_exact_del_table_entry_by_id(struct rvu *rvu, u32 seq_id) +{ + struct npc_exact_table_entry *entry = NULL; + struct npc_exact_table *table; + bool disable_cam = false; + u32 drop_mcam_idx = -1; + int *cnt; + bool rc; + + table = rvu->hw->table; + + mutex_lock(&table->lock); + + /* Lookup for entry which needs to be updated */ + entry = __rvu_npc_exact_find_entry_by_seq_id(rvu, seq_id); + if (!entry) { + dev_dbg(rvu->dev, "%s: failed to find entry for id=%d\n", __func__, seq_id); + mutex_unlock(&table->lock); + return -ENODATA; + } + + cnt = (entry->opc_type == NPC_EXACT_OPC_CAM) ? &table->cam_tbl_entry_cnt : + &table->mem_tbl_entry_cnt; + + /* delete from lists */ + list_del_init(&entry->list); + list_del_init(&entry->glist); + + (*cnt)--; + + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, entry->cgx_id, + entry->lmac_id, &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + dev_dbg(rvu->dev, "%s: failed to retrieve drop info for id=0x%x\n", + __func__, seq_id); + mutex_unlock(&table->lock); + return -ENODATA; + } + + if (entry->cmd) + __rvu_npc_exact_cmd_rules_cnt_update(rvu, drop_mcam_idx, -1, &disable_cam); + + /* No dmac filter rules; disable drop on hit rule */ + if (disable_cam) { + rvu_npc_enable_mcam_by_entry_index(rvu, drop_mcam_idx, NIX_INTF_RX, false); + dev_dbg(rvu->dev, "%s: Disabling mcam idx %d\n", + __func__, drop_mcam_idx); + } + + mutex_unlock(&table->lock); + + rvu_npc_exact_dealloc_table_entry(rvu, entry->opc_type, entry->ways, entry->index); + + rvu_npc_exact_free_id(rvu, seq_id); + + dev_dbg(rvu->dev, "%s: delete entry success for id=0x%x, mca=%pM\n", + __func__, seq_id, entry->mac); + kfree(entry); + + return 0; +} + +/** + * rvu_npc_exact_add_table_entry - Adds a table entry + * @rvu: resource virtualization unit. + * @cgx_id: cgx identifier. + * @lmac_id: lmac identifier. + * @mac: MAC address. + * @chan: Channel number. + * @ctype: Channel Type. + * @seq_id: Sequence number. + * @cmd: Whether it is invoked by ethtool cmd. + * @mcam_idx: NPC mcam index corresponding to MAC + * @pcifunc: PCI func. + * + * Creates a new exact match table entry in either CAM or + * MEM table. + * Return: 0 upon success. + */ +static int rvu_npc_exact_add_table_entry(struct rvu *rvu, u8 cgx_id, u8 lmac_id, u8 *mac, + u16 chan, u8 ctype, u32 *seq_id, bool cmd, + u32 mcam_idx, u16 pcifunc) +{ + int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + enum npc_exact_opc_type opc_type; + bool enable_cam = false; + u32 drop_mcam_idx; + u32 index; + u64 mdata; + bool rc; + int err; + u8 ways; + + ctype = 0; + + err = rvu_npc_exact_alloc_table_entry(rvu, mac, chan, ctype, &index, &ways, &opc_type); + if (err) { + dev_err(rvu->dev, "%s: Could not alloc in exact match table\n", __func__); + return err; + } + + /* Write mdata to table */ + mdata = rvu_exact_prepare_table_entry(rvu, true, ctype, chan, mac); + + if (opc_type == NPC_EXACT_OPC_CAM) + rvu_npc_exact_cam_table_write(rvu, blkaddr, index, mdata); + else + rvu_npc_exact_mem_table_write(rvu, blkaddr, ways, index, mdata); + + /* Insert entry to linked list */ + err = rvu_npc_exact_add_to_list(rvu, opc_type, ways, index, cgx_id, lmac_id, + mac, chan, ctype, seq_id, cmd, mcam_idx, pcifunc); + if (err) { + rvu_npc_exact_dealloc_table_entry(rvu, opc_type, ways, index); + dev_err(rvu->dev, "%s: could not add to exact match table\n", __func__); + return err; + } + + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, + &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + rvu_npc_exact_dealloc_table_entry(rvu, opc_type, ways, index); + dev_dbg(rvu->dev, "%s: failed to get drop rule info cgx=%d lmac=%d\n", + __func__, cgx_id, lmac_id); + return -EINVAL; + } + + if (cmd) + __rvu_npc_exact_cmd_rules_cnt_update(rvu, drop_mcam_idx, 1, &enable_cam); + + /* First command rule; enable drop on hit rule */ + if (enable_cam) { + rvu_npc_enable_mcam_by_entry_index(rvu, drop_mcam_idx, NIX_INTF_RX, true); + dev_dbg(rvu->dev, "%s: Enabling mcam idx %d\n", + __func__, drop_mcam_idx); + } + + dev_dbg(rvu->dev, + "%s: Successfully added entry (index=%d, dmac=%pM, ways=%d opc_type=%d\n", + __func__, index, mac, ways, opc_type); + + return 0; +} + +/** + * rvu_npc_exact_update_table_entry - Update exact match table. + * @rvu: resource virtualization unit. + * @cgx_id: CGX identifier. + * @lmac_id: LMAC identifier. + * @old_mac: Existing MAC address entry. + * @new_mac: New MAC address entry. + * @seq_id: Sequence identifier of the entry. + * + * Updates MAC address of an entry. If entry is in MEM table, new + * hash value may not match with old one. + * Return: 0 upon success. + */ +static int rvu_npc_exact_update_table_entry(struct rvu *rvu, u8 cgx_id, u8 lmac_id, + u8 *old_mac, u8 *new_mac, u32 *seq_id) +{ + int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + struct npc_exact_table_entry *entry; + struct npc_exact_table *table; + u32 hash_index; + u64 mdata; + + table = rvu->hw->table; + + mutex_lock(&table->lock); + + /* Lookup for entry which needs to be updated */ + entry = __rvu_npc_exact_find_entry_by_seq_id(rvu, *seq_id); + if (!entry) { + mutex_unlock(&table->lock); + dev_dbg(rvu->dev, + "%s: failed to find entry for cgx_id=%d lmac_id=%d old_mac=%pM\n", + __func__, cgx_id, lmac_id, old_mac); + return -ENODATA; + } + + /* If entry is in mem table and new hash index is different than old + * hash index, we cannot update the entry. Fail in these scenarios. + */ + if (entry->opc_type == NPC_EXACT_OPC_MEM) { + hash_index = rvu_exact_calculate_hash(rvu, entry->chan, entry->ctype, + new_mac, table->mem_table.mask, + table->mem_table.depth); + if (hash_index != entry->index) { + dev_dbg(rvu->dev, + "%s: Update failed due to index mismatch(new=0x%x, old=%x)\n", + __func__, hash_index, entry->index); + mutex_unlock(&table->lock); + return -EINVAL; + } + } + + mdata = rvu_exact_prepare_table_entry(rvu, true, entry->ctype, entry->chan, new_mac); + + if (entry->opc_type == NPC_EXACT_OPC_MEM) + rvu_npc_exact_mem_table_write(rvu, blkaddr, entry->ways, entry->index, mdata); + else + rvu_npc_exact_cam_table_write(rvu, blkaddr, entry->index, mdata); + + /* Update entry fields */ + ether_addr_copy(entry->mac, new_mac); + *seq_id = entry->seq_id; + + dev_dbg(rvu->dev, + "%s: Successfully updated entry (index=%d, dmac=%pM, ways=%d opc_type=%d\n", + __func__, entry->index, entry->mac, entry->ways, entry->opc_type); + + dev_dbg(rvu->dev, "%s: Successfully updated entry (old mac=%pM new_mac=%pM\n", + __func__, old_mac, new_mac); + + mutex_unlock(&table->lock); + return 0; +} + +/** + * rvu_npc_exact_promisc_disable - Disable promiscuous mode. + * @rvu: resource virtualization unit. + * @pcifunc: pcifunc + * + * Drop rule is against each PF. We dont support DMAC filter for + * VF. + * Return: 0 upon success + */ + +int rvu_npc_exact_promisc_disable(struct rvu *rvu, u16 pcifunc) +{ + struct npc_exact_table *table; + int pf = rvu_get_pf(pcifunc); + u8 cgx_id, lmac_id; + u32 drop_mcam_idx; + bool *promisc; + bool rc; + u32 cnt; + + table = rvu->hw->table; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, + &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + dev_dbg(rvu->dev, "%s: failed to get drop rule info cgx=%d lmac=%d\n", + __func__, cgx_id, lmac_id); + return -EINVAL; + } + + mutex_lock(&table->lock); + promisc = &table->promisc_mode[drop_mcam_idx]; + + if (!*promisc) { + mutex_unlock(&table->lock); + dev_dbg(rvu->dev, "%s: Err Already promisc mode disabled (cgx=%d lmac=%d)\n", + __func__, cgx_id, lmac_id); + return LMAC_AF_ERR_INVALID_PARAM; + } + *promisc = false; + cnt = __rvu_npc_exact_cmd_rules_cnt_update(rvu, drop_mcam_idx, 0, NULL); + mutex_unlock(&table->lock); + + /* If no dmac filter entries configured, disable drop rule */ + if (!cnt) + rvu_npc_enable_mcam_by_entry_index(rvu, drop_mcam_idx, NIX_INTF_RX, false); + else + rvu_npc_enable_mcam_by_entry_index(rvu, drop_mcam_idx, NIX_INTF_RX, !*promisc); + + dev_dbg(rvu->dev, "%s: disabled promisc mode (cgx=%d lmac=%d, cnt=%d)\n", + __func__, cgx_id, lmac_id, cnt); + return 0; +} + +/** + * rvu_npc_exact_promisc_enable - Enable promiscuous mode. + * @rvu: resource virtualization unit. + * @pcifunc: pcifunc. + * Return: 0 upon success + */ +int rvu_npc_exact_promisc_enable(struct rvu *rvu, u16 pcifunc) +{ + struct npc_exact_table *table; + int pf = rvu_get_pf(pcifunc); + u8 cgx_id, lmac_id; + u32 drop_mcam_idx; + bool *promisc; + bool rc; + u32 cnt; + + table = rvu->hw->table; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + rc = rvu_npc_exact_get_drop_rule_info(rvu, NIX_INTF_TYPE_CGX, cgx_id, lmac_id, + &drop_mcam_idx, NULL, NULL, NULL); + if (!rc) { + dev_dbg(rvu->dev, "%s: failed to get drop rule info cgx=%d lmac=%d\n", + __func__, cgx_id, lmac_id); + return -EINVAL; + } + + mutex_lock(&table->lock); + promisc = &table->promisc_mode[drop_mcam_idx]; + + if (*promisc) { + mutex_unlock(&table->lock); + dev_dbg(rvu->dev, "%s: Already in promisc mode (cgx=%d lmac=%d)\n", + __func__, cgx_id, lmac_id); + return LMAC_AF_ERR_INVALID_PARAM; + } + *promisc = true; + cnt = __rvu_npc_exact_cmd_rules_cnt_update(rvu, drop_mcam_idx, 0, NULL); + mutex_unlock(&table->lock); + + /* If no dmac filter entries configured, disable drop rule */ + if (!cnt) + rvu_npc_enable_mcam_by_entry_index(rvu, drop_mcam_idx, NIX_INTF_RX, false); + else + rvu_npc_enable_mcam_by_entry_index(rvu, drop_mcam_idx, NIX_INTF_RX, !*promisc); + + dev_dbg(rvu->dev, "%s: Enabled promisc mode (cgx=%d lmac=%d cnt=%d)\n", + __func__, cgx_id, lmac_id, cnt); + return 0; +} + +/** + * rvu_npc_exact_mac_addr_reset - Delete PF mac address. + * @rvu: resource virtualization unit. + * @req: Reset request + * @rsp: Reset response. + * Return: 0 upon success + */ +int rvu_npc_exact_mac_addr_reset(struct rvu *rvu, struct cgx_mac_addr_reset_req *req, + struct msg_rsp *rsp) +{ + int pf = rvu_get_pf(req->hdr.pcifunc); + u32 seq_id = req->index; + struct rvu_pfvf *pfvf; + u8 cgx_id, lmac_id; + int rc; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + + pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + + rc = rvu_npc_exact_del_table_entry_by_id(rvu, seq_id); + if (rc) { + /* TODO: how to handle this error case ? */ + dev_err(rvu->dev, "%s MAC (%pM) del PF=%d failed\n", __func__, pfvf->mac_addr, pf); + return 0; + } + + dev_dbg(rvu->dev, "%s MAC (%pM) del PF=%d success (seq_id=%u)\n", + __func__, pfvf->mac_addr, pf, seq_id); + return 0; +} + +/** + * rvu_npc_exact_mac_addr_update - Update mac address field with new value. + * @rvu: resource virtualization unit. + * @req: Update request. + * @rsp: Update response. + * Return: 0 upon success + */ +int rvu_npc_exact_mac_addr_update(struct rvu *rvu, + struct cgx_mac_addr_update_req *req, + struct cgx_mac_addr_update_rsp *rsp) +{ + int pf = rvu_get_pf(req->hdr.pcifunc); + struct npc_exact_table_entry *entry; + struct npc_exact_table *table; + struct rvu_pfvf *pfvf; + u32 seq_id, mcam_idx; + u8 old_mac[ETH_ALEN]; + u8 cgx_id, lmac_id; + int rc; + + if (!is_cgx_config_permitted(rvu, req->hdr.pcifunc)) + return LMAC_AF_ERR_PERM_DENIED; + + dev_dbg(rvu->dev, "%s: Update request for seq_id=%d, mac=%pM\n", + __func__, req->index, req->mac_addr); + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + + pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + + table = rvu->hw->table; + + mutex_lock(&table->lock); + + /* Lookup for entry which needs to be updated */ + entry = __rvu_npc_exact_find_entry_by_seq_id(rvu, req->index); + if (!entry) { + dev_err(rvu->dev, "%s: failed to find entry for id=0x%x\n", __func__, req->index); + mutex_unlock(&table->lock); + return LMAC_AF_ERR_EXACT_MATCH_TBL_LOOK_UP_FAILED; + } + ether_addr_copy(old_mac, entry->mac); + seq_id = entry->seq_id; + mcam_idx = entry->mcam_idx; + mutex_unlock(&table->lock); + + rc = rvu_npc_exact_update_table_entry(rvu, cgx_id, lmac_id, old_mac, + req->mac_addr, &seq_id); + if (!rc) { + rsp->index = seq_id; + dev_dbg(rvu->dev, "%s mac:%pM (pfvf:%pM default:%pM) update to PF=%d success\n", + __func__, req->mac_addr, pfvf->mac_addr, pfvf->default_mac, pf); + ether_addr_copy(pfvf->mac_addr, req->mac_addr); + return 0; + } + + /* Try deleting and adding it again */ + rc = rvu_npc_exact_del_table_entry_by_id(rvu, req->index); + if (rc) { + /* This could be a new entry */ + dev_dbg(rvu->dev, "%s MAC (%pM) del PF=%d failed\n", __func__, + pfvf->mac_addr, pf); + } + + rc = rvu_npc_exact_add_table_entry(rvu, cgx_id, lmac_id, req->mac_addr, + pfvf->rx_chan_base, 0, &seq_id, true, + mcam_idx, req->hdr.pcifunc); + if (rc) { + dev_err(rvu->dev, "%s MAC (%pM) add PF=%d failed\n", __func__, + req->mac_addr, pf); + return LMAC_AF_ERR_EXACT_MATCH_TBL_ADD_FAILED; + } + + rsp->index = seq_id; + dev_dbg(rvu->dev, + "%s MAC (new:%pM, old=%pM default:%pM) del and add to PF=%d success (seq_id=%u)\n", + __func__, req->mac_addr, pfvf->mac_addr, pfvf->default_mac, pf, seq_id); + + ether_addr_copy(pfvf->mac_addr, req->mac_addr); + return 0; +} + +/** + * rvu_npc_exact_mac_addr_add - Adds MAC address to exact match table. + * @rvu: resource virtualization unit. + * @req: Add request. + * @rsp: Add response. + * Return: 0 upon success + */ +int rvu_npc_exact_mac_addr_add(struct rvu *rvu, + struct cgx_mac_addr_add_req *req, + struct cgx_mac_addr_add_rsp *rsp) +{ + int pf = rvu_get_pf(req->hdr.pcifunc); + struct rvu_pfvf *pfvf; + u8 cgx_id, lmac_id; + int rc = 0; + u32 seq_id; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + + rc = rvu_npc_exact_add_table_entry(rvu, cgx_id, lmac_id, req->mac_addr, + pfvf->rx_chan_base, 0, &seq_id, + true, -1, req->hdr.pcifunc); + + if (!rc) { + rsp->index = seq_id; + dev_dbg(rvu->dev, "%s MAC (%pM) add to PF=%d success (seq_id=%u)\n", + __func__, req->mac_addr, pf, seq_id); + return 0; + } + + dev_err(rvu->dev, "%s MAC (%pM) add to PF=%d failed\n", __func__, + req->mac_addr, pf); + return LMAC_AF_ERR_EXACT_MATCH_TBL_ADD_FAILED; +} + +/** + * rvu_npc_exact_mac_addr_del - Delete DMAC filter + * @rvu: resource virtualization unit. + * @req: Delete request. + * @rsp: Delete response. + * Return: 0 upon success + */ +int rvu_npc_exact_mac_addr_del(struct rvu *rvu, + struct cgx_mac_addr_del_req *req, + struct msg_rsp *rsp) +{ + int pf = rvu_get_pf(req->hdr.pcifunc); + int rc; + + rc = rvu_npc_exact_del_table_entry_by_id(rvu, req->index); + if (!rc) { + dev_dbg(rvu->dev, "%s del to PF=%d success (seq_id=%u)\n", + __func__, pf, req->index); + return 0; + } + + dev_err(rvu->dev, "%s del to PF=%d failed (seq_id=%u)\n", + __func__, pf, req->index); + return LMAC_AF_ERR_EXACT_MATCH_TBL_DEL_FAILED; +} + +/** + * rvu_npc_exact_mac_addr_set - Add PF mac address to dmac filter. + * @rvu: resource virtualization unit. + * @req: Set request. + * @rsp: Set response. + * Return: 0 upon success + */ +int rvu_npc_exact_mac_addr_set(struct rvu *rvu, struct cgx_mac_addr_set_or_get *req, + struct cgx_mac_addr_set_or_get *rsp) +{ + int pf = rvu_get_pf(req->hdr.pcifunc); + u32 seq_id = req->index; + struct rvu_pfvf *pfvf; + u8 cgx_id, lmac_id; + u32 mcam_idx = -1; + int rc, nixlf; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); + + pfvf = &rvu->pf[pf]; + + /* If table does not have an entry; both update entry and del table entry API + * below fails. Those are not failure conditions. + */ + rc = rvu_npc_exact_update_table_entry(rvu, cgx_id, lmac_id, pfvf->mac_addr, + req->mac_addr, &seq_id); + if (!rc) { + rsp->index = seq_id; + ether_addr_copy(pfvf->mac_addr, req->mac_addr); + ether_addr_copy(rsp->mac_addr, req->mac_addr); + dev_dbg(rvu->dev, "%s MAC (%pM) update to PF=%d success\n", + __func__, req->mac_addr, pf); + return 0; + } + + /* Try deleting and adding it again */ + rc = rvu_npc_exact_del_table_entry_by_id(rvu, req->index); + if (rc) { + dev_dbg(rvu->dev, "%s MAC (%pM) del PF=%d failed\n", + __func__, pfvf->mac_addr, pf); + } + + /* find mcam entry if exist */ + rc = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, NULL); + if (!rc) { + mcam_idx = npc_get_nixlf_mcam_index(&rvu->hw->mcam, req->hdr.pcifunc, + nixlf, NIXLF_UCAST_ENTRY); + } + + rc = rvu_npc_exact_add_table_entry(rvu, cgx_id, lmac_id, req->mac_addr, + pfvf->rx_chan_base, 0, &seq_id, + true, mcam_idx, req->hdr.pcifunc); + if (rc) { + dev_err(rvu->dev, "%s MAC (%pM) add PF=%d failed\n", + __func__, req->mac_addr, pf); + return LMAC_AF_ERR_EXACT_MATCH_TBL_ADD_FAILED; + } + + rsp->index = seq_id; + ether_addr_copy(rsp->mac_addr, req->mac_addr); + ether_addr_copy(pfvf->mac_addr, req->mac_addr); + dev_dbg(rvu->dev, + "%s MAC (%pM) del and add to PF=%d success (seq_id=%u)\n", + __func__, req->mac_addr, pf, seq_id); + return 0; +} + +/** + * rvu_npc_exact_can_disable_feature - Check if feature can be disabled. + * @rvu: resource virtualization unit. + * Return: True if exact match feature is supported. + */ +bool rvu_npc_exact_can_disable_feature(struct rvu *rvu) +{ + struct npc_exact_table *table = rvu->hw->table; + bool empty; + + if (!rvu->hw->cap.npc_exact_match_enabled) + return false; + + mutex_lock(&table->lock); + empty = list_empty(&table->lhead_gbl); + mutex_unlock(&table->lock); + + return empty; +} + +/** + * rvu_npc_exact_disable_feature - Disable feature. + * @rvu: resource virtualization unit. + */ +void rvu_npc_exact_disable_feature(struct rvu *rvu) +{ + rvu->hw->cap.npc_exact_match_enabled = false; +} + +/** + * rvu_npc_exact_reset - Delete and free all entry which match pcifunc. + * @rvu: resource virtualization unit. + * @pcifunc: PCI func to match. + */ +void rvu_npc_exact_reset(struct rvu *rvu, u16 pcifunc) +{ + struct npc_exact_table *table = rvu->hw->table; + struct npc_exact_table_entry *tmp, *iter; + u32 seq_id; + + mutex_lock(&table->lock); + list_for_each_entry_safe(iter, tmp, &table->lhead_gbl, glist) { + if (pcifunc != iter->pcifunc) + continue; + + seq_id = iter->seq_id; + dev_dbg(rvu->dev, "%s: resetting pcifun=%d seq_id=%u\n", __func__, + pcifunc, seq_id); + + mutex_unlock(&table->lock); + rvu_npc_exact_del_table_entry_by_id(rvu, seq_id); + mutex_lock(&table->lock); + } + mutex_unlock(&table->lock); +} + +/** + * rvu_npc_exact_init - initialize exact match table + * @rvu: resource virtualization unit. + * + * Initialize HW and SW resources to manage 4way-2K table and fully + * associative 32-entry mcam table. + * Return: 0 upon success. + */ +int rvu_npc_exact_init(struct rvu *rvu) +{ + u64 bcast_mcast_val, bcast_mcast_mask; + struct npc_exact_table *table; + u64 exact_val, exact_mask; + u64 chan_val, chan_mask; + u8 cgx_id, lmac_id; + u32 *drop_mcam_idx; + u16 max_lmac_cnt; + u64 npc_const3; + int table_size; + int blkaddr; + u16 pcifunc; + int err, i; + u64 cfg; + bool rc; + + /* Read NPC_AF_CONST3 and check for have exact + * match functionality is present + */ + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return -EINVAL; + } + + /* Check exact match feature is supported */ + npc_const3 = rvu_read64(rvu, blkaddr, NPC_AF_CONST3); + if (!(npc_const3 & BIT_ULL(62))) { + dev_info(rvu->dev, "%s: No support for exact match support\n", + __func__); + return 0; + } + + /* Check if kex profile has enabled EXACT match nibble */ + cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX)); + if (!(cfg & NPC_EXACT_NIBBLE_HIT)) { + dev_info(rvu->dev, "%s: NPC exact match nibble not enabled in KEX profile\n", + __func__); + return 0; + } + + /* Set capability to true */ + rvu->hw->cap.npc_exact_match_enabled = true; + + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + return -ENOMEM; + + dev_dbg(rvu->dev, "%s: Memory allocation for table success\n", __func__); + memset(table, 0, sizeof(*table)); + rvu->hw->table = table; + + /* Read table size, ways and depth */ + table->mem_table.depth = FIELD_GET(GENMASK_ULL(31, 24), npc_const3); + table->mem_table.ways = FIELD_GET(GENMASK_ULL(19, 16), npc_const3); + table->cam_table.depth = FIELD_GET(GENMASK_ULL(15, 0), npc_const3); + + dev_dbg(rvu->dev, "%s: NPC exact match 4way_2k table(ways=%d, depth=%d)\n", + __func__, table->mem_table.ways, table->cam_table.depth); + + /* Check if depth of table is not a sequre of 2 + * TODO: why _builtin_popcount() is not working ? + */ + if ((table->mem_table.depth & (table->mem_table.depth - 1)) != 0) { + dev_err(rvu->dev, + "%s: NPC exact match 4way_2k table depth(%d) is not square of 2\n", + __func__, table->mem_table.depth); + return -EINVAL; + } + + table_size = table->mem_table.depth * table->mem_table.ways; + + /* Allocate bitmap for 4way 2K table */ + table->mem_table.bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(table_size), + sizeof(long), GFP_KERNEL); + if (!table->mem_table.bmap) + return -ENOMEM; + + dev_dbg(rvu->dev, "%s: Allocated bitmap for 4way 2K entry table\n", __func__); + + /* Allocate bitmap for 32 entry mcam */ + table->cam_table.bmap = devm_kcalloc(rvu->dev, 1, sizeof(long), GFP_KERNEL); + + if (!table->cam_table.bmap) + return -ENOMEM; + + dev_dbg(rvu->dev, "%s: Allocated bitmap for 32 entry cam\n", __func__); + + table->tot_ids = (table->mem_table.depth * table->mem_table.ways) + table->cam_table.depth; + table->id_bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(table->tot_ids), + table->tot_ids, GFP_KERNEL); + + if (!table->id_bmap) + return -ENOMEM; + + dev_dbg(rvu->dev, "%s: Allocated bitmap for id map (total=%d)\n", + __func__, table->tot_ids); + + /* Initialize list heads for npc_exact_table entries. + * This entry is used by debugfs to show entries in + * exact match table. + */ + for (i = 0; i < NPC_EXACT_TBL_MAX_WAYS; i++) + INIT_LIST_HEAD(&table->lhead_mem_tbl_entry[i]); + + INIT_LIST_HEAD(&table->lhead_cam_tbl_entry); + INIT_LIST_HEAD(&table->lhead_gbl); + + mutex_init(&table->lock); + + rvu_exact_config_secret_key(rvu); + rvu_exact_config_search_key(rvu); + + rvu_exact_config_table_mask(rvu); + rvu_exact_config_result_ctrl(rvu, table->mem_table.depth); + + /* - No drop rule for LBK + * - Drop rules for SDP and each LMAC. + */ + exact_val = !NPC_EXACT_RESULT_HIT; + exact_mask = NPC_EXACT_RESULT_HIT; + + /* nibble - 3 2 1 0 + * L3B L3M L2B L2M + */ + bcast_mcast_val = 0b0000; + bcast_mcast_mask = 0b0011; + + /* Install SDP drop rule */ + drop_mcam_idx = &table->num_drop_rules; + + max_lmac_cnt = rvu->cgx_cnt_max * MAX_LMAC_PER_CGX + PF_CGXMAP_BASE; + for (i = PF_CGXMAP_BASE; i < max_lmac_cnt; i++) { + if (rvu->pf2cgxlmac_map[i] == 0xFF) + continue; + + rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[i], &cgx_id, &lmac_id); + + rc = rvu_npc_exact_calc_drop_rule_chan_and_mask(rvu, NIX_INTF_TYPE_CGX, cgx_id, + lmac_id, &chan_val, &chan_mask); + if (!rc) { + dev_err(rvu->dev, + "%s: failed, info chan_val=0x%llx chan_mask=0x%llx rule_id=%d\n", + __func__, chan_val, chan_mask, *drop_mcam_idx); + return -EINVAL; + } + + /* Filter rules are only for PF */ + pcifunc = RVU_PFFUNC(i, 0); + + dev_dbg(rvu->dev, + "%s:Drop rule cgx=%d lmac=%d chan(val=0x%llx, mask=0x%llx\n", + __func__, cgx_id, lmac_id, chan_val, chan_mask); + + rc = rvu_npc_exact_save_drop_rule_chan_and_mask(rvu, table->num_drop_rules, + chan_val, chan_mask, pcifunc); + if (!rc) { + dev_err(rvu->dev, + "%s: failed to set drop info for cgx=%d, lmac=%d, chan=%llx\n", + __func__, cgx_id, lmac_id, chan_val); + return -EINVAL; + } + + err = npc_install_mcam_drop_rule(rvu, *drop_mcam_idx, + &table->counter_idx[*drop_mcam_idx], + chan_val, chan_mask, + exact_val, exact_mask, + bcast_mcast_val, bcast_mcast_mask); + if (err) { + dev_err(rvu->dev, + "failed to configure drop rule (cgx=%d lmac=%d)\n", + cgx_id, lmac_id); + return err; + } + + (*drop_mcam_idx)++; + } + + dev_info(rvu->dev, "initialized exact match table successfully\n"); + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h new file mode 100644 index 000000000000..3efeb09c58de --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2022 Marvell. + * + */ + +#ifndef __RVU_NPC_HASH_H +#define __RVU_NPC_HASH_H + +#define RVU_NPC_HASH_SECRET_KEY0 0xa9d5af4c9fbc76b1 +#define RVU_NPC_HASH_SECRET_KEY1 0xa9d5af4c9fbc87b4 +#define RVU_NPC_HASH_SECRET_KEY2 0x5954c9e7 + +#define NPC_MAX_HASH 2 +#define NPC_MAX_HASH_MASK 2 + +#define KEX_LD_CFG_USE_HASH(use_hash, bytesm1, hdr_ofs, ena, flags_ena, key_ofs) \ + ((use_hash) << 20 | ((bytesm1) << 16) | ((hdr_ofs) << 8) | \ + ((ena) << 7) | ((flags_ena) << 6) | ((key_ofs) & 0x3F)) +#define KEX_LD_CFG_HASH(hdr_ofs, bytesm1, lt_en, lid_en, lid, ltype_match, ltype_mask) \ + (((hdr_ofs) << 32) | ((bytesm1) << 16) | \ + ((lt_en) << 12) | ((lid_en) << 11) | ((lid) << 8) | \ + ((ltype_match) << 4) | ((ltype_mask) & 0xF)) + +#define SET_KEX_LD_HASH(intf, ld, cfg) \ + rvu_write64(rvu, blkaddr, \ + NPC_AF_INTFX_HASHX_CFG(intf, ld), cfg) + +#define SET_KEX_LD_HASH_MASK(intf, ld, mask_idx, cfg) \ + rvu_write64(rvu, blkaddr, \ + NPC_AF_INTFX_HASHX_MASKX(intf, ld, mask_idx), cfg) + +#define SET_KEX_LD_HASH_CTRL(intf, ld, cfg) \ + rvu_write64(rvu, blkaddr, \ + NPC_AF_INTFX_HASHX_RESULT_CTRL(intf, ld), cfg) + +struct npc_mcam_kex_hash { + /* NPC_AF_INTF(0..1)_LID(0..7)_LT(0..15)_LD(0..1)_CFG */ + bool lid_lt_ld_hash_en[NPC_MAX_INTF][NPC_MAX_LID][NPC_MAX_LT][NPC_MAX_LD]; + /* NPC_AF_INTF(0..1)_HASH(0..1)_CFG */ + u64 hash[NPC_MAX_INTF][NPC_MAX_HASH]; + /* NPC_AF_INTF(0..1)_HASH(0..1)_MASK(0..1) */ + u64 hash_mask[NPC_MAX_INTF][NPC_MAX_HASH][NPC_MAX_HASH_MASK]; + /* NPC_AF_INTF(0..1)_HASH(0..1)_RESULT_CTRL */ + u64 hash_ctrl[NPC_MAX_INTF][NPC_MAX_HASH]; +} __packed; + +void npc_update_field_hash(struct rvu *rvu, u8 intf, + struct mcam_entry *entry, + int blkaddr, + u64 features, + struct flow_msg *pkt, + struct flow_msg *mask, + struct flow_msg *opkt, + struct flow_msg *omask); +void npc_config_secret_key(struct rvu *rvu, int blkaddr); +void npc_program_mkex_hash(struct rvu *rvu, int blkaddr); +u32 npc_field_hash_calc(u64 *ldata, struct npc_mcam_kex_hash *mkex_hash, + u64 *secret_key, u8 intf, u8 hash_idx); + +static struct npc_mcam_kex_hash npc_mkex_hash_default __maybe_unused = { + .lid_lt_ld_hash_en = { + [NIX_INTF_RX] = { + [NPC_LID_LC] = { + [NPC_LT_LC_IP6] = { + true, + true, + }, + }, + }, + + [NIX_INTF_TX] = { + [NPC_LID_LC] = { + [NPC_LT_LC_IP6] = { + true, + true, + }, + }, + }, + }, + + .hash = { + [NIX_INTF_RX] = { + KEX_LD_CFG_HASH(0x8ULL, 0xf, 0x1, 0x1, NPC_LID_LC, NPC_LT_LC_IP6, 0xf), + KEX_LD_CFG_HASH(0x18ULL, 0xf, 0x1, 0x1, NPC_LID_LC, NPC_LT_LC_IP6, 0xf), + }, + + [NIX_INTF_TX] = { + KEX_LD_CFG_HASH(0x8ULL, 0xf, 0x1, 0x1, NPC_LID_LC, NPC_LT_LC_IP6, 0xf), + KEX_LD_CFG_HASH(0x18ULL, 0xf, 0x1, 0x1, NPC_LID_LC, NPC_LT_LC_IP6, 0xf), + }, + }, + + .hash_mask = { + [NIX_INTF_RX] = { + [0] = { + GENMASK_ULL(63, 0), + GENMASK_ULL(63, 0), + }, + [1] = { + GENMASK_ULL(63, 0), + GENMASK_ULL(63, 0), + }, + }, + + [NIX_INTF_TX] = { + [0] = { + GENMASK_ULL(63, 0), + GENMASK_ULL(63, 0), + }, + [1] = { + GENMASK_ULL(63, 0), + GENMASK_ULL(63, 0), + }, + }, + }, + + .hash_ctrl = { + [NIX_INTF_RX] = { + [0] = GENMASK_ULL(63, 32), /* MSB 32 bit is mask and LSB 32 bit is offset. */ + [1] = GENMASK_ULL(63, 32), /* MSB 32 bit is mask and LSB 32 bit is offset. */ + }, + + [NIX_INTF_TX] = { + [0] = GENMASK_ULL(63, 32), /* MSB 32 bit is mask and LSB 32 bit is offset. */ + [1] = GENMASK_ULL(63, 32), /* MSB 32 bit is mask and LSB 32 bit is offset. */ + }, + }, +}; + +/* If exact match table support is enabled, enable drop rules */ +#define NPC_MCAM_DROP_RULE_MAX 30 +#define NPC_MCAM_SDP_DROP_RULE_IDX 0 + +#define RVU_PFFUNC(pf, func) \ + ((((pf) & RVU_PFVF_PF_MASK) << RVU_PFVF_PF_SHIFT) | \ + (((func) & RVU_PFVF_FUNC_MASK) << RVU_PFVF_FUNC_SHIFT)) + +enum npc_exact_opc_type { + NPC_EXACT_OPC_MEM, + NPC_EXACT_OPC_CAM, +}; + +struct npc_exact_table_entry { + struct list_head list; + struct list_head glist; + u32 seq_id; /* Sequence number of entry */ + u32 index; /* Mem table or cam table index */ + u32 mcam_idx; + /* Mcam index. This is valid only if "cmd" field is false */ + enum npc_exact_opc_type opc_type; + u16 chan; + u16 pcifunc; + u8 ways; + u8 mac[ETH_ALEN]; + u8 ctype; + u8 cgx_id; + u8 lmac_id; + bool cmd; /* Is added by ethtool command ? */ +}; + +struct npc_exact_table { + struct mutex lock; /* entries update lock */ + unsigned long *id_bmap; + int num_drop_rules; + u32 tot_ids; + u16 cnt_cmd_rules[NPC_MCAM_DROP_RULE_MAX]; + u16 counter_idx[NPC_MCAM_DROP_RULE_MAX]; + bool promisc_mode[NPC_MCAM_DROP_RULE_MAX]; + struct { + int ways; + int depth; + unsigned long *bmap; + u64 mask; // Masks before hash calculation. + u16 hash_mask; // 11 bits for hash mask + u16 hash_offset; // 11 bits offset + } mem_table; + + struct { + int depth; + unsigned long *bmap; + } cam_table; + + struct { + bool valid; + u16 chan_val; + u16 chan_mask; + u16 pcifunc; + u8 drop_rule_idx; + } drop_rule_map[NPC_MCAM_DROP_RULE_MAX]; + +#define NPC_EXACT_TBL_MAX_WAYS 4 + + struct list_head lhead_mem_tbl_entry[NPC_EXACT_TBL_MAX_WAYS]; + int mem_tbl_entry_cnt; + + struct list_head lhead_cam_tbl_entry; + int cam_tbl_entry_cnt; + + struct list_head lhead_gbl; +}; + +bool rvu_npc_exact_has_match_table(struct rvu *rvu); +u32 rvu_npc_exact_get_max_entries(struct rvu *rvu); +int rvu_npc_exact_init(struct rvu *rvu); +int rvu_npc_exact_mac_addr_reset(struct rvu *rvu, struct cgx_mac_addr_reset_req *req, + struct msg_rsp *rsp); + +int rvu_npc_exact_mac_addr_update(struct rvu *rvu, + struct cgx_mac_addr_update_req *req, + struct cgx_mac_addr_update_rsp *rsp); + +int rvu_npc_exact_mac_addr_add(struct rvu *rvu, + struct cgx_mac_addr_add_req *req, + struct cgx_mac_addr_add_rsp *rsp); + +int rvu_npc_exact_mac_addr_del(struct rvu *rvu, + struct cgx_mac_addr_del_req *req, + struct msg_rsp *rsp); + +int rvu_npc_exact_mac_addr_set(struct rvu *rvu, struct cgx_mac_addr_set_or_get *req, + struct cgx_mac_addr_set_or_get *rsp); + +void rvu_npc_exact_reset(struct rvu *rvu, u16 pcifunc); + +bool rvu_npc_exact_can_disable_feature(struct rvu *rvu); +void rvu_npc_exact_disable_feature(struct rvu *rvu); +void rvu_npc_exact_reset(struct rvu *rvu, u16 pcifunc); +u16 rvu_npc_exact_drop_rule_to_pcifunc(struct rvu *rvu, u32 drop_rule_idx); +int rvu_npc_exact_promisc_disable(struct rvu *rvu, u16 pcifunc); +int rvu_npc_exact_promisc_enable(struct rvu *rvu, u16 pcifunc); +#endif /* RVU_NPC_HASH_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h index 22cd751613cd..0e0d536645ac 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h @@ -266,6 +266,7 @@ #define NIX_AF_TX_NPC_CAPTURE_CONFIG (0x0660) #define NIX_AF_TX_NPC_CAPTURE_INFO (0x0670) #define NIX_AF_SEB_CFG (0x05F0) +#define NIX_PTP_1STEP_EN BIT_ULL(2) #define NIX_AF_DEBUG_NPC_RESP_DATAX(a) (0x680 | (a) << 3) #define NIX_AF_SMQX_CFG(a) (0x700 | (a) << 16) @@ -565,7 +566,13 @@ #define NPC_AF_PCK_DEF_OIP4 (0x00620) #define NPC_AF_PCK_DEF_OIP6 (0x00630) #define NPC_AF_PCK_DEF_IIP4 (0x00640) +#define NPC_AF_INTFX_HASHX_RESULT_CTRL(a, b) (0x006c0 | (a) << 4 | (b) << 3) +#define NPC_AF_INTFX_HASHX_MASKX(a, b, c) (0x00700 | (a) << 5 | (b) << 4 | (c) << 3) #define NPC_AF_KEX_LDATAX_FLAGS_CFG(a) (0x00800 | (a) << 3) +#define NPC_AF_INTFX_HASHX_CFG(a, b) (0x00b00 | (a) << 6 | (b) << 4) +#define NPC_AF_INTFX_SECRET_KEY0(a) (0x00e00 | (a) << 3) +#define NPC_AF_INTFX_SECRET_KEY1(a) (0x00e20 | (a) << 3) +#define NPC_AF_INTFX_SECRET_KEY2(a) (0x00e40 | (a) << 3) #define NPC_AF_INTFX_KEX_CFG(a) (0x01010 | (a) << 8) #define NPC_AF_PKINDX_ACTION0(a) (0x80000ull | (a) << 6) #define NPC_AF_PKINDX_ACTION1(a) (0x80008ull | (a) << 6) @@ -599,6 +606,15 @@ #define NPC_AF_DBG_DATAX(a) (0x3001400 | (a) << 4) #define NPC_AF_DBG_RESULTX(a) (0x3001800 | (a) << 4) +#define NPC_AF_EXACT_MEM_ENTRY(a, b) (0x300000 | (a) << 15 | (b) << 3) +#define NPC_AF_EXACT_CAM_ENTRY(a) (0xC00 | (a) << 3) +#define NPC_AF_INTFX_EXACT_MASK(a) (0x660 | (a) << 3) +#define NPC_AF_INTFX_EXACT_RESULT_CTL(a)(0x680 | (a) << 3) +#define NPC_AF_INTFX_EXACT_CFG(a) (0xA00 | (a) << 3) +#define NPC_AF_INTFX_EXACT_SECRET0(a) (0xE00 | (a) << 3) +#define NPC_AF_INTFX_EXACT_SECRET1(a) (0xE20 | (a) << 3) +#define NPC_AF_INTFX_EXACT_SECRET2(a) (0xE40 | (a) << 3) + #define NPC_AF_MCAMEX_BANKX_CAMX_INTF(a, b, c) ({ \ u64 offset; \ \ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile index 0048b5946712..73fdb8798614 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile @@ -11,4 +11,8 @@ rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \ otx2_devlink.o rvu_nicvf-y := otx2_vf.o otx2_devlink.o +rvu_nicpf-$(CONFIG_DCB) += otx2_dcbnl.o +rvu_nicvf-$(CONFIG_DCB) += otx2_dcbnl.o +rvu_nicpf-$(CONFIG_MACSEC) += cn10k_macsec.o + ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c index fd4f083c699e..826f691de259 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -86,8 +86,7 @@ int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ aq->sq.cq_ena = 1; aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); aq->sq.default_chan = pfvf->hw.tx_chan_base; aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c new file mode 100644 index 000000000000..9ec5f38d38a8 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -0,0 +1,1671 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MACSEC hardware offload driver + * + * Copyright (C) 2022 Marvell. + */ + +#include <linux/rtnetlink.h> +#include <linux/bitfield.h> +#include <net/macsec.h> +#include "otx2_common.h" + +#define MCS_TCAM0_MAC_SA_MASK GENMASK_ULL(63, 48) +#define MCS_TCAM1_MAC_SA_MASK GENMASK_ULL(31, 0) +#define MCS_TCAM1_ETYPE_MASK GENMASK_ULL(47, 32) + +#define MCS_SA_MAP_MEM_SA_USE BIT_ULL(9) + +#define MCS_RX_SECY_PLCY_RW_MASK GENMASK_ULL(49, 18) +#define MCS_RX_SECY_PLCY_RP BIT_ULL(17) +#define MCS_RX_SECY_PLCY_AUTH_ENA BIT_ULL(16) +#define MCS_RX_SECY_PLCY_CIP GENMASK_ULL(8, 5) +#define MCS_RX_SECY_PLCY_VAL GENMASK_ULL(2, 1) +#define MCS_RX_SECY_PLCY_ENA BIT_ULL(0) + +#define MCS_TX_SECY_PLCY_MTU GENMASK_ULL(43, 28) +#define MCS_TX_SECY_PLCY_ST_TCI GENMASK_ULL(27, 22) +#define MCS_TX_SECY_PLCY_ST_OFFSET GENMASK_ULL(21, 15) +#define MCS_TX_SECY_PLCY_INS_MODE BIT_ULL(14) +#define MCS_TX_SECY_PLCY_AUTH_ENA BIT_ULL(13) +#define MCS_TX_SECY_PLCY_CIP GENMASK_ULL(5, 2) +#define MCS_TX_SECY_PLCY_PROTECT BIT_ULL(1) +#define MCS_TX_SECY_PLCY_ENA BIT_ULL(0) + +#define MCS_GCM_AES_128 0 +#define MCS_GCM_AES_256 1 +#define MCS_GCM_AES_XPN_128 2 +#define MCS_GCM_AES_XPN_256 3 + +#define MCS_TCI_ES 0x40 /* end station */ +#define MCS_TCI_SC 0x20 /* SCI present */ +#define MCS_TCI_SCB 0x10 /* epon */ +#define MCS_TCI_E 0x08 /* encryption */ +#define MCS_TCI_C 0x04 /* changed text */ + +static struct cn10k_mcs_txsc *cn10k_mcs_get_txsc(struct cn10k_mcs_cfg *cfg, + struct macsec_secy *secy) +{ + struct cn10k_mcs_txsc *txsc; + + list_for_each_entry(txsc, &cfg->txsc_list, entry) { + if (txsc->sw_secy == secy) + return txsc; + } + + return NULL; +} + +static struct cn10k_mcs_rxsc *cn10k_mcs_get_rxsc(struct cn10k_mcs_cfg *cfg, + struct macsec_secy *secy, + struct macsec_rx_sc *rx_sc) +{ + struct cn10k_mcs_rxsc *rxsc; + + list_for_each_entry(rxsc, &cfg->rxsc_list, entry) { + if (rxsc->sw_rxsc == rx_sc && rxsc->sw_secy == secy) + return rxsc; + } + + return NULL; +} + +static const char *rsrc_name(enum mcs_rsrc_type rsrc_type) +{ + switch (rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + return "FLOW"; + case MCS_RSRC_TYPE_SC: + return "SC"; + case MCS_RSRC_TYPE_SECY: + return "SECY"; + case MCS_RSRC_TYPE_SA: + return "SA"; + default: + return "Unknown"; + }; + + return "Unknown"; +} + +static int cn10k_mcs_alloc_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, + enum mcs_rsrc_type type, u16 *rsrc_id) +{ + struct mbox *mbox = &pfvf->mbox; + struct mcs_alloc_rsrc_req *req; + struct mcs_alloc_rsrc_rsp *rsp; + int ret = -ENOMEM; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_alloc_resources(mbox); + if (!req) + goto fail; + + req->rsrc_type = type; + req->rsrc_cnt = 1; + req->dir = dir; + + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_alloc_rsrc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp) || req->rsrc_cnt != rsp->rsrc_cnt || + req->rsrc_type != rsp->rsrc_type || req->dir != rsp->dir) { + ret = -EINVAL; + goto fail; + } + + switch (rsp->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + *rsrc_id = rsp->flow_ids[0]; + break; + case MCS_RSRC_TYPE_SC: + *rsrc_id = rsp->sc_ids[0]; + break; + case MCS_RSRC_TYPE_SECY: + *rsrc_id = rsp->secy_ids[0]; + break; + case MCS_RSRC_TYPE_SA: + *rsrc_id = rsp->sa_ids[0]; + break; + default: + ret = -EINVAL; + goto fail; + } + + mutex_unlock(&mbox->lock); + + return 0; +fail: + dev_err(pfvf->dev, "Failed to allocate %s %s resource\n", + dir == MCS_TX ? "TX" : "RX", rsrc_name(type)); + mutex_unlock(&mbox->lock); + return ret; +} + +static void cn10k_mcs_free_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, + enum mcs_rsrc_type type, u16 hw_rsrc_id, + bool all) +{ + struct mbox *mbox = &pfvf->mbox; + struct mcs_free_rsrc_req *req; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_free_resources(mbox); + if (!req) + goto fail; + + req->rsrc_id = hw_rsrc_id; + req->rsrc_type = type; + req->dir = dir; + if (all) + req->all = 1; + + if (otx2_sync_mbox_msg(&pfvf->mbox)) + goto fail; + + mutex_unlock(&mbox->lock); + + return; +fail: + dev_err(pfvf->dev, "Failed to free %s %s resource\n", + dir == MCS_TX ? "TX" : "RX", rsrc_name(type)); + mutex_unlock(&mbox->lock); +} + +static int cn10k_mcs_alloc_txsa(struct otx2_nic *pfvf, u16 *hw_sa_id) +{ + return cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id); +} + +static int cn10k_mcs_alloc_rxsa(struct otx2_nic *pfvf, u16 *hw_sa_id) +{ + return cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id); +} + +static void cn10k_mcs_free_txsa(struct otx2_nic *pfvf, u16 hw_sa_id) +{ + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id, false); +} + +static void cn10k_mcs_free_rxsa(struct otx2_nic *pfvf, u16 hw_sa_id) +{ + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id, false); +} + +static int cn10k_mcs_write_rx_secy(struct otx2_nic *pfvf, + struct macsec_secy *secy, u8 hw_secy_id) +{ + struct mcs_secy_plcy_write_req *req; + struct mbox *mbox = &pfvf->mbox; + u64 policy; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + policy = FIELD_PREP(MCS_RX_SECY_PLCY_RW_MASK, secy->replay_window); + if (secy->replay_protect) + policy |= MCS_RX_SECY_PLCY_RP; + + policy |= MCS_RX_SECY_PLCY_AUTH_ENA; + policy |= FIELD_PREP(MCS_RX_SECY_PLCY_CIP, MCS_GCM_AES_128); + policy |= FIELD_PREP(MCS_RX_SECY_PLCY_VAL, secy->validate_frames); + + policy |= MCS_RX_SECY_PLCY_ENA; + + req->plcy = policy; + req->secy_id = hw_secy_id; + req->dir = MCS_RX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_flowid(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id) +{ + struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc; + struct mcs_flowid_entry_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->data[1] = FIELD_PREP(MCS_TCAM1_ETYPE_MASK, ETH_P_MACSEC); + req->mask[1] = ~0ULL; + req->mask[1] &= ~MCS_TCAM1_ETYPE_MASK; + + req->mask[0] = ~0ULL; + req->mask[2] = ~0ULL; + req->mask[3] = ~0ULL; + + req->flow_id = rxsc->hw_flow_id; + req->secy_id = hw_secy_id; + req->sc_id = rxsc->hw_sc_id; + req->dir = MCS_RX; + + if (sw_rx_sc->active) + req->ena = 1; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_sc_cam(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id) +{ + struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc; + struct mcs_rx_sc_cam_write_req *sc_req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + sc_req = otx2_mbox_alloc_msg_mcs_rx_sc_cam_write(mbox); + if (!sc_req) { + ret = -ENOMEM; + goto fail; + } + + sc_req->sci = (__force u64)cpu_to_be64((__force u64)sw_rx_sc->sci); + sc_req->sc_id = rxsc->hw_sc_id; + sc_req->secy_id = hw_secy_id; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_sa_plcy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_rxsc *rxsc, + u8 assoc_num, bool sa_in_use) +{ + unsigned char *src = rxsc->sa_key[assoc_num]; + struct mcs_sa_plcy_write_req *plcy_req; + struct mcs_rx_sc_sa_map *map_req; + struct mbox *mbox = &pfvf->mbox; + u8 reg, key_len; + int ret; + + mutex_lock(&mbox->lock); + + plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox); + if (!plcy_req) { + ret = -ENOMEM; + goto fail; + } + + map_req = otx2_mbox_alloc_msg_mcs_rx_sc_sa_map_write(mbox); + if (!map_req) { + otx2_mbox_reset(&mbox->mbox, 0); + ret = -ENOMEM; + goto fail; + } + + for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) { + memcpy((u8 *)&plcy_req->plcy[0][reg], + (src + reg * 8), 8); + reg++; + } + + plcy_req->sa_index[0] = rxsc->hw_sa_id[assoc_num]; + plcy_req->sa_cnt = 1; + plcy_req->dir = MCS_RX; + + map_req->sa_index = rxsc->hw_sa_id[assoc_num]; + map_req->sa_in_use = sa_in_use; + map_req->sc_id = rxsc->hw_sc_id; + map_req->an = assoc_num; + + /* Send two messages together */ + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_sa_pn(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, + u8 assoc_num, u64 next_pn) +{ + struct mcs_pn_table_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->pn_id = rxsc->hw_sa_id[assoc_num]; + req->next_pn = next_pn; + req->dir = MCS_RX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_secy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct mcs_secy_plcy_write_req *req; + struct mbox *mbox = &pfvf->mbox; + struct macsec_tx_sc *sw_tx_sc; + /* Insert SecTag after 12 bytes (DA+SA)*/ + u8 tag_offset = 12; + u8 sectag_tci = 0; + u64 policy; + int ret; + + sw_tx_sc = &secy->tx_sc; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + if (sw_tx_sc->send_sci) { + sectag_tci |= MCS_TCI_SC; + } else { + if (sw_tx_sc->end_station) + sectag_tci |= MCS_TCI_ES; + if (sw_tx_sc->scb) + sectag_tci |= MCS_TCI_SCB; + } + + if (sw_tx_sc->encrypt) + sectag_tci |= (MCS_TCI_E | MCS_TCI_C); + + policy = FIELD_PREP(MCS_TX_SECY_PLCY_MTU, secy->netdev->mtu); + /* Write SecTag excluding AN bits(1..0) */ + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_TCI, sectag_tci >> 2); + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_OFFSET, tag_offset); + policy |= MCS_TX_SECY_PLCY_INS_MODE; + policy |= MCS_TX_SECY_PLCY_AUTH_ENA; + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_CIP, MCS_GCM_AES_128); + + if (secy->protect_frames) + policy |= MCS_TX_SECY_PLCY_PROTECT; + + /* If the encodingsa does not exist/active and protect is + * not set then frames can be sent out as it is. Hence enable + * the policy irrespective of secy operational when !protect. + */ + if (!secy->protect_frames || secy->operational) + policy |= MCS_TX_SECY_PLCY_ENA; + + req->plcy = policy; + req->secy_id = txsc->hw_secy_id_tx; + req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_flowid(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct mcs_flowid_entry_write_req *req; + struct mbox *mbox = &pfvf->mbox; + u64 mac_sa; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + mac_sa = ether_addr_to_u64(secy->netdev->dev_addr); + + req->data[0] = FIELD_PREP(MCS_TCAM0_MAC_SA_MASK, mac_sa); + req->data[1] = FIELD_PREP(MCS_TCAM1_MAC_SA_MASK, mac_sa >> 16); + + req->mask[0] = ~0ULL; + req->mask[0] &= ~MCS_TCAM0_MAC_SA_MASK; + + req->mask[1] = ~0ULL; + req->mask[1] &= ~MCS_TCAM1_MAC_SA_MASK; + + req->mask[2] = ~0ULL; + req->mask[3] = ~0ULL; + + req->flow_id = txsc->hw_flow_id; + req->secy_id = txsc->hw_secy_id_tx; + req->sc_id = txsc->hw_sc_id; + req->sci = (__force u64)cpu_to_be64((__force u64)secy->sci); + req->dir = MCS_TX; + /* This can be enabled since stack xmits packets only when interface is up */ + req->ena = 1; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_link_tx_sa2sc(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + u8 sa_num, bool sa_active) +{ + struct mcs_tx_sc_sa_map *map_req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + /* Link the encoding_sa only to SC out of all SAs */ + if (txsc->encoding_sa != sa_num) + return 0; + + mutex_lock(&mbox->lock); + + map_req = otx2_mbox_alloc_msg_mcs_tx_sc_sa_map_write(mbox); + if (!map_req) { + otx2_mbox_reset(&mbox->mbox, 0); + ret = -ENOMEM; + goto fail; + } + + map_req->sa_index0 = txsc->hw_sa_id[sa_num]; + map_req->sa_index0_vld = sa_active; + map_req->sectag_sci = (__force u64)cpu_to_be64((__force u64)secy->sci); + map_req->sc_id = txsc->hw_sc_id; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_sa_plcy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + u8 assoc_num) +{ + unsigned char *src = txsc->sa_key[assoc_num]; + struct mcs_sa_plcy_write_req *plcy_req; + struct mbox *mbox = &pfvf->mbox; + u8 reg, key_len; + int ret; + + mutex_lock(&mbox->lock); + + plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox); + if (!plcy_req) { + ret = -ENOMEM; + goto fail; + } + + for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) { + memcpy((u8 *)&plcy_req->plcy[0][reg], (src + reg * 8), 8); + reg++; + } + + plcy_req->plcy[0][8] = assoc_num; + plcy_req->sa_index[0] = txsc->hw_sa_id[assoc_num]; + plcy_req->sa_cnt = 1; + plcy_req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_write_tx_sa_pn(struct otx2_nic *pfvf, + struct cn10k_mcs_txsc *txsc, + u8 assoc_num, u64 next_pn) +{ + struct mcs_pn_table_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->pn_id = txsc->hw_sa_id[assoc_num]; + req->next_pn = next_pn; + req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_ena_dis_flowid(struct otx2_nic *pfvf, u16 hw_flow_id, + bool enable, enum mcs_direction dir) +{ + struct mcs_flowid_ena_dis_entry *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_ena_entry(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->flow_id = hw_flow_id; + req->ena = enable; + req->dir = dir; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_sa_stats(struct otx2_nic *pfvf, u8 hw_sa_id, + struct mcs_sa_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_stats_req *req; + struct mcs_sa_stats *rsp; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_sa_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_sa_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_sa_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SA; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_sa_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_sc_stats(struct otx2_nic *pfvf, u8 hw_sc_id, + struct mcs_sc_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_stats_req *req; + struct mcs_sc_stats *rsp; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_sc_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_sc_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_sc_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SC; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_sc_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_secy_stats(struct otx2_nic *pfvf, u8 hw_secy_id, + struct mcs_secy_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_secy_stats *rsp; + struct mcs_stats_req *req; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_secy_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_secy_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_secy_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SECY; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_secy_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static struct cn10k_mcs_txsc *cn10k_mcs_create_txsc(struct otx2_nic *pfvf) +{ + struct cn10k_mcs_txsc *txsc; + int ret; + + txsc = kzalloc(sizeof(*txsc), GFP_KERNEL); + if (!txsc) + return ERR_PTR(-ENOMEM); + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + &txsc->hw_flow_id); + if (ret) + goto fail; + + /* For a SecY, one TX secy and one RX secy HW resources are needed */ + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + &txsc->hw_secy_id_tx); + if (ret) + goto free_flowid; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + &txsc->hw_secy_id_rx); + if (ret) + goto free_tx_secy; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC, + &txsc->hw_sc_id); + if (ret) + goto free_rx_secy; + + return txsc; +free_rx_secy: + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_rx, false); +free_tx_secy: + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_tx, false); +free_flowid: + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + txsc->hw_flow_id, false); +fail: + kfree(txsc); + return ERR_PTR(ret); +} + +/* Free Tx SC and its SAs(if any) resources to AF + */ +static void cn10k_mcs_delete_txsc(struct otx2_nic *pfvf, + struct cn10k_mcs_txsc *txsc) +{ + u8 sa_bmap = txsc->sa_bmap; + u8 sa_num = 0; + + while (sa_bmap) { + if (sa_bmap & 1) { + cn10k_mcs_write_tx_sa_plcy(pfvf, txsc->sw_secy, + txsc, sa_num); + cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]); + } + sa_num++; + sa_bmap >>= 1; + } + + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC, + txsc->hw_sc_id, false); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_rx, false); + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_tx, false); + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + txsc->hw_flow_id, false); +} + +static struct cn10k_mcs_rxsc *cn10k_mcs_create_rxsc(struct otx2_nic *pfvf) +{ + struct cn10k_mcs_rxsc *rxsc; + int ret; + + rxsc = kzalloc(sizeof(*rxsc), GFP_KERNEL); + if (!rxsc) + return ERR_PTR(-ENOMEM); + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + &rxsc->hw_flow_id); + if (ret) + goto fail; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC, + &rxsc->hw_sc_id); + if (ret) + goto free_flowid; + + return rxsc; +free_flowid: + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + rxsc->hw_flow_id, false); +fail: + kfree(rxsc); + return ERR_PTR(ret); +} + +/* Free Rx SC and its SAs(if any) resources to AF + */ +static void cn10k_mcs_delete_rxsc(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc) +{ + u8 sa_bmap = rxsc->sa_bmap; + u8 sa_num = 0; + + while (sa_bmap) { + if (sa_bmap & 1) { + cn10k_mcs_write_rx_sa_plcy(pfvf, rxsc->sw_secy, rxsc, + sa_num, false); + cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]); + } + sa_num++; + sa_bmap >>= 1; + } + + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC, + rxsc->hw_sc_id, false); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + rxsc->hw_flow_id, false); +} + +static int cn10k_mcs_secy_tx_cfg(struct otx2_nic *pfvf, struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + struct macsec_tx_sa *sw_tx_sa, u8 sa_num) +{ + if (sw_tx_sa) { + cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num); + cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, sa_num, + sw_tx_sa->active); + } + + cn10k_mcs_write_tx_secy(pfvf, secy, txsc); + cn10k_mcs_write_tx_flowid(pfvf, secy, txsc); + /* When updating secy, change RX secy also */ + cn10k_mcs_write_rx_secy(pfvf, secy, txsc->hw_secy_id_rx); + + return 0; +} + +static int cn10k_mcs_secy_rx_cfg(struct otx2_nic *pfvf, + struct macsec_secy *secy, u8 hw_secy_id) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *mcs_rx_sc; + struct macsec_rx_sc *sw_rx_sc; + struct macsec_rx_sa *sw_rx_sa; + u8 sa_num; + + for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active; + sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) { + mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (unlikely(!mcs_rx_sc)) + continue; + + for (sa_num = 0; sa_num < CN10K_MCS_SA_PER_SC; sa_num++) { + sw_rx_sa = rcu_dereference_bh(sw_rx_sc->sa[sa_num]); + if (!sw_rx_sa) + continue; + + cn10k_mcs_write_rx_sa_plcy(pfvf, secy, mcs_rx_sc, + sa_num, sw_rx_sa->active); + cn10k_mcs_write_rx_sa_pn(pfvf, mcs_rx_sc, sa_num, + sw_rx_sa->next_pn_halves.lower); + } + + cn10k_mcs_write_rx_flowid(pfvf, mcs_rx_sc, hw_secy_id); + cn10k_mcs_write_sc_cam(pfvf, mcs_rx_sc, hw_secy_id); + } + + return 0; +} + +static int cn10k_mcs_disable_rxscs(struct otx2_nic *pfvf, + struct macsec_secy *secy, + bool delete) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *mcs_rx_sc; + struct macsec_rx_sc *sw_rx_sc; + int ret; + + for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active; + sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) { + mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (unlikely(!mcs_rx_sc)) + continue; + + ret = cn10k_mcs_ena_dis_flowid(pfvf, mcs_rx_sc->hw_flow_id, + false, MCS_RX); + if (ret) + dev_err(pfvf->dev, "Failed to disable TCAM for SC %d\n", + mcs_rx_sc->hw_sc_id); + if (delete) { + cn10k_mcs_delete_rxsc(pfvf, mcs_rx_sc); + list_del(&mcs_rx_sc->entry); + kfree(mcs_rx_sc); + } + } + + return 0; +} + +static void cn10k_mcs_sync_stats(struct otx2_nic *pfvf, struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_secy_stats rx_rsp = { 0 }; + struct mcs_sc_stats sc_rsp = { 0 }; + struct cn10k_mcs_rxsc *rxsc; + + /* Because of shared counters for some stats in the hardware, when + * updating secy policy take a snapshot of current stats and reset them. + * Below are the effected stats because of shared counters. + */ + + /* Check if sync is really needed */ + if (secy->validate_frames == txsc->last_validate_frames && + secy->protect_frames == txsc->last_protect_frames) + return; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true); + + txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt; + txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt; + txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt; + if (txsc->last_validate_frames == MACSEC_VALIDATE_STRICT) + txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt; + else + txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt; + + list_for_each_entry(rxsc, &cfg->rxsc_list, entry) { + cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &sc_rsp, MCS_RX, true); + + rxsc->stats.InOctetsValidated += sc_rsp.octet_validate_cnt; + rxsc->stats.InOctetsDecrypted += sc_rsp.octet_decrypt_cnt; + + rxsc->stats.InPktsInvalid += sc_rsp.pkt_invalid_cnt; + rxsc->stats.InPktsNotValid += sc_rsp.pkt_notvalid_cnt; + + if (txsc->last_protect_frames) + rxsc->stats.InPktsLate += sc_rsp.pkt_late_cnt; + else + rxsc->stats.InPktsDelayed += sc_rsp.pkt_late_cnt; + + if (txsc->last_validate_frames == MACSEC_VALIDATE_CHECK) + rxsc->stats.InPktsUnchecked += sc_rsp.pkt_unchecked_cnt; + else + rxsc->stats.InPktsOK += sc_rsp.pkt_unchecked_cnt; + } + + txsc->last_validate_frames = secy->validate_frames; + txsc->last_protect_frames = secy->protect_frames; +} + +static int cn10k_mdo_open(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct macsec_tx_sa *sw_tx_sa; + struct cn10k_mcs_txsc *txsc; + u8 sa_num; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + sa_num = txsc->encoding_sa; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]); + + err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num); + if (err) + return err; + + return cn10k_mcs_secy_rx_cfg(pfvf, secy, txsc->hw_secy_id_rx); +} + +static int cn10k_mdo_stop(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + err = cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX); + if (err) + return err; + + return cn10k_mcs_disable_rxscs(pfvf, ctx->secy, false); +} + +static int cn10k_mdo_add_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_txsc *txsc; + + if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) + return -EOPNOTSUPP; + + /* Stick to 16 bytes key len until XPN support is added */ + if (secy->key_len != 16) + return -EOPNOTSUPP; + + if (secy->xpn) + return -EOPNOTSUPP; + + txsc = cn10k_mcs_create_txsc(pfvf); + if (IS_ERR(txsc)) + return -ENOSPC; + + txsc->sw_secy = secy; + txsc->encoding_sa = secy->tx_sc.encoding_sa; + txsc->last_validate_frames = secy->validate_frames; + txsc->last_protect_frames = secy->protect_frames; + + list_add(&txsc->entry, &cfg->txsc_list); + + if (netif_running(secy->netdev)) + return cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, NULL, 0); + + return 0; +} + +static int cn10k_mdo_upd_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct macsec_tx_sa *sw_tx_sa; + struct cn10k_mcs_txsc *txsc; + u8 sa_num; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + txsc->encoding_sa = secy->tx_sc.encoding_sa; + + sa_num = txsc->encoding_sa; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]); + + if (netif_running(secy->netdev)) { + cn10k_mcs_sync_stats(pfvf, secy, txsc); + + err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX); + cn10k_mcs_disable_rxscs(pfvf, ctx->secy, true); + cn10k_mcs_delete_txsc(pfvf, txsc); + list_del(&txsc->entry); + kfree(txsc); + + return 0; +} + +static int cn10k_mdo_add_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa; + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (cn10k_mcs_alloc_txsa(pfvf, &txsc->hw_sa_id[sa_num])) + return -ENOSPC; + + memcpy(&txsc->sa_key[sa_num], ctx->sa.key, secy->key_len); + txsc->sa_bmap |= 1 << sa_num; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num); + if (err) + return err; + + err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + if (err) + return err; + + err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, + sa_num, sw_tx_sa->active); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa; + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (netif_running(secy->netdev)) { + /* Keys cannot be changed after creation */ + err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + if (err) + return err; + + err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, + sa_num, sw_tx_sa->active); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]); + txsc->sa_bmap &= ~(1 << sa_num); + + return 0; +} + +static int cn10k_mdo_add_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_rxsc *rxsc; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + rxsc = cn10k_mcs_create_rxsc(pfvf); + if (IS_ERR(rxsc)) + return -ENOSPC; + + rxsc->sw_secy = ctx->secy; + rxsc->sw_rxsc = ctx->rx_sc; + list_add(&rxsc->entry, &cfg->rxsc_list); + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_flowid(pfvf, rxsc, txsc->hw_secy_id_rx); + if (err) + return err; + + err = cn10k_mcs_write_sc_cam(pfvf, rxsc, txsc->hw_secy_id_rx); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + bool enable = ctx->rx_sc->active; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + if (netif_running(secy->netdev)) + return cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, + enable, MCS_RX); + + return 0; +} + +static int cn10k_mdo_del_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, false, MCS_RX); + cn10k_mcs_delete_rxsc(pfvf, rxsc); + list_del(&rxsc->entry); + kfree(rxsc); + + return 0; +} + +static int cn10k_mdo_add_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa; + u64 next_pn = rx_sa->next_pn_halves.lower; + struct macsec_secy *secy = ctx->secy; + bool sa_in_use = rx_sa->active; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + int err; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (cn10k_mcs_alloc_rxsa(pfvf, &rxsc->hw_sa_id[sa_num])) + return -ENOSPC; + + memcpy(&rxsc->sa_key[sa_num], ctx->sa.key, ctx->secy->key_len); + rxsc->sa_bmap |= 1 << sa_num; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, + sa_num, sa_in_use); + if (err) + return err; + + err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa; + u64 next_pn = rx_sa->next_pn_halves.lower; + struct macsec_secy *secy = ctx->secy; + bool sa_in_use = rx_sa->active; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + int err; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, sa_num, sa_in_use); + if (err) + return err; + + err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_write_rx_sa_plcy(pfvf, ctx->secy, rxsc, sa_num, false); + cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]); + + rxsc->sa_bmap &= ~(1 << sa_num); + + return 0; +} + +static int cn10k_mdo_get_dev_stats(struct macsec_context *ctx) +{ + struct mcs_secy_stats tx_rsp = { 0 }, rx_rsp = { 0 }; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_tx, &tx_rsp, MCS_TX, false); + ctx->stats.dev_stats->OutPktsUntagged = tx_rsp.pkt_untagged_cnt; + ctx->stats.dev_stats->OutPktsTooLong = tx_rsp.pkt_toolong_cnt; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true); + txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt; + txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt; + txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt; + if (secy->validate_frames == MACSEC_VALIDATE_STRICT) + txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt; + else + txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt; + txsc->stats.InPktsOverrun = 0; + + ctx->stats.dev_stats->InPktsNoTag = txsc->stats.InPktsNoTag; + ctx->stats.dev_stats->InPktsUntagged = txsc->stats.InPktsUntagged; + ctx->stats.dev_stats->InPktsBadTag = txsc->stats.InPktsBadTag; + ctx->stats.dev_stats->InPktsUnknownSCI = txsc->stats.InPktsUnknownSCI; + ctx->stats.dev_stats->InPktsNoSCI = txsc->stats.InPktsNoSCI; + ctx->stats.dev_stats->InPktsOverrun = txsc->stats.InPktsOverrun; + + return 0; +} + +static int cn10k_mdo_get_tx_sc_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sc_stats rsp = { 0 }; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_sc_stats(pfvf, txsc->hw_sc_id, &rsp, MCS_TX, false); + + ctx->stats.tx_sc_stats->OutPktsProtected = rsp.pkt_protected_cnt; + ctx->stats.tx_sc_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt; + ctx->stats.tx_sc_stats->OutOctetsProtected = rsp.octet_protected_cnt; + ctx->stats.tx_sc_stats->OutOctetsEncrypted = rsp.octet_encrypt_cnt; + + return 0; +} + +static int cn10k_mdo_get_tx_sa_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sa_stats rsp = { 0 }; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_sa_stats(pfvf, txsc->hw_sa_id[sa_num], &rsp, MCS_TX, false); + + ctx->stats.tx_sa_stats->OutPktsProtected = rsp.pkt_protected_cnt; + ctx->stats.tx_sa_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt; + + return 0; +} + +static int cn10k_mdo_get_rx_sc_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct mcs_sc_stats rsp = { 0 }; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &rsp, MCS_RX, true); + + rxsc->stats.InOctetsValidated += rsp.octet_validate_cnt; + rxsc->stats.InOctetsDecrypted += rsp.octet_decrypt_cnt; + + rxsc->stats.InPktsInvalid += rsp.pkt_invalid_cnt; + rxsc->stats.InPktsNotValid += rsp.pkt_notvalid_cnt; + + if (secy->protect_frames) + rxsc->stats.InPktsLate += rsp.pkt_late_cnt; + else + rxsc->stats.InPktsDelayed += rsp.pkt_late_cnt; + + if (secy->validate_frames == MACSEC_VALIDATE_CHECK) + rxsc->stats.InPktsUnchecked += rsp.pkt_unchecked_cnt; + else + rxsc->stats.InPktsOK += rsp.pkt_unchecked_cnt; + + ctx->stats.rx_sc_stats->InOctetsValidated = rxsc->stats.InOctetsValidated; + ctx->stats.rx_sc_stats->InOctetsDecrypted = rxsc->stats.InOctetsDecrypted; + ctx->stats.rx_sc_stats->InPktsInvalid = rxsc->stats.InPktsInvalid; + ctx->stats.rx_sc_stats->InPktsNotValid = rxsc->stats.InPktsNotValid; + ctx->stats.rx_sc_stats->InPktsLate = rxsc->stats.InPktsLate; + ctx->stats.rx_sc_stats->InPktsDelayed = rxsc->stats.InPktsDelayed; + ctx->stats.rx_sc_stats->InPktsUnchecked = rxsc->stats.InPktsUnchecked; + ctx->stats.rx_sc_stats->InPktsOK = rxsc->stats.InPktsOK; + + return 0; +} + +static int cn10k_mdo_get_rx_sa_stats(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sa_stats rsp = { 0 }; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_sa_stats(pfvf, rxsc->hw_sa_id[sa_num], &rsp, MCS_RX, false); + + ctx->stats.rx_sa_stats->InPktsOK = rsp.pkt_ok_cnt; + ctx->stats.rx_sa_stats->InPktsInvalid = rsp.pkt_invalid_cnt; + ctx->stats.rx_sa_stats->InPktsNotValid = rsp.pkt_notvalid_cnt; + ctx->stats.rx_sa_stats->InPktsNotUsingSA = rsp.pkt_nosaerror_cnt; + ctx->stats.rx_sa_stats->InPktsUnusedSA = rsp.pkt_nosa_cnt; + + return 0; +} + +static const struct macsec_ops cn10k_mcs_ops = { + .mdo_dev_open = cn10k_mdo_open, + .mdo_dev_stop = cn10k_mdo_stop, + .mdo_add_secy = cn10k_mdo_add_secy, + .mdo_upd_secy = cn10k_mdo_upd_secy, + .mdo_del_secy = cn10k_mdo_del_secy, + .mdo_add_rxsc = cn10k_mdo_add_rxsc, + .mdo_upd_rxsc = cn10k_mdo_upd_rxsc, + .mdo_del_rxsc = cn10k_mdo_del_rxsc, + .mdo_add_rxsa = cn10k_mdo_add_rxsa, + .mdo_upd_rxsa = cn10k_mdo_upd_rxsa, + .mdo_del_rxsa = cn10k_mdo_del_rxsa, + .mdo_add_txsa = cn10k_mdo_add_txsa, + .mdo_upd_txsa = cn10k_mdo_upd_txsa, + .mdo_del_txsa = cn10k_mdo_del_txsa, + .mdo_get_dev_stats = cn10k_mdo_get_dev_stats, + .mdo_get_tx_sc_stats = cn10k_mdo_get_tx_sc_stats, + .mdo_get_tx_sa_stats = cn10k_mdo_get_tx_sa_stats, + .mdo_get_rx_sc_stats = cn10k_mdo_get_rx_sc_stats, + .mdo_get_rx_sa_stats = cn10k_mdo_get_rx_sa_stats, +}; + +void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_tx_sa *sw_tx_sa = NULL; + struct macsec_secy *secy = NULL; + struct cn10k_mcs_txsc *txsc; + u8 an; + + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return; + + if (!(event->intr_mask & MCS_CPM_TX_PACKET_XPN_EQ0_INT)) + return; + + /* Find the SecY to which the expired hardware SA is mapped */ + list_for_each_entry(txsc, &cfg->txsc_list, entry) { + for (an = 0; an < CN10K_MCS_SA_PER_SC; an++) + if (txsc->hw_sa_id[an] == event->sa_id) { + secy = txsc->sw_secy; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[an]); + } + } + + if (secy && sw_tx_sa) + macsec_pn_wrapped(secy, sw_tx_sa); +} + +int cn10k_mcs_init(struct otx2_nic *pfvf) +{ + struct mbox *mbox = &pfvf->mbox; + struct cn10k_mcs_cfg *cfg; + struct mcs_intr_cfg *req; + + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return 0; + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return -ENOMEM; + + INIT_LIST_HEAD(&cfg->txsc_list); + INIT_LIST_HEAD(&cfg->rxsc_list); + pfvf->macsec_cfg = cfg; + + pfvf->netdev->features |= NETIF_F_HW_MACSEC; + pfvf->netdev->macsec_ops = &cn10k_mcs_ops; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_intr_cfg(mbox); + if (!req) + goto fail; + + req->intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + if (otx2_sync_mbox_msg(mbox)) + goto fail; + + mutex_unlock(&mbox->lock); + + return 0; +fail: + dev_err(pfvf->dev, "Cannot notify PN wrapped event\n"); + mutex_unlock(&mbox->lock); + return 0; +} + +void cn10k_mcs_free(struct otx2_nic *pfvf) +{ + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return; + + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, 0, true); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, 0, true); + kfree(pfvf->macsec_cfg); + pfvf->macsec_cfg = NULL; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 66da31f30d3e..9e10e7471b88 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -97,11 +97,6 @@ void otx2_get_dev_stats(struct otx2_nic *pfvf) { struct otx2_dev_stats *dev_stats = &pfvf->hw.dev_stats; -#define OTX2_GET_RX_STATS(reg) \ - otx2_read64(pfvf, NIX_LF_RX_STATX(reg)) -#define OTX2_GET_TX_STATS(reg) \ - otx2_read64(pfvf, NIX_LF_TX_STATX(reg)) - dev_stats->rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); dev_stats->rx_drops = OTX2_GET_RX_STATS(RX_DROP); dev_stats->rx_bcast_frames = OTX2_GET_RX_STATS(RX_BCAST); @@ -222,8 +217,11 @@ EXPORT_SYMBOL(otx2_set_mac_address); int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu) { struct nix_frs_cfg *req; + u16 maxlen; int err; + maxlen = otx2_get_max_mtu(pfvf) + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; + mutex_lock(&pfvf->mbox.lock); req = otx2_mbox_alloc_msg_nix_set_hw_frs(&pfvf->mbox); if (!req) { @@ -233,6 +231,10 @@ int otx2_hw_set_mtu(struct otx2_nic *pfvf, int mtu) req->maxlen = pfvf->netdev->mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN; + /* Use max receive length supported by hardware for loopback devices */ + if (is_otx2_lbkvf(pfvf->pdev)) + req->maxlen = maxlen; + err = otx2_sync_mbox_msg(&pfvf->mbox); mutex_unlock(&pfvf->mbox.lock); return err; @@ -262,6 +264,7 @@ unlock: mutex_unlock(&pfvf->mbox.lock); return err; } +EXPORT_SYMBOL(otx2_config_pause_frm); int otx2_set_flowkey_cfg(struct otx2_nic *pfvf) { @@ -583,8 +586,9 @@ void otx2_get_mac_from_af(struct net_device *netdev) } EXPORT_SYMBOL(otx2_get_mac_from_af); -int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) +int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc) { + u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC]; struct otx2_hw *hw = &pfvf->hw; struct nix_txschq_config *req; u64 schq, parent; @@ -599,7 +603,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->lvl = lvl; req->num_regs = 1; - schq = hw->txschq_list[lvl][0]; + schq_list = hw->txschq_list; +#ifdef CONFIG_DCB + if (txschq_for_pfc) + schq_list = pfvf->pfc_schq_list; +#endif + + schq = schq_list[lvl][prio]; /* Set topology e.t.c configuration */ if (lvl == NIX_TXSCH_LVL_SMQ) { req->reg[0] = NIX_AF_SMQX_CFG(schq); @@ -608,7 +618,7 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) (0x2ULL << 36); req->num_regs++; /* MDQ config */ - parent = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; + parent = schq_list[NIX_TXSCH_LVL_TL4][prio]; req->reg[1] = NIX_AF_MDQX_PARENT(schq); req->regval[1] = parent << 16; req->num_regs++; @@ -616,21 +626,29 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); req->regval[2] = dwrr_val; } else if (lvl == NIX_TXSCH_LVL_TL4) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL3][0]; + parent = schq_list[NIX_TXSCH_LVL_TL3][prio]; req->reg[0] = NIX_AF_TL4X_PARENT(schq); req->regval[0] = parent << 16; req->num_regs++; req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); req->regval[1] = dwrr_val; } else if (lvl == NIX_TXSCH_LVL_TL3) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL2][0]; + parent = schq_list[NIX_TXSCH_LVL_TL2][prio]; req->reg[0] = NIX_AF_TL3X_PARENT(schq); req->regval[0] = parent << 16; req->num_regs++; req->reg[1] = NIX_AF_TL3X_SCHEDULE(schq); req->regval[1] = dwrr_val; + if (lvl == hw->txschq_link_cfg_lvl) { + req->num_regs++; + req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); + /* Enable this queue and backpressure + * and set relative channel + */ + req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; + } } else if (lvl == NIX_TXSCH_LVL_TL2) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL1][0]; + parent = schq_list[NIX_TXSCH_LVL_TL1][prio]; req->reg[0] = NIX_AF_TL2X_PARENT(schq); req->regval[0] = parent << 16; @@ -638,11 +656,14 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->reg[1] = NIX_AF_TL2X_SCHEDULE(schq); req->regval[1] = TXSCH_TL1_DFLT_RR_PRIO << 24 | dwrr_val; - req->num_regs++; - req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); - /* Enable this queue and backpressure */ - req->regval[2] = BIT_ULL(13) | BIT_ULL(12); - + if (lvl == hw->txschq_link_cfg_lvl) { + req->num_regs++; + req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); + /* Enable this queue and backpressure + * and set relative channel + */ + req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; + } } else if (lvl == NIX_TXSCH_LVL_TL1) { /* Default config for TL1. * For VF this is always ignored. @@ -666,6 +687,31 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) return otx2_sync_mbox_msg(&pfvf->mbox); } +EXPORT_SYMBOL(otx2_txschq_config); + +int otx2_smq_flush(struct otx2_nic *pfvf, int smq) +{ + struct nix_txschq_config *req; + int rc; + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + req->lvl = NIX_TXSCH_LVL_SMQ; + req->reg[0] = NIX_AF_SMQX_CFG(smq); + req->regval[0] |= BIT_ULL(49); + req->num_regs++; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + mutex_unlock(&pfvf->mbox.lock); + return rc; +} +EXPORT_SYMBOL(otx2_smq_flush); int otx2_txsch_alloc(struct otx2_nic *pfvf) { @@ -796,8 +842,7 @@ int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ aq->sq.cq_ena = 1; aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); aq->sq.default_chan = pfvf->hw.tx_chan_base; aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ @@ -853,6 +898,7 @@ static int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) } sq->head = 0; + sq->cons_head = 0; sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; /* Set SQE threshold to 10% of total SQEs */ @@ -931,7 +977,11 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) if (!is_otx2_lbkvf(pfvf->pdev)) { /* Enable receive CQ backpressure */ aq->cq.bp_ena = 1; +#ifdef CONFIG_DCB + aq->cq.bpid = pfvf->bpid[pfvf->queue_to_pfc_map[qidx]]; +#else aq->cq.bpid = pfvf->bpid[0]; +#endif /* Set backpressure level is same as cq pass level */ aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); @@ -1036,7 +1086,7 @@ int otx2_config_nix(struct otx2_nic *pfvf) struct nix_lf_alloc_rsp *rsp; int err; - pfvf->qset.xqe_size = NIX_XQESZ_W16 ? 128 : 512; + pfvf->qset.xqe_size = pfvf->hw.xqe_size; /* Get memory to put this msg */ nixlf = otx2_mbox_alloc_msg_nix_lf_alloc(&pfvf->mbox); @@ -1049,7 +1099,7 @@ int otx2_config_nix(struct otx2_nic *pfvf) nixlf->cq_cnt = pfvf->qset.cq_cnt; nixlf->rss_sz = MAX_RSS_INDIR_TBL_SIZE; nixlf->rss_grps = MAX_RSS_GROUPS; - nixlf->xqe_sz = NIX_XQESZ_W16; + nixlf->xqe_sz = pfvf->hw.xqe_size == 128 ? NIX_XQESZ_W16 : NIX_XQESZ_W64; /* We don't know absolute NPA LF idx attached. * AF will replace 'RVU_DEFAULT_PF_FUNC' with * NPA LF attached to this RVU PF/VF. @@ -1211,7 +1261,11 @@ static int otx2_aura_init(struct otx2_nic *pfvf, int aura_id, */ if (pfvf->nix_blkaddr == BLKADDR_NIX1) aq->aura.bp_ena = 1; +#ifdef CONFIG_DCB + aq->aura.nix0_bpid = pfvf->bpid[pfvf->queue_to_pfc_map[aura_id]]; +#else aq->aura.nix0_bpid = pfvf->bpid[0]; +#endif /* Set backpressure level for RQ's Aura */ aq->aura.bp = RQ_BP_LVL_AURA; @@ -1538,11 +1592,18 @@ int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable) return -ENOMEM; req->chan_base = 0; - req->chan_cnt = 1; +#ifdef CONFIG_DCB + req->chan_cnt = pfvf->pfc_en ? IEEE_8021QAZ_MAX_TCS : 1; + req->bpid_per_chan = pfvf->pfc_en ? 1 : 0; +#else + req->chan_cnt = 1; req->bpid_per_chan = 0; +#endif + return otx2_sync_mbox_msg(&pfvf->mbox); } +EXPORT_SYMBOL(otx2_nix_config_bp); /* Mbox message handlers */ void mbox_handler_cgx_stats(struct otx2_nic *pfvf, @@ -1573,6 +1634,8 @@ void mbox_handler_nix_txsch_alloc(struct otx2_nic *pf, for (schq = 0; schq < rsp->schq[lvl]; schq++) pf->hw.txschq_list[lvl][schq] = rsp->schq_list[lvl][schq]; + + pf->hw.txschq_link_cfg_lvl = rsp->link_cfg_lvl; } EXPORT_SYMBOL(mbox_handler_nix_txsch_alloc); @@ -1704,6 +1767,56 @@ out: } EXPORT_SYMBOL(otx2_get_max_mtu); +int otx2_handle_ntuple_tc_features(struct net_device *netdev, netdev_features_t features) +{ + netdev_features_t changed = features ^ netdev->features; + struct otx2_nic *pfvf = netdev_priv(netdev); + bool ntuple = !!(features & NETIF_F_NTUPLE); + bool tc = !!(features & NETIF_F_HW_TC); + + if ((changed & NETIF_F_NTUPLE) && !ntuple) + otx2_destroy_ntuple_flows(pfvf); + + if ((changed & NETIF_F_NTUPLE) && ntuple) { + if (!pfvf->flow_cfg->max_flows) { + netdev_err(netdev, + "Can't enable NTUPLE, MCAM entries not allocated\n"); + return -EINVAL; + } + } + + if ((changed & NETIF_F_HW_TC) && tc) { + if (!pfvf->flow_cfg->max_flows) { + netdev_err(netdev, + "Can't enable TC, MCAM entries not allocated\n"); + return -EINVAL; + } + } + + if ((changed & NETIF_F_HW_TC) && !tc && + pfvf->flow_cfg && pfvf->flow_cfg->nr_flows) { + netdev_err(netdev, "Can't disable TC hardware offload while flows are active\n"); + return -EBUSY; + } + + if ((changed & NETIF_F_NTUPLE) && ntuple && + (netdev->features & NETIF_F_HW_TC) && !(changed & NETIF_F_HW_TC)) { + netdev_err(netdev, + "Can't enable NTUPLE when TC is active, disable TC and retry\n"); + return -EINVAL; + } + + if ((changed & NETIF_F_HW_TC) && tc && + (netdev->features & NETIF_F_NTUPLE) && !(changed & NETIF_F_NTUPLE)) { + netdev_err(netdev, + "Can't enable TC when NTUPLE is active, disable NTUPLE and retry\n"); + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL(otx2_handle_ntuple_tc_features); + #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ int __weak \ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ @@ -1715,4 +1828,5 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ } \ EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 61e52812983f..282db6fe3b08 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -17,6 +17,9 @@ #include <linux/soc/marvell/octeontx2/asm.h> #include <net/pkt_cls.h> #include <net/devlink.h> +#include <linux/time64.h> +#include <linux/dim.h> +#include <uapi/linux/if_macsec.h> #include <mbox.h> #include <npc.h> @@ -31,6 +34,7 @@ #define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8 #define PCI_SUBSYS_DEVID_96XX_RVU_PFVF 0xB200 +#define PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF 0xBD00 /* PCI BAR nos */ #define PCI_CFG_REG_BAR_NUM 2 @@ -38,6 +42,11 @@ #define NAME_SIZE 32 +#ifdef CONFIG_DCB +/* Max priority supported for PFC */ +#define NIX_PF_PFC_PRIO_MAX 8 +#endif + enum arua_mapped_qtypes { AURA_NIX_RQ, AURA_NIX_SQ, @@ -53,6 +62,11 @@ enum arua_mapped_qtypes { /* Send skid of 2000 packets required for CQ size of 4K CQEs. */ #define SEND_CQ_SKID 2000 +#define OTX2_GET_RX_STATS(reg) \ + otx2_read64(pfvf, NIX_LF_RX_STATX(reg)) +#define OTX2_GET_TX_STATS(reg) \ + otx2_read64(pfvf, NIX_LF_TX_STATX(reg)) + struct otx2_lmt_info { u64 lmt_addr; u16 lmt_id; @@ -178,13 +192,18 @@ struct otx2_hw { u16 rqpool_cnt; u16 sqpool_cnt; +#define OTX2_DEFAULT_RBUF_LEN 2048 + u16 rbuf_len; + u32 xqe_size; + /* NPA */ u32 stack_pg_ptrs; /* No of ptrs per stack page */ u32 stack_pg_bytes; /* Size of stack page */ u16 sqb_size; /* NIX */ - u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + u8 txschq_link_cfg_lvl; + u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; u16 matchall_ipolicer; u32 dwrr_mtu; @@ -226,6 +245,8 @@ struct otx2_hw { #define CN10K_MBOX 1 #define CN10K_LMTST 2 #define CN10K_RPM 3 +#define CN10K_PTP_ONESTEP 4 +#define CN10K_HW_MACSEC 5 unsigned long cap_flag; #define LMT_LINE_SIZE 128 @@ -259,6 +280,13 @@ struct refill_work { struct otx2_nic *pf; }; +/* PTPv2 originTimestamp structure */ +struct ptpv2_tstamp { + __be16 seconds_msb; /* 16 bits + */ + __be32 seconds_lsb; /* 32 bits = 48 bits*/ + __be32 nanoseconds; +} __packed; + struct otx2_ptp { struct ptp_clock_info ptp_info; struct ptp_clock *ptp_clock; @@ -272,6 +300,11 @@ struct otx2_ptp { u64 thresh; struct ptp_pin_desc extts_config; + u64 (*convert_rx_ptp_tstmp)(u64 timestamp); + u64 (*convert_tx_ptp_tstmp)(u64 timestamp); + struct delayed_work synctstamp_work; + u64 tstamp; + u32 base_ns; }; #define OTX2_HW_TIMESTAMP_LEN 8 @@ -301,8 +334,8 @@ struct otx2_flow_config { #define OTX2_VF_VLAN_TX_INDEX 1 u16 max_flows; u8 dmacflt_max_flows; - u8 *bmap_to_dmacindex; - unsigned long dmacflt_bmap; + u32 *bmap_to_dmacindex; + unsigned long *dmacflt_bmap; struct list_head flow_list; }; @@ -321,6 +354,66 @@ struct dev_hw_ops { void (*aura_freeptr)(void *dev, int aura, u64 buf); }; +#define CN10K_MCS_SA_PER_SC 4 + +/* Stats which need to be accumulated in software because + * of shared counters in hardware. + */ +struct cn10k_txsc_stats { + u64 InPktsUntagged; + u64 InPktsNoTag; + u64 InPktsBadTag; + u64 InPktsUnknownSCI; + u64 InPktsNoSCI; + u64 InPktsOverrun; +}; + +struct cn10k_rxsc_stats { + u64 InOctetsValidated; + u64 InOctetsDecrypted; + u64 InPktsUnchecked; + u64 InPktsDelayed; + u64 InPktsOK; + u64 InPktsInvalid; + u64 InPktsLate; + u64 InPktsNotValid; + u64 InPktsNotUsingSA; + u64 InPktsUnusedSA; +}; + +struct cn10k_mcs_txsc { + struct macsec_secy *sw_secy; + struct cn10k_txsc_stats stats; + struct list_head entry; + enum macsec_validation_type last_validate_frames; + bool last_protect_frames; + u16 hw_secy_id_tx; + u16 hw_secy_id_rx; + u16 hw_flow_id; + u16 hw_sc_id; + u16 hw_sa_id[CN10K_MCS_SA_PER_SC]; + u8 sa_bmap; + u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN]; + u8 encoding_sa; +}; + +struct cn10k_mcs_rxsc { + struct macsec_secy *sw_secy; + struct macsec_rx_sc *sw_rxsc; + struct cn10k_rxsc_stats stats; + struct list_head entry; + u16 hw_flow_id; + u16 hw_sc_id; + u16 hw_sa_id[CN10K_MCS_SA_PER_SC]; + u8 sa_bmap; + u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN]; +}; + +struct cn10k_mcs_cfg { + struct list_head txsc_list; + struct list_head rxsc_list; +}; + struct otx2_nic { void __iomem *reg_base; struct net_device *netdev; @@ -344,6 +437,8 @@ struct otx2_nic { #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12) #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13) #define OTX2_FLAG_DMACFLTR_SUPPORT BIT_ULL(14) +#define OTX2_FLAG_PTP_ONESTEP_SYNC BIT_ULL(15) +#define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16) u64 flags; u64 *cq_op_addr; @@ -396,6 +491,20 @@ struct otx2_nic { /* Devlink */ struct otx2_devlink *dl; +#ifdef CONFIG_DCB + /* PFC */ + u8 pfc_en; + u8 *queue_to_pfc_map; + u16 pfc_schq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + bool pfc_alloc_status[NIX_PF_PFC_PRIO_MAX]; +#endif + + /* napi event count. It is needed for adaptive irq coalescing. */ + u32 napi_events; + +#if IS_ENABLED(CONFIG_MACSEC) + struct cn10k_mcs_cfg *macsec_cfg; +#endif }; static inline bool is_otx2_lbkvf(struct pci_dev *pdev) @@ -435,6 +544,11 @@ static inline bool is_dev_otx2(struct pci_dev *pdev) midr == PCI_REVISION_ID_95XXMM || midr == PCI_REVISION_ID_95XXO); } +static inline bool is_dev_cn10kb(struct pci_dev *pdev) +{ + return pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF; +} + static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf) { struct otx2_hw *hw = &pfvf->hw; @@ -464,7 +578,11 @@ static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf) __set_bit(CN10K_MBOX, &hw->cap_flag); __set_bit(CN10K_LMTST, &hw->cap_flag); __set_bit(CN10K_RPM, &hw->cap_flag); + __set_bit(CN10K_PTP_ONESTEP, &hw->cap_flag); } + + if (is_dev_cn10kb(pfvf->pdev)) + __set_bit(CN10K_HW_MACSEC, &hw->cap_flag); } /* Register read/write APIs */ @@ -603,6 +721,7 @@ static inline void __cn10k_aura_freeptr(struct otx2_nic *pfvf, u64 aura, size++; tar_addr |= ((size - 1) & 0x7) << 4; } + dma_wmb(); memcpy((u64 *)lmt_info->lmt_addr, ptrs, sizeof(u64) * num_ptrs); /* Perform LMTST flush */ cn10k_lmt_flush(val, tar_addr); @@ -719,6 +838,7 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ struct _rsp_type *rsp); \ MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M /* Time to wait before watchdog kicks off */ @@ -761,6 +881,16 @@ static inline void otx2_dma_unmap_page(struct otx2_nic *pfvf, dir, DMA_ATTR_SKIP_CPU_SYNC); } +static inline u16 otx2_get_smq_idx(struct otx2_nic *pfvf, u16 qidx) +{ +#ifdef CONFIG_DCB + if (pfvf->pfc_alloc_status[qidx]) + return pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][qidx]; +#endif + + return pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; +} + /* MSI-X APIs */ void otx2_free_cints(struct otx2_nic *pfvf, int n); void otx2_set_cints_affinity(struct otx2_nic *pfvf); @@ -783,7 +913,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type); void otx2_sq_free_sqbs(struct otx2_nic *pfvf); int otx2_config_nix(struct otx2_nic *pfvf); int otx2_config_nix_queues(struct otx2_nic *pfvf); -int otx2_txschq_config(struct otx2_nic *pfvf, int lvl); +int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool pfc_en); int otx2_txsch_alloc(struct otx2_nic *pfvf); int otx2_txschq_stop(struct otx2_nic *pfvf); void otx2_sqb_flush(struct otx2_nic *pfvf); @@ -862,6 +992,10 @@ int otx2_enable_rxvlan(struct otx2_nic *pf, bool enable); int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf); bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx); u16 otx2_get_max_mtu(struct otx2_nic *pfvf); +int otx2_handle_ntuple_tc_features(struct net_device *netdev, + netdev_features_t features); +int otx2_smq_flush(struct otx2_nic *pfvf, int smq); + /* tc support */ int otx2_init_tc(struct otx2_nic *nic); void otx2_shutdown_tc(struct otx2_nic *nic); @@ -870,9 +1004,35 @@ int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic); /* CGX/RPM DMAC filters support */ int otx2_dmacflt_get_max_cnt(struct otx2_nic *pf); -int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u8 bit_pos); -int otx2_dmacflt_remove(struct otx2_nic *pf, const u8 *mac, u8 bit_pos); -int otx2_dmacflt_update(struct otx2_nic *pf, u8 *mac, u8 bit_pos); +int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u32 bit_pos); +int otx2_dmacflt_remove(struct otx2_nic *pf, const u8 *mac, u32 bit_pos); +int otx2_dmacflt_update(struct otx2_nic *pf, u8 *mac, u32 bit_pos); void otx2_dmacflt_reinstall_flows(struct otx2_nic *pf); void otx2_dmacflt_update_pfmac_flow(struct otx2_nic *pfvf); + +#ifdef CONFIG_DCB +/* DCB support*/ +void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx, bool pfc_enable); +int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf); +int otx2_dcbnl_set_ops(struct net_device *dev); +/* PFC support */ +int otx2_pfc_txschq_config(struct otx2_nic *pfvf); +int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf); +int otx2_pfc_txschq_update(struct otx2_nic *pfvf); +int otx2_pfc_txschq_stop(struct otx2_nic *pfvf); +#endif + +#if IS_ENABLED(CONFIG_MACSEC) +/* MACSEC offload support */ +int cn10k_mcs_init(struct otx2_nic *pfvf); +void cn10k_mcs_free(struct otx2_nic *pfvf); +void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event); +#else +static inline int cn10k_mcs_init(struct otx2_nic *pfvf) { return 0; } +static inline void cn10k_mcs_free(struct otx2_nic *pfvf) {} +static inline void cn10k_handle_mcs_event(struct otx2_nic *pfvf, + struct mcs_intr_info *event) +{} +#endif /* CONFIG_MACSEC */ + #endif /* OTX2_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c new file mode 100644 index 000000000000..ccaf97bb1ce0 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell RVU Ethernet driver + * + * Copyright (C) 2021 Marvell. + * + */ + +#include "otx2_common.h" + +static int otx2_check_pfc_config(struct otx2_nic *pfvf) +{ + u8 tx_queues = pfvf->hw.tx_queues, prio; + u8 pfc_en = pfvf->pfc_en; + + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + if ((pfc_en & (1 << prio)) && + prio > tx_queues - 1) { + dev_warn(pfvf->dev, + "Increase number of tx queues from %d to %d to support PFC.\n", + tx_queues, prio + 1); + return -EINVAL; + } + } + + return 0; +} + +int otx2_pfc_txschq_config(struct otx2_nic *pfvf) +{ + u8 pfc_en, pfc_bit_set; + int prio, lvl, err; + + pfc_en = pfvf->pfc_en; + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + /* Either PFC bit is not set + * or tx scheduler is not allocated for the priority + */ + if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio]) + continue; + + /* configure the scheduler for the tls*/ + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { + err = otx2_txschq_config(pfvf, lvl, prio, true); + if (err) { + dev_err(pfvf->dev, + "%s configure PFC tx schq for lvl:%d, prio:%d failed!\n", + __func__, lvl, prio); + return err; + } + } + } + + return 0; +} + +static int otx2_pfc_txschq_alloc_one(struct otx2_nic *pfvf, u8 prio) +{ + struct nix_txsch_alloc_req *req; + struct nix_txsch_alloc_rsp *rsp; + int lvl, rc; + + /* Get memory to put this msg */ + req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox); + if (!req) + return -ENOMEM; + + /* Request one schq per level upto max level as configured + * link config level. These rest of the scheduler can be + * same as hw.txschq_list. + */ + for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) + req->schq[lvl] = 1; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + if (rc) + return rc; + + rsp = (struct nix_txsch_alloc_rsp *) + otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); + if (IS_ERR(rsp)) + return PTR_ERR(rsp); + + /* Setup transmit scheduler list */ + for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) { + if (!rsp->schq[lvl]) + return -ENOSPC; + + pfvf->pfc_schq_list[lvl][prio] = rsp->schq_list[lvl][0]; + } + + /* Set the Tx schedulers for rest of the levels same as + * hw.txschq_list as those will be common for all. + */ + for (; lvl < NIX_TXSCH_LVL_CNT; lvl++) + pfvf->pfc_schq_list[lvl][prio] = pfvf->hw.txschq_list[lvl][0]; + + pfvf->pfc_alloc_status[prio] = true; + return 0; +} + +int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf) +{ + u8 pfc_en = pfvf->pfc_en; + u8 pfc_bit_set; + int err, prio; + + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + if (!pfc_bit_set || pfvf->pfc_alloc_status[prio]) + continue; + + /* Add new scheduler to the priority */ + err = otx2_pfc_txschq_alloc_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, "%s failed to allocate PFC TX schedulers\n", __func__); + return err; + } + } + + return 0; +} + +static int otx2_pfc_txschq_stop_one(struct otx2_nic *pfvf, u8 prio) +{ + struct nix_txsch_free_req *free_req; + + mutex_lock(&pfvf->mbox.lock); + /* free PFC TLx nodes */ + free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox); + if (!free_req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + free_req->flags = TXSCHQ_FREE_ALL; + otx2_sync_mbox_msg(&pfvf->mbox); + mutex_unlock(&pfvf->mbox.lock); + + pfvf->pfc_alloc_status[prio] = false; + return 0; +} + +static int otx2_pfc_update_sq_smq_mapping(struct otx2_nic *pfvf, int prio) +{ + struct nix_cn10k_aq_enq_req *cn10k_sq_aq; + struct net_device *dev = pfvf->netdev; + bool if_up = netif_running(dev); + struct nix_aq_enq_req *sq_aq; + + if (if_up) { + if (pfvf->pfc_alloc_status[prio]) + netif_tx_stop_all_queues(pfvf->netdev); + else + netif_tx_stop_queue(netdev_get_tx_queue(dev, prio)); + } + + if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { + cn10k_sq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!cn10k_sq_aq) + return -ENOMEM; + + /* Fill AQ info */ + cn10k_sq_aq->qidx = prio; + cn10k_sq_aq->ctype = NIX_AQ_CTYPE_SQ; + cn10k_sq_aq->op = NIX_AQ_INSTOP_WRITE; + + /* Fill fields to update */ + cn10k_sq_aq->sq.ena = 1; + cn10k_sq_aq->sq_mask.ena = 1; + cn10k_sq_aq->sq_mask.smq = GENMASK(9, 0); + cn10k_sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio); + } else { + sq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); + if (!sq_aq) + return -ENOMEM; + + /* Fill AQ info */ + sq_aq->qidx = prio; + sq_aq->ctype = NIX_AQ_CTYPE_SQ; + sq_aq->op = NIX_AQ_INSTOP_WRITE; + + /* Fill fields to update */ + sq_aq->sq.ena = 1; + sq_aq->sq_mask.ena = 1; + sq_aq->sq_mask.smq = GENMASK(8, 0); + sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio); + } + + otx2_sync_mbox_msg(&pfvf->mbox); + + if (if_up) { + if (pfvf->pfc_alloc_status[prio]) + netif_tx_start_all_queues(pfvf->netdev); + else + netif_tx_start_queue(netdev_get_tx_queue(dev, prio)); + } + + return 0; +} + +int otx2_pfc_txschq_update(struct otx2_nic *pfvf) +{ + bool if_up = netif_running(pfvf->netdev); + u8 pfc_en = pfvf->pfc_en, pfc_bit_set; + struct mbox *mbox = &pfvf->mbox; + int err, prio; + + mutex_lock(&mbox->lock); + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + /* tx scheduler was created but user wants to disable now */ + if (!pfc_bit_set && pfvf->pfc_alloc_status[prio]) { + mutex_unlock(&mbox->lock); + if (if_up) + netif_tx_stop_all_queues(pfvf->netdev); + + otx2_smq_flush(pfvf, pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][prio]); + if (if_up) + netif_tx_start_all_queues(pfvf->netdev); + + /* delete the schq */ + err = otx2_pfc_txschq_stop_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, + "%s failed to stop PFC tx schedulers for priority: %d\n", + __func__, prio); + return err; + } + + mutex_lock(&mbox->lock); + goto update_sq_smq_map; + } + + /* Either PFC bit is not set + * or Tx scheduler is already mapped for the priority + */ + if (!pfc_bit_set || pfvf->pfc_alloc_status[prio]) + continue; + + /* Add new scheduler to the priority */ + err = otx2_pfc_txschq_alloc_one(pfvf, prio); + if (err) { + mutex_unlock(&mbox->lock); + dev_err(pfvf->dev, + "%s failed to allocate PFC tx schedulers for priority: %d\n", + __func__, prio); + return err; + } + +update_sq_smq_map: + err = otx2_pfc_update_sq_smq_mapping(pfvf, prio); + if (err) { + mutex_unlock(&mbox->lock); + dev_err(pfvf->dev, "%s failed PFC Tx schq sq:%d mapping", __func__, prio); + return err; + } + } + + err = otx2_pfc_txschq_config(pfvf); + mutex_unlock(&mbox->lock); + if (err) + return err; + + return 0; +} + +int otx2_pfc_txschq_stop(struct otx2_nic *pfvf) +{ + u8 pfc_en, pfc_bit_set; + int prio, err; + + pfc_en = pfvf->pfc_en; + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio]) + continue; + + /* Delete the existing scheduler */ + err = otx2_pfc_txschq_stop_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, "%s failed to stop PFC TX schedulers\n", __func__); + return err; + } + } + + return 0; +} + +int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf) +{ + struct cgx_pfc_cfg *req; + struct cgx_pfc_rsp *rsp; + int err = 0; + + if (is_otx2_lbkvf(pfvf->pdev)) + return 0; + + mutex_lock(&pfvf->mbox.lock); + req = otx2_mbox_alloc_msg_cgx_prio_flow_ctrl_cfg(&pfvf->mbox); + if (!req) { + err = -ENOMEM; + goto unlock; + } + + if (pfvf->pfc_en) { + req->rx_pause = true; + req->tx_pause = true; + } else { + req->rx_pause = false; + req->tx_pause = false; + } + req->pfc_en = pfvf->pfc_en; + + if (!otx2_sync_mbox_msg(&pfvf->mbox)) { + rsp = (struct cgx_pfc_rsp *) + otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); + if (req->rx_pause != rsp->rx_pause || req->tx_pause != rsp->tx_pause) { + dev_warn(pfvf->dev, + "Failed to config PFC\n"); + err = -EPERM; + } + } +unlock: + mutex_unlock(&pfvf->mbox.lock); + return err; +} + +void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx, + bool pfc_enable) +{ + bool if_up = netif_running(pfvf->netdev); + struct npa_aq_enq_req *npa_aq; + struct nix_aq_enq_req *aq; + int err = 0; + + if (pfvf->queue_to_pfc_map[qidx] && pfc_enable) { + dev_warn(pfvf->dev, + "PFC enable not permitted as Priority %d already mapped to Queue %d\n", + pfvf->queue_to_pfc_map[qidx], qidx); + return; + } + + if (if_up) { + netif_tx_stop_all_queues(pfvf->netdev); + netif_carrier_off(pfvf->netdev); + } + + pfvf->queue_to_pfc_map[qidx] = vlan_prio; + + aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); + if (!aq) { + err = -ENOMEM; + goto out; + } + + aq->cq.bpid = pfvf->bpid[vlan_prio]; + aq->cq_mask.bpid = GENMASK(8, 0); + + /* Fill AQ info */ + aq->qidx = qidx; + aq->ctype = NIX_AQ_CTYPE_CQ; + aq->op = NIX_AQ_INSTOP_WRITE; + + otx2_sync_mbox_msg(&pfvf->mbox); + + npa_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); + if (!npa_aq) { + err = -ENOMEM; + goto out; + } + npa_aq->aura.nix0_bpid = pfvf->bpid[vlan_prio]; + npa_aq->aura_mask.nix0_bpid = GENMASK(8, 0); + + /* Fill NPA AQ info */ + npa_aq->aura_id = qidx; + npa_aq->ctype = NPA_AQ_CTYPE_AURA; + npa_aq->op = NPA_AQ_INSTOP_WRITE; + otx2_sync_mbox_msg(&pfvf->mbox); + +out: + if (if_up) { + netif_carrier_on(pfvf->netdev); + netif_tx_start_all_queues(pfvf->netdev); + } + + if (err) + dev_warn(pfvf->dev, + "Updating BPIDs in CQ and Aura contexts of RQ%d failed with err %d\n", + qidx, err); +} + +static int otx2_dcbnl_ieee_getpfc(struct net_device *dev, struct ieee_pfc *pfc) +{ + struct otx2_nic *pfvf = netdev_priv(dev); + + pfc->pfc_cap = IEEE_8021QAZ_MAX_TCS; + pfc->pfc_en = pfvf->pfc_en; + + return 0; +} + +static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc) +{ + struct otx2_nic *pfvf = netdev_priv(dev); + int err; + + /* Save PFC configuration to interface */ + pfvf->pfc_en = pfc->pfc_en; + + if (pfvf->hw.tx_queues >= NIX_PF_PFC_PRIO_MAX) + goto process_pfc; + + /* Check if the PFC configuration can be + * supported by the tx queue configuration + */ + err = otx2_check_pfc_config(pfvf); + if (err) + return err; + +process_pfc: + err = otx2_config_priority_flow_ctrl(pfvf); + if (err) + return err; + + /* Request Per channel Bpids */ + if (pfc->pfc_en) + otx2_nix_config_bp(pfvf, true); + + err = otx2_pfc_txschq_update(pfvf); + if (err) { + dev_err(pfvf->dev, "%s failed to update TX schedulers\n", __func__); + return err; + } + + return 0; +} + +static u8 otx2_dcbnl_getdcbx(struct net_device __always_unused *dev) +{ + return DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; +} + +static u8 otx2_dcbnl_setdcbx(struct net_device __always_unused *dev, u8 mode) +{ + return (mode != (DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE)) ? 1 : 0; +} + +static const struct dcbnl_rtnl_ops otx2_dcbnl_ops = { + .ieee_getpfc = otx2_dcbnl_ieee_getpfc, + .ieee_setpfc = otx2_dcbnl_ieee_setpfc, + .getdcbx = otx2_dcbnl_getdcbx, + .setdcbx = otx2_dcbnl_setdcbx, +}; + +int otx2_dcbnl_set_ops(struct net_device *dev) +{ + struct otx2_nic *pfvf = netdev_priv(dev); + + pfvf->queue_to_pfc_map = devm_kzalloc(pfvf->dev, pfvf->hw.rx_queues, + GFP_KERNEL); + if (!pfvf->queue_to_pfc_map) + return -ENOMEM; + dev->dcbnl_ops = &otx2_dcbnl_ops; + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c index 2ec800f741d8..80d853b343f9 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dmac_flt.c @@ -8,7 +8,7 @@ #include "otx2_common.h" static int otx2_dmacflt_do_add(struct otx2_nic *pf, const u8 *mac, - u8 *dmac_index) + u32 *dmac_index) { struct cgx_mac_addr_add_req *req; struct cgx_mac_addr_add_rsp *rsp; @@ -35,9 +35,10 @@ static int otx2_dmacflt_do_add(struct otx2_nic *pf, const u8 *mac, return err; } -static int otx2_dmacflt_add_pfmac(struct otx2_nic *pf) +static int otx2_dmacflt_add_pfmac(struct otx2_nic *pf, u32 *dmac_index) { struct cgx_mac_addr_set_or_get *req; + struct cgx_mac_addr_set_or_get *rsp; int err; mutex_lock(&pf->mbox.lock); @@ -48,16 +49,31 @@ static int otx2_dmacflt_add_pfmac(struct otx2_nic *pf) return -ENOMEM; } + req->index = *dmac_index; + ether_addr_copy(req->mac_addr, pf->netdev->dev_addr); err = otx2_sync_mbox_msg(&pf->mbox); + if (err) + goto out; + + rsp = (struct cgx_mac_addr_set_or_get *) + otx2_mbox_get_rsp(&pf->mbox.mbox, 0, &req->hdr); + + if (IS_ERR_OR_NULL(rsp)) { + err = -EINVAL; + goto out; + } + + *dmac_index = rsp->index; +out: mutex_unlock(&pf->mbox.lock); return err; } -int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u8 bit_pos) +int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u32 bit_pos) { - u8 *dmacindex; + u32 *dmacindex; /* Store dmacindex returned by CGX/RPM driver which will * be used for macaddr update/remove @@ -65,13 +81,13 @@ int otx2_dmacflt_add(struct otx2_nic *pf, const u8 *mac, u8 bit_pos) dmacindex = &pf->flow_cfg->bmap_to_dmacindex[bit_pos]; if (ether_addr_equal(mac, pf->netdev->dev_addr)) - return otx2_dmacflt_add_pfmac(pf); + return otx2_dmacflt_add_pfmac(pf, dmacindex); else return otx2_dmacflt_do_add(pf, mac, dmacindex); } static int otx2_dmacflt_do_remove(struct otx2_nic *pfvf, const u8 *mac, - u8 dmac_index) + u32 dmac_index) { struct cgx_mac_addr_del_req *req; int err; @@ -91,9 +107,9 @@ static int otx2_dmacflt_do_remove(struct otx2_nic *pfvf, const u8 *mac, return err; } -static int otx2_dmacflt_remove_pfmac(struct otx2_nic *pf) +static int otx2_dmacflt_remove_pfmac(struct otx2_nic *pf, u32 dmac_index) { - struct msg_req *req; + struct cgx_mac_addr_reset_req *req; int err; mutex_lock(&pf->mbox.lock); @@ -102,6 +118,7 @@ static int otx2_dmacflt_remove_pfmac(struct otx2_nic *pf) mutex_unlock(&pf->mbox.lock); return -ENOMEM; } + req->index = dmac_index; err = otx2_sync_mbox_msg(&pf->mbox); @@ -110,12 +127,12 @@ static int otx2_dmacflt_remove_pfmac(struct otx2_nic *pf) } int otx2_dmacflt_remove(struct otx2_nic *pf, const u8 *mac, - u8 bit_pos) + u32 bit_pos) { - u8 dmacindex = pf->flow_cfg->bmap_to_dmacindex[bit_pos]; + u32 dmacindex = pf->flow_cfg->bmap_to_dmacindex[bit_pos]; if (ether_addr_equal(mac, pf->netdev->dev_addr)) - return otx2_dmacflt_remove_pfmac(pf); + return otx2_dmacflt_remove_pfmac(pf, dmacindex); else return otx2_dmacflt_do_remove(pf, mac, dmacindex); } @@ -144,6 +161,12 @@ int otx2_dmacflt_get_max_cnt(struct otx2_nic *pf) rsp = (struct cgx_max_dmac_entries_get_rsp *) otx2_mbox_get_rsp(&pf->mbox.mbox, 0, &msg->hdr); + + if (IS_ERR_OR_NULL(rsp)) { + err = -EINVAL; + goto out; + } + pf->flow_cfg->dmacflt_max_flows = rsp->max_dmac_filters; out: @@ -151,9 +174,10 @@ out: return err; } -int otx2_dmacflt_update(struct otx2_nic *pf, u8 *mac, u8 bit_pos) +int otx2_dmacflt_update(struct otx2_nic *pf, u8 *mac, u32 bit_pos) { struct cgx_mac_addr_update_req *req; + struct cgx_mac_addr_update_rsp *rsp; int rc; mutex_lock(&pf->mbox.lock); @@ -167,8 +191,19 @@ int otx2_dmacflt_update(struct otx2_nic *pf, u8 *mac, u8 bit_pos) ether_addr_copy(req->mac_addr, mac); req->index = pf->flow_cfg->bmap_to_dmacindex[bit_pos]; + + /* check the response and change index */ + rc = otx2_sync_mbox_msg(&pf->mbox); + if (rc) + goto out; + + rsp = (struct cgx_mac_addr_update_rsp *) + otx2_mbox_get_rsp(&pf->mbox.mbox, 0, &req->hdr); + pf->flow_cfg->bmap_to_dmacindex[bit_pos] = rsp->index; + +out: mutex_unlock(&pf->mbox.lock); return rc; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 80d4ce61f442..0eb74e8c553d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -76,8 +76,8 @@ static void otx2_get_drvinfo(struct net_device *netdev, { struct otx2_nic *pfvf = netdev_priv(netdev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info)); } static void otx2_get_qset_strings(struct otx2_nic *pfvf, u8 **data, int qset) @@ -360,7 +360,9 @@ static int otx2_set_pauseparam(struct net_device *netdev, } static void otx2_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct otx2_nic *pfvf = netdev_priv(netdev); struct otx2_qset *qs = &pfvf->qset; @@ -369,12 +371,19 @@ static void otx2_get_ringparam(struct net_device *netdev, ring->rx_pending = qs->rqe_cnt ? qs->rqe_cnt : Q_COUNT(Q_SIZE_256); ring->tx_max_pending = Q_COUNT(Q_SIZE_MAX); ring->tx_pending = qs->sqe_cnt ? qs->sqe_cnt : Q_COUNT(Q_SIZE_4K); + kernel_ring->rx_buf_len = pfvf->hw.rbuf_len; + kernel_ring->cqe_size = pfvf->hw.xqe_size; } static int otx2_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct otx2_nic *pfvf = netdev_priv(netdev); + u32 rx_buf_len = kernel_ring->rx_buf_len; + u32 old_rx_buf_len = pfvf->hw.rbuf_len; + u32 xqe_size = kernel_ring->cqe_size; bool if_up = netif_running(netdev); struct otx2_qset *qs = &pfvf->qset; u32 rx_count, tx_count; @@ -382,6 +391,21 @@ static int otx2_set_ringparam(struct net_device *netdev, if (ring->rx_mini_pending || ring->rx_jumbo_pending) return -EINVAL; + /* Hardware supports max size of 32k for a receive buffer + * and 1536 is typical ethernet frame size. + */ + if (rx_buf_len && (rx_buf_len < 1536 || rx_buf_len > 32768)) { + netdev_err(netdev, + "Receive buffer range is 1536 - 32768"); + return -EINVAL; + } + + if (xqe_size != 128 && xqe_size != 512) { + netdev_err(netdev, + "Completion event size must be 128 or 512"); + return -EINVAL; + } + /* Permitted lengths are 16 64 256 1K 4K 16K 64K 256K 1M */ rx_count = ring->rx_pending; /* On some silicon variants a skid or reserved CQEs are @@ -399,7 +423,8 @@ static int otx2_set_ringparam(struct net_device *netdev, Q_COUNT(Q_SIZE_4K), Q_COUNT(Q_SIZE_MAX)); tx_count = Q_COUNT(Q_SIZE(tx_count, 3)); - if (tx_count == qs->sqe_cnt && rx_count == qs->rqe_cnt) + if (tx_count == qs->sqe_cnt && rx_count == qs->rqe_cnt && + rx_buf_len == old_rx_buf_len && xqe_size == pfvf->hw.xqe_size) return 0; if (if_up) @@ -409,6 +434,9 @@ static int otx2_set_ringparam(struct net_device *netdev, qs->sqe_cnt = tx_count; qs->rqe_cnt = rx_count; + pfvf->hw.rbuf_len = rx_buf_len; + pfvf->hw.xqe_size = xqe_size; + if (if_up) return netdev->netdev_ops->ndo_open(netdev); @@ -427,6 +455,14 @@ static int otx2_get_coalesce(struct net_device *netdev, cmd->rx_max_coalesced_frames = hw->cq_ecount_wait; cmd->tx_coalesce_usecs = hw->cq_time_wait; cmd->tx_max_coalesced_frames = hw->cq_ecount_wait; + if ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) == + OTX2_FLAG_ADPTV_INT_COAL_ENABLED) { + cmd->use_adaptive_rx_coalesce = 1; + cmd->use_adaptive_tx_coalesce = 1; + } else { + cmd->use_adaptive_rx_coalesce = 0; + cmd->use_adaptive_tx_coalesce = 0; + } return 0; } @@ -438,11 +474,30 @@ static int otx2_set_coalesce(struct net_device *netdev, { struct otx2_nic *pfvf = netdev_priv(netdev); struct otx2_hw *hw = &pfvf->hw; + u8 priv_coalesce_status; int qidx; if (!ec->rx_max_coalesced_frames || !ec->tx_max_coalesced_frames) return 0; + if (ec->use_adaptive_rx_coalesce != ec->use_adaptive_tx_coalesce) { + netdev_err(netdev, + "adaptive-rx should be same as adaptive-tx"); + return -EINVAL; + } + + /* Check and update coalesce status */ + if ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) == + OTX2_FLAG_ADPTV_INT_COAL_ENABLED) { + priv_coalesce_status = 1; + if (!ec->use_adaptive_rx_coalesce) + pfvf->flags &= ~OTX2_FLAG_ADPTV_INT_COAL_ENABLED; + } else { + priv_coalesce_status = 0; + if (ec->use_adaptive_rx_coalesce) + pfvf->flags |= OTX2_FLAG_ADPTV_INT_COAL_ENABLED; + } + /* 'cq_time_wait' is 8bit and is in multiple of 100ns, * so clamp the user given value to the range of 1 to 25usec. */ @@ -466,9 +521,9 @@ static int otx2_set_coalesce(struct net_device *netdev, * so clamp the user given value to the range of 1 to 64k. */ ec->rx_max_coalesced_frames = clamp_t(u32, ec->rx_max_coalesced_frames, - 1, U16_MAX); + 1, NAPI_POLL_WEIGHT); ec->tx_max_coalesced_frames = clamp_t(u32, ec->tx_max_coalesced_frames, - 1, U16_MAX); + 1, NAPI_POLL_WEIGHT); /* Rx and Tx are mapped to same CQ, check which one * is changed, if both then choose the min. @@ -481,6 +536,17 @@ static int otx2_set_coalesce(struct net_device *netdev, hw->cq_ecount_wait = min_t(u16, ec->rx_max_coalesced_frames, ec->tx_max_coalesced_frames); + /* Reset 'cq_time_wait' and 'cq_ecount_wait' to + * default values if coalesce status changed from + * 'on' to 'off'. + */ + if (priv_coalesce_status && + ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) != + OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) { + hw->cq_time_wait = CQ_TIMER_THRESH_DEFAULT; + hw->cq_ecount_wait = CQ_CQE_THRESH_DEFAULT; + } + if (netif_running(netdev)) { for (qidx = 0; qidx < pfvf->hw.cint_cnt; qidx++) otx2_config_irq_coalescing(pfvf, qidx); @@ -897,10 +963,12 @@ static int otx2_get_ts_info(struct net_device *netdev, info->phc_index = otx2_ptp_clock_index(pfvf); - info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + if (test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) + info->tx_types |= BIT(HWTSTAMP_TX_ONESTEP_SYNC); - info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | - (1 << HWTSTAMP_FILTER_ALL); + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_ALL); return 0; } @@ -1202,7 +1270,10 @@ end: static const struct ethtool_ops otx2_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_MAX_FRAMES, + ETHTOOL_COALESCE_MAX_FRAMES | + ETHTOOL_COALESCE_USE_ADAPTIVE, + .supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN | + ETHTOOL_RING_USE_CQE_SIZE, .get_link = otx2_get_link, .get_drvinfo = otx2_get_drvinfo, .get_strings = otx2_get_strings, @@ -1244,8 +1315,8 @@ static void otx2vf_get_drvinfo(struct net_device *netdev, { struct otx2_nic *vf = netdev_priv(netdev); - strlcpy(info->driver, DRV_VF_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_VF_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info)); } static void otx2vf_get_strings(struct net_device *netdev, u32 sset, u8 *data) @@ -1321,7 +1392,10 @@ static int otx2vf_get_link_ksettings(struct net_device *netdev, static const struct ethtool_ops otx2vf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_MAX_FRAMES, + ETHTOOL_COALESCE_MAX_FRAMES | + ETHTOOL_COALESCE_USE_ADAPTIVE, + .supported_ring_params = ETHTOOL_RING_USE_RX_BUF_LEN | + ETHTOOL_RING_USE_CQE_SIZE, .get_link = otx2_get_link, .get_drvinfo = otx2vf_get_drvinfo, .get_strings = otx2vf_get_strings, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c index 77a13fb555fb..709fc0114fbd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c @@ -18,11 +18,13 @@ struct otx2_flow { struct ethtool_rx_flow_spec flow_spec; struct list_head list; u32 location; - u16 entry; + u32 entry; bool is_vf; u8 rss_ctx_id; +#define DMAC_FILTER_RULE BIT(0) +#define PFC_FLOWCTRL_RULE BIT(1) + u16 rule_type; int vf; - bool dmac_filter; }; enum dmac_req { @@ -230,6 +232,9 @@ static int otx2_mcam_entry_init(struct otx2_nic *pfvf) return 0; } +/* TODO : revisit on size */ +#define OTX2_DMAC_FLTR_BITMAP_SZ (4 * 2048 + 32) + int otx2vf_mcam_flow_init(struct otx2_nic *pfvf) { struct otx2_flow_config *flow_cfg; @@ -240,6 +245,12 @@ int otx2vf_mcam_flow_init(struct otx2_nic *pfvf) if (!pfvf->flow_cfg) return -ENOMEM; + pfvf->flow_cfg->dmacflt_bmap = devm_kcalloc(pfvf->dev, + BITS_TO_LONGS(OTX2_DMAC_FLTR_BITMAP_SZ), + sizeof(long), GFP_KERNEL); + if (!pfvf->flow_cfg->dmacflt_bmap) + return -ENOMEM; + flow_cfg = pfvf->flow_cfg; INIT_LIST_HEAD(&flow_cfg->flow_list); flow_cfg->max_flows = 0; @@ -257,6 +268,12 @@ int otx2_mcam_flow_init(struct otx2_nic *pf) if (!pf->flow_cfg) return -ENOMEM; + pf->flow_cfg->dmacflt_bmap = devm_kcalloc(pf->dev, + BITS_TO_LONGS(OTX2_DMAC_FLTR_BITMAP_SZ), + sizeof(long), GFP_KERNEL); + if (!pf->flow_cfg->dmacflt_bmap) + return -ENOMEM; + INIT_LIST_HEAD(&pf->flow_cfg->flow_list); /* Allocate bare minimum number of MCAM entries needed for @@ -282,7 +299,7 @@ int otx2_mcam_flow_init(struct otx2_nic *pf) return 0; pf->flow_cfg->bmap_to_dmacindex = - devm_kzalloc(pf->dev, sizeof(u8) * + devm_kzalloc(pf->dev, sizeof(u32) * pf->flow_cfg->dmacflt_max_flows, GFP_KERNEL); @@ -353,7 +370,7 @@ int otx2_add_macfilter(struct net_device *netdev, const u8 *mac) { struct otx2_nic *pf = netdev_priv(netdev); - if (bitmap_weight(&pf->flow_cfg->dmacflt_bmap, + if (!bitmap_empty(pf->flow_cfg->dmacflt_bmap, pf->flow_cfg->dmacflt_max_flows)) netdev_warn(netdev, "Add %pM to CGX/RPM DMAC filters list as well\n", @@ -436,7 +453,7 @@ int otx2_get_maxflows(struct otx2_flow_config *flow_cfg) return 0; if (flow_cfg->nr_flows == flow_cfg->max_flows || - bitmap_weight(&flow_cfg->dmacflt_bmap, + !bitmap_empty(flow_cfg->dmacflt_bmap, flow_cfg->dmacflt_max_flows)) return flow_cfg->max_flows + flow_cfg->dmacflt_max_flows; else @@ -899,6 +916,9 @@ static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf, static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) { u64 ring_cookie = flow->flow_spec.ring_cookie; +#ifdef CONFIG_DCB + int vlan_prio, qidx, pfc_rule = 0; +#endif struct npc_install_flow_req *req; int err, vf = 0; @@ -940,6 +960,24 @@ static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) mutex_unlock(&pfvf->mbox.lock); return -EINVAL; } + +#ifdef CONFIG_DCB + /* Identify PFC rule if PFC enabled and ntuple rule is vlan */ + if (!vf && (req->features & BIT_ULL(NPC_OUTER_VID)) && + pfvf->pfc_en && req->op != NIX_RX_ACTIONOP_RSS) { + vlan_prio = ntohs(req->packet.vlan_tci) & + ntohs(req->mask.vlan_tci); + + /* Get the priority */ + vlan_prio >>= 13; + flow->rule_type |= PFC_FLOWCTRL_RULE; + /* Check if PFC enabled for this priority */ + if (pfvf->pfc_en & BIT(vlan_prio)) { + pfc_rule = true; + qidx = req->index; + } + } +#endif } /* ethtool ring_cookie has (VF + 1) for VF */ @@ -951,6 +989,12 @@ static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) /* Send message to AF */ err = otx2_sync_mbox_msg(&pfvf->mbox); + +#ifdef CONFIG_DCB + if (!err && pfc_rule) + otx2_update_bpid_in_rqctx(pfvf, vlan_prio, qidx, true); +#endif + mutex_unlock(&pfvf->mbox.lock); return err; } @@ -966,7 +1010,7 @@ static int otx2_add_flow_with_pfmac(struct otx2_nic *pfvf, return -ENOMEM; pf_mac->entry = 0; - pf_mac->dmac_filter = true; + pf_mac->rule_type |= DMAC_FILTER_RULE; pf_mac->location = pfvf->flow_cfg->max_flows; memcpy(&pf_mac->flow_spec, &flow->flow_spec, sizeof(struct ethtool_rx_flow_spec)); @@ -981,7 +1025,7 @@ static int otx2_add_flow_with_pfmac(struct otx2_nic *pfvf, otx2_add_flow_to_list(pfvf, pf_mac); pfvf->flow_cfg->nr_flows++; - set_bit(0, &pfvf->flow_cfg->dmacflt_bmap); + set_bit(0, pfvf->flow_cfg->dmacflt_bmap); return 0; } @@ -1031,11 +1075,11 @@ int otx2_add_flow(struct otx2_nic *pfvf, struct ethtool_rxnfc *nfc) eth_hdr = &flow->flow_spec.h_u.ether_spec; /* Sync dmac filter table with updated fields */ - if (flow->dmac_filter) + if (flow->rule_type & DMAC_FILTER_RULE) return otx2_dmacflt_update(pfvf, eth_hdr->h_dest, flow->entry); - if (bitmap_full(&flow_cfg->dmacflt_bmap, + if (bitmap_full(flow_cfg->dmacflt_bmap, flow_cfg->dmacflt_max_flows)) { netdev_warn(pfvf->netdev, "Can't insert the rule %d as max allowed dmac filters are %d\n", @@ -1049,17 +1093,17 @@ int otx2_add_flow(struct otx2_nic *pfvf, struct ethtool_rxnfc *nfc) } /* Install PF mac address to DMAC filter list */ - if (!test_bit(0, &flow_cfg->dmacflt_bmap)) + if (!test_bit(0, flow_cfg->dmacflt_bmap)) otx2_add_flow_with_pfmac(pfvf, flow); - flow->dmac_filter = true; - flow->entry = find_first_zero_bit(&flow_cfg->dmacflt_bmap, + flow->rule_type |= DMAC_FILTER_RULE; + flow->entry = find_first_zero_bit(flow_cfg->dmacflt_bmap, flow_cfg->dmacflt_max_flows); fsp->location = flow_cfg->max_flows + flow->entry; flow->flow_spec.location = fsp->location; flow->location = fsp->location; - set_bit(flow->entry, &flow_cfg->dmacflt_bmap); + set_bit(flow->entry, flow_cfg->dmacflt_bmap); otx2_dmacflt_add(pfvf, eth_hdr->h_dest, flow->entry); } else { @@ -1120,16 +1164,17 @@ static void otx2_update_rem_pfmac(struct otx2_nic *pfvf, int req) bool found = false; list_for_each_entry(iter, &pfvf->flow_cfg->flow_list, list) { - if (iter->dmac_filter && iter->entry == 0) { + if ((iter->rule_type & DMAC_FILTER_RULE) && iter->entry == 0) { eth_hdr = &iter->flow_spec.h_u.ether_spec; if (req == DMAC_ADDR_DEL) { otx2_dmacflt_remove(pfvf, eth_hdr->h_dest, 0); - clear_bit(0, &pfvf->flow_cfg->dmacflt_bmap); + clear_bit(0, pfvf->flow_cfg->dmacflt_bmap); found = true; } else { ether_addr_copy(eth_hdr->h_dest, pfvf->netdev->dev_addr); + otx2_dmacflt_update(pfvf, eth_hdr->h_dest, 0); } break; @@ -1156,7 +1201,7 @@ int otx2_remove_flow(struct otx2_nic *pfvf, u32 location) if (!flow) return -ENOENT; - if (flow->dmac_filter) { + if (flow->rule_type & DMAC_FILTER_RULE) { struct ethhdr *eth_hdr = &flow->flow_spec.h_u.ether_spec; /* user not allowed to remove dmac filter with interface mac */ @@ -1165,15 +1210,22 @@ int otx2_remove_flow(struct otx2_nic *pfvf, u32 location) err = otx2_dmacflt_remove(pfvf, eth_hdr->h_dest, flow->entry); - clear_bit(flow->entry, &flow_cfg->dmacflt_bmap); + clear_bit(flow->entry, flow_cfg->dmacflt_bmap); /* If all dmac filters are removed delete macfilter with * interface mac address and configure CGX/RPM block in * promiscuous mode */ - if (bitmap_weight(&flow_cfg->dmacflt_bmap, + if (bitmap_weight(flow_cfg->dmacflt_bmap, flow_cfg->dmacflt_max_flows) == 1) otx2_update_rem_pfmac(pfvf, DMAC_ADDR_DEL); } else { +#ifdef CONFIG_DCB + if (flow->rule_type & PFC_FLOWCTRL_RULE) + otx2_update_bpid_in_rqctx(pfvf, 0, + flow->flow_spec.ring_cookie, + false); +#endif + err = otx2_remove_flow_msg(pfvf, flow->entry, false); } @@ -1383,7 +1435,7 @@ void otx2_dmacflt_reinstall_flows(struct otx2_nic *pf) struct ethhdr *eth_hdr; list_for_each_entry(iter, &pf->flow_cfg->flow_list, list) { - if (iter->dmac_filter) { + if (iter->rule_type & DMAC_FILTER_RULE) { eth_hdr = &iter->flow_spec.h_u.ether_spec; otx2_dmacflt_add(pf, eth_hdr->h_dest, iter->entry); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 1e0d0c9c1dac..303930499a4c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -15,6 +15,7 @@ #include <net/ip.h> #include <linux/bpf.h> #include <linux/bpf_trace.h> +#include <linux/bitfield.h> #include "otx2_reg.h" #include "otx2_common.h" @@ -394,7 +395,12 @@ static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf, dst_mdev->msg_size = mbox_hdr->msg_size; dst_mdev->num_msgs = num_msgs; err = otx2_sync_mbox_msg(dst_mbox); - if (err) { + /* Error code -EIO indicate there is a communication failure + * to the AF. Rest of the error codes indicate that AF processed + * VF messages and set the error codes in response messages + * (if any) so simply forward responses to VF. + */ + if (err == -EIO) { dev_warn(pf->dev, "AF not responding to VF%d messages\n", vf); /* restore PF mbase and exit */ @@ -853,6 +859,15 @@ static void otx2_handle_link_event(struct otx2_nic *pf) } } +int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf, + struct mcs_intr_info *event, + struct msg_rsp *rsp) +{ + cn10k_handle_mcs_event(pf, event); + + return 0; +} + int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf, struct cgx_link_info_msg *msg, struct msg_rsp *rsp) @@ -912,6 +927,7 @@ static int otx2_process_mbox_msg_up(struct otx2_nic *pf, return err; \ } MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M break; default: @@ -1115,7 +1131,7 @@ static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable) struct msg_req *msg; int err; - if (enable && bitmap_weight(&pf->flow_cfg->dmacflt_bmap, + if (enable && !bitmap_empty(pf->flow_cfg->dmacflt_bmap, pf->flow_cfg->dmacflt_max_flows)) netdev_warn(pf->netdev, "CGX/RPM internal loopback might not work as DMAC filters are active\n"); @@ -1156,6 +1172,59 @@ int otx2_set_real_num_queues(struct net_device *netdev, } EXPORT_SYMBOL(otx2_set_real_num_queues); +static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = { + "NIX_SQOPERR_OOR", + "NIX_SQOPERR_CTX_FAULT", + "NIX_SQOPERR_CTX_POISON", + "NIX_SQOPERR_DISABLED", + "NIX_SQOPERR_SIZE_ERR", + "NIX_SQOPERR_OFLOW", + "NIX_SQOPERR_SQB_NULL", + "NIX_SQOPERR_SQB_FAULT", + "NIX_SQOPERR_SQE_SZ_ZERO", +}; + +static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = { + "NIX_MNQERR_SQ_CTX_FAULT", + "NIX_MNQERR_SQ_CTX_POISON", + "NIX_MNQERR_SQB_FAULT", + "NIX_MNQERR_SQB_POISON", + "NIX_MNQERR_TOTAL_ERR", + "NIX_MNQERR_LSO_ERR", + "NIX_MNQERR_CQ_QUERY_ERR", + "NIX_MNQERR_MAX_SQE_SIZE_ERR", + "NIX_MNQERR_MAXLEN_ERR", + "NIX_MNQERR_SQE_SIZEM1_ZERO", +}; + +static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = { + "NIX_SND_STATUS_GOOD", + "NIX_SND_STATUS_SQ_CTX_FAULT", + "NIX_SND_STATUS_SQ_CTX_POISON", + "NIX_SND_STATUS_SQB_FAULT", + "NIX_SND_STATUS_SQB_POISON", + "NIX_SND_STATUS_HDR_ERR", + "NIX_SND_STATUS_EXT_ERR", + "NIX_SND_STATUS_JUMP_FAULT", + "NIX_SND_STATUS_JUMP_POISON", + "NIX_SND_STATUS_CRC_ERR", + "NIX_SND_STATUS_IMM_ERR", + "NIX_SND_STATUS_SG_ERR", + "NIX_SND_STATUS_MEM_ERR", + "NIX_SND_STATUS_INVALID_SUBDC", + "NIX_SND_STATUS_SUBDC_ORDER_ERR", + "NIX_SND_STATUS_DATA_FAULT", + "NIX_SND_STATUS_DATA_POISON", + "NIX_SND_STATUS_NPC_DROP_ACTION", + "NIX_SND_STATUS_LOCK_VIOL", + "NIX_SND_STATUS_NPC_UCAST_CHAN_ERR", + "NIX_SND_STATUS_NPC_MCAST_CHAN_ERR", + "NIX_SND_STATUS_NPC_MCAST_ABORT", + "NIX_SND_STATUS_NPC_VTAG_PTR_ERR", + "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR", + "NIX_SND_STATUS_SEND_STATS_ERR", +}; + static irqreturn_t otx2_q_intr_handler(int irq, void *data) { struct otx2_nic *pf = data; @@ -1189,46 +1258,67 @@ static irqreturn_t otx2_q_intr_handler(int irq, void *data) /* SQ */ for (qidx = 0; qidx < pf->hw.tot_tx_queues; qidx++) { + u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg; + u8 sq_op_err_code, mnq_err_code, snd_err_code; + + /* Below debug registers captures first errors corresponding to + * those registers. We don't have to check against SQ qid as + * these are fatal errors. + */ + ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT); val = otx2_atomic64_add((qidx << 44), ptr); otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) | (val & NIX_SQINT_BITS)); - if (!(val & (NIX_SQINT_BITS | BIT_ULL(42)))) - continue; - if (val & BIT_ULL(42)) { netdev_err(pf->netdev, "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n", qidx, otx2_read64(pf, NIX_LF_ERR_INT)); - } else { - if (val & BIT_ULL(NIX_SQINT_LMT_ERR)) { - netdev_err(pf->netdev, "SQ%lld: LMT store error NIX_LF_SQ_OP_ERR_DBG:0x%llx", - qidx, - otx2_read64(pf, - NIX_LF_SQ_OP_ERR_DBG)); - otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_MNQ_ERR)) { - netdev_err(pf->netdev, "SQ%lld: Meta-descriptor enqueue error NIX_LF_MNQ_ERR_DGB:0x%llx\n", - qidx, - otx2_read64(pf, NIX_LF_MNQ_ERR_DBG)); - otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_SEND_ERR)) { - netdev_err(pf->netdev, "SQ%lld: Send error, NIX_LF_SEND_ERR_DBG 0x%llx", - qidx, - otx2_read64(pf, - NIX_LF_SEND_ERR_DBG)); - otx2_write64(pf, NIX_LF_SEND_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) - netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", - qidx); + goto done; + } + + sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG); + if (!(sq_op_err_dbg & BIT(44))) + goto chk_mnq_err_dbg; + + sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(%llx) err=%s\n", + qidx, sq_op_err_dbg, nix_sqoperr_e_str[sq_op_err_code]); + + otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44)); + + if (sq_op_err_code == NIX_SQOPERR_SQB_NULL) + goto chk_mnq_err_dbg; + + /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure. + * TODO: But we are in irq context. How to call mbox functions which does sleep + */ + +chk_mnq_err_dbg: + mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG); + if (!(mnq_err_dbg & BIT(44))) + goto chk_snd_err_dbg; + + mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_MNQ_ERR_DBG(%llx) err=%s\n", + qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code]); + otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44)); + +chk_snd_err_dbg: + snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG); + if (snd_err_dbg & BIT(44)) { + snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s\n", + qidx, snd_err_dbg, nix_snd_status_e_str[snd_err_code]); + otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44)); } +done: + /* Print values and reset */ + if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) + netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", + qidx); + schedule_work(&pf->reset_task); } @@ -1249,6 +1339,7 @@ static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq) otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0)); /* Schedule NAPI */ + pf->napi_events++; napi_schedule_irqoff(&cq_poll->napi); return IRQ_HANDLED; @@ -1262,6 +1353,7 @@ static void otx2_disable_napi(struct otx2_nic *pf) for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) { cq_poll = &qset->napi[qidx]; + cancel_work_sync(&cq_poll->dim.work); napi_disable(&cq_poll->napi); netif_napi_del(&cq_poll->napi); } @@ -1306,6 +1398,9 @@ static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu) int total_size; int rbuf_size; + if (pf->hw.rbuf_len) + return ALIGN(pf->hw.rbuf_len, OTX2_ALIGN) + OTX2_HEAD_ROOM; + /* The data transferred by NIX to memory consists of actual packet * plus additional data which has timestamp and/or EDSA/HIGIG2 * headers if interface is configured in corresponding modes. @@ -1379,18 +1474,40 @@ static int otx2_init_hw_resources(struct otx2_nic *pf) goto err_free_sq_ptrs; } +#ifdef CONFIG_DCB + if (pf->pfc_en) { + err = otx2_pfc_txschq_alloc(pf); + if (err) { + mutex_unlock(&mbox->lock); + goto err_free_sq_ptrs; + } + } +#endif + err = otx2_config_nix_queues(pf); if (err) { mutex_unlock(&mbox->lock); goto err_free_txsch; } + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { - err = otx2_txschq_config(pf, lvl); + err = otx2_txschq_config(pf, lvl, 0, false); if (err) { mutex_unlock(&mbox->lock); goto err_free_nix_queues; } } + +#ifdef CONFIG_DCB + if (pf->pfc_en) { + err = otx2_pfc_txschq_config(pf); + if (err) { + mutex_unlock(&mbox->lock); + goto err_free_nix_queues; + } + } +#endif + mutex_unlock(&mbox->lock); return err; @@ -1445,6 +1562,11 @@ static void otx2_free_hw_resources(struct otx2_nic *pf) if (err) dev_err(pf->dev, "RVUPF: Failed to stop/free TX schedulers\n"); +#ifdef CONFIG_DCB + if (pf->pfc_en) + otx2_pfc_txschq_stop(pf); +#endif + mutex_lock(&mbox->lock); /* Disable backpressure */ if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK)) @@ -1538,6 +1660,24 @@ static void otx2_do_set_rx_mode(struct otx2_nic *pf) mutex_unlock(&pf->mbox.lock); } +static void otx2_dim_work(struct work_struct *w) +{ + struct dim_cq_moder cur_moder; + struct otx2_cq_poll *cq_poll; + struct otx2_nic *pfvf; + struct dim *dim; + + dim = container_of(w, struct dim, work); + cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); + cq_poll = container_of(dim, struct otx2_cq_poll, dim); + pfvf = (struct otx2_nic *)cq_poll->dev; + pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ? + CQ_TIMER_THRESH_MAX : cur_moder.usec; + pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ? + NAPI_POLL_WEIGHT : cur_moder.pkts; + dim->state = DIM_START_MEASURE; +} + int otx2_open(struct net_device *netdev) { struct otx2_nic *pf = netdev_priv(netdev); @@ -1604,8 +1744,9 @@ int otx2_open(struct net_device *netdev) cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ; cq_poll->dev = (void *)pf; - netif_napi_add(netdev, &cq_poll->napi, - otx2_napi_handler, NAPI_POLL_WEIGHT); + cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; + INIT_WORK(&cq_poll->dim.work, otx2_dim_work); + netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler); napi_enable(&cq_poll->napi); } @@ -1689,9 +1830,6 @@ int otx2_open(struct net_device *netdev) if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK)) otx2_handle_link_event(pf); - /* Restore pause frame settings */ - otx2_config_pause_frm(pf); - /* Install DMAC Filters */ if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT) otx2_dmacflt_reinstall_flows(pf); @@ -1713,7 +1851,6 @@ err_free_cints: vec = pci_irq_vector(pf->pdev, pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START); otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0)); - synchronize_irq(vec); free_irq(vec, pf); err_disable_napi: otx2_disable_napi(pf); @@ -1757,7 +1894,6 @@ int otx2_stop(struct net_device *netdev) vec = pci_irq_vector(pf->pdev, pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START); otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0)); - synchronize_irq(vec); free_irq(vec, pf); /* Cleanup CQ NAPI and IRQ */ @@ -1791,8 +1927,7 @@ int otx2_stop(struct net_device *netdev) kfree(qset->rq); kfree(qset->napi); /* Do not clear RQ/SQ ringsize settings */ - memset((void *)qset + offsetof(struct otx2_qset, sqe_cnt), 0, - sizeof(*qset) - offsetof(struct otx2_qset, sqe_cnt)); + memset_startat(qset, 0, sqe_cnt); return 0; } EXPORT_SYMBOL(otx2_stop); @@ -1829,6 +1964,30 @@ static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +static u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb, + struct net_device *sb_dev) +{ +#ifdef CONFIG_DCB + struct otx2_nic *pf = netdev_priv(netdev); + u8 vlan_prio; +#endif + +#ifdef CONFIG_DCB + if (!skb->vlan_present) + goto pick_tx; + + vlan_prio = skb->vlan_tci >> 13; + if ((vlan_prio > pf->hw.tx_queues - 1) || + !pf->pfc_alloc_status[vlan_prio]) + goto pick_tx; + + return vlan_prio; + +pick_tx: +#endif + return netdev_pick_tx(netdev, skb, NULL); +} + static netdev_features_t otx2_fix_features(struct net_device *dev, netdev_features_t features) { @@ -1858,9 +2017,7 @@ static int otx2_set_features(struct net_device *netdev, netdev_features_t features) { netdev_features_t changed = features ^ netdev->features; - bool ntuple = !!(features & NETIF_F_NTUPLE); struct otx2_nic *pf = netdev_priv(netdev); - bool tc = !!(features & NETIF_F_HW_TC); if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev)) return otx2_cgx_config_loopback(pf, @@ -1870,46 +2027,7 @@ static int otx2_set_features(struct net_device *netdev, return otx2_enable_rxvlan(pf, features & NETIF_F_HW_VLAN_CTAG_RX); - if ((changed & NETIF_F_NTUPLE) && !ntuple) - otx2_destroy_ntuple_flows(pf); - - if ((changed & NETIF_F_NTUPLE) && ntuple) { - if (!pf->flow_cfg->max_flows) { - netdev_err(netdev, - "Can't enable NTUPLE, MCAM entries not allocated\n"); - return -EINVAL; - } - } - - if ((changed & NETIF_F_HW_TC) && tc) { - if (!pf->flow_cfg->max_flows) { - netdev_err(netdev, - "Can't enable TC, MCAM entries not allocated\n"); - return -EINVAL; - } - } - - if ((changed & NETIF_F_HW_TC) && !tc && - pf->flow_cfg && pf->flow_cfg->nr_flows) { - netdev_err(netdev, "Can't disable TC hardware offload while flows are active\n"); - return -EBUSY; - } - - if ((changed & NETIF_F_NTUPLE) && ntuple && - (netdev->features & NETIF_F_HW_TC) && !(changed & NETIF_F_HW_TC)) { - netdev_err(netdev, - "Can't enable NTUPLE when TC is active, disable TC and retry\n"); - return -EINVAL; - } - - if ((changed & NETIF_F_HW_TC) && tc && - (netdev->features & NETIF_F_NTUPLE) && !(changed & NETIF_F_NTUPLE)) { - netdev_err(netdev, - "Can't enable TC when NTUPLE is active, disable NTUPLE and retry\n"); - return -EINVAL; - } - - return 0; + return otx2_handle_ntuple_tc_features(netdev, features); } static void otx2_reset_task(struct work_struct *work) @@ -2002,14 +2120,21 @@ int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr) if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) return -EFAULT; - /* reserved for future extensions */ - if (config.flags) - return -EINVAL; - switch (config.tx_type) { case HWTSTAMP_TX_OFF: + if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC) + pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC; + + cancel_delayed_work(&pfvf->ptp->synctstamp_work); otx2_config_hw_tx_tstamp(pfvf, false); break; + case HWTSTAMP_TX_ONESTEP_SYNC: + if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) + return -ERANGE; + pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC; + schedule_delayed_work(&pfvf->ptp->synctstamp_work, + msecs_to_jiffies(500)); + fallthrough; case HWTSTAMP_TX_ON: otx2_config_hw_tx_tstamp(pfvf, true); break; @@ -2468,6 +2593,7 @@ static const struct net_device_ops otx2_netdev_ops = { .ndo_open = otx2_open, .ndo_stop = otx2_stop, .ndo_start_xmit = otx2_xmit, + .ndo_select_queue = otx2_select_queue, .ndo_fix_features = otx2_fix_features, .ndo_set_mac_address = otx2_set_mac_address, .ndo_change_mtu = otx2_change_mtu, @@ -2624,6 +2750,9 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) hw->tx_queues = qcount; hw->tot_tx_queues = qcount; hw->max_queues = qcount; + hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN; + /* Use CQE of 128 byte descriptor size by default */ + hw->xqe_size = 128; num_vec = pci_msix_vec_count(pdev); hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE, @@ -2720,6 +2849,10 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_ptp_destroy; + err = cn10k_mcs_init(pf); + if (err) + goto err_del_mcam_entries; + if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT) netdev->hw_features |= NETIF_F_NTUPLE; @@ -2741,7 +2874,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL; - netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; + netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS); netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2_netdev_ops; @@ -2752,7 +2885,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_del_mcam_entries; + goto err_mcs_free; } err = otx2_wq_init(pf); @@ -2777,9 +2910,11 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) /* Enable link notifications */ otx2_cgx_config_linkevents(pf, true); - /* Enable pause frames by default */ - pf->flags |= OTX2_FLAG_RX_PAUSE_ENABLED; - pf->flags |= OTX2_FLAG_TX_PAUSE_ENABLED; +#ifdef CONFIG_DCB + err = otx2_dcbnl_set_ops(netdev); + if (err) + goto err_pf_sriov_init; +#endif return 0; @@ -2789,6 +2924,8 @@ err_mcam_flow_del: otx2_mcam_flow_del(pf); err_unreg_netdev: unregister_netdev(netdev); +err_mcs_free: + cn10k_mcs_free(pf); err_del_mcam_entries: otx2_mcam_flow_del(pf); err_ptp_destroy: @@ -2924,6 +3061,23 @@ static void otx2_remove(struct pci_dev *pdev) if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) otx2_config_hw_rx_tstamp(pf, false); + /* Disable 802.3x pause frames */ + if (pf->flags & OTX2_FLAG_RX_PAUSE_ENABLED || + (pf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) { + pf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED; + pf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED; + otx2_config_pause_frm(pf); + } + + cn10k_mcs_free(pf); + +#ifdef CONFIG_DCB + /* Disable PFC config */ + if (pf->pfc_en) { + pf->pfc_en = 0; + otx2_config_priority_flow_ctrl(pf); + } +#endif cancel_work_sync(&pf->reset_task); /* Disable link notifications */ otx2_cgx_config_linkevents(pf, false); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c index 61c20907315f..896b2f9bac34 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c @@ -10,6 +10,33 @@ #include "otx2_common.h" #include "otx2_ptp.h" +static u64 otx2_ptp_get_clock(struct otx2_ptp *ptp) +{ + struct ptp_req *req; + struct ptp_rsp *rsp; + int err; + + if (!ptp->nic) + return 0; + + req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox); + if (!req) + return 0; + + req->op = PTP_OP_GET_CLOCK; + + err = otx2_sync_mbox_msg(&ptp->nic->mbox); + if (err) + return 0; + + rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0, + &req->hdr); + if (IS_ERR(rsp)) + return 0; + + return rsp->clk; +} + static int otx2_ptp_adjfine(struct ptp_clock_info *ptp_info, long scaled_ppm) { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, @@ -46,32 +73,28 @@ static int ptp_set_thresh(struct otx2_ptp *ptp, u64 thresh) return otx2_sync_mbox_msg(&ptp->nic->mbox); } -static u64 ptp_cc_read(const struct cyclecounter *cc) +static int ptp_extts_on(struct otx2_ptp *ptp, int on) { - struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter); struct ptp_req *req; - struct ptp_rsp *rsp; - int err; if (!ptp->nic) - return 0; + return -ENODEV; req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox); if (!req) - return 0; + return -ENOMEM; - req->op = PTP_OP_GET_CLOCK; + req->op = PTP_OP_EXTTS_ON; + req->extts_on = on; - err = otx2_sync_mbox_msg(&ptp->nic->mbox); - if (err) - return 0; + return otx2_sync_mbox_msg(&ptp->nic->mbox); +} - rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0, - &req->hdr); - if (IS_ERR(rsp)) - return 0; +static u64 ptp_cc_read(const struct cyclecounter *cc) +{ + struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter); - return rsp->clk; + return otx2_ptp_get_clock(ptp); } static u64 ptp_tstmp_read(struct otx2_ptp *ptp) @@ -101,6 +124,15 @@ static u64 ptp_tstmp_read(struct otx2_ptp *ptp) return rsp->clk; } +static void otx2_get_ptpclock(struct otx2_ptp *ptp, u64 *tstamp) +{ + struct otx2_nic *pfvf = ptp->nic; + + mutex_lock(&pfvf->mbox.lock); + *tstamp = timecounter_read(&ptp->time_counter); + mutex_unlock(&pfvf->mbox.lock); +} + static int otx2_ptp_adjtime(struct ptp_clock_info *ptp_info, s64 delta) { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, @@ -119,14 +151,10 @@ static int otx2_ptp_gettime(struct ptp_clock_info *ptp_info, { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, ptp_info); - struct otx2_nic *pfvf = ptp->nic; - u64 nsec; + u64 tstamp; - mutex_lock(&pfvf->mbox.lock); - nsec = timecounter_read(&ptp->time_counter); - mutex_unlock(&pfvf->mbox.lock); - - *ts = ns_to_timespec64(nsec); + otx2_get_ptpclock(ptp, &tstamp); + *ts = ns_to_timespec64(tstamp); return 0; } @@ -178,8 +206,6 @@ static void otx2_ptp_extts_check(struct work_struct *work) event.index = 0; event.timestamp = timecounter_cyc2time(&ptp->time_counter, tstmp); ptp_clock_event(ptp->ptp_clock, &event); - ptp->last_extts = tstmp; - new_thresh = tstmp % 500000000; if (ptp->thresh != new_thresh) { mutex_lock(&ptp->nic->mbox.lock); @@ -187,10 +213,28 @@ static void otx2_ptp_extts_check(struct work_struct *work) mutex_unlock(&ptp->nic->mbox.lock); ptp->thresh = new_thresh; } + ptp->last_extts = tstmp; } schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200)); } +static void otx2_sync_tstamp(struct work_struct *work) +{ + struct otx2_ptp *ptp = container_of(work, struct otx2_ptp, + synctstamp_work.work); + struct otx2_nic *pfvf = ptp->nic; + u64 tstamp; + + mutex_lock(&pfvf->mbox.lock); + tstamp = otx2_ptp_get_clock(ptp); + mutex_unlock(&pfvf->mbox.lock); + + ptp->tstamp = timecounter_cyc2time(&pfvf->ptp->time_counter, tstamp); + ptp->base_ns = tstamp % NSEC_PER_SEC; + + schedule_delayed_work(&ptp->synctstamp_work, msecs_to_jiffies(250)); +} + static int otx2_ptp_enable(struct ptp_clock_info *ptp_info, struct ptp_clock_request *rq, int on) { @@ -207,10 +251,13 @@ static int otx2_ptp_enable(struct ptp_clock_info *ptp_info, rq->extts.index); if (pin < 0) return -EBUSY; - if (on) + if (on) { + ptp_extts_on(ptp, on); schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200)); - else + } else { + ptp_extts_on(ptp, on); cancel_delayed_work_sync(&ptp->extts_work); + } return 0; default: break; @@ -294,6 +341,16 @@ int otx2_ptp_init(struct otx2_nic *pfvf) goto error; } + if (is_dev_otx2(pfvf->pdev)) { + ptp_ptr->convert_rx_ptp_tstmp = &otx2_ptp_convert_rx_timestamp; + ptp_ptr->convert_tx_ptp_tstmp = &otx2_ptp_convert_tx_timestamp; + } else { + ptp_ptr->convert_rx_ptp_tstmp = &cn10k_ptp_convert_timestamp; + ptp_ptr->convert_tx_ptp_tstmp = &cn10k_ptp_convert_timestamp; + } + + INIT_DELAYED_WORK(&ptp_ptr->synctstamp_work, otx2_sync_tstamp); + pfvf->ptp = ptp_ptr; error: @@ -308,6 +365,8 @@ void otx2_ptp_destroy(struct otx2_nic *pfvf) if (!ptp) return; + cancel_delayed_work(&pfvf->ptp->synctstamp_work); + ptp_clock_unregister(ptp->ptp_clock); kfree(ptp); pfvf->ptp = NULL; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.h index 6ff284211d7b..7ff41927ceaf 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.h @@ -8,6 +8,21 @@ #ifndef OTX2_PTP_H #define OTX2_PTP_H +static inline u64 otx2_ptp_convert_rx_timestamp(u64 timestamp) +{ + return be64_to_cpu(*(__be64 *)×tamp); +} + +static inline u64 otx2_ptp_convert_tx_timestamp(u64 timestamp) +{ + return timestamp; +} + +static inline u64 cn10k_ptp_convert_timestamp(u64 timestamp) +{ + return ((timestamp >> 32) * NSEC_PER_SEC) + (timestamp & 0xFFFFFFFFUL); +} + int otx2_ptp_init(struct otx2_nic *pfvf); void otx2_ptp_destroy(struct otx2_nic *pfvf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h index 4bbd12ff26e6..fa37b9f312ca 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h @@ -236,8 +236,15 @@ struct nix_sqe_sg_s { /* NIX send memory subdescriptor structure */ struct nix_sqe_mem_s { - u64 offset : 16; /* W0 */ - u64 rsvd_51_16 : 36; + u64 start_offset : 8; + u64 rsvd_11_8 : 4; + u64 rsvd_12 : 1; + u64 udp_csum_crt : 1; + u64 update64 : 1; + u64 rsvd_15_16 : 1; + u64 base_ns : 32; + u64 step_type : 1; + u64 rsvd_51_49 : 3; u64 per_lso_seg : 1; u64 wmem : 1; u64 dsz : 2; @@ -274,4 +281,61 @@ enum nix_sqint_e { BIT_ULL(NIX_SQINT_SEND_ERR) | \ BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) +enum nix_sqoperr_e { + NIX_SQOPERR_OOR = 0, + NIX_SQOPERR_CTX_FAULT = 1, + NIX_SQOPERR_CTX_POISON = 2, + NIX_SQOPERR_DISABLED = 3, + NIX_SQOPERR_SIZE_ERR = 4, + NIX_SQOPERR_OFLOW = 5, + NIX_SQOPERR_SQB_NULL = 6, + NIX_SQOPERR_SQB_FAULT = 7, + NIX_SQOPERR_SQE_SZ_ZERO = 8, + NIX_SQOPERR_MAX, +}; + +enum nix_mnqerr_e { + NIX_MNQERR_SQ_CTX_FAULT = 0, + NIX_MNQERR_SQ_CTX_POISON = 1, + NIX_MNQERR_SQB_FAULT = 2, + NIX_MNQERR_SQB_POISON = 3, + NIX_MNQERR_TOTAL_ERR = 4, + NIX_MNQERR_LSO_ERR = 5, + NIX_MNQERR_CQ_QUERY_ERR = 6, + NIX_MNQERR_MAX_SQE_SIZE_ERR = 7, + NIX_MNQERR_MAXLEN_ERR = 8, + NIX_MNQERR_SQE_SIZEM1_ZERO = 9, + NIX_MNQERR_MAX, +}; + +enum nix_snd_status_e { + NIX_SND_STATUS_GOOD = 0x0, + NIX_SND_STATUS_SQ_CTX_FAULT = 0x1, + NIX_SND_STATUS_SQ_CTX_POISON = 0x2, + NIX_SND_STATUS_SQB_FAULT = 0x3, + NIX_SND_STATUS_SQB_POISON = 0x4, + NIX_SND_STATUS_HDR_ERR = 0x5, + NIX_SND_STATUS_EXT_ERR = 0x6, + NIX_SND_STATUS_JUMP_FAULT = 0x7, + NIX_SND_STATUS_JUMP_POISON = 0x8, + NIX_SND_STATUS_CRC_ERR = 0x9, + NIX_SND_STATUS_IMM_ERR = 0x10, + NIX_SND_STATUS_SG_ERR = 0x11, + NIX_SND_STATUS_MEM_ERR = 0x12, + NIX_SND_STATUS_INVALID_SUBDC = 0x13, + NIX_SND_STATUS_SUBDC_ORDER_ERR = 0x14, + NIX_SND_STATUS_DATA_FAULT = 0x15, + NIX_SND_STATUS_DATA_POISON = 0x16, + NIX_SND_STATUS_NPC_DROP_ACTION = 0x17, + NIX_SND_STATUS_LOCK_VIOL = 0x18, + NIX_SND_STATUS_NPC_UCAST_CHAN_ERR = 0x19, + NIX_SND_STATUS_NPC_MCAST_CHAN_ERR = 0x20, + NIX_SND_STATUS_NPC_MCAST_ABORT = 0x21, + NIX_SND_STATUS_NPC_VTAG_PTR_ERR = 0x22, + NIX_SND_STATUS_NPC_VTAG_SIZE_ERR = 0x23, + NIX_SND_STATUS_SEND_MEM_FAULT = 0x24, + NIX_SND_STATUS_SEND_STATS_ERR = 0x25, + NIX_SND_STATUS_MAX, +}; + #endif /* OTX2_STRUCT_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 626961a41089..e64318c110fd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -28,6 +28,9 @@ #define MAX_RATE_EXPONENT 0x0FULL #define MAX_RATE_MANTISSA 0xFFULL +#define CN10K_MAX_BURST_MANTISSA 0x7FFFULL +#define CN10K_MAX_BURST_SIZE 8453888ULL + /* Bitfields in NIX_TLX_PIR register */ #define TLX_RATE_MANTISSA GENMASK_ULL(8, 1) #define TLX_RATE_EXPONENT GENMASK_ULL(12, 9) @@ -35,6 +38,9 @@ #define TLX_BURST_MANTISSA GENMASK_ULL(36, 29) #define TLX_BURST_EXPONENT GENMASK_ULL(40, 37) +#define CN10K_TLX_BURST_MANTISSA GENMASK_ULL(43, 29) +#define CN10K_TLX_BURST_EXPONENT GENMASK_ULL(47, 44) + struct otx2_tc_flow_stats { u64 bytes; u64 pkts; @@ -58,7 +64,7 @@ int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic) { struct otx2_tc_info *tc = &nic->tc_info; - if (!nic->flow_cfg->max_flows || is_otx2_vf(nic->pcifunc)) + if (!nic->flow_cfg->max_flows) return 0; /* Max flows changed, free the existing bitmap */ @@ -77,33 +83,42 @@ int otx2_tc_alloc_ent_bitmap(struct otx2_nic *nic) } EXPORT_SYMBOL(otx2_tc_alloc_ent_bitmap); -static void otx2_get_egress_burst_cfg(u32 burst, u32 *burst_exp, - u32 *burst_mantissa) +static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, + u32 *burst_exp, u32 *burst_mantissa) { + int max_burst, max_mantissa; unsigned int tmp; + if (is_dev_otx2(nic->pdev)) { + max_burst = MAX_BURST_SIZE; + max_mantissa = MAX_BURST_MANTISSA; + } else { + max_burst = CN10K_MAX_BURST_SIZE; + max_mantissa = CN10K_MAX_BURST_MANTISSA; + } + /* Burst is calculated as * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256 * Max supported burst size is 130,816 bytes. */ - burst = min_t(u32, burst, MAX_BURST_SIZE); + burst = min_t(u32, burst, max_burst); if (burst) { *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0; tmp = burst - rounddown_pow_of_two(burst); - if (burst < MAX_BURST_MANTISSA) + if (burst < max_mantissa) *burst_mantissa = tmp * 2; else *burst_mantissa = tmp / (1ULL << (*burst_exp - 7)); } else { *burst_exp = MAX_BURST_EXPONENT; - *burst_mantissa = MAX_BURST_MANTISSA; + *burst_mantissa = max_mantissa; } } -static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp, +static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp, u32 *mantissa, u32 *div_exp) { - unsigned int tmp; + u64 tmp; /* Rate calculation by hardware * @@ -132,21 +147,44 @@ static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp, } } -static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 maxrate) +static u64 otx2_get_txschq_rate_regval(struct otx2_nic *nic, + u64 maxrate, u32 burst) { - struct otx2_hw *hw = &nic->hw; - struct nix_txschq_config *req; u32 burst_exp, burst_mantissa; u32 exp, mantissa, div_exp; + u64 regval = 0; + + /* Get exponent and mantissa values from the desired rate */ + otx2_get_egress_burst_cfg(nic, burst, &burst_exp, &burst_mantissa); + otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); + + if (is_dev_otx2(nic->pdev)) { + regval = FIELD_PREP(TLX_BURST_EXPONENT, (u64)burst_exp) | + FIELD_PREP(TLX_BURST_MANTISSA, (u64)burst_mantissa) | + FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | + FIELD_PREP(TLX_RATE_EXPONENT, exp) | + FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); + } else { + regval = FIELD_PREP(CN10K_TLX_BURST_EXPONENT, (u64)burst_exp) | + FIELD_PREP(CN10K_TLX_BURST_MANTISSA, (u64)burst_mantissa) | + FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | + FIELD_PREP(TLX_RATE_EXPONENT, exp) | + FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); + } + + return regval; +} + +static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, + u32 burst, u64 maxrate) +{ + struct otx2_hw *hw = &nic->hw; + struct nix_txschq_config *req; int txschq, err; /* All SQs share the same TL4, so pick the first scheduler */ txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; - /* Get exponent and mantissa values from the desired rate */ - otx2_get_egress_burst_cfg(burst, &burst_exp, &burst_mantissa); - otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp); - mutex_lock(&nic->mbox.lock); req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox); if (!req) { @@ -157,11 +195,7 @@ static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 ma req->lvl = NIX_TXSCH_LVL_TL4; req->num_regs = 1; req->reg[0] = NIX_AF_TL4X_PIR(txschq); - req->regval[0] = FIELD_PREP(TLX_BURST_EXPONENT, burst_exp) | - FIELD_PREP(TLX_BURST_MANTISSA, burst_mantissa) | - FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) | - FIELD_PREP(TLX_RATE_EXPONENT, exp) | - FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0); + req->regval[0] = otx2_get_txschq_rate_regval(nic, maxrate, burst); err = otx2_sync_mbox_msg(&nic->mbox); mutex_unlock(&nic->mbox.lock); @@ -190,13 +224,47 @@ static int otx2_tc_validate_flow(struct otx2_nic *nic, return 0; } +static int otx2_policer_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack) +{ + if (act->police.exceed.act_id != FLOW_ACTION_DROP) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when exceed action is not drop"); + return -EOPNOTSUPP; + } + + if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is not pipe or ok"); + return -EOPNOTSUPP; + } + + if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && + !flow_action_is_last_entry(action, act)) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is ok, but action is not last"); + return -EOPNOTSUPP; + } + + if (act->police.peakrate_bytes_ps || + act->police.avrate || act->police.overhead) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when peakrate/avrate/overhead is configured"); + return -EOPNOTSUPP; + } + + return 0; +} + static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, struct tc_cls_matchall_offload *cls) { struct netlink_ext_ack *extack = cls->common.extack; struct flow_action *actions = &cls->rule->action; struct flow_action_entry *entry; - u32 rate; + u64 rate; int err; err = otx2_tc_validate_flow(nic, actions, extack); @@ -212,13 +280,17 @@ static int otx2_tc_egress_matchall_install(struct otx2_nic *nic, entry = &cls->rule->action.entries[0]; switch (entry->id) { case FLOW_ACTION_POLICE: + err = otx2_policer_validate(&cls->rule->action, entry, extack); + if (err) + return err; + if (entry->police.rate_pkt_ps) { NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second"); return -EOPNOTSUPP; } /* Convert bytes per second to Mbps */ rate = entry->police.rate_bytes_ps * 8; - rate = max_t(u32, rate / 1000000, 1); + rate = max_t(u64, rate / 1000000, 1); err = otx2_set_matchall_egress_rate(nic, entry->police.burst, rate); if (err) return err; @@ -315,6 +387,7 @@ static int otx2_tc_parse_actions(struct otx2_nic *nic, u8 nr_police = 0; bool pps = false; u64 rate; + int err; int i; if (!flow_action_has_entries(flow_action)) { @@ -355,6 +428,10 @@ static int otx2_tc_parse_actions(struct otx2_nic *nic, return -EOPNOTSUPP; } + err = otx2_policer_validate(flow_action, act, extack); + if (err) + return err; + if (act->police.rate_bytes_ps > 0) { rate = act->police.rate_bytes_ps * 8; burst = act->police.burst; @@ -571,21 +648,27 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, flow_spec->dport = match.key->dst; flow_mask->dport = match.mask->dst; - if (ip_proto == IPPROTO_UDP) - req->features |= BIT_ULL(NPC_DPORT_UDP); - else if (ip_proto == IPPROTO_TCP) - req->features |= BIT_ULL(NPC_DPORT_TCP); - else if (ip_proto == IPPROTO_SCTP) - req->features |= BIT_ULL(NPC_DPORT_SCTP); + + if (flow_mask->dport) { + if (ip_proto == IPPROTO_UDP) + req->features |= BIT_ULL(NPC_DPORT_UDP); + else if (ip_proto == IPPROTO_TCP) + req->features |= BIT_ULL(NPC_DPORT_TCP); + else if (ip_proto == IPPROTO_SCTP) + req->features |= BIT_ULL(NPC_DPORT_SCTP); + } flow_spec->sport = match.key->src; flow_mask->sport = match.mask->src; - if (ip_proto == IPPROTO_UDP) - req->features |= BIT_ULL(NPC_SPORT_UDP); - else if (ip_proto == IPPROTO_TCP) - req->features |= BIT_ULL(NPC_SPORT_TCP); - else if (ip_proto == IPPROTO_SCTP) - req->features |= BIT_ULL(NPC_SPORT_SCTP); + + if (flow_mask->sport) { + if (ip_proto == IPPROTO_UDP) + req->features |= BIT_ULL(NPC_SPORT_UDP); + else if (ip_proto == IPPROTO_TCP) + req->features |= BIT_ULL(NPC_SPORT_TCP); + else if (ip_proto == IPPROTO_SCTP) + req->features |= BIT_ULL(NPC_SPORT_SCTP); + } } return otx2_tc_parse_actions(nic, &rule->action, req, f, node); @@ -1023,6 +1106,7 @@ int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type, return -EOPNOTSUPP; } } +EXPORT_SYMBOL(otx2_setup_tc); static const struct rhashtable_params tc_flow_ht_params = { .head_offset = offsetof(struct otx2_tc_flow, node), @@ -1052,6 +1136,7 @@ int otx2_init_tc(struct otx2_nic *nic) tc->flow_ht_params = tc_flow_ht_params; return rhashtable_init(&tc->flow_table, &tc->flow_ht_params); } +EXPORT_SYMBOL(otx2_init_tc); void otx2_shutdown_tc(struct otx2_nic *nic) { @@ -1060,3 +1145,4 @@ void otx2_shutdown_tc(struct otx2_nic *nic) kfree(tc->tc_entries_bitmap); rhashtable_destroy(&tc->flow_table); } +EXPORT_SYMBOL(otx2_shutdown_tc); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 0cc6353254bf..ef10aef3cda0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -19,6 +19,12 @@ #include "cn10k.h" #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx))) +#define PTP_PORT 0x13F +/* PTPv2 header Original Timestamp starts at byte offset 34 and + * contains 6 byte seconds field and 4 byte nano seconds field. + */ +#define PTP_SYNC_SEC_OFFSET 34 + static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, struct bpf_prog *prog, struct nix_cqe_rx_s *cqe, @@ -148,6 +154,7 @@ static void otx2_snd_pkt_handler(struct otx2_nic *pfvf, if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) { timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id]; if (timestamp != 1) { + timestamp = pfvf->ptp->convert_tx_ptp_tstmp(timestamp); err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns); if (!err) { memset(&ts, 0, sizeof(ts)); @@ -167,14 +174,15 @@ static void otx2_snd_pkt_handler(struct otx2_nic *pfvf, static void otx2_set_rxtstamp(struct otx2_nic *pfvf, struct sk_buff *skb, void *data) { - u64 tsns; + u64 timestamp, tsns; int err; if (!(pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)) return; + timestamp = pfvf->ptp->convert_rx_ptp_tstmp(*(u64 *)data); /* The first 8 bytes is the timestamp */ - err = otx2_ptp_tstamp2time(pfvf, be64_to_cpu(*(__be64 *)data), &tsns); + err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns); if (err) return; @@ -433,6 +441,7 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int budget) { int tx_pkts = 0, tx_bytes = 0, qidx; + struct otx2_snd_queue *sq; struct nix_cqe_tx_s *cqe; int processed_cqe = 0; @@ -443,6 +452,9 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, return 0; process_cqe: + qidx = cq->cq_idx - pfvf->hw.rx_queues; + sq = &pfvf->qset.sq[qidx]; + while (likely(processed_cqe < budget) && cq->pend_cqe) { cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq); if (unlikely(!cqe)) { @@ -450,18 +462,20 @@ process_cqe: return 0; break; } + if (cq->cq_type == CQ_XDP) { - qidx = cq->cq_idx - pfvf->hw.rx_queues; - otx2_xdp_snd_pkt_handler(pfvf, &pfvf->qset.sq[qidx], - cqe); + otx2_xdp_snd_pkt_handler(pfvf, sq, cqe); } else { - otx2_snd_pkt_handler(pfvf, cq, - &pfvf->qset.sq[cq->cint_idx], - cqe, budget, &tx_pkts, &tx_bytes); + otx2_snd_pkt_handler(pfvf, cq, sq, cqe, budget, + &tx_pkts, &tx_bytes); } + cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID; processed_cqe++; cq->pend_cqe--; + + sq->cons_head++; + sq->cons_head &= (sq->sqe_cnt - 1); } /* Free CQEs to HW */ @@ -482,6 +496,18 @@ process_cqe: return 0; } +static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll) +{ + struct dim_sample dim_sample; + u64 rx_frames, rx_bytes; + + rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) + + OTX2_GET_RX_STATS(RX_UCAST); + rx_bytes = OTX2_GET_RX_STATS(RX_OCTS); + dim_update_sample(pfvf->napi_events, rx_frames, rx_bytes, &dim_sample); + net_dim(&cq_poll->dim, dim_sample); +} + int otx2_napi_handler(struct napi_struct *napi, int budget) { struct otx2_cq_queue *rx_cq = NULL; @@ -519,6 +545,17 @@ int otx2_napi_handler(struct napi_struct *napi, int budget) if (pfvf->flags & OTX2_FLAG_INTF_DOWN) return workdone; + /* Check for adaptive interrupt coalesce */ + if (workdone != 0 && + ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) == + OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) { + /* Adjust irq coalese using net_dim */ + otx2_adjust_adaptive_coalese(pfvf, cq_poll); + /* Update irq coalescing */ + for (i = 0; i < pfvf->hw.cint_cnt; i++) + otx2_config_irq_coalescing(pfvf, i); + } + /* Re-enable interrupts */ otx2_write64(pfvf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0)); @@ -599,7 +636,7 @@ static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, ext->subdc = NIX_SUBDC_EXT; if (skb_shinfo(skb)->gso_size) { ext->lso = 1; - ext->lso_sb = skb_transport_offset(skb) + tcp_hdrlen(skb); + ext->lso_sb = skb_tcp_all_headers(skb); ext->lso_mps = skb_shinfo(skb)->gso_size; /* Only TSOv4 and TSOv6 GSO offloads are supported */ @@ -661,7 +698,8 @@ static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, } static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, - int alg, u64 iova) + int alg, u64 iova, int ptp_offset, + u64 base_ns, int udp_csum) { struct nix_sqe_mem_s *mem; @@ -671,6 +709,13 @@ static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, mem->wmem = 1; /* wait for the memory operation */ mem->addr = iova; + if (ptp_offset) { + mem->start_offset = ptp_offset; + mem->udp_csum_crt = udp_csum; + mem->base_ns = base_ns; + mem->step_type = 1; + } + *offset += sizeof(*mem); } @@ -906,7 +951,7 @@ static bool is_hw_tso_supported(struct otx2_nic *pfvf, * be correctly modified, hence don't offload such TSO segments. */ - payload_len = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + payload_len = skb->len - skb_tcp_all_headers(skb); last_seg_size = payload_len % skb_shinfo(skb)->gso_size; if (last_seg_size && last_seg_size < 16) return false; @@ -927,16 +972,102 @@ static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb) return skb_shinfo(skb)->gso_segs; } +static bool otx2_validate_network_transport(struct sk_buff *skb) +{ + if ((ip_hdr(skb)->protocol == IPPROTO_UDP) || + (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) { + struct udphdr *udph = udp_hdr(skb); + + if (udph->source == htons(PTP_PORT) && + udph->dest == htons(PTP_PORT)) + return true; + } + + return false; +} + +static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, int *udp_csum) +{ + struct ethhdr *eth = (struct ethhdr *)(skb->data); + u16 nix_offload_hlen = 0, inner_vhlen = 0; + u8 *data = skb->data, *msgtype; + __be16 proto = eth->h_proto; + int network_depth = 0; + + /* NIX is programmed to offload outer VLAN header + * in case of single vlan protocol field holds Network header ETH_IP/V6 + * in case of stacked vlan protocol field holds Inner vlan (8100) + */ + if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX && + skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) { + if (skb->vlan_proto == htons(ETH_P_8021AD)) { + /* Get vlan protocol */ + proto = __vlan_get_protocol(skb, eth->h_proto, NULL); + /* SKB APIs like skb_transport_offset does not include + * offloaded vlan header length. Need to explicitly add + * the length + */ + nix_offload_hlen = VLAN_HLEN; + inner_vhlen = VLAN_HLEN; + } else if (skb->vlan_proto == htons(ETH_P_8021Q)) { + nix_offload_hlen = VLAN_HLEN; + } + } else if (eth_type_vlan(eth->h_proto)) { + proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth); + } + + switch (ntohs(proto)) { + case ETH_P_1588: + if (network_depth) + *offset = network_depth; + else + *offset = ETH_HLEN + nix_offload_hlen + + inner_vhlen; + break; + case ETH_P_IP: + case ETH_P_IPV6: + if (!otx2_validate_network_transport(skb)) + return false; + + *udp_csum = 1; + *offset = nix_offload_hlen + skb_transport_offset(skb) + + sizeof(struct udphdr); + } + + msgtype = data + *offset; + + /* Check PTP messageId is SYNC or not */ + return (*msgtype & 0xf) == 0; +} + static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb, struct otx2_snd_queue *sq, int *offset) { + struct ptpv2_tstamp *origin_tstamp; + int ptp_offset = 0, udp_csum = 0; + struct timespec64 ts; u64 iova; - if (!skb_shinfo(skb)->gso_size && - skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + if (unlikely(!skb_shinfo(skb)->gso_size && + (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) { + if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)) { + if (otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum)) { + origin_tstamp = (struct ptpv2_tstamp *) + ((u8 *)skb->data + ptp_offset + + PTP_SYNC_SEC_OFFSET); + ts = ns_to_timespec64(pfvf->ptp->tstamp); + origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff); + origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff); + origin_tstamp->nanoseconds = htonl(ts.tv_nsec); + /* Point to correction field in PTP packet */ + ptp_offset += 8; + } + } else { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + } iova = sq->timestamps->iova + (sq->head * sizeof(u64)); - otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova); + otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova, + ptp_offset, pfvf->ptp->base_ns, udp_csum); } else { skb_tx_timestamp(skb); } @@ -947,17 +1078,17 @@ bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq, { struct netdev_queue *txq = netdev_get_tx_queue(netdev, qidx); struct otx2_nic *pfvf = netdev_priv(netdev); - int offset, num_segs, free_sqe; + int offset, num_segs, free_desc; struct nix_sqe_hdr_s *sqe_hdr; - /* Check if there is room for new SQE. - * 'Num of SQBs freed to SQ's pool - SQ's Aura count' - * will give free SQE count. + /* Check if there is enough room between producer + * and consumer index. */ - free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb; + free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1); + if (free_desc < sq->sqe_thresh) + return false; - if (free_sqe < sq->sqe_thresh || - free_sqe < otx2_get_sqe_count(pfvf, skb)) + if (free_desc < otx2_get_sqe_count(pfvf, skb)) return false; num_segs = skb_shinfo(skb)->nr_frags + 1; @@ -1198,7 +1329,7 @@ static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, put_page(page); break; default: - bpf_warn_invalid_xdp_action(act); + bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act); break; case XDP_ABORTED: trace_xdp_exception(pfvf->netdev, prog, act); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h index f1a04cf9210c..93cac2c2664c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h @@ -21,7 +21,7 @@ #define OTX2_HEAD_ROOM OTX2_ALIGN #define OTX2_ETH_HLEN (VLAN_ETH_HLEN + VLAN_HLEN) -#define OTX2_MIN_MTU 64 +#define OTX2_MIN_MTU 60 #define OTX2_MAX_GSO_SEGS 255 #define OTX2_MAX_FRAGS_IN_SQE 9 @@ -79,6 +79,7 @@ struct sg_list { struct otx2_snd_queue { u8 aura_id; u16 head; + u16 cons_head; u16 sqe_size; u32 sqe_cnt; u16 num_sqbs; @@ -109,6 +110,7 @@ struct otx2_cq_poll { #define CINT_INVALID_CQ 255 u8 cint_idx; u8 cq_ids[CQS_PER_CINT]; + struct dim dim; struct napi_struct napi; }; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index 78944ad3492f..86653bb8e403 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -472,23 +472,7 @@ static void otx2vf_reset_task(struct work_struct *work) static int otx2vf_set_features(struct net_device *netdev, netdev_features_t features) { - netdev_features_t changed = features ^ netdev->features; - bool ntuple_enabled = !!(features & NETIF_F_NTUPLE); - struct otx2_nic *vf = netdev_priv(netdev); - - if (changed & NETIF_F_NTUPLE) { - if (!ntuple_enabled) { - otx2_mcam_flow_del(vf); - return 0; - } - - if (!otx2_get_maxflows(vf->flow_cfg)) { - netdev_err(netdev, - "Can't enable NTUPLE, MCAM entries not allocated\n"); - return -EINVAL; - } - } - return 0; + return otx2_handle_ntuple_tc_features(netdev, features); } static const struct net_device_ops otx2vf_netdev_ops = { @@ -502,6 +486,7 @@ static const struct net_device_ops otx2vf_netdev_ops = { .ndo_get_stats64 = otx2_get_stats64, .ndo_tx_timeout = otx2_tx_timeout, .ndo_eth_ioctl = otx2_ioctl, + .ndo_setup_tc = otx2_setup_tc, }; static int otx2_wq_init(struct otx2_nic *vf) @@ -586,6 +571,9 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) hw->tx_queues = qcount; hw->max_queues = qcount; hw->tot_tx_queues = qcount; + hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN; + /* Use CQE of 128 byte descriptor size by default */ + hw->xqe_size = 128; hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE, GFP_KERNEL); @@ -662,8 +650,9 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->hw_features |= NETIF_F_NTUPLE; netdev->hw_features |= NETIF_F_RXALL; + netdev->hw_features |= NETIF_F_HW_TC; - netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; + netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS); netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2vf_netdev_ops; @@ -684,7 +673,7 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_detach_rsrc; + goto err_ptp_destroy; } err = otx2_wq_init(vf); @@ -697,18 +686,28 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_unreg_netdev; - err = otx2_register_dl(vf); + err = otx2_init_tc(vf); if (err) goto err_unreg_netdev; - /* Enable pause frames by default */ - vf->flags |= OTX2_FLAG_RX_PAUSE_ENABLED; - vf->flags |= OTX2_FLAG_TX_PAUSE_ENABLED; + err = otx2_register_dl(vf); + if (err) + goto err_shutdown_tc; + +#ifdef CONFIG_DCB + err = otx2_dcbnl_set_ops(netdev); + if (err) + goto err_shutdown_tc; +#endif return 0; +err_shutdown_tc: + otx2_shutdown_tc(vf); err_unreg_netdev: unregister_netdev(netdev); +err_ptp_destroy: + otx2_ptp_destroy(vf); err_detach_rsrc: if (test_bit(CN10K_LMTST, &vf->hw.cap_flag)) qmem_free(vf->dev, vf->dync_lmt); @@ -737,11 +736,28 @@ static void otx2vf_remove(struct pci_dev *pdev) vf = netdev_priv(netdev); + /* Disable 802.3x pause frames */ + if (vf->flags & OTX2_FLAG_RX_PAUSE_ENABLED || + (vf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) { + vf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED; + vf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED; + otx2_config_pause_frm(vf); + } + +#ifdef CONFIG_DCB + /* Disable PFC config */ + if (vf->pfc_en) { + vf->pfc_en = 0; + otx2_config_priority_flow_ctrl(vf); + } +#endif + cancel_work_sync(&vf->reset_task); otx2_unregister_dl(vf); unregister_netdev(netdev); if (vf->otx2_wq) destroy_workqueue(vf->otx2_wq); + otx2_ptp_destroy(vf); otx2vf_disable_mbox_intr(vf); otx2_detach_resources(&vf->mbox); if (test_bit(CN10K_LMTST, &vf->hw.cap_flag)) diff --git a/drivers/net/ethernet/marvell/prestera/Kconfig b/drivers/net/ethernet/marvell/prestera/Kconfig index b6f20e2034c6..f2f7663c3d10 100644 --- a/drivers/net/ethernet/marvell/prestera/Kconfig +++ b/drivers/net/ethernet/marvell/prestera/Kconfig @@ -8,6 +8,7 @@ config PRESTERA depends on NET_SWITCHDEV && VLAN_8021Q depends on BRIDGE || BRIDGE=n select NET_DEVLINK + select PHYLINK help This driver supports Marvell Prestera Switch ASICs family. diff --git a/drivers/net/ethernet/marvell/prestera/Makefile b/drivers/net/ethernet/marvell/prestera/Makefile index 0609df8b913d..df14cee80153 100644 --- a/drivers/net/ethernet/marvell/prestera/Makefile +++ b/drivers/net/ethernet/marvell/prestera/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_PRESTERA) += prestera.o prestera-objs := prestera_main.o prestera_hw.o prestera_dsa.o \ prestera_rxtx.o prestera_devlink.o prestera_ethtool.o \ prestera_switchdev.o prestera_acl.o prestera_flow.o \ - prestera_flower.o prestera_span.o + prestera_flower.o prestera_span.o prestera_counter.o \ + prestera_router.o prestera_router_hw.o prestera_matchall.o obj-$(CONFIG_PRESTERA_PCI) += prestera_pci.o diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h index 2a4c14c704c0..35554ee805cd 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera.h +++ b/drivers/net/ethernet/marvell/prestera/prestera.h @@ -7,6 +7,7 @@ #include <linux/notifier.h> #include <linux/skbuff.h> #include <linux/workqueue.h> +#include <linux/phylink.h> #include <net/devlink.h> #include <uapi/linux/if_ether.h> @@ -20,6 +21,26 @@ struct prestera_fw_rev { u16 sub; }; +struct prestera_flood_domain { + struct prestera_switch *sw; + struct list_head flood_domain_port_list; + u32 idx; +}; + +struct prestera_mdb_entry { + struct prestera_switch *sw; + struct prestera_flood_domain *flood_domain; + unsigned char addr[ETH_ALEN]; + u16 vid; +}; + +struct prestera_flood_domain_port { + struct prestera_flood_domain *flood_domain; + struct net_device *dev; + struct list_head flood_domain_port_node; + u16 vid; +}; + struct prestera_port_stats { u64 good_octets_received; u64 bad_octets_received; @@ -72,6 +93,7 @@ struct prestera_lag { struct prestera_flow_block; struct prestera_port_mac_state { + bool valid; u32 mode; u32 speed; bool oper; @@ -107,7 +129,8 @@ struct prestera_port_phy_config { struct prestera_port { struct net_device *dev; struct prestera_switch *sw; - struct prestera_flow_block *flow_block; + struct prestera_flow_block *ingress_flow_block; + struct prestera_flow_block *egress_flow_block; struct devlink_port dl_port; struct list_head lag_member; struct prestera_lag *lag; @@ -130,6 +153,13 @@ struct prestera_port { struct prestera_port_phy_config cfg_phy; struct prestera_port_mac_state state_mac; struct prestera_port_phy_state state_phy; + + struct phylink_config phy_config; + struct phylink *phy_link; + struct phylink_pcs phylink_pcs; + + /* protects state_mac */ + spinlock_t state_mac_lock; }; struct prestera_device { @@ -225,6 +255,29 @@ struct prestera_event { }; }; +enum prestera_if_type { + /* the interface is of port type (dev,port) */ + PRESTERA_IF_PORT_E = 0, + + /* the interface is of lag type (lag-id) */ + PRESTERA_IF_LAG_E = 1, + + /* the interface is of Vid type (vlan-id) */ + PRESTERA_IF_VID_E = 3, +}; + +struct prestera_iface { + enum prestera_if_type type; + struct { + u32 hw_dev_num; + u32 port_num; + } dev_port; + u32 hw_dev_num; + u16 vr_id; + u16 lag_id; + u16 vlan_id; +}; + struct prestera_switchdev; struct prestera_span; struct prestera_rxtx; @@ -247,9 +300,33 @@ struct prestera_switch { u32 mtu_min; u32 mtu_max; u8 id; + struct device_node *np; + struct prestera_router *router; struct prestera_lag *lags; + struct prestera_counter *counter; u8 lag_member_max; u8 lag_max; + u32 size_tbl_router_nexthop; +}; + +struct prestera_router { + struct prestera_switch *sw; + struct list_head vr_list; + struct list_head rif_entry_list; + struct rhashtable nh_neigh_ht; + struct rhashtable nexthop_group_ht; + struct rhashtable fib_ht; + struct rhashtable kern_neigh_cache_ht; + struct rhashtable kern_fib_cache_ht; + struct notifier_block inetaddr_nb; + struct notifier_block inetaddr_valid_nb; + struct notifier_block fib_nb; + struct notifier_block netevent_nb; + u8 *nhgrp_hw_state_cache; /* Bitmap cached hw state of nhs */ + unsigned long nhgrp_hw_cache_kick; /* jiffies */ + struct { + struct delayed_work dw; + } neighs_update; }; struct prestera_rxtx_params { @@ -279,8 +356,13 @@ struct prestera_port *prestera_port_find_by_hwid(struct prestera_switch *sw, int prestera_port_autoneg_set(struct prestera_port *port, u64 link_modes); +int prestera_router_init(struct prestera_switch *sw); +void prestera_router_fini(struct prestera_switch *sw); + struct prestera_port *prestera_find_port(struct prestera_switch *sw, u32 id); +struct prestera_switch *prestera_switch_get(struct net_device *dev); + int prestera_port_cfg_mac_read(struct prestera_port *port, struct prestera_port_mac_config *cfg); @@ -289,14 +371,47 @@ int prestera_port_cfg_mac_write(struct prestera_port *port, struct prestera_port *prestera_port_dev_lower_find(struct net_device *dev); +void prestera_queue_work(struct work_struct *work); +void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay); +void prestera_queue_drain(void); + +int prestera_port_learning_set(struct prestera_port *port, bool learn_enable); +int prestera_port_uc_flood_set(struct prestera_port *port, bool flood); +int prestera_port_mc_flood_set(struct prestera_port *port, bool flood); + +int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked); + int prestera_port_pvid_set(struct prestera_port *port, u16 vid); bool prestera_netdev_check(const struct net_device *dev); +int prestera_is_valid_mac_addr(struct prestera_port *port, const u8 *addr); + bool prestera_port_is_lag_member(const struct prestera_port *port); +int prestera_lag_id(struct prestera_switch *sw, + struct net_device *lag_dev, u16 *lag_id); struct prestera_lag *prestera_lag_by_id(struct prestera_switch *sw, u16 id); u16 prestera_port_lag_id(const struct prestera_port *port); +struct prestera_mdb_entry * +prestera_mdb_entry_create(struct prestera_switch *sw, + const unsigned char *addr, u16 vid); +void prestera_mdb_entry_destroy(struct prestera_mdb_entry *mdb_entry); + +struct prestera_flood_domain * +prestera_flood_domain_create(struct prestera_switch *sw); +void prestera_flood_domain_destroy(struct prestera_flood_domain *flood_domain); + +int +prestera_flood_domain_port_create(struct prestera_flood_domain *flood_domain, + struct net_device *dev, + u16 vid); +void +prestera_flood_domain_port_destroy(struct prestera_flood_domain_port *port); +struct prestera_flood_domain_port * +prestera_flood_domain_port_find(struct prestera_flood_domain *flood_domain, + struct net_device *dev, u16 vid); + #endif /* _PRESTERA_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c index 83c75ffb1a1c..cba89fda504b 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c @@ -1,35 +1,88 @@ // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 -/* Copyright (c) 2020 Marvell International Ltd. All rights reserved */ +/* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved */ #include <linux/rhashtable.h> -#include "prestera.h" -#include "prestera_hw.h" #include "prestera_acl.h" -#include "prestera_span.h" +#include "prestera_flow.h" +#include "prestera_hw.h" +#include "prestera.h" + +#define ACL_KEYMASK_SIZE \ + (sizeof(__be32) * __PRESTERA_ACL_RULE_MATCH_TYPE_MAX) struct prestera_acl { struct prestera_switch *sw; + struct list_head vtcam_list; struct list_head rules; + struct rhashtable ruleset_ht; + struct rhashtable acl_rule_entry_ht; + struct idr uid; +}; + +struct prestera_acl_ruleset_ht_key { + struct prestera_flow_block *block; + u32 chain_index; +}; + +struct prestera_acl_rule_entry { + struct rhash_head ht_node; + struct prestera_acl_rule_entry_key key; + u32 hw_id; + u32 vtcam_id; + struct { + struct { + u8 valid:1; + } accept, drop, trap; + struct { + u8 valid:1; + struct prestera_acl_action_police i; + } police; + struct { + struct prestera_acl_action_jump i; + u8 valid:1; + } jump; + struct { + u32 id; + struct prestera_counter_block *block; + } counter; + }; }; struct prestera_acl_ruleset { + struct rhash_head ht_node; /* Member of acl HT */ + struct prestera_acl_ruleset_ht_key ht_key; struct rhashtable rule_ht; - struct prestera_switch *sw; - u16 id; + struct prestera_acl *acl; + struct { + u32 min; + u32 max; + } prio; + unsigned long rule_count; + refcount_t refcount; + void *keymask; + u32 vtcam_id; + u32 index; + u16 pcl_id; + bool offload; + bool ingress; }; -struct prestera_acl_rule { - struct rhash_head ht_node; +struct prestera_acl_vtcam { struct list_head list; - struct list_head match_list; - struct list_head action_list; - struct prestera_flow_block *block; - unsigned long cookie; - u32 priority; - u8 n_actions; - u8 n_matches; + __be32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + refcount_t refcount; u32 id; + bool is_keymask_set; + u8 lookup; + u8 direction; +}; + +static const struct rhashtable_params prestera_acl_ruleset_ht_params = { + .key_len = sizeof(struct prestera_acl_ruleset_ht_key), + .key_offset = offsetof(struct prestera_acl_ruleset, ht_key), + .head_offset = offsetof(struct prestera_acl_ruleset, ht_node), + .automatic_shrinking = true, }; static const struct rhashtable_params prestera_acl_rule_ht_params = { @@ -39,28 +92,92 @@ static const struct rhashtable_params prestera_acl_rule_ht_params = { .automatic_shrinking = true, }; +static const struct rhashtable_params __prestera_acl_rule_entry_ht_params = { + .key_offset = offsetof(struct prestera_acl_rule_entry, key), + .head_offset = offsetof(struct prestera_acl_rule_entry, ht_node), + .key_len = sizeof(struct prestera_acl_rule_entry_key), + .automatic_shrinking = true, +}; + +int prestera_acl_chain_to_client(u32 chain_index, bool ingress, u32 *client) +{ + static const u32 ingress_client_map[] = { + PRESTERA_HW_COUNTER_CLIENT_INGRESS_LOOKUP_0, + PRESTERA_HW_COUNTER_CLIENT_INGRESS_LOOKUP_1, + PRESTERA_HW_COUNTER_CLIENT_INGRESS_LOOKUP_2 + }; + + if (!ingress) { + /* prestera supports only one chain on egress */ + if (chain_index > 0) + return -EINVAL; + + *client = PRESTERA_HW_COUNTER_CLIENT_EGRESS_LOOKUP; + return 0; + } + + if (chain_index >= ARRAY_SIZE(ingress_client_map)) + return -EINVAL; + + *client = ingress_client_map[chain_index]; + return 0; +} + +static bool prestera_acl_chain_is_supported(u32 chain_index, bool ingress) +{ + if (!ingress) + /* prestera supports only one chain on egress */ + return chain_index == 0; + + return (chain_index & ~PRESTERA_ACL_CHAIN_MASK) == 0; +} + static struct prestera_acl_ruleset * -prestera_acl_ruleset_create(struct prestera_switch *sw) +prestera_acl_ruleset_create(struct prestera_acl *acl, + struct prestera_flow_block *block, + u32 chain_index) { struct prestera_acl_ruleset *ruleset; + u32 uid = 0; int err; + if (!prestera_acl_chain_is_supported(chain_index, block->ingress)) + return ERR_PTR(-EINVAL); + ruleset = kzalloc(sizeof(*ruleset), GFP_KERNEL); if (!ruleset) return ERR_PTR(-ENOMEM); + ruleset->acl = acl; + ruleset->ingress = block->ingress; + ruleset->ht_key.block = block; + ruleset->ht_key.chain_index = chain_index; + refcount_set(&ruleset->refcount, 1); + err = rhashtable_init(&ruleset->rule_ht, &prestera_acl_rule_ht_params); if (err) goto err_rhashtable_init; - err = prestera_hw_acl_ruleset_create(sw, &ruleset->id); + err = idr_alloc_u32(&acl->uid, NULL, &uid, U8_MAX, GFP_KERNEL); if (err) goto err_ruleset_create; - ruleset->sw = sw; + /* make pcl-id based on uid */ + ruleset->pcl_id = PRESTERA_ACL_PCL_ID_MAKE((u8)uid, chain_index); + ruleset->index = uid; + + ruleset->prio.min = UINT_MAX; + ruleset->prio.max = 0; + + err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node, + prestera_acl_ruleset_ht_params); + if (err) + goto err_ruleset_ht_insert; return ruleset; +err_ruleset_ht_insert: + idr_remove(&acl->uid, uid); err_ruleset_create: rhashtable_destroy(&ruleset->rule_ht); err_rhashtable_init: @@ -68,271 +185,367 @@ err_rhashtable_init: return ERR_PTR(err); } -static void prestera_acl_ruleset_destroy(struct prestera_acl_ruleset *ruleset) -{ - prestera_hw_acl_ruleset_del(ruleset->sw, ruleset->id); - rhashtable_destroy(&ruleset->rule_ht); - kfree(ruleset); -} - -struct prestera_flow_block * -prestera_acl_block_create(struct prestera_switch *sw, struct net *net) -{ - struct prestera_flow_block *block; - - block = kzalloc(sizeof(*block), GFP_KERNEL); - if (!block) - return NULL; - INIT_LIST_HEAD(&block->binding_list); - block->net = net; - block->sw = sw; - - block->ruleset = prestera_acl_ruleset_create(sw); - if (IS_ERR(block->ruleset)) { - kfree(block); - return NULL; - } - - return block; -} - -void prestera_acl_block_destroy(struct prestera_flow_block *block) +int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, + void *keymask) { - prestera_acl_ruleset_destroy(block->ruleset); - WARN_ON(!list_empty(&block->binding_list)); - kfree(block); -} - -static struct prestera_flow_block_binding * -prestera_acl_block_lookup(struct prestera_flow_block *block, - struct prestera_port *port) -{ - struct prestera_flow_block_binding *binding; - - list_for_each_entry(binding, &block->binding_list, list) - if (binding->port == port) - return binding; + ruleset->keymask = kmemdup(keymask, ACL_KEYMASK_SIZE, GFP_KERNEL); + if (!ruleset->keymask) + return -ENOMEM; - return NULL; + return 0; } -int prestera_acl_block_bind(struct prestera_flow_block *block, - struct prestera_port *port) +int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset) { - struct prestera_flow_block_binding *binding; + struct prestera_acl_iface iface; + u32 vtcam_id; + int dir; int err; - if (WARN_ON(prestera_acl_block_lookup(block, port))) - return -EEXIST; + dir = ruleset->ingress ? + PRESTERA_HW_VTCAM_DIR_INGRESS : PRESTERA_HW_VTCAM_DIR_EGRESS; - binding = kzalloc(sizeof(*binding), GFP_KERNEL); - if (!binding) - return -ENOMEM; - binding->span_id = PRESTERA_SPAN_INVALID_ID; - binding->port = port; + if (ruleset->offload) + return -EEXIST; - err = prestera_hw_acl_port_bind(port, block->ruleset->id); + err = prestera_acl_vtcam_id_get(ruleset->acl, + ruleset->ht_key.chain_index, + dir, + ruleset->keymask, &vtcam_id); if (err) - goto err_rules_bind; + goto err_vtcam_create; + + if (ruleset->ht_key.chain_index) { + /* for chain > 0, bind iface index to pcl-id to be able + * to jump from any other ruleset to this one using the index. + */ + iface.index = ruleset->index; + iface.type = PRESTERA_ACL_IFACE_TYPE_INDEX; + err = prestera_hw_vtcam_iface_bind(ruleset->acl->sw, &iface, + vtcam_id, ruleset->pcl_id); + if (err) + goto err_ruleset_bind; + } - list_add(&binding->list, &block->binding_list); + ruleset->vtcam_id = vtcam_id; + ruleset->offload = true; return 0; -err_rules_bind: - kfree(binding); +err_ruleset_bind: + prestera_acl_vtcam_id_put(ruleset->acl, ruleset->vtcam_id); +err_vtcam_create: return err; } -int prestera_acl_block_unbind(struct prestera_flow_block *block, - struct prestera_port *port) +static void prestera_acl_ruleset_destroy(struct prestera_acl_ruleset *ruleset) { - struct prestera_flow_block_binding *binding; + struct prestera_acl *acl = ruleset->acl; + u8 uid = ruleset->pcl_id & PRESTERA_ACL_KEYMASK_PCL_ID_USER; + int err; - binding = prestera_acl_block_lookup(block, port); - if (!binding) - return -ENOENT; + rhashtable_remove_fast(&acl->ruleset_ht, &ruleset->ht_node, + prestera_acl_ruleset_ht_params); + + if (ruleset->offload) { + if (ruleset->ht_key.chain_index) { + struct prestera_acl_iface iface = { + .type = PRESTERA_ACL_IFACE_TYPE_INDEX, + .index = ruleset->index + }; + err = prestera_hw_vtcam_iface_unbind(acl->sw, &iface, + ruleset->vtcam_id); + WARN_ON(err); + } + WARN_ON(prestera_acl_vtcam_id_put(acl, ruleset->vtcam_id)); + } - list_del(&binding->list); + idr_remove(&acl->uid, uid); + rhashtable_destroy(&ruleset->rule_ht); + kfree(ruleset->keymask); + kfree(ruleset); +} - prestera_hw_acl_port_unbind(port, block->ruleset->id); +static struct prestera_acl_ruleset * +__prestera_acl_ruleset_lookup(struct prestera_acl *acl, + struct prestera_flow_block *block, + u32 chain_index) +{ + struct prestera_acl_ruleset_ht_key ht_key; - kfree(binding); - return 0; + memset(&ht_key, 0, sizeof(ht_key)); + ht_key.block = block; + ht_key.chain_index = chain_index; + return rhashtable_lookup_fast(&acl->ruleset_ht, &ht_key, + prestera_acl_ruleset_ht_params); } struct prestera_acl_ruleset * -prestera_acl_block_ruleset_get(struct prestera_flow_block *block) +prestera_acl_ruleset_lookup(struct prestera_acl *acl, + struct prestera_flow_block *block, + u32 chain_index) { - return block->ruleset; + struct prestera_acl_ruleset *ruleset; + + ruleset = __prestera_acl_ruleset_lookup(acl, block, chain_index); + if (!ruleset) + return ERR_PTR(-ENOENT); + + refcount_inc(&ruleset->refcount); + return ruleset; } -u16 prestera_acl_rule_ruleset_id_get(const struct prestera_acl_rule *rule) +struct prestera_acl_ruleset * +prestera_acl_ruleset_get(struct prestera_acl *acl, + struct prestera_flow_block *block, + u32 chain_index) { - return rule->block->ruleset->id; + struct prestera_acl_ruleset *ruleset; + + ruleset = __prestera_acl_ruleset_lookup(acl, block, chain_index); + if (ruleset) { + refcount_inc(&ruleset->refcount); + return ruleset; + } + + return prestera_acl_ruleset_create(acl, block, chain_index); } -struct net *prestera_acl_block_net(struct prestera_flow_block *block) +void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset) { - return block->net; + if (!refcount_dec_and_test(&ruleset->refcount)) + return; + + prestera_acl_ruleset_destroy(ruleset); } -struct prestera_switch *prestera_acl_block_sw(struct prestera_flow_block *block) +int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port) { - return block->sw; + struct prestera_acl_iface iface = { + .type = PRESTERA_ACL_IFACE_TYPE_PORT, + .port = port + }; + + return prestera_hw_vtcam_iface_bind(port->sw, &iface, ruleset->vtcam_id, + ruleset->pcl_id); } -struct prestera_acl_rule * -prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, - unsigned long cookie) +int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port) { - return rhashtable_lookup_fast(&ruleset->rule_ht, &cookie, - prestera_acl_rule_ht_params); + struct prestera_acl_iface iface = { + .type = PRESTERA_ACL_IFACE_TYPE_PORT, + .port = port + }; + + return prestera_hw_vtcam_iface_unbind(port->sw, &iface, + ruleset->vtcam_id); } -struct prestera_acl_rule * -prestera_acl_rule_create(struct prestera_flow_block *block, - unsigned long cookie) +static int prestera_acl_ruleset_block_bind(struct prestera_acl_ruleset *ruleset, + struct prestera_flow_block *block) { - struct prestera_acl_rule *rule; + struct prestera_flow_block_binding *binding; + int err; - rule = kzalloc(sizeof(*rule), GFP_KERNEL); - if (!rule) - return ERR_PTR(-ENOMEM); + block->ruleset_zero = ruleset; + list_for_each_entry(binding, &block->binding_list, list) { + err = prestera_acl_ruleset_bind(ruleset, binding->port); + if (err) + goto rollback; + } + return 0; - INIT_LIST_HEAD(&rule->match_list); - INIT_LIST_HEAD(&rule->action_list); - rule->cookie = cookie; - rule->block = block; +rollback: + list_for_each_entry_continue_reverse(binding, &block->binding_list, + list) + err = prestera_acl_ruleset_unbind(ruleset, binding->port); + block->ruleset_zero = NULL; - return rule; + return err; } -struct list_head * -prestera_acl_rule_match_list_get(struct prestera_acl_rule *rule) +static void +prestera_acl_ruleset_block_unbind(struct prestera_acl_ruleset *ruleset, + struct prestera_flow_block *block) { - return &rule->match_list; + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + prestera_acl_ruleset_unbind(ruleset, binding->port); + block->ruleset_zero = NULL; } -struct list_head * -prestera_acl_rule_action_list_get(struct prestera_acl_rule *rule) +static void +prestera_acl_ruleset_prio_refresh(struct prestera_acl *acl, + struct prestera_acl_ruleset *ruleset) { - return &rule->action_list; + struct prestera_acl_rule *rule; + + ruleset->prio.min = UINT_MAX; + ruleset->prio.max = 0; + + list_for_each_entry(rule, &acl->rules, list) { + if (ruleset->ingress != rule->ruleset->ingress) + continue; + if (ruleset->ht_key.chain_index != rule->chain_index) + continue; + + ruleset->prio.min = min(ruleset->prio.min, rule->priority); + ruleset->prio.max = max(ruleset->prio.max, rule->priority); + } } -int prestera_acl_rule_action_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_action_entry *entry) +void +prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id) { - struct prestera_acl_rule_action_entry *a_entry; + struct prestera_acl_match *r_match = &rule->re_key.match; + __be16 pcl_id_mask = htons(PRESTERA_ACL_KEYMASK_PCL_ID); + __be16 pcl_id_key = htons(pcl_id); - a_entry = kmalloc(sizeof(*a_entry), GFP_KERNEL); - if (!a_entry) - return -ENOMEM; - - memcpy(a_entry, entry, sizeof(*entry)); - list_add(&a_entry->list, &rule->action_list); + rule_match_set(r_match->key, PCL_ID, pcl_id_key); + rule_match_set(r_match->mask, PCL_ID, pcl_id_mask); +} - rule->n_actions++; - return 0; +struct prestera_acl_rule * +prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, + unsigned long cookie) +{ + return rhashtable_lookup_fast(&ruleset->rule_ht, &cookie, + prestera_acl_rule_ht_params); } -u8 prestera_acl_rule_action_len(struct prestera_acl_rule *rule) +u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset) { - return rule->n_actions; + return ruleset->index; } -u32 prestera_acl_rule_priority_get(struct prestera_acl_rule *rule) +void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset, + u32 *prio_min, u32 *prio_max) { - return rule->priority; + *prio_min = ruleset->prio.min; + *prio_max = ruleset->prio.max; } -void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, - u32 priority) +bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset) { - rule->priority = priority; + return ruleset->offload; } -int prestera_acl_rule_match_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_match_entry *entry) +struct prestera_acl_rule * +prestera_acl_rule_create(struct prestera_acl_ruleset *ruleset, + unsigned long cookie, u32 chain_index) { - struct prestera_acl_rule_match_entry *m_entry; + struct prestera_acl_rule *rule; - m_entry = kmalloc(sizeof(*m_entry), GFP_KERNEL); - if (!m_entry) - return -ENOMEM; + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return ERR_PTR(-ENOMEM); - memcpy(m_entry, entry, sizeof(*entry)); - list_add(&m_entry->list, &rule->match_list); + rule->ruleset = ruleset; + rule->cookie = cookie; + rule->chain_index = chain_index; - rule->n_matches++; - return 0; + refcount_inc(&ruleset->refcount); + + return rule; } -u8 prestera_acl_rule_match_len(struct prestera_acl_rule *rule) +void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, + u32 priority) { - return rule->n_matches; + rule->priority = priority; } void prestera_acl_rule_destroy(struct prestera_acl_rule *rule) { - struct prestera_acl_rule_action_entry *a_entry; - struct prestera_acl_rule_match_entry *m_entry; - struct list_head *pos, *n; - - list_for_each_safe(pos, n, &rule->match_list) { - m_entry = list_entry(pos, typeof(*m_entry), list); - list_del(pos); - kfree(m_entry); - } - - list_for_each_safe(pos, n, &rule->action_list) { - a_entry = list_entry(pos, typeof(*a_entry), list); - list_del(pos); - kfree(a_entry); - } + if (rule->jump_ruleset) + /* release ruleset kept by jump action */ + prestera_acl_ruleset_put(rule->jump_ruleset); + prestera_acl_ruleset_put(rule->ruleset); kfree(rule); } +static void prestera_acl_ruleset_prio_update(struct prestera_acl_ruleset *ruleset, + u32 prio) +{ + ruleset->prio.min = min(ruleset->prio.min, prio); + ruleset->prio.max = max(ruleset->prio.max, prio); +} + int prestera_acl_rule_add(struct prestera_switch *sw, struct prestera_acl_rule *rule) { - u32 rule_id; int err; + struct prestera_acl_ruleset *ruleset = rule->ruleset; + struct prestera_flow_block *block = ruleset->ht_key.block; /* try to add rule to hash table first */ - err = rhashtable_insert_fast(&rule->block->ruleset->rule_ht, - &rule->ht_node, + err = rhashtable_insert_fast(&ruleset->rule_ht, &rule->ht_node, prestera_acl_rule_ht_params); if (err) - return err; + goto err_ht_insert; - /* add rule to hw */ - err = prestera_hw_acl_rule_add(sw, rule, &rule_id); + prestera_acl_rule_keymask_pcl_id_set(rule, ruleset->pcl_id); + rule->re_arg.vtcam_id = ruleset->vtcam_id; + rule->re_key.prio = rule->priority; + + rule->re = prestera_acl_rule_entry_find(sw->acl, &rule->re_key); + err = WARN_ON(rule->re) ? -EEXIST : 0; if (err) goto err_rule_add; - rule->id = rule_id; + rule->re = prestera_acl_rule_entry_create(sw->acl, &rule->re_key, + &rule->re_arg); + err = !rule->re ? -EINVAL : 0; + if (err) + goto err_rule_add; - list_add_tail(&rule->list, &sw->acl->rules); + /* bind the block (all ports) to chain index 0, rest of + * the chains are bound to goto action + */ + if (!ruleset->ht_key.chain_index && !ruleset->rule_count) { + err = prestera_acl_ruleset_block_bind(ruleset, block); + if (err) + goto err_acl_block_bind; + } + list_add_tail(&rule->list, &sw->acl->rules); + ruleset->rule_count++; + prestera_acl_ruleset_prio_update(ruleset, rule->priority); return 0; +err_acl_block_bind: + prestera_acl_rule_entry_destroy(sw->acl, rule->re); err_rule_add: - rhashtable_remove_fast(&rule->block->ruleset->rule_ht, &rule->ht_node, + rule->re = NULL; + rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node, prestera_acl_rule_ht_params); +err_ht_insert: return err; } void prestera_acl_rule_del(struct prestera_switch *sw, struct prestera_acl_rule *rule) { - rhashtable_remove_fast(&rule->block->ruleset->rule_ht, &rule->ht_node, + struct prestera_acl_ruleset *ruleset = rule->ruleset; + struct prestera_flow_block *block = ruleset->ht_key.block; + + rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node, prestera_acl_rule_ht_params); + ruleset->rule_count--; list_del(&rule->list); - prestera_hw_acl_rule_del(sw, rule->id); + + prestera_acl_rule_entry_destroy(sw->acl, rule->re); + prestera_acl_ruleset_prio_refresh(sw->acl, ruleset); + + /* unbind block (all ports) */ + if (!ruleset->ht_key.chain_index && !ruleset->rule_count) + prestera_acl_ruleset_block_unbind(ruleset, block); } -int prestera_acl_rule_get_stats(struct prestera_switch *sw, +int prestera_acl_rule_get_stats(struct prestera_acl *acl, struct prestera_acl_rule *rule, u64 *packets, u64 *bytes, u64 *last_use) { @@ -340,8 +553,10 @@ int prestera_acl_rule_get_stats(struct prestera_switch *sw, u64 current_bytes; int err; - err = prestera_hw_acl_rule_stats_get(sw, rule->id, ¤t_packets, - ¤t_bytes); + err = prestera_counter_stats_get(acl->sw->counter, + rule->re->counter.block, + rule->re->counter.id, + ¤t_packets, ¤t_bytes); if (err) return err; @@ -352,25 +567,361 @@ int prestera_acl_rule_get_stats(struct prestera_switch *sw, return 0; } +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_find(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key) +{ + return rhashtable_lookup_fast(&acl->acl_rule_entry_ht, key, + __prestera_acl_rule_entry_ht_params); +} + +static int __prestera_acl_rule_entry2hw_del(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e) +{ + return prestera_hw_vtcam_rule_del(sw, e->vtcam_id, e->hw_id); +} + +static int __prestera_acl_rule_entry2hw_add(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e) +{ + struct prestera_acl_hw_action_info act_hw[PRESTERA_ACL_RULE_ACTION_MAX]; + int act_num; + + memset(&act_hw, 0, sizeof(act_hw)); + act_num = 0; + + /* accept */ + if (e->accept.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_ACCEPT; + act_num++; + } + /* drop */ + if (e->drop.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_DROP; + act_num++; + } + /* trap */ + if (e->trap.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_TRAP; + act_num++; + } + /* police */ + if (e->police.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_POLICE; + act_hw[act_num].police = e->police.i; + act_num++; + } + /* jump */ + if (e->jump.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_JUMP; + act_hw[act_num].jump = e->jump.i; + act_num++; + } + /* counter */ + if (e->counter.block) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_COUNT; + act_hw[act_num].count.id = e->counter.id; + act_num++; + } + + return prestera_hw_vtcam_rule_add(sw, e->vtcam_id, e->key.prio, + e->key.match.key, e->key.match.mask, + act_hw, act_num, &e->hw_id); +} + +static void +__prestera_acl_rule_entry_act_destruct(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e) +{ + /* counter */ + prestera_counter_put(sw->counter, e->counter.block, e->counter.id); + /* police */ + if (e->police.valid) + prestera_hw_policer_release(sw, e->police.i.id); +} + +void prestera_acl_rule_entry_destroy(struct prestera_acl *acl, + struct prestera_acl_rule_entry *e) +{ + int ret; + + rhashtable_remove_fast(&acl->acl_rule_entry_ht, &e->ht_node, + __prestera_acl_rule_entry_ht_params); + + ret = __prestera_acl_rule_entry2hw_del(acl->sw, e); + WARN_ON(ret && ret != -ENODEV); + + __prestera_acl_rule_entry_act_destruct(acl->sw, e); + kfree(e); +} + +static int +__prestera_acl_rule_entry_act_construct(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e, + struct prestera_acl_rule_entry_arg *arg) +{ + int err; + + /* accept */ + e->accept.valid = arg->accept.valid; + /* drop */ + e->drop.valid = arg->drop.valid; + /* trap */ + e->trap.valid = arg->trap.valid; + /* jump */ + e->jump.valid = arg->jump.valid; + e->jump.i = arg->jump.i; + /* police */ + if (arg->police.valid) { + u8 type = arg->police.ingress ? PRESTERA_POLICER_TYPE_INGRESS : + PRESTERA_POLICER_TYPE_EGRESS; + + err = prestera_hw_policer_create(sw, type, &e->police.i.id); + if (err) + goto err_out; + + err = prestera_hw_policer_sr_tcm_set(sw, e->police.i.id, + arg->police.rate, + arg->police.burst); + if (err) { + prestera_hw_policer_release(sw, e->police.i.id); + goto err_out; + } + e->police.valid = arg->police.valid; + } + /* counter */ + if (arg->count.valid) { + err = prestera_counter_get(sw->counter, arg->count.client, + &e->counter.block, + &e->counter.id); + if (err) + goto err_out; + } + + return 0; + +err_out: + __prestera_acl_rule_entry_act_destruct(sw, e); + return -EINVAL; +} + +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_create(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key, + struct prestera_acl_rule_entry_arg *arg) +{ + struct prestera_acl_rule_entry *e; + int err; + + e = kzalloc(sizeof(*e), GFP_KERNEL); + if (!e) + goto err_kzalloc; + + memcpy(&e->key, key, sizeof(*key)); + e->vtcam_id = arg->vtcam_id; + err = __prestera_acl_rule_entry_act_construct(acl->sw, e, arg); + if (err) + goto err_act_construct; + + err = __prestera_acl_rule_entry2hw_add(acl->sw, e); + if (err) + goto err_hw_add; + + err = rhashtable_insert_fast(&acl->acl_rule_entry_ht, &e->ht_node, + __prestera_acl_rule_entry_ht_params); + if (err) + goto err_ht_insert; + + return e; + +err_ht_insert: + WARN_ON(__prestera_acl_rule_entry2hw_del(acl->sw, e)); +err_hw_add: + __prestera_acl_rule_entry_act_destruct(acl->sw, e); +err_act_construct: + kfree(e); +err_kzalloc: + return NULL; +} + +static int __prestera_acl_vtcam_id_try_fit(struct prestera_acl *acl, u8 lookup, + void *keymask, u32 *vtcam_id) +{ + struct prestera_acl_vtcam *vtcam; + int i; + + list_for_each_entry(vtcam, &acl->vtcam_list, list) { + if (lookup != vtcam->lookup) + continue; + + if (!keymask && !vtcam->is_keymask_set) + goto vtcam_found; + + if (!(keymask && vtcam->is_keymask_set)) + continue; + + /* try to fit with vtcam keymask */ + for (i = 0; i < __PRESTERA_ACL_RULE_MATCH_TYPE_MAX; i++) { + __be32 __keymask = ((__be32 *)keymask)[i]; + + if (!__keymask) + /* vtcam keymask in not interested */ + continue; + + if (__keymask & ~vtcam->keymask[i]) + /* keymask does not fit the vtcam keymask */ + break; + } + + if (i == __PRESTERA_ACL_RULE_MATCH_TYPE_MAX) + /* keymask fits vtcam keymask, return it */ + goto vtcam_found; + } + + /* nothing is found */ + return -ENOENT; + +vtcam_found: + refcount_inc(&vtcam->refcount); + *vtcam_id = vtcam->id; + return 0; +} + +int prestera_acl_vtcam_id_get(struct prestera_acl *acl, u8 lookup, u8 dir, + void *keymask, u32 *vtcam_id) +{ + struct prestera_acl_vtcam *vtcam; + u32 new_vtcam_id; + int err; + + /* find the vtcam that suits keymask. We do not expect to have + * a big number of vtcams, so, the list type for vtcam list is + * fine for now + */ + list_for_each_entry(vtcam, &acl->vtcam_list, list) { + if (lookup != vtcam->lookup || + dir != vtcam->direction) + continue; + + if (!keymask && !vtcam->is_keymask_set) { + refcount_inc(&vtcam->refcount); + goto vtcam_found; + } + + if (keymask && vtcam->is_keymask_set && + !memcmp(keymask, vtcam->keymask, sizeof(vtcam->keymask))) { + refcount_inc(&vtcam->refcount); + goto vtcam_found; + } + } + + /* vtcam not found, try to create new one */ + vtcam = kzalloc(sizeof(*vtcam), GFP_KERNEL); + if (!vtcam) + return -ENOMEM; + + err = prestera_hw_vtcam_create(acl->sw, lookup, keymask, &new_vtcam_id, + dir); + if (err) { + kfree(vtcam); + + /* cannot create new, try to fit into existing vtcam */ + if (__prestera_acl_vtcam_id_try_fit(acl, lookup, + keymask, &new_vtcam_id)) + return err; + + *vtcam_id = new_vtcam_id; + return 0; + } + + vtcam->direction = dir; + vtcam->id = new_vtcam_id; + vtcam->lookup = lookup; + if (keymask) { + memcpy(vtcam->keymask, keymask, sizeof(vtcam->keymask)); + vtcam->is_keymask_set = true; + } + refcount_set(&vtcam->refcount, 1); + list_add_rcu(&vtcam->list, &acl->vtcam_list); + +vtcam_found: + *vtcam_id = vtcam->id; + return 0; +} + +int prestera_acl_vtcam_id_put(struct prestera_acl *acl, u32 vtcam_id) +{ + struct prestera_acl_vtcam *vtcam; + int err; + + list_for_each_entry(vtcam, &acl->vtcam_list, list) { + if (vtcam_id != vtcam->id) + continue; + + if (!refcount_dec_and_test(&vtcam->refcount)) + return 0; + + err = prestera_hw_vtcam_destroy(acl->sw, vtcam->id); + if (err && err != -ENODEV) { + refcount_set(&vtcam->refcount, 1); + return err; + } + + list_del(&vtcam->list); + kfree(vtcam); + return 0; + } + + return -ENOENT; +} + int prestera_acl_init(struct prestera_switch *sw) { struct prestera_acl *acl; + int err; acl = kzalloc(sizeof(*acl), GFP_KERNEL); if (!acl) return -ENOMEM; + acl->sw = sw; INIT_LIST_HEAD(&acl->rules); + INIT_LIST_HEAD(&acl->vtcam_list); + idr_init(&acl->uid); + + err = rhashtable_init(&acl->acl_rule_entry_ht, + &__prestera_acl_rule_entry_ht_params); + if (err) + goto err_acl_rule_entry_ht_init; + + err = rhashtable_init(&acl->ruleset_ht, + &prestera_acl_ruleset_ht_params); + if (err) + goto err_ruleset_ht_init; + sw->acl = acl; - acl->sw = sw; return 0; + +err_ruleset_ht_init: + rhashtable_destroy(&acl->acl_rule_entry_ht); +err_acl_rule_entry_ht_init: + kfree(acl); + return err; } void prestera_acl_fini(struct prestera_switch *sw) { struct prestera_acl *acl = sw->acl; + WARN_ON(!idr_is_empty(&acl->uid)); + idr_destroy(&acl->uid); + + WARN_ON(!list_empty(&acl->vtcam_list)); WARN_ON(!list_empty(&acl->rules)); + + rhashtable_destroy(&acl->ruleset_ht); + rhashtable_destroy(&acl->acl_rule_entry_ht); + kfree(acl); } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.h b/drivers/net/ethernet/marvell/prestera/prestera_acl.h index 39b7869be659..a35cc0609a1d 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.h @@ -1,114 +1,162 @@ /* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ -/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ +/* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved. */ #ifndef _PRESTERA_ACL_H_ #define _PRESTERA_ACL_H_ -enum prestera_acl_rule_match_entry_type { - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE = 1, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_SRC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_DST, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE +#include <linux/types.h> +#include "prestera_counter.h" + +#define PRESTERA_ACL_KEYMASK_PCL_ID 0x3FF +#define PRESTERA_ACL_KEYMASK_PCL_ID_USER \ + (PRESTERA_ACL_KEYMASK_PCL_ID & 0x00FF) +#define PRESTERA_ACL_KEYMASK_PCL_ID_CHAIN \ + (PRESTERA_ACL_KEYMASK_PCL_ID & 0xFF00) +#define PRESTERA_ACL_CHAIN_MASK \ + (PRESTERA_ACL_KEYMASK_PCL_ID >> 8) + +#define PRESTERA_ACL_PCL_ID_MAKE(uid, chain_id) \ + (((uid) & PRESTERA_ACL_KEYMASK_PCL_ID_USER) | \ + (((chain_id) << 8) & PRESTERA_ACL_KEYMASK_PCL_ID_CHAIN)) + +#define rule_match_set_n(match_p, type, val_p, size) \ + memcpy(&(match_p)[PRESTERA_ACL_RULE_MATCH_TYPE_##type], \ + val_p, size) +#define rule_match_set(match_p, type, val) \ + memcpy(&(match_p)[PRESTERA_ACL_RULE_MATCH_TYPE_##type], \ + &(val), sizeof(val)) + +enum prestera_acl_match_type { + PRESTERA_ACL_RULE_MATCH_TYPE_PCL_ID, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_TYPE, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_DMAC_0, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_DMAC_1, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_SMAC_0, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_SMAC_1, + PRESTERA_ACL_RULE_MATCH_TYPE_IP_PROTO, + PRESTERA_ACL_RULE_MATCH_TYPE_SYS_PORT, + PRESTERA_ACL_RULE_MATCH_TYPE_SYS_DEV, + PRESTERA_ACL_RULE_MATCH_TYPE_IP_SRC, + PRESTERA_ACL_RULE_MATCH_TYPE_IP_DST, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_SRC, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_DST, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_RANGE_SRC, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_RANGE_DST, + PRESTERA_ACL_RULE_MATCH_TYPE_VLAN_ID, + PRESTERA_ACL_RULE_MATCH_TYPE_VLAN_TPID, + PRESTERA_ACL_RULE_MATCH_TYPE_ICMP_TYPE, + PRESTERA_ACL_RULE_MATCH_TYPE_ICMP_CODE, + + __PRESTERA_ACL_RULE_MATCH_TYPE_MAX }; enum prestera_acl_rule_action { - PRESTERA_ACL_RULE_ACTION_ACCEPT, - PRESTERA_ACL_RULE_ACTION_DROP, - PRESTERA_ACL_RULE_ACTION_TRAP + PRESTERA_ACL_RULE_ACTION_ACCEPT = 0, + PRESTERA_ACL_RULE_ACTION_DROP = 1, + PRESTERA_ACL_RULE_ACTION_TRAP = 2, + PRESTERA_ACL_RULE_ACTION_JUMP = 5, + PRESTERA_ACL_RULE_ACTION_COUNT = 7, + PRESTERA_ACL_RULE_ACTION_POLICE = 8, + + PRESTERA_ACL_RULE_ACTION_MAX }; -struct prestera_switch; -struct prestera_port; -struct prestera_acl_rule; -struct prestera_acl_ruleset; +enum { + PRESTERA_ACL_IFACE_TYPE_PORT, + PRESTERA_ACL_IFACE_TYPE_INDEX +}; -struct prestera_flow_block_binding { - struct list_head list; - struct prestera_port *port; - int span_id; +struct prestera_acl_match { + __be32 key[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + __be32 mask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; }; -struct prestera_flow_block { - struct list_head binding_list; - struct prestera_switch *sw; - struct net *net; - struct prestera_acl_ruleset *ruleset; - struct flow_block_cb *block_cb; +struct prestera_acl_action_jump { + u32 index; }; -struct prestera_acl_rule_action_entry { - struct list_head list; - enum prestera_acl_rule_action id; +struct prestera_acl_action_police { + u32 id; }; -struct prestera_acl_rule_match_entry { - struct list_head list; - enum prestera_acl_rule_match_entry_type type; +struct prestera_acl_action_count { + u32 id; +}; + +struct prestera_acl_rule_entry_key { + u32 prio; + struct prestera_acl_match match; +}; + +struct prestera_acl_hw_action_info { + enum prestera_acl_rule_action id; union { + struct prestera_acl_action_police police; + struct prestera_acl_action_count count; + struct prestera_acl_action_jump jump; + }; +}; + +/* This struct (arg) used only to be passed as parameter for + * acl_rule_entry_create. Must be flat. Can contain object keys, which will be + * resolved to object links, before saving to acl_rule_entry struct + */ +struct prestera_acl_rule_entry_arg { + u32 vtcam_id; + struct { struct { - u8 key; - u8 mask; - } u8; - struct { - u16 key; - u16 mask; - } u16; + u8 valid:1; + } accept, drop, trap; struct { - u32 key; - u32 mask; - } u32; + struct prestera_acl_action_jump i; + u8 valid:1; + } jump; struct { - u64 key; - u64 mask; - } u64; + u8 valid:1; + u64 rate; + u64 burst; + bool ingress; + } police; struct { - u8 key[ETH_ALEN]; - u8 mask[ETH_ALEN]; - } mac; - } keymask; + u8 valid:1; + u32 client; + } count; + }; +}; + +struct prestera_acl_rule { + struct rhash_head ht_node; /* Member of acl HT */ + struct list_head list; + struct prestera_acl_ruleset *ruleset; + struct prestera_acl_ruleset *jump_ruleset; + unsigned long cookie; + u32 chain_index; + u32 priority; + struct prestera_acl_rule_entry_key re_key; + struct prestera_acl_rule_entry_arg re_arg; + struct prestera_acl_rule_entry *re; }; +struct prestera_acl_iface { + union { + struct prestera_port *port; + u32 index; + }; + u8 type; +}; + +struct prestera_acl; +struct prestera_switch; +struct prestera_flow_block; + int prestera_acl_init(struct prestera_switch *sw); void prestera_acl_fini(struct prestera_switch *sw); -struct prestera_flow_block * -prestera_acl_block_create(struct prestera_switch *sw, struct net *net); -void prestera_acl_block_destroy(struct prestera_flow_block *block); -struct net *prestera_acl_block_net(struct prestera_flow_block *block); -struct prestera_switch *prestera_acl_block_sw(struct prestera_flow_block *block); -int prestera_acl_block_bind(struct prestera_flow_block *block, - struct prestera_port *port); -int prestera_acl_block_unbind(struct prestera_flow_block *block, - struct prestera_port *port); -struct prestera_acl_ruleset * -prestera_acl_block_ruleset_get(struct prestera_flow_block *block); + struct prestera_acl_rule * -prestera_acl_rule_create(struct prestera_flow_block *block, - unsigned long cookie); -u32 prestera_acl_rule_priority_get(struct prestera_acl_rule *rule); +prestera_acl_rule_create(struct prestera_acl_ruleset *ruleset, + unsigned long cookie, u32 chain_index); void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, u32 priority); -u16 prestera_acl_rule_ruleset_id_get(const struct prestera_acl_rule *rule); -struct list_head * -prestera_acl_rule_action_list_get(struct prestera_acl_rule *rule); -u8 prestera_acl_rule_action_len(struct prestera_acl_rule *rule); -u8 prestera_acl_rule_match_len(struct prestera_acl_rule *rule); -int prestera_acl_rule_action_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_action_entry *entry); -struct list_head * -prestera_acl_rule_match_list_get(struct prestera_acl_rule *rule); -int prestera_acl_rule_match_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_match_entry *entry); void prestera_acl_rule_destroy(struct prestera_acl_rule *rule); struct prestera_acl_rule * prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, @@ -117,8 +165,45 @@ int prestera_acl_rule_add(struct prestera_switch *sw, struct prestera_acl_rule *rule); void prestera_acl_rule_del(struct prestera_switch *sw, struct prestera_acl_rule *rule); -int prestera_acl_rule_get_stats(struct prestera_switch *sw, +int prestera_acl_rule_get_stats(struct prestera_acl *acl, struct prestera_acl_rule *rule, u64 *packets, u64 *bytes, u64 *last_use); +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_find(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key); +void prestera_acl_rule_entry_destroy(struct prestera_acl *acl, + struct prestera_acl_rule_entry *e); +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_create(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key, + struct prestera_acl_rule_entry_arg *arg); +struct prestera_acl_ruleset * +prestera_acl_ruleset_get(struct prestera_acl *acl, + struct prestera_flow_block *block, + u32 chain_index); +struct prestera_acl_ruleset * +prestera_acl_ruleset_lookup(struct prestera_acl *acl, + struct prestera_flow_block *block, + u32 chain_index); +int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, + void *keymask); +bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset); +int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset); +void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset); +int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port); +int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port); +u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset); +void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset, + u32 *prio_min, u32 *prio_max); +void +prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, + u16 pcl_id); + +int prestera_acl_vtcam_id_get(struct prestera_acl *acl, u8 lookup, u8 dir, + void *keymask, u32 *vtcam_id); +int prestera_acl_vtcam_id_put(struct prestera_acl *acl, u32 vtcam_id); +int prestera_acl_chain_to_client(u32 chain_index, bool ingress, u32 *client); #endif /* _PRESTERA_ACL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_counter.c b/drivers/net/ethernet/marvell/prestera/prestera_counter.c new file mode 100644 index 000000000000..4cd53a2dae46 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_counter.c @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2021 Marvell International Ltd. All rights reserved */ + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_acl.h" +#include "prestera_counter.h" + +#define COUNTER_POLL_TIME (msecs_to_jiffies(1000)) +#define COUNTER_RESCHED_TIME (msecs_to_jiffies(50)) +#define COUNTER_BULK_SIZE (256) + +struct prestera_counter { + struct prestera_switch *sw; + struct delayed_work stats_dw; + struct mutex mtx; /* protect block_list */ + struct prestera_counter_block **block_list; + u32 total_read; + u32 block_list_len; + u32 curr_idx; + bool is_fetching; +}; + +struct prestera_counter_block { + struct list_head list; + u32 id; + u32 offset; + u32 num_counters; + u32 client; + struct idr counter_idr; + refcount_t refcnt; + struct mutex mtx; /* protect stats and counter_idr */ + struct prestera_counter_stats *stats; + u8 *counter_flag; + bool is_updating; + bool full; +}; + +enum { + COUNTER_FLAG_READY = 0, + COUNTER_FLAG_INVALID = 1 +}; + +static bool +prestera_counter_is_ready(struct prestera_counter_block *block, u32 id) +{ + return block->counter_flag[id - block->offset] == COUNTER_FLAG_READY; +} + +static void prestera_counter_lock(struct prestera_counter *counter) +{ + mutex_lock(&counter->mtx); +} + +static void prestera_counter_unlock(struct prestera_counter *counter) +{ + mutex_unlock(&counter->mtx); +} + +static void prestera_counter_block_lock(struct prestera_counter_block *block) +{ + mutex_lock(&block->mtx); +} + +static void prestera_counter_block_unlock(struct prestera_counter_block *block) +{ + mutex_unlock(&block->mtx); +} + +static bool prestera_counter_block_incref(struct prestera_counter_block *block) +{ + return refcount_inc_not_zero(&block->refcnt); +} + +static bool prestera_counter_block_decref(struct prestera_counter_block *block) +{ + return refcount_dec_and_test(&block->refcnt); +} + +/* must be called with prestera_counter_block_lock() */ +static void prestera_counter_stats_clear(struct prestera_counter_block *block, + u32 counter_id) +{ + memset(&block->stats[counter_id - block->offset], 0, + sizeof(*block->stats)); +} + +static struct prestera_counter_block * +prestera_counter_block_lookup_not_full(struct prestera_counter *counter, + u32 client) +{ + u32 i; + + prestera_counter_lock(counter); + for (i = 0; i < counter->block_list_len; i++) { + if (counter->block_list[i] && + counter->block_list[i]->client == client && + !counter->block_list[i]->full && + prestera_counter_block_incref(counter->block_list[i])) { + prestera_counter_unlock(counter); + return counter->block_list[i]; + } + } + prestera_counter_unlock(counter); + + return NULL; +} + +static int prestera_counter_block_list_add(struct prestera_counter *counter, + struct prestera_counter_block *block) +{ + struct prestera_counter_block **arr; + u32 i; + + prestera_counter_lock(counter); + + for (i = 0; i < counter->block_list_len; i++) { + if (counter->block_list[i]) + continue; + + counter->block_list[i] = block; + prestera_counter_unlock(counter); + return 0; + } + + arr = krealloc(counter->block_list, (counter->block_list_len + 1) * + sizeof(*counter->block_list), GFP_KERNEL); + if (!arr) { + prestera_counter_unlock(counter); + return -ENOMEM; + } + + counter->block_list = arr; + counter->block_list[counter->block_list_len] = block; + counter->block_list_len++; + prestera_counter_unlock(counter); + return 0; +} + +static struct prestera_counter_block * +prestera_counter_block_get(struct prestera_counter *counter, u32 client) +{ + struct prestera_counter_block *block; + int err; + + block = prestera_counter_block_lookup_not_full(counter, client); + if (block) + return block; + + block = kzalloc(sizeof(*block), GFP_KERNEL); + if (!block) + return ERR_PTR(-ENOMEM); + + err = prestera_hw_counter_block_get(counter->sw, client, + &block->id, &block->offset, + &block->num_counters); + if (err) + goto err_block; + + block->stats = kcalloc(block->num_counters, + sizeof(*block->stats), GFP_KERNEL); + if (!block->stats) { + err = -ENOMEM; + goto err_stats; + } + + block->counter_flag = kcalloc(block->num_counters, + sizeof(*block->counter_flag), + GFP_KERNEL); + if (!block->counter_flag) { + err = -ENOMEM; + goto err_flag; + } + + block->client = client; + mutex_init(&block->mtx); + refcount_set(&block->refcnt, 1); + idr_init_base(&block->counter_idr, block->offset); + + err = prestera_counter_block_list_add(counter, block); + if (err) + goto err_list_add; + + return block; + +err_list_add: + idr_destroy(&block->counter_idr); + mutex_destroy(&block->mtx); + kfree(block->counter_flag); +err_flag: + kfree(block->stats); +err_stats: + prestera_hw_counter_block_release(counter->sw, block->id); +err_block: + kfree(block); + return ERR_PTR(err); +} + +static void prestera_counter_block_put(struct prestera_counter *counter, + struct prestera_counter_block *block) +{ + u32 i; + + if (!prestera_counter_block_decref(block)) + return; + + prestera_counter_lock(counter); + for (i = 0; i < counter->block_list_len; i++) { + if (counter->block_list[i] && + counter->block_list[i]->id == block->id) { + counter->block_list[i] = NULL; + break; + } + } + prestera_counter_unlock(counter); + + WARN_ON(!idr_is_empty(&block->counter_idr)); + + prestera_hw_counter_block_release(counter->sw, block->id); + idr_destroy(&block->counter_idr); + mutex_destroy(&block->mtx); + kfree(block->stats); + kfree(block); +} + +static int prestera_counter_get_vacant(struct prestera_counter_block *block, + u32 *id) +{ + int free_id; + + if (block->full) + return -ENOSPC; + + prestera_counter_block_lock(block); + free_id = idr_alloc_cyclic(&block->counter_idr, NULL, block->offset, + block->offset + block->num_counters, + GFP_KERNEL); + if (free_id < 0) { + if (free_id == -ENOSPC) + block->full = true; + + prestera_counter_block_unlock(block); + return free_id; + } + *id = free_id; + prestera_counter_block_unlock(block); + + return 0; +} + +int prestera_counter_get(struct prestera_counter *counter, u32 client, + struct prestera_counter_block **bl, u32 *counter_id) +{ + struct prestera_counter_block *block; + int err; + u32 id; + +get_next_block: + block = prestera_counter_block_get(counter, client); + if (IS_ERR(block)) + return PTR_ERR(block); + + err = prestera_counter_get_vacant(block, &id); + if (err) { + prestera_counter_block_put(counter, block); + + if (err == -ENOSPC) + goto get_next_block; + + return err; + } + + prestera_counter_block_lock(block); + if (block->is_updating) + block->counter_flag[id - block->offset] = COUNTER_FLAG_INVALID; + prestera_counter_block_unlock(block); + + *counter_id = id; + *bl = block; + + return 0; +} + +void prestera_counter_put(struct prestera_counter *counter, + struct prestera_counter_block *block, u32 counter_id) +{ + if (!block) + return; + + prestera_counter_block_lock(block); + idr_remove(&block->counter_idr, counter_id); + block->full = false; + prestera_counter_stats_clear(block, counter_id); + prestera_counter_block_unlock(block); + + prestera_hw_counter_clear(counter->sw, block->id, counter_id); + prestera_counter_block_put(counter, block); +} + +static u32 prestera_counter_block_idx_next(struct prestera_counter *counter, + u32 curr_idx) +{ + u32 idx, i, start = curr_idx + 1; + + prestera_counter_lock(counter); + for (i = 0; i < counter->block_list_len; i++) { + idx = (start + i) % counter->block_list_len; + if (!counter->block_list[idx]) + continue; + + prestera_counter_unlock(counter); + return idx; + } + prestera_counter_unlock(counter); + + return 0; +} + +static struct prestera_counter_block * +prestera_counter_block_get_by_idx(struct prestera_counter *counter, u32 idx) +{ + if (idx >= counter->block_list_len) + return NULL; + + prestera_counter_lock(counter); + + if (!counter->block_list[idx] || + !prestera_counter_block_incref(counter->block_list[idx])) { + prestera_counter_unlock(counter); + return NULL; + } + + prestera_counter_unlock(counter); + return counter->block_list[idx]; +} + +static void prestera_counter_stats_work(struct work_struct *work) +{ + struct delayed_work *dl_work = + container_of(work, struct delayed_work, work); + struct prestera_counter *counter = + container_of(dl_work, struct prestera_counter, stats_dw); + struct prestera_counter_block *block; + u32 resched_time = COUNTER_POLL_TIME; + u32 count = COUNTER_BULK_SIZE; + bool done = false; + int err; + u32 i; + + block = prestera_counter_block_get_by_idx(counter, counter->curr_idx); + if (!block) { + if (counter->is_fetching) + goto abort; + + goto next; + } + + if (!counter->is_fetching) { + err = prestera_hw_counter_trigger(counter->sw, block->id); + if (err) + goto abort; + + prestera_counter_block_lock(block); + block->is_updating = true; + prestera_counter_block_unlock(block); + + counter->is_fetching = true; + counter->total_read = 0; + resched_time = COUNTER_RESCHED_TIME; + goto resched; + } + + prestera_counter_block_lock(block); + err = prestera_hw_counters_get(counter->sw, counter->total_read, + &count, &done, + &block->stats[counter->total_read]); + prestera_counter_block_unlock(block); + if (err) + goto abort; + + counter->total_read += count; + if (!done || counter->total_read < block->num_counters) { + resched_time = COUNTER_RESCHED_TIME; + goto resched; + } + + for (i = 0; i < block->num_counters; i++) { + if (block->counter_flag[i] == COUNTER_FLAG_INVALID) { + prestera_counter_block_lock(block); + block->counter_flag[i] = COUNTER_FLAG_READY; + memset(&block->stats[i], 0, sizeof(*block->stats)); + prestera_counter_block_unlock(block); + } + } + + prestera_counter_block_lock(block); + block->is_updating = false; + prestera_counter_block_unlock(block); + + goto next; +abort: + prestera_hw_counter_abort(counter->sw); +next: + counter->is_fetching = false; + counter->curr_idx = + prestera_counter_block_idx_next(counter, counter->curr_idx); +resched: + if (block) + prestera_counter_block_put(counter, block); + + schedule_delayed_work(&counter->stats_dw, resched_time); +} + +/* Can be executed without rtnl_lock(). + * So pay attention when something changing. + */ +int prestera_counter_stats_get(struct prestera_counter *counter, + struct prestera_counter_block *block, + u32 counter_id, u64 *packets, u64 *bytes) +{ + if (!block || !prestera_counter_is_ready(block, counter_id)) { + *packets = 0; + *bytes = 0; + return 0; + } + + prestera_counter_block_lock(block); + *packets = block->stats[counter_id - block->offset].packets; + *bytes = block->stats[counter_id - block->offset].bytes; + + prestera_counter_stats_clear(block, counter_id); + prestera_counter_block_unlock(block); + + return 0; +} + +int prestera_counter_init(struct prestera_switch *sw) +{ + struct prestera_counter *counter; + + counter = kzalloc(sizeof(*counter), GFP_KERNEL); + if (!counter) + return -ENOMEM; + + counter->block_list = kzalloc(sizeof(*counter->block_list), GFP_KERNEL); + if (!counter->block_list) { + kfree(counter); + return -ENOMEM; + } + + mutex_init(&counter->mtx); + counter->block_list_len = 1; + counter->sw = sw; + sw->counter = counter; + + INIT_DELAYED_WORK(&counter->stats_dw, prestera_counter_stats_work); + schedule_delayed_work(&counter->stats_dw, COUNTER_POLL_TIME); + + return 0; +} + +void prestera_counter_fini(struct prestera_switch *sw) +{ + struct prestera_counter *counter = sw->counter; + u32 i; + + cancel_delayed_work_sync(&counter->stats_dw); + + for (i = 0; i < counter->block_list_len; i++) + WARN_ON(counter->block_list[i]); + + mutex_destroy(&counter->mtx); + kfree(counter->block_list); + kfree(counter); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_counter.h b/drivers/net/ethernet/marvell/prestera/prestera_counter.h new file mode 100644 index 000000000000..ad6b73907799 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_counter.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2021 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_COUNTER_H_ +#define _PRESTERA_COUNTER_H_ + +#include <linux/types.h> + +struct prestera_counter_stats { + u64 packets; + u64 bytes; +}; + +struct prestera_switch; +struct prestera_counter; +struct prestera_counter_block; + +int prestera_counter_init(struct prestera_switch *sw); +void prestera_counter_fini(struct prestera_switch *sw); + +int prestera_counter_get(struct prestera_counter *counter, u32 client, + struct prestera_counter_block **block, + u32 *counter_id); +void prestera_counter_put(struct prestera_counter *counter, + struct prestera_counter_block *block, u32 counter_id); +int prestera_counter_stats_get(struct prestera_counter *counter, + struct prestera_counter_block *block, + u32 counter_id, u64 *packets, u64 *bytes); + +#endif /* _PRESTERA_COUNTER_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c index 40d5b89573bb..2f52daba58e6 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c @@ -300,8 +300,8 @@ static void prestera_ethtool_get_drvinfo(struct net_device *dev, struct prestera_port *port = netdev_priv(dev); struct prestera_switch *sw = port->sw; - strlcpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, dev_name(prestera_dev(sw)), + strscpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, dev_name(prestera_dev(sw)), sizeof(drvinfo->bus_info)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", @@ -521,6 +521,9 @@ prestera_ethtool_get_link_ksettings(struct net_device *dev, ecmd->base.speed = SPEED_UNKNOWN; ecmd->base.duplex = DUPLEX_UNKNOWN; + if (port->phy_link) + return phylink_ethtool_ksettings_get(port->phy_link, ecmd); + ecmd->base.autoneg = port->autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE; if (port->caps.type == PRESTERA_PORT_TYPE_TP) { @@ -648,6 +651,9 @@ prestera_ethtool_set_link_ksettings(struct net_device *dev, u8 adver_fec; int err; + if (port->phy_link) + return phylink_ethtool_ksettings_set(port->phy_link, ecmd); + err = prestera_port_type_set(ecmd, port); if (err) return err; @@ -782,28 +788,6 @@ static int prestera_ethtool_nway_reset(struct net_device *dev) return -EINVAL; } -void prestera_ethtool_port_state_changed(struct prestera_port *port, - struct prestera_port_event *evt) -{ - struct prestera_port_mac_state *smac = &port->state_mac; - - smac->oper = evt->data.mac.oper; - - if (smac->oper) { - smac->mode = evt->data.mac.mode; - smac->speed = evt->data.mac.speed; - smac->duplex = evt->data.mac.duplex; - smac->fc = evt->data.mac.fc; - smac->fec = evt->data.mac.fec; - } else { - smac->mode = PRESTERA_MAC_MODE_MAX; - smac->speed = SPEED_UNKNOWN; - smac->duplex = DUPLEX_UNKNOWN; - smac->fc = 0; - smac->fec = 0; - } -} - const struct ethtool_ops prestera_ethtool_ops = { .get_drvinfo = prestera_ethtool_get_drvinfo, .get_link_ksettings = prestera_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h index 9eb18e99dea6..bd5600886bc6 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.h @@ -11,7 +11,4 @@ struct prestera_port; extern const struct ethtool_ops prestera_ethtool_ops; -void prestera_ethtool_port_state_changed(struct prestera_port *port, - struct prestera_port_event *evt); - #endif /* _PRESTERA_ETHTOOL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.c b/drivers/net/ethernet/marvell/prestera/prestera_flow.c index c9891e968259..9f4267f326b0 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.c @@ -7,8 +7,9 @@ #include "prestera.h" #include "prestera_acl.h" #include "prestera_flow.h" -#include "prestera_span.h" #include "prestera_flower.h" +#include "prestera_matchall.h" +#include "prestera_span.h" static LIST_HEAD(prestera_block_cb_list); @@ -17,9 +18,9 @@ static int prestera_flow_block_mall_cb(struct prestera_flow_block *block, { switch (f->command) { case TC_CLSMATCHALL_REPLACE: - return prestera_span_replace(block, f); + return prestera_mall_replace(block, f); case TC_CLSMATCHALL_DESTROY: - prestera_span_destroy(block); + prestera_mall_destroy(block); return 0; default: return -EOPNOTSUPP; @@ -29,9 +30,6 @@ static int prestera_flow_block_mall_cb(struct prestera_flow_block *block, static int prestera_flow_block_flower_cb(struct prestera_flow_block *block, struct flow_cls_offload *f) { - if (f->common.chain_index != 0) - return -EOPNOTSUPP; - switch (f->command) { case FLOW_CLS_REPLACE: return prestera_flower_replace(block, f); @@ -40,6 +38,11 @@ static int prestera_flow_block_flower_cb(struct prestera_flow_block *block, return 0; case FLOW_CLS_STATS: return prestera_flower_stats(block, f); + case FLOW_CLS_TMPLT_CREATE: + return prestera_flower_tmplt_create(block, f); + case FLOW_CLS_TMPLT_DESTROY: + prestera_flower_tmplt_destroy(block, f); + return 0; default: return -EOPNOTSUPP; } @@ -60,17 +63,117 @@ static int prestera_flow_block_cb(enum tc_setup_type type, } } +static void prestera_flow_block_destroy(void *cb_priv) +{ + struct prestera_flow_block *block = cb_priv; + + prestera_flower_template_cleanup(block); + + WARN_ON(!list_empty(&block->template_list)); + WARN_ON(!list_empty(&block->binding_list)); + + kfree(block); +} + +static struct prestera_flow_block * +prestera_flow_block_create(struct prestera_switch *sw, + struct net *net, + bool ingress) +{ + struct prestera_flow_block *block; + + block = kzalloc(sizeof(*block), GFP_KERNEL); + if (!block) + return NULL; + + INIT_LIST_HEAD(&block->binding_list); + INIT_LIST_HEAD(&block->template_list); + block->net = net; + block->sw = sw; + block->mall.prio_min = UINT_MAX; + block->mall.prio_max = 0; + block->mall.bound = false; + block->ingress = ingress; + + return block; +} + static void prestera_flow_block_release(void *cb_priv) { struct prestera_flow_block *block = cb_priv; - prestera_acl_block_destroy(block); + prestera_flow_block_destroy(block); +} + +static bool +prestera_flow_block_is_bound(const struct prestera_flow_block *block) +{ + return block->ruleset_zero; +} + +static struct prestera_flow_block_binding * +prestera_flow_block_lookup(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + if (binding->port == port) + return binding; + + return NULL; +} + +static int prestera_flow_block_bind(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + int err; + + binding = kzalloc(sizeof(*binding), GFP_KERNEL); + if (!binding) + return -ENOMEM; + + binding->span_id = PRESTERA_SPAN_INVALID_ID; + binding->port = port; + + if (prestera_flow_block_is_bound(block)) { + err = prestera_acl_ruleset_bind(block->ruleset_zero, port); + if (err) + goto err_ruleset_bind; + } + + list_add(&binding->list, &block->binding_list); + return 0; + +err_ruleset_bind: + kfree(binding); + return err; +} + +static int prestera_flow_block_unbind(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + + binding = prestera_flow_block_lookup(block, port); + if (!binding) + return -ENOENT; + + list_del(&binding->list); + + if (prestera_flow_block_is_bound(block)) + prestera_acl_ruleset_unbind(block->ruleset_zero, port); + + kfree(binding); + return 0; } static struct prestera_flow_block * prestera_flow_block_get(struct prestera_switch *sw, struct flow_block_offload *f, - bool *register_block) + bool *register_block, + bool ingress) { struct prestera_flow_block *block; struct flow_block_cb *block_cb; @@ -78,7 +181,7 @@ prestera_flow_block_get(struct prestera_switch *sw, block_cb = flow_block_cb_lookup(f->block, prestera_flow_block_cb, sw); if (!block_cb) { - block = prestera_acl_block_create(sw, f->net); + block = prestera_flow_block_create(sw, f->net, ingress); if (!block) return ERR_PTR(-ENOMEM); @@ -86,7 +189,7 @@ prestera_flow_block_get(struct prestera_switch *sw, sw, block, prestera_flow_block_release); if (IS_ERR(block_cb)) { - prestera_acl_block_destroy(block); + prestera_flow_block_destroy(block); return ERR_CAST(block_cb); } @@ -110,11 +213,11 @@ static void prestera_flow_block_put(struct prestera_flow_block *block) return; flow_block_cb_free(block_cb); - prestera_acl_block_destroy(block); + prestera_flow_block_destroy(block); } static int prestera_setup_flow_block_bind(struct prestera_port *port, - struct flow_block_offload *f) + struct flow_block_offload *f, bool ingress) { struct prestera_switch *sw = port->sw; struct prestera_flow_block *block; @@ -122,13 +225,13 @@ static int prestera_setup_flow_block_bind(struct prestera_port *port, bool register_block; int err; - block = prestera_flow_block_get(sw, f, ®ister_block); + block = prestera_flow_block_get(sw, f, ®ister_block, ingress); if (IS_ERR(block)) return PTR_ERR(block); block_cb = block->block_cb; - err = prestera_acl_block_bind(block, port); + err = prestera_flow_block_bind(block, port); if (err) goto err_block_bind; @@ -137,7 +240,11 @@ static int prestera_setup_flow_block_bind(struct prestera_port *port, list_add_tail(&block_cb->driver_list, &prestera_block_cb_list); } - port->flow_block = block; + if (ingress) + port->ingress_flow_block = block; + else + port->egress_flow_block = block; + return 0; err_block_bind: @@ -147,7 +254,7 @@ err_block_bind: } static void prestera_setup_flow_block_unbind(struct prestera_port *port, - struct flow_block_offload *f) + struct flow_block_offload *f, bool ingress) { struct prestera_switch *sw = port->sw; struct prestera_flow_block *block; @@ -160,9 +267,9 @@ static void prestera_setup_flow_block_unbind(struct prestera_port *port, block = flow_block_cb_priv(block_cb); - prestera_span_destroy(block); + prestera_mall_destroy(block); - err = prestera_acl_block_unbind(block, port); + err = prestera_flow_block_unbind(block, port); if (err) goto error; @@ -171,24 +278,38 @@ static void prestera_setup_flow_block_unbind(struct prestera_port *port, list_del(&block_cb->driver_list); } error: - port->flow_block = NULL; + if (ingress) + port->ingress_flow_block = NULL; + else + port->egress_flow_block = NULL; } -int prestera_flow_block_setup(struct prestera_port *port, - struct flow_block_offload *f) +static int prestera_setup_flow_block_clsact(struct prestera_port *port, + struct flow_block_offload *f, + bool ingress) { - if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) - return -EOPNOTSUPP; - f->driver_block_list = &prestera_block_cb_list; switch (f->command) { case FLOW_BLOCK_BIND: - return prestera_setup_flow_block_bind(port, f); + return prestera_setup_flow_block_bind(port, f, ingress); case FLOW_BLOCK_UNBIND: - prestera_setup_flow_block_unbind(port, f); + prestera_setup_flow_block_unbind(port, f, ingress); return 0; default: return -EOPNOTSUPP; } } + +int prestera_flow_block_setup(struct prestera_port *port, + struct flow_block_offload *f) +{ + switch (f->binder_type) { + case FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS: + return prestera_setup_flow_block_clsact(port, f, true); + case FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS: + return prestera_setup_flow_block_clsact(port, f, false); + default: + return -EOPNOTSUPP; + } +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.h b/drivers/net/ethernet/marvell/prestera/prestera_flow.h index 467c7038cace..a85a3eb40279 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.h @@ -7,6 +7,29 @@ #include <net/flow_offload.h> struct prestera_port; +struct prestera_switch; + +struct prestera_flow_block_binding { + struct list_head list; + struct prestera_port *port; + int span_id; +}; + +struct prestera_flow_block { + struct list_head binding_list; + struct prestera_switch *sw; + struct net *net; + struct prestera_acl_ruleset *ruleset_zero; + struct flow_block_cb *block_cb; + struct list_head template_list; + struct { + u32 prio_min; + u32 prio_max; + bool bound; + } mall; + unsigned int rule_count; + bool ingress; +}; int prestera_flow_block_setup(struct prestera_port *port, struct flow_block_offload *f); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index e571ba09ec08..91a478b75cbf 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -3,42 +3,134 @@ #include "prestera.h" #include "prestera_acl.h" +#include "prestera_flow.h" #include "prestera_flower.h" +#include "prestera_matchall.h" + +struct prestera_flower_template { + struct prestera_acl_ruleset *ruleset; + struct list_head list; + u32 chain_index; +}; + +static void +prestera_flower_template_free(struct prestera_flower_template *template) +{ + prestera_acl_ruleset_put(template->ruleset); + list_del(&template->list); + kfree(template); +} + +void prestera_flower_template_cleanup(struct prestera_flow_block *block) +{ + struct prestera_flower_template *template, *tmp; + + /* put the reference to all rulesets kept in tmpl create */ + list_for_each_entry_safe(template, tmp, &block->template_list, list) + prestera_flower_template_free(template); +} + +static int +prestera_flower_parse_goto_action(struct prestera_flow_block *block, + struct prestera_acl_rule *rule, + u32 chain_index, + const struct flow_action_entry *act) +{ + struct prestera_acl_ruleset *ruleset; + + if (act->chain_index <= chain_index) + /* we can jump only forward */ + return -EINVAL; + + if (rule->re_arg.jump.valid) + return -EEXIST; + + ruleset = prestera_acl_ruleset_get(block->sw->acl, block, + act->chain_index); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + rule->re_arg.jump.valid = 1; + rule->re_arg.jump.i.index = prestera_acl_ruleset_index_get(ruleset); + + rule->jump_ruleset = ruleset; + + return 0; +} static int prestera_flower_parse_actions(struct prestera_flow_block *block, struct prestera_acl_rule *rule, struct flow_action *flow_action, + u32 chain_index, struct netlink_ext_ack *extack) { - struct prestera_acl_rule_action_entry a_entry; const struct flow_action_entry *act; int err, i; + /* whole struct (rule->re_arg) must be initialized with 0 */ if (!flow_action_has_entries(flow_action)) return 0; - flow_action_for_each(i, act, flow_action) { - memset(&a_entry, 0, sizeof(a_entry)); + if (!flow_action_mixed_hw_stats_check(flow_action, extack)) + return -EOPNOTSUPP; + + act = flow_action_first_entry_get(flow_action); + if (act->hw_stats & FLOW_ACTION_HW_STATS_DISABLED) { + /* Nothing to do */ + } else if (act->hw_stats & FLOW_ACTION_HW_STATS_DELAYED) { + /* setup counter first */ + rule->re_arg.count.valid = true; + err = prestera_acl_chain_to_client(chain_index, block->ingress, + &rule->re_arg.count.client); + if (err) + return err; + } else { + NL_SET_ERR_MSG_MOD(extack, "Unsupported action HW stats type"); + return -EOPNOTSUPP; + } + flow_action_for_each(i, act, flow_action) { switch (act->id) { case FLOW_ACTION_ACCEPT: - a_entry.id = PRESTERA_ACL_RULE_ACTION_ACCEPT; + if (rule->re_arg.accept.valid) + return -EEXIST; + + rule->re_arg.accept.valid = 1; break; case FLOW_ACTION_DROP: - a_entry.id = PRESTERA_ACL_RULE_ACTION_DROP; + if (rule->re_arg.drop.valid) + return -EEXIST; + + rule->re_arg.drop.valid = 1; break; case FLOW_ACTION_TRAP: - a_entry.id = PRESTERA_ACL_RULE_ACTION_TRAP; + if (rule->re_arg.trap.valid) + return -EEXIST; + + rule->re_arg.trap.valid = 1; + break; + case FLOW_ACTION_POLICE: + if (rule->re_arg.police.valid) + return -EEXIST; + + rule->re_arg.police.valid = 1; + rule->re_arg.police.rate = + act->police.rate_bytes_ps; + rule->re_arg.police.burst = act->police.burst; + rule->re_arg.police.ingress = block->ingress; + break; + case FLOW_ACTION_GOTO: + err = prestera_flower_parse_goto_action(block, rule, + chain_index, + act); + if (err) + return err; break; default: NL_SET_ERR_MSG_MOD(extack, "Unsupported action"); pr_err("Unsupported action\n"); return -EOPNOTSUPP; } - - err = prestera_acl_rule_action_add(rule, &a_entry); - if (err) - return err; } return 0; @@ -49,10 +141,11 @@ static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, struct prestera_flow_block *block) { struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); - struct prestera_acl_rule_match_entry m_entry = {0}; + struct prestera_acl_match *r_match = &rule->re_key.match; + struct prestera_port *port; struct net_device *ingress_dev; struct flow_match_meta match; - struct prestera_port *port; + __be16 key, mask; flow_rule_match_meta(f_rule, &match); if (match.mask->ingress_ifindex != 0xFFFFFFFF) { @@ -61,7 +154,7 @@ static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, return -EINVAL; } - ingress_dev = __dev_get_by_index(prestera_acl_block_net(block), + ingress_dev = __dev_get_by_index(block->net, match.key->ingress_ifindex); if (!ingress_dev) { NL_SET_ERR_MSG_MOD(f->common.extack, @@ -76,11 +169,17 @@ static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, } port = netdev_priv(ingress_dev); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT; - m_entry.keymask.u64.key = port->hw_id | ((u64)port->dev_id << 32); - m_entry.keymask.u64.mask = ~(u64)0; + mask = htons(0x1FFF << 3); + key = htons(port->hw_id << 3); + rule_match_set(r_match->key, SYS_PORT, key); + rule_match_set(r_match->mask, SYS_PORT, mask); + + mask = htons(0x3FF); + key = htons(port->dev_id); + rule_match_set(r_match->key, SYS_DEV, key); + rule_match_set(r_match->mask, SYS_DEV, mask); - return prestera_acl_rule_match_add(rule, &m_entry); + return 0; } static int prestera_flower_parse(struct prestera_flow_block *block, @@ -89,9 +188,9 @@ static int prestera_flower_parse(struct prestera_flow_block *block, { struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); struct flow_dissector *dissector = f_rule->match.dissector; - struct prestera_acl_rule_match_entry m_entry; - u16 n_proto_mask = 0; - u16 n_proto_key = 0; + struct prestera_acl_match *r_match = &rule->re_key.match; + __be16 n_proto_mask = 0; + __be16 n_proto_key = 0; u16 addr_type = 0; u8 ip_proto = 0; int err; @@ -105,6 +204,7 @@ static int prestera_flower_parse(struct prestera_flow_block *block, BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | BIT(FLOW_DISSECTOR_KEY_ICMP) | BIT(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_PORTS_RANGE) | BIT(FLOW_DISSECTOR_KEY_VLAN))) { NL_SET_ERR_MSG_MOD(f->common.extack, "Unsupported key"); return -EOPNOTSUPP; @@ -129,32 +229,19 @@ static int prestera_flower_parse(struct prestera_flow_block *block, struct flow_match_basic match; flow_rule_match_basic(f_rule, &match); - n_proto_key = ntohs(match.key->n_proto); - n_proto_mask = ntohs(match.mask->n_proto); + n_proto_key = match.key->n_proto; + n_proto_mask = match.mask->n_proto; - if (n_proto_key == ETH_P_ALL) { + if (ntohs(match.key->n_proto) == ETH_P_ALL) { n_proto_key = 0; n_proto_mask = 0; } - /* add eth type key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE; - m_entry.keymask.u16.key = n_proto_key; - m_entry.keymask.u16.mask = n_proto_mask; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; - - /* add ip proto key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO; - m_entry.keymask.u8.key = match.key->ip_proto; - m_entry.keymask.u8.mask = match.mask->ip_proto; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, ETH_TYPE, n_proto_key); + rule_match_set(r_match->mask, ETH_TYPE, n_proto_mask); + rule_match_set(r_match->key, IP_PROTO, match.key->ip_proto); + rule_match_set(r_match->mask, IP_PROTO, match.mask->ip_proto); ip_proto = match.key->ip_proto; } @@ -163,27 +250,27 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_eth_addrs(f_rule, &match); - /* add ethernet dst key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC; - memcpy(&m_entry.keymask.mac.key, - &match.key->dst, sizeof(match.key->dst)); - memcpy(&m_entry.keymask.mac.mask, - &match.mask->dst, sizeof(match.mask->dst)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; - - /* add ethernet src key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC; - memcpy(&m_entry.keymask.mac.key, - &match.key->src, sizeof(match.key->src)); - memcpy(&m_entry.keymask.mac.mask, - &match.mask->src, sizeof(match.mask->src)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + /* DA key, mask */ + rule_match_set_n(r_match->key, + ETH_DMAC_0, &match.key->dst[0], 4); + rule_match_set_n(r_match->key, + ETH_DMAC_1, &match.key->dst[4], 2); + + rule_match_set_n(r_match->mask, + ETH_DMAC_0, &match.mask->dst[0], 4); + rule_match_set_n(r_match->mask, + ETH_DMAC_1, &match.mask->dst[4], 2); + + /* SA key, mask */ + rule_match_set_n(r_match->key, + ETH_SMAC_0, &match.key->src[0], 4); + rule_match_set_n(r_match->key, + ETH_SMAC_1, &match.key->src[4], 2); + + rule_match_set_n(r_match->mask, + ETH_SMAC_0, &match.mask->src[0], 4); + rule_match_set_n(r_match->mask, + ETH_SMAC_1, &match.mask->src[4], 2); } if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { @@ -191,25 +278,11 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_ipv4_addrs(f_rule, &match); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC; - memcpy(&m_entry.keymask.u32.key, - &match.key->src, sizeof(match.key->src)); - memcpy(&m_entry.keymask.u32.mask, - &match.mask->src, sizeof(match.mask->src)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, IP_SRC, match.key->src); + rule_match_set(r_match->mask, IP_SRC, match.mask->src); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST; - memcpy(&m_entry.keymask.u32.key, - &match.key->dst, sizeof(match.key->dst)); - memcpy(&m_entry.keymask.u32.mask, - &match.mask->dst, sizeof(match.mask->dst)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, IP_DST, match.key->dst); + rule_match_set(r_match->mask, IP_DST, match.mask->dst); } if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_PORTS)) { @@ -224,21 +297,34 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_ports(f_rule, &match); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC; - m_entry.keymask.u16.key = ntohs(match.key->src); - m_entry.keymask.u16.mask = ntohs(match.mask->src); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, L4_PORT_SRC, match.key->src); + rule_match_set(r_match->mask, L4_PORT_SRC, match.mask->src); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST; - m_entry.keymask.u16.key = ntohs(match.key->dst); - m_entry.keymask.u16.mask = ntohs(match.mask->dst); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, L4_PORT_DST, match.key->dst); + rule_match_set(r_match->mask, L4_PORT_DST, match.mask->dst); + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_PORTS_RANGE)) { + struct flow_match_ports_range match; + __be32 tp_key, tp_mask; + + flow_rule_match_ports_range(f_rule, &match); + + /* src port range (min, max) */ + tp_key = htonl(ntohs(match.key->tp_min.src) | + (ntohs(match.key->tp_max.src) << 16)); + tp_mask = htonl(ntohs(match.mask->tp_min.src) | + (ntohs(match.mask->tp_max.src) << 16)); + rule_match_set(r_match->key, L4_PORT_RANGE_SRC, tp_key); + rule_match_set(r_match->mask, L4_PORT_RANGE_SRC, tp_mask); + + /* dst port range (min, max) */ + tp_key = htonl(ntohs(match.key->tp_min.dst) | + (ntohs(match.key->tp_max.dst) << 16)); + tp_mask = htonl(ntohs(match.mask->tp_min.dst) | + (ntohs(match.mask->tp_max.dst) << 16)); + rule_match_set(r_match->key, L4_PORT_RANGE_DST, tp_key); + rule_match_set(r_match->mask, L4_PORT_RANGE_DST, tp_mask); } if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_VLAN)) { @@ -247,22 +333,15 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_vlan(f_rule, &match); if (match.mask->vlan_id != 0) { - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID; - m_entry.keymask.u16.key = match.key->vlan_id; - m_entry.keymask.u16.mask = match.mask->vlan_id; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + __be16 key = cpu_to_be16(match.key->vlan_id); + __be16 mask = cpu_to_be16(match.mask->vlan_id); + + rule_match_set(r_match->key, VLAN_ID, key); + rule_match_set(r_match->mask, VLAN_ID, mask); } - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID; - m_entry.keymask.u16.key = ntohs(match.key->vlan_tpid); - m_entry.keymask.u16.mask = ntohs(match.mask->vlan_tpid); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, VLAN_TPID, match.key->vlan_tpid); + rule_match_set(r_match->mask, VLAN_TPID, match.mask->vlan_tpid); } if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_ICMP)) { @@ -270,90 +349,227 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_icmp(f_rule, &match); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE; - m_entry.keymask.u8.key = match.key->type; - m_entry.keymask.u8.mask = match.mask->type; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, ICMP_TYPE, match.key->type); + rule_match_set(r_match->mask, ICMP_TYPE, match.mask->type); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE; - m_entry.keymask.u8.key = match.key->code; - m_entry.keymask.u8.mask = match.mask->code; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, ICMP_CODE, match.key->code); + rule_match_set(r_match->mask, ICMP_CODE, match.mask->code); } - return prestera_flower_parse_actions(block, rule, - &f->rule->action, + return prestera_flower_parse_actions(block, rule, &f->rule->action, + f->common.chain_index, f->common.extack); } +static int prestera_flower_prio_check(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + u32 mall_prio_min; + u32 mall_prio_max; + int err; + + err = prestera_mall_prio_get(block, &mall_prio_min, &mall_prio_max); + if (err == -ENOENT) + /* No matchall filters installed on this chain. */ + return 0; + + if (err) { + NL_SET_ERR_MSG(f->common.extack, "Failed to get matchall priorities"); + return err; + } + + if (f->common.prio <= mall_prio_max && block->ingress) { + NL_SET_ERR_MSG(f->common.extack, + "Failed to add in front of existing matchall rules"); + return -EOPNOTSUPP; + } + if (f->common.prio >= mall_prio_min && !block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing matchall rules"); + return -EOPNOTSUPP; + } + + return 0; +} + +int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index, + u32 *prio_min, u32 *prio_max) +{ + struct prestera_acl_ruleset *ruleset; + + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block, chain_index); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + prestera_acl_ruleset_prio_get(ruleset, prio_min, prio_max); + return 0; +} + int prestera_flower_replace(struct prestera_flow_block *block, struct flow_cls_offload *f) { - struct prestera_switch *sw = prestera_acl_block_sw(block); + struct prestera_acl_ruleset *ruleset; + struct prestera_acl *acl = block->sw->acl; struct prestera_acl_rule *rule; int err; - rule = prestera_acl_rule_create(block, f->cookie); - if (IS_ERR(rule)) - return PTR_ERR(rule); + err = prestera_flower_prio_check(block, f); + if (err) + return err; + + ruleset = prestera_acl_ruleset_get(acl, block, f->common.chain_index); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + /* increments the ruleset reference */ + rule = prestera_acl_rule_create(ruleset, f->cookie, + f->common.chain_index); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + goto err_rule_create; + } err = prestera_flower_parse(block, rule, f); if (err) - goto err_flower_parse; + goto err_rule_add; - err = prestera_acl_rule_add(sw, rule); + if (!prestera_acl_ruleset_is_offload(ruleset)) { + err = prestera_acl_ruleset_offload(ruleset); + if (err) + goto err_ruleset_offload; + } + + err = prestera_acl_rule_add(block->sw, rule); if (err) goto err_rule_add; + prestera_acl_ruleset_put(ruleset); return 0; +err_ruleset_offload: err_rule_add: -err_flower_parse: prestera_acl_rule_destroy(rule); +err_rule_create: + prestera_acl_ruleset_put(ruleset); return err; } void prestera_flower_destroy(struct prestera_flow_block *block, struct flow_cls_offload *f) { + struct prestera_acl_ruleset *ruleset; struct prestera_acl_rule *rule; - struct prestera_switch *sw; - rule = prestera_acl_rule_lookup(prestera_acl_block_ruleset_get(block), - f->cookie); + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block, + f->common.chain_index); + if (IS_ERR(ruleset)) + return; + + rule = prestera_acl_rule_lookup(ruleset, f->cookie); if (rule) { - sw = prestera_acl_block_sw(block); - prestera_acl_rule_del(sw, rule); + prestera_acl_rule_del(block->sw, rule); prestera_acl_rule_destroy(rule); } + prestera_acl_ruleset_put(ruleset); +} + +int prestera_flower_tmplt_create(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + struct prestera_flower_template *template; + struct prestera_acl_ruleset *ruleset; + struct prestera_acl_rule rule; + int err; + + memset(&rule, 0, sizeof(rule)); + err = prestera_flower_parse(block, &rule, f); + if (err) + return err; + + template = kmalloc(sizeof(*template), GFP_KERNEL); + if (!template) { + err = -ENOMEM; + goto err_malloc; + } + + prestera_acl_rule_keymask_pcl_id_set(&rule, 0); + ruleset = prestera_acl_ruleset_get(block->sw->acl, block, + f->common.chain_index); + if (IS_ERR_OR_NULL(ruleset)) { + err = -EINVAL; + goto err_ruleset_get; + } + + /* preserve keymask/template to this ruleset */ + err = prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask); + if (err) + goto err_ruleset_keymask_set; + + /* skip error, as it is not possible to reject template operation, + * so, keep the reference to the ruleset for rules to be added + * to that ruleset later. In case of offload fail, the ruleset + * will be offloaded again during adding a new rule. Also, + * unlikly possble that ruleset is already offloaded at this staage. + */ + prestera_acl_ruleset_offload(ruleset); + + /* keep the reference to the ruleset */ + template->ruleset = ruleset; + template->chain_index = f->common.chain_index; + list_add_rcu(&template->list, &block->template_list); + return 0; + +err_ruleset_keymask_set: + prestera_acl_ruleset_put(ruleset); +err_ruleset_get: + kfree(template); +err_malloc: + NL_SET_ERR_MSG_MOD(f->common.extack, "Create chain template failed"); + return err; +} + +void prestera_flower_tmplt_destroy(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + struct prestera_flower_template *template, *tmp; + + list_for_each_entry_safe(template, tmp, &block->template_list, list) + if (template->chain_index == f->common.chain_index) { + /* put the reference to the ruleset kept in create */ + prestera_flower_template_free(template); + return; + } } int prestera_flower_stats(struct prestera_flow_block *block, struct flow_cls_offload *f) { - struct prestera_switch *sw = prestera_acl_block_sw(block); + struct prestera_acl_ruleset *ruleset; struct prestera_acl_rule *rule; u64 packets; u64 lastuse; u64 bytes; int err; - rule = prestera_acl_rule_lookup(prestera_acl_block_ruleset_get(block), - f->cookie); - if (!rule) - return -EINVAL; + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block, + f->common.chain_index); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + rule = prestera_acl_rule_lookup(ruleset, f->cookie); + if (!rule) { + err = -EINVAL; + goto err_rule_get_stats; + } - err = prestera_acl_rule_get_stats(sw, rule, &packets, &bytes, &lastuse); + err = prestera_acl_rule_get_stats(block->sw->acl, rule, &packets, + &bytes, &lastuse); if (err) - return err; + goto err_rule_get_stats; flow_stats_update(&f->stats, bytes, packets, 0, lastuse, - FLOW_ACTION_HW_STATS_IMMEDIATE); - return 0; + FLOW_ACTION_HW_STATS_DELAYED); + +err_rule_get_stats: + prestera_acl_ruleset_put(ruleset); + return err; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.h b/drivers/net/ethernet/marvell/prestera/prestera_flower.h index 91e045eec58b..1181115fe6fa 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ -/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ +/* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved. */ #ifndef _PRESTERA_FLOWER_H_ #define _PRESTERA_FLOWER_H_ @@ -14,5 +14,12 @@ void prestera_flower_destroy(struct prestera_flow_block *block, struct flow_cls_offload *f); int prestera_flower_stats(struct prestera_flow_block *block, struct flow_cls_offload *f); +int prestera_flower_tmplt_create(struct prestera_flow_block *block, + struct flow_cls_offload *f); +void prestera_flower_tmplt_destroy(struct prestera_flow_block *block, + struct flow_cls_offload *f); +void prestera_flower_template_cleanup(struct prestera_flow_block *block); +int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index, + u32 *prio_min, u32 *prio_max); #endif /* _PRESTERA_FLOWER_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_hw.c index 9b8b1ed474fc..fc6f7d2746e8 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.c @@ -9,11 +9,15 @@ #include "prestera.h" #include "prestera_hw.h" #include "prestera_acl.h" +#include "prestera_counter.h" +#include "prestera_router_hw.h" #define PRESTERA_SWITCH_INIT_TIMEOUT_MS (30 * 1000) #define PRESTERA_MIN_MTU 64 +#define PRESTERA_MSG_CHUNK_SIZE 1024 + enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_SWITCH_INIT = 0x1, PRESTERA_CMD_TYPE_SWITCH_ATTR_SET = 0x2, @@ -38,13 +42,38 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_BRIDGE_PORT_ADD = 0x402, PRESTERA_CMD_TYPE_BRIDGE_PORT_DELETE = 0x403, - PRESTERA_CMD_TYPE_ACL_RULE_ADD = 0x500, - PRESTERA_CMD_TYPE_ACL_RULE_DELETE = 0x501, - PRESTERA_CMD_TYPE_ACL_RULE_STATS_GET = 0x510, - PRESTERA_CMD_TYPE_ACL_RULESET_CREATE = 0x520, - PRESTERA_CMD_TYPE_ACL_RULESET_DELETE = 0x521, - PRESTERA_CMD_TYPE_ACL_PORT_BIND = 0x530, - PRESTERA_CMD_TYPE_ACL_PORT_UNBIND = 0x531, + PRESTERA_CMD_TYPE_COUNTER_GET = 0x510, + PRESTERA_CMD_TYPE_COUNTER_ABORT = 0x511, + PRESTERA_CMD_TYPE_COUNTER_TRIGGER = 0x512, + PRESTERA_CMD_TYPE_COUNTER_BLOCK_GET = 0x513, + PRESTERA_CMD_TYPE_COUNTER_BLOCK_RELEASE = 0x514, + PRESTERA_CMD_TYPE_COUNTER_CLEAR = 0x515, + + PRESTERA_CMD_TYPE_VTCAM_CREATE = 0x540, + PRESTERA_CMD_TYPE_VTCAM_DESTROY = 0x541, + PRESTERA_CMD_TYPE_VTCAM_RULE_ADD = 0x550, + PRESTERA_CMD_TYPE_VTCAM_RULE_DELETE = 0x551, + PRESTERA_CMD_TYPE_VTCAM_IFACE_BIND = 0x560, + PRESTERA_CMD_TYPE_VTCAM_IFACE_UNBIND = 0x561, + + PRESTERA_CMD_TYPE_ROUTER_RIF_CREATE = 0x600, + PRESTERA_CMD_TYPE_ROUTER_RIF_DELETE = 0x601, + PRESTERA_CMD_TYPE_ROUTER_LPM_ADD = 0x610, + PRESTERA_CMD_TYPE_ROUTER_LPM_DELETE = 0x611, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET = 0x622, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET = 0x645, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD = 0x623, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE = 0x624, + PRESTERA_CMD_TYPE_ROUTER_VR_CREATE = 0x630, + PRESTERA_CMD_TYPE_ROUTER_VR_DELETE = 0x631, + + PRESTERA_CMD_TYPE_FLOOD_DOMAIN_CREATE = 0x700, + PRESTERA_CMD_TYPE_FLOOD_DOMAIN_DESTROY = 0x701, + PRESTERA_CMD_TYPE_FLOOD_DOMAIN_PORTS_SET = 0x702, + PRESTERA_CMD_TYPE_FLOOD_DOMAIN_PORTS_RESET = 0x703, + + PRESTERA_CMD_TYPE_MDB_CREATE = 0x704, + PRESTERA_CMD_TYPE_MDB_DESTROY = 0x705, PRESTERA_CMD_TYPE_RXTX_INIT = 0x800, @@ -56,9 +85,15 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_STP_PORT_SET = 0x1000, PRESTERA_CMD_TYPE_SPAN_GET = 0x1100, - PRESTERA_CMD_TYPE_SPAN_BIND = 0x1101, - PRESTERA_CMD_TYPE_SPAN_UNBIND = 0x1102, + PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND = 0x1101, + PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND = 0x1102, PRESTERA_CMD_TYPE_SPAN_RELEASE = 0x1103, + PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND = 0x1104, + PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND = 0x1105, + + PRESTERA_CMD_TYPE_POLICER_CREATE = 0x1500, + PRESTERA_CMD_TYPE_POLICER_RELEASE = 0x1501, + PRESTERA_CMD_TYPE_POLICER_SET = 0x1502, PRESTERA_CMD_TYPE_CPU_CODE_COUNTERS_GET = 0x2000, @@ -75,6 +110,7 @@ enum { PRESTERA_CMD_PORT_ATTR_LEARNING = 7, PRESTERA_CMD_PORT_ATTR_FLOOD = 8, PRESTERA_CMD_PORT_ATTR_CAPABILITY = 9, + PRESTERA_CMD_PORT_ATTR_LOCKED = 10, PRESTERA_CMD_PORT_ATTR_PHY_MODE = 12, PRESTERA_CMD_PORT_ATTR_TYPE = 13, PRESTERA_CMD_PORT_ATTR_STATS = 17, @@ -150,6 +186,10 @@ enum { }; enum { + PRESTERA_POLICER_MODE_SR_TCM +}; + +enum { PRESTERA_HW_FDB_ENTRY_TYPE_REG_PORT = 0, PRESTERA_HW_FDB_ENTRY_TYPE_LAG = 1, PRESTERA_HW_FDB_ENTRY_TYPE_MAX = 2, @@ -163,6 +203,12 @@ struct prestera_fw_event_handler { void *arg; }; +enum { + PRESTERA_HW_FLOOD_DOMAIN_PORT_TYPE_REG_PORT = 0, + PRESTERA_HW_FLOOD_DOMAIN_PORT_TYPE_LAG = 1, + PRESTERA_HW_FLOOD_DOMAIN_PORT_TYPE_MAX = 2, +}; + struct prestera_msg_cmd { __le32 type; }; @@ -249,6 +295,7 @@ union prestera_msg_port_param { u8 duplex; u8 fec; u8 fc; + u8 br_locked; union { struct { u8 admin; @@ -359,76 +406,90 @@ struct prestera_msg_bridge_resp { u8 pad[2]; }; -struct prestera_msg_acl_action { - __le32 id; - __le32 reserved[5]; +struct prestera_msg_vtcam_create_req { + struct prestera_msg_cmd cmd; + __le32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + u8 direction; + u8 lookup; + u8 pad[2]; }; -struct prestera_msg_acl_match { - __le32 type; - __le32 __reserved; - union { - struct { - u8 key; - u8 mask; - } u8; - struct { - __le16 key; - __le16 mask; - } u16; - struct { - __le32 key; - __le32 mask; - } u32; - struct { - __le64 key; - __le64 mask; - } u64; - struct { - u8 key[ETH_ALEN]; - u8 mask[ETH_ALEN]; - } mac; - } keymask; +struct prestera_msg_vtcam_destroy_req { + struct prestera_msg_cmd cmd; + __le32 vtcam_id; }; -struct prestera_msg_acl_rule_req { +struct prestera_msg_vtcam_rule_add_req { struct prestera_msg_cmd cmd; - __le32 id; - __le32 priority; - __le16 ruleset_id; - u8 n_actions; - u8 n_matches; + __le32 key[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + __le32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + __le32 vtcam_id; + __le32 prio; + __le32 n_act; }; -struct prestera_msg_acl_rule_resp { - struct prestera_msg_ret ret; +struct prestera_msg_vtcam_rule_del_req { + struct prestera_msg_cmd cmd; + __le32 vtcam_id; __le32 id; }; -struct prestera_msg_acl_rule_stats_resp { +struct prestera_msg_vtcam_bind_req { + struct prestera_msg_cmd cmd; + union { + struct { + __le32 hw_id; + __le32 dev_id; + } port; + __le32 index; + }; + __le32 vtcam_id; + __le16 pcl_id; + __le16 type; +}; + +struct prestera_msg_vtcam_resp { struct prestera_msg_ret ret; - __le64 packets; - __le64 bytes; + __le32 vtcam_id; + __le32 rule_id; }; -struct prestera_msg_acl_ruleset_bind_req { - struct prestera_msg_cmd cmd; - __le32 port; - __le32 dev; - __le16 ruleset_id; - u8 pad[2]; +struct prestera_msg_acl_action { + __le32 id; + __le32 __reserved; + union { + struct { + __le32 index; + } jump; + struct { + __le32 id; + } police; + struct { + __le32 id; + } count; + __le32 reserved[6]; + }; }; -struct prestera_msg_acl_ruleset_req { +struct prestera_msg_counter_req { struct prestera_msg_cmd cmd; - __le16 id; - u8 pad[2]; + __le32 client; + __le32 block_id; + __le32 num_counters; +}; + +struct prestera_msg_counter_stats { + __le64 packets; + __le64 bytes; }; -struct prestera_msg_acl_ruleset_resp { +struct prestera_msg_counter_resp { struct prestera_msg_ret ret; - __le16 id; - u8 pad[2]; + __le32 block_id; + __le32 offset; + __le32 num_counters; + __le32 done; + struct prestera_msg_counter_stats stats[]; }; struct prestera_msg_span_req { @@ -465,6 +526,102 @@ struct prestera_msg_rxtx_resp { __le32 map_addr; }; +struct prestera_msg_iface { + union { + struct { + __le32 dev; + __le32 port; + }; + __le16 lag_id; + }; + __le16 vr_id; + __le16 vid; + u8 type; + u8 __pad[3]; +}; + +struct prestera_msg_ip_addr { + union { + __be32 ipv4; + __be32 ipv6[4]; + } u; + u8 v; /* e.g. PRESTERA_IPV4 */ + u8 __pad[3]; +}; + +struct prestera_msg_nh { + struct prestera_msg_iface oif; + __le32 hw_id; + u8 mac[ETH_ALEN]; + u8 is_active; + u8 pad; +}; + +struct prestera_msg_rif_req { + struct prestera_msg_cmd cmd; + struct prestera_msg_iface iif; + __le32 mtu; + __le16 rif_id; + __le16 __reserved; + u8 mac[ETH_ALEN]; + u8 __pad[2]; +}; + +struct prestera_msg_rif_resp { + struct prestera_msg_ret ret; + __le16 rif_id; + u8 __pad[2]; +}; + +struct prestera_msg_lpm_req { + struct prestera_msg_cmd cmd; + struct prestera_msg_ip_addr dst; + __le32 grp_id; + __le32 dst_len; + __le16 vr_id; + u8 __pad[2]; +}; + +struct prestera_msg_nh_req { + struct prestera_msg_cmd cmd; + struct prestera_msg_nh nh[PRESTERA_NHGR_SIZE_MAX]; + __le32 size; + __le32 grp_id; +}; + +struct prestera_msg_nh_chunk_req { + struct prestera_msg_cmd cmd; + __le32 offset; +}; + +struct prestera_msg_nh_chunk_resp { + struct prestera_msg_ret ret; + u8 hw_state[PRESTERA_MSG_CHUNK_SIZE]; +}; + +struct prestera_msg_nh_grp_req { + struct prestera_msg_cmd cmd; + __le32 grp_id; + __le32 size; +}; + +struct prestera_msg_nh_grp_resp { + struct prestera_msg_ret ret; + __le32 grp_id; +}; + +struct prestera_msg_vr_req { + struct prestera_msg_cmd cmd; + __le16 vr_id; + u8 __pad[2]; +}; + +struct prestera_msg_vr_resp { + struct prestera_msg_ret ret; + __le16 vr_id; + u8 __pad[2]; +}; + struct prestera_msg_lag_req { struct prestera_msg_cmd cmd; __le32 port; @@ -485,6 +642,26 @@ struct mvsw_msg_cpu_code_counter_ret { __le64 packet_count; }; +struct prestera_msg_policer_req { + struct prestera_msg_cmd cmd; + __le32 id; + union { + struct { + __le64 cir; + __le32 cbs; + } __packed sr_tcm; /* make sure always 12 bytes size */ + __le32 reserved[6]; + }; + u8 mode; + u8 type; + u8 pad[2]; +}; + +struct prestera_msg_policer_resp { + struct prestera_msg_ret ret; + __le32 id; +}; + struct prestera_msg_event { __le16 type; __le16 id; @@ -511,6 +688,57 @@ struct prestera_msg_event_fdb { u8 dest_type; }; +struct prestera_msg_flood_domain_create_req { + struct prestera_msg_cmd cmd; +}; + +struct prestera_msg_flood_domain_create_resp { + struct prestera_msg_ret ret; + __le32 flood_domain_idx; +}; + +struct prestera_msg_flood_domain_destroy_req { + struct prestera_msg_cmd cmd; + __le32 flood_domain_idx; +}; + +struct prestera_msg_flood_domain_ports_set_req { + struct prestera_msg_cmd cmd; + __le32 flood_domain_idx; + __le32 ports_num; +}; + +struct prestera_msg_flood_domain_ports_reset_req { + struct prestera_msg_cmd cmd; + __le32 flood_domain_idx; +}; + +struct prestera_msg_flood_domain_port { + union { + struct { + __le32 port_num; + __le32 dev_num; + }; + __le16 lag_id; + }; + __le16 vid; + __le16 port_type; +}; + +struct prestera_msg_mdb_create_req { + struct prestera_msg_cmd cmd; + __le32 flood_domain_idx; + __le16 vid; + u8 mac[ETH_ALEN]; +}; + +struct prestera_msg_mdb_destroy_req { + struct prestera_msg_cmd cmd; + __le32 flood_domain_idx; + __le16 vid; + u8 mac[ETH_ALEN]; +}; + static void prestera_hw_build_tests(void) { /* check requests */ @@ -521,14 +749,38 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_vlan_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_fdb_req) != 28); BUILD_BUG_ON(sizeof(struct prestera_msg_bridge_req) != 16); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_rule_req) != 16); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_ruleset_bind_req) != 16); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_ruleset_req) != 8); BUILD_BUG_ON(sizeof(struct prestera_msg_span_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_stp_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_rxtx_req) != 8); BUILD_BUG_ON(sizeof(struct prestera_msg_lag_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_cpu_code_counter_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_create_req) != 84); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_destroy_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_rule_add_req) != 168); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_rule_del_req) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_bind_req) != 20); + BUILD_BUG_ON(sizeof(struct prestera_msg_acl_action) != 32); + BUILD_BUG_ON(sizeof(struct prestera_msg_counter_req) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_counter_stats) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_rif_req) != 36); + BUILD_BUG_ON(sizeof(struct prestera_msg_vr_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_lpm_req) != 36); + BUILD_BUG_ON(sizeof(struct prestera_msg_policer_req) != 36); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_create_req) != 4); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_destroy_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_ports_set_req) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_ports_reset_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_create_req) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_destroy_req) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_req) != 124); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_req) != 12); + + /* structure that are part of req/resp fw messages */ + BUILD_BUG_ON(sizeof(struct prestera_msg_iface) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_ip_addr) != 20); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_port) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh) != 28); /* check responses */ BUILD_BUG_ON(sizeof(struct prestera_msg_common_resp) != 8); @@ -537,11 +789,16 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_port_stats_resp) != 248); BUILD_BUG_ON(sizeof(struct prestera_msg_port_info_resp) != 20); BUILD_BUG_ON(sizeof(struct prestera_msg_bridge_resp) != 12); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_rule_resp) != 12); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_rule_stats_resp) != 24); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_ruleset_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_span_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_rxtx_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_resp) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_counter_resp) != 24); + BUILD_BUG_ON(sizeof(struct prestera_msg_rif_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_vr_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_policer_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_create_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_resp) != 1032); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_resp) != 12); /* check events */ BUILD_BUG_ON(sizeof(struct prestera_msg_event_port) != 20); @@ -819,6 +1076,8 @@ int prestera_hw_switch_init(struct prestera_switch *sw) sw->id = resp.switch_id; sw->lag_member_max = resp.lag_member_max; sw->lag_max = resp.lag_max; + sw->size_tbl_router_nexthop = + __le32_to_cpu(resp.size_tbl_router_nexthop); return 0; } @@ -1044,225 +1303,168 @@ static void prestera_hw_remote_fc_to_eth(u8 fc, bool *pause, bool *asym_pause) } } -int prestera_hw_acl_ruleset_create(struct prestera_switch *sw, u16 *ruleset_id) +int prestera_hw_vtcam_create(struct prestera_switch *sw, + u8 lookup, const u32 *keymask, u32 *vtcam_id, + enum prestera_hw_vtcam_direction_t dir) { - struct prestera_msg_acl_ruleset_resp resp; - struct prestera_msg_acl_ruleset_req req; int err; + struct prestera_msg_vtcam_resp resp; + struct prestera_msg_vtcam_create_req req = { + .lookup = lookup, + .direction = dir, + }; + + if (keymask) + memcpy(req.keymask, keymask, sizeof(req.keymask)); + else + memset(req.keymask, 0, sizeof(req.keymask)); - err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULESET_CREATE, + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_VTCAM_CREATE, &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); if (err) return err; - *ruleset_id = __le16_to_cpu(resp.id); - + *vtcam_id = __le32_to_cpu(resp.vtcam_id); return 0; } -int prestera_hw_acl_ruleset_del(struct prestera_switch *sw, u16 ruleset_id) +int prestera_hw_vtcam_destroy(struct prestera_switch *sw, u32 vtcam_id) { - struct prestera_msg_acl_ruleset_req req = { - .id = __cpu_to_le16(ruleset_id), + struct prestera_msg_vtcam_destroy_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), }; - return prestera_cmd(sw, PRESTERA_CMD_TYPE_ACL_RULESET_DELETE, + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_DESTROY, &req.cmd, sizeof(req)); } -static int prestera_hw_acl_actions_put(struct prestera_msg_acl_action *action, - struct prestera_acl_rule *rule) +static int +prestera_acl_rule_add_put_action(struct prestera_msg_acl_action *action, + struct prestera_acl_hw_action_info *info) { - struct list_head *a_list = prestera_acl_rule_action_list_get(rule); - struct prestera_acl_rule_action_entry *a_entry; - int i = 0; - - list_for_each_entry(a_entry, a_list, list) { - action[i].id = __cpu_to_le32(a_entry->id); - - switch (a_entry->id) { - case PRESTERA_ACL_RULE_ACTION_ACCEPT: - case PRESTERA_ACL_RULE_ACTION_DROP: - case PRESTERA_ACL_RULE_ACTION_TRAP: - /* just rule action id, no specific data */ - break; - default: - return -EINVAL; - } - - i++; - } - - return 0; -} - -static int prestera_hw_acl_matches_put(struct prestera_msg_acl_match *match, - struct prestera_acl_rule *rule) -{ - struct list_head *m_list = prestera_acl_rule_match_list_get(rule); - struct prestera_acl_rule_match_entry *m_entry; - int i = 0; - - list_for_each_entry(m_entry, m_list, list) { - match[i].type = __cpu_to_le32(m_entry->type); - - switch (m_entry->type) { - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID: - match[i].keymask.u16.key = - __cpu_to_le16(m_entry->keymask.u16.key); - match[i].keymask.u16.mask = - __cpu_to_le16(m_entry->keymask.u16.mask); - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO: - match[i].keymask.u8.key = m_entry->keymask.u8.key; - match[i].keymask.u8.mask = m_entry->keymask.u8.mask; - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC: - memcpy(match[i].keymask.mac.key, - m_entry->keymask.mac.key, - sizeof(match[i].keymask.mac.key)); - memcpy(match[i].keymask.mac.mask, - m_entry->keymask.mac.mask, - sizeof(match[i].keymask.mac.mask)); - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_SRC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_DST: - match[i].keymask.u32.key = - __cpu_to_le32(m_entry->keymask.u32.key); - match[i].keymask.u32.mask = - __cpu_to_le32(m_entry->keymask.u32.mask); - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT: - match[i].keymask.u64.key = - __cpu_to_le64(m_entry->keymask.u64.key); - match[i].keymask.u64.mask = - __cpu_to_le64(m_entry->keymask.u64.mask); - break; - default: - return -EINVAL; - } + action->id = __cpu_to_le32(info->id); - i++; + switch (info->id) { + case PRESTERA_ACL_RULE_ACTION_ACCEPT: + case PRESTERA_ACL_RULE_ACTION_DROP: + case PRESTERA_ACL_RULE_ACTION_TRAP: + /* just rule action id, no specific data */ + break; + case PRESTERA_ACL_RULE_ACTION_JUMP: + action->jump.index = __cpu_to_le32(info->jump.index); + break; + case PRESTERA_ACL_RULE_ACTION_POLICE: + action->police.id = __cpu_to_le32(info->police.id); + break; + case PRESTERA_ACL_RULE_ACTION_COUNT: + action->count.id = __cpu_to_le32(info->count.id); + break; + default: + return -EINVAL; } return 0; } -int prestera_hw_acl_rule_add(struct prestera_switch *sw, - struct prestera_acl_rule *rule, - u32 *rule_id) +int prestera_hw_vtcam_rule_add(struct prestera_switch *sw, + u32 vtcam_id, u32 prio, void *key, void *keymask, + struct prestera_acl_hw_action_info *act, + u8 n_act, u32 *rule_id) { - struct prestera_msg_acl_action *actions; - struct prestera_msg_acl_match *matches; - struct prestera_msg_acl_rule_resp resp; - struct prestera_msg_acl_rule_req *req; - u8 n_actions; - u8 n_matches; + struct prestera_msg_acl_action *actions_msg; + struct prestera_msg_vtcam_rule_add_req *req; + struct prestera_msg_vtcam_resp resp; void *buff; u32 size; int err; + u8 i; - n_actions = prestera_acl_rule_action_len(rule); - n_matches = prestera_acl_rule_match_len(rule); - - size = sizeof(*req) + sizeof(*actions) * n_actions + - sizeof(*matches) * n_matches; + size = sizeof(*req) + sizeof(*actions_msg) * n_act; buff = kzalloc(size, GFP_KERNEL); if (!buff) return -ENOMEM; req = buff; - actions = buff + sizeof(*req); - matches = buff + sizeof(*req) + sizeof(*actions) * n_actions; - - /* put acl actions into the message */ - err = prestera_hw_acl_actions_put(actions, rule); - if (err) - goto free_buff; + req->n_act = __cpu_to_le32(n_act); + actions_msg = buff + sizeof(*req); /* put acl matches into the message */ - err = prestera_hw_acl_matches_put(matches, rule); - if (err) - goto free_buff; + memcpy(req->key, key, sizeof(req->key)); + memcpy(req->keymask, keymask, sizeof(req->keymask)); - req->ruleset_id = __cpu_to_le16(prestera_acl_rule_ruleset_id_get(rule)); - req->priority = __cpu_to_le32(prestera_acl_rule_priority_get(rule)); - req->n_actions = prestera_acl_rule_action_len(rule); - req->n_matches = prestera_acl_rule_match_len(rule); + /* put acl actions into the message */ + for (i = 0; i < n_act; i++) { + err = prestera_acl_rule_add_put_action(&actions_msg[i], + &act[i]); + if (err) + goto free_buff; + } - err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULE_ADD, + req->vtcam_id = __cpu_to_le32(vtcam_id); + req->prio = __cpu_to_le32(prio); + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_VTCAM_RULE_ADD, &req->cmd, size, &resp.ret, sizeof(resp)); if (err) goto free_buff; - *rule_id = __le32_to_cpu(resp.id); + *rule_id = __le32_to_cpu(resp.rule_id); free_buff: kfree(buff); return err; } -int prestera_hw_acl_rule_del(struct prestera_switch *sw, u32 rule_id) +int prestera_hw_vtcam_rule_del(struct prestera_switch *sw, + u32 vtcam_id, u32 rule_id) { - struct prestera_msg_acl_rule_req req = { + struct prestera_msg_vtcam_rule_del_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), .id = __cpu_to_le32(rule_id) }; - return prestera_cmd(sw, PRESTERA_CMD_TYPE_ACL_RULE_DELETE, + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_RULE_DELETE, &req.cmd, sizeof(req)); } -int prestera_hw_acl_rule_stats_get(struct prestera_switch *sw, u32 rule_id, - u64 *packets, u64 *bytes) +int prestera_hw_vtcam_iface_bind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id, u16 pcl_id) { - struct prestera_msg_acl_rule_stats_resp resp; - struct prestera_msg_acl_rule_req req = { - .id = __cpu_to_le32(rule_id) + struct prestera_msg_vtcam_bind_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), + .type = __cpu_to_le16(iface->type), + .pcl_id = __cpu_to_le16(pcl_id) }; - int err; - - err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULE_STATS_GET, - &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); - if (err) - return err; - - *packets = __le64_to_cpu(resp.packets); - *bytes = __le64_to_cpu(resp.bytes); - - return 0; -} -int prestera_hw_acl_port_bind(const struct prestera_port *port, u16 ruleset_id) -{ - struct prestera_msg_acl_ruleset_bind_req req = { - .port = __cpu_to_le32(port->hw_id), - .dev = __cpu_to_le32(port->dev_id), - .ruleset_id = __cpu_to_le16(ruleset_id), - }; + if (iface->type == PRESTERA_ACL_IFACE_TYPE_PORT) { + req.port.dev_id = __cpu_to_le32(iface->port->dev_id); + req.port.hw_id = __cpu_to_le32(iface->port->hw_id); + } else { + req.index = __cpu_to_le32(iface->index); + } - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_ACL_PORT_BIND, + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_IFACE_BIND, &req.cmd, sizeof(req)); } -int prestera_hw_acl_port_unbind(const struct prestera_port *port, - u16 ruleset_id) +int prestera_hw_vtcam_iface_unbind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id) { - struct prestera_msg_acl_ruleset_bind_req req = { - .port = __cpu_to_le32(port->hw_id), - .dev = __cpu_to_le32(port->dev_id), - .ruleset_id = __cpu_to_le16(ruleset_id), + struct prestera_msg_vtcam_bind_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), + .type = __cpu_to_le16(iface->type) }; - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_ACL_PORT_UNBIND, + if (iface->type == PRESTERA_ACL_IFACE_TYPE_PORT) { + req.port.dev_id = __cpu_to_le32(iface->port->dev_id); + req.port.hw_id = __cpu_to_le32(iface->port->hw_id); + } else { + req.index = __cpu_to_le32(iface->index); + } + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_IFACE_UNBIND, &req.cmd, sizeof(req)); } @@ -1285,27 +1487,39 @@ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id) return 0; } -int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id) +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id, + bool ingress) { struct prestera_msg_span_req req = { .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), .id = span_id, }; + enum prestera_cmd_type_t cmd_type; + + if (ingress) + cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND; + else + cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND; + + return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req)); - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_BIND, - &req.cmd, sizeof(req)); } -int prestera_hw_span_unbind(const struct prestera_port *port) +int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress) { struct prestera_msg_span_req req = { .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), }; + enum prestera_cmd_type_t cmd_type; - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_UNBIND, - &req.cmd, sizeof(req)); + if (ingress) + cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND; + else + cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND; + + return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req)); } int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id) @@ -1457,7 +1671,7 @@ int prestera_hw_port_learning_set(struct prestera_port *port, bool enable) &req.cmd, sizeof(req)); } -static int prestera_hw_port_uc_flood_set(struct prestera_port *port, bool flood) +int prestera_hw_port_uc_flood_set(const struct prestera_port *port, bool flood) { struct prestera_msg_port_attr_req req = { .attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_FLOOD), @@ -1475,7 +1689,7 @@ static int prestera_hw_port_uc_flood_set(struct prestera_port *port, bool flood) &req.cmd, sizeof(req)); } -static int prestera_hw_port_mc_flood_set(struct prestera_port *port, bool flood) +int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood) { struct prestera_msg_port_attr_req req = { .attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_FLOOD), @@ -1493,14 +1707,15 @@ static int prestera_hw_port_mc_flood_set(struct prestera_port *port, bool flood) &req.cmd, sizeof(req)); } -static int prestera_hw_port_flood_set_v2(struct prestera_port *port, bool flood) +int prestera_hw_port_br_locked_set(const struct prestera_port *port, + bool br_locked) { struct prestera_msg_port_attr_req req = { - .attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_FLOOD), + .attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_LOCKED), .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), .param = { - .flood = flood, + .br_locked = br_locked, } }; @@ -1508,41 +1723,6 @@ static int prestera_hw_port_flood_set_v2(struct prestera_port *port, bool flood) &req.cmd, sizeof(req)); } -int prestera_hw_port_flood_set(struct prestera_port *port, unsigned long mask, - unsigned long val) -{ - int err; - - if (port->sw->dev->fw_rev.maj <= 2) { - if (!(mask & BR_FLOOD)) - return 0; - - return prestera_hw_port_flood_set_v2(port, val & BR_FLOOD); - } - - if (mask & BR_FLOOD) { - err = prestera_hw_port_uc_flood_set(port, val & BR_FLOOD); - if (err) - goto err_uc_flood; - } - - if (mask & BR_MCAST_FLOOD) { - err = prestera_hw_port_mc_flood_set(port, val & BR_MCAST_FLOOD); - if (err) - goto err_mc_flood; - } - - return 0; - -err_mc_flood: - prestera_hw_port_mc_flood_set(port, 0); -err_uc_flood: - if (mask & BR_FLOOD) - prestera_hw_port_uc_flood_set(port, 0); - - return err; -} - int prestera_hw_vlan_create(struct prestera_switch *sw, u16 vid) { struct prestera_msg_vlan_req req = { @@ -1796,6 +1976,197 @@ int prestera_hw_bridge_port_delete(struct prestera_port *port, u16 bridge_id) &req.cmd, sizeof(req)); } +static int prestera_iface_to_msg(struct prestera_iface *iface, + struct prestera_msg_iface *msg_if) +{ + switch (iface->type) { + case PRESTERA_IF_PORT_E: + case PRESTERA_IF_VID_E: + msg_if->port = __cpu_to_le32(iface->dev_port.port_num); + msg_if->dev = __cpu_to_le32(iface->dev_port.hw_dev_num); + break; + case PRESTERA_IF_LAG_E: + msg_if->lag_id = __cpu_to_le16(iface->lag_id); + break; + default: + return -EOPNOTSUPP; + } + + msg_if->vr_id = __cpu_to_le16(iface->vr_id); + msg_if->vid = __cpu_to_le16(iface->vlan_id); + msg_if->type = iface->type; + return 0; +} + +int prestera_hw_rif_create(struct prestera_switch *sw, + struct prestera_iface *iif, u8 *mac, u16 *rif_id) +{ + struct prestera_msg_rif_resp resp; + struct prestera_msg_rif_req req; + int err; + + memcpy(req.mac, mac, ETH_ALEN); + + err = prestera_iface_to_msg(iif, &req.iif); + if (err) + return err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ROUTER_RIF_CREATE, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *rif_id = __le16_to_cpu(resp.rif_id); + return err; +} + +int prestera_hw_rif_delete(struct prestera_switch *sw, u16 rif_id, + struct prestera_iface *iif) +{ + struct prestera_msg_rif_req req = { + .rif_id = __cpu_to_le16(rif_id), + }; + int err; + + err = prestera_iface_to_msg(iif, &req.iif); + if (err) + return err; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_RIF_DELETE, &req.cmd, + sizeof(req)); +} + +int prestera_hw_vr_create(struct prestera_switch *sw, u16 *vr_id) +{ + struct prestera_msg_vr_resp resp; + struct prestera_msg_vr_req req; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ROUTER_VR_CREATE, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *vr_id = __le16_to_cpu(resp.vr_id); + return err; +} + +int prestera_hw_vr_delete(struct prestera_switch *sw, u16 vr_id) +{ + struct prestera_msg_vr_req req = { + .vr_id = __cpu_to_le16(vr_id), + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_VR_DELETE, &req.cmd, + sizeof(req)); +} + +int prestera_hw_lpm_add(struct prestera_switch *sw, u16 vr_id, + __be32 dst, u32 dst_len, u32 grp_id) +{ + struct prestera_msg_lpm_req req = { + .dst_len = __cpu_to_le32(dst_len), + .vr_id = __cpu_to_le16(vr_id), + .grp_id = __cpu_to_le32(grp_id), + .dst.u.ipv4 = dst + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_LPM_ADD, &req.cmd, + sizeof(req)); +} + +int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id, + __be32 dst, u32 dst_len) +{ + struct prestera_msg_lpm_req req = { + .dst_len = __cpu_to_le32(dst_len), + .vr_id = __cpu_to_le16(vr_id), + .dst.u.ipv4 = dst + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_LPM_DELETE, &req.cmd, + sizeof(req)); +} + +int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count, + struct prestera_neigh_info *nhs, u32 grp_id) +{ + struct prestera_msg_nh_req req = { .size = __cpu_to_le32((u32)count), + .grp_id = __cpu_to_le32(grp_id) }; + int i, err; + + for (i = 0; i < count; i++) { + req.nh[i].is_active = nhs[i].connected; + memcpy(&req.nh[i].mac, nhs[i].ha, ETH_ALEN); + err = prestera_iface_to_msg(&nhs[i].iface, &req.nh[i].oif); + if (err) + return err; + } + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET, &req.cmd, + sizeof(req)); +} + +int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw, + u8 *hw_state, u32 buf_size /* Buffer in bytes */) +{ + static struct prestera_msg_nh_chunk_resp resp; + struct prestera_msg_nh_chunk_req req; + u32 buf_offset; + int err; + + memset(&hw_state[0], 0, buf_size); + buf_offset = 0; + while (1) { + if (buf_offset >= buf_size) + break; + + memset(&req, 0, sizeof(req)); + req.offset = __cpu_to_le32(buf_offset * 8); /* 8 bits in u8 */ + err = prestera_cmd_ret(sw, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET, + &req.cmd, sizeof(req), &resp.ret, + sizeof(resp)); + if (err) + return err; + + memcpy(&hw_state[buf_offset], &resp.hw_state[0], + buf_offset + PRESTERA_MSG_CHUNK_SIZE > buf_size ? + buf_size - buf_offset : PRESTERA_MSG_CHUNK_SIZE); + buf_offset += PRESTERA_MSG_CHUNK_SIZE; + } + + return 0; +} + +int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count, + u32 *grp_id) +{ + struct prestera_msg_nh_grp_req req = { .size = __cpu_to_le32((u32)nh_count) }; + struct prestera_msg_nh_grp_resp resp; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *grp_id = __le32_to_cpu(resp.grp_id); + return err; +} + +int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count, + u32 grp_id) +{ + struct prestera_msg_nh_grp_req req = { + .grp_id = __cpu_to_le32(grp_id), + .size = __cpu_to_le32(nh_count) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE, + &req.cmd, sizeof(req)); +} + int prestera_hw_rxtx_init(struct prestera_switch *sw, struct prestera_rxtx_params *params) { @@ -1916,3 +2287,275 @@ void prestera_hw_event_handler_unregister(struct prestera_switch *sw, list_del_rcu(&eh->list); kfree_rcu(eh, rcu); } + +int prestera_hw_counter_trigger(struct prestera_switch *sw, u32 block_id) +{ + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(block_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_TRIGGER, + &req.cmd, sizeof(req)); +} + +int prestera_hw_counter_abort(struct prestera_switch *sw) +{ + struct prestera_msg_counter_req req; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_ABORT, + &req.cmd, sizeof(req)); +} + +int prestera_hw_counters_get(struct prestera_switch *sw, u32 idx, + u32 *len, bool *done, + struct prestera_counter_stats *stats) +{ + struct prestera_msg_counter_resp *resp; + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(idx), + .num_counters = __cpu_to_le32(*len), + }; + size_t size = struct_size(resp, stats, *len); + int err, i; + + resp = kmalloc(size, GFP_KERNEL); + if (!resp) + return -ENOMEM; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_COUNTER_GET, + &req.cmd, sizeof(req), &resp->ret, size); + if (err) + goto free_buff; + + for (i = 0; i < __le32_to_cpu(resp->num_counters); i++) { + stats[i].packets += __le64_to_cpu(resp->stats[i].packets); + stats[i].bytes += __le64_to_cpu(resp->stats[i].bytes); + } + + *len = __le32_to_cpu(resp->num_counters); + *done = __le32_to_cpu(resp->done); + +free_buff: + kfree(resp); + return err; +} + +int prestera_hw_counter_block_get(struct prestera_switch *sw, + u32 client, u32 *block_id, u32 *offset, + u32 *num_counters) +{ + struct prestera_msg_counter_resp resp; + struct prestera_msg_counter_req req = { + .client = __cpu_to_le32(client) + }; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_COUNTER_BLOCK_GET, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *block_id = __le32_to_cpu(resp.block_id); + *offset = __le32_to_cpu(resp.offset); + *num_counters = __le32_to_cpu(resp.num_counters); + + return 0; +} + +int prestera_hw_counter_block_release(struct prestera_switch *sw, + u32 block_id) +{ + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(block_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_BLOCK_RELEASE, + &req.cmd, sizeof(req)); +} + +int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id, + u32 counter_id) +{ + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(block_id), + .num_counters = __cpu_to_le32(counter_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_CLEAR, + &req.cmd, sizeof(req)); +} + +int prestera_hw_policer_create(struct prestera_switch *sw, u8 type, + u32 *policer_id) +{ + struct prestera_msg_policer_resp resp; + struct prestera_msg_policer_req req = { + .type = type + }; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_POLICER_CREATE, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *policer_id = __le32_to_cpu(resp.id); + return 0; +} + +int prestera_hw_policer_release(struct prestera_switch *sw, + u32 policer_id) +{ + struct prestera_msg_policer_req req = { + .id = __cpu_to_le32(policer_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_POLICER_RELEASE, + &req.cmd, sizeof(req)); +} + +int prestera_hw_policer_sr_tcm_set(struct prestera_switch *sw, + u32 policer_id, u64 cir, u32 cbs) +{ + struct prestera_msg_policer_req req = { + .mode = PRESTERA_POLICER_MODE_SR_TCM, + .id = __cpu_to_le32(policer_id), + .sr_tcm = { + .cir = __cpu_to_le64(cir), + .cbs = __cpu_to_le32(cbs) + } + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_POLICER_SET, + &req.cmd, sizeof(req)); +} + +int prestera_hw_flood_domain_create(struct prestera_flood_domain *domain) +{ + struct prestera_msg_flood_domain_create_resp resp; + struct prestera_msg_flood_domain_create_req req; + int err; + + err = prestera_cmd_ret(domain->sw, + PRESTERA_CMD_TYPE_FLOOD_DOMAIN_CREATE, &req.cmd, + sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + domain->idx = __le32_to_cpu(resp.flood_domain_idx); + + return 0; +} + +int prestera_hw_flood_domain_destroy(struct prestera_flood_domain *domain) +{ + struct prestera_msg_flood_domain_destroy_req req = { + .flood_domain_idx = __cpu_to_le32(domain->idx), + }; + + return prestera_cmd(domain->sw, PRESTERA_CMD_TYPE_FLOOD_DOMAIN_DESTROY, + &req.cmd, sizeof(req)); +} + +int prestera_hw_flood_domain_ports_set(struct prestera_flood_domain *domain) +{ + struct prestera_flood_domain_port *flood_domain_port; + struct prestera_msg_flood_domain_ports_set_req *req; + struct prestera_msg_flood_domain_port *ports; + struct prestera_switch *sw = domain->sw; + struct prestera_port *port; + u32 ports_num = 0; + int buf_size; + void *buff; + u16 lag_id; + int err; + + list_for_each_entry(flood_domain_port, &domain->flood_domain_port_list, + flood_domain_port_node) + ports_num++; + + if (!ports_num) + return -EINVAL; + + buf_size = sizeof(*req) + sizeof(*ports) * ports_num; + + buff = kmalloc(buf_size, GFP_KERNEL); + if (!buff) + return -ENOMEM; + + req = buff; + ports = buff + sizeof(*req); + + req->flood_domain_idx = __cpu_to_le32(domain->idx); + req->ports_num = __cpu_to_le32(ports_num); + + list_for_each_entry(flood_domain_port, &domain->flood_domain_port_list, + flood_domain_port_node) { + if (netif_is_lag_master(flood_domain_port->dev)) { + if (prestera_lag_id(sw, flood_domain_port->dev, + &lag_id)) { + kfree(buff); + return -EINVAL; + } + + ports->port_type = + __cpu_to_le16(PRESTERA_HW_FLOOD_DOMAIN_PORT_TYPE_LAG); + ports->lag_id = __cpu_to_le16(lag_id); + } else { + port = prestera_port_dev_lower_find(flood_domain_port->dev); + + ports->port_type = + __cpu_to_le16(PRESTERA_HW_FDB_ENTRY_TYPE_REG_PORT); + ports->dev_num = __cpu_to_le32(port->dev_id); + ports->port_num = __cpu_to_le32(port->hw_id); + } + + ports->vid = __cpu_to_le16(flood_domain_port->vid); + + ports++; + } + + err = prestera_cmd(sw, PRESTERA_CMD_TYPE_FLOOD_DOMAIN_PORTS_SET, + &req->cmd, buf_size); + + kfree(buff); + + return err; +} + +int prestera_hw_flood_domain_ports_reset(struct prestera_flood_domain *domain) +{ + struct prestera_msg_flood_domain_ports_reset_req req = { + .flood_domain_idx = __cpu_to_le32(domain->idx), + }; + + return prestera_cmd(domain->sw, + PRESTERA_CMD_TYPE_FLOOD_DOMAIN_PORTS_RESET, &req.cmd, + sizeof(req)); +} + +int prestera_hw_mdb_create(struct prestera_mdb_entry *mdb) +{ + struct prestera_msg_mdb_create_req req = { + .flood_domain_idx = __cpu_to_le32(mdb->flood_domain->idx), + .vid = __cpu_to_le16(mdb->vid), + }; + + memcpy(req.mac, mdb->addr, ETH_ALEN); + + return prestera_cmd(mdb->sw, PRESTERA_CMD_TYPE_MDB_CREATE, &req.cmd, + sizeof(req)); +} + +int prestera_hw_mdb_destroy(struct prestera_mdb_entry *mdb) +{ + struct prestera_msg_mdb_destroy_req req = { + .flood_domain_idx = __cpu_to_le32(mdb->flood_domain->idx), + .vid = __cpu_to_le16(mdb->vid), + }; + + memcpy(req.mac, mdb->addr, ETH_ALEN); + + return prestera_cmd(mdb->sw, PRESTERA_CMD_TYPE_MDB_DESTROY, &req.cmd, + sizeof(req)); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_hw.h index 57a3c2e5b112..0a929279e1ce 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.h @@ -5,6 +5,7 @@ #define _PRESTERA_HW_H_ #include <linux/types.h> +#include "prestera_acl.h" enum prestera_accept_frm_type { PRESTERA_ACCEPT_FRAME_TYPE_TAGGED, @@ -106,23 +107,46 @@ enum { PRESTERA_STP_FORWARD, }; +enum { + PRESTERA_POLICER_TYPE_INGRESS, + PRESTERA_POLICER_TYPE_EGRESS +}; + enum prestera_hw_cpu_code_cnt_t { PRESTERA_HW_CPU_CODE_CNT_TYPE_DROP = 0, PRESTERA_HW_CPU_CODE_CNT_TYPE_TRAP = 1, }; +enum prestera_hw_vtcam_direction_t { + PRESTERA_HW_VTCAM_DIR_INGRESS = 0, + PRESTERA_HW_VTCAM_DIR_EGRESS = 1, +}; + +enum { + PRESTERA_HW_COUNTER_CLIENT_INGRESS_LOOKUP_0 = 0, + PRESTERA_HW_COUNTER_CLIENT_INGRESS_LOOKUP_1 = 1, + PRESTERA_HW_COUNTER_CLIENT_INGRESS_LOOKUP_2 = 2, + PRESTERA_HW_COUNTER_CLIENT_EGRESS_LOOKUP = 3, +}; + struct prestera_switch; struct prestera_port; struct prestera_port_stats; struct prestera_port_caps; enum prestera_event_type; struct prestera_event; -struct prestera_acl_rule; typedef void (*prestera_event_cb_t) (struct prestera_switch *sw, struct prestera_event *evt, void *arg); struct prestera_rxtx_params; +struct prestera_acl_hw_action_info; +struct prestera_acl_iface; +struct prestera_counter_stats; +struct prestera_iface; +struct prestera_flood_domain; +struct prestera_mdb_entry; +struct prestera_neigh_info; /* Switch API */ int prestera_hw_switch_init(struct prestera_switch *sw); @@ -158,8 +182,10 @@ int prestera_hw_port_stats_get(const struct prestera_port *port, struct prestera_port_stats *stats); int prestera_hw_port_speed_get(const struct prestera_port *port, u32 *speed); int prestera_hw_port_learning_set(struct prestera_port *port, bool enable); -int prestera_hw_port_flood_set(struct prestera_port *port, unsigned long mask, - unsigned long val); +int prestera_hw_port_uc_flood_set(const struct prestera_port *port, bool flood); +int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood); +int prestera_hw_port_br_locked_set(const struct prestera_port *port, + bool br_locked); int prestera_hw_port_accept_frm_type(struct prestera_port *port, enum prestera_accept_frm_type type); /* Vlan API */ @@ -186,28 +212,71 @@ int prestera_hw_bridge_delete(struct prestera_switch *sw, u16 bridge_id); int prestera_hw_bridge_port_add(struct prestera_port *port, u16 bridge_id); int prestera_hw_bridge_port_delete(struct prestera_port *port, u16 bridge_id); -/* ACL API */ -int prestera_hw_acl_ruleset_create(struct prestera_switch *sw, - u16 *ruleset_id); -int prestera_hw_acl_ruleset_del(struct prestera_switch *sw, - u16 ruleset_id); -int prestera_hw_acl_rule_add(struct prestera_switch *sw, - struct prestera_acl_rule *rule, - u32 *rule_id); -int prestera_hw_acl_rule_del(struct prestera_switch *sw, u32 rule_id); -int prestera_hw_acl_rule_stats_get(struct prestera_switch *sw, - u32 rule_id, u64 *packets, u64 *bytes); -int prestera_hw_acl_port_bind(const struct prestera_port *port, - u16 ruleset_id); -int prestera_hw_acl_port_unbind(const struct prestera_port *port, - u16 ruleset_id); +/* vTCAM API */ +int prestera_hw_vtcam_create(struct prestera_switch *sw, + u8 lookup, const u32 *keymask, u32 *vtcam_id, + enum prestera_hw_vtcam_direction_t direction); +int prestera_hw_vtcam_rule_add(struct prestera_switch *sw, u32 vtcam_id, + u32 prio, void *key, void *keymask, + struct prestera_acl_hw_action_info *act, + u8 n_act, u32 *rule_id); +int prestera_hw_vtcam_rule_del(struct prestera_switch *sw, + u32 vtcam_id, u32 rule_id); +int prestera_hw_vtcam_destroy(struct prestera_switch *sw, u32 vtcam_id); +int prestera_hw_vtcam_iface_bind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id, u16 pcl_id); +int prestera_hw_vtcam_iface_unbind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id); + +/* Counter API */ +int prestera_hw_counter_trigger(struct prestera_switch *sw, u32 block_id); +int prestera_hw_counter_abort(struct prestera_switch *sw); +int prestera_hw_counters_get(struct prestera_switch *sw, u32 idx, + u32 *len, bool *done, + struct prestera_counter_stats *stats); +int prestera_hw_counter_block_get(struct prestera_switch *sw, + u32 client, u32 *block_id, u32 *offset, + u32 *num_counters); +int prestera_hw_counter_block_release(struct prestera_switch *sw, + u32 block_id); +int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id, + u32 counter_id); /* SPAN API */ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id); -int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id); -int prestera_hw_span_unbind(const struct prestera_port *port); +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id, + bool ingress); +int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress); int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id); +/* Router API */ +int prestera_hw_rif_create(struct prestera_switch *sw, + struct prestera_iface *iif, u8 *mac, u16 *rif_id); +int prestera_hw_rif_delete(struct prestera_switch *sw, u16 rif_id, + struct prestera_iface *iif); + +/* Virtual Router API */ +int prestera_hw_vr_create(struct prestera_switch *sw, u16 *vr_id); +int prestera_hw_vr_delete(struct prestera_switch *sw, u16 vr_id); + +/* LPM PI */ +int prestera_hw_lpm_add(struct prestera_switch *sw, u16 vr_id, + __be32 dst, u32 dst_len, u32 grp_id); +int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id, + __be32 dst, u32 dst_len); + +/* NH API */ +int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count, + struct prestera_neigh_info *nhs, u32 grp_id); +int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw, + u8 *hw_state, u32 buf_size /* Buffer in bytes */); +int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count, + u32 *grp_id); +int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count, + u32 grp_id); + /* Event handlers */ int prestera_hw_event_handler_register(struct prestera_switch *sw, enum prestera_event_type type, @@ -241,4 +310,21 @@ prestera_hw_cpu_code_counters_get(struct prestera_switch *sw, u8 code, enum prestera_hw_cpu_code_cnt_t counter_type, u64 *packet_count); +/* Policer API */ +int prestera_hw_policer_create(struct prestera_switch *sw, u8 type, + u32 *policer_id); +int prestera_hw_policer_release(struct prestera_switch *sw, + u32 policer_id); +int prestera_hw_policer_sr_tcm_set(struct prestera_switch *sw, + u32 policer_id, u64 cir, u32 cbs); + +/* Flood domain / MDB API */ +int prestera_hw_flood_domain_create(struct prestera_flood_domain *domain); +int prestera_hw_flood_domain_destroy(struct prestera_flood_domain *domain); +int prestera_hw_flood_domain_ports_set(struct prestera_flood_domain *domain); +int prestera_hw_flood_domain_ports_reset(struct prestera_flood_domain *domain); + +int prestera_hw_mdb_create(struct prestera_mdb_entry *mdb); +int prestera_hw_mdb_destroy(struct prestera_mdb_entry *mdb); + #endif /* _PRESTERA_HW_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index 4369a3ffad45..24f9d6024745 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -9,6 +9,7 @@ #include <linux/of.h> #include <linux/of_net.h> #include <linux/if_vlan.h> +#include <linux/phylink.h> #include "prestera.h" #include "prestera_hw.h" @@ -18,6 +19,7 @@ #include "prestera_rxtx.h" #include "prestera_devlink.h" #include "prestera_ethtool.h" +#include "prestera_counter.h" #include "prestera_switchdev.h" #define PRESTERA_MTU_DEFAULT 1536 @@ -27,6 +29,43 @@ #define PRESTERA_MAC_ADDR_NUM_MAX 255 static struct workqueue_struct *prestera_wq; +static struct workqueue_struct *prestera_owq; + +void prestera_queue_work(struct work_struct *work) +{ + queue_work(prestera_owq, work); +} + +void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay) +{ + queue_delayed_work(prestera_wq, work, delay); +} + +void prestera_queue_drain(void) +{ + drain_workqueue(prestera_wq); + drain_workqueue(prestera_owq); +} + +int prestera_port_learning_set(struct prestera_port *port, bool learn) +{ + return prestera_hw_port_learning_set(port, learn); +} + +int prestera_port_uc_flood_set(struct prestera_port *port, bool flood) +{ + return prestera_hw_port_uc_flood_set(port, flood); +} + +int prestera_port_mc_flood_set(struct prestera_port *port, bool flood) +{ + return prestera_hw_port_mc_flood_set(port, flood); +} + +int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked) +{ + return prestera_hw_port_br_locked_set(port, br_locked); +} int prestera_port_pvid_set(struct prestera_port *port, u16 vid) { @@ -54,12 +93,14 @@ int prestera_port_pvid_set(struct prestera_port *port, u16 vid) struct prestera_port *prestera_port_find_by_hwid(struct prestera_switch *sw, u32 dev_id, u32 hw_id) { - struct prestera_port *port = NULL; + struct prestera_port *port = NULL, *tmp; read_lock(&sw->port_list_lock); - list_for_each_entry(port, &sw->port_list, list) { - if (port->dev_id == dev_id && port->hw_id == hw_id) + list_for_each_entry(tmp, &sw->port_list, list) { + if (tmp->dev_id == dev_id && tmp->hw_id == hw_id) { + port = tmp; break; + } } read_unlock(&sw->port_list_lock); @@ -68,18 +109,28 @@ struct prestera_port *prestera_port_find_by_hwid(struct prestera_switch *sw, struct prestera_port *prestera_find_port(struct prestera_switch *sw, u32 id) { - struct prestera_port *port = NULL; + struct prestera_port *port = NULL, *tmp; read_lock(&sw->port_list_lock); - list_for_each_entry(port, &sw->port_list, list) { - if (port->id == id) + list_for_each_entry(tmp, &sw->port_list, list) { + if (tmp->id == id) { + port = tmp; break; + } } read_unlock(&sw->port_list_lock); return port; } +struct prestera_switch *prestera_switch_get(struct net_device *dev) +{ + struct prestera_port *port; + + port = prestera_port_dev_lower_find(dev); + return port ? port->sw : NULL; +} + int prestera_port_cfg_mac_read(struct prestera_port *port, struct prestera_port_mac_config *cfg) { @@ -108,18 +159,24 @@ static int prestera_port_open(struct net_device *dev) struct prestera_port_mac_config cfg_mac; int err = 0; - if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { - err = prestera_port_cfg_mac_read(port, &cfg_mac); - if (!err) { - cfg_mac.admin = true; - err = prestera_port_cfg_mac_write(port, &cfg_mac); - } + if (port->phy_link) { + phylink_start(port->phy_link); } else { - port->cfg_phy.admin = true; - err = prestera_hw_port_phy_mode_set(port, true, port->autoneg, - port->cfg_phy.mode, - port->adver_link_modes, - port->cfg_phy.mdix); + if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { + err = prestera_port_cfg_mac_read(port, &cfg_mac); + if (!err) { + cfg_mac.admin = true; + err = prestera_port_cfg_mac_write(port, + &cfg_mac); + } + } else { + port->cfg_phy.admin = true; + err = prestera_hw_port_phy_mode_set(port, true, + port->autoneg, + port->cfg_phy.mode, + port->adver_link_modes, + port->cfg_phy.mdix); + } } netif_start_queue(dev); @@ -135,30 +192,268 @@ static int prestera_port_close(struct net_device *dev) netif_stop_queue(dev); - if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { + if (port->phy_link) { + phylink_stop(port->phy_link); + phylink_disconnect_phy(port->phy_link); err = prestera_port_cfg_mac_read(port, &cfg_mac); if (!err) { cfg_mac.admin = false; prestera_port_cfg_mac_write(port, &cfg_mac); } } else { - port->cfg_phy.admin = false; - err = prestera_hw_port_phy_mode_set(port, false, port->autoneg, - port->cfg_phy.mode, - port->adver_link_modes, - port->cfg_phy.mdix); + if (port->caps.transceiver == PRESTERA_PORT_TCVR_SFP) { + err = prestera_port_cfg_mac_read(port, &cfg_mac); + if (!err) { + cfg_mac.admin = false; + prestera_port_cfg_mac_write(port, &cfg_mac); + } + } else { + port->cfg_phy.admin = false; + err = prestera_hw_port_phy_mode_set(port, false, port->autoneg, + port->cfg_phy.mode, + port->adver_link_modes, + port->cfg_phy.mdix); + } } return err; } +static void +prestera_port_mac_state_cache_read(struct prestera_port *port, + struct prestera_port_mac_state *state) +{ + spin_lock(&port->state_mac_lock); + *state = port->state_mac; + spin_unlock(&port->state_mac_lock); +} + +static void +prestera_port_mac_state_cache_write(struct prestera_port *port, + struct prestera_port_mac_state *state) +{ + spin_lock(&port->state_mac_lock); + port->state_mac = *state; + spin_unlock(&port->state_mac_lock); +} + +static struct prestera_port *prestera_pcs_to_port(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct prestera_port, phylink_pcs); +} + +static void prestera_mac_config(struct phylink_config *config, + unsigned int an_mode, + const struct phylink_link_state *state) +{ +} + +static void prestera_mac_link_down(struct phylink_config *config, + unsigned int mode, phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct prestera_port *port = netdev_priv(ndev); + struct prestera_port_mac_state state_mac; + + /* Invalidate. Parameters will update on next link event. */ + memset(&state_mac, 0, sizeof(state_mac)); + state_mac.valid = false; + prestera_port_mac_state_cache_write(port, &state_mac); +} + +static void prestera_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ +} + +static struct phylink_pcs * +prestera_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct net_device *dev = to_net_dev(config->dev); + struct prestera_port *port = netdev_priv(dev); + + return &port->phylink_pcs; +} + +static void prestera_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct prestera_port *port = container_of(pcs, struct prestera_port, + phylink_pcs); + struct prestera_port_mac_state smac; + + prestera_port_mac_state_cache_read(port, &smac); + + if (smac.valid) { + state->link = smac.oper ? 1 : 0; + /* AN is completed, when port is up */ + state->an_complete = (smac.oper && port->autoneg) ? 1 : 0; + state->speed = smac.speed; + state->duplex = smac.duplex; + } else { + state->link = 0; + state->an_complete = 0; + } +} + +static int prestera_pcs_config(struct phylink_pcs *pcs, + unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct prestera_port *port = prestera_pcs_to_port(pcs); + struct prestera_port_mac_config cfg_mac; + int err; + + err = prestera_port_cfg_mac_read(port, &cfg_mac); + if (err) + return err; + + cfg_mac.admin = true; + cfg_mac.fec = PRESTERA_PORT_FEC_OFF; + + switch (interface) { + case PHY_INTERFACE_MODE_10GBASER: + cfg_mac.speed = SPEED_10000; + cfg_mac.inband = 0; + cfg_mac.mode = PRESTERA_MAC_MODE_SR_LR; + break; + case PHY_INTERFACE_MODE_2500BASEX: + cfg_mac.speed = SPEED_2500; + cfg_mac.duplex = DUPLEX_FULL; + cfg_mac.inband = test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + advertising); + cfg_mac.mode = PRESTERA_MAC_MODE_SGMII; + break; + case PHY_INTERFACE_MODE_SGMII: + cfg_mac.inband = 1; + cfg_mac.mode = PRESTERA_MAC_MODE_SGMII; + break; + case PHY_INTERFACE_MODE_1000BASEX: + default: + cfg_mac.speed = SPEED_1000; + cfg_mac.duplex = DUPLEX_FULL; + cfg_mac.inband = test_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + advertising); + cfg_mac.mode = PRESTERA_MAC_MODE_1000BASE_X; + break; + } + + err = prestera_port_cfg_mac_write(port, &cfg_mac); + if (err) + return err; + + return 0; +} + +static void prestera_pcs_an_restart(struct phylink_pcs *pcs) +{ + /* TODO: add 1000basex AN restart support + * (Currently FW has no support for 1000baseX AN restart, but it will in the future, + * so as for now the function would stay empty.) + */ +} + +static const struct phylink_mac_ops prestera_mac_ops = { + .validate = phylink_generic_validate, + .mac_select_pcs = prestera_mac_select_pcs, + .mac_config = prestera_mac_config, + .mac_link_down = prestera_mac_link_down, + .mac_link_up = prestera_mac_link_up, +}; + +static const struct phylink_pcs_ops prestera_pcs_ops = { + .pcs_get_state = prestera_pcs_get_state, + .pcs_config = prestera_pcs_config, + .pcs_an_restart = prestera_pcs_an_restart, +}; + +static int prestera_port_sfp_bind(struct prestera_port *port) +{ + struct prestera_switch *sw = port->sw; + struct device_node *ports, *node; + struct fwnode_handle *fwnode; + struct phylink *phy_link; + int err; + + if (!sw->np) + return 0; + + of_node_get(sw->np); + ports = of_find_node_by_name(sw->np, "ports"); + + for_each_child_of_node(ports, node) { + int num; + + err = of_property_read_u32(node, "prestera,port-num", &num); + if (err) { + dev_err(sw->dev->dev, + "device node %pOF has no valid reg property: %d\n", + node, err); + goto out; + } + + if (port->fp_id != num) + continue; + + port->phylink_pcs.ops = &prestera_pcs_ops; + + port->phy_config.dev = &port->dev->dev; + port->phy_config.type = PHYLINK_NETDEV; + + fwnode = of_fwnode_handle(node); + + __set_bit(PHY_INTERFACE_MODE_10GBASER, + port->phy_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + port->phy_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + port->phy_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + port->phy_config.supported_interfaces); + + port->phy_config.mac_capabilities = + MAC_1000 | MAC_2500FD | MAC_10000FD; + + phy_link = phylink_create(&port->phy_config, fwnode, + PHY_INTERFACE_MODE_INTERNAL, + &prestera_mac_ops); + if (IS_ERR(phy_link)) { + netdev_err(port->dev, "failed to create phylink\n"); + err = PTR_ERR(phy_link); + goto out; + } + + port->phy_link = phy_link; + break; + } + +out: + of_node_put(node); + of_node_put(ports); + return err; +} + +static int prestera_port_sfp_unbind(struct prestera_port *port) +{ + if (port->phy_link) + phylink_destroy(port->phy_link); + + return 0; +} + static netdev_tx_t prestera_port_xmit(struct sk_buff *skb, struct net_device *dev) { return prestera_rxtx_xmit(netdev_priv(dev), skb); } -static int prestera_is_valid_mac_addr(struct prestera_port *port, u8 *addr) +int prestera_is_valid_mac_addr(struct prestera_port *port, const u8 *addr) { if (!is_valid_ether_addr(addr)) return -EADDRNOTAVAIL; @@ -332,6 +627,8 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) port->id = id; port->sw = sw; + spin_lock_init(&port->state_mac_lock); + err = prestera_hw_port_info_get(port, &port->dev_id, &port->hw_id, &port->fp_id); if (err) { @@ -346,8 +643,10 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_HW_TC; dev->netdev_ops = &prestera_netdev_ops; dev->ethtool_ops = &prestera_ethtool_ops; + SET_NETDEV_DEV(dev, sw->dev->dev); - netif_carrier_off(dev); + if (port->caps.transceiver != PRESTERA_PORT_TCVR_SFP) + netif_carrier_off(dev); dev->mtu = min_t(unsigned int, sw->mtu_max, PRESTERA_MTU_DEFAULT); dev->min_mtu = sw->mtu_min; @@ -398,7 +697,7 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) cfg_mac.admin = false; cfg_mac.mode = PRESTERA_MAC_MODE_MAX; } - cfg_mac.inband = false; + cfg_mac.inband = 0; cfg_mac.speed = 0; cfg_mac.duplex = DUPLEX_UNKNOWN; cfg_mac.fec = PRESTERA_PORT_FEC_OFF; @@ -440,8 +739,13 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) prestera_devlink_port_set(port); + err = prestera_port_sfp_bind(port); + if (err) + goto err_sfp_bind; + return 0; +err_sfp_bind: err_register_netdev: prestera_port_list_del(port); err_port_init: @@ -487,8 +791,10 @@ static int prestera_create_ports(struct prestera_switch *sw) return 0; err_port_create: - list_for_each_entry_safe(port, tmp, &sw->port_list, list) + list_for_each_entry_safe(port, tmp, &sw->port_list, list) { + prestera_port_sfp_unbind(port); prestera_port_destroy(port); + } return err; } @@ -496,25 +802,45 @@ err_port_create: static void prestera_port_handle_event(struct prestera_switch *sw, struct prestera_event *evt, void *arg) { + struct prestera_port_mac_state smac; + struct prestera_port_event *pevt; struct delayed_work *caching_dw; struct prestera_port *port; - port = prestera_find_port(sw, evt->port_evt.port_id); - if (!port || !port->dev) - return; - - caching_dw = &port->cached_hw_stats.caching_dw; - - prestera_ethtool_port_state_changed(port, &evt->port_evt); - if (evt->id == PRESTERA_PORT_EVENT_MAC_STATE_CHANGED) { + pevt = &evt->port_evt; + port = prestera_find_port(sw, pevt->port_id); + if (!port || !port->dev) + return; + + caching_dw = &port->cached_hw_stats.caching_dw; + + memset(&smac, 0, sizeof(smac)); + smac.valid = true; + smac.oper = pevt->data.mac.oper; + if (smac.oper) { + smac.mode = pevt->data.mac.mode; + smac.speed = pevt->data.mac.speed; + smac.duplex = pevt->data.mac.duplex; + smac.fc = pevt->data.mac.fc; + smac.fec = pevt->data.mac.fec; + } + prestera_port_mac_state_cache_write(port, &smac); + if (port->state_mac.oper) { - netif_carrier_on(port->dev); + if (port->phy_link) + phylink_mac_change(port->phy_link, true); + else + netif_carrier_on(port->dev); + if (!delayed_work_pending(caching_dw)) queue_delayed_work(prestera_wq, caching_dw, 0); - } else if (netif_running(port->dev) && - netif_carrier_ok(port->dev)) { - netif_carrier_off(port->dev); + } else { + if (port->phy_link) + phylink_mac_change(port->phy_link, false); + else if (netif_running(port->dev) && netif_carrier_ok(port->dev)) + netif_carrier_off(port->dev); + if (delayed_work_pending(caching_dw)) cancel_delayed_work(caching_dw); } @@ -537,18 +863,20 @@ static void prestera_event_handlers_unregister(struct prestera_switch *sw) static int prestera_switch_set_base_mac_addr(struct prestera_switch *sw) { struct device_node *base_mac_np; - struct device_node *np; - int ret; + int ret = 0; - np = of_find_compatible_node(NULL, NULL, "marvell,prestera"); - base_mac_np = of_parse_phandle(np, "base-mac-provider", 0); + if (sw->np) { + base_mac_np = of_parse_phandle(sw->np, "base-mac-provider", 0); + if (base_mac_np) { + ret = of_get_mac_address(base_mac_np, sw->base_mac); + of_node_put(base_mac_np); + } + } - ret = of_get_mac_address(base_mac_np, sw->base_mac); - if (ret) { + if (!is_valid_ether_addr(sw->base_mac) || ret) { eth_random_addr(sw->base_mac); dev_info(prestera_dev(sw), "using random base mac address\n"); } - of_node_put(base_mac_np); return prestera_hw_switch_mac_set(sw, sw->base_mac); } @@ -573,6 +901,30 @@ static struct prestera_lag *prestera_lag_by_dev(struct prestera_switch *sw, return NULL; } +int prestera_lag_id(struct prestera_switch *sw, + struct net_device *lag_dev, u16 *lag_id) +{ + struct prestera_lag *lag; + int free_id = -1; + int id; + + for (id = 0; id < sw->lag_max; id++) { + lag = prestera_lag_by_id(sw, id); + if (lag->member_count) { + if (lag->dev == lag_dev) { + *lag_id = id; + return 0; + } + } else if (free_id < 0) { + free_id = id; + } + } + if (free_id < 0) + return -ENOSPC; + *lag_id = free_id; + return 0; +} + static struct prestera_lag *prestera_lag_create(struct prestera_switch *sw, struct net_device *lag_dev) { @@ -764,23 +1116,27 @@ static int prestera_netdev_port_event(struct net_device *lower, struct net_device *dev, unsigned long event, void *ptr) { - struct netdev_notifier_changeupper_info *info = ptr; + struct netdev_notifier_info *info = ptr; + struct netdev_notifier_changeupper_info *cu_info; struct prestera_port *port = netdev_priv(dev); struct netlink_ext_ack *extack; struct net_device *upper; - extack = netdev_notifier_info_to_extack(&info->info); - upper = info->upper_dev; + extack = netdev_notifier_info_to_extack(info); + cu_info = container_of(info, + struct netdev_notifier_changeupper_info, + info); switch (event) { case NETDEV_PRECHANGEUPPER: + upper = cu_info->upper_dev; if (!netif_is_bridge_master(upper) && !netif_is_lag_master(upper)) { NL_SET_ERR_MSG_MOD(extack, "Unknown upper device type"); return -EINVAL; } - if (!info->linking) + if (!cu_info->linking) break; if (netdev_has_any_upper_dev(upper)) { @@ -789,7 +1145,7 @@ static int prestera_netdev_port_event(struct net_device *lower, } if (netif_is_lag_master(upper) && - !prestera_lag_master_check(upper, info->upper_info, extack)) + !prestera_lag_master_check(upper, cu_info->upper_info, extack)) return -EOPNOTSUPP; if (netif_is_lag_master(upper) && vlan_uses_dev(dev)) { NL_SET_ERR_MSG_MOD(extack, @@ -805,14 +1161,15 @@ static int prestera_netdev_port_event(struct net_device *lower, break; case NETDEV_CHANGEUPPER: + upper = cu_info->upper_dev; if (netif_is_bridge_master(upper)) { - if (info->linking) + if (cu_info->linking) return prestera_bridge_port_join(upper, port, extack); else prestera_bridge_port_leave(upper, port); } else if (netif_is_lag_master(upper)) { - if (info->linking) + if (cu_info->linking) return prestera_lag_port_add(port, upper); else prestera_lag_port_del(port); @@ -859,6 +1216,150 @@ static int prestera_netdev_event_handler(struct notifier_block *nb, return notifier_from_errno(err); } +struct prestera_mdb_entry * +prestera_mdb_entry_create(struct prestera_switch *sw, + const unsigned char *addr, u16 vid) +{ + struct prestera_flood_domain *flood_domain; + struct prestera_mdb_entry *mdb_entry; + + mdb_entry = kzalloc(sizeof(*mdb_entry), GFP_KERNEL); + if (!mdb_entry) + goto err_mdb_alloc; + + flood_domain = prestera_flood_domain_create(sw); + if (!flood_domain) + goto err_flood_domain_create; + + mdb_entry->sw = sw; + mdb_entry->vid = vid; + mdb_entry->flood_domain = flood_domain; + ether_addr_copy(mdb_entry->addr, addr); + + if (prestera_hw_mdb_create(mdb_entry)) + goto err_mdb_hw_create; + + return mdb_entry; + +err_mdb_hw_create: + prestera_flood_domain_destroy(flood_domain); +err_flood_domain_create: + kfree(mdb_entry); +err_mdb_alloc: + return NULL; +} + +void prestera_mdb_entry_destroy(struct prestera_mdb_entry *mdb_entry) +{ + prestera_hw_mdb_destroy(mdb_entry); + prestera_flood_domain_destroy(mdb_entry->flood_domain); + kfree(mdb_entry); +} + +struct prestera_flood_domain * +prestera_flood_domain_create(struct prestera_switch *sw) +{ + struct prestera_flood_domain *domain; + + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) + return NULL; + + domain->sw = sw; + + if (prestera_hw_flood_domain_create(domain)) { + kfree(domain); + return NULL; + } + + INIT_LIST_HEAD(&domain->flood_domain_port_list); + + return domain; +} + +void prestera_flood_domain_destroy(struct prestera_flood_domain *flood_domain) +{ + WARN_ON(!list_empty(&flood_domain->flood_domain_port_list)); + WARN_ON_ONCE(prestera_hw_flood_domain_destroy(flood_domain)); + kfree(flood_domain); +} + +int +prestera_flood_domain_port_create(struct prestera_flood_domain *flood_domain, + struct net_device *dev, + u16 vid) +{ + struct prestera_flood_domain_port *flood_domain_port; + bool is_first_port_in_list = false; + int err; + + flood_domain_port = kzalloc(sizeof(*flood_domain_port), GFP_KERNEL); + if (!flood_domain_port) { + err = -ENOMEM; + goto err_port_alloc; + } + + flood_domain_port->vid = vid; + + if (list_empty(&flood_domain->flood_domain_port_list)) + is_first_port_in_list = true; + + list_add(&flood_domain_port->flood_domain_port_node, + &flood_domain->flood_domain_port_list); + + flood_domain_port->flood_domain = flood_domain; + flood_domain_port->dev = dev; + + if (!is_first_port_in_list) { + err = prestera_hw_flood_domain_ports_reset(flood_domain); + if (err) + goto err_prestera_mdb_port_create_hw; + } + + err = prestera_hw_flood_domain_ports_set(flood_domain); + if (err) + goto err_prestera_mdb_port_create_hw; + + return 0; + +err_prestera_mdb_port_create_hw: + list_del(&flood_domain_port->flood_domain_port_node); + kfree(flood_domain_port); +err_port_alloc: + return err; +} + +void +prestera_flood_domain_port_destroy(struct prestera_flood_domain_port *port) +{ + struct prestera_flood_domain *flood_domain = port->flood_domain; + + list_del(&port->flood_domain_port_node); + + WARN_ON_ONCE(prestera_hw_flood_domain_ports_reset(flood_domain)); + + if (!list_empty(&flood_domain->flood_domain_port_list)) + WARN_ON_ONCE(prestera_hw_flood_domain_ports_set(flood_domain)); + + kfree(port); +} + +struct prestera_flood_domain_port * +prestera_flood_domain_port_find(struct prestera_flood_domain *flood_domain, + struct net_device *dev, u16 vid) +{ + struct prestera_flood_domain_port *flood_domain_port; + + list_for_each_entry(flood_domain_port, + &flood_domain->flood_domain_port_list, + flood_domain_port_node) + if (flood_domain_port->dev == dev && + vid == flood_domain_port->vid) + return flood_domain_port; + + return NULL; +} + static int prestera_netdev_event_handler_register(struct prestera_switch *sw) { sw->netdev_nb.notifier_call = prestera_netdev_event_handler; @@ -875,6 +1376,8 @@ static int prestera_switch_init(struct prestera_switch *sw) { int err; + sw->np = of_find_compatible_node(NULL, NULL, "marvell,prestera"); + err = prestera_hw_switch_init(sw); if (err) { dev_err(prestera_dev(sw), "Failed to init Switch device\n"); @@ -892,6 +1395,10 @@ static int prestera_switch_init(struct prestera_switch *sw) if (err) return err; + err = prestera_router_init(sw); + if (err) + goto err_router_init; + err = prestera_switchdev_init(sw); if (err) goto err_swdev_register; @@ -904,6 +1411,10 @@ static int prestera_switch_init(struct prestera_switch *sw) if (err) goto err_handlers_register; + err = prestera_counter_init(sw); + if (err) + goto err_counter_init; + err = prestera_acl_init(sw); if (err) goto err_acl_init; @@ -936,12 +1447,16 @@ err_dl_register: err_span_init: prestera_acl_fini(sw); err_acl_init: + prestera_counter_fini(sw); +err_counter_init: prestera_event_handlers_unregister(sw); err_handlers_register: prestera_rxtx_switch_fini(sw); err_rxtx_register: prestera_switchdev_fini(sw); err_swdev_register: + prestera_router_fini(sw); +err_router_init: prestera_netdev_event_handler_unregister(sw); prestera_hw_switch_fini(sw); @@ -956,11 +1471,14 @@ static void prestera_switch_fini(struct prestera_switch *sw) prestera_devlink_traps_unregister(sw); prestera_span_fini(sw); prestera_acl_fini(sw); + prestera_counter_fini(sw); prestera_event_handlers_unregister(sw); prestera_rxtx_switch_fini(sw); prestera_switchdev_fini(sw); + prestera_router_fini(sw); prestera_netdev_event_handler_unregister(sw); prestera_hw_switch_fini(sw); + of_node_put(sw->np); } int prestera_device_register(struct prestera_device *dev) @@ -1000,12 +1518,19 @@ static int __init prestera_module_init(void) if (!prestera_wq) return -ENOMEM; + prestera_owq = alloc_ordered_workqueue("prestera_ordered", 0); + if (!prestera_owq) { + destroy_workqueue(prestera_wq); + return -ENOMEM; + } + return 0; } static void __exit prestera_module_exit(void) { destroy_workqueue(prestera_wq); + destroy_workqueue(prestera_owq); } module_init(prestera_module_init); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.c b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c new file mode 100644 index 000000000000..1da9c1bc1ee9 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2019-2022 Marvell International Ltd. All rights reserved */ + +#include <linux/kernel.h> +#include <linux/list.h> + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_flow.h" +#include "prestera_flower.h" +#include "prestera_matchall.h" +#include "prestera_span.h" + +static int prestera_mall_prio_check(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + u32 flower_prio_min; + u32 flower_prio_max; + int err; + + err = prestera_flower_prio_get(block, f->common.chain_index, + &flower_prio_min, &flower_prio_max); + if (err == -ENOENT) + /* No flower filters installed on this chain. */ + return 0; + + if (err) { + NL_SET_ERR_MSG(f->common.extack, "Failed to get flower priorities"); + return err; + } + + if (f->common.prio <= flower_prio_max && !block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add in front of existing flower rules"); + return -EOPNOTSUPP; + } + if (f->common.prio >= flower_prio_min && block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing flower rules"); + return -EOPNOTSUPP; + } + + return 0; +} + +int prestera_mall_prio_get(struct prestera_flow_block *block, + u32 *prio_min, u32 *prio_max) +{ + if (!block->mall.bound) + return -ENOENT; + + *prio_min = block->mall.prio_min; + *prio_max = block->mall.prio_max; + return 0; +} + +static void prestera_mall_prio_update(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + block->mall.prio_min = min(block->mall.prio_min, f->common.prio); + block->mall.prio_max = max(block->mall.prio_max, f->common.prio); +} + +int prestera_mall_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + struct prestera_flow_block_binding *binding; + __be16 protocol = f->common.protocol; + struct flow_action_entry *act; + struct prestera_port *port; + int err; + + if (!flow_offload_has_one_action(&f->rule->action)) { + NL_SET_ERR_MSG(f->common.extack, + "Only singular actions are supported"); + return -EOPNOTSUPP; + } + + act = &f->rule->action.entries[0]; + + if (!prestera_netdev_check(act->dev)) { + NL_SET_ERR_MSG(f->common.extack, + "Only Marvell Prestera port is supported"); + return -EINVAL; + } + if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) + return -EOPNOTSUPP; + if (act->id != FLOW_ACTION_MIRRED) + return -EOPNOTSUPP; + if (protocol != htons(ETH_P_ALL)) + return -EOPNOTSUPP; + + err = prestera_mall_prio_check(block, f); + if (err) + return err; + + port = netdev_priv(act->dev); + + list_for_each_entry(binding, &block->binding_list, list) { + err = prestera_span_rule_add(binding, port, block->ingress); + if (err == -EEXIST) + return err; + if (err) + goto rollback; + } + + prestera_mall_prio_update(block, f); + + block->mall.bound = true; + return 0; + +rollback: + list_for_each_entry_continue_reverse(binding, + &block->binding_list, list) + prestera_span_rule_del(binding, block->ingress); + return err; +} + +void prestera_mall_destroy(struct prestera_flow_block *block) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + prestera_span_rule_del(binding, block->ingress); + + block->mall.prio_min = UINT_MAX; + block->mall.prio_max = 0; + block->mall.bound = false; +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.h b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h new file mode 100644 index 000000000000..fed08be80257 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2022 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_MATCHALL_H_ +#define _PRESTERA_MATCHALL_H_ + +#include <net/pkt_cls.h> + +struct prestera_flow_block; + +int prestera_mall_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f); +void prestera_mall_destroy(struct prestera_flow_block *block); +int prestera_mall_prio_get(struct prestera_flow_block *block, + u32 *prio_min, u32 *prio_max); + +#endif /* _PRESTERA_MATCHALL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_pci.c b/drivers/net/ethernet/marvell/prestera/prestera_pci.c index f538a749ebd4..59470d99f522 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_pci.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_pci.c @@ -872,6 +872,7 @@ static void prestera_pci_remove(struct pci_dev *pdev) static const struct pci_device_id prestera_pci_devices[] = { { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xC804) }, { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xC80C) }, + { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xCC1E) }, { } }; MODULE_DEVICE_TABLE(pci, prestera_pci_devices); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router.c b/drivers/net/ethernet/marvell/prestera/prestera_router.c new file mode 100644 index 000000000000..4046be0e86ff --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_router.c @@ -0,0 +1,1645 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2019-2021 Marvell International Ltd. All rights reserved */ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/inetdevice.h> +#include <net/inet_dscp.h> +#include <net/switchdev.h> +#include <linux/rhashtable.h> +#include <net/nexthop.h> +#include <net/arp.h> +#include <linux/if_vlan.h> +#include <linux/if_macvlan.h> +#include <net/netevent.h> + +#include "prestera.h" +#include "prestera_router_hw.h" + +#define PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH +#define PRESTERA_NH_PROBE_INTERVAL 5000 /* ms */ + +struct prestera_kern_neigh_cache_key { + struct prestera_ip_addr addr; + struct net_device *dev; +}; + +struct prestera_kern_neigh_cache { + struct prestera_kern_neigh_cache_key key; + struct rhash_head ht_node; + struct list_head kern_fib_cache_list; + /* Hold prepared nh_neigh info if is in_kernel */ + struct prestera_neigh_info nh_neigh_info; + /* Indicate if neighbour is reachable by direct route */ + bool reachable; + /* Lock cache if neigh is present in kernel */ + bool in_kernel; +}; + +struct prestera_kern_fib_cache_key { + struct prestera_ip_addr addr; + u32 prefix_len; + u32 kern_tb_id; /* tb_id from kernel (not fixed) */ +}; + +/* Subscribing on neighbours in kernel */ +struct prestera_kern_fib_cache { + struct prestera_kern_fib_cache_key key; + struct { + struct prestera_fib_key fib_key; + enum prestera_fib_type fib_type; + struct prestera_nexthop_group_key nh_grp_key; + } lpm_info; /* hold prepared lpm info */ + /* Indicate if route is not overlapped by another table */ + struct rhash_head ht_node; /* node of prestera_router */ + struct prestera_kern_neigh_cache_head { + struct prestera_kern_fib_cache *this; + struct list_head head; + struct prestera_kern_neigh_cache *n_cache; + } kern_neigh_cache_head[PRESTERA_NHGR_SIZE_MAX]; + union { + struct fib_notifier_info info; /* point to any of 4/6 */ + struct fib_entry_notifier_info fen4_info; + }; + bool reachable; +}; + +static const struct rhashtable_params __prestera_kern_neigh_cache_ht_params = { + .key_offset = offsetof(struct prestera_kern_neigh_cache, key), + .head_offset = offsetof(struct prestera_kern_neigh_cache, ht_node), + .key_len = sizeof(struct prestera_kern_neigh_cache_key), + .automatic_shrinking = true, +}; + +static const struct rhashtable_params __prestera_kern_fib_cache_ht_params = { + .key_offset = offsetof(struct prestera_kern_fib_cache, key), + .head_offset = offsetof(struct prestera_kern_fib_cache, ht_node), + .key_len = sizeof(struct prestera_kern_fib_cache_key), + .automatic_shrinking = true, +}; + +/* This util to be used, to convert kernel rules for default vr in hw_vr */ +static u32 prestera_fix_tb_id(u32 tb_id) +{ + if (tb_id == RT_TABLE_UNSPEC || + tb_id == RT_TABLE_LOCAL || + tb_id == RT_TABLE_DEFAULT) + tb_id = RT_TABLE_MAIN; + + return tb_id; +} + +static void +prestera_util_fen_info2fib_cache_key(struct fib_notifier_info *info, + struct prestera_kern_fib_cache_key *key) +{ + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + memset(key, 0, sizeof(*key)); + key->addr.v = PRESTERA_IPV4; + key->addr.u.ipv4 = cpu_to_be32(fen_info->dst); + key->prefix_len = fen_info->dst_len; + key->kern_tb_id = fen_info->tb_id; +} + +static int prestera_util_nhc2nc_key(struct prestera_switch *sw, + struct fib_nh_common *nhc, + struct prestera_kern_neigh_cache_key *nk) +{ + memset(nk, 0, sizeof(*nk)); + if (nhc->nhc_gw_family == AF_INET) { + nk->addr.v = PRESTERA_IPV4; + nk->addr.u.ipv4 = nhc->nhc_gw.ipv4; + } else { + nk->addr.v = PRESTERA_IPV6; + nk->addr.u.ipv6 = nhc->nhc_gw.ipv6; + } + + nk->dev = nhc->nhc_dev; + return 0; +} + +static void +prestera_util_nc_key2nh_key(struct prestera_kern_neigh_cache_key *ck, + struct prestera_nh_neigh_key *nk) +{ + memset(nk, 0, sizeof(*nk)); + nk->addr = ck->addr; + nk->rif = (void *)ck->dev; +} + +static bool +prestera_util_nhc_eq_n_cache_key(struct prestera_switch *sw, + struct fib_nh_common *nhc, + struct prestera_kern_neigh_cache_key *nk) +{ + struct prestera_kern_neigh_cache_key tk; + int err; + + err = prestera_util_nhc2nc_key(sw, nhc, &tk); + if (err) + return false; + + if (memcmp(&tk, nk, sizeof(tk))) + return false; + + return true; +} + +static int +prestera_util_neigh2nc_key(struct prestera_switch *sw, struct neighbour *n, + struct prestera_kern_neigh_cache_key *key) +{ + memset(key, 0, sizeof(*key)); + if (n->tbl->family == AF_INET) { + key->addr.v = PRESTERA_IPV4; + key->addr.u.ipv4 = *(__be32 *)n->primary_key; + } else { + return -ENOENT; + } + + key->dev = n->dev; + + return 0; +} + +static bool __prestera_fi_is_direct(struct fib_info *fi) +{ + struct fib_nh *fib_nh; + + if (fib_info_num_path(fi) == 1) { + fib_nh = fib_info_nh(fi, 0); + if (fib_nh->fib_nh_gw_family == AF_UNSPEC) + return true; + } + + return false; +} + +static bool prestera_fi_is_direct(struct fib_info *fi) +{ + if (fi->fib_type != RTN_UNICAST) + return false; + + return __prestera_fi_is_direct(fi); +} + +static bool prestera_fi_is_nh(struct fib_info *fi) +{ + if (fi->fib_type != RTN_UNICAST) + return false; + + return !__prestera_fi_is_direct(fi); +} + +static bool __prestera_fi6_is_direct(struct fib6_info *fi) +{ + if (!fi->fib6_nh->nh_common.nhc_gw_family) + return true; + + return false; +} + +static bool prestera_fi6_is_direct(struct fib6_info *fi) +{ + if (fi->fib6_type != RTN_UNICAST) + return false; + + return __prestera_fi6_is_direct(fi); +} + +static bool prestera_fi6_is_nh(struct fib6_info *fi) +{ + if (fi->fib6_type != RTN_UNICAST) + return false; + + return !__prestera_fi6_is_direct(fi); +} + +static bool prestera_fib_info_is_direct(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info = + container_of(info, struct fib6_entry_notifier_info, info); + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + if (info->family == AF_INET) + return prestera_fi_is_direct(fen_info->fi); + else + return prestera_fi6_is_direct(fen6_info->rt); +} + +static bool prestera_fib_info_is_nh(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info = + container_of(info, struct fib6_entry_notifier_info, info); + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + if (info->family == AF_INET) + return prestera_fi_is_nh(fen_info->fi); + else + return prestera_fi6_is_nh(fen6_info->rt); +} + +/* must be called with rcu_read_lock() */ +static int prestera_util_kern_get_route(struct fib_result *res, u32 tb_id, + __be32 *addr) +{ + struct flowi4 fl4; + + /* TODO: walkthrough appropriate tables in kernel + * to know if the same prefix exists in several tables + */ + memset(&fl4, 0, sizeof(fl4)); + fl4.daddr = *addr; + return fib_lookup(&init_net, &fl4, res, 0 /* FIB_LOOKUP_NOREF */); +} + +static bool +__prestera_util_kern_n_is_reachable_v4(u32 tb_id, __be32 *addr, + struct net_device *dev) +{ + struct fib_nh *fib_nh; + struct fib_result res; + bool reachable; + + reachable = false; + + if (!prestera_util_kern_get_route(&res, tb_id, addr)) + if (prestera_fi_is_direct(res.fi)) { + fib_nh = fib_info_nh(res.fi, 0); + if (dev == fib_nh->fib_nh_dev) + reachable = true; + } + + return reachable; +} + +/* Check if neigh route is reachable */ +static bool +prestera_util_kern_n_is_reachable(u32 tb_id, + struct prestera_ip_addr *addr, + struct net_device *dev) +{ + if (addr->v == PRESTERA_IPV4) + return __prestera_util_kern_n_is_reachable_v4(tb_id, + &addr->u.ipv4, + dev); + else + return false; +} + +static void prestera_util_kern_set_neigh_offload(struct neighbour *n, + bool offloaded) +{ + if (offloaded) + n->flags |= NTF_OFFLOADED; + else + n->flags &= ~NTF_OFFLOADED; +} + +static void +prestera_util_kern_set_nh_offload(struct fib_nh_common *nhc, bool offloaded, bool trap) +{ + if (offloaded) + nhc->nhc_flags |= RTNH_F_OFFLOAD; + else + nhc->nhc_flags &= ~RTNH_F_OFFLOAD; + + if (trap) + nhc->nhc_flags |= RTNH_F_TRAP; + else + nhc->nhc_flags &= ~RTNH_F_TRAP; +} + +static struct fib_nh_common * +prestera_kern_fib_info_nhc(struct fib_notifier_info *info, int n) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + struct fib6_info *iter; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return &fib_info_nh(fen4_info->fi, n)->nh_common; + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + if (!n) + return &fen6_info->rt->fib6_nh->nh_common; + + list_for_each_entry(iter, &fen6_info->rt->fib6_siblings, + fib6_siblings) { + if (!--n) + return &iter->fib6_nh->nh_common; + } + } + + /* if family is incorrect - than upper functions has BUG */ + /* if doesn't find requested index - there is alsi bug, because + * valid index must be produced by nhs, which checks list length + */ + WARN(1, "Invalid parameters passed to %s n=%d i=%p", + __func__, n, info); + return NULL; +} + +static int prestera_kern_fib_info_nhs(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return fib_info_num_path(fen4_info->fi); + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + return fen6_info->rt->fib6_nsiblings + 1; + } + + return 0; +} + +static unsigned char +prestera_kern_fib_info_type(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return fen4_info->fi->fib_type; + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + /* TODO: ECMP in ipv6 is several routes. + * Every route has single nh. + */ + return fen6_info->rt->fib6_type; + } + + return RTN_UNSPEC; +} + +/* Decided, that uc_nh route with key==nh is obviously neighbour route */ +static bool +prestera_fib_node_util_is_neighbour(struct prestera_fib_node *fib_node) +{ + if (fib_node->info.type != PRESTERA_FIB_TYPE_UC_NH) + return false; + + if (fib_node->info.nh_grp->nh_neigh_head[1].neigh) + return false; + + if (!fib_node->info.nh_grp->nh_neigh_head[0].neigh) + return false; + + if (memcmp(&fib_node->info.nh_grp->nh_neigh_head[0].neigh->key.addr, + &fib_node->key.addr, sizeof(struct prestera_ip_addr))) + return false; + + return true; +} + +static int prestera_dev_if_type(const struct net_device *dev) +{ + struct macvlan_dev *vlan; + + if (is_vlan_dev(dev) && + netif_is_bridge_master(vlan_dev_real_dev(dev))) { + return PRESTERA_IF_VID_E; + } else if (netif_is_bridge_master(dev)) { + return PRESTERA_IF_VID_E; + } else if (netif_is_lag_master(dev)) { + return PRESTERA_IF_LAG_E; + } else if (netif_is_macvlan(dev)) { + vlan = netdev_priv(dev); + return prestera_dev_if_type(vlan->lowerdev); + } else { + return PRESTERA_IF_PORT_E; + } +} + +static int +prestera_neigh_iface_init(struct prestera_switch *sw, + struct prestera_iface *iface, + struct neighbour *n) +{ + struct prestera_port *port; + + iface->vlan_id = 0; /* TODO: vlan egress */ + iface->type = prestera_dev_if_type(n->dev); + if (iface->type != PRESTERA_IF_PORT_E) + return -EINVAL; + + if (!prestera_netdev_check(n->dev)) + return -EINVAL; + + port = netdev_priv(n->dev); + iface->dev_port.hw_dev_num = port->dev_id; + iface->dev_port.port_num = port->hw_id; + + return 0; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_find(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + + n_cache = + rhashtable_lookup_fast(&sw->router->kern_neigh_cache_ht, key, + __prestera_kern_neigh_cache_ht_params); + return IS_ERR(n_cache) ? NULL : n_cache; +} + +static void +__prestera_kern_neigh_cache_destruct(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + dev_put(n_cache->key.dev); +} + +static void +__prestera_kern_neigh_cache_destroy(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + rhashtable_remove_fast(&sw->router->kern_neigh_cache_ht, + &n_cache->ht_node, + __prestera_kern_neigh_cache_ht_params); + __prestera_kern_neigh_cache_destruct(sw, n_cache); + kfree(n_cache); +} + +static struct prestera_kern_neigh_cache * +__prestera_kern_neigh_cache_create(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + int err; + + n_cache = kzalloc(sizeof(*n_cache), GFP_KERNEL); + if (!n_cache) + goto err_kzalloc; + + memcpy(&n_cache->key, key, sizeof(*key)); + dev_hold(n_cache->key.dev); + + INIT_LIST_HEAD(&n_cache->kern_fib_cache_list); + err = rhashtable_insert_fast(&sw->router->kern_neigh_cache_ht, + &n_cache->ht_node, + __prestera_kern_neigh_cache_ht_params); + if (err) + goto err_ht_insert; + + return n_cache; + +err_ht_insert: + dev_put(n_cache->key.dev); + kfree(n_cache); +err_kzalloc: + return NULL; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_get(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + + n_cache = prestera_kern_neigh_cache_find(sw, key); + if (!n_cache) + n_cache = __prestera_kern_neigh_cache_create(sw, key); + + return n_cache; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_put(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + if (!n_cache->in_kernel && + list_empty(&n_cache->kern_fib_cache_list)) { + __prestera_kern_neigh_cache_destroy(sw, n_cache); + return NULL; + } + + return n_cache; +} + +static struct prestera_kern_fib_cache * +prestera_kern_fib_cache_find(struct prestera_switch *sw, + struct prestera_kern_fib_cache_key *key) +{ + struct prestera_kern_fib_cache *fib_cache; + + fib_cache = + rhashtable_lookup_fast(&sw->router->kern_fib_cache_ht, key, + __prestera_kern_fib_cache_ht_params); + return fib_cache; +} + +static void +__prestera_kern_fib_cache_destruct(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fib_cache) +{ + struct prestera_kern_neigh_cache *n_cache; + int i; + + for (i = 0; i < PRESTERA_NHGR_SIZE_MAX; i++) { + n_cache = fib_cache->kern_neigh_cache_head[i].n_cache; + if (n_cache) { + list_del(&fib_cache->kern_neigh_cache_head[i].head); + prestera_kern_neigh_cache_put(sw, n_cache); + } + } + + fib_info_put(fib_cache->fen4_info.fi); +} + +static void +prestera_kern_fib_cache_destroy(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fib_cache) +{ + rhashtable_remove_fast(&sw->router->kern_fib_cache_ht, + &fib_cache->ht_node, + __prestera_kern_fib_cache_ht_params); + __prestera_kern_fib_cache_destruct(sw, fib_cache); + kfree(fib_cache); +} + +static int +__prestera_kern_fib_cache_create_nhs(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + struct prestera_kern_neigh_cache_key nc_key; + struct prestera_kern_neigh_cache *n_cache; + struct fib_nh_common *nhc; + int i, nhs, err; + + if (!prestera_fib_info_is_nh(&fc->info)) + return 0; + + nhs = prestera_kern_fib_info_nhs(&fc->info); + if (nhs > PRESTERA_NHGR_SIZE_MAX) + return 0; + + for (i = 0; i < nhs; i++) { + nhc = prestera_kern_fib_info_nhc(&fc->fen4_info.info, i); + err = prestera_util_nhc2nc_key(sw, nhc, &nc_key); + if (err) + return 0; + + n_cache = prestera_kern_neigh_cache_get(sw, &nc_key); + if (!n_cache) + return 0; + + fc->kern_neigh_cache_head[i].this = fc; + fc->kern_neigh_cache_head[i].n_cache = n_cache; + list_add(&fc->kern_neigh_cache_head[i].head, + &n_cache->kern_fib_cache_list); + } + + return 0; +} + +/* Operations on fi (offload, etc) must be wrapped in utils. + * This function just create storage. + */ +static struct prestera_kern_fib_cache * +prestera_kern_fib_cache_create(struct prestera_switch *sw, + struct prestera_kern_fib_cache_key *key, + struct fib_notifier_info *info) +{ + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + struct prestera_kern_fib_cache *fib_cache; + int err; + + fib_cache = kzalloc(sizeof(*fib_cache), GFP_KERNEL); + if (!fib_cache) + goto err_kzalloc; + + memcpy(&fib_cache->key, key, sizeof(*key)); + fib_info_hold(fen_info->fi); + memcpy(&fib_cache->fen4_info, fen_info, sizeof(*fen_info)); + + err = rhashtable_insert_fast(&sw->router->kern_fib_cache_ht, + &fib_cache->ht_node, + __prestera_kern_fib_cache_ht_params); + if (err) + goto err_ht_insert; + + /* Handle nexthops */ + err = __prestera_kern_fib_cache_create_nhs(sw, fib_cache); + if (err) + goto out; /* Not critical */ + +out: + return fib_cache; + +err_ht_insert: + fib_info_put(fen_info->fi); + kfree(fib_cache); +err_kzalloc: + return NULL; +} + +static void +__prestera_k_arb_fib_nh_offload_set(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fibc, + struct prestera_kern_neigh_cache *nc, + bool offloaded, bool trap) +{ + struct fib_nh_common *nhc; + int i, nhs; + + nhs = prestera_kern_fib_info_nhs(&fibc->info); + for (i = 0; i < nhs; i++) { + nhc = prestera_kern_fib_info_nhc(&fibc->info, i); + if (!nc) { + prestera_util_kern_set_nh_offload(nhc, offloaded, trap); + continue; + } + + if (prestera_util_nhc_eq_n_cache_key(sw, nhc, &nc->key)) { + prestera_util_kern_set_nh_offload(nhc, offloaded, trap); + break; + } + } +} + +static void +__prestera_k_arb_n_offload_set(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc, + bool offloaded) +{ + struct neighbour *n; + + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + if (!n) + return; + + prestera_util_kern_set_neigh_offload(n, offloaded); + neigh_release(n); +} + +static void +__prestera_k_arb_fib_lpm_offload_set(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc, + bool fail, bool offload, bool trap) +{ + struct fib_rt_info fri; + + switch (fc->key.addr.v) { + case PRESTERA_IPV4: + fri.fi = fc->fen4_info.fi; + fri.tb_id = fc->key.kern_tb_id; + fri.dst = fc->key.addr.u.ipv4; + fri.dst_len = fc->key.prefix_len; + fri.dscp = fc->fen4_info.dscp; + fri.type = fc->fen4_info.type; + /* flags begin */ + fri.offload = offload; + fri.trap = trap; + fri.offload_failed = fail; + /* flags end */ + fib_alias_hw_flags_set(&init_net, &fri); + return; + case PRESTERA_IPV6: + /* TODO */ + return; + } +} + +static void +__prestera_k_arb_n_lpm_set(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache, + bool enabled) +{ + struct prestera_nexthop_group_key nh_grp_key; + struct prestera_kern_fib_cache_key fc_key; + struct prestera_kern_fib_cache *fib_cache; + struct prestera_fib_node *fib_node; + struct prestera_fib_key fib_key; + + /* Exception for fc with prefix 32: LPM entry is already used by fib */ + memset(&fc_key, 0, sizeof(fc_key)); + fc_key.addr = n_cache->key.addr; + fc_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v); + /* But better to use tb_id of route, which pointed to this neighbour. */ + /* We take it from rif, because rif inconsistent. + * Must be separated in_rif and out_rif. + * Also note: for each fib pointed to this neigh should be separated + * neigh lpm entry (for each ingress vr) + */ + fc_key.kern_tb_id = l3mdev_fib_table(n_cache->key.dev); + fib_cache = prestera_kern_fib_cache_find(sw, &fc_key); + memset(&fib_key, 0, sizeof(fib_key)); + fib_key.addr = n_cache->key.addr; + fib_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v); + fib_key.tb_id = prestera_fix_tb_id(fc_key.kern_tb_id); + fib_node = prestera_fib_node_find(sw, &fib_key); + if (!fib_cache || !fib_cache->reachable) { + if (!enabled && fib_node) { + if (prestera_fib_node_util_is_neighbour(fib_node)) + prestera_fib_node_destroy(sw, fib_node); + return; + } + } + + if (enabled && !fib_node) { + memset(&nh_grp_key, 0, sizeof(nh_grp_key)); + prestera_util_nc_key2nh_key(&n_cache->key, + &nh_grp_key.neigh[0]); + fib_node = prestera_fib_node_create(sw, &fib_key, + PRESTERA_FIB_TYPE_UC_NH, + &nh_grp_key); + if (!fib_node) + pr_err("%s failed ip=%pI4n", "prestera_fib_node_create", + &fib_key.addr.u.ipv4); + return; + } +} + +static void +__prestera_k_arb_nc_kern_fib_fetch(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + if (prestera_util_kern_n_is_reachable(l3mdev_fib_table(nc->key.dev), + &nc->key.addr, nc->key.dev)) + nc->reachable = true; + else + nc->reachable = false; +} + +/* Kernel neighbour -> neigh_cache info */ +static void +__prestera_k_arb_nc_kern_n_fetch(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct neighbour *n; + int err; + + memset(&nc->nh_neigh_info, 0, sizeof(nc->nh_neigh_info)); + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, nc->key.dev); + if (!n) + goto out; + + read_lock_bh(&n->lock); + if (n->nud_state & NUD_VALID && !n->dead) { + err = prestera_neigh_iface_init(sw, &nc->nh_neigh_info.iface, + n); + if (err) + goto n_read_out; + + memcpy(&nc->nh_neigh_info.ha[0], &n->ha[0], ETH_ALEN); + nc->nh_neigh_info.connected = true; + } +n_read_out: + read_unlock_bh(&n->lock); +out: + nc->in_kernel = nc->nh_neigh_info.connected; + if (n) + neigh_release(n); +} + +/* neigh_cache info -> lpm update */ +static void +__prestera_k_arb_nc_apply(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct prestera_kern_neigh_cache_head *nhead; + struct prestera_nh_neigh_key nh_key; + struct prestera_nh_neigh *nh_neigh; + int err; + + __prestera_k_arb_n_lpm_set(sw, nc, nc->reachable && nc->in_kernel); + __prestera_k_arb_n_offload_set(sw, nc, nc->reachable && nc->in_kernel); + + prestera_util_nc_key2nh_key(&nc->key, &nh_key); + nh_neigh = prestera_nh_neigh_find(sw, &nh_key); + if (!nh_neigh) + goto out; + + /* Do hw update only if something changed to prevent nh flap */ + if (memcmp(&nc->nh_neigh_info, &nh_neigh->info, + sizeof(nh_neigh->info))) { + memcpy(&nh_neigh->info, &nc->nh_neigh_info, + sizeof(nh_neigh->info)); + err = prestera_nh_neigh_set(sw, nh_neigh); + if (err) { + pr_err("%s failed with err=%d ip=%pI4n mac=%pM", + "prestera_nh_neigh_set", err, + &nh_neigh->key.addr.u.ipv4, + &nh_neigh->info.ha[0]); + goto out; + } + } + +out: + list_for_each_entry(nhead, &nc->kern_fib_cache_list, head) { + __prestera_k_arb_fib_nh_offload_set(sw, nhead->this, nc, + nc->in_kernel, + !nc->in_kernel); + } +} + +static int +__prestera_pr_k_arb_fc_lpm_info_calc(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + struct fib_nh_common *nhc; + int nh_cnt; + + memset(&fc->lpm_info, 0, sizeof(fc->lpm_info)); + + switch (prestera_kern_fib_info_type(&fc->info)) { + case RTN_UNICAST: + if (prestera_fib_info_is_direct(&fc->info) && + fc->key.prefix_len == + PRESTERA_IP_ADDR_PLEN(fc->key.addr.v)) { + /* This is special case. + * When prefix is 32. Than we will have conflict in lpm + * for direct route - once TRAP added, there is no + * place for neighbour entry. So represent direct route + * with prefix 32, as NH. So neighbour will be resolved + * as nexthop of this route. + */ + nhc = prestera_kern_fib_info_nhc(&fc->info, 0); + fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_UC_NH; + fc->lpm_info.nh_grp_key.neigh[0].addr = + fc->key.addr; + fc->lpm_info.nh_grp_key.neigh[0].rif = + nhc->nhc_dev; + + break; + } + + /* We can also get nh_grp_key from fi. This will be correct to + * because cache not always represent, what actually written to + * lpm. But we use nh cache, as well for now (for this case). + */ + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + if (!fc->kern_neigh_cache_head[nh_cnt].n_cache) + break; + + fc->lpm_info.nh_grp_key.neigh[nh_cnt].addr = + fc->kern_neigh_cache_head[nh_cnt].n_cache->key.addr; + fc->lpm_info.nh_grp_key.neigh[nh_cnt].rif = + fc->kern_neigh_cache_head[nh_cnt].n_cache->key.dev; + } + + fc->lpm_info.fib_type = nh_cnt ? + PRESTERA_FIB_TYPE_UC_NH : + PRESTERA_FIB_TYPE_TRAP; + break; + /* Unsupported. Leave it for kernel: */ + case RTN_BROADCAST: + case RTN_MULTICAST: + /* Routes we must trap by design: */ + case RTN_LOCAL: + case RTN_UNREACHABLE: + case RTN_PROHIBIT: + fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_TRAP; + break; + case RTN_BLACKHOLE: + fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_DROP; + break; + default: + dev_err(sw->dev->dev, "Unsupported fib_type"); + return -EOPNOTSUPP; + } + + fc->lpm_info.fib_key.addr = fc->key.addr; + fc->lpm_info.fib_key.prefix_len = fc->key.prefix_len; + fc->lpm_info.fib_key.tb_id = prestera_fix_tb_id(fc->key.kern_tb_id); + + return 0; +} + +static int __prestera_k_arb_f_lpm_set(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc, + bool enabled) +{ + struct prestera_fib_node *fib_node; + + fib_node = prestera_fib_node_find(sw, &fc->lpm_info.fib_key); + if (fib_node) + prestera_fib_node_destroy(sw, fib_node); + + if (!enabled) + return 0; + + fib_node = prestera_fib_node_create(sw, &fc->lpm_info.fib_key, + fc->lpm_info.fib_type, + &fc->lpm_info.nh_grp_key); + + if (!fib_node) { + dev_err(sw->dev->dev, "fib_node=NULL %pI4n/%d kern_tb_id = %d", + &fc->key.addr.u.ipv4, fc->key.prefix_len, + fc->key.kern_tb_id); + return -ENOENT; + } + + return 0; +} + +static int __prestera_k_arb_fc_apply(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + int err; + + err = __prestera_pr_k_arb_fc_lpm_info_calc(sw, fc); + if (err) + return err; + + err = __prestera_k_arb_f_lpm_set(sw, fc, fc->reachable); + if (err) { + __prestera_k_arb_fib_lpm_offload_set(sw, fc, + true, false, false); + return err; + } + + switch (fc->lpm_info.fib_type) { + case PRESTERA_FIB_TYPE_UC_NH: + __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, + fc->reachable, false); + break; + case PRESTERA_FIB_TYPE_TRAP: + __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, + false, fc->reachable); + break; + case PRESTERA_FIB_TYPE_DROP: + __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, true, + fc->reachable); + break; + case PRESTERA_FIB_TYPE_INVALID: + break; + } + + return 0; +} + +static struct prestera_kern_fib_cache * +__prestera_k_arb_util_fib_overlaps(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + struct prestera_kern_fib_cache_key fc_key; + struct prestera_kern_fib_cache *rfc; + + /* TODO: parse kernel rules */ + rfc = NULL; + if (fc->key.kern_tb_id == RT_TABLE_LOCAL) { + memcpy(&fc_key, &fc->key, sizeof(fc_key)); + fc_key.kern_tb_id = RT_TABLE_MAIN; + rfc = prestera_kern_fib_cache_find(sw, &fc_key); + } + + return rfc; +} + +static struct prestera_kern_fib_cache * +__prestera_k_arb_util_fib_overlapped(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + struct prestera_kern_fib_cache_key fc_key; + struct prestera_kern_fib_cache *rfc; + + /* TODO: parse kernel rules */ + rfc = NULL; + if (fc->key.kern_tb_id == RT_TABLE_MAIN) { + memcpy(&fc_key, &fc->key, sizeof(fc_key)); + fc_key.kern_tb_id = RT_TABLE_LOCAL; + rfc = prestera_kern_fib_cache_find(sw, &fc_key); + } + + return rfc; +} + +static void __prestera_k_arb_hw_state_upd(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct prestera_nh_neigh_key nh_key; + struct prestera_nh_neigh *nh_neigh; + struct neighbour *n; + bool hw_active; + + prestera_util_nc_key2nh_key(&nc->key, &nh_key); + nh_neigh = prestera_nh_neigh_find(sw, &nh_key); + if (!nh_neigh) { + pr_err("Cannot find nh_neigh for cached %pI4n", + &nc->key.addr.u.ipv4); + return; + } + + hw_active = prestera_nh_neigh_util_hw_state(sw, nh_neigh); + +#ifdef PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH + if (!hw_active && nc->in_kernel) + goto out; +#else /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */ + if (!hw_active) + goto out; +#endif /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */ + + if (nc->key.addr.v == PRESTERA_IPV4) { + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + if (!n) + n = neigh_create(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + } else { + n = NULL; + } + + if (!IS_ERR(n) && n) { + neigh_event_send(n, NULL); + neigh_release(n); + } else { + pr_err("Cannot create neighbour %pI4n", &nc->key.addr.u.ipv4); + } + +out: + return; +} + +/* Propagate hw state to kernel */ +static void prestera_k_arb_hw_evt(struct prestera_switch *sw) +{ + struct prestera_kern_neigh_cache *n_cache; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter); + rhashtable_walk_start(&iter); + while (1) { + n_cache = rhashtable_walk_next(&iter); + + if (!n_cache) + break; + + if (IS_ERR(n_cache)) + continue; + + rhashtable_walk_stop(&iter); + __prestera_k_arb_hw_state_upd(sw, n_cache); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + +/* Propagate kernel event to hw */ +static void prestera_k_arb_n_evt(struct prestera_switch *sw, + struct neighbour *n) +{ + struct prestera_kern_neigh_cache_key n_key; + struct prestera_kern_neigh_cache *n_cache; + int err; + + err = prestera_util_neigh2nc_key(sw, n, &n_key); + if (err) + return; + + n_cache = prestera_kern_neigh_cache_find(sw, &n_key); + if (!n_cache) { + n_cache = prestera_kern_neigh_cache_get(sw, &n_key); + if (!n_cache) + return; + __prestera_k_arb_nc_kern_fib_fetch(sw, n_cache); + } + + __prestera_k_arb_nc_kern_n_fetch(sw, n_cache); + __prestera_k_arb_nc_apply(sw, n_cache); + + prestera_kern_neigh_cache_put(sw, n_cache); +} + +static void __prestera_k_arb_fib_evt2nc(struct prestera_switch *sw) +{ + struct prestera_kern_neigh_cache *n_cache; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter); + rhashtable_walk_start(&iter); + while (1) { + n_cache = rhashtable_walk_next(&iter); + + if (!n_cache) + break; + + if (IS_ERR(n_cache)) + continue; + + rhashtable_walk_stop(&iter); + __prestera_k_arb_nc_kern_fib_fetch(sw, n_cache); + __prestera_k_arb_nc_apply(sw, n_cache); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + +static int +prestera_k_arb_fib_evt(struct prestera_switch *sw, + bool replace, /* replace or del */ + struct fib_notifier_info *info) +{ + struct prestera_kern_fib_cache *tfib_cache, *bfib_cache; /* top/btm */ + struct prestera_kern_fib_cache_key fc_key; + struct prestera_kern_fib_cache *fib_cache; + int err; + + prestera_util_fen_info2fib_cache_key(info, &fc_key); + fib_cache = prestera_kern_fib_cache_find(sw, &fc_key); + if (fib_cache) { + fib_cache->reachable = false; + err = __prestera_k_arb_fc_apply(sw, fib_cache); + if (err) + dev_err(sw->dev->dev, + "Applying destroyed fib_cache failed"); + + bfib_cache = __prestera_k_arb_util_fib_overlaps(sw, fib_cache); + tfib_cache = __prestera_k_arb_util_fib_overlapped(sw, fib_cache); + if (!tfib_cache && bfib_cache) { + bfib_cache->reachable = true; + err = __prestera_k_arb_fc_apply(sw, bfib_cache); + if (err) + dev_err(sw->dev->dev, + "Applying fib_cache btm failed"); + } + + prestera_kern_fib_cache_destroy(sw, fib_cache); + } + + if (replace) { + fib_cache = prestera_kern_fib_cache_create(sw, &fc_key, info); + if (!fib_cache) { + dev_err(sw->dev->dev, "fib_cache == NULL"); + return -ENOENT; + } + + bfib_cache = __prestera_k_arb_util_fib_overlaps(sw, fib_cache); + tfib_cache = __prestera_k_arb_util_fib_overlapped(sw, fib_cache); + if (!tfib_cache) + fib_cache->reachable = true; + + if (bfib_cache) { + bfib_cache->reachable = false; + err = __prestera_k_arb_fc_apply(sw, bfib_cache); + if (err) + dev_err(sw->dev->dev, + "Applying fib_cache btm failed"); + } + + err = __prestera_k_arb_fc_apply(sw, fib_cache); + if (err) + dev_err(sw->dev->dev, "Applying fib_cache failed"); + } + + /* Update all neighs to resolve overlapped and apply related */ + __prestera_k_arb_fib_evt2nc(sw); + + return 0; +} + +static void __prestera_k_arb_abort_neigh_ht_cb(void *ptr, void *arg) +{ + struct prestera_kern_neigh_cache *n_cache = ptr; + struct prestera_switch *sw = arg; + + if (!list_empty(&n_cache->kern_fib_cache_list)) { + WARN_ON(1); /* BUG */ + return; + } + __prestera_k_arb_n_offload_set(sw, n_cache, false); + n_cache->in_kernel = false; + /* No need to destroy lpm. + * It will be aborted by destroy_ht + */ + __prestera_kern_neigh_cache_destruct(sw, n_cache); + kfree(n_cache); +} + +static void __prestera_k_arb_abort_fib_ht_cb(void *ptr, void *arg) +{ + struct prestera_kern_fib_cache *fib_cache = ptr; + struct prestera_switch *sw = arg; + + __prestera_k_arb_fib_lpm_offload_set(sw, fib_cache, + false, false, + false); + __prestera_k_arb_fib_nh_offload_set(sw, fib_cache, NULL, + false, false); + /* No need to destroy lpm. + * It will be aborted by destroy_ht + */ + __prestera_kern_fib_cache_destruct(sw, fib_cache); + kfree(fib_cache); +} + +static void prestera_k_arb_abort(struct prestera_switch *sw) +{ + /* Function to remove all arbiter entries and related hw objects. */ + /* Sequence: + * 1) Clear arbiter tables, but don't touch hw + * 2) Clear hw + * We use such approach, because arbiter object is not directly mapped + * to hw. So deletion of one arbiter object may even lead to creation of + * hw object (e.g. in case of overlapped routes). + */ + rhashtable_free_and_destroy(&sw->router->kern_fib_cache_ht, + __prestera_k_arb_abort_fib_ht_cb, + sw); + rhashtable_free_and_destroy(&sw->router->kern_neigh_cache_ht, + __prestera_k_arb_abort_neigh_ht_cb, + sw); +} + +static int __prestera_inetaddr_port_event(struct net_device *port_dev, + unsigned long event, + struct netlink_ext_ack *extack) +{ + struct prestera_port *port = netdev_priv(port_dev); + struct prestera_rif_entry_key re_key = {}; + struct prestera_rif_entry *re; + u32 kern_tb_id; + int err; + + err = prestera_is_valid_mac_addr(port, port_dev->dev_addr); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "RIF MAC must have the same prefix"); + return err; + } + + kern_tb_id = l3mdev_fib_table(port_dev); + re_key.iface.type = PRESTERA_IF_PORT_E; + re_key.iface.dev_port.hw_dev_num = port->dev_id; + re_key.iface.dev_port.port_num = port->hw_id; + re = prestera_rif_entry_find(port->sw, &re_key); + + switch (event) { + case NETDEV_UP: + if (re) { + NL_SET_ERR_MSG_MOD(extack, "RIF already exist"); + return -EEXIST; + } + re = prestera_rif_entry_create(port->sw, &re_key, + prestera_fix_tb_id(kern_tb_id), + port_dev->dev_addr); + if (!re) { + NL_SET_ERR_MSG_MOD(extack, "Can't create RIF"); + return -EINVAL; + } + dev_hold(port_dev); + break; + case NETDEV_DOWN: + if (!re) { + NL_SET_ERR_MSG_MOD(extack, "Can't find RIF"); + return -EEXIST; + } + prestera_rif_entry_destroy(port->sw, re); + dev_put(port_dev); + break; + } + + return 0; +} + +static int __prestera_inetaddr_event(struct prestera_switch *sw, + struct net_device *dev, + unsigned long event, + struct netlink_ext_ack *extack) +{ + if (!prestera_netdev_check(dev) || netif_is_any_bridge_port(dev) || + netif_is_lag_port(dev)) + return 0; + + return __prestera_inetaddr_port_event(dev, event, extack); +} + +static int __prestera_inetaddr_cb(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct in_ifaddr *ifa = (struct in_ifaddr *)ptr; + struct net_device *dev = ifa->ifa_dev->dev; + struct prestera_router *router = container_of(nb, + struct prestera_router, + inetaddr_nb); + struct in_device *idev; + int err = 0; + + if (event != NETDEV_DOWN) + goto out; + + /* Ignore if this is not latest address */ + idev = __in_dev_get_rtnl(dev); + if (idev && idev->ifa_list) + goto out; + + err = __prestera_inetaddr_event(router->sw, dev, event, NULL); +out: + return notifier_from_errno(err); +} + +static int __prestera_inetaddr_valid_cb(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct in_validator_info *ivi = (struct in_validator_info *)ptr; + struct net_device *dev = ivi->ivi_dev->dev; + struct prestera_router *router = container_of(nb, + struct prestera_router, + inetaddr_valid_nb); + struct in_device *idev; + int err = 0; + + if (event != NETDEV_UP) + goto out; + + /* Ignore if this is not first address */ + idev = __in_dev_get_rtnl(dev); + if (idev && idev->ifa_list) + goto out; + + if (ipv4_is_multicast(ivi->ivi_addr)) { + NL_SET_ERR_MSG_MOD(ivi->extack, + "Multicast addr on RIF is not supported"); + err = -EINVAL; + goto out; + } + + err = __prestera_inetaddr_event(router->sw, dev, event, ivi->extack); +out: + return notifier_from_errno(err); +} + +struct prestera_fib_event_work { + struct work_struct work; + struct prestera_switch *sw; + struct fib_entry_notifier_info fen_info; + unsigned long event; +}; + +static void __prestera_router_fib_event_work(struct work_struct *work) +{ + struct prestera_fib_event_work *fib_work = + container_of(work, struct prestera_fib_event_work, work); + struct prestera_switch *sw = fib_work->sw; + int err; + + rtnl_lock(); + + switch (fib_work->event) { + case FIB_EVENT_ENTRY_REPLACE: + err = prestera_k_arb_fib_evt(sw, true, + &fib_work->fen_info.info); + if (err) + goto err_out; + + break; + case FIB_EVENT_ENTRY_DEL: + err = prestera_k_arb_fib_evt(sw, false, + &fib_work->fen_info.info); + if (err) + goto err_out; + + break; + } + + goto out; + +err_out: + dev_err(sw->dev->dev, "Error when processing %pI4h/%d", + &fib_work->fen_info.dst, + fib_work->fen_info.dst_len); +out: + fib_info_put(fib_work->fen_info.fi); + rtnl_unlock(); + kfree(fib_work); +} + +/* Called with rcu_read_lock() */ +static int __prestera_router_fib_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct prestera_fib_event_work *fib_work; + struct fib_entry_notifier_info *fen_info; + struct fib_notifier_info *info = ptr; + struct prestera_router *router; + + if (info->family != AF_INET) + return NOTIFY_DONE; + + router = container_of(nb, struct prestera_router, fib_nb); + + switch (event) { + case FIB_EVENT_ENTRY_REPLACE: + case FIB_EVENT_ENTRY_DEL: + fen_info = container_of(info, struct fib_entry_notifier_info, + info); + if (!fen_info->fi) + return NOTIFY_DONE; + + fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); + if (WARN_ON(!fib_work)) + return NOTIFY_BAD; + + fib_info_hold(fen_info->fi); + fib_work->fen_info = *fen_info; + fib_work->event = event; + fib_work->sw = router->sw; + INIT_WORK(&fib_work->work, __prestera_router_fib_event_work); + prestera_queue_work(&fib_work->work); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_DONE; +} + +struct prestera_netevent_work { + struct work_struct work; + struct prestera_switch *sw; + struct neighbour *n; +}; + +static void prestera_router_neigh_event_work(struct work_struct *work) +{ + struct prestera_netevent_work *net_work = + container_of(work, struct prestera_netevent_work, work); + struct prestera_switch *sw = net_work->sw; + struct neighbour *n = net_work->n; + + /* neigh - its not hw related object. It stored only in kernel. So... */ + rtnl_lock(); + + prestera_k_arb_n_evt(sw, n); + + neigh_release(n); + rtnl_unlock(); + kfree(net_work); +} + +static int prestera_router_netevent_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct prestera_netevent_work *net_work; + struct prestera_router *router; + struct neighbour *n = ptr; + + router = container_of(nb, struct prestera_router, netevent_nb); + + switch (event) { + case NETEVENT_NEIGH_UPDATE: + if (n->tbl->family != AF_INET) + return NOTIFY_DONE; + + net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); + if (WARN_ON(!net_work)) + return NOTIFY_BAD; + + neigh_clone(n); + net_work->n = n; + net_work->sw = router->sw; + INIT_WORK(&net_work->work, prestera_router_neigh_event_work); + prestera_queue_work(&net_work->work); + } + + return NOTIFY_DONE; +} + +static void prestera_router_update_neighs_work(struct work_struct *work) +{ + struct prestera_router *router; + + router = container_of(work, struct prestera_router, + neighs_update.dw.work); + rtnl_lock(); + + prestera_k_arb_hw_evt(router->sw); + + rtnl_unlock(); + prestera_queue_delayed_work(&router->neighs_update.dw, + msecs_to_jiffies(PRESTERA_NH_PROBE_INTERVAL)); +} + +static int prestera_neigh_work_init(struct prestera_switch *sw) +{ + INIT_DELAYED_WORK(&sw->router->neighs_update.dw, + prestera_router_update_neighs_work); + prestera_queue_delayed_work(&sw->router->neighs_update.dw, 0); + return 0; +} + +static void prestera_neigh_work_fini(struct prestera_switch *sw) +{ + cancel_delayed_work_sync(&sw->router->neighs_update.dw); +} + +int prestera_router_init(struct prestera_switch *sw) +{ + struct prestera_router *router; + int err, nhgrp_cache_bytes; + + router = kzalloc(sizeof(*sw->router), GFP_KERNEL); + if (!router) + return -ENOMEM; + + sw->router = router; + router->sw = sw; + + err = prestera_router_hw_init(sw); + if (err) + goto err_router_lib_init; + + err = rhashtable_init(&router->kern_fib_cache_ht, + &__prestera_kern_fib_cache_ht_params); + if (err) + goto err_kern_fib_cache_ht_init; + + err = rhashtable_init(&router->kern_neigh_cache_ht, + &__prestera_kern_neigh_cache_ht_params); + if (err) + goto err_kern_neigh_cache_ht_init; + + nhgrp_cache_bytes = sw->size_tbl_router_nexthop / 8 + 1; + router->nhgrp_hw_state_cache = kzalloc(nhgrp_cache_bytes, GFP_KERNEL); + if (!router->nhgrp_hw_state_cache) { + err = -ENOMEM; + goto err_nh_state_cache_alloc; + } + + err = prestera_neigh_work_init(sw); + if (err) + goto err_neigh_work_init; + + router->inetaddr_valid_nb.notifier_call = __prestera_inetaddr_valid_cb; + err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); + if (err) + goto err_register_inetaddr_validator_notifier; + + router->inetaddr_nb.notifier_call = __prestera_inetaddr_cb; + err = register_inetaddr_notifier(&router->inetaddr_nb); + if (err) + goto err_register_inetaddr_notifier; + + router->netevent_nb.notifier_call = prestera_router_netevent_event; + err = register_netevent_notifier(&router->netevent_nb); + if (err) + goto err_register_netevent_notifier; + + router->fib_nb.notifier_call = __prestera_router_fib_event; + err = register_fib_notifier(&init_net, &router->fib_nb, + /* TODO: flush fib entries */ NULL, NULL); + if (err) + goto err_register_fib_notifier; + + return 0; + +err_register_fib_notifier: + unregister_netevent_notifier(&router->netevent_nb); +err_register_netevent_notifier: + unregister_inetaddr_notifier(&router->inetaddr_nb); +err_register_inetaddr_notifier: + unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); +err_register_inetaddr_validator_notifier: + prestera_neigh_work_fini(sw); +err_neigh_work_init: + kfree(router->nhgrp_hw_state_cache); +err_nh_state_cache_alloc: + rhashtable_destroy(&router->kern_neigh_cache_ht); +err_kern_neigh_cache_ht_init: + rhashtable_destroy(&router->kern_fib_cache_ht); +err_kern_fib_cache_ht_init: + prestera_router_hw_fini(sw); +err_router_lib_init: + kfree(sw->router); + return err; +} + +void prestera_router_fini(struct prestera_switch *sw) +{ + unregister_fib_notifier(&init_net, &sw->router->fib_nb); + unregister_netevent_notifier(&sw->router->netevent_nb); + unregister_inetaddr_notifier(&sw->router->inetaddr_nb); + unregister_inetaddr_validator_notifier(&sw->router->inetaddr_valid_nb); + prestera_neigh_work_fini(sw); + prestera_queue_drain(); + + prestera_k_arb_abort(sw); + + kfree(sw->router->nhgrp_hw_state_cache); + rhashtable_destroy(&sw->router->kern_fib_cache_ht); + prestera_router_hw_fini(sw); + kfree(sw->router); + sw->router = NULL; +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c new file mode 100644 index 000000000000..aa080dc57ff0 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2019-2021 Marvell International Ltd. All rights reserved */ + +#include <linux/rhashtable.h> + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_router_hw.h" +#include "prestera_acl.h" + +/* Nexthop is pointed + * to port (not rif) + * +-------+ + * +>|nexthop| + * | +-------+ + * | + * +--+ +-----++ + * +------->|vr|<-+ +>|nh_grp| + * | +--+ | | +------+ + * | | | + * +-+-------+ +--+---+-+ + * |rif_entry| |fib_node| + * +---------+ +--------+ + * Rif is Fib - is exit point + * used as + * entry point + * for vr in hw + */ + +#define PRESTERA_NHGR_UNUSED (0) +#define PRESTERA_NHGR_DROP (0xFFFFFFFF) +/* Need to merge it with router_manager */ +#define PRESTERA_NH_ACTIVE_JIFFER_FILTER 3000 /* ms */ + +static const struct rhashtable_params __prestera_fib_ht_params = { + .key_offset = offsetof(struct prestera_fib_node, key), + .head_offset = offsetof(struct prestera_fib_node, ht_node), + .key_len = sizeof(struct prestera_fib_key), + .automatic_shrinking = true, +}; + +static const struct rhashtable_params __prestera_nh_neigh_ht_params = { + .key_offset = offsetof(struct prestera_nh_neigh, key), + .key_len = sizeof(struct prestera_nh_neigh_key), + .head_offset = offsetof(struct prestera_nh_neigh, ht_node), +}; + +static const struct rhashtable_params __prestera_nexthop_group_ht_params = { + .key_offset = offsetof(struct prestera_nexthop_group, key), + .key_len = sizeof(struct prestera_nexthop_group_key), + .head_offset = offsetof(struct prestera_nexthop_group, ht_node), +}; + +static int prestera_nexthop_group_set(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp); +static bool +prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp); +static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg); + +/* TODO: move to router.h as macros */ +static bool prestera_nh_neigh_key_is_valid(struct prestera_nh_neigh_key *key) +{ + return memchr_inv(key, 0, sizeof(*key)) ? true : false; +} + +int prestera_router_hw_init(struct prestera_switch *sw) +{ + int err; + + err = rhashtable_init(&sw->router->nh_neigh_ht, + &__prestera_nh_neigh_ht_params); + if (err) + goto err_nh_neigh_ht_init; + + err = rhashtable_init(&sw->router->nexthop_group_ht, + &__prestera_nexthop_group_ht_params); + if (err) + goto err_nexthop_grp_ht_init; + + err = rhashtable_init(&sw->router->fib_ht, + &__prestera_fib_ht_params); + if (err) + goto err_fib_ht_init; + + INIT_LIST_HEAD(&sw->router->vr_list); + INIT_LIST_HEAD(&sw->router->rif_entry_list); + + return 0; + +err_fib_ht_init: + rhashtable_destroy(&sw->router->nexthop_group_ht); +err_nexthop_grp_ht_init: + rhashtable_destroy(&sw->router->nh_neigh_ht); +err_nh_neigh_ht_init: + return 0; +} + +void prestera_router_hw_fini(struct prestera_switch *sw) +{ + rhashtable_free_and_destroy(&sw->router->fib_ht, + prestera_fib_node_destroy_ht_cb, sw); + WARN_ON(!list_empty(&sw->router->vr_list)); + WARN_ON(!list_empty(&sw->router->rif_entry_list)); + rhashtable_destroy(&sw->router->fib_ht); + rhashtable_destroy(&sw->router->nexthop_group_ht); + rhashtable_destroy(&sw->router->nh_neigh_ht); +} + +static struct prestera_vr *__prestera_vr_find(struct prestera_switch *sw, + u32 tb_id) +{ + struct prestera_vr *vr; + + list_for_each_entry(vr, &sw->router->vr_list, router_node) { + if (vr->tb_id == tb_id) + return vr; + } + + return NULL; +} + +static struct prestera_vr *__prestera_vr_create(struct prestera_switch *sw, + u32 tb_id, + struct netlink_ext_ack *extack) +{ + struct prestera_vr *vr; + int err; + + vr = kzalloc(sizeof(*vr), GFP_KERNEL); + if (!vr) { + err = -ENOMEM; + goto err_alloc_vr; + } + + vr->tb_id = tb_id; + + err = prestera_hw_vr_create(sw, &vr->hw_vr_id); + if (err) + goto err_hw_create; + + list_add(&vr->router_node, &sw->router->vr_list); + + return vr; + +err_hw_create: + kfree(vr); +err_alloc_vr: + return ERR_PTR(err); +} + +static void __prestera_vr_destroy(struct prestera_switch *sw, + struct prestera_vr *vr) +{ + list_del(&vr->router_node); + prestera_hw_vr_delete(sw, vr->hw_vr_id); + kfree(vr); +} + +static struct prestera_vr *prestera_vr_get(struct prestera_switch *sw, u32 tb_id, + struct netlink_ext_ack *extack) +{ + struct prestera_vr *vr; + + vr = __prestera_vr_find(sw, tb_id); + if (vr) { + refcount_inc(&vr->refcount); + } else { + vr = __prestera_vr_create(sw, tb_id, extack); + if (IS_ERR(vr)) + return ERR_CAST(vr); + + refcount_set(&vr->refcount, 1); + } + + return vr; +} + +static void prestera_vr_put(struct prestera_switch *sw, struct prestera_vr *vr) +{ + if (refcount_dec_and_test(&vr->refcount)) + __prestera_vr_destroy(sw, vr); +} + +/* iface is overhead struct. vr_id also can be removed. */ +static int +__prestera_rif_entry_key_copy(const struct prestera_rif_entry_key *in, + struct prestera_rif_entry_key *out) +{ + memset(out, 0, sizeof(*out)); + + switch (in->iface.type) { + case PRESTERA_IF_PORT_E: + out->iface.dev_port.hw_dev_num = in->iface.dev_port.hw_dev_num; + out->iface.dev_port.port_num = in->iface.dev_port.port_num; + break; + case PRESTERA_IF_LAG_E: + out->iface.lag_id = in->iface.lag_id; + break; + case PRESTERA_IF_VID_E: + out->iface.vlan_id = in->iface.vlan_id; + break; + default: + WARN(1, "Unsupported iface type"); + return -EINVAL; + } + + out->iface.type = in->iface.type; + return 0; +} + +struct prestera_rif_entry * +prestera_rif_entry_find(const struct prestera_switch *sw, + const struct prestera_rif_entry_key *k) +{ + struct prestera_rif_entry *rif_entry; + struct prestera_rif_entry_key lk; /* lookup key */ + + if (__prestera_rif_entry_key_copy(k, &lk)) + return NULL; + + list_for_each_entry(rif_entry, &sw->router->rif_entry_list, + router_node) { + if (!memcmp(k, &rif_entry->key, sizeof(*k))) + return rif_entry; + } + + return NULL; +} + +void prestera_rif_entry_destroy(struct prestera_switch *sw, + struct prestera_rif_entry *e) +{ + struct prestera_iface iface; + + list_del(&e->router_node); + + memcpy(&iface, &e->key.iface, sizeof(iface)); + iface.vr_id = e->vr->hw_vr_id; + prestera_hw_rif_delete(sw, e->hw_id, &iface); + + prestera_vr_put(sw, e->vr); + kfree(e); +} + +struct prestera_rif_entry * +prestera_rif_entry_create(struct prestera_switch *sw, + struct prestera_rif_entry_key *k, + u32 tb_id, const unsigned char *addr) +{ + int err; + struct prestera_rif_entry *e; + struct prestera_iface iface; + + e = kzalloc(sizeof(*e), GFP_KERNEL); + if (!e) + goto err_kzalloc; + + if (__prestera_rif_entry_key_copy(k, &e->key)) + goto err_key_copy; + + e->vr = prestera_vr_get(sw, tb_id, NULL); + if (IS_ERR(e->vr)) + goto err_vr_get; + + memcpy(&e->addr, addr, sizeof(e->addr)); + + /* HW */ + memcpy(&iface, &e->key.iface, sizeof(iface)); + iface.vr_id = e->vr->hw_vr_id; + err = prestera_hw_rif_create(sw, &iface, e->addr, &e->hw_id); + if (err) + goto err_hw_create; + + list_add(&e->router_node, &sw->router->rif_entry_list); + + return e; + +err_hw_create: + prestera_vr_put(sw, e->vr); +err_vr_get: +err_key_copy: + kfree(e); +err_kzalloc: + return NULL; +} + +static void __prestera_nh_neigh_destroy(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + rhashtable_remove_fast(&sw->router->nh_neigh_ht, + &neigh->ht_node, + __prestera_nh_neigh_ht_params); + kfree(neigh); +} + +static struct prestera_nh_neigh * +__prestera_nh_neigh_create(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *neigh; + int err; + + neigh = kzalloc(sizeof(*neigh), GFP_KERNEL); + if (!neigh) + goto err_kzalloc; + + memcpy(&neigh->key, key, sizeof(*key)); + neigh->info.connected = false; + INIT_LIST_HEAD(&neigh->nexthop_group_list); + err = rhashtable_insert_fast(&sw->router->nh_neigh_ht, + &neigh->ht_node, + __prestera_nh_neigh_ht_params); + if (err) + goto err_rhashtable_insert; + + return neigh; + +err_rhashtable_insert: + kfree(neigh); +err_kzalloc: + return NULL; +} + +struct prestera_nh_neigh * +prestera_nh_neigh_find(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *nh_neigh; + + nh_neigh = rhashtable_lookup_fast(&sw->router->nh_neigh_ht, + key, __prestera_nh_neigh_ht_params); + return IS_ERR(nh_neigh) ? NULL : nh_neigh; +} + +struct prestera_nh_neigh * +prestera_nh_neigh_get(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *neigh; + + neigh = prestera_nh_neigh_find(sw, key); + if (!neigh) + return __prestera_nh_neigh_create(sw, key); + + return neigh; +} + +void prestera_nh_neigh_put(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + if (list_empty(&neigh->nexthop_group_list)) + __prestera_nh_neigh_destroy(sw, neigh); +} + +/* Updates new prestera_neigh_info */ +int prestera_nh_neigh_set(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + struct prestera_nh_neigh_head *nh_head; + struct prestera_nexthop_group *nh_grp; + int err; + + list_for_each_entry(nh_head, &neigh->nexthop_group_list, head) { + nh_grp = nh_head->this; + err = prestera_nexthop_group_set(sw, nh_grp); + if (err) + return err; + } + + return 0; +} + +bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, + struct prestera_nh_neigh *nh_neigh) +{ + bool state; + struct prestera_nh_neigh_head *nh_head, *tmp; + + state = false; + list_for_each_entry_safe(nh_head, tmp, + &nh_neigh->nexthop_group_list, head) { + state = prestera_nexthop_group_util_hw_state(sw, nh_head->this); + if (state) + goto out; + } + +out: + return state; +} + +static struct prestera_nexthop_group * +__prestera_nexthop_group_create(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + struct prestera_nh_neigh *nh_neigh; + int nh_cnt, err, gid; + + nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); + if (!nh_grp) + goto err_kzalloc; + + memcpy(&nh_grp->key, key, sizeof(*key)); + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + if (!prestera_nh_neigh_key_is_valid(&nh_grp->key.neigh[nh_cnt])) + break; + + nh_neigh = prestera_nh_neigh_get(sw, + &nh_grp->key.neigh[nh_cnt]); + if (!nh_neigh) + goto err_nh_neigh_get; + + nh_grp->nh_neigh_head[nh_cnt].neigh = nh_neigh; + nh_grp->nh_neigh_head[nh_cnt].this = nh_grp; + list_add(&nh_grp->nh_neigh_head[nh_cnt].head, + &nh_neigh->nexthop_group_list); + } + + err = prestera_hw_nh_group_create(sw, nh_cnt, &nh_grp->grp_id); + if (err) + goto err_nh_group_create; + + err = prestera_nexthop_group_set(sw, nh_grp); + if (err) + goto err_nexthop_group_set; + + err = rhashtable_insert_fast(&sw->router->nexthop_group_ht, + &nh_grp->ht_node, + __prestera_nexthop_group_ht_params); + if (err) + goto err_ht_insert; + + /* reset cache for created group */ + gid = nh_grp->grp_id; + sw->router->nhgrp_hw_state_cache[gid / 8] &= ~BIT(gid % 8); + + return nh_grp; + +err_ht_insert: +err_nexthop_group_set: + prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); +err_nh_group_create: +err_nh_neigh_get: + for (nh_cnt--; nh_cnt >= 0; nh_cnt--) { + list_del(&nh_grp->nh_neigh_head[nh_cnt].head); + prestera_nh_neigh_put(sw, nh_grp->nh_neigh_head[nh_cnt].neigh); + } + + kfree(nh_grp); +err_kzalloc: + return NULL; +} + +static void +__prestera_nexthop_group_destroy(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + struct prestera_nh_neigh *nh_neigh; + int nh_cnt; + + rhashtable_remove_fast(&sw->router->nexthop_group_ht, + &nh_grp->ht_node, + __prestera_nexthop_group_ht_params); + + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + nh_neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; + if (!nh_neigh) + break; + + list_del(&nh_grp->nh_neigh_head[nh_cnt].head); + prestera_nh_neigh_put(sw, nh_neigh); + } + + prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); + kfree(nh_grp); +} + +static struct prestera_nexthop_group * +__prestera_nexthop_group_find(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + + nh_grp = rhashtable_lookup_fast(&sw->router->nexthop_group_ht, + key, __prestera_nexthop_group_ht_params); + return IS_ERR(nh_grp) ? NULL : nh_grp; +} + +static struct prestera_nexthop_group * +prestera_nexthop_group_get(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + + nh_grp = __prestera_nexthop_group_find(sw, key); + if (nh_grp) { + refcount_inc(&nh_grp->refcount); + } else { + nh_grp = __prestera_nexthop_group_create(sw, key); + if (!nh_grp) + return ERR_PTR(-ENOMEM); + + refcount_set(&nh_grp->refcount, 1); + } + + return nh_grp; +} + +static void prestera_nexthop_group_put(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + if (refcount_dec_and_test(&nh_grp->refcount)) + __prestera_nexthop_group_destroy(sw, nh_grp); +} + +/* Updates with new nh_neigh's info */ +static int prestera_nexthop_group_set(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + struct prestera_neigh_info info[PRESTERA_NHGR_SIZE_MAX]; + struct prestera_nh_neigh *neigh; + int nh_cnt; + + memset(&info[0], 0, sizeof(info)); + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; + if (!neigh) + break; + + memcpy(&info[nh_cnt], &neigh->info, sizeof(neigh->info)); + } + + return prestera_hw_nh_entries_set(sw, nh_cnt, &info[0], nh_grp->grp_id); +} + +static bool +prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + int err; + u32 buf_size = sw->size_tbl_router_nexthop / 8 + 1; + u32 gid = nh_grp->grp_id; + u8 *cache = sw->router->nhgrp_hw_state_cache; + + /* Antijitter + * Prevent situation, when we read state of nh_grp twice in short time, + * and state bit is still cleared on second call. So just stuck active + * state for PRESTERA_NH_ACTIVE_JIFFER_FILTER, after last occurred. + */ + if (!time_before(jiffies, sw->router->nhgrp_hw_cache_kick + + msecs_to_jiffies(PRESTERA_NH_ACTIVE_JIFFER_FILTER))) { + err = prestera_hw_nhgrp_blk_get(sw, cache, buf_size); + if (err) { + pr_err("Failed to get hw state nh_grp's"); + return false; + } + + sw->router->nhgrp_hw_cache_kick = jiffies; + } + + if (cache[gid / 8] & BIT(gid % 8)) + return true; + + return false; +} + +struct prestera_fib_node * +prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key) +{ + struct prestera_fib_node *fib_node; + + fib_node = rhashtable_lookup_fast(&sw->router->fib_ht, key, + __prestera_fib_ht_params); + return fib_node; +} + +static void __prestera_fib_node_destruct(struct prestera_switch *sw, + struct prestera_fib_node *fib_node) +{ + struct prestera_vr *vr; + + vr = fib_node->info.vr; + prestera_hw_lpm_del(sw, vr->hw_vr_id, fib_node->key.addr.u.ipv4, + fib_node->key.prefix_len); + switch (fib_node->info.type) { + case PRESTERA_FIB_TYPE_UC_NH: + prestera_nexthop_group_put(sw, fib_node->info.nh_grp); + break; + case PRESTERA_FIB_TYPE_TRAP: + break; + case PRESTERA_FIB_TYPE_DROP: + break; + default: + pr_err("Unknown fib_node->info.type = %d", + fib_node->info.type); + } + + prestera_vr_put(sw, vr); +} + +void prestera_fib_node_destroy(struct prestera_switch *sw, + struct prestera_fib_node *fib_node) +{ + __prestera_fib_node_destruct(sw, fib_node); + rhashtable_remove_fast(&sw->router->fib_ht, &fib_node->ht_node, + __prestera_fib_ht_params); + kfree(fib_node); +} + +static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg) +{ + struct prestera_fib_node *node = ptr; + struct prestera_switch *sw = arg; + + __prestera_fib_node_destruct(sw, node); + kfree(node); +} + +struct prestera_fib_node * +prestera_fib_node_create(struct prestera_switch *sw, + struct prestera_fib_key *key, + enum prestera_fib_type fib_type, + struct prestera_nexthop_group_key *nh_grp_key) +{ + struct prestera_fib_node *fib_node; + u32 grp_id; + struct prestera_vr *vr; + int err; + + fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); + if (!fib_node) + goto err_kzalloc; + + memcpy(&fib_node->key, key, sizeof(*key)); + fib_node->info.type = fib_type; + + vr = prestera_vr_get(sw, key->tb_id, NULL); + if (IS_ERR(vr)) + goto err_vr_get; + + fib_node->info.vr = vr; + + switch (fib_type) { + case PRESTERA_FIB_TYPE_TRAP: + grp_id = PRESTERA_NHGR_UNUSED; + break; + case PRESTERA_FIB_TYPE_DROP: + grp_id = PRESTERA_NHGR_DROP; + break; + case PRESTERA_FIB_TYPE_UC_NH: + fib_node->info.nh_grp = prestera_nexthop_group_get(sw, + nh_grp_key); + if (IS_ERR(fib_node->info.nh_grp)) + goto err_nh_grp_get; + + grp_id = fib_node->info.nh_grp->grp_id; + break; + default: + pr_err("Unsupported fib_type %d", fib_type); + goto err_nh_grp_get; + } + + err = prestera_hw_lpm_add(sw, vr->hw_vr_id, key->addr.u.ipv4, + key->prefix_len, grp_id); + if (err) + goto err_lpm_add; + + err = rhashtable_insert_fast(&sw->router->fib_ht, &fib_node->ht_node, + __prestera_fib_ht_params); + if (err) + goto err_ht_insert; + + return fib_node; + +err_ht_insert: + prestera_hw_lpm_del(sw, vr->hw_vr_id, key->addr.u.ipv4, + key->prefix_len); +err_lpm_add: + if (fib_type == PRESTERA_FIB_TYPE_UC_NH) + prestera_nexthop_group_put(sw, fib_node->info.nh_grp); +err_nh_grp_get: + prestera_vr_put(sw, vr); +err_vr_get: + kfree(fib_node); +err_kzalloc: + return NULL; +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h new file mode 100644 index 000000000000..9ca97919c863 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2019-2021 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_ROUTER_HW_H_ +#define _PRESTERA_ROUTER_HW_H_ + +struct prestera_vr { + struct list_head router_node; + refcount_t refcount; + u32 tb_id; /* key (kernel fib table id) */ + u16 hw_vr_id; /* virtual router ID */ + u8 __pad[2]; +}; + +struct prestera_rif_entry { + struct prestera_rif_entry_key { + struct prestera_iface iface; + } key; + struct prestera_vr *vr; + unsigned char addr[ETH_ALEN]; + u16 hw_id; /* rif_id */ + struct list_head router_node; /* ht */ +}; + +struct prestera_ip_addr { + union { + __be32 ipv4; + struct in6_addr ipv6; + } u; + enum { + PRESTERA_IPV4 = 0, + PRESTERA_IPV6 + } v; +#define PRESTERA_IP_ADDR_PLEN(V) ((V) == PRESTERA_IPV4 ? 32 : \ + /* (V) == PRESTERA_IPV6 ? */ 128 /* : 0 */) +}; + +struct prestera_nh_neigh_key { + struct prestera_ip_addr addr; + /* Seems like rif is obsolete, because there is iface in info ? + * Key can contain functional fields, or fields, which is used to + * filter duplicate objects on logical level (before you pass it to + * HW)... also key can be used to cover hardware restrictions. + * In our case rif - is logical interface (even can be VLAN), which + * is used in combination with IP address (which is also not related to + * hardware nexthop) to provide logical compression of created nexthops. + * You even can imagine, that rif+IPaddr is just cookie. + */ + /* struct prestera_rif *rif; */ + /* Use just as cookie, to divide ARP domains (in order with addr) */ + void *rif; +}; + +/* Used for hw call */ +struct prestera_neigh_info { + struct prestera_iface iface; + unsigned char ha[ETH_ALEN]; + u8 connected; /* bool. indicate, if mac/oif valid */ + u8 __pad[1]; +}; + +/* Used to notify nh about neigh change */ +struct prestera_nh_neigh { + struct prestera_nh_neigh_key key; + struct prestera_neigh_info info; + struct rhash_head ht_node; /* node of prestera_vr */ + struct list_head nexthop_group_list; +}; + +#define PRESTERA_NHGR_SIZE_MAX 4 + +struct prestera_nexthop_group { + struct prestera_nexthop_group_key { + struct prestera_nh_neigh_key neigh[PRESTERA_NHGR_SIZE_MAX]; + } key; + /* Store intermediate object here. + * This prevent overhead kzalloc call. + */ + /* nh_neigh is used only to notify nexthop_group */ + struct prestera_nh_neigh_head { + struct prestera_nexthop_group *this; + struct list_head head; + /* ptr to neigh is not necessary. + * It used to prevent lookup of nh_neigh by key (n) on destroy + */ + struct prestera_nh_neigh *neigh; + } nh_neigh_head[PRESTERA_NHGR_SIZE_MAX]; + struct rhash_head ht_node; /* node of prestera_vr */ + refcount_t refcount; + u32 grp_id; /* hw */ +}; + +struct prestera_fib_key { + struct prestera_ip_addr addr; + u32 prefix_len; + u32 tb_id; +}; + +struct prestera_fib_info { + struct prestera_vr *vr; + struct list_head vr_node; + enum prestera_fib_type { + PRESTERA_FIB_TYPE_INVALID = 0, + /* must be pointer to nh_grp id */ + PRESTERA_FIB_TYPE_UC_NH, + /* It can be connected route + * and will be overlapped with neighbours + */ + PRESTERA_FIB_TYPE_TRAP, + PRESTERA_FIB_TYPE_DROP + } type; + /* Valid only if type = UC_NH*/ + struct prestera_nexthop_group *nh_grp; +}; + +struct prestera_fib_node { + struct rhash_head ht_node; /* node of prestera_vr */ + struct prestera_fib_key key; + struct prestera_fib_info info; /* action related info */ +}; + +struct prestera_rif_entry * +prestera_rif_entry_find(const struct prestera_switch *sw, + const struct prestera_rif_entry_key *k); +void prestera_rif_entry_destroy(struct prestera_switch *sw, + struct prestera_rif_entry *e); +struct prestera_rif_entry * +prestera_rif_entry_create(struct prestera_switch *sw, + struct prestera_rif_entry_key *k, + u32 tb_id, const unsigned char *addr); +struct prestera_nh_neigh * +prestera_nh_neigh_find(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key); +struct prestera_nh_neigh * +prestera_nh_neigh_get(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key); +void prestera_nh_neigh_put(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh); +int prestera_nh_neigh_set(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh); +bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, + struct prestera_nh_neigh *nh_neigh); +struct prestera_fib_node *prestera_fib_node_find(struct prestera_switch *sw, + struct prestera_fib_key *key); +void prestera_fib_node_destroy(struct prestera_switch *sw, + struct prestera_fib_node *fib_node); +struct prestera_fib_node * +prestera_fib_node_create(struct prestera_switch *sw, + struct prestera_fib_key *key, + enum prestera_fib_type fib_type, + struct prestera_nexthop_group_key *nh_grp_key); +int prestera_router_hw_init(struct prestera_switch *sw); +void prestera_router_hw_fini(struct prestera_switch *sw); + +#endif /* _PRESTERA_ROUTER_HW_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c index e452cdeaf703..9277a8fd1339 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c @@ -102,7 +102,7 @@ struct prestera_sdma { struct net_device napi_dev; u32 map_addr; u64 dma_mask; - /* protect SDMA with concurrrent access from multiple CPUs */ + /* protect SDMA with concurrent access from multiple CPUs */ spinlock_t tx_lock; }; @@ -659,7 +659,7 @@ static int prestera_sdma_switch_init(struct prestera_switch *sw) init_dummy_netdev(&sdma->napi_dev); - netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll, 64); + netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll); napi_enable(&sdma->rx_napi); return 0; @@ -776,6 +776,7 @@ tx_done: int prestera_rxtx_switch_init(struct prestera_switch *sw) { struct prestera_rxtx *rxtx; + int err; rxtx = kzalloc(sizeof(*rxtx), GFP_KERNEL); if (!rxtx) @@ -783,7 +784,11 @@ int prestera_rxtx_switch_init(struct prestera_switch *sw) sw->rxtx = rxtx; - return prestera_sdma_switch_init(sw); + err = prestera_sdma_switch_init(sw); + if (err) + kfree(rxtx); + + return err; } void prestera_rxtx_switch_fini(struct prestera_switch *sw) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.c b/drivers/net/ethernet/marvell/prestera/prestera_span.c index 3cafca827bb7..1005182ce3bc 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.c @@ -7,6 +7,7 @@ #include "prestera.h" #include "prestera_hw.h" #include "prestera_acl.h" +#include "prestera_flow.h" #include "prestera_span.h" struct prestera_span_entry { @@ -106,7 +107,7 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id) entry = prestera_span_entry_find_by_id(sw->span, span_id); if (!entry) - return false; + return -ENOENT; if (!refcount_dec_and_test(&entry->ref_count)) return 0; @@ -119,8 +120,9 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id) return 0; } -static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, - struct prestera_port *to_port) +int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port, + bool ingress) { struct prestera_switch *sw = binding->port->sw; u8 span_id; @@ -134,7 +136,7 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, if (err) return err; - err = prestera_hw_span_bind(binding->port, span_id); + err = prestera_hw_span_bind(binding->port, span_id, ingress); if (err) { prestera_span_put(sw, span_id); return err; @@ -144,11 +146,15 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, return 0; } -static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) +int prestera_span_rule_del(struct prestera_flow_block_binding *binding, + bool ingress) { int err; - err = prestera_hw_span_unbind(binding->port); + if (binding->span_id == PRESTERA_SPAN_INVALID_ID) + return -ENOENT; + + err = prestera_hw_span_unbind(binding->port, ingress); if (err) return err; @@ -160,60 +166,6 @@ static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) return 0; } -int prestera_span_replace(struct prestera_flow_block *block, - struct tc_cls_matchall_offload *f) -{ - struct prestera_flow_block_binding *binding; - __be16 protocol = f->common.protocol; - struct flow_action_entry *act; - struct prestera_port *port; - int err; - - if (!flow_offload_has_one_action(&f->rule->action)) { - NL_SET_ERR_MSG(f->common.extack, - "Only singular actions are supported"); - return -EOPNOTSUPP; - } - - act = &f->rule->action.entries[0]; - - if (!prestera_netdev_check(act->dev)) { - NL_SET_ERR_MSG(f->common.extack, - "Only Marvell Prestera port is supported"); - return -EINVAL; - } - if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) - return -EOPNOTSUPP; - if (act->id != FLOW_ACTION_MIRRED) - return -EOPNOTSUPP; - if (protocol != htons(ETH_P_ALL)) - return -EOPNOTSUPP; - - port = netdev_priv(act->dev); - - list_for_each_entry(binding, &block->binding_list, list) { - err = prestera_span_rule_add(binding, port); - if (err) - goto rollback; - } - - return 0; - -rollback: - list_for_each_entry_continue_reverse(binding, - &block->binding_list, list) - prestera_span_rule_del(binding); - return err; -} - -void prestera_span_destroy(struct prestera_flow_block *block) -{ - struct prestera_flow_block_binding *binding; - - list_for_each_entry(binding, &block->binding_list, list) - prestera_span_rule_del(binding); -} - int prestera_span_init(struct prestera_switch *sw) { struct prestera_span *span; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.h b/drivers/net/ethernet/marvell/prestera/prestera_span.h index f0644521f78a..493b68524bcb 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.h @@ -8,13 +8,17 @@ #define PRESTERA_SPAN_INVALID_ID -1 +struct prestera_port; struct prestera_switch; -struct prestera_flow_block; +struct prestera_flow_block_binding; int prestera_span_init(struct prestera_switch *sw); void prestera_span_fini(struct prestera_switch *sw); -int prestera_span_replace(struct prestera_flow_block *block, - struct tc_cls_matchall_offload *f); -void prestera_span_destroy(struct prestera_flow_block *block); + +int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port, + bool ingress); +int prestera_span_rule_del(struct prestera_flow_block_binding *binding, + bool ingress); #endif /* _PRESTERA_SPAN_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c index b4599fe4ca8d..e548cd32582e 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c @@ -39,7 +39,10 @@ struct prestera_bridge { struct net_device *dev; struct prestera_switchdev *swdev; struct list_head port_list; + struct list_head br_mdb_entry_list; + bool mrouter_exist; bool vlan_enabled; + bool multicast_enabled; u16 bridge_id; }; @@ -48,8 +51,10 @@ struct prestera_bridge_port { struct net_device *dev; struct prestera_bridge *bridge; struct list_head vlan_list; + struct list_head br_mdb_port_list; refcount_t ref_count; unsigned long flags; + bool mrouter; u8 stp_state; }; @@ -67,6 +72,20 @@ struct prestera_port_vlan { u16 vid; }; +struct prestera_br_mdb_port { + struct prestera_bridge_port *br_port; + struct list_head br_mdb_port_node; +}; + +/* Software representation of MDB table. */ +struct prestera_br_mdb_entry { + struct prestera_bridge *bridge; + struct prestera_mdb_entry *mdb; + struct list_head br_mdb_port_list; + struct list_head br_mdb_entry_node; + bool enabled; +}; + static struct workqueue_struct *swdev_wq; static void prestera_bridge_port_put(struct prestera_bridge_port *br_port); @@ -74,6 +93,88 @@ static void prestera_bridge_port_put(struct prestera_bridge_port *br_port); static int prestera_port_vid_stp_set(struct prestera_port *port, u16 vid, u8 state); +static struct prestera_bridge * +prestera_bridge_find(const struct prestera_switch *sw, + const struct net_device *br_dev) +{ + struct prestera_bridge *bridge; + + list_for_each_entry(bridge, &sw->swdev->bridge_list, head) + if (bridge->dev == br_dev) + return bridge; + + return NULL; +} + +static struct prestera_bridge_port * +__prestera_bridge_port_find(const struct prestera_bridge *bridge, + const struct net_device *brport_dev) +{ + struct prestera_bridge_port *br_port; + + list_for_each_entry(br_port, &bridge->port_list, head) + if (br_port->dev == brport_dev) + return br_port; + + return NULL; +} + +static struct prestera_bridge_port * +prestera_bridge_port_find(struct prestera_switch *sw, + struct net_device *brport_dev) +{ + struct net_device *br_dev = netdev_master_upper_dev_get(brport_dev); + struct prestera_bridge *bridge; + + if (!br_dev) + return NULL; + + bridge = prestera_bridge_find(sw, br_dev); + if (!bridge) + return NULL; + + return __prestera_bridge_port_find(bridge, brport_dev); +} + +static void +prestera_br_port_flags_reset(struct prestera_bridge_port *br_port, + struct prestera_port *port) +{ + prestera_port_uc_flood_set(port, false); + prestera_port_mc_flood_set(port, false); + prestera_port_learning_set(port, false); + prestera_port_br_locked_set(port, false); +} + +static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port, + struct prestera_port *port) +{ + int err; + + err = prestera_port_uc_flood_set(port, br_port->flags & BR_FLOOD); + if (err) + goto err_out; + + err = prestera_port_mc_flood_set(port, br_port->flags & BR_MCAST_FLOOD); + if (err) + goto err_out; + + err = prestera_port_learning_set(port, br_port->flags & BR_LEARNING); + if (err) + goto err_out; + + err = prestera_port_br_locked_set(port, + br_port->flags & BR_PORT_LOCKED); + if (err) + goto err_out; + + return 0; + +err_out: + prestera_br_port_flags_reset(br_port, port); + return err; +} + static struct prestera_bridge_vlan * prestera_bridge_vlan_create(struct prestera_bridge_port *br_port, u16 vid) { @@ -220,6 +321,70 @@ static int prestera_fdb_flush_port(struct prestera_port *port, u32 mode) } static void +prestera_mdb_port_del(struct prestera_mdb_entry *mdb, + struct net_device *orig_dev) +{ + struct prestera_flood_domain *fl_domain = mdb->flood_domain; + struct prestera_flood_domain_port *flood_domain_port; + + flood_domain_port = prestera_flood_domain_port_find(fl_domain, + orig_dev, + mdb->vid); + if (flood_domain_port) + prestera_flood_domain_port_destroy(flood_domain_port); +} + +static void +prestera_br_mdb_entry_put(struct prestera_br_mdb_entry *br_mdb) +{ + struct prestera_bridge_port *br_port; + + if (list_empty(&br_mdb->br_mdb_port_list)) { + list_for_each_entry(br_port, &br_mdb->bridge->port_list, head) + prestera_mdb_port_del(br_mdb->mdb, br_port->dev); + + prestera_mdb_entry_destroy(br_mdb->mdb); + list_del(&br_mdb->br_mdb_entry_node); + kfree(br_mdb); + } +} + +static void +prestera_br_mdb_port_del(struct prestera_br_mdb_entry *br_mdb, + struct prestera_bridge_port *br_port) +{ + struct prestera_br_mdb_port *br_mdb_port, *tmp; + + list_for_each_entry_safe(br_mdb_port, tmp, &br_mdb->br_mdb_port_list, + br_mdb_port_node) { + if (br_mdb_port->br_port == br_port) { + list_del(&br_mdb_port->br_mdb_port_node); + kfree(br_mdb_port); + } + } +} + +static void +prestera_mdb_flush_bridge_port(struct prestera_bridge_port *br_port) +{ + struct prestera_br_mdb_port *br_mdb_port, *tmp_port; + struct prestera_br_mdb_entry *br_mdb, *br_mdb_tmp; + struct prestera_bridge *br_dev = br_port->bridge; + + list_for_each_entry_safe(br_mdb, br_mdb_tmp, &br_dev->br_mdb_entry_list, + br_mdb_entry_node) { + list_for_each_entry_safe(br_mdb_port, tmp_port, + &br_mdb->br_mdb_port_list, + br_mdb_port_node) { + prestera_mdb_port_del(br_mdb->mdb, + br_mdb_port->br_port->dev); + prestera_br_mdb_port_del(br_mdb, br_mdb_port->br_port); + } + prestera_br_mdb_entry_put(br_mdb); + } +} + +static void prestera_port_vlan_bridge_leave(struct prestera_port_vlan *port_vlan) { u32 fdb_flush_mode = PRESTERA_FDB_FLUSH_MODE_DYNAMIC; @@ -244,6 +409,8 @@ prestera_port_vlan_bridge_leave(struct prestera_port_vlan *port_vlan) else prestera_fdb_flush_port_vlan(port, vid, fdb_flush_mode); + prestera_mdb_flush_bridge_port(br_port); + list_del(&port_vlan->br_vlan_head); prestera_bridge_vlan_put(br_vlan); prestera_bridge_port_put(br_port); @@ -295,8 +462,10 @@ prestera_bridge_create(struct prestera_switchdev *swdev, struct net_device *dev) bridge->vlan_enabled = vlan_enabled; bridge->swdev = swdev; bridge->dev = dev; + bridge->multicast_enabled = br_multicast_enabled(dev); INIT_LIST_HEAD(&bridge->port_list); + INIT_LIST_HEAD(&bridge->br_mdb_entry_list); list_add(&bridge->head, &swdev->bridge_list); @@ -314,6 +483,7 @@ static void prestera_bridge_destroy(struct prestera_bridge *bridge) else prestera_hw_bridge_delete(swdev->sw, bridge->bridge_id); + WARN_ON(!list_empty(&bridge->br_mdb_entry_list)); WARN_ON(!list_empty(&bridge->port_list)); kfree(bridge); } @@ -405,6 +575,7 @@ prestera_bridge_port_create(struct prestera_bridge *bridge, INIT_LIST_HEAD(&br_port->vlan_list); list_add(&br_port->head, &bridge->port_list); + INIT_LIST_HEAD(&br_port->br_mdb_port_list); return br_port; } @@ -414,6 +585,7 @@ prestera_bridge_port_destroy(struct prestera_bridge_port *br_port) { list_del(&br_port->head); WARN_ON(!list_empty(&br_port->vlan_list)); + WARN_ON(!list_empty(&br_port->br_mdb_port_list)); kfree(br_port); } @@ -461,19 +633,13 @@ prestera_bridge_1d_port_join(struct prestera_bridge_port *br_port) if (err) return err; - err = prestera_hw_port_flood_set(port, BR_FLOOD | BR_MCAST_FLOOD, - br_port->flags); - if (err) - goto err_port_flood_set; - - err = prestera_hw_port_learning_set(port, br_port->flags & BR_LEARNING); + err = prestera_br_port_flags_set(br_port, port); if (err) - goto err_port_learning_set; + goto err_flags2port_set; return 0; -err_port_learning_set: -err_port_flood_set: +err_flags2port_set: prestera_hw_bridge_port_delete(port, bridge->bridge_id); return err; @@ -592,8 +758,9 @@ void prestera_bridge_port_leave(struct net_device *br_dev, switchdev_bridge_port_unoffload(br_port->dev, NULL, NULL, NULL); - prestera_hw_port_learning_set(port, false); - prestera_hw_port_flood_set(port, BR_FLOOD | BR_MCAST_FLOOD, 0); + prestera_mdb_flush_bridge_port(br_port); + + prestera_br_port_flags_reset(br_port, port); prestera_port_vid_stp_set(port, PRESTERA_VID_ALL, BR_STATE_FORWARDING); prestera_bridge_port_put(br_port); } @@ -603,26 +770,14 @@ static int prestera_port_attr_br_flags_set(struct prestera_port *port, struct switchdev_brport_flags flags) { struct prestera_bridge_port *br_port; - int err; br_port = prestera_bridge_port_by_dev(port->sw->swdev, dev); if (!br_port) return 0; - err = prestera_hw_port_flood_set(port, flags.mask, flags.val); - if (err) - return err; - - if (flags.mask & BR_LEARNING) { - err = prestera_hw_port_learning_set(port, - flags.val & BR_LEARNING); - if (err) - return err; - } - - memcpy(&br_port->flags, &flags.val, sizeof(flags.val)); - - return 0; + br_port->flags &= ~flags.mask; + br_port->flags |= flags.val & flags.mask; + return prestera_br_port_flags_set(br_port, port); } static int prestera_port_attr_br_ageing_set(struct prestera_port *port, @@ -716,6 +871,290 @@ err_port_stp_set: return err; } +static int +prestera_br_port_lag_mdb_mc_enable_sync(struct prestera_bridge_port *br_port, + bool enabled) +{ + struct prestera_port *pr_port; + struct prestera_switch *sw; + u16 lag_id; + int err; + + pr_port = prestera_port_dev_lower_find(br_port->dev); + if (!pr_port) + return 0; + + sw = pr_port->sw; + err = prestera_lag_id(sw, br_port->dev, &lag_id); + if (err) + return err; + + list_for_each_entry(pr_port, &sw->port_list, list) { + if (pr_port->lag->lag_id == lag_id) { + err = prestera_port_mc_flood_set(pr_port, enabled); + if (err) + return err; + } + } + + return 0; +} + +static int prestera_br_mdb_mc_enable_sync(struct prestera_bridge *br_dev) +{ + struct prestera_bridge_port *br_port; + struct prestera_port *port; + bool enabled; + int err; + + /* if mrouter exists: + * - make sure every mrouter receives unreg mcast traffic; + * if mrouter doesn't exists: + * - make sure every port receives unreg mcast traffic; + */ + list_for_each_entry(br_port, &br_dev->port_list, head) { + if (br_dev->multicast_enabled && br_dev->mrouter_exist) + enabled = br_port->mrouter; + else + enabled = br_port->flags & BR_MCAST_FLOOD; + + if (netif_is_lag_master(br_port->dev)) { + err = prestera_br_port_lag_mdb_mc_enable_sync(br_port, + enabled); + if (err) + return err; + continue; + } + + port = prestera_port_dev_lower_find(br_port->dev); + if (!port) + continue; + + err = prestera_port_mc_flood_set(port, enabled); + if (err) + return err; + } + + return 0; +} + +static bool +prestera_br_mdb_port_is_member(struct prestera_br_mdb_entry *br_mdb, + struct net_device *orig_dev) +{ + struct prestera_br_mdb_port *tmp_port; + + list_for_each_entry(tmp_port, &br_mdb->br_mdb_port_list, + br_mdb_port_node) + if (tmp_port->br_port->dev == orig_dev) + return true; + + return false; +} + +static int +prestera_mdb_port_add(struct prestera_mdb_entry *mdb, + struct net_device *orig_dev, + const unsigned char addr[ETH_ALEN], u16 vid) +{ + struct prestera_flood_domain *flood_domain = mdb->flood_domain; + int err; + + if (!prestera_flood_domain_port_find(flood_domain, + orig_dev, vid)) { + err = prestera_flood_domain_port_create(flood_domain, orig_dev, + vid); + if (err) + return err; + } + + return 0; +} + +/* Sync bridge mdb (software table) with HW table (if MC is enabled). */ +static int prestera_br_mdb_sync(struct prestera_bridge *br_dev) +{ + struct prestera_br_mdb_port *br_mdb_port; + struct prestera_bridge_port *br_port; + struct prestera_br_mdb_entry *br_mdb; + struct prestera_mdb_entry *mdb; + struct prestera_port *pr_port; + int err = 0; + + if (!br_dev->multicast_enabled) + return 0; + + list_for_each_entry(br_mdb, &br_dev->br_mdb_entry_list, + br_mdb_entry_node) { + mdb = br_mdb->mdb; + /* Make sure every port that explicitly been added to the mdb + * joins the specified group. + */ + list_for_each_entry(br_mdb_port, &br_mdb->br_mdb_port_list, + br_mdb_port_node) { + br_port = br_mdb_port->br_port; + pr_port = prestera_port_dev_lower_find(br_port->dev); + + /* Match only mdb and br_mdb ports that belong to the + * same broadcast domain. + */ + if (br_dev->vlan_enabled && + !prestera_port_vlan_by_vid(pr_port, + mdb->vid)) + continue; + + /* If port is not in MDB or there's no Mrouter + * clear HW mdb. + */ + if (prestera_br_mdb_port_is_member(br_mdb, + br_mdb_port->br_port->dev) && + br_dev->mrouter_exist) + err = prestera_mdb_port_add(mdb, br_port->dev, + mdb->addr, + mdb->vid); + else + prestera_mdb_port_del(mdb, br_port->dev); + + if (err) + return err; + } + + /* Make sure that every mrouter port joins every MC group int + * broadcast domain. If it's not an mrouter - it should leave + */ + list_for_each_entry(br_port, &br_dev->port_list, head) { + pr_port = prestera_port_dev_lower_find(br_port->dev); + + /* Make sure mrouter woudln't receive traffci from + * another broadcast domain (e.g. from a vlan, which + * mrouter port is not a member of). + */ + if (br_dev->vlan_enabled && + !prestera_port_vlan_by_vid(pr_port, + mdb->vid)) + continue; + + if (br_port->mrouter) { + err = prestera_mdb_port_add(mdb, br_port->dev, + mdb->addr, + mdb->vid); + if (err) + return err; + } else if (!br_port->mrouter && + !prestera_br_mdb_port_is_member + (br_mdb, br_port->dev)) { + prestera_mdb_port_del(mdb, br_port->dev); + } + } + } + + return 0; +} + +static int +prestera_mdb_enable_set(struct prestera_br_mdb_entry *br_mdb, bool enable) +{ + int err; + + if (enable != br_mdb->enabled) { + if (enable) + err = prestera_hw_mdb_create(br_mdb->mdb); + else + err = prestera_hw_mdb_destroy(br_mdb->mdb); + + if (err) + return err; + + br_mdb->enabled = enable; + } + + return 0; +} + +static int +prestera_br_mdb_enable_set(struct prestera_bridge *br_dev, bool enable) +{ + struct prestera_br_mdb_entry *br_mdb; + int err; + + list_for_each_entry(br_mdb, &br_dev->br_mdb_entry_list, + br_mdb_entry_node) { + err = prestera_mdb_enable_set(br_mdb, enable); + if (err) + return err; + } + + return 0; +} + +static int prestera_port_attr_br_mc_disabled_set(struct prestera_port *port, + struct net_device *orig_dev, + bool mc_disabled) +{ + struct prestera_switch *sw = port->sw; + struct prestera_bridge *br_dev; + + br_dev = prestera_bridge_find(sw, orig_dev); + if (!br_dev) + return 0; + + br_dev->multicast_enabled = !mc_disabled; + + /* There's no point in enabling mdb back if router is missing. */ + WARN_ON(prestera_br_mdb_enable_set(br_dev, br_dev->multicast_enabled && + br_dev->mrouter_exist)); + + WARN_ON(prestera_br_mdb_sync(br_dev)); + + WARN_ON(prestera_br_mdb_mc_enable_sync(br_dev)); + + return 0; +} + +static bool +prestera_bridge_mdb_mc_mrouter_exists(struct prestera_bridge *br_dev) +{ + struct prestera_bridge_port *br_port; + + list_for_each_entry(br_port, &br_dev->port_list, head) + if (br_port->mrouter) + return true; + + return false; +} + +static int +prestera_port_attr_mrouter_set(struct prestera_port *port, + struct net_device *orig_dev, + bool is_port_mrouter) +{ + struct prestera_bridge_port *br_port; + struct prestera_bridge *br_dev; + + br_port = prestera_bridge_port_find(port->sw, orig_dev); + if (!br_port) + return 0; + + br_dev = br_port->bridge; + br_port->mrouter = is_port_mrouter; + + br_dev->mrouter_exist = prestera_bridge_mdb_mc_mrouter_exists(br_dev); + + /* Enable MDB processing if both mrouter exists and mc is enabled. + * In case if MC enabled, but there is no mrouter, device would flood + * all multicast traffic (even if MDB table is not empty) with the use + * of bridge's flood capabilities (without the use of flood_domain). + */ + WARN_ON(prestera_br_mdb_enable_set(br_dev, br_dev->multicast_enabled && + br_dev->mrouter_exist)); + + WARN_ON(prestera_br_mdb_sync(br_dev)); + + WARN_ON(prestera_br_mdb_mc_enable_sync(br_dev)); + + return 0; +} + static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) @@ -730,7 +1169,7 @@ static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx, break; case SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS: if (attr->u.brport_flags.mask & - ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD)) + ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD | BR_PORT_LOCKED)) err = -EINVAL; break; case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS: @@ -745,6 +1184,14 @@ static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx, err = prestera_port_attr_br_vlan_set(port, attr->orig_dev, attr->u.vlan_filtering); break; + case SWITCHDEV_ATTR_ID_PORT_MROUTER: + err = prestera_port_attr_mrouter_set(port, attr->orig_dev, + attr->u.mrouter); + break; + case SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED: + err = prestera_port_attr_br_mc_disabled_set(port, attr->orig_dev, + attr->u.mc_disabled); + break; default: err = -EOPNOTSUPP; } @@ -918,14 +1365,9 @@ prestera_port_vlan_bridge_join(struct prestera_port_vlan *port_vlan, if (port_vlan->br_port) return 0; - err = prestera_hw_port_flood_set(port, BR_FLOOD | BR_MCAST_FLOOD, - br_port->flags); + err = prestera_br_port_flags_set(br_port, port); if (err) - return err; - - err = prestera_hw_port_learning_set(port, br_port->flags & BR_LEARNING); - if (err) - goto err_port_learning_set; + goto err_flags2port_set; err = prestera_port_vid_stp_set(port, vid, br_port->stp_state); if (err) @@ -950,8 +1392,8 @@ prestera_port_vlan_bridge_join(struct prestera_port_vlan *port_vlan, err_bridge_vlan_get: prestera_port_vid_stp_set(port, vid, BR_STATE_FORWARDING); err_port_vid_stp_set: - prestera_hw_port_learning_set(port, false); -err_port_learning_set: + prestera_br_port_flags_reset(br_port, port); +err_flags2port_set: return err; } @@ -1048,20 +1490,162 @@ static int prestera_port_vlans_add(struct prestera_port *port, flag_pvid, extack); } +static struct prestera_br_mdb_entry * +prestera_br_mdb_entry_create(struct prestera_switch *sw, + struct prestera_bridge *br_dev, + const unsigned char *addr, u16 vid) +{ + struct prestera_br_mdb_entry *br_mdb_entry; + struct prestera_mdb_entry *mdb_entry; + + br_mdb_entry = kzalloc(sizeof(*br_mdb_entry), GFP_KERNEL); + if (!br_mdb_entry) + return NULL; + + mdb_entry = prestera_mdb_entry_create(sw, addr, vid); + if (!mdb_entry) + goto err_mdb_alloc; + + br_mdb_entry->mdb = mdb_entry; + br_mdb_entry->bridge = br_dev; + br_mdb_entry->enabled = true; + INIT_LIST_HEAD(&br_mdb_entry->br_mdb_port_list); + + list_add(&br_mdb_entry->br_mdb_entry_node, &br_dev->br_mdb_entry_list); + + return br_mdb_entry; + +err_mdb_alloc: + kfree(br_mdb_entry); + return NULL; +} + +static int prestera_br_mdb_port_add(struct prestera_br_mdb_entry *br_mdb, + struct prestera_bridge_port *br_port) +{ + struct prestera_br_mdb_port *br_mdb_port; + + list_for_each_entry(br_mdb_port, &br_mdb->br_mdb_port_list, + br_mdb_port_node) + if (br_mdb_port->br_port == br_port) + return 0; + + br_mdb_port = kzalloc(sizeof(*br_mdb_port), GFP_KERNEL); + if (!br_mdb_port) + return -ENOMEM; + + br_mdb_port->br_port = br_port; + list_add(&br_mdb_port->br_mdb_port_node, + &br_mdb->br_mdb_port_list); + + return 0; +} + +static struct prestera_br_mdb_entry * +prestera_br_mdb_entry_find(struct prestera_bridge *br_dev, + const unsigned char *addr, u16 vid) +{ + struct prestera_br_mdb_entry *br_mdb; + + list_for_each_entry(br_mdb, &br_dev->br_mdb_entry_list, + br_mdb_entry_node) + if (ether_addr_equal(&br_mdb->mdb->addr[0], addr) && + vid == br_mdb->mdb->vid) + return br_mdb; + + return NULL; +} + +static struct prestera_br_mdb_entry * +prestera_br_mdb_entry_get(struct prestera_switch *sw, + struct prestera_bridge *br_dev, + const unsigned char *addr, u16 vid) +{ + struct prestera_br_mdb_entry *br_mdb; + + br_mdb = prestera_br_mdb_entry_find(br_dev, addr, vid); + if (br_mdb) + return br_mdb; + + return prestera_br_mdb_entry_create(sw, br_dev, addr, vid); +} + +static int +prestera_mdb_port_addr_obj_add(const struct switchdev_obj_port_mdb *mdb) +{ + struct prestera_br_mdb_entry *br_mdb; + struct prestera_bridge_port *br_port; + struct prestera_bridge *br_dev; + struct prestera_switch *sw; + struct prestera_port *port; + int err; + + sw = prestera_switch_get(mdb->obj.orig_dev); + port = prestera_port_dev_lower_find(mdb->obj.orig_dev); + + br_port = prestera_bridge_port_find(sw, mdb->obj.orig_dev); + if (!br_port) + return 0; + + br_dev = br_port->bridge; + + if (mdb->vid && !prestera_port_vlan_by_vid(port, mdb->vid)) + return 0; + + if (mdb->vid) + br_mdb = prestera_br_mdb_entry_get(sw, br_dev, &mdb->addr[0], + mdb->vid); + else + br_mdb = prestera_br_mdb_entry_get(sw, br_dev, &mdb->addr[0], + br_dev->bridge_id); + + if (!br_mdb) + return -ENOMEM; + + /* Make sure newly allocated MDB entry gets disabled if either MC is + * disabled, or the mrouter does not exist. + */ + WARN_ON(prestera_mdb_enable_set(br_mdb, br_dev->multicast_enabled && + br_dev->mrouter_exist)); + + err = prestera_br_mdb_port_add(br_mdb, br_port); + if (err) { + prestera_br_mdb_entry_put(br_mdb); + return err; + } + + err = prestera_br_mdb_sync(br_dev); + if (err) + return err; + + return 0; +} + static int prestera_port_obj_add(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) { struct prestera_port *port = netdev_priv(dev); const struct switchdev_obj_port_vlan *vlan; + const struct switchdev_obj_port_mdb *mdb; + int err = 0; switch (obj->id) { case SWITCHDEV_OBJ_ID_PORT_VLAN: vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); return prestera_port_vlans_add(port, vlan, extack); + case SWITCHDEV_OBJ_ID_PORT_MDB: + mdb = SWITCHDEV_OBJ_PORT_MDB(obj); + err = prestera_mdb_port_addr_obj_add(mdb); + break; + case SWITCHDEV_OBJ_ID_HOST_MDB: + fallthrough; default: - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + break; } + + return err; } static int prestera_port_vlans_del(struct prestera_port *port, @@ -1086,17 +1670,71 @@ static int prestera_port_vlans_del(struct prestera_port *port, return 0; } +static int +prestera_mdb_port_addr_obj_del(struct prestera_port *port, + const struct switchdev_obj_port_mdb *mdb) +{ + struct prestera_br_mdb_entry *br_mdb; + struct prestera_bridge_port *br_port; + struct prestera_bridge *br_dev; + int err; + + /* Bridge port no longer exists - and so does this MDB entry */ + br_port = prestera_bridge_port_find(port->sw, mdb->obj.orig_dev); + if (!br_port) + return 0; + + /* Removing MDB with non-existing VLAN - not supported; */ + if (mdb->vid && !prestera_port_vlan_by_vid(port, mdb->vid)) + return 0; + + br_dev = br_port->bridge; + + if (br_port->bridge->vlan_enabled) + br_mdb = prestera_br_mdb_entry_find(br_dev, &mdb->addr[0], + mdb->vid); + else + br_mdb = prestera_br_mdb_entry_find(br_dev, &mdb->addr[0], + br_port->bridge->bridge_id); + + if (!br_mdb) + return 0; + + /* Since there might be a situation that this port was the last in the + * MDB group, we have to both remove this port from software and HW MDB, + * sync MDB table, and then destroy software MDB (if needed). + */ + prestera_br_mdb_port_del(br_mdb, br_port); + + prestera_br_mdb_entry_put(br_mdb); + + err = prestera_br_mdb_sync(br_dev); + if (err) + return err; + + return 0; +} + static int prestera_port_obj_del(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj) { struct prestera_port *port = netdev_priv(dev); + const struct switchdev_obj_port_mdb *mdb; + int err = 0; switch (obj->id) { case SWITCHDEV_OBJ_ID_PORT_VLAN: return prestera_port_vlans_del(port, SWITCHDEV_OBJ_PORT_VLAN(obj)); + case SWITCHDEV_OBJ_ID_PORT_MDB: + mdb = SWITCHDEV_OBJ_PORT_MDB(obj); + err = prestera_mdb_port_addr_obj_del(port, mdb); + break; default: - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + break; } + + return err; } static int prestera_switchdev_blk_event(struct notifier_block *unused, diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c index 1d607bc6b59e..cf456d62677f 100644 --- a/drivers/net/ethernet/marvell/pxa168_eth.c +++ b/drivers/net/ethernet/marvell/pxa168_eth.c @@ -1354,10 +1354,10 @@ static void pxa168_eth_netpoll(struct net_device *dev) static void pxa168_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); - strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); - strlcpy(info->bus_info, "N/A", sizeof(info->bus_info)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); + strscpy(info->bus_info, "N/A", sizeof(info->bus_info)); } static const struct ethtool_ops pxa168_ethtool_ops = { @@ -1388,7 +1388,6 @@ static int pxa168_eth_probe(struct platform_device *pdev) { struct pxa168_eth_private *pep = NULL; struct net_device *dev = NULL; - struct resource *res; struct clk *clk; struct device_node *np; int err; @@ -1419,9 +1418,11 @@ static int pxa168_eth_probe(struct platform_device *pdev) goto err_netdev; } - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - BUG_ON(!res); - dev->irq = res->start; + err = platform_get_irq(pdev, 0); + if (err == -EPROBE_DEFER) + goto err_netdev; + BUG_ON(dev->irq < 0); + dev->irq = err; dev->netdev_ops = &pxa168_eth_netdev_ops; dev->watchdog_timeo = 2 * HZ; dev->base_addr = 0; @@ -1485,7 +1486,8 @@ static int pxa168_eth_probe(struct platform_device *pdev) /* Hardware supports only 3 ports */ BUG_ON(pep->port_num > 2); - netif_napi_add(dev, &pep->napi, pxa168_rx_poll, pep->rx_ring_size); + netif_napi_add_weight(dev, &pep->napi, pxa168_rx_poll, + pep->rx_ring_size); memset(&pep->timeout, 0, sizeof(struct timer_list)); timer_setup(&pep->timeout, rxq_refill_timer_wrapper, 0); diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 0c864e5bf0a6..1b43704baceb 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -50,7 +50,6 @@ #define PHY_RETRIES 1000 #define ETH_JUMBO_MTU 9000 #define TX_WATCHDOG (5 * HZ) -#define NAPI_WEIGHT 64 #define BLINK_MS 250 #define LINK_HZ HZ @@ -395,9 +394,9 @@ static void skge_get_drvinfo(struct net_device *dev, { struct skge_port *skge = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(skge->hw->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(skge->hw->pdev), sizeof(info->bus_info)); } @@ -492,7 +491,9 @@ static void skge_get_strings(struct net_device *dev, u32 stringset, u8 *data) } static void skge_get_ring_param(struct net_device *dev, - struct ethtool_ringparam *p) + struct ethtool_ringparam *p, + struct kernel_ethtool_ringparam *kernel_p, + struct netlink_ext_ack *extack) { struct skge_port *skge = netdev_priv(dev); @@ -504,7 +505,9 @@ static void skge_get_ring_param(struct net_device *dev, } static int skge_set_ring_param(struct net_device *dev, - struct ethtool_ringparam *p) + struct ethtool_ringparam *p, + struct kernel_ethtool_ringparam *kernel_p, + struct netlink_ext_ack *extack) { struct skge_port *skge = netdev_priv(dev); int err = 0; @@ -3829,7 +3832,7 @@ static struct net_device *skge_devinit(struct skge_hw *hw, int port, dev->features |= NETIF_F_HIGHDMA; skge = netdev_priv(dev); - netif_napi_add(dev, &skge->napi, skge_poll, NAPI_WEIGHT); + netif_napi_add(dev, &skge->napi, skge_poll); skge->netdev = dev; skge->hw = hw; skge->msg_enable = netif_msg_init(debug, default_msg); diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 28b5b9341145..ab33ba1c3023 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -63,7 +63,6 @@ #define TX_DEF_PENDING 63 #define TX_WATCHDOG (5 * HZ) -#define NAPI_WEIGHT 64 #define PHY_RETRIES 1000 #define SKY2_EEPROM_MAGIC 0x9955aabb @@ -1864,7 +1863,7 @@ static netdev_tx_t sky2_xmit_frame(struct sk_buff *skb, if (mss != 0) { if (!(hw->flags & SKY2_HW_NEW_LE)) - mss += ETH_HLEN + ip_hdrlen(skb) + tcp_hdrlen(skb); + mss += skb_tcp_all_headers(skb); if (mss != sky2->tx_last_mss) { le = get_tx_le(sky2, &slot); @@ -3688,9 +3687,9 @@ static void sky2_get_drvinfo(struct net_device *dev, { struct sky2_port *sky2 = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(sky2->hw->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(sky2->hw->pdev), sizeof(info->bus_info)); } @@ -4149,7 +4148,9 @@ static unsigned long roundup_ring_size(unsigned long pending) } static void sky2_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct sky2_port *sky2 = netdev_priv(dev); @@ -4161,7 +4162,9 @@ static void sky2_get_ringparam(struct net_device *dev, } static int sky2_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct sky2_port *sky2 = netdev_priv(dev); @@ -4266,96 +4269,36 @@ static int sky2_get_eeprom_len(struct net_device *dev) return 1 << ( ((reg2 & PCI_VPD_ROM_SZ) >> 14) + 8); } -static int sky2_vpd_wait(const struct sky2_hw *hw, int cap, u16 busy) -{ - unsigned long start = jiffies; - - while ( (sky2_pci_read16(hw, cap + PCI_VPD_ADDR) & PCI_VPD_ADDR_F) == busy) { - /* Can take up to 10.6 ms for write */ - if (time_after(jiffies, start + HZ/4)) { - dev_err(&hw->pdev->dev, "VPD cycle timed out\n"); - return -ETIMEDOUT; - } - msleep(1); - } - - return 0; -} - -static int sky2_vpd_read(struct sky2_hw *hw, int cap, void *data, - u16 offset, size_t length) -{ - int rc = 0; - - while (length > 0) { - u32 val; - - sky2_pci_write16(hw, cap + PCI_VPD_ADDR, offset); - rc = sky2_vpd_wait(hw, cap, 0); - if (rc) - break; - - val = sky2_pci_read32(hw, cap + PCI_VPD_DATA); - - memcpy(data, &val, min(sizeof(val), length)); - offset += sizeof(u32); - data += sizeof(u32); - length -= sizeof(u32); - } - - return rc; -} - -static int sky2_vpd_write(struct sky2_hw *hw, int cap, const void *data, - u16 offset, unsigned int length) -{ - unsigned int i; - int rc = 0; - - for (i = 0; i < length; i += sizeof(u32)) { - u32 val = *(u32 *)(data + i); - - sky2_pci_write32(hw, cap + PCI_VPD_DATA, val); - sky2_pci_write32(hw, cap + PCI_VPD_ADDR, offset | PCI_VPD_ADDR_F); - - rc = sky2_vpd_wait(hw, cap, PCI_VPD_ADDR_F); - if (rc) - break; - } - return rc; -} - static int sky2_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data) { struct sky2_port *sky2 = netdev_priv(dev); - int cap = pci_find_capability(sky2->hw->pdev, PCI_CAP_ID_VPD); - - if (!cap) - return -EINVAL; + int rc; eeprom->magic = SKY2_EEPROM_MAGIC; + rc = pci_read_vpd_any(sky2->hw->pdev, eeprom->offset, eeprom->len, + data); + if (rc < 0) + return rc; + + eeprom->len = rc; - return sky2_vpd_read(sky2->hw, cap, data, eeprom->offset, eeprom->len); + return 0; } static int sky2_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data) { struct sky2_port *sky2 = netdev_priv(dev); - int cap = pci_find_capability(sky2->hw->pdev, PCI_CAP_ID_VPD); - - if (!cap) - return -EINVAL; + int rc; if (eeprom->magic != SKY2_EEPROM_MAGIC) return -EINVAL; - /* Partial writes not supported */ - if ((eeprom->offset & 3) || (eeprom->len & 3)) - return -EINVAL; + rc = pci_write_vpd_any(sky2->hw->pdev, eeprom->offset, eeprom->len, + data); - return sky2_vpd_write(sky2->hw, cap, data, eeprom->offset, eeprom->len); + return rc < 0 ? rc : 0; } static netdev_features_t sky2_fix_features(struct net_device *dev, @@ -4768,7 +4711,7 @@ static irqreturn_t sky2_test_intr(int irq, void *dev_id) return IRQ_HANDLED; } -/* Test interrupt path by forcing a a software IRQ */ +/* Test interrupt path by forcing a software IRQ */ static int sky2_test_msi(struct sky2_hw *hw) { struct pci_dev *pdev = hw->pdev; @@ -4994,7 +4937,7 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - netif_napi_add(dev, &hw->napi, sky2_poll, NAPI_WEIGHT); + netif_napi_add(dev, &hw->napi, sky2_poll); err = register_netdev(dev); if (err) { |