diff options
Diffstat (limited to 'drivers/net/ethernet')
610 files changed, 51244 insertions, 10643 deletions
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c index 741c67e546d4..7d7d3ffe25c3 100644 --- a/drivers/net/ethernet/3com/3c59x.c +++ b/drivers/net/ethernet/3com/3c59x.c @@ -1464,7 +1464,7 @@ static int vortex_probe1(struct device *gendev, void __iomem *ioaddr, int irq, if (pdev) { vp->pm_state_valid = 1; pci_save_state(pdev); - acpi_set_WOL(dev); + acpi_set_WOL(dev); } retval = register_netdev(dev); if (retval == 0) diff --git a/drivers/net/ethernet/8390/axnet_cs.c b/drivers/net/ethernet/8390/axnet_cs.c index 2488bfdb9133..8c321dfc7b3b 100644 --- a/drivers/net/ethernet/8390/axnet_cs.c +++ b/drivers/net/ethernet/8390/axnet_cs.c @@ -767,7 +767,7 @@ module_pcmcia_driver(axnet_cs_driver); Paul Gortmaker : tweak ANK's above multicast changes a bit. Paul Gortmaker : update packet statistics for v2.1.x Alan Cox : support arbitrary stupid port mappings on the - 68K Macintosh. Support >16bit I/O spaces + 68K Macintosh. Support >16bit I/O spaces Paul Gortmaker : add kmod support for auto-loading of the 8390 module by all drivers that require it. Alan Cox : Spinlocking work, added 'BUG_83C690' @@ -1091,7 +1091,7 @@ static irqreturn_t ax_interrupt(int irq, void *dev_id) long e8390_base; int interrupts, nr_serviced = 0, i; struct ei_device *ei_local; - int handled = 0; + int handled = 0; unsigned long flags; e8390_base = dev->base_addr; @@ -1587,12 +1587,12 @@ static void do_set_multicast_list(struct net_device *dev) } outb_p(E8390_NODMA + E8390_PAGE0, e8390_base + E8390_CMD); - if(dev->flags&IFF_PROMISC) - outb_p(E8390_RXCONFIG | 0x58, e8390_base + EN0_RXCR); + if(dev->flags&IFF_PROMISC) + outb_p(E8390_RXCONFIG | 0x58, e8390_base + EN0_RXCR); else if (dev->flags & IFF_ALLMULTI || !netdev_mc_empty(dev)) - outb_p(E8390_RXCONFIG | 0x48, e8390_base + EN0_RXCR); - else - outb_p(E8390_RXCONFIG | 0x40, e8390_base + EN0_RXCR); + outb_p(E8390_RXCONFIG | 0x48, e8390_base + EN0_RXCR); + else + outb_p(E8390_RXCONFIG | 0x40, e8390_base + EN0_RXCR); outb_p(E8390_NODMA+E8390_PAGE0+E8390_START, e8390_base+E8390_CMD); } diff --git a/drivers/net/ethernet/8390/pcnet_cs.c b/drivers/net/ethernet/8390/pcnet_cs.c index 9d3b1e0e425c..cac036706382 100644 --- a/drivers/net/ethernet/8390/pcnet_cs.c +++ b/drivers/net/ethernet/8390/pcnet_cs.c @@ -1527,7 +1527,7 @@ static const struct pcmcia_device_id pcnet_ids[] = { PCMCIA_DEVICE_PROD_ID12("ACCTON", "EN2216-PCMCIA-ETHERNET", 0xdfc6b5b2, 0x5542bfff), PCMCIA_DEVICE_PROD_ID12("Allied Telesis, K.K.", "CentreCOM LA100-PCM-T V2 100/10M LAN PC Card", 0xbb7fbdd7, 0xcd91cc68), PCMCIA_DEVICE_PROD_ID12("Allied Telesis K.K.", "LA100-PCM V2", 0x36634a66, 0xc6d05997), - PCMCIA_DEVICE_PROD_ID12("Allied Telesis, K.K.", "CentreCOM LA-PCM_V2", 0xbb7fBdd7, 0x28e299f8), + PCMCIA_DEVICE_PROD_ID12("Allied Telesis, K.K.", "CentreCOM LA-PCM_V2", 0xbb7fBdd7, 0x28e299f8), PCMCIA_DEVICE_PROD_ID12("Allied Telesis K.K.", "LA-PCM V3", 0x36634a66, 0x62241d96), PCMCIA_DEVICE_PROD_ID12("AmbiCom", "AMB8010", 0x5070a7f9, 0x82f96e96), PCMCIA_DEVICE_PROD_ID12("AmbiCom", "AMB8610", 0x5070a7f9, 0x86741224), diff --git a/drivers/net/ethernet/8390/smc-ultra.c b/drivers/net/ethernet/8390/smc-ultra.c index 3fe3b4dfa7c5..1d8ed7357b7f 100644 --- a/drivers/net/ethernet/8390/smc-ultra.c +++ b/drivers/net/ethernet/8390/smc-ultra.c @@ -347,11 +347,11 @@ static int __init ultra_probe_isapnp(struct net_device *dev) idev))) { /* Avoid already found cards from previous calls */ if (pnp_device_attach(idev) < 0) - continue; + continue; if (pnp_activate_dev(idev) < 0) { __again: - pnp_device_detach(idev); - continue; + pnp_device_detach(idev); + continue; } /* if no io and irq, search for next */ if (!pnp_port_valid(idev, 0) || !pnp_irq_valid(idev, 0)) diff --git a/drivers/net/ethernet/8390/stnic.c b/drivers/net/ethernet/8390/stnic.c index 1f0670cd3ea3..fbbd7f22c142 100644 --- a/drivers/net/ethernet/8390/stnic.c +++ b/drivers/net/ethernet/8390/stnic.c @@ -114,7 +114,7 @@ static int __init stnic_probe(void) /* New style probing API */ dev = alloc_ei_netdev(); if (!dev) - return -ENOMEM; + return -ENOMEM; #ifdef CONFIG_SH_STANDARD_BIOS sh_bios_get_node_addr (stnic_eadr); diff --git a/drivers/net/ethernet/aeroflex/greth.c b/drivers/net/ethernet/aeroflex/greth.c index d77fafbc1530..c560ad06f0be 100644 --- a/drivers/net/ethernet/aeroflex/greth.c +++ b/drivers/net/ethernet/aeroflex/greth.c @@ -1539,10 +1539,11 @@ static int greth_of_remove(struct platform_device *of_dev) mdiobus_unregister(greth->mdio); unregister_netdev(ndev); - free_netdev(ndev); of_iounmap(&of_dev->resource[0], greth->regs, resource_size(&of_dev->resource[0])); + free_netdev(ndev); + return 0; } diff --git a/drivers/net/ethernet/alteon/acenic.c b/drivers/net/ethernet/alteon/acenic.c index 1a7e4df9b3e9..9dc12b13061f 100644 --- a/drivers/net/ethernet/alteon/acenic.c +++ b/drivers/net/ethernet/alteon/acenic.c @@ -1883,16 +1883,16 @@ static u32 ace_handle_event(struct net_device *dev, u32 evtcsm, u32 evtprd) } } - if (ACE_IS_TIGON_I(ap)) { - struct cmd cmd; - cmd.evt = C_SET_RX_JUMBO_PRD_IDX; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(ap->regs, &cmd); - } else { - writel(0, &((ap->regs)->RxJumboPrd)); - wmb(); - } + if (ACE_IS_TIGON_I(ap)) { + struct cmd cmd; + cmd.evt = C_SET_RX_JUMBO_PRD_IDX; + cmd.code = 0; + cmd.idx = 0; + ace_issue_cmd(ap->regs, &cmd); + } else { + writel(0, &((ap->regs)->RxJumboPrd)); + wmb(); + } ap->jumbo = 0; ap->rx_jumbo_skbprd = 0; @@ -2489,9 +2489,9 @@ restart: } } - wmb(); - ap->tx_prd = idx; - ace_set_txprd(regs, ap, idx); + wmb(); + ap->tx_prd = idx; + ace_set_txprd(regs, ap, idx); if (flagsize & BD_FLG_COAL_NOW) { netif_stop_queue(dev); diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h index 4164eacc5c28..f5ec35fa4c63 100644 --- a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h +++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h @@ -1042,8 +1042,6 @@ enum ena_admin_aenq_group { }; enum ena_admin_aenq_notification_syndrome { - ENA_ADMIN_SUSPEND = 0, - ENA_ADMIN_RESUME = 1, ENA_ADMIN_UPDATE_HINTS = 2, }; diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c index 764852ead1d6..ab413fc1f68e 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_com.c @@ -1979,7 +1979,8 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, if (rc) return rc; - if (get_resp.u.max_queue_ext.version != ENA_FEATURE_MAX_QUEUE_EXT_VER) + if (get_resp.u.max_queue_ext.version != + ENA_FEATURE_MAX_QUEUE_EXT_VER) return -EINVAL; memcpy(&get_feat_ctx->max_queue_ext, &get_resp.u.max_queue_ext, diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c index c3be751e7379..3d6f0a466a9e 100644 --- a/drivers/net/ethernet/amazon/ena/ena_eth_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c @@ -151,11 +151,14 @@ static int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq) return 0; /* bounce buffer was used, so write it and get a new one */ - if (pkt_ctrl->idx) { + if (likely(pkt_ctrl->idx)) { rc = ena_com_write_bounce_buffer_to_dev(io_sq, pkt_ctrl->curr_bounce_buf); - if (unlikely(rc)) + if (unlikely(rc)) { + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, + "Failed to write bounce buffer to device\n"); return rc; + } pkt_ctrl->curr_bounce_buf = ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); @@ -185,8 +188,11 @@ static int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq) if (!pkt_ctrl->descs_left_in_line) { rc = ena_com_write_bounce_buffer_to_dev(io_sq, pkt_ctrl->curr_bounce_buf); - if (unlikely(rc)) + if (unlikely(rc)) { + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, + "Failed to write bounce buffer to device\n"); return rc; + } pkt_ctrl->curr_bounce_buf = ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); @@ -406,8 +412,11 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, } if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && - !buffer_to_push)) + !buffer_to_push)) { + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, + "Push header wasn't provided in LLQ mode\n"); return -EINVAL; + } rc = ena_com_write_header_to_bounce(io_sq, buffer_to_push, header_len); if (unlikely(rc)) @@ -423,6 +432,9 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, /* If the caller doesn't want to send packets */ if (unlikely(!num_bufs && !header_len)) { rc = ena_com_close_bounce_buffer(io_sq); + if (rc) + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, + "Failed to write buffers to LLQ\n"); *nb_hw_desc = io_sq->tail - start_tail; return rc; } @@ -482,8 +494,11 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, /* The first desc share the same desc as the header */ if (likely(i != 0)) { rc = ena_com_sq_update_tail(io_sq); - if (unlikely(rc)) + if (unlikely(rc)) { + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, + "Failed to update sq tail\n"); return rc; + } desc = get_sq_desc(io_sq); if (unlikely(!desc)) @@ -512,8 +527,11 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, desc->len_ctrl |= ENA_ETH_IO_TX_DESC_LAST_MASK; rc = ena_com_sq_update_tail(io_sq); - if (unlikely(rc)) + if (unlikely(rc)) { + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, + "Failed to update sq tail of the last descriptor\n"); return rc; + } rc = ena_com_close_bounce_buffer(io_sq); diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index 2fe7ccee55b2..27dae632efcb 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -233,10 +233,13 @@ int ena_get_sset_count(struct net_device *netdev, int sset) { struct ena_adapter *adapter = netdev_priv(netdev); - if (sset != ETH_SS_STATS) - return -EOPNOTSUPP; + switch (sset) { + case ETH_SS_STATS: + return ena_get_sw_stats_count(adapter) + + ena_get_hw_stats_count(adapter); + } - return ena_get_sw_stats_count(adapter) + ena_get_hw_stats_count(adapter); + return -EOPNOTSUPP; } static void ena_queue_strings(struct ena_adapter *adapter, u8 **data) @@ -314,10 +317,11 @@ static void ena_get_ethtool_strings(struct net_device *netdev, { struct ena_adapter *adapter = netdev_priv(netdev); - if (sset != ETH_SS_STATS) - return; - - ena_get_strings(adapter, data, adapter->eni_stats_supported); + switch (sset) { + case ETH_SS_STATS: + ena_get_strings(adapter, data, adapter->eni_stats_supported); + break; + } } static int ena_get_link_ksettings(struct net_device *netdev, diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 881f88754bf6..0e43000614ab 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -35,9 +35,6 @@ MODULE_LICENSE("GPL"); #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \ NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR) -static int debug = -1; -module_param(debug, int, 0); -MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); static struct ena_aenq_handlers aenq_handlers; @@ -89,6 +86,12 @@ static void ena_increase_stat(u64 *statp, u64 cnt, u64_stats_update_end(syncp); } +static void ena_ring_tx_doorbell(struct ena_ring *tx_ring) +{ + ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); + ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp); +} + static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct ena_adapter *adapter = netdev_priv(dev); @@ -147,7 +150,7 @@ static int ena_xmit_common(struct net_device *dev, netif_dbg(adapter, tx_queued, dev, "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", ring->qid); - ena_com_write_sq_doorbell(ring->ena_com_io_sq); + ena_ring_tx_doorbell(ring); } /* prepare the packet's descriptors to dma engine */ @@ -197,7 +200,6 @@ static int ena_xdp_io_poll(struct napi_struct *napi, int budget) int ret; xdp_ring = ena_napi->xdp_ring; - xdp_ring->first_interrupt = ena_napi->first_interrupt; xdp_budget = budget; @@ -229,6 +231,7 @@ static int ena_xdp_io_poll(struct napi_struct *napi, int budget) xdp_ring->tx_stats.napi_comp += napi_comp_call; xdp_ring->tx_stats.tx_poll++; u64_stats_update_end(&xdp_ring->syncp); + xdp_ring->tx_stats.last_napi_jiffies = jiffies; return ret; } @@ -236,36 +239,48 @@ static int ena_xdp_io_poll(struct napi_struct *napi, int budget) static int ena_xdp_tx_map_frame(struct ena_ring *xdp_ring, struct ena_tx_buffer *tx_info, struct xdp_frame *xdpf, - void **push_hdr, - u32 *push_len) + struct ena_com_tx_ctx *ena_tx_ctx) { struct ena_adapter *adapter = xdp_ring->adapter; struct ena_com_buf *ena_buf; - dma_addr_t dma = 0; + int push_len = 0; + dma_addr_t dma; + void *data; u32 size; tx_info->xdpf = xdpf; + data = tx_info->xdpf->data; size = tx_info->xdpf->len; - ena_buf = tx_info->bufs; - /* llq push buffer */ - *push_len = min_t(u32, size, xdp_ring->tx_max_header_size); - *push_hdr = tx_info->xdpf->data; + if (xdp_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { + /* Designate part of the packet for LLQ */ + push_len = min_t(u32, size, xdp_ring->tx_max_header_size); + + ena_tx_ctx->push_header = data; - if (size - *push_len > 0) { + size -= push_len; + data += push_len; + } + + ena_tx_ctx->header_len = push_len; + + if (size > 0) { dma = dma_map_single(xdp_ring->dev, - *push_hdr + *push_len, - size - *push_len, + data, + size, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(xdp_ring->dev, dma))) goto error_report_dma_error; - tx_info->map_linear_data = 1; - tx_info->num_of_bufs = 1; - } + tx_info->map_linear_data = 0; - ena_buf->paddr = dma; - ena_buf->len = size; + ena_buf = tx_info->bufs; + ena_buf->paddr = dma; + ena_buf->len = size; + + ena_tx_ctx->ena_bufs = ena_buf; + ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1; + } return 0; @@ -274,10 +289,6 @@ error_report_dma_error: &xdp_ring->syncp); netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n"); - xdp_return_frame_rx_napi(tx_info->xdpf); - tx_info->xdpf = NULL; - tx_info->num_of_bufs = 0; - return -EINVAL; } @@ -289,8 +300,6 @@ static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring, struct ena_com_tx_ctx ena_tx_ctx = {}; struct ena_tx_buffer *tx_info; u16 next_to_use, req_id; - void *push_hdr; - u32 push_len; int rc; next_to_use = xdp_ring->next_to_use; @@ -298,15 +307,11 @@ static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring, tx_info = &xdp_ring->tx_buffer_info[req_id]; tx_info->num_of_bufs = 0; - rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &push_hdr, &push_len); + rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &ena_tx_ctx); if (unlikely(rc)) return rc; - ena_tx_ctx.ena_bufs = tx_info->bufs; - ena_tx_ctx.push_header = push_hdr; - ena_tx_ctx.num_bufs = tx_info->num_of_bufs; ena_tx_ctx.req_id = req_id; - ena_tx_ctx.header_len = push_len; rc = ena_xmit_common(dev, xdp_ring, @@ -316,14 +321,12 @@ static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring, xdpf->len); if (rc) goto error_unmap_dma; - /* trigger the dma engine. ena_com_write_sq_doorbell() - * has a mb + + /* trigger the dma engine. ena_ring_tx_doorbell() + * calls a memory barrier inside it. */ - if (flags & XDP_XMIT_FLUSH) { - ena_com_write_sq_doorbell(xdp_ring->ena_com_io_sq); - ena_increase_stat(&xdp_ring->tx_stats.doorbells, 1, - &xdp_ring->syncp); - } + if (flags & XDP_XMIT_FLUSH) + ena_ring_tx_doorbell(xdp_ring); return rc; @@ -364,11 +367,8 @@ static int ena_xdp_xmit(struct net_device *dev, int n, } /* Ring doorbell to make device aware of the packets */ - if (flags & XDP_XMIT_FLUSH) { - ena_com_write_sq_doorbell(xdp_ring->ena_com_io_sq); - ena_increase_stat(&xdp_ring->tx_stats.doorbells, 1, - &xdp_ring->syncp); - } + if (flags & XDP_XMIT_FLUSH) + ena_ring_tx_doorbell(xdp_ring); spin_unlock(&xdp_ring->xdp_tx_lock); @@ -383,9 +383,7 @@ static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) u32 verdict = XDP_PASS; struct xdp_frame *xdpf; u64 *xdp_stat; - int qid; - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog); if (!xdp_prog) @@ -404,8 +402,7 @@ static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) } /* Find xmit queue */ - qid = rx_ring->qid + rx_ring->adapter->num_io_queues; - xdp_ring = &rx_ring->adapter->tx_ring[qid]; + xdp_ring = rx_ring->xdp_ring; /* The XDP queues are shared between XDP_TX and XDP_REDIRECT */ spin_lock(&xdp_ring->xdp_tx_lock); @@ -443,8 +440,6 @@ static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp) ena_increase_stat(xdp_stat, 1, &rx_ring->syncp); out: - rcu_read_unlock(); - return verdict; } @@ -532,7 +527,7 @@ static void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter, rx_ring->rx_headroom = XDP_PACKET_HEADROOM; } else { ena_xdp_unregister_rxq_info(rx_ring); - rx_ring->rx_headroom = 0; + rx_ring->rx_headroom = NET_SKB_PAD; } } } @@ -681,7 +676,6 @@ static void ena_init_io_rings_common(struct ena_adapter *adapter, ring->ena_dev = adapter->ena_dev; ring->per_napi_packets = 0; ring->cpu = 0; - ring->first_interrupt = false; ring->no_interrupt_event_cnt = 0; u64_stats_init(&ring->syncp); } @@ -724,7 +718,9 @@ static void ena_init_io_rings(struct ena_adapter *adapter, rxr->smoothed_interval = ena_com_get_nonadaptive_moderation_interval_rx(ena_dev); rxr->empty_rx_queue = 0; + rxr->rx_headroom = NET_SKB_PAD; adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues]; } } } @@ -978,47 +974,66 @@ static void ena_free_all_io_rx_resources(struct ena_adapter *adapter) ena_free_rx_resources(adapter, i); } -static int ena_alloc_rx_page(struct ena_ring *rx_ring, - struct ena_rx_buffer *rx_info, gfp_t gfp) +static struct page *ena_alloc_map_page(struct ena_ring *rx_ring, + dma_addr_t *dma) { - int headroom = rx_ring->rx_headroom; - struct ena_com_buf *ena_buf; struct page *page; - dma_addr_t dma; - /* restore page offset value in case it has been changed by device */ - rx_info->page_offset = headroom; - - /* if previous allocated page is not used */ - if (unlikely(rx_info->page)) - return 0; - - page = alloc_page(gfp); - if (unlikely(!page)) { + /* This would allocate the page on the same NUMA node the executing code + * is running on. + */ + page = dev_alloc_page(); + if (!page) { ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, &rx_ring->syncp); - return -ENOMEM; + return ERR_PTR(-ENOSPC); } /* To enable NIC-side port-mirroring, AKA SPAN port, * we make the buffer readable from the nic as well */ - dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE, - DMA_BIDIRECTIONAL); - if (unlikely(dma_mapping_error(rx_ring->dev, dma))) { + *dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) { ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1, &rx_ring->syncp); - __free_page(page); - return -EIO; + return ERR_PTR(-EIO); } + + return page; +} + +static int ena_alloc_rx_buffer(struct ena_ring *rx_ring, + struct ena_rx_buffer *rx_info) +{ + int headroom = rx_ring->rx_headroom; + struct ena_com_buf *ena_buf; + struct page *page; + dma_addr_t dma; + int tailroom; + + /* restore page offset value in case it has been changed by device */ + rx_info->page_offset = headroom; + + /* if previous allocated page is not used */ + if (unlikely(rx_info->page)) + return 0; + + /* We handle DMA here */ + page = ena_alloc_map_page(rx_ring, &dma); + if (unlikely(IS_ERR(page))) + return PTR_ERR(page); + netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, "Allocate page %p, rx_info %p\n", page, rx_info); + tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + rx_info->page = page; ena_buf = &rx_info->ena_buf; ena_buf->paddr = dma + headroom; - ena_buf->len = ENA_PAGE_SIZE - headroom; + ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom; return 0; } @@ -1065,8 +1080,7 @@ static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num) rx_info = &rx_ring->rx_buffer_info[req_id]; - rc = ena_alloc_rx_page(rx_ring, rx_info, - GFP_ATOMIC | __GFP_COMP); + rc = ena_alloc_rx_buffer(rx_ring, rx_info); if (unlikely(rc < 0)) { netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev, "Failed to allocate buffer for rx queue %d\n", @@ -1384,21 +1398,23 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) return tx_pkts; } -static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, bool frags) +static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag) { struct sk_buff *skb; - if (frags) - skb = napi_get_frags(rx_ring->napi); - else + if (!first_frag) skb = netdev_alloc_skb_ip_align(rx_ring->netdev, rx_ring->rx_copybreak); + else + skb = build_skb(first_frag, ENA_PAGE_SIZE); if (unlikely(!skb)) { ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1, &rx_ring->syncp); + netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev, - "Failed to allocate skb. frags: %d\n", frags); + "Failed to allocate skb. first_frag %s\n", + first_frag ? "provided" : "not provided"); return NULL; } @@ -1410,10 +1426,12 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, u32 descs, u16 *next_to_clean) { - struct sk_buff *skb; struct ena_rx_buffer *rx_info; u16 len, req_id, buf = 0; - void *va; + struct sk_buff *skb; + void *page_addr; + u32 page_offset; + void *data_addr; len = ena_bufs[buf].len; req_id = ena_bufs[buf].req_id; @@ -1431,12 +1449,14 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, rx_info, rx_info->page); /* save virt address of first buffer */ - va = page_address(rx_info->page) + rx_info->page_offset; + page_addr = page_address(rx_info->page); + page_offset = rx_info->page_offset; + data_addr = page_addr + page_offset; - prefetch(va); + prefetch(data_addr); if (len <= rx_ring->rx_copybreak) { - skb = ena_alloc_skb(rx_ring, false); + skb = ena_alloc_skb(rx_ring, NULL); if (unlikely(!skb)) return NULL; @@ -1449,7 +1469,7 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, dma_unmap_addr(&rx_info->ena_buf, paddr), len, DMA_FROM_DEVICE); - skb_copy_to_linear_data(skb, va, len); + skb_copy_to_linear_data(skb, data_addr, len); dma_sync_single_for_device(rx_ring->dev, dma_unmap_addr(&rx_info->ena_buf, paddr), len, @@ -1463,16 +1483,18 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, return skb; } - skb = ena_alloc_skb(rx_ring, true); + ena_unmap_rx_buff(rx_ring, rx_info); + + skb = ena_alloc_skb(rx_ring, page_addr); if (unlikely(!skb)) return NULL; - do { - ena_unmap_rx_buff(rx_ring, rx_info); - - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page, - rx_info->page_offset, len, ENA_PAGE_SIZE); + /* Populate skb's linear part */ + skb_reserve(skb, page_offset); + skb_put(skb, len); + skb->protocol = eth_type_trans(skb, rx_ring->netdev); + do { netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, "RX skb updated. len %d. data_len %d\n", skb->len, skb->data_len); @@ -1491,6 +1513,12 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, req_id = ena_bufs[buf].req_id; rx_info = &rx_ring->rx_buffer_info[req_id]; + + ena_unmap_rx_buff(rx_ring, rx_info); + + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page, + rx_info->page_offset, len, ENA_PAGE_SIZE); + } while (1); return skb; @@ -1703,14 +1731,12 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, skb_record_rx_queue(skb, rx_ring->qid); - if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) { - total_len += rx_ring->ena_bufs[0].len; + if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak) rx_copybreak_pkt++; - napi_gro_receive(napi, skb); - } else { - total_len += skb->len; - napi_gro_frags(napi); - } + + total_len += skb->len; + + napi_gro_receive(napi, skb); res_budget--; } while (likely(res_budget)); @@ -1922,9 +1948,6 @@ static int ena_io_poll(struct napi_struct *napi, int budget) tx_ring = ena_napi->tx_ring; rx_ring = ena_napi->rx_ring; - tx_ring->first_interrupt = ena_napi->first_interrupt; - rx_ring->first_interrupt = ena_napi->first_interrupt; - tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER; if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || @@ -1979,6 +2002,8 @@ static int ena_io_poll(struct napi_struct *napi, int budget) tx_ring->tx_stats.tx_poll++; u64_stats_update_end(&tx_ring->syncp); + tx_ring->tx_stats.last_napi_jiffies = jiffies; + return ret; } @@ -2003,7 +2028,8 @@ static irqreturn_t ena_intr_msix_io(int irq, void *data) { struct ena_napi *ena_napi = data; - ena_napi->first_interrupt = true; + /* Used to check HW health */ + WRITE_ONCE(ena_napi->first_interrupt, true); WRITE_ONCE(ena_napi->interrupts_masked, true); smp_wmb(); /* write interrupts_masked before calling napi */ @@ -3089,14 +3115,11 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev) } } - if (netif_xmit_stopped(txq) || !netdev_xmit_more()) { - /* trigger the dma engine. ena_com_write_sq_doorbell() - * has a mb + if (netif_xmit_stopped(txq) || !netdev_xmit_more()) + /* trigger the dma engine. ena_ring_tx_doorbell() + * calls a memory barrier inside it. */ - ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); - ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, - &tx_ring->syncp); - } + ena_ring_tx_doorbell(tx_ring); return NETDEV_TX_OK; @@ -3346,7 +3369,7 @@ static int ena_set_queues_placement_policy(struct pci_dev *pdev, llq_feature_mask = 1 << ENA_ADMIN_LLQ; if (!(ena_dev->supported_features & llq_feature_mask)) { - dev_err(&pdev->dev, + dev_warn(&pdev->dev, "LLQ is not supported Fallback to host mode policy.\n"); ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; return 0; @@ -3657,7 +3680,9 @@ static void ena_fw_reset_device(struct work_struct *work) static int check_for_rx_interrupt_queue(struct ena_adapter *adapter, struct ena_ring *rx_ring) { - if (likely(rx_ring->first_interrupt)) + struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi); + + if (likely(READ_ONCE(ena_napi->first_interrupt))) return 0; if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) @@ -3681,6 +3706,10 @@ static int check_for_rx_interrupt_queue(struct ena_adapter *adapter, static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, struct ena_ring *tx_ring) { + struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi); + unsigned int time_since_last_napi; + unsigned int missing_tx_comp_to; + bool is_tx_comp_time_expired; struct ena_tx_buffer *tx_buf; unsigned long last_jiffies; u32 missed_tx = 0; @@ -3694,8 +3723,10 @@ static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, /* no pending Tx at this location */ continue; - if (unlikely(!tx_ring->first_interrupt && time_is_before_jiffies(last_jiffies + - 2 * adapter->missing_tx_completion_to))) { + is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies + + 2 * adapter->missing_tx_completion_to); + + if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) { /* If after graceful period interrupt is still not * received, we schedule a reset */ @@ -3708,12 +3739,17 @@ static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, return -EIO; } - if (unlikely(time_is_before_jiffies(last_jiffies + - adapter->missing_tx_completion_to))) { - if (!tx_buf->print_once) + is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies + + adapter->missing_tx_completion_to); + + if (unlikely(is_tx_comp_time_expired)) { + if (!tx_buf->print_once) { + time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies); + missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to); netif_notice(adapter, tx_err, adapter->netdev, - "Found a Tx that wasn't completed on time, qid %d, index %d.\n", - tx_ring->qid, i); + "Found a Tx that wasn't completed on time, qid %d, index %d. %u usecs have passed since last napi execution. Missing Tx timeout value %u msecs\n", + tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to); + } tx_buf->print_once = 1; missed_tx++; @@ -4244,7 +4280,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->ena_dev = ena_dev; adapter->netdev = netdev; adapter->pdev = pdev; - adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); + adapter->msg_enable = DEFAULT_MSG_ENABLE; ena_dev->net_device = netdev; diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h index 74af15d62ee1..0c39fc2fa345 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.h +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h @@ -55,12 +55,6 @@ #define ENA_TX_WAKEUP_THRESH (MAX_SKB_FRAGS + 2) #define ENA_DEFAULT_RX_COPYBREAK (256 - NET_IP_ALIGN) -/* limit the buffer size to 600 bytes to handle MTU changes from very - * small to very large, in which case the number of buffers per packet - * could exceed ENA_PKT_MAX_BUFS - */ -#define ENA_DEFAULT_MIN_RX_BUFF_ALLOC_SIZE 600 - #define ENA_MIN_MTU 128 #define ENA_NAME_MAX_LEN 20 @@ -135,12 +129,12 @@ struct ena_irq { }; struct ena_napi { - struct napi_struct napi ____cacheline_aligned; + u8 first_interrupt ____cacheline_aligned; + u8 interrupts_masked; + struct napi_struct napi; struct ena_ring *tx_ring; struct ena_ring *rx_ring; struct ena_ring *xdp_ring; - bool first_interrupt; - bool interrupts_masked; u32 qid; struct dim dim; }; @@ -212,6 +206,7 @@ struct ena_stats_tx { u64 llq_buffer_copy; u64 missed_tx; u64 unmask_interrupt; + u64 last_napi_jiffies; }; struct ena_stats_rx { @@ -259,6 +254,10 @@ struct ena_ring { struct bpf_prog *xdp_bpf_prog; struct xdp_rxq_info xdp_rxq; spinlock_t xdp_tx_lock; /* synchronize XDP TX/Redirect traffic */ + /* Used for rx queues only to point to the xdp tx ring, to + * which traffic should be redirected from this rx ring. + */ + struct ena_ring *xdp_ring; u16 next_to_use; u16 next_to_clean; @@ -271,7 +270,6 @@ struct ena_ring { /* The maximum header length the device can handle */ u8 tx_max_header_size; - bool first_interrupt; bool disable_meta_caching; u16 no_interrupt_event_cnt; @@ -414,11 +412,6 @@ enum ena_xdp_errors_t { ENA_XDP_NO_ENOUGH_QUEUES, }; -static inline bool ena_xdp_queues_present(struct ena_adapter *adapter) -{ - return adapter->xdp_first_ring != 0; -} - static inline bool ena_xdp_present(struct ena_adapter *adapter) { return !!adapter->xdp_bpf_prog; diff --git a/drivers/net/ethernet/amd/amd8111e.c b/drivers/net/ethernet/amd/amd8111e.c index 4a1220cc6f10..9cac5aa75a73 100644 --- a/drivers/net/ethernet/amd/amd8111e.c +++ b/drivers/net/ethernet/amd/amd8111e.c @@ -19,14 +19,14 @@ Module Name: Abstract: - AMD8111 based 10/100 Ethernet Controller Driver. + AMD8111 based 10/100 Ethernet Controller Driver. Environment: Kernel Mode Revision History: - 3.0.0 + 3.0.0 Initial Revision. 3.0.1 1. Dynamic interrupt coalescing. diff --git a/drivers/net/ethernet/amd/amd8111e.h b/drivers/net/ethernet/amd/amd8111e.h index 493f154eccf4..37da79da5f5e 100644 --- a/drivers/net/ethernet/amd/amd8111e.h +++ b/drivers/net/ethernet/amd/amd8111e.h @@ -10,14 +10,14 @@ Module Name: Abstract: - AMD8111 based 10/100 Ethernet Controller driver definitions. + AMD8111 based 10/100 Ethernet Controller driver definitions. Environment: Kernel Mode Revision History: - 3.0.0 + 3.0.0 Initial Revision. 3.0.1 */ @@ -692,7 +692,7 @@ enum coal_type{ }; enum coal_mode{ - RX_INTR_COAL, + RX_INTR_COAL, TX_INTR_COAL, DISABLE_COAL, ENABLE_COAL, diff --git a/drivers/net/ethernet/amd/atarilance.c b/drivers/net/ethernet/amd/atarilance.c index c1eab916438f..36f54d13a2eb 100644 --- a/drivers/net/ethernet/amd/atarilance.c +++ b/drivers/net/ethernet/amd/atarilance.c @@ -706,7 +706,7 @@ static void lance_init_ring( struct net_device *dev ) CHECK_OFFSET(offset); MEM->tx_head[i].base = offset; MEM->tx_head[i].flag = TMD1_OWN_HOST; - MEM->tx_head[i].base_hi = 0; + MEM->tx_head[i].base_hi = 0; MEM->tx_head[i].length = 0; MEM->tx_head[i].misc = 0; offset += PKT_BUF_SZ; diff --git a/drivers/net/ethernet/amd/declance.c b/drivers/net/ethernet/amd/declance.c index 7282ce55ffb8..493b0cefcc2a 100644 --- a/drivers/net/ethernet/amd/declance.c +++ b/drivers/net/ethernet/amd/declance.c @@ -937,7 +937,7 @@ static netdev_tx_t lance_start_xmit(struct sk_buff *skb, struct net_device *dev) dev_kfree_skb(skb); - return NETDEV_TX_OK; + return NETDEV_TX_OK; } static void lance_load_multicast(struct net_device *dev) diff --git a/drivers/net/ethernet/amd/lance.c b/drivers/net/ethernet/amd/lance.c index aff44241988c..2178e6b89dbd 100644 --- a/drivers/net/ethernet/amd/lance.c +++ b/drivers/net/ethernet/amd/lance.c @@ -780,7 +780,7 @@ lance_open(struct net_device *dev) outw(0x0002, ioaddr+LANCE_ADDR); /* Only touch autoselect bit. */ outw(inw(ioaddr+LANCE_BUS_IF) | 0x0002, ioaddr+LANCE_BUS_IF); - } + } if (lance_debug > 1) printk("%s: lance_open() irq %d dma %d tx/rx rings %#x/%#x init %#x.\n", @@ -812,7 +812,7 @@ lance_open(struct net_device *dev) * We used to clear the InitDone bit, 0x0100, here but Mark Stockton * reports that doing so triggers a bug in the '974. */ - outw(0x0042, ioaddr+LANCE_DATA); + outw(0x0042, ioaddr+LANCE_DATA); if (lance_debug > 2) printk("%s: LANCE open after %d ticks, init block %#x csr0 %4.4x.\n", diff --git a/drivers/net/ethernet/amd/ni65.c b/drivers/net/ethernet/amd/ni65.c index c38edf6f03a3..5c1cfb0c4a42 100644 --- a/drivers/net/ethernet/amd/ni65.c +++ b/drivers/net/ethernet/amd/ni65.c @@ -193,7 +193,7 @@ static struct card { .vendor_id = ni_vendor, .cardname = "ni6510", .config = 0x1, - }, + }, { .id0 = NI65_EB_ID0, .id1 = NI65_EB_ID1, @@ -204,7 +204,7 @@ static struct card { .vendor_id = ni_vendor, .cardname = "ni6510 EtherBlaster", .config = 0x2, - }, + }, { .id0 = NE2100_ID0, .id1 = NE2100_ID1, @@ -1232,15 +1232,15 @@ MODULE_PARM_DESC(dma, "ni6510 ISA DMA channel (ignored for some cards)"); int __init init_module(void) { - dev_ni65 = ni65_probe(-1); + dev_ni65 = ni65_probe(-1); return PTR_ERR_OR_ZERO(dev_ni65); } void __exit cleanup_module(void) { - unregister_netdev(dev_ni65); - cleanup_card(dev_ni65); - free_netdev(dev_ni65); + unregister_netdev(dev_ni65); + cleanup_card(dev_ni65); + free_netdev(dev_ni65); } #endif /* MODULE */ diff --git a/drivers/net/ethernet/amd/nmclan_cs.c b/drivers/net/ethernet/amd/nmclan_cs.c index 11c0b13edd30..4019cab87505 100644 --- a/drivers/net/ethernet/amd/nmclan_cs.c +++ b/drivers/net/ethernet/amd/nmclan_cs.c @@ -541,7 +541,7 @@ static int mace_init(mace_private *lp, unsigned int ioaddr, char *enet_addr) if(++ct > 500) { pr_err("reset failed, card removed?\n"); - return -1; + return -1; } udelay(1); } @@ -585,11 +585,11 @@ static int mace_init(mace_private *lp, unsigned int ioaddr, char *enet_addr) ct = 0; while (mace_read(lp, ioaddr, MACE_IAC) & MACE_IAC_ADDRCHG) { - if(++ ct > 500) - { + if(++ ct > 500) + { pr_err("ADDRCHG timeout, card removed?\n"); - return -1; - } + return -1; + } } /* Set PADR register */ for (i = 0; i < ETH_ALEN; i++) @@ -655,7 +655,7 @@ static int nmclan_config(struct pcmcia_device *link) } if(mace_init(lp, ioaddr, dev->dev_addr) == -1) - goto failed; + goto failed; /* The if_port symbol can be set when the module is loaded */ if (if_port <= 2) diff --git a/drivers/net/ethernet/amd/sun3lance.c b/drivers/net/ethernet/amd/sun3lance.c index 00ae1081254d..f8d7a9387a56 100644 --- a/drivers/net/ethernet/amd/sun3lance.c +++ b/drivers/net/ethernet/amd/sun3lance.c @@ -150,7 +150,7 @@ struct lance_memory { struct lance_private { volatile unsigned short *iobase; struct lance_memory *mem; - int new_rx, new_tx; /* The next free ring entry */ + int new_rx, new_tx; /* The next free ring entry */ int old_tx, old_rx; /* ring entry to be processed */ /* These two must be longs for set_bit() */ long tx_full; @@ -465,7 +465,7 @@ static void lance_init_ring( struct net_device *dev ) for( i = 0; i < TX_RING_SIZE; i++ ) { MEM->tx_head[i].base = dvma_vtob(MEM->tx_data[i]); MEM->tx_head[i].flag = 0; - MEM->tx_head[i].base_hi = + MEM->tx_head[i].base_hi = (dvma_vtob(MEM->tx_data[i])) >>16; MEM->tx_head[i].length = 0; MEM->tx_head[i].misc = 0; @@ -581,8 +581,8 @@ lance_start_xmit(struct sk_buff *skb, struct net_device *dev) } AREG = CSR0; - DPRINTK( 2, ( "%s: lance_start_xmit() called, csr0 %4.4x.\n", - dev->name, DREG )); + DPRINTK( 2, ( "%s: lance_start_xmit() called, csr0 %4.4x.\n", + dev->name, DREG )); #ifdef CONFIG_SUN3X /* this weirdness doesn't appear on sun3... */ @@ -636,8 +636,8 @@ lance_start_xmit(struct sk_buff *skb, struct net_device *dev) /* Trigger an immediate send poll. */ REGA(CSR0) = CSR0_INEA | CSR0_TDMD | CSR0_STRT; AREG = CSR0; - DPRINTK( 2, ( "%s: lance_start_xmit() exiting, csr0 %4.4x.\n", - dev->name, DREG )); + DPRINTK( 2, ( "%s: lance_start_xmit() exiting, csr0 %4.4x.\n", + dev->name, DREG )); dev_kfree_skb(skb); lp->lock = 0; diff --git a/drivers/net/ethernet/apple/bmac.c b/drivers/net/ethernet/apple/bmac.c index 1e4e402f07d7..a989d2df59ad 100644 --- a/drivers/net/ethernet/apple/bmac.c +++ b/drivers/net/ethernet/apple/bmac.c @@ -477,26 +477,26 @@ static int bmac_suspend(struct macio_dev *mdev, pm_message_t state) config = bmread(dev, RXCFG); bmwrite(dev, RXCFG, (config & ~RxMACEnable)); config = bmread(dev, TXCFG); - bmwrite(dev, TXCFG, (config & ~TxMACEnable)); + bmwrite(dev, TXCFG, (config & ~TxMACEnable)); bmwrite(dev, INTDISABLE, DisableAll); /* disable all intrs */ - /* disable rx and tx dma */ + /* disable rx and tx dma */ rd->control = cpu_to_le32(DBDMA_CLEAR(RUN|PAUSE|FLUSH|WAKE)); /* clear run bit */ td->control = cpu_to_le32(DBDMA_CLEAR(RUN|PAUSE|FLUSH|WAKE)); /* clear run bit */ - /* free some skb's */ - for (i=0; i<N_RX_RING; i++) { - if (bp->rx_bufs[i] != NULL) { - dev_kfree_skb(bp->rx_bufs[i]); - bp->rx_bufs[i] = NULL; - } - } - for (i = 0; i<N_TX_RING; i++) { + /* free some skb's */ + for (i=0; i<N_RX_RING; i++) { + if (bp->rx_bufs[i] != NULL) { + dev_kfree_skb(bp->rx_bufs[i]); + bp->rx_bufs[i] = NULL; + } + } + for (i = 0; i<N_TX_RING; i++) { if (bp->tx_bufs[i] != NULL) { dev_kfree_skb(bp->tx_bufs[i]); bp->tx_bufs[i] = NULL; } } } - pmac_call_feature(PMAC_FTR_BMAC_ENABLE, macio_get_of_node(bp->mdev), 0, 0); + pmac_call_feature(PMAC_FTR_BMAC_ENABLE, macio_get_of_node(bp->mdev), 0, 0); return 0; } @@ -510,9 +510,9 @@ static int bmac_resume(struct macio_dev *mdev) bmac_reset_and_enable(dev); enable_irq(dev->irq); - enable_irq(bp->tx_dma_intr); - enable_irq(bp->rx_dma_intr); - netif_device_attach(dev); + enable_irq(bp->tx_dma_intr); + enable_irq(bp->rx_dma_intr); + netif_device_attach(dev); return 0; } @@ -1599,7 +1599,7 @@ static int bmac_remove(struct macio_dev *mdev) unregister_netdev(dev); - free_irq(dev->irq, dev); + free_irq(dev->irq, dev); free_irq(bp->tx_dma_intr, dev); free_irq(bp->rx_dma_intr, dev); diff --git a/drivers/net/ethernet/apple/mace.c b/drivers/net/ethernet/apple/mace.c index 9e5006e59215..4b80e3a52a19 100644 --- a/drivers/net/ethernet/apple/mace.c +++ b/drivers/net/ethernet/apple/mace.c @@ -364,9 +364,9 @@ static void mace_reset(struct net_device *dev) out_8(&mb->iac, 0); if (mp->port_aaui) - out_8(&mb->plscc, PORTSEL_AUI + ENPLSIO); + out_8(&mb->plscc, PORTSEL_AUI + ENPLSIO); else - out_8(&mb->plscc, PORTSEL_GPSI + ENPLSIO); + out_8(&mb->plscc, PORTSEL_GPSI + ENPLSIO); } static void __mace_set_address(struct net_device *dev, void *addr) @@ -378,9 +378,9 @@ static void __mace_set_address(struct net_device *dev, void *addr) /* load up the hardware address */ if (mp->chipid == BROKEN_ADDRCHG_REV) - out_8(&mb->iac, PHYADDR); + out_8(&mb->iac, PHYADDR); else { - out_8(&mb->iac, ADDRCHG | PHYADDR); + out_8(&mb->iac, ADDRCHG | PHYADDR); while ((in_8(&mb->iac) & ADDRCHG) != 0) ; } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.h b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.h index f5fba8b8cdea..a47e2710487e 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.h @@ -91,7 +91,7 @@ struct aq_macsec_txsc { u32 hw_sc_idx; unsigned long tx_sa_idx_busy; const struct macsec_secy *sw_secy; - u8 tx_sa_key[MACSEC_NUM_AN][MACSEC_KEYID_LEN]; + u8 tx_sa_key[MACSEC_NUM_AN][MACSEC_MAX_KEY_LEN]; struct aq_macsec_tx_sc_stats stats; struct aq_macsec_tx_sa_stats tx_sa_stats[MACSEC_NUM_AN]; }; @@ -101,7 +101,7 @@ struct aq_macsec_rxsc { unsigned long rx_sa_idx_busy; const struct macsec_secy *sw_secy; const struct macsec_rx_sc *sw_rxsc; - u8 rx_sa_key[MACSEC_NUM_AN][MACSEC_KEYID_LEN]; + u8 rx_sa_key[MACSEC_NUM_AN][MACSEC_MAX_KEY_LEN]; struct aq_macsec_rx_sa_stats rx_sa_stats[MACSEC_NUM_AN]; }; diff --git a/drivers/net/ethernet/arc/emac_rockchip.c b/drivers/net/ethernet/arc/emac_rockchip.c index 48ecdf15eddc..1c9ca3bcb871 100644 --- a/drivers/net/ethernet/arc/emac_rockchip.c +++ b/drivers/net/ethernet/arc/emac_rockchip.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/** +/* * emac-rockchip.c - Rockchip EMAC specific glue layer * * Copyright (C) 2014 Romain Perier <romain.perier@gmail.com> diff --git a/drivers/net/ethernet/atheros/alx/alx.h b/drivers/net/ethernet/atheros/alx/alx.h index 9d0e74f6b089..693006c5a498 100644 --- a/drivers/net/ethernet/atheros/alx/alx.h +++ b/drivers/net/ethernet/atheros/alx/alx.h @@ -137,6 +137,8 @@ struct alx_priv { /* protects hw.stats */ spinlock_t stats_lock; + + struct mutex mtx; }; extern const struct ethtool_ops alx_ethtool_ops; diff --git a/drivers/net/ethernet/atheros/alx/ethtool.c b/drivers/net/ethernet/atheros/alx/ethtool.c index 2f4eabf652e8..b716adacd815 100644 --- a/drivers/net/ethernet/atheros/alx/ethtool.c +++ b/drivers/net/ethernet/atheros/alx/ethtool.c @@ -163,8 +163,10 @@ static int alx_get_link_ksettings(struct net_device *netdev, } } + mutex_lock(&alx->mtx); cmd->base.speed = hw->link_speed; cmd->base.duplex = hw->duplex; + mutex_unlock(&alx->mtx); ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, supported); @@ -181,8 +183,7 @@ static int alx_set_link_ksettings(struct net_device *netdev, struct alx_hw *hw = &alx->hw; u32 adv_cfg; u32 advertising; - - ASSERT_RTNL(); + int ret; ethtool_convert_link_mode_to_legacy_u32(&advertising, cmd->link_modes.advertising); @@ -200,7 +201,12 @@ static int alx_set_link_ksettings(struct net_device *netdev, } hw->adv_cfg = adv_cfg; - return alx_setup_speed_duplex(hw, adv_cfg, hw->flowctrl); + + mutex_lock(&alx->mtx); + ret = alx_setup_speed_duplex(hw, adv_cfg, hw->flowctrl); + mutex_unlock(&alx->mtx); + + return ret; } static void alx_get_pauseparam(struct net_device *netdev, @@ -209,10 +215,12 @@ static void alx_get_pauseparam(struct net_device *netdev, struct alx_priv *alx = netdev_priv(netdev); struct alx_hw *hw = &alx->hw; + mutex_lock(&alx->mtx); pause->autoneg = !!(hw->flowctrl & ALX_FC_ANEG && hw->adv_cfg & ADVERTISED_Autoneg); pause->tx_pause = !!(hw->flowctrl & ALX_FC_TX); pause->rx_pause = !!(hw->flowctrl & ALX_FC_RX); + mutex_unlock(&alx->mtx); } @@ -232,7 +240,7 @@ static int alx_set_pauseparam(struct net_device *netdev, if (pause->autoneg) fc |= ALX_FC_ANEG; - ASSERT_RTNL(); + mutex_lock(&alx->mtx); /* restart auto-neg for auto-mode */ if (hw->adv_cfg & ADVERTISED_Autoneg) { @@ -245,8 +253,10 @@ static int alx_set_pauseparam(struct net_device *netdev, if (reconfig_phy) { err = alx_setup_speed_duplex(hw, hw->adv_cfg, fc); - if (err) + if (err) { + mutex_unlock(&alx->mtx); return err; + } } /* flow control on mac */ @@ -254,6 +264,7 @@ static int alx_set_pauseparam(struct net_device *netdev, alx_cfg_mac_flowcontrol(hw, fc); hw->flowctrl = fc; + mutex_unlock(&alx->mtx); return 0; } diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c index b3d74332ed33..11ef1fbe7aee 100644 --- a/drivers/net/ethernet/atheros/alx/main.c +++ b/drivers/net/ethernet/atheros/alx/main.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Johannes Berg <johannes@sipsolutions.net> + * Copyright (c) 2013, 2021 Johannes Berg <johannes@sipsolutions.net> * * This file is free software: you may copy, redistribute and/or modify it * under the terms of the GNU General Public License as published by the @@ -1091,8 +1091,9 @@ static int alx_init_sw(struct alx_priv *alx) ALX_MAC_CTRL_RXFC_EN | ALX_MAC_CTRL_TXFC_EN | 7 << ALX_MAC_CTRL_PRMBLEN_SHIFT; + mutex_init(&alx->mtx); - return err; + return 0; } @@ -1122,6 +1123,8 @@ static void alx_halt(struct alx_priv *alx) { struct alx_hw *hw = &alx->hw; + lockdep_assert_held(&alx->mtx); + alx_netif_stop(alx); hw->link_speed = SPEED_UNKNOWN; hw->duplex = DUPLEX_UNKNOWN; @@ -1147,6 +1150,8 @@ static void alx_configure(struct alx_priv *alx) static void alx_activate(struct alx_priv *alx) { + lockdep_assert_held(&alx->mtx); + /* hardware setting lost, restore it */ alx_reinit_rings(alx); alx_configure(alx); @@ -1161,7 +1166,7 @@ static void alx_activate(struct alx_priv *alx) static void alx_reinit(struct alx_priv *alx) { - ASSERT_RTNL(); + lockdep_assert_held(&alx->mtx); alx_halt(alx); alx_activate(alx); @@ -1249,6 +1254,8 @@ out_disable_adv_intr: static void __alx_stop(struct alx_priv *alx) { + lockdep_assert_held(&alx->mtx); + alx_free_irq(alx); cancel_work_sync(&alx->link_check_wk); @@ -1284,6 +1291,8 @@ static void alx_check_link(struct alx_priv *alx) int old_speed; int err; + lockdep_assert_held(&alx->mtx); + /* clear PHY internal interrupt status, otherwise the main * interrupt status will be asserted forever */ @@ -1338,12 +1347,24 @@ reset: static int alx_open(struct net_device *netdev) { - return __alx_open(netdev_priv(netdev), false); + struct alx_priv *alx = netdev_priv(netdev); + int ret; + + mutex_lock(&alx->mtx); + ret = __alx_open(alx, false); + mutex_unlock(&alx->mtx); + + return ret; } static int alx_stop(struct net_device *netdev) { - __alx_stop(netdev_priv(netdev)); + struct alx_priv *alx = netdev_priv(netdev); + + mutex_lock(&alx->mtx); + __alx_stop(alx); + mutex_unlock(&alx->mtx); + return 0; } @@ -1353,18 +1374,18 @@ static void alx_link_check(struct work_struct *work) alx = container_of(work, struct alx_priv, link_check_wk); - rtnl_lock(); + mutex_lock(&alx->mtx); alx_check_link(alx); - rtnl_unlock(); + mutex_unlock(&alx->mtx); } static void alx_reset(struct work_struct *work) { struct alx_priv *alx = container_of(work, struct alx_priv, reset_wk); - rtnl_lock(); + mutex_lock(&alx->mtx); alx_reinit(alx); - rtnl_unlock(); + mutex_unlock(&alx->mtx); } static int alx_tpd_req(struct sk_buff *skb) @@ -1771,6 +1792,8 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_unmap; } + mutex_lock(&alx->mtx); + alx_reset_pcie(hw); phy_configured = alx_phy_configured(hw); @@ -1781,7 +1804,7 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err = alx_reset_mac(hw); if (err) { dev_err(&pdev->dev, "MAC Reset failed, error = %d\n", err); - goto out_unmap; + goto out_unlock; } /* setup link to put it in a known good starting state */ @@ -1791,7 +1814,7 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) dev_err(&pdev->dev, "failed to configure PHY speed/duplex (err=%d)\n", err); - goto out_unmap; + goto out_unlock; } } @@ -1824,9 +1847,11 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!alx_get_phy_info(hw)) { dev_err(&pdev->dev, "failed to identify PHY\n"); err = -EIO; - goto out_unmap; + goto out_unlock; } + mutex_unlock(&alx->mtx); + INIT_WORK(&alx->link_check_wk, alx_link_check); INIT_WORK(&alx->reset_wk, alx_reset); netif_carrier_off(netdev); @@ -1843,12 +1868,15 @@ static int alx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; +out_unlock: + mutex_unlock(&alx->mtx); out_unmap: iounmap(hw->hw_addr); out_free_netdev: free_netdev(netdev); out_pci_release: pci_release_mem_regions(pdev); + pci_disable_pcie_error_reporting(pdev); out_pci_disable: pci_disable_device(pdev); return err; @@ -1869,6 +1897,8 @@ static void alx_remove(struct pci_dev *pdev) pci_disable_pcie_error_reporting(pdev); pci_disable_device(pdev); + mutex_destroy(&alx->mtx); + free_netdev(alx->dev); } @@ -1880,7 +1910,11 @@ static int alx_suspend(struct device *dev) if (!netif_running(alx->dev)) return 0; netif_device_detach(alx->dev); + + mutex_lock(&alx->mtx); __alx_stop(alx); + mutex_unlock(&alx->mtx); + return 0; } @@ -1890,20 +1924,23 @@ static int alx_resume(struct device *dev) struct alx_hw *hw = &alx->hw; int err; + mutex_lock(&alx->mtx); alx_reset_phy(hw); - if (!netif_running(alx->dev)) - return 0; + if (!netif_running(alx->dev)) { + err = 0; + goto unlock; + } - rtnl_lock(); err = __alx_open(alx, true); - rtnl_unlock(); if (err) - return err; + goto unlock; netif_device_attach(alx->dev); - return 0; +unlock: + mutex_unlock(&alx->mtx); + return err; } static SIMPLE_DEV_PM_OPS(alx_pm_ops, alx_suspend, alx_resume); @@ -1922,7 +1959,7 @@ static pci_ers_result_t alx_pci_error_detected(struct pci_dev *pdev, dev_info(&pdev->dev, "pci error detected\n"); - rtnl_lock(); + mutex_lock(&alx->mtx); if (netif_running(netdev)) { netif_device_detach(netdev); @@ -1934,7 +1971,7 @@ static pci_ers_result_t alx_pci_error_detected(struct pci_dev *pdev, else pci_disable_device(pdev); - rtnl_unlock(); + mutex_unlock(&alx->mtx); return rc; } @@ -1947,7 +1984,7 @@ static pci_ers_result_t alx_pci_error_slot_reset(struct pci_dev *pdev) dev_info(&pdev->dev, "pci error slot reset\n"); - rtnl_lock(); + mutex_lock(&alx->mtx); if (pci_enable_device(pdev)) { dev_err(&pdev->dev, "Failed to re-enable PCI device after reset\n"); @@ -1960,7 +1997,7 @@ static pci_ers_result_t alx_pci_error_slot_reset(struct pci_dev *pdev) if (!alx_reset_mac(hw)) rc = PCI_ERS_RESULT_RECOVERED; out: - rtnl_unlock(); + mutex_unlock(&alx->mtx); return rc; } @@ -1972,14 +2009,14 @@ static void alx_pci_error_resume(struct pci_dev *pdev) dev_info(&pdev->dev, "pci error resume\n"); - rtnl_lock(); + mutex_lock(&alx->mtx); if (netif_running(netdev)) { alx_activate(alx); netif_device_attach(netdev); } - rtnl_unlock(); + mutex_unlock(&alx->mtx); } static const struct pci_error_handlers alx_err_handlers = { diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c.h b/drivers/net/ethernet/atheros/atl1c/atl1c.h index 28ae5c16831e..43d821fe7a54 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c.h +++ b/drivers/net/ethernet/atheros/atl1c/atl1c.h @@ -63,7 +63,7 @@ #define AT_MAX_RECEIVE_QUEUE 4 #define AT_DEF_RECEIVE_QUEUE 1 -#define AT_MAX_TRANSMIT_QUEUE 2 +#define AT_MAX_TRANSMIT_QUEUE 4 #define AT_DMA_HI_ADDR_MASK 0xffffffff00000000ULL #define AT_DMA_LO_ADDR_MASK 0x00000000ffffffffULL @@ -241,6 +241,8 @@ struct atl1c_tpd_ext_desc { #define RRS_PACKET_PROT_IS_IPV6_ONLY(word) \ ((((word) >> RRS_PROT_ID_SHIFT) & RRS_PROT_ID_MASK) == 6) +#define RRS_MT_PROT_ID_TCPUDP BIT(19) + struct atl1c_recv_ret_status { __le32 word0; __le32 rss_hash; @@ -289,11 +291,7 @@ enum atl1c_nic_type { athr_l2c_b2, athr_l1d, athr_l1d_2, -}; - -enum atl1c_trans_queue { - atl1c_trans_normal = 0, - atl1c_trans_high = 1 + athr_mt, }; struct atl1c_hw_stats { @@ -472,13 +470,16 @@ struct atl1c_buffer { /* transimit packet descriptor (tpd) ring */ struct atl1c_tpd_ring { + struct atl1c_adapter *adapter; void *desc; /* descriptor ring virtual address */ dma_addr_t dma; /* descriptor ring physical address */ + u16 num; u16 size; /* descriptor ring length in bytes */ u16 count; /* number of descriptors in the ring */ u16 next_to_use; atomic_t next_to_clean; struct atl1c_buffer *buffer_info; + struct napi_struct napi; }; /* receive free descriptor (rfd) ring */ @@ -494,27 +495,30 @@ struct atl1c_rfd_ring { /* receive return descriptor (rrd) ring */ struct atl1c_rrd_ring { + struct atl1c_adapter *adapter; void *desc; /* descriptor ring virtual address */ dma_addr_t dma; /* descriptor ring physical address */ + u16 num; u16 size; /* descriptor ring length in bytes */ u16 count; /* number of descriptors in the ring */ u16 next_to_use; u16 next_to_clean; + struct napi_struct napi; + struct page *rx_page; + unsigned int rx_page_offset; }; /* board specific private data structure */ struct atl1c_adapter { struct net_device *netdev; struct pci_dev *pdev; - struct napi_struct napi; - struct napi_struct tx_napi; - struct page *rx_page; - unsigned int rx_page_offset; unsigned int rx_frag_size; struct atl1c_hw hw; struct atl1c_hw_stats hw_stats; struct mii_if_info mii; /* MII interface info */ u16 rx_buffer_len; + unsigned int tx_queue_count; + unsigned int rx_queue_count; unsigned long flags; #define __AT_TESTING 0x0001 @@ -540,8 +544,8 @@ struct atl1c_adapter { /* All Descriptor memory */ struct atl1c_ring_header ring_header; struct atl1c_tpd_ring tpd_ring[AT_MAX_TRANSMIT_QUEUE]; - struct atl1c_rfd_ring rfd_ring; - struct atl1c_rrd_ring rrd_ring; + struct atl1c_rfd_ring rfd_ring[AT_MAX_RECEIVE_QUEUE]; + struct atl1c_rrd_ring rrd_ring[AT_MAX_RECEIVE_QUEUE]; u32 bd_number; /* board number;*/ }; diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_hw.c b/drivers/net/ethernet/atheros/atl1c/atl1c_hw.c index 140358dcf61e..7dff20350865 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_hw.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_hw.c @@ -636,6 +636,23 @@ int atl1c_phy_init(struct atl1c_hw *hw) return 0; } +bool atl1c_get_link_status(struct atl1c_hw *hw) +{ + u16 phy_data; + + if (hw->nic_type == athr_mt) { + u32 spd; + + AT_READ_REG(hw, REG_MT_SPEED, &spd); + return !!spd; + } + + /* MII_BMSR must be read twice */ + atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); + atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); + return !!(phy_data & BMSR_LSTATUS); +} + /* * Detects the current speed and duplex settings of the hardware. * @@ -648,6 +665,15 @@ int atl1c_get_speed_and_duplex(struct atl1c_hw *hw, u16 *speed, u16 *duplex) int err; u16 phy_data; + if (hw->nic_type == athr_mt) { + u32 spd; + + AT_READ_REG(hw, REG_MT_SPEED, &spd); + *speed = spd; + *duplex = FULL_DUPLEX; + return 0; + } + /* Read PHY Specific Status Register (17) */ err = atl1c_read_phy_reg(hw, MII_GIGA_PSSR, &phy_data); if (err) @@ -686,15 +712,12 @@ int atl1c_phy_to_ps_link(struct atl1c_hw *hw) int ret = 0; u16 autoneg_advertised = ADVERTISED_10baseT_Half; u16 save_autoneg_advertised; - u16 phy_data; u16 mii_lpa_data; u16 speed = SPEED_0; u16 duplex = FULL_DUPLEX; int i; - atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); - atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); - if (phy_data & BMSR_LSTATUS) { + if (atl1c_get_link_status(hw)) { atl1c_read_phy_reg(hw, MII_LPA, &mii_lpa_data); if (mii_lpa_data & LPA_10FULL) autoneg_advertised = ADVERTISED_10baseT_Full; @@ -717,9 +740,7 @@ int atl1c_phy_to_ps_link(struct atl1c_hw *hw) if (mii_lpa_data) { for (i = 0; i < AT_SUSPEND_LINK_TIMEOUT; i++) { mdelay(100); - atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); - atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); - if (phy_data & BMSR_LSTATUS) { + if (atl1c_get_link_status(hw)) { if (atl1c_get_speed_and_duplex(hw, &speed, &duplex) != 0) dev_dbg(&pdev->dev, diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h b/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h index ce1a123dce2c..c567c920628f 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_hw.h @@ -26,6 +26,7 @@ void atl1c_phy_disable(struct atl1c_hw *hw); void atl1c_hw_set_mac_addr(struct atl1c_hw *hw, u8 *mac_addr); int atl1c_phy_reset(struct atl1c_hw *hw); int atl1c_read_mac_addr(struct atl1c_hw *hw); +bool atl1c_get_link_status(struct atl1c_hw *hw); int atl1c_get_speed_and_duplex(struct atl1c_hw *hw, u16 *speed, u16 *duplex); u32 atl1c_hash_mc_addr(struct atl1c_hw *hw, u8 *mc_addr); void atl1c_hash_set(struct atl1c_hw *hw, u32 hash_value); @@ -527,15 +528,24 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed); #define REG_RX_BASE_ADDR_HI 0x1540 #define REG_TX_BASE_ADDR_HI 0x1544 #define REG_RFD0_HEAD_ADDR_LO 0x1550 +#define REG_RFD1_HEAD_ADDR_LO 0x1554 +#define REG_RFD2_HEAD_ADDR_LO 0x1558 +#define REG_RFD3_HEAD_ADDR_LO 0x155C #define REG_RFD_RING_SIZE 0x1560 #define RFD_RING_SIZE_MASK 0x0FFF #define REG_RX_BUF_SIZE 0x1564 #define RX_BUF_SIZE_MASK 0xFFFF #define REG_RRD0_HEAD_ADDR_LO 0x1568 +#define REG_RRD1_HEAD_ADDR_LO 0x156C +#define REG_RRD2_HEAD_ADDR_LO 0x1570 +#define REG_RRD3_HEAD_ADDR_LO 0x1574 #define REG_RRD_RING_SIZE 0x1578 #define RRD_RING_SIZE_MASK 0x0FFF #define REG_TPD_PRI1_ADDR_LO 0x157C #define REG_TPD_PRI0_ADDR_LO 0x1580 +#define REG_TPD_PRI2_ADDR_LO 0x1F10 +#define REG_TPD_PRI3_ADDR_LO 0x1F14 + #define REG_TPD_RING_SIZE 0x1584 #define TPD_RING_SIZE_MASK 0xFFFF @@ -654,15 +664,26 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed); /* Mail box */ #define MB_RFDX_PROD_IDX_MASK 0xFFFF #define REG_MB_RFD0_PROD_IDX 0x15E0 +#define REG_MB_RFD1_PROD_IDX 0x15E4 +#define REG_MB_RFD2_PROD_IDX 0x15E8 +#define REG_MB_RFD3_PROD_IDX 0x15EC #define REG_TPD_PRI1_PIDX 0x15F0 /* 16bit,hi-tpd producer idx */ #define REG_TPD_PRI0_PIDX 0x15F2 /* 16bit,lo-tpd producer idx */ #define REG_TPD_PRI1_CIDX 0x15F4 /* 16bit,hi-tpd consumer idx */ #define REG_TPD_PRI0_CIDX 0x15F6 /* 16bit,lo-tpd consumer idx */ +#define REG_TPD_PRI3_PIDX 0x1F18 +#define REG_TPD_PRI2_PIDX 0x1F1A +#define REG_TPD_PRI3_CIDX 0x1F1C +#define REG_TPD_PRI2_CIDX 0x1F1E + #define REG_MB_RFD01_CONS_IDX 0x15F8 #define MB_RFD0_CONS_IDX_MASK 0x0000FFFF #define MB_RFD1_CONS_IDX_MASK 0xFFFF0000 +#define REG_MB_RFD23_CONS_IDX 0x15FC +#define MB_RFD2_CONS_IDX_MASK 0x0000FFFF +#define MB_RFD3_CONS_IDX_MASK 0xFFFF0000 /* Interrupt Status Register */ #define REG_ISR 0x1600 @@ -686,7 +707,7 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed); /* GPHY low power state interrupt */ #define ISR_GPHY_LPW 0x00002000 #define ISR_TXQ_TO_RST 0x00004000 -#define ISR_TX_PKT 0x00008000 +#define ISR_TX_PKT_0 0x00008000 #define ISR_RX_PKT_0 0x00010000 #define ISR_RX_PKT_1 0x00020000 #define ISR_RX_PKT_2 0x00040000 @@ -698,6 +719,9 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed); #define ISR_NFERR_DETECTED 0x01000000 #define ISR_CERR_DETECTED 0x02000000 #define ISR_PHY_LINKDOWN 0x04000000 +#define ISR_TX_PKT_1 0x10000000 +#define ISR_TX_PKT_2 0x20000000 +#define ISR_TX_PKT_3 0x40000000 #define ISR_DIS_INT 0x80000000 /* Interrupt Mask Register */ @@ -712,11 +736,15 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed); ISR_TXQ_TO_RST |\ ISR_DMAW_TO_RST |\ ISR_GPHY |\ - ISR_TX_PKT |\ - ISR_RX_PKT_0 |\ ISR_GPHY_LPW |\ ISR_PHY_LINKDOWN) +#define ISR_TX_PKT ( \ + ISR_TX_PKT_0 | \ + ISR_TX_PKT_1 | \ + ISR_TX_PKT_2 | \ + ISR_TX_PKT_3) + #define ISR_RX_PKT (\ ISR_RX_PKT_0 |\ ISR_RX_PKT_1 |\ @@ -764,6 +792,14 @@ void atl1c_post_phy_linkchg(struct atl1c_hw *hw, u16 link_speed); #define REG_DEBUG_DATA0 0x1900 #define REG_DEBUG_DATA1 0x1904 +#define REG_MT_MAGIC 0x1F00 +#define REG_MT_MODE 0x1F04 +#define REG_MT_SPEED 0x1F08 +#define REG_MT_VERSION 0x1F0C + +#define MT_MAGIC 0xaabb1234 +#define MT_MODE_4Q BIT(0) + #define L1D_MPW_PHYID1 0xD01C /* V7 */ #define L1D_MPW_PHYID2 0xD01D /* V1-V6 */ #define L1D_MPW_PHYID3 0xD01E /* V8 */ diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index c6263cf8d3c0..1c6246a5dc22 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -36,18 +36,51 @@ MODULE_AUTHOR("Qualcomm Atheros Inc."); MODULE_DESCRIPTION("Qualcomm Atheros 100/1000M Ethernet Network Driver"); MODULE_LICENSE("GPL"); +struct atl1c_qregs { + u16 tpd_addr_lo; + u16 tpd_prod; + u16 tpd_cons; + u16 rfd_addr_lo; + u16 rrd_addr_lo; + u16 rfd_prod; + u32 tx_isr; + u32 rx_isr; +}; + +static struct atl1c_qregs atl1c_qregs[AT_MAX_TRANSMIT_QUEUE] = { + { + REG_TPD_PRI0_ADDR_LO, REG_TPD_PRI0_PIDX, REG_TPD_PRI0_CIDX, + REG_RFD0_HEAD_ADDR_LO, REG_RRD0_HEAD_ADDR_LO, + REG_MB_RFD0_PROD_IDX, ISR_TX_PKT_0, ISR_RX_PKT_0 + }, + { + REG_TPD_PRI1_ADDR_LO, REG_TPD_PRI1_PIDX, REG_TPD_PRI1_CIDX, + REG_RFD1_HEAD_ADDR_LO, REG_RRD1_HEAD_ADDR_LO, + REG_MB_RFD1_PROD_IDX, ISR_TX_PKT_1, ISR_RX_PKT_1 + }, + { + REG_TPD_PRI2_ADDR_LO, REG_TPD_PRI2_PIDX, REG_TPD_PRI2_CIDX, + REG_RFD2_HEAD_ADDR_LO, REG_RRD2_HEAD_ADDR_LO, + REG_MB_RFD2_PROD_IDX, ISR_TX_PKT_2, ISR_RX_PKT_2 + }, + { + REG_TPD_PRI3_ADDR_LO, REG_TPD_PRI3_PIDX, REG_TPD_PRI3_CIDX, + REG_RFD3_HEAD_ADDR_LO, REG_RRD3_HEAD_ADDR_LO, + REG_MB_RFD3_PROD_IDX, ISR_TX_PKT_3, ISR_RX_PKT_3 + }, +}; + static int atl1c_stop_mac(struct atl1c_hw *hw); static void atl1c_disable_l0s_l1(struct atl1c_hw *hw); static void atl1c_set_aspm(struct atl1c_hw *hw, u16 link_speed); static void atl1c_start_mac(struct atl1c_adapter *adapter); -static void atl1c_clean_rx_irq(struct atl1c_adapter *adapter, - int *work_done, int work_to_do); static int atl1c_up(struct atl1c_adapter *adapter); static void atl1c_down(struct atl1c_adapter *adapter); static int atl1c_reset_mac(struct atl1c_hw *hw); static void atl1c_reset_dma_ring(struct atl1c_adapter *adapter); static int atl1c_configure(struct atl1c_adapter *adapter); -static int atl1c_alloc_rx_buffer(struct atl1c_adapter *adapter, bool napi_mode); +static int atl1c_alloc_rx_buffer(struct atl1c_adapter *adapter, u32 queue, + bool napi_mode); static const u32 atl1c_default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | @@ -232,15 +265,14 @@ static void atl1c_check_link_status(struct atl1c_adapter *adapter) struct pci_dev *pdev = adapter->pdev; int err; unsigned long flags; - u16 speed, duplex, phy_data; + u16 speed, duplex; + bool link; spin_lock_irqsave(&adapter->mdio_lock, flags); - /* MII_BMSR must read twise */ - atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); - atl1c_read_phy_reg(hw, MII_BMSR, &phy_data); + link = atl1c_get_link_status(hw); spin_unlock_irqrestore(&adapter->mdio_lock, flags); - if ((phy_data & BMSR_LSTATUS) == 0) { + if (!link) { /* link down */ netif_carrier_off(netdev); hw->hibernate = true; @@ -284,16 +316,13 @@ static void atl1c_link_chg_event(struct atl1c_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct pci_dev *pdev = adapter->pdev; - u16 phy_data; - u16 link_up; + bool link; spin_lock(&adapter->mdio_lock); - atl1c_read_phy_reg(&adapter->hw, MII_BMSR, &phy_data); - atl1c_read_phy_reg(&adapter->hw, MII_BMSR, &phy_data); + link = atl1c_get_link_status(&adapter->hw); spin_unlock(&adapter->mdio_lock); - link_up = phy_data & BMSR_LSTATUS; /* notify upper layer link down ASAP */ - if (!link_up) { + if (!link) { if (netif_carrier_ok(netdev)) { /* old link state: Up */ netif_carrier_off(netdev); @@ -436,7 +465,7 @@ static void atl1c_restore_vlan(struct atl1c_adapter *adapter) } /** - * atl1c_set_mac - Change the Ethernet Address of the NIC + * atl1c_set_mac_addr - Change the Ethernet Address of the NIC * @netdev: network interface device structure * @p: pointer to an address structure * @@ -478,6 +507,9 @@ static void atl1c_set_rxbufsize(struct atl1c_adapter *adapter, static netdev_features_t atl1c_fix_features(struct net_device *netdev, netdev_features_t features) { + struct atl1c_adapter *adapter = netdev_priv(netdev); + struct atl1c_hw *hw = &adapter->hw; + /* * Since there is no support for separate rx/tx vlan accel * enable/disable make sure tx flag is always in same state as rx. @@ -487,8 +519,10 @@ static netdev_features_t atl1c_fix_features(struct net_device *netdev, else features &= ~NETIF_F_HW_VLAN_CTAG_TX; - if (netdev->mtu > MAX_TSO_FRAME_SIZE) - features &= ~(NETIF_F_TSO | NETIF_F_TSO6); + if (hw->nic_type != athr_mt) { + if (netdev->mtu > MAX_TSO_FRAME_SIZE) + features &= ~(NETIF_F_TSO | NETIF_F_TSO6); + } return features; } @@ -515,9 +549,12 @@ static void atl1c_set_max_mtu(struct net_device *netdev) case athr_l1d: case athr_l1d_2: netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); + (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); + break; + case athr_mt: + netdev->max_mtu = 9500; break; - /* The 10/100 devices don't support jumbo packets, max_mtu 1500 */ + /* The 10/100 devices don't support jumbo packets, max_mtu 1500 */ default: netdev->max_mtu = ETH_DATA_LEN; break; @@ -642,29 +679,26 @@ static int atl1c_alloc_queues(struct atl1c_adapter *adapter) return 0; } -static void atl1c_set_mac_type(struct atl1c_hw *hw) +static enum atl1c_nic_type atl1c_get_mac_type(struct pci_dev *pdev, + u8 __iomem *hw_addr) { - switch (hw->device_id) { + switch (pdev->device) { case PCI_DEVICE_ID_ATTANSIC_L2C: - hw->nic_type = athr_l2c; - break; + return athr_l2c; case PCI_DEVICE_ID_ATTANSIC_L1C: - hw->nic_type = athr_l1c; - break; + return athr_l1c; case PCI_DEVICE_ID_ATHEROS_L2C_B: - hw->nic_type = athr_l2c_b; - break; + return athr_l2c_b; case PCI_DEVICE_ID_ATHEROS_L2C_B2: - hw->nic_type = athr_l2c_b2; - break; + return athr_l2c_b2; case PCI_DEVICE_ID_ATHEROS_L1D: - hw->nic_type = athr_l1d; - break; + return athr_l1d; case PCI_DEVICE_ID_ATHEROS_L1D_2_0: - hw->nic_type = athr_l1d_2; - break; + if (readl(hw_addr + REG_MT_MAGIC) == MT_MAGIC) + return athr_mt; + return athr_l1d_2; default: - break; + return athr_l1c; } } @@ -672,7 +706,6 @@ static int atl1c_setup_mac_funcs(struct atl1c_hw *hw) { u32 link_ctrl_data; - atl1c_set_mac_type(hw); AT_READ_REG(hw, REG_LINK_CTRL, &link_ctrl_data); hw->ctrl_flags = ATL1C_INTR_MODRT_ENABLE | @@ -763,14 +796,14 @@ static int atl1c_sw_init(struct atl1c_adapter *adapter) struct atl1c_hw *hw = &adapter->hw; struct pci_dev *pdev = adapter->pdev; u32 revision; - + int i; adapter->wol = 0; device_set_wakeup_enable(&pdev->dev, false); adapter->link_speed = SPEED_0; adapter->link_duplex = FULL_DUPLEX; adapter->tpd_ring[0].count = 1024; - adapter->rfd_ring.count = 512; + adapter->rfd_ring[0].count = 512; hw->vendor_id = pdev->vendor; hw->device_id = pdev->device; @@ -788,6 +821,10 @@ static int atl1c_sw_init(struct atl1c_adapter *adapter) atl1c_patch_assign(hw); hw->intr_mask = IMR_NORMAL_MASK; + for (i = 0; i < adapter->tx_queue_count; ++i) + hw->intr_mask |= atl1c_qregs[i].tx_isr; + for (i = 0; i < adapter->rx_queue_count; ++i) + hw->intr_mask |= atl1c_qregs[i].rx_isr; hw->phy_configured = false; hw->preamble_len = 7; hw->max_frame_size = adapter->netdev->mtu; @@ -847,12 +884,12 @@ static inline void atl1c_clean_buffer(struct pci_dev *pdev, /** * atl1c_clean_tx_ring - Free Tx-skb * @adapter: board private structure - * @type: type of transmit queue + * @queue: idx of transmit queue */ static void atl1c_clean_tx_ring(struct atl1c_adapter *adapter, - enum atl1c_trans_queue type) + u32 queue) { - struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[type]; + struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[queue]; struct atl1c_buffer *buffer_info; struct pci_dev *pdev = adapter->pdev; u16 index, ring_count; @@ -875,11 +912,12 @@ static void atl1c_clean_tx_ring(struct atl1c_adapter *adapter, /** * atl1c_clean_rx_ring - Free rx-reservation skbs * @adapter: board private structure + * @queue: idx of transmit queue */ -static void atl1c_clean_rx_ring(struct atl1c_adapter *adapter) +static void atl1c_clean_rx_ring(struct atl1c_adapter *adapter, u32 queue) { - struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring; - struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring; + struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring[queue]; + struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring[queue]; struct atl1c_buffer *buffer_info; struct pci_dev *pdev = adapter->pdev; int j; @@ -902,26 +940,28 @@ static void atl1c_clean_rx_ring(struct atl1c_adapter *adapter) static void atl1c_init_ring_ptrs(struct atl1c_adapter *adapter) { struct atl1c_tpd_ring *tpd_ring = adapter->tpd_ring; - struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring; - struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring; + struct atl1c_rfd_ring *rfd_ring = adapter->rfd_ring; + struct atl1c_rrd_ring *rrd_ring = adapter->rrd_ring; struct atl1c_buffer *buffer_info; int i, j; - for (i = 0; i < AT_MAX_TRANSMIT_QUEUE; i++) { + for (i = 0; i < adapter->tx_queue_count; i++) { tpd_ring[i].next_to_use = 0; atomic_set(&tpd_ring[i].next_to_clean, 0); buffer_info = tpd_ring[i].buffer_info; for (j = 0; j < tpd_ring->count; j++) ATL1C_SET_BUFFER_STATE(&buffer_info[i], - ATL1C_BUFFER_FREE); - } - rfd_ring->next_to_use = 0; - rfd_ring->next_to_clean = 0; - rrd_ring->next_to_use = 0; - rrd_ring->next_to_clean = 0; - for (j = 0; j < rfd_ring->count; j++) { - buffer_info = &rfd_ring->buffer_info[j]; - ATL1C_SET_BUFFER_STATE(buffer_info, ATL1C_BUFFER_FREE); + ATL1C_BUFFER_FREE); + } + for (i = 0; i < adapter->rx_queue_count; i++) { + rfd_ring[i].next_to_use = 0; + rfd_ring[i].next_to_clean = 0; + rrd_ring[i].next_to_use = 0; + rrd_ring[i].next_to_clean = 0; + for (j = 0; j < rfd_ring[i].count; j++) { + buffer_info = &rfd_ring[i].buffer_info[j]; + ATL1C_SET_BUFFER_STATE(buffer_info, ATL1C_BUFFER_FREE); + } } } @@ -934,25 +974,29 @@ static void atl1c_init_ring_ptrs(struct atl1c_adapter *adapter) static void atl1c_free_ring_resources(struct atl1c_adapter *adapter) { struct pci_dev *pdev = adapter->pdev; + int i; dma_free_coherent(&pdev->dev, adapter->ring_header.size, adapter->ring_header.desc, adapter->ring_header.dma); adapter->ring_header.desc = NULL; /* Note: just free tdp_ring.buffer_info, - * it contain rfd_ring.buffer_info, do not double free */ + * it contain rfd_ring.buffer_info, do not double free + */ if (adapter->tpd_ring[0].buffer_info) { kfree(adapter->tpd_ring[0].buffer_info); adapter->tpd_ring[0].buffer_info = NULL; } - if (adapter->rx_page) { - put_page(adapter->rx_page); - adapter->rx_page = NULL; + for (i = 0; i < adapter->rx_queue_count; ++i) { + if (adapter->rrd_ring[i].rx_page) { + put_page(adapter->rrd_ring[i].rx_page); + adapter->rrd_ring[i].rx_page = NULL; + } } } /** - * atl1c_setup_mem_resources - allocate Tx / RX descriptor resources + * atl1c_setup_ring_resources - allocate Tx / RX descriptor resources * @adapter: board private structure * * Return 0 on success, negative on failure @@ -961,37 +1005,46 @@ static int atl1c_setup_ring_resources(struct atl1c_adapter *adapter) { struct pci_dev *pdev = adapter->pdev; struct atl1c_tpd_ring *tpd_ring = adapter->tpd_ring; - struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring; - struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring; + struct atl1c_rfd_ring *rfd_ring = adapter->rfd_ring; + struct atl1c_rrd_ring *rrd_ring = adapter->rrd_ring; struct atl1c_ring_header *ring_header = &adapter->ring_header; + int tqc = adapter->tx_queue_count; + int rqc = adapter->rx_queue_count; int size; int i; int count = 0; - int rx_desc_count = 0; u32 offset = 0; - rrd_ring->count = rfd_ring->count; - for (i = 1; i < AT_MAX_TRANSMIT_QUEUE; i++) + /* Even though only one tpd queue is actually used, the "high" + * priority tpd queue also gets initialized + */ + if (tqc == 1) + tqc = 2; + + for (i = 1; i < tqc; i++) tpd_ring[i].count = tpd_ring[0].count; - /* 2 tpd queue, one high priority queue, - * another normal priority queue */ - size = sizeof(struct atl1c_buffer) * (tpd_ring->count * 2 + - rfd_ring->count); + size = sizeof(struct atl1c_buffer) * (tpd_ring->count * tqc + + rfd_ring->count * rqc); tpd_ring->buffer_info = kzalloc(size, GFP_KERNEL); if (unlikely(!tpd_ring->buffer_info)) goto err_nomem; - for (i = 0; i < AT_MAX_TRANSMIT_QUEUE; i++) { - tpd_ring[i].buffer_info = - (tpd_ring->buffer_info + count); + for (i = 0; i < tqc; i++) { + tpd_ring[i].adapter = adapter; + tpd_ring[i].num = i; + tpd_ring[i].buffer_info = (tpd_ring->buffer_info + count); count += tpd_ring[i].count; } - rfd_ring->buffer_info = - (tpd_ring->buffer_info + count); - count += rfd_ring->count; - rx_desc_count += rfd_ring->count; + for (i = 0; i < rqc; i++) { + rrd_ring[i].adapter = adapter; + rrd_ring[i].num = i; + rrd_ring[i].count = rfd_ring[0].count; + rfd_ring[i].count = rfd_ring[0].count; + rfd_ring[i].buffer_info = (tpd_ring->buffer_info + count); + count += rfd_ring->count; + } /* * real ring DMA buffer @@ -999,9 +1052,9 @@ static int atl1c_setup_ring_resources(struct atl1c_adapter *adapter) * additional bytes tacked onto the end. */ ring_header->size = size = - sizeof(struct atl1c_tpd_desc) * tpd_ring->count * 2 + - sizeof(struct atl1c_rx_free_desc) * rx_desc_count + - sizeof(struct atl1c_recv_ret_status) * rx_desc_count + + sizeof(struct atl1c_tpd_desc) * tpd_ring->count * tqc + + sizeof(struct atl1c_rx_free_desc) * rfd_ring->count * rqc + + sizeof(struct atl1c_recv_ret_status) * rfd_ring->count * rqc + 8 * 4; ring_header->desc = dma_alloc_coherent(&pdev->dev, ring_header->size, @@ -1014,25 +1067,28 @@ static int atl1c_setup_ring_resources(struct atl1c_adapter *adapter) tpd_ring[0].dma = roundup(ring_header->dma, 8); offset = tpd_ring[0].dma - ring_header->dma; - for (i = 0; i < AT_MAX_TRANSMIT_QUEUE; i++) { + for (i = 0; i < tqc; i++) { tpd_ring[i].dma = ring_header->dma + offset; - tpd_ring[i].desc = (u8 *) ring_header->desc + offset; + tpd_ring[i].desc = (u8 *)ring_header->desc + offset; tpd_ring[i].size = sizeof(struct atl1c_tpd_desc) * tpd_ring[i].count; offset += roundup(tpd_ring[i].size, 8); } - /* init RFD ring */ - rfd_ring->dma = ring_header->dma + offset; - rfd_ring->desc = (u8 *) ring_header->desc + offset; - rfd_ring->size = sizeof(struct atl1c_rx_free_desc) * rfd_ring->count; - offset += roundup(rfd_ring->size, 8); + for (i = 0; i < rqc; i++) { + /* init RFD ring */ + rfd_ring[i].dma = ring_header->dma + offset; + rfd_ring[i].desc = (u8 *)ring_header->desc + offset; + rfd_ring[i].size = sizeof(struct atl1c_rx_free_desc) * + rfd_ring[i].count; + offset += roundup(rfd_ring[i].size, 8); - /* init RRD ring */ - rrd_ring->dma = ring_header->dma + offset; - rrd_ring->desc = (u8 *) ring_header->desc + offset; - rrd_ring->size = sizeof(struct atl1c_recv_ret_status) * - rrd_ring->count; - offset += roundup(rrd_ring->size, 8); + /* init RRD ring */ + rrd_ring[i].dma = ring_header->dma + offset; + rrd_ring[i].desc = (u8 *)ring_header->desc + offset; + rrd_ring[i].size = sizeof(struct atl1c_recv_ret_status) * + rrd_ring[i].count; + offset += roundup(rrd_ring[i].size, 8); + } return 0; @@ -1044,31 +1100,34 @@ err_nomem: static void atl1c_configure_des_ring(struct atl1c_adapter *adapter) { struct atl1c_hw *hw = &adapter->hw; - struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring; - struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring; - struct atl1c_tpd_ring *tpd_ring = (struct atl1c_tpd_ring *) - adapter->tpd_ring; + struct atl1c_rfd_ring *rfd_ring = adapter->rfd_ring; + struct atl1c_rrd_ring *rrd_ring = adapter->rrd_ring; + struct atl1c_tpd_ring *tpd_ring = adapter->tpd_ring; + int i; + int tx_queue_count = adapter->tx_queue_count; + + if (tx_queue_count == 1) + tx_queue_count = 2; /* TPD */ AT_WRITE_REG(hw, REG_TX_BASE_ADDR_HI, - (u32)((tpd_ring[atl1c_trans_normal].dma & - AT_DMA_HI_ADDR_MASK) >> 32)); + (u32)((tpd_ring[0].dma & AT_DMA_HI_ADDR_MASK) >> 32)); /* just enable normal priority TX queue */ - AT_WRITE_REG(hw, REG_TPD_PRI0_ADDR_LO, - (u32)(tpd_ring[atl1c_trans_normal].dma & - AT_DMA_LO_ADDR_MASK)); - AT_WRITE_REG(hw, REG_TPD_PRI1_ADDR_LO, - (u32)(tpd_ring[atl1c_trans_high].dma & - AT_DMA_LO_ADDR_MASK)); + for (i = 0; i < tx_queue_count; i++) { + AT_WRITE_REG(hw, atl1c_qregs[i].tpd_addr_lo, + (u32)(tpd_ring[i].dma & AT_DMA_LO_ADDR_MASK)); + } AT_WRITE_REG(hw, REG_TPD_RING_SIZE, (u32)(tpd_ring[0].count & TPD_RING_SIZE_MASK)); /* RFD */ AT_WRITE_REG(hw, REG_RX_BASE_ADDR_HI, - (u32)((rfd_ring->dma & AT_DMA_HI_ADDR_MASK) >> 32)); - AT_WRITE_REG(hw, REG_RFD0_HEAD_ADDR_LO, - (u32)(rfd_ring->dma & AT_DMA_LO_ADDR_MASK)); + (u32)((rfd_ring->dma & AT_DMA_HI_ADDR_MASK) >> 32)); + for (i = 0; i < adapter->rx_queue_count; i++) { + AT_WRITE_REG(hw, atl1c_qregs[i].rfd_addr_lo, + (u32)(rfd_ring[i].dma & AT_DMA_LO_ADDR_MASK)); + } AT_WRITE_REG(hw, REG_RFD_RING_SIZE, rfd_ring->count & RFD_RING_SIZE_MASK); @@ -1076,8 +1135,10 @@ static void atl1c_configure_des_ring(struct atl1c_adapter *adapter) adapter->rx_buffer_len & RX_BUF_SIZE_MASK); /* RRD */ - AT_WRITE_REG(hw, REG_RRD0_HEAD_ADDR_LO, - (u32)(rrd_ring->dma & AT_DMA_LO_ADDR_MASK)); + for (i = 0; i < adapter->rx_queue_count; i++) { + AT_WRITE_REG(hw, atl1c_qregs[i].rrd_addr_lo, + (u32)(rrd_ring[i].dma & AT_DMA_LO_ADDR_MASK)); + } AT_WRITE_REG(hw, REG_RRD_RING_SIZE, (rrd_ring->count & RRD_RING_SIZE_MASK)); @@ -1358,7 +1419,7 @@ static void atl1c_set_aspm(struct atl1c_hw *hw, u16 link_speed) } /** - * atl1c_configure - Configure Transmit&Receive Unit after Reset + * atl1c_configure_mac - Configure Transmit&Receive Unit after Reset * @adapter: board private structure * * Configure the Tx /Rx unit of the MAC after a reset. @@ -1430,14 +1491,28 @@ static int atl1c_configure(struct atl1c_adapter *adapter) { struct net_device *netdev = adapter->netdev; int num; + int i; + + if (adapter->hw.nic_type == athr_mt) { + u32 mode; + + AT_READ_REG(&adapter->hw, REG_MT_MODE, &mode); + if (adapter->rx_queue_count == 4) + mode |= MT_MODE_4Q; + else + mode &= ~MT_MODE_4Q; + AT_WRITE_REG(&adapter->hw, REG_MT_MODE, mode); + } atl1c_init_ring_ptrs(adapter); atl1c_set_multi(netdev); atl1c_restore_vlan(adapter); - num = atl1c_alloc_rx_buffer(adapter, false); - if (unlikely(num == 0)) - return -ENOMEM; + for (i = 0; i < adapter->rx_queue_count; ++i) { + num = atl1c_alloc_rx_buffer(adapter, i, false); + if (unlikely(num == 0)) + return -ENOMEM; + } if (atl1c_configure_mac(adapter)) return -EIO; @@ -1533,9 +1608,11 @@ static inline void atl1c_clear_phy_int(struct atl1c_adapter *adapter) static int atl1c_clean_tx(struct napi_struct *napi, int budget) { - struct atl1c_adapter *adapter = - container_of(napi, struct atl1c_adapter, tx_napi); - struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[atl1c_trans_normal]; + struct atl1c_tpd_ring *tpd_ring = + container_of(napi, struct atl1c_tpd_ring, napi); + struct atl1c_adapter *adapter = tpd_ring->adapter; + struct netdev_queue *txq = + netdev_get_tx_queue(napi->dev, tpd_ring->num); struct atl1c_buffer *buffer_info; struct pci_dev *pdev = adapter->pdev; u16 next_to_clean = atomic_read(&tpd_ring->next_to_clean); @@ -1543,7 +1620,8 @@ static int atl1c_clean_tx(struct napi_struct *napi, int budget) unsigned int total_bytes = 0, total_packets = 0; unsigned long flags; - AT_READ_REGW(&adapter->hw, REG_TPD_PRI0_CIDX, &hw_next_to_clean); + AT_READ_REGW(&adapter->hw, atl1c_qregs[tpd_ring->num].tpd_cons, + &hw_next_to_clean); while (next_to_clean != hw_next_to_clean) { buffer_info = &tpd_ring->buffer_info[next_to_clean]; @@ -1557,17 +1635,15 @@ static int atl1c_clean_tx(struct napi_struct *napi, int budget) atomic_set(&tpd_ring->next_to_clean, next_to_clean); } - netdev_completed_queue(adapter->netdev, total_packets, total_bytes); + netdev_tx_completed_queue(txq, total_packets, total_bytes); - if (netif_queue_stopped(adapter->netdev) && - netif_carrier_ok(adapter->netdev)) { - netif_wake_queue(adapter->netdev); - } + if (netif_tx_queue_stopped(txq) && netif_carrier_ok(adapter->netdev)) + netif_tx_wake_queue(txq); if (total_packets < budget) { napi_complete_done(napi, total_packets); spin_lock_irqsave(&adapter->hw.intr_mask_lock, flags); - adapter->hw.intr_mask |= ISR_TX_PKT; + adapter->hw.intr_mask |= atl1c_qregs[tpd_ring->num].tx_isr; AT_WRITE_REG(&adapter->hw, REG_IMR, adapter->hw.intr_mask); spin_unlock_irqrestore(&adapter->hw.intr_mask_lock, flags); return total_packets; @@ -1575,6 +1651,38 @@ static int atl1c_clean_tx(struct napi_struct *napi, int budget) return budget; } +static void atl1c_intr_rx_tx(struct atl1c_adapter *adapter, u32 status) +{ + struct atl1c_hw *hw = &adapter->hw; + u32 intr_mask; + int i; + + spin_lock(&hw->intr_mask_lock); + intr_mask = hw->intr_mask; + for (i = 0; i < adapter->rx_queue_count; ++i) { + if (!(status & atl1c_qregs[i].rx_isr)) + continue; + if (napi_schedule_prep(&adapter->rrd_ring[i].napi)) { + intr_mask &= ~atl1c_qregs[i].rx_isr; + __napi_schedule(&adapter->rrd_ring[i].napi); + } + } + for (i = 0; i < adapter->tx_queue_count; ++i) { + if (!(status & atl1c_qregs[i].tx_isr)) + continue; + if (napi_schedule_prep(&adapter->tpd_ring[i].napi)) { + intr_mask &= ~atl1c_qregs[i].tx_isr; + __napi_schedule(&adapter->tpd_ring[i].napi); + } + } + + if (hw->intr_mask != intr_mask) { + hw->intr_mask = intr_mask; + AT_WRITE_REG(hw, REG_IMR, hw->intr_mask); + } + spin_unlock(&hw->intr_mask_lock); +} + /** * atl1c_intr - Interrupt Handler * @irq: interrupt number @@ -1605,24 +1713,8 @@ static irqreturn_t atl1c_intr(int irq, void *data) atl1c_clear_phy_int(adapter); /* Ack ISR */ AT_WRITE_REG(hw, REG_ISR, status | ISR_DIS_INT); - if (status & ISR_RX_PKT) { - if (likely(napi_schedule_prep(&adapter->napi))) { - spin_lock(&hw->intr_mask_lock); - hw->intr_mask &= ~ISR_RX_PKT; - AT_WRITE_REG(hw, REG_IMR, hw->intr_mask); - spin_unlock(&hw->intr_mask_lock); - __napi_schedule(&adapter->napi); - } - } - if (status & ISR_TX_PKT) { - if (napi_schedule_prep(&adapter->tx_napi)) { - spin_lock(&hw->intr_mask_lock); - hw->intr_mask &= ~ISR_TX_PKT; - AT_WRITE_REG(hw, REG_IMR, hw->intr_mask); - spin_unlock(&hw->intr_mask_lock); - __napi_schedule(&adapter->tx_napi); - } - } + if (status & (ISR_RX_PKT | ISR_TX_PKT)) + atl1c_intr_rx_tx(adapter, status); handled = IRQ_HANDLED; /* check if PCIE PHY Link down */ @@ -1659,6 +1751,11 @@ static irqreturn_t atl1c_intr(int irq, void *data) static inline void atl1c_rx_checksum(struct atl1c_adapter *adapter, struct sk_buff *skb, struct atl1c_recv_ret_status *prrs) { + if (adapter->hw.nic_type == athr_mt) { + if (prrs->word3 & RRS_MT_PROT_ID_TCPUDP) + skb->ip_summed = CHECKSUM_UNNECESSARY; + return; + } /* * The pid field in RRS in not correct sometimes, so we * cannot figure out if the packet is fragmented or not, @@ -1668,44 +1765,47 @@ static inline void atl1c_rx_checksum(struct atl1c_adapter *adapter, } static struct sk_buff *atl1c_alloc_skb(struct atl1c_adapter *adapter, - bool napi_mode) + u32 queue, bool napi_mode) { + struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring[queue]; struct sk_buff *skb; struct page *page; if (adapter->rx_frag_size > PAGE_SIZE) { if (likely(napi_mode)) - return napi_alloc_skb(&adapter->napi, + return napi_alloc_skb(&rrd_ring->napi, adapter->rx_buffer_len); else return netdev_alloc_skb_ip_align(adapter->netdev, adapter->rx_buffer_len); } - page = adapter->rx_page; + page = rrd_ring->rx_page; if (!page) { - adapter->rx_page = page = alloc_page(GFP_ATOMIC); + page = alloc_page(GFP_ATOMIC); if (unlikely(!page)) return NULL; - adapter->rx_page_offset = 0; + rrd_ring->rx_page = page; + rrd_ring->rx_page_offset = 0; } - skb = build_skb(page_address(page) + adapter->rx_page_offset, + skb = build_skb(page_address(page) + rrd_ring->rx_page_offset, adapter->rx_frag_size); if (likely(skb)) { skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); - adapter->rx_page_offset += adapter->rx_frag_size; - if (adapter->rx_page_offset >= PAGE_SIZE) - adapter->rx_page = NULL; + rrd_ring->rx_page_offset += adapter->rx_frag_size; + if (rrd_ring->rx_page_offset >= PAGE_SIZE) + rrd_ring->rx_page = NULL; else get_page(page); } return skb; } -static int atl1c_alloc_rx_buffer(struct atl1c_adapter *adapter, bool napi_mode) +static int atl1c_alloc_rx_buffer(struct atl1c_adapter *adapter, u32 queue, + bool napi_mode) { - struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring; + struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring[queue]; struct pci_dev *pdev = adapter->pdev; struct atl1c_buffer *buffer_info, *next_info; struct sk_buff *skb; @@ -1724,7 +1824,7 @@ static int atl1c_alloc_rx_buffer(struct atl1c_adapter *adapter, bool napi_mode) while (next_info->flags & ATL1C_BUFFER_FREE) { rfd_desc = ATL1C_RFD_DESC(rfd_ring, rfd_next_to_use); - skb = atl1c_alloc_skb(adapter, napi_mode); + skb = atl1c_alloc_skb(adapter, queue, napi_mode); if (unlikely(!skb)) { if (netif_msg_rx_err(adapter)) dev_warn(&pdev->dev, "alloc rx buffer failed\n"); @@ -1766,8 +1866,8 @@ static int atl1c_alloc_rx_buffer(struct atl1c_adapter *adapter, bool napi_mode) /* TODO: update mailbox here */ wmb(); rfd_ring->next_to_use = rfd_next_to_use; - AT_WRITE_REG(&adapter->hw, REG_MB_RFD0_PROD_IDX, - rfd_ring->next_to_use & MB_RFDX_PROD_IDX_MASK); + AT_WRITE_REG(&adapter->hw, atl1c_qregs[queue].rfd_prod, + rfd_ring->next_to_use & MB_RFDX_PROD_IDX_MASK); } return num_alloc; @@ -1805,22 +1905,33 @@ static void atl1c_clean_rfd(struct atl1c_rfd_ring *rfd_ring, rfd_ring->next_to_clean = rfd_index; } -static void atl1c_clean_rx_irq(struct atl1c_adapter *adapter, - int *work_done, int work_to_do) +/** + * atl1c_clean_rx - NAPI Rx polling callback + * @napi: napi info + * @budget: limit of packets to clean + */ +static int atl1c_clean_rx(struct napi_struct *napi, int budget) { + struct atl1c_rrd_ring *rrd_ring = + container_of(napi, struct atl1c_rrd_ring, napi); + struct atl1c_adapter *adapter = rrd_ring->adapter; u16 rfd_num, rfd_index; - u16 count = 0; u16 length; struct pci_dev *pdev = adapter->pdev; struct net_device *netdev = adapter->netdev; - struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring; - struct atl1c_rrd_ring *rrd_ring = &adapter->rrd_ring; + struct atl1c_rfd_ring *rfd_ring = &adapter->rfd_ring[rrd_ring->num]; struct sk_buff *skb; struct atl1c_recv_ret_status *rrs; struct atl1c_buffer *buffer_info; + int work_done = 0; + unsigned long flags; + + /* Keep link state information with original netdev */ + if (!netif_carrier_ok(adapter->netdev)) + goto quit_polling; while (1) { - if (*work_done >= work_to_do) + if (work_done >= budget) break; rrs = ATL1C_RRD_DESC(rrd_ring, rrd_ring->next_to_clean); if (likely(RRS_RXD_IS_VALID(rrs->word3))) { @@ -1874,38 +1985,18 @@ rrs_checked: vlan = le16_to_cpu(vlan); __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan); } - napi_gro_receive(&adapter->napi, skb); + napi_gro_receive(napi, skb); - (*work_done)++; - count++; + work_done++; } - if (count) - atl1c_alloc_rx_buffer(adapter, true); -} - -/** - * atl1c_clean - NAPI Rx polling callback - * @napi: napi info - * @budget: limit of packets to clean - */ -static int atl1c_clean(struct napi_struct *napi, int budget) -{ - struct atl1c_adapter *adapter = - container_of(napi, struct atl1c_adapter, napi); - int work_done = 0; - unsigned long flags; - - /* Keep link state information with original netdev */ - if (!netif_carrier_ok(adapter->netdev)) - goto quit_polling; - /* just enable one RXQ */ - atl1c_clean_rx_irq(adapter, &work_done, budget); + if (work_done) + atl1c_alloc_rx_buffer(adapter, rrd_ring->num, true); if (work_done < budget) { quit_polling: napi_complete_done(napi, work_done); spin_lock_irqsave(&adapter->hw.intr_mask_lock, flags); - adapter->hw.intr_mask |= ISR_RX_PKT; + adapter->hw.intr_mask |= atl1c_qregs[rrd_ring->num].rx_isr; AT_WRITE_REG(&adapter->hw, REG_IMR, adapter->hw.intr_mask); spin_unlock_irqrestore(&adapter->hw.intr_mask_lock, flags); } @@ -1929,9 +2020,9 @@ static void atl1c_netpoll(struct net_device *netdev) } #endif -static inline u16 atl1c_tpd_avail(struct atl1c_adapter *adapter, enum atl1c_trans_queue type) +static inline u16 atl1c_tpd_avail(struct atl1c_adapter *adapter, u32 queue) { - struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[type]; + struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[queue]; u16 next_to_use = 0; u16 next_to_clean = 0; @@ -1949,9 +2040,9 @@ static inline u16 atl1c_tpd_avail(struct atl1c_adapter *adapter, enum atl1c_tran * there is enough tpd to use */ static struct atl1c_tpd_desc *atl1c_get_tpd(struct atl1c_adapter *adapter, - enum atl1c_trans_queue type) + u32 queue) { - struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[type]; + struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[queue]; struct atl1c_tpd_desc *tpd_desc; u16 next_to_use = 0; @@ -1993,7 +2084,7 @@ static u16 atl1c_cal_tpd_req(const struct sk_buff *skb) static int atl1c_tso_csum(struct atl1c_adapter *adapter, struct sk_buff *skb, struct atl1c_tpd_desc **tpd, - enum atl1c_trans_queue type) + u32 queue) { struct pci_dev *pdev = adapter->pdev; unsigned short offload_type; @@ -2038,7 +2129,7 @@ static int atl1c_tso_csum(struct atl1c_adapter *adapter, *(struct atl1c_tpd_ext_desc **)(tpd); memset(etpd, 0, sizeof(struct atl1c_tpd_ext_desc)); - *tpd = atl1c_get_tpd(adapter, type); + *tpd = atl1c_get_tpd(adapter, queue); ipv6_hdr(skb)->payload_len = 0; /* check payload == 0 byte ? */ hdr_len = (skb_transport_offset(skb) + tcp_hdrlen(skb)); @@ -2090,9 +2181,9 @@ check_sum: static void atl1c_tx_rollback(struct atl1c_adapter *adpt, struct atl1c_tpd_desc *first_tpd, - enum atl1c_trans_queue type) + u32 queue) { - struct atl1c_tpd_ring *tpd_ring = &adpt->tpd_ring[type]; + struct atl1c_tpd_ring *tpd_ring = &adpt->tpd_ring[queue]; struct atl1c_buffer *buffer_info; struct atl1c_tpd_desc *tpd; u16 first_index, index; @@ -2111,8 +2202,8 @@ static void atl1c_tx_rollback(struct atl1c_adapter *adpt, } static int atl1c_tx_map(struct atl1c_adapter *adapter, - struct sk_buff *skb, struct atl1c_tpd_desc *tpd, - enum atl1c_trans_queue type) + struct sk_buff *skb, struct atl1c_tpd_desc *tpd, + u32 queue) { struct atl1c_tpd_desc *use_tpd = NULL; struct atl1c_buffer *buffer_info = NULL; @@ -2152,7 +2243,7 @@ static int atl1c_tx_map(struct atl1c_adapter *adapter, if (mapped_len == 0) use_tpd = tpd; else { - use_tpd = atl1c_get_tpd(adapter, type); + use_tpd = atl1c_get_tpd(adapter, queue); memcpy(use_tpd, tpd, sizeof(struct atl1c_tpd_desc)); } buffer_info = atl1c_get_tx_buffer(adapter, use_tpd); @@ -2174,7 +2265,7 @@ static int atl1c_tx_map(struct atl1c_adapter *adapter, for (f = 0; f < nr_frags; f++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; - use_tpd = atl1c_get_tpd(adapter, type); + use_tpd = atl1c_get_tpd(adapter, queue); memcpy(use_tpd, tpd, sizeof(struct atl1c_tpd_desc)); buffer_info = atl1c_get_tx_buffer(adapter, use_tpd); @@ -2207,23 +2298,22 @@ err_dma: return -1; } -static void atl1c_tx_queue(struct atl1c_adapter *adapter, struct sk_buff *skb, - struct atl1c_tpd_desc *tpd, enum atl1c_trans_queue type) +static void atl1c_tx_queue(struct atl1c_adapter *adapter, u32 queue) { - struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[type]; - u16 reg; + struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[queue]; - reg = type == atl1c_trans_high ? REG_TPD_PRI1_PIDX : REG_TPD_PRI0_PIDX; - AT_WRITE_REGW(&adapter->hw, reg, tpd_ring->next_to_use); + AT_WRITE_REGW(&adapter->hw, atl1c_qregs[queue].tpd_prod, + tpd_ring->next_to_use); } static netdev_tx_t atl1c_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct atl1c_adapter *adapter = netdev_priv(netdev); - u16 tpd_req; + u32 queue = skb_get_queue_mapping(skb); + struct netdev_queue *txq = netdev_get_tx_queue(netdev, queue); struct atl1c_tpd_desc *tpd; - enum atl1c_trans_queue type = atl1c_trans_normal; + u16 tpd_req; if (test_bit(__AT_DOWN, &adapter->flags)) { dev_kfree_skb_any(skb); @@ -2232,16 +2322,18 @@ static netdev_tx_t atl1c_xmit_frame(struct sk_buff *skb, tpd_req = atl1c_cal_tpd_req(skb); - if (atl1c_tpd_avail(adapter, type) < tpd_req) { + if (atl1c_tpd_avail(adapter, queue) < tpd_req) { /* no enough descriptor, just stop queue */ - netif_stop_queue(netdev); + atl1c_tx_queue(adapter, queue); + netif_tx_stop_queue(txq); return NETDEV_TX_BUSY; } - tpd = atl1c_get_tpd(adapter, type); + tpd = atl1c_get_tpd(adapter, queue); /* do TSO and check sum */ - if (atl1c_tso_csum(adapter, skb, &tpd, type) != 0) { + if (atl1c_tso_csum(adapter, skb, &tpd, queue) != 0) { + atl1c_tx_queue(adapter, queue); dev_kfree_skb_any(skb); return NETDEV_TX_OK; } @@ -2259,15 +2351,17 @@ static netdev_tx_t atl1c_xmit_frame(struct sk_buff *skb, if (skb_network_offset(skb) != ETH_HLEN) tpd->word1 |= 1 << TPD_ETH_TYPE_SHIFT; /* Ethernet frame */ - if (atl1c_tx_map(adapter, skb, tpd, type) < 0) { + if (atl1c_tx_map(adapter, skb, tpd, queue) < 0) { netif_info(adapter, tx_done, adapter->netdev, "tx-skb dropped due to dma error\n"); /* roll back tpd/buffer */ - atl1c_tx_rollback(adapter, tpd, type); + atl1c_tx_rollback(adapter, tpd, queue); dev_kfree_skb_any(skb); } else { - netdev_sent_queue(adapter->netdev, skb->len); - atl1c_tx_queue(adapter, skb, tpd, type); + bool more = netdev_xmit_more(); + + if (__netdev_tx_sent_queue(txq, skb->len, more)) + atl1c_tx_queue(adapter, queue); } return NETDEV_TX_OK; @@ -2321,16 +2415,19 @@ static int atl1c_request_irq(struct atl1c_adapter *adapter) static void atl1c_reset_dma_ring(struct atl1c_adapter *adapter) { + int i; /* release tx-pending skbs and reset tx/rx ring index */ - atl1c_clean_tx_ring(adapter, atl1c_trans_normal); - atl1c_clean_tx_ring(adapter, atl1c_trans_high); - atl1c_clean_rx_ring(adapter); + for (i = 0; i < adapter->tx_queue_count; ++i) + atl1c_clean_tx_ring(adapter, i); + for (i = 0; i < adapter->rx_queue_count; ++i) + atl1c_clean_rx_ring(adapter, i); } static int atl1c_up(struct atl1c_adapter *adapter) { struct net_device *netdev = adapter->netdev; int err; + int i; netif_carrier_off(netdev); @@ -2344,20 +2441,24 @@ static int atl1c_up(struct atl1c_adapter *adapter) atl1c_check_link_status(adapter); clear_bit(__AT_DOWN, &adapter->flags); - napi_enable(&adapter->napi); - napi_enable(&adapter->tx_napi); + for (i = 0; i < adapter->tx_queue_count; ++i) + napi_enable(&adapter->tpd_ring[i].napi); + for (i = 0; i < adapter->rx_queue_count; ++i) + napi_enable(&adapter->rrd_ring[i].napi); atl1c_irq_enable(adapter); netif_start_queue(netdev); return err; err_up: - atl1c_clean_rx_ring(adapter); + for (i = 0; i < adapter->rx_queue_count; ++i) + atl1c_clean_rx_ring(adapter, i); return err; } static void atl1c_down(struct atl1c_adapter *adapter) { struct net_device *netdev = adapter->netdev; + int i; atl1c_del_timer(adapter); adapter->work_event = 0; /* clear all event */ @@ -2365,8 +2466,10 @@ static void atl1c_down(struct atl1c_adapter *adapter) * reschedule our watchdog timer */ set_bit(__AT_DOWN, &adapter->flags); netif_carrier_off(netdev); - napi_disable(&adapter->napi); - napi_disable(&adapter->tx_napi); + for (i = 0; i < adapter->tx_queue_count; ++i) + napi_disable(&adapter->tpd_ring[i].napi); + for (i = 0; i < adapter->rx_queue_count; ++i) + napi_disable(&adapter->rrd_ring[i].napi); atl1c_irq_disable(adapter); atl1c_free_irq(adapter); /* disable ASPM if device inactive */ @@ -2551,8 +2654,11 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct net_device *netdev; struct atl1c_adapter *adapter; static int cards_found; - + u8 __iomem *hw_addr; + enum atl1c_nic_type nic_type; + u32 queue_count = 1; int err = 0; + int i; /* enable device (incl. PCI PM wakeup and hotplug setup) */ err = pci_enable_device_mem(pdev); @@ -2585,7 +2691,18 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_master(pdev); - netdev = alloc_etherdev(sizeof(struct atl1c_adapter)); + hw_addr = pci_ioremap_bar(pdev, 0); + if (!hw_addr) { + err = -EIO; + dev_err(&pdev->dev, "cannot map device registers\n"); + goto err_ioremap; + } + + nic_type = atl1c_get_mac_type(pdev, hw_addr); + if (nic_type == athr_mt) + queue_count = 4; + + netdev = alloc_etherdev_mq(sizeof(struct atl1c_adapter), queue_count); if (netdev == NULL) { err = -ENOMEM; goto err_alloc_etherdev; @@ -2601,13 +2718,11 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->netdev = netdev; adapter->pdev = pdev; adapter->hw.adapter = adapter; + adapter->hw.nic_type = nic_type; adapter->msg_enable = netif_msg_init(-1, atl1c_default_msg); - adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); - if (!adapter->hw.hw_addr) { - err = -EIO; - dev_err(&pdev->dev, "cannot map device registers\n"); - goto err_ioremap; - } + adapter->hw.hw_addr = hw_addr; + adapter->tx_queue_count = queue_count; + adapter->rx_queue_count = queue_count; /* init mii data */ adapter->mii.dev = netdev; @@ -2616,8 +2731,12 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->mii.phy_id_mask = 0x1f; adapter->mii.reg_num_mask = MDIO_CTRL_REG_MASK; dev_set_threaded(netdev, true); - netif_napi_add(netdev, &adapter->napi, atl1c_clean, 64); - netif_napi_add(netdev, &adapter->tx_napi, atl1c_clean_tx, 64); + for (i = 0; i < adapter->rx_queue_count; ++i) + netif_napi_add(netdev, &adapter->rrd_ring[i].napi, + atl1c_clean_rx, 64); + for (i = 0; i < adapter->tx_queue_count; ++i) + netif_napi_add(netdev, &adapter->tpd_ring[i].napi, + atl1c_clean_tx, 64); timer_setup(&adapter->phy_config_timer, atl1c_phy_config, 0); /* setup the private structure */ err = atl1c_sw_init(adapter); @@ -2670,11 +2789,11 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_reset: err_register: err_sw_init: - iounmap(adapter->hw.hw_addr); err_init_netdev: -err_ioremap: free_netdev(netdev); err_alloc_etherdev: + iounmap(hw_addr); +err_ioremap: pci_release_regions(pdev); err_pci_reg: err_dma: diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c index ff9f96de74b8..2eb0a2ab69f6 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c @@ -357,7 +357,7 @@ static void atl1e_restore_vlan(struct atl1e_adapter *adapter) } /** - * atl1e_set_mac - Change the Ethernet Address of the NIC + * atl1e_set_mac_addr - Change the Ethernet Address of the NIC * @netdev: network interface device structure * @p: pointer to an address structure * @@ -787,7 +787,7 @@ static void atl1e_free_ring_resources(struct atl1e_adapter *adapter) } /** - * atl1e_setup_mem_resources - allocate Tx / RX descriptor resources + * atl1e_setup_ring_resources - allocate Tx / RX descriptor resources * @adapter: board private structure * * Return 0 on success, negative on failure diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index eaf96d002fa5..c67201a13cf5 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -1011,7 +1011,7 @@ static int atl1_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) } /** - * atl1_setup_mem_resources - allocate Tx / RX descriptor resources + * atl1_setup_ring_resources - allocate Tx / RX descriptor resources * @adapter: board private structure * * Return 0 on success, negative on failure diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index cb88ffb8f12f..1a02ca600b71 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -206,6 +206,7 @@ config SYSTEMPORT config BNXT tristate "Broadcom NetXtreme-C/E support" depends on PCI + imply PTP_1588_CLOCK select FW_LOADER select LIBCRC32C select NET_DEVLINK diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c index b455b60a5434..ad2655efe423 100644 --- a/drivers/net/ethernet/broadcom/b44.c +++ b/drivers/net/ethernet/broadcom/b44.c @@ -1556,8 +1556,8 @@ static void b44_setup_pseudo_magicp(struct b44 *bp) plen0 = b44_magic_pattern(bp->dev->dev_addr, pwol_pattern, pwol_mask, B44_ETHIPV4UDP_HLEN); - bwfilter_table(bp, pwol_pattern, B44_PATTERN_SIZE, B44_PATTERN_BASE); - bwfilter_table(bp, pwol_mask, B44_PMASK_SIZE, B44_PMASK_BASE); + bwfilter_table(bp, pwol_pattern, B44_PATTERN_SIZE, B44_PATTERN_BASE); + bwfilter_table(bp, pwol_mask, B44_PMASK_SIZE, B44_PMASK_BASE); /* Raw ethernet II magic packet pattern - pattern 1 */ memset(pwol_pattern, 0, B44_PATTERN_SIZE); @@ -1565,9 +1565,9 @@ static void b44_setup_pseudo_magicp(struct b44 *bp) plen1 = b44_magic_pattern(bp->dev->dev_addr, pwol_pattern, pwol_mask, ETH_HLEN); - bwfilter_table(bp, pwol_pattern, B44_PATTERN_SIZE, + bwfilter_table(bp, pwol_pattern, B44_PATTERN_SIZE, B44_PATTERN_BASE + B44_PATTERN_SIZE); - bwfilter_table(bp, pwol_mask, B44_PMASK_SIZE, + bwfilter_table(bp, pwol_mask, B44_PMASK_SIZE, B44_PMASK_BASE + B44_PMASK_SIZE); /* Ipv6 magic packet pattern - pattern 2 */ @@ -1576,9 +1576,9 @@ static void b44_setup_pseudo_magicp(struct b44 *bp) plen2 = b44_magic_pattern(bp->dev->dev_addr, pwol_pattern, pwol_mask, B44_ETHIPV6UDP_HLEN); - bwfilter_table(bp, pwol_pattern, B44_PATTERN_SIZE, + bwfilter_table(bp, pwol_pattern, B44_PATTERN_SIZE, B44_PATTERN_BASE + B44_PATTERN_SIZE + B44_PATTERN_SIZE); - bwfilter_table(bp, pwol_mask, B44_PMASK_SIZE, + bwfilter_table(bp, pwol_mask, B44_PMASK_SIZE, B44_PMASK_BASE + B44_PMASK_SIZE + B44_PMASK_SIZE); kfree(pwol_pattern); @@ -1631,9 +1631,9 @@ static void b44_setup_wol(struct b44 *bp) val = br32(bp, B44_DEVCTRL); bw32(bp, B44_DEVCTRL, val | DEVCTRL_MPM | DEVCTRL_PFE); - } else { - b44_setup_pseudo_magicp(bp); - } + } else { + b44_setup_pseudo_magicp(bp); + } b44_setup_wol_pci(bp); } @@ -1757,7 +1757,7 @@ static void __b44_set_rx_mode(struct net_device *dev) __b44_cam_write(bp, zero, i); bw32(bp, B44_RXCONFIG, val); - val = br32(bp, B44_CAM_CTRL); + val = br32(bp, B44_CAM_CTRL); bw32(bp, B44_CAM_CTRL, val | CAM_CTRL_ENABLE); } } diff --git a/drivers/net/ethernet/broadcom/bcm4908_enet.c b/drivers/net/ethernet/broadcom/bcm4908_enet.c index 60d908507f51..02a569500234 100644 --- a/drivers/net/ethernet/broadcom/bcm4908_enet.c +++ b/drivers/net/ethernet/broadcom/bcm4908_enet.c @@ -174,9 +174,6 @@ static int bcm4908_dma_alloc_buf_descs(struct bcm4908_enet *enet, if (!ring->slots) goto err_free_buf_descs; - ring->read_idx = 0; - ring->write_idx = 0; - return 0; err_free_buf_descs: @@ -304,6 +301,9 @@ static void bcm4908_enet_dma_ring_init(struct bcm4908_enet *enet, enet_write(enet, ring->st_ram_block + ENET_DMA_CH_STATE_RAM_BASE_DESC_PTR, (uint32_t)ring->dma_addr); + + ring->read_idx = 0; + ring->write_idx = 0; } static void bcm4908_enet_dma_uninit(struct bcm4908_enet *enet) diff --git a/drivers/net/ethernet/broadcom/bgmac-platform.c b/drivers/net/ethernet/broadcom/bgmac-platform.c index 9834b77cf4b6..4ab5bf64d353 100644 --- a/drivers/net/ethernet/broadcom/bgmac-platform.c +++ b/drivers/net/ethernet/broadcom/bgmac-platform.c @@ -172,7 +172,6 @@ static int bgmac_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct bgmac *bgmac; - struct resource *regs; int ret; bgmac = bgmac_alloc(&pdev->dev); @@ -206,21 +205,15 @@ static int bgmac_probe(struct platform_device *pdev) if (IS_ERR(bgmac->plat.base)) return PTR_ERR(bgmac->plat.base); - regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "idm_base"); - if (regs) { - bgmac->plat.idm_base = devm_ioremap_resource(&pdev->dev, regs); - if (IS_ERR(bgmac->plat.idm_base)) - return PTR_ERR(bgmac->plat.idm_base); + bgmac->plat.idm_base = devm_platform_ioremap_resource_byname(pdev, "idm_base"); + if (IS_ERR(bgmac->plat.idm_base)) + return PTR_ERR(bgmac->plat.idm_base); + else bgmac->feature_flags &= ~BGMAC_FEAT_IDM_MASK; - } - regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nicpm_base"); - if (regs) { - bgmac->plat.nicpm_base = devm_ioremap_resource(&pdev->dev, - regs); - if (IS_ERR(bgmac->plat.nicpm_base)) - return PTR_ERR(bgmac->plat.nicpm_base); - } + bgmac->plat.nicpm_base = devm_platform_ioremap_resource_byname(pdev, "nicpm_base"); + if (IS_ERR(bgmac->plat.nicpm_base)) + return PTR_ERR(bgmac->plat.nicpm_base); bgmac->read = platform_bgmac_read; bgmac->write = platform_bgmac_write; diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index c0986096c701..bee6cfad9fc6 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -572,7 +572,7 @@ bnx2_write_phy(struct bnx2 *bp, u32 reg, u32 val) } if (val1 & BNX2_EMAC_MDIO_COMM_START_BUSY) - ret = -EBUSY; + ret = -EBUSY; else ret = 0; @@ -3599,7 +3599,7 @@ bnx2_set_rx_mode(struct net_device *dev) for (i = 0; i < NUM_MC_HASH_REGISTERS; i++) { BNX2_WR(bp, BNX2_EMAC_MULTICAST_HASH0 + (i * 4), 0xffffffff); - } + } sort_mode |= BNX2_RPM_SORT_USER0_MC_EN; } else { @@ -4674,7 +4674,7 @@ bnx2_nvram_write(struct bnx2 *bp, u32 offset, u8 *data_buf, if (addr == page_end-4) { cmd_flags = BNX2_NVM_COMMAND_LAST; - } + } rc = bnx2_nvram_write_dword(bp, addr, &flash_buffer[i], cmd_flags); @@ -8247,9 +8247,9 @@ bnx2_init_board(struct pci_dev *pdev, struct net_device *dev) BNX2_WR(bp, PCI_COMMAND, reg); } else if ((BNX2_CHIP_ID(bp) == BNX2_CHIP_ID_5706_A1) && !(bp->flags & BNX2_FLAG_PCIX)) { - dev_err(&pdev->dev, "5706 A1 can only be used in a PCIX bus, aborting\n"); + rc = -EPERM; goto err_out_unmap; } diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 281b1c2e04a7..2acbc73dcd18 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -13586,7 +13586,7 @@ static int bnx2x_set_qm_cid_count(struct bnx2x *bp) } /** - * bnx2x_get_num_none_def_sbs - return the number of none default SBs + * bnx2x_get_num_non_def_sbs - return the number of none default SBs * @pdev: pci device * @cnic_cnt: count * diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c index 6cd1523ad9e5..542c69822649 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sp.c @@ -4152,7 +4152,7 @@ void bnx2x_init_mcast_obj(struct bnx2x *bp, /*************************** Credit handling **********************************/ /** - * atomic_add_ifless - add if the result is less than a given value. + * __atomic_add_ifless - add if the result is less than a given value. * * @v: pointer of type atomic_t * @a: the amount to add to v... @@ -4180,7 +4180,7 @@ static inline bool __atomic_add_ifless(atomic_t *v, int a, int u) } /** - * atomic_dec_ifmoe - dec if the result is more or equal than a given value. + * __atomic_dec_ifmoe - dec if the result is more or equal than a given value. * * @v: pointer of type atomic_t * @a: the amount to dec from v... diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c index d21f085044cd..27943b0446c2 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c @@ -1223,8 +1223,10 @@ int bnx2x_iov_init_one(struct bnx2x *bp, int int_mode_param, goto failed; /* SR-IOV capability was enabled but there are no VFs*/ - if (iov->total == 0) + if (iov->total == 0) { + err = -EINVAL; goto failed; + } iov->nr_virtfn = min_t(u16, iov->total, num_vfs_param); diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h index 3a716c015415..966d5722c5e2 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h @@ -504,7 +504,6 @@ enum sample_bulletin_result bnx2x_sample_bulletin(struct bnx2x *bp); /* VF side vfpf channel functions */ int bnx2x_vfpf_acquire(struct bnx2x *bp, u8 tx_count, u8 rx_count); int bnx2x_vfpf_release(struct bnx2x *bp); -int bnx2x_vfpf_release(struct bnx2x *bp); int bnx2x_vfpf_init(struct bnx2x *bp); void bnx2x_vfpf_close_vf(struct bnx2x *bp); int bnx2x_vfpf_setup_q(struct bnx2x *bp, struct bnx2x_fastpath *fp, diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile index cb97ec56fdec..2b8ae687b3c1 100644 --- a/drivers/net/ethernet/broadcom/bnxt/Makefile +++ b/drivers/net/ethernet/broadcom/bnxt/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_BNXT) += bnxt_en.o -bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o +bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_ptp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o bnxt_en-$(CONFIG_DEBUG_FS) += bnxt_debugfs.o diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 2985844634c8..f56245eeef7b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -49,6 +49,8 @@ #include <linux/log2.h> #include <linux/aer.h> #include <linux/bitmap.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/timecounter.h> #include <linux/cpu_rmap.h> #include <linux/cpumask.h> #include <net/pkt_cls.h> @@ -63,6 +65,7 @@ #include "bnxt_ethtool.h" #include "bnxt_dcb.h" #include "bnxt_xdp.h" +#include "bnxt_ptp.h" #include "bnxt_vfr.h" #include "bnxt_tc.h" #include "bnxt_devlink.h" @@ -282,7 +285,8 @@ static bool bnxt_vf_pciid(enum board_idx idx) { return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || - idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF); + idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || + idx == NETXTREME_E_P5_VF_HV); } #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) @@ -417,12 +421,25 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; } - if (unlikely(skb->no_fcs)) { - lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); - goto normal_tx; + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + + if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) && + atomic_dec_if_positive(&ptp->tx_avail) >= 0) { + if (!bnxt_ptp_parse(skb, &ptp->tx_seqid)) { + lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + } else { + atomic_inc(&bp->ptp_cfg->tx_avail); + } + } } - if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) { + if (unlikely(skb->no_fcs)) + lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); + + if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && + !lflags) { struct tx_push_buffer *tx_push_buf = txr->tx_push; struct tx_push_bd *tx_push = &tx_push_buf->push_bd; struct tx_bd_ext *tx_push1 = &tx_push->txbd2; @@ -589,6 +606,8 @@ normal_tx: netdev_tx_sent_queue(txq, skb->len); + skb_tx_timestamp(skb); + /* Sync BD data before updating doorbell */ wmb(); @@ -618,6 +637,9 @@ tx_done: return NETDEV_TX_OK; tx_dma_error: + if (BNXT_TX_PTP_IS_SET(lflags)) + atomic_inc(&bp->ptp_cfg->tx_avail); + last_frag = i; /* start back at beginning and unmap skb */ @@ -652,6 +674,7 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) for (i = 0; i < nr_pkts; i++) { struct bnxt_sw_tx_bd *tx_buf; + bool compl_deferred = false; struct sk_buff *skb; int j, last; @@ -678,12 +701,21 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) skb_frag_size(&skb_shinfo(skb)->frags[j]), PCI_DMA_TODEVICE); } + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { + if (bp->flags & BNXT_FLAG_CHIP_P5) { + if (!bnxt_get_tx_ts_p5(bp, skb)) + compl_deferred = true; + else + atomic_inc(&bp->ptp_cfg->tx_avail); + } + } next_tx_int: cons = NEXT_TX(cons); tx_bytes += skb->len; - dev_kfree_skb_any(skb); + if (!compl_deferred) + dev_kfree_skb_any(skb); } netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); @@ -1705,9 +1737,9 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, u8 *data_ptr, agg_bufs, cmp_type; dma_addr_t dma_addr; struct sk_buff *skb; + u32 flags, misc; void *data; int rc = 0; - u32 misc; rxcmp = (struct rx_cmp *) &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; @@ -1805,7 +1837,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, goto next_rx_no_len; } - len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT; + flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); + len = flags >> RX_CMP_LEN_SHIFT; dma_addr = rx_buf->mapping; if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) { @@ -1882,6 +1915,24 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } } + if (unlikely((flags & RX_CMP_FLAGS_ITYPES_MASK) == + RX_CMP_FLAGS_ITYPE_PTP_W_TS)) { + if (bp->flags & BNXT_FLAG_CHIP_P5) { + u32 cmpl_ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); + u64 ns, ts; + + if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + + spin_lock_bh(&ptp->ptp_lock); + ns = timecounter_cyc2time(&ptp->tc, ts); + spin_unlock_bh(&ptp->ptp_lock); + memset(skb_hwtstamps(skb), 0, + sizeof(*skb_hwtstamps(skb))); + skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); + } + } + } bnxt_deliver_skb(bp, bnapi, skb); rc = 1; @@ -2183,6 +2234,7 @@ static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: bnxt_async_event_process(bp, (struct hwrm_async_event_cmpl *)txcmp); + break; default: break; @@ -6932,17 +6984,10 @@ ctx_err: static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, __le64 *pg_dir) { - u8 pg_size = 0; - if (!rmem->nr_pages) return; - if (BNXT_PAGE_SHIFT == 13) - pg_size = 1 << 4; - else if (BNXT_PAGE_SIZE == 16) - pg_size = 2 << 4; - - *pg_attr = pg_size; + BNXT_SET_CTX_PAGE_ATTR(*pg_attr); if (rmem->depth >= 1) { if (rmem->depth == 2) *pg_attr |= 2; @@ -7314,7 +7359,7 @@ skip_rdma: entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries) + min; entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple); - entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries; + entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries); entries = roundup(entries, ctx->tqm_entries_multiple); entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring); for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { @@ -7397,6 +7442,56 @@ hwrm_func_resc_qcaps_exit: return rc; } +/* bp->hwrm_cmd_lock already held. */ +static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) +{ + struct hwrm_port_mac_ptp_qcfg_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_port_mac_ptp_qcfg_input req = {0}; + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + u8 flags; + int rc; + + if (bp->hwrm_spec_code < 0x10801) { + rc = -ENODEV; + goto no_ptp; + } + + req.port_id = cpu_to_le16(bp->pf.port_id); + bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_MAC_PTP_QCFG, -1, -1); + rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); + if (rc) + goto no_ptp; + + flags = resp->flags; + if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { + rc = -ENODEV; + goto no_ptp; + } + if (!ptp) { + ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); + if (!ptp) + return -ENOMEM; + ptp->bp = bp; + bp->ptp_cfg = ptp; + } + if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) { + ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); + ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); + } else if (bp->flags & BNXT_FLAG_CHIP_P5) { + ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; + ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; + } else { + rc = -ENODEV; + goto no_ptp; + } + return 0; + +no_ptp: + kfree(ptp); + bp->ptp_cfg = NULL; + return rc; +} + static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) { int rc = 0; @@ -7468,6 +7563,8 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) bp->flags &= ~BNXT_FLAG_WOL_CAP; if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) bp->flags |= BNXT_FLAG_WOL_CAP; + if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) + __bnxt_hwrm_ptp_qcfg(bp); } else { #ifdef CONFIG_BNXT_SRIOV struct bnxt_vf_info *vf = &bp->vf; @@ -10026,6 +10123,7 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) } } + bnxt_ptp_start(bp); rc = bnxt_init_nic(bp, irq_re_init); if (rc) { netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); @@ -10341,6 +10439,12 @@ static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, mdio->val_in); + case SIOCSHWTSTAMP: + return bnxt_hwtstamp_set(dev, ifr); + + case SIOCGHWTSTAMP: + return bnxt_hwtstamp_get(dev, ifr); + default: /* do nothing */ break; @@ -10785,37 +10889,125 @@ static int bnxt_set_features(struct net_device *dev, netdev_features_t features) return rc; } +static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, + u8 **nextp) +{ + struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); + int hdr_count = 0; + u8 *nexthdr; + int start; + + /* Check that there are at most 2 IPv6 extension headers, no + * fragment header, and each is <= 64 bytes. + */ + start = nw_off + sizeof(*ip6h); + nexthdr = &ip6h->nexthdr; + while (ipv6_ext_hdr(*nexthdr)) { + struct ipv6_opt_hdr *hp; + int hdrlen; + + if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || + *nexthdr == NEXTHDR_FRAGMENT) + return false; + hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, + skb_headlen(skb), NULL); + if (!hp) + return false; + if (*nexthdr == NEXTHDR_AUTH) + hdrlen = ipv6_authlen(hp); + else + hdrlen = ipv6_optlen(hp); + + if (hdrlen > 64) + return false; + nexthdr = &hp->nexthdr; + start += hdrlen; + hdr_count++; + } + if (nextp) { + /* Caller will check inner protocol */ + if (skb->encapsulation) { + *nextp = nexthdr; + return true; + } + *nextp = NULL; + } + /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ + return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; +} + +/* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ +static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) +{ + struct udphdr *uh = udp_hdr(skb); + __be16 udp_port = uh->dest; + + if (udp_port != bp->vxlan_port && udp_port != bp->nge_port) + return false; + if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) { + struct ethhdr *eh = inner_eth_hdr(skb); + + switch (eh->h_proto) { + case htons(ETH_P_IP): + return true; + case htons(ETH_P_IPV6): + return bnxt_exthdr_check(bp, skb, + skb_inner_network_offset(skb), + NULL); + } + } + return false; +} + +static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) +{ + switch (l4_proto) { + case IPPROTO_UDP: + return bnxt_udp_tunl_check(bp, skb); + case IPPROTO_IPIP: + return true; + case IPPROTO_GRE: { + switch (skb->inner_protocol) { + default: + return false; + case htons(ETH_P_IP): + return true; + case htons(ETH_P_IPV6): + fallthrough; + } + } + case IPPROTO_IPV6: + /* Check ext headers of inner ipv6 */ + return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), + NULL); + } + return false; +} + static netdev_features_t bnxt_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) { - struct bnxt *bp; - __be16 udp_port; - u8 l4_proto = 0; + struct bnxt *bp = netdev_priv(dev); + u8 *l4_proto; features = vlan_features_check(skb, features); - if (!skb->encapsulation) - return features; - switch (vlan_get_protocol(skb)) { case htons(ETH_P_IP): - l4_proto = ip_hdr(skb)->protocol; + if (!skb->encapsulation) + return features; + l4_proto = &ip_hdr(skb)->protocol; + if (bnxt_tunl_check(bp, skb, *l4_proto)) + return features; break; case htons(ETH_P_IPV6): - l4_proto = ipv6_hdr(skb)->nexthdr; + if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), + &l4_proto)) + break; + if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) + return features; break; - default: - return features; } - - if (l4_proto != IPPROTO_UDP) - return features; - - bp = netdev_priv(dev); - /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ - udp_port = udp_hdr(skb)->dest; - if (udp_port == bp->vxlan_port || udp_port == bp->nge_port) - return features; return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); } @@ -11668,6 +11860,8 @@ static void bnxt_fw_init_one_p3(struct bnxt *bp) bnxt_hwrm_coal_params_qcaps(bp); } +static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); + static int bnxt_fw_init_one(struct bnxt *bp) { int rc; @@ -11682,6 +11876,9 @@ static int bnxt_fw_init_one(struct bnxt *bp) netdev_err(bp->dev, "Firmware init phase 2 failed\n"); return rc; } + rc = bnxt_probe_phy(bp, false); + if (rc) + return rc; rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); if (rc) return rc; @@ -12464,6 +12661,8 @@ static void bnxt_remove_one(struct pci_dev *pdev) if (BNXT_PF(bp)) devlink_port_type_clear(&bp->dl_port); + + bnxt_ptp_clear(bp); pci_disable_pcie_error_reporting(pdev); unregister_netdev(dev); clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); @@ -12484,6 +12683,8 @@ static void bnxt_remove_one(struct pci_dev *pdev) bnxt_dcb_free(bp); kfree(bp->edev); bp->edev = NULL; + kfree(bp->ptp_cfg); + bp->ptp_cfg = NULL; kfree(bp->fw_health); bp->fw_health = NULL; bnxt_cleanup_pci(bp); @@ -13045,6 +13246,11 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) rc); } + if (bnxt_ptp_init(bp)) { + netdev_warn(dev, "PTP initialization failed.\n"); + kfree(bp->ptp_cfg); + bp->ptp_cfg = NULL; + } bnxt_inv_fw_health_reg(bp); bnxt_dl_register(bp); @@ -13073,6 +13279,9 @@ init_err_pci_clean: bnxt_hwrm_func_drv_unrgtr(bp); bnxt_free_hwrm_short_cmd_req(bp); bnxt_free_hwrm_resources(bp); + bnxt_ethtool_free(bp); + kfree(bp->ptp_cfg); + bp->ptp_cfg = NULL; kfree(bp->fw_health); bp->fw_health = NULL; bnxt_cleanup_pci(bp); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 98e0cef4532c..bcf8d00b8c80 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -89,6 +89,8 @@ struct tx_bd_ext { #define TX_BD_CFA_META_KEY_VLAN (1 << 28) }; +#define BNXT_TX_PTP_IS_SET(lflags) ((lflags) & cpu_to_le32(TX_BD_FLAGS_STAMP)) + struct rx_bd { __le32 rx_bd_len_flags_type; #define RX_BD_TYPE (0x3f << 0) @@ -159,6 +161,7 @@ struct rx_cmp { #define RX_CMP_FLAGS_RSS_VALID (1 << 10) #define RX_CMP_FLAGS_UNUSED (1 << 11) #define RX_CMP_FLAGS_ITYPES_SHIFT 12 + #define RX_CMP_FLAGS_ITYPES_MASK 0xf000 #define RX_CMP_FLAGS_ITYPE_UNKNOWN (0 << 12) #define RX_CMP_FLAGS_ITYPE_IP (1 << 12) #define RX_CMP_FLAGS_ITYPE_TCP (2 << 12) @@ -240,7 +243,7 @@ struct rx_cmp_ext { #define RX_CMPL_CFA_CODE_MASK (0xffff << 16) #define RX_CMPL_CFA_CODE_SFT 16 - __le32 rx_cmp_unused3; + __le32 rx_cmp_timestamp; }; #define RX_CMP_L2_ERRORS \ @@ -1362,6 +1365,9 @@ struct bnxt_test_info { #define BNXT_GRC_REG_CHIP_NUM 0x48 #define BNXT_GRC_REG_BASE 0x260000 +#define BNXT_TS_REG_TIMESYNC_TS0_LOWER 0x640180c +#define BNXT_TS_REG_TIMESYNC_TS0_UPPER 0x6401810 + #define BNXT_GRC_BASE_MASK 0xfffff000 #define BNXT_GRC_OFFSET_MASK 0x00000ffc @@ -1457,6 +1463,16 @@ struct bnxt_ctx_pg_info { #define BNXT_BACKING_STORE_CFG_LEGACY_LEN 256 +#define BNXT_SET_CTX_PAGE_ATTR(attr) \ +do { \ + if (BNXT_PAGE_SIZE == 0x2000) \ + attr = FUNC_BACKING_STORE_CFG_REQ_SRQ_PG_SIZE_PG_8K; \ + else if (BNXT_PAGE_SIZE == 0x10000) \ + attr = FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_64K; \ + else \ + attr = FUNC_BACKING_STORE_CFG_REQ_QPC_PG_SIZE_PG_4K; \ +} while (0) + struct bnxt_ctx_mem_info { u32 qp_max_entries; u16 qp_min_qp1_entries; @@ -2032,6 +2048,8 @@ struct bnxt { struct bpf_prog *xdp_prog; + struct bnxt_ptp_cfg *ptp_cfg; + /* devlink interface and vf-rep structs */ struct devlink *dl; struct devlink_port dl_port; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index c664ec52ebcf..786ca51e669b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -19,9 +19,13 @@ #include <linux/firmware.h> #include <linux/utsname.h> #include <linux/time.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/net_tstamp.h> +#include <linux/timecounter.h> #include "bnxt_hsi.h" #include "bnxt.h" #include "bnxt_xdp.h" +#include "bnxt_ptp.h" #include "bnxt_ethtool.h" #include "bnxt_nvm_defs.h" /* NVRAM content constant and structure defs */ #include "bnxt_fw_hdr.h" /* Firmware hdr constant and structure defs */ @@ -3926,6 +3930,35 @@ static int bnxt_get_dump_data(struct net_device *dev, struct ethtool_dump *dump, return 0; } +static int bnxt_get_ts_info(struct net_device *dev, + struct ethtool_ts_info *info) +{ + struct bnxt *bp = netdev_priv(dev); + struct bnxt_ptp_cfg *ptp; + + ptp = bp->ptp_cfg; + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE; + + info->phc_index = -1; + if (!ptp) + return 0; + + info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + if (ptp->ptp_clock) + info->phc_index = ptp_clock_index(ptp->ptp_clock); + + info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); + + info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | + (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | + (1 << HWTSTAMP_FILTER_PTP_V2_L4_EVENT); + return 0; +} + void bnxt_ethtool_init(struct bnxt *bp) { struct hwrm_selftest_qlist_output *resp = bp->hwrm_cmd_resp_addr; @@ -4172,6 +4205,7 @@ const struct ethtool_ops bnxt_ethtool_ops = { .nway_reset = bnxt_nway_reset, .set_phys_id = bnxt_set_phys_id, .self_test = bnxt_self_test, + .get_ts_info = bnxt_get_ts_info, .reset = bnxt_reset, .set_dump = bnxt_set_dump, .get_dump_flag = bnxt_get_dump_flag, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h index 6199f125bc13..3fc6781c5b98 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h @@ -189,6 +189,8 @@ struct cmd_nums { #define HWRM_QUEUE_VLANPRI_QCAPS 0x83UL #define HWRM_QUEUE_VLANPRI2PRI_QCFG 0x84UL #define HWRM_QUEUE_VLANPRI2PRI_CFG 0x85UL + #define HWRM_QUEUE_GLOBAL_CFG 0x86UL + #define HWRM_QUEUE_GLOBAL_QCFG 0x87UL #define HWRM_CFA_L2_FILTER_ALLOC 0x90UL #define HWRM_CFA_L2_FILTER_FREE 0x91UL #define HWRM_CFA_L2_FILTER_CFG 0x92UL @@ -250,6 +252,8 @@ struct cmd_nums { #define HWRM_PORT_SFP_SIDEBAND_QCFG 0xd7UL #define HWRM_FW_STATE_UNQUIESCE 0xd8UL #define HWRM_PORT_DSC_DUMP 0xd9UL + #define HWRM_PORT_EP_TX_QCFG 0xdaUL + #define HWRM_PORT_EP_TX_CFG 0xdbUL #define HWRM_TEMP_MONITOR_QUERY 0xe0UL #define HWRM_REG_POWER_QUERY 0xe1UL #define HWRM_CORE_FREQUENCY_QUERY 0xe2UL @@ -305,6 +309,8 @@ struct cmd_nums { #define HWRM_CFA_EEM_OP 0x123UL #define HWRM_CFA_ADV_FLOW_MGNT_QCAPS 0x124UL #define HWRM_CFA_TFLIB 0x125UL + #define HWRM_CFA_LAG_GROUP_MEMBER_RGTR 0x126UL + #define HWRM_CFA_LAG_GROUP_MEMBER_UNRGTR 0x127UL #define HWRM_ENGINE_CKV_STATUS 0x12eUL #define HWRM_ENGINE_CKV_CKEK_ADD 0x12fUL #define HWRM_ENGINE_CKV_CKEK_DELETE 0x130UL @@ -356,6 +362,12 @@ struct cmd_nums { #define HWRM_STAT_EXT_CTX_QUERY 0x199UL #define HWRM_FUNC_SPD_CFG 0x19aUL #define HWRM_FUNC_SPD_QCFG 0x19bUL + #define HWRM_FUNC_PTP_PIN_QCFG 0x19cUL + #define HWRM_FUNC_PTP_PIN_CFG 0x19dUL + #define HWRM_FUNC_PTP_CFG 0x19eUL + #define HWRM_FUNC_PTP_TS_QUERY 0x19fUL + #define HWRM_FUNC_PTP_EXT_CFG 0x1a0UL + #define HWRM_FUNC_PTP_EXT_QCFG 0x1a1UL #define HWRM_SELFTEST_QLIST 0x200UL #define HWRM_SELFTEST_EXEC 0x201UL #define HWRM_SELFTEST_IRQ 0x202UL @@ -373,6 +385,10 @@ struct cmd_nums { #define HWRM_MFG_PARAM_SEEPROM_SYNC 0x20eUL #define HWRM_MFG_PARAM_SEEPROM_READ 0x20fUL #define HWRM_MFG_PARAM_SEEPROM_HEALTH 0x210UL + #define HWRM_MFG_PRVSN_EXPORT_CSR 0x211UL + #define HWRM_MFG_PRVSN_IMPORT_CERT 0x212UL + #define HWRM_MFG_PRVSN_GET_STATE 0x213UL + #define HWRM_MFG_GET_NVM_MEASUREMENT 0x214UL #define HWRM_TF 0x2bcUL #define HWRM_TF_VERSION_GET 0x2bdUL #define HWRM_TF_SESSION_OPEN 0x2c6UL @@ -385,6 +401,7 @@ struct cmd_nums { #define HWRM_TF_SESSION_RESC_ALLOC 0x2cdUL #define HWRM_TF_SESSION_RESC_FREE 0x2ceUL #define HWRM_TF_SESSION_RESC_FLUSH 0x2cfUL + #define HWRM_TF_SESSION_RESC_INFO 0x2d0UL #define HWRM_TF_TBL_TYPE_GET 0x2daUL #define HWRM_TF_TBL_TYPE_SET 0x2dbUL #define HWRM_TF_TBL_TYPE_BULK_GET 0x2dcUL @@ -399,6 +416,7 @@ struct cmd_nums { #define HWRM_TF_EM_INSERT 0x2eaUL #define HWRM_TF_EM_DELETE 0x2ebUL #define HWRM_TF_EM_HASH_INSERT 0x2ecUL + #define HWRM_TF_EM_MOVE 0x2edUL #define HWRM_TF_TCAM_SET 0x2f8UL #define HWRM_TF_TCAM_GET 0x2f9UL #define HWRM_TF_TCAM_MOVE 0x2faUL @@ -427,6 +445,16 @@ struct cmd_nums { #define HWRM_DBG_QCAPS 0xff20UL #define HWRM_DBG_QCFG 0xff21UL #define HWRM_DBG_CRASHDUMP_MEDIUM_CFG 0xff22UL + #define HWRM_DBG_USEQ_ALLOC 0xff23UL + #define HWRM_DBG_USEQ_FREE 0xff24UL + #define HWRM_DBG_USEQ_FLUSH 0xff25UL + #define HWRM_DBG_USEQ_QCAPS 0xff26UL + #define HWRM_DBG_USEQ_CW_CFG 0xff27UL + #define HWRM_DBG_USEQ_SCHED_CFG 0xff28UL + #define HWRM_DBG_USEQ_RUN 0xff29UL + #define HWRM_DBG_USEQ_DELIVERY_REQ 0xff2aUL + #define HWRM_DBG_USEQ_RESP_HDR 0xff2bUL + #define HWRM_NVM_DEFRAG 0xffecUL #define HWRM_NVM_REQ_ARBITRATION 0xffedUL #define HWRM_NVM_FACTORY_DEFAULTS 0xffeeUL #define HWRM_NVM_VALIDATE_OPTION 0xffefUL @@ -471,6 +499,7 @@ struct ret_codes { #define HWRM_ERR_CODE_HWRM_ERROR 0xfUL #define HWRM_ERR_CODE_BUSY 0x10UL #define HWRM_ERR_CODE_RESOURCE_LOCKED 0x11UL + #define HWRM_ERR_CODE_PF_UNAVAILABLE 0x12UL #define HWRM_ERR_CODE_TLV_ENCAPSULATED_RESPONSE 0x8000UL #define HWRM_ERR_CODE_UNKNOWN_ERR 0xfffeUL #define HWRM_ERR_CODE_CMD_NOT_SUPPORTED 0xffffUL @@ -502,8 +531,8 @@ struct hwrm_err_output { #define HWRM_VERSION_MAJOR 1 #define HWRM_VERSION_MINOR 10 #define HWRM_VERSION_UPDATE 2 -#define HWRM_VERSION_RSVD 16 -#define HWRM_VERSION_STR "1.10.2.16" +#define HWRM_VERSION_RSVD 47 +#define HWRM_VERSION_STR "1.10.2.47" /* hwrm_ver_get_input (size:192b/24B) */ struct hwrm_ver_get_input { @@ -604,7 +633,8 @@ struct hwrm_ver_get_output { __le16 roce_fw_build; __le16 roce_fw_patch; __le16 max_ext_req_len; - u8 unused_1[5]; + __le16 max_req_timeout; + u8 unused_1[3]; u8 valid; }; @@ -725,7 +755,10 @@ struct hwrm_async_event_cmpl { #define ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE 0x40UL #define ASYNC_EVENT_CMPL_EVENT_ID_PFC_WATCHDOG_CFG_CHANGE 0x41UL #define ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST 0x42UL - #define ASYNC_EVENT_CMPL_EVENT_ID_MAX_RGTR_EVENT_ID 0x43UL + #define ASYNC_EVENT_CMPL_EVENT_ID_PHC_MASTER 0x43UL + #define ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP 0x44UL + #define ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT 0x45UL + #define ASYNC_EVENT_CMPL_EVENT_ID_MAX_RGTR_EVENT_ID 0x46UL #define ASYNC_EVENT_CMPL_EVENT_ID_FW_TRACE_MSG 0xfeUL #define ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR 0xffUL #define ASYNC_EVENT_CMPL_EVENT_ID_LAST ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR @@ -919,6 +952,8 @@ struct hwrm_async_event_cmpl_vf_cfg_change { #define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE 0x33UL #define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_LAST ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE __le32 event_data2; + #define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA2_VF_ID_MASK 0xffffUL + #define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA2_VF_ID_SFT 0 u8 opaque_v; #define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_V 0x1UL #define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_MASK 0xfeUL @@ -1074,6 +1109,223 @@ struct hwrm_async_event_cmpl_echo_request { __le32 event_data1; }; +/* hwrm_async_event_cmpl_phc_master (size:128b/16B) */ +struct hwrm_async_event_cmpl_phc_master { + __le16 type; + #define ASYNC_EVENT_CMPL_PHC_MASTER_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_PHC_MASTER_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_PHC_MASTER_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_PHC_MASTER_TYPE_LAST ASYNC_EVENT_CMPL_PHC_MASTER_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_ID_PHC_MASTER 0x43UL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_ID_LAST ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_ID_PHC_MASTER + __le32 event_data2; + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA2_PHC_MASTER_FID_MASK 0xffffUL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA2_PHC_MASTER_FID_SFT 0 + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA2_PHC_SEC_FID_MASK 0xffff0000UL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA2_PHC_SEC_FID_SFT 16 + u8 opaque_v; + #define ASYNC_EVENT_CMPL_PHC_MASTER_V 0x1UL + #define ASYNC_EVENT_CMPL_PHC_MASTER_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_PHC_MASTER_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_MASK 0xfUL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_SFT 0 + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_PHC_MASTER 0x1UL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_PHC_SECONDARY 0x2UL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_PHC_FAILOVER 0x3UL + #define ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_LAST ASYNC_EVENT_CMPL_PHC_MASTER_EVENT_DATA1_FLAGS_PHC_FAILOVER +}; + +/* hwrm_async_event_cmpl_pps_timestamp (size:128b/16B) */ +struct hwrm_async_event_cmpl_pps_timestamp { + __le16 type; + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_LAST ASYNC_EVENT_CMPL_PPS_TIMESTAMP_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_ID_PPS_TIMESTAMP 0x44UL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_ID_LAST ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_ID_PPS_TIMESTAMP + __le32 event_data2; + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE 0x1UL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_INTERNAL 0x0UL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_EXTERNAL 0x1UL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_LAST ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_EVENT_TYPE_EXTERNAL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PIN_NUMBER_MASK 0xeUL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PIN_NUMBER_SFT 1 + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PPS_TIMESTAMP_UPPER_MASK 0xffff0UL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA2_PPS_TIMESTAMP_UPPER_SFT 4 + u8 opaque_v; + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_V 0x1UL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA1_PPS_TIMESTAMP_LOWER_MASK 0xffffffffUL + #define ASYNC_EVENT_CMPL_PPS_TIMESTAMP_EVENT_DATA1_PPS_TIMESTAMP_LOWER_SFT 0 +}; + +/* hwrm_async_event_cmpl_error_report (size:128b/16B) */ +struct hwrm_async_event_cmpl_error_report { + __le16 type; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_ID_ERROR_REPORT 0x45UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_ID_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_ID_ERROR_REPORT + __le32 event_data2; + u8 opaque_v; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_V 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_DATA1_ERROR_TYPE_MASK 0xffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_EVENT_DATA1_ERROR_TYPE_SFT 0 +}; + +/* hwrm_async_event_cmpl_hwrm_error (size:128b/16B) */ +struct hwrm_async_event_cmpl_hwrm_error { + __le16 type; + #define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_LAST ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR 0xffUL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_LAST ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR + __le32 event_data2; + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_MASK 0xffUL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_SFT 0 + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_WARNING 0x0UL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_NONFATAL 0x1UL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL 0x2UL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_LAST ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL + u8 opaque_v; + #define ASYNC_EVENT_CMPL_HWRM_ERROR_V 0x1UL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_HWRM_ERROR_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA1_TIMESTAMP 0x1UL +}; + +/* hwrm_async_event_cmpl_error_report_base (size:128b/16B) */ +struct hwrm_async_event_cmpl_error_report_base { + __le16 type; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_ID_ERROR_REPORT 0x45UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_ID_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_ID_ERROR_REPORT + __le32 event_data2; + u8 opaque_v; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_V 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_MASK 0xffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_RESERVED 0x0UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL 0x2UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_NVM 0x3UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_NVM +}; + +/* hwrm_async_event_cmpl_error_report_pause_storm (size:128b/16B) */ +struct hwrm_async_event_cmpl_error_report_pause_storm { + __le16 type; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_ID_ERROR_REPORT 0x45UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_ID_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_ID_ERROR_REPORT + __le32 event_data2; + u8 opaque_v; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_V 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_MASK 0xffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_PAUSE_STORM_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM +}; + +/* hwrm_async_event_cmpl_error_report_invalid_signal (size:128b/16B) */ +struct hwrm_async_event_cmpl_error_report_invalid_signal { + __le16 type; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_ID_ERROR_REPORT 0x45UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_ID_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_ID_ERROR_REPORT + __le32 event_data2; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_MASK 0xffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_SFT 0 + u8 opaque_v; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_V 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_MASK 0xffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL 0x2UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL +}; + +/* hwrm_async_event_cmpl_error_report_nvm (size:128b/16B) */ +struct hwrm_async_event_cmpl_error_report_nvm { + __le16 type; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_MASK 0x3fUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_HWRM_ASYNC_EVENT 0x2eUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_TYPE_HWRM_ASYNC_EVENT + __le16 event_id; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_ID_ERROR_REPORT 0x45UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_ID_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_ID_ERROR_REPORT + __le32 event_data2; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_MASK 0xffffffffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_SFT 0 + u8 opaque_v; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_V 0x1UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_OPAQUE_MASK 0xfeUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_OPAQUE_SFT 1 + u8 timestamp_lo; + __le16 timestamp_hi; + __le32 event_data1; + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_MASK 0xffUL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_SFT 0 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_NVM_ERROR 0x3UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_ERROR_TYPE_NVM_ERROR + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_MASK 0xff00UL + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_SFT 8 + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_WRITE (0x1UL << 8) + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_ERASE (0x2UL << 8) + #define ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_LAST ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_ERASE +}; + /* hwrm_func_reset_input (size:192b/24B) */ struct hwrm_func_reset_input { __le16 req_type; @@ -1302,7 +1554,7 @@ struct hwrm_func_qcaps_output { __le32 max_flow_id; __le32 max_hw_ring_grps; __le16 max_sp_tx_rings; - u8 unused_0[2]; + __le16 max_msix_vfs; __le32 flags_ext; #define FUNC_QCAPS_RESP_FLAGS_EXT_ECN_MARK_SUPPORTED 0x1UL #define FUNC_QCAPS_RESP_FLAGS_EXT_ECN_STATS_SUPPORTED 0x2UL @@ -1320,6 +1572,14 @@ struct hwrm_func_qcaps_output { #define FUNC_QCAPS_RESP_FLAGS_EXT_NVM_OPTION_ACTION_SUPPORTED 0x2000UL #define FUNC_QCAPS_RESP_FLAGS_EXT_BD_METADATA_SUPPORTED 0x4000UL #define FUNC_QCAPS_RESP_FLAGS_EXT_ECHO_REQUEST_SUPPORTED 0x8000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_NPAR_1_2_SUPPORTED 0x10000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PTM_SUPPORTED 0x20000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED 0x40000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_VF_CFG_ASYNC_FOR_PF_SUPPORTED 0x80000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_PARTITION_BW_SUPPORTED 0x100000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED 0x200000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_KTLS_SUPPORTED 0x400000UL + #define FUNC_QCAPS_RESP_FLAGS_EXT_EP_RATE_CONTROL 0x800000UL u8 max_schqs; u8 mpc_chnls_cap; #define FUNC_QCAPS_RESP_MPC_CHNLS_CAP_TCE 0x1UL @@ -1342,7 +1602,7 @@ struct hwrm_func_qcfg_input { u8 unused_0[6]; }; -/* hwrm_func_qcfg_output (size:768b/96B) */ +/* hwrm_func_qcfg_output (size:832b/104B) */ struct hwrm_func_qcfg_output { __le16 error_code; __le16 req_type; @@ -1366,6 +1626,7 @@ struct hwrm_func_qcfg_output { #define FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED 0x800UL #define FUNC_QCFG_RESP_FLAGS_FAST_RESET_ALLOWED 0x1000UL #define FUNC_QCFG_RESP_FLAGS_MULTI_ROOT 0x2000UL + #define FUNC_QCFG_RESP_FLAGS_ENABLE_RDMA_SRIOV 0x4000UL u8 mac_address[6]; __le16 pci_id; __le16 alloc_rsscos_ctx; @@ -1374,7 +1635,7 @@ struct hwrm_func_qcfg_output { __le16 alloc_rx_rings; __le16 alloc_l2_ctx; __le16 alloc_vnics; - __le16 mtu; + __le16 admin_mtu; __le16 mru; __le16 stat_ctx_id; u8 port_partition_type; @@ -1383,6 +1644,7 @@ struct hwrm_func_qcfg_output { #define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0 0x2UL #define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5 0x3UL #define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0 0x4UL + #define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_2 0x5UL #define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN 0xffUL #define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_LAST FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN u8 port_pf_cnt; @@ -1463,11 +1725,35 @@ struct hwrm_func_qcfg_output { #define FUNC_QCFG_RESP_MPC_CHNLS_TE_CFA_ENABLED 0x4UL #define FUNC_QCFG_RESP_MPC_CHNLS_RE_CFA_ENABLED 0x8UL #define FUNC_QCFG_RESP_MPC_CHNLS_PRIMATE_ENABLED 0x10UL - u8 unused_2[6]; + u8 unused_2[3]; + __le32 partition_min_bw; + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_MASK 0xfffffffUL + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_SFT 0 + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE 0x10000000UL + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BITS (0x0UL << 28) + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BYTES (0x1UL << 28) + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_LAST FUNC_QCFG_RESP_PARTITION_MIN_BW_SCALE_BYTES + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_SFT 29 + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29) + #define FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_LAST FUNC_QCFG_RESP_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 + __le32 partition_max_bw; + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_MASK 0xfffffffUL + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_SFT 0 + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE 0x10000000UL + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BITS (0x0UL << 28) + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BYTES (0x1UL << 28) + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_LAST FUNC_QCFG_RESP_PARTITION_MAX_BW_SCALE_BYTES + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_SFT 29 + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29) + #define FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_LAST FUNC_QCFG_RESP_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 + __le16 host_mtu; + u8 unused_3; u8 valid; }; -/* hwrm_func_cfg_input (size:768b/96B) */ +/* hwrm_func_cfg_input (size:832b/104B) */ struct hwrm_func_cfg_input { __le16 req_type; __le16 cmpl_ring; @@ -1504,7 +1790,7 @@ struct hwrm_func_cfg_input { #define FUNC_CFG_REQ_FLAGS_BD_METADATA_ENABLE 0x20000000UL #define FUNC_CFG_REQ_FLAGS_BD_METADATA_DISABLE 0x40000000UL __le32 enables; - #define FUNC_CFG_REQ_ENABLES_MTU 0x1UL + #define FUNC_CFG_REQ_ENABLES_ADMIN_MTU 0x1UL #define FUNC_CFG_REQ_ENABLES_MRU 0x2UL #define FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS 0x4UL #define FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS 0x8UL @@ -1530,7 +1816,11 @@ struct hwrm_func_cfg_input { #define FUNC_CFG_REQ_ENABLES_HOT_RESET_IF_SUPPORT 0x800000UL #define FUNC_CFG_REQ_ENABLES_SCHQ_ID 0x1000000UL #define FUNC_CFG_REQ_ENABLES_MPC_CHNLS 0x2000000UL - __le16 mtu; + #define FUNC_CFG_REQ_ENABLES_PARTITION_MIN_BW 0x4000000UL + #define FUNC_CFG_REQ_ENABLES_PARTITION_MAX_BW 0x8000000UL + #define FUNC_CFG_REQ_ENABLES_TPID 0x10000000UL + #define FUNC_CFG_REQ_ENABLES_HOST_MTU 0x20000000UL + __le16 admin_mtu; __le16 mru; __le16 num_rsscos_ctxs; __le16 num_cmpl_rings; @@ -1615,7 +1905,30 @@ struct hwrm_func_cfg_input { #define FUNC_CFG_REQ_MPC_CHNLS_RE_CFA_DISABLE 0x80UL #define FUNC_CFG_REQ_MPC_CHNLS_PRIMATE_ENABLE 0x100UL #define FUNC_CFG_REQ_MPC_CHNLS_PRIMATE_DISABLE 0x200UL - u8 unused_0[4]; + __le32 partition_min_bw; + #define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_MASK 0xfffffffUL + #define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_SFT 0 + #define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE 0x10000000UL + #define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_BITS (0x0UL << 28) + #define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_BYTES (0x1UL << 28) + #define FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_LAST FUNC_CFG_REQ_PARTITION_MIN_BW_SCALE_BYTES + #define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL + #define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_SFT 29 + #define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29) + #define FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_LAST FUNC_CFG_REQ_PARTITION_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 + __le32 partition_max_bw; + #define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_MASK 0xfffffffUL + #define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_SFT 0 + #define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE 0x10000000UL + #define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_BITS (0x0UL << 28) + #define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_BYTES (0x1UL << 28) + #define FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_LAST FUNC_CFG_REQ_PARTITION_MAX_BW_SCALE_BYTES + #define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL + #define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_SFT 29 + #define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29) + #define FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_LAST FUNC_CFG_REQ_PARTITION_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 + __be16 tpid; + __le16 host_mtu; }; /* hwrm_func_cfg_output (size:128b/16B) */ @@ -1777,14 +2090,15 @@ struct hwrm_func_drv_rgtr_input { __le16 target_id; __le64 resp_addr; __le32 flags; - #define FUNC_DRV_RGTR_REQ_FLAGS_FWD_ALL_MODE 0x1UL - #define FUNC_DRV_RGTR_REQ_FLAGS_FWD_NONE_MODE 0x2UL - #define FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE 0x4UL - #define FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE 0x8UL - #define FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT 0x10UL - #define FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT 0x20UL - #define FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT 0x40UL - #define FUNC_DRV_RGTR_REQ_FLAGS_FAST_RESET_SUPPORT 0x80UL + #define FUNC_DRV_RGTR_REQ_FLAGS_FWD_ALL_MODE 0x1UL + #define FUNC_DRV_RGTR_REQ_FLAGS_FWD_NONE_MODE 0x2UL + #define FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE 0x4UL + #define FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE 0x8UL + #define FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT 0x10UL + #define FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT 0x20UL + #define FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT 0x40UL + #define FUNC_DRV_RGTR_REQ_FLAGS_FAST_RESET_SUPPORT 0x80UL + #define FUNC_DRV_RGTR_REQ_FLAGS_RSS_STRICT_HASH_TYPE_SUPPORT 0x100UL __le32 enables; #define FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE 0x1UL #define FUNC_DRV_RGTR_REQ_ENABLES_VER 0x2UL @@ -2047,7 +2361,7 @@ struct hwrm_func_backing_store_qcaps_input { __le64 resp_addr; }; -/* hwrm_func_backing_store_qcaps_output (size:704b/88B) */ +/* hwrm_func_backing_store_qcaps_output (size:832b/104B) */ struct hwrm_func_backing_store_qcaps_output { __le16 error_code; __le16 req_type; @@ -2085,6 +2399,8 @@ struct hwrm_func_backing_store_qcaps_output { #define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_VNIC 0x8UL #define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_STAT 0x10UL #define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_MRAV 0x20UL + #define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_TKC 0x40UL + #define FUNC_BACKING_STORE_QCAPS_RESP_CTX_INIT_MASK_RKC 0x80UL u8 qp_init_offset; u8 srq_init_offset; u8 cq_init_offset; @@ -2093,7 +2409,13 @@ struct hwrm_func_backing_store_qcaps_output { u8 stat_init_offset; u8 mrav_init_offset; u8 tqm_fp_rings_count_ext; - u8 rsvd[5]; + u8 tkc_init_offset; + u8 rkc_init_offset; + __le16 tkc_entry_size; + __le16 rkc_entry_size; + __le32 tkc_max_entries; + __le32 rkc_max_entries; + u8 rsvd[7]; u8 valid; }; @@ -2120,7 +2442,7 @@ struct tqm_fp_ring_cfg { __le64 tqm_ring_page_dir; }; -/* hwrm_func_backing_store_cfg_input (size:2432b/304B) */ +/* hwrm_func_backing_store_cfg_input (size:2688b/336B) */ struct hwrm_func_backing_store_cfg_input { __le16 req_type; __le16 cmpl_ring; @@ -2150,6 +2472,8 @@ struct hwrm_func_backing_store_cfg_input { #define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING8 0x10000UL #define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING9 0x20000UL #define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_RING10 0x40000UL + #define FUNC_BACKING_STORE_CFG_REQ_ENABLES_TKC 0x80000UL + #define FUNC_BACKING_STORE_CFG_REQ_ENABLES_RKC 0x100000UL u8 qpc_pg_size_qpc_lvl; #define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_MASK 0xfUL #define FUNC_BACKING_STORE_CFG_REQ_QPC_LVL_SFT 0 @@ -2508,6 +2832,45 @@ struct hwrm_func_backing_store_cfg_input { u8 ring10_unused[3]; __le32 tqm_ring10_num_entries; __le64 tqm_ring10_page_dir; + __le32 tkc_num_entries; + __le32 rkc_num_entries; + __le64 tkc_page_dir; + __le64 rkc_page_dir; + __le16 tkc_entry_size; + __le16 rkc_entry_size; + u8 tkc_pg_size_tkc_lvl; + #define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_MASK 0xfUL + #define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_SFT 0 + #define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_0 0x0UL + #define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_1 0x1UL + #define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_2 0x2UL + #define FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LAST FUNC_BACKING_STORE_CFG_REQ_TKC_LVL_LVL_2 + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_MASK 0xf0UL + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_SFT 4 + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_4K (0x0UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_8K (0x1UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_64K (0x2UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_2M (0x3UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_8M (0x4UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_1G (0x5UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_LAST FUNC_BACKING_STORE_CFG_REQ_TKC_PG_SIZE_PG_1G + u8 rkc_pg_size_rkc_lvl; + #define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_MASK 0xfUL + #define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_SFT 0 + #define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_0 0x0UL + #define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_1 0x1UL + #define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_2 0x2UL + #define FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LAST FUNC_BACKING_STORE_CFG_REQ_RKC_LVL_LVL_2 + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_MASK 0xf0UL + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_SFT 4 + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_4K (0x0UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_8K (0x1UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_64K (0x2UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_2M (0x3UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_8M (0x4UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_1G (0x5UL << 4) + #define FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_LAST FUNC_BACKING_STORE_CFG_REQ_RKC_PG_SIZE_PG_1G + u8 rsvd[2]; }; /* hwrm_func_backing_store_cfg_output (size:128b/16B) */ @@ -2634,6 +2997,212 @@ struct hwrm_func_echo_response_output { u8 valid; }; +/* hwrm_func_ptp_pin_qcfg_input (size:192b/24B) */ +struct hwrm_func_ptp_pin_qcfg_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + u8 unused_0[8]; +}; + +/* hwrm_func_ptp_pin_qcfg_output (size:128b/16B) */ +struct hwrm_func_ptp_pin_qcfg_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 num_pins; + u8 state; + #define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN0_ENABLED 0x1UL + #define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN1_ENABLED 0x2UL + #define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN2_ENABLED 0x4UL + #define FUNC_PTP_PIN_QCFG_RESP_STATE_PIN3_ENABLED 0x8UL + u8 pin0_usage; + #define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_LAST FUNC_PTP_PIN_QCFG_RESP_PIN0_USAGE_SYNC_OUT + u8 pin1_usage; + #define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_LAST FUNC_PTP_PIN_QCFG_RESP_PIN1_USAGE_SYNC_OUT + u8 pin2_usage; + #define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_LAST FUNC_PTP_PIN_QCFG_RESP_PIN2_USAGE_SYNC_OUT + u8 pin3_usage; + #define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_LAST FUNC_PTP_PIN_QCFG_RESP_PIN3_USAGE_SYNC_OUT + u8 unused_0; + u8 valid; +}; + +/* hwrm_func_ptp_pin_cfg_input (size:256b/32B) */ +struct hwrm_func_ptp_pin_cfg_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le32 enables; + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN0_STATE 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN0_USAGE 0x2UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN1_STATE 0x4UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN1_USAGE 0x8UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN2_STATE 0x10UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN2_USAGE 0x20UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN3_STATE 0x40UL + #define FUNC_PTP_PIN_CFG_REQ_ENABLES_PIN3_USAGE 0x80UL + u8 pin0_state; + #define FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_DISABLED 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_ENABLED 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_LAST FUNC_PTP_PIN_CFG_REQ_PIN0_STATE_ENABLED + u8 pin0_usage; + #define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_LAST FUNC_PTP_PIN_CFG_REQ_PIN0_USAGE_SYNC_OUT + u8 pin1_state; + #define FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_DISABLED 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_ENABLED 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_LAST FUNC_PTP_PIN_CFG_REQ_PIN1_STATE_ENABLED + u8 pin1_usage; + #define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_LAST FUNC_PTP_PIN_CFG_REQ_PIN1_USAGE_SYNC_OUT + u8 pin2_state; + #define FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_DISABLED 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_ENABLED 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_LAST FUNC_PTP_PIN_CFG_REQ_PIN2_STATE_ENABLED + u8 pin2_usage; + #define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_LAST FUNC_PTP_PIN_CFG_REQ_PIN2_USAGE_SYNC_OUT + u8 pin3_state; + #define FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_DISABLED 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_ENABLED 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_LAST FUNC_PTP_PIN_CFG_REQ_PIN3_STATE_ENABLED + u8 pin3_usage; + #define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_NONE 0x0UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_PPS_IN 0x1UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_PPS_OUT 0x2UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNC_IN 0x3UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNC_OUT 0x4UL + #define FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_LAST FUNC_PTP_PIN_CFG_REQ_PIN3_USAGE_SYNC_OUT + u8 unused_0[4]; +}; + +/* hwrm_func_ptp_pin_cfg_output (size:128b/16B) */ +struct hwrm_func_ptp_pin_cfg_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + +/* hwrm_func_ptp_cfg_input (size:320b/40B) */ +struct hwrm_func_ptp_cfg_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le16 enables; + #define FUNC_PTP_CFG_REQ_ENABLES_PTP_PPS_EVENT 0x1UL + #define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_DLL_SOURCE 0x2UL + #define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_DLL_PHASE 0x4UL + #define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_EXT_PERIOD 0x8UL + #define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_EXT_UP 0x10UL + #define FUNC_PTP_CFG_REQ_ENABLES_PTP_FREQ_ADJ_EXT_PHASE 0x20UL + u8 ptp_pps_event; + #define FUNC_PTP_CFG_REQ_PTP_PPS_EVENT_INTERNAL 0x1UL + #define FUNC_PTP_CFG_REQ_PTP_PPS_EVENT_EXTERNAL 0x2UL + u8 ptp_freq_adj_dll_source; + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_NONE 0x0UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_0 0x1UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_1 0x2UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_2 0x3UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_TSIO_3 0x4UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_0 0x5UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_1 0x6UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_2 0x7UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_PORT_3 0x8UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_INVALID 0xffUL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_LAST FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_SOURCE_INVALID + u8 ptp_freq_adj_dll_phase; + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_NONE 0x0UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_4K 0x1UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_8K 0x2UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_10M 0x3UL + #define FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_LAST FUNC_PTP_CFG_REQ_PTP_FREQ_ADJ_DLL_PHASE_10M + u8 unused_0[3]; + __le32 ptp_freq_adj_ext_period; + __le32 ptp_freq_adj_ext_up; + __le32 ptp_freq_adj_ext_phase_lower; + __le32 ptp_freq_adj_ext_phase_upper; +}; + +/* hwrm_func_ptp_cfg_output (size:128b/16B) */ +struct hwrm_func_ptp_cfg_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + +/* hwrm_func_ptp_ts_query_input (size:192b/24B) */ +struct hwrm_func_ptp_ts_query_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le32 flags; + #define FUNC_PTP_TS_QUERY_REQ_FLAGS_PPS_TIME 0x1UL + #define FUNC_PTP_TS_QUERY_REQ_FLAGS_PTM_TIME 0x2UL + u8 unused_0[4]; +}; + +/* hwrm_func_ptp_ts_query_output (size:320b/40B) */ +struct hwrm_func_ptp_ts_query_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le64 pps_event_ts; + __le64 ptm_res_local_ts; + __le64 ptm_pmstr_ts; + __le32 ptm_mstr_prop_dly; + u8 unused_0[3]; + u8 valid; +}; + /* hwrm_func_drv_if_change_input (size:192b/24B) */ struct hwrm_func_drv_if_change_input { __le16 req_type; @@ -3156,6 +3725,7 @@ struct hwrm_port_mac_cfg_input { #define PORT_MAC_CFG_REQ_ENABLES_TX_TS_CAPTURE_PTP_MSG_TYPE 0x80UL #define PORT_MAC_CFG_REQ_ENABLES_COS_FIELD_CFG 0x100UL #define PORT_MAC_CFG_REQ_ENABLES_PTP_FREQ_ADJ_PPB 0x200UL + #define PORT_MAC_CFG_REQ_ENABLES_PTP_ADJ_PHASE 0x400UL __le16 port_id; u8 ipg; u8 lpbk; @@ -3188,8 +3758,8 @@ struct hwrm_port_mac_cfg_input { #define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_MASK 0xe0UL #define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_SFT 5 u8 unused_0[3]; - __s32 ptp_freq_adj_ppb; - u8 unused_1[4]; + __le32 ptp_freq_adj_ppb; + __le32 ptp_adj_phase; }; /* hwrm_port_mac_cfg_output (size:128b/16B) */ @@ -3221,16 +3791,17 @@ struct hwrm_port_mac_ptp_qcfg_input { u8 unused_0[6]; }; -/* hwrm_port_mac_ptp_qcfg_output (size:640b/80B) */ +/* hwrm_port_mac_ptp_qcfg_output (size:704b/88B) */ struct hwrm_port_mac_ptp_qcfg_output { __le16 error_code; __le16 req_type; __le16 seq_id; __le16 resp_len; u8 flags; - #define PORT_MAC_PTP_QCFG_RESP_FLAGS_DIRECT_ACCESS 0x1UL - #define PORT_MAC_PTP_QCFG_RESP_FLAGS_ONE_STEP_TX_TS 0x4UL - #define PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS 0x8UL + #define PORT_MAC_PTP_QCFG_RESP_FLAGS_DIRECT_ACCESS 0x1UL + #define PORT_MAC_PTP_QCFG_RESP_FLAGS_ONE_STEP_TX_TS 0x4UL + #define PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS 0x8UL + #define PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK 0x10UL u8 unused_0[3]; __le32 rx_ts_reg_off_lower; __le32 rx_ts_reg_off_upper; @@ -3247,6 +3818,8 @@ struct hwrm_port_mac_ptp_qcfg_output { __le32 tx_ts_reg_off_seq_id; __le32 tx_ts_reg_off_fifo; __le32 tx_ts_reg_off_granularity; + __le32 ts_ref_clock_reg_lower; + __le32 ts_ref_clock_reg_upper; u8 unused_1[7]; u8 valid; }; @@ -3647,7 +4220,7 @@ struct hwrm_port_lpbk_clr_stats_output { u8 valid; }; -/* hwrm_port_ts_query_input (size:192b/24B) */ +/* hwrm_port_ts_query_input (size:256b/32B) */ struct hwrm_port_ts_query_input { __le16 req_type; __le16 cmpl_ring; @@ -3662,6 +4235,11 @@ struct hwrm_port_ts_query_input { #define PORT_TS_QUERY_REQ_FLAGS_CURRENT_TIME 0x2UL __le16 port_id; u8 unused_0[2]; + __le16 enables; + #define PORT_TS_QUERY_REQ_ENABLES_TS_REQ_TIMEOUT 0x1UL + #define PORT_TS_QUERY_REQ_ENABLES_PTP_SEQ_ID 0x2UL + __le16 ts_req_timeout; + __le32 ptp_seq_id; }; /* hwrm_port_ts_query_output (size:192b/24B) */ @@ -4215,7 +4793,8 @@ struct hwrm_queue_qportcfg_output { u8 max_configurable_lossless_queues; u8 queue_cfg_allowed; u8 queue_cfg_info; - #define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG 0x1UL + #define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG 0x1UL + #define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_USE_PROFILE_TYPE 0x2UL u8 queue_pfcenable_cfg_allowed; u8 queue_pri2cos_cfg_allowed; u8 queue_cos2bw_cfg_allowed; @@ -5467,6 +6046,7 @@ struct hwrm_vnic_qcaps_output { #define VNIC_QCAPS_RESP_FLAGS_VNIC_STATE_CAP 0x400UL #define VNIC_QCAPS_RESP_FLAGS_VIRTIO_NET_VNIC_ALLOC_CAP 0x800UL #define VNIC_QCAPS_RESP_FLAGS_METADATA_FORMAT_CAP 0x1000UL + #define VNIC_QCAPS_RESP_FLAGS_RSS_STRICT_HASH_TYPE_CAP 0x2000UL __le16 max_aggs_supported; u8 unused_1[5]; u8 valid; @@ -7224,6 +7804,7 @@ struct hwrm_cfa_adv_flow_mgnt_qcaps_output { #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ETHERTYPE_IP_SUPPORTED 0x4000UL #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TRUFLOW_CAPABLE 0x8000UL #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_FILTER_TRAFFIC_TYPE_L2_ROCE_SUPPORTED 0x10000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_LAG_SUPPORTED 0x20000UL u8 unused_0[3]; u8 valid; }; @@ -7914,11 +8495,14 @@ struct hwrm_temp_monitor_query_output { u8 phy_temp; u8 om_temp; u8 flags; - #define TEMP_MONITOR_QUERY_RESP_FLAGS_TEMP_NOT_AVAILABLE 0x1UL - #define TEMP_MONITOR_QUERY_RESP_FLAGS_PHY_TEMP_NOT_AVAILABLE 0x2UL - #define TEMP_MONITOR_QUERY_RESP_FLAGS_OM_NOT_PRESENT 0x4UL - #define TEMP_MONITOR_QUERY_RESP_FLAGS_OM_TEMP_NOT_AVAILABLE 0x8UL - u8 unused_0[3]; + #define TEMP_MONITOR_QUERY_RESP_FLAGS_TEMP_NOT_AVAILABLE 0x1UL + #define TEMP_MONITOR_QUERY_RESP_FLAGS_PHY_TEMP_NOT_AVAILABLE 0x2UL + #define TEMP_MONITOR_QUERY_RESP_FLAGS_OM_NOT_PRESENT 0x4UL + #define TEMP_MONITOR_QUERY_RESP_FLAGS_OM_TEMP_NOT_AVAILABLE 0x8UL + #define TEMP_MONITOR_QUERY_RESP_FLAGS_EXT_TEMP_FIELDS_AVAILABLE 0x10UL + u8 temp2; + u8 phy_temp2; + u8 om_temp2; u8 valid; }; @@ -8109,6 +8693,7 @@ struct hwrm_dbg_qcaps_output { #define DBG_QCAPS_RESP_FLAGS_CRASHDUMP_NVM 0x1UL #define DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR 0x2UL #define DBG_QCAPS_RESP_FLAGS_CRASHDUMP_SOC_DDR 0x4UL + #define DBG_QCAPS_RESP_FLAGS_USEQ 0x8UL u8 unused_1[3]; u8 valid; }; @@ -8632,10 +9217,11 @@ struct hwrm_nvm_install_update_output { /* hwrm_nvm_install_update_cmd_err (size:64b/8B) */ struct hwrm_nvm_install_update_cmd_err { u8 code; - #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN 0x0UL - #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_FRAG_ERR 0x1UL - #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE 0x2UL - #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_LAST NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE + #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN 0x0UL + #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_FRAG_ERR 0x1UL + #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE 0x2UL + #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_ANTI_ROLLBACK 0x3UL + #define NVM_INSTALL_UPDATE_CMD_ERR_CODE_LAST NVM_INSTALL_UPDATE_CMD_ERR_CODE_ANTI_ROLLBACK u8 unused_0[7]; }; @@ -8876,6 +9462,7 @@ struct fw_status_reg { #define FW_STATUS_REG_CRASHDUMP_COMPLETE 0x80000UL #define FW_STATUS_REG_SHUTDOWN 0x100000UL #define FW_STATUS_REG_CRASHED_NO_MASTER 0x200000UL + #define FW_STATUS_REG_RECOVERING 0x400000UL }; /* hcomm_status (size:64b/8B) */ diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c new file mode 100644 index 000000000000..f698b6bd4ff8 --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c @@ -0,0 +1,473 @@ +/* Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2021 Broadcom Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/etherdevice.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/net_tstamp.h> +#include <linux/timecounter.h> +#include <linux/timekeeping.h> +#include <linux/ptp_classify.h> +#include "bnxt_hsi.h" +#include "bnxt.h" +#include "bnxt_ptp.h" + +int bnxt_ptp_parse(struct sk_buff *skb, u16 *seq_id) +{ + unsigned int ptp_class; + struct ptp_header *hdr; + + ptp_class = ptp_classify_raw(skb); + + switch (ptp_class & PTP_CLASS_VMASK) { + case PTP_CLASS_V1: + case PTP_CLASS_V2: + hdr = ptp_parse_header(skb, ptp_class); + if (!hdr) + return -EINVAL; + + *seq_id = ntohs(hdr->sequence_id); + return 0; + default: + return -ERANGE; + } +} + +static int bnxt_ptp_settime(struct ptp_clock_info *ptp_info, + const struct timespec64 *ts) +{ + struct bnxt_ptp_cfg *ptp = container_of(ptp_info, struct bnxt_ptp_cfg, + ptp_info); + u64 ns = timespec64_to_ns(ts); + + spin_lock_bh(&ptp->ptp_lock); + timecounter_init(&ptp->tc, &ptp->cc, ns); + spin_unlock_bh(&ptp->ptp_lock); + return 0; +} + +/* Caller holds ptp_lock */ +static u64 bnxt_refclk_read(struct bnxt *bp, struct ptp_system_timestamp *sts) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + u64 ns; + + ptp_read_system_prets(sts); + ns = readl(bp->bar0 + ptp->refclk_mapped_regs[0]); + ptp_read_system_postts(sts); + ns |= (u64)readl(bp->bar0 + ptp->refclk_mapped_regs[1]) << 32; + return ns; +} + +static void bnxt_ptp_get_current_time(struct bnxt *bp) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + + if (!ptp) + return; + spin_lock_bh(&ptp->ptp_lock); + WRITE_ONCE(ptp->old_time, ptp->current_time); + ptp->current_time = bnxt_refclk_read(bp, NULL); + spin_unlock_bh(&ptp->ptp_lock); +} + +static int bnxt_hwrm_port_ts_query(struct bnxt *bp, u32 flags, u64 *ts) +{ + struct hwrm_port_ts_query_output *resp = bp->hwrm_cmd_resp_addr; + struct hwrm_port_ts_query_input req = {0}; + int rc; + + bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_TS_QUERY, -1, -1); + req.flags = cpu_to_le32(flags); + if ((flags & PORT_TS_QUERY_REQ_FLAGS_PATH) == + PORT_TS_QUERY_REQ_FLAGS_PATH_TX) { + req.enables = cpu_to_le16(BNXT_PTP_QTS_TX_ENABLES); + req.ptp_seq_id = cpu_to_le32(bp->ptp_cfg->tx_seqid); + req.ts_req_timeout = cpu_to_le16(BNXT_PTP_QTS_TIMEOUT); + } + mutex_lock(&bp->hwrm_cmd_lock); + rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); + if (!rc) + *ts = le64_to_cpu(resp->ptp_msg_ts); + mutex_unlock(&bp->hwrm_cmd_lock); + return rc; +} + +static int bnxt_ptp_gettimex(struct ptp_clock_info *ptp_info, + struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct bnxt_ptp_cfg *ptp = container_of(ptp_info, struct bnxt_ptp_cfg, + ptp_info); + u64 ns, cycles; + + spin_lock_bh(&ptp->ptp_lock); + cycles = bnxt_refclk_read(ptp->bp, sts); + ns = timecounter_cyc2time(&ptp->tc, cycles); + spin_unlock_bh(&ptp->ptp_lock); + *ts = ns_to_timespec64(ns); + + return 0; +} + +static int bnxt_ptp_adjtime(struct ptp_clock_info *ptp_info, s64 delta) +{ + struct bnxt_ptp_cfg *ptp = container_of(ptp_info, struct bnxt_ptp_cfg, + ptp_info); + + spin_lock_bh(&ptp->ptp_lock); + timecounter_adjtime(&ptp->tc, delta); + spin_unlock_bh(&ptp->ptp_lock); + return 0; +} + +static int bnxt_ptp_adjfreq(struct ptp_clock_info *ptp_info, s32 ppb) +{ + struct bnxt_ptp_cfg *ptp = container_of(ptp_info, struct bnxt_ptp_cfg, + ptp_info); + struct hwrm_port_mac_cfg_input req = {0}; + struct bnxt *bp = ptp->bp; + int rc; + + bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_MAC_CFG, -1, -1); + req.ptp_freq_adj_ppb = cpu_to_le32(ppb); + req.enables = cpu_to_le32(PORT_MAC_CFG_REQ_ENABLES_PTP_FREQ_ADJ_PPB); + rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); + if (rc) + netdev_err(ptp->bp->dev, + "ptp adjfreq failed. rc = %d\n", rc); + return rc; +} + +static int bnxt_ptp_enable(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + return -EOPNOTSUPP; +} + +static int bnxt_hwrm_ptp_cfg(struct bnxt *bp) +{ + struct hwrm_port_mac_cfg_input req = {0}; + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + u32 flags = 0; + + bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_MAC_CFG, -1, -1); + if (ptp->rx_filter) + flags |= PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_ENABLE; + else + flags |= PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_DISABLE; + if (ptp->tx_tstamp_en) + flags |= PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_ENABLE; + else + flags |= PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_DISABLE; + req.flags = cpu_to_le32(flags); + req.enables = cpu_to_le32(PORT_MAC_CFG_REQ_ENABLES_RX_TS_CAPTURE_PTP_MSG_TYPE); + req.rx_ts_capture_ptp_msg_type = cpu_to_le16(ptp->rxctl); + + return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); +} + +int bnxt_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) +{ + struct bnxt *bp = netdev_priv(dev); + struct hwtstamp_config stmpconf; + struct bnxt_ptp_cfg *ptp; + u16 old_rxctl; + int old_rx_filter, rc; + u8 old_tx_tstamp_en; + + ptp = bp->ptp_cfg; + if (!ptp) + return -EOPNOTSUPP; + + if (copy_from_user(&stmpconf, ifr->ifr_data, sizeof(stmpconf))) + return -EFAULT; + + if (stmpconf.flags) + return -EINVAL; + + if (stmpconf.tx_type != HWTSTAMP_TX_ON && + stmpconf.tx_type != HWTSTAMP_TX_OFF) + return -ERANGE; + + old_rx_filter = ptp->rx_filter; + old_rxctl = ptp->rxctl; + old_tx_tstamp_en = ptp->tx_tstamp_en; + switch (stmpconf.rx_filter) { + case HWTSTAMP_FILTER_NONE: + ptp->rxctl = 0; + ptp->rx_filter = HWTSTAMP_FILTER_NONE; + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + ptp->rxctl = BNXT_PTP_MSG_EVENTS; + ptp->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + ptp->rxctl = BNXT_PTP_MSG_SYNC; + ptp->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; + break; + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + ptp->rxctl = BNXT_PTP_MSG_DELAY_REQ; + ptp->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; + break; + default: + return -ERANGE; + } + + if (stmpconf.tx_type == HWTSTAMP_TX_ON) + ptp->tx_tstamp_en = 1; + else + ptp->tx_tstamp_en = 0; + + rc = bnxt_hwrm_ptp_cfg(bp); + if (rc) + goto ts_set_err; + + stmpconf.rx_filter = ptp->rx_filter; + return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ? + -EFAULT : 0; + +ts_set_err: + ptp->rx_filter = old_rx_filter; + ptp->rxctl = old_rxctl; + ptp->tx_tstamp_en = old_tx_tstamp_en; + return rc; +} + +int bnxt_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) +{ + struct bnxt *bp = netdev_priv(dev); + struct hwtstamp_config stmpconf; + struct bnxt_ptp_cfg *ptp; + + ptp = bp->ptp_cfg; + if (!ptp) + return -EOPNOTSUPP; + + stmpconf.flags = 0; + stmpconf.tx_type = ptp->tx_tstamp_en ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF; + + stmpconf.rx_filter = ptp->rx_filter; + return copy_to_user(ifr->ifr_data, &stmpconf, sizeof(stmpconf)) ? + -EFAULT : 0; +} + +static int bnxt_map_regs(struct bnxt *bp, u32 *reg_arr, int count, int reg_win) +{ + u32 reg_base = *reg_arr & BNXT_GRC_BASE_MASK; + u32 win_off; + int i; + + for (i = 0; i < count; i++) { + if ((reg_arr[i] & BNXT_GRC_BASE_MASK) != reg_base) + return -ERANGE; + } + win_off = BNXT_GRCPF_REG_WINDOW_BASE_OUT + (reg_win - 1) * 4; + writel(reg_base, bp->bar0 + win_off); + return 0; +} + +static int bnxt_map_ptp_regs(struct bnxt *bp) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + u32 *reg_arr; + int rc, i; + + reg_arr = ptp->refclk_regs; + if (bp->flags & BNXT_FLAG_CHIP_P5) { + rc = bnxt_map_regs(bp, reg_arr, 2, BNXT_PTP_GRC_WIN); + if (rc) + return rc; + for (i = 0; i < 2; i++) + ptp->refclk_mapped_regs[i] = BNXT_PTP_GRC_WIN_BASE + + (ptp->refclk_regs[i] & BNXT_GRC_OFFSET_MASK); + return 0; + } + return -ENODEV; +} + +static void bnxt_unmap_ptp_regs(struct bnxt *bp) +{ + writel(0, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + + (BNXT_PTP_GRC_WIN - 1) * 4); +} + +static u64 bnxt_cc_read(const struct cyclecounter *cc) +{ + struct bnxt_ptp_cfg *ptp = container_of(cc, struct bnxt_ptp_cfg, cc); + + return bnxt_refclk_read(ptp->bp, NULL); +} + +static void bnxt_stamp_tx_skb(struct bnxt *bp, struct sk_buff *skb) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + struct skb_shared_hwtstamps timestamp; + u64 ts = 0, ns = 0; + int rc; + + rc = bnxt_hwrm_port_ts_query(bp, PORT_TS_QUERY_REQ_FLAGS_PATH_TX, &ts); + if (!rc) { + memset(×tamp, 0, sizeof(timestamp)); + spin_lock_bh(&ptp->ptp_lock); + ns = timecounter_cyc2time(&ptp->tc, ts); + spin_unlock_bh(&ptp->ptp_lock); + timestamp.hwtstamp = ns_to_ktime(ns); + skb_tstamp_tx(ptp->tx_skb, ×tamp); + } else { + netdev_err(bp->dev, "TS query for TX timer failed rc = %x\n", + rc); + } + + dev_kfree_skb_any(ptp->tx_skb); + ptp->tx_skb = NULL; + atomic_inc(&ptp->tx_avail); +} + +static long bnxt_ptp_ts_aux_work(struct ptp_clock_info *ptp_info) +{ + struct bnxt_ptp_cfg *ptp = container_of(ptp_info, struct bnxt_ptp_cfg, + ptp_info); + unsigned long now = jiffies; + struct bnxt *bp = ptp->bp; + + if (ptp->tx_skb) + bnxt_stamp_tx_skb(bp, ptp->tx_skb); + + if (!time_after_eq(now, ptp->next_period)) + return ptp->next_period - now; + + bnxt_ptp_get_current_time(bp); + ptp->next_period = now + HZ; + return HZ; +} + +int bnxt_get_tx_ts_p5(struct bnxt *bp, struct sk_buff *skb) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + + if (ptp->tx_skb) { + netdev_err(bp->dev, "deferring skb:one SKB is still outstanding\n"); + return -EBUSY; + } + ptp->tx_skb = skb; + ptp_schedule_worker(ptp->ptp_clock, 0); + return 0; +} + +int bnxt_get_rx_ts_p5(struct bnxt *bp, u64 *ts, u32 pkt_ts) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + u64 time; + + if (!ptp) + return -ENODEV; + + BNXT_READ_TIME64(ptp, time, ptp->old_time); + *ts = (time & BNXT_HI_TIMER_MASK) | pkt_ts; + if (pkt_ts < (time & BNXT_LO_TIMER_MASK)) + *ts += BNXT_LO_TIMER_MASK + 1; + + return 0; +} + +void bnxt_ptp_start(struct bnxt *bp) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + + if (!ptp) + return; + + if (bp->flags & BNXT_FLAG_CHIP_P5) { + spin_lock_bh(&ptp->ptp_lock); + ptp->current_time = bnxt_refclk_read(bp, NULL); + WRITE_ONCE(ptp->old_time, ptp->current_time); + spin_unlock_bh(&ptp->ptp_lock); + ptp_schedule_worker(ptp->ptp_clock, 0); + } +} + +static const struct ptp_clock_info bnxt_ptp_caps = { + .owner = THIS_MODULE, + .name = "bnxt clock", + .max_adj = BNXT_MAX_PHC_DRIFT, + .n_alarm = 0, + .n_ext_ts = 0, + .n_per_out = 0, + .n_pins = 0, + .pps = 0, + .adjfreq = bnxt_ptp_adjfreq, + .adjtime = bnxt_ptp_adjtime, + .do_aux_work = bnxt_ptp_ts_aux_work, + .gettimex64 = bnxt_ptp_gettimex, + .settime64 = bnxt_ptp_settime, + .enable = bnxt_ptp_enable, +}; + +int bnxt_ptp_init(struct bnxt *bp) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + int rc; + + if (!ptp) + return 0; + + rc = bnxt_map_ptp_regs(bp); + if (rc) + return rc; + + atomic_set(&ptp->tx_avail, BNXT_MAX_TX_TS); + spin_lock_init(&ptp->ptp_lock); + + memset(&ptp->cc, 0, sizeof(ptp->cc)); + ptp->cc.read = bnxt_cc_read; + ptp->cc.mask = CYCLECOUNTER_MASK(48); + ptp->cc.shift = 0; + ptp->cc.mult = 1; + + timecounter_init(&ptp->tc, &ptp->cc, ktime_to_ns(ktime_get_real())); + + ptp->ptp_info = bnxt_ptp_caps; + ptp->ptp_clock = ptp_clock_register(&ptp->ptp_info, &bp->pdev->dev); + if (IS_ERR(ptp->ptp_clock)) { + int err = PTR_ERR(ptp->ptp_clock); + + ptp->ptp_clock = NULL; + bnxt_unmap_ptp_regs(bp); + return err; + } + + return 0; +} + +void bnxt_ptp_clear(struct bnxt *bp) +{ + struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; + + if (!ptp) + return; + + if (ptp->ptp_clock) + ptp_clock_unregister(ptp->ptp_clock); + + ptp->ptp_clock = NULL; + if (ptp->tx_skb) { + dev_kfree_skb_any(ptp->tx_skb); + ptp->tx_skb = NULL; + } + bnxt_unmap_ptp_regs(bp); +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h new file mode 100644 index 000000000000..6b6245750e20 --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.h @@ -0,0 +1,81 @@ +/* Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2021 Broadcom Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#ifndef BNXT_PTP_H +#define BNXT_PTP_H + +#define BNXT_PTP_GRC_WIN 5 +#define BNXT_PTP_GRC_WIN_BASE 0x5000 + +#define BNXT_MAX_PHC_DRIFT 31000000 +#define BNXT_LO_TIMER_MASK 0x0000ffffffffUL +#define BNXT_HI_TIMER_MASK 0xffff00000000UL + +#define BNXT_PTP_QTS_TIMEOUT 1000 +#define BNXT_PTP_QTS_TX_ENABLES (PORT_TS_QUERY_REQ_ENABLES_PTP_SEQ_ID | \ + PORT_TS_QUERY_REQ_ENABLES_TS_REQ_TIMEOUT) + +struct bnxt_ptp_cfg { + struct ptp_clock_info ptp_info; + struct ptp_clock *ptp_clock; + struct cyclecounter cc; + struct timecounter tc; + /* serialize timecounter access */ + spinlock_t ptp_lock; + struct sk_buff *tx_skb; + u64 current_time; + u64 old_time; + unsigned long next_period; + u16 tx_seqid; + struct bnxt *bp; + atomic_t tx_avail; +#define BNXT_MAX_TX_TS 1 + u16 rxctl; +#define BNXT_PTP_MSG_SYNC (1 << 0) +#define BNXT_PTP_MSG_DELAY_REQ (1 << 1) +#define BNXT_PTP_MSG_PDELAY_REQ (1 << 2) +#define BNXT_PTP_MSG_PDELAY_RESP (1 << 3) +#define BNXT_PTP_MSG_FOLLOW_UP (1 << 8) +#define BNXT_PTP_MSG_DELAY_RESP (1 << 9) +#define BNXT_PTP_MSG_PDELAY_RESP_FOLLOW_UP (1 << 10) +#define BNXT_PTP_MSG_ANNOUNCE (1 << 11) +#define BNXT_PTP_MSG_SIGNALING (1 << 12) +#define BNXT_PTP_MSG_MANAGEMENT (1 << 13) +#define BNXT_PTP_MSG_EVENTS (BNXT_PTP_MSG_SYNC | \ + BNXT_PTP_MSG_DELAY_REQ | \ + BNXT_PTP_MSG_PDELAY_REQ | \ + BNXT_PTP_MSG_PDELAY_RESP) + u8 tx_tstamp_en:1; + int rx_filter; + + u32 refclk_regs[2]; + u32 refclk_mapped_regs[2]; +}; + +#if BITS_PER_LONG == 32 +#define BNXT_READ_TIME64(ptp, dst, src) \ +do { \ + spin_lock_bh(&(ptp)->ptp_lock); \ + (dst) = (src); \ + spin_unlock_bh(&(ptp)->ptp_lock); \ +} while (0) +#else +#define BNXT_READ_TIME64(ptp, dst, src) \ + ((dst) = READ_ONCE(src)) +#endif + +int bnxt_ptp_parse(struct sk_buff *skb, u16 *seq_id); +int bnxt_hwtstamp_set(struct net_device *dev, struct ifreq *ifr); +int bnxt_hwtstamp_get(struct net_device *dev, struct ifreq *ifr); +int bnxt_get_tx_ts_p5(struct bnxt *bp, struct sk_buff *skb); +int bnxt_get_rx_ts_p5(struct bnxt *bp, u64 *ts, u32 pkt_ts); +void bnxt_ptp_start(struct bnxt *bp); +int bnxt_ptp_init(struct bnxt *bp); +void bnxt_ptp_clear(struct bnxt *bp); +#endif diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c index eb00a219aa51..7fa881e1cd80 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c @@ -632,7 +632,7 @@ static int bnxt_hwrm_func_cfg(struct bnxt *bp, int num_vfs) vf_vnics = (hw_resc->max_vnics - bp->nr_vnics) / num_vfs; vf_vnics = min_t(u16, vf_vnics, vf_rx_rings); - req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_MTU | + req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ADMIN_MTU | FUNC_CFG_REQ_ENABLES_MRU | FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS | FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS | @@ -645,7 +645,7 @@ static int bnxt_hwrm_func_cfg(struct bnxt *bp, int num_vfs) mtu = bp->dev->mtu + ETH_HLEN + VLAN_HLEN; req.mru = cpu_to_le16(mtu); - req.mtu = cpu_to_le16(mtu); + req.admin_mtu = cpu_to_le16(mtu); req.num_rsscos_ctxs = cpu_to_le16(1); req.num_cmpl_rings = cpu_to_le16(vf_cp_rings); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index ec9564e584e0..bee6e091a997 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -138,9 +138,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, xdp_prepare_buff(&xdp, *data_ptr - offset, offset, *len, false); orig_data = xdp.data; - rcu_read_lock(); act = bpf_prog_run_xdp(xdp_prog, &xdp); - rcu_read_unlock(); tx_avail = bnxt_tx_avail(bp, txr); /* If the tx ring is not full, we must not update the rx producer yet diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index fcca023f22e5..41f7f078cd27 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -4296,3 +4296,4 @@ MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver"); MODULE_ALIAS("platform:bcmgenet"); MODULE_LICENSE("GPL"); +MODULE_SOFTDEP("pre: mdio-bcm-unimac"); diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 5335244e4577..89d16c587bb7 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -423,6 +423,10 @@ static int bcmgenet_mii_register(struct bcmgenet_priv *priv) int id, ret; pres = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!pres) { + dev_err(&pdev->dev, "Invalid resource\n"); + return -EINVAL; + } memset(&res, 0, sizeof(res)); memset(&ppd, 0, sizeof(ppd)); diff --git a/drivers/net/ethernet/brocade/bna/bfa_cee.c b/drivers/net/ethernet/brocade/bna/bfa_cee.c index 06f221c44802..eeb05e31713f 100644 --- a/drivers/net/ethernet/brocade/bna/bfa_cee.c +++ b/drivers/net/ethernet/brocade/bna/bfa_cee.c @@ -82,7 +82,7 @@ bfa_cee_get_attr_isr(struct bfa_cee *cee, enum bfa_status status) } /** - * bfa_cee_get_attr_isr - CEE ISR for get-stats responses from f/w + * bfa_cee_get_stats_isr - CEE ISR for get-stats responses from f/w * * @cee: Pointer to the CEE module * @status: Return status from the f/w diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 6bc7d41d519b..7d2fe13a52f8 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -2867,6 +2867,9 @@ static struct net_device_stats *gem_get_stats(struct macb *bp) struct gem_stats *hwstat = &bp->hw_stats.gem; struct net_device_stats *nstat = &bp->dev->stats; + if (!netif_running(bp->dev)) + return nstat; + gem_update_stats(bp); nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + @@ -4652,8 +4655,7 @@ static int macb_probe(struct platform_device *pdev) struct macb *bp; int err, val; - regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mem = devm_ioremap_resource(&pdev->dev, regs); + mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); if (IS_ERR(mem)) return PTR_ERR(mem); diff --git a/drivers/net/ethernet/cadence/macb_pci.c b/drivers/net/ethernet/cadence/macb_pci.c index 353393dea639..8b7b59908a1a 100644 --- a/drivers/net/ethernet/cadence/macb_pci.c +++ b/drivers/net/ethernet/cadence/macb_pci.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/** +/* * DOC: Cadence GEM PCI wrapper. * * Copyright (C) 2016 Cadence Design Systems - https://www.cadence.com diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c index 283918aeb741..5c368a9cbbbc 100644 --- a/drivers/net/ethernet/cadence/macb_ptp.c +++ b/drivers/net/ethernet/cadence/macb_ptp.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/** +/* * 1588 PTP support for Cadence GEM device. * * Copyright (C) 2017 Cadence Design Systems - https://www.cadence.com diff --git a/drivers/net/ethernet/calxeda/xgmac.c b/drivers/net/ethernet/calxeda/xgmac.c index bbb453c6a5f7..b6a066404f4b 100644 --- a/drivers/net/ethernet/calxeda/xgmac.c +++ b/drivers/net/ethernet/calxeda/xgmac.c @@ -711,7 +711,7 @@ static void xgmac_rx_refill(struct xgmac_priv *priv) } /** - * init_xgmac_dma_desc_rings - init the RX/TX descriptor rings + * xgmac_dma_desc_rings_init - init the RX/TX descriptor rings * @dev: net device structure * Description: this function initializes the DMA RX/TX descriptors * and allocates the socket buffers. @@ -859,7 +859,7 @@ static void xgmac_free_dma_desc_rings(struct xgmac_priv *priv) } /** - * xgmac_tx: + * xgmac_tx_complete: * @priv: private driver structure * Description: it reclaims resources after transmission completes. */ @@ -1040,7 +1040,7 @@ static int xgmac_open(struct net_device *dev) } /** - * xgmac_release - close entry point of the driver + * xgmac_stop - close entry point of the driver * @dev : device pointer. * Description: * This is the stop entry point of the driver. @@ -1812,7 +1812,7 @@ err_alloc: } /** - * xgmac_dvr_remove + * xgmac_remove * @pdev: platform device pointer * Description: this function resets the TX/RX processes, disables the MAC RX/TX * changes the link status, releases the DMA descriptor rings, diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index 7c5af4beedc6..591229b96257 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -1153,7 +1153,7 @@ static void octeon_destroy_resources(struct octeon_device *oct) * @lio: per-network private data * @start_stop: whether to start or stop */ -static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) +static int send_rx_ctrl_cmd(struct lio *lio, int start_stop) { struct octeon_soft_command *sc; union octnet_cmd *ncmd; @@ -1161,15 +1161,15 @@ static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) int retval; if (oct->props[lio->ifidx].rx_on == start_stop) - return; + return 0; sc = (struct octeon_soft_command *) octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 16, 0); if (!sc) { netif_info(lio, rx_err, lio->netdev, - "Failed to allocate octeon_soft_command\n"); - return; + "Failed to allocate octeon_soft_command struct\n"); + return -ENOMEM; } ncmd = (union octnet_cmd *)sc->virtdptr; @@ -1192,18 +1192,19 @@ static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) if (retval == IQ_SEND_FAILED) { netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n"); octeon_free_soft_command(oct, sc); - return; } else { /* Sleep on a wait queue till the cond flag indicates that the * response arrived or timed-out. */ retval = wait_for_sc_completion_timeout(oct, sc, 0); if (retval) - return; + return retval; oct->props[lio->ifidx].rx_on = start_stop; WRITE_ONCE(sc->caller_is_done, true); } + + return retval; } /** @@ -1778,6 +1779,7 @@ static int liquidio_open(struct net_device *netdev) struct octeon_device_priv *oct_priv = (struct octeon_device_priv *)oct->priv; struct napi_struct *napi, *n; + int ret = 0; if (oct->props[lio->ifidx].napi_enabled == 0) { tasklet_disable(&oct_priv->droq_tasklet); @@ -1813,7 +1815,9 @@ static int liquidio_open(struct net_device *netdev) netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n"); /* tell Octeon to start forwarding packets to host */ - send_rx_ctrl_cmd(lio, 1); + ret = send_rx_ctrl_cmd(lio, 1); + if (ret) + return ret; /* start periodical statistics fetch */ INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats); @@ -1824,7 +1828,7 @@ static int liquidio_open(struct net_device *netdev) dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name); - return 0; + return ret; } /** @@ -1838,6 +1842,7 @@ static int liquidio_stop(struct net_device *netdev) struct octeon_device_priv *oct_priv = (struct octeon_device_priv *)oct->priv; struct napi_struct *napi, *n; + int ret = 0; ifstate_reset(lio, LIO_IFSTATE_RUNNING); @@ -1854,7 +1859,9 @@ static int liquidio_stop(struct net_device *netdev) lio->link_changes++; /* Tell Octeon that nic interface is down. */ - send_rx_ctrl_cmd(lio, 0); + ret = send_rx_ctrl_cmd(lio, 0); + if (ret) + return ret; if (OCTEON_CN23XX_PF(oct)) { if (!oct->msix_on) @@ -1889,7 +1896,7 @@ static int liquidio_stop(struct net_device *netdev) dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); - return 0; + return ret; } /** diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c index 516f166ceff8..ffddb3126a32 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c @@ -595,7 +595,7 @@ static void octeon_destroy_resources(struct octeon_device *oct) * @lio: per-network private data * @start_stop: whether to start or stop */ -static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) +static int send_rx_ctrl_cmd(struct lio *lio, int start_stop) { struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; struct octeon_soft_command *sc; @@ -603,11 +603,16 @@ static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) int retval; if (oct->props[lio->ifidx].rx_on == start_stop) - return; + return 0; sc = (struct octeon_soft_command *) octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 16, 0); + if (!sc) { + netif_info(lio, rx_err, lio->netdev, + "Failed to allocate octeon_soft_command struct\n"); + return -ENOMEM; + } ncmd = (union octnet_cmd *)sc->virtdptr; @@ -635,11 +640,13 @@ static void send_rx_ctrl_cmd(struct lio *lio, int start_stop) */ retval = wait_for_sc_completion_timeout(oct, sc, 0); if (retval) - return; + return retval; oct->props[lio->ifidx].rx_on = start_stop; WRITE_ONCE(sc->caller_is_done, true); } + + return retval; } /** @@ -906,6 +913,7 @@ static int liquidio_open(struct net_device *netdev) struct octeon_device_priv *oct_priv = (struct octeon_device_priv *)oct->priv; struct napi_struct *napi, *n; + int ret = 0; if (!oct->props[lio->ifidx].napi_enabled) { tasklet_disable(&oct_priv->droq_tasklet); @@ -932,11 +940,13 @@ static int liquidio_open(struct net_device *netdev) (LIQUIDIO_NDEV_STATS_POLL_TIME_MS)); /* tell Octeon to start forwarding packets to host */ - send_rx_ctrl_cmd(lio, 1); + ret = send_rx_ctrl_cmd(lio, 1); + if (ret) + return ret; dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name); - return 0; + return ret; } /** @@ -950,9 +960,12 @@ static int liquidio_stop(struct net_device *netdev) struct octeon_device_priv *oct_priv = (struct octeon_device_priv *)oct->priv; struct napi_struct *napi, *n; + int ret = 0; /* tell Octeon to stop forwarding packets to host */ - send_rx_ctrl_cmd(lio, 0); + ret = send_rx_ctrl_cmd(lio, 0); + if (ret) + return ret; netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n"); /* Inform that netif carrier is down */ @@ -986,7 +999,7 @@ static int liquidio_stop(struct net_device *netdev) dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); - return 0; + return ret; } /** diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index c33b4e837515..e2b290135fd9 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -555,9 +555,7 @@ static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog, xdp_prepare_buff(&xdp, hard_start, data - hard_start, len, false); orig_data = xdp.data; - rcu_read_lock(); action = bpf_prog_run_xdp(prog, &xdp); - rcu_read_unlock(); len = xdp.data_end - xdp.data; /* Check if XDP program has changed headers */ diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c index 0c783aadf393..c36fed9c3d73 100644 --- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c +++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c @@ -594,9 +594,6 @@ static void bgx_lmac_handler(struct net_device *netdev) struct phy_device *phydev; int link_changed = 0; - if (!lmac) - return; - phydev = lmac->phydev; if (!phydev->link && lmac->last_link) diff --git a/drivers/net/ethernet/chelsio/cxgb3/adapter.h b/drivers/net/ethernet/chelsio/cxgb3/adapter.h index f80fbd81b609..6d682b7c7aac 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/adapter.h +++ b/drivers/net/ethernet/chelsio/cxgb3/adapter.h @@ -178,7 +178,7 @@ struct sge_txq { /* state for an SGE Tx queue */ unsigned int token; /* WR token */ dma_addr_t phys_addr; /* physical address of the ring */ struct sk_buff_head sendq; /* List of backpressured offload packets */ - struct tasklet_struct qresume_tsk; /* restarts the queue */ + struct work_struct qresume_task; /* restarts the queue */ unsigned int cntxt_id; /* SGE context id for the Tx q */ unsigned long stops; /* # of times q has been stopped */ unsigned long restarts; /* # of queue restarts */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/common.h b/drivers/net/ethernet/chelsio/cxgb3/common.h index 1bd7d89666c4..b706f2fbe4f4 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/common.h +++ b/drivers/net/ethernet/chelsio/cxgb3/common.h @@ -770,4 +770,6 @@ int t3_xaui_direct_phy_prep(struct cphy *phy, struct adapter *adapter, int phy_addr, const struct mdio_ops *mdio_ops); int t3_aq100x_phy_prep(struct cphy *phy, struct adapter *adapter, int phy_addr, const struct mdio_ops *mdio_ops); + +extern struct workqueue_struct *cxgb3_wq; #endif /* __CHELSIO_COMMON_H */ diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index 84ad7261e243..57f210c53afc 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -1273,14 +1273,14 @@ static int cxgb_up(struct adapter *adap) free_irq(adap->msix_info[0].vec, adap); goto irq_err; } - } else if ((err = request_irq(adap->pdev->irq, - t3_intr_handler(adap, - adap->sge.qs[0].rspq. - polling), - (adap->flags & USING_MSI) ? - 0 : IRQF_SHARED, - adap->name, adap))) - goto irq_err; + } else { + err = request_irq(adap->pdev->irq, + t3_intr_handler(adap, adap->sge.qs[0].rspq.polling), + (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, + adap->name, adap); + if (err) + goto irq_err; + } enable_all_napi(adap); t3_sge_start(adap); @@ -3098,8 +3098,9 @@ static void set_nqsets(struct adapter *adap) nqsets = num_cpus; if (nqsets < 1 || hwports == 4) nqsets = 1; - } else + } else { nqsets = 1; + } for_each_port(adap, i) { struct port_info *pi = adap2pinfo(adap, i); diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge.c b/drivers/net/ethernet/chelsio/cxgb3/sge.c index 1cc3c51eff71..cb5c79c43bc9 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb3/sge.c @@ -665,7 +665,7 @@ static void t3_reset_qset(struct sge_qset *q) /** - * free_qset - free the resources of an SGE queue set + * t3_free_qset - free the resources of an SGE queue set * @adapter: the adapter owning the queue set * @q: the queue set * @@ -1256,7 +1256,7 @@ static inline void t3_stop_tx_queue(struct netdev_queue *txq, } /** - * eth_xmit - add a packet to the Ethernet Tx queue + * t3_eth_xmit - add a packet to the Ethernet Tx queue * @skb: the packet * @dev: the egress net device * @@ -1518,14 +1518,15 @@ static int ctrl_xmit(struct adapter *adap, struct sge_txq *q, /** * restart_ctrlq - restart a suspended control queue - * @t: pointer to the tasklet associated with this handler + * @w: pointer to the work associated with this handler * * Resumes transmission on a suspended Tx control queue. */ -static void restart_ctrlq(struct tasklet_struct *t) +static void restart_ctrlq(struct work_struct *w) { struct sk_buff *skb; - struct sge_qset *qs = from_tasklet(qs, t, txq[TXQ_CTRL].qresume_tsk); + struct sge_qset *qs = container_of(w, struct sge_qset, + txq[TXQ_CTRL].qresume_task); struct sge_txq *q = &qs->txq[TXQ_CTRL]; spin_lock(&q->lock); @@ -1736,14 +1737,15 @@ again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK); /** * restart_offloadq - restart a suspended offload queue - * @t: pointer to the tasklet associated with this handler + * @w: pointer to the work associated with this handler * * Resumes transmission on a suspended Tx offload queue. */ -static void restart_offloadq(struct tasklet_struct *t) +static void restart_offloadq(struct work_struct *w) { struct sk_buff *skb; - struct sge_qset *qs = from_tasklet(qs, t, txq[TXQ_OFLD].qresume_tsk); + struct sge_qset *qs = container_of(w, struct sge_qset, + txq[TXQ_OFLD].qresume_task); struct sge_txq *q = &qs->txq[TXQ_OFLD]; const struct port_info *pi = netdev_priv(qs->netdev); struct adapter *adap = pi->adapter; @@ -1998,13 +2000,17 @@ static void restart_tx(struct sge_qset *qs) should_restart_tx(&qs->txq[TXQ_OFLD]) && test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { qs->txq[TXQ_OFLD].restarts++; - tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk); + + /* The work can be quite lengthy so we use driver's own queue */ + queue_work(cxgb3_wq, &qs->txq[TXQ_OFLD].qresume_task); } if (test_bit(TXQ_CTRL, &qs->txq_stopped) && should_restart_tx(&qs->txq[TXQ_CTRL]) && test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { qs->txq[TXQ_CTRL].restarts++; - tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk); + + /* The work can be quite lengthy so we use driver's own queue */ + queue_work(cxgb3_wq, &qs->txq[TXQ_CTRL].qresume_task); } } @@ -3085,8 +3091,8 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, skb_queue_head_init(&q->txq[i].sendq); } - tasklet_setup(&q->txq[TXQ_OFLD].qresume_tsk, restart_offloadq); - tasklet_setup(&q->txq[TXQ_CTRL].qresume_tsk, restart_ctrlq); + INIT_WORK(&q->txq[TXQ_OFLD].qresume_task, restart_offloadq); + INIT_WORK(&q->txq[TXQ_CTRL].qresume_task, restart_ctrlq); q->fl[0].gen = q->fl[1].gen = 1; q->fl[0].size = p->fl_size; @@ -3276,11 +3282,11 @@ void t3_sge_start(struct adapter *adap) * * Can be invoked from interrupt context e.g. error handler. * - * Note that this function cannot disable the restart of tasklets as + * Note that this function cannot disable the restart of works as * it cannot wait if called from interrupt context, however the - * tasklets will have no effect since the doorbells are disabled. The + * works will have no effect since the doorbells are disabled. The * driver will call tg3_sge_stop() later from process context, at - * which time the tasklets will be stopped if they are still running. + * which time the works will be stopped if they are still running. */ void t3_sge_stop_dma(struct adapter *adap) { @@ -3292,7 +3298,7 @@ void t3_sge_stop_dma(struct adapter *adap) * @adap: the adapter * * Called from process context. Disables the DMA engine and any - * pending queue restart tasklets. + * pending queue restart works. */ void t3_sge_stop(struct adapter *adap) { @@ -3303,8 +3309,8 @@ void t3_sge_stop(struct adapter *adap) for (i = 0; i < SGE_QSETS; ++i) { struct sge_qset *qs = &adap->sge.qs[i]; - tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk); - tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk); + cancel_work_sync(&qs->txq[TXQ_OFLD].qresume_task); + cancel_work_sync(&qs->txq[TXQ_CTRL].qresume_task); } } @@ -3371,7 +3377,7 @@ void t3_sge_prep(struct adapter *adap, struct sge_params *p) q->coalesce_usecs = 5; q->rspq_size = 1024; q->fl_size = 1024; - q->jumbo_size = 512; + q->jumbo_size = 512; q->txq_size[TXQ_ETH] = 1024; q->txq_size[TXQ_OFLD] = 1024; q->txq_size[TXQ_CTRL] = 256; diff --git a/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c b/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c index 12fcf84d67ad..163efab27e9b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c +++ b/drivers/net/ethernet/chelsio/cxgb4/clip_tbl.c @@ -106,8 +106,7 @@ int cxgb4_clip_get(const struct net_device *dev, const u32 *lip, u8 v6) if (!list_empty(&ctbl->ce_free_head)) { ce = list_first_entry(&ctbl->ce_free_head, struct clip_entry, list); - list_del(&ce->list); - INIT_LIST_HEAD(&ce->list); + list_del_init(&ce->list); spin_lock_init(&ce->lock); refcount_set(&ce->refcnt, 0); atomic_dec(&ctbl->nfree); @@ -179,8 +178,7 @@ found: write_lock_bh(&ctbl->lock); spin_lock_bh(&ce->lock); if (refcount_dec_and_test(&ce->refcnt)) { - list_del(&ce->list); - INIT_LIST_HEAD(&ce->list); + list_del_init(&ce->list); list_add_tail(&ce->list, &ctbl->ce_free_head); atomic_inc(&ctbl->nfree); if (v6) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h index 314f8d806723..9058f09f921e 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -2177,8 +2177,6 @@ int cxgb4_update_mac_filt(struct port_info *pi, unsigned int viid, bool persistent, u8 *smt_idx); int cxgb4_get_msix_idx_from_bmap(struct adapter *adap); void cxgb4_free_msix_idx_in_bmap(struct adapter *adap, u32 msix_idx); -int cxgb_open(struct net_device *dev); -int cxgb_close(struct net_device *dev); void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q); void cxgb4_quiesce_rx(struct sge_rspq *q); int cxgb4_port_mirror_alloc(struct net_device *dev); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 61ea3ec5c3fc..83ed10ac8660 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -1337,13 +1337,27 @@ static int cxgb4_ethtool_flash_phy(struct net_device *netdev, return ret; } - spin_lock_bh(&adap->win0_lock); + /* We have to RESET the chip/firmware because we need the + * chip in uninitialized state for loading new PHY image. + * Otherwise, the running firmware will only store the PHY + * image in local RAM which will be lost after next reset. + */ + ret = t4_fw_reset(adap, adap->mbox, PIORSTMODE_F | PIORST_F); + if (ret < 0) { + dev_err(adap->pdev_dev, + "Set FW to RESET for flashing PHY FW failed. ret: %d\n", + ret); + return ret; + } + ret = t4_load_phy_fw(adap, MEMWIN_NIC, NULL, data, size); - spin_unlock_bh(&adap->win0_lock); - if (ret) - dev_err(adap->pdev_dev, "Failed to load PHY FW\n"); + if (ret < 0) { + dev_err(adap->pdev_dev, "Failed to load PHY FW. ret: %d\n", + ret); + return ret; + } - return ret; + return 0; } static int cxgb4_ethtool_flash_fw(struct net_device *netdev, @@ -1610,16 +1624,14 @@ static struct filter_entry *cxgb4_get_filter_entry(struct adapter *adap, u32 ftid) { struct tid_info *t = &adap->tids; - struct filter_entry *f; - if (ftid < t->nhpftids) - f = &adap->tids.hpftid_tab[ftid]; - else if (ftid < t->nftids) - f = &adap->tids.ftid_tab[ftid - t->nhpftids]; - else - f = lookup_tid(&adap->tids, ftid); + if (ftid >= t->hpftid_base && ftid < t->hpftid_base + t->nhpftids) + return &t->hpftid_tab[ftid - t->hpftid_base]; - return f; + if (ftid >= t->ftid_base && ftid < t->ftid_base + t->nftids) + return &t->ftid_tab[ftid - t->ftid_base]; + + return lookup_tid(t, ftid); } static void cxgb4_fill_filter_rule(struct ethtool_rx_flow_spec *fs, @@ -1826,6 +1838,11 @@ static int cxgb4_ntuple_del_filter(struct net_device *dev, filter_id = filter_info->loc_array[cmd->fs.location]; f = cxgb4_get_filter_entry(adapter, filter_id); + if (f->fs.prio) + filter_id -= adapter->tids.hpftid_base; + else if (!f->fs.hash) + filter_id -= (adapter->tids.ftid_base - adapter->tids.nhpftids); + ret = cxgb4_flow_rule_destroy(dev, f->fs.tc_prio, &f->fs, filter_id); if (ret) goto err; @@ -1885,6 +1902,11 @@ static int cxgb4_ntuple_set_filter(struct net_device *netdev, filter_info = &adapter->ethtool_filters->port[pi->port_id]; + if (fs.prio) + tid += adapter->tids.hpftid_base; + else if (!fs.hash) + tid += (adapter->tids.ftid_base - adapter->tids.nhpftids); + filter_info->loc_array[cmd->fs.location] = tid; set_bit(cmd->fs.location, filter_info->bmap); filter_info->in_use++; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c index bc581b149b11..6260b3bebd2b 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c @@ -198,7 +198,7 @@ static void set_nat_params(struct adapter *adap, struct filter_entry *f, WORD_MASK, f->fs.nat_lip[3] | f->fs.nat_lip[2] << 8 | f->fs.nat_lip[1] << 16 | - (u64)f->fs.nat_lip[0] << 25, 1); + (u64)f->fs.nat_lip[0] << 24, 1); } } @@ -1042,7 +1042,7 @@ void clear_all_filters(struct adapter *adapter) cxgb4_del_filter(dev, f->tid, &f->fs); } - sb = t4_read_reg(adapter, LE_DB_SRVR_START_INDEX_A); + sb = adapter->tids.stid_base; for (i = 0; i < sb; i++) { f = (struct filter_entry *)adapter->tids.tid_tab[i]; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index 6264bc66a4fc..9a2b166d651e 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -2834,7 +2834,7 @@ static void cxgb_down(struct adapter *adapter) /* * net_device operations */ -int cxgb_open(struct net_device *dev) +static int cxgb_open(struct net_device *dev) { struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; @@ -2882,7 +2882,7 @@ out_unlock: return err; } -int cxgb_close(struct net_device *dev) +static int cxgb_close(struct net_device *dev) { struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; @@ -3894,7 +3894,6 @@ static const struct net_device_ops cxgb4_mgmt_netdev_ops = { .ndo_set_vf_vlan = cxgb4_mgmt_set_vf_vlan, .ndo_set_vf_link_state = cxgb4_mgmt_set_vf_link_state, }; -#endif static void cxgb4_mgmt_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) @@ -3909,6 +3908,7 @@ static void cxgb4_mgmt_get_drvinfo(struct net_device *dev, static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { .get_drvinfo = cxgb4_mgmt_get_drvinfo, }; +#endif static void notify_fatal_err(struct work_struct *work) { @@ -4424,10 +4424,8 @@ static int adap_init0_phy(struct adapter *adap) /* Load PHY Firmware onto adapter. */ - spin_lock_bh(&adap->win0_lock); ret = t4_load_phy_fw(adap, MEMWIN_NIC, phy_info->phy_fw_version, (u8 *)phyf->data, phyf->size); - spin_unlock_bh(&adap->win0_lock); if (ret < 0) dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", -ret); @@ -6480,9 +6478,9 @@ static void cxgb4_ktls_dev_del(struct net_device *netdev, adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_del(netdev, tls_ctx, direction); - cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); out_unlock: + cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); mutex_unlock(&uld_mutex); } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c index 70dbee89118e..5bf117d2179f 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ptp.c @@ -446,7 +446,7 @@ void cxgb4_ptp_init(struct adapter *adapter) } /** - * cxgb4_ptp_remove - disable PTP device and stop the overflow check + * cxgb4_ptp_stop - disable PTP device and stop the overflow check * @adapter: board private structure * * Stop the PTP support. diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c index 1b88bd1c2dbe..dd9be229819a 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c @@ -997,20 +997,16 @@ int cxgb4_tc_flower_destroy(struct net_device *dev, if (!ch_flower) return -ENOENT; + rhashtable_remove_fast(&adap->flower_tbl, &ch_flower->node, + adap->flower_ht_params); + ret = cxgb4_flow_rule_destroy(dev, ch_flower->fs.tc_prio, &ch_flower->fs, ch_flower->filter_id); if (ret) - goto err; + netdev_err(dev, "Flow rule destroy failed for tid: %u, ret: %d", + ch_flower->filter_id, ret); - ret = rhashtable_remove_fast(&adap->flower_tbl, &ch_flower->node, - adap->flower_ht_params); - if (ret) { - netdev_err(dev, "Flow remove from rhashtable failed"); - goto err; - } kfree_rcu(ch_flower, rcu); - -err: return ret; } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c index 6c259de96f96..338b04f339b3 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_mqprio.c @@ -589,7 +589,8 @@ int cxgb4_setup_tc_mqprio(struct net_device *dev, * down before configuring tc params. */ if (netif_running(dev)) { - cxgb_close(dev); + netif_tx_stop_all_queues(dev); + netif_carrier_off(dev); needs_bring_up = true; } @@ -615,8 +616,10 @@ int cxgb4_setup_tc_mqprio(struct net_device *dev, } out: - if (needs_bring_up) - cxgb_open(dev); + if (needs_bring_up) { + netif_tx_start_all_queues(dev); + netif_carrier_on(dev); + } mutex_unlock(&adap->tc_mqprio->mqprio_mutex); return ret; diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index 1e5f2edb70cf..6a099cb34b12 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -2556,6 +2556,12 @@ int cxgb4_ethofld_send_flowc(struct net_device *dev, u32 eotid, u32 tc) if (!eosw_txq) return -ENOMEM; + if (!(adap->flags & CXGB4_FW_OK)) { + /* Don't stall caller when access to FW is lost */ + complete(&eosw_txq->completion); + return -EIO; + } + skb = alloc_skb(len, GFP_KERNEL); if (!skb) return -ENOMEM; diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c index 9428ef1f04a8..6606fb8b3e42 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c @@ -3060,16 +3060,19 @@ int t4_read_flash(struct adapter *adapter, unsigned int addr, * @addr: the start address to write * @n: length of data to write in bytes * @data: the data to write + * @byte_oriented: whether to store data as bytes or as words * * Writes up to a page of data (256 bytes) to the serial flash starting * at the given address. All the data must be written to the same page. + * If @byte_oriented is set the write data is stored as byte stream + * (i.e. matches what on disk), otherwise in big-endian. */ static int t4_write_flash(struct adapter *adapter, unsigned int addr, - unsigned int n, const u8 *data) + unsigned int n, const u8 *data, bool byte_oriented) { - int ret; - u32 buf[64]; unsigned int i, c, left, val, offset = addr & 0xff; + u32 buf[64]; + int ret; if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE) return -EINVAL; @@ -3080,10 +3083,14 @@ static int t4_write_flash(struct adapter *adapter, unsigned int addr, (ret = sf1_write(adapter, 4, 1, 1, val)) != 0) goto unlock; - for (left = n; left; left -= c) { + for (left = n; left; left -= c, data += c) { c = min(left, 4U); - for (val = 0, i = 0; i < c; ++i) - val = (val << 8) + *data++; + for (val = 0, i = 0; i < c; ++i) { + if (byte_oriented) + val = (val << 8) + data[i]; + else + val = (val << 8) + data[c - i - 1]; + } ret = sf1_write(adapter, c, c != left, 1, val); if (ret) @@ -3096,7 +3103,8 @@ static int t4_write_flash(struct adapter *adapter, unsigned int addr, t4_write_reg(adapter, SF_OP_A, 0); /* unlock SF */ /* Read the page to verify the write succeeded */ - ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1); + ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, + byte_oriented); if (ret) return ret; @@ -3692,7 +3700,7 @@ int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size) */ memcpy(first_page, fw_data, SF_PAGE_SIZE); ((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff); - ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page); + ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page, true); if (ret) goto out; @@ -3700,14 +3708,14 @@ int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size) for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) { addr += SF_PAGE_SIZE; fw_data += SF_PAGE_SIZE; - ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data); + ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, true); if (ret) goto out; } - ret = t4_write_flash(adap, - fw_start + offsetof(struct fw_hdr, fw_ver), - sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver); + ret = t4_write_flash(adap, fw_start + offsetof(struct fw_hdr, fw_ver), + sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, + true); out: if (ret) dev_err(adap->pdev_dev, "firmware download failed, error %d\n", @@ -3812,9 +3820,11 @@ int t4_load_phy_fw(struct adapter *adap, int win, /* Copy the supplied PHY Firmware image to the adapter memory location * allocated by the adapter firmware. */ + spin_lock_bh(&adap->win0_lock); ret = t4_memory_rw(adap, win, mtype, maddr, phy_fw_size, (__be32 *)phy_fw_data, T4_MEMORY_WRITE); + spin_unlock_bh(&adap->win0_lock); if (ret) return ret; @@ -6983,7 +6993,7 @@ int t4_fw_bye(struct adapter *adap, unsigned int mbox) } /** - * t4_init_cmd - ask FW to initialize the device + * t4_early_init - ask FW to initialize the device * @adap: the adapter * @mbox: mailbox to use for the FW command * @@ -7782,7 +7792,6 @@ int t4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, int idx, bool sleep_ok) { struct fw_vi_mac_exact *p; - u8 addr[] = {0, 0, 0, 0, 0, 0}; struct fw_vi_mac_cmd c; int ret = 0; u32 exact; @@ -7799,7 +7808,7 @@ int t4_free_encap_mac_filt(struct adapter *adap, unsigned int viid, p = c.u.exact; p->valid_to_idx = cpu_to_be16(FW_VI_MAC_CMD_VALID_F | FW_VI_MAC_CMD_IDX_V(idx)); - memcpy(p->macaddr, addr, sizeof(p->macaddr)); + eth_zero_addr(p->macaddr); ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok); return ret; } @@ -10208,7 +10217,7 @@ int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size) n = size - i; else n = SF_PAGE_SIZE; - ret = t4_write_flash(adap, addr, n, cfg_data); + ret = t4_write_flash(adap, addr, n, cfg_data, true); if (ret) goto out; @@ -10224,7 +10233,7 @@ out: } /** - * t4_set_vf_mac - Set MAC address for the specified VF + * t4_set_vf_mac_acl - Set MAC address for the specified VF * @adapter: The adapter * @vf: one of the VFs instantiated by the specified PF * @naddr: the number of MAC addresses @@ -10677,13 +10686,14 @@ int t4_load_boot(struct adapter *adap, u8 *boot_data, for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) { addr += SF_PAGE_SIZE; boot_data += SF_PAGE_SIZE; - ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, boot_data); + ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, boot_data, + false); if (ret) goto out; } ret = t4_write_flash(adap, boot_sector, SF_PAGE_SIZE, - (const u8 *)header); + (const u8 *)header, false); out: if (ret) @@ -10758,7 +10768,7 @@ int t4_load_bootcfg(struct adapter *adap, const u8 *cfg_data, unsigned int size) for (i = 0; i < size; i += SF_PAGE_SIZE) { n = min_t(u32, size - i, SF_PAGE_SIZE); - ret = t4_write_flash(adap, addr, n, cfg_data); + ret = t4_write_flash(adap, addr, n, cfg_data, false); if (ret) goto out; @@ -10770,7 +10780,8 @@ int t4_load_bootcfg(struct adapter *adap, const u8 *cfg_data, unsigned int size) for (i = 0; i < npad; i++) { u8 data = 0; - ret = t4_write_flash(adap, cfg_addr + size + i, 1, &data); + ret = t4_write_flash(adap, cfg_addr + size + i, 1, &data, + false); if (ret) goto out; } diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c index 95657da0aa4b..7bc80eeb2c21 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c @@ -954,7 +954,7 @@ static void write_sgl(const struct sk_buff *skb, struct sge_txq *tq, } /** - * check_ring_tx_db - check and potentially ring a TX queue's doorbell + * ring_tx_db - check and potentially ring a TX queue's doorbell * @adapter: the adapter * @tq: the TX queue * @n: number of new descriptors to give to HW diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c index ef3f1e92632f..59683f79959c 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c @@ -59,6 +59,7 @@ static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len) } static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); +static void clear_conn_resources(struct chcr_ktls_info *tx_info); /* * chcr_ktls_save_keys: calculate and save crypto keys. * @tx_info - driver specific tls info. @@ -364,10 +365,14 @@ static void chcr_ktls_dev_del(struct net_device *netdev, chcr_get_ktls_tx_context(tls_ctx); struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; struct ch_ktls_port_stats_debug *port_stats; + struct chcr_ktls_uld_ctx *u_ctx; if (!tx_info) return; + u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle; + if (u_ctx && u_ctx->detach) + return; /* clear l2t entry */ if (tx_info->l2te) cxgb4_l2t_release(tx_info->l2te); @@ -384,6 +389,8 @@ static void chcr_ktls_dev_del(struct net_device *netdev, if (tx_info->tid != -1) { cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, tx_info->tid, tx_info->ip_family); + + xa_erase(&u_ctx->tid_list, tx_info->tid); } port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; @@ -411,6 +418,7 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, struct tls_context *tls_ctx = tls_get_ctx(sk); struct ch_ktls_port_stats_debug *port_stats; struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_uld_ctx *u_ctx; struct chcr_ktls_info *tx_info; struct dst_entry *dst; struct adapter *adap; @@ -425,6 +433,7 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, adap = pi->adapter; port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id]; atomic64_inc(&port_stats->ktls_tx_connection_open); + u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; if (direction == TLS_OFFLOAD_CTX_DIR_RX) { pr_err("not expecting for RX direction\n"); @@ -434,6 +443,9 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, if (tx_ctx->chcr_info) goto out; + if (u_ctx && u_ctx->detach) + goto out; + tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); if (!tx_info) goto out; @@ -569,6 +581,8 @@ free_tid: cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, tx_info->tid, tx_info->ip_family); + xa_erase(&u_ctx->tid_list, tx_info->tid); + put_module: /* release module refcount */ module_put(THIS_MODULE); @@ -633,8 +647,12 @@ static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, { const struct cpl_act_open_rpl *p = (void *)input; struct chcr_ktls_info *tx_info = NULL; + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_uld_ctx *u_ctx; unsigned int atid, tid, status; + struct tls_context *tls_ctx; struct tid_info *t; + int ret = 0; tid = GET_TID(p); status = AOPEN_STATUS_G(ntohl(p->atid_status)); @@ -666,14 +684,29 @@ static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, if (!status) { tx_info->tid = tid; cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); + /* Adding tid */ + tls_ctx = tls_get_ctx(tx_info->sk); + tx_ctx = chcr_get_ktls_tx_context(tls_ctx); + u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; + if (u_ctx) { + ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx, + GFP_NOWAIT); + if (ret < 0) { + pr_err("%s: Failed to allocate tid XA entry = %d\n", + __func__, tx_info->tid); + tx_info->open_state = CH_KTLS_OPEN_FAILURE; + goto out; + } + } tx_info->open_state = CH_KTLS_OPEN_SUCCESS; } else { tx_info->open_state = CH_KTLS_OPEN_FAILURE; } +out: spin_unlock(&tx_info->lock); complete(&tx_info->completion); - return 0; + return ret; } /* @@ -2090,6 +2123,8 @@ static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi) goto out; } u_ctx->lldi = *lldi; + u_ctx->detach = false; + xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH); out: return u_ctx; } @@ -2123,6 +2158,45 @@ static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp, return 0; } +static void clear_conn_resources(struct chcr_ktls_info *tx_info) +{ + /* clear l2t entry */ + if (tx_info->l2te) + cxgb4_l2t_release(tx_info->l2te); + +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(tx_info->netdev, (const u32 *) + &tx_info->sk->sk_v6_rcv_saddr, + 1); +#endif + + /* clear tid */ + if (tx_info->tid != -1) + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tx_info->tid, tx_info->ip_family); +} + +static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx) +{ + struct ch_ktls_port_stats_debug *port_stats; + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_info *tx_info; + unsigned long index; + + xa_for_each(&u_ctx->tid_list, index, tx_ctx) { + tx_info = tx_ctx->chcr_info; + clear_conn_resources(tx_info); + port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; + atomic64_inc(&port_stats->ktls_tx_connection_close); + kvfree(tx_info); + tx_ctx->chcr_info = NULL; + /* release module refcount */ + module_put(THIS_MODULE); + } +} + static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state) { struct chcr_ktls_uld_ctx *u_ctx = handle; @@ -2139,7 +2213,10 @@ static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state) case CXGB4_STATE_DETACH: pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev)); mutex_lock(&dev_mutex); + u_ctx->detach = true; list_del(&u_ctx->entry); + ch_ktls_reset_all_conn(u_ctx); + xa_destroy(&u_ctx->tid_list); mutex_unlock(&dev_mutex); break; default: @@ -2178,6 +2255,7 @@ static void __exit chcr_ktls_exit(void) adap = pci_get_drvdata(u_ctx->lldi.pdev); memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats)); list_del(&u_ctx->entry); + xa_destroy(&u_ctx->tid_list); kfree(u_ctx); } mutex_unlock(&dev_mutex); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h index 18b3b1f02415..10572dc55365 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h @@ -75,6 +75,8 @@ struct chcr_ktls_ofld_ctx_tx { struct chcr_ktls_uld_ctx { struct list_head entry; struct cxgb4_lld_info lldi; + struct xarray tid_list; + bool detach; }; static inline struct chcr_ktls_ofld_ctx_tx * diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c index 19dc7dc054a2..bcad69c48074 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -2134,7 +2134,7 @@ static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb) sk->sk_err = ETIMEDOUT; if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_error_report(sk); + sk_error_report(sk); if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb)) return; diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c index 188d871f6b8c..c320cc8ca68d 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c @@ -1564,8 +1564,10 @@ found_ok_skb: cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE, sizeof(thdr->type), &thdr->type); - if (cerr && thdr->type != TLS_RECORD_TYPE_DATA) - return -EIO; + if (cerr && thdr->type != TLS_RECORD_TYPE_DATA) { + copied = -EIO; + break; + } /* don't send tls header, skip copy */ goto skip_copy; } diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 8df6f081f244..c2ebb3388789 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -2356,8 +2356,6 @@ static int gemini_ethernet_port_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct gemini_ethernet *geth; struct net_device *netdev; - struct resource *gmacres; - struct resource *dmares; struct device *parent; unsigned int id; int irq; @@ -2390,24 +2388,18 @@ static int gemini_ethernet_port_probe(struct platform_device *pdev) port->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); /* DMA memory */ - dmares = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!dmares) { - dev_err(dev, "no DMA resource\n"); - return -ENODEV; - } - port->dma_base = devm_ioremap_resource(dev, dmares); - if (IS_ERR(port->dma_base)) + port->dma_base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(port->dma_base)) { + dev_err(dev, "get DMA address failed\n"); return PTR_ERR(port->dma_base); + } /* GMAC config memory */ - gmacres = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!gmacres) { - dev_err(dev, "no GMAC resource\n"); - return -ENODEV; - } - port->gmac_base = devm_ioremap_resource(dev, gmacres); - if (IS_ERR(port->gmac_base)) + port->gmac_base = devm_platform_get_and_ioremap_resource(pdev, 1, NULL); + if (IS_ERR(port->gmac_base)) { + dev_err(dev, "get GMAC address failed\n"); return PTR_ERR(port->gmac_base); + } /* Interrupt */ irq = platform_get_irq(pdev, 0); @@ -2502,10 +2494,6 @@ static int gemini_ethernet_port_probe(struct platform_device *pdev) if (ret) goto unprepare; - netdev_info(netdev, - "irq %d, DMA @ 0x%pap, GMAC @ 0x%pap\n", - port->irq, &dmares->start, - &gmacres->start); return 0; unprepare: @@ -2544,17 +2532,13 @@ static int gemini_ethernet_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct gemini_ethernet *geth; unsigned int retry = 5; - struct resource *res; u32 val; /* Global registers */ geth = devm_kzalloc(dev, sizeof(*geth), GFP_KERNEL); if (!geth) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - geth->base = devm_ioremap_resource(dev, res); + geth->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); if (IS_ERR(geth->base)) return PTR_ERR(geth->base); geth->dev = dev; diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c index b018195f0243..117c26fa5909 100644 --- a/drivers/net/ethernet/dec/tulip/de2104x.c +++ b/drivers/net/ethernet/dec/tulip/de2104x.c @@ -832,8 +832,8 @@ static struct net_device_stats *de_get_stats(struct net_device *dev) /* The chip only need report frame silently dropped. */ spin_lock_irq(&de->lock); - if (netif_running(dev) && netif_device_present(dev)) - __de_get_stats(de); + if (netif_running(dev) && netif_device_present(dev)) + __de_get_stats(de); spin_unlock_irq(&de->lock); return &dev->stats; diff --git a/drivers/net/ethernet/dec/tulip/de4x5.c b/drivers/net/ethernet/dec/tulip/de4x5.c index 683e328b5461..b125d7faefdf 100644 --- a/drivers/net/ethernet/dec/tulip/de4x5.c +++ b/drivers/net/ethernet/dec/tulip/de4x5.c @@ -396,7 +396,7 @@ <earl@exis.net>. Updated the PCI interface to conform with the latest version. I hope nothing is broken... - Add TX done interrupt modification from suggestion + Add TX done interrupt modification from suggestion by <Austin.Donnelly@cl.cam.ac.uk>. Fix is_anc_capable() bug reported by <Austin.Donnelly@cl.cam.ac.uk>. @@ -1499,7 +1499,7 @@ de4x5_queue_pkt(struct sk_buff *skb, struct net_device *dev) spin_lock_irqsave(&lp->lock, flags); netif_stop_queue(dev); load_packet(dev, skb->data, TD_IC | TD_LS | TD_FS | skb->len, skb); - lp->stats.tx_bytes += skb->len; + lp->stats.tx_bytes += skb->len; outl(POLL_DEMAND, DE4X5_TPD);/* Start the TX */ lp->tx_new = (lp->tx_new + 1) % lp->txRingSize; @@ -1651,7 +1651,7 @@ de4x5_rx(struct net_device *dev) /* Update stats */ lp->stats.rx_packets++; - lp->stats.rx_bytes += pkt_len; + lp->stats.rx_bytes += pkt_len; } } diff --git a/drivers/net/ethernet/dec/tulip/dmfe.c b/drivers/net/ethernet/dec/tulip/dmfe.c index 87a27fe2992d..c763b692e164 100644 --- a/drivers/net/ethernet/dec/tulip/dmfe.c +++ b/drivers/net/ethernet/dec/tulip/dmfe.c @@ -518,7 +518,7 @@ static void dmfe_remove_one(struct pci_dev *pdev) DMFE_DBUG(0, "dmfe_remove_one()", 0); - if (dev) { + if (dev) { unregister_netdev(dev); pci_iounmap(db->pdev, db->ioaddr); @@ -567,10 +567,10 @@ static int dmfe_open(struct net_device *dev) /* CR6 operation mode decision */ if ( !chkmode || (db->chip_id == PCI_DM9132_ID) || (db->chip_revision >= 0x30) ) { - db->cr6_data |= DMFE_TXTH_256; + db->cr6_data |= DMFE_TXTH_256; db->cr0_data = CR0_DEFAULT; db->dm910x_chk_mode=4; /* Enter the normal mode */ - } else { + } else { db->cr6_data |= CR6_SFT; /* Store & Forward mode */ db->cr0_data = 0; db->dm910x_chk_mode = 1; /* Enter the check mode */ @@ -903,7 +903,7 @@ static void dmfe_free_tx_pkt(struct net_device *dev, struct dmfe_board_info *db) } } - txptr = txptr->next_tx_desc; + txptr = txptr->next_tx_desc; }/* End of while */ /* Update TX remove pointer to next */ @@ -1121,7 +1121,7 @@ static void dmfe_timer(struct timer_list *t) void __iomem *ioaddr = db->ioaddr; u32 tmp_cr8; unsigned char tmp_cr12; - unsigned long flags; + unsigned long flags; int link_ok, link_ok_phy; @@ -1217,7 +1217,7 @@ static void dmfe_timer(struct timer_list *t) if (link_ok_phy != link_ok) { DMFE_DBUG (0, "PHY and chip report different link status", 0); link_ok = link_ok | link_ok_phy; - } + } if ( !link_ok && netif_carrier_ok(dev)) { /* Link Failed */ @@ -1699,14 +1699,14 @@ static void dmfe_set_phyxcer(struct dmfe_board_info *db) if (db->chip_id == PCI_DM9009_ID) phy_reg &= 0x61; } - /* Write new capability to Phyxcer Reg4 */ + /* Write new capability to Phyxcer Reg4 */ if ( !(phy_reg & 0x01e0)) { phy_reg|=db->PHY_reg4; db->media_mode|=DMFE_AUTO; } dmfe_phy_write(db->ioaddr, db->phy_addr, 4, phy_reg, db->chip_id); - /* Restart Auto-Negotiation */ + /* Restart Auto-Negotiation */ if ( db->chip_type && (db->chip_id == PCI_DM9102_ID) ) dmfe_phy_write(db->ioaddr, db->phy_addr, 0, 0x1800, db->chip_id); if ( !db->chip_type ) @@ -1754,7 +1754,7 @@ static void dmfe_process_mode(struct dmfe_board_info *db) } dmfe_phy_write(db->ioaddr, db->phy_addr, 0, phy_reg, db->chip_id); - if ( db->chip_type && (db->chip_id == PCI_DM9102_ID) ) + if ( db->chip_type && (db->chip_id == PCI_DM9102_ID) ) mdelay(20); dmfe_phy_write(db->ioaddr, db->phy_addr, 0, phy_reg, db->chip_id); diff --git a/drivers/net/ethernet/dec/tulip/pnic2.c b/drivers/net/ethernet/dec/tulip/pnic2.c index 412adaa7fdf8..72a09156b48b 100644 --- a/drivers/net/ethernet/dec/tulip/pnic2.c +++ b/drivers/net/ethernet/dec/tulip/pnic2.c @@ -351,7 +351,7 @@ void pnic2_lnk_change(struct net_device *dev, int csr5) del_timer_sync(&tp->timer); pnic2_start_nway(dev); tp->timer.expires = RUN_AT(3*HZ); - add_timer(&tp->timer); + add_timer(&tp->timer); } return; @@ -375,7 +375,7 @@ void pnic2_lnk_change(struct net_device *dev, int csr5) del_timer_sync(&tp->timer); pnic2_start_nway(dev); tp->timer.expires = RUN_AT(3*HZ); - add_timer(&tp->timer); + add_timer(&tp->timer); } return; diff --git a/drivers/net/ethernet/dec/tulip/tulip.h b/drivers/net/ethernet/dec/tulip/tulip.h index 815907259048..0ed598dc7569 100644 --- a/drivers/net/ethernet/dec/tulip/tulip.h +++ b/drivers/net/ethernet/dec/tulip/tulip.h @@ -478,7 +478,6 @@ void t21142_lnk_change(struct net_device *dev, int csr5); void pnic2_lnk_change(struct net_device *dev, int csr5); void pnic2_timer(struct timer_list *t); void pnic2_start_nway(struct net_device *dev); -void pnic2_lnk_change(struct net_device *dev, int csr5); /* eeprom.c */ void tulip_parse_eeprom(struct net_device *dev); diff --git a/drivers/net/ethernet/dec/tulip/uli526x.c b/drivers/net/ethernet/dec/tulip/uli526x.c index 13e73ed15ef0..d67ef7d02d6b 100644 --- a/drivers/net/ethernet/dec/tulip/uli526x.c +++ b/drivers/net/ethernet/dec/tulip/uli526x.c @@ -780,7 +780,7 @@ static void uli526x_free_tx_pkt(struct net_device *dev, } } - txptr = txptr->next_tx_desc; + txptr = txptr->next_tx_desc; }/* End of while */ /* Update TX remove pointer to next */ @@ -1015,7 +1015,7 @@ static void uli526x_timer(struct timer_list *t) struct net_device *dev = pci_get_drvdata(db->pdev); struct uli_phy_ops *phy = &db->phy; void __iomem *ioaddr = db->ioaddr; - unsigned long flags; + unsigned long flags; u8 tmp_cr12 = 0; u32 tmp_cr8; @@ -1535,14 +1535,14 @@ static void uli526x_set_phyxcer(struct uli526x_board_info *db) } - /* Write new capability to Phyxcer Reg4 */ + /* Write new capability to Phyxcer Reg4 */ if ( !(phy_reg & 0x01e0)) { phy_reg|=db->PHY_reg4; db->media_mode|=ULI526X_AUTO; } phy->write(db, db->phy_addr, 4, phy_reg); - /* Restart Auto-Negotiation */ + /* Restart Auto-Negotiation */ phy->write(db, db->phy_addr, 0, 0x1200); udelay(50); } @@ -1550,7 +1550,7 @@ static void uli526x_set_phyxcer(struct uli526x_board_info *db) /* * Process op-mode - AUTO mode : PHY controller in Auto-negotiation Mode + AUTO mode : PHY controller in Auto-negotiation Mode * Force mode: PHY controller in force mode with HUB * N-way force capability with SWITCH */ diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c index 514df170ec5d..f6ff1f76eacb 100644 --- a/drivers/net/ethernet/dec/tulip/winbond-840.c +++ b/drivers/net/ethernet/dec/tulip/winbond-840.c @@ -36,7 +36,7 @@ power management. support for big endian descriptors Copyright (C) 2001 Manfred Spraul - * ethtool support (jgarzik) + * ethtool support (jgarzik) * Replace some MII-related magic numbers with constants (jgarzik) TODO: @@ -1479,7 +1479,7 @@ static int netdev_close(struct net_device *dev) np->cur_rx, np->dirty_rx); } - /* Stop the chip's Tx and Rx processes. */ + /* Stop the chip's Tx and Rx processes. */ spin_lock_irq(&np->lock); netif_device_detach(dev); update_csr6(dev, 0); diff --git a/drivers/net/ethernet/dlink/sundance.c b/drivers/net/ethernet/dlink/sundance.c index ce61f79f3b7c..ee0ca712dd1c 100644 --- a/drivers/net/ethernet/dlink/sundance.c +++ b/drivers/net/ethernet/dlink/sundance.c @@ -1847,20 +1847,20 @@ static int netdev_close(struct net_device *dev) /* Stop the chip's Tx and Rx processes. */ iowrite16(TxDisable | RxDisable | StatsDisable, ioaddr + MACCtrl1); - for (i = 2000; i > 0; i--) { - if ((ioread32(ioaddr + DMACtrl) & 0xc000) == 0) + for (i = 2000; i > 0; i--) { + if ((ioread32(ioaddr + DMACtrl) & 0xc000) == 0) break; mdelay(1); - } + } - iowrite16(GlobalReset | DMAReset | FIFOReset | NetworkReset, + iowrite16(GlobalReset | DMAReset | FIFOReset | NetworkReset, ioaddr + ASIC_HI_WORD(ASICCtrl)); - for (i = 2000; i > 0; i--) { + for (i = 2000; i > 0; i--) { if ((ioread16(ioaddr + ASIC_HI_WORD(ASICCtrl)) & ResetBusy) == 0) break; mdelay(1); - } + } #ifdef __i386__ if (netif_msg_hw(np)) { diff --git a/drivers/net/ethernet/ec_bhf.c b/drivers/net/ethernet/ec_bhf.c index 46b0dbab8aad..7c992172933b 100644 --- a/drivers/net/ethernet/ec_bhf.c +++ b/drivers/net/ethernet/ec_bhf.c @@ -576,10 +576,12 @@ static void ec_bhf_remove(struct pci_dev *dev) struct ec_bhf_priv *priv = netdev_priv(net_dev); unregister_netdev(net_dev); - free_netdev(net_dev); pci_iounmap(dev, priv->dma_io); pci_iounmap(dev, priv->io); + + free_netdev(net_dev); + pci_release_regions(dev); pci_clear_master(dev); pci_disable_device(dev); diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c index 701c12c9e033..649c5c429bd7 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.c +++ b/drivers/net/ethernet/emulex/benet/be_cmds.c @@ -550,7 +550,7 @@ int be_process_mcc(struct be_adapter *adapter) int num = 0, status = 0; struct be_mcc_obj *mcc_obj = &adapter->mcc_obj; - spin_lock_bh(&adapter->mcc_cq_lock); + spin_lock(&adapter->mcc_cq_lock); while ((compl = be_mcc_compl_get(adapter))) { if (compl->flags & CQE_FLAGS_ASYNC_MASK) { @@ -566,7 +566,7 @@ int be_process_mcc(struct be_adapter *adapter) if (num) be_cq_notify(adapter, mcc_obj->cq.id, mcc_obj->rearm_cq, num); - spin_unlock_bh(&adapter->mcc_cq_lock); + spin_unlock(&adapter->mcc_cq_lock); return status; } @@ -581,7 +581,9 @@ static int be_mcc_wait_compl(struct be_adapter *adapter) if (be_check_error(adapter, BE_ERROR_ANY)) return -EIO; + local_bh_disable(); status = be_process_mcc(adapter); + local_bh_enable(); if (atomic_read(&mcc_obj->q.used) == 0) break; diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index b6eba29d8e99..361c1c87c183 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -5501,7 +5501,9 @@ static void be_worker(struct work_struct *work) * mcc completions */ if (!netif_running(adapter->netdev)) { + local_bh_disable(); be_process_mcc(adapter); + local_bh_enable(); goto reschedule; } @@ -5897,6 +5899,7 @@ drv_cleanup: unmap_bars: be_unmap_pci_bars(adapter); free_netdev: + pci_disable_pcie_error_reporting(pdev); free_netdev(netdev); rel_reg: pci_release_regions(pdev); diff --git a/drivers/net/ethernet/ezchip/nps_enet.c b/drivers/net/ethernet/ezchip/nps_enet.c index e3954d8835e7..f9a288a6ec8c 100644 --- a/drivers/net/ethernet/ezchip/nps_enet.c +++ b/drivers/net/ethernet/ezchip/nps_enet.c @@ -607,7 +607,7 @@ static s32 nps_enet_probe(struct platform_device *pdev) /* Get IRQ number */ priv->irq = platform_get_irq(pdev, 0); - if (!priv->irq) { + if (priv->irq < 0) { dev_err(dev, "failed to retrieve <irq Rx-Tx> value from device tree\n"); err = -ENODEV; goto out_netdev; @@ -630,8 +630,7 @@ static s32 nps_enet_probe(struct platform_device *pdev) out_netif_api: netif_napi_del(&priv->napi); out_netdev: - if (err) - free_netdev(ndev); + free_netdev(ndev); return err; } @@ -642,8 +641,8 @@ static s32 nps_enet_remove(struct platform_device *pdev) struct nps_enet_priv *priv = netdev_priv(ndev); unregister_netdev(ndev); - free_netdev(ndev); netif_napi_del(&priv->napi); + free_netdev(ndev); return 0; } diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index 04421aec2dfd..11dbbfd38770 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1830,14 +1830,17 @@ static int ftgmac100_probe(struct platform_device *pdev) if (np && of_get_property(np, "use-ncsi", NULL)) { if (!IS_ENABLED(CONFIG_NET_NCSI)) { dev_err(&pdev->dev, "NCSI stack not enabled\n"); + err = -EINVAL; goto err_phy_connect; } dev_info(&pdev->dev, "Using NCSI interface\n"); priv->use_ncsi = true; priv->ndev = ncsi_register_dev(netdev, ftgmac100_ncsi_handler); - if (!priv->ndev) + if (!priv->ndev) { + err = -EINVAL; goto err_phy_connect; + } } else if (np && of_get_property(np, "phy-handle", NULL)) { struct phy_device *phy; @@ -1856,6 +1859,7 @@ static int ftgmac100_probe(struct platform_device *pdev) &ftgmac100_adjust_link); if (!phy) { dev_err(&pdev->dev, "Failed to connect to phy\n"); + err = -EINVAL; goto err_phy_connect; } diff --git a/drivers/net/ethernet/fealnx.c b/drivers/net/ethernet/fealnx.c index 0908771aa9ac..0f141c14d72d 100644 --- a/drivers/net/ethernet/fealnx.c +++ b/drivers/net/ethernet/fealnx.c @@ -144,7 +144,7 @@ struct chip_info { }; static const struct chip_info skel_netdrv_tbl[] = { - { "100/10M Ethernet PCI Adapter", HAS_MII_XCVR }, + { "100/10M Ethernet PCI Adapter", HAS_MII_XCVR }, { "100/10M Ethernet PCI Adapter", HAS_CHIP_XCVR }, { "1000/100/10M Ethernet PCI Adapter", HAS_MII_XCVR }, }; diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 177c020bf34a..e6826561cf11 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2558,13 +2558,9 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, u32 xdp_act; int err; - rcu_read_lock(); - xdp_prog = READ_ONCE(priv->xdp_prog); - if (!xdp_prog) { - rcu_read_unlock(); + if (!xdp_prog) return XDP_PASS; - } xdp_init_buff(&xdp, DPAA_BP_RAW_SIZE - DPAA_TX_PRIV_DATA_SIZE, &dpaa_fq->xdp_rxq); @@ -2638,8 +2634,6 @@ static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr, break; } - rcu_read_unlock(); - return xdp_act; } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.c index b87db0846e10..8356af4631fd 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth-debugfs.c @@ -121,10 +121,14 @@ DEFINE_SHOW_ATTRIBUTE(dpaa2_dbg_ch); void dpaa2_dbg_add(struct dpaa2_eth_priv *priv) { + struct fsl_mc_device *dpni_dev; struct dentry *dir; + char name[10]; /* Create a directory for the interface */ - dir = debugfs_create_dir(priv->net_dev->name, dpaa2_dbg_root); + dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); + snprintf(name, 10, "dpni.%d", dpni_dev->obj_desc.id); + dir = debugfs_create_dir(name, dpaa2_dbg_root); priv->dbg.dir = dir; /* per-cpu stats file */ diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index e0c3c58e2ac7..973352393bd4 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -352,8 +352,6 @@ static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, u32 xdp_act = XDP_PASS; int err, offset; - rcu_read_lock(); - xdp_prog = READ_ONCE(ch->xdp.prog); if (!xdp_prog) goto out; @@ -414,7 +412,6 @@ static u32 dpaa2_eth_run_xdp(struct dpaa2_eth_priv *priv, ch->xdp.res |= xdp_act; out: - rcu_read_unlock(); return xdp_act; } @@ -4164,10 +4161,11 @@ static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) if (dpaa2_eth_is_type_phy(priv)) { err = dpaa2_mac_connect(mac); - if (err) { - netdev_err(priv->net_dev, "Error connecting to the MAC endpoint\n"); + if (err && err != -EPROBE_DEFER) + netdev_err(priv->net_dev, "Error connecting to the MAC endpoint: %pe", + ERR_PTR(err)); + if (err) goto err_close_mac; - } } return 0; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c index ccaf7e35abeb..ae6d382d8735 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* Copyright 2019 NXP */ +#include <linux/acpi.h> +#include <linux/property.h> + #include "dpaa2-eth.h" #include "dpaa2-mac.h" @@ -34,39 +37,51 @@ static int phy_mode(enum dpmac_eth_if eth_if, phy_interface_t *if_mode) return 0; } -/* Caller must call of_node_put on the returned value */ -static struct device_node *dpaa2_mac_get_node(u16 dpmac_id) +static struct fwnode_handle *dpaa2_mac_get_node(struct device *dev, + u16 dpmac_id) { - struct device_node *dpmacs, *dpmac = NULL; - u32 id; + struct fwnode_handle *fwnode, *parent, *child = NULL; + struct device_node *dpmacs = NULL; int err; + u32 id; - dpmacs = of_find_node_by_name(NULL, "dpmacs"); - if (!dpmacs) - return NULL; + fwnode = dev_fwnode(dev->parent); + if (is_of_node(fwnode)) { + dpmacs = of_find_node_by_name(NULL, "dpmacs"); + if (!dpmacs) + return NULL; + parent = of_fwnode_handle(dpmacs); + } else if (is_acpi_node(fwnode)) { + parent = fwnode; + } - while ((dpmac = of_get_next_child(dpmacs, dpmac)) != NULL) { - err = of_property_read_u32(dpmac, "reg", &id); + fwnode_for_each_child_node(parent, child) { + err = -EINVAL; + if (is_acpi_device_node(child)) + err = acpi_get_local_address(ACPI_HANDLE_FWNODE(child), &id); + else if (is_of_node(child)) + err = of_property_read_u32(to_of_node(child), "reg", &id); if (err) continue; - if (id == dpmac_id) - break; - } + if (id == dpmac_id) { + of_node_put(dpmacs); + return child; + } + } of_node_put(dpmacs); - - return dpmac; + return NULL; } -static int dpaa2_mac_get_if_mode(struct device_node *node, +static int dpaa2_mac_get_if_mode(struct fwnode_handle *dpmac_node, struct dpmac_attr attr) { phy_interface_t if_mode; int err; - err = of_get_phy_mode(node, &if_mode); - if (!err) - return if_mode; + err = fwnode_get_phy_mode(dpmac_node); + if (err > 0) + return err; err = phy_mode(attr.eth_if, &if_mode); if (!err) @@ -235,26 +250,27 @@ static const struct phylink_mac_ops dpaa2_mac_phylink_ops = { }; static int dpaa2_pcs_create(struct dpaa2_mac *mac, - struct device_node *dpmac_node, int id) + struct fwnode_handle *dpmac_node, + int id) { struct mdio_device *mdiodev; - struct device_node *node; + struct fwnode_handle *node; - node = of_parse_phandle(dpmac_node, "pcs-handle", 0); - if (!node) { + node = fwnode_find_reference(dpmac_node, "pcs-handle", 0); + if (IS_ERR(node)) { /* do not error out on old DTS files */ netdev_warn(mac->net_dev, "pcs-handle node not found\n"); return 0; } - if (!of_device_is_available(node)) { + if (!fwnode_device_is_available(node)) { netdev_err(mac->net_dev, "pcs-handle node not available\n"); - of_node_put(node); + fwnode_handle_put(node); return -ENODEV; } - mdiodev = of_mdio_find_device(node); - of_node_put(node); + mdiodev = fwnode_mdio_find_device(node); + fwnode_handle_put(node); if (!mdiodev) return -EPROBE_DEFER; @@ -283,36 +299,33 @@ static void dpaa2_pcs_destroy(struct dpaa2_mac *mac) int dpaa2_mac_connect(struct dpaa2_mac *mac) { struct net_device *net_dev = mac->net_dev; - struct device_node *dpmac_node; + struct fwnode_handle *dpmac_node; struct phylink *phylink; int err; mac->if_link_type = mac->attr.link_type; - dpmac_node = dpaa2_mac_get_node(mac->attr.id); + dpmac_node = mac->fw_node; if (!dpmac_node) { netdev_err(net_dev, "No dpmac@%d node found.\n", mac->attr.id); return -ENODEV; } err = dpaa2_mac_get_if_mode(dpmac_node, mac->attr); - if (err < 0) { - err = -EINVAL; - goto err_put_node; - } + if (err < 0) + return -EINVAL; mac->if_mode = err; /* The MAC does not have the capability to add RGMII delays so * error out if the interface mode requests them and there is no PHY * to act upon them */ - if (of_phy_is_fixed_link(dpmac_node) && + if (of_phy_is_fixed_link(to_of_node(dpmac_node)) && (mac->if_mode == PHY_INTERFACE_MODE_RGMII_ID || mac->if_mode == PHY_INTERFACE_MODE_RGMII_RXID || mac->if_mode == PHY_INTERFACE_MODE_RGMII_TXID)) { netdev_err(net_dev, "RGMII delay not supported\n"); - err = -EINVAL; - goto err_put_node; + return -EINVAL; } if ((mac->attr.link_type == DPMAC_LINK_TYPE_PHY && @@ -320,14 +333,14 @@ int dpaa2_mac_connect(struct dpaa2_mac *mac) mac->attr.link_type == DPMAC_LINK_TYPE_BACKPLANE) { err = dpaa2_pcs_create(mac, dpmac_node, mac->attr.id); if (err) - goto err_put_node; + return err; } mac->phylink_config.dev = &net_dev->dev; mac->phylink_config.type = PHYLINK_NETDEV; phylink = phylink_create(&mac->phylink_config, - of_fwnode_handle(dpmac_node), mac->if_mode, + dpmac_node, mac->if_mode, &dpaa2_mac_phylink_ops); if (IS_ERR(phylink)) { err = PTR_ERR(phylink); @@ -338,22 +351,18 @@ int dpaa2_mac_connect(struct dpaa2_mac *mac) if (mac->pcs) phylink_set_pcs(mac->phylink, &mac->pcs->pcs); - err = phylink_of_phy_connect(mac->phylink, dpmac_node, 0); + err = phylink_fwnode_phy_connect(mac->phylink, dpmac_node, 0); if (err) { - netdev_err(net_dev, "phylink_of_phy_connect() = %d\n", err); + netdev_err(net_dev, "phylink_fwnode_phy_connect() = %d\n", err); goto err_phylink_destroy; } - of_node_put(dpmac_node); - return 0; err_phylink_destroy: phylink_destroy(mac->phylink); err_pcs_destroy: dpaa2_pcs_destroy(mac); -err_put_node: - of_node_put(dpmac_node); return err; } @@ -388,6 +397,12 @@ int dpaa2_mac_open(struct dpaa2_mac *mac) goto err_close_dpmac; } + /* Find the device node representing the MAC device and link the device + * behind the associated netdev to it. + */ + mac->fw_node = dpaa2_mac_get_node(&mac->mc_dev->dev, mac->attr.id); + net_dev->dev.of_node = to_of_node(mac->fw_node); + return 0; err_close_dpmac: @@ -400,6 +415,8 @@ void dpaa2_mac_close(struct dpaa2_mac *mac) struct fsl_mc_device *dpmac_dev = mac->mc_dev; dpmac_close(mac->mc_io, 0, dpmac_dev->mc_handle); + if (mac->fw_node) + fwnode_handle_put(mac->fw_node); } static char dpaa2_mac_ethtool_stats[][ETH_GSTRING_LEN] = { diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h index 13d42dd58ec9..7842cbb2207a 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h @@ -24,6 +24,7 @@ struct dpaa2_mac { phy_interface_t if_mode; enum dpmac_link_type if_link_type; struct lynx_pcs *pcs; + struct fwnode_handle *fw_node; }; bool dpaa2_mac_is_type_fixed(struct fsl_mc_device *dpmac_dev, diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 05de37c3b64c..f3d12d0714fb 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -1625,7 +1625,7 @@ static int dpaa2_switch_port_bridge_flags(struct net_device *netdev, return 0; } -static int dpaa2_switch_port_attr_set(struct net_device *netdev, +static int dpaa2_switch_port_attr_set(struct net_device *netdev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) { diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ierb.c b/drivers/net/ethernet/freescale/enetc/enetc_ierb.c index 8b356c485507..ee1468e3eaa3 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ierb.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ierb.c @@ -99,15 +99,13 @@ EXPORT_SYMBOL(enetc_ierb_register_pf); static int enetc_ierb_probe(struct platform_device *pdev) { struct enetc_ierb *ierb; - struct resource *res; void __iomem *regs; ierb = devm_kzalloc(&pdev->dev, sizeof(*ierb), GFP_KERNEL); if (!ierb) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - regs = devm_ioremap_resource(&pdev->dev, res); + regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); if (IS_ERR(regs)) return PTR_ERR(regs); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index 31274325159a..c84f6c226743 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* Copyright 2017-2019 NXP */ +#include <asm/unaligned.h> #include <linux/mdio.h> #include <linux/module.h> #include <linux/fsl/enetc_mdio.h> @@ -17,15 +18,15 @@ static void enetc_pf_get_primary_mac_addr(struct enetc_hw *hw, int si, u8 *addr) u32 upper = __raw_readl(hw->port + ENETC_PSIPMAR0(si)); u16 lower = __raw_readw(hw->port + ENETC_PSIPMAR1(si)); - *(u32 *)addr = upper; - *(u16 *)(addr + 4) = lower; + put_unaligned_le32(upper, addr); + put_unaligned_le16(lower, addr + 4); } static void enetc_pf_set_primary_mac_addr(struct enetc_hw *hw, int si, const u8 *addr) { - u32 upper = *(const u32 *)addr; - u16 lower = *(const u16 *)(addr + 4); + u32 upper = get_unaligned_le32(addr); + u16 lower = get_unaligned_le16(addr + 4); __raw_writel(upper, hw->port + ENETC_PSIPMAR0(si)); __raw_writew(lower, hw->port + ENETC_PSIPMAR1(si)); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_qos.c b/drivers/net/ethernet/freescale/enetc/enetc_qos.c index af699f2ad095..4577226d3c6a 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_qos.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_qos.c @@ -465,8 +465,13 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv, struct streamid_conf *si_conf; u16 data_size; dma_addr_t dma; + int port; int err; + port = enetc_pf_to_port(priv->si->pdev); + if (port < 0) + return -EINVAL; + if (sid->index >= priv->psfp_cap.max_streamid) return -EINVAL; @@ -499,7 +504,7 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv, si_conf = &cbd.sid_set; /* Only one port supported for one entry, set itself */ - si_conf->iports = cpu_to_le32(1 << enetc_pf_to_port(priv->si->pdev)); + si_conf->iports = cpu_to_le32(1 << port); si_conf->id_type = 1; si_conf->oui[2] = 0x0; si_conf->oui[1] = 0x80; @@ -524,7 +529,7 @@ static int enetc_streamid_hw_set(struct enetc_ndev_priv *priv, si_conf->en = 0x80; si_conf->stream_handle = cpu_to_le32(sid->handle); - si_conf->iports = cpu_to_le32(1 << enetc_pf_to_port(priv->si->pdev)); + si_conf->iports = cpu_to_le32(1 << port); si_conf->id_type = sid->filtertype; si_conf->oui[2] = 0x0; si_conf->oui[1] = 0x80; @@ -567,6 +572,11 @@ static int enetc_streamfilter_hw_set(struct enetc_ndev_priv *priv, { struct enetc_cbd cbd = {.cmd = 0}; struct sfi_conf *sfi_config; + int port; + + port = enetc_pf_to_port(priv->si->pdev); + if (port < 0) + return -EINVAL; cbd.index = cpu_to_le16(sfi->index); cbd.cls = BDCR_CMD_STREAM_FILTER; @@ -586,8 +596,7 @@ static int enetc_streamfilter_hw_set(struct enetc_ndev_priv *priv, } sfi_config->sg_inst_table_index = cpu_to_le16(sfi->gate_id); - sfi_config->input_ports = - cpu_to_le32(1 << enetc_pf_to_port(priv->si->pdev)); + sfi_config->input_ports = cpu_to_le32(1 << port); /* The priority value which may be matched against the * frame’s priority value to determine a match for this entry. @@ -1548,7 +1557,7 @@ int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data) { struct enetc_ndev_priv *priv = netdev_priv(ndev); struct flow_block_offload *f = type_data; - int err; + int port, err; err = flow_block_cb_setup_simple(f, &enetc_block_cb_list, enetc_setup_tc_block_cb, @@ -1558,10 +1567,18 @@ int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data) switch (f->command) { case FLOW_BLOCK_BIND: - set_bit(enetc_pf_to_port(priv->si->pdev), &epsfp.dev_bitmap); + port = enetc_pf_to_port(priv->si->pdev); + if (port < 0) + return -EINVAL; + + set_bit(port, &epsfp.dev_bitmap); break; case FLOW_BLOCK_UNBIND: - clear_bit(enetc_pf_to_port(priv->si->pdev), &epsfp.dev_bitmap); + port = enetc_pf_to_port(priv->si->pdev); + if (port < 0) + return -EINVAL; + + clear_bit(port, &epsfp.dev_bitmap); if (!epsfp.dev_bitmap) clean_psfp_all(); break; diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h index 0602d5d5d2ee..2e002e4b4b4a 100644 --- a/drivers/net/ethernet/freescale/fec.h +++ b/drivers/net/ethernet/freescale/fec.h @@ -467,6 +467,11 @@ struct bufdesc_ex { */ #define FEC_QUIRK_NO_HARD_RESET (1 << 18) +/* i.MX6SX ENET IP supports multiple queues (3 queues), use this quirk to + * represents this ENET IP. + */ +#define FEC_QUIRK_HAS_MULTI_QUEUES (1 << 19) + struct bufdesc_prop { int qid; /* Address of Rx and Tx buffers */ diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index f2065f9d02e6..8aea707a65a7 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -76,6 +76,8 @@ static void fec_enet_itr_coal_init(struct net_device *ndev); #define DRIVER_NAME "fec" +static const u16 fec_enet_vlan_pri_to_queue[8] = {0, 0, 1, 1, 1, 2, 2, 2}; + /* Pause frame feild and FIFO threshold */ #define FEC_ENET_FCE (1 << 5) #define FEC_ENET_RSEM_V 0x84 @@ -122,7 +124,7 @@ static const struct fec_devinfo fec_imx6x_info = { FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE | FEC_QUIRK_HAS_RACC | FEC_QUIRK_HAS_COALESCE | - FEC_QUIRK_CLEAR_SETUP_MII, + FEC_QUIRK_CLEAR_SETUP_MII | FEC_QUIRK_HAS_MULTI_QUEUES, }; static const struct fec_devinfo fec_imx6ul_info = { @@ -421,6 +423,7 @@ fec_enet_txq_submit_frag_skb(struct fec_enet_priv_tx_q *txq, estatus |= FEC_TX_BD_FTYPE(txq->bd.qid); if (skb->ip_summed == CHECKSUM_PARTIAL) estatus |= BD_ENET_TX_PINS | BD_ENET_TX_IINS; + ebdp->cbd_bdu = 0; ebdp->cbd_esc = cpu_to_fec32(estatus); } @@ -954,7 +957,7 @@ fec_restart(struct net_device *ndev) * For i.MX6SX SOC, enet use AXI bus, we use disable MAC * instead of reset MAC itself. */ - if (fep->quirks & FEC_QUIRK_HAS_AVB || + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { writel(0, fep->hwp + FEC_ECNTRL); } else { @@ -1165,7 +1168,7 @@ fec_stop(struct net_device *ndev) * instead of reset MAC itself. */ if (!(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { - if (fep->quirks & FEC_QUIRK_HAS_AVB) { + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { writel(0, fep->hwp + FEC_ECNTRL); } else { writel(1, fep->hwp + FEC_ECNTRL); @@ -1662,7 +1665,7 @@ static int fec_enet_rx_napi(struct napi_struct *napi, int budget) } /* ------------------------------------------------------------------------- */ -static void fec_get_mac(struct net_device *ndev) +static int fec_get_mac(struct net_device *ndev) { struct fec_enet_private *fep = netdev_priv(ndev); unsigned char *iap, tmpaddr[ETH_ALEN]; @@ -1685,6 +1688,8 @@ static void fec_get_mac(struct net_device *ndev) ret = of_get_mac_address(np, tmpaddr); if (!ret) iap = tmpaddr; + else if (ret == -EPROBE_DEFER) + return ret; } } @@ -1723,7 +1728,7 @@ static void fec_get_mac(struct net_device *ndev) eth_hw_addr_random(ndev); dev_info(&fep->pdev->dev, "Using random MAC address: %pM\n", ndev->dev_addr); - return; + return 0; } memcpy(ndev->dev_addr, iap, ETH_ALEN); @@ -1731,6 +1736,8 @@ static void fec_get_mac(struct net_device *ndev) /* Adjust MAC if using macaddr */ if (iap == macaddr) ndev->dev_addr[ETH_ALEN-1] = macaddr[ETH_ALEN-1] + fep->dev_id; + + return 0; } /* ------------------------------------------------------------------------- */ @@ -2566,7 +2573,7 @@ static void fec_enet_itr_coal_set(struct net_device *ndev) writel(tx_itr, fep->hwp + FEC_TXIC0); writel(rx_itr, fep->hwp + FEC_RXIC0); - if (fep->quirks & FEC_QUIRK_HAS_AVB) { + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { writel(tx_itr, fep->hwp + FEC_TXIC1); writel(rx_itr, fep->hwp + FEC_RXIC1); writel(tx_itr, fep->hwp + FEC_TXIC2); @@ -3235,10 +3242,40 @@ static int fec_set_features(struct net_device *netdev, return 0; } +static u16 fec_enet_get_raw_vlan_tci(struct sk_buff *skb) +{ + struct vlan_ethhdr *vhdr; + unsigned short vlan_TCI = 0; + + if (skb->protocol == htons(ETH_P_ALL)) { + vhdr = (struct vlan_ethhdr *)(skb->data); + vlan_TCI = ntohs(vhdr->h_vlan_TCI); + } + + return vlan_TCI; +} + +static u16 fec_enet_select_queue(struct net_device *ndev, struct sk_buff *skb, + struct net_device *sb_dev) +{ + struct fec_enet_private *fep = netdev_priv(ndev); + u16 vlan_tag; + + if (!(fep->quirks & FEC_QUIRK_HAS_AVB)) + return netdev_pick_tx(ndev, skb, NULL); + + vlan_tag = fec_enet_get_raw_vlan_tci(skb); + if (!vlan_tag) + return vlan_tag; + + return fec_enet_vlan_pri_to_queue[vlan_tag >> 13]; +} + static const struct net_device_ops fec_netdev_ops = { .ndo_open = fec_enet_open, .ndo_stop = fec_enet_close, .ndo_start_xmit = fec_enet_start_xmit, + .ndo_select_queue = fec_enet_select_queue, .ndo_set_rx_mode = set_multicast_list, .ndo_validate_addr = eth_validate_addr, .ndo_tx_timeout = fec_timeout, @@ -3290,7 +3327,9 @@ static int fec_enet_init(struct net_device *ndev) return ret; } - fec_enet_alloc_queue(ndev); + ret = fec_enet_alloc_queue(ndev); + if (ret) + return ret; bd_size = (fep->total_tx_ring_size + fep->total_rx_ring_size) * dsize; @@ -3298,11 +3337,15 @@ static int fec_enet_init(struct net_device *ndev) cbd_base = dmam_alloc_coherent(&fep->pdev->dev, bd_size, &bd_dma, GFP_KERNEL); if (!cbd_base) { - return -ENOMEM; + ret = -ENOMEM; + goto free_queue_mem; } /* Get the Ethernet address */ - fec_get_mac(ndev); + ret = fec_get_mac(ndev); + if (ret) + goto free_queue_mem; + /* make sure MAC we just acquired is programmed into the hw */ fec_set_mac_address(ndev, NULL); @@ -3361,7 +3404,7 @@ static int fec_enet_init(struct net_device *ndev) fep->csum_flags |= FLAG_RX_CSUM_ENABLED; } - if (fep->quirks & FEC_QUIRK_HAS_AVB) { + if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { fep->tx_align = 0; fep->rx_align = 0x3f; } @@ -3376,6 +3419,10 @@ static int fec_enet_init(struct net_device *ndev) fec_enet_update_ethtool_stats(ndev); return 0; + +free_queue_mem: + fec_enet_free_queue(ndev); + return ret; } #ifdef CONFIG_OF diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index 1753807cbf97..d71eac7e1924 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -215,15 +215,13 @@ static u64 fec_ptp_read(const struct cyclecounter *cc) { struct fec_enet_private *fep = container_of(cc, struct fec_enet_private, cc); - const struct platform_device_id *id_entry = - platform_get_device_id(fep->pdev); u32 tempval; tempval = readl(fep->hwp + FEC_ATIME_CTRL); tempval |= FEC_T_CTRL_CAPTURE; writel(tempval, fep->hwp + FEC_ATIME_CTRL); - if (id_entry->driver_data & FEC_QUIRK_BUG_CAPTURE) + if (fep->quirks & FEC_QUIRK_BUG_CAPTURE) udelay(1); return readl(fep->hwp + FEC_ATIME); @@ -604,6 +602,10 @@ void fec_ptp_init(struct platform_device *pdev, int irq_idx) fep->ptp_caps.enable = fec_ptp_enable; fep->cycle_speed = clk_get_rate(fep->clk_ptp); + if (!fep->cycle_speed) { + fep->cycle_speed = NSEC_PER_SEC; + dev_err(&fep->pdev->dev, "clk_ptp clock rate is zero\n"); + } fep->ptp_inc = NSEC_PER_SEC / fep->cycle_speed; spin_lock_init(&fep->tmreg_lock); diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c index f2945abdb041..9646483137c4 100644 --- a/drivers/net/ethernet/freescale/gianfar.c +++ b/drivers/net/ethernet/freescale/gianfar.c @@ -274,32 +274,44 @@ static void gfar_configure_coalescing_all(struct gfar_private *priv) gfar_configure_coalescing(priv, 0xFF, 0xFF); } -static struct net_device_stats *gfar_get_stats(struct net_device *dev) +static void gfar_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) { struct gfar_private *priv = netdev_priv(dev); - unsigned long rx_packets = 0, rx_bytes = 0, rx_dropped = 0; - unsigned long tx_packets = 0, tx_bytes = 0; int i; for (i = 0; i < priv->num_rx_queues; i++) { - rx_packets += priv->rx_queue[i]->stats.rx_packets; - rx_bytes += priv->rx_queue[i]->stats.rx_bytes; - rx_dropped += priv->rx_queue[i]->stats.rx_dropped; + stats->rx_packets += priv->rx_queue[i]->stats.rx_packets; + stats->rx_bytes += priv->rx_queue[i]->stats.rx_bytes; + stats->rx_dropped += priv->rx_queue[i]->stats.rx_dropped; } - dev->stats.rx_packets = rx_packets; - dev->stats.rx_bytes = rx_bytes; - dev->stats.rx_dropped = rx_dropped; - for (i = 0; i < priv->num_tx_queues; i++) { - tx_bytes += priv->tx_queue[i]->stats.tx_bytes; - tx_packets += priv->tx_queue[i]->stats.tx_packets; + stats->tx_bytes += priv->tx_queue[i]->stats.tx_bytes; + stats->tx_packets += priv->tx_queue[i]->stats.tx_packets; } - dev->stats.tx_bytes = tx_bytes; - dev->stats.tx_packets = tx_packets; + if (priv->device_flags & FSL_GIANFAR_DEV_HAS_RMON) { + struct rmon_mib __iomem *rmon = &priv->gfargrp[0].regs->rmon; + unsigned long flags; + u32 rdrp, car, car_before; + u64 rdrp_offset; + + spin_lock_irqsave(&priv->rmon_overflow.lock, flags); + car = gfar_read(&rmon->car1) & CAR1_C1RDR; + do { + car_before = car; + rdrp = gfar_read(&rmon->rdrp); + car = gfar_read(&rmon->car1) & CAR1_C1RDR; + } while (car != car_before); + if (car) { + priv->rmon_overflow.rdrp++; + gfar_write(&rmon->car1, car); + } + rdrp_offset = priv->rmon_overflow.rdrp; + spin_unlock_irqrestore(&priv->rmon_overflow.lock, flags); - return &dev->stats; + stats->rx_missed_errors = rdrp + (rdrp_offset << 16); + } } /* Set the appropriate hash bit for the given addr */ @@ -390,7 +402,8 @@ static void gfar_ints_enable(struct gfar_private *priv) for (i = 0; i < priv->num_grps; i++) { struct gfar __iomem *regs = priv->gfargrp[i].regs; /* Unmask the interrupts we look for */ - gfar_write(®s->imask, IMASK_DEFAULT); + gfar_write(®s->imask, + IMASK_DEFAULT | priv->rmon_overflow.imask); } } @@ -2298,7 +2311,7 @@ static irqreturn_t gfar_receive(int irq, void *grp_id) if (likely(napi_schedule_prep(&grp->napi_rx))) { spin_lock_irqsave(&grp->grplock, flags); imask = gfar_read(&grp->regs->imask); - imask &= IMASK_RX_DISABLED; + imask &= IMASK_RX_DISABLED | grp->priv->rmon_overflow.imask; gfar_write(&grp->regs->imask, imask); spin_unlock_irqrestore(&grp->grplock, flags); __napi_schedule(&grp->napi_rx); @@ -2322,7 +2335,7 @@ static irqreturn_t gfar_transmit(int irq, void *grp_id) if (likely(napi_schedule_prep(&grp->napi_tx))) { spin_lock_irqsave(&grp->grplock, flags); imask = gfar_read(&grp->regs->imask); - imask &= IMASK_TX_DISABLED; + imask &= IMASK_TX_DISABLED | grp->priv->rmon_overflow.imask; gfar_write(&grp->regs->imask, imask); spin_unlock_irqrestore(&grp->grplock, flags); __napi_schedule(&grp->napi_tx); @@ -2693,6 +2706,18 @@ static irqreturn_t gfar_error(int irq, void *grp_id) } netif_dbg(priv, tx_err, dev, "Transmit Error\n"); } + if (events & IEVENT_MSRO) { + struct rmon_mib __iomem *rmon = ®s->rmon; + u32 car; + + spin_lock(&priv->rmon_overflow.lock); + car = gfar_read(&rmon->car1) & CAR1_C1RDR; + if (car) { + priv->rmon_overflow.rdrp++; + gfar_write(&rmon->car1, car); + } + spin_unlock(&priv->rmon_overflow.lock); + } if (events & IEVENT_BSY) { dev->stats.rx_over_errors++; atomic64_inc(&priv->extra_stats.rx_bsy); @@ -3109,11 +3134,14 @@ static void gfar_hw_init(struct gfar_private *priv) /* Zero out the rmon mib registers if it has them */ if (priv->device_flags & FSL_GIANFAR_DEV_HAS_RMON) { - memset_io(&(regs->rmon), 0, sizeof(struct rmon_mib)); + memset_io(®s->rmon, 0, offsetof(struct rmon_mib, car1)); /* Mask off the CAM interrupts */ gfar_write(®s->rmon.cam1, 0xffffffff); gfar_write(®s->rmon.cam2, 0xffffffff); + /* Clear the CAR registers (w1c style) */ + gfar_write(®s->rmon.car1, 0xffffffff); + gfar_write(®s->rmon.car2, 0xffffffff); } /* Initialize ECNTRL */ @@ -3157,7 +3185,7 @@ static const struct net_device_ops gfar_netdev_ops = { .ndo_set_rx_mode = gfar_set_multi, .ndo_tx_timeout = gfar_timeout, .ndo_do_ioctl = gfar_ioctl, - .ndo_get_stats = gfar_get_stats, + .ndo_get_stats64 = gfar_get_stats64, .ndo_change_carrier = fixed_phy_change_carrier, .ndo_set_mac_address = gfar_set_mac_addr, .ndo_validate_addr = eth_validate_addr, @@ -3267,6 +3295,14 @@ static int gfar_probe(struct platform_device *ofdev) gfar_hw_init(priv); + if (priv->device_flags & FSL_GIANFAR_DEV_HAS_RMON) { + struct rmon_mib __iomem *rmon = &priv->gfargrp[0].regs->rmon; + + spin_lock_init(&priv->rmon_overflow.lock); + priv->rmon_overflow.imask = IMASK_MSRO; + gfar_write(&rmon->cam1, gfar_read(&rmon->cam1) & ~CAM1_M1RDR); + } + /* Carrier starts down, phylib will bring it up */ netif_carrier_off(dev); diff --git a/drivers/net/ethernet/freescale/gianfar.h b/drivers/net/ethernet/freescale/gianfar.h index 5ea47df93e5e..ca5e14f908fe 100644 --- a/drivers/net/ethernet/freescale/gianfar.h +++ b/drivers/net/ethernet/freescale/gianfar.h @@ -445,6 +445,60 @@ struct ethtool_rx_list { #define RQFPR_PER 0x00000002 #define RQFPR_EER 0x00000001 +/* CAR1 bits */ +#define CAR1_C164 0x80000000 +#define CAR1_C1127 0x40000000 +#define CAR1_C1255 0x20000000 +#define CAR1_C1511 0x10000000 +#define CAR1_C11K 0x08000000 +#define CAR1_C1MAX 0x04000000 +#define CAR1_C1MGV 0x02000000 +#define CAR1_C1REJ 0x00020000 +#define CAR1_C1RBY 0x00010000 +#define CAR1_C1RPK 0x00008000 +#define CAR1_C1RFC 0x00004000 +#define CAR1_C1RMC 0x00002000 +#define CAR1_C1RBC 0x00001000 +#define CAR1_C1RXC 0x00000800 +#define CAR1_C1RXP 0x00000400 +#define CAR1_C1RXU 0x00000200 +#define CAR1_C1RAL 0x00000100 +#define CAR1_C1RFL 0x00000080 +#define CAR1_C1RCD 0x00000040 +#define CAR1_C1RCS 0x00000020 +#define CAR1_C1RUN 0x00000010 +#define CAR1_C1ROV 0x00000008 +#define CAR1_C1RFR 0x00000004 +#define CAR1_C1RJB 0x00000002 +#define CAR1_C1RDR 0x00000001 + +/* CAM1 bits */ +#define CAM1_M164 0x80000000 +#define CAM1_M1127 0x40000000 +#define CAM1_M1255 0x20000000 +#define CAM1_M1511 0x10000000 +#define CAM1_M11K 0x08000000 +#define CAM1_M1MAX 0x04000000 +#define CAM1_M1MGV 0x02000000 +#define CAM1_M1REJ 0x00020000 +#define CAM1_M1RBY 0x00010000 +#define CAM1_M1RPK 0x00008000 +#define CAM1_M1RFC 0x00004000 +#define CAM1_M1RMC 0x00002000 +#define CAM1_M1RBC 0x00001000 +#define CAM1_M1RXC 0x00000800 +#define CAM1_M1RXP 0x00000400 +#define CAM1_M1RXU 0x00000200 +#define CAM1_M1RAL 0x00000100 +#define CAM1_M1RFL 0x00000080 +#define CAM1_M1RCD 0x00000040 +#define CAM1_M1RCS 0x00000020 +#define CAM1_M1RUN 0x00000010 +#define CAM1_M1ROV 0x00000008 +#define CAM1_M1RFR 0x00000004 +#define CAM1_M1RJB 0x00000002 +#define CAM1_M1RDR 0x00000001 + /* TxBD status field bits */ #define TXBD_READY 0x8000 #define TXBD_PADCRC 0x4000 @@ -609,6 +663,15 @@ struct rmon_mib u32 cam2; /* 0x.73c - Carry Mask Register Two */ }; +struct rmon_overflow { + /* lock for synchronization of the rdrp field of this struct, and + * CAR1/CAR2 registers + */ + spinlock_t lock; + u32 imask; + u64 rdrp; +}; + struct gfar_extra_stats { atomic64_t rx_alloc_err; atomic64_t rx_large; @@ -913,8 +976,8 @@ enum { * Per TX queue stats */ struct tx_q_stats { - unsigned long tx_packets; - unsigned long tx_bytes; + u64 tx_packets; + u64 tx_bytes; }; /** @@ -963,9 +1026,9 @@ struct gfar_priv_tx_q { * Per RX queue stats */ struct rx_q_stats { - unsigned long rx_packets; - unsigned long rx_bytes; - unsigned long rx_dropped; + u64 rx_packets; + u64 rx_bytes; + u64 rx_dropped; }; struct gfar_rx_buff { @@ -1096,6 +1159,7 @@ struct gfar_private { /* Network Statistics */ struct gfar_extra_stats extra_stats; + struct rmon_overflow rmon_overflow; /* PHY stuff */ phy_interface_t interface; diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c index e0936510fa34..0acfafb73db1 100644 --- a/drivers/net/ethernet/freescale/ucc_geth.c +++ b/drivers/net/ethernet/freescale/ucc_geth.c @@ -3590,10 +3590,9 @@ static int ucc_geth_probe(struct platform_device* ofdev) if ((ucc_num < 0) || (ucc_num > 7)) return -ENODEV; - ug_info = kmalloc(sizeof(*ug_info), GFP_KERNEL); + ug_info = kmemdup(&ugeth_primary_info, sizeof(*ug_info), GFP_KERNEL); if (ug_info == NULL) return -ENOMEM; - memcpy(ug_info, &ugeth_primary_info, sizeof(*ug_info)); ug_info->uf_info.ucc_num = ucc_num; diff --git a/drivers/net/ethernet/freescale/xgmac_mdio.c b/drivers/net/ethernet/freescale/xgmac_mdio.c index bfa2826c5545..0b68852379da 100644 --- a/drivers/net/ethernet/freescale/xgmac_mdio.c +++ b/drivers/net/ethernet/freescale/xgmac_mdio.c @@ -2,6 +2,7 @@ * QorIQ 10G MDIO Controller * * Copyright 2012 Freescale Semiconductor, Inc. + * Copyright 2021 NXP * * Authors: Andy Fleming <afleming@freescale.com> * Timur Tabi <timur@freescale.com> @@ -11,15 +12,17 @@ * kind, whether express or implied. */ -#include <linux/kernel.h> -#include <linux/slab.h> +#include <linux/acpi.h> +#include <linux/acpi_mdio.h> #include <linux/interrupt.h> -#include <linux/module.h> -#include <linux/phy.h> +#include <linux/kernel.h> #include <linux/mdio.h> +#include <linux/module.h> #include <linux/of_address.h> -#include <linux/of_platform.h> #include <linux/of_mdio.h> +#include <linux/of_platform.h> +#include <linux/phy.h> +#include <linux/slab.h> /* Number of microseconds to wait for a register to respond */ #define TIMEOUT 1000 @@ -243,10 +246,10 @@ static int xgmac_mdio_read(struct mii_bus *bus, int phy_id, int regnum) static int xgmac_mdio_probe(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; - struct mii_bus *bus; - struct resource *res; + struct fwnode_handle *fwnode; struct mdio_fsl_priv *priv; + struct resource *res; + struct mii_bus *bus; int ret; /* In DPAA-1, MDIO is one of the many FMan sub-devices. The FMan @@ -279,13 +282,22 @@ static int xgmac_mdio_probe(struct platform_device *pdev) goto err_ioremap; } + /* For both ACPI and DT cases, endianness of MDIO controller + * needs to be specified using "little-endian" property. + */ priv->is_little_endian = device_property_read_bool(&pdev->dev, "little-endian"); priv->has_a011043 = device_property_read_bool(&pdev->dev, "fsl,erratum-a011043"); - ret = of_mdiobus_register(bus, np); + fwnode = pdev->dev.fwnode; + if (is_of_node(fwnode)) + ret = of_mdiobus_register(bus, to_of_node(fwnode)); + else if (is_acpi_node(fwnode)) + ret = acpi_mdiobus_register(bus, fwnode); + else + ret = -EINVAL; if (ret) { dev_err(&pdev->dev, "cannot register MDIO bus\n"); goto err_registration; diff --git a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c index a7b7a4aace79..62c0bed82ced 100644 --- a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c +++ b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c @@ -548,8 +548,8 @@ static int fmvj18x_get_hwinfo(struct pcmcia_device *link, u_char *node_id) base = ioremap(link->resource[2]->start, resource_size(link->resource[2])); if (!base) { - pcmcia_release_window(link, link->resource[2]); - return -ENOMEM; + pcmcia_release_window(link, link->resource[2]); + return -1; } pcmcia_map_mem_page(link, link->resource[2], 0); @@ -812,9 +812,9 @@ static netdev_tx_t fjn_start_xmit(struct sk_buff *skb, if (length < ETH_ZLEN) { - if (skb_padto(skb, ETH_ZLEN)) - return NETDEV_TX_OK; - length = ETH_ZLEN; + if (skb_padto(skb, ETH_ZLEN)) + return NETDEV_TX_OK; + length = ETH_ZLEN; } netif_stop_queue(dev); diff --git a/drivers/net/ethernet/google/Kconfig b/drivers/net/ethernet/google/Kconfig index b8f04d052fda..8641a00f8e63 100644 --- a/drivers/net/ethernet/google/Kconfig +++ b/drivers/net/ethernet/google/Kconfig @@ -17,7 +17,7 @@ if NET_VENDOR_GOOGLE config GVE tristate "Google Virtual NIC (gVNIC) support" - depends on PCI_MSI + depends on (PCI_MSI && (X86 || CPU_LITTLE_ENDIAN)) help This driver supports Google Virtual NIC (gVNIC)" diff --git a/drivers/net/ethernet/google/gve/Makefile b/drivers/net/ethernet/google/gve/Makefile index 3354ce40eb97..b9a6be76531b 100644 --- a/drivers/net/ethernet/google/gve/Makefile +++ b/drivers/net/ethernet/google/gve/Makefile @@ -1,4 +1,4 @@ # Makefile for the Google virtual Ethernet (gve) driver obj-$(CONFIG_GVE) += gve.o -gve-objs := gve_main.o gve_tx.o gve_rx.o gve_ethtool.o gve_adminq.o +gve-objs := gve_main.o gve_tx.o gve_tx_dqo.o gve_rx.o gve_rx_dqo.o gve_ethtool.o gve_adminq.o gve_utils.o diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h index daf07c0f790b..1d3188e8e3b3 100644 --- a/drivers/net/ethernet/google/gve/gve.h +++ b/drivers/net/ethernet/google/gve/gve.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: (GPL-2.0 OR MIT) * Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #ifndef _GVE_H_ @@ -11,7 +11,9 @@ #include <linux/netdevice.h> #include <linux/pci.h> #include <linux/u64_stats_sync.h> + #include "gve_desc.h" +#include "gve_desc_dqo.h" #ifndef PCI_VENDOR_ID_GOOGLE #define PCI_VENDOR_ID_GOOGLE 0x1ae0 @@ -40,6 +42,11 @@ #define GVE_DATA_SLOT_ADDR_PAGE_MASK (~(PAGE_SIZE - 1)) +/* PTYPEs are always 10 bits. */ +#define GVE_NUM_PTYPES 1024 + +#define GVE_RX_BUFFER_SIZE_DQO 2048 + /* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ struct gve_rx_desc_queue { struct gve_rx_desc *desc_ring; /* the descriptor ring */ @@ -51,7 +58,8 @@ struct gve_rx_desc_queue { struct gve_rx_slot_page_info { struct page *page; void *page_address; - u8 page_offset; /* flipped to second half? */ + u32 page_offset; /* offset to write to in page */ + int pagecnt_bias; /* expected pagecnt if only the driver has a ref */ u8 can_flip; }; @@ -76,17 +84,117 @@ struct gve_rx_data_queue { struct gve_priv; -/* An RX ring that contains a power-of-two sized desc and data ring. */ +/* RX buffer queue for posting buffers to HW. + * Each RX (completion) queue has a corresponding buffer queue. + */ +struct gve_rx_buf_queue_dqo { + struct gve_rx_desc_dqo *desc_ring; + dma_addr_t bus; + u32 head; /* Pointer to start cleaning buffers at. */ + u32 tail; /* Last posted buffer index + 1 */ + u32 mask; /* Mask for indices to the size of the ring */ +}; + +/* RX completion queue to receive packets from HW. */ +struct gve_rx_compl_queue_dqo { + struct gve_rx_compl_desc_dqo *desc_ring; + dma_addr_t bus; + + /* Number of slots which did not have a buffer posted yet. We should not + * post more buffers than the queue size to avoid HW overrunning the + * queue. + */ + int num_free_slots; + + /* HW uses a "generation bit" to notify SW of new descriptors. When a + * descriptor's generation bit is different from the current generation, + * that descriptor is ready to be consumed by SW. + */ + u8 cur_gen_bit; + + /* Pointer into desc_ring where the next completion descriptor will be + * received. + */ + u32 head; + u32 mask; /* Mask for indices to the size of the ring */ +}; + +/* Stores state for tracking buffers posted to HW */ +struct gve_rx_buf_state_dqo { + /* The page posted to HW. */ + struct gve_rx_slot_page_info page_info; + + /* The DMA address corresponding to `page_info`. */ + dma_addr_t addr; + + /* Last offset into the page when it only had a single reference, at + * which point every other offset is free to be reused. + */ + u32 last_single_ref_offset; + + /* Linked list index to next element in the list, or -1 if none */ + s16 next; +}; + +/* `head` and `tail` are indices into an array, or -1 if empty. */ +struct gve_index_list { + s16 head; + s16 tail; +}; + +/* Contains datapath state used to represent an RX queue. */ struct gve_rx_ring { struct gve_priv *gve; - struct gve_rx_desc_queue desc; - struct gve_rx_data_queue data; + union { + /* GQI fields */ + struct { + struct gve_rx_desc_queue desc; + struct gve_rx_data_queue data; + + /* threshold for posting new buffs and descs */ + u32 db_threshold; + }; + + /* DQO fields. */ + struct { + struct gve_rx_buf_queue_dqo bufq; + struct gve_rx_compl_queue_dqo complq; + + struct gve_rx_buf_state_dqo *buf_states; + u16 num_buf_states; + + /* Linked list of gve_rx_buf_state_dqo. Index into + * buf_states, or -1 if empty. + */ + s16 free_buf_states; + + /* Linked list of gve_rx_buf_state_dqo. Indexes into + * buf_states, or -1 if empty. + * + * This list contains buf_states which are pointing to + * valid buffers. + * + * We use a FIFO here in order to increase the + * probability that buffers can be reused by increasing + * the time between usages. + */ + struct gve_index_list recycled_buf_states; + + /* Linked list of gve_rx_buf_state_dqo. Indexes into + * buf_states, or -1 if empty. + * + * This list contains buf_states which have buffers + * which cannot be reused yet. + */ + struct gve_index_list used_buf_states; + } dqo; + }; + u64 rbytes; /* free-running bytes received */ u64 rpackets; /* free-running packets received */ u32 cnt; /* free-running total number of completed packets */ u32 fill_cnt; /* free-running total number of descs and buffs posted */ u32 mask; /* masks the cnt and fill_cnt to the size of the ring */ - u32 db_threshold; /* threshold for posting new buffs and descs */ u64 rx_copybreak_pkt; /* free-running count of copybreak packets */ u64 rx_copied_pkt; /* free-running total number of copied packets */ u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */ @@ -97,6 +205,10 @@ struct gve_rx_ring { struct gve_queue_resources *q_resources; /* head and tail pointer idx */ dma_addr_t q_resources_bus; /* dma address for the queue resources */ struct u64_stats_sync statss; /* sync stats for 32bit archs */ + + /* head and tail of skb chain for the current packet or NULL if none */ + struct sk_buff *skb_head; + struct sk_buff *skb_tail; }; /* A TX desc ring entry */ @@ -137,23 +249,161 @@ struct gve_tx_fifo { struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */ }; -/* A TX ring that contains a power-of-two sized desc ring and a FIFO buffer */ +/* TX descriptor for DQO format */ +union gve_tx_desc_dqo { + struct gve_tx_pkt_desc_dqo pkt; + struct gve_tx_tso_context_desc_dqo tso_ctx; + struct gve_tx_general_context_desc_dqo general_ctx; +}; + +enum gve_packet_state { + /* Packet is in free list, available to be allocated. + * This should always be zero since state is not explicitly initialized. + */ + GVE_PACKET_STATE_UNALLOCATED, + /* Packet is expecting a regular data completion or miss completion */ + GVE_PACKET_STATE_PENDING_DATA_COMPL, + /* Packet has received a miss completion and is expecting a + * re-injection completion. + */ + GVE_PACKET_STATE_PENDING_REINJECT_COMPL, + /* No valid completion received within the specified timeout. */ + GVE_PACKET_STATE_TIMED_OUT_COMPL, +}; + +struct gve_tx_pending_packet_dqo { + struct sk_buff *skb; /* skb for this packet */ + + /* 0th element corresponds to the linear portion of `skb`, should be + * unmapped with `dma_unmap_single`. + * + * All others correspond to `skb`'s frags and should be unmapped with + * `dma_unmap_page`. + */ + struct gve_tx_dma_buf bufs[MAX_SKB_FRAGS + 1]; + u16 num_bufs; + + /* Linked list index to next element in the list, or -1 if none */ + s16 next; + + /* Linked list index to prev element in the list, or -1 if none. + * Used for tracking either outstanding miss completions or prematurely + * freed packets. + */ + s16 prev; + + /* Identifies the current state of the packet as defined in + * `enum gve_packet_state`. + */ + u8 state; + + /* If packet is an outstanding miss completion, then the packet is + * freed if the corresponding re-injection completion is not received + * before kernel jiffies exceeds timeout_jiffies. + */ + unsigned long timeout_jiffies; +}; + +/* Contains datapath state used to represent a TX queue. */ struct gve_tx_ring { /* Cacheline 0 -- Accessed & dirtied during transmit */ - struct gve_tx_fifo tx_fifo; - u32 req; /* driver tracked head pointer */ - u32 done; /* driver tracked tail pointer */ + union { + /* GQI fields */ + struct { + struct gve_tx_fifo tx_fifo; + u32 req; /* driver tracked head pointer */ + u32 done; /* driver tracked tail pointer */ + }; + + /* DQO fields. */ + struct { + /* Linked list of gve_tx_pending_packet_dqo. Index into + * pending_packets, or -1 if empty. + * + * This is a consumer list owned by the TX path. When it + * runs out, the producer list is stolen from the + * completion handling path + * (dqo_compl.free_pending_packets). + */ + s16 free_pending_packets; + + /* Cached value of `dqo_compl.hw_tx_head` */ + u32 head; + u32 tail; /* Last posted buffer index + 1 */ + + /* Index of the last descriptor with "report event" bit + * set. + */ + u32 last_re_idx; + } dqo_tx; + }; /* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */ - __be32 last_nic_done ____cacheline_aligned; /* NIC tail pointer */ + union { + /* GQI fields */ + struct { + /* NIC tail pointer */ + __be32 last_nic_done; + }; + + /* DQO fields. */ + struct { + u32 head; /* Last read on compl_desc */ + + /* Tracks the current gen bit of compl_q */ + u8 cur_gen_bit; + + /* Linked list of gve_tx_pending_packet_dqo. Index into + * pending_packets, or -1 if empty. + * + * This is the producer list, owned by the completion + * handling path. When the consumer list + * (dqo_tx.free_pending_packets) is runs out, this list + * will be stolen. + */ + atomic_t free_pending_packets; + + /* Last TX ring index fetched by HW */ + atomic_t hw_tx_head; + + /* List to track pending packets which received a miss + * completion but not a corresponding reinjection. + */ + struct gve_index_list miss_completions; + + /* List to track pending packets that were completed + * before receiving a valid completion because they + * reached a specified timeout. + */ + struct gve_index_list timed_out_completions; + } dqo_compl; + } ____cacheline_aligned; u64 pkt_done; /* free-running - total packets completed */ u64 bytes_done; /* free-running - total bytes completed */ u64 dropped_pkt; /* free-running - total packets dropped */ u64 dma_mapping_error; /* count of dma mapping errors */ /* Cacheline 2 -- Read-mostly fields */ - union gve_tx_desc *desc ____cacheline_aligned; - struct gve_tx_buffer_state *info; /* Maps 1:1 to a desc */ + union { + /* GQI fields */ + struct { + union gve_tx_desc *desc; + + /* Maps 1:1 to a desc */ + struct gve_tx_buffer_state *info; + }; + + /* DQO fields. */ + struct { + union gve_tx_desc_dqo *tx_ring; + struct gve_tx_compl_desc *compl_ring; + + struct gve_tx_pending_packet_dqo *pending_packets; + s16 num_pending_packets; + + u32 complq_mask; /* complq size is complq_mask + 1 */ + } dqo; + } ____cacheline_aligned; struct netdev_queue *netdev_txq; struct gve_queue_resources *q_resources; /* head and tail pointer idx */ struct device *dev; @@ -167,6 +417,7 @@ struct gve_tx_ring { u32 ntfy_id; /* notification block index */ dma_addr_t bus; /* dma address of the descr ring */ dma_addr_t q_resources_bus; /* dma address of the queue resources */ + dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */ struct u64_stats_sync statss; /* sync stats for 32bit archs */ } ____cacheline_aligned; @@ -194,6 +445,31 @@ struct gve_qpl_config { unsigned long *qpl_id_map; /* bitmap of used qpl ids */ }; +struct gve_options_dqo_rda { + u16 tx_comp_ring_entries; /* number of tx_comp descriptors */ + u16 rx_buff_ring_entries; /* number of rx_buff descriptors */ +}; + +struct gve_ptype { + u8 l3_type; /* `gve_l3_type` in gve_adminq.h */ + u8 l4_type; /* `gve_l4_type` in gve_adminq.h */ +}; + +struct gve_ptype_lut { + struct gve_ptype ptypes[GVE_NUM_PTYPES]; +}; + +/* GVE_QUEUE_FORMAT_UNSPECIFIED must be zero since 0 is the default value + * when the entire configure_device_resources command is zeroed out and the + * queue_format is not specified. + */ +enum gve_queue_format { + GVE_QUEUE_FORMAT_UNSPECIFIED = 0x0, + GVE_GQI_RDA_FORMAT = 0x1, + GVE_GQI_QPL_FORMAT = 0x2, + GVE_DQO_RDA_FORMAT = 0x3, +}; + struct gve_priv { struct net_device *dev; struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */ @@ -216,7 +492,6 @@ struct gve_priv { u64 num_registered_pages; /* num pages registered with NIC */ u32 rx_copybreak; /* copy packets smaller than this */ u16 default_num_queues; /* default num queues to set up */ - u8 raw_addressing; /* 1 if this dev supports raw addressing, 0 otherwise */ struct gve_queue_config tx_cfg; struct gve_queue_config rx_cfg; @@ -251,6 +526,7 @@ struct gve_priv { u32 adminq_set_driver_parameter_cnt; u32 adminq_report_stats_cnt; u32 adminq_report_link_speed_cnt; + u32 adminq_get_ptype_map_cnt; /* Global stats */ u32 interface_up_cnt; /* count of times interface turned up since last reset */ @@ -275,6 +551,14 @@ struct gve_priv { /* Gvnic device link speed from hypervisor. */ u64 link_speed; + + struct gve_options_dqo_rda options_dqo_rda; + struct gve_ptype_lut *ptype_lut_dqo; + + /* Must be a power of two. */ + int data_buffer_size_dqo; + + enum gve_queue_format queue_format; }; enum gve_service_task_flags_bit { @@ -454,14 +738,20 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) */ static inline u32 gve_num_tx_qpls(struct gve_priv *priv) { - return priv->raw_addressing ? 0 : priv->tx_cfg.num_queues; + if (priv->queue_format != GVE_GQI_QPL_FORMAT) + return 0; + + return priv->tx_cfg.num_queues; } /* Returns the number of rx queue page lists */ static inline u32 gve_num_rx_qpls(struct gve_priv *priv) { - return priv->raw_addressing ? 0 : priv->rx_cfg.num_queues; + if (priv->queue_format != GVE_GQI_QPL_FORMAT) + return 0; + + return priv->rx_cfg.num_queues; } /* Returns a pointer to the next available tx qpl in the list of qpls @@ -515,6 +805,12 @@ static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv, return DMA_FROM_DEVICE; } +static inline bool gve_is_gqi(struct gve_priv *priv) +{ + return priv->queue_format == GVE_GQI_RDA_FORMAT || + priv->queue_format == GVE_GQI_QPL_FORMAT; +} + /* buffers */ int gve_alloc_page(struct gve_priv *priv, struct device *dev, struct page **page, dma_addr_t *dma, @@ -525,14 +821,14 @@ void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev); bool gve_tx_poll(struct gve_notify_block *block, int budget); int gve_tx_alloc_rings(struct gve_priv *priv); -void gve_tx_free_rings(struct gve_priv *priv); +void gve_tx_free_rings_gqi(struct gve_priv *priv); __be32 gve_tx_load_event_counter(struct gve_priv *priv, struct gve_tx_ring *tx); /* rx handling */ void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx); bool gve_rx_poll(struct gve_notify_block *block, int budget); int gve_rx_alloc_rings(struct gve_priv *priv); -void gve_rx_free_rings(struct gve_priv *priv); +void gve_rx_free_rings_gqi(struct gve_priv *priv); bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget, netdev_features_t feat); /* Reset */ diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c index 53864f200599..5bb56b454541 100644 --- a/drivers/net/ethernet/google/gve/gve_adminq.c +++ b/drivers/net/ethernet/google/gve/gve_adminq.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #include <linux/etherdevice.h> @@ -18,6 +18,8 @@ "Expected: length=%d, feature_mask=%x.\n" \ "Actual: length=%d, feature_mask=%x.\n" +#define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n" + static struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor, struct gve_device_option *option) @@ -33,28 +35,81 @@ struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *desc static void gve_parse_device_option(struct gve_priv *priv, struct gve_device_descriptor *device_descriptor, - struct gve_device_option *option) + struct gve_device_option *option, + struct gve_device_option_gqi_rda **dev_op_gqi_rda, + struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, + struct gve_device_option_dqo_rda **dev_op_dqo_rda) { + u32 req_feat_mask = be32_to_cpu(option->required_features_mask); u16 option_length = be16_to_cpu(option->option_length); u16 option_id = be16_to_cpu(option->option_id); + /* If the length or feature mask doesn't match, continue without + * enabling the feature. + */ switch (option_id) { - case GVE_DEV_OPT_ID_RAW_ADDRESSING: - /* If the length or feature mask doesn't match, - * continue without enabling the feature. - */ - if (option_length != GVE_DEV_OPT_LEN_RAW_ADDRESSING || - option->feat_mask != cpu_to_be32(GVE_DEV_OPT_FEAT_MASK_RAW_ADDRESSING)) { - dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, "Raw Addressing", - GVE_DEV_OPT_LEN_RAW_ADDRESSING, - cpu_to_be32(GVE_DEV_OPT_FEAT_MASK_RAW_ADDRESSING), - option_length, option->feat_mask); - priv->raw_addressing = 0; - } else { - dev_info(&priv->pdev->dev, - "Raw addressing device option enabled.\n"); - priv->raw_addressing = 1; + case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING: + if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING || + req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) { + dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, + "Raw Addressing", + GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING, + GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING, + option_length, req_feat_mask); + break; + } + + dev_info(&priv->pdev->dev, + "Gqi raw addressing device option enabled.\n"); + priv->queue_format = GVE_GQI_RDA_FORMAT; + break; + case GVE_DEV_OPT_ID_GQI_RDA: + if (option_length < sizeof(**dev_op_gqi_rda) || + req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) { + dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, + "GQI RDA", (int)sizeof(**dev_op_gqi_rda), + GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA, + option_length, req_feat_mask); + break; + } + + if (option_length > sizeof(**dev_op_gqi_rda)) { + dev_warn(&priv->pdev->dev, + GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA"); + } + *dev_op_gqi_rda = (void *)(option + 1); + break; + case GVE_DEV_OPT_ID_GQI_QPL: + if (option_length < sizeof(**dev_op_gqi_qpl) || + req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) { + dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, + "GQI QPL", (int)sizeof(**dev_op_gqi_qpl), + GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL, + option_length, req_feat_mask); + break; + } + + if (option_length > sizeof(**dev_op_gqi_qpl)) { + dev_warn(&priv->pdev->dev, + GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL"); + } + *dev_op_gqi_qpl = (void *)(option + 1); + break; + case GVE_DEV_OPT_ID_DQO_RDA: + if (option_length < sizeof(**dev_op_dqo_rda) || + req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) { + dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, + "DQO RDA", (int)sizeof(**dev_op_dqo_rda), + GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA, + option_length, req_feat_mask); + break; + } + + if (option_length > sizeof(**dev_op_dqo_rda)) { + dev_warn(&priv->pdev->dev, + GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA"); } + *dev_op_dqo_rda = (void *)(option + 1); break; default: /* If we don't recognize the option just continue @@ -65,6 +120,39 @@ void gve_parse_device_option(struct gve_priv *priv, } } +/* Process all device options for a given describe device call. */ +static int +gve_process_device_options(struct gve_priv *priv, + struct gve_device_descriptor *descriptor, + struct gve_device_option_gqi_rda **dev_op_gqi_rda, + struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, + struct gve_device_option_dqo_rda **dev_op_dqo_rda) +{ + const int num_options = be16_to_cpu(descriptor->num_device_options); + struct gve_device_option *dev_opt; + int i; + + /* The options struct directly follows the device descriptor. */ + dev_opt = (void *)(descriptor + 1); + for (i = 0; i < num_options; i++) { + struct gve_device_option *next_opt; + + next_opt = gve_get_next_option(descriptor, dev_opt); + if (!next_opt) { + dev_err(&priv->dev->dev, + "options exceed device_descriptor's total length.\n"); + return -EINVAL; + } + + gve_parse_device_option(priv, descriptor, dev_opt, + dev_op_gqi_rda, dev_op_gqi_qpl, + dev_op_dqo_rda); + dev_opt = next_opt; + } + + return 0; +} + int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) { priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE, @@ -88,6 +176,7 @@ int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) priv->adminq_set_driver_parameter_cnt = 0; priv->adminq_report_stats_cnt = 0; priv->adminq_report_link_speed_cnt = 0; + priv->adminq_get_ptype_map_cnt = 0; /* Setup Admin queue with the device */ iowrite32be(priv->adminq_bus_addr / PAGE_SIZE, @@ -293,6 +382,9 @@ static int gve_adminq_issue_cmd(struct gve_priv *priv, case GVE_ADMINQ_REPORT_LINK_SPEED: priv->adminq_report_link_speed_cnt++; break; + case GVE_ADMINQ_GET_PTYPE_MAP: + priv->adminq_get_ptype_map_cnt++; + break; default: dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); } @@ -305,7 +397,8 @@ static int gve_adminq_issue_cmd(struct gve_priv *priv, * The caller is also responsible for making sure there are no commands * waiting to be executed. */ -static int gve_adminq_execute_cmd(struct gve_priv *priv, union gve_adminq_command *cmd_orig) +static int gve_adminq_execute_cmd(struct gve_priv *priv, + union gve_adminq_command *cmd_orig) { u32 tail, head; int err; @@ -350,6 +443,7 @@ int gve_adminq_configure_device_resources(struct gve_priv *priv, .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])), .ntfy_blk_msix_base_idx = cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX), + .queue_format = priv->queue_format, }; return gve_adminq_execute_cmd(priv, &cmd); @@ -369,27 +463,32 @@ static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) { struct gve_tx_ring *tx = &priv->tx[queue_index]; union gve_adminq_command cmd; - u32 qpl_id; - int err; - qpl_id = priv->raw_addressing ? GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; memset(&cmd, 0, sizeof(cmd)); cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE); cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) { .queue_id = cpu_to_be32(queue_index), - .reserved = 0, .queue_resources_addr = cpu_to_be64(tx->q_resources_bus), .tx_ring_addr = cpu_to_be64(tx->bus), - .queue_page_list_id = cpu_to_be32(qpl_id), .ntfy_id = cpu_to_be32(tx->ntfy_id), }; - err = gve_adminq_issue_cmd(priv, &cmd); - if (err) - return err; + if (gve_is_gqi(priv)) { + u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? + GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; + + cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); + } else { + cmd.create_tx_queue.tx_ring_size = + cpu_to_be16(priv->tx_desc_cnt); + cmd.create_tx_queue.tx_comp_ring_addr = + cpu_to_be64(tx->complq_bus_dqo); + cmd.create_tx_queue.tx_comp_ring_size = + cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries); + } - return 0; + return gve_adminq_issue_cmd(priv, &cmd); } int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues) @@ -410,28 +509,41 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) { struct gve_rx_ring *rx = &priv->rx[queue_index]; union gve_adminq_command cmd; - u32 qpl_id; - int err; - qpl_id = priv->raw_addressing ? GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; memset(&cmd, 0, sizeof(cmd)); cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) { .queue_id = cpu_to_be32(queue_index), - .index = cpu_to_be32(queue_index), - .reserved = 0, .ntfy_id = cpu_to_be32(rx->ntfy_id), .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), - .rx_desc_ring_addr = cpu_to_be64(rx->desc.bus), - .rx_data_ring_addr = cpu_to_be64(rx->data.data_bus), - .queue_page_list_id = cpu_to_be32(qpl_id), }; - err = gve_adminq_issue_cmd(priv, &cmd); - if (err) - return err; + if (gve_is_gqi(priv)) { + u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? + GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; + + cmd.create_rx_queue.rx_desc_ring_addr = + cpu_to_be64(rx->desc.bus), + cmd.create_rx_queue.rx_data_ring_addr = + cpu_to_be64(rx->data.data_bus), + cmd.create_rx_queue.index = cpu_to_be32(queue_index); + cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); + } else { + cmd.create_rx_queue.rx_ring_size = + cpu_to_be16(priv->rx_desc_cnt); + cmd.create_rx_queue.rx_desc_ring_addr = + cpu_to_be64(rx->dqo.complq.bus); + cmd.create_rx_queue.rx_data_ring_addr = + cpu_to_be64(rx->dqo.bufq.bus); + cmd.create_rx_queue.packet_buffer_size = + cpu_to_be16(priv->data_buffer_size_dqo); + cmd.create_rx_queue.rx_buff_ring_size = + cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries); + cmd.create_rx_queue.enable_rsc = + !!(priv->dev->features & NETIF_F_LRO); + } - return 0; + return gve_adminq_issue_cmd(priv, &cmd); } int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) @@ -512,17 +624,51 @@ int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues) return gve_adminq_kick_and_wait(priv); } +static int gve_set_desc_cnt(struct gve_priv *priv, + struct gve_device_descriptor *descriptor) +{ + priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); + if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) { + dev_err(&priv->pdev->dev, "Tx desc count %d too low\n", + priv->tx_desc_cnt); + return -EINVAL; + } + priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); + if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0]) + < PAGE_SIZE) { + dev_err(&priv->pdev->dev, "Rx desc count %d too low\n", + priv->rx_desc_cnt); + return -EINVAL; + } + return 0; +} + +static int +gve_set_desc_cnt_dqo(struct gve_priv *priv, + const struct gve_device_descriptor *descriptor, + const struct gve_device_option_dqo_rda *dev_op_dqo_rda) +{ + priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); + priv->options_dqo_rda.tx_comp_ring_entries = + be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries); + priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); + priv->options_dqo_rda.rx_buff_ring_entries = + be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries); + + return 0; +} + int gve_adminq_describe_device(struct gve_priv *priv) { + struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; + struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; + struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; struct gve_device_descriptor *descriptor; - struct gve_device_option *dev_opt; union gve_adminq_command cmd; dma_addr_t descriptor_bus; - u16 num_options; int err = 0; u8 *mac; u16 mtu; - int i; memset(&cmd, 0, sizeof(cmd)); descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE, @@ -540,21 +686,41 @@ int gve_adminq_describe_device(struct gve_priv *priv) if (err) goto free_device_descriptor; - priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); - if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) { - dev_err(&priv->pdev->dev, "Tx desc count %d too low\n", priv->tx_desc_cnt); - err = -EINVAL; + err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, + &dev_op_gqi_qpl, &dev_op_dqo_rda); + if (err) goto free_device_descriptor; + + /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format + * is not set to GqiRda, choose the queue format in a priority order: + * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default. + */ + if (priv->queue_format == GVE_GQI_RDA_FORMAT) { + dev_info(&priv->pdev->dev, + "Driver is running with GQI RDA queue format.\n"); + } else if (dev_op_dqo_rda) { + priv->queue_format = GVE_DQO_RDA_FORMAT; + dev_info(&priv->pdev->dev, + "Driver is running with DQO RDA queue format.\n"); + } else if (dev_op_gqi_rda) { + priv->queue_format = GVE_GQI_RDA_FORMAT; + dev_info(&priv->pdev->dev, + "Driver is running with GQI RDA queue format.\n"); + } else { + priv->queue_format = GVE_GQI_QPL_FORMAT; + dev_info(&priv->pdev->dev, + "Driver is running with GQI QPL queue format.\n"); } - priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); - if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0]) - < PAGE_SIZE || - priv->rx_desc_cnt * sizeof(priv->rx->data.data_ring[0]) - < PAGE_SIZE) { - dev_err(&priv->pdev->dev, "Rx desc count %d too low\n", priv->rx_desc_cnt); - err = -EINVAL; - goto free_device_descriptor; + if (gve_is_gqi(priv)) { + err = gve_set_desc_cnt(priv, descriptor); + } else { + /* DQO supports LRO. */ + priv->dev->hw_features |= NETIF_F_LRO; + err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda); } + if (err) + goto free_device_descriptor; + priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages); mtu = be16_to_cpu(descriptor->mtu); @@ -570,32 +736,16 @@ int gve_adminq_describe_device(struct gve_priv *priv) dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl); - if (priv->rx_data_slot_cnt < priv->rx_desc_cnt) { + + if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) { dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n", priv->rx_data_slot_cnt); priv->rx_desc_cnt = priv->rx_data_slot_cnt; } priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); - dev_opt = (void *)(descriptor + 1); - - num_options = be16_to_cpu(descriptor->num_device_options); - for (i = 0; i < num_options; i++) { - struct gve_device_option *next_opt; - - next_opt = gve_get_next_option(descriptor, dev_opt); - if (!next_opt) { - dev_err(&priv->dev->dev, - "options exceed device_descriptor's total length.\n"); - err = -EINVAL; - goto free_device_descriptor; - } - - gve_parse_device_option(priv, descriptor, dev_opt); - dev_opt = next_opt; - } free_device_descriptor: - dma_free_coherent(&priv->pdev->dev, sizeof(*descriptor), descriptor, + dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor, descriptor_bus); return err; } @@ -701,3 +851,41 @@ int gve_adminq_report_link_speed(struct gve_priv *priv) link_speed_region_bus); return err; } + +int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, + struct gve_ptype_lut *ptype_lut) +{ + struct gve_ptype_map *ptype_map; + union gve_adminq_command cmd; + dma_addr_t ptype_map_bus; + int err = 0; + int i; + + memset(&cmd, 0, sizeof(cmd)); + ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map), + &ptype_map_bus, GFP_KERNEL); + if (!ptype_map) + return -ENOMEM; + + cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP); + cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) { + .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)), + .ptype_map_addr = cpu_to_be64(ptype_map_bus), + }; + + err = gve_adminq_execute_cmd(priv, &cmd); + if (err) + goto err; + + /* Populate ptype_lut. */ + for (i = 0; i < GVE_NUM_PTYPES; i++) { + ptype_lut->ptypes[i].l3_type = + ptype_map->ptypes[i].l3_type; + ptype_lut->ptypes[i].l4_type = + ptype_map->ptypes[i].l4_type; + } +err: + dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map, + ptype_map_bus); + return err; +} diff --git a/drivers/net/ethernet/google/gve/gve_adminq.h b/drivers/net/ethernet/google/gve/gve_adminq.h index d320c2ffd87c..47c3d8f313fc 100644 --- a/drivers/net/ethernet/google/gve/gve_adminq.h +++ b/drivers/net/ethernet/google/gve/gve_adminq.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: (GPL-2.0 OR MIT) * Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #ifndef _GVE_ADMINQ_H @@ -22,7 +22,8 @@ enum gve_adminq_opcodes { GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES = 0x9, GVE_ADMINQ_SET_DRIVER_PARAMETER = 0xB, GVE_ADMINQ_REPORT_STATS = 0xC, - GVE_ADMINQ_REPORT_LINK_SPEED = 0xD + GVE_ADMINQ_REPORT_LINK_SPEED = 0xD, + GVE_ADMINQ_GET_PTYPE_MAP = 0xE, }; /* Admin queue status codes */ @@ -82,14 +83,54 @@ static_assert(sizeof(struct gve_device_descriptor) == 40); struct gve_device_option { __be16 option_id; __be16 option_length; - __be32 feat_mask; + __be32 required_features_mask; }; static_assert(sizeof(struct gve_device_option) == 8); -#define GVE_DEV_OPT_ID_RAW_ADDRESSING 0x1 -#define GVE_DEV_OPT_LEN_RAW_ADDRESSING 0x0 -#define GVE_DEV_OPT_FEAT_MASK_RAW_ADDRESSING 0x0 +struct gve_device_option_gqi_rda { + __be32 supported_features_mask; +}; + +static_assert(sizeof(struct gve_device_option_gqi_rda) == 4); + +struct gve_device_option_gqi_qpl { + __be32 supported_features_mask; +}; + +static_assert(sizeof(struct gve_device_option_gqi_qpl) == 4); + +struct gve_device_option_dqo_rda { + __be32 supported_features_mask; + __be16 tx_comp_ring_entries; + __be16 rx_buff_ring_entries; +}; + +static_assert(sizeof(struct gve_device_option_dqo_rda) == 8); + +/* Terminology: + * + * RDA - Raw DMA Addressing - Buffers associated with SKBs are directly DMA + * mapped and read/updated by the device. + * + * QPL - Queue Page Lists - Driver uses bounce buffers which are DMA mapped with + * the device for read/write and data is copied from/to SKBs. + */ +enum gve_dev_opt_id { + GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING = 0x1, + GVE_DEV_OPT_ID_GQI_RDA = 0x2, + GVE_DEV_OPT_ID_GQI_QPL = 0x3, + GVE_DEV_OPT_ID_DQO_RDA = 0x4, +}; + +enum gve_dev_opt_req_feat_mask { + GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING = 0x0, + GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA = 0x0, + GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL = 0x0, + GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA = 0x0, +}; + +#define GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING 0x0 struct gve_adminq_configure_device_resources { __be64 counter_array; @@ -98,9 +139,11 @@ struct gve_adminq_configure_device_resources { __be32 num_irq_dbs; __be32 irq_db_stride; __be32 ntfy_blk_msix_base_idx; + u8 queue_format; + u8 padding[7]; }; -static_assert(sizeof(struct gve_adminq_configure_device_resources) == 32); +static_assert(sizeof(struct gve_adminq_configure_device_resources) == 40); struct gve_adminq_register_page_list { __be32 page_list_id; @@ -125,9 +168,13 @@ struct gve_adminq_create_tx_queue { __be64 tx_ring_addr; __be32 queue_page_list_id; __be32 ntfy_id; + __be64 tx_comp_ring_addr; + __be16 tx_ring_size; + __be16 tx_comp_ring_size; + u8 padding[4]; }; -static_assert(sizeof(struct gve_adminq_create_tx_queue) == 32); +static_assert(sizeof(struct gve_adminq_create_tx_queue) == 48); struct gve_adminq_create_rx_queue { __be32 queue_id; @@ -138,10 +185,14 @@ struct gve_adminq_create_rx_queue { __be64 rx_desc_ring_addr; __be64 rx_data_ring_addr; __be32 queue_page_list_id; - u8 padding[4]; + __be16 rx_ring_size; + __be16 packet_buffer_size; + __be16 rx_buff_ring_size; + u8 enable_rsc; + u8 padding[5]; }; -static_assert(sizeof(struct gve_adminq_create_rx_queue) == 48); +static_assert(sizeof(struct gve_adminq_create_rx_queue) == 56); /* Queue resources that are shared with the device */ struct gve_queue_resources { @@ -226,6 +277,41 @@ enum gve_stat_names { RX_DROPS_INVALID_CHECKSUM = 68, }; +enum gve_l3_type { + /* Must be zero so zero initialized LUT is unknown. */ + GVE_L3_TYPE_UNKNOWN = 0, + GVE_L3_TYPE_OTHER, + GVE_L3_TYPE_IPV4, + GVE_L3_TYPE_IPV6, +}; + +enum gve_l4_type { + /* Must be zero so zero initialized LUT is unknown. */ + GVE_L4_TYPE_UNKNOWN = 0, + GVE_L4_TYPE_OTHER, + GVE_L4_TYPE_TCP, + GVE_L4_TYPE_UDP, + GVE_L4_TYPE_ICMP, + GVE_L4_TYPE_SCTP, +}; + +/* These are control path types for PTYPE which are the same as the data path + * types. + */ +struct gve_ptype_entry { + u8 l3_type; + u8 l4_type; +}; + +struct gve_ptype_map { + struct gve_ptype_entry ptypes[1 << 10]; /* PTYPES are always 10 bits. */ +}; + +struct gve_adminq_get_ptype_map { + __be64 ptype_map_len; + __be64 ptype_map_addr; +}; + union gve_adminq_command { struct { __be32 opcode; @@ -243,6 +329,7 @@ union gve_adminq_command { struct gve_adminq_set_driver_parameter set_driver_param; struct gve_adminq_report_stats report_stats; struct gve_adminq_report_link_speed report_link_speed; + struct gve_adminq_get_ptype_map get_ptype_map; }; }; u8 reserved[64]; @@ -271,4 +358,9 @@ int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu); int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, dma_addr_t stats_report_addr, u64 interval); int gve_adminq_report_link_speed(struct gve_priv *priv); + +struct gve_ptype_lut; +int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, + struct gve_ptype_lut *ptype_lut); + #endif /* _GVE_ADMINQ_H */ diff --git a/drivers/net/ethernet/google/gve/gve_desc_dqo.h b/drivers/net/ethernet/google/gve/gve_desc_dqo.h new file mode 100644 index 000000000000..e8fe9adef7f2 --- /dev/null +++ b/drivers/net/ethernet/google/gve/gve_desc_dqo.h @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) + * Google virtual Ethernet (gve) driver + * + * Copyright (C) 2015-2021 Google, Inc. + */ + +/* GVE DQO Descriptor formats */ + +#ifndef _GVE_DESC_DQO_H_ +#define _GVE_DESC_DQO_H_ + +#include <linux/build_bug.h> + +#define GVE_TX_MAX_HDR_SIZE_DQO 255 +#define GVE_TX_MIN_TSO_MSS_DQO 88 + +#ifndef __LITTLE_ENDIAN_BITFIELD +#error "Only little endian supported" +#endif + +/* Basic TX descriptor (DTYPE 0x0C) */ +struct gve_tx_pkt_desc_dqo { + __le64 buf_addr; + + /* Must be GVE_TX_PKT_DESC_DTYPE_DQO (0xc) */ + u8 dtype: 5; + + /* Denotes the last descriptor of a packet. */ + u8 end_of_packet: 1; + u8 checksum_offload_enable: 1; + + /* If set, will generate a descriptor completion for this descriptor. */ + u8 report_event: 1; + u8 reserved0; + __le16 reserved1; + + /* The TX completion associated with this packet will contain this tag. + */ + __le16 compl_tag; + u16 buf_size: 14; + u16 reserved2: 2; +} __packed; +static_assert(sizeof(struct gve_tx_pkt_desc_dqo) == 16); + +#define GVE_TX_PKT_DESC_DTYPE_DQO 0xc +#define GVE_TX_MAX_BUF_SIZE_DQO ((16 * 1024) - 1) + +/* Maximum number of data descriptors allowed per packet, or per-TSO segment. */ +#define GVE_TX_MAX_DATA_DESCS 10 + +/* Min gap between tail and head to avoid cacheline overlap */ +#define GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP 4 + +/* "report_event" on TX packet descriptors may only be reported on the last + * descriptor of a TX packet, and they must be spaced apart with at least this + * value. + */ +#define GVE_TX_MIN_RE_INTERVAL 32 + +struct gve_tx_context_cmd_dtype { + u8 dtype: 5; + u8 tso: 1; + u8 reserved1: 2; + + u8 reserved2; +}; + +static_assert(sizeof(struct gve_tx_context_cmd_dtype) == 2); + +/* TX Native TSO Context DTYPE (0x05) + * + * "flex" fields allow the driver to send additional packet context to HW. + */ +struct gve_tx_tso_context_desc_dqo { + /* The L4 payload bytes that should be segmented. */ + u32 tso_total_len: 24; + u32 flex10: 8; + + /* Max segment size in TSO excluding headers. */ + u16 mss: 14; + u16 reserved: 2; + + u8 header_len; /* Header length to use for TSO offload */ + u8 flex11; + struct gve_tx_context_cmd_dtype cmd_dtype; + u8 flex0; + u8 flex5; + u8 flex6; + u8 flex7; + u8 flex8; + u8 flex9; +} __packed; +static_assert(sizeof(struct gve_tx_tso_context_desc_dqo) == 16); + +#define GVE_TX_TSO_CTX_DESC_DTYPE_DQO 0x5 + +/* General context descriptor for sending metadata. */ +struct gve_tx_general_context_desc_dqo { + u8 flex4; + u8 flex5; + u8 flex6; + u8 flex7; + u8 flex8; + u8 flex9; + u8 flex10; + u8 flex11; + struct gve_tx_context_cmd_dtype cmd_dtype; + u16 reserved; + u8 flex0; + u8 flex1; + u8 flex2; + u8 flex3; +} __packed; +static_assert(sizeof(struct gve_tx_general_context_desc_dqo) == 16); + +#define GVE_TX_GENERAL_CTX_DESC_DTYPE_DQO 0x4 + +/* Logical structure of metadata which is packed into context descriptor flex + * fields. + */ +struct gve_tx_metadata_dqo { + union { + struct { + u8 version; + + /* If `skb->l4_hash` is set, this value should be + * derived from `skb->hash`. + * + * A zero value means no l4_hash was associated with the + * skb. + */ + u16 path_hash: 15; + + /* Should be set to 1 if the flow associated with the + * skb had a rehash from the TCP stack. + */ + u16 rehash_event: 1; + } __packed; + u8 bytes[12]; + }; +} __packed; +static_assert(sizeof(struct gve_tx_metadata_dqo) == 12); + +#define GVE_TX_METADATA_VERSION_DQO 0 + +/* TX completion descriptor */ +struct gve_tx_compl_desc { + /* For types 0-4 this is the TX queue ID associated with this + * completion. + */ + u16 id: 11; + + /* See: GVE_COMPL_TYPE_DQO* */ + u16 type: 3; + u16 reserved0: 1; + + /* Flipped by HW to notify the descriptor is populated. */ + u16 generation: 1; + union { + /* For descriptor completions, this is the last index fetched + * by HW + 1. + */ + __le16 tx_head; + + /* For packet completions, this is the completion tag set on the + * TX packet descriptors. + */ + __le16 completion_tag; + }; + __le32 reserved1; +} __packed; +static_assert(sizeof(struct gve_tx_compl_desc) == 8); + +#define GVE_COMPL_TYPE_DQO_PKT 0x2 /* Packet completion */ +#define GVE_COMPL_TYPE_DQO_DESC 0x4 /* Descriptor completion */ +#define GVE_COMPL_TYPE_DQO_MISS 0x1 /* Miss path completion */ +#define GVE_COMPL_TYPE_DQO_REINJECTION 0x3 /* Re-injection completion */ + +/* Descriptor to post buffers to HW on buffer queue. */ +struct gve_rx_desc_dqo { + __le16 buf_id; /* ID returned in Rx completion descriptor */ + __le16 reserved0; + __le32 reserved1; + __le64 buf_addr; /* DMA address of the buffer */ + __le64 header_buf_addr; + __le64 reserved2; +} __packed; +static_assert(sizeof(struct gve_rx_desc_dqo) == 32); + +/* Descriptor for HW to notify SW of new packets received on RX queue. */ +struct gve_rx_compl_desc_dqo { + /* Must be 1 */ + u8 rxdid: 4; + u8 reserved0: 4; + + /* Packet originated from this system rather than the network. */ + u8 loopback: 1; + /* Set when IPv6 packet contains a destination options header or routing + * header. + */ + u8 ipv6_ex_add: 1; + /* Invalid packet was received. */ + u8 rx_error: 1; + u8 reserved1: 5; + + u16 packet_type: 10; + u16 ip_hdr_err: 1; + u16 udp_len_err: 1; + u16 raw_cs_invalid: 1; + u16 reserved2: 3; + + u16 packet_len: 14; + /* Flipped by HW to notify the descriptor is populated. */ + u16 generation: 1; + /* Should be zero. */ + u16 buffer_queue_id: 1; + + u16 header_len: 10; + u16 rsc: 1; + u16 split_header: 1; + u16 reserved3: 4; + + u8 descriptor_done: 1; + u8 end_of_packet: 1; + u8 header_buffer_overflow: 1; + u8 l3_l4_processed: 1; + u8 csum_ip_err: 1; + u8 csum_l4_err: 1; + u8 csum_external_ip_err: 1; + u8 csum_external_udp_err: 1; + + u8 status_error1; + + __le16 reserved5; + __le16 buf_id; /* Buffer ID which was sent on the buffer queue. */ + + union { + /* Packet checksum. */ + __le16 raw_cs; + /* Segment length for RSC packets. */ + __le16 rsc_seg_len; + }; + __le32 hash; + __le32 reserved6; + __le64 reserved7; +} __packed; + +static_assert(sizeof(struct gve_rx_compl_desc_dqo) == 32); + +/* Ringing the doorbell too often can hurt performance. + * + * HW requires this value to be at least 8. + */ +#define GVE_RX_BUF_THRESH_DQO 32 + +#endif /* _GVE_DESC_DQO_H_ */ diff --git a/drivers/net/ethernet/google/gve/gve_dqo.h b/drivers/net/ethernet/google/gve/gve_dqo.h new file mode 100644 index 000000000000..836042364124 --- /dev/null +++ b/drivers/net/ethernet/google/gve/gve_dqo.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) + * Google virtual Ethernet (gve) driver + * + * Copyright (C) 2015-2021 Google, Inc. + */ + +#ifndef _GVE_DQO_H_ +#define _GVE_DQO_H_ + +#include "gve_adminq.h" + +#define GVE_ITR_ENABLE_BIT_DQO BIT(0) +#define GVE_ITR_CLEAR_PBA_BIT_DQO BIT(1) +#define GVE_ITR_NO_UPDATE_DQO (3 << 3) + +#define GVE_ITR_INTERVAL_DQO_SHIFT 5 +#define GVE_ITR_INTERVAL_DQO_MASK ((1 << 12) - 1) + +#define GVE_TX_IRQ_RATELIMIT_US_DQO 50 +#define GVE_RX_IRQ_RATELIMIT_US_DQO 20 + +/* Timeout in seconds to wait for a reinjection completion after receiving + * its corresponding miss completion. + */ +#define GVE_REINJECT_COMPL_TIMEOUT 1 + +/* Timeout in seconds to deallocate the completion tag for a packet that was + * prematurely freed for not receiving a valid completion. This should be large + * enough to rule out the possibility of receiving the corresponding valid + * completion after this interval. + */ +#define GVE_DEALLOCATE_COMPL_TIMEOUT 60 + +netdev_tx_t gve_tx_dqo(struct sk_buff *skb, struct net_device *dev); +bool gve_tx_poll_dqo(struct gve_notify_block *block, bool do_clean); +int gve_rx_poll_dqo(struct gve_notify_block *block, int budget); +int gve_tx_alloc_rings_dqo(struct gve_priv *priv); +void gve_tx_free_rings_dqo(struct gve_priv *priv); +int gve_rx_alloc_rings_dqo(struct gve_priv *priv); +void gve_rx_free_rings_dqo(struct gve_priv *priv); +int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, + struct napi_struct *napi); +void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx); +void gve_rx_write_doorbell_dqo(const struct gve_priv *priv, int queue_idx); + +static inline void +gve_tx_put_doorbell_dqo(const struct gve_priv *priv, + const struct gve_queue_resources *q_resources, u32 val) +{ + u64 index; + + index = be32_to_cpu(q_resources->db_index); + iowrite32(val, &priv->db_bar2[index]); +} + +/* Builds register value to write to DQO IRQ doorbell to enable with specified + * ratelimit. + */ +static inline u32 gve_set_itr_ratelimit_dqo(u32 ratelimit_us) +{ + u32 result = GVE_ITR_ENABLE_BIT_DQO; + + /* Interval has 2us granularity. */ + ratelimit_us >>= 1; + + ratelimit_us &= GVE_ITR_INTERVAL_DQO_MASK; + result |= (ratelimit_us << GVE_ITR_INTERVAL_DQO_SHIFT); + + return result; +} + +static inline void +gve_write_irq_doorbell_dqo(const struct gve_priv *priv, + const struct gve_notify_block *block, u32 val) +{ + u32 index = be32_to_cpu(block->irq_db_index); + + iowrite32(val, &priv->db_bar2[index]); +} + +#endif /* _GVE_DQO_H_ */ diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index 5fb05cf36b49..716e6240305d 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #include <linux/ethtool.h> @@ -311,8 +311,16 @@ gve_get_ethtool_stats(struct net_device *netdev, for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { struct gve_tx_ring *tx = &priv->tx[ring]; - data[i++] = tx->req; - data[i++] = tx->done; + if (gve_is_gqi(priv)) { + data[i++] = tx->req; + data[i++] = tx->done; + } else { + /* DQO doesn't currently support + * posted/completed descriptor counts; + */ + data[i++] = 0; + data[i++] = 0; + } do { start = u64_stats_fetch_begin(&priv->tx[ring].statss); @@ -453,11 +461,16 @@ static int gve_set_tunable(struct net_device *netdev, switch (etuna->id) { case ETHTOOL_RX_COPYBREAK: + { + u32 max_copybreak = gve_is_gqi(priv) ? + (PAGE_SIZE / 2) : priv->data_buffer_size_dqo; + len = *(u32 *)value; - if (len > PAGE_SIZE / 2) + if (len > max_copybreak) return -EINVAL; priv->rx_copybreak = len; return 0; + } default: return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 7302498c6df3..867e87af3432 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #include <linux/cpumask.h> @@ -14,6 +14,7 @@ #include <linux/workqueue.h> #include <net/sch_generic.h> #include "gve.h" +#include "gve_dqo.h" #include "gve_adminq.h" #include "gve_register.h" @@ -26,6 +27,16 @@ const char gve_version_str[] = GVE_VERSION; static const char gve_version_prefix[] = GVE_VERSION_PREFIX; +static netdev_tx_t gve_start_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct gve_priv *priv = netdev_priv(dev); + + if (gve_is_gqi(priv)) + return gve_tx(skb, dev); + else + return gve_tx_dqo(skb, dev); +} + static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) { struct gve_priv *priv = netdev_priv(dev); @@ -155,6 +166,15 @@ static irqreturn_t gve_intr(int irq, void *arg) return IRQ_HANDLED; } +static irqreturn_t gve_intr_dqo(int irq, void *arg) +{ + struct gve_notify_block *block = arg; + + /* Interrupts are automatically masked */ + napi_schedule_irqoff(&block->napi); + return IRQ_HANDLED; +} + static int gve_napi_poll(struct napi_struct *napi, int budget) { struct gve_notify_block *block; @@ -180,7 +200,7 @@ static int gve_napi_poll(struct napi_struct *napi, int budget) /* Double check we have no extra work. * Ensure unmask synchronizes with checking for work. */ - dma_rmb(); + mb(); if (block->tx) reschedule |= gve_tx_poll(block, -1); if (block->rx) @@ -191,6 +211,54 @@ static int gve_napi_poll(struct napi_struct *napi, int budget) return 0; } +static int gve_napi_poll_dqo(struct napi_struct *napi, int budget) +{ + struct gve_notify_block *block = + container_of(napi, struct gve_notify_block, napi); + struct gve_priv *priv = block->priv; + bool reschedule = false; + int work_done = 0; + + /* Clear PCI MSI-X Pending Bit Array (PBA) + * + * This bit is set if an interrupt event occurs while the vector is + * masked. If this bit is set and we reenable the interrupt, it will + * fire again. Since we're just about to poll the queue state, we don't + * need it to fire again. + * + * Under high softirq load, it's possible that the interrupt condition + * is triggered twice before we got the chance to process it. + */ + gve_write_irq_doorbell_dqo(priv, block, + GVE_ITR_NO_UPDATE_DQO | GVE_ITR_CLEAR_PBA_BIT_DQO); + + if (block->tx) + reschedule |= gve_tx_poll_dqo(block, /*do_clean=*/true); + + if (block->rx) { + work_done = gve_rx_poll_dqo(block, budget); + reschedule |= work_done == budget; + } + + if (reschedule) + return budget; + + if (likely(napi_complete_done(napi, work_done))) { + /* Enable interrupts again. + * + * We don't need to repoll afterwards because HW supports the + * PCI MSI-X PBA feature. + * + * Another interrupt would be triggered if a new event came in + * since the last one. + */ + gve_write_irq_doorbell_dqo(priv, block, + GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO); + } + + return work_done; +} + static int gve_alloc_notify_blocks(struct gve_priv *priv) { int num_vecs_requested = priv->num_ntfy_blks + 1; @@ -220,6 +288,7 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) int vecs_left = new_num_ntfy_blks % 2; priv->num_ntfy_blks = new_num_ntfy_blks; + priv->mgmt_msix_idx = priv->num_ntfy_blks; priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues, vecs_per_type); priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues, @@ -263,7 +332,8 @@ static int gve_alloc_notify_blocks(struct gve_priv *priv) name, i); block->priv = priv; err = request_irq(priv->msix_vectors[msix_idx].vector, - gve_intr, 0, block->name, block); + gve_is_gqi(priv) ? gve_intr : gve_intr_dqo, + 0, block->name, block); if (err) { dev_err(&priv->pdev->dev, "Failed to receive msix vector %d\n", i); @@ -300,20 +370,22 @@ static void gve_free_notify_blocks(struct gve_priv *priv) { int i; - /* Free the irqs */ - for (i = 0; i < priv->num_ntfy_blks; i++) { - struct gve_notify_block *block = &priv->ntfy_blocks[i]; - int msix_idx = i; + if (priv->msix_vectors) { + /* Free the irqs */ + for (i = 0; i < priv->num_ntfy_blks; i++) { + struct gve_notify_block *block = &priv->ntfy_blocks[i]; + int msix_idx = i; - irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, - NULL); - free_irq(priv->msix_vectors[msix_idx].vector, block); + irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, + NULL); + free_irq(priv->msix_vectors[msix_idx].vector, block); + } + free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); } dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks), priv->ntfy_blocks, priv->ntfy_block_bus); priv->ntfy_blocks = NULL; - free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); pci_disable_msix(priv->pdev); kvfree(priv->msix_vectors); priv->msix_vectors = NULL; @@ -343,6 +415,22 @@ static int gve_setup_device_resources(struct gve_priv *priv) err = -ENXIO; goto abort_with_stats_report; } + + if (priv->queue_format == GVE_DQO_RDA_FORMAT) { + priv->ptype_lut_dqo = kvzalloc(sizeof(*priv->ptype_lut_dqo), + GFP_KERNEL); + if (!priv->ptype_lut_dqo) { + err = -ENOMEM; + goto abort_with_stats_report; + } + err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo); + if (err) { + dev_err(&priv->pdev->dev, + "Failed to get ptype map: err=%d\n", err); + goto abort_with_ptype_lut; + } + } + err = gve_adminq_report_stats(priv, priv->stats_report_len, priv->stats_report_bus, GVE_STATS_REPORT_TIMER_PERIOD); @@ -351,12 +439,17 @@ static int gve_setup_device_resources(struct gve_priv *priv) "Failed to report stats: err=%d\n", err); gve_set_device_resources_ok(priv); return 0; + +abort_with_ptype_lut: + kvfree(priv->ptype_lut_dqo); + priv->ptype_lut_dqo = NULL; abort_with_stats_report: gve_free_stats_report(priv); abort_with_ntfy_blocks: gve_free_notify_blocks(priv); abort_with_counter: gve_free_counter_array(priv); + return err; } @@ -383,17 +476,22 @@ static void gve_teardown_device_resources(struct gve_priv *priv) gve_trigger_reset(priv); } } + + kvfree(priv->ptype_lut_dqo); + priv->ptype_lut_dqo = NULL; + gve_free_counter_array(priv); gve_free_notify_blocks(priv); gve_free_stats_report(priv); gve_clear_device_resources_ok(priv); } -static void gve_add_napi(struct gve_priv *priv, int ntfy_idx) +static void gve_add_napi(struct gve_priv *priv, int ntfy_idx, + int (*gve_poll)(struct napi_struct *, int)) { struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; - netif_napi_add(priv->dev, &block->napi, gve_napi_poll, + netif_napi_add(priv->dev, &block->napi, gve_poll, NAPI_POLL_WEIGHT); } @@ -473,31 +571,75 @@ static int gve_create_rings(struct gve_priv *priv) netif_dbg(priv, drv, priv->dev, "created %d rx queues\n", priv->rx_cfg.num_queues); - /* Rx data ring has been prefilled with packet buffers at queue - * allocation time. - * Write the doorbell to provide descriptor slots and packet buffers - * to the NIC. - */ - for (i = 0; i < priv->rx_cfg.num_queues; i++) - gve_rx_write_doorbell(priv, &priv->rx[i]); + if (gve_is_gqi(priv)) { + /* Rx data ring has been prefilled with packet buffers at queue + * allocation time. + * + * Write the doorbell to provide descriptor slots and packet + * buffers to the NIC. + */ + for (i = 0; i < priv->rx_cfg.num_queues; i++) + gve_rx_write_doorbell(priv, &priv->rx[i]); + } else { + for (i = 0; i < priv->rx_cfg.num_queues; i++) { + /* Post buffers and ring doorbell. */ + gve_rx_post_buffers_dqo(&priv->rx[i]); + } + } return 0; } +static void add_napi_init_sync_stats(struct gve_priv *priv, + int (*napi_poll)(struct napi_struct *napi, + int budget)) +{ + int i; + + /* Add tx napi & init sync stats*/ + for (i = 0; i < priv->tx_cfg.num_queues; i++) { + int ntfy_idx = gve_tx_idx_to_ntfy(priv, i); + + u64_stats_init(&priv->tx[i].statss); + priv->tx[i].ntfy_id = ntfy_idx; + gve_add_napi(priv, ntfy_idx, napi_poll); + } + /* Add rx napi & init sync stats*/ + for (i = 0; i < priv->rx_cfg.num_queues; i++) { + int ntfy_idx = gve_rx_idx_to_ntfy(priv, i); + + u64_stats_init(&priv->rx[i].statss); + priv->rx[i].ntfy_id = ntfy_idx; + gve_add_napi(priv, ntfy_idx, napi_poll); + } +} + +static void gve_tx_free_rings(struct gve_priv *priv) +{ + if (gve_is_gqi(priv)) { + gve_tx_free_rings_gqi(priv); + } else { + gve_tx_free_rings_dqo(priv); + } +} + static int gve_alloc_rings(struct gve_priv *priv) { - int ntfy_idx; int err; - int i; /* Setup tx rings */ priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx), GFP_KERNEL); if (!priv->tx) return -ENOMEM; - err = gve_tx_alloc_rings(priv); + + if (gve_is_gqi(priv)) + err = gve_tx_alloc_rings(priv); + else + err = gve_tx_alloc_rings_dqo(priv); if (err) goto free_tx; + /* Setup rx rings */ priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx), GFP_KERNEL); @@ -505,21 +647,18 @@ static int gve_alloc_rings(struct gve_priv *priv) err = -ENOMEM; goto free_tx_queue; } - err = gve_rx_alloc_rings(priv); + + if (gve_is_gqi(priv)) + err = gve_rx_alloc_rings(priv); + else + err = gve_rx_alloc_rings_dqo(priv); if (err) goto free_rx; - /* Add tx napi & init sync stats*/ - for (i = 0; i < priv->tx_cfg.num_queues; i++) { - u64_stats_init(&priv->tx[i].statss); - ntfy_idx = gve_tx_idx_to_ntfy(priv, i); - gve_add_napi(priv, ntfy_idx); - } - /* Add rx napi & init sync stats*/ - for (i = 0; i < priv->rx_cfg.num_queues; i++) { - u64_stats_init(&priv->rx[i].statss); - ntfy_idx = gve_rx_idx_to_ntfy(priv, i); - gve_add_napi(priv, ntfy_idx); - } + + if (gve_is_gqi(priv)) + add_napi_init_sync_stats(priv, gve_napi_poll); + else + add_napi_init_sync_stats(priv, gve_napi_poll_dqo); return 0; @@ -557,6 +696,14 @@ static int gve_destroy_rings(struct gve_priv *priv) return 0; } +static void gve_rx_free_rings(struct gve_priv *priv) +{ + if (gve_is_gqi(priv)) + gve_rx_free_rings_gqi(priv); + else + gve_rx_free_rings_dqo(priv); +} + static void gve_free_rings(struct gve_priv *priv) { int ntfy_idx; @@ -678,7 +825,7 @@ static int gve_alloc_qpls(struct gve_priv *priv) int err; /* Raw addressing means no QPLs */ - if (priv->raw_addressing) + if (priv->queue_format == GVE_GQI_RDA_FORMAT) return 0; priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL); @@ -722,7 +869,7 @@ static void gve_free_qpls(struct gve_priv *priv) int i; /* Raw addressing means no QPLs */ - if (priv->raw_addressing) + if (priv->queue_format == GVE_GQI_RDA_FORMAT) return; kvfree(priv->qpl_cfg.qpl_id_map); @@ -756,6 +903,7 @@ static int gve_open(struct net_device *dev) err = gve_alloc_qpls(priv); if (err) return err; + err = gve_alloc_rings(priv); if (err) goto free_qpls; @@ -770,9 +918,17 @@ static int gve_open(struct net_device *dev) err = gve_register_qpls(priv); if (err) goto reset; + + if (!gve_is_gqi(priv)) { + /* Hard code this for now. This may be tuned in the future for + * performance. + */ + priv->data_buffer_size_dqo = GVE_RX_BUFFER_SIZE_DQO; + } err = gve_create_rings(priv); if (err) goto reset; + gve_set_device_rings_ok(priv); if (gve_get_report_stats(priv)) @@ -921,14 +1077,26 @@ static void gve_turnup(struct gve_priv *priv) struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; napi_enable(&block->napi); - iowrite32be(0, gve_irq_doorbell(priv, block)); + if (gve_is_gqi(priv)) { + iowrite32be(0, gve_irq_doorbell(priv, block)); + } else { + u32 val = gve_set_itr_ratelimit_dqo(GVE_TX_IRQ_RATELIMIT_US_DQO); + + gve_write_irq_doorbell_dqo(priv, block, val); + } } for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; napi_enable(&block->napi); - iowrite32be(0, gve_irq_doorbell(priv, block)); + if (gve_is_gqi(priv)) { + iowrite32be(0, gve_irq_doorbell(priv, block)); + } else { + u32 val = gve_set_itr_ratelimit_dqo(GVE_RX_IRQ_RATELIMIT_US_DQO); + + gve_write_irq_doorbell_dqo(priv, block, val); + } } gve_set_napi_enabled(priv); @@ -942,12 +1110,49 @@ static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue) priv->tx_timeo_cnt++; } +static int gve_set_features(struct net_device *netdev, + netdev_features_t features) +{ + const netdev_features_t orig_features = netdev->features; + struct gve_priv *priv = netdev_priv(netdev); + int err; + + if ((netdev->features & NETIF_F_LRO) != (features & NETIF_F_LRO)) { + netdev->features ^= NETIF_F_LRO; + if (netif_carrier_ok(netdev)) { + /* To make this process as simple as possible we + * teardown the device, set the new configuration, + * and then bring the device up again. + */ + err = gve_close(netdev); + /* We have already tried to reset in close, just fail + * at this point. + */ + if (err) + goto err; + + err = gve_open(netdev); + if (err) + goto err; + } + } + + return 0; +err: + /* Reverts the change on error. */ + netdev->features = orig_features; + netif_err(priv, drv, netdev, + "Set features failed! !!! DISABLING ALL QUEUES !!!\n"); + return err; +} + static const struct net_device_ops gve_netdev_ops = { - .ndo_start_xmit = gve_tx, + .ndo_start_xmit = gve_start_xmit, .ndo_open = gve_open, .ndo_stop = gve_close, .ndo_get_stats64 = gve_get_stats, .ndo_tx_timeout = gve_tx_timeout, + .ndo_set_features = gve_set_features, }; static void gve_handle_status(struct gve_priv *priv, u32 status) @@ -991,6 +1196,15 @@ void gve_handle_report_stats(struct gve_priv *priv) /* tx stats */ if (priv->tx) { for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { + u32 last_completion = 0; + u32 tx_frames = 0; + + /* DQO doesn't currently support these metrics. */ + if (gve_is_gqi(priv)) { + last_completion = priv->tx[idx].done; + tx_frames = priv->tx[idx].req; + } + do { start = u64_stats_fetch_begin(&priv->tx[idx].statss); tx_bytes = priv->tx[idx].bytes_done; @@ -1007,7 +1221,7 @@ void gve_handle_report_stats(struct gve_priv *priv) }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_FRAMES_SENT), - .value = cpu_to_be64(priv->tx[idx].req), + .value = cpu_to_be64(tx_frames), .queue_id = cpu_to_be32(idx), }; stats[stats_idx++] = (struct stats) { @@ -1017,7 +1231,7 @@ void gve_handle_report_stats(struct gve_priv *priv) }; stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_LAST_COMPLETION_PROCESSED), - .value = cpu_to_be64(priv->tx[idx].done), + .value = cpu_to_be64(last_completion), .queue_id = cpu_to_be32(idx), }; } @@ -1085,7 +1299,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) if (skip_describe_device) goto setup_device; - priv->raw_addressing = false; + priv->queue_format = GVE_QUEUE_FORMAT_UNSPECIFIED; /* Get the initial information we need from the device */ err = gve_adminq_describe_device(priv); if (err) { @@ -1093,7 +1307,7 @@ static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) "Could not get device information: err=%d\n", err); goto err; } - if (priv->dev->max_mtu > PAGE_SIZE) { + if (gve_is_gqi(priv) && priv->dev->max_mtu > PAGE_SIZE) { priv->dev->max_mtu = PAGE_SIZE; err = gve_adminq_set_mtu(priv, priv->dev->mtu); if (err) { @@ -1292,8 +1506,8 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) gve_write_version(®_bar->driver_version); /* Get max queues to alloc etherdev */ - max_rx_queues = ioread32be(®_bar->max_tx_queues); - max_tx_queues = ioread32be(®_bar->max_rx_queues); + max_tx_queues = ioread32be(®_bar->max_tx_queues); + max_rx_queues = ioread32be(®_bar->max_rx_queues); /* Alloc and setup the netdev and priv */ dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues); if (!dev) { @@ -1304,7 +1518,12 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, dev); dev->ethtool_ops = &gve_ethtool_ops; dev->netdev_ops = &gve_netdev_ops; - /* advertise features */ + + /* Set default and supported features. + * + * Features might be set in other locations as well (such as + * `gve_adminq_describe_device`). + */ dev->hw_features = NETIF_F_HIGHDMA; dev->hw_features |= NETIF_F_SG; dev->hw_features |= NETIF_F_HW_CSUM; @@ -1349,6 +1568,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto abort_with_wq; dev_info(&pdev->dev, "GVE version %s\n", gve_version_str); + dev_info(&pdev->dev, "GVE queue format %d\n", (int)priv->queue_format); gve_clear_probe_in_progress(priv); queue_work(priv->gve_wq, &priv->service_task); return 0; diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c index bf123fe524c4..bb8261368250 100644 --- a/drivers/net/ethernet/google/gve/gve_rx.c +++ b/drivers/net/ethernet/google/gve/gve_rx.c @@ -1,21 +1,14 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #include "gve.h" #include "gve_adminq.h" +#include "gve_utils.h" #include <linux/etherdevice.h> -static void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx) -{ - struct gve_notify_block *block = - &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)]; - - block->rx = NULL; -} - static void gve_rx_free_buffer(struct device *dev, struct gve_rx_slot_page_info *page_info, union gve_rx_data_slot *data_slot) @@ -137,16 +130,6 @@ alloc_err: return err; } -static void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx) -{ - u32 ntfy_idx = gve_rx_idx_to_ntfy(priv, queue_idx); - struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; - struct gve_rx_ring *rx = &priv->rx[queue_idx]; - - block->rx = rx; - rx->ntfy_id = ntfy_idx; -} - static int gve_rx_alloc_ring(struct gve_priv *priv, int idx) { struct gve_rx_ring *rx = &priv->rx[idx]; @@ -165,7 +148,7 @@ static int gve_rx_alloc_ring(struct gve_priv *priv, int idx) slots = priv->rx_data_slot_cnt; rx->mask = slots - 1; - rx->data.raw_addressing = priv->raw_addressing; + rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT; /* alloc rx data ring */ bytes = sizeof(*rx->data.data_ring) * slots; @@ -255,7 +238,7 @@ int gve_rx_alloc_rings(struct gve_priv *priv) return err; } -void gve_rx_free_rings(struct gve_priv *priv) +void gve_rx_free_rings_gqi(struct gve_priv *priv) { int i; @@ -279,27 +262,6 @@ static enum pkt_hash_types gve_rss_type(__be16 pkt_flags) return PKT_HASH_TYPE_L2; } -static struct sk_buff *gve_rx_copy(struct net_device *dev, - struct napi_struct *napi, - struct gve_rx_slot_page_info *page_info, - u16 len) -{ - struct sk_buff *skb = napi_alloc_skb(napi, len); - void *va = page_info->page_address + GVE_RX_PAD + - (page_info->page_offset ? PAGE_SIZE / 2 : 0); - - if (unlikely(!skb)) - return NULL; - - __skb_put(skb, len); - - skb_copy_to_linear_data(skb, va, len); - - skb->protocol = eth_type_trans(skb, dev); - - return skb; -} - static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi, struct gve_rx_slot_page_info *page_info, u16 len) @@ -310,7 +272,7 @@ static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi, return NULL; skb_add_rx_frag(skb, 0, page_info->page, - (page_info->page_offset ? PAGE_SIZE / 2 : 0) + + page_info->page_offset + GVE_RX_PAD, len, PAGE_SIZE / 2); return skb; @@ -321,7 +283,7 @@ static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *sl const __be64 offset = cpu_to_be64(PAGE_SIZE / 2); /* "flip" to other packet buffer on this page */ - page_info->page_offset ^= 0x1; + page_info->page_offset ^= PAGE_SIZE / 2; *(slot_addr) ^= offset; } @@ -388,7 +350,7 @@ gve_rx_qpl(struct device *dev, struct net_device *netdev, gve_rx_flip_buff(page_info, &data_slot->qpl_offset); } } else { - skb = gve_rx_copy(netdev, napi, page_info, len); + skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD); if (skb) { u64_stats_update_begin(&rx->statss); rx->rx_copied_pkt++; @@ -430,7 +392,7 @@ static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc, if (len <= priv->rx_copybreak) { /* Just copy small packets */ - skb = gve_rx_copy(dev, napi, page_info, len); + skb = gve_rx_copy(dev, napi, page_info, len, GVE_RX_PAD); u64_stats_update_begin(&rx->statss); rx->rx_copied_pkt++; rx->rx_copybreak_pkt++; diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c new file mode 100644 index 000000000000..77bb8227f89b --- /dev/null +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -0,0 +1,763 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Google virtual Ethernet (gve) driver + * + * Copyright (C) 2015-2021 Google, Inc. + */ + +#include "gve.h" +#include "gve_dqo.h" +#include "gve_adminq.h" +#include "gve_utils.h" +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/skbuff.h> +#include <linux/slab.h> +#include <net/ip6_checksum.h> +#include <net/ipv6.h> +#include <net/tcp.h> + +static int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs) +{ + return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias; +} + +static void gve_free_page_dqo(struct gve_priv *priv, + struct gve_rx_buf_state_dqo *bs) +{ + page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1); + gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr, + DMA_FROM_DEVICE); + bs->page_info.page = NULL; +} + +static struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx) +{ + struct gve_rx_buf_state_dqo *buf_state; + s16 buffer_id; + + buffer_id = rx->dqo.free_buf_states; + if (unlikely(buffer_id == -1)) + return NULL; + + buf_state = &rx->dqo.buf_states[buffer_id]; + + /* Remove buf_state from free list */ + rx->dqo.free_buf_states = buf_state->next; + + /* Point buf_state to itself to mark it as allocated */ + buf_state->next = buffer_id; + + return buf_state; +} + +static bool gve_buf_state_is_allocated(struct gve_rx_ring *rx, + struct gve_rx_buf_state_dqo *buf_state) +{ + s16 buffer_id = buf_state - rx->dqo.buf_states; + + return buf_state->next == buffer_id; +} + +static void gve_free_buf_state(struct gve_rx_ring *rx, + struct gve_rx_buf_state_dqo *buf_state) +{ + s16 buffer_id = buf_state - rx->dqo.buf_states; + + buf_state->next = rx->dqo.free_buf_states; + rx->dqo.free_buf_states = buffer_id; +} + +static struct gve_rx_buf_state_dqo * +gve_dequeue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list) +{ + struct gve_rx_buf_state_dqo *buf_state; + s16 buffer_id; + + buffer_id = list->head; + if (unlikely(buffer_id == -1)) + return NULL; + + buf_state = &rx->dqo.buf_states[buffer_id]; + + /* Remove buf_state from list */ + list->head = buf_state->next; + if (buf_state->next == -1) + list->tail = -1; + + /* Point buf_state to itself to mark it as allocated */ + buf_state->next = buffer_id; + + return buf_state; +} + +static void gve_enqueue_buf_state(struct gve_rx_ring *rx, + struct gve_index_list *list, + struct gve_rx_buf_state_dqo *buf_state) +{ + s16 buffer_id = buf_state - rx->dqo.buf_states; + + buf_state->next = -1; + + if (list->head == -1) { + list->head = buffer_id; + list->tail = buffer_id; + } else { + int tail = list->tail; + + rx->dqo.buf_states[tail].next = buffer_id; + list->tail = buffer_id; + } +} + +static struct gve_rx_buf_state_dqo * +gve_get_recycled_buf_state(struct gve_rx_ring *rx) +{ + struct gve_rx_buf_state_dqo *buf_state; + int i; + + /* Recycled buf states are immediately usable. */ + buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states); + if (likely(buf_state)) + return buf_state; + + if (unlikely(rx->dqo.used_buf_states.head == -1)) + return NULL; + + /* Used buf states are only usable when ref count reaches 0, which means + * no SKBs refer to them. + * + * Search a limited number before giving up. + */ + for (i = 0; i < 5; i++) { + buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); + if (gve_buf_ref_cnt(buf_state) == 0) + return buf_state; + + gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); + } + + /* If there are no free buf states discard an entry from + * `used_buf_states` so it can be used. + */ + if (unlikely(rx->dqo.free_buf_states == -1)) { + buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); + if (gve_buf_ref_cnt(buf_state) == 0) + return buf_state; + + gve_free_page_dqo(rx->gve, buf_state); + gve_free_buf_state(rx, buf_state); + } + + return NULL; +} + +static int gve_alloc_page_dqo(struct gve_priv *priv, + struct gve_rx_buf_state_dqo *buf_state) +{ + int err; + + err = gve_alloc_page(priv, &priv->pdev->dev, &buf_state->page_info.page, + &buf_state->addr, DMA_FROM_DEVICE); + if (err) + return err; + + buf_state->page_info.page_offset = 0; + buf_state->page_info.page_address = + page_address(buf_state->page_info.page); + buf_state->last_single_ref_offset = 0; + + /* The page already has 1 ref. */ + page_ref_add(buf_state->page_info.page, INT_MAX - 1); + buf_state->page_info.pagecnt_bias = INT_MAX; + + return 0; +} + +static void gve_rx_free_ring_dqo(struct gve_priv *priv, int idx) +{ + struct gve_rx_ring *rx = &priv->rx[idx]; + struct device *hdev = &priv->pdev->dev; + size_t completion_queue_slots; + size_t buffer_queue_slots; + size_t size; + int i; + + completion_queue_slots = rx->dqo.complq.mask + 1; + buffer_queue_slots = rx->dqo.bufq.mask + 1; + + gve_rx_remove_from_block(priv, idx); + + if (rx->q_resources) { + dma_free_coherent(hdev, sizeof(*rx->q_resources), + rx->q_resources, rx->q_resources_bus); + rx->q_resources = NULL; + } + + for (i = 0; i < rx->dqo.num_buf_states; i++) { + struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i]; + + if (bs->page_info.page) + gve_free_page_dqo(priv, bs); + } + + if (rx->dqo.bufq.desc_ring) { + size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots; + dma_free_coherent(hdev, size, rx->dqo.bufq.desc_ring, + rx->dqo.bufq.bus); + rx->dqo.bufq.desc_ring = NULL; + } + + if (rx->dqo.complq.desc_ring) { + size = sizeof(rx->dqo.complq.desc_ring[0]) * + completion_queue_slots; + dma_free_coherent(hdev, size, rx->dqo.complq.desc_ring, + rx->dqo.complq.bus); + rx->dqo.complq.desc_ring = NULL; + } + + kvfree(rx->dqo.buf_states); + rx->dqo.buf_states = NULL; + + netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx); +} + +static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx) +{ + struct gve_rx_ring *rx = &priv->rx[idx]; + struct device *hdev = &priv->pdev->dev; + size_t size; + int i; + + const u32 buffer_queue_slots = + priv->options_dqo_rda.rx_buff_ring_entries; + const u32 completion_queue_slots = priv->rx_desc_cnt; + + netif_dbg(priv, drv, priv->dev, "allocating rx ring DQO\n"); + + memset(rx, 0, sizeof(*rx)); + rx->gve = priv; + rx->q_num = idx; + rx->dqo.bufq.mask = buffer_queue_slots - 1; + rx->dqo.complq.num_free_slots = completion_queue_slots; + rx->dqo.complq.mask = completion_queue_slots - 1; + rx->skb_head = NULL; + rx->skb_tail = NULL; + + rx->dqo.num_buf_states = min_t(s16, S16_MAX, buffer_queue_slots * 4); + rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states, + sizeof(rx->dqo.buf_states[0]), + GFP_KERNEL); + if (!rx->dqo.buf_states) + return -ENOMEM; + + /* Set up linked list of buffer IDs */ + for (i = 0; i < rx->dqo.num_buf_states - 1; i++) + rx->dqo.buf_states[i].next = i + 1; + + rx->dqo.buf_states[rx->dqo.num_buf_states - 1].next = -1; + rx->dqo.recycled_buf_states.head = -1; + rx->dqo.recycled_buf_states.tail = -1; + rx->dqo.used_buf_states.head = -1; + rx->dqo.used_buf_states.tail = -1; + + /* Allocate RX completion queue */ + size = sizeof(rx->dqo.complq.desc_ring[0]) * + completion_queue_slots; + rx->dqo.complq.desc_ring = + dma_alloc_coherent(hdev, size, &rx->dqo.complq.bus, GFP_KERNEL); + if (!rx->dqo.complq.desc_ring) + goto err; + + /* Allocate RX buffer queue */ + size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots; + rx->dqo.bufq.desc_ring = + dma_alloc_coherent(hdev, size, &rx->dqo.bufq.bus, GFP_KERNEL); + if (!rx->dqo.bufq.desc_ring) + goto err; + + rx->q_resources = dma_alloc_coherent(hdev, sizeof(*rx->q_resources), + &rx->q_resources_bus, GFP_KERNEL); + if (!rx->q_resources) + goto err; + + gve_rx_add_to_block(priv, idx); + + return 0; + +err: + gve_rx_free_ring_dqo(priv, idx); + return -ENOMEM; +} + +void gve_rx_write_doorbell_dqo(const struct gve_priv *priv, int queue_idx) +{ + const struct gve_rx_ring *rx = &priv->rx[queue_idx]; + u64 index = be32_to_cpu(rx->q_resources->db_index); + + iowrite32(rx->dqo.bufq.tail, &priv->db_bar2[index]); +} + +int gve_rx_alloc_rings_dqo(struct gve_priv *priv) +{ + int err = 0; + int i; + + for (i = 0; i < priv->rx_cfg.num_queues; i++) { + err = gve_rx_alloc_ring_dqo(priv, i); + if (err) { + netif_err(priv, drv, priv->dev, + "Failed to alloc rx ring=%d: err=%d\n", + i, err); + goto err; + } + } + + return 0; + +err: + for (i--; i >= 0; i--) + gve_rx_free_ring_dqo(priv, i); + + return err; +} + +void gve_rx_free_rings_dqo(struct gve_priv *priv) +{ + int i; + + for (i = 0; i < priv->rx_cfg.num_queues; i++) + gve_rx_free_ring_dqo(priv, i); +} + +void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx) +{ + struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq; + struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq; + struct gve_priv *priv = rx->gve; + u32 num_avail_slots; + u32 num_full_slots; + u32 num_posted = 0; + + num_full_slots = (bufq->tail - bufq->head) & bufq->mask; + num_avail_slots = bufq->mask - num_full_slots; + + num_avail_slots = min_t(u32, num_avail_slots, complq->num_free_slots); + while (num_posted < num_avail_slots) { + struct gve_rx_desc_dqo *desc = &bufq->desc_ring[bufq->tail]; + struct gve_rx_buf_state_dqo *buf_state; + + buf_state = gve_get_recycled_buf_state(rx); + if (unlikely(!buf_state)) { + buf_state = gve_alloc_buf_state(rx); + if (unlikely(!buf_state)) + break; + + if (unlikely(gve_alloc_page_dqo(priv, buf_state))) { + u64_stats_update_begin(&rx->statss); + rx->rx_buf_alloc_fail++; + u64_stats_update_end(&rx->statss); + gve_free_buf_state(rx, buf_state); + break; + } + } + + desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states); + desc->buf_addr = cpu_to_le64(buf_state->addr + + buf_state->page_info.page_offset); + + bufq->tail = (bufq->tail + 1) & bufq->mask; + complq->num_free_slots--; + num_posted++; + + if ((bufq->tail & (GVE_RX_BUF_THRESH_DQO - 1)) == 0) + gve_rx_write_doorbell_dqo(priv, rx->q_num); + } + + rx->fill_cnt += num_posted; +} + +static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx, + struct gve_rx_buf_state_dqo *buf_state) +{ + const int data_buffer_size = priv->data_buffer_size_dqo; + int pagecount; + + /* Can't reuse if we only fit one buffer per page */ + if (data_buffer_size * 2 > PAGE_SIZE) + goto mark_used; + + pagecount = gve_buf_ref_cnt(buf_state); + + /* Record the offset when we have a single remaining reference. + * + * When this happens, we know all of the other offsets of the page are + * usable. + */ + if (pagecount == 1) { + buf_state->last_single_ref_offset = + buf_state->page_info.page_offset; + } + + /* Use the next buffer sized chunk in the page. */ + buf_state->page_info.page_offset += data_buffer_size; + buf_state->page_info.page_offset &= (PAGE_SIZE - 1); + + /* If we wrap around to the same offset without ever dropping to 1 + * reference, then we don't know if this offset was ever freed. + */ + if (buf_state->page_info.page_offset == + buf_state->last_single_ref_offset) { + goto mark_used; + } + + gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state); + return; + +mark_used: + gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); +} + +static void gve_rx_skb_csum(struct sk_buff *skb, + const struct gve_rx_compl_desc_dqo *desc, + struct gve_ptype ptype) +{ + skb->ip_summed = CHECKSUM_NONE; + + /* HW did not identify and process L3 and L4 headers. */ + if (unlikely(!desc->l3_l4_processed)) + return; + + if (ptype.l3_type == GVE_L3_TYPE_IPV4) { + if (unlikely(desc->csum_ip_err || desc->csum_external_ip_err)) + return; + } else if (ptype.l3_type == GVE_L3_TYPE_IPV6) { + /* Checksum should be skipped if this flag is set. */ + if (unlikely(desc->ipv6_ex_add)) + return; + } + + if (unlikely(desc->csum_l4_err)) + return; + + switch (ptype.l4_type) { + case GVE_L4_TYPE_TCP: + case GVE_L4_TYPE_UDP: + case GVE_L4_TYPE_ICMP: + case GVE_L4_TYPE_SCTP: + skb->ip_summed = CHECKSUM_UNNECESSARY; + break; + default: + break; + } +} + +static void gve_rx_skb_hash(struct sk_buff *skb, + const struct gve_rx_compl_desc_dqo *compl_desc, + struct gve_ptype ptype) +{ + enum pkt_hash_types hash_type = PKT_HASH_TYPE_L2; + + if (ptype.l4_type != GVE_L4_TYPE_UNKNOWN) + hash_type = PKT_HASH_TYPE_L4; + else if (ptype.l3_type != GVE_L3_TYPE_UNKNOWN) + hash_type = PKT_HASH_TYPE_L3; + + skb_set_hash(skb, le32_to_cpu(compl_desc->hash), hash_type); +} + +static void gve_rx_free_skb(struct gve_rx_ring *rx) +{ + if (!rx->skb_head) + return; + + dev_kfree_skb_any(rx->skb_head); + rx->skb_head = NULL; + rx->skb_tail = NULL; +} + +/* Chains multi skbs for single rx packet. + * Returns 0 if buffer is appended, -1 otherwise. + */ +static int gve_rx_append_frags(struct napi_struct *napi, + struct gve_rx_buf_state_dqo *buf_state, + u16 buf_len, struct gve_rx_ring *rx, + struct gve_priv *priv) +{ + int num_frags = skb_shinfo(rx->skb_tail)->nr_frags; + + if (unlikely(num_frags == MAX_SKB_FRAGS)) { + struct sk_buff *skb; + + skb = napi_alloc_skb(napi, 0); + if (!skb) + return -1; + + skb_shinfo(rx->skb_tail)->frag_list = skb; + rx->skb_tail = skb; + num_frags = 0; + } + if (rx->skb_tail != rx->skb_head) { + rx->skb_head->len += buf_len; + rx->skb_head->data_len += buf_len; + rx->skb_head->truesize += priv->data_buffer_size_dqo; + } + + skb_add_rx_frag(rx->skb_tail, num_frags, + buf_state->page_info.page, + buf_state->page_info.page_offset, + buf_len, priv->data_buffer_size_dqo); + gve_dec_pagecnt_bias(&buf_state->page_info); + + return 0; +} + +/* Returns 0 if descriptor is completed successfully. + * Returns -EINVAL if descriptor is invalid. + * Returns -ENOMEM if data cannot be copied to skb. + */ +static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, + const struct gve_rx_compl_desc_dqo *compl_desc, + int queue_idx) +{ + const u16 buffer_id = le16_to_cpu(compl_desc->buf_id); + const bool eop = compl_desc->end_of_packet != 0; + struct gve_rx_buf_state_dqo *buf_state; + struct gve_priv *priv = rx->gve; + u16 buf_len; + + if (unlikely(buffer_id >= rx->dqo.num_buf_states)) { + net_err_ratelimited("%s: Invalid RX buffer_id=%u\n", + priv->dev->name, buffer_id); + return -EINVAL; + } + buf_state = &rx->dqo.buf_states[buffer_id]; + if (unlikely(!gve_buf_state_is_allocated(rx, buf_state))) { + net_err_ratelimited("%s: RX buffer_id is not allocated: %u\n", + priv->dev->name, buffer_id); + return -EINVAL; + } + + if (unlikely(compl_desc->rx_error)) { + gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, + buf_state); + return -EINVAL; + } + + buf_len = compl_desc->packet_len; + + /* Page might have not been used for awhile and was likely last written + * by a different thread. + */ + prefetch(buf_state->page_info.page); + + /* Sync the portion of dma buffer for CPU to read. */ + dma_sync_single_range_for_cpu(&priv->pdev->dev, buf_state->addr, + buf_state->page_info.page_offset, + buf_len, DMA_FROM_DEVICE); + + /* Append to current skb if one exists. */ + if (rx->skb_head) { + if (unlikely(gve_rx_append_frags(napi, buf_state, buf_len, rx, + priv)) != 0) { + goto error; + } + + gve_try_recycle_buf(priv, rx, buf_state); + return 0; + } + + /* Prefetch the payload header. */ + prefetch((char *)buf_state->addr + buf_state->page_info.page_offset); +#if L1_CACHE_BYTES < 128 + prefetch((char *)buf_state->addr + buf_state->page_info.page_offset + + L1_CACHE_BYTES); +#endif + + if (eop && buf_len <= priv->rx_copybreak) { + rx->skb_head = gve_rx_copy(priv->dev, napi, + &buf_state->page_info, buf_len, 0); + if (unlikely(!rx->skb_head)) + goto error; + rx->skb_tail = rx->skb_head; + + u64_stats_update_begin(&rx->statss); + rx->rx_copied_pkt++; + rx->rx_copybreak_pkt++; + u64_stats_update_end(&rx->statss); + + gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, + buf_state); + return 0; + } + + rx->skb_head = napi_get_frags(napi); + if (unlikely(!rx->skb_head)) + goto error; + rx->skb_tail = rx->skb_head; + + skb_add_rx_frag(rx->skb_head, 0, buf_state->page_info.page, + buf_state->page_info.page_offset, buf_len, + priv->data_buffer_size_dqo); + gve_dec_pagecnt_bias(&buf_state->page_info); + + gve_try_recycle_buf(priv, rx, buf_state); + return 0; + +error: + gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state); + return -ENOMEM; +} + +static int gve_rx_complete_rsc(struct sk_buff *skb, + const struct gve_rx_compl_desc_dqo *desc, + struct gve_ptype ptype) +{ + struct skb_shared_info *shinfo = skb_shinfo(skb); + + /* Only TCP is supported right now. */ + if (ptype.l4_type != GVE_L4_TYPE_TCP) + return -EINVAL; + + switch (ptype.l3_type) { + case GVE_L3_TYPE_IPV4: + shinfo->gso_type = SKB_GSO_TCPV4; + break; + case GVE_L3_TYPE_IPV6: + shinfo->gso_type = SKB_GSO_TCPV6; + break; + default: + return -EINVAL; + } + + shinfo->gso_size = le16_to_cpu(desc->rsc_seg_len); + return 0; +} + +/* Returns 0 if skb is completed successfully, -1 otherwise. */ +static int gve_rx_complete_skb(struct gve_rx_ring *rx, struct napi_struct *napi, + const struct gve_rx_compl_desc_dqo *desc, + netdev_features_t feat) +{ + struct gve_ptype ptype = + rx->gve->ptype_lut_dqo->ptypes[desc->packet_type]; + int err; + + skb_record_rx_queue(rx->skb_head, rx->q_num); + + if (feat & NETIF_F_RXHASH) + gve_rx_skb_hash(rx->skb_head, desc, ptype); + + if (feat & NETIF_F_RXCSUM) + gve_rx_skb_csum(rx->skb_head, desc, ptype); + + /* RSC packets must set gso_size otherwise the TCP stack will complain + * that packets are larger than MTU. + */ + if (desc->rsc) { + err = gve_rx_complete_rsc(rx->skb_head, desc, ptype); + if (err < 0) + return err; + } + + if (skb_headlen(rx->skb_head) == 0) + napi_gro_frags(napi); + else + napi_gro_receive(napi, rx->skb_head); + + return 0; +} + +int gve_rx_poll_dqo(struct gve_notify_block *block, int budget) +{ + struct napi_struct *napi = &block->napi; + netdev_features_t feat = napi->dev->features; + + struct gve_rx_ring *rx = block->rx; + struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq; + + u32 work_done = 0; + u64 bytes = 0; + int err; + + while (work_done < budget) { + struct gve_rx_compl_desc_dqo *compl_desc = + &complq->desc_ring[complq->head]; + u32 pkt_bytes; + + /* No more new packets */ + if (compl_desc->generation == complq->cur_gen_bit) + break; + + /* Prefetch the next two descriptors. */ + prefetch(&complq->desc_ring[(complq->head + 1) & complq->mask]); + prefetch(&complq->desc_ring[(complq->head + 2) & complq->mask]); + + /* Do not read data until we own the descriptor */ + dma_rmb(); + + err = gve_rx_dqo(napi, rx, compl_desc, rx->q_num); + if (err < 0) { + gve_rx_free_skb(rx); + u64_stats_update_begin(&rx->statss); + if (err == -ENOMEM) + rx->rx_skb_alloc_fail++; + else if (err == -EINVAL) + rx->rx_desc_err_dropped_pkt++; + u64_stats_update_end(&rx->statss); + } + + complq->head = (complq->head + 1) & complq->mask; + complq->num_free_slots++; + + /* When the ring wraps, the generation bit is flipped. */ + complq->cur_gen_bit ^= (complq->head == 0); + + /* Receiving a completion means we have space to post another + * buffer on the buffer queue. + */ + { + struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq; + + bufq->head = (bufq->head + 1) & bufq->mask; + } + + /* Free running counter of completed descriptors */ + rx->cnt++; + + if (!rx->skb_head) + continue; + + if (!compl_desc->end_of_packet) + continue; + + work_done++; + pkt_bytes = rx->skb_head->len; + /* The ethernet header (first ETH_HLEN bytes) is snipped off + * by eth_type_trans. + */ + if (skb_headlen(rx->skb_head)) + pkt_bytes += ETH_HLEN; + + /* gve_rx_complete_skb() will consume skb if successful */ + if (gve_rx_complete_skb(rx, napi, compl_desc, feat) != 0) { + gve_rx_free_skb(rx); + u64_stats_update_begin(&rx->statss); + rx->rx_desc_err_dropped_pkt++; + u64_stats_update_end(&rx->statss); + continue; + } + + bytes += pkt_bytes; + rx->skb_head = NULL; + rx->skb_tail = NULL; + } + + gve_rx_post_buffers_dqo(rx); + + u64_stats_update_begin(&rx->statss); + rx->rpackets += work_done; + rx->rbytes += bytes; + u64_stats_update_end(&rx->statss); + + return work_done; +} diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c index 6938f3a939d6..665ac795a1ad 100644 --- a/drivers/net/ethernet/google/gve/gve_tx.c +++ b/drivers/net/ethernet/google/gve/gve_tx.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: (GPL-2.0 OR MIT) /* Google virtual Ethernet (gve) driver * - * Copyright (C) 2015-2019 Google, Inc. + * Copyright (C) 2015-2021 Google, Inc. */ #include "gve.h" #include "gve_adminq.h" +#include "gve_utils.h" #include <linux/ip.h> #include <linux/tcp.h> #include <linux/vmalloc.h> @@ -131,14 +132,6 @@ static void gve_tx_free_fifo(struct gve_tx_fifo *fifo, size_t bytes) atomic_add(bytes, &fifo->available); } -static void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx) -{ - struct gve_notify_block *block = - &priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)]; - - block->tx = NULL; -} - static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx, u32 to_do, bool try_to_wake); @@ -174,16 +167,6 @@ static void gve_tx_free_ring(struct gve_priv *priv, int idx) netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); } -static void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx) -{ - int ntfy_idx = gve_tx_idx_to_ntfy(priv, queue_idx); - struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; - struct gve_tx_ring *tx = &priv->tx[queue_idx]; - - block->tx = tx; - tx->ntfy_id = ntfy_idx; -} - static int gve_tx_alloc_ring(struct gve_priv *priv, int idx) { struct gve_tx_ring *tx = &priv->tx[idx]; @@ -208,14 +191,15 @@ static int gve_tx_alloc_ring(struct gve_priv *priv, int idx) if (!tx->desc) goto abort_with_info; - tx->raw_addressing = priv->raw_addressing; + tx->raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT; tx->dev = &priv->pdev->dev; if (!tx->raw_addressing) { tx->tx_fifo.qpl = gve_assign_tx_qpl(priv); - + if (!tx->tx_fifo.qpl) + goto abort_with_desc; /* map Tx FIFO */ if (gve_tx_fifo_init(priv, &tx->tx_fifo)) - goto abort_with_desc; + goto abort_with_qpl; } tx->q_resources = @@ -236,6 +220,9 @@ static int gve_tx_alloc_ring(struct gve_priv *priv, int idx) abort_with_fifo: if (!tx->raw_addressing) gve_tx_fifo_release(priv, &tx->tx_fifo); +abort_with_qpl: + if (!tx->raw_addressing) + gve_unassign_qpl(priv, tx->tx_fifo.qpl->id); abort_with_desc: dma_free_coherent(hdev, bytes, tx->desc, tx->bus); tx->desc = NULL; @@ -269,7 +256,7 @@ int gve_tx_alloc_rings(struct gve_priv *priv) return err; } -void gve_tx_free_rings(struct gve_priv *priv) +void gve_tx_free_rings_gqi(struct gve_priv *priv) { int i; @@ -589,7 +576,7 @@ netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev) struct gve_tx_ring *tx; int nsegs; - WARN(skb_get_queue_mapping(skb) > priv->tx_cfg.num_queues, + WARN(skb_get_queue_mapping(skb) >= priv->tx_cfg.num_queues, "skb queue index out of range"); tx = &priv->tx[skb_get_queue_mapping(skb)]; if (unlikely(gve_maybe_stop_tx(tx, skb))) { diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c new file mode 100644 index 000000000000..05ddb6a75c38 --- /dev/null +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -0,0 +1,1030 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Google virtual Ethernet (gve) driver + * + * Copyright (C) 2015-2021 Google, Inc. + */ + +#include "gve.h" +#include "gve_adminq.h" +#include "gve_utils.h" +#include "gve_dqo.h" +#include <linux/tcp.h> +#include <linux/slab.h> +#include <linux/skbuff.h> + +/* Returns true if a gve_tx_pending_packet_dqo object is available. */ +static bool gve_has_pending_packet(struct gve_tx_ring *tx) +{ + /* Check TX path's list. */ + if (tx->dqo_tx.free_pending_packets != -1) + return true; + + /* Check completion handler's list. */ + if (atomic_read_acquire(&tx->dqo_compl.free_pending_packets) != -1) + return true; + + return false; +} + +static struct gve_tx_pending_packet_dqo * +gve_alloc_pending_packet(struct gve_tx_ring *tx) +{ + struct gve_tx_pending_packet_dqo *pending_packet; + s16 index; + + index = tx->dqo_tx.free_pending_packets; + + /* No pending_packets available, try to steal the list from the + * completion handler. + */ + if (unlikely(index == -1)) { + tx->dqo_tx.free_pending_packets = + atomic_xchg(&tx->dqo_compl.free_pending_packets, -1); + index = tx->dqo_tx.free_pending_packets; + + if (unlikely(index == -1)) + return NULL; + } + + pending_packet = &tx->dqo.pending_packets[index]; + + /* Remove pending_packet from free list */ + tx->dqo_tx.free_pending_packets = pending_packet->next; + pending_packet->state = GVE_PACKET_STATE_PENDING_DATA_COMPL; + + return pending_packet; +} + +static void +gve_free_pending_packet(struct gve_tx_ring *tx, + struct gve_tx_pending_packet_dqo *pending_packet) +{ + s16 index = pending_packet - tx->dqo.pending_packets; + + pending_packet->state = GVE_PACKET_STATE_UNALLOCATED; + while (true) { + s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_pending_packets); + + pending_packet->next = old_head; + if (atomic_cmpxchg(&tx->dqo_compl.free_pending_packets, + old_head, index) == old_head) { + break; + } + } +} + +/* gve_tx_free_desc - Cleans up all pending tx requests and buffers. + */ +static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) +{ + int i; + + for (i = 0; i < tx->dqo.num_pending_packets; i++) { + struct gve_tx_pending_packet_dqo *cur_state = + &tx->dqo.pending_packets[i]; + int j; + + for (j = 0; j < cur_state->num_bufs; j++) { + struct gve_tx_dma_buf *buf = &cur_state->bufs[j]; + + if (j == 0) { + dma_unmap_single(tx->dev, + dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), + DMA_TO_DEVICE); + } else { + dma_unmap_page(tx->dev, + dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), + DMA_TO_DEVICE); + } + } + if (cur_state->skb) { + dev_consume_skb_any(cur_state->skb); + cur_state->skb = NULL; + } + } +} + +static void gve_tx_free_ring_dqo(struct gve_priv *priv, int idx) +{ + struct gve_tx_ring *tx = &priv->tx[idx]; + struct device *hdev = &priv->pdev->dev; + size_t bytes; + + gve_tx_remove_from_block(priv, idx); + + if (tx->q_resources) { + dma_free_coherent(hdev, sizeof(*tx->q_resources), + tx->q_resources, tx->q_resources_bus); + tx->q_resources = NULL; + } + + if (tx->dqo.compl_ring) { + bytes = sizeof(tx->dqo.compl_ring[0]) * + (tx->dqo.complq_mask + 1); + dma_free_coherent(hdev, bytes, tx->dqo.compl_ring, + tx->complq_bus_dqo); + tx->dqo.compl_ring = NULL; + } + + if (tx->dqo.tx_ring) { + bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); + dma_free_coherent(hdev, bytes, tx->dqo.tx_ring, tx->bus); + tx->dqo.tx_ring = NULL; + } + + kvfree(tx->dqo.pending_packets); + tx->dqo.pending_packets = NULL; + + netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); +} + +static int gve_tx_alloc_ring_dqo(struct gve_priv *priv, int idx) +{ + struct gve_tx_ring *tx = &priv->tx[idx]; + struct device *hdev = &priv->pdev->dev; + int num_pending_packets; + size_t bytes; + int i; + + memset(tx, 0, sizeof(*tx)); + tx->q_num = idx; + tx->dev = &priv->pdev->dev; + tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx); + atomic_set_release(&tx->dqo_compl.hw_tx_head, 0); + + /* Queue sizes must be a power of 2 */ + tx->mask = priv->tx_desc_cnt - 1; + tx->dqo.complq_mask = priv->options_dqo_rda.tx_comp_ring_entries - 1; + + /* The max number of pending packets determines the maximum number of + * descriptors which maybe written to the completion queue. + * + * We must set the number small enough to make sure we never overrun the + * completion queue. + */ + num_pending_packets = tx->dqo.complq_mask + 1; + + /* Reserve space for descriptor completions, which will be reported at + * most every GVE_TX_MIN_RE_INTERVAL packets. + */ + num_pending_packets -= + (tx->dqo.complq_mask + 1) / GVE_TX_MIN_RE_INTERVAL; + + /* Each packet may have at most 2 buffer completions if it receives both + * a miss and reinjection completion. + */ + num_pending_packets /= 2; + + tx->dqo.num_pending_packets = min_t(int, num_pending_packets, S16_MAX); + tx->dqo.pending_packets = kvcalloc(tx->dqo.num_pending_packets, + sizeof(tx->dqo.pending_packets[0]), + GFP_KERNEL); + if (!tx->dqo.pending_packets) + goto err; + + /* Set up linked list of pending packets */ + for (i = 0; i < tx->dqo.num_pending_packets - 1; i++) + tx->dqo.pending_packets[i].next = i + 1; + + tx->dqo.pending_packets[tx->dqo.num_pending_packets - 1].next = -1; + atomic_set_release(&tx->dqo_compl.free_pending_packets, -1); + tx->dqo_compl.miss_completions.head = -1; + tx->dqo_compl.miss_completions.tail = -1; + tx->dqo_compl.timed_out_completions.head = -1; + tx->dqo_compl.timed_out_completions.tail = -1; + + bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); + tx->dqo.tx_ring = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL); + if (!tx->dqo.tx_ring) + goto err; + + bytes = sizeof(tx->dqo.compl_ring[0]) * (tx->dqo.complq_mask + 1); + tx->dqo.compl_ring = dma_alloc_coherent(hdev, bytes, + &tx->complq_bus_dqo, + GFP_KERNEL); + if (!tx->dqo.compl_ring) + goto err; + + tx->q_resources = dma_alloc_coherent(hdev, sizeof(*tx->q_resources), + &tx->q_resources_bus, GFP_KERNEL); + if (!tx->q_resources) + goto err; + + gve_tx_add_to_block(priv, idx); + + return 0; + +err: + gve_tx_free_ring_dqo(priv, idx); + return -ENOMEM; +} + +int gve_tx_alloc_rings_dqo(struct gve_priv *priv) +{ + int err = 0; + int i; + + for (i = 0; i < priv->tx_cfg.num_queues; i++) { + err = gve_tx_alloc_ring_dqo(priv, i); + if (err) { + netif_err(priv, drv, priv->dev, + "Failed to alloc tx ring=%d: err=%d\n", + i, err); + goto err; + } + } + + return 0; + +err: + for (i--; i >= 0; i--) + gve_tx_free_ring_dqo(priv, i); + + return err; +} + +void gve_tx_free_rings_dqo(struct gve_priv *priv) +{ + int i; + + for (i = 0; i < priv->tx_cfg.num_queues; i++) { + struct gve_tx_ring *tx = &priv->tx[i]; + + gve_clean_tx_done_dqo(priv, tx, /*napi=*/NULL); + netdev_tx_reset_queue(tx->netdev_txq); + gve_tx_clean_pending_packets(tx); + + gve_tx_free_ring_dqo(priv, i); + } +} + +/* Returns the number of slots available in the ring */ +static u32 num_avail_tx_slots(const struct gve_tx_ring *tx) +{ + u32 num_used = (tx->dqo_tx.tail - tx->dqo_tx.head) & tx->mask; + + return tx->mask - num_used; +} + +/* Stops the queue if available descriptors is less than 'count'. + * Return: 0 if stop is not required. + */ +static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx, int count) +{ + if (likely(gve_has_pending_packet(tx) && + num_avail_tx_slots(tx) >= count)) + return 0; + + /* Update cached TX head pointer */ + tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); + + if (likely(gve_has_pending_packet(tx) && + num_avail_tx_slots(tx) >= count)) + return 0; + + /* No space, so stop the queue */ + tx->stop_queue++; + netif_tx_stop_queue(tx->netdev_txq); + + /* Sync with restarting queue in `gve_tx_poll_dqo()` */ + mb(); + + /* After stopping queue, check if we can transmit again in order to + * avoid TOCTOU bug. + */ + tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); + + if (likely(!gve_has_pending_packet(tx) || + num_avail_tx_slots(tx) < count)) + return -EBUSY; + + netif_tx_start_queue(tx->netdev_txq); + tx->wake_queue++; + return 0; +} + +static void gve_extract_tx_metadata_dqo(const struct sk_buff *skb, + struct gve_tx_metadata_dqo *metadata) +{ + memset(metadata, 0, sizeof(*metadata)); + metadata->version = GVE_TX_METADATA_VERSION_DQO; + + if (skb->l4_hash) { + u16 path_hash = skb->hash ^ (skb->hash >> 16); + + path_hash &= (1 << 15) - 1; + if (unlikely(path_hash == 0)) + path_hash = ~path_hash; + + metadata->path_hash = path_hash; + } +} + +static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx, + struct sk_buff *skb, u32 len, u64 addr, + s16 compl_tag, bool eop, bool is_gso) +{ + const bool checksum_offload_en = skb->ip_summed == CHECKSUM_PARTIAL; + + while (len > 0) { + struct gve_tx_pkt_desc_dqo *desc = + &tx->dqo.tx_ring[*desc_idx].pkt; + u32 cur_len = min_t(u32, len, GVE_TX_MAX_BUF_SIZE_DQO); + bool cur_eop = eop && cur_len == len; + + *desc = (struct gve_tx_pkt_desc_dqo){ + .buf_addr = cpu_to_le64(addr), + .dtype = GVE_TX_PKT_DESC_DTYPE_DQO, + .end_of_packet = cur_eop, + .checksum_offload_enable = checksum_offload_en, + .compl_tag = cpu_to_le16(compl_tag), + .buf_size = cur_len, + }; + + addr += cur_len; + len -= cur_len; + *desc_idx = (*desc_idx + 1) & tx->mask; + } +} + +/* Validates and prepares `skb` for TSO. + * + * Returns header length, or < 0 if invalid. + */ +static int gve_prep_tso(struct sk_buff *skb) +{ + struct tcphdr *tcp; + int header_len; + u32 paylen; + int err; + + /* Note: HW requires MSS (gso_size) to be <= 9728 and the total length + * of the TSO to be <= 262143. + * + * However, we don't validate these because: + * - Hypervisor enforces a limit of 9K MTU + * - Kernel will not produce a TSO larger than 64k + */ + + if (unlikely(skb_shinfo(skb)->gso_size < GVE_TX_MIN_TSO_MSS_DQO)) + return -1; + + /* Needed because we will modify header. */ + err = skb_cow_head(skb, 0); + if (err < 0) + return err; + + tcp = tcp_hdr(skb); + + /* Remove payload length from checksum. */ + paylen = skb->len - skb_transport_offset(skb); + + switch (skb_shinfo(skb)->gso_type) { + case SKB_GSO_TCPV4: + case SKB_GSO_TCPV6: + csum_replace_by_diff(&tcp->check, + (__force __wsum)htonl(paylen)); + + /* Compute length of segmentation header. */ + header_len = skb_transport_offset(skb) + tcp_hdrlen(skb); + break; + default: + return -EINVAL; + } + + if (unlikely(header_len > GVE_TX_MAX_HDR_SIZE_DQO)) + return -EINVAL; + + return header_len; +} + +static void gve_tx_fill_tso_ctx_desc(struct gve_tx_tso_context_desc_dqo *desc, + const struct sk_buff *skb, + const struct gve_tx_metadata_dqo *metadata, + int header_len) +{ + *desc = (struct gve_tx_tso_context_desc_dqo){ + .header_len = header_len, + .cmd_dtype = { + .dtype = GVE_TX_TSO_CTX_DESC_DTYPE_DQO, + .tso = 1, + }, + .flex0 = metadata->bytes[0], + .flex5 = metadata->bytes[5], + .flex6 = metadata->bytes[6], + .flex7 = metadata->bytes[7], + .flex8 = metadata->bytes[8], + .flex9 = metadata->bytes[9], + .flex10 = metadata->bytes[10], + .flex11 = metadata->bytes[11], + }; + desc->tso_total_len = skb->len - header_len; + desc->mss = skb_shinfo(skb)->gso_size; +} + +static void +gve_tx_fill_general_ctx_desc(struct gve_tx_general_context_desc_dqo *desc, + const struct gve_tx_metadata_dqo *metadata) +{ + *desc = (struct gve_tx_general_context_desc_dqo){ + .flex0 = metadata->bytes[0], + .flex1 = metadata->bytes[1], + .flex2 = metadata->bytes[2], + .flex3 = metadata->bytes[3], + .flex4 = metadata->bytes[4], + .flex5 = metadata->bytes[5], + .flex6 = metadata->bytes[6], + .flex7 = metadata->bytes[7], + .flex8 = metadata->bytes[8], + .flex9 = metadata->bytes[9], + .flex10 = metadata->bytes[10], + .flex11 = metadata->bytes[11], + .cmd_dtype = {.dtype = GVE_TX_GENERAL_CTX_DESC_DTYPE_DQO}, + }; +} + +/* Returns 0 on success, or < 0 on error. + * + * Before this function is called, the caller must ensure + * gve_has_pending_packet(tx) returns true. + */ +static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx, + struct sk_buff *skb) +{ + const struct skb_shared_info *shinfo = skb_shinfo(skb); + const bool is_gso = skb_is_gso(skb); + u32 desc_idx = tx->dqo_tx.tail; + + struct gve_tx_pending_packet_dqo *pending_packet; + struct gve_tx_metadata_dqo metadata; + s16 completion_tag; + int i; + + pending_packet = gve_alloc_pending_packet(tx); + pending_packet->skb = skb; + pending_packet->num_bufs = 0; + completion_tag = pending_packet - tx->dqo.pending_packets; + + gve_extract_tx_metadata_dqo(skb, &metadata); + if (is_gso) { + int header_len = gve_prep_tso(skb); + + if (unlikely(header_len < 0)) + goto err; + + gve_tx_fill_tso_ctx_desc(&tx->dqo.tx_ring[desc_idx].tso_ctx, + skb, &metadata, header_len); + desc_idx = (desc_idx + 1) & tx->mask; + } + + gve_tx_fill_general_ctx_desc(&tx->dqo.tx_ring[desc_idx].general_ctx, + &metadata); + desc_idx = (desc_idx + 1) & tx->mask; + + /* Note: HW requires that the size of a non-TSO packet be within the + * range of [17, 9728]. + * + * We don't double check because + * - We limited `netdev->min_mtu` to ETH_MIN_MTU. + * - Hypervisor won't allow MTU larger than 9216. + */ + + /* Map the linear portion of skb */ + { + struct gve_tx_dma_buf *buf = + &pending_packet->bufs[pending_packet->num_bufs]; + u32 len = skb_headlen(skb); + dma_addr_t addr; + + addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(tx->dev, addr))) + goto err; + + dma_unmap_len_set(buf, len, len); + dma_unmap_addr_set(buf, dma, addr); + ++pending_packet->num_bufs; + + gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr, + completion_tag, + /*eop=*/shinfo->nr_frags == 0, is_gso); + } + + for (i = 0; i < shinfo->nr_frags; i++) { + struct gve_tx_dma_buf *buf = + &pending_packet->bufs[pending_packet->num_bufs]; + const skb_frag_t *frag = &shinfo->frags[i]; + bool is_eop = i == (shinfo->nr_frags - 1); + u32 len = skb_frag_size(frag); + dma_addr_t addr; + + addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(tx->dev, addr))) + goto err; + + dma_unmap_len_set(buf, len, len); + dma_unmap_addr_set(buf, dma, addr); + ++pending_packet->num_bufs; + + gve_tx_fill_pkt_desc_dqo(tx, &desc_idx, skb, len, addr, + completion_tag, is_eop, is_gso); + } + + /* Commit the changes to our state */ + tx->dqo_tx.tail = desc_idx; + + /* Request a descriptor completion on the last descriptor of the + * packet if we are allowed to by the HW enforced interval. + */ + { + u32 last_desc_idx = (desc_idx - 1) & tx->mask; + u32 last_report_event_interval = + (last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask; + + if (unlikely(last_report_event_interval >= + GVE_TX_MIN_RE_INTERVAL)) { + tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true; + tx->dqo_tx.last_re_idx = last_desc_idx; + } + } + + return 0; + +err: + for (i = 0; i < pending_packet->num_bufs; i++) { + struct gve_tx_dma_buf *buf = &pending_packet->bufs[i]; + + if (i == 0) { + dma_unmap_single(tx->dev, dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), + DMA_TO_DEVICE); + } else { + dma_unmap_page(tx->dev, dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), DMA_TO_DEVICE); + } + } + + pending_packet->skb = NULL; + pending_packet->num_bufs = 0; + gve_free_pending_packet(tx, pending_packet); + + return -1; +} + +static int gve_num_descs_per_buf(size_t size) +{ + return DIV_ROUND_UP(size, GVE_TX_MAX_BUF_SIZE_DQO); +} + +static int gve_num_buffer_descs_needed(const struct sk_buff *skb) +{ + const struct skb_shared_info *shinfo = skb_shinfo(skb); + int num_descs; + int i; + + num_descs = gve_num_descs_per_buf(skb_headlen(skb)); + + for (i = 0; i < shinfo->nr_frags; i++) { + unsigned int frag_size = skb_frag_size(&shinfo->frags[i]); + + num_descs += gve_num_descs_per_buf(frag_size); + } + + return num_descs; +} + +/* Returns true if HW is capable of sending TSO represented by `skb`. + * + * Each segment must not span more than GVE_TX_MAX_DATA_DESCS buffers. + * - The header is counted as one buffer for every single segment. + * - A buffer which is split between two segments is counted for both. + * - If a buffer contains both header and payload, it is counted as two buffers. + */ +static bool gve_can_send_tso(const struct sk_buff *skb) +{ + const int header_len = skb_checksum_start_offset(skb) + tcp_hdrlen(skb); + const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1; + const struct skb_shared_info *shinfo = skb_shinfo(skb); + const int gso_size = shinfo->gso_size; + int cur_seg_num_bufs; + int cur_seg_size; + int i; + + cur_seg_size = skb_headlen(skb) - header_len; + cur_seg_num_bufs = cur_seg_size > 0; + + for (i = 0; i < shinfo->nr_frags; i++) { + if (cur_seg_size >= gso_size) { + cur_seg_size %= gso_size; + cur_seg_num_bufs = cur_seg_size > 0; + } + + if (unlikely(++cur_seg_num_bufs > max_bufs_per_seg)) + return false; + + cur_seg_size += skb_frag_size(&shinfo->frags[i]); + } + + return true; +} + +/* Attempt to transmit specified SKB. + * + * Returns 0 if the SKB was transmitted or dropped. + * Returns -1 if there is not currently enough space to transmit the SKB. + */ +static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx, + struct sk_buff *skb) +{ + int num_buffer_descs; + int total_num_descs; + + if (skb_is_gso(skb)) { + /* If TSO doesn't meet HW requirements, attempt to linearize the + * packet. + */ + if (unlikely(!gve_can_send_tso(skb) && + skb_linearize(skb) < 0)) { + net_err_ratelimited("%s: Failed to transmit TSO packet\n", + priv->dev->name); + goto drop; + } + + num_buffer_descs = gve_num_buffer_descs_needed(skb); + } else { + num_buffer_descs = gve_num_buffer_descs_needed(skb); + + if (unlikely(num_buffer_descs > GVE_TX_MAX_DATA_DESCS)) { + if (unlikely(skb_linearize(skb) < 0)) + goto drop; + + num_buffer_descs = 1; + } + } + + /* Metadata + (optional TSO) + data descriptors. */ + total_num_descs = 1 + skb_is_gso(skb) + num_buffer_descs; + if (unlikely(gve_maybe_stop_tx_dqo(tx, total_num_descs + + GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP))) { + return -1; + } + + if (unlikely(gve_tx_add_skb_no_copy_dqo(tx, skb) < 0)) + goto drop; + + netdev_tx_sent_queue(tx->netdev_txq, skb->len); + skb_tx_timestamp(skb); + return 0; + +drop: + tx->dropped_pkt++; + dev_kfree_skb_any(skb); + return 0; +} + +/* Transmit a given skb and ring the doorbell. */ +netdev_tx_t gve_tx_dqo(struct sk_buff *skb, struct net_device *dev) +{ + struct gve_priv *priv = netdev_priv(dev); + struct gve_tx_ring *tx; + + tx = &priv->tx[skb_get_queue_mapping(skb)]; + if (unlikely(gve_try_tx_skb(priv, tx, skb) < 0)) { + /* We need to ring the txq doorbell -- we have stopped the Tx + * queue for want of resources, but prior calls to gve_tx() + * may have added descriptors without ringing the doorbell. + */ + gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); + return NETDEV_TX_BUSY; + } + + if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more()) + return NETDEV_TX_OK; + + gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); + return NETDEV_TX_OK; +} + +static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list, + struct gve_tx_pending_packet_dqo *pending_packet) +{ + s16 old_tail, index; + + index = pending_packet - tx->dqo.pending_packets; + old_tail = list->tail; + list->tail = index; + if (old_tail == -1) + list->head = index; + else + tx->dqo.pending_packets[old_tail].next = index; + + pending_packet->next = -1; + pending_packet->prev = old_tail; +} + +static void remove_from_list(struct gve_tx_ring *tx, + struct gve_index_list *list, + struct gve_tx_pending_packet_dqo *pending_packet) +{ + s16 prev_index, next_index; + + prev_index = pending_packet->prev; + next_index = pending_packet->next; + + if (prev_index == -1) { + /* Node is head */ + list->head = next_index; + } else { + tx->dqo.pending_packets[prev_index].next = next_index; + } + if (next_index == -1) { + /* Node is tail */ + list->tail = prev_index; + } else { + tx->dqo.pending_packets[next_index].prev = prev_index; + } +} + +static void gve_unmap_packet(struct device *dev, + struct gve_tx_pending_packet_dqo *pending_packet) +{ + struct gve_tx_dma_buf *buf; + int i; + + /* SKB linear portion is guaranteed to be mapped */ + buf = &pending_packet->bufs[0]; + dma_unmap_single(dev, dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), DMA_TO_DEVICE); + for (i = 1; i < pending_packet->num_bufs; i++) { + buf = &pending_packet->bufs[i]; + dma_unmap_page(dev, dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), DMA_TO_DEVICE); + } + pending_packet->num_bufs = 0; +} + +/* Completion types and expected behavior: + * No Miss compl + Packet compl = Packet completed normally. + * Miss compl + Re-inject compl = Packet completed normally. + * No Miss compl + Re-inject compl = Skipped i.e. packet not completed. + * Miss compl + Packet compl = Skipped i.e. packet not completed. + */ +static void gve_handle_packet_completion(struct gve_priv *priv, + struct gve_tx_ring *tx, bool is_napi, + u16 compl_tag, u64 *bytes, u64 *pkts, + bool is_reinjection) +{ + struct gve_tx_pending_packet_dqo *pending_packet; + + if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { + net_err_ratelimited("%s: Invalid TX completion tag: %d\n", + priv->dev->name, (int)compl_tag); + return; + } + + pending_packet = &tx->dqo.pending_packets[compl_tag]; + + if (unlikely(is_reinjection)) { + if (unlikely(pending_packet->state == + GVE_PACKET_STATE_TIMED_OUT_COMPL)) { + net_err_ratelimited("%s: Re-injection completion: %d received after timeout.\n", + priv->dev->name, (int)compl_tag); + /* Packet was already completed as a result of timeout, + * so just remove from list and free pending packet. + */ + remove_from_list(tx, + &tx->dqo_compl.timed_out_completions, + pending_packet); + gve_free_pending_packet(tx, pending_packet); + return; + } + if (unlikely(pending_packet->state != + GVE_PACKET_STATE_PENDING_REINJECT_COMPL)) { + /* No outstanding miss completion but packet allocated + * implies packet receives a re-injection completion + * without a a prior miss completion. Return without + * completing the packet. + */ + net_err_ratelimited("%s: Re-injection completion received without corresponding miss completion: %d\n", + priv->dev->name, (int)compl_tag); + return; + } + remove_from_list(tx, &tx->dqo_compl.miss_completions, + pending_packet); + } else { + /* Packet is allocated but not a pending data completion. */ + if (unlikely(pending_packet->state != + GVE_PACKET_STATE_PENDING_DATA_COMPL)) { + net_err_ratelimited("%s: No pending data completion: %d\n", + priv->dev->name, (int)compl_tag); + return; + } + } + gve_unmap_packet(tx->dev, pending_packet); + + *bytes += pending_packet->skb->len; + (*pkts)++; + napi_consume_skb(pending_packet->skb, is_napi); + pending_packet->skb = NULL; + gve_free_pending_packet(tx, pending_packet); +} + +static void gve_handle_miss_completion(struct gve_priv *priv, + struct gve_tx_ring *tx, u16 compl_tag, + u64 *bytes, u64 *pkts) +{ + struct gve_tx_pending_packet_dqo *pending_packet; + + if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { + net_err_ratelimited("%s: Invalid TX completion tag: %d\n", + priv->dev->name, (int)compl_tag); + return; + } + + pending_packet = &tx->dqo.pending_packets[compl_tag]; + if (unlikely(pending_packet->state != + GVE_PACKET_STATE_PENDING_DATA_COMPL)) { + net_err_ratelimited("%s: Unexpected packet state: %d for completion tag : %d\n", + priv->dev->name, (int)pending_packet->state, + (int)compl_tag); + return; + } + + pending_packet->state = GVE_PACKET_STATE_PENDING_REINJECT_COMPL; + /* jiffies can wraparound but time comparisons can handle overflows. */ + pending_packet->timeout_jiffies = + jiffies + + msecs_to_jiffies(GVE_REINJECT_COMPL_TIMEOUT * + MSEC_PER_SEC); + add_to_list(tx, &tx->dqo_compl.miss_completions, pending_packet); + + *bytes += pending_packet->skb->len; + (*pkts)++; +} + +static void remove_miss_completions(struct gve_priv *priv, + struct gve_tx_ring *tx) +{ + struct gve_tx_pending_packet_dqo *pending_packet; + s16 next_index; + + next_index = tx->dqo_compl.miss_completions.head; + while (next_index != -1) { + pending_packet = &tx->dqo.pending_packets[next_index]; + next_index = pending_packet->next; + /* Break early because packets should timeout in order. */ + if (time_is_after_jiffies(pending_packet->timeout_jiffies)) + break; + + remove_from_list(tx, &tx->dqo_compl.miss_completions, + pending_packet); + /* Unmap buffers and free skb but do not unallocate packet i.e. + * the completion tag is not freed to ensure that the driver + * can take appropriate action if a corresponding valid + * completion is received later. + */ + gve_unmap_packet(tx->dev, pending_packet); + /* This indicates the packet was dropped. */ + dev_kfree_skb_any(pending_packet->skb); + pending_packet->skb = NULL; + tx->dropped_pkt++; + net_err_ratelimited("%s: No reinjection completion was received for: %d.\n", + priv->dev->name, + (int)(pending_packet - tx->dqo.pending_packets)); + + pending_packet->state = GVE_PACKET_STATE_TIMED_OUT_COMPL; + pending_packet->timeout_jiffies = + jiffies + + msecs_to_jiffies(GVE_DEALLOCATE_COMPL_TIMEOUT * + MSEC_PER_SEC); + /* Maintain pending packet in another list so the packet can be + * unallocated at a later time. + */ + add_to_list(tx, &tx->dqo_compl.timed_out_completions, + pending_packet); + } +} + +static void remove_timed_out_completions(struct gve_priv *priv, + struct gve_tx_ring *tx) +{ + struct gve_tx_pending_packet_dqo *pending_packet; + s16 next_index; + + next_index = tx->dqo_compl.timed_out_completions.head; + while (next_index != -1) { + pending_packet = &tx->dqo.pending_packets[next_index]; + next_index = pending_packet->next; + /* Break early because packets should timeout in order. */ + if (time_is_after_jiffies(pending_packet->timeout_jiffies)) + break; + + remove_from_list(tx, &tx->dqo_compl.timed_out_completions, + pending_packet); + gve_free_pending_packet(tx, pending_packet); + } +} + +int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, + struct napi_struct *napi) +{ + u64 reinject_compl_bytes = 0; + u64 reinject_compl_pkts = 0; + int num_descs_cleaned = 0; + u64 miss_compl_bytes = 0; + u64 miss_compl_pkts = 0; + u64 pkt_compl_bytes = 0; + u64 pkt_compl_pkts = 0; + + /* Limit in order to avoid blocking for too long */ + while (!napi || pkt_compl_pkts < napi->weight) { + struct gve_tx_compl_desc *compl_desc = + &tx->dqo.compl_ring[tx->dqo_compl.head]; + u16 type; + + if (compl_desc->generation == tx->dqo_compl.cur_gen_bit) + break; + + /* Prefetch the next descriptor. */ + prefetch(&tx->dqo.compl_ring[(tx->dqo_compl.head + 1) & + tx->dqo.complq_mask]); + + /* Do not read data until we own the descriptor */ + dma_rmb(); + type = compl_desc->type; + + if (type == GVE_COMPL_TYPE_DQO_DESC) { + /* This is the last descriptor fetched by HW plus one */ + u16 tx_head = le16_to_cpu(compl_desc->tx_head); + + atomic_set_release(&tx->dqo_compl.hw_tx_head, tx_head); + } else if (type == GVE_COMPL_TYPE_DQO_PKT) { + u16 compl_tag = le16_to_cpu(compl_desc->completion_tag); + + gve_handle_packet_completion(priv, tx, !!napi, + compl_tag, + &pkt_compl_bytes, + &pkt_compl_pkts, + /*is_reinjection=*/false); + } else if (type == GVE_COMPL_TYPE_DQO_MISS) { + u16 compl_tag = le16_to_cpu(compl_desc->completion_tag); + + gve_handle_miss_completion(priv, tx, compl_tag, + &miss_compl_bytes, + &miss_compl_pkts); + } else if (type == GVE_COMPL_TYPE_DQO_REINJECTION) { + u16 compl_tag = le16_to_cpu(compl_desc->completion_tag); + + gve_handle_packet_completion(priv, tx, !!napi, + compl_tag, + &reinject_compl_bytes, + &reinject_compl_pkts, + /*is_reinjection=*/true); + } + + tx->dqo_compl.head = + (tx->dqo_compl.head + 1) & tx->dqo.complq_mask; + /* Flip the generation bit when we wrap around */ + tx->dqo_compl.cur_gen_bit ^= tx->dqo_compl.head == 0; + num_descs_cleaned++; + } + + netdev_tx_completed_queue(tx->netdev_txq, + pkt_compl_pkts + miss_compl_pkts, + pkt_compl_bytes + miss_compl_bytes); + + remove_miss_completions(priv, tx); + remove_timed_out_completions(priv, tx); + + u64_stats_update_begin(&tx->statss); + tx->bytes_done += pkt_compl_bytes + reinject_compl_bytes; + tx->pkt_done += pkt_compl_pkts + reinject_compl_pkts; + u64_stats_update_end(&tx->statss); + return num_descs_cleaned; +} + +bool gve_tx_poll_dqo(struct gve_notify_block *block, bool do_clean) +{ + struct gve_tx_compl_desc *compl_desc; + struct gve_tx_ring *tx = block->tx; + struct gve_priv *priv = block->priv; + + if (do_clean) { + int num_descs_cleaned = gve_clean_tx_done_dqo(priv, tx, + &block->napi); + + /* Sync with queue being stopped in `gve_maybe_stop_tx_dqo()` */ + mb(); + + if (netif_tx_queue_stopped(tx->netdev_txq) && + num_descs_cleaned > 0) { + tx->wake_queue++; + netif_tx_wake_queue(tx->netdev_txq); + } + } + + /* Return true if we still have work. */ + compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head]; + return compl_desc->generation != tx->dqo_compl.cur_gen_bit; +} diff --git a/drivers/net/ethernet/google/gve/gve_utils.c b/drivers/net/ethernet/google/gve/gve_utils.c new file mode 100644 index 000000000000..93f3dcbeeea9 --- /dev/null +++ b/drivers/net/ethernet/google/gve/gve_utils.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Google virtual Ethernet (gve) driver + * + * Copyright (C) 2015-2021 Google, Inc. + */ + +#include "gve.h" +#include "gve_adminq.h" +#include "gve_utils.h" + +void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx) +{ + struct gve_notify_block *block = + &priv->ntfy_blocks[gve_tx_idx_to_ntfy(priv, queue_idx)]; + + block->tx = NULL; +} + +void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx) +{ + int ntfy_idx = gve_tx_idx_to_ntfy(priv, queue_idx); + struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; + struct gve_tx_ring *tx = &priv->tx[queue_idx]; + + block->tx = tx; + tx->ntfy_id = ntfy_idx; +} + +void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx) +{ + struct gve_notify_block *block = + &priv->ntfy_blocks[gve_rx_idx_to_ntfy(priv, queue_idx)]; + + block->rx = NULL; +} + +void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx) +{ + u32 ntfy_idx = gve_rx_idx_to_ntfy(priv, queue_idx); + struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; + struct gve_rx_ring *rx = &priv->rx[queue_idx]; + + block->rx = rx; + rx->ntfy_id = ntfy_idx; +} + +struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi, + struct gve_rx_slot_page_info *page_info, u16 len, + u16 pad) +{ + struct sk_buff *skb = napi_alloc_skb(napi, len); + void *va = page_info->page_address + pad + + page_info->page_offset; + + if (unlikely(!skb)) + return NULL; + + __skb_put(skb, len); + + skb_copy_to_linear_data(skb, va, len); + + skb->protocol = eth_type_trans(skb, dev); + + return skb; +} + +void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info) +{ + page_info->pagecnt_bias--; + if (page_info->pagecnt_bias == 0) { + int pagecount = page_count(page_info->page); + + /* If we have run out of bias - set it back up to INT_MAX + * minus the existing refs. + */ + page_info->pagecnt_bias = INT_MAX - pagecount; + + /* Set pagecount back up to max. */ + page_ref_add(page_info->page, INT_MAX - pagecount); + } +} diff --git a/drivers/net/ethernet/google/gve/gve_utils.h b/drivers/net/ethernet/google/gve/gve_utils.h new file mode 100644 index 000000000000..79595940b351 --- /dev/null +++ b/drivers/net/ethernet/google/gve/gve_utils.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) + * Google virtual Ethernet (gve) driver + * + * Copyright (C) 2015-2021 Google, Inc. + */ + +#ifndef _GVE_UTILS_H +#define _GVE_UTILS_H + +#include <linux/etherdevice.h> + +#include "gve.h" + +void gve_tx_remove_from_block(struct gve_priv *priv, int queue_idx); +void gve_tx_add_to_block(struct gve_priv *priv, int queue_idx); + +void gve_rx_remove_from_block(struct gve_priv *priv, int queue_idx); +void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx); + +struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi, + struct gve_rx_slot_page_info *page_info, u16 len, + u16 pad); + +/* Decrement pagecnt_bias. Set it back to INT_MAX if it reached zero. */ +void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info); + +#endif /* _GVE_UTILS_H */ + diff --git a/drivers/net/ethernet/hisilicon/Kconfig b/drivers/net/ethernet/hisilicon/Kconfig index 44f9279cdde1..bb062b02fb85 100644 --- a/drivers/net/ethernet/hisilicon/Kconfig +++ b/drivers/net/ethernet/hisilicon/Kconfig @@ -102,6 +102,7 @@ config HNS3_HCLGE tristate "Hisilicon HNS3 HCLGE Acceleration Engine & Compatibility Layer Support" default m depends on PCI_MSI + imply PTP_1588_CLOCK help This selects the HNS3_HCLGE network acceleration engine & its hardware compatibility layer. The engine would be used in Hisilicon hip08 family of @@ -130,6 +131,7 @@ config HNS3_ENET default m depends on 64BIT && PCI depends on INET + select DIMLIB help This selects the Ethernet Driver for Hisilicon Network Subsystem 3 for hip08 family of SoCs. This module depends upon HNAE3 driver to access the HNAE3 diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c b/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c index c615fbf9094e..75e4ec569da8 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_ae_adapt.c @@ -462,8 +462,6 @@ static void hns_ae_adjust_link(struct hnae_handle *handle, int speed, default: break; } - - return; } static void hns_ae_get_ring_bdnum_limit(struct hnae_queue *queue, diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c index f4cf569a2599..f41379de2186 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c @@ -111,7 +111,7 @@ int hns_mac_get_port_info(struct hns_mac_cb *mac_cb, } /** - *hns_mac_is_adjust_link - check is need change mac speed and duplex register + *hns_mac_need_adjust_link - check is need change mac speed and duplex register *@mac_cb: mac device *@speed: phy device speed *@duplex:phy device duplex @@ -374,7 +374,7 @@ static void hns_mac_param_get(struct mac_params *param, } /** - * hns_mac_queue_config_bc_en - set broadcast rx&tx enable + * hns_mac_port_config_bc_en - set broadcast rx&tx enable * @mac_cb: mac device * @port_num: queue number * @vlan_id: vlan id` @@ -597,7 +597,7 @@ int hns_mac_set_autoneg(struct hns_mac_cb *mac_cb, u8 enable) } /** - * hns_mac_set_autoneg - set rx & tx pause parameter + * hns_mac_set_pauseparam - set rx & tx pause parameter * @mac_cb: mac control block * @rx_en: rx enable or not * @tx_en: tx enable or not @@ -914,8 +914,7 @@ static int hns_mac_get_info(struct hns_mac_cb *mac_cb) } } else if (is_acpi_node(mac_cb->fw_port)) { ret = hns_mac_register_phy(mac_cb); - /* - * Mac can work well if there is phy or not.If the port don't + /* Mac can work well if there is phy or not.If the port don't * connect with phy, the return value will be ignored. Only * when there is phy but can't find mdio bus, the return value * will be handled. diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c index c2a60612f503..fcaf5132b865 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c @@ -227,7 +227,7 @@ hns_dsaf_reg_cnt_clr_ce(struct dsaf_device *dsaf_dev, u32 reg_cnt_clr_ce) } /** - * hns_ppe_qid_cfg - config ppe qid + * hns_dsaf_ppe_qid_cfg - config ppe qid * @dsaf_dev: dsa fabric id * @qid_cfg: value array */ @@ -613,7 +613,7 @@ static void hns_dsaf_tbl_tcam_data_cfg( } /** - * dsaf_tbl_tcam_mcast_cfg - tbl + * hns_dsaf_tbl_tcam_mcast_cfg - tbl * @dsaf_dev: dsa fabric id * @mcast: addr */ @@ -1213,7 +1213,7 @@ void hns_dsaf_get_rx_mac_pause_en(struct dsaf_device *dsaf_dev, int mac_id, } /** - * hns_dsaf_tbl_tcam_init - INT + * hns_dsaf_comm_init - INT * @dsaf_dev: dsa fabric id */ static void hns_dsaf_comm_init(struct dsaf_device *dsaf_dev) @@ -2111,7 +2111,7 @@ static void hns_dsaf_free_dev(struct dsaf_device *dsaf_dev) } /** - * dsaf_pfc_unit_cnt - set pfc unit count + * hns_dsaf_pfc_unit_cnt - set pfc unit count * @dsaf_dev: dsa fabric id * @mac_id: id in use * @rate: value array @@ -2142,7 +2142,7 @@ static void hns_dsaf_pfc_unit_cnt(struct dsaf_device *dsaf_dev, int mac_id, } /** - * dsaf_port_work_rate_cfg - fifo + * hns_dsaf_port_work_rate_cfg - fifo * @dsaf_dev: dsa fabric id * @mac_id: mac contrl block * @rate_mode: value array @@ -2738,7 +2738,7 @@ void hns_dsaf_get_strings(int stringset, u8 *data, int port, } /** - *hns_dsaf_get_sset_count - get dsaf regs count + *hns_dsaf_get_regs_count - get dsaf regs count *return dsaf regs count */ int hns_dsaf_get_regs_count(void) @@ -2949,7 +2949,7 @@ int hns_dsaf_wait_pkt_clean(struct dsaf_device *dsaf_dev, int port) } /** - * dsaf_probe - probo dsaf dev + * hns_dsaf_probe - probo dsaf dev * @pdev: dasf platform device * return 0 - success , negative --fail */ @@ -3004,7 +3004,7 @@ free_dev: } /** - * dsaf_remove - remove dsaf dev + * hns_dsaf_remove - remove dsaf dev * @pdev: dasf platform device */ static int hns_dsaf_remove(struct platform_device *pdev) diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c index 325e81d30cfd..23d9cbf262c3 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c @@ -56,31 +56,31 @@ static u32 dsaf_read_sub(struct dsaf_device *dsaf_dev, u32 reg) } static void hns_dsaf_acpi_ledctrl_by_port(struct hns_mac_cb *mac_cb, u8 op_type, - u32 link, u32 port, u32 act) + u32 link, u32 port, u32 act) { - union acpi_object *obj; - union acpi_object obj_args[3], argv4; + union acpi_object *obj; + union acpi_object obj_args[3], argv4; - obj_args[0].integer.type = ACPI_TYPE_INTEGER; - obj_args[0].integer.value = link; - obj_args[1].integer.type = ACPI_TYPE_INTEGER; - obj_args[1].integer.value = port; - obj_args[2].integer.type = ACPI_TYPE_INTEGER; - obj_args[2].integer.value = act; + obj_args[0].integer.type = ACPI_TYPE_INTEGER; + obj_args[0].integer.value = link; + obj_args[1].integer.type = ACPI_TYPE_INTEGER; + obj_args[1].integer.value = port; + obj_args[2].integer.type = ACPI_TYPE_INTEGER; + obj_args[2].integer.value = act; - argv4.type = ACPI_TYPE_PACKAGE; - argv4.package.count = 3; - argv4.package.elements = obj_args; + argv4.type = ACPI_TYPE_PACKAGE; + argv4.package.count = 3; + argv4.package.elements = obj_args; - obj = acpi_evaluate_dsm(ACPI_HANDLE(mac_cb->dev), - &hns_dsaf_acpi_dsm_guid, 0, op_type, &argv4); - if (!obj) { - dev_warn(mac_cb->dev, "ledctrl fail, link:%d port:%d act:%d!\n", - link, port, act); - return; - } + obj = acpi_evaluate_dsm(ACPI_HANDLE(mac_cb->dev), + &hns_dsaf_acpi_dsm_guid, 0, op_type, &argv4); + if (!obj) { + dev_warn(mac_cb->dev, "ledctrl fail, link:%d port:%d act:%d!\n", + link, port, act); + return; + } - ACPI_FREE(obj); + ACPI_FREE(obj); } static void hns_dsaf_acpi_locate_ledctrl_by_port(struct hns_mac_cb *mac_cb, @@ -151,15 +151,15 @@ static void hns_cpld_set_led(struct hns_mac_cb *mac_cb, int link_status, } static void hns_cpld_set_led_acpi(struct hns_mac_cb *mac_cb, int link_status, - u16 speed, int data) + u16 speed, int data) { - if (!mac_cb) { - pr_err("cpld_led_set mac_cb is null!\n"); - return; - } + if (!mac_cb) { + pr_err("cpld_led_set mac_cb is null!\n"); + return; + } - hns_dsaf_acpi_ledctrl_by_port(mac_cb, HNS_OP_LED_SET_FUNC, - link_status, mac_cb->mac_id, data); + hns_dsaf_acpi_ledctrl_by_port(mac_cb, HNS_OP_LED_SET_FUNC, + link_status, mac_cb->mac_id, data); } static void cpld_led_reset(struct hns_mac_cb *mac_cb) @@ -174,16 +174,16 @@ static void cpld_led_reset(struct hns_mac_cb *mac_cb) static void cpld_led_reset_acpi(struct hns_mac_cb *mac_cb) { - if (!mac_cb) { - pr_err("cpld_led_reset mac_cb is null!\n"); - return; - } + if (!mac_cb) { + pr_err("cpld_led_reset mac_cb is null!\n"); + return; + } - if (mac_cb->media_type != HNAE_MEDIA_TYPE_FIBER) - return; + if (mac_cb->media_type != HNAE_MEDIA_TYPE_FIBER) + return; - hns_dsaf_acpi_ledctrl_by_port(mac_cb, HNS_OP_LED_SET_FUNC, - 0, mac_cb->mac_id, 0); + hns_dsaf_acpi_ledctrl_by_port(mac_cb, HNS_OP_LED_SET_FUNC, + 0, mac_cb->mac_id, 0); } static int cpld_set_led_id(struct hns_mac_cb *mac_cb, @@ -351,7 +351,7 @@ hns_dsaf_srst_chns(struct dsaf_device *dsaf_dev, u32 msk, bool dereset) } /** - * hns_dsaf_srst_chns - reset dsaf channels + * hns_dsaf_srst_chns_acpi - reset dsaf channels * @dsaf_dev: dsaf device struct pointer * @msk: xbar channels mask value: * @dereset: false - request reset , true - drop reset @@ -501,7 +501,7 @@ static void hns_ppe_com_srst(struct dsaf_device *dsaf_dev, bool dereset) } /** - * hns_mac_get_sds_mode - get phy ifterface form serdes mode + * hns_mac_get_phy_if - get phy ifterface form serdes mode * @mac_cb: mac control block * retuen phy interface */ @@ -521,7 +521,7 @@ static phy_interface_t hns_mac_get_phy_if(struct hns_mac_cb *mac_cb) reg = HNS_MAC_HILINK4_REG; else reg = HNS_MAC_HILINK3_REG; - } else{ + } else { if (!HNS_DSAF_IS_DEBUG(mac_cb->dsaf_dev) && mac_id <= 3) reg = HNS_MAC_HILINK4V2_REG; else diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c index ff03cafccb66..a7eb87da4e70 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c @@ -296,7 +296,7 @@ int hns_ppe_wait_tx_fifo_clean(struct hns_ppe_cb *ppe_cb) } /** - * ppe_init_hw - init ppe + * hns_ppe_init_hw - init ppe * @ppe_cb: ppe device */ static void hns_ppe_init_hw(struct hns_ppe_cb *ppe_cb) @@ -343,7 +343,7 @@ static void hns_ppe_init_hw(struct hns_ppe_cb *ppe_cb) } /** - * ppe_uninit_hw - uninit ppe + * hns_ppe_uninit_hw - uninit ppe * @ppe_cb: ppe device */ static void hns_ppe_uninit_hw(struct hns_ppe_cb *ppe_cb) @@ -382,7 +382,7 @@ void hns_ppe_uninit(struct dsaf_device *dsaf_dev) } /** - * hns_ppe_reset - reinit ppe/rcb hw + * hns_ppe_reset_common - reinit ppe/rcb hw * @dsaf_dev: dasf device * @ppe_common_index: the index * return void @@ -455,7 +455,7 @@ int hns_ppe_get_regs_count(void) } /** - * ppe_get_strings - get ppe srting + * hns_ppe_get_strings - get ppe srting * @ppe_cb: ppe device * @stringset: string set type * @data: output string diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c index 5d5dc6942232..e2ff3ca198d1 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c @@ -913,7 +913,7 @@ int hns_rcb_get_common_regs_count(void) } /** - *rcb_get_sset_count - rcb ring regs count + *hns_rcb_get_ring_regs_count - rcb ring regs count *return regs count */ int hns_rcb_get_ring_regs_count(void) diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c index be52acd448f9..401fef5f1d07 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_xgmac.c @@ -104,7 +104,7 @@ static void hns_xgmac_rx_enable(struct mac_driver *drv, u32 value) } /** - * hns_xgmac_tx_lf_rf_insert - insert lf rf control about xgmac + * hns_xgmac_lf_rf_insert - insert lf rf control about xgmac * @mac_drv: mac driver * @mode: inserf rf or lf */ @@ -115,7 +115,7 @@ static void hns_xgmac_lf_rf_insert(struct mac_driver *mac_drv, u32 mode) } /** - * hns_xgmac__lf_rf_control_init - initial the lf rf control register + * hns_xgmac_lf_rf_control_init - initial the lf rf control register * @mac_drv: mac driver */ static void hns_xgmac_lf_rf_control_init(struct mac_driver *mac_drv) diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c index 5e349c0bdecc..ad534f9e41ab 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c @@ -770,7 +770,7 @@ static u32 smooth_alg(u32 new_param, u32 old_param) } /** - * hns_nic_adp_coalesce - self adapte coalesce according to rx rate + * hns_nic_adpt_coalesce - self adapte coalesce according to rx rate * @ring_data: pointer to hns_nic_ring_data **/ static void hns_nic_adpt_coalesce(struct hns_nic_ring_data *ring_data) diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c index da48c05435ea..7e62dcff2426 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c @@ -192,7 +192,7 @@ static int hns_nic_get_link_ksettings(struct net_device *net_dev, } /** - *hns_nic_set_link_settings - implement ethtool set link ksettings + *hns_nic_set_link_ksettings - implement ethtool set link ksettings *@net_dev: net_device *@cmd: ethtool_link_ksettings *retuen 0 - success , negative --fail @@ -827,7 +827,7 @@ hns_get_channels(struct net_device *net_dev, struct ethtool_channels *ch) } /** - * get_ethtool_stats - get detail statistics. + * hns_get_ethtool_stats - get detail statistics. * @netdev: net device * @stats: statistics info. * @data: statistics data. @@ -885,7 +885,7 @@ static void hns_get_ethtool_stats(struct net_device *netdev, } /** - * get_strings: Return a set of strings that describe the requested objects + * hns_get_strings: Return a set of strings that describe the requested objects * @netdev: net device * @stringset: string set ID. * @data: objects data. diff --git a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h index a2c17af57fde..0a6cda309b24 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h +++ b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h @@ -20,7 +20,7 @@ enum HCLGE_MBX_OPCODE { HCLGE_MBX_API_NEGOTIATE, /* (VF -> PF) negotiate API version */ HCLGE_MBX_GET_QINFO, /* (VF -> PF) get queue config */ HCLGE_MBX_GET_QDEPTH, /* (VF -> PF) get queue depth */ - HCLGE_MBX_GET_TCINFO, /* (VF -> PF) get TC config */ + HCLGE_MBX_GET_BASIC_INFO, /* (VF -> PF) get basic info */ HCLGE_MBX_GET_RETA, /* (VF -> PF) get RETA */ HCLGE_MBX_GET_RSS_KEY, /* (VF -> PF) get RSS key */ HCLGE_MBX_GET_MAC_ADDR, /* (VF -> PF) get MAC addr */ @@ -69,6 +69,7 @@ enum hclge_mbx_vlan_cfg_subcode { HCLGE_MBX_VLAN_RX_OFF_CFG, /* set rx side vlan offload */ HCLGE_MBX_PORT_BASE_VLAN_CFG, /* set port based vlan configuration */ HCLGE_MBX_GET_PORT_BASE_VLAN_STATE, /* get port based vlan state */ + HCLGE_MBX_ENABLE_VLAN_FILTER, }; enum hclge_mbx_tbl_cfg_subcode { @@ -85,6 +86,13 @@ struct hclge_ring_chain_param { u8 int_gl_index; }; +struct hclge_basic_info { + u8 hw_tc_map; + u8 rsv; + u16 mbx_api_version; + u32 pf_caps; +}; + struct hclgevf_mbx_resp_status { struct mutex mbx_mutex; /* protects against contending sync cmd resp */ u32 origin_mbx_msg; diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index 1d2189047781..e0b7c3c44e7b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -91,6 +91,10 @@ enum HNAE3_DEV_CAP_BITS { HNAE3_DEV_SUPPORT_STASH_B, HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, HNAE3_DEV_SUPPORT_PAUSE_B, + HNAE3_DEV_SUPPORT_RAS_IMP_B, + HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B, + HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, + HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, }; #define hnae3_dev_fd_supported(hdev) \ @@ -126,6 +130,9 @@ enum HNAE3_DEV_CAP_BITS { #define hnae3_dev_phy_imp_supported(hdev) \ test_bit(HNAE3_DEV_SUPPORT_PHY_IMP_B, (hdev)->ae_dev->caps) +#define hnae3_dev_ras_imp_supported(hdev) \ + test_bit(HNAE3_DEV_SUPPORT_RAS_IMP_B, (hdev)->ae_dev->caps) + #define hnae3_dev_tqp_txrx_indep_supported(hdev) \ test_bit(HNAE3_DEV_SUPPORT_TQP_TXRX_INDEP_B, (hdev)->ae_dev->caps) @@ -141,18 +148,17 @@ enum HNAE3_DEV_CAP_BITS { #define hnae3_ae_dev_tqp_txrx_indep_supported(ae_dev) \ test_bit(HNAE3_DEV_SUPPORT_TQP_TXRX_INDEP_B, (ae_dev)->caps) +#define hnae3_ae_dev_rxd_adv_layout_supported(ae_dev) \ + test_bit(HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B, (ae_dev)->caps) + +enum HNAE3_PF_CAP_BITS { + HNAE3_PF_SUPPORT_VLAN_FLTR_MDF_B = 0, +}; #define ring_ptr_move_fw(ring, p) \ ((ring)->p = ((ring)->p + 1) % (ring)->desc_num) #define ring_ptr_move_bw(ring, p) \ ((ring)->p = ((ring)->p - 1 + (ring)->desc_num) % (ring)->desc_num) -enum hns_desc_type { - DESC_TYPE_UNKNOWN, - DESC_TYPE_SKB, - DESC_TYPE_FRAGLIST_SKB, - DESC_TYPE_PAGE, -}; - struct hnae3_handle; struct hnae3_queue { @@ -234,7 +240,6 @@ enum hnae3_reset_type { HNAE3_FUNC_RESET, HNAE3_GLOBAL_RESET, HNAE3_IMP_RESET, - HNAE3_UNKNOWN_RESET, HNAE3_NONE_RESET, HNAE3_MAX_RESET, }; @@ -246,6 +251,52 @@ enum hnae3_port_base_vlan_state { HNAE3_PORT_BASE_VLAN_NOCHANGE, }; +enum hnae3_dbg_cmd { + HNAE3_DBG_CMD_TM_NODES, + HNAE3_DBG_CMD_TM_PRI, + HNAE3_DBG_CMD_TM_QSET, + HNAE3_DBG_CMD_TM_MAP, + HNAE3_DBG_CMD_TM_PG, + HNAE3_DBG_CMD_TM_PORT, + HNAE3_DBG_CMD_TC_SCH_INFO, + HNAE3_DBG_CMD_QOS_PAUSE_CFG, + HNAE3_DBG_CMD_QOS_PRI_MAP, + HNAE3_DBG_CMD_QOS_BUF_CFG, + HNAE3_DBG_CMD_DEV_INFO, + HNAE3_DBG_CMD_TX_BD, + HNAE3_DBG_CMD_RX_BD, + HNAE3_DBG_CMD_MAC_UC, + HNAE3_DBG_CMD_MAC_MC, + HNAE3_DBG_CMD_MNG_TBL, + HNAE3_DBG_CMD_LOOPBACK, + HNAE3_DBG_CMD_PTP_INFO, + HNAE3_DBG_CMD_INTERRUPT_INFO, + HNAE3_DBG_CMD_RESET_INFO, + HNAE3_DBG_CMD_IMP_INFO, + HNAE3_DBG_CMD_NCL_CONFIG, + HNAE3_DBG_CMD_REG_BIOS_COMMON, + HNAE3_DBG_CMD_REG_SSU, + HNAE3_DBG_CMD_REG_IGU_EGU, + HNAE3_DBG_CMD_REG_RPU, + HNAE3_DBG_CMD_REG_NCSI, + HNAE3_DBG_CMD_REG_RTC, + HNAE3_DBG_CMD_REG_PPP, + HNAE3_DBG_CMD_REG_RCB, + HNAE3_DBG_CMD_REG_TQP, + HNAE3_DBG_CMD_REG_MAC, + HNAE3_DBG_CMD_REG_DCB, + HNAE3_DBG_CMD_VLAN_CONFIG, + HNAE3_DBG_CMD_QUEUE_MAP, + HNAE3_DBG_CMD_RX_QUEUE_INFO, + HNAE3_DBG_CMD_TX_QUEUE_INFO, + HNAE3_DBG_CMD_FD_TCAM, + HNAE3_DBG_CMD_FD_COUNTER, + HNAE3_DBG_CMD_MAC_TNL_STATUS, + HNAE3_DBG_CMD_SERV_INFO, + HNAE3_DBG_CMD_UMV_INFO, + HNAE3_DBG_CMD_UNKNOWN, +}; + struct hnae3_vector_info { u8 __iomem *io_addr; int vector; @@ -470,6 +521,12 @@ struct hnae3_ae_dev { * Check if any cls flower rule exist * dbg_read_cmd * Execute debugfs read command. + * set_tx_hwts_info + * Save information for 1588 tx packet + * get_rx_hwts + * Get 1588 rx hwstamp + * get_ts_info + * Get phc info */ struct hnae3_ae_ops { int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev); @@ -585,7 +642,7 @@ struct hnae3_ae_ops { void (*get_mdix_mode)(struct hnae3_handle *handle, u8 *tp_mdix_ctrl, u8 *tp_mdix); - void (*enable_vlan_filter)(struct hnae3_handle *handle, bool enable); + int (*enable_vlan_filter)(struct hnae3_handle *handle, bool enable); int (*set_vlan_filter)(struct hnae3_handle *handle, __be16 proto, u16 vlan_id, bool is_kill); int (*set_vf_vlan_filter)(struct hnae3_handle *handle, int vfid, @@ -622,8 +679,7 @@ struct hnae3_ae_ops { void (*enable_fd)(struct hnae3_handle *handle, bool enable); int (*add_arfs_entry)(struct hnae3_handle *handle, u16 queue_id, u16 flow_id, struct flow_keys *fkeys); - int (*dbg_run_cmd)(struct hnae3_handle *handle, const char *cmd_buf); - int (*dbg_read_cmd)(struct hnae3_handle *handle, const char *cmd_buf, + int (*dbg_read_cmd)(struct hnae3_handle *handle, enum hnae3_dbg_cmd cmd, char *buf, int len); pci_ers_result_t (*handle_hw_ras_error)(struct hnae3_ae_dev *ae_dev); bool (*get_hw_reset_stat)(struct hnae3_handle *handle); @@ -656,6 +712,12 @@ struct hnae3_ae_ops { struct ethtool_link_ksettings *cmd); int (*set_phy_link_ksettings)(struct hnae3_handle *handle, const struct ethtool_link_ksettings *cmd); + bool (*set_tx_hwts_info)(struct hnae3_handle *handle, + struct sk_buff *skb); + void (*get_rx_hwts)(struct hnae3_handle *handle, struct sk_buff *skb, + u32 nsec, u32 sec); + int (*get_ts_info)(struct hnae3_handle *handle, + struct ethtool_ts_info *info); }; struct hnae3_dcb_ops { @@ -700,6 +762,7 @@ struct hnae3_knic_private_info { u16 rx_buf_len; u16 num_tx_desc; u16 num_rx_desc; + u32 tx_spare_buf_size; struct hnae3_tc_info tc_info; @@ -738,7 +801,6 @@ struct hnae3_roce_private_info { #define HNAE3_BPE BIT(2) /* broadcast promisc enable */ #define HNAE3_OVERFLOW_UPE BIT(3) /* unicast mac vlan overflow */ #define HNAE3_OVERFLOW_MPE BIT(4) /* multicast mac vlan overflow */ -#define HNAE3_VLAN_FLTR BIT(5) /* enable vlan filter */ #define HNAE3_UPE (HNAE3_USER_UPE | HNAE3_OVERFLOW_UPE) #define HNAE3_MPE (HNAE3_USER_MPE | HNAE3_OVERFLOW_MPE) @@ -786,10 +848,6 @@ struct hnae3_handle { #define hnae3_get_bit(origin, shift) \ hnae3_get_field(origin, 0x1 << (shift), shift) -#define HNAE3_DBG_TM_NODES "tm_nodes" -#define HNAE3_DBG_TM_PRI "tm_priority" -#define HNAE3_DBG_TM_QSET "tm_qset" - int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev); void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 9d702bd0c7c1..532523069d74 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -5,46 +5,539 @@ #include <linux/device.h> #include "hnae3.h" +#include "hns3_debugfs.h" #include "hns3_enet.h" -#define HNS3_DBG_READ_LEN 65536 -#define HNS3_DBG_WRITE_LEN 1024 - static struct dentry *hns3_dbgfs_root; -static int hns3_dbg_queue_info(struct hnae3_handle *h, - const char *cmd_buf) +static struct hns3_dbg_dentry_info hns3_dbg_dentry[] = { + { + .name = "tm" + }, + { + .name = "tx_bd_info" + }, + { + .name = "rx_bd_info" + }, + { + .name = "mac_list" + }, + { + .name = "reg" + }, + { + .name = "queue" + }, + { + .name = "fd" + }, + /* keep common at the bottom and add new directory above */ + { + .name = "common" + }, +}; + +static int hns3_dbg_bd_file_init(struct hnae3_handle *handle, unsigned int cmd); +static int hns3_dbg_common_file_init(struct hnae3_handle *handle, + unsigned int cmd); + +static struct hns3_dbg_cmd_info hns3_dbg_cmd[] = { + { + .name = "tm_nodes", + .cmd = HNAE3_DBG_CMD_TM_NODES, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tm_priority", + .cmd = HNAE3_DBG_CMD_TM_PRI, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tm_qset", + .cmd = HNAE3_DBG_CMD_TM_QSET, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tm_map", + .cmd = HNAE3_DBG_CMD_TM_MAP, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN_1MB, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tm_pg", + .cmd = HNAE3_DBG_CMD_TM_PG, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tm_port", + .cmd = HNAE3_DBG_CMD_TM_PORT, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tc_sch_info", + .cmd = HNAE3_DBG_CMD_TC_SCH_INFO, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "qos_pause_cfg", + .cmd = HNAE3_DBG_CMD_QOS_PAUSE_CFG, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "qos_pri_map", + .cmd = HNAE3_DBG_CMD_QOS_PRI_MAP, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "qos_buf_cfg", + .cmd = HNAE3_DBG_CMD_QOS_BUF_CFG, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "dev_info", + .cmd = HNAE3_DBG_CMD_DEV_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tx_bd_queue", + .cmd = HNAE3_DBG_CMD_TX_BD, + .dentry = HNS3_DBG_DENTRY_TX_BD, + .buf_len = HNS3_DBG_READ_LEN_4MB, + .init = hns3_dbg_bd_file_init, + }, + { + .name = "rx_bd_queue", + .cmd = HNAE3_DBG_CMD_RX_BD, + .dentry = HNS3_DBG_DENTRY_RX_BD, + .buf_len = HNS3_DBG_READ_LEN_4MB, + .init = hns3_dbg_bd_file_init, + }, + { + .name = "uc", + .cmd = HNAE3_DBG_CMD_MAC_UC, + .dentry = HNS3_DBG_DENTRY_MAC, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "mc", + .cmd = HNAE3_DBG_CMD_MAC_MC, + .dentry = HNS3_DBG_DENTRY_MAC, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "mng_tbl", + .cmd = HNAE3_DBG_CMD_MNG_TBL, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "loopback", + .cmd = HNAE3_DBG_CMD_LOOPBACK, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "interrupt_info", + .cmd = HNAE3_DBG_CMD_INTERRUPT_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "reset_info", + .cmd = HNAE3_DBG_CMD_RESET_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "imp_info", + .cmd = HNAE3_DBG_CMD_IMP_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "ncl_config", + .cmd = HNAE3_DBG_CMD_NCL_CONFIG, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN_128KB, + .init = hns3_dbg_common_file_init, + }, + { + .name = "mac_tnl_status", + .cmd = HNAE3_DBG_CMD_MAC_TNL_STATUS, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "bios_common", + .cmd = HNAE3_DBG_CMD_REG_BIOS_COMMON, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "ssu", + .cmd = HNAE3_DBG_CMD_REG_SSU, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "igu_egu", + .cmd = HNAE3_DBG_CMD_REG_IGU_EGU, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "rpu", + .cmd = HNAE3_DBG_CMD_REG_RPU, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "ncsi", + .cmd = HNAE3_DBG_CMD_REG_NCSI, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "rtc", + .cmd = HNAE3_DBG_CMD_REG_RTC, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "ppp", + .cmd = HNAE3_DBG_CMD_REG_PPP, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "rcb", + .cmd = HNAE3_DBG_CMD_REG_RCB, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tqp", + .cmd = HNAE3_DBG_CMD_REG_TQP, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "mac", + .cmd = HNAE3_DBG_CMD_REG_MAC, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "dcb", + .cmd = HNAE3_DBG_CMD_REG_DCB, + .dentry = HNS3_DBG_DENTRY_REG, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "queue_map", + .cmd = HNAE3_DBG_CMD_QUEUE_MAP, + .dentry = HNS3_DBG_DENTRY_QUEUE, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "rx_queue_info", + .cmd = HNAE3_DBG_CMD_RX_QUEUE_INFO, + .dentry = HNS3_DBG_DENTRY_QUEUE, + .buf_len = HNS3_DBG_READ_LEN_1MB, + .init = hns3_dbg_common_file_init, + }, + { + .name = "tx_queue_info", + .cmd = HNAE3_DBG_CMD_TX_QUEUE_INFO, + .dentry = HNS3_DBG_DENTRY_QUEUE, + .buf_len = HNS3_DBG_READ_LEN_1MB, + .init = hns3_dbg_common_file_init, + }, + { + .name = "fd_tcam", + .cmd = HNAE3_DBG_CMD_FD_TCAM, + .dentry = HNS3_DBG_DENTRY_FD, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "service_task_info", + .cmd = HNAE3_DBG_CMD_SERV_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "vlan_config", + .cmd = HNAE3_DBG_CMD_VLAN_CONFIG, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "ptp_info", + .cmd = HNAE3_DBG_CMD_PTP_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "fd_counter", + .cmd = HNAE3_DBG_CMD_FD_COUNTER, + .dentry = HNS3_DBG_DENTRY_FD, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, + { + .name = "umv_info", + .cmd = HNAE3_DBG_CMD_UMV_INFO, + .dentry = HNS3_DBG_DENTRY_COMMON, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, +}; + +static struct hns3_dbg_cap_info hns3_dbg_cap[] = { + { + .name = "support FD", + .cap_bit = HNAE3_DEV_SUPPORT_FD_B, + }, { + .name = "support GRO", + .cap_bit = HNAE3_DEV_SUPPORT_GRO_B, + }, { + .name = "support FEC", + .cap_bit = HNAE3_DEV_SUPPORT_FEC_B, + }, { + .name = "support UDP GSO", + .cap_bit = HNAE3_DEV_SUPPORT_UDP_GSO_B, + }, { + .name = "support PTP", + .cap_bit = HNAE3_DEV_SUPPORT_PTP_B, + }, { + .name = "support INT QL", + .cap_bit = HNAE3_DEV_SUPPORT_INT_QL_B, + }, { + .name = "support HW TX csum", + .cap_bit = HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, + }, { + .name = "support UDP tunnel csum", + .cap_bit = HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, + }, { + .name = "support TX push", + .cap_bit = HNAE3_DEV_SUPPORT_TX_PUSH_B, + }, { + .name = "support imp-controlled PHY", + .cap_bit = HNAE3_DEV_SUPPORT_PHY_IMP_B, + }, { + .name = "support imp-controlled RAS", + .cap_bit = HNAE3_DEV_SUPPORT_RAS_IMP_B, + }, { + .name = "support rxd advanced layout", + .cap_bit = HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B, + }, { + .name = "support port vlan bypass", + .cap_bit = HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, + }, { + .name = "support modify vlan filter state", + .cap_bit = HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, + } +}; + +static void hns3_dbg_fill_content(char *content, u16 len, + const struct hns3_dbg_item *items, + const char **result, u16 size) +{ + char *pos = content; + u16 i; + + memset(content, ' ', len); + for (i = 0; i < size; i++) { + if (result) + strncpy(pos, result[i], strlen(result[i])); + else + strncpy(pos, items[i].name, strlen(items[i].name)); + + pos += strlen(items[i].name) + items[i].interval; + } + + *pos++ = '\n'; + *pos++ = '\0'; +} + +static const struct hns3_dbg_item tx_spare_info_items[] = { + { "QUEUE_ID", 2 }, + { "COPYBREAK", 2 }, + { "LEN", 7 }, + { "NTU", 4 }, + { "NTC", 4 }, + { "LTC", 4 }, + { "DMA", 17 }, +}; + +static void hns3_dbg_tx_spare_info(struct hns3_enet_ring *ring, char *buf, + int len, u32 ring_num, int *pos) +{ + char data_str[ARRAY_SIZE(tx_spare_info_items)][HNS3_DBG_DATA_STR_LEN]; + struct hns3_tx_spare *tx_spare = ring->tx_spare; + char *result[ARRAY_SIZE(tx_spare_info_items)]; + char content[HNS3_DBG_INFO_LEN]; + u32 i, j; + + if (!tx_spare) { + *pos += scnprintf(buf + *pos, len - *pos, + "tx spare buffer is not enabled\n"); + return; + } + + for (i = 0; i < ARRAY_SIZE(tx_spare_info_items); i++) + result[i] = &data_str[i][0]; + + *pos += scnprintf(buf + *pos, len - *pos, "tx spare buffer info\n"); + hns3_dbg_fill_content(content, sizeof(content), tx_spare_info_items, + NULL, ARRAY_SIZE(tx_spare_info_items)); + *pos += scnprintf(buf + *pos, len - *pos, "%s", content); + + for (i = 0; i < ring_num; i++) { + j = 0; + sprintf(result[j++], "%8u", i); + sprintf(result[j++], "%9u", ring->tx_copybreak); + sprintf(result[j++], "%3u", tx_spare->len); + sprintf(result[j++], "%3u", tx_spare->next_to_use); + sprintf(result[j++], "%3u", tx_spare->next_to_clean); + sprintf(result[j++], "%3u", tx_spare->last_to_clean); + sprintf(result[j++], "%pad", &tx_spare->dma); + hns3_dbg_fill_content(content, sizeof(content), + tx_spare_info_items, + (const char **)result, + ARRAY_SIZE(tx_spare_info_items)); + *pos += scnprintf(buf + *pos, len - *pos, "%s", content); + } +} + +static const struct hns3_dbg_item rx_queue_info_items[] = { + { "QUEUE_ID", 2 }, + { "BD_NUM", 2 }, + { "BD_LEN", 2 }, + { "TAIL", 2 }, + { "HEAD", 2 }, + { "FBDNUM", 2 }, + { "PKTNUM", 2 }, + { "COPYBREAK", 2 }, + { "RING_EN", 2 }, + { "RX_RING_EN", 2 }, + { "BASE_ADDR", 10 }, +}; + +static void hns3_dump_rx_queue_info(struct hns3_enet_ring *ring, + struct hnae3_ae_dev *ae_dev, char **result, + u32 index) { + u32 base_add_l, base_add_h; + u32 j = 0; + + sprintf(result[j++], "%8u", index); + + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_BD_NUM_REG)); + + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_BD_LEN_REG)); + + sprintf(result[j++], "%4u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_TAIL_REG)); + + sprintf(result[j++], "%4u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_HEAD_REG)); + + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_FBDNUM_REG)); + + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_PKTNUM_RECORD_REG)); + sprintf(result[j++], "%9u", ring->rx_copybreak); + + sprintf(result[j++], "%7s", readl_relaxed(ring->tqp->io_base + + HNS3_RING_EN_REG) ? "on" : "off"); + + if (hnae3_ae_dev_tqp_txrx_indep_supported(ae_dev)) + sprintf(result[j++], "%10s", readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_EN_REG) ? "on" : "off"); + else + sprintf(result[j++], "%10s", "NA"); + + base_add_h = readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_BASEADDR_H_REG); + base_add_l = readl_relaxed(ring->tqp->io_base + + HNS3_RING_RX_RING_BASEADDR_L_REG); + sprintf(result[j++], "0x%08x%08x", base_add_h, base_add_l); +} + +static int hns3_dbg_rx_queue_info(struct hnae3_handle *h, + char *buf, int len) +{ + char data_str[ARRAY_SIZE(rx_queue_info_items)][HNS3_DBG_DATA_STR_LEN]; struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); + char *result[ARRAY_SIZE(rx_queue_info_items)]; struct hns3_nic_priv *priv = h->priv; + char content[HNS3_DBG_INFO_LEN]; struct hns3_enet_ring *ring; - u32 base_add_l, base_add_h; - u32 queue_num, queue_max; - u32 value, i; - int cnt; + int pos = 0; + u32 i; if (!priv->ring) { dev_err(&h->pdev->dev, "priv->ring is NULL\n"); return -EFAULT; } - queue_max = h->kinfo.num_tqps; - cnt = kstrtouint(&cmd_buf[11], 0, &queue_num); - if (cnt) - queue_num = 0; - else - queue_max = queue_num + 1; + for (i = 0; i < ARRAY_SIZE(rx_queue_info_items); i++) + result[i] = &data_str[i][0]; - dev_info(&h->pdev->dev, "queue info\n"); - - if (queue_num >= h->kinfo.num_tqps) { - dev_err(&h->pdev->dev, - "Queue number(%u) is out of range(0-%u)\n", queue_num, - h->kinfo.num_tqps - 1); - return -EINVAL; - } - - for (i = queue_num; i < queue_max; i++) { + hns3_dbg_fill_content(content, sizeof(content), rx_queue_info_items, + NULL, ARRAY_SIZE(rx_queue_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + for (i = 0; i < h->kinfo.num_tqps; i++) { /* Each cycle needs to determine whether the instance is reset, * to prevent reference to invalid memory. And need to ensure * that the following code is executed within 100ms. @@ -54,491 +547,524 @@ static int hns3_dbg_queue_info(struct hnae3_handle *h, return -EPERM; ring = &priv->ring[(u32)(i + h->kinfo.num_tqps)]; - base_add_h = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_BASEADDR_H_REG); - base_add_l = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_BASEADDR_L_REG); - dev_info(&h->pdev->dev, "RX(%u) BASE ADD: 0x%08x%08x\n", i, - base_add_h, base_add_l); + hns3_dump_rx_queue_info(ring, ae_dev, result, i); + hns3_dbg_fill_content(content, sizeof(content), + rx_queue_info_items, + (const char **)result, + ARRAY_SIZE(rx_queue_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + } + + return 0; +} - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_BD_NUM_REG); - dev_info(&h->pdev->dev, "RX(%u) RING BD NUM: %u\n", i, value); +static const struct hns3_dbg_item tx_queue_info_items[] = { + { "QUEUE_ID", 2 }, + { "BD_NUM", 2 }, + { "TC", 2 }, + { "TAIL", 2 }, + { "HEAD", 2 }, + { "FBDNUM", 2 }, + { "OFFSET", 2 }, + { "PKTNUM", 2 }, + { "RING_EN", 2 }, + { "TX_RING_EN", 2 }, + { "BASE_ADDR", 10 }, +}; - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_BD_LEN_REG); - dev_info(&h->pdev->dev, "RX(%u) RING BD LEN: %u\n", i, value); +static void hns3_dump_tx_queue_info(struct hns3_enet_ring *ring, + struct hnae3_ae_dev *ae_dev, char **result, + u32 index) +{ + u32 base_add_l, base_add_h; + u32 j = 0; - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_TAIL_REG); - dev_info(&h->pdev->dev, "RX(%u) RING TAIL: %u\n", i, value); + sprintf(result[j++], "%8u", index); + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_BD_NUM_REG)); - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_HEAD_REG); - dev_info(&h->pdev->dev, "RX(%u) RING HEAD: %u\n", i, value); + sprintf(result[j++], "%2u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_TC_REG)); - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_FBDNUM_REG); - dev_info(&h->pdev->dev, "RX(%u) RING FBDNUM: %u\n", i, value); + sprintf(result[j++], "%4u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_TAIL_REG)); - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_RING_PKTNUM_RECORD_REG); - dev_info(&h->pdev->dev, "RX(%u) RING PKTNUM: %u\n", i, value); + sprintf(result[j++], "%4u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_HEAD_REG)); - ring = &priv->ring[i]; - base_add_h = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_BASEADDR_H_REG); - base_add_l = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_BASEADDR_L_REG); - dev_info(&h->pdev->dev, "TX(%u) BASE ADD: 0x%08x%08x\n", i, - base_add_h, base_add_l); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_BD_NUM_REG); - dev_info(&h->pdev->dev, "TX(%u) RING BD NUM: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_TC_REG); - dev_info(&h->pdev->dev, "TX(%u) RING TC: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_TAIL_REG); - dev_info(&h->pdev->dev, "TX(%u) RING TAIL: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_HEAD_REG); - dev_info(&h->pdev->dev, "TX(%u) RING HEAD: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_FBDNUM_REG); - dev_info(&h->pdev->dev, "TX(%u) RING FBDNUM: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_OFFSET_REG); - dev_info(&h->pdev->dev, "TX(%u) RING OFFSET: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_RING_PKTNUM_RECORD_REG); - dev_info(&h->pdev->dev, "TX(%u) RING PKTNUM: %u\n", i, value); - - value = readl_relaxed(ring->tqp->io_base + HNS3_RING_EN_REG); - dev_info(&h->pdev->dev, "TX/RX(%u) RING EN: %s\n", i, - value ? "enable" : "disable"); - - if (hnae3_ae_dev_tqp_txrx_indep_supported(ae_dev)) { - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_TX_EN_REG); - dev_info(&h->pdev->dev, "TX(%u) RING EN: %s\n", i, - value ? "enable" : "disable"); - - value = readl_relaxed(ring->tqp->io_base + - HNS3_RING_RX_EN_REG); - dev_info(&h->pdev->dev, "RX(%u) RING EN: %s\n", i, - value ? "enable" : "disable"); - } + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_FBDNUM_REG)); + + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_OFFSET_REG)); + + sprintf(result[j++], "%6u", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_PKTNUM_RECORD_REG)); + + sprintf(result[j++], "%7s", readl_relaxed(ring->tqp->io_base + + HNS3_RING_EN_REG) ? "on" : "off"); + + if (hnae3_ae_dev_tqp_txrx_indep_supported(ae_dev)) + sprintf(result[j++], "%10s", readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_EN_REG) ? "on" : "off"); + else + sprintf(result[j++], "%10s", "NA"); + + base_add_h = readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_BASEADDR_H_REG); + base_add_l = readl_relaxed(ring->tqp->io_base + + HNS3_RING_TX_RING_BASEADDR_L_REG); + sprintf(result[j++], "0x%08x%08x", base_add_h, base_add_l); +} + +static int hns3_dbg_tx_queue_info(struct hnae3_handle *h, + char *buf, int len) +{ + char data_str[ARRAY_SIZE(tx_queue_info_items)][HNS3_DBG_DATA_STR_LEN]; + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); + char *result[ARRAY_SIZE(tx_queue_info_items)]; + struct hns3_nic_priv *priv = h->priv; + char content[HNS3_DBG_INFO_LEN]; + struct hns3_enet_ring *ring; + int pos = 0; + u32 i; + + if (!priv->ring) { + dev_err(&h->pdev->dev, "priv->ring is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < ARRAY_SIZE(tx_queue_info_items); i++) + result[i] = &data_str[i][0]; + + hns3_dbg_fill_content(content, sizeof(content), tx_queue_info_items, + NULL, ARRAY_SIZE(tx_queue_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + + for (i = 0; i < h->kinfo.num_tqps; i++) { + /* Each cycle needs to determine whether the instance is reset, + * to prevent reference to invalid memory. And need to ensure + * that the following code is executed within 100ms. + */ + if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state) || + test_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) + return -EPERM; - dev_info(&h->pdev->dev, "\n"); + ring = &priv->ring[i]; + hns3_dump_tx_queue_info(ring, ae_dev, result, i); + hns3_dbg_fill_content(content, sizeof(content), + tx_queue_info_items, + (const char **)result, + ARRAY_SIZE(tx_queue_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); } + hns3_dbg_tx_spare_info(ring, buf, len, h->kinfo.num_tqps, &pos); + return 0; } -static int hns3_dbg_queue_map(struct hnae3_handle *h) +static const struct hns3_dbg_item queue_map_items[] = { + { "local_queue_id", 2 }, + { "global_queue_id", 2 }, + { "vector_id", 2 }, +}; + +static int hns3_dbg_queue_map(struct hnae3_handle *h, char *buf, int len) { + char data_str[ARRAY_SIZE(queue_map_items)][HNS3_DBG_DATA_STR_LEN]; + char *result[ARRAY_SIZE(queue_map_items)]; struct hns3_nic_priv *priv = h->priv; - int i; + char content[HNS3_DBG_INFO_LEN]; + int pos = 0; + int j; + u32 i; if (!h->ae_algo->ops->get_global_queue_id) return -EOPNOTSUPP; - dev_info(&h->pdev->dev, "map info for queue id and vector id\n"); - dev_info(&h->pdev->dev, - "local queue id | global queue id | vector id\n"); - for (i = 0; i < h->kinfo.num_tqps; i++) { - u16 global_qid; + for (i = 0; i < ARRAY_SIZE(queue_map_items); i++) + result[i] = &data_str[i][0]; - global_qid = h->ae_algo->ops->get_global_queue_id(h, i); + hns3_dbg_fill_content(content, sizeof(content), queue_map_items, + NULL, ARRAY_SIZE(queue_map_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + for (i = 0; i < h->kinfo.num_tqps; i++) { if (!priv->ring || !priv->ring[i].tqp_vector) continue; - - dev_info(&h->pdev->dev, - " %4d %4u %4d\n", - i, global_qid, priv->ring[i].tqp_vector->vector_irq); + j = 0; + sprintf(result[j++], "%u", i); + sprintf(result[j++], "%u", + h->ae_algo->ops->get_global_queue_id(h, i)); + sprintf(result[j++], "%u", + priv->ring[i].tqp_vector->vector_irq); + hns3_dbg_fill_content(content, sizeof(content), queue_map_items, + (const char **)result, + ARRAY_SIZE(queue_map_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); } return 0; } -static int hns3_dbg_bd_info(struct hnae3_handle *h, const char *cmd_buf) +static const struct hns3_dbg_item rx_bd_info_items[] = { + { "BD_IDX", 3 }, + { "L234_INFO", 2 }, + { "PKT_LEN", 3 }, + { "SIZE", 4 }, + { "RSS_HASH", 4 }, + { "FD_ID", 2 }, + { "VLAN_TAG", 2 }, + { "O_DM_VLAN_ID_FB", 2 }, + { "OT_VLAN_TAG", 2 }, + { "BD_BASE_INFO", 2 }, + { "PTYPE", 2 }, + { "HW_CSUM", 2 }, +}; + +static void hns3_dump_rx_bd_info(struct hns3_nic_priv *priv, + struct hns3_desc *desc, char **result, int idx) { - struct hns3_nic_priv *priv = h->priv; - struct hns3_desc *rx_desc, *tx_desc; - struct device *dev = &h->pdev->dev; - struct hns3_enet_ring *ring; - u32 tx_index, rx_index; - u32 q_num, value; - dma_addr_t addr; - u16 mss_hw_csum; - u32 l234info; - int cnt; - - cnt = sscanf(&cmd_buf[8], "%u %u", &q_num, &tx_index); - if (cnt == 2) { - rx_index = tx_index; - } else if (cnt != 1) { - dev_err(dev, "bd info: bad command string, cnt=%d\n", cnt); - return -EINVAL; + unsigned int j = 0; + + sprintf(result[j++], "%5d", idx); + sprintf(result[j++], "%#x", le32_to_cpu(desc->rx.l234_info)); + sprintf(result[j++], "%7u", le16_to_cpu(desc->rx.pkt_len)); + sprintf(result[j++], "%4u", le16_to_cpu(desc->rx.size)); + sprintf(result[j++], "%#x", le32_to_cpu(desc->rx.rss_hash)); + sprintf(result[j++], "%5u", le16_to_cpu(desc->rx.fd_id)); + sprintf(result[j++], "%8u", le16_to_cpu(desc->rx.vlan_tag)); + sprintf(result[j++], "%15u", le16_to_cpu(desc->rx.o_dm_vlan_id_fb)); + sprintf(result[j++], "%11u", le16_to_cpu(desc->rx.ot_vlan_tag)); + sprintf(result[j++], "%#x", le32_to_cpu(desc->rx.bd_base_info)); + if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) { + u32 ol_info = le32_to_cpu(desc->rx.ol_info); + + sprintf(result[j++], "%5lu", hnae3_get_field(ol_info, + HNS3_RXD_PTYPE_M, + HNS3_RXD_PTYPE_S)); + sprintf(result[j++], "%7u", le16_to_cpu(desc->csum)); + } else { + sprintf(result[j++], "NA"); + sprintf(result[j++], "NA"); } +} + +static int hns3_dbg_rx_bd_info(struct hns3_dbg_data *d, char *buf, int len) +{ + char data_str[ARRAY_SIZE(rx_bd_info_items)][HNS3_DBG_DATA_STR_LEN]; + struct hns3_nic_priv *priv = d->handle->priv; + char *result[ARRAY_SIZE(rx_bd_info_items)]; + char content[HNS3_DBG_INFO_LEN]; + struct hns3_enet_ring *ring; + struct hns3_desc *desc; + unsigned int i; + int pos = 0; - if (q_num >= h->kinfo.num_tqps) { - dev_err(dev, "Queue number(%u) is out of range(0-%u)\n", q_num, - h->kinfo.num_tqps - 1); + if (d->qid >= d->handle->kinfo.num_tqps) { + dev_err(&d->handle->pdev->dev, + "queue%u is not in use\n", d->qid); return -EINVAL; } - ring = &priv->ring[q_num]; - value = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG); - tx_index = (cnt == 1) ? value : tx_index; + for (i = 0; i < ARRAY_SIZE(rx_bd_info_items); i++) + result[i] = &data_str[i][0]; - if (tx_index >= ring->desc_num) { - dev_err(dev, "bd index(%u) is out of range(0-%u)\n", tx_index, - ring->desc_num - 1); - return -EINVAL; - } + pos += scnprintf(buf + pos, len - pos, + "Queue %u rx bd info:\n", d->qid); + hns3_dbg_fill_content(content, sizeof(content), rx_bd_info_items, + NULL, ARRAY_SIZE(rx_bd_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); - tx_desc = &ring->desc[tx_index]; - addr = le64_to_cpu(tx_desc->addr); - mss_hw_csum = le16_to_cpu(tx_desc->tx.mss_hw_csum); - dev_info(dev, "TX Queue Num: %u, BD Index: %u\n", q_num, tx_index); - dev_info(dev, "(TX)addr: %pad\n", &addr); - dev_info(dev, "(TX)vlan_tag: %u\n", le16_to_cpu(tx_desc->tx.vlan_tag)); - dev_info(dev, "(TX)send_size: %u\n", - le16_to_cpu(tx_desc->tx.send_size)); - - if (mss_hw_csum & BIT(HNS3_TXD_HW_CS_B)) { - u32 offset = le32_to_cpu(tx_desc->tx.ol_type_vlan_len_msec); - u32 start = le32_to_cpu(tx_desc->tx.type_cs_vlan_tso_len); - - dev_info(dev, "(TX)csum start: %u\n", - hnae3_get_field(start, - HNS3_TXD_CSUM_START_M, - HNS3_TXD_CSUM_START_S)); - dev_info(dev, "(TX)csum offset: %u\n", - hnae3_get_field(offset, - HNS3_TXD_CSUM_OFFSET_M, - HNS3_TXD_CSUM_OFFSET_S)); - } else { - dev_info(dev, "(TX)vlan_tso: %u\n", - tx_desc->tx.type_cs_vlan_tso); - dev_info(dev, "(TX)l2_len: %u\n", tx_desc->tx.l2_len); - dev_info(dev, "(TX)l3_len: %u\n", tx_desc->tx.l3_len); - dev_info(dev, "(TX)l4_len: %u\n", tx_desc->tx.l4_len); - dev_info(dev, "(TX)vlan_msec: %u\n", - tx_desc->tx.ol_type_vlan_msec); - dev_info(dev, "(TX)ol2_len: %u\n", tx_desc->tx.ol2_len); - dev_info(dev, "(TX)ol3_len: %u\n", tx_desc->tx.ol3_len); - dev_info(dev, "(TX)ol4_len: %u\n", tx_desc->tx.ol4_len); - } + ring = &priv->ring[d->qid + d->handle->kinfo.num_tqps]; + for (i = 0; i < ring->desc_num; i++) { + desc = &ring->desc[i]; - dev_info(dev, "(TX)vlan_tag: %u\n", - le16_to_cpu(tx_desc->tx.outer_vlan_tag)); - dev_info(dev, "(TX)tv: %u\n", le16_to_cpu(tx_desc->tx.tv)); - dev_info(dev, "(TX)paylen_ol4cs: %u\n", - le32_to_cpu(tx_desc->tx.paylen_ol4cs)); - dev_info(dev, "(TX)vld_ra_ri: %u\n", - le16_to_cpu(tx_desc->tx.bdtp_fe_sc_vld_ra_ri)); - dev_info(dev, "(TX)mss_hw_csum: %u\n", mss_hw_csum); - - ring = &priv->ring[q_num + h->kinfo.num_tqps]; - value = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_TAIL_REG); - rx_index = (cnt == 1) ? value : tx_index; - rx_desc = &ring->desc[rx_index]; - - addr = le64_to_cpu(rx_desc->addr); - l234info = le32_to_cpu(rx_desc->rx.l234_info); - dev_info(dev, "RX Queue Num: %u, BD Index: %u\n", q_num, rx_index); - dev_info(dev, "(RX)addr: %pad\n", &addr); - dev_info(dev, "(RX)l234_info: %u\n", l234info); - - if (l234info & BIT(HNS3_RXD_L2_CSUM_B)) { - u32 lo, hi; - - lo = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_L_M, - HNS3_RXD_L2_CSUM_L_S); - hi = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_H_M, - HNS3_RXD_L2_CSUM_H_S); - dev_info(dev, "(RX)csum: %u\n", lo | hi << 8); + hns3_dump_rx_bd_info(priv, desc, result, i); + hns3_dbg_fill_content(content, sizeof(content), + rx_bd_info_items, (const char **)result, + ARRAY_SIZE(rx_bd_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); } - dev_info(dev, "(RX)pkt_len: %u\n", le16_to_cpu(rx_desc->rx.pkt_len)); - dev_info(dev, "(RX)size: %u\n", le16_to_cpu(rx_desc->rx.size)); - dev_info(dev, "(RX)rss_hash: %u\n", le32_to_cpu(rx_desc->rx.rss_hash)); - dev_info(dev, "(RX)fd_id: %u\n", le16_to_cpu(rx_desc->rx.fd_id)); - dev_info(dev, "(RX)vlan_tag: %u\n", le16_to_cpu(rx_desc->rx.vlan_tag)); - dev_info(dev, "(RX)o_dm_vlan_id_fb: %u\n", - le16_to_cpu(rx_desc->rx.o_dm_vlan_id_fb)); - dev_info(dev, "(RX)ot_vlan_tag: %u\n", - le16_to_cpu(rx_desc->rx.ot_vlan_tag)); - dev_info(dev, "(RX)bd_base_info: %u\n", - le32_to_cpu(rx_desc->rx.bd_base_info)); - return 0; } -static void hns3_dbg_help(struct hnae3_handle *h) -{ -#define HNS3_DBG_BUF_LEN 256 - - char printf_buf[HNS3_DBG_BUF_LEN]; - - dev_info(&h->pdev->dev, "available commands\n"); - dev_info(&h->pdev->dev, "queue info <number>\n"); - dev_info(&h->pdev->dev, "queue map\n"); - dev_info(&h->pdev->dev, "bd info <q_num> <bd index>\n"); - dev_info(&h->pdev->dev, "dev capability\n"); - dev_info(&h->pdev->dev, "dev spec\n"); - - if (!hns3_is_phys_func(h->pdev)) - return; - - dev_info(&h->pdev->dev, "dump fd tcam\n"); - dev_info(&h->pdev->dev, "dump tc\n"); - dev_info(&h->pdev->dev, "dump tm map <q_num>\n"); - dev_info(&h->pdev->dev, "dump tm\n"); - dev_info(&h->pdev->dev, "dump qos pause cfg\n"); - dev_info(&h->pdev->dev, "dump qos pri map\n"); - dev_info(&h->pdev->dev, "dump qos buf cfg\n"); - dev_info(&h->pdev->dev, "dump mng tbl\n"); - dev_info(&h->pdev->dev, "dump reset info\n"); - dev_info(&h->pdev->dev, "dump m7 info\n"); - dev_info(&h->pdev->dev, "dump ncl_config <offset> <length>(in hex)\n"); - dev_info(&h->pdev->dev, "dump mac tnl status\n"); - dev_info(&h->pdev->dev, "dump loopback\n"); - dev_info(&h->pdev->dev, "dump qs shaper [qs id]\n"); - dev_info(&h->pdev->dev, "dump uc mac list <func id>\n"); - dev_info(&h->pdev->dev, "dump mc mac list <func id>\n"); - dev_info(&h->pdev->dev, "dump intr\n"); - - memset(printf_buf, 0, HNS3_DBG_BUF_LEN); - strncat(printf_buf, "dump reg [[bios common] [ssu <port_id>]", - HNS3_DBG_BUF_LEN - 1); - strncat(printf_buf + strlen(printf_buf), - " [igu egu <port_id>] [rpu <tc_queue_num>]", - HNS3_DBG_BUF_LEN - strlen(printf_buf) - 1); - strncat(printf_buf + strlen(printf_buf), - " [rtc] [ppp] [rcb] [tqp <queue_num>] [mac]]\n", - HNS3_DBG_BUF_LEN - strlen(printf_buf) - 1); - dev_info(&h->pdev->dev, "%s", printf_buf); - - memset(printf_buf, 0, HNS3_DBG_BUF_LEN); - strncat(printf_buf, "dump reg dcb <port_id> <pri_id> <pg_id>", - HNS3_DBG_BUF_LEN - 1); - strncat(printf_buf + strlen(printf_buf), " <rq_id> <nq_id> <qset_id>\n", - HNS3_DBG_BUF_LEN - strlen(printf_buf) - 1); - dev_info(&h->pdev->dev, "%s", printf_buf); -} +static const struct hns3_dbg_item tx_bd_info_items[] = { + { "BD_IDX", 5 }, + { "ADDRESS", 2 }, + { "VLAN_TAG", 2 }, + { "SIZE", 2 }, + { "T_CS_VLAN_TSO", 2 }, + { "OT_VLAN_TAG", 3 }, + { "TV", 2 }, + { "OLT_VLAN_LEN", 2}, + { "PAYLEN_OL4CS", 2}, + { "BD_FE_SC_VLD", 2}, + { "MSS_HW_CSUM", 0}, +}; -static void hns3_dbg_dev_caps(struct hnae3_handle *h) +static void hns3_dump_tx_bd_info(struct hns3_nic_priv *priv, + struct hns3_desc *desc, char **result, int idx) { - struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); - unsigned long *caps; - - caps = ae_dev->caps; - - dev_info(&h->pdev->dev, "support FD: %s\n", - test_bit(HNAE3_DEV_SUPPORT_FD_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support GRO: %s\n", - test_bit(HNAE3_DEV_SUPPORT_GRO_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support FEC: %s\n", - test_bit(HNAE3_DEV_SUPPORT_FEC_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support UDP GSO: %s\n", - test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support PTP: %s\n", - test_bit(HNAE3_DEV_SUPPORT_PTP_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support INT QL: %s\n", - test_bit(HNAE3_DEV_SUPPORT_INT_QL_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support HW TX csum: %s\n", - test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, caps) ? "yes" : "no"); - dev_info(&h->pdev->dev, "support UDP tunnel csum: %s\n", - test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, caps) ? - "yes" : "no"); - dev_info(&h->pdev->dev, "support PAUSE: %s\n", - test_bit(HNAE3_DEV_SUPPORT_PAUSE_B, ae_dev->caps) ? - "yes" : "no"); - dev_info(&h->pdev->dev, "support imp-controlled PHY: %s\n", - test_bit(HNAE3_DEV_SUPPORT_PHY_IMP_B, caps) ? "yes" : "no"); + unsigned int j = 0; + + sprintf(result[j++], "%6d", idx); + sprintf(result[j++], "%#llx", le64_to_cpu(desc->addr)); + sprintf(result[j++], "%5u", le16_to_cpu(desc->tx.vlan_tag)); + sprintf(result[j++], "%5u", le16_to_cpu(desc->tx.send_size)); + sprintf(result[j++], "%#x", + le32_to_cpu(desc->tx.type_cs_vlan_tso_len)); + sprintf(result[j++], "%5u", le16_to_cpu(desc->tx.outer_vlan_tag)); + sprintf(result[j++], "%5u", le16_to_cpu(desc->tx.tv)); + sprintf(result[j++], "%10u", + le32_to_cpu(desc->tx.ol_type_vlan_len_msec)); + sprintf(result[j++], "%#x", le32_to_cpu(desc->tx.paylen_ol4cs)); + sprintf(result[j++], "%#x", le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri)); + sprintf(result[j++], "%5u", le16_to_cpu(desc->tx.mss_hw_csum)); } -static void hns3_dbg_dev_specs(struct hnae3_handle *h) +static int hns3_dbg_tx_bd_info(struct hns3_dbg_data *d, char *buf, int len) { - struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); - struct hnae3_dev_specs *dev_specs = &ae_dev->dev_specs; - struct hnae3_knic_private_info *kinfo = &h->kinfo; - struct hns3_nic_priv *priv = h->priv; - - dev_info(priv->dev, "MAC entry num: %u\n", dev_specs->mac_entry_num); - dev_info(priv->dev, "MNG entry num: %u\n", dev_specs->mng_entry_num); - dev_info(priv->dev, "MAX non tso bd num: %u\n", - dev_specs->max_non_tso_bd_num); - dev_info(priv->dev, "RSS ind tbl size: %u\n", - dev_specs->rss_ind_tbl_size); - dev_info(priv->dev, "RSS key size: %u\n", dev_specs->rss_key_size); - dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size); - dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size); - dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps); - - dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len); - dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc); - dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc); - dev_info(priv->dev, "Total number of enabled TCs: %u\n", - kinfo->tc_info.num_tc); - dev_info(priv->dev, "MAX INT QL: %u\n", dev_specs->int_ql_max); - dev_info(priv->dev, "MAX INT GL: %u\n", dev_specs->max_int_gl); - dev_info(priv->dev, "MAX frame size: %u\n", dev_specs->max_frm_size); - dev_info(priv->dev, "MAX TM RATE: %uMbps\n", dev_specs->max_tm_rate); - dev_info(priv->dev, "MAX QSET number: %u\n", dev_specs->max_qset_num); -} + char data_str[ARRAY_SIZE(tx_bd_info_items)][HNS3_DBG_DATA_STR_LEN]; + struct hns3_nic_priv *priv = d->handle->priv; + char *result[ARRAY_SIZE(tx_bd_info_items)]; + char content[HNS3_DBG_INFO_LEN]; + struct hns3_enet_ring *ring; + struct hns3_desc *desc; + unsigned int i; + int pos = 0; -static ssize_t hns3_dbg_cmd_read(struct file *filp, char __user *buffer, - size_t count, loff_t *ppos) -{ - int uncopy_bytes; - char *buf; - int len; + if (d->qid >= d->handle->kinfo.num_tqps) { + dev_err(&d->handle->pdev->dev, + "queue%u is not in use\n", d->qid); + return -EINVAL; + } - if (*ppos != 0) - return 0; + for (i = 0; i < ARRAY_SIZE(tx_bd_info_items); i++) + result[i] = &data_str[i][0]; - if (count < HNS3_DBG_READ_LEN) - return -ENOSPC; + pos += scnprintf(buf + pos, len - pos, + "Queue %u tx bd info:\n", d->qid); + hns3_dbg_fill_content(content, sizeof(content), tx_bd_info_items, + NULL, ARRAY_SIZE(tx_bd_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); - buf = kzalloc(HNS3_DBG_READ_LEN, GFP_KERNEL); - if (!buf) - return -ENOMEM; + ring = &priv->ring[d->qid]; + for (i = 0; i < ring->desc_num; i++) { + desc = &ring->desc[i]; - len = scnprintf(buf, HNS3_DBG_READ_LEN, "%s\n", - "Please echo help to cmd to get help information"); - uncopy_bytes = copy_to_user(buffer, buf, len); + hns3_dump_tx_bd_info(priv, desc, result, i); + hns3_dbg_fill_content(content, sizeof(content), + tx_bd_info_items, (const char **)result, + ARRAY_SIZE(tx_bd_info_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + } - kfree(buf); + return 0; +} - if (uncopy_bytes) - return -EFAULT; +static void +hns3_dbg_dev_caps(struct hnae3_handle *h, char *buf, int len, int *pos) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); + static const char * const str[] = {"no", "yes"}; + unsigned long *caps = ae_dev->caps; + u32 i, state; + + *pos += scnprintf(buf + *pos, len - *pos, "dev capability:\n"); + + for (i = 0; i < ARRAY_SIZE(hns3_dbg_cap); i++) { + state = test_bit(hns3_dbg_cap[i].cap_bit, caps); + *pos += scnprintf(buf + *pos, len - *pos, "%s: %s\n", + hns3_dbg_cap[i].name, str[state]); + } - return (*ppos = len); + *pos += scnprintf(buf + *pos, len - *pos, "\n"); } -static int hns3_dbg_check_cmd(struct hnae3_handle *handle, char *cmd_buf) +static void +hns3_dbg_dev_specs(struct hnae3_handle *h, char *buf, int len, int *pos) { - int ret = 0; - - if (strncmp(cmd_buf, "help", 4) == 0) - hns3_dbg_help(handle); - else if (strncmp(cmd_buf, "queue info", 10) == 0) - ret = hns3_dbg_queue_info(handle, cmd_buf); - else if (strncmp(cmd_buf, "queue map", 9) == 0) - ret = hns3_dbg_queue_map(handle); - else if (strncmp(cmd_buf, "bd info", 7) == 0) - ret = hns3_dbg_bd_info(handle, cmd_buf); - else if (strncmp(cmd_buf, "dev capability", 14) == 0) - hns3_dbg_dev_caps(handle); - else if (strncmp(cmd_buf, "dev spec", 8) == 0) - hns3_dbg_dev_specs(handle); - else if (handle->ae_algo->ops->dbg_run_cmd) - ret = handle->ae_algo->ops->dbg_run_cmd(handle, cmd_buf); - else - ret = -EOPNOTSUPP; + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); + struct hnae3_dev_specs *dev_specs = &ae_dev->dev_specs; + struct hnae3_knic_private_info *kinfo = &h->kinfo; - return ret; + *pos += scnprintf(buf + *pos, len - *pos, "dev_spec:\n"); + *pos += scnprintf(buf + *pos, len - *pos, "MAC entry num: %u\n", + dev_specs->mac_entry_num); + *pos += scnprintf(buf + *pos, len - *pos, "MNG entry num: %u\n", + dev_specs->mng_entry_num); + *pos += scnprintf(buf + *pos, len - *pos, "MAX non tso bd num: %u\n", + dev_specs->max_non_tso_bd_num); + *pos += scnprintf(buf + *pos, len - *pos, "RSS ind tbl size: %u\n", + dev_specs->rss_ind_tbl_size); + *pos += scnprintf(buf + *pos, len - *pos, "RSS key size: %u\n", + dev_specs->rss_key_size); + *pos += scnprintf(buf + *pos, len - *pos, "RSS size: %u\n", + kinfo->rss_size); + *pos += scnprintf(buf + *pos, len - *pos, "Allocated RSS size: %u\n", + kinfo->req_rss_size); + *pos += scnprintf(buf + *pos, len - *pos, + "Task queue pairs numbers: %u\n", + kinfo->num_tqps); + *pos += scnprintf(buf + *pos, len - *pos, "RX buffer length: %u\n", + kinfo->rx_buf_len); + *pos += scnprintf(buf + *pos, len - *pos, "Desc num per TX queue: %u\n", + kinfo->num_tx_desc); + *pos += scnprintf(buf + *pos, len - *pos, "Desc num per RX queue: %u\n", + kinfo->num_rx_desc); + *pos += scnprintf(buf + *pos, len - *pos, + "Total number of enabled TCs: %u\n", + kinfo->tc_info.num_tc); + *pos += scnprintf(buf + *pos, len - *pos, "MAX INT QL: %u\n", + dev_specs->int_ql_max); + *pos += scnprintf(buf + *pos, len - *pos, "MAX INT GL: %u\n", + dev_specs->max_int_gl); + *pos += scnprintf(buf + *pos, len - *pos, "MAX TM RATE: %u\n", + dev_specs->max_tm_rate); + *pos += scnprintf(buf + *pos, len - *pos, "MAX QSET number: %u\n", + dev_specs->max_qset_num); } -static ssize_t hns3_dbg_cmd_write(struct file *filp, const char __user *buffer, - size_t count, loff_t *ppos) +static int hns3_dbg_dev_info(struct hnae3_handle *h, char *buf, int len) { - struct hnae3_handle *handle = filp->private_data; - struct hns3_nic_priv *priv = handle->priv; - char *cmd_buf, *cmd_buf_tmp; - int uncopied_bytes; - int ret; + int pos = 0; - if (*ppos != 0) - return 0; + hns3_dbg_dev_caps(h, buf, len, &pos); - /* Judge if the instance is being reset. */ - if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state) || - test_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) - return 0; + hns3_dbg_dev_specs(h, buf, len, &pos); - if (count > HNS3_DBG_WRITE_LEN) - return -ENOSPC; + return 0; +} - cmd_buf = kzalloc(count + 1, GFP_KERNEL); - if (!cmd_buf) - return count; +static int hns3_dbg_get_cmd_index(struct hnae3_handle *handle, + const unsigned char *name, u32 *index) +{ + u32 i; - uncopied_bytes = copy_from_user(cmd_buf, buffer, count); - if (uncopied_bytes) { - kfree(cmd_buf); - return -EFAULT; + for (i = 0; i < ARRAY_SIZE(hns3_dbg_cmd); i++) { + if (!strncmp(name, hns3_dbg_cmd[i].name, + strlen(hns3_dbg_cmd[i].name))) { + *index = i; + return 0; + } } - cmd_buf[count] = '\0'; + dev_err(&handle->pdev->dev, "unknown command(%s)\n", name); + return -EINVAL; +} - cmd_buf_tmp = strchr(cmd_buf, '\n'); - if (cmd_buf_tmp) { - *cmd_buf_tmp = '\0'; - count = cmd_buf_tmp - cmd_buf + 1; - } +static const struct hns3_dbg_func hns3_dbg_cmd_func[] = { + { + .cmd = HNAE3_DBG_CMD_QUEUE_MAP, + .dbg_dump = hns3_dbg_queue_map, + }, + { + .cmd = HNAE3_DBG_CMD_DEV_INFO, + .dbg_dump = hns3_dbg_dev_info, + }, + { + .cmd = HNAE3_DBG_CMD_TX_BD, + .dbg_dump_bd = hns3_dbg_tx_bd_info, + }, + { + .cmd = HNAE3_DBG_CMD_RX_BD, + .dbg_dump_bd = hns3_dbg_rx_bd_info, + }, + { + .cmd = HNAE3_DBG_CMD_RX_QUEUE_INFO, + .dbg_dump = hns3_dbg_rx_queue_info, + }, + { + .cmd = HNAE3_DBG_CMD_TX_QUEUE_INFO, + .dbg_dump = hns3_dbg_tx_queue_info, + }, +}; - ret = hns3_dbg_check_cmd(handle, cmd_buf); - if (ret) - hns3_dbg_help(handle); +static int hns3_dbg_read_cmd(struct hns3_dbg_data *dbg_data, + enum hnae3_dbg_cmd cmd, char *buf, int len) +{ + const struct hnae3_ae_ops *ops = dbg_data->handle->ae_algo->ops; + const struct hns3_dbg_func *cmd_func; + u32 i; + + for (i = 0; i < ARRAY_SIZE(hns3_dbg_cmd_func); i++) { + if (cmd == hns3_dbg_cmd_func[i].cmd) { + cmd_func = &hns3_dbg_cmd_func[i]; + if (cmd_func->dbg_dump) + return cmd_func->dbg_dump(dbg_data->handle, buf, + len); + else + return cmd_func->dbg_dump_bd(dbg_data, buf, + len); + } + } - kfree(cmd_buf); - cmd_buf = NULL; + if (!ops->dbg_read_cmd) + return -EOPNOTSUPP; - return count; + return ops->dbg_read_cmd(dbg_data->handle, cmd, buf, len); } static ssize_t hns3_dbg_read(struct file *filp, char __user *buffer, size_t count, loff_t *ppos) { - struct hnae3_handle *handle = filp->private_data; - const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + struct hns3_dbg_data *dbg_data = filp->private_data; + struct hnae3_handle *handle = dbg_data->handle; struct hns3_nic_priv *priv = handle->priv; - char *cmd_buf, *read_buf; ssize_t size = 0; - int ret = 0; - - read_buf = kzalloc(HNS3_DBG_READ_LEN, GFP_KERNEL); - if (!read_buf) - return -ENOMEM; + char **save_buf; + char *read_buf; + u32 index; + int ret; - cmd_buf = filp->f_path.dentry->d_iname; + ret = hns3_dbg_get_cmd_index(handle, filp->f_path.dentry->d_iname, + &index); + if (ret) + return ret; - if (ops->dbg_read_cmd) - ret = ops->dbg_read_cmd(handle, cmd_buf, read_buf, - HNS3_DBG_READ_LEN); + save_buf = &hns3_dbg_cmd[index].buf; - if (ret) { - dev_info(priv->dev, "unknown command\n"); + if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state) || + test_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) { + ret = -EBUSY; goto out; } + if (*save_buf) { + read_buf = *save_buf; + } else { + read_buf = kvzalloc(hns3_dbg_cmd[index].buf_len, GFP_KERNEL); + if (!read_buf) + return -ENOMEM; + + /* save the buffer addr until the last read operation */ + *save_buf = read_buf; + } + + /* get data ready for the first time to read */ + if (!*ppos) { + ret = hns3_dbg_read_cmd(dbg_data, hns3_dbg_cmd[index].cmd, + read_buf, hns3_dbg_cmd[index].buf_len); + if (ret) + goto out; + } + size = simple_read_from_buffer(buffer, count, ppos, read_buf, strlen(read_buf)); + if (size > 0) + return size; out: - kfree(read_buf); - return size; -} + /* free the buffer for the last read operation */ + if (*save_buf) { + kvfree(*save_buf); + *save_buf = NULL; + } -static const struct file_operations hns3_dbg_cmd_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = hns3_dbg_cmd_read, - .write = hns3_dbg_cmd_write, -}; + return ret; +} static const struct file_operations hns3_dbg_fops = { .owner = THIS_MODULE, @@ -546,29 +1072,108 @@ static const struct file_operations hns3_dbg_fops = { .read = hns3_dbg_read, }; -void hns3_dbg_init(struct hnae3_handle *handle) +static int hns3_dbg_bd_file_init(struct hnae3_handle *handle, u32 cmd) +{ + struct dentry *entry_dir; + struct hns3_dbg_data *data; + u16 max_queue_num; + unsigned int i; + + entry_dir = hns3_dbg_dentry[hns3_dbg_cmd[cmd].dentry].dentry; + max_queue_num = hns3_get_max_available_channels(handle); + data = devm_kzalloc(&handle->pdev->dev, max_queue_num * sizeof(*data), + GFP_KERNEL); + if (!data) + return -ENOMEM; + + for (i = 0; i < max_queue_num; i++) { + char name[HNS3_DBG_FILE_NAME_LEN]; + + data[i].handle = handle; + data[i].qid = i; + sprintf(name, "%s%u", hns3_dbg_cmd[cmd].name, i); + debugfs_create_file(name, 0400, entry_dir, &data[i], + &hns3_dbg_fops); + } + + return 0; +} + +static int +hns3_dbg_common_file_init(struct hnae3_handle *handle, u32 cmd) +{ + struct hns3_dbg_data *data; + struct dentry *entry_dir; + + data = devm_kzalloc(&handle->pdev->dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->handle = handle; + entry_dir = hns3_dbg_dentry[hns3_dbg_cmd[cmd].dentry].dentry; + debugfs_create_file(hns3_dbg_cmd[cmd].name, 0400, entry_dir, + data, &hns3_dbg_fops); + + return 0; +} + +int hns3_dbg_init(struct hnae3_handle *handle) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev); const char *name = pci_name(handle->pdev); - struct dentry *entry_dir; + int ret; + u32 i; + + hns3_dbg_dentry[HNS3_DBG_DENTRY_COMMON].dentry = + debugfs_create_dir(name, hns3_dbgfs_root); + handle->hnae3_dbgfs = hns3_dbg_dentry[HNS3_DBG_DENTRY_COMMON].dentry; + + for (i = 0; i < HNS3_DBG_DENTRY_COMMON; i++) + hns3_dbg_dentry[i].dentry = + debugfs_create_dir(hns3_dbg_dentry[i].name, + handle->hnae3_dbgfs); + + for (i = 0; i < ARRAY_SIZE(hns3_dbg_cmd); i++) { + if ((hns3_dbg_cmd[i].cmd == HNAE3_DBG_CMD_TM_NODES && + ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2) || + (hns3_dbg_cmd[i].cmd == HNAE3_DBG_CMD_PTP_INFO && + !test_bit(HNAE3_DEV_SUPPORT_PTP_B, ae_dev->caps))) + continue; - handle->hnae3_dbgfs = debugfs_create_dir(name, hns3_dbgfs_root); + if (!hns3_dbg_cmd[i].init) { + dev_err(&handle->pdev->dev, + "cmd %s lack of init func\n", + hns3_dbg_cmd[i].name); + ret = -EINVAL; + goto out; + } - debugfs_create_file("cmd", 0600, handle->hnae3_dbgfs, handle, - &hns3_dbg_cmd_fops); + ret = hns3_dbg_cmd[i].init(handle, i); + if (ret) { + dev_err(&handle->pdev->dev, "failed to init cmd %s\n", + hns3_dbg_cmd[i].name); + goto out; + } + } - entry_dir = debugfs_create_dir("tm", handle->hnae3_dbgfs); - if (ae_dev->dev_version > HNAE3_DEVICE_VERSION_V2) - debugfs_create_file(HNAE3_DBG_TM_NODES, 0600, entry_dir, handle, - &hns3_dbg_fops); - debugfs_create_file(HNAE3_DBG_TM_PRI, 0600, entry_dir, handle, - &hns3_dbg_fops); - debugfs_create_file(HNAE3_DBG_TM_QSET, 0600, entry_dir, handle, - &hns3_dbg_fops); + return 0; + +out: + debugfs_remove_recursive(handle->hnae3_dbgfs); + handle->hnae3_dbgfs = NULL; + return ret; } void hns3_dbg_uninit(struct hnae3_handle *handle) { + u32 i; + + for (i = 0; i < ARRAY_SIZE(hns3_dbg_cmd); i++) + if (hns3_dbg_cmd[i].buf) { + kvfree(hns3_dbg_cmd[i].buf); + hns3_dbg_cmd[i].buf = NULL; + } + debugfs_remove_recursive(handle->hnae3_dbgfs); handle->hnae3_dbgfs = NULL; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h new file mode 100644 index 000000000000..f3766ff38bb7 --- /dev/null +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Copyright (c) 2021 Hisilicon Limited. */ + +#ifndef __HNS3_DEBUGFS_H +#define __HNS3_DEBUGFS_H + +#define HNS3_DBG_READ_LEN 65536 +#define HNS3_DBG_READ_LEN_128KB 0x20000 +#define HNS3_DBG_READ_LEN_1MB 0x100000 +#define HNS3_DBG_READ_LEN_4MB 0x400000 +#define HNS3_DBG_WRITE_LEN 1024 + +#define HNS3_DBG_DATA_STR_LEN 32 +#define HNS3_DBG_INFO_LEN 256 +#define HNS3_DBG_ITEM_NAME_LEN 32 +#define HNS3_DBG_FILE_NAME_LEN 16 + +struct hns3_dbg_item { + char name[HNS3_DBG_ITEM_NAME_LEN]; + u16 interval; /* blank numbers after the item */ +}; + +struct hns3_dbg_data { + struct hnae3_handle *handle; + u16 qid; +}; + +enum hns3_dbg_dentry_type { + HNS3_DBG_DENTRY_TM, + HNS3_DBG_DENTRY_TX_BD, + HNS3_DBG_DENTRY_RX_BD, + HNS3_DBG_DENTRY_MAC, + HNS3_DBG_DENTRY_REG, + HNS3_DBG_DENTRY_QUEUE, + HNS3_DBG_DENTRY_FD, + HNS3_DBG_DENTRY_COMMON, +}; + +struct hns3_dbg_dentry_info { + const char *name; + struct dentry *dentry; +}; + +struct hns3_dbg_cmd_info { + const char *name; + enum hnae3_dbg_cmd cmd; + enum hns3_dbg_dentry_type dentry; + u32 buf_len; + char *buf; + int (*init)(struct hnae3_handle *handle, unsigned int cmd); +}; + +struct hns3_dbg_func { + enum hnae3_dbg_cmd cmd; + int (*dbg_dump)(struct hnae3_handle *handle, char *buf, int len); + int (*dbg_dump_bd)(struct hns3_dbg_data *data, char *buf, int len); +}; + +struct hns3_dbg_cap_info { + const char *name; + enum HNAE3_DEV_CAP_BITS cap_bit; +}; + +#endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 783fdaf8f8d6..cdb5f14fb6bc 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -53,6 +53,19 @@ static int debug = -1; module_param(debug, int, 0); MODULE_PARM_DESC(debug, " Network interface message level setting"); +static unsigned int tx_spare_buf_size; +module_param(tx_spare_buf_size, uint, 0400); +MODULE_PARM_DESC(tx_spare_buf_size, "Size used to allocate tx spare buffer"); + +static unsigned int tx_sgl = 1; +module_param(tx_sgl, uint, 0600); +MODULE_PARM_DESC(tx_sgl, "Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping"); + +#define HNS3_SGL_SIZE(nfrag) (sizeof(struct scatterlist) * (nfrag) + \ + sizeof(struct sg_table)) +#define HNS3_MAX_SGL_SIZE ALIGN(HNS3_SGL_SIZE(HNS3_MAX_TSO_BD_NUM),\ + dma_get_cache_alignment()) + #define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \ NETIF_MSG_IFDOWN | NETIF_MSG_IFUP) @@ -91,11 +104,284 @@ static const struct pci_device_id hns3_pci_tbl[] = { }; MODULE_DEVICE_TABLE(pci, hns3_pci_tbl); +#define HNS3_RX_PTYPE_ENTRY(ptype, l, s, t) \ + { ptype, \ + l, \ + CHECKSUM_##s, \ + HNS3_L3_TYPE_##t, \ + 1 } + +#define HNS3_RX_PTYPE_UNUSED_ENTRY(ptype) \ + { ptype, 0, CHECKSUM_NONE, HNS3_L3_TYPE_PARSE_FAIL, 0 } + +static const struct hns3_rx_ptype hns3_rx_ptype_tbl[] = { + HNS3_RX_PTYPE_UNUSED_ENTRY(0), + HNS3_RX_PTYPE_ENTRY(1, 0, COMPLETE, ARP), + HNS3_RX_PTYPE_ENTRY(2, 0, COMPLETE, RARP), + HNS3_RX_PTYPE_ENTRY(3, 0, COMPLETE, LLDP), + HNS3_RX_PTYPE_ENTRY(4, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(5, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(6, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(7, 0, COMPLETE, CNM), + HNS3_RX_PTYPE_ENTRY(8, 0, NONE, PARSE_FAIL), + HNS3_RX_PTYPE_UNUSED_ENTRY(9), + HNS3_RX_PTYPE_UNUSED_ENTRY(10), + HNS3_RX_PTYPE_UNUSED_ENTRY(11), + HNS3_RX_PTYPE_UNUSED_ENTRY(12), + HNS3_RX_PTYPE_UNUSED_ENTRY(13), + HNS3_RX_PTYPE_UNUSED_ENTRY(14), + HNS3_RX_PTYPE_UNUSED_ENTRY(15), + HNS3_RX_PTYPE_ENTRY(16, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(17, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(18, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(19, 0, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(20, 0, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(21, 0, NONE, IPV4), + HNS3_RX_PTYPE_ENTRY(22, 0, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(23, 0, NONE, IPV4), + HNS3_RX_PTYPE_ENTRY(24, 0, NONE, IPV4), + HNS3_RX_PTYPE_ENTRY(25, 0, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_UNUSED_ENTRY(26), + HNS3_RX_PTYPE_UNUSED_ENTRY(27), + HNS3_RX_PTYPE_UNUSED_ENTRY(28), + HNS3_RX_PTYPE_ENTRY(29, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(30, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(31, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(32, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(33, 1, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(34, 1, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(35, 1, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(36, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(37, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_UNUSED_ENTRY(38), + HNS3_RX_PTYPE_ENTRY(39, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(40, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(41, 1, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(42, 1, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(43, 1, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(44, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(45, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_UNUSED_ENTRY(46), + HNS3_RX_PTYPE_UNUSED_ENTRY(47), + HNS3_RX_PTYPE_UNUSED_ENTRY(48), + HNS3_RX_PTYPE_UNUSED_ENTRY(49), + HNS3_RX_PTYPE_UNUSED_ENTRY(50), + HNS3_RX_PTYPE_UNUSED_ENTRY(51), + HNS3_RX_PTYPE_UNUSED_ENTRY(52), + HNS3_RX_PTYPE_UNUSED_ENTRY(53), + HNS3_RX_PTYPE_UNUSED_ENTRY(54), + HNS3_RX_PTYPE_UNUSED_ENTRY(55), + HNS3_RX_PTYPE_UNUSED_ENTRY(56), + HNS3_RX_PTYPE_UNUSED_ENTRY(57), + HNS3_RX_PTYPE_UNUSED_ENTRY(58), + HNS3_RX_PTYPE_UNUSED_ENTRY(59), + HNS3_RX_PTYPE_UNUSED_ENTRY(60), + HNS3_RX_PTYPE_UNUSED_ENTRY(61), + HNS3_RX_PTYPE_UNUSED_ENTRY(62), + HNS3_RX_PTYPE_UNUSED_ENTRY(63), + HNS3_RX_PTYPE_UNUSED_ENTRY(64), + HNS3_RX_PTYPE_UNUSED_ENTRY(65), + HNS3_RX_PTYPE_UNUSED_ENTRY(66), + HNS3_RX_PTYPE_UNUSED_ENTRY(67), + HNS3_RX_PTYPE_UNUSED_ENTRY(68), + HNS3_RX_PTYPE_UNUSED_ENTRY(69), + HNS3_RX_PTYPE_UNUSED_ENTRY(70), + HNS3_RX_PTYPE_UNUSED_ENTRY(71), + HNS3_RX_PTYPE_UNUSED_ENTRY(72), + HNS3_RX_PTYPE_UNUSED_ENTRY(73), + HNS3_RX_PTYPE_UNUSED_ENTRY(74), + HNS3_RX_PTYPE_UNUSED_ENTRY(75), + HNS3_RX_PTYPE_UNUSED_ENTRY(76), + HNS3_RX_PTYPE_UNUSED_ENTRY(77), + HNS3_RX_PTYPE_UNUSED_ENTRY(78), + HNS3_RX_PTYPE_UNUSED_ENTRY(79), + HNS3_RX_PTYPE_UNUSED_ENTRY(80), + HNS3_RX_PTYPE_UNUSED_ENTRY(81), + HNS3_RX_PTYPE_UNUSED_ENTRY(82), + HNS3_RX_PTYPE_UNUSED_ENTRY(83), + HNS3_RX_PTYPE_UNUSED_ENTRY(84), + HNS3_RX_PTYPE_UNUSED_ENTRY(85), + HNS3_RX_PTYPE_UNUSED_ENTRY(86), + HNS3_RX_PTYPE_UNUSED_ENTRY(87), + HNS3_RX_PTYPE_UNUSED_ENTRY(88), + HNS3_RX_PTYPE_UNUSED_ENTRY(89), + HNS3_RX_PTYPE_UNUSED_ENTRY(90), + HNS3_RX_PTYPE_UNUSED_ENTRY(91), + HNS3_RX_PTYPE_UNUSED_ENTRY(92), + HNS3_RX_PTYPE_UNUSED_ENTRY(93), + HNS3_RX_PTYPE_UNUSED_ENTRY(94), + HNS3_RX_PTYPE_UNUSED_ENTRY(95), + HNS3_RX_PTYPE_UNUSED_ENTRY(96), + HNS3_RX_PTYPE_UNUSED_ENTRY(97), + HNS3_RX_PTYPE_UNUSED_ENTRY(98), + HNS3_RX_PTYPE_UNUSED_ENTRY(99), + HNS3_RX_PTYPE_UNUSED_ENTRY(100), + HNS3_RX_PTYPE_UNUSED_ENTRY(101), + HNS3_RX_PTYPE_UNUSED_ENTRY(102), + HNS3_RX_PTYPE_UNUSED_ENTRY(103), + HNS3_RX_PTYPE_UNUSED_ENTRY(104), + HNS3_RX_PTYPE_UNUSED_ENTRY(105), + HNS3_RX_PTYPE_UNUSED_ENTRY(106), + HNS3_RX_PTYPE_UNUSED_ENTRY(107), + HNS3_RX_PTYPE_UNUSED_ENTRY(108), + HNS3_RX_PTYPE_UNUSED_ENTRY(109), + HNS3_RX_PTYPE_UNUSED_ENTRY(110), + HNS3_RX_PTYPE_ENTRY(111, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(112, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(113, 0, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(114, 0, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(115, 0, NONE, IPV6), + HNS3_RX_PTYPE_ENTRY(116, 0, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(117, 0, NONE, IPV6), + HNS3_RX_PTYPE_ENTRY(118, 0, NONE, IPV6), + HNS3_RX_PTYPE_ENTRY(119, 0, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_UNUSED_ENTRY(120), + HNS3_RX_PTYPE_UNUSED_ENTRY(121), + HNS3_RX_PTYPE_UNUSED_ENTRY(122), + HNS3_RX_PTYPE_ENTRY(123, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(124, 0, COMPLETE, PARSE_FAIL), + HNS3_RX_PTYPE_ENTRY(125, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(126, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(127, 1, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(128, 1, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(129, 1, UNNECESSARY, IPV4), + HNS3_RX_PTYPE_ENTRY(130, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_ENTRY(131, 0, COMPLETE, IPV4), + HNS3_RX_PTYPE_UNUSED_ENTRY(132), + HNS3_RX_PTYPE_ENTRY(133, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(134, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(135, 1, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(136, 1, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(137, 1, UNNECESSARY, IPV6), + HNS3_RX_PTYPE_ENTRY(138, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_ENTRY(139, 0, COMPLETE, IPV6), + HNS3_RX_PTYPE_UNUSED_ENTRY(140), + HNS3_RX_PTYPE_UNUSED_ENTRY(141), + HNS3_RX_PTYPE_UNUSED_ENTRY(142), + HNS3_RX_PTYPE_UNUSED_ENTRY(143), + HNS3_RX_PTYPE_UNUSED_ENTRY(144), + HNS3_RX_PTYPE_UNUSED_ENTRY(145), + HNS3_RX_PTYPE_UNUSED_ENTRY(146), + HNS3_RX_PTYPE_UNUSED_ENTRY(147), + HNS3_RX_PTYPE_UNUSED_ENTRY(148), + HNS3_RX_PTYPE_UNUSED_ENTRY(149), + HNS3_RX_PTYPE_UNUSED_ENTRY(150), + HNS3_RX_PTYPE_UNUSED_ENTRY(151), + HNS3_RX_PTYPE_UNUSED_ENTRY(152), + HNS3_RX_PTYPE_UNUSED_ENTRY(153), + HNS3_RX_PTYPE_UNUSED_ENTRY(154), + HNS3_RX_PTYPE_UNUSED_ENTRY(155), + HNS3_RX_PTYPE_UNUSED_ENTRY(156), + HNS3_RX_PTYPE_UNUSED_ENTRY(157), + HNS3_RX_PTYPE_UNUSED_ENTRY(158), + HNS3_RX_PTYPE_UNUSED_ENTRY(159), + HNS3_RX_PTYPE_UNUSED_ENTRY(160), + HNS3_RX_PTYPE_UNUSED_ENTRY(161), + HNS3_RX_PTYPE_UNUSED_ENTRY(162), + HNS3_RX_PTYPE_UNUSED_ENTRY(163), + HNS3_RX_PTYPE_UNUSED_ENTRY(164), + HNS3_RX_PTYPE_UNUSED_ENTRY(165), + HNS3_RX_PTYPE_UNUSED_ENTRY(166), + HNS3_RX_PTYPE_UNUSED_ENTRY(167), + HNS3_RX_PTYPE_UNUSED_ENTRY(168), + HNS3_RX_PTYPE_UNUSED_ENTRY(169), + HNS3_RX_PTYPE_UNUSED_ENTRY(170), + HNS3_RX_PTYPE_UNUSED_ENTRY(171), + HNS3_RX_PTYPE_UNUSED_ENTRY(172), + HNS3_RX_PTYPE_UNUSED_ENTRY(173), + HNS3_RX_PTYPE_UNUSED_ENTRY(174), + HNS3_RX_PTYPE_UNUSED_ENTRY(175), + HNS3_RX_PTYPE_UNUSED_ENTRY(176), + HNS3_RX_PTYPE_UNUSED_ENTRY(177), + HNS3_RX_PTYPE_UNUSED_ENTRY(178), + HNS3_RX_PTYPE_UNUSED_ENTRY(179), + HNS3_RX_PTYPE_UNUSED_ENTRY(180), + HNS3_RX_PTYPE_UNUSED_ENTRY(181), + HNS3_RX_PTYPE_UNUSED_ENTRY(182), + HNS3_RX_PTYPE_UNUSED_ENTRY(183), + HNS3_RX_PTYPE_UNUSED_ENTRY(184), + HNS3_RX_PTYPE_UNUSED_ENTRY(185), + HNS3_RX_PTYPE_UNUSED_ENTRY(186), + HNS3_RX_PTYPE_UNUSED_ENTRY(187), + HNS3_RX_PTYPE_UNUSED_ENTRY(188), + HNS3_RX_PTYPE_UNUSED_ENTRY(189), + HNS3_RX_PTYPE_UNUSED_ENTRY(190), + HNS3_RX_PTYPE_UNUSED_ENTRY(191), + HNS3_RX_PTYPE_UNUSED_ENTRY(192), + HNS3_RX_PTYPE_UNUSED_ENTRY(193), + HNS3_RX_PTYPE_UNUSED_ENTRY(194), + HNS3_RX_PTYPE_UNUSED_ENTRY(195), + HNS3_RX_PTYPE_UNUSED_ENTRY(196), + HNS3_RX_PTYPE_UNUSED_ENTRY(197), + HNS3_RX_PTYPE_UNUSED_ENTRY(198), + HNS3_RX_PTYPE_UNUSED_ENTRY(199), + HNS3_RX_PTYPE_UNUSED_ENTRY(200), + HNS3_RX_PTYPE_UNUSED_ENTRY(201), + HNS3_RX_PTYPE_UNUSED_ENTRY(202), + HNS3_RX_PTYPE_UNUSED_ENTRY(203), + HNS3_RX_PTYPE_UNUSED_ENTRY(204), + HNS3_RX_PTYPE_UNUSED_ENTRY(205), + HNS3_RX_PTYPE_UNUSED_ENTRY(206), + HNS3_RX_PTYPE_UNUSED_ENTRY(207), + HNS3_RX_PTYPE_UNUSED_ENTRY(208), + HNS3_RX_PTYPE_UNUSED_ENTRY(209), + HNS3_RX_PTYPE_UNUSED_ENTRY(210), + HNS3_RX_PTYPE_UNUSED_ENTRY(211), + HNS3_RX_PTYPE_UNUSED_ENTRY(212), + HNS3_RX_PTYPE_UNUSED_ENTRY(213), + HNS3_RX_PTYPE_UNUSED_ENTRY(214), + HNS3_RX_PTYPE_UNUSED_ENTRY(215), + HNS3_RX_PTYPE_UNUSED_ENTRY(216), + HNS3_RX_PTYPE_UNUSED_ENTRY(217), + HNS3_RX_PTYPE_UNUSED_ENTRY(218), + HNS3_RX_PTYPE_UNUSED_ENTRY(219), + HNS3_RX_PTYPE_UNUSED_ENTRY(220), + HNS3_RX_PTYPE_UNUSED_ENTRY(221), + HNS3_RX_PTYPE_UNUSED_ENTRY(222), + HNS3_RX_PTYPE_UNUSED_ENTRY(223), + HNS3_RX_PTYPE_UNUSED_ENTRY(224), + HNS3_RX_PTYPE_UNUSED_ENTRY(225), + HNS3_RX_PTYPE_UNUSED_ENTRY(226), + HNS3_RX_PTYPE_UNUSED_ENTRY(227), + HNS3_RX_PTYPE_UNUSED_ENTRY(228), + HNS3_RX_PTYPE_UNUSED_ENTRY(229), + HNS3_RX_PTYPE_UNUSED_ENTRY(230), + HNS3_RX_PTYPE_UNUSED_ENTRY(231), + HNS3_RX_PTYPE_UNUSED_ENTRY(232), + HNS3_RX_PTYPE_UNUSED_ENTRY(233), + HNS3_RX_PTYPE_UNUSED_ENTRY(234), + HNS3_RX_PTYPE_UNUSED_ENTRY(235), + HNS3_RX_PTYPE_UNUSED_ENTRY(236), + HNS3_RX_PTYPE_UNUSED_ENTRY(237), + HNS3_RX_PTYPE_UNUSED_ENTRY(238), + HNS3_RX_PTYPE_UNUSED_ENTRY(239), + HNS3_RX_PTYPE_UNUSED_ENTRY(240), + HNS3_RX_PTYPE_UNUSED_ENTRY(241), + HNS3_RX_PTYPE_UNUSED_ENTRY(242), + HNS3_RX_PTYPE_UNUSED_ENTRY(243), + HNS3_RX_PTYPE_UNUSED_ENTRY(244), + HNS3_RX_PTYPE_UNUSED_ENTRY(245), + HNS3_RX_PTYPE_UNUSED_ENTRY(246), + HNS3_RX_PTYPE_UNUSED_ENTRY(247), + HNS3_RX_PTYPE_UNUSED_ENTRY(248), + HNS3_RX_PTYPE_UNUSED_ENTRY(249), + HNS3_RX_PTYPE_UNUSED_ENTRY(250), + HNS3_RX_PTYPE_UNUSED_ENTRY(251), + HNS3_RX_PTYPE_UNUSED_ENTRY(252), + HNS3_RX_PTYPE_UNUSED_ENTRY(253), + HNS3_RX_PTYPE_UNUSED_ENTRY(254), + HNS3_RX_PTYPE_UNUSED_ENTRY(255), +}; + +#define HNS3_INVALID_PTYPE \ + ARRAY_SIZE(hns3_rx_ptype_tbl) + static irqreturn_t hns3_irq_handle(int irq, void *vector) { struct hns3_enet_tqp_vector *tqp_vector = vector; napi_schedule_irqoff(&tqp_vector->napi); + tqp_vector->event_cnt++; return IRQ_HANDLED; } @@ -199,6 +485,8 @@ static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector) disable_irq(tqp_vector->vector_irq); napi_disable(&tqp_vector->napi); + cancel_work_sync(&tqp_vector->rx_group.dim.work); + cancel_work_sync(&tqp_vector->tx_group.dim.work); } void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector, @@ -264,22 +552,17 @@ static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector, struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev); struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal; struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal; + struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal; + struct hns3_enet_coalesce *prx_coal = &priv->rx_coal; - /* initialize the configuration for interrupt coalescing. - * 1. GL (Interrupt Gap Limiter) - * 2. RL (Interrupt Rate Limiter) - * 3. QL (Interrupt Quantity Limiter) - * - * Default: enable interrupt coalescing self-adaptive and GL - */ - tx_coal->adapt_enable = 1; - rx_coal->adapt_enable = 1; + tx_coal->adapt_enable = ptx_coal->adapt_enable; + rx_coal->adapt_enable = prx_coal->adapt_enable; - tx_coal->int_gl = HNS3_INT_GL_50K; - rx_coal->int_gl = HNS3_INT_GL_50K; + tx_coal->int_gl = ptx_coal->int_gl; + rx_coal->int_gl = prx_coal->int_gl; - rx_coal->flow_level = HNS3_FLOW_LOW; - tx_coal->flow_level = HNS3_FLOW_LOW; + rx_coal->flow_level = prx_coal->flow_level; + tx_coal->flow_level = ptx_coal->flow_level; /* device version above V3(include V3), GL can configure 1us * unit, so uses 1us unit. @@ -294,8 +577,8 @@ static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector, rx_coal->ql_enable = 1; tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max; rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max; - tx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG; - rx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG; + tx_coal->int_ql = ptx_coal->int_ql; + rx_coal->int_ql = prx_coal->int_ql; } } @@ -362,7 +645,7 @@ static int hns3_nic_set_real_num_queue(struct net_device *netdev) return 0; } -static u16 hns3_get_max_available_channels(struct hnae3_handle *h) +u16 hns3_get_max_available_channels(struct hnae3_handle *h) { u16 alloc_tqps, max_rss_size, rss_size; @@ -638,13 +921,10 @@ static u8 hns3_get_netdev_flags(struct net_device *netdev) { u8 flags = 0; - if (netdev->flags & IFF_PROMISC) { + if (netdev->flags & IFF_PROMISC) flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE; - } else { - flags |= HNAE3_VLAN_FLTR; - if (netdev->flags & IFF_ALLMULTI) - flags |= HNAE3_USER_MPE; - } + else if (netdev->flags & IFF_ALLMULTI) + flags = HNAE3_USER_MPE; return flags; } @@ -674,23 +954,202 @@ void hns3_request_update_promisc_mode(struct hnae3_handle *handle) ops->request_update_promisc_mode(handle); } -void hns3_enable_vlan_filter(struct net_device *netdev, bool enable) +static u32 hns3_tx_spare_space(struct hns3_enet_ring *ring) { - struct hns3_nic_priv *priv = netdev_priv(netdev); - struct hnae3_handle *h = priv->ae_handle; - struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); - bool last_state; + struct hns3_tx_spare *tx_spare = ring->tx_spare; + u32 ntc, ntu; - if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2 && - h->ae_algo->ops->enable_vlan_filter) { - last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false; - if (enable != last_state) { - netdev_info(netdev, - "%s vlan filter\n", - enable ? "enable" : "disable"); - h->ae_algo->ops->enable_vlan_filter(h, enable); + /* This smp_load_acquire() pairs with smp_store_release() in + * hns3_tx_spare_update() called in tx desc cleaning process. + */ + ntc = smp_load_acquire(&tx_spare->last_to_clean); + ntu = tx_spare->next_to_use; + + if (ntc > ntu) + return ntc - ntu - 1; + + /* The free tx buffer is divided into two part, so pick the + * larger one. + */ + return (ntc > (tx_spare->len - ntu) ? ntc : + (tx_spare->len - ntu)) - 1; +} + +static void hns3_tx_spare_update(struct hns3_enet_ring *ring) +{ + struct hns3_tx_spare *tx_spare = ring->tx_spare; + + if (!tx_spare || + tx_spare->last_to_clean == tx_spare->next_to_clean) + return; + + /* This smp_store_release() pairs with smp_load_acquire() in + * hns3_tx_spare_space() called in xmit process. + */ + smp_store_release(&tx_spare->last_to_clean, + tx_spare->next_to_clean); +} + +static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring, + struct sk_buff *skb, + u32 space) +{ + u32 len = skb->len <= ring->tx_copybreak ? skb->len : + skb_headlen(skb); + + if (len > ring->tx_copybreak) + return false; + + if (ALIGN(len, dma_get_cache_alignment()) > space) { + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_spare_full++; + u64_stats_update_end(&ring->syncp); + return false; + } + + return true; +} + +static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring, + struct sk_buff *skb, + u32 space) +{ + if (skb->len <= ring->tx_copybreak || !tx_sgl || + (!skb_has_frag_list(skb) && + skb_shinfo(skb)->nr_frags < tx_sgl)) + return false; + + if (space < HNS3_MAX_SGL_SIZE) { + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_spare_full++; + u64_stats_update_end(&ring->syncp); + return false; + } + + return true; +} + +static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) +{ + struct hns3_tx_spare *tx_spare; + struct page *page; + u32 alloc_size; + dma_addr_t dma; + int order; + + alloc_size = tx_spare_buf_size ? tx_spare_buf_size : + ring->tqp->handle->kinfo.tx_spare_buf_size; + if (!alloc_size) + return; + + order = get_order(alloc_size); + tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare), + GFP_KERNEL); + if (!tx_spare) { + /* The driver still work without the tx spare buffer */ + dev_warn(ring_to_dev(ring), "failed to allocate hns3_tx_spare\n"); + return; + } + + page = alloc_pages_node(dev_to_node(ring_to_dev(ring)), + GFP_KERNEL, order); + if (!page) { + dev_warn(ring_to_dev(ring), "failed to allocate tx spare pages\n"); + devm_kfree(ring_to_dev(ring), tx_spare); + return; + } + + dma = dma_map_page(ring_to_dev(ring), page, 0, + PAGE_SIZE << order, DMA_TO_DEVICE); + if (dma_mapping_error(ring_to_dev(ring), dma)) { + dev_warn(ring_to_dev(ring), "failed to map pages for tx spare\n"); + put_page(page); + devm_kfree(ring_to_dev(ring), tx_spare); + return; + } + + tx_spare->dma = dma; + tx_spare->buf = page_address(page); + tx_spare->len = PAGE_SIZE << order; + ring->tx_spare = tx_spare; +} + +/* Use hns3_tx_spare_space() to make sure there is enough buffer + * before calling below function to allocate tx buffer. + */ +static void *hns3_tx_spare_alloc(struct hns3_enet_ring *ring, + unsigned int size, dma_addr_t *dma, + u32 *cb_len) +{ + struct hns3_tx_spare *tx_spare = ring->tx_spare; + u32 ntu = tx_spare->next_to_use; + + size = ALIGN(size, dma_get_cache_alignment()); + *cb_len = size; + + /* Tx spare buffer wraps back here because the end of + * freed tx buffer is not enough. + */ + if (ntu + size > tx_spare->len) { + *cb_len += (tx_spare->len - ntu); + ntu = 0; + } + + tx_spare->next_to_use = ntu + size; + if (tx_spare->next_to_use == tx_spare->len) + tx_spare->next_to_use = 0; + + *dma = tx_spare->dma + ntu; + + return tx_spare->buf + ntu; +} + +static void hns3_tx_spare_rollback(struct hns3_enet_ring *ring, u32 len) +{ + struct hns3_tx_spare *tx_spare = ring->tx_spare; + + if (len > tx_spare->next_to_use) { + len -= tx_spare->next_to_use; + tx_spare->next_to_use = tx_spare->len - len; + } else { + tx_spare->next_to_use -= len; + } +} + +static void hns3_tx_spare_reclaim_cb(struct hns3_enet_ring *ring, + struct hns3_desc_cb *cb) +{ + struct hns3_tx_spare *tx_spare = ring->tx_spare; + u32 ntc = tx_spare->next_to_clean; + u32 len = cb->length; + + tx_spare->next_to_clean += len; + + if (tx_spare->next_to_clean >= tx_spare->len) { + tx_spare->next_to_clean -= tx_spare->len; + + if (tx_spare->next_to_clean) { + ntc = 0; + len = tx_spare->next_to_clean; } } + + /* This tx spare buffer is only really reclaimed after calling + * hns3_tx_spare_update(), so it is still safe to use the info in + * the tx buffer to do the dma sync or sg unmapping after + * tx_spare->next_to_clean is moved forword. + */ + if (cb->type & (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) { + dma_addr_t dma = tx_spare->dma + ntc; + + dma_sync_single_for_cpu(ring_to_dev(ring), dma, len, + DMA_TO_DEVICE); + } else { + struct sg_table *sgt = tx_spare->buf + ntc; + + dma_unmap_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents, + DMA_TO_DEVICE); + } } static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs, @@ -846,8 +1305,6 @@ static bool hns3_tunnel_csum_bug(struct sk_buff *skb) l4.udp->dest == htons(4790)))) return false; - skb_checksum_help(skb); - return true; } @@ -924,8 +1381,7 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, /* the stack computes the IP header already, * driver calculate l4 checksum when not TSO. */ - skb_checksum_help(skb); - return 0; + return skb_checksum_help(skb); } hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec); @@ -970,7 +1426,7 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, break; case IPPROTO_UDP: if (hns3_tunnel_csum_bug(skb)) - break; + return skb_checksum_help(skb); hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1); hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S, @@ -995,8 +1451,7 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, /* the stack computes the IP header already, * driver calculate l4 checksum when not TSO. */ - skb_checksum_help(skb); - return 0; + return skb_checksum_help(skb); } return 0; @@ -1168,40 +1623,14 @@ out_hw_tx_csum: return 0; } -static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, - unsigned int size, enum hns_desc_type type) +static int hns3_fill_desc(struct hns3_enet_ring *ring, dma_addr_t dma, + unsigned int size) { #define HNS3_LIKELY_BD_NUM 1 - struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; struct hns3_desc *desc = &ring->desc[ring->next_to_use]; - struct device *dev = ring_to_dev(ring); - skb_frag_t *frag; unsigned int frag_buf_num; int k, sizeoflast; - dma_addr_t dma; - - if (type == DESC_TYPE_FRAGLIST_SKB || - type == DESC_TYPE_SKB) { - struct sk_buff *skb = (struct sk_buff *)priv; - - dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); - } else { - frag = (skb_frag_t *)priv; - dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); - } - - if (unlikely(dma_mapping_error(dev, dma))) { - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); - return -ENOMEM; - } - - desc_cb->priv = priv; - desc_cb->length = size; - desc_cb->dma = dma; - desc_cb->type = type; if (likely(size <= HNS3_MAX_BD_SIZE)) { desc->addr = cpu_to_le64(dma); @@ -1237,6 +1666,52 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv, return frag_buf_num; } +static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv, + unsigned int type) +{ + struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; + struct device *dev = ring_to_dev(ring); + unsigned int size; + dma_addr_t dma; + + if (type & (DESC_TYPE_FRAGLIST_SKB | DESC_TYPE_SKB)) { + struct sk_buff *skb = (struct sk_buff *)priv; + + size = skb_headlen(skb); + if (!size) + return 0; + + dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); + } else if (type & DESC_TYPE_BOUNCE_HEAD) { + /* Head data has been filled in hns3_handle_tx_bounce(), + * just return 0 here. + */ + return 0; + } else { + skb_frag_t *frag = (skb_frag_t *)priv; + + size = skb_frag_size(frag); + if (!size) + return 0; + + dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); + } + + if (unlikely(dma_mapping_error(dev, dma))) { + u64_stats_update_begin(&ring->syncp); + ring->stats.sw_err_cnt++; + u64_stats_update_end(&ring->syncp); + return -ENOMEM; + } + + desc_cb->priv = priv; + desc_cb->length = size; + desc_cb->dma = dma; + desc_cb->type = type; + + return hns3_fill_desc(ring, dma, size); +} + static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size, unsigned int bd_num) { @@ -1460,6 +1935,7 @@ static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig) for (i = 0; i < ring->desc_num; i++) { struct hns3_desc *desc = &ring->desc[ring->next_to_use]; + struct hns3_desc_cb *desc_cb; memset(desc, 0, sizeof(*desc)); @@ -1470,52 +1946,44 @@ static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig) /* rollback one */ ring_ptr_move_bw(ring, next_to_use); - if (!ring->desc_cb[ring->next_to_use].dma) + desc_cb = &ring->desc_cb[ring->next_to_use]; + + if (!desc_cb->dma) continue; /* unmap the descriptor dma address */ - if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB || - ring->desc_cb[ring->next_to_use].type == - DESC_TYPE_FRAGLIST_SKB) - dma_unmap_single(dev, - ring->desc_cb[ring->next_to_use].dma, - ring->desc_cb[ring->next_to_use].length, - DMA_TO_DEVICE); - else if (ring->desc_cb[ring->next_to_use].length) - dma_unmap_page(dev, - ring->desc_cb[ring->next_to_use].dma, - ring->desc_cb[ring->next_to_use].length, + if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB)) + dma_unmap_single(dev, desc_cb->dma, desc_cb->length, + DMA_TO_DEVICE); + else if (desc_cb->type & + (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) + hns3_tx_spare_rollback(ring, desc_cb->length); + else if (desc_cb->length) + dma_unmap_page(dev, desc_cb->dma, desc_cb->length, DMA_TO_DEVICE); - ring->desc_cb[ring->next_to_use].length = 0; - ring->desc_cb[ring->next_to_use].dma = 0; - ring->desc_cb[ring->next_to_use].type = DESC_TYPE_UNKNOWN; + desc_cb->length = 0; + desc_cb->dma = 0; + desc_cb->type = DESC_TYPE_UNKNOWN; } } static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring, - struct sk_buff *skb, enum hns_desc_type type) + struct sk_buff *skb, unsigned int type) { - unsigned int size = skb_headlen(skb); struct sk_buff *frag_skb; int i, ret, bd_num = 0; - if (size) { - ret = hns3_fill_desc(ring, skb, size, type); - if (unlikely(ret < 0)) - return ret; + ret = hns3_map_and_fill_desc(ring, skb, type); + if (unlikely(ret < 0)) + return ret; - bd_num += ret; - } + bd_num += ret; for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - size = skb_frag_size(frag); - if (!size) - continue; - - ret = hns3_fill_desc(ring, frag, size, DESC_TYPE_PAGE); + ret = hns3_map_and_fill_desc(ring, frag, DESC_TYPE_PAGE); if (unlikely(ret < 0)) return ret; @@ -1555,6 +2023,153 @@ static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num, WRITE_ONCE(ring->last_to_use, ring->next_to_use); } +static void hns3_tsyn(struct net_device *netdev, struct sk_buff *skb, + struct hns3_desc *desc) +{ + struct hnae3_handle *h = hns3_get_handle(netdev); + + if (!(h->ae_algo->ops->set_tx_hwts_info && + h->ae_algo->ops->set_tx_hwts_info(h, skb))) + return; + + desc->tx.bdtp_fe_sc_vld_ra_ri |= cpu_to_le16(BIT(HNS3_TXD_TSYN_B)); +} + +static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring, + struct sk_buff *skb) +{ + struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; + unsigned int type = DESC_TYPE_BOUNCE_HEAD; + unsigned int size = skb_headlen(skb); + dma_addr_t dma; + int bd_num = 0; + u32 cb_len; + void *buf; + int ret; + + if (skb->len <= ring->tx_copybreak) { + size = skb->len; + type = DESC_TYPE_BOUNCE_ALL; + } + + /* hns3_can_use_tx_bounce() is called to ensure the below + * function can always return the tx buffer. + */ + buf = hns3_tx_spare_alloc(ring, size, &dma, &cb_len); + + ret = skb_copy_bits(skb, 0, buf, size); + if (unlikely(ret < 0)) { + hns3_tx_spare_rollback(ring, cb_len); + u64_stats_update_begin(&ring->syncp); + ring->stats.copy_bits_err++; + u64_stats_update_end(&ring->syncp); + return ret; + } + + desc_cb->priv = skb; + desc_cb->length = cb_len; + desc_cb->dma = dma; + desc_cb->type = type; + + bd_num += hns3_fill_desc(ring, dma, size); + + if (type == DESC_TYPE_BOUNCE_HEAD) { + ret = hns3_fill_skb_to_desc(ring, skb, + DESC_TYPE_BOUNCE_HEAD); + if (unlikely(ret < 0)) + return ret; + + bd_num += ret; + } + + dma_sync_single_for_device(ring_to_dev(ring), dma, size, + DMA_TO_DEVICE); + + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_bounce++; + u64_stats_update_end(&ring->syncp); + return bd_num; +} + +static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring, + struct sk_buff *skb) +{ + struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; + u32 nfrag = skb_shinfo(skb)->nr_frags + 1; + struct sg_table *sgt; + int i, bd_num = 0; + dma_addr_t dma; + u32 cb_len; + int nents; + + if (skb_has_frag_list(skb)) + nfrag = HNS3_MAX_TSO_BD_NUM; + + /* hns3_can_use_tx_sgl() is called to ensure the below + * function can always return the tx buffer. + */ + sgt = hns3_tx_spare_alloc(ring, HNS3_SGL_SIZE(nfrag), + &dma, &cb_len); + + /* scatterlist follows by the sg table */ + sgt->sgl = (struct scatterlist *)(sgt + 1); + sg_init_table(sgt->sgl, nfrag); + nents = skb_to_sgvec(skb, sgt->sgl, 0, skb->len); + if (unlikely(nents < 0)) { + hns3_tx_spare_rollback(ring, cb_len); + u64_stats_update_begin(&ring->syncp); + ring->stats.skb2sgl_err++; + u64_stats_update_end(&ring->syncp); + return -ENOMEM; + } + + sgt->orig_nents = nents; + sgt->nents = dma_map_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents, + DMA_TO_DEVICE); + if (unlikely(!sgt->nents)) { + hns3_tx_spare_rollback(ring, cb_len); + u64_stats_update_begin(&ring->syncp); + ring->stats.map_sg_err++; + u64_stats_update_end(&ring->syncp); + return -ENOMEM; + } + + desc_cb->priv = skb; + desc_cb->length = cb_len; + desc_cb->dma = dma; + desc_cb->type = DESC_TYPE_SGL_SKB; + + for (i = 0; i < sgt->nents; i++) + bd_num += hns3_fill_desc(ring, sg_dma_address(sgt->sgl + i), + sg_dma_len(sgt->sgl + i)); + + u64_stats_update_begin(&ring->syncp); + ring->stats.tx_sgl++; + u64_stats_update_end(&ring->syncp); + + return bd_num; +} + +static int hns3_handle_desc_filling(struct hns3_enet_ring *ring, + struct sk_buff *skb) +{ + u32 space; + + if (!ring->tx_spare) + goto out; + + space = hns3_tx_spare_space(ring); + + if (hns3_can_use_tx_sgl(ring, skb, space)) + return hns3_handle_tx_sgl(ring, skb); + + if (hns3_can_use_tx_bounce(ring, skb, space)) + return hns3_handle_tx_bounce(ring, skb); + +out: + return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB); +} + netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); @@ -1601,16 +2216,22 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) * zero, which is unlikely, and 'ret > 0' means how many tx desc * need to be notified to the hw. */ - ret = hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB); + ret = hns3_handle_desc_filling(ring, skb); if (unlikely(ret <= 0)) goto fill_err; pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) : (ring->desc_num - 1); + + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + hns3_tsyn(netdev, skb, &ring->desc[pre_ntu]); + ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |= cpu_to_le16(BIT(HNS3_TXD_FE_B)); trace_hns3_tx_desc(ring, pre_ntu); + skb_tx_timestamp(skb); + /* Complete translate all packets */ dev_queue = netdev_get_tx_queue(netdev, ring->queue_index); doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes, @@ -1714,6 +2335,14 @@ static int hns3_nic_set_features(struct net_device *netdev, return -EINVAL; } + if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) && + h->ae_algo->ops->enable_vlan_filter) { + enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER); + ret = h->ae_algo->ops->enable_vlan_filter(h, enable); + if (ret) + return ret; + } + netdev->features = features; return 0; } @@ -1789,6 +2418,9 @@ static void hns3_nic_get_stats64(struct net_device *netdev, tx_drop += ring->stats.tx_tso_err; tx_drop += ring->stats.over_max_recursion; tx_drop += ring->stats.hw_limitation; + tx_drop += ring->stats.copy_bits_err; + tx_drop += ring->stats.skb2sgl_err; + tx_drop += ring->stats.map_sg_err; tx_errors += ring->stats.sw_err_cnt; tx_errors += ring->stats.tx_vlan_err; tx_errors += ring->stats.tx_l4_proto_err; @@ -1796,6 +2428,9 @@ static void hns3_nic_get_stats64(struct net_device *netdev, tx_errors += ring->stats.tx_tso_err; tx_errors += ring->stats.over_max_recursion; tx_errors += ring->stats.hw_limitation; + tx_errors += ring->stats.copy_bits_err; + tx_errors += ring->stats.skb2sgl_err; + tx_errors += ring->stats.map_sg_err; } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); /* fetch the rx stats */ @@ -2559,6 +3194,9 @@ static void hns3_set_default_feature(struct net_device *netdev) netdev->hw_features |= NETIF_F_HW_TC; netdev->features |= NETIF_F_HW_TC; } + + if (test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps)) + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; } static int hns3_alloc_buffer(struct hns3_enet_ring *ring, @@ -2586,7 +3224,8 @@ static int hns3_alloc_buffer(struct hns3_enet_ring *ring, static void hns3_free_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb, int budget) { - if (cb->type == DESC_TYPE_SKB) + if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_HEAD | + DESC_TYPE_BOUNCE_ALL | DESC_TYPE_SGL_SKB)) napi_consume_skb(cb->priv, budget); else if (!HNAE3_IS_TX_RING(ring) && cb->pagecnt_bias) __page_frag_cache_drain(cb->priv, cb->pagecnt_bias); @@ -2607,12 +3246,15 @@ static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb) static void hns3_unmap_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb) { - if (cb->type == DESC_TYPE_SKB || cb->type == DESC_TYPE_FRAGLIST_SKB) + if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB)) dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length, ring_to_dma_dir(ring)); - else if (cb->length) + else if ((cb->type & DESC_TYPE_PAGE) && cb->length) dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length, ring_to_dma_dir(ring)); + else if (cb->type & (DESC_TYPE_BOUNCE_ALL | DESC_TYPE_BOUNCE_HEAD | + DESC_TYPE_SGL_SKB)) + hns3_tx_spare_reclaim_cb(ring, cb); } static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i) @@ -2764,7 +3406,9 @@ static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring, desc_cb = &ring->desc_cb[ntc]; - if (desc_cb->type == DESC_TYPE_SKB) { + if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_ALL | + DESC_TYPE_BOUNCE_HEAD | + DESC_TYPE_SGL_SKB)) { (*pkts)++; (*bytes) += desc_cb->send_bytes; } @@ -2787,6 +3431,9 @@ static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring, * ring_space called by hns3_nic_net_xmit. */ smp_store_release(&ring->next_to_clean, ntc); + + hns3_tx_spare_update(ring); + return true; } @@ -2878,7 +3525,7 @@ static void hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, static bool hns3_can_reuse_page(struct hns3_desc_cb *cb) { - return (page_count(cb->priv) - cb->pagecnt_bias) == 1; + return page_count(cb->priv) == cb->pagecnt_bias; } static void hns3_nic_reuse_page(struct sk_buff *skb, int i, @@ -2886,40 +3533,74 @@ static void hns3_nic_reuse_page(struct sk_buff *skb, int i, struct hns3_desc_cb *desc_cb) { struct hns3_desc *desc = &ring->desc[ring->next_to_clean]; + u32 frag_offset = desc_cb->page_offset + pull_len; int size = le16_to_cpu(desc->rx.size); u32 truesize = hns3_buf_size(ring); + u32 frag_size = size - pull_len; + bool reused; - desc_cb->pagecnt_bias--; - skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len, - size - pull_len, truesize); + /* Avoid re-using remote or pfmem page */ + if (unlikely(!dev_page_is_reusable(desc_cb->priv))) + goto out; - /* Avoid re-using remote and pfmemalloc pages, or the stack is still - * using the page when page_offset rollback to zero, flag default - * unreuse + reused = hns3_can_reuse_page(desc_cb); + + /* Rx page can be reused when: + * 1. Rx page is only owned by the driver when page_offset + * is zero, which means 0 @ truesize will be used by + * stack after skb_add_rx_frag() is called, and the rest + * of rx page can be reused by driver. + * Or + * 2. Rx page is only owned by the driver when page_offset + * is non-zero, which means page_offset @ truesize will + * be used by stack after skb_add_rx_frag() is called, + * and 0 @ truesize can be reused by driver. */ - if (!dev_page_is_reusable(desc_cb->priv) || - (!desc_cb->page_offset && !hns3_can_reuse_page(desc_cb))) { - __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias); - return; - } + if ((!desc_cb->page_offset && reused) || + ((desc_cb->page_offset + truesize + truesize) <= + hns3_page_size(ring) && desc_cb->page_offset)) { + desc_cb->page_offset += truesize; + desc_cb->reuse_flag = 1; + } else if (desc_cb->page_offset && reused) { + desc_cb->page_offset = 0; + desc_cb->reuse_flag = 1; + } else if (frag_size <= ring->rx_copybreak) { + void *frag = napi_alloc_frag(frag_size); - /* Move offset up to the next cache line */ - desc_cb->page_offset += truesize; + if (unlikely(!frag)) { + u64_stats_update_begin(&ring->syncp); + ring->stats.frag_alloc_err++; + u64_stats_update_end(&ring->syncp); + + hns3_rl_err(ring_to_netdev(ring), + "failed to allocate rx frag\n"); + goto out; + } - if (desc_cb->page_offset + truesize <= hns3_page_size(ring)) { - desc_cb->reuse_flag = 1; - } else if (hns3_can_reuse_page(desc_cb)) { desc_cb->reuse_flag = 1; - desc_cb->page_offset = 0; - } else if (desc_cb->pagecnt_bias) { - __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias); + memcpy(frag, desc_cb->buf + frag_offset, frag_size); + skb_add_rx_frag(skb, i, virt_to_page(frag), + offset_in_page(frag), frag_size, frag_size); + + u64_stats_update_begin(&ring->syncp); + ring->stats.frag_alloc++; + u64_stats_update_end(&ring->syncp); return; } +out: + desc_cb->pagecnt_bias--; + if (unlikely(!desc_cb->pagecnt_bias)) { page_ref_add(desc_cb->priv, USHRT_MAX); desc_cb->pagecnt_bias = USHRT_MAX; } + + skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset, + frag_size, truesize); + + if (unlikely(!desc_cb->reuse_flag)) + __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias); } static int hns3_gro_complete(struct sk_buff *skb, u32 l234info) @@ -2980,51 +3661,31 @@ static int hns3_gro_complete(struct sk_buff *skb, u32 l234info) return 0; } -static void hns3_checksum_complete(struct hns3_enet_ring *ring, - struct sk_buff *skb, u32 l234info) +static bool hns3_checksum_complete(struct hns3_enet_ring *ring, + struct sk_buff *skb, u32 ptype, u16 csum) { - u32 lo, hi; + if (ptype == HNS3_INVALID_PTYPE || + hns3_rx_ptype_tbl[ptype].ip_summed != CHECKSUM_COMPLETE) + return false; u64_stats_update_begin(&ring->syncp); ring->stats.csum_complete++; u64_stats_update_end(&ring->syncp); skb->ip_summed = CHECKSUM_COMPLETE; - lo = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_L_M, - HNS3_RXD_L2_CSUM_L_S); - hi = hnae3_get_field(l234info, HNS3_RXD_L2_CSUM_H_M, - HNS3_RXD_L2_CSUM_H_S); - skb->csum = csum_unfold((__force __sum16)(lo | hi << 8)); + skb->csum = csum_unfold((__force __sum16)csum); + + return true; } -static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, - u32 l234info, u32 bd_base_info, u32 ol_info) +static void hns3_rx_handle_csum(struct sk_buff *skb, u32 l234info, + u32 ol_info, u32 ptype) { - struct net_device *netdev = ring_to_netdev(ring); int l3_type, l4_type; int ol4_type; - skb->ip_summed = CHECKSUM_NONE; - - skb_checksum_none_assert(skb); - - if (!(netdev->features & NETIF_F_RXCSUM)) - return; - - if (l234info & BIT(HNS3_RXD_L2_CSUM_B)) { - hns3_checksum_complete(ring, skb, l234info); - return; - } - - /* check if hardware has done checksum */ - if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B))) - return; - - if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) | - BIT(HNS3_RXD_OL3E_B) | - BIT(HNS3_RXD_OL4E_B)))) { - u64_stats_update_begin(&ring->syncp); - ring->stats.l3l4_csum_err++; - u64_stats_update_end(&ring->syncp); + if (ptype != HNS3_INVALID_PTYPE) { + skb->csum_level = hns3_rx_ptype_tbl[ptype].csum_level; + skb->ip_summed = hns3_rx_ptype_tbl[ptype].ip_summed; return; } @@ -3054,6 +3715,45 @@ static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, } } +static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, + u32 l234info, u32 bd_base_info, u32 ol_info, + u16 csum) +{ + struct net_device *netdev = ring_to_netdev(ring); + struct hns3_nic_priv *priv = netdev_priv(netdev); + u32 ptype = HNS3_INVALID_PTYPE; + + skb->ip_summed = CHECKSUM_NONE; + + skb_checksum_none_assert(skb); + + if (!(netdev->features & NETIF_F_RXCSUM)) + return; + + if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) + ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M, + HNS3_RXD_PTYPE_S); + + if (hns3_checksum_complete(ring, skb, ptype, csum)) + return; + + /* check if hardware has done checksum */ + if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B))) + return; + + if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) | + BIT(HNS3_RXD_OL3E_B) | + BIT(HNS3_RXD_OL4E_B)))) { + u64_stats_update_begin(&ring->syncp); + ring->stats.l3l4_csum_err++; + u64_stats_update_end(&ring->syncp); + + return; + } + + hns3_rx_handle_csum(skb, l234info, ol_info, ptype); +} + static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb) { if (skb_has_frag_list(skb)) @@ -3235,8 +3935,10 @@ static int hns3_add_frag(struct hns3_enet_ring *ring) static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, u32 l234info, - u32 bd_base_info, u32 ol_info) + u32 bd_base_info, u32 ol_info, u16 csum) { + struct net_device *netdev = ring_to_netdev(ring); + struct hns3_nic_priv *priv = netdev_priv(netdev); u32 l3_type; skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info, @@ -3244,7 +3946,8 @@ static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring, HNS3_RXD_GRO_SIZE_S); /* if there is no HW GRO, do not set gro params */ if (!skb_shinfo(skb)->gso_size) { - hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info); + hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info, + csum); return 0; } @@ -3252,7 +3955,16 @@ static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring, HNS3_RXD_GRO_COUNT_M, HNS3_RXD_GRO_COUNT_S); - l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S); + if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) { + u32 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M, + HNS3_RXD_PTYPE_S); + + l3_type = hns3_rx_ptype_tbl[ptype].l3_type; + } else { + l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, + HNS3_RXD_L3ID_S); + } + if (l3_type == HNS3_L3_TYPE_IPV4) skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; else if (l3_type == HNS3_L3_TYPE_IPV6) @@ -3285,6 +3997,7 @@ static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) struct hns3_desc *desc; unsigned int len; int pre_ntc, ret; + u16 csum; /* bdinfo handled below is only valid on the last BD of the * current packet, and ring->next_to_clean indicates the first @@ -3296,6 +4009,16 @@ static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) bd_base_info = le32_to_cpu(desc->rx.bd_base_info); l234info = le32_to_cpu(desc->rx.l234_info); ol_info = le32_to_cpu(desc->rx.ol_info); + csum = le16_to_cpu(desc->csum); + + if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) { + struct hnae3_handle *h = hns3_get_handle(netdev); + u32 nsec = le32_to_cpu(desc->ts_nsec); + u32 sec = le32_to_cpu(desc->ts_sec); + + if (h->ae_algo->ops->get_rx_hwts) + h->ae_algo->ops->get_rx_hwts(h, skb, nsec, sec); + } /* Based on hw strategy, the tag offloaded will be stored at * ot_vlan_tag in two layer tag case, and stored at vlan_tag @@ -3328,7 +4051,7 @@ static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) /* This is needed in order to enable forwarding support */ ret = hns3_set_gro_and_checksum(ring, skb, l234info, - bd_base_info, ol_info); + bd_base_info, ol_info, csum); if (unlikely(ret)) { u64_stats_update_begin(&ring->syncp); ring->stats.rx_err_cnt++; @@ -3467,139 +4190,30 @@ out: return recv_pkts; } -static bool hns3_get_new_flow_lvl(struct hns3_enet_ring_group *ring_group) +static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector) { -#define HNS3_RX_LOW_BYTE_RATE 10000 -#define HNS3_RX_MID_BYTE_RATE 20000 -#define HNS3_RX_ULTRA_PACKET_RATE 40 - - enum hns3_flow_level_range new_flow_level; - struct hns3_enet_tqp_vector *tqp_vector; - int packets_per_msecs, bytes_per_msecs; - u32 time_passed_ms; - - tqp_vector = ring_group->ring->tqp_vector; - time_passed_ms = - jiffies_to_msecs(jiffies - tqp_vector->last_jiffies); - if (!time_passed_ms) - return false; - - do_div(ring_group->total_packets, time_passed_ms); - packets_per_msecs = ring_group->total_packets; - - do_div(ring_group->total_bytes, time_passed_ms); - bytes_per_msecs = ring_group->total_bytes; - - new_flow_level = ring_group->coal.flow_level; - - /* Simple throttlerate management - * 0-10MB/s lower (50000 ints/s) - * 10-20MB/s middle (20000 ints/s) - * 20-1249MB/s high (18000 ints/s) - * > 40000pps ultra (8000 ints/s) - */ - switch (new_flow_level) { - case HNS3_FLOW_LOW: - if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE) - new_flow_level = HNS3_FLOW_MID; - break; - case HNS3_FLOW_MID: - if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE) - new_flow_level = HNS3_FLOW_HIGH; - else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE) - new_flow_level = HNS3_FLOW_LOW; - break; - case HNS3_FLOW_HIGH: - case HNS3_FLOW_ULTRA: - default: - if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE) - new_flow_level = HNS3_FLOW_MID; - break; - } - - if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE && - &tqp_vector->rx_group == ring_group) - new_flow_level = HNS3_FLOW_ULTRA; - - ring_group->total_bytes = 0; - ring_group->total_packets = 0; - ring_group->coal.flow_level = new_flow_level; - - return true; -} - -static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group) -{ - struct hns3_enet_tqp_vector *tqp_vector; - u16 new_int_gl; - - if (!ring_group->ring) - return false; - - tqp_vector = ring_group->ring->tqp_vector; - if (!tqp_vector->last_jiffies) - return false; - - if (ring_group->total_packets == 0) { - ring_group->coal.int_gl = HNS3_INT_GL_50K; - ring_group->coal.flow_level = HNS3_FLOW_LOW; - return true; - } - - if (!hns3_get_new_flow_lvl(ring_group)) - return false; + struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group; + struct dim_sample sample = {}; - new_int_gl = ring_group->coal.int_gl; - switch (ring_group->coal.flow_level) { - case HNS3_FLOW_LOW: - new_int_gl = HNS3_INT_GL_50K; - break; - case HNS3_FLOW_MID: - new_int_gl = HNS3_INT_GL_20K; - break; - case HNS3_FLOW_HIGH: - new_int_gl = HNS3_INT_GL_18K; - break; - case HNS3_FLOW_ULTRA: - new_int_gl = HNS3_INT_GL_8K; - break; - default: - break; - } + if (!rx_group->coal.adapt_enable) + return; - if (new_int_gl != ring_group->coal.int_gl) { - ring_group->coal.int_gl = new_int_gl; - return true; - } - return false; + dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets, + rx_group->total_bytes, &sample); + net_dim(&rx_group->dim, sample); } -static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector) +static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector) { - struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group; struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group; - bool rx_update, tx_update; + struct dim_sample sample = {}; - /* update param every 1000ms */ - if (time_before(jiffies, - tqp_vector->last_jiffies + msecs_to_jiffies(1000))) + if (!tx_group->coal.adapt_enable) return; - if (rx_group->coal.adapt_enable) { - rx_update = hns3_get_new_int_gl(rx_group); - if (rx_update) - hns3_set_vector_coalesce_rx_gl(tqp_vector, - rx_group->coal.int_gl); - } - - if (tx_group->coal.adapt_enable) { - tx_update = hns3_get_new_int_gl(tx_group); - if (tx_update) - hns3_set_vector_coalesce_tx_gl(tqp_vector, - tx_group->coal.int_gl); - } - - tqp_vector->last_jiffies = jiffies; + dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets, + tx_group->total_bytes, &sample); + net_dim(&tx_group->dim, sample); } static int hns3_nic_common_poll(struct napi_struct *napi, int budget) @@ -3644,7 +4258,9 @@ static int hns3_nic_common_poll(struct napi_struct *napi, int budget) if (napi_complete(napi) && likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) { - hns3_update_new_int_gl(tqp_vector); + hns3_update_rx_int_coalesce(tqp_vector); + hns3_update_tx_int_coalesce(tqp_vector); + hns3_mask_vector_irq(tqp_vector, 1); } @@ -3775,6 +4391,54 @@ static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv) } } +static void hns3_rx_dim_work(struct work_struct *work) +{ + struct dim *dim = container_of(work, struct dim, work); + struct hns3_enet_ring_group *group = container_of(dim, + struct hns3_enet_ring_group, dim); + struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector; + struct dim_cq_moder cur_moder = + net_dim_get_rx_moderation(dim->mode, dim->profile_ix); + + hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec); + tqp_vector->rx_group.coal.int_gl = cur_moder.usec; + + if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) { + hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts); + tqp_vector->rx_group.coal.int_ql = cur_moder.pkts; + } + + dim->state = DIM_START_MEASURE; +} + +static void hns3_tx_dim_work(struct work_struct *work) +{ + struct dim *dim = container_of(work, struct dim, work); + struct hns3_enet_ring_group *group = container_of(dim, + struct hns3_enet_ring_group, dim); + struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector; + struct dim_cq_moder cur_moder = + net_dim_get_tx_moderation(dim->mode, dim->profile_ix); + + hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec); + tqp_vector->tx_group.coal.int_gl = cur_moder.usec; + + if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) { + hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts); + tqp_vector->tx_group.coal.int_ql = cur_moder.pkts; + } + + dim->state = DIM_START_MEASURE; +} + +static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector) +{ + INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work); + tqp_vector->rx_group.dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work); + tqp_vector->tx_group.dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; +} + static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) { struct hnae3_handle *h = priv->ae_handle; @@ -3788,6 +4452,7 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) tqp_vector = &priv->tqp_vector[i]; hns3_vector_coalesce_init_hw(tqp_vector, priv); tqp_vector->num_tqps = 0; + hns3_nic_init_dim(tqp_vector); } for (i = 0; i < h->kinfo.num_tqps; i++) { @@ -3844,6 +4509,34 @@ map_ring_fail: return ret; } +static void hns3_nic_init_coal_cfg(struct hns3_nic_priv *priv) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev); + struct hns3_enet_coalesce *tx_coal = &priv->tx_coal; + struct hns3_enet_coalesce *rx_coal = &priv->rx_coal; + + /* initialize the configuration for interrupt coalescing. + * 1. GL (Interrupt Gap Limiter) + * 2. RL (Interrupt Rate Limiter) + * 3. QL (Interrupt Quantity Limiter) + * + * Default: enable interrupt coalescing self-adaptive and GL + */ + tx_coal->adapt_enable = 1; + rx_coal->adapt_enable = 1; + + tx_coal->int_gl = HNS3_INT_GL_50K; + rx_coal->int_gl = HNS3_INT_GL_50K; + + rx_coal->flow_level = HNS3_FLOW_LOW; + tx_coal->flow_level = HNS3_FLOW_LOW; + + if (ae_dev->dev_specs.int_ql_max) { + tx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG; + rx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG; + } +} + static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv) { struct hnae3_handle *h = priv->ae_handle; @@ -3955,10 +4648,13 @@ static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, ring = &priv->ring[q->tqp_index]; desc_num = priv->ae_handle->kinfo.num_tx_desc; ring->queue_index = q->tqp_index; + ring->tx_copybreak = priv->tx_copybreak; + ring->last_to_use = 0; } else { ring = &priv->ring[q->tqp_index + queue_num]; desc_num = priv->ae_handle->kinfo.num_rx_desc; ring->queue_index = q->tqp_index; + ring->rx_copybreak = priv->rx_copybreak; } hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type); @@ -3972,7 +4668,6 @@ static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, ring->desc_num = desc_num; ring->next_to_use = 0; ring->next_to_clean = 0; - ring->last_to_use = 0; } static void hns3_queue_to_ring(struct hnae3_queue *tqp, @@ -4032,6 +4727,8 @@ static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring) ret = hns3_alloc_ring_buffers(ring); if (ret) goto out_with_desc; + } else { + hns3_init_tx_spare_buffer(ring); } return 0; @@ -4054,9 +4751,18 @@ void hns3_fini_ring(struct hns3_enet_ring *ring) ring->next_to_use = 0; ring->last_to_use = 0; ring->pending_buf = 0; - if (ring->skb) { + if (!HNAE3_IS_TX_RING(ring) && ring->skb) { dev_kfree_skb_any(ring->skb); ring->skb = NULL; + } else if (HNAE3_IS_TX_RING(ring) && ring->tx_spare) { + struct hns3_tx_spare *tx_spare = ring->tx_spare; + + dma_unmap_page(ring_to_dev(ring), tx_spare->dma, tx_spare->len, + DMA_TO_DEVICE); + free_pages((unsigned long)tx_spare->buf, + get_order(tx_spare->len)); + devm_kfree(ring_to_dev(ring), tx_spare); + ring->tx_spare = NULL; } } @@ -4295,6 +5001,8 @@ static int hns3_client_init(struct hnae3_handle *handle) goto out_get_ring_cfg; } + hns3_nic_init_coal_cfg(priv); + ret = hns3_nic_alloc_vector_data(priv); if (ret) { ret = -ENOMEM; @@ -4317,12 +5025,6 @@ static int hns3_client_init(struct hnae3_handle *handle) if (ret) goto out_init_phy; - ret = register_netdev(netdev); - if (ret) { - dev_err(priv->dev, "probe register netdev fail!\n"); - goto out_reg_netdev_fail; - } - /* the device can work without cpu rmap, only aRFS needs it */ ret = hns3_set_rx_cpu_rmap(netdev); if (ret) @@ -4343,29 +5045,43 @@ static int hns3_client_init(struct hnae3_handle *handle) hns3_dcbnl_setup(handle); - hns3_dbg_init(handle); + ret = hns3_dbg_init(handle); + if (ret) { + dev_err(priv->dev, "failed to init debugfs, ret = %d\n", + ret); + goto out_client_start; + } netdev->max_mtu = HNS3_MAX_MTU(ae_dev->dev_specs.max_frm_size); if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps)) set_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state); + if (hnae3_ae_dev_rxd_adv_layout_supported(ae_dev)) + set_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state); + set_bit(HNS3_NIC_STATE_INITED, &priv->state); if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->supported_pflags); + ret = register_netdev(netdev); + if (ret) { + dev_err(priv->dev, "probe register netdev fail!\n"); + goto out_reg_netdev_fail; + } + if (netif_msg_drv(handle)) hns3_info_show(priv); return ret; +out_reg_netdev_fail: + hns3_dbg_uninit(handle); out_client_start: hns3_free_rx_cpu_rmap(netdev); hns3_nic_uninit_irq(priv); out_init_irq_fail: - unregister_netdev(netdev); -out_reg_netdev_fail: hns3_uninit_phy(netdev); out_init_phy: hns3_uninit_all_ring(priv); @@ -4571,31 +5287,6 @@ int hns3_nic_reset_all_ring(struct hnae3_handle *h) return 0; } -static void hns3_store_coal(struct hns3_nic_priv *priv) -{ - /* ethtool only support setting and querying one coal - * configuration for now, so save the vector 0' coal - * configuration here in order to restore it. - */ - memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal, - sizeof(struct hns3_enet_coalesce)); - memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal, - sizeof(struct hns3_enet_coalesce)); -} - -static void hns3_restore_coal(struct hns3_nic_priv *priv) -{ - u16 vector_num = priv->vector_num; - int i; - - for (i = 0; i < vector_num; i++) { - memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal, - sizeof(struct hns3_enet_coalesce)); - memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal, - sizeof(struct hns3_enet_coalesce)); - } -} - static int hns3_reset_notify_down_enet(struct hnae3_handle *handle) { struct hnae3_knic_private_info *kinfo = &handle->kinfo; @@ -4654,8 +5345,6 @@ static int hns3_reset_notify_init_enet(struct hnae3_handle *handle) if (ret) goto err_put_ring; - hns3_restore_coal(priv); - ret = hns3_nic_init_vector_data(priv); if (ret) goto err_dealloc_vector; @@ -4721,8 +5410,6 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) hns3_nic_uninit_vector_data(priv); - hns3_store_coal(priv); - hns3_nic_dealloc_vector_data(priv); hns3_uninit_all_ring(priv); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index daa04aeb0942..15af3d93857b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -4,6 +4,7 @@ #ifndef __HNS3_ENET_H #define __HNS3_ENET_H +#include <linux/dim.h> #include <linux/if_vlan.h> #include "hnae3.h" @@ -19,6 +20,7 @@ enum hns3_nic_state { HNS3_NIC_STATE_SERVICE_SCHED, HNS3_NIC_STATE2_RESET_REQUESTED, HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, + HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, HNS3_NIC_STATE_MAX }; @@ -82,12 +84,6 @@ enum hns3_nic_state { #define HNS3_RXD_STRP_TAGP_S 13 #define HNS3_RXD_STRP_TAGP_M (0x3 << HNS3_RXD_STRP_TAGP_S) -#define HNS3_RXD_L2_CSUM_B 15 -#define HNS3_RXD_L2_CSUM_L_S 4 -#define HNS3_RXD_L2_CSUM_L_M (0xff << HNS3_RXD_L2_CSUM_L_S) -#define HNS3_RXD_L2_CSUM_H_S 24 -#define HNS3_RXD_L2_CSUM_H_M (0xff << HNS3_RXD_L2_CSUM_H_S) - #define HNS3_RXD_L2E_B 16 #define HNS3_RXD_L3E_B 17 #define HNS3_RXD_L4E_B 18 @@ -114,6 +110,9 @@ enum hns3_nic_state { #define HNS3_RXD_FBLI_S 14 #define HNS3_RXD_FBLI_M (0x3 << HNS3_RXD_FBLI_S) +#define HNS3_RXD_PTYPE_S 4 +#define HNS3_RXD_PTYPE_M GENMASK(11, 4) + #define HNS3_RXD_BDTYPE_S 0 #define HNS3_RXD_BDTYPE_M (0xf << HNS3_RXD_BDTYPE_S) #define HNS3_RXD_VLD_B 4 @@ -123,8 +122,9 @@ enum hns3_nic_state { #define HNS3_RXD_LUM_B 9 #define HNS3_RXD_CRCP_B 10 #define HNS3_RXD_L3L4P_B 11 -#define HNS3_RXD_TSIND_S 12 -#define HNS3_RXD_TSIND_M (0x7 << HNS3_RXD_TSIND_S) +#define HNS3_RXD_TSIDX_S 12 +#define HNS3_RXD_TSIDX_M (0x3 << HNS3_RXD_TSIDX_S) +#define HNS3_RXD_TS_VLD_B 14 #define HNS3_RXD_LKBK_B 15 #define HNS3_RXD_GRO_SIZE_S 16 #define HNS3_RXD_GRO_SIZE_M (0x3fff << HNS3_RXD_GRO_SIZE_S) @@ -238,7 +238,14 @@ enum hns3_pkt_tun_type { /* hardware spec ring buffer format */ struct __packed hns3_desc { - __le64 addr; + union { + __le64 addr; + __le16 csum; + struct { + __le32 ts_nsec; + __le32 ts_sec; + }; + }; union { struct { __le16 vlan_tag; @@ -292,6 +299,16 @@ struct __packed hns3_desc { }; }; +enum hns3_desc_type { + DESC_TYPE_UNKNOWN = 0, + DESC_TYPE_SKB = 1 << 0, + DESC_TYPE_FRAGLIST_SKB = 1 << 1, + DESC_TYPE_PAGE = 1 << 2, + DESC_TYPE_BOUNCE_ALL = 1 << 3, + DESC_TYPE_BOUNCE_HEAD = 1 << 4, + DESC_TYPE_SGL_SKB = 1 << 5, +}; + struct hns3_desc_cb { dma_addr_t dma; /* dma address of this desc */ void *buf; /* cpu addr for a desc */ @@ -366,6 +383,14 @@ enum hns3_pkt_ol4type { HNS3_OL4_TYPE_UNKNOWN }; +struct hns3_rx_ptype { + u32 ptype:8; + u32 csum_level:2; + u32 ip_summed:2; + u32 l3_type:4; + u32 valid:1; +}; + struct ring_stats { u64 sw_err_cnt; u64 seg_pkt_cnt; @@ -383,6 +408,12 @@ struct ring_stats { u64 tx_tso_err; u64 over_max_recursion; u64 hw_limitation; + u64 tx_bounce; + u64 tx_spare_full; + u64 copy_bits_err; + u64 tx_sgl; + u64 skb2sgl_err; + u64 map_sg_err; }; struct { u64 rx_pkts; @@ -396,10 +427,22 @@ struct ring_stats { u64 csum_complete; u64 rx_multicast; u64 non_reuse_pg; + u64 frag_alloc_err; + u64 frag_alloc; }; + __le16 csum; }; }; +struct hns3_tx_spare { + dma_addr_t dma; + void *buf; + u32 next_to_use; + u32 next_to_clean; + u32 last_to_clean; + u32 len; +}; + struct hns3_enet_ring { struct hns3_desc *desc; /* dma map address space */ struct hns3_desc_cb *desc_cb; @@ -422,18 +465,29 @@ struct hns3_enet_ring { * next_to_use */ int next_to_clean; - union { - int last_to_use; /* last idx used by xmit */ - u32 pull_len; /* memcpy len for current rx packet */ - }; - u32 frag_num; - void *va; /* first buffer address for current packet */ - u32 flag; /* ring attribute */ int pending_buf; - struct sk_buff *skb; - struct sk_buff *tail_skb; + union { + /* for Tx ring */ + struct { + u32 fd_qb_tx_sample; + int last_to_use; /* last idx used by xmit */ + u32 tx_copybreak; + struct hns3_tx_spare *tx_spare; + }; + + /* for Rx ring */ + struct { + u32 pull_len; /* memcpy len for current rx packet */ + u32 rx_copybreak; + u32 frag_num; + /* first buffer address for current packet */ + unsigned char *va; + struct sk_buff *skb; + struct sk_buff *tail_skb; + }; + }; } ____cacheline_internodealigned_in_smp; enum hns3_flow_level_range { @@ -472,6 +526,7 @@ struct hns3_enet_ring_group { u64 total_packets; /* total packets processed this group */ u16 count; struct hns3_enet_coalesce coal; + struct dim dim; }; struct hns3_enet_tqp_vector { @@ -493,7 +548,7 @@ struct hns3_enet_tqp_vector { char name[HNAE3_INT_NAME_LEN]; - unsigned long last_jiffies; + u64 event_cnt; } ____cacheline_internodealigned_in_smp; struct hns3_nic_priv { @@ -516,6 +571,8 @@ struct hns3_nic_priv { struct hns3_enet_coalesce tx_coal; struct hns3_enet_coalesce rx_coal; + u32 tx_copybreak; + u32 rx_copybreak; }; union l3_hdr_info { @@ -631,7 +688,6 @@ void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector, void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector, u32 ql_value); -void hns3_enable_vlan_filter(struct net_device *netdev, bool enable); void hns3_request_update_promisc_mode(struct hnae3_handle *handle); #ifdef CONFIG_HNS3_DCB @@ -640,9 +696,10 @@ void hns3_dcbnl_setup(struct hnae3_handle *handle); static inline void hns3_dcbnl_setup(struct hnae3_handle *handle) {} #endif -void hns3_dbg_init(struct hnae3_handle *handle); +int hns3_dbg_init(struct hnae3_handle *handle); void hns3_dbg_uninit(struct hnae3_handle *handle); void hns3_dbg_register_debugfs(const char *debugfs_dir_name); void hns3_dbg_unregister_debugfs(void); void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size); +u16 hns3_get_max_available_channels(struct hnae3_handle *h); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index b48faf769b1c..82061ab6930f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -46,6 +46,12 @@ static const struct hns3_stats hns3_txq_stats[] = { HNS3_TQP_STAT("tso_err", tx_tso_err), HNS3_TQP_STAT("over_max_recursion", over_max_recursion), HNS3_TQP_STAT("hw_limitation", hw_limitation), + HNS3_TQP_STAT("bounce", tx_bounce), + HNS3_TQP_STAT("spare_full", tx_spare_full), + HNS3_TQP_STAT("copy_bits_err", copy_bits_err), + HNS3_TQP_STAT("sgl", tx_sgl), + HNS3_TQP_STAT("skb2sgl_err", skb2sgl_err), + HNS3_TQP_STAT("map_sg_err", map_sg_err), }; #define HNS3_TXQ_STATS_COUNT ARRAY_SIZE(hns3_txq_stats) @@ -65,6 +71,8 @@ static const struct hns3_stats hns3_rxq_stats[] = { HNS3_TQP_STAT("csum_complete", csum_complete), HNS3_TQP_STAT("multicast", rx_multicast), HNS3_TQP_STAT("non_reuse_pg", non_reuse_pg), + HNS3_TQP_STAT("frag_alloc_err", frag_alloc_err), + HNS3_TQP_STAT("frag_alloc", frag_alloc), }; #define HNS3_PRIV_FLAGS_LEN ARRAY_SIZE(hns3_priv_flags) @@ -88,7 +96,6 @@ static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop, bool en) { struct hnae3_handle *h = hns3_get_handle(ndev); struct hnae3_ae_dev *ae_dev = pci_get_drvdata(h->pdev); - bool vlan_filter_enable; int ret; if (!h->ae_algo->ops->set_loopback || @@ -110,14 +117,11 @@ static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop, bool en) if (ret || ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) return ret; - if (en) { + if (en) h->ae_algo->ops->set_promisc_mode(h, true, true); - } else { + else /* recover promisc mode before loopback test */ hns3_request_update_promisc_mode(h); - vlan_filter_enable = ndev->flags & IFF_PROMISC ? false : true; - hns3_enable_vlan_filter(ndev, vlan_filter_enable); - } return ret; } @@ -1134,50 +1138,32 @@ static void hns3_get_channels(struct net_device *netdev, h->ae_algo->ops->get_channels(h, ch); } -static int hns3_get_coalesce_per_queue(struct net_device *netdev, u32 queue, - struct ethtool_coalesce *cmd) +static int hns3_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *cmd) { - struct hns3_enet_tqp_vector *tx_vector, *rx_vector; struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hns3_enet_coalesce *tx_coal = &priv->tx_coal; + struct hns3_enet_coalesce *rx_coal = &priv->rx_coal; struct hnae3_handle *h = priv->ae_handle; - u16 queue_num = h->kinfo.num_tqps; if (hns3_nic_resetting(netdev)) return -EBUSY; - if (queue >= queue_num) { - netdev_err(netdev, - "Invalid queue value %u! Queue max id=%u\n", - queue, queue_num - 1); - return -EINVAL; - } - - tx_vector = priv->ring[queue].tqp_vector; - rx_vector = priv->ring[queue_num + queue].tqp_vector; - - cmd->use_adaptive_tx_coalesce = - tx_vector->tx_group.coal.adapt_enable; - cmd->use_adaptive_rx_coalesce = - rx_vector->rx_group.coal.adapt_enable; + cmd->use_adaptive_tx_coalesce = tx_coal->adapt_enable; + cmd->use_adaptive_rx_coalesce = rx_coal->adapt_enable; - cmd->tx_coalesce_usecs = tx_vector->tx_group.coal.int_gl; - cmd->rx_coalesce_usecs = rx_vector->rx_group.coal.int_gl; + cmd->tx_coalesce_usecs = tx_coal->int_gl; + cmd->rx_coalesce_usecs = rx_coal->int_gl; cmd->tx_coalesce_usecs_high = h->kinfo.int_rl_setting; cmd->rx_coalesce_usecs_high = h->kinfo.int_rl_setting; - cmd->tx_max_coalesced_frames = tx_vector->tx_group.coal.int_ql; - cmd->rx_max_coalesced_frames = rx_vector->rx_group.coal.int_ql; + cmd->tx_max_coalesced_frames = tx_coal->int_ql; + cmd->rx_max_coalesced_frames = rx_coal->int_ql; return 0; } -static int hns3_get_coalesce(struct net_device *netdev, - struct ethtool_coalesce *cmd) -{ - return hns3_get_coalesce_per_queue(netdev, 0, cmd); -} - static int hns3_check_gl_coalesce_para(struct net_device *netdev, struct ethtool_coalesce *cmd) { @@ -1292,19 +1278,7 @@ static int hns3_check_coalesce_para(struct net_device *netdev, return ret; } - ret = hns3_check_ql_coalesce_param(netdev, cmd); - if (ret) - return ret; - - if (cmd->use_adaptive_tx_coalesce == 1 || - cmd->use_adaptive_rx_coalesce == 1) { - netdev_info(netdev, - "adaptive-tx=%u and adaptive-rx=%u, tx_usecs or rx_usecs will changed dynamically.\n", - cmd->use_adaptive_tx_coalesce, - cmd->use_adaptive_rx_coalesce); - } - - return 0; + return hns3_check_ql_coalesce_param(netdev, cmd); } static void hns3_set_coalesce_per_queue(struct net_device *netdev, @@ -1350,6 +1324,9 @@ static int hns3_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *cmd) { struct hnae3_handle *h = hns3_get_handle(netdev); + struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hns3_enet_coalesce *tx_coal = &priv->tx_coal; + struct hns3_enet_coalesce *rx_coal = &priv->rx_coal; u16 queue_num = h->kinfo.num_tqps; int ret; int i; @@ -1364,6 +1341,15 @@ static int hns3_set_coalesce(struct net_device *netdev, h->kinfo.int_rl_setting = hns3_rl_round_down(cmd->rx_coalesce_usecs_high); + tx_coal->adapt_enable = cmd->use_adaptive_tx_coalesce; + rx_coal->adapt_enable = cmd->use_adaptive_rx_coalesce; + + tx_coal->int_gl = cmd->tx_coalesce_usecs; + rx_coal->int_gl = cmd->rx_coalesce_usecs; + + tx_coal->int_ql = cmd->tx_max_coalesced_frames; + rx_coal->int_ql = cmd->rx_max_coalesced_frames; + for (i = 0; i < queue_num; i++) hns3_set_coalesce_per_queue(netdev, cmd, i); @@ -1614,12 +1600,77 @@ static int hns3_set_priv_flags(struct net_device *netdev, u32 pflags) return 0; } +static int hns3_get_tunable(struct net_device *netdev, + const struct ethtool_tunable *tuna, + void *data) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + int ret = 0; + + switch (tuna->id) { + case ETHTOOL_TX_COPYBREAK: + /* all the tx rings have the same tx_copybreak */ + *(u32 *)data = priv->tx_copybreak; + break; + case ETHTOOL_RX_COPYBREAK: + *(u32 *)data = priv->rx_copybreak; + break; + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + +static int hns3_set_tunable(struct net_device *netdev, + const struct ethtool_tunable *tuna, + const void *data) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hnae3_handle *h = priv->ae_handle; + int i, ret = 0; + + switch (tuna->id) { + case ETHTOOL_TX_COPYBREAK: + priv->tx_copybreak = *(u32 *)data; + + for (i = 0; i < h->kinfo.num_tqps; i++) + priv->ring[i].tx_copybreak = priv->tx_copybreak; + + break; + case ETHTOOL_RX_COPYBREAK: + priv->rx_copybreak = *(u32 *)data; + + for (i = h->kinfo.num_tqps; i < h->kinfo.num_tqps * 2; i++) + priv->ring[i].rx_copybreak = priv->rx_copybreak; + + break; + default: + ret = -EOPNOTSUPP; + break; + } + + return ret; +} + #define HNS3_ETHTOOL_COALESCE (ETHTOOL_COALESCE_USECS | \ ETHTOOL_COALESCE_USE_ADAPTIVE | \ ETHTOOL_COALESCE_RX_USECS_HIGH | \ ETHTOOL_COALESCE_TX_USECS_HIGH | \ ETHTOOL_COALESCE_MAX_FRAMES) +static int hns3_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *info) +{ + struct hnae3_handle *handle = hns3_get_handle(netdev); + + if (handle->ae_algo->ops->get_ts_info) + return handle->ae_algo->ops->get_ts_info(handle, info); + + return ethtool_op_get_ts_info(netdev, info); +} + static const struct ethtool_ops hns3vf_ethtool_ops = { .supported_coalesce_params = HNS3_ETHTOOL_COALESCE, .get_drvinfo = hns3_get_drvinfo, @@ -1646,6 +1697,8 @@ static const struct ethtool_ops hns3vf_ethtool_ops = { .set_msglevel = hns3_set_msglevel, .get_priv_flags = hns3_get_priv_flags, .set_priv_flags = hns3_set_priv_flags, + .get_tunable = hns3_get_tunable, + .set_tunable = hns3_set_tunable, }; static const struct ethtool_ops hns3_ethtool_ops = { @@ -1684,6 +1737,9 @@ static const struct ethtool_ops hns3_ethtool_ops = { .get_module_eeprom = hns3_get_module_eeprom, .get_priv_flags = hns3_get_priv_flags, .set_priv_flags = hns3_set_priv_flags, + .get_ts_info = hns3_get_ts_info, + .get_tunable = hns3_get_tunable, + .set_tunable = hns3_set_tunable, }; void hns3_ethtool_set_ops(struct net_device *netdev) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile b/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile index 6c28c8f6292c..a685392dbfe9 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile @@ -7,6 +7,6 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3 ccflags-y += -I $(srctree)/$(src) obj-$(CONFIG_HNS3_HCLGE) += hclge.o -hclge-objs = hclge_main.o hclge_cmd.o hclge_mdio.o hclge_tm.o hclge_mbx.o hclge_err.o hclge_debugfs.o +hclge-objs = hclge_main.o hclge_cmd.o hclge_mdio.o hclge_tm.o hclge_mbx.o hclge_err.o hclge_debugfs.o hclge_ptp.o hclge-$(CONFIG_HNS3_DCB) += hclge_dcb.o diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c index 76a482456f1f..887297e37cf3 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.c @@ -178,7 +178,8 @@ static bool hclge_is_special_opcode(u16 opcode) HCLGE_QUERY_CLEAR_MPF_RAS_INT, HCLGE_QUERY_CLEAR_PF_RAS_INT, HCLGE_QUERY_CLEAR_ALL_MPF_MSIX_INT, - HCLGE_QUERY_CLEAR_ALL_PF_MSIX_INT}; + HCLGE_QUERY_CLEAR_ALL_PF_MSIX_INT, + HCLGE_QUERY_ALL_ERR_INFO}; int i; for (i = 0; i < ARRAY_SIZE(spec_opcode); i++) { @@ -386,6 +387,14 @@ static void hclge_parse_capability(struct hclge_dev *hdev, set_bit(HNAE3_DEV_SUPPORT_PAUSE_B, ae_dev->caps); if (hnae3_get_bit(caps, HCLGE_CAP_PHY_IMP_B)) set_bit(HNAE3_DEV_SUPPORT_PHY_IMP_B, ae_dev->caps); + if (hnae3_get_bit(caps, HCLGE_CAP_RAS_IMP_B)) + set_bit(HNAE3_DEV_SUPPORT_RAS_IMP_B, ae_dev->caps); + if (hnae3_get_bit(caps, HCLGE_CAP_RXD_ADV_LAYOUT_B)) + set_bit(HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B, ae_dev->caps); + if (hnae3_get_bit(caps, HCLGE_CAP_PORT_VLAN_BYPASS_B)) { + set_bit(HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, ae_dev->caps); + set_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps); + } } static __le32 hclge_build_api_caps(void) @@ -469,7 +478,7 @@ static int hclge_firmware_compat_config(struct hclge_dev *hdev) struct hclge_desc desc; u32 compat = 0; - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_M7_COMPAT_CFG, false); + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_IMP_COMPAT_CFG, false); req = (struct hclge_firmware_compat_cmd *)desc.data; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index c6fc22e29581..18bde77ef944 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -130,6 +130,10 @@ enum hclge_opcode_type { HCLGE_OPC_COMMON_LOOPBACK = 0x0315, HCLGE_OPC_CONFIG_FEC_MODE = 0x031A, + /* PTP commands */ + HCLGE_OPC_PTP_INT_EN = 0x0501, + HCLGE_OPC_PTP_MODE_CFG = 0x0507, + /* PFC/Pause commands */ HCLGE_OPC_CFG_MAC_PAUSE_EN = 0x0701, HCLGE_OPC_CFG_PFC_PAUSE_EN = 0x0702, @@ -236,6 +240,7 @@ enum hclge_opcode_type { HCLGE_OPC_VLAN_FILTER_CTRL = 0x1100, HCLGE_OPC_VLAN_FILTER_PF_CFG = 0x1101, HCLGE_OPC_VLAN_FILTER_VF_CFG = 0x1102, + HCLGE_OPC_PORT_VLAN_BYPASS = 0x1103, /* Flow Director commands */ HCLGE_OPC_FD_MODE_CTRL = 0x1200, @@ -243,6 +248,7 @@ enum hclge_opcode_type { HCLGE_OPC_FD_KEY_CONFIG = 0x1202, HCLGE_OPC_FD_TCAM_OP = 0x1203, HCLGE_OPC_FD_AD_OP = 0x1204, + HCLGE_OPC_FD_CNT_OP = 0x1205, HCLGE_OPC_FD_USER_DEF_OP = 0x1207, /* MDIO command */ @@ -267,10 +273,10 @@ enum hclge_opcode_type { /* NCL config command */ HCLGE_OPC_QUERY_NCL_CONFIG = 0x7011, - /* M7 stats command */ - HCLGE_OPC_M7_STATS_BD = 0x7012, - HCLGE_OPC_M7_STATS_INFO = 0x7013, - HCLGE_OPC_M7_COMPAT_CFG = 0x701A, + /* IMP stats command */ + HCLGE_OPC_IMP_STATS_BD = 0x7012, + HCLGE_OPC_IMP_STATS_INFO = 0x7013, + HCLGE_OPC_IMP_COMPAT_CFG = 0x701A, /* SFP command */ HCLGE_OPC_GET_SFP_EEPROM = 0x7100, @@ -292,6 +298,8 @@ enum hclge_opcode_type { HCLGE_QUERY_MSIX_INT_STS_BD_NUM = 0x1513, HCLGE_QUERY_CLEAR_ALL_MPF_MSIX_INT = 0x1514, HCLGE_QUERY_CLEAR_ALL_PF_MSIX_INT = 0x1515, + HCLGE_QUERY_ALL_ERR_BD_NUM = 0x1516, + HCLGE_QUERY_ALL_ERR_INFO = 0x1517, HCLGE_CONFIG_ROCEE_RAS_INT_EN = 0x1580, HCLGE_QUERY_CLEAR_ROCEE_RAS_INT = 0x1581, HCLGE_ROCEE_PF_RAS_INT_CMD = 0x1584, @@ -389,8 +397,11 @@ enum HCLGE_CAP_BITS { HCLGE_CAP_HW_PAD_B, HCLGE_CAP_STASH_B, HCLGE_CAP_UDP_TUNNEL_CSUM_B, + HCLGE_CAP_RAS_IMP_B = 12, HCLGE_CAP_FEC_B = 13, HCLGE_CAP_PAUSE_B = 14, + HCLGE_CAP_RXD_ADV_LAYOUT_B = 15, + HCLGE_CAP_PORT_VLAN_BYPASS_B = 17, }; enum HCLGE_API_CAP_BITS { @@ -526,10 +537,14 @@ struct hclge_pf_res_cmd { #define HCLGE_CFG_SPEED_ABILITY_M GENMASK(7, 0) #define HCLGE_CFG_SPEED_ABILITY_EXT_S 10 #define HCLGE_CFG_SPEED_ABILITY_EXT_M GENMASK(15, 10) +#define HCLGE_CFG_VLAN_FLTR_CAP_S 8 +#define HCLGE_CFG_VLAN_FLTR_CAP_M GENMASK(9, 8) #define HCLGE_CFG_UMV_TBL_SPACE_S 16 #define HCLGE_CFG_UMV_TBL_SPACE_M GENMASK(31, 16) #define HCLGE_CFG_PF_RSS_SIZE_S 0 #define HCLGE_CFG_PF_RSS_SIZE_M GENMASK(3, 0) +#define HCLGE_CFG_TX_SPARE_BUF_SIZE_S 4 +#define HCLGE_CFG_TX_SPARE_BUF_SIZE_M GENMASK(15, 4) #define HCLGE_CFG_CMD_CNT 4 @@ -810,6 +825,14 @@ struct hclge_vlan_filter_vf_cfg_cmd { u8 vf_bitmap[HCLGE_MAX_VF_BYTES]; }; +#define HCLGE_INGRESS_BYPASS_B 0 +struct hclge_port_vlan_filter_bypass_cmd { + u8 bypass_state; + u8 rsv1[3]; + u8 vf_id; + u8 rsv2[19]; +}; + #define HCLGE_SWITCH_ANTI_SPOOF_B 0U #define HCLGE_SWITCH_ALW_LPBK_B 1U #define HCLGE_SWITCH_ALW_LCL_LPBK_B 2U @@ -1087,6 +1110,14 @@ struct hclge_fd_ad_config_cmd { u8 rsv2[8]; }; +struct hclge_fd_ad_cnt_read_cmd { + u8 rsv0[4]; + __le16 index; + u8 rsv1[2]; + __le64 cnt; + u8 rsv2[8]; +}; + #define HCLGE_FD_USER_DEF_OFT_S 0 #define HCLGE_FD_USER_DEF_OFT_M GENMASK(14, 0) #define HCLGE_FD_USER_DEF_EN_B 15 @@ -1100,7 +1131,7 @@ struct hclge_fd_user_def_cfg_cmd { u8 rsv[12]; }; -struct hclge_get_m7_bd_cmd { +struct hclge_get_imp_bd_cmd { __le32 bd_num; u8 rsv[20]; }; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c index 85d306459e36..288788186ecc 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c @@ -4,74 +4,110 @@ #include <linux/device.h> #include "hclge_debugfs.h" +#include "hclge_err.h" #include "hclge_main.h" #include "hclge_tm.h" #include "hnae3.h" +static const char * const state_str[] = { "off", "on" }; +static const char * const hclge_mac_state_str[] = { + "TO_ADD", "TO_DEL", "ACTIVE" +}; + static const struct hclge_dbg_reg_type_info hclge_dbg_reg_info[] = { - { .reg_type = "bios common", + { .cmd = HNAE3_DBG_CMD_REG_BIOS_COMMON, .dfx_msg = &hclge_dbg_bios_common_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_bios_common_reg), .offset = HCLGE_DBG_DFX_BIOS_OFFSET, .cmd = HCLGE_OPC_DFX_BIOS_COMMON_REG } }, - { .reg_type = "ssu", + { .cmd = HNAE3_DBG_CMD_REG_SSU, .dfx_msg = &hclge_dbg_ssu_reg_0[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_ssu_reg_0), .offset = HCLGE_DBG_DFX_SSU_0_OFFSET, .cmd = HCLGE_OPC_DFX_SSU_REG_0 } }, - { .reg_type = "ssu", + { .cmd = HNAE3_DBG_CMD_REG_SSU, .dfx_msg = &hclge_dbg_ssu_reg_1[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_ssu_reg_1), .offset = HCLGE_DBG_DFX_SSU_1_OFFSET, .cmd = HCLGE_OPC_DFX_SSU_REG_1 } }, - { .reg_type = "ssu", + { .cmd = HNAE3_DBG_CMD_REG_SSU, .dfx_msg = &hclge_dbg_ssu_reg_2[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_ssu_reg_2), .offset = HCLGE_DBG_DFX_SSU_2_OFFSET, .cmd = HCLGE_OPC_DFX_SSU_REG_2 } }, - { .reg_type = "igu egu", + { .cmd = HNAE3_DBG_CMD_REG_IGU_EGU, .dfx_msg = &hclge_dbg_igu_egu_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_igu_egu_reg), .offset = HCLGE_DBG_DFX_IGU_OFFSET, .cmd = HCLGE_OPC_DFX_IGU_EGU_REG } }, - { .reg_type = "rpu", + { .cmd = HNAE3_DBG_CMD_REG_RPU, .dfx_msg = &hclge_dbg_rpu_reg_0[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_rpu_reg_0), .offset = HCLGE_DBG_DFX_RPU_0_OFFSET, .cmd = HCLGE_OPC_DFX_RPU_REG_0 } }, - { .reg_type = "rpu", + { .cmd = HNAE3_DBG_CMD_REG_RPU, .dfx_msg = &hclge_dbg_rpu_reg_1[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_rpu_reg_1), .offset = HCLGE_DBG_DFX_RPU_1_OFFSET, .cmd = HCLGE_OPC_DFX_RPU_REG_1 } }, - { .reg_type = "ncsi", + { .cmd = HNAE3_DBG_CMD_REG_NCSI, .dfx_msg = &hclge_dbg_ncsi_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_ncsi_reg), .offset = HCLGE_DBG_DFX_NCSI_OFFSET, .cmd = HCLGE_OPC_DFX_NCSI_REG } }, - { .reg_type = "rtc", + { .cmd = HNAE3_DBG_CMD_REG_RTC, .dfx_msg = &hclge_dbg_rtc_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_rtc_reg), .offset = HCLGE_DBG_DFX_RTC_OFFSET, .cmd = HCLGE_OPC_DFX_RTC_REG } }, - { .reg_type = "ppp", + { .cmd = HNAE3_DBG_CMD_REG_PPP, .dfx_msg = &hclge_dbg_ppp_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_ppp_reg), .offset = HCLGE_DBG_DFX_PPP_OFFSET, .cmd = HCLGE_OPC_DFX_PPP_REG } }, - { .reg_type = "rcb", + { .cmd = HNAE3_DBG_CMD_REG_RCB, .dfx_msg = &hclge_dbg_rcb_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_rcb_reg), .offset = HCLGE_DBG_DFX_RCB_OFFSET, .cmd = HCLGE_OPC_DFX_RCB_REG } }, - { .reg_type = "tqp", + { .cmd = HNAE3_DBG_CMD_REG_TQP, .dfx_msg = &hclge_dbg_tqp_reg[0], .reg_msg = { .msg_num = ARRAY_SIZE(hclge_dbg_tqp_reg), .offset = HCLGE_DBG_DFX_TQP_OFFSET, .cmd = HCLGE_OPC_DFX_TQP_REG } }, }; -static int hclge_dbg_get_dfx_bd_num(struct hclge_dev *hdev, int offset) +static void hclge_dbg_fill_content(char *content, u16 len, + const struct hclge_dbg_item *items, + const char **result, u16 size) +{ + char *pos = content; + u16 i; + + memset(content, ' ', len); + for (i = 0; i < size; i++) { + if (result) + strncpy(pos, result[i], strlen(result[i])); + else + strncpy(pos, items[i].name, strlen(items[i].name)); + pos += strlen(items[i].name) + items[i].interval; + } + *pos++ = '\n'; + *pos++ = '\0'; +} + +static char *hclge_dbg_get_func_id_str(char *buf, u8 id) +{ + if (id) + sprintf(buf, "vf%u", id - 1); + else + sprintf(buf, "pf"); + + return buf; +} + +static int hclge_dbg_get_dfx_bd_num(struct hclge_dev *hdev, int offset, + u32 *bd_num) { struct hclge_desc desc[HCLGE_GET_DFX_REG_TYPE_CNT]; int entries_per_desc; @@ -81,13 +117,21 @@ static int hclge_dbg_get_dfx_bd_num(struct hclge_dev *hdev, int offset) ret = hclge_query_bd_num_cmd_send(hdev, desc); if (ret) { dev_err(&hdev->pdev->dev, - "get dfx bdnum fail, ret = %d\n", ret); + "failed to get dfx bd_num, offset = %d, ret = %d\n", + offset, ret); return ret; } entries_per_desc = ARRAY_SIZE(desc[0].data); index = offset % entries_per_desc; - return le32_to_cpu(desc[offset / entries_per_desc].data[index]); + + *bd_num = le32_to_cpu(desc[offset / entries_per_desc].data[index]); + if (!(*bd_num)) { + dev_err(&hdev->pdev->dev, "The value of dfx bd_num is 0!\n"); + return -EINVAL; + } + + return 0; } static int hclge_dbg_cmd_send(struct hclge_dev *hdev, @@ -114,66 +158,108 @@ static int hclge_dbg_cmd_send(struct hclge_dev *hdev, return ret; } -static void hclge_dbg_dump_reg_common(struct hclge_dev *hdev, - const struct hclge_dbg_reg_type_info *reg_info, - const char *cmd_buf) +static int +hclge_dbg_dump_reg_tqp(struct hclge_dev *hdev, + const struct hclge_dbg_reg_type_info *reg_info, + char *buf, int len, int *pos) { -#define IDX_OFFSET 1 - - const char *s = &cmd_buf[strlen(reg_info->reg_type) + IDX_OFFSET]; const struct hclge_dbg_dfx_message *dfx_message = reg_info->dfx_msg; const struct hclge_dbg_reg_common_msg *reg_msg = ®_info->reg_msg; struct hclge_desc *desc_src; + u32 index, entry, i, cnt; + int bd_num, min_num, ret; struct hclge_desc *desc; - int entries_per_desc; - int bd_num, buf_len; - int index = 0; - int min_num; - int ret, i; - if (*s) { - ret = kstrtouint(s, 0, &index); - index = (ret != 0) ? 0 : index; - } + ret = hclge_dbg_get_dfx_bd_num(hdev, reg_msg->offset, &bd_num); + if (ret) + return ret; + + desc_src = kcalloc(bd_num, sizeof(struct hclge_desc), GFP_KERNEL); + if (!desc_src) + return -ENOMEM; + + min_num = min_t(int, bd_num * HCLGE_DESC_DATA_LEN, reg_msg->msg_num); + + for (i = 0, cnt = 0; i < min_num; i++, dfx_message++) + *pos += scnprintf(buf + *pos, len - *pos, "item%u = %s\n", + cnt++, dfx_message->message); + + for (i = 0; i < cnt; i++) + *pos += scnprintf(buf + *pos, len - *pos, "item%u\t", i); - bd_num = hclge_dbg_get_dfx_bd_num(hdev, reg_msg->offset); - if (bd_num <= 0) { - dev_err(&hdev->pdev->dev, "get cmd(%d) bd num(%d) failed\n", - reg_msg->offset, bd_num); - return; + *pos += scnprintf(buf + *pos, len - *pos, "\n"); + + for (index = 0; index < hdev->vport[0].alloc_tqps; index++) { + dfx_message = reg_info->dfx_msg; + desc = desc_src; + ret = hclge_dbg_cmd_send(hdev, desc, index, bd_num, + reg_msg->cmd); + if (ret) + break; + + for (i = 0; i < min_num; i++, dfx_message++) { + entry = i % HCLGE_DESC_DATA_LEN; + if (i > 0 && !entry) + desc++; + + *pos += scnprintf(buf + *pos, len - *pos, "%#x\t", + le32_to_cpu(desc->data[entry])); + } + *pos += scnprintf(buf + *pos, len - *pos, "\n"); } - buf_len = sizeof(struct hclge_desc) * bd_num; - desc_src = kzalloc(buf_len, GFP_KERNEL); + kfree(desc_src); + return ret; +} + +static int +hclge_dbg_dump_reg_common(struct hclge_dev *hdev, + const struct hclge_dbg_reg_type_info *reg_info, + char *buf, int len, int *pos) +{ + const struct hclge_dbg_reg_common_msg *reg_msg = ®_info->reg_msg; + const struct hclge_dbg_dfx_message *dfx_message = reg_info->dfx_msg; + struct hclge_desc *desc_src; + int bd_num, min_num, ret; + struct hclge_desc *desc; + u32 entry, i; + + ret = hclge_dbg_get_dfx_bd_num(hdev, reg_msg->offset, &bd_num); + if (ret) + return ret; + + desc_src = kcalloc(bd_num, sizeof(struct hclge_desc), GFP_KERNEL); if (!desc_src) - return; + return -ENOMEM; desc = desc_src; - ret = hclge_dbg_cmd_send(hdev, desc, index, bd_num, reg_msg->cmd); + + ret = hclge_dbg_cmd_send(hdev, desc, 0, bd_num, reg_msg->cmd); if (ret) { - kfree(desc_src); - return; + kfree(desc); + return ret; } - entries_per_desc = ARRAY_SIZE(desc->data); - min_num = min_t(int, bd_num * entries_per_desc, reg_msg->msg_num); + min_num = min_t(int, bd_num * HCLGE_DESC_DATA_LEN, reg_msg->msg_num); - desc = desc_src; - for (i = 0; i < min_num; i++) { - if (i > 0 && (i % entries_per_desc) == 0) + for (i = 0; i < min_num; i++, dfx_message++) { + entry = i % HCLGE_DESC_DATA_LEN; + if (i > 0 && !entry) desc++; - if (dfx_message->flag) - dev_info(&hdev->pdev->dev, "%s: 0x%x\n", - dfx_message->message, - le32_to_cpu(desc->data[i % entries_per_desc])); + if (!dfx_message->flag) + continue; - dfx_message++; + *pos += scnprintf(buf + *pos, len - *pos, "%s: %#x\n", + dfx_message->message, + le32_to_cpu(desc->data[entry])); } kfree(desc_src); + return 0; } -static void hclge_dbg_dump_mac_enable_status(struct hclge_dev *hdev) +static int hclge_dbg_dump_mac_enable_status(struct hclge_dev *hdev, char *buf, + int len, int *pos) { struct hclge_config_mac_mode_cmd *req; struct hclge_desc desc; @@ -186,43 +272,51 @@ static void hclge_dbg_dump_mac_enable_status(struct hclge_dev *hdev) if (ret) { dev_err(&hdev->pdev->dev, "failed to dump mac enable status, ret = %d\n", ret); - return; + return ret; } req = (struct hclge_config_mac_mode_cmd *)desc.data; loop_en = le32_to_cpu(req->txrx_pad_fcs_loop_en); - dev_info(&hdev->pdev->dev, "config_mac_trans_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_TX_EN_B)); - dev_info(&hdev->pdev->dev, "config_mac_rcv_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_EN_B)); - dev_info(&hdev->pdev->dev, "config_pad_trans_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_PAD_TX_B)); - dev_info(&hdev->pdev->dev, "config_pad_rcv_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_PAD_RX_B)); - dev_info(&hdev->pdev->dev, "config_1588_trans_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_1588_TX_B)); - dev_info(&hdev->pdev->dev, "config_1588_rcv_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_1588_RX_B)); - dev_info(&hdev->pdev->dev, "config_mac_app_loop_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_APP_LP_B)); - dev_info(&hdev->pdev->dev, "config_mac_line_loop_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_LINE_LP_B)); - dev_info(&hdev->pdev->dev, "config_mac_fcs_tx_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_FCS_TX_B)); - dev_info(&hdev->pdev->dev, "config_mac_rx_oversize_truncate_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B)); - dev_info(&hdev->pdev->dev, "config_mac_rx_fcs_strip_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_FCS_STRIP_B)); - dev_info(&hdev->pdev->dev, "config_mac_rx_fcs_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_FCS_B)); - dev_info(&hdev->pdev->dev, "config_mac_tx_under_min_err_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_TX_UNDER_MIN_ERR_B)); - dev_info(&hdev->pdev->dev, "config_mac_tx_oversize_truncate_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B)); -} - -static void hclge_dbg_dump_mac_frame_size(struct hclge_dev *hdev) + *pos += scnprintf(buf + *pos, len - *pos, "mac_trans_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_TX_EN_B)); + *pos += scnprintf(buf + *pos, len - *pos, "mac_rcv_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_RX_EN_B)); + *pos += scnprintf(buf + *pos, len - *pos, "pad_trans_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_PAD_TX_B)); + *pos += scnprintf(buf + *pos, len - *pos, "pad_rcv_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_PAD_RX_B)); + *pos += scnprintf(buf + *pos, len - *pos, "1588_trans_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_1588_TX_B)); + *pos += scnprintf(buf + *pos, len - *pos, "1588_rcv_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_1588_RX_B)); + *pos += scnprintf(buf + *pos, len - *pos, "mac_app_loop_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_APP_LP_B)); + *pos += scnprintf(buf + *pos, len - *pos, "mac_line_loop_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_LINE_LP_B)); + *pos += scnprintf(buf + *pos, len - *pos, "mac_fcs_tx_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_FCS_TX_B)); + *pos += scnprintf(buf + *pos, len - *pos, + "mac_rx_oversize_truncate_en: %#x\n", + hnae3_get_bit(loop_en, + HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B)); + *pos += scnprintf(buf + *pos, len - *pos, "mac_rx_fcs_strip_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_RX_FCS_STRIP_B)); + *pos += scnprintf(buf + *pos, len - *pos, "mac_rx_fcs_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_RX_FCS_B)); + *pos += scnprintf(buf + *pos, len - *pos, + "mac_tx_under_min_err_en: %#x\n", + hnae3_get_bit(loop_en, HCLGE_MAC_TX_UNDER_MIN_ERR_B)); + *pos += scnprintf(buf + *pos, len - *pos, + "mac_tx_oversize_truncate_en: %#x\n", + hnae3_get_bit(loop_en, + HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B)); + + return 0; +} + +static int hclge_dbg_dump_mac_frame_size(struct hclge_dev *hdev, char *buf, + int len, int *pos) { struct hclge_config_max_frm_size_cmd *req; struct hclge_desc desc; @@ -234,17 +328,21 @@ static void hclge_dbg_dump_mac_frame_size(struct hclge_dev *hdev) if (ret) { dev_err(&hdev->pdev->dev, "failed to dump mac frame size, ret = %d\n", ret); - return; + return ret; } req = (struct hclge_config_max_frm_size_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "max_frame_size: %u\n", - le16_to_cpu(req->max_frm_size)); - dev_info(&hdev->pdev->dev, "min_frame_size: %u\n", req->min_frm_size); + *pos += scnprintf(buf + *pos, len - *pos, "max_frame_size: %u\n", + le16_to_cpu(req->max_frm_size)); + *pos += scnprintf(buf + *pos, len - *pos, "min_frame_size: %u\n", + req->min_frm_size); + + return 0; } -static void hclge_dbg_dump_mac_speed_duplex(struct hclge_dev *hdev) +static int hclge_dbg_dump_mac_speed_duplex(struct hclge_dev *hdev, char *buf, + int len, int *pos) { #define HCLGE_MAC_SPEED_SHIFT 0 #define HCLGE_MAC_SPEED_MASK GENMASK(5, 0) @@ -260,543 +358,540 @@ static void hclge_dbg_dump_mac_speed_duplex(struct hclge_dev *hdev) if (ret) { dev_err(&hdev->pdev->dev, "failed to dump mac speed duplex, ret = %d\n", ret); - return; + return ret; } req = (struct hclge_config_mac_speed_dup_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "speed: %#lx\n", - hnae3_get_field(req->speed_dup, HCLGE_MAC_SPEED_MASK, - HCLGE_MAC_SPEED_SHIFT)); - dev_info(&hdev->pdev->dev, "duplex: %#x\n", - hnae3_get_bit(req->speed_dup, HCLGE_MAC_DUPLEX_SHIFT)); + *pos += scnprintf(buf + *pos, len - *pos, "speed: %#lx\n", + hnae3_get_field(req->speed_dup, HCLGE_MAC_SPEED_MASK, + HCLGE_MAC_SPEED_SHIFT)); + *pos += scnprintf(buf + *pos, len - *pos, "duplex: %#x\n", + hnae3_get_bit(req->speed_dup, + HCLGE_MAC_DUPLEX_SHIFT)); + return 0; } -static void hclge_dbg_dump_mac(struct hclge_dev *hdev) +static int hclge_dbg_dump_mac(struct hclge_dev *hdev, char *buf, int len) { - hclge_dbg_dump_mac_enable_status(hdev); + int pos = 0; + int ret; - hclge_dbg_dump_mac_frame_size(hdev); + ret = hclge_dbg_dump_mac_enable_status(hdev, buf, len, &pos); + if (ret) + return ret; - hclge_dbg_dump_mac_speed_duplex(hdev); + ret = hclge_dbg_dump_mac_frame_size(hdev, buf, len, &pos); + if (ret) + return ret; + + return hclge_dbg_dump_mac_speed_duplex(hdev, buf, len, &pos); } -static void hclge_dbg_dump_dcb(struct hclge_dev *hdev, const char *cmd_buf) +static int hclge_dbg_dump_dcb_qset(struct hclge_dev *hdev, char *buf, int len, + int *pos) { - struct device *dev = &hdev->pdev->dev; struct hclge_dbg_bitmap_cmd *bitmap; - enum hclge_opcode_type cmd; - int rq_id, pri_id, qset_id; - int port_id, nq_id, pg_id; - struct hclge_desc desc[2]; - - int cnt, ret; - - cnt = sscanf(cmd_buf, "%i %i %i %i %i %i", - &port_id, &pri_id, &pg_id, &rq_id, &nq_id, &qset_id); - if (cnt != 6) { - dev_err(&hdev->pdev->dev, - "dump dcb: bad command parameter, cnt=%d\n", cnt); - return; - } + struct hclge_desc desc; + u16 qset_id, qset_num; + int ret; - cmd = HCLGE_OPC_QSET_DFX_STS; - ret = hclge_dbg_cmd_send(hdev, desc, qset_id, 1, cmd); + ret = hclge_tm_get_qset_num(hdev, &qset_num); if (ret) - goto err_dcb_cmd_send; + return ret; - bitmap = (struct hclge_dbg_bitmap_cmd *)&desc[0].data[1]; - dev_info(dev, "roce_qset_mask: 0x%x\n", bitmap->bit0); - dev_info(dev, "nic_qs_mask: 0x%x\n", bitmap->bit1); - dev_info(dev, "qs_shaping_pass: 0x%x\n", bitmap->bit2); - dev_info(dev, "qs_bp_sts: 0x%x\n", bitmap->bit3); + *pos += scnprintf(buf + *pos, len - *pos, + "qset_id roce_qset_mask nic_qset_mask qset_shaping_pass qset_bp_status\n"); + for (qset_id = 0; qset_id < qset_num; qset_id++) { + ret = hclge_dbg_cmd_send(hdev, &desc, qset_id, 1, + HCLGE_OPC_QSET_DFX_STS); + if (ret) + return ret; - cmd = HCLGE_OPC_PRI_DFX_STS; - ret = hclge_dbg_cmd_send(hdev, desc, pri_id, 1, cmd); - if (ret) - goto err_dcb_cmd_send; + bitmap = (struct hclge_dbg_bitmap_cmd *)&desc.data[1]; - bitmap = (struct hclge_dbg_bitmap_cmd *)&desc[0].data[1]; - dev_info(dev, "pri_mask: 0x%x\n", bitmap->bit0); - dev_info(dev, "pri_cshaping_pass: 0x%x\n", bitmap->bit1); - dev_info(dev, "pri_pshaping_pass: 0x%x\n", bitmap->bit2); + *pos += scnprintf(buf + *pos, len - *pos, + "%04u %#x %#x %#x %#x\n", + qset_id, bitmap->bit0, bitmap->bit1, + bitmap->bit2, bitmap->bit3); + } - cmd = HCLGE_OPC_PG_DFX_STS; - ret = hclge_dbg_cmd_send(hdev, desc, pg_id, 1, cmd); - if (ret) - goto err_dcb_cmd_send; + return 0; +} - bitmap = (struct hclge_dbg_bitmap_cmd *)&desc[0].data[1]; - dev_info(dev, "pg_mask: 0x%x\n", bitmap->bit0); - dev_info(dev, "pg_cshaping_pass: 0x%x\n", bitmap->bit1); - dev_info(dev, "pg_pshaping_pass: 0x%x\n", bitmap->bit2); +static int hclge_dbg_dump_dcb_pri(struct hclge_dev *hdev, char *buf, int len, + int *pos) +{ + struct hclge_dbg_bitmap_cmd *bitmap; + struct hclge_desc desc; + u8 pri_id, pri_num; + int ret; - cmd = HCLGE_OPC_PORT_DFX_STS; - ret = hclge_dbg_cmd_send(hdev, desc, port_id, 1, cmd); + ret = hclge_tm_get_pri_num(hdev, &pri_num); if (ret) - goto err_dcb_cmd_send; - - bitmap = (struct hclge_dbg_bitmap_cmd *)&desc[0].data[1]; - dev_info(dev, "port_mask: 0x%x\n", bitmap->bit0); - dev_info(dev, "port_shaping_pass: 0x%x\n", bitmap->bit1); + return ret; - cmd = HCLGE_OPC_SCH_NQ_CNT; - ret = hclge_dbg_cmd_send(hdev, desc, nq_id, 1, cmd); - if (ret) - goto err_dcb_cmd_send; + *pos += scnprintf(buf + *pos, len - *pos, + "pri_id pri_mask pri_cshaping_pass pri_pshaping_pass\n"); + for (pri_id = 0; pri_id < pri_num; pri_id++) { + ret = hclge_dbg_cmd_send(hdev, &desc, pri_id, 1, + HCLGE_OPC_PRI_DFX_STS); + if (ret) + return ret; - dev_info(dev, "sch_nq_cnt: 0x%x\n", le32_to_cpu(desc[0].data[1])); + bitmap = (struct hclge_dbg_bitmap_cmd *)&desc.data[1]; - cmd = HCLGE_OPC_SCH_RQ_CNT; - ret = hclge_dbg_cmd_send(hdev, desc, nq_id, 1, cmd); - if (ret) - goto err_dcb_cmd_send; + *pos += scnprintf(buf + *pos, len - *pos, + "%03u %#x %#x %#x\n", + pri_id, bitmap->bit0, bitmap->bit1, + bitmap->bit2); + } - dev_info(dev, "sch_rq_cnt: 0x%x\n", le32_to_cpu(desc[0].data[1])); + return 0; +} - cmd = HCLGE_OPC_TM_INTERNAL_STS; - ret = hclge_dbg_cmd_send(hdev, desc, 0, 2, cmd); - if (ret) - goto err_dcb_cmd_send; - - dev_info(dev, "pri_bp: 0x%x\n", le32_to_cpu(desc[0].data[1])); - dev_info(dev, "fifo_dfx_info: 0x%x\n", le32_to_cpu(desc[0].data[2])); - dev_info(dev, "sch_roce_fifo_afull_gap: 0x%x\n", - le32_to_cpu(desc[0].data[3])); - dev_info(dev, "tx_private_waterline: 0x%x\n", - le32_to_cpu(desc[0].data[4])); - dev_info(dev, "tm_bypass_en: 0x%x\n", le32_to_cpu(desc[0].data[5])); - dev_info(dev, "SSU_TM_BYPASS_EN: 0x%x\n", le32_to_cpu(desc[1].data[0])); - dev_info(dev, "SSU_RESERVE_CFG: 0x%x\n", le32_to_cpu(desc[1].data[1])); - - cmd = HCLGE_OPC_TM_INTERNAL_CNT; - ret = hclge_dbg_cmd_send(hdev, desc, port_id, 1, cmd); - if (ret) - goto err_dcb_cmd_send; +static int hclge_dbg_dump_dcb_pg(struct hclge_dev *hdev, char *buf, int len, + int *pos) +{ + struct hclge_dbg_bitmap_cmd *bitmap; + struct hclge_desc desc; + u8 pg_id; + int ret; - dev_info(dev, "SCH_NIC_NUM: 0x%x\n", le32_to_cpu(desc[0].data[1])); - dev_info(dev, "SCH_ROCE_NUM: 0x%x\n", le32_to_cpu(desc[0].data[2])); + *pos += scnprintf(buf + *pos, len - *pos, + "pg_id pg_mask pg_cshaping_pass pg_pshaping_pass\n"); + for (pg_id = 0; pg_id < hdev->tm_info.num_pg; pg_id++) { + ret = hclge_dbg_cmd_send(hdev, &desc, pg_id, 1, + HCLGE_OPC_PG_DFX_STS); + if (ret) + return ret; - cmd = HCLGE_OPC_TM_INTERNAL_STS_1; - ret = hclge_dbg_cmd_send(hdev, desc, port_id, 1, cmd); - if (ret) - goto err_dcb_cmd_send; + bitmap = (struct hclge_dbg_bitmap_cmd *)&desc.data[1]; - dev_info(dev, "TC_MAP_SEL: 0x%x\n", le32_to_cpu(desc[0].data[1])); - dev_info(dev, "IGU_PFC_PRI_EN: 0x%x\n", le32_to_cpu(desc[0].data[2])); - dev_info(dev, "MAC_PFC_PRI_EN: 0x%x\n", le32_to_cpu(desc[0].data[3])); - dev_info(dev, "IGU_PRI_MAP_TC_CFG: 0x%x\n", - le32_to_cpu(desc[0].data[4])); - dev_info(dev, "IGU_TX_PRI_MAP_TC_CFG: 0x%x\n", - le32_to_cpu(desc[0].data[5])); - return; + *pos += scnprintf(buf + *pos, len - *pos, + "%03u %#x %#x %#x\n", + pg_id, bitmap->bit0, bitmap->bit1, + bitmap->bit2); + } -err_dcb_cmd_send: - dev_err(&hdev->pdev->dev, - "failed to dump dcb dfx, cmd = %#x, ret = %d\n", - cmd, ret); + return 0; } -static void hclge_dbg_dump_reg_cmd(struct hclge_dev *hdev, const char *cmd_buf) +static int hclge_dbg_dump_dcb_queue(struct hclge_dev *hdev, char *buf, int len, + int *pos) { - const struct hclge_dbg_reg_type_info *reg_info; - bool has_dump = false; - int i; + struct hclge_desc desc; + u16 nq_id; + int ret; - for (i = 0; i < ARRAY_SIZE(hclge_dbg_reg_info); i++) { - reg_info = &hclge_dbg_reg_info[i]; - if (!strncmp(cmd_buf, reg_info->reg_type, - strlen(reg_info->reg_type))) { - hclge_dbg_dump_reg_common(hdev, reg_info, cmd_buf); - has_dump = true; - } - } + *pos += scnprintf(buf + *pos, len - *pos, + "nq_id sch_nic_queue_cnt sch_roce_queue_cnt\n"); + for (nq_id = 0; nq_id < hdev->num_tqps; nq_id++) { + ret = hclge_dbg_cmd_send(hdev, &desc, nq_id, 1, + HCLGE_OPC_SCH_NQ_CNT); + if (ret) + return ret; - if (strncmp(cmd_buf, "mac", strlen("mac")) == 0) { - hclge_dbg_dump_mac(hdev); - has_dump = true; - } + *pos += scnprintf(buf + *pos, len - *pos, "%04u %#x", + nq_id, le32_to_cpu(desc.data[1])); - if (strncmp(cmd_buf, "dcb", 3) == 0) { - hclge_dbg_dump_dcb(hdev, &cmd_buf[sizeof("dcb")]); - has_dump = true; - } + ret = hclge_dbg_cmd_send(hdev, &desc, nq_id, 1, + HCLGE_OPC_SCH_RQ_CNT); + if (ret) + return ret; - if (!has_dump) { - dev_info(&hdev->pdev->dev, "unknown command\n"); - return; + *pos += scnprintf(buf + *pos, len - *pos, + " %#x\n", + le32_to_cpu(desc.data[1])); } -} -static void hclge_print_tc_info(struct hclge_dev *hdev, bool flag, int index) -{ - if (flag) - dev_info(&hdev->pdev->dev, "tc(%d): no sp mode weight: %u\n", - index, hdev->tm_info.pg_info[0].tc_dwrr[index]); - else - dev_info(&hdev->pdev->dev, "tc(%d): sp mode\n", index); + return 0; } -static void hclge_dbg_dump_tc(struct hclge_dev *hdev) +static int hclge_dbg_dump_dcb_port(struct hclge_dev *hdev, char *buf, int len, + int *pos) { - struct hclge_ets_tc_weight_cmd *ets_weight; + struct hclge_dbg_bitmap_cmd *bitmap; struct hclge_desc desc; - int i, ret; - - if (!hnae3_dev_dcb_supported(hdev)) { - dev_info(&hdev->pdev->dev, - "Only DCB-supported dev supports tc\n"); - return; - } - - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_ETS_TC_WEIGHT, true); + u8 port_id = 0; + int ret; - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) { - dev_err(&hdev->pdev->dev, "dump tc fail, ret = %d\n", ret); - return; - } + ret = hclge_dbg_cmd_send(hdev, &desc, port_id, 1, + HCLGE_OPC_PORT_DFX_STS); + if (ret) + return ret; - ets_weight = (struct hclge_ets_tc_weight_cmd *)desc.data; + bitmap = (struct hclge_dbg_bitmap_cmd *)&desc.data[1]; - dev_info(&hdev->pdev->dev, "dump tc: %u tc enabled\n", - hdev->tm_info.num_tc); - dev_info(&hdev->pdev->dev, "weight_offset: %u\n", - ets_weight->weight_offset); + *pos += scnprintf(buf + *pos, len - *pos, "port_mask: %#x\n", + bitmap->bit0); + *pos += scnprintf(buf + *pos, len - *pos, "port_shaping_pass: %#x\n", + bitmap->bit1); - for (i = 0; i < HNAE3_MAX_TC; i++) - hclge_print_tc_info(hdev, ets_weight->tc_weight[i], i); + return 0; } -static void hclge_dbg_dump_tm_pg(struct hclge_dev *hdev) +static int hclge_dbg_dump_dcb_tm(struct hclge_dev *hdev, char *buf, int len, + int *pos) { - struct hclge_port_shapping_cmd *port_shap_cfg_cmd; - struct hclge_bp_to_qs_map_cmd *bp_to_qs_map_cmd; - struct hclge_pg_shapping_cmd *pg_shap_cfg_cmd; - enum hclge_opcode_type cmd; - struct hclge_desc desc; + struct hclge_desc desc[2]; + u8 port_id = 0; int ret; - cmd = HCLGE_OPC_TM_PG_C_SHAPPING; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + ret = hclge_dbg_cmd_send(hdev, desc, port_id, 1, + HCLGE_OPC_TM_INTERNAL_CNT); if (ret) - goto err_tm_pg_cmd_send; + return ret; - pg_shap_cfg_cmd = (struct hclge_pg_shapping_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PG_C pg_id: %u\n", pg_shap_cfg_cmd->pg_id); - dev_info(&hdev->pdev->dev, "PG_C pg_shapping: 0x%x\n", - le32_to_cpu(pg_shap_cfg_cmd->pg_shapping_para)); + *pos += scnprintf(buf + *pos, len - *pos, "SCH_NIC_NUM: %#x\n", + le32_to_cpu(desc[0].data[1])); + *pos += scnprintf(buf + *pos, len - *pos, "SCH_ROCE_NUM: %#x\n", + le32_to_cpu(desc[0].data[2])); - cmd = HCLGE_OPC_TM_PG_P_SHAPPING; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + ret = hclge_dbg_cmd_send(hdev, desc, port_id, 2, + HCLGE_OPC_TM_INTERNAL_STS); if (ret) - goto err_tm_pg_cmd_send; - - pg_shap_cfg_cmd = (struct hclge_pg_shapping_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PG_P pg_id: %u\n", pg_shap_cfg_cmd->pg_id); - dev_info(&hdev->pdev->dev, "PG_P pg_shapping: 0x%x\n", - le32_to_cpu(pg_shap_cfg_cmd->pg_shapping_para)); - dev_info(&hdev->pdev->dev, "PG_P flag: %#x\n", pg_shap_cfg_cmd->flag); - dev_info(&hdev->pdev->dev, "PG_P pg_rate: %u(Mbps)\n", - le32_to_cpu(pg_shap_cfg_cmd->pg_rate)); - - cmd = HCLGE_OPC_TM_PORT_SHAPPING; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + return ret; + + *pos += scnprintf(buf + *pos, len - *pos, "pri_bp: %#x\n", + le32_to_cpu(desc[0].data[1])); + *pos += scnprintf(buf + *pos, len - *pos, "fifo_dfx_info: %#x\n", + le32_to_cpu(desc[0].data[2])); + *pos += scnprintf(buf + *pos, len - *pos, + "sch_roce_fifo_afull_gap: %#x\n", + le32_to_cpu(desc[0].data[3])); + *pos += scnprintf(buf + *pos, len - *pos, + "tx_private_waterline: %#x\n", + le32_to_cpu(desc[0].data[4])); + *pos += scnprintf(buf + *pos, len - *pos, "tm_bypass_en: %#x\n", + le32_to_cpu(desc[0].data[5])); + *pos += scnprintf(buf + *pos, len - *pos, "SSU_TM_BYPASS_EN: %#x\n", + le32_to_cpu(desc[1].data[0])); + *pos += scnprintf(buf + *pos, len - *pos, "SSU_RESERVE_CFG: %#x\n", + le32_to_cpu(desc[1].data[1])); + + if (hdev->hw.mac.media_type == HNAE3_MEDIA_TYPE_COPPER) + return 0; + + ret = hclge_dbg_cmd_send(hdev, desc, port_id, 1, + HCLGE_OPC_TM_INTERNAL_STS_1); if (ret) - goto err_tm_pg_cmd_send; + return ret; - port_shap_cfg_cmd = (struct hclge_port_shapping_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PORT port_shapping: 0x%x\n", - le32_to_cpu(port_shap_cfg_cmd->port_shapping_para)); - dev_info(&hdev->pdev->dev, "PORT flag: %#x\n", port_shap_cfg_cmd->flag); - dev_info(&hdev->pdev->dev, "PORT port_rate: %u(Mbps)\n", - le32_to_cpu(port_shap_cfg_cmd->port_rate)); + *pos += scnprintf(buf + *pos, len - *pos, "TC_MAP_SEL: %#x\n", + le32_to_cpu(desc[0].data[1])); + *pos += scnprintf(buf + *pos, len - *pos, "IGU_PFC_PRI_EN: %#x\n", + le32_to_cpu(desc[0].data[2])); + *pos += scnprintf(buf + *pos, len - *pos, "MAC_PFC_PRI_EN: %#x\n", + le32_to_cpu(desc[0].data[3])); + *pos += scnprintf(buf + *pos, len - *pos, "IGU_PRI_MAP_TC_CFG: %#x\n", + le32_to_cpu(desc[0].data[4])); + *pos += scnprintf(buf + *pos, len - *pos, + "IGU_TX_PRI_MAP_TC_CFG: %#x\n", + le32_to_cpu(desc[0].data[5])); - cmd = HCLGE_OPC_TM_PG_SCH_MODE_CFG; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + return 0; +} + +static int hclge_dbg_dump_dcb(struct hclge_dev *hdev, char *buf, int len) +{ + int pos = 0; + int ret; + + ret = hclge_dbg_dump_dcb_qset(hdev, buf, len, &pos); if (ret) - goto err_tm_pg_cmd_send; + return ret; - dev_info(&hdev->pdev->dev, "PG_SCH pg_id: %u\n", - le32_to_cpu(desc.data[0])); + ret = hclge_dbg_dump_dcb_pri(hdev, buf, len, &pos); + if (ret) + return ret; - cmd = HCLGE_OPC_TM_PRI_SCH_MODE_CFG; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + ret = hclge_dbg_dump_dcb_pg(hdev, buf, len, &pos); if (ret) - goto err_tm_pg_cmd_send; + return ret; - dev_info(&hdev->pdev->dev, "PRI_SCH pri_id: %u\n", - le32_to_cpu(desc.data[0])); + ret = hclge_dbg_dump_dcb_queue(hdev, buf, len, &pos); + if (ret) + return ret; - cmd = HCLGE_OPC_TM_QS_SCH_MODE_CFG; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + ret = hclge_dbg_dump_dcb_port(hdev, buf, len, &pos); if (ret) - goto err_tm_pg_cmd_send; + return ret; - dev_info(&hdev->pdev->dev, "QS_SCH qs_id: %u\n", - le32_to_cpu(desc.data[0])); + return hclge_dbg_dump_dcb_tm(hdev, buf, len, &pos); +} - if (!hnae3_dev_dcb_supported(hdev)) { - dev_info(&hdev->pdev->dev, - "Only DCB-supported dev supports tm mapping\n"); - return; +static int hclge_dbg_dump_reg_cmd(struct hclge_dev *hdev, + enum hnae3_dbg_cmd cmd, char *buf, int len) +{ + const struct hclge_dbg_reg_type_info *reg_info; + int pos = 0, ret = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(hclge_dbg_reg_info); i++) { + reg_info = &hclge_dbg_reg_info[i]; + if (cmd == reg_info->cmd) { + if (cmd == HNAE3_DBG_CMD_REG_TQP) + return hclge_dbg_dump_reg_tqp(hdev, reg_info, + buf, len, &pos); + + ret = hclge_dbg_dump_reg_common(hdev, reg_info, buf, + len, &pos); + if (ret) + break; + } } - cmd = HCLGE_OPC_TM_BP_TO_QSET_MAPPING; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_pg_cmd_send; - - bp_to_qs_map_cmd = (struct hclge_bp_to_qs_map_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "BP_TO_QSET tc_id: %u\n", - bp_to_qs_map_cmd->tc_id); - dev_info(&hdev->pdev->dev, "BP_TO_QSET qs_group_id: 0x%x\n", - bp_to_qs_map_cmd->qs_group_id); - dev_info(&hdev->pdev->dev, "BP_TO_QSET qs_bit_map: 0x%x\n", - le32_to_cpu(bp_to_qs_map_cmd->qs_bit_map)); - return; - -err_tm_pg_cmd_send: - dev_err(&hdev->pdev->dev, "dump tm_pg fail(0x%x), ret = %d\n", - cmd, ret); -} - -static void hclge_dbg_dump_tm(struct hclge_dev *hdev) -{ - struct hclge_priority_weight_cmd *priority_weight; - struct hclge_pg_to_pri_link_cmd *pg_to_pri_map; - struct hclge_qs_to_pri_link_cmd *qs_to_pri_map; - struct hclge_nq_to_qs_link_cmd *nq_to_qs_map; - struct hclge_pri_shapping_cmd *shap_cfg_cmd; - struct hclge_pg_weight_cmd *pg_weight; - struct hclge_qs_weight_cmd *qs_weight; - enum hclge_opcode_type cmd; + return ret; +} + +static int hclge_dbg_dump_tc(struct hclge_dev *hdev, char *buf, int len) +{ + struct hclge_ets_tc_weight_cmd *ets_weight; struct hclge_desc desc; + char *sch_mode_str; + int pos = 0; int ret; + u8 i; - cmd = HCLGE_OPC_TM_PG_TO_PRI_LINK; - hclge_cmd_setup_basic_desc(&desc, cmd, true); + if (!hnae3_dev_dcb_supported(hdev)) { + dev_err(&hdev->pdev->dev, + "Only DCB-supported dev supports tc\n"); + return -EOPNOTSUPP; + } + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_ETS_TC_WEIGHT, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; + if (ret) { + dev_err(&hdev->pdev->dev, "failed to get tc weight, ret = %d\n", + ret); + return ret; + } - pg_to_pri_map = (struct hclge_pg_to_pri_link_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "dump tm\n"); - dev_info(&hdev->pdev->dev, "PG_TO_PRI gp_id: %u\n", - pg_to_pri_map->pg_id); - dev_info(&hdev->pdev->dev, "PG_TO_PRI map: 0x%x\n", - pg_to_pri_map->pri_bit_map); + ets_weight = (struct hclge_ets_tc_weight_cmd *)desc.data; - cmd = HCLGE_OPC_TM_QS_TO_PRI_LINK; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; - - qs_to_pri_map = (struct hclge_qs_to_pri_link_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "QS_TO_PRI qs_id: %u\n", - le16_to_cpu(qs_to_pri_map->qs_id)); - dev_info(&hdev->pdev->dev, "QS_TO_PRI priority: %u\n", - qs_to_pri_map->priority); - dev_info(&hdev->pdev->dev, "QS_TO_PRI link_vld: %u\n", - qs_to_pri_map->link_vld); - - cmd = HCLGE_OPC_TM_NQ_TO_QS_LINK; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; + pos += scnprintf(buf + pos, len - pos, "enabled tc number: %u\n", + hdev->tm_info.num_tc); + pos += scnprintf(buf + pos, len - pos, "weight_offset: %u\n", + ets_weight->weight_offset); + + pos += scnprintf(buf + pos, len - pos, "TC MODE WEIGHT\n"); + for (i = 0; i < HNAE3_MAX_TC; i++) { + sch_mode_str = ets_weight->tc_weight[i] ? "dwrr" : "sp"; + pos += scnprintf(buf + pos, len - pos, "%u %4s %3u\n", + i, sch_mode_str, + hdev->tm_info.pg_info[0].tc_dwrr[i]); + } - nq_to_qs_map = (struct hclge_nq_to_qs_link_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "NQ_TO_QS nq_id: %u\n", - le16_to_cpu(nq_to_qs_map->nq_id)); - dev_info(&hdev->pdev->dev, "NQ_TO_QS qset_id: 0x%x\n", - le16_to_cpu(nq_to_qs_map->qset_id)); + return 0; +} - cmd = HCLGE_OPC_TM_PG_WEIGHT; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; +static const struct hclge_dbg_item tm_pg_items[] = { + { "ID", 2 }, + { "PRI_MAP", 2 }, + { "MODE", 2 }, + { "DWRR", 2 }, + { "C_IR_B", 2 }, + { "C_IR_U", 2 }, + { "C_IR_S", 2 }, + { "C_BS_B", 2 }, + { "C_BS_S", 2 }, + { "C_FLAG", 2 }, + { "C_RATE(Mbps)", 2 }, + { "P_IR_B", 2 }, + { "P_IR_U", 2 }, + { "P_IR_S", 2 }, + { "P_BS_B", 2 }, + { "P_BS_S", 2 }, + { "P_FLAG", 2 }, + { "P_RATE(Mbps)", 0 } +}; - pg_weight = (struct hclge_pg_weight_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PG pg_id: %u\n", pg_weight->pg_id); - dev_info(&hdev->pdev->dev, "PG dwrr: %u\n", pg_weight->dwrr); +static void hclge_dbg_fill_shaper_content(struct hclge_tm_shaper_para *para, + char **result, u8 *index) +{ + sprintf(result[(*index)++], "%3u", para->ir_b); + sprintf(result[(*index)++], "%3u", para->ir_u); + sprintf(result[(*index)++], "%3u", para->ir_s); + sprintf(result[(*index)++], "%3u", para->bs_b); + sprintf(result[(*index)++], "%3u", para->bs_s); + sprintf(result[(*index)++], "%3u", para->flag); + sprintf(result[(*index)++], "%6u", para->rate); +} - cmd = HCLGE_OPC_TM_QS_WEIGHT; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; +static int hclge_dbg_dump_tm_pg(struct hclge_dev *hdev, char *buf, int len) +{ + char data_str[ARRAY_SIZE(tm_pg_items)][HCLGE_DBG_DATA_STR_LEN]; + struct hclge_tm_shaper_para c_shaper_para, p_shaper_para; + char *result[ARRAY_SIZE(tm_pg_items)], *sch_mode_str; + u8 pg_id, sch_mode, weight, pri_bit_map, i, j; + char content[HCLGE_DBG_TM_INFO_LEN]; + int pos = 0; + int ret; - qs_weight = (struct hclge_qs_weight_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "QS qs_id: %u\n", - le16_to_cpu(qs_weight->qs_id)); - dev_info(&hdev->pdev->dev, "QS dwrr: %u\n", qs_weight->dwrr); + for (i = 0; i < ARRAY_SIZE(tm_pg_items); i++) + result[i] = &data_str[i][0]; - cmd = HCLGE_OPC_TM_PRI_WEIGHT; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; + hclge_dbg_fill_content(content, sizeof(content), tm_pg_items, + NULL, ARRAY_SIZE(tm_pg_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); - priority_weight = (struct hclge_priority_weight_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PRI pri_id: %u\n", priority_weight->pri_id); - dev_info(&hdev->pdev->dev, "PRI dwrr: %u\n", priority_weight->dwrr); + for (pg_id = 0; pg_id < hdev->tm_info.num_pg; pg_id++) { + ret = hclge_tm_get_pg_to_pri_map(hdev, pg_id, &pri_bit_map); + if (ret) + return ret; - cmd = HCLGE_OPC_TM_PRI_C_SHAPPING; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; - - shap_cfg_cmd = (struct hclge_pri_shapping_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PRI_C pri_id: %u\n", shap_cfg_cmd->pri_id); - dev_info(&hdev->pdev->dev, "PRI_C pri_shapping: 0x%x\n", - le32_to_cpu(shap_cfg_cmd->pri_shapping_para)); - dev_info(&hdev->pdev->dev, "PRI_C flag: %#x\n", shap_cfg_cmd->flag); - dev_info(&hdev->pdev->dev, "PRI_C pri_rate: %u(Mbps)\n", - le32_to_cpu(shap_cfg_cmd->pri_rate)); - - cmd = HCLGE_OPC_TM_PRI_P_SHAPPING; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_cmd_send; + ret = hclge_tm_get_pg_sch_mode(hdev, pg_id, &sch_mode); + if (ret) + return ret; - shap_cfg_cmd = (struct hclge_pri_shapping_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "PRI_P pri_id: %u\n", shap_cfg_cmd->pri_id); - dev_info(&hdev->pdev->dev, "PRI_P pri_shapping: 0x%x\n", - le32_to_cpu(shap_cfg_cmd->pri_shapping_para)); - dev_info(&hdev->pdev->dev, "PRI_P flag: %#x\n", shap_cfg_cmd->flag); - dev_info(&hdev->pdev->dev, "PRI_P pri_rate: %u(Mbps)\n", - le32_to_cpu(shap_cfg_cmd->pri_rate)); + ret = hclge_tm_get_pg_weight(hdev, pg_id, &weight); + if (ret) + return ret; - hclge_dbg_dump_tm_pg(hdev); + ret = hclge_tm_get_pg_shaper(hdev, pg_id, + HCLGE_OPC_TM_PG_C_SHAPPING, + &c_shaper_para); + if (ret) + return ret; + + ret = hclge_tm_get_pg_shaper(hdev, pg_id, + HCLGE_OPC_TM_PG_P_SHAPPING, + &p_shaper_para); + if (ret) + return ret; - return; + sch_mode_str = sch_mode & HCLGE_TM_TX_SCHD_DWRR_MSK ? "dwrr" : + "sp"; + + j = 0; + sprintf(result[j++], "%02u", pg_id); + sprintf(result[j++], "0x%02x", pri_bit_map); + sprintf(result[j++], "%4s", sch_mode_str); + sprintf(result[j++], "%3u", weight); + hclge_dbg_fill_shaper_content(&c_shaper_para, result, &j); + hclge_dbg_fill_shaper_content(&p_shaper_para, result, &j); + + hclge_dbg_fill_content(content, sizeof(content), tm_pg_items, + (const char **)result, + ARRAY_SIZE(tm_pg_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + } -err_tm_cmd_send: - dev_err(&hdev->pdev->dev, "dump tm fail(0x%x), ret = %d\n", - cmd, ret); + return 0; } -static void hclge_dbg_dump_tm_map(struct hclge_dev *hdev, - const char *cmd_buf) +static int hclge_dbg_dump_tm_port(struct hclge_dev *hdev, char *buf, int len) { - struct hclge_bp_to_qs_map_cmd *bp_to_qs_map_cmd; - struct hclge_nq_to_qs_link_cmd *nq_to_qs_map; - u32 qset_mapping[HCLGE_BP_EXT_GRP_NUM]; - struct hclge_qs_to_pri_link_cmd *map; - struct hclge_tqp_tx_queue_tc_cmd *tc; - u16 group_id, queue_id, qset_id; - enum hclge_opcode_type cmd; - u8 grp_num, pri_id, tc_id; - struct hclge_desc desc; - u16 qs_id_l; - u16 qs_id_h; + struct hclge_tm_shaper_para shaper_para; + int pos = 0; int ret; - u32 i; - ret = kstrtou16(cmd_buf, 0, &queue_id); - queue_id = (ret != 0) ? 0 : queue_id; - - cmd = HCLGE_OPC_TM_NQ_TO_QS_LINK; - nq_to_qs_map = (struct hclge_nq_to_qs_link_cmd *)desc.data; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - nq_to_qs_map->nq_id = cpu_to_le16(queue_id); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); + ret = hclge_tm_get_port_shaper(hdev, &shaper_para); if (ret) - goto err_tm_map_cmd_send; - qset_id = le16_to_cpu(nq_to_qs_map->qset_id); - - /* convert qset_id to the following format, drop the vld bit - * | qs_id_h | vld | qs_id_l | - * qset_id: | 15 ~ 11 | 10 | 9 ~ 0 | - * \ \ / / - * \ \ / / - * qset_id: | 15 | 14 ~ 10 | 9 ~ 0 | - */ - qs_id_l = hnae3_get_field(qset_id, HCLGE_TM_QS_ID_L_MSK, - HCLGE_TM_QS_ID_L_S); - qs_id_h = hnae3_get_field(qset_id, HCLGE_TM_QS_ID_H_EXT_MSK, - HCLGE_TM_QS_ID_H_EXT_S); - qset_id = 0; - hnae3_set_field(qset_id, HCLGE_TM_QS_ID_L_MSK, HCLGE_TM_QS_ID_L_S, - qs_id_l); - hnae3_set_field(qset_id, HCLGE_TM_QS_ID_H_MSK, HCLGE_TM_QS_ID_H_S, - qs_id_h); - - cmd = HCLGE_OPC_TM_QS_TO_PRI_LINK; - map = (struct hclge_qs_to_pri_link_cmd *)desc.data; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - map->qs_id = cpu_to_le16(qset_id); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_map_cmd_send; - pri_id = map->priority; + return ret; - cmd = HCLGE_OPC_TQP_TX_QUEUE_TC; - tc = (struct hclge_tqp_tx_queue_tc_cmd *)desc.data; - hclge_cmd_setup_basic_desc(&desc, cmd, true); - tc->queue_id = cpu_to_le16(queue_id); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_map_cmd_send; - tc_id = tc->tc_id & 0x7; + pos += scnprintf(buf + pos, len - pos, + "IR_B IR_U IR_S BS_B BS_S FLAG RATE(Mbps)\n"); + pos += scnprintf(buf + pos, len - pos, + "%3u %3u %3u %3u %3u %1u %6u\n", + shaper_para.ir_b, shaper_para.ir_u, shaper_para.ir_s, + shaper_para.bs_b, shaper_para.bs_s, shaper_para.flag, + shaper_para.rate); - dev_info(&hdev->pdev->dev, "queue_id | qset_id | pri_id | tc_id\n"); - dev_info(&hdev->pdev->dev, "%04u | %04u | %02u | %02u\n", - queue_id, qset_id, pri_id, tc_id); + return 0; +} - if (!hnae3_dev_dcb_supported(hdev)) { - dev_info(&hdev->pdev->dev, - "Only DCB-supported dev supports tm mapping\n"); - return; - } +static int hclge_dbg_dump_tm_bp_qset_map(struct hclge_dev *hdev, u8 tc_id, + char *buf, int len) +{ + u32 qset_mapping[HCLGE_BP_EXT_GRP_NUM]; + struct hclge_bp_to_qs_map_cmd *map; + struct hclge_desc desc; + int pos = 0; + u8 group_id; + u8 grp_num; + u16 i = 0; + int ret; grp_num = hdev->num_tqps <= HCLGE_TQP_MAX_SIZE_DEV_V2 ? HCLGE_BP_GRP_NUM : HCLGE_BP_EXT_GRP_NUM; - cmd = HCLGE_OPC_TM_BP_TO_QSET_MAPPING; - bp_to_qs_map_cmd = (struct hclge_bp_to_qs_map_cmd *)desc.data; + map = (struct hclge_bp_to_qs_map_cmd *)desc.data; for (group_id = 0; group_id < grp_num; group_id++) { - hclge_cmd_setup_basic_desc(&desc, cmd, true); - bp_to_qs_map_cmd->tc_id = tc_id; - bp_to_qs_map_cmd->qs_group_id = group_id; + hclge_cmd_setup_basic_desc(&desc, + HCLGE_OPC_TM_BP_TO_QSET_MAPPING, + true); + map->tc_id = tc_id; + map->qs_group_id = group_id; ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) - goto err_tm_map_cmd_send; + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get bp to qset map, ret = %d\n", + ret); + return ret; + } - qset_mapping[group_id] = - le32_to_cpu(bp_to_qs_map_cmd->qs_bit_map); + qset_mapping[group_id] = le32_to_cpu(map->qs_bit_map); } - dev_info(&hdev->pdev->dev, "index | tm bp qset maping:\n"); - - i = 0; + pos += scnprintf(buf + pos, len - pos, "INDEX | TM BP QSET MAPPING:\n"); for (group_id = 0; group_id < grp_num / 8; group_id++) { - dev_info(&hdev->pdev->dev, + pos += scnprintf(buf + pos, len - pos, "%04d | %08x:%08x:%08x:%08x:%08x:%08x:%08x:%08x\n", - group_id * 256, qset_mapping[(u32)(i + 7)], - qset_mapping[(u32)(i + 6)], qset_mapping[(u32)(i + 5)], - qset_mapping[(u32)(i + 4)], qset_mapping[(u32)(i + 3)], - qset_mapping[(u32)(i + 2)], qset_mapping[(u32)(i + 1)], + group_id * 256, qset_mapping[i + 7], + qset_mapping[i + 6], qset_mapping[i + 5], + qset_mapping[i + 4], qset_mapping[i + 3], + qset_mapping[i + 2], qset_mapping[i + 1], qset_mapping[i]); i += 8; } - return; + return pos; +} + +static int hclge_dbg_dump_tm_map(struct hclge_dev *hdev, char *buf, int len) +{ + u16 queue_id; + u16 qset_id; + u8 link_vld; + int pos = 0; + u8 pri_id; + u8 tc_id; + int ret; + + for (queue_id = 0; queue_id < hdev->num_tqps; queue_id++) { + ret = hclge_tm_get_q_to_qs_map(hdev, queue_id, &qset_id); + if (ret) + return ret; + + ret = hclge_tm_get_qset_map_pri(hdev, qset_id, &pri_id, + &link_vld); + if (ret) + return ret; + + ret = hclge_tm_get_q_to_tc(hdev, queue_id, &tc_id); + if (ret) + return ret; + + pos += scnprintf(buf + pos, len - pos, + "QUEUE_ID QSET_ID PRI_ID TC_ID\n"); + pos += scnprintf(buf + pos, len - pos, + "%04u %4u %3u %2u\n", + queue_id, qset_id, pri_id, tc_id); + + if (!hnae3_dev_dcb_supported(hdev)) + continue; + + ret = hclge_dbg_dump_tm_bp_qset_map(hdev, tc_id, buf + pos, + len - pos); + if (ret < 0) + return ret; + pos += ret; + + pos += scnprintf(buf + pos, len - pos, "\n"); + } -err_tm_map_cmd_send: - dev_err(&hdev->pdev->dev, "dump tqp map fail(0x%x), ret = %d\n", - cmd, ret); + return 0; } static int hclge_dbg_dump_tm_nodes(struct hclge_dev *hdev, char *buf, int len) @@ -833,8 +928,8 @@ static int hclge_dbg_dump_tm_nodes(struct hclge_dev *hdev, char *buf, int len) static int hclge_dbg_dump_tm_pri(struct hclge_dev *hdev, char *buf, int len) { - struct hclge_pri_shaper_para c_shaper_para; - struct hclge_pri_shaper_para p_shaper_para; + struct hclge_tm_shaper_para c_shaper_para; + struct hclge_tm_shaper_para p_shaper_para; u8 pri_num, sch_mode, weight; char *sch_mode_str; int pos = 0; @@ -896,19 +991,42 @@ static int hclge_dbg_dump_tm_pri(struct hclge_dev *hdev, char *buf, int len) return 0; } +static const struct hclge_dbg_item tm_qset_items[] = { + { "ID", 4 }, + { "MAP_PRI", 2 }, + { "LINK_VLD", 2 }, + { "MODE", 2 }, + { "DWRR", 2 }, + { "IR_B", 2 }, + { "IR_U", 2 }, + { "IR_S", 2 }, + { "BS_B", 2 }, + { "BS_S", 2 }, + { "FLAG", 2 }, + { "RATE(Mbps)", 0 } +}; + static int hclge_dbg_dump_tm_qset(struct hclge_dev *hdev, char *buf, int len) { + char data_str[ARRAY_SIZE(tm_qset_items)][HCLGE_DBG_DATA_STR_LEN]; + char *result[ARRAY_SIZE(tm_qset_items)], *sch_mode_str; u8 priority, link_vld, sch_mode, weight; - char *sch_mode_str; + struct hclge_tm_shaper_para shaper_para; + char content[HCLGE_DBG_TM_INFO_LEN]; + u16 qset_num, i; int ret, pos; - u16 qset_num; - u16 i; + u8 j; ret = hclge_tm_get_qset_num(hdev, &qset_num); if (ret) return ret; - pos = scnprintf(buf, len, "ID MAP_PRI LINK_VLD MODE DWRR\n"); + for (i = 0; i < ARRAY_SIZE(tm_qset_items); i++) + result[i] = &data_str[i][0]; + + hclge_dbg_fill_content(content, sizeof(content), tm_qset_items, + NULL, ARRAY_SIZE(tm_qset_items)); + pos = scnprintf(buf, len, "%s", content); for (i = 0; i < qset_num; i++) { ret = hclge_tm_get_qset_map_pri(hdev, i, &priority, &link_vld); @@ -923,280 +1041,326 @@ static int hclge_dbg_dump_tm_qset(struct hclge_dev *hdev, char *buf, int len) if (ret) return ret; + ret = hclge_tm_get_qset_shaper(hdev, i, &shaper_para); + if (ret) + return ret; + sch_mode_str = sch_mode & HCLGE_TM_TX_SCHD_DWRR_MSK ? "dwrr" : "sp"; - pos += scnprintf(buf + pos, len - pos, - "%04u %4u %1u %4s %3u\n", - i, priority, link_vld, sch_mode_str, weight); + + j = 0; + sprintf(result[j++], "%04u", i); + sprintf(result[j++], "%4u", priority); + sprintf(result[j++], "%4u", link_vld); + sprintf(result[j++], "%4s", sch_mode_str); + sprintf(result[j++], "%3u", weight); + hclge_dbg_fill_shaper_content(&shaper_para, result, &j); + + hclge_dbg_fill_content(content, sizeof(content), tm_qset_items, + (const char **)result, + ARRAY_SIZE(tm_qset_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); } return 0; } -static void hclge_dbg_dump_qos_pause_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_qos_pause_cfg(struct hclge_dev *hdev, char *buf, + int len) { struct hclge_cfg_pause_param_cmd *pause_param; struct hclge_desc desc; + int pos = 0; int ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_MAC_PARA, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { - dev_err(&hdev->pdev->dev, "dump checksum fail, ret = %d\n", - ret); - return; + dev_err(&hdev->pdev->dev, + "failed to dump qos pause, ret = %d\n", ret); + return ret; } pause_param = (struct hclge_cfg_pause_param_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "dump qos pause cfg\n"); - dev_info(&hdev->pdev->dev, "pause_trans_gap: 0x%x\n", - pause_param->pause_trans_gap); - dev_info(&hdev->pdev->dev, "pause_trans_time: 0x%x\n", - le16_to_cpu(pause_param->pause_trans_time)); + + pos += scnprintf(buf + pos, len - pos, "pause_trans_gap: 0x%x\n", + pause_param->pause_trans_gap); + pos += scnprintf(buf + pos, len - pos, "pause_trans_time: 0x%x\n", + le16_to_cpu(pause_param->pause_trans_time)); + return 0; } -static void hclge_dbg_dump_qos_pri_map(struct hclge_dev *hdev) +static int hclge_dbg_dump_qos_pri_map(struct hclge_dev *hdev, char *buf, + int len) { +#define HCLGE_DBG_TC_MASK 0x0F +#define HCLGE_DBG_TC_BIT_WIDTH 4 + struct hclge_qos_pri_map_cmd *pri_map; struct hclge_desc desc; + int pos = 0; + u8 *pri_tc; + u8 tc, i; int ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PRI_TO_TC_MAPPING, true); - ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { dev_err(&hdev->pdev->dev, - "dump qos pri map fail, ret = %d\n", ret); - return; + "failed to dump qos pri map, ret = %d\n", ret); + return ret; } pri_map = (struct hclge_qos_pri_map_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "dump qos pri map\n"); - dev_info(&hdev->pdev->dev, "vlan_to_pri: 0x%x\n", pri_map->vlan_pri); - dev_info(&hdev->pdev->dev, "pri_0_to_tc: 0x%x\n", pri_map->pri0_tc); - dev_info(&hdev->pdev->dev, "pri_1_to_tc: 0x%x\n", pri_map->pri1_tc); - dev_info(&hdev->pdev->dev, "pri_2_to_tc: 0x%x\n", pri_map->pri2_tc); - dev_info(&hdev->pdev->dev, "pri_3_to_tc: 0x%x\n", pri_map->pri3_tc); - dev_info(&hdev->pdev->dev, "pri_4_to_tc: 0x%x\n", pri_map->pri4_tc); - dev_info(&hdev->pdev->dev, "pri_5_to_tc: 0x%x\n", pri_map->pri5_tc); - dev_info(&hdev->pdev->dev, "pri_6_to_tc: 0x%x\n", pri_map->pri6_tc); - dev_info(&hdev->pdev->dev, "pri_7_to_tc: 0x%x\n", pri_map->pri7_tc); + + pos += scnprintf(buf + pos, len - pos, "vlan_to_pri: 0x%x\n", + pri_map->vlan_pri); + pos += scnprintf(buf + pos, len - pos, "PRI TC\n"); + + pri_tc = (u8 *)pri_map; + for (i = 0; i < HNAE3_MAX_TC; i++) { + tc = pri_tc[i >> 1] >> ((i & 1) * HCLGE_DBG_TC_BIT_WIDTH); + tc &= HCLGE_DBG_TC_MASK; + pos += scnprintf(buf + pos, len - pos, "%u %u\n", i, tc); + } + + return 0; } -static int hclge_dbg_dump_tx_buf_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_tx_buf_cfg(struct hclge_dev *hdev, char *buf, int len) { struct hclge_tx_buff_alloc_cmd *tx_buf_cmd; struct hclge_desc desc; + int pos = 0; int i, ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TX_BUFF_ALLOC, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump tx buf, ret = %d\n", ret); return ret; + } - dev_info(&hdev->pdev->dev, "dump qos buf cfg\n"); tx_buf_cmd = (struct hclge_tx_buff_alloc_cmd *)desc.data; for (i = 0; i < HCLGE_MAX_TC_NUM; i++) - dev_info(&hdev->pdev->dev, "tx_packet_buf_tc_%d: 0x%x\n", i, - le16_to_cpu(tx_buf_cmd->tx_pkt_buff[i])); + pos += scnprintf(buf + pos, len - pos, + "tx_packet_buf_tc_%d: 0x%x\n", i, + le16_to_cpu(tx_buf_cmd->tx_pkt_buff[i])); - return 0; + return pos; } -static int hclge_dbg_dump_rx_priv_buf_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_rx_priv_buf_cfg(struct hclge_dev *hdev, char *buf, + int len) { struct hclge_rx_priv_buff_cmd *rx_buf_cmd; struct hclge_desc desc; + int pos = 0; int i, ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RX_PRIV_BUFF_ALLOC, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump rx priv buf, ret = %d\n", ret); return ret; + } + + pos += scnprintf(buf + pos, len - pos, "\n"); - dev_info(&hdev->pdev->dev, "\n"); rx_buf_cmd = (struct hclge_rx_priv_buff_cmd *)desc.data; for (i = 0; i < HCLGE_MAX_TC_NUM; i++) - dev_info(&hdev->pdev->dev, "rx_packet_buf_tc_%d: 0x%x\n", i, - le16_to_cpu(rx_buf_cmd->buf_num[i])); + pos += scnprintf(buf + pos, len - pos, + "rx_packet_buf_tc_%d: 0x%x\n", i, + le16_to_cpu(rx_buf_cmd->buf_num[i])); - dev_info(&hdev->pdev->dev, "rx_share_buf: 0x%x\n", - le16_to_cpu(rx_buf_cmd->shared_buf)); + pos += scnprintf(buf + pos, len - pos, "rx_share_buf: 0x%x\n", + le16_to_cpu(rx_buf_cmd->shared_buf)); - return 0; + return pos; } -static int hclge_dbg_dump_rx_common_wl_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_rx_common_wl_cfg(struct hclge_dev *hdev, char *buf, + int len) { struct hclge_rx_com_wl *rx_com_wl; struct hclge_desc desc; + int pos = 0; int ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RX_COM_WL_ALLOC, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump rx common wl, ret = %d\n", ret); return ret; + } rx_com_wl = (struct hclge_rx_com_wl *)desc.data; - dev_info(&hdev->pdev->dev, "\n"); - dev_info(&hdev->pdev->dev, "rx_com_wl: high: 0x%x, low: 0x%x\n", - le16_to_cpu(rx_com_wl->com_wl.high), - le16_to_cpu(rx_com_wl->com_wl.low)); + pos += scnprintf(buf + pos, len - pos, "\n"); + pos += scnprintf(buf + pos, len - pos, + "rx_com_wl: high: 0x%x, low: 0x%x\n", + le16_to_cpu(rx_com_wl->com_wl.high), + le16_to_cpu(rx_com_wl->com_wl.low)); - return 0; + return pos; } -static int hclge_dbg_dump_rx_global_pkt_cnt(struct hclge_dev *hdev) +static int hclge_dbg_dump_rx_global_pkt_cnt(struct hclge_dev *hdev, char *buf, + int len) { struct hclge_rx_com_wl *rx_packet_cnt; struct hclge_desc desc; + int pos = 0; int ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_RX_GBL_PKT_CNT, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump rx global pkt cnt, ret = %d\n", ret); return ret; + } rx_packet_cnt = (struct hclge_rx_com_wl *)desc.data; - dev_info(&hdev->pdev->dev, - "rx_global_packet_cnt: high: 0x%x, low: 0x%x\n", - le16_to_cpu(rx_packet_cnt->com_wl.high), - le16_to_cpu(rx_packet_cnt->com_wl.low)); + pos += scnprintf(buf + pos, len - pos, + "rx_global_packet_cnt: high: 0x%x, low: 0x%x\n", + le16_to_cpu(rx_packet_cnt->com_wl.high), + le16_to_cpu(rx_packet_cnt->com_wl.low)); - return 0; + return pos; } -static int hclge_dbg_dump_rx_priv_wl_buf_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_rx_priv_wl_buf_cfg(struct hclge_dev *hdev, char *buf, + int len) { struct hclge_rx_priv_wl_buf *rx_priv_wl; struct hclge_desc desc[2]; + int pos = 0; int i, ret; hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_RX_PRIV_WL_ALLOC, true); desc[0].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT); hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_RX_PRIV_WL_ALLOC, true); ret = hclge_cmd_send(&hdev->hw, desc, 2); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump rx priv wl buf, ret = %d\n", ret); return ret; + } rx_priv_wl = (struct hclge_rx_priv_wl_buf *)desc[0].data; for (i = 0; i < HCLGE_TC_NUM_ONE_DESC; i++) - dev_info(&hdev->pdev->dev, + pos += scnprintf(buf + pos, len - pos, "rx_priv_wl_tc_%d: high: 0x%x, low: 0x%x\n", i, le16_to_cpu(rx_priv_wl->tc_wl[i].high), le16_to_cpu(rx_priv_wl->tc_wl[i].low)); rx_priv_wl = (struct hclge_rx_priv_wl_buf *)desc[1].data; for (i = 0; i < HCLGE_TC_NUM_ONE_DESC; i++) - dev_info(&hdev->pdev->dev, + pos += scnprintf(buf + pos, len - pos, "rx_priv_wl_tc_%d: high: 0x%x, low: 0x%x\n", i + HCLGE_TC_NUM_ONE_DESC, le16_to_cpu(rx_priv_wl->tc_wl[i].high), le16_to_cpu(rx_priv_wl->tc_wl[i].low)); - return 0; + return pos; } -static int hclge_dbg_dump_rx_common_threshold_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_rx_common_threshold_cfg(struct hclge_dev *hdev, + char *buf, int len) { struct hclge_rx_com_thrd *rx_com_thrd; struct hclge_desc desc[2]; + int pos = 0; int i, ret; hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_RX_COM_THRD_ALLOC, true); desc[0].flag |= cpu_to_le16(HCLGE_CMD_FLAG_NEXT); hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_RX_COM_THRD_ALLOC, true); ret = hclge_cmd_send(&hdev->hw, desc, 2); - if (ret) + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump rx common threshold, ret = %d\n", ret); return ret; + } - dev_info(&hdev->pdev->dev, "\n"); + pos += scnprintf(buf + pos, len - pos, "\n"); rx_com_thrd = (struct hclge_rx_com_thrd *)desc[0].data; for (i = 0; i < HCLGE_TC_NUM_ONE_DESC; i++) - dev_info(&hdev->pdev->dev, + pos += scnprintf(buf + pos, len - pos, "rx_com_thrd_tc_%d: high: 0x%x, low: 0x%x\n", i, le16_to_cpu(rx_com_thrd->com_thrd[i].high), le16_to_cpu(rx_com_thrd->com_thrd[i].low)); rx_com_thrd = (struct hclge_rx_com_thrd *)desc[1].data; for (i = 0; i < HCLGE_TC_NUM_ONE_DESC; i++) - dev_info(&hdev->pdev->dev, + pos += scnprintf(buf + pos, len - pos, "rx_com_thrd_tc_%d: high: 0x%x, low: 0x%x\n", i + HCLGE_TC_NUM_ONE_DESC, le16_to_cpu(rx_com_thrd->com_thrd[i].high), le16_to_cpu(rx_com_thrd->com_thrd[i].low)); - return 0; + return pos; } -static void hclge_dbg_dump_qos_buf_cfg(struct hclge_dev *hdev) +static int hclge_dbg_dump_qos_buf_cfg(struct hclge_dev *hdev, char *buf, + int len) { - enum hclge_opcode_type cmd; + int pos = 0; int ret; - cmd = HCLGE_OPC_TX_BUFF_ALLOC; - ret = hclge_dbg_dump_tx_buf_cfg(hdev); - if (ret) - goto err_qos_cmd_send; - - cmd = HCLGE_OPC_RX_PRIV_BUFF_ALLOC; - ret = hclge_dbg_dump_rx_priv_buf_cfg(hdev); - if (ret) - goto err_qos_cmd_send; + ret = hclge_dbg_dump_tx_buf_cfg(hdev, buf + pos, len - pos); + if (ret < 0) + return ret; + pos += ret; - cmd = HCLGE_OPC_RX_COM_WL_ALLOC; - ret = hclge_dbg_dump_rx_common_wl_cfg(hdev); - if (ret) - goto err_qos_cmd_send; + ret = hclge_dbg_dump_rx_priv_buf_cfg(hdev, buf + pos, len - pos); + if (ret < 0) + return ret; + pos += ret; - cmd = HCLGE_OPC_RX_GBL_PKT_CNT; - ret = hclge_dbg_dump_rx_global_pkt_cnt(hdev); - if (ret) - goto err_qos_cmd_send; + ret = hclge_dbg_dump_rx_common_wl_cfg(hdev, buf + pos, len - pos); + if (ret < 0) + return ret; + pos += ret; - dev_info(&hdev->pdev->dev, "\n"); - if (!hnae3_dev_dcb_supported(hdev)) { - dev_info(&hdev->pdev->dev, - "Only DCB-supported dev supports rx priv wl\n"); - return; - } + ret = hclge_dbg_dump_rx_global_pkt_cnt(hdev, buf + pos, len - pos); + if (ret < 0) + return ret; + pos += ret; - cmd = HCLGE_OPC_RX_PRIV_WL_ALLOC; - ret = hclge_dbg_dump_rx_priv_wl_buf_cfg(hdev); - if (ret) - goto err_qos_cmd_send; + pos += scnprintf(buf + pos, len - pos, "\n"); + if (!hnae3_dev_dcb_supported(hdev)) + return 0; - cmd = HCLGE_OPC_RX_COM_THRD_ALLOC; - ret = hclge_dbg_dump_rx_common_threshold_cfg(hdev); - if (ret) - goto err_qos_cmd_send; + ret = hclge_dbg_dump_rx_priv_wl_buf_cfg(hdev, buf + pos, len - pos); + if (ret < 0) + return ret; + pos += ret; - return; + ret = hclge_dbg_dump_rx_common_threshold_cfg(hdev, buf + pos, + len - pos); + if (ret < 0) + return ret; -err_qos_cmd_send: - dev_err(&hdev->pdev->dev, - "dump qos buf cfg fail(0x%x), ret = %d\n", cmd, ret); + return 0; } -static void hclge_dbg_dump_mng_table(struct hclge_dev *hdev) +static int hclge_dbg_dump_mng_table(struct hclge_dev *hdev, char *buf, int len) { struct hclge_mac_ethertype_idx_rd_cmd *req0; - char printf_buf[HCLGE_DBG_BUF_LEN]; struct hclge_desc desc; u32 msg_egress_port; + int pos = 0; int ret, i; - dev_info(&hdev->pdev->dev, "mng tab:\n"); - memset(printf_buf, 0, HCLGE_DBG_BUF_LEN); - strncat(printf_buf, - "entry|mac_addr |mask|ether|mask|vlan|mask", - HCLGE_DBG_BUF_LEN - 1); - strncat(printf_buf + strlen(printf_buf), - "|i_map|i_dir|e_type|pf_id|vf_id|q_id|drop\n", - HCLGE_DBG_BUF_LEN - strlen(printf_buf) - 1); - - dev_info(&hdev->pdev->dev, "%s", printf_buf); + pos += scnprintf(buf + pos, len - pos, + "entry mac_addr mask ether "); + pos += scnprintf(buf + pos, len - pos, + "mask vlan mask i_map i_dir e_type "); + pos += scnprintf(buf + pos, len - pos, "pf_id vf_id q_id drop\n"); for (i = 0; i < HCLGE_DBG_MNG_TBL_MAX; i++) { hclge_cmd_setup_basic_desc(&desc, HCLGE_MAC_ETHERTYPE_IDX_RD, @@ -1207,52 +1371,53 @@ static void hclge_dbg_dump_mng_table(struct hclge_dev *hdev) ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { dev_err(&hdev->pdev->dev, - "call hclge_cmd_send fail, ret = %d\n", ret); - return; + "failed to dump manage table, ret = %d\n", ret); + return ret; } if (!req0->resp_code) continue; - memset(printf_buf, 0, HCLGE_DBG_BUF_LEN); - snprintf(printf_buf, HCLGE_DBG_BUF_LEN, - "%02u |%02x:%02x:%02x:%02x:%02x:%02x|", - le16_to_cpu(req0->index), - req0->mac_addr[0], req0->mac_addr[1], - req0->mac_addr[2], req0->mac_addr[3], - req0->mac_addr[4], req0->mac_addr[5]); - - snprintf(printf_buf + strlen(printf_buf), - HCLGE_DBG_BUF_LEN - strlen(printf_buf), - "%x |%04x |%x |%04x|%x |%02x |%02x |", - !!(req0->flags & HCLGE_DBG_MNG_MAC_MASK_B), - le16_to_cpu(req0->ethter_type), - !!(req0->flags & HCLGE_DBG_MNG_ETHER_MASK_B), - le16_to_cpu(req0->vlan_tag) & HCLGE_DBG_MNG_VLAN_TAG, - !!(req0->flags & HCLGE_DBG_MNG_VLAN_MASK_B), - req0->i_port_bitmap, req0->i_port_direction); + pos += scnprintf(buf + pos, len - pos, "%02u %pM ", + le16_to_cpu(req0->index), req0->mac_addr); + + pos += scnprintf(buf + pos, len - pos, + "%x %04x %x %04x ", + !!(req0->flags & HCLGE_DBG_MNG_MAC_MASK_B), + le16_to_cpu(req0->ethter_type), + !!(req0->flags & HCLGE_DBG_MNG_ETHER_MASK_B), + le16_to_cpu(req0->vlan_tag) & + HCLGE_DBG_MNG_VLAN_TAG); + + pos += scnprintf(buf + pos, len - pos, + "%x %02x %02x ", + !!(req0->flags & HCLGE_DBG_MNG_VLAN_MASK_B), + req0->i_port_bitmap, req0->i_port_direction); msg_egress_port = le16_to_cpu(req0->egress_port); - snprintf(printf_buf + strlen(printf_buf), - HCLGE_DBG_BUF_LEN - strlen(printf_buf), - "%x |%x |%02x |%04x|%x\n", - !!(msg_egress_port & HCLGE_DBG_MNG_E_TYPE_B), - msg_egress_port & HCLGE_DBG_MNG_PF_ID, - (msg_egress_port >> 3) & HCLGE_DBG_MNG_VF_ID, - le16_to_cpu(req0->egress_queue), - !!(msg_egress_port & HCLGE_DBG_MNG_DROP_B)); - - dev_info(&hdev->pdev->dev, "%s", printf_buf); + pos += scnprintf(buf + pos, len - pos, + "%x %x %02x %04x %x\n", + !!(msg_egress_port & HCLGE_DBG_MNG_E_TYPE_B), + msg_egress_port & HCLGE_DBG_MNG_PF_ID, + (msg_egress_port >> 3) & HCLGE_DBG_MNG_VF_ID, + le16_to_cpu(req0->egress_queue), + !!(msg_egress_port & HCLGE_DBG_MNG_DROP_B)); } + + return 0; } -static int hclge_dbg_fd_tcam_read(struct hclge_dev *hdev, u8 stage, - bool sel_x, u32 loc) +#define HCLGE_DBG_TCAM_BUF_SIZE 256 + +static int hclge_dbg_fd_tcam_read(struct hclge_dev *hdev, bool sel_x, + char *tcam_buf, + struct hclge_dbg_tcam_msg tcam_msg) { struct hclge_fd_tcam_config_1_cmd *req1; struct hclge_fd_tcam_config_2_cmd *req2; struct hclge_fd_tcam_config_3_cmd *req3; struct hclge_desc desc[3]; + int pos = 0; int ret, i; u32 *req; @@ -1266,31 +1431,35 @@ static int hclge_dbg_fd_tcam_read(struct hclge_dev *hdev, u8 stage, req2 = (struct hclge_fd_tcam_config_2_cmd *)desc[1].data; req3 = (struct hclge_fd_tcam_config_3_cmd *)desc[2].data; - req1->stage = stage; + req1->stage = tcam_msg.stage; req1->xy_sel = sel_x ? 1 : 0; - req1->index = cpu_to_le32(loc); + req1->index = cpu_to_le32(tcam_msg.loc); ret = hclge_cmd_send(&hdev->hw, desc, 3); if (ret) return ret; - dev_info(&hdev->pdev->dev, " read result tcam key %s(%u):\n", - sel_x ? "x" : "y", loc); + pos += scnprintf(tcam_buf + pos, HCLGE_DBG_TCAM_BUF_SIZE - pos, + "read result tcam key %s(%u):\n", sel_x ? "x" : "y", + tcam_msg.loc); /* tcam_data0 ~ tcam_data1 */ req = (u32 *)req1->tcam_data; for (i = 0; i < 2; i++) - dev_info(&hdev->pdev->dev, "%08x\n", *req++); + pos += scnprintf(tcam_buf + pos, HCLGE_DBG_TCAM_BUF_SIZE - pos, + "%08x\n", *req++); /* tcam_data2 ~ tcam_data7 */ req = (u32 *)req2->tcam_data; for (i = 0; i < 6; i++) - dev_info(&hdev->pdev->dev, "%08x\n", *req++); + pos += scnprintf(tcam_buf + pos, HCLGE_DBG_TCAM_BUF_SIZE - pos, + "%08x\n", *req++); /* tcam_data8 ~ tcam_data12 */ req = (u32 *)req3->tcam_data; for (i = 0; i < 5; i++) - dev_info(&hdev->pdev->dev, "%08x\n", *req++); + pos += scnprintf(tcam_buf + pos, HCLGE_DBG_TCAM_BUF_SIZE - pos, + "%08x\n", *req++); return ret; } @@ -1308,265 +1477,348 @@ static int hclge_dbg_get_rules_location(struct hclge_dev *hdev, u16 *rule_locs) } spin_unlock_bh(&hdev->fd_rule_lock); - if (cnt != hdev->hclge_fd_rule_num) + if (cnt != hdev->hclge_fd_rule_num || cnt == 0) return -EINVAL; return cnt; } -static void hclge_dbg_fd_tcam(struct hclge_dev *hdev) +static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len) { + u32 rule_num = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; + struct hclge_dbg_tcam_msg tcam_msg; int i, ret, rule_cnt; u16 *rule_locs; + char *tcam_buf; + int pos = 0; if (!hnae3_dev_fd_supported(hdev)) { dev_err(&hdev->pdev->dev, "Only FD-supported dev supports dump fd tcam\n"); - return; + return -EOPNOTSUPP; } - if (!hdev->hclge_fd_rule_num || - !hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) - return; + if (!hdev->hclge_fd_rule_num || !rule_num) + return 0; - rule_locs = kcalloc(hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1], - sizeof(u16), GFP_KERNEL); + rule_locs = kcalloc(rule_num, sizeof(u16), GFP_KERNEL); if (!rule_locs) - return; + return -ENOMEM; + + tcam_buf = kzalloc(HCLGE_DBG_TCAM_BUF_SIZE, GFP_KERNEL); + if (!tcam_buf) { + kfree(rule_locs); + return -ENOMEM; + } rule_cnt = hclge_dbg_get_rules_location(hdev, rule_locs); - if (rule_cnt <= 0) { + if (rule_cnt < 0) { + ret = rule_cnt; dev_err(&hdev->pdev->dev, - "failed to get rule number, ret = %d\n", rule_cnt); - kfree(rule_locs); - return; + "failed to get rule number, ret = %d\n", ret); + goto out; } + ret = 0; for (i = 0; i < rule_cnt; i++) { - ret = hclge_dbg_fd_tcam_read(hdev, 0, true, rule_locs[i]); + tcam_msg.stage = HCLGE_FD_STAGE_1; + tcam_msg.loc = rule_locs[i]; + + ret = hclge_dbg_fd_tcam_read(hdev, true, tcam_buf, tcam_msg); if (ret) { dev_err(&hdev->pdev->dev, "failed to get fd tcam key x, ret = %d\n", ret); - kfree(rule_locs); - return; + goto out; } - ret = hclge_dbg_fd_tcam_read(hdev, 0, false, rule_locs[i]); + pos += scnprintf(buf + pos, len - pos, "%s", tcam_buf); + + ret = hclge_dbg_fd_tcam_read(hdev, false, tcam_buf, tcam_msg); if (ret) { dev_err(&hdev->pdev->dev, "failed to get fd tcam key y, ret = %d\n", ret); - kfree(rule_locs); - return; + goto out; } + + pos += scnprintf(buf + pos, len - pos, "%s", tcam_buf); } +out: + kfree(tcam_buf); kfree(rule_locs); + return ret; } -void hclge_dbg_dump_rst_info(struct hclge_dev *hdev) -{ - dev_info(&hdev->pdev->dev, "PF reset count: %u\n", - hdev->rst_stats.pf_rst_cnt); - dev_info(&hdev->pdev->dev, "FLR reset count: %u\n", - hdev->rst_stats.flr_rst_cnt); - dev_info(&hdev->pdev->dev, "GLOBAL reset count: %u\n", - hdev->rst_stats.global_rst_cnt); - dev_info(&hdev->pdev->dev, "IMP reset count: %u\n", - hdev->rst_stats.imp_rst_cnt); - dev_info(&hdev->pdev->dev, "reset done count: %u\n", - hdev->rst_stats.reset_done_cnt); - dev_info(&hdev->pdev->dev, "HW reset done count: %u\n", - hdev->rst_stats.hw_reset_done_cnt); - dev_info(&hdev->pdev->dev, "reset count: %u\n", - hdev->rst_stats.reset_cnt); - dev_info(&hdev->pdev->dev, "reset fail count: %u\n", - hdev->rst_stats.reset_fail_cnt); - dev_info(&hdev->pdev->dev, "vector0 interrupt enable status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_REG_BASE)); - dev_info(&hdev->pdev->dev, "reset interrupt source: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG)); - dev_info(&hdev->pdev->dev, "reset interrupt status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_INT_STS)); - dev_info(&hdev->pdev->dev, "hardware reset status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG)); - dev_info(&hdev->pdev->dev, "handshake status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_NIC_CSQ_DEPTH_REG)); - dev_info(&hdev->pdev->dev, "function reset status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_FUN_RST_ING)); - dev_info(&hdev->pdev->dev, "hdev state: 0x%lx\n", hdev->state); -} - -static void hclge_dbg_dump_serv_info(struct hclge_dev *hdev) -{ - dev_info(&hdev->pdev->dev, "last_serv_processed: %lu\n", - hdev->last_serv_processed); - dev_info(&hdev->pdev->dev, "last_serv_cnt: %lu\n", - hdev->serv_processed_cnt); -} - -static void hclge_dbg_dump_interrupt(struct hclge_dev *hdev) -{ - dev_info(&hdev->pdev->dev, "num_nic_msi: %u\n", hdev->num_nic_msi); - dev_info(&hdev->pdev->dev, "num_roce_msi: %u\n", hdev->num_roce_msi); - dev_info(&hdev->pdev->dev, "num_msi_used: %u\n", hdev->num_msi_used); - dev_info(&hdev->pdev->dev, "num_msi_left: %u\n", hdev->num_msi_left); -} - -static void hclge_dbg_get_m7_stats_info(struct hclge_dev *hdev) -{ - struct hclge_desc *desc_src, *desc_tmp; - struct hclge_get_m7_bd_cmd *req; +static int hclge_dbg_dump_fd_counter(struct hclge_dev *hdev, char *buf, int len) +{ + u8 func_num = pci_num_vf(hdev->pdev) + 1; /* pf and enabled vf num */ + struct hclge_fd_ad_cnt_read_cmd *req; + char str_id[HCLGE_DBG_ID_LEN]; struct hclge_desc desc; - u32 bd_num, buf_len; - int ret, i; + int pos = 0; + int ret; + u64 cnt; + u8 i; + + pos += scnprintf(buf + pos, len - pos, + "func_id\thit_times\n"); + + for (i = 0; i < func_num; i++) { + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_CNT_OP, true); + req = (struct hclge_fd_ad_cnt_read_cmd *)desc.data; + req->index = cpu_to_le16(i); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, "failed to get fd counter, ret = %d\n", + ret); + return ret; + } + cnt = le64_to_cpu(req->cnt); + hclge_dbg_get_func_id_str(str_id, i); + pos += scnprintf(buf + pos, len - pos, + "%s\t%llu\n", str_id, cnt); + } + + return 0; +} + +int hclge_dbg_dump_rst_info(struct hclge_dev *hdev, char *buf, int len) +{ + int pos = 0; + + pos += scnprintf(buf + pos, len - pos, "PF reset count: %u\n", + hdev->rst_stats.pf_rst_cnt); + pos += scnprintf(buf + pos, len - pos, "FLR reset count: %u\n", + hdev->rst_stats.flr_rst_cnt); + pos += scnprintf(buf + pos, len - pos, "GLOBAL reset count: %u\n", + hdev->rst_stats.global_rst_cnt); + pos += scnprintf(buf + pos, len - pos, "IMP reset count: %u\n", + hdev->rst_stats.imp_rst_cnt); + pos += scnprintf(buf + pos, len - pos, "reset done count: %u\n", + hdev->rst_stats.reset_done_cnt); + pos += scnprintf(buf + pos, len - pos, "HW reset done count: %u\n", + hdev->rst_stats.hw_reset_done_cnt); + pos += scnprintf(buf + pos, len - pos, "reset count: %u\n", + hdev->rst_stats.reset_cnt); + pos += scnprintf(buf + pos, len - pos, "reset fail count: %u\n", + hdev->rst_stats.reset_fail_cnt); + pos += scnprintf(buf + pos, len - pos, + "vector0 interrupt enable status: 0x%x\n", + hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_REG_BASE)); + pos += scnprintf(buf + pos, len - pos, "reset interrupt source: 0x%x\n", + hclge_read_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG)); + pos += scnprintf(buf + pos, len - pos, "reset interrupt status: 0x%x\n", + hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_INT_STS)); + pos += scnprintf(buf + pos, len - pos, "RAS interrupt status: 0x%x\n", + hclge_read_dev(&hdev->hw, + HCLGE_RAS_PF_OTHER_INT_STS_REG)); + pos += scnprintf(buf + pos, len - pos, "hardware reset status: 0x%x\n", + hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG)); + pos += scnprintf(buf + pos, len - pos, "handshake status: 0x%x\n", + hclge_read_dev(&hdev->hw, HCLGE_NIC_CSQ_DEPTH_REG)); + pos += scnprintf(buf + pos, len - pos, "function reset status: 0x%x\n", + hclge_read_dev(&hdev->hw, HCLGE_FUN_RST_ING)); + pos += scnprintf(buf + pos, len - pos, "hdev state: 0x%lx\n", + hdev->state); + + return 0; +} + +static int hclge_dbg_dump_serv_info(struct hclge_dev *hdev, char *buf, int len) +{ + unsigned long rem_nsec; + int pos = 0; + u64 lc; + + lc = local_clock(); + rem_nsec = do_div(lc, HCLGE_BILLION_NANO_SECONDS); + + pos += scnprintf(buf + pos, len - pos, "local_clock: [%5lu.%06lu]\n", + (unsigned long)lc, rem_nsec / 1000); + pos += scnprintf(buf + pos, len - pos, "delta: %u(ms)\n", + jiffies_to_msecs(jiffies - hdev->last_serv_processed)); + pos += scnprintf(buf + pos, len - pos, + "last_service_task_processed: %lu(jiffies)\n", + hdev->last_serv_processed); + pos += scnprintf(buf + pos, len - pos, "last_service_task_cnt: %lu\n", + hdev->serv_processed_cnt); + + return 0; +} + +static int hclge_dbg_dump_interrupt(struct hclge_dev *hdev, char *buf, int len) +{ + int pos = 0; + + pos += scnprintf(buf + pos, len - pos, "num_nic_msi: %u\n", + hdev->num_nic_msi); + pos += scnprintf(buf + pos, len - pos, "num_roce_msi: %u\n", + hdev->num_roce_msi); + pos += scnprintf(buf + pos, len - pos, "num_msi_used: %u\n", + hdev->num_msi_used); + pos += scnprintf(buf + pos, len - pos, "num_msi_left: %u\n", + hdev->num_msi_left); + + return 0; +} - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_M7_STATS_BD, true); +static void hclge_dbg_imp_info_data_print(struct hclge_desc *desc_src, + char *buf, int len, u32 bd_num) +{ +#define HCLGE_DBG_IMP_INFO_PRINT_OFFSET 0x2 + + struct hclge_desc *desc_index = desc_src; + u32 offset = 0; + int pos = 0; + u32 i, j; - req = (struct hclge_get_m7_bd_cmd *)desc.data; + pos += scnprintf(buf + pos, len - pos, "offset | data\n"); + + for (i = 0; i < bd_num; i++) { + j = 0; + while (j < HCLGE_DESC_DATA_LEN - 1) { + pos += scnprintf(buf + pos, len - pos, "0x%04x | ", + offset); + pos += scnprintf(buf + pos, len - pos, "0x%08x ", + le32_to_cpu(desc_index->data[j++])); + pos += scnprintf(buf + pos, len - pos, "0x%08x\n", + le32_to_cpu(desc_index->data[j++])); + offset += sizeof(u32) * HCLGE_DBG_IMP_INFO_PRINT_OFFSET; + } + desc_index++; + } +} + +static int +hclge_dbg_get_imp_stats_info(struct hclge_dev *hdev, char *buf, int len) +{ + struct hclge_get_imp_bd_cmd *req; + struct hclge_desc *desc_src; + struct hclge_desc desc; + u32 bd_num; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_IMP_STATS_BD, true); + + req = (struct hclge_get_imp_bd_cmd *)desc.data; ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { dev_err(&hdev->pdev->dev, - "get firmware statistics bd number failed, ret = %d\n", + "failed to get imp statistics bd number, ret = %d\n", ret); - return; + return ret; } bd_num = le32_to_cpu(req->bd_num); - buf_len = sizeof(struct hclge_desc) * bd_num; - desc_src = kzalloc(buf_len, GFP_KERNEL); + desc_src = kcalloc(bd_num, sizeof(struct hclge_desc), GFP_KERNEL); if (!desc_src) - return; + return -ENOMEM; - desc_tmp = desc_src; - ret = hclge_dbg_cmd_send(hdev, desc_tmp, 0, bd_num, - HCLGE_OPC_M7_STATS_INFO); + ret = hclge_dbg_cmd_send(hdev, desc_src, 0, bd_num, + HCLGE_OPC_IMP_STATS_INFO); if (ret) { kfree(desc_src); dev_err(&hdev->pdev->dev, - "get firmware statistics failed, ret = %d\n", ret); - return; + "failed to get imp statistics, ret = %d\n", ret); + return ret; } - for (i = 0; i < bd_num; i++) { - dev_info(&hdev->pdev->dev, "0x%08x 0x%08x 0x%08x\n", - le32_to_cpu(desc_tmp->data[0]), - le32_to_cpu(desc_tmp->data[1]), - le32_to_cpu(desc_tmp->data[2])); - dev_info(&hdev->pdev->dev, "0x%08x 0x%08x 0x%08x\n", - le32_to_cpu(desc_tmp->data[3]), - le32_to_cpu(desc_tmp->data[4]), - le32_to_cpu(desc_tmp->data[5])); - - desc_tmp++; - } + hclge_dbg_imp_info_data_print(desc_src, buf, len, bd_num); kfree(desc_src); + + return 0; } #define HCLGE_CMD_NCL_CONFIG_BD_NUM 5 +#define HCLGE_MAX_NCL_CONFIG_LENGTH 16384 -static void hclge_ncl_config_data_print(struct hclge_dev *hdev, - struct hclge_desc *desc, int *offset, - int *length) +static void hclge_ncl_config_data_print(struct hclge_desc *desc, int *index, + char *buf, int *len, int *pos) { #define HCLGE_CMD_DATA_NUM 6 - int i; - int j; + int offset = HCLGE_MAX_NCL_CONFIG_LENGTH - *index; + int i, j; for (i = 0; i < HCLGE_CMD_NCL_CONFIG_BD_NUM; i++) { for (j = 0; j < HCLGE_CMD_DATA_NUM; j++) { if (i == 0 && j == 0) continue; - dev_info(&hdev->pdev->dev, "0x%04x | 0x%08x\n", - *offset, - le32_to_cpu(desc[i].data[j])); - *offset += sizeof(u32); - *length -= sizeof(u32); - if (*length <= 0) + *pos += scnprintf(buf + *pos, *len - *pos, + "0x%04x | 0x%08x\n", offset, + le32_to_cpu(desc[i].data[j])); + + offset += sizeof(u32); + *index -= sizeof(u32); + + if (*index <= 0) return; } } } -/* hclge_dbg_dump_ncl_config: print specified range of NCL_CONFIG file - * @hdev: pointer to struct hclge_dev - * @cmd_buf: string that contains offset and length - */ -static void hclge_dbg_dump_ncl_config(struct hclge_dev *hdev, - const char *cmd_buf) +static int +hclge_dbg_dump_ncl_config(struct hclge_dev *hdev, char *buf, int len) { -#define HCLGE_MAX_NCL_CONFIG_OFFSET 4096 #define HCLGE_NCL_CONFIG_LENGTH_IN_EACH_CMD (20 + 24 * 4) -#define HCLGE_NCL_CONFIG_PARAM_NUM 2 struct hclge_desc desc[HCLGE_CMD_NCL_CONFIG_BD_NUM]; int bd_num = HCLGE_CMD_NCL_CONFIG_BD_NUM; - int offset; - int length; - int data0; + int index = HCLGE_MAX_NCL_CONFIG_LENGTH; + int pos = 0; + u32 data0; int ret; - ret = sscanf(cmd_buf, "%x %x", &offset, &length); - if (ret != HCLGE_NCL_CONFIG_PARAM_NUM) { - dev_err(&hdev->pdev->dev, - "Too few parameters, num = %d.\n", ret); - return; - } - - if (offset < 0 || offset >= HCLGE_MAX_NCL_CONFIG_OFFSET || - length <= 0 || length > HCLGE_MAX_NCL_CONFIG_OFFSET - offset) { - dev_err(&hdev->pdev->dev, - "Invalid input, offset = %d, length = %d.\n", - offset, length); - return; - } + pos += scnprintf(buf + pos, len - pos, "offset | data\n"); - dev_info(&hdev->pdev->dev, "offset | data\n"); - - while (length > 0) { - data0 = offset; - if (length >= HCLGE_NCL_CONFIG_LENGTH_IN_EACH_CMD) + while (index > 0) { + data0 = HCLGE_MAX_NCL_CONFIG_LENGTH - index; + if (index >= HCLGE_NCL_CONFIG_LENGTH_IN_EACH_CMD) data0 |= HCLGE_NCL_CONFIG_LENGTH_IN_EACH_CMD << 16; else - data0 |= length << 16; + data0 |= (u32)index << 16; ret = hclge_dbg_cmd_send(hdev, desc, data0, bd_num, HCLGE_OPC_QUERY_NCL_CONFIG); if (ret) - return; + return ret; - hclge_ncl_config_data_print(hdev, desc, &offset, &length); + hclge_ncl_config_data_print(desc, &index, buf, &len, &pos); } + + return 0; } -static void hclge_dbg_dump_loopback(struct hclge_dev *hdev) +static int hclge_dbg_dump_loopback(struct hclge_dev *hdev, char *buf, int len) { struct phy_device *phydev = hdev->hw.mac.phydev; struct hclge_config_mac_mode_cmd *req_app; struct hclge_common_lb_cmd *req_common; struct hclge_desc desc; u8 loopback_en; + int pos = 0; int ret; req_app = (struct hclge_config_mac_mode_cmd *)desc.data; req_common = (struct hclge_common_lb_cmd *)desc.data; - dev_info(&hdev->pdev->dev, "mac id: %u\n", hdev->hw.mac.mac_id); + pos += scnprintf(buf + pos, len - pos, "mac id: %u\n", + hdev->hw.mac.mac_id); hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAC_MODE, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { dev_err(&hdev->pdev->dev, "failed to dump app loopback status, ret = %d\n", ret); - return; + return ret; } loopback_en = hnae3_get_bit(le32_to_cpu(req_app->txrx_pad_fcs_loop_en), HCLGE_MAC_APP_LP_B); - dev_info(&hdev->pdev->dev, "app loopback: %s\n", - loopback_en ? "on" : "off"); + pos += scnprintf(buf + pos, len - pos, "app loopback: %s\n", + state_str[loopback_en]); hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_COMMON_LOOPBACK, true); ret = hclge_cmd_send(&hdev->hw, &desc, 1); @@ -1574,247 +1826,647 @@ static void hclge_dbg_dump_loopback(struct hclge_dev *hdev) dev_err(&hdev->pdev->dev, "failed to dump common loopback status, ret = %d\n", ret); - return; + return ret; } loopback_en = req_common->enable & HCLGE_CMD_SERDES_SERIAL_INNER_LOOP_B; - dev_info(&hdev->pdev->dev, "serdes serial loopback: %s\n", - loopback_en ? "on" : "off"); + pos += scnprintf(buf + pos, len - pos, "serdes serial loopback: %s\n", + state_str[loopback_en]); loopback_en = req_common->enable & - HCLGE_CMD_SERDES_PARALLEL_INNER_LOOP_B; - dev_info(&hdev->pdev->dev, "serdes parallel loopback: %s\n", - loopback_en ? "on" : "off"); + HCLGE_CMD_SERDES_PARALLEL_INNER_LOOP_B ? 1 : 0; + pos += scnprintf(buf + pos, len - pos, "serdes parallel loopback: %s\n", + state_str[loopback_en]); if (phydev) { - dev_info(&hdev->pdev->dev, "phy loopback: %s\n", - phydev->loopback_enabled ? "on" : "off"); + loopback_en = phydev->loopback_enabled; + pos += scnprintf(buf + pos, len - pos, "phy loopback: %s\n", + state_str[loopback_en]); } else if (hnae3_dev_phy_imp_supported(hdev)) { loopback_en = req_common->enable & HCLGE_CMD_GE_PHY_INNER_LOOP_B; - dev_info(&hdev->pdev->dev, "phy loopback: %s\n", - loopback_en ? "on" : "off"); + pos += scnprintf(buf + pos, len - pos, "phy loopback: %s\n", + state_str[loopback_en]); } + + return 0; } /* hclge_dbg_dump_mac_tnl_status: print message about mac tnl interrupt * @hdev: pointer to struct hclge_dev */ -static void hclge_dbg_dump_mac_tnl_status(struct hclge_dev *hdev) +static int +hclge_dbg_dump_mac_tnl_status(struct hclge_dev *hdev, char *buf, int len) { -#define HCLGE_BILLION_NANO_SECONDS 1000000000 - struct hclge_mac_tnl_stats stats; unsigned long rem_nsec; + int pos = 0; - dev_info(&hdev->pdev->dev, "Recently generated mac tnl interruption:\n"); + pos += scnprintf(buf + pos, len - pos, + "Recently generated mac tnl interruption:\n"); while (kfifo_get(&hdev->mac_tnl_log, &stats)) { rem_nsec = do_div(stats.time, HCLGE_BILLION_NANO_SECONDS); - dev_info(&hdev->pdev->dev, "[%07lu.%03lu] status = 0x%x\n", - (unsigned long)stats.time, rem_nsec / 1000, - stats.status); + + pos += scnprintf(buf + pos, len - pos, + "[%07lu.%03lu] status = 0x%x\n", + (unsigned long)stats.time, rem_nsec / 1000, + stats.status); + } + + return 0; +} + + +static const struct hclge_dbg_item mac_list_items[] = { + { "FUNC_ID", 2 }, + { "MAC_ADDR", 12 }, + { "STATE", 2 }, +}; + +static void hclge_dbg_dump_mac_list(struct hclge_dev *hdev, char *buf, int len, + bool is_unicast) +{ + char data_str[ARRAY_SIZE(mac_list_items)][HCLGE_DBG_DATA_STR_LEN]; + char content[HCLGE_DBG_INFO_LEN], str_id[HCLGE_DBG_ID_LEN]; + char *result[ARRAY_SIZE(mac_list_items)]; + struct hclge_mac_node *mac_node, *tmp; + struct hclge_vport *vport; + struct list_head *list; + u32 func_id; + int pos = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(mac_list_items); i++) + result[i] = &data_str[i][0]; + + pos += scnprintf(buf + pos, len - pos, "%s MAC_LIST:\n", + is_unicast ? "UC" : "MC"); + hclge_dbg_fill_content(content, sizeof(content), mac_list_items, + NULL, ARRAY_SIZE(mac_list_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + + for (func_id = 0; func_id < hdev->num_alloc_vport; func_id++) { + vport = &hdev->vport[func_id]; + list = is_unicast ? &vport->uc_mac_list : &vport->mc_mac_list; + spin_lock_bh(&vport->mac_list_lock); + list_for_each_entry_safe(mac_node, tmp, list, node) { + i = 0; + result[i++] = hclge_dbg_get_func_id_str(str_id, + func_id); + sprintf(result[i++], "%pM", mac_node->mac_addr); + sprintf(result[i++], "%5s", + hclge_mac_state_str[mac_node->state]); + hclge_dbg_fill_content(content, sizeof(content), + mac_list_items, + (const char **)result, + ARRAY_SIZE(mac_list_items)); + pos += scnprintf(buf + pos, len - pos, "%s", content); + } + spin_unlock_bh(&vport->mac_list_lock); } } -static void hclge_dbg_dump_qs_shaper_single(struct hclge_dev *hdev, u16 qsid) +static int hclge_dbg_dump_umv_info(struct hclge_dev *hdev, char *buf, int len) { - struct hclge_qs_shapping_cmd *shap_cfg_cmd; - u8 ir_u, ir_b, ir_s, bs_b, bs_s; + u8 func_num = pci_num_vf(hdev->pdev) + 1; + struct hclge_vport *vport; + int pos = 0; + u8 i; + + pos += scnprintf(buf, len, "num_alloc_vport : %u\n", + hdev->num_alloc_vport); + pos += scnprintf(buf + pos, len - pos, "max_umv_size : %u\n", + hdev->max_umv_size); + pos += scnprintf(buf + pos, len - pos, "wanted_umv_size : %u\n", + hdev->wanted_umv_size); + pos += scnprintf(buf + pos, len - pos, "priv_umv_size : %u\n", + hdev->priv_umv_size); + + mutex_lock(&hdev->vport_lock); + pos += scnprintf(buf + pos, len - pos, "share_umv_size : %u\n", + hdev->share_umv_size); + for (i = 0; i < func_num; i++) { + vport = &hdev->vport[i]; + pos += scnprintf(buf + pos, len - pos, + "vport(%u) used_umv_num : %u\n", + i, vport->used_umv_num); + } + mutex_unlock(&hdev->vport_lock); + + return 0; +} + +static int hclge_get_vlan_rx_offload_cfg(struct hclge_dev *hdev, u8 vf_id, + struct hclge_dbg_vlan_cfg *vlan_cfg) +{ + struct hclge_vport_vtag_rx_cfg_cmd *req; struct hclge_desc desc; - u32 shapping_para; - u32 rate; + u16 bmap_index; + u8 rx_cfg; int ret; - hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QCN_SHAPPING_CFG, true); + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_VLAN_PORT_RX_CFG, true); - shap_cfg_cmd = (struct hclge_qs_shapping_cmd *)desc.data; - shap_cfg_cmd->qs_id = cpu_to_le16(qsid); + req = (struct hclge_vport_vtag_rx_cfg_cmd *)desc.data; + req->vf_offset = vf_id / HCLGE_VF_NUM_PER_CMD; + bmap_index = vf_id % HCLGE_VF_NUM_PER_CMD / HCLGE_VF_NUM_PER_BYTE; + req->vf_bitmap[bmap_index] = 1U << (vf_id % HCLGE_VF_NUM_PER_BYTE); ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { dev_err(&hdev->pdev->dev, - "qs%u failed to get tx_rate, ret=%d\n", - qsid, ret); - return; + "failed to get vport%u rxvlan cfg, ret = %d\n", + vf_id, ret); + return ret; } - shapping_para = le32_to_cpu(shap_cfg_cmd->qs_shapping_para); - ir_b = hclge_tm_get_field(shapping_para, IR_B); - ir_u = hclge_tm_get_field(shapping_para, IR_U); - ir_s = hclge_tm_get_field(shapping_para, IR_S); - bs_b = hclge_tm_get_field(shapping_para, BS_B); - bs_s = hclge_tm_get_field(shapping_para, BS_S); - rate = le32_to_cpu(shap_cfg_cmd->qs_rate); + rx_cfg = req->vport_vlan_cfg; + vlan_cfg->strip_tag1 = hnae3_get_bit(rx_cfg, HCLGE_REM_TAG1_EN_B); + vlan_cfg->strip_tag2 = hnae3_get_bit(rx_cfg, HCLGE_REM_TAG2_EN_B); + vlan_cfg->drop_tag1 = hnae3_get_bit(rx_cfg, HCLGE_DISCARD_TAG1_EN_B); + vlan_cfg->drop_tag2 = hnae3_get_bit(rx_cfg, HCLGE_DISCARD_TAG2_EN_B); + vlan_cfg->pri_only1 = hnae3_get_bit(rx_cfg, HCLGE_SHOW_TAG1_EN_B); + vlan_cfg->pri_only2 = hnae3_get_bit(rx_cfg, HCLGE_SHOW_TAG2_EN_B); - dev_info(&hdev->pdev->dev, - "qs%u ir_b:%u, ir_u:%u, ir_s:%u, bs_b:%u, bs_s:%u, flag:%#x, rate:%u(Mbps)\n", - qsid, ir_b, ir_u, ir_s, bs_b, bs_s, shap_cfg_cmd->flag, rate); + return 0; } -static void hclge_dbg_dump_qs_shaper_all(struct hclge_dev *hdev) +static int hclge_get_vlan_tx_offload_cfg(struct hclge_dev *hdev, u8 vf_id, + struct hclge_dbg_vlan_cfg *vlan_cfg) { - struct hnae3_knic_private_info *kinfo; - struct hclge_vport *vport; - int vport_id, i; + struct hclge_vport_vtag_tx_cfg_cmd *req; + struct hclge_desc desc; + u16 bmap_index; + u8 tx_cfg; + int ret; - for (vport_id = 0; vport_id <= pci_num_vf(hdev->pdev); vport_id++) { - vport = &hdev->vport[vport_id]; - kinfo = &vport->nic.kinfo; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_VLAN_PORT_TX_CFG, true); + req = (struct hclge_vport_vtag_tx_cfg_cmd *)desc.data; + req->vf_offset = vf_id / HCLGE_VF_NUM_PER_CMD; + bmap_index = vf_id % HCLGE_VF_NUM_PER_CMD / HCLGE_VF_NUM_PER_BYTE; + req->vf_bitmap[bmap_index] = 1U << (vf_id % HCLGE_VF_NUM_PER_BYTE); - dev_info(&hdev->pdev->dev, "qs cfg of vport%d:\n", vport_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get vport%u txvlan cfg, ret = %d\n", + vf_id, ret); + return ret; + } - for (i = 0; i < kinfo->tc_info.num_tc; i++) { - u16 qsid = vport->qs_offset + i; + tx_cfg = req->vport_vlan_cfg; + vlan_cfg->pvid = le16_to_cpu(req->def_vlan_tag1); - hclge_dbg_dump_qs_shaper_single(hdev, qsid); - } - } + vlan_cfg->accept_tag1 = hnae3_get_bit(tx_cfg, HCLGE_ACCEPT_TAG1_B); + vlan_cfg->accept_tag2 = hnae3_get_bit(tx_cfg, HCLGE_ACCEPT_TAG2_B); + vlan_cfg->accept_untag1 = hnae3_get_bit(tx_cfg, HCLGE_ACCEPT_UNTAG1_B); + vlan_cfg->accept_untag2 = hnae3_get_bit(tx_cfg, HCLGE_ACCEPT_UNTAG2_B); + vlan_cfg->insert_tag1 = hnae3_get_bit(tx_cfg, HCLGE_PORT_INS_TAG1_EN_B); + vlan_cfg->insert_tag2 = hnae3_get_bit(tx_cfg, HCLGE_PORT_INS_TAG2_EN_B); + vlan_cfg->shift_tag = hnae3_get_bit(tx_cfg, HCLGE_TAG_SHIFT_MODE_EN_B); + + return 0; } -static void hclge_dbg_dump_qs_shaper(struct hclge_dev *hdev, - const char *cmd_buf) +static int hclge_get_vlan_filter_config_cmd(struct hclge_dev *hdev, + u8 vlan_type, u8 vf_id, + struct hclge_desc *desc) { - u16 qsid; + struct hclge_vlan_filter_ctrl_cmd *req; int ret; - ret = kstrtou16(cmd_buf, 0, &qsid); - if (ret) { - hclge_dbg_dump_qs_shaper_all(hdev); - return; - } + hclge_cmd_setup_basic_desc(desc, HCLGE_OPC_VLAN_FILTER_CTRL, true); + req = (struct hclge_vlan_filter_ctrl_cmd *)desc->data; + req->vlan_type = vlan_type; + req->vf_id = vf_id; - if (qsid >= hdev->ae_dev->dev_specs.max_qset_num) { - dev_err(&hdev->pdev->dev, "qsid(%u) out of range[0-%u]\n", - qsid, hdev->ae_dev->dev_specs.max_qset_num - 1); - return; - } + ret = hclge_cmd_send(&hdev->hw, desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to get vport%u vlan filter config, ret = %d.\n", + vf_id, ret); - hclge_dbg_dump_qs_shaper_single(hdev, qsid); + return ret; } -static int hclge_dbg_dump_mac_list(struct hclge_dev *hdev, const char *cmd_buf, - bool is_unicast) +static int hclge_get_vlan_filter_state(struct hclge_dev *hdev, u8 vlan_type, + u8 vf_id, u8 *vlan_fe) { - struct hclge_mac_node *mac_node, *tmp; - struct hclge_vport *vport; - struct list_head *list; - u32 func_id; + struct hclge_vlan_filter_ctrl_cmd *req; + struct hclge_desc desc; + int ret; + + ret = hclge_get_vlan_filter_config_cmd(hdev, vlan_type, vf_id, &desc); + if (ret) + return ret; + + req = (struct hclge_vlan_filter_ctrl_cmd *)desc.data; + *vlan_fe = req->vlan_fe; + + return 0; +} + +static int hclge_get_port_vlan_filter_bypass_state(struct hclge_dev *hdev, + u8 vf_id, u8 *bypass_en) +{ + struct hclge_port_vlan_filter_bypass_cmd *req; + struct hclge_desc desc; int ret; - ret = kstrtouint(cmd_buf, 0, &func_id); - if (ret < 0) { + if (!test_bit(HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, hdev->ae_dev->caps)) + return 0; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PORT_VLAN_BYPASS, true); + req = (struct hclge_port_vlan_filter_bypass_cmd *)desc.data; + req->vf_id = vf_id; + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { dev_err(&hdev->pdev->dev, - "dump mac list: bad command string, ret = %d\n", ret); - return -EINVAL; + "failed to get vport%u port vlan filter bypass state, ret = %d.\n", + vf_id, ret); + return ret; } - if (func_id >= hdev->num_alloc_vport) { - dev_err(&hdev->pdev->dev, - "function id(%u) is out of range(0-%u)\n", func_id, - hdev->num_alloc_vport - 1); - return -EINVAL; + *bypass_en = hnae3_get_bit(req->bypass_state, HCLGE_INGRESS_BYPASS_B); + + return 0; +} + +static const struct hclge_dbg_item vlan_filter_items[] = { + { "FUNC_ID", 2 }, + { "I_VF_VLAN_FILTER", 2 }, + { "E_VF_VLAN_FILTER", 2 }, + { "PORT_VLAN_FILTER_BYPASS", 0 } +}; + +static const struct hclge_dbg_item vlan_offload_items[] = { + { "FUNC_ID", 2 }, + { "PVID", 4 }, + { "ACCEPT_TAG1", 2 }, + { "ACCEPT_TAG2", 2 }, + { "ACCEPT_UNTAG1", 2 }, + { "ACCEPT_UNTAG2", 2 }, + { "INSERT_TAG1", 2 }, + { "INSERT_TAG2", 2 }, + { "SHIFT_TAG", 2 }, + { "STRIP_TAG1", 2 }, + { "STRIP_TAG2", 2 }, + { "DROP_TAG1", 2 }, + { "DROP_TAG2", 2 }, + { "PRI_ONLY_TAG1", 2 }, + { "PRI_ONLY_TAG2", 0 } +}; + +static int hclge_dbg_dump_vlan_filter_config(struct hclge_dev *hdev, char *buf, + int len, int *pos) +{ + char content[HCLGE_DBG_VLAN_FLTR_INFO_LEN], str_id[HCLGE_DBG_ID_LEN]; + const char *result[ARRAY_SIZE(vlan_filter_items)]; + u8 i, j, vlan_fe, bypass, ingress, egress; + u8 func_num = pci_num_vf(hdev->pdev) + 1; /* pf and enabled vf num */ + int ret; + + ret = hclge_get_vlan_filter_state(hdev, HCLGE_FILTER_TYPE_PORT, 0, + &vlan_fe); + if (ret) + return ret; + ingress = vlan_fe & HCLGE_FILTER_FE_NIC_INGRESS_B; + egress = vlan_fe & HCLGE_FILTER_FE_NIC_EGRESS_B ? 1 : 0; + + *pos += scnprintf(buf, len, "I_PORT_VLAN_FILTER: %s\n", + state_str[ingress]); + *pos += scnprintf(buf + *pos, len - *pos, "E_PORT_VLAN_FILTER: %s\n", + state_str[egress]); + + hclge_dbg_fill_content(content, sizeof(content), vlan_filter_items, + NULL, ARRAY_SIZE(vlan_filter_items)); + *pos += scnprintf(buf + *pos, len - *pos, "%s", content); + + for (i = 0; i < func_num; i++) { + ret = hclge_get_vlan_filter_state(hdev, HCLGE_FILTER_TYPE_VF, i, + &vlan_fe); + if (ret) + return ret; + + ingress = vlan_fe & HCLGE_FILTER_FE_NIC_INGRESS_B; + egress = vlan_fe & HCLGE_FILTER_FE_NIC_EGRESS_B ? 1 : 0; + ret = hclge_get_port_vlan_filter_bypass_state(hdev, i, &bypass); + if (ret) + return ret; + j = 0; + result[j++] = hclge_dbg_get_func_id_str(str_id, i); + result[j++] = state_str[ingress]; + result[j++] = state_str[egress]; + result[j++] = + test_bit(HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, + hdev->ae_dev->caps) ? state_str[bypass] : "NA"; + hclge_dbg_fill_content(content, sizeof(content), + vlan_filter_items, result, + ARRAY_SIZE(vlan_filter_items)); + *pos += scnprintf(buf + *pos, len - *pos, "%s", content); } + *pos += scnprintf(buf + *pos, len - *pos, "\n"); + + return 0; +} - vport = &hdev->vport[func_id]; +static int hclge_dbg_dump_vlan_offload_config(struct hclge_dev *hdev, char *buf, + int len, int *pos) +{ + char str_id[HCLGE_DBG_ID_LEN], str_pvid[HCLGE_DBG_ID_LEN]; + const char *result[ARRAY_SIZE(vlan_offload_items)]; + char content[HCLGE_DBG_VLAN_OFFLOAD_INFO_LEN]; + u8 func_num = pci_num_vf(hdev->pdev) + 1; /* pf and enabled vf num */ + struct hclge_dbg_vlan_cfg vlan_cfg; + int ret; + u8 i, j; - list = is_unicast ? &vport->uc_mac_list : &vport->mc_mac_list; + hclge_dbg_fill_content(content, sizeof(content), vlan_offload_items, + NULL, ARRAY_SIZE(vlan_offload_items)); + *pos += scnprintf(buf + *pos, len - *pos, "%s", content); - dev_info(&hdev->pdev->dev, "vport %u %s mac list:\n", - func_id, is_unicast ? "uc" : "mc"); - dev_info(&hdev->pdev->dev, "mac address state\n"); + for (i = 0; i < func_num; i++) { + ret = hclge_get_vlan_tx_offload_cfg(hdev, i, &vlan_cfg); + if (ret) + return ret; - spin_lock_bh(&vport->mac_list_lock); + ret = hclge_get_vlan_rx_offload_cfg(hdev, i, &vlan_cfg); + if (ret) + return ret; - list_for_each_entry_safe(mac_node, tmp, list, node) { - dev_info(&hdev->pdev->dev, "%pM %d\n", - mac_node->mac_addr, mac_node->state); + sprintf(str_pvid, "%u", vlan_cfg.pvid); + j = 0; + result[j++] = hclge_dbg_get_func_id_str(str_id, i); + result[j++] = str_pvid; + result[j++] = state_str[vlan_cfg.accept_tag1]; + result[j++] = state_str[vlan_cfg.accept_tag2]; + result[j++] = state_str[vlan_cfg.accept_untag1]; + result[j++] = state_str[vlan_cfg.accept_untag2]; + result[j++] = state_str[vlan_cfg.insert_tag1]; + result[j++] = state_str[vlan_cfg.insert_tag2]; + result[j++] = state_str[vlan_cfg.shift_tag]; + result[j++] = state_str[vlan_cfg.strip_tag1]; + result[j++] = state_str[vlan_cfg.strip_tag2]; + result[j++] = state_str[vlan_cfg.drop_tag1]; + result[j++] = state_str[vlan_cfg.drop_tag2]; + result[j++] = state_str[vlan_cfg.pri_only1]; + result[j++] = state_str[vlan_cfg.pri_only2]; + + hclge_dbg_fill_content(content, sizeof(content), + vlan_offload_items, result, + ARRAY_SIZE(vlan_offload_items)); + *pos += scnprintf(buf + *pos, len - *pos, "%s", content); } - spin_unlock_bh(&vport->mac_list_lock); + return 0; +} + +static int hclge_dbg_dump_vlan_config(struct hclge_dev *hdev, char *buf, + int len) +{ + int pos = 0; + int ret; + + ret = hclge_dbg_dump_vlan_filter_config(hdev, buf, len, &pos); + if (ret) + return ret; + + return hclge_dbg_dump_vlan_offload_config(hdev, buf, len, &pos); +} + +static int hclge_dbg_dump_ptp_info(struct hclge_dev *hdev, char *buf, int len) +{ + struct hclge_ptp *ptp = hdev->ptp; + u32 sw_cfg = ptp->ptp_cfg; + unsigned int tx_start; + unsigned int last_rx; + int pos = 0; + u32 hw_cfg; + int ret; + + pos += scnprintf(buf + pos, len - pos, "phc %s's debug info:\n", + ptp->info.name); + pos += scnprintf(buf + pos, len - pos, "ptp enable: %s\n", + test_bit(HCLGE_PTP_FLAG_EN, &ptp->flags) ? + "yes" : "no"); + pos += scnprintf(buf + pos, len - pos, "ptp tx enable: %s\n", + test_bit(HCLGE_PTP_FLAG_TX_EN, &ptp->flags) ? + "yes" : "no"); + pos += scnprintf(buf + pos, len - pos, "ptp rx enable: %s\n", + test_bit(HCLGE_PTP_FLAG_RX_EN, &ptp->flags) ? + "yes" : "no"); + + last_rx = jiffies_to_msecs(ptp->last_rx); + pos += scnprintf(buf + pos, len - pos, "last rx time: %lu.%lu\n", + last_rx / MSEC_PER_SEC, last_rx % MSEC_PER_SEC); + pos += scnprintf(buf + pos, len - pos, "rx count: %lu\n", ptp->rx_cnt); + + tx_start = jiffies_to_msecs(ptp->tx_start); + pos += scnprintf(buf + pos, len - pos, "last tx start time: %lu.%lu\n", + tx_start / MSEC_PER_SEC, tx_start % MSEC_PER_SEC); + pos += scnprintf(buf + pos, len - pos, "tx count: %lu\n", ptp->tx_cnt); + pos += scnprintf(buf + pos, len - pos, "tx skipped count: %lu\n", + ptp->tx_skipped); + pos += scnprintf(buf + pos, len - pos, "tx timeout count: %lu\n", + ptp->tx_timeout); + pos += scnprintf(buf + pos, len - pos, "last tx seqid: %u\n", + ptp->last_tx_seqid); + + ret = hclge_ptp_cfg_qry(hdev, &hw_cfg); + if (ret) + return ret; + + pos += scnprintf(buf + pos, len - pos, "sw_cfg: %#x, hw_cfg: %#x\n", + sw_cfg, hw_cfg); + + pos += scnprintf(buf + pos, len - pos, "tx type: %d, rx filter: %d\n", + ptp->ts_cfg.tx_type, ptp->ts_cfg.rx_filter); return 0; } -int hclge_dbg_run_cmd(struct hnae3_handle *handle, const char *cmd_buf) +static int hclge_dbg_dump_mac_uc(struct hclge_dev *hdev, char *buf, int len) { -#define DUMP_REG "dump reg" -#define DUMP_TM_MAP "dump tm map" -#define DUMP_LOOPBACK "dump loopback" -#define DUMP_INTERRUPT "dump intr" + hclge_dbg_dump_mac_list(hdev, buf, len, true); - struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; + return 0; +} - if (strncmp(cmd_buf, "dump fd tcam", 12) == 0) { - hclge_dbg_fd_tcam(hdev); - } else if (strncmp(cmd_buf, "dump tc", 7) == 0) { - hclge_dbg_dump_tc(hdev); - } else if (strncmp(cmd_buf, DUMP_TM_MAP, strlen(DUMP_TM_MAP)) == 0) { - hclge_dbg_dump_tm_map(hdev, &cmd_buf[sizeof(DUMP_TM_MAP)]); - } else if (strncmp(cmd_buf, "dump tm", 7) == 0) { - hclge_dbg_dump_tm(hdev); - } else if (strncmp(cmd_buf, "dump qos pause cfg", 18) == 0) { - hclge_dbg_dump_qos_pause_cfg(hdev); - } else if (strncmp(cmd_buf, "dump qos pri map", 16) == 0) { - hclge_dbg_dump_qos_pri_map(hdev); - } else if (strncmp(cmd_buf, "dump qos buf cfg", 16) == 0) { - hclge_dbg_dump_qos_buf_cfg(hdev); - } else if (strncmp(cmd_buf, "dump mng tbl", 12) == 0) { - hclge_dbg_dump_mng_table(hdev); - } else if (strncmp(cmd_buf, DUMP_REG, strlen(DUMP_REG)) == 0) { - hclge_dbg_dump_reg_cmd(hdev, &cmd_buf[sizeof(DUMP_REG)]); - } else if (strncmp(cmd_buf, "dump reset info", 15) == 0) { - hclge_dbg_dump_rst_info(hdev); - } else if (strncmp(cmd_buf, "dump serv info", 14) == 0) { - hclge_dbg_dump_serv_info(hdev); - } else if (strncmp(cmd_buf, "dump m7 info", 12) == 0) { - hclge_dbg_get_m7_stats_info(hdev); - } else if (strncmp(cmd_buf, "dump ncl_config", 15) == 0) { - hclge_dbg_dump_ncl_config(hdev, - &cmd_buf[sizeof("dump ncl_config")]); - } else if (strncmp(cmd_buf, "dump mac tnl status", 19) == 0) { - hclge_dbg_dump_mac_tnl_status(hdev); - } else if (strncmp(cmd_buf, DUMP_LOOPBACK, - strlen(DUMP_LOOPBACK)) == 0) { - hclge_dbg_dump_loopback(hdev); - } else if (strncmp(cmd_buf, "dump qs shaper", 14) == 0) { - hclge_dbg_dump_qs_shaper(hdev, - &cmd_buf[sizeof("dump qs shaper")]); - } else if (strncmp(cmd_buf, "dump uc mac list", 16) == 0) { - hclge_dbg_dump_mac_list(hdev, - &cmd_buf[sizeof("dump uc mac list")], - true); - } else if (strncmp(cmd_buf, "dump mc mac list", 16) == 0) { - hclge_dbg_dump_mac_list(hdev, - &cmd_buf[sizeof("dump mc mac list")], - false); - } else if (strncmp(cmd_buf, DUMP_INTERRUPT, - strlen(DUMP_INTERRUPT)) == 0) { - hclge_dbg_dump_interrupt(hdev); - } else { - dev_info(&hdev->pdev->dev, "unknown command\n"); - return -EINVAL; - } +static int hclge_dbg_dump_mac_mc(struct hclge_dev *hdev, char *buf, int len) +{ + hclge_dbg_dump_mac_list(hdev, buf, len, false); return 0; } -int hclge_dbg_read_cmd(struct hnae3_handle *handle, const char *cmd_buf, +static const struct hclge_dbg_func hclge_dbg_cmd_func[] = { + { + .cmd = HNAE3_DBG_CMD_TM_NODES, + .dbg_dump = hclge_dbg_dump_tm_nodes, + }, + { + .cmd = HNAE3_DBG_CMD_TM_PRI, + .dbg_dump = hclge_dbg_dump_tm_pri, + }, + { + .cmd = HNAE3_DBG_CMD_TM_QSET, + .dbg_dump = hclge_dbg_dump_tm_qset, + }, + { + .cmd = HNAE3_DBG_CMD_TM_MAP, + .dbg_dump = hclge_dbg_dump_tm_map, + }, + { + .cmd = HNAE3_DBG_CMD_TM_PG, + .dbg_dump = hclge_dbg_dump_tm_pg, + }, + { + .cmd = HNAE3_DBG_CMD_TM_PORT, + .dbg_dump = hclge_dbg_dump_tm_port, + }, + { + .cmd = HNAE3_DBG_CMD_TC_SCH_INFO, + .dbg_dump = hclge_dbg_dump_tc, + }, + { + .cmd = HNAE3_DBG_CMD_QOS_PAUSE_CFG, + .dbg_dump = hclge_dbg_dump_qos_pause_cfg, + }, + { + .cmd = HNAE3_DBG_CMD_QOS_PRI_MAP, + .dbg_dump = hclge_dbg_dump_qos_pri_map, + }, + { + .cmd = HNAE3_DBG_CMD_QOS_BUF_CFG, + .dbg_dump = hclge_dbg_dump_qos_buf_cfg, + }, + { + .cmd = HNAE3_DBG_CMD_MAC_UC, + .dbg_dump = hclge_dbg_dump_mac_uc, + }, + { + .cmd = HNAE3_DBG_CMD_MAC_MC, + .dbg_dump = hclge_dbg_dump_mac_mc, + }, + { + .cmd = HNAE3_DBG_CMD_MNG_TBL, + .dbg_dump = hclge_dbg_dump_mng_table, + }, + { + .cmd = HNAE3_DBG_CMD_LOOPBACK, + .dbg_dump = hclge_dbg_dump_loopback, + }, + { + .cmd = HNAE3_DBG_CMD_PTP_INFO, + .dbg_dump = hclge_dbg_dump_ptp_info, + }, + { + .cmd = HNAE3_DBG_CMD_INTERRUPT_INFO, + .dbg_dump = hclge_dbg_dump_interrupt, + }, + { + .cmd = HNAE3_DBG_CMD_RESET_INFO, + .dbg_dump = hclge_dbg_dump_rst_info, + }, + { + .cmd = HNAE3_DBG_CMD_IMP_INFO, + .dbg_dump = hclge_dbg_get_imp_stats_info, + }, + { + .cmd = HNAE3_DBG_CMD_NCL_CONFIG, + .dbg_dump = hclge_dbg_dump_ncl_config, + }, + { + .cmd = HNAE3_DBG_CMD_REG_BIOS_COMMON, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_SSU, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_IGU_EGU, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_RPU, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_NCSI, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_RTC, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_PPP, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_RCB, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_TQP, + .dbg_dump_reg = hclge_dbg_dump_reg_cmd, + }, + { + .cmd = HNAE3_DBG_CMD_REG_MAC, + .dbg_dump = hclge_dbg_dump_mac, + }, + { + .cmd = HNAE3_DBG_CMD_REG_DCB, + .dbg_dump = hclge_dbg_dump_dcb, + }, + { + .cmd = HNAE3_DBG_CMD_FD_TCAM, + .dbg_dump = hclge_dbg_dump_fd_tcam, + }, + { + .cmd = HNAE3_DBG_CMD_MAC_TNL_STATUS, + .dbg_dump = hclge_dbg_dump_mac_tnl_status, + }, + { + .cmd = HNAE3_DBG_CMD_SERV_INFO, + .dbg_dump = hclge_dbg_dump_serv_info, + }, + { + .cmd = HNAE3_DBG_CMD_VLAN_CONFIG, + .dbg_dump = hclge_dbg_dump_vlan_config, + }, + { + .cmd = HNAE3_DBG_CMD_FD_COUNTER, + .dbg_dump = hclge_dbg_dump_fd_counter, + }, + { + .cmd = HNAE3_DBG_CMD_UMV_INFO, + .dbg_dump = hclge_dbg_dump_umv_info, + }, +}; + +int hclge_dbg_read_cmd(struct hnae3_handle *handle, enum hnae3_dbg_cmd cmd, char *buf, int len) { struct hclge_vport *vport = hclge_get_vport(handle); + const struct hclge_dbg_func *cmd_func; struct hclge_dev *hdev = vport->back; + u32 i; - if (strncmp(cmd_buf, HNAE3_DBG_TM_NODES, - strlen(HNAE3_DBG_TM_NODES)) == 0) - return hclge_dbg_dump_tm_nodes(hdev, buf, len); - else if (strncmp(cmd_buf, HNAE3_DBG_TM_PRI, - strlen(HNAE3_DBG_TM_PRI)) == 0) - return hclge_dbg_dump_tm_pri(hdev, buf, len); - else if (strncmp(cmd_buf, HNAE3_DBG_TM_QSET, - strlen(HNAE3_DBG_TM_QSET)) == 0) - return hclge_dbg_dump_tm_qset(hdev, buf, len); + for (i = 0; i < ARRAY_SIZE(hclge_dbg_cmd_func); i++) { + if (cmd == hclge_dbg_cmd_func[i].cmd) { + cmd_func = &hclge_dbg_cmd_func[i]; + if (cmd_func->dbg_dump) + return cmd_func->dbg_dump(hdev, buf, len); + else + return cmd_func->dbg_dump_reg(hdev, cmd, buf, + len); + } + } + dev_err(&hdev->pdev->dev, "invalid command(%d)\n", cmd); return -EINVAL; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h index ca2ab6cf84d9..c526591a7240 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h @@ -7,7 +7,6 @@ #include <linux/etherdevice.h> #include "hclge_cmd.h" -#define HCLGE_DBG_BUF_LEN 256 #define HCLGE_DBG_MNG_TBL_MAX 64 #define HCLGE_DBG_MNG_VLAN_MASK_B BIT(0) @@ -70,6 +69,11 @@ struct hclge_dbg_reg_common_msg { enum hclge_opcode_type cmd; }; +struct hclge_dbg_tcam_msg { + u8 stage; + u32 loc; +}; + #define HCLGE_DBG_MAX_DFX_MSG_LEN 60 struct hclge_dbg_dfx_message { int flag; @@ -78,11 +82,18 @@ struct hclge_dbg_dfx_message { #define HCLGE_DBG_MAC_REG_TYPE_LEN 32 struct hclge_dbg_reg_type_info { - const char *reg_type; + enum hnae3_dbg_cmd cmd; const struct hclge_dbg_dfx_message *dfx_msg; struct hclge_dbg_reg_common_msg reg_msg; }; +struct hclge_dbg_func { + enum hnae3_dbg_cmd cmd; + int (*dbg_dump)(struct hclge_dev *hdev, char *buf, int len); + int (*dbg_dump_reg)(struct hclge_dev *hdev, enum hnae3_dbg_cmd cmd, + char *buf, int len); +}; + static const struct hclge_dbg_dfx_message hclge_dbg_bios_common_reg[] = { {false, "Reserved"}, {true, "BP_CPU_STATE"}, @@ -723,4 +734,36 @@ static const struct hclge_dbg_dfx_message hclge_dbg_tqp_reg[] = { {true, "RCB_CFG_TX_RING_EBDNUM"}, }; +#define HCLGE_DBG_INFO_LEN 256 +#define HCLGE_DBG_VLAN_FLTR_INFO_LEN 256 +#define HCLGE_DBG_VLAN_OFFLOAD_INFO_LEN 512 +#define HCLGE_DBG_ID_LEN 16 +#define HCLGE_DBG_ITEM_NAME_LEN 32 +#define HCLGE_DBG_DATA_STR_LEN 32 +#define HCLGE_DBG_TM_INFO_LEN 256 + +#define HCLGE_BILLION_NANO_SECONDS 1000000000 + +struct hclge_dbg_item { + char name[HCLGE_DBG_ITEM_NAME_LEN]; + u16 interval; /* blank numbers after the item */ +}; + +struct hclge_dbg_vlan_cfg { + u16 pvid; + u8 accept_tag1; + u8 accept_tag2; + u8 accept_untag1; + u8 accept_untag2; + u8 insert_tag1; + u8 insert_tag2; + u8 shift_tag; + u8 strip_tag1; + u8 strip_tag2; + u8 drop_tag1; + u8 drop_tag2; + u8 pri_only1; + u8 pri_only2; +}; + #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c index 8223d699cd94..ec9a7f8bc3fe 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.c @@ -631,6 +631,134 @@ static const struct hclge_hw_error hclge_rocee_qmm_ovf_err_int[] = { { /* sentinel */ } }; +static const struct hclge_hw_module_id hclge_hw_module_id_st[] = { + { + .module_id = MODULE_NONE, + .msg = "MODULE_NONE" + }, { + .module_id = MODULE_BIOS_COMMON, + .msg = "MODULE_BIOS_COMMON" + }, { + .module_id = MODULE_GE, + .msg = "MODULE_GE" + }, { + .module_id = MODULE_IGU_EGU, + .msg = "MODULE_IGU_EGU" + }, { + .module_id = MODULE_LGE, + .msg = "MODULE_LGE" + }, { + .module_id = MODULE_NCSI, + .msg = "MODULE_NCSI" + }, { + .module_id = MODULE_PPP, + .msg = "MODULE_PPP" + }, { + .module_id = MODULE_QCN, + .msg = "MODULE_QCN" + }, { + .module_id = MODULE_RCB_RX, + .msg = "MODULE_RCB_RX" + }, { + .module_id = MODULE_RTC, + .msg = "MODULE_RTC" + }, { + .module_id = MODULE_SSU, + .msg = "MODULE_SSU" + }, { + .module_id = MODULE_TM, + .msg = "MODULE_TM" + }, { + .module_id = MODULE_RCB_TX, + .msg = "MODULE_RCB_TX" + }, { + .module_id = MODULE_TXDMA, + .msg = "MODULE_TXDMA" + }, { + .module_id = MODULE_MASTER, + .msg = "MODULE_MASTER" + }, { + .module_id = MODULE_ROCEE_TOP, + .msg = "MODULE_ROCEE_TOP" + }, { + .module_id = MODULE_ROCEE_TIMER, + .msg = "MODULE_ROCEE_TIMER" + }, { + .module_id = MODULE_ROCEE_MDB, + .msg = "MODULE_ROCEE_MDB" + }, { + .module_id = MODULE_ROCEE_TSP, + .msg = "MODULE_ROCEE_TSP" + }, { + .module_id = MODULE_ROCEE_TRP, + .msg = "MODULE_ROCEE_TRP" + }, { + .module_id = MODULE_ROCEE_SCC, + .msg = "MODULE_ROCEE_SCC" + }, { + .module_id = MODULE_ROCEE_CAEP, + .msg = "MODULE_ROCEE_CAEP" + }, { + .module_id = MODULE_ROCEE_GEN_AC, + .msg = "MODULE_ROCEE_GEN_AC" + }, { + .module_id = MODULE_ROCEE_QMM, + .msg = "MODULE_ROCEE_QMM" + }, { + .module_id = MODULE_ROCEE_LSAN, + .msg = "MODULE_ROCEE_LSAN" + } +}; + +static const struct hclge_hw_type_id hclge_hw_type_id_st[] = { + { + .type_id = NONE_ERROR, + .msg = "none_error" + }, { + .type_id = FIFO_ERROR, + .msg = "fifo_error" + }, { + .type_id = MEMORY_ERROR, + .msg = "memory_error" + }, { + .type_id = POISON_ERROR, + .msg = "poison_error" + }, { + .type_id = MSIX_ECC_ERROR, + .msg = "msix_ecc_error" + }, { + .type_id = TQP_INT_ECC_ERROR, + .msg = "tqp_int_ecc_error" + }, { + .type_id = PF_ABNORMAL_INT_ERROR, + .msg = "pf_abnormal_int_error" + }, { + .type_id = MPF_ABNORMAL_INT_ERROR, + .msg = "mpf_abnormal_int_error" + }, { + .type_id = COMMON_ERROR, + .msg = "common_error" + }, { + .type_id = PORT_ERROR, + .msg = "port_error" + }, { + .type_id = ETS_ERROR, + .msg = "ets_error" + }, { + .type_id = NCSI_ERROR, + .msg = "ncsi_error" + }, { + .type_id = GLB_ERROR, + .msg = "glb_error" + }, { + .type_id = ROCEE_NORMAL_ERR, + .msg = "rocee_normal_error" + }, { + .type_id = ROCEE_OVF_ERR, + .msg = "rocee_ovf_error" + } +}; + static void hclge_log_error(struct device *dev, char *reg, const struct hclge_hw_error *err, u32 err_sts, unsigned long *reset_requests) @@ -1611,11 +1739,27 @@ static const struct hclge_hw_blk hw_blk[] = { { /* sentinel */ } }; +static void hclge_config_all_msix_error(struct hclge_dev *hdev, bool enable) +{ + u32 reg_val; + + reg_val = hclge_read_dev(&hdev->hw, HCLGE_PF_OTHER_INT_REG); + + if (enable) + reg_val |= BIT(HCLGE_VECTOR0_ALL_MSIX_ERR_B); + else + reg_val &= ~BIT(HCLGE_VECTOR0_ALL_MSIX_ERR_B); + + hclge_write_dev(&hdev->hw, HCLGE_PF_OTHER_INT_REG, reg_val); +} + int hclge_config_nic_hw_error(struct hclge_dev *hdev, bool state) { const struct hclge_hw_blk *module = hw_blk; int ret = 0; + hclge_config_all_msix_error(hdev, state); + while (module->name) { if (module->config_err_int) { ret = module->config_err_int(hdev, state); @@ -1876,11 +2020,8 @@ static int hclge_handle_pf_msix_error(struct hclge_dev *hdev, static int hclge_handle_all_hw_msix_error(struct hclge_dev *hdev, unsigned long *reset_requests) { - struct hclge_mac_tnl_stats mac_tnl_stats; - struct device *dev = &hdev->pdev->dev; u32 mpf_bd_num, pf_bd_num, bd_num; struct hclge_desc *desc; - u32 status; int ret; /* query the number of bds for the MSIx int status */ @@ -1903,16 +2044,45 @@ static int hclge_handle_all_hw_msix_error(struct hclge_dev *hdev, if (ret) goto msi_error; + ret = hclge_handle_mac_tnl(hdev); + +msi_error: + kfree(desc); +out: + return ret; +} + +int hclge_handle_hw_msix_error(struct hclge_dev *hdev, + unsigned long *reset_requests) +{ + struct device *dev = &hdev->pdev->dev; + + if (!test_bit(HCLGE_STATE_SERVICE_INITED, &hdev->state)) { + dev_err(dev, + "failed to handle msix error during dev init\n"); + return -EAGAIN; + } + + return hclge_handle_all_hw_msix_error(hdev, reset_requests); +} + +int hclge_handle_mac_tnl(struct hclge_dev *hdev) +{ + struct hclge_mac_tnl_stats mac_tnl_stats; + struct device *dev = &hdev->pdev->dev; + struct hclge_desc desc; + u32 status; + int ret; + /* query and clear mac tnl interruptions */ - hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QUERY_MAC_TNL_INT, - true); - ret = hclge_cmd_send(&hdev->hw, &desc[0], 1); + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_MAC_TNL_INT, true); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { - dev_err(dev, "query mac tnl int cmd failed (%d)\n", ret); - goto msi_error; + dev_err(dev, "failed to query mac tnl int, ret = %d.\n", ret); + return ret; } - status = le32_to_cpu(desc->data[0]); + status = le32_to_cpu(desc.data[0]); if (status) { /* When mac tnl interrupt occurs, we record current time and * register status here in a fifo, then clear the status. So @@ -1924,33 +2094,15 @@ static int hclge_handle_all_hw_msix_error(struct hclge_dev *hdev, kfifo_put(&hdev->mac_tnl_log, mac_tnl_stats); ret = hclge_clear_mac_tnl_int(hdev); if (ret) - dev_err(dev, "clear mac tnl int failed (%d)\n", ret); + dev_err(dev, "failed to clear mac tnl int, ret = %d.\n", + ret); } -msi_error: - kfree(desc); -out: return ret; } -int hclge_handle_hw_msix_error(struct hclge_dev *hdev, - unsigned long *reset_requests) -{ - struct device *dev = &hdev->pdev->dev; - - if (!test_bit(HCLGE_STATE_SERVICE_INITED, &hdev->state)) { - dev_err(dev, - "Can't handle - MSIx error reported during dev init\n"); - return 0; - } - - return hclge_handle_all_hw_msix_error(hdev, reset_requests); -} - void hclge_handle_all_hns_hw_errors(struct hnae3_ae_dev *ae_dev) { -#define HCLGE_DESC_NO_DATA_LEN 8 - struct hclge_dev *hdev = ae_dev->priv; struct device *dev = &hdev->pdev->dev; u32 mpf_bd_num, pf_bd_num, bd_num; @@ -1999,3 +2151,207 @@ void hclge_handle_all_hns_hw_errors(struct hnae3_ae_dev *ae_dev) msi_error: kfree(desc); } + +bool hclge_find_error_source(struct hclge_dev *hdev) +{ + u32 msix_src_flag, hw_err_src_flag; + + msix_src_flag = hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_INT_STS) & + HCLGE_VECTOR0_REG_MSIX_MASK; + + hw_err_src_flag = hclge_read_dev(&hdev->hw, + HCLGE_RAS_PF_OTHER_INT_STS_REG) & + HCLGE_RAS_REG_ERR_MASK; + + return msix_src_flag || hw_err_src_flag; +} + +void hclge_handle_occurred_error(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + + if (hclge_find_error_source(hdev)) + hclge_handle_error_info_log(ae_dev); +} + +static void +hclge_handle_error_type_reg_log(struct device *dev, + struct hclge_mod_err_info *mod_info, + struct hclge_type_reg_err_info *type_reg_info) +{ +#define HCLGE_ERR_TYPE_MASK 0x7F +#define HCLGE_ERR_TYPE_IS_RAS_OFFSET 7 + + u8 mod_id, total_module, type_id, total_type, i, is_ras; + u8 index_module = MODULE_NONE; + u8 index_type = NONE_ERROR; + + mod_id = mod_info->mod_id; + type_id = type_reg_info->type_id & HCLGE_ERR_TYPE_MASK; + is_ras = type_reg_info->type_id >> HCLGE_ERR_TYPE_IS_RAS_OFFSET; + + total_module = ARRAY_SIZE(hclge_hw_module_id_st); + total_type = ARRAY_SIZE(hclge_hw_type_id_st); + + for (i = 0; i < total_module; i++) { + if (mod_id == hclge_hw_module_id_st[i].module_id) { + index_module = i; + break; + } + } + + for (i = 0; i < total_type; i++) { + if (type_id == hclge_hw_type_id_st[i].type_id) { + index_type = i; + break; + } + } + + if (index_module != MODULE_NONE && index_type != NONE_ERROR) + dev_err(dev, + "found %s %s, is %s error.\n", + hclge_hw_module_id_st[index_module].msg, + hclge_hw_type_id_st[index_type].msg, + is_ras ? "ras" : "msix"); + else + dev_err(dev, + "unknown module[%u] or type[%u].\n", mod_id, type_id); + + dev_err(dev, "reg_value:\n"); + for (i = 0; i < type_reg_info->reg_num; i++) + dev_err(dev, "0x%08x\n", type_reg_info->hclge_reg[i]); +} + +static void hclge_handle_error_module_log(struct hnae3_ae_dev *ae_dev, + const u32 *buf, u32 buf_size) +{ + struct hclge_type_reg_err_info *type_reg_info; + struct hclge_dev *hdev = ae_dev->priv; + struct device *dev = &hdev->pdev->dev; + struct hclge_mod_err_info *mod_info; + struct hclge_sum_err_info *sum_info; + u8 mod_num, err_num, i; + u32 offset = 0; + + sum_info = (struct hclge_sum_err_info *)&buf[offset++]; + if (sum_info->reset_type && + sum_info->reset_type != HNAE3_NONE_RESET) + set_bit(sum_info->reset_type, &ae_dev->hw_err_reset_req); + mod_num = sum_info->mod_num; + + while (mod_num--) { + if (offset >= buf_size) { + dev_err(dev, "The offset(%u) exceeds buf's size(%u).\n", + offset, buf_size); + return; + } + mod_info = (struct hclge_mod_err_info *)&buf[offset++]; + err_num = mod_info->err_num; + + for (i = 0; i < err_num; i++) { + if (offset >= buf_size) { + dev_err(dev, + "The offset(%u) exceeds buf size(%u).\n", + offset, buf_size); + return; + } + + type_reg_info = (struct hclge_type_reg_err_info *) + &buf[offset++]; + hclge_handle_error_type_reg_log(dev, mod_info, + type_reg_info); + + offset += type_reg_info->reg_num; + } + } +} + +static int hclge_query_all_err_bd_num(struct hclge_dev *hdev, u32 *bd_num) +{ + struct device *dev = &hdev->pdev->dev; + struct hclge_desc desc_bd; + int ret; + + hclge_cmd_setup_basic_desc(&desc_bd, HCLGE_QUERY_ALL_ERR_BD_NUM, true); + ret = hclge_cmd_send(&hdev->hw, &desc_bd, 1); + if (ret) { + dev_err(dev, "failed to query error bd_num, ret = %d.\n", ret); + return ret; + } + + *bd_num = le32_to_cpu(desc_bd.data[0]); + if (!(*bd_num)) { + dev_err(dev, "The value of bd_num is 0!\n"); + return -EINVAL; + } + + return 0; +} + +static int hclge_query_all_err_info(struct hclge_dev *hdev, + struct hclge_desc *desc, u32 bd_num) +{ + struct device *dev = &hdev->pdev->dev; + int ret; + + hclge_cmd_setup_basic_desc(desc, HCLGE_QUERY_ALL_ERR_INFO, true); + ret = hclge_cmd_send(&hdev->hw, desc, bd_num); + if (ret) + dev_err(dev, "failed to query error info, ret = %d.\n", ret); + + return ret; +} + +int hclge_handle_error_info_log(struct hnae3_ae_dev *ae_dev) +{ + u32 bd_num, desc_len, buf_len, buf_size, i; + struct hclge_dev *hdev = ae_dev->priv; + struct hclge_desc *desc; + __le32 *desc_data; + u32 *buf; + int ret; + + ret = hclge_query_all_err_bd_num(hdev, &bd_num); + if (ret) + goto out; + + desc_len = bd_num * sizeof(struct hclge_desc); + desc = kzalloc(desc_len, GFP_KERNEL); + if (!desc) { + ret = -ENOMEM; + goto out; + } + + ret = hclge_query_all_err_info(hdev, desc, bd_num); + if (ret) + goto err_desc; + + buf_len = bd_num * sizeof(struct hclge_desc) - HCLGE_DESC_NO_DATA_LEN; + buf_size = buf_len / sizeof(u32); + + desc_data = kzalloc(buf_len, GFP_KERNEL); + if (!desc_data) { + ret = -ENOMEM; + goto err_desc; + } + + buf = kzalloc(buf_len, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_buf_alloc; + } + + memcpy(desc_data, &desc[0].data[0], buf_len); + for (i = 0; i < buf_size; i++) + buf[i] = le32_to_cpu(desc_data[i]); + + hclge_handle_error_module_log(ae_dev, buf, buf_size); + kfree(buf); + +err_buf_alloc: + kfree(desc_data); +err_desc: + kfree(desc); +out: + return ret; +} diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h index d647f3c84134..07987fb8332e 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_err.h @@ -15,6 +15,8 @@ #define HCLGE_RAS_PF_OTHER_INT_STS_REG 0x20B00 #define HCLGE_RAS_REG_NFE_MASK 0xFF00 #define HCLGE_RAS_REG_ROCEE_ERR_MASK 0x3000000 +#define HCLGE_RAS_REG_ERR_MASK \ + (HCLGE_RAS_REG_NFE_MASK | HCLGE_RAS_REG_ROCEE_ERR_MASK) #define HCLGE_VECTOR0_REG_MSIX_MASK 0x1FF00 @@ -107,6 +109,10 @@ #define HCLGE_ROCEE_OVF_ERR_INT_MASK 0x10000 #define HCLGE_ROCEE_OVF_ERR_TYPE_MASK 0x3F +#define HCLGE_DESC_DATA_MAX 8 +#define HCLGE_REG_NUM_MAX 256 +#define HCLGE_DESC_NO_DATA_LEN 8 + enum hclge_err_int_type { HCLGE_ERR_INT_MSIX = 0, HCLGE_ERR_INT_RAS_CE = 1, @@ -114,6 +120,56 @@ enum hclge_err_int_type { HCLGE_ERR_INT_RAS_FE = 3, }; +enum hclge_mod_name_list { + MODULE_NONE = 0, + MODULE_BIOS_COMMON = 1, + MODULE_GE = 2, + MODULE_IGU_EGU = 3, + MODULE_LGE = 4, + MODULE_NCSI = 5, + MODULE_PPP = 6, + MODULE_QCN = 7, + MODULE_RCB_RX = 8, + MODULE_RTC = 9, + MODULE_SSU = 10, + MODULE_TM = 11, + MODULE_RCB_TX = 12, + MODULE_TXDMA = 13, + MODULE_MASTER = 14, + /* add new MODULE NAME for NIC here in order */ + MODULE_ROCEE_TOP = 40, + MODULE_ROCEE_TIMER = 41, + MODULE_ROCEE_MDB = 42, + MODULE_ROCEE_TSP = 43, + MODULE_ROCEE_TRP = 44, + MODULE_ROCEE_SCC = 45, + MODULE_ROCEE_CAEP = 46, + MODULE_ROCEE_GEN_AC = 47, + MODULE_ROCEE_QMM = 48, + MODULE_ROCEE_LSAN = 49, + /* add new MODULE NAME for RoCEE here in order */ +}; + +enum hclge_err_type_list { + NONE_ERROR = 0, + FIFO_ERROR = 1, + MEMORY_ERROR = 2, + POISON_ERROR = 3, + MSIX_ECC_ERROR = 4, + TQP_INT_ECC_ERROR = 5, + PF_ABNORMAL_INT_ERROR = 6, + MPF_ABNORMAL_INT_ERROR = 7, + COMMON_ERROR = 8, + PORT_ERROR = 9, + ETS_ERROR = 10, + NCSI_ERROR = 11, + GLB_ERROR = 12, + /* add new ERROR TYPE for NIC here in order */ + ROCEE_NORMAL_ERR = 40, + ROCEE_OVF_ERR = 41, + /* add new ERROR TYPE for ROCEE here in order */ +}; + struct hclge_hw_blk { u32 msk; const char *name; @@ -126,11 +182,44 @@ struct hclge_hw_error { enum hnae3_reset_type reset_level; }; +struct hclge_hw_module_id { + enum hclge_mod_name_list module_id; + const char *msg; +}; + +struct hclge_hw_type_id { + enum hclge_err_type_list type_id; + const char *msg; +}; + +struct hclge_sum_err_info { + u8 reset_type; + u8 mod_num; + u8 rsv[2]; +}; + +struct hclge_mod_err_info { + u8 mod_id; + u8 err_num; + u8 rsv[2]; +}; + +struct hclge_type_reg_err_info { + u8 type_id; + u8 reg_num; + u8 rsv[2]; + u32 hclge_reg[HCLGE_REG_NUM_MAX]; +}; + int hclge_config_mac_tnl_int(struct hclge_dev *hdev, bool en); int hclge_config_nic_hw_error(struct hclge_dev *hdev, bool state); int hclge_config_rocee_ras_interrupt(struct hclge_dev *hdev, bool en); void hclge_handle_all_hns_hw_errors(struct hnae3_ae_dev *ae_dev); +bool hclge_find_error_source(struct hclge_dev *hdev); +void hclge_handle_occurred_error(struct hclge_dev *hdev); pci_ers_result_t hclge_handle_hw_ras_error(struct hnae3_ae_dev *ae_dev); int hclge_handle_hw_msix_error(struct hclge_dev *hdev, unsigned long *reset_requests); +int hclge_handle_error_info_log(struct hnae3_ae_dev *ae_dev); +int hclge_handle_mac_tnl(struct hclge_dev *hdev); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 6304aed49f22..dd3354a57c62 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -1279,6 +1279,7 @@ static u32 hclge_get_max_speed(u16 speed_ability) static void hclge_parse_cfg(struct hclge_cfg *cfg, struct hclge_desc *desc) { +#define HCLGE_TX_SPARE_SIZE_UNIT 4096 #define SPEED_ABILITY_EXT_SHIFT 8 struct hclge_cfg_param_cmd *req; @@ -1334,6 +1335,10 @@ static void hclge_parse_cfg(struct hclge_cfg *cfg, struct hclge_desc *desc) HCLGE_CFG_SPEED_ABILITY_EXT_S); cfg->speed_ability |= speed_ability_ext << SPEED_ABILITY_EXT_SHIFT; + cfg->vlan_fliter_cap = hnae3_get_field(__le32_to_cpu(req->param[1]), + HCLGE_CFG_VLAN_FLTR_CAP_M, + HCLGE_CFG_VLAN_FLTR_CAP_S); + cfg->umv_space = hnae3_get_field(__le32_to_cpu(req->param[1]), HCLGE_CFG_UMV_TBL_SPACE_M, HCLGE_CFG_UMV_TBL_SPACE_S); @@ -1354,6 +1359,15 @@ static void hclge_parse_cfg(struct hclge_cfg *cfg, struct hclge_desc *desc) cfg->pf_rss_size_max = cfg->pf_rss_size_max ? 1U << cfg->pf_rss_size_max : cfg->vf_rss_size_max; + + /* The unit of the tx spare buffer size queried from configuration + * file is HCLGE_TX_SPARE_SIZE_UNIT(4096) bytes, so a conversion is + * needed here. + */ + cfg->tx_spare_buf_size = hnae3_get_field(__le32_to_cpu(req->param[2]), + HCLGE_CFG_TX_SPARE_BUF_SIZE_M, + HCLGE_CFG_TX_SPARE_BUF_SIZE_S); + cfg->tx_spare_buf_size *= HCLGE_TX_SPARE_SIZE_UNIT; } /* hclge_get_cfg: query the static parameter from flash @@ -1513,6 +1527,7 @@ static void hclge_init_kdump_kernel_config(struct hclge_dev *hdev) static int hclge_configure(struct hclge_dev *hdev) { + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); struct hclge_cfg cfg; unsigned int i; int ret; @@ -1534,6 +1549,9 @@ static int hclge_configure(struct hclge_dev *hdev) hdev->tc_max = cfg.tc_num; hdev->tm_info.hw_pfc_map = 0; hdev->wanted_umv_size = cfg.umv_space; + hdev->tx_spare_buf_size = cfg.tx_spare_buf_size; + if (cfg.vlan_fliter_cap == HCLGE_VLAN_FLTR_CAN_MDF) + set_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps); if (hnae3_dev_fd_supported(hdev)) { hdev->fd_en = true; @@ -1729,6 +1747,7 @@ static int hclge_knic_setup(struct hclge_vport *vport, u16 num_tqps, kinfo->num_rx_desc = num_rx_desc; kinfo->rx_buf_len = hdev->rx_buf_len; + kinfo->tx_spare_buf_size = hdev->tx_spare_buf_size; kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, num_tqps, sizeof(struct hnae3_queue *), GFP_KERNEL); @@ -1843,6 +1862,7 @@ static int hclge_alloc_vport(struct hclge_dev *hdev) vport->mps = HCLGE_MAC_DEFAULT_FRAME; vport->port_base_vlan_cfg.state = HNAE3_PORT_BASE_VLAN_DISABLE; vport->rxvlan_cfg.rx_vlan_offload_en = true; + vport->req_vlan_fltr_en = true; INIT_LIST_HEAD(&vport->vlan_list); INIT_LIST_HEAD(&vport->uc_mac_list); INIT_LIST_HEAD(&vport->mc_mac_list); @@ -2835,6 +2855,14 @@ static void hclge_reset_task_schedule(struct hclge_dev *hdev) hclge_wq, &hdev->service_task, 0); } +static void hclge_errhand_task_schedule(struct hclge_dev *hdev) +{ + if (!test_bit(HCLGE_STATE_REMOVING, &hdev->state) && + !test_and_set_bit(HCLGE_STATE_ERR_SERVICE_SCHED, &hdev->state)) + mod_delayed_work_on(cpumask_first(&hdev->affinity_mask), + hclge_wq, &hdev->service_task, 0); +} + void hclge_task_schedule(struct hclge_dev *hdev, unsigned long delay_time) { if (!test_bit(HCLGE_STATE_REMOVING, &hdev->state) && @@ -3291,11 +3319,13 @@ static int hclge_set_vf_link_state(struct hnae3_handle *handle, int vf, static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval) { - u32 cmdq_src_reg, msix_src_reg; + u32 cmdq_src_reg, msix_src_reg, hw_err_src_reg; /* fetch the events from their corresponding regs */ cmdq_src_reg = hclge_read_dev(&hdev->hw, HCLGE_VECTOR0_CMDQ_SRC_REG); msix_src_reg = hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_INT_STS); + hw_err_src_reg = hclge_read_dev(&hdev->hw, + HCLGE_RAS_PF_OTHER_INT_STS_REG); /* Assumption: If by any chance reset and mailbox events are reported * together then we will only process reset event in this go and will @@ -3323,10 +3353,15 @@ static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval) return HCLGE_VECTOR0_EVENT_RST; } - /* check for vector0 msix event source */ - if (msix_src_reg & HCLGE_VECTOR0_REG_MSIX_MASK) { - *clearval = msix_src_reg; + /* check for vector0 msix event and hardware error event source */ + if (msix_src_reg & HCLGE_VECTOR0_REG_MSIX_MASK || + hw_err_src_reg & HCLGE_RAS_REG_ERR_MASK) return HCLGE_VECTOR0_EVENT_ERR; + + /* check for vector0 ptp event source */ + if (BIT(HCLGE_VECTOR0_REG_PTP_INT_B) & msix_src_reg) { + *clearval = msix_src_reg; + return HCLGE_VECTOR0_EVENT_PTP; } /* check for vector0 mailbox(=CMDQ RX) event source */ @@ -3338,9 +3373,8 @@ static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval) /* print other vector0 event source */ dev_info(&hdev->pdev->dev, - "CMDQ INT status:0x%x, other INT status:0x%x\n", - cmdq_src_reg, msix_src_reg); - *clearval = msix_src_reg; + "INT status: CMDQ(%#x) HW errors(%#x) other(%#x)\n", + cmdq_src_reg, hw_err_src_reg, msix_src_reg); return HCLGE_VECTOR0_EVENT_OTHER; } @@ -3349,6 +3383,7 @@ static void hclge_clear_event_cause(struct hclge_dev *hdev, u32 event_type, u32 regclr) { switch (event_type) { + case HCLGE_VECTOR0_EVENT_PTP: case HCLGE_VECTOR0_EVENT_RST: hclge_write_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG, regclr); break; @@ -3377,6 +3412,7 @@ static void hclge_enable_vector(struct hclge_misc_vector *vector, bool enable) static irqreturn_t hclge_misc_irq_handle(int irq, void *data) { struct hclge_dev *hdev = data; + unsigned long flags; u32 clearval = 0; u32 event_cause; @@ -3386,21 +3422,16 @@ static irqreturn_t hclge_misc_irq_handle(int irq, void *data) /* vector 0 interrupt is shared with reset and mailbox source events.*/ switch (event_cause) { case HCLGE_VECTOR0_EVENT_ERR: - /* we do not know what type of reset is required now. This could - * only be decided after we fetch the type of errors which - * caused this event. Therefore, we will do below for now: - * 1. Assert HNAE3_UNKNOWN_RESET type of reset. This means we - * have defered type of reset to be used. - * 2. Schedule the reset service task. - * 3. When service task receives HNAE3_UNKNOWN_RESET type it - * will fetch the correct type of reset. This would be done - * by first decoding the types of errors. - */ - set_bit(HNAE3_UNKNOWN_RESET, &hdev->reset_request); - fallthrough; + hclge_errhand_task_schedule(hdev); + break; case HCLGE_VECTOR0_EVENT_RST: hclge_reset_task_schedule(hdev); break; + case HCLGE_VECTOR0_EVENT_PTP: + spin_lock_irqsave(&hdev->ptp->lock, flags); + hclge_ptp_clean_tx_hwts(hdev); + spin_unlock_irqrestore(&hdev->ptp->lock, flags); + break; case HCLGE_VECTOR0_EVENT_MBX: /* If we are here then, * 1. Either we are not handling any mbx task and we are not @@ -3421,15 +3452,11 @@ static irqreturn_t hclge_misc_irq_handle(int irq, void *data) hclge_clear_event_cause(hdev, event_cause, clearval); - /* Enable interrupt if it is not cause by reset. And when - * clearval equal to 0, it means interrupt status may be - * cleared by hardware before driver reads status register. - * For this case, vector0 interrupt also should be enabled. - */ - if (!clearval || - event_cause == HCLGE_VECTOR0_EVENT_MBX) { + /* Enable interrupt if it is not caused by reset event or error event */ + if (event_cause == HCLGE_VECTOR0_EVENT_PTP || + event_cause == HCLGE_VECTOR0_EVENT_MBX || + event_cause == HCLGE_VECTOR0_EVENT_OTHER) hclge_enable_vector(&hdev->misc_vector, true); - } return IRQ_HANDLED; } @@ -3786,28 +3813,6 @@ static enum hnae3_reset_type hclge_get_reset_level(struct hnae3_ae_dev *ae_dev, enum hnae3_reset_type rst_level = HNAE3_NONE_RESET; struct hclge_dev *hdev = ae_dev->priv; - /* first, resolve any unknown reset type to the known type(s) */ - if (test_bit(HNAE3_UNKNOWN_RESET, addr)) { - u32 msix_sts_reg = hclge_read_dev(&hdev->hw, - HCLGE_MISC_VECTOR_INT_STS); - /* we will intentionally ignore any errors from this function - * as we will end up in *some* reset request in any case - */ - if (hclge_handle_hw_msix_error(hdev, addr)) - dev_info(&hdev->pdev->dev, "received msix interrupt 0x%x\n", - msix_sts_reg); - - clear_bit(HNAE3_UNKNOWN_RESET, addr); - /* We defered the clearing of the error event which caused - * interrupt since it was not posssible to do that in - * interrupt context (and this is the reason we introduced - * new UNKNOWN reset type). Now, the errors have been - * handled and cleared in hardware we can safely enable - * interrupts. This is an exception to the norm. - */ - hclge_enable_vector(&hdev->misc_vector, true); - } - /* return the highest priority reset level amongst all */ if (test_bit(HNAE3_IMP_RESET, addr)) { rst_level = HNAE3_IMP_RESET; @@ -3936,6 +3941,21 @@ static int hclge_reset_prepare_wait(struct hclge_dev *hdev) return ret; } +static void hclge_show_rst_info(struct hclge_dev *hdev) +{ + char *buf; + + buf = kzalloc(HCLGE_DBG_RESET_INFO_LEN, GFP_KERNEL); + if (!buf) + return; + + hclge_dbg_dump_rst_info(hdev, buf, HCLGE_DBG_RESET_INFO_LEN); + + dev_info(&hdev->pdev->dev, "dump reset info:\n%s", buf); + + kfree(buf); +} + static bool hclge_reset_err_handle(struct hclge_dev *hdev) { #define MAX_RESET_FAIL_CNT 5 @@ -3966,7 +3986,7 @@ static bool hclge_reset_err_handle(struct hclge_dev *hdev) dev_err(&hdev->pdev->dev, "Reset fail!\n"); - hclge_dbg_dump_rst_info(hdev); + hclge_show_rst_info(hdev); set_bit(HCLGE_STATE_RST_FAIL, &hdev->state); @@ -4241,6 +4261,68 @@ static void hclge_reset_subtask(struct hclge_dev *hdev) hdev->reset_type = HNAE3_NONE_RESET; } +static void hclge_handle_err_reset_request(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + enum hnae3_reset_type reset_type; + + if (ae_dev->hw_err_reset_req) { + reset_type = hclge_get_reset_level(ae_dev, + &ae_dev->hw_err_reset_req); + hclge_set_def_reset_request(ae_dev, reset_type); + } + + if (hdev->default_reset_request && ae_dev->ops->reset_event) + ae_dev->ops->reset_event(hdev->pdev, NULL); + + /* enable interrupt after error handling complete */ + hclge_enable_vector(&hdev->misc_vector, true); +} + +static void hclge_handle_err_recovery(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + + ae_dev->hw_err_reset_req = 0; + + if (hclge_find_error_source(hdev)) { + hclge_handle_error_info_log(ae_dev); + hclge_handle_mac_tnl(hdev); + } + + hclge_handle_err_reset_request(hdev); +} + +static void hclge_misc_err_recovery(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + struct device *dev = &hdev->pdev->dev; + u32 msix_sts_reg; + + msix_sts_reg = hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_INT_STS); + if (msix_sts_reg & HCLGE_VECTOR0_REG_MSIX_MASK) { + if (hclge_handle_hw_msix_error + (hdev, &hdev->default_reset_request)) + dev_info(dev, "received msix interrupt 0x%x\n", + msix_sts_reg); + } + + hclge_handle_hw_ras_error(ae_dev); + + hclge_handle_err_reset_request(hdev); +} + +static void hclge_errhand_service_task(struct hclge_dev *hdev) +{ + if (!test_and_clear_bit(HCLGE_STATE_ERR_SERVICE_SCHED, &hdev->state)) + return; + + if (hnae3_dev_ras_imp_supported(hdev)) + hclge_handle_err_recovery(hdev); + else + hclge_misc_err_recovery(hdev); +} + static void hclge_reset_service_task(struct hclge_dev *hdev) { if (!test_and_clear_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state)) @@ -4319,19 +4401,43 @@ out: hclge_task_schedule(hdev, delta); } +static void hclge_ptp_service_task(struct hclge_dev *hdev) +{ + unsigned long flags; + + if (!test_bit(HCLGE_STATE_PTP_EN, &hdev->state) || + !test_bit(HCLGE_STATE_PTP_TX_HANDLING, &hdev->state) || + !time_is_before_jiffies(hdev->ptp->tx_start + HZ)) + return; + + /* to prevent concurrence with the irq handler */ + spin_lock_irqsave(&hdev->ptp->lock, flags); + + /* check HCLGE_STATE_PTP_TX_HANDLING here again, since the irq + * handler may handle it just before spin_lock_irqsave(). + */ + if (test_bit(HCLGE_STATE_PTP_TX_HANDLING, &hdev->state)) + hclge_ptp_clean_tx_hwts(hdev); + + spin_unlock_irqrestore(&hdev->ptp->lock, flags); +} + static void hclge_service_task(struct work_struct *work) { struct hclge_dev *hdev = container_of(work, struct hclge_dev, service_task.work); + hclge_errhand_service_task(hdev); hclge_reset_service_task(hdev); + hclge_ptp_service_task(hdev); hclge_mailbox_service_task(hdev); hclge_periodic_service_task(hdev); - /* Handle reset and mbx again in case periodical task delays the - * handling by calling hclge_task_schedule() in + /* Handle error recovery, reset and mbx again in case periodical task + * delays the handling by calling hclge_task_schedule() in * hclge_periodic_service_task(). */ + hclge_errhand_service_task(hdev); hclge_reset_service_task(hdev); hclge_mailbox_service_task(hdev); } @@ -5168,9 +5274,8 @@ static int hclge_set_promisc_mode(struct hnae3_handle *handle, bool en_uc_pmc, static void hclge_request_update_promisc_mode(struct hnae3_handle *handle) { struct hclge_vport *vport = hclge_get_vport(handle); - struct hclge_dev *hdev = vport->back; - set_bit(HCLGE_STATE_PROMISC_CHANGED, &hdev->state); + set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); } static void hclge_sync_fd_state(struct hclge_dev *hdev) @@ -5895,8 +6000,14 @@ static int hclge_config_action(struct hclge_dev *hdev, u8 stage, ad_data.queue_id = rule->queue_id; } - ad_data.use_counter = false; - ad_data.counter_id = 0; + if (hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]) { + ad_data.use_counter = true; + ad_data.counter_id = rule->vf_id % + hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]; + } else { + ad_data.use_counter = false; + ad_data.counter_id = 0; + } ad_data.use_next_stage = false; ad_data.next_input_key = 0; @@ -8035,6 +8146,7 @@ int hclge_vport_start(struct hclge_vport *vport) struct hclge_dev *hdev = vport->back; set_bit(HCLGE_VPORT_STATE_ALIVE, &vport->state); + set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); vport->last_active_jiffies = jiffies; if (test_bit(vport->vport_id, hdev->vport_config_block)) { @@ -8776,8 +8888,7 @@ static bool hclge_sync_from_add_list(struct list_head *add_list, kfree(mac_node); } else if (mac_node->state == HCLGE_MAC_ACTIVE) { mac_node->state = HCLGE_MAC_TO_DEL; - list_del(&mac_node->node); - list_add_tail(&mac_node->node, mac_list); + list_move_tail(&mac_node->node, mac_list); } else { list_del(&mac_node->node); kfree(mac_node); @@ -8806,8 +8917,7 @@ static void hclge_sync_from_del_list(struct list_head *del_list, list_del(&mac_node->node); kfree(mac_node); } else { - list_del(&mac_node->node); - list_add_tail(&mac_node->node, mac_list); + list_move_tail(&mac_node->node, mac_list); } } } @@ -8851,8 +8961,7 @@ static void hclge_sync_vport_mac_table(struct hclge_vport *vport, list_for_each_entry_safe(mac_node, tmp, list, node) { switch (mac_node->state) { case HCLGE_MAC_TO_DEL: - list_del(&mac_node->node); - list_add_tail(&mac_node->node, &tmp_del_list); + list_move_tail(&mac_node->node, &tmp_del_list); break; case HCLGE_MAC_TO_ADD: new_node = kzalloc(sizeof(*new_node), GFP_ATOMIC); @@ -8934,8 +9043,7 @@ static void hclge_build_del_list(struct list_head *list, switch (mac_cfg->state) { case HCLGE_MAC_TO_DEL: case HCLGE_MAC_ACTIVE: - list_del(&mac_cfg->node); - list_add_tail(&mac_cfg->node, tmp_del_list); + list_move_tail(&mac_cfg->node, tmp_del_list); break; case HCLGE_MAC_TO_ADD: if (is_del_list) { @@ -9030,8 +9138,7 @@ static void hclge_uninit_vport_mac_list(struct hclge_vport *vport, switch (mac_node->state) { case HCLGE_MAC_TO_DEL: case HCLGE_MAC_ACTIVE: - list_del(&mac_node->node); - list_add_tail(&mac_node->node, &tmp_del_list); + list_move_tail(&mac_node->node, &tmp_del_list); break; case HCLGE_MAC_TO_ADD: list_del(&mac_node->node); @@ -9360,12 +9467,41 @@ static int hclge_do_ioctl(struct hnae3_handle *handle, struct ifreq *ifr, struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - if (!hdev->hw.mac.phydev) - return hclge_mii_ioctl(hdev, ifr, cmd); + switch (cmd) { + case SIOCGHWTSTAMP: + return hclge_ptp_get_cfg(hdev, ifr); + case SIOCSHWTSTAMP: + return hclge_ptp_set_cfg(hdev, ifr); + default: + if (!hdev->hw.mac.phydev) + return hclge_mii_ioctl(hdev, ifr, cmd); + } return phy_mii_ioctl(hdev->hw.mac.phydev, ifr, cmd); } +static int hclge_set_port_vlan_filter_bypass(struct hclge_dev *hdev, u8 vf_id, + bool bypass_en) +{ + struct hclge_port_vlan_filter_bypass_cmd *req; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PORT_VLAN_BYPASS, false); + req = (struct hclge_port_vlan_filter_bypass_cmd *)desc.data; + req->vf_id = vf_id; + hnae3_set_bit(req->bypass_state, HCLGE_INGRESS_BYPASS_B, + bypass_en ? 1 : 0); + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to set vport%u port vlan filter bypass state, ret = %d.\n", + vf_id, ret); + + return ret; +} + static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type, u8 fe_type, bool filter_en, u8 vf_id) { @@ -9399,37 +9535,99 @@ static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type, return ret; } -#define HCLGE_FILTER_TYPE_VF 0 -#define HCLGE_FILTER_TYPE_PORT 1 -#define HCLGE_FILTER_FE_EGRESS_V1_B BIT(0) -#define HCLGE_FILTER_FE_NIC_INGRESS_B BIT(0) -#define HCLGE_FILTER_FE_NIC_EGRESS_B BIT(1) -#define HCLGE_FILTER_FE_ROCE_INGRESS_B BIT(2) -#define HCLGE_FILTER_FE_ROCE_EGRESS_B BIT(3) -#define HCLGE_FILTER_FE_EGRESS (HCLGE_FILTER_FE_NIC_EGRESS_B \ - | HCLGE_FILTER_FE_ROCE_EGRESS_B) -#define HCLGE_FILTER_FE_INGRESS (HCLGE_FILTER_FE_NIC_INGRESS_B \ - | HCLGE_FILTER_FE_ROCE_INGRESS_B) +static int hclge_set_vport_vlan_filter(struct hclge_vport *vport, bool enable) +{ + struct hclge_dev *hdev = vport->back; + struct hnae3_ae_dev *ae_dev = hdev->ae_dev; + int ret; + + if (hdev->ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2) + return hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, + HCLGE_FILTER_FE_EGRESS_V1_B, + enable, vport->vport_id); + + ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, + HCLGE_FILTER_FE_EGRESS, enable, + vport->vport_id); + if (ret) + return ret; + + if (test_bit(HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B, ae_dev->caps)) + ret = hclge_set_port_vlan_filter_bypass(hdev, vport->vport_id, + !enable); + else if (!vport->vport_id) + ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, + HCLGE_FILTER_FE_INGRESS, + enable, 0); + + return ret; +} -static void hclge_enable_vlan_filter(struct hnae3_handle *handle, bool enable) +static bool hclge_need_enable_vport_vlan_filter(struct hclge_vport *vport) { - struct hclge_vport *vport = hclge_get_vport(handle); + struct hnae3_handle *handle = &vport->nic; + struct hclge_vport_vlan_cfg *vlan, *tmp; struct hclge_dev *hdev = vport->back; - if (hdev->ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) { - hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, - HCLGE_FILTER_FE_EGRESS, enable, 0); - hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, - HCLGE_FILTER_FE_INGRESS, enable, 0); - } else { - hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, - HCLGE_FILTER_FE_EGRESS_V1_B, enable, - 0); + if (vport->vport_id) { + if (vport->port_base_vlan_cfg.state != + HNAE3_PORT_BASE_VLAN_DISABLE) + return true; + + if (vport->vf_info.trusted && vport->vf_info.request_uc_en) + return false; + } else if (handle->netdev_flags & HNAE3_USER_UPE) { + return false; } - if (enable) - handle->netdev_flags |= HNAE3_VLAN_FLTR; - else - handle->netdev_flags &= ~HNAE3_VLAN_FLTR; + + if (!vport->req_vlan_fltr_en) + return false; + + /* compatible with former device, always enable vlan filter */ + if (!test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, hdev->ae_dev->caps)) + return true; + + list_for_each_entry_safe(vlan, tmp, &vport->vlan_list, node) + if (vlan->vlan_id != 0) + return true; + + return false; +} + +int hclge_enable_vport_vlan_filter(struct hclge_vport *vport, bool request_en) +{ + struct hclge_dev *hdev = vport->back; + bool need_en; + int ret; + + mutex_lock(&hdev->vport_lock); + + vport->req_vlan_fltr_en = request_en; + + need_en = hclge_need_enable_vport_vlan_filter(vport); + if (need_en == vport->cur_vlan_fltr_en) { + mutex_unlock(&hdev->vport_lock); + return 0; + } + + ret = hclge_set_vport_vlan_filter(vport, need_en); + if (ret) { + mutex_unlock(&hdev->vport_lock); + return ret; + } + + vport->cur_vlan_fltr_en = need_en; + + mutex_unlock(&hdev->vport_lock); + + return 0; +} + +static int hclge_enable_vlan_filter(struct hnae3_handle *handle, bool enable) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + + return hclge_enable_vport_vlan_filter(vport, enable); } static int hclge_set_vf_vlan_filter_cmd(struct hclge_dev *hdev, u16 vfid, @@ -9709,7 +9907,7 @@ static int hclge_set_vlan_rx_offload_cfg(struct hclge_vport *vport) static int hclge_vlan_offload_cfg(struct hclge_vport *vport, u16 port_base_vlan_state, - u16 vlan_tag) + u16 vlan_tag, u8 qos) { int ret; @@ -9723,7 +9921,8 @@ static int hclge_vlan_offload_cfg(struct hclge_vport *vport, vport->txvlan_cfg.accept_tag1 = ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3; vport->txvlan_cfg.insert_tag1_en = true; - vport->txvlan_cfg.default_tag1 = vlan_tag; + vport->txvlan_cfg.default_tag1 = (qos << VLAN_PRIO_SHIFT) | + vlan_tag; } vport->txvlan_cfg.accept_untag1 = true; @@ -9822,6 +10021,7 @@ static int hclge_init_vlan_config(struct hclge_dev *hdev) vport->vport_id); if (ret) return ret; + vport->cur_vlan_fltr_en = true; } ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, @@ -9837,8 +10037,6 @@ static int hclge_init_vlan_config(struct hclge_dev *hdev) return ret; } - handle->netdev_flags |= HNAE3_VLAN_FLTR; - hdev->vlan_type_cfg.rx_in_fst_vlan_type = HCLGE_DEF_VLAN_TYPE; hdev->vlan_type_cfg.rx_in_sec_vlan_type = HCLGE_DEF_VLAN_TYPE; hdev->vlan_type_cfg.rx_ot_fst_vlan_type = HCLGE_DEF_VLAN_TYPE; @@ -9852,13 +10050,15 @@ static int hclge_init_vlan_config(struct hclge_dev *hdev) for (i = 0; i < hdev->num_alloc_vport; i++) { u16 vlan_tag; + u8 qos; vport = &hdev->vport[i]; vlan_tag = vport->port_base_vlan_cfg.vlan_info.vlan_tag; + qos = vport->port_base_vlan_cfg.vlan_info.qos; ret = hclge_vlan_offload_cfg(vport, vport->port_base_vlan_cfg.state, - vlan_tag); + vlan_tag, qos); if (ret) return ret; } @@ -10033,7 +10233,6 @@ static void hclge_restore_hw_table(struct hclge_dev *hdev) hclge_restore_mac_table_common(vport); hclge_restore_vport_vlan_table(vport); - set_bit(HCLGE_STATE_PROMISC_CHANGED, &hdev->state); set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state); hclge_restore_fd_entries(handle); } @@ -10060,6 +10259,14 @@ int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable) return hclge_set_vlan_rx_offload_cfg(vport); } +static void hclge_set_vport_vlan_fltr_change(struct hclge_vport *vport) +{ + struct hclge_dev *hdev = vport->back; + + if (test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, hdev->ae_dev->caps)) + set_bit(HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, &vport->state); +} + static int hclge_update_vlan_filter_entries(struct hclge_vport *vport, u16 port_base_vlan_state, struct hclge_vlan_info *new_info, @@ -10070,6 +10277,10 @@ static int hclge_update_vlan_filter_entries(struct hclge_vport *vport, if (port_base_vlan_state == HNAE3_PORT_BASE_VLAN_ENABLE) { hclge_rm_vport_all_vlan_table(vport, false); + /* force clear VLAN 0 */ + ret = hclge_set_vf_vlan_common(hdev, vport->vport_id, true, 0); + if (ret) + return ret; return hclge_set_vlan_filter_hw(hdev, htons(new_info->vlan_proto), vport->vport_id, @@ -10077,6 +10288,11 @@ static int hclge_update_vlan_filter_entries(struct hclge_vport *vport, false); } + /* force add VLAN 0 */ + ret = hclge_set_vf_vlan_common(hdev, vport->vport_id, false, 0); + if (ret) + return ret; + ret = hclge_set_vlan_filter_hw(hdev, htons(old_info->vlan_proto), vport->vport_id, old_info->vlan_tag, true); @@ -10086,6 +10302,18 @@ static int hclge_update_vlan_filter_entries(struct hclge_vport *vport, return hclge_add_vport_all_vlan_table(vport); } +static bool hclge_need_update_vlan_filter(const struct hclge_vlan_info *new_cfg, + const struct hclge_vlan_info *old_cfg) +{ + if (new_cfg->vlan_tag != old_cfg->vlan_tag) + return true; + + if (new_cfg->vlan_tag == 0 && (new_cfg->qos == 0 || old_cfg->qos == 0)) + return true; + + return false; +} + int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, struct hclge_vlan_info *vlan_info) { @@ -10096,10 +10324,14 @@ int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, old_vlan_info = &vport->port_base_vlan_cfg.vlan_info; - ret = hclge_vlan_offload_cfg(vport, state, vlan_info->vlan_tag); + ret = hclge_vlan_offload_cfg(vport, state, vlan_info->vlan_tag, + vlan_info->qos); if (ret) return ret; + if (!hclge_need_update_vlan_filter(vlan_info, old_vlan_info)) + goto out; + if (state == HNAE3_PORT_BASE_VLAN_MODIFY) { /* add new VLAN tag */ ret = hclge_set_vlan_filter_hw(hdev, @@ -10111,15 +10343,23 @@ int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, return ret; /* remove old VLAN tag */ - ret = hclge_set_vlan_filter_hw(hdev, - htons(old_vlan_info->vlan_proto), - vport->vport_id, - old_vlan_info->vlan_tag, - true); - if (ret) + if (old_vlan_info->vlan_tag == 0) + ret = hclge_set_vf_vlan_common(hdev, vport->vport_id, + true, 0); + else + ret = hclge_set_vlan_filter_hw(hdev, + htons(ETH_P_8021Q), + vport->vport_id, + old_vlan_info->vlan_tag, + true); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to clear vport%u port base vlan %u, ret = %d.\n", + vport->vport_id, old_vlan_info->vlan_tag, ret); return ret; + } - goto update; + goto out; } ret = hclge_update_vlan_filter_entries(vport, state, vlan_info, @@ -10127,38 +10367,38 @@ int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, if (ret) return ret; - /* update state only when disable/enable port based VLAN */ +out: vport->port_base_vlan_cfg.state = state; if (state == HNAE3_PORT_BASE_VLAN_DISABLE) nic->port_base_vlan_state = HNAE3_PORT_BASE_VLAN_DISABLE; else nic->port_base_vlan_state = HNAE3_PORT_BASE_VLAN_ENABLE; -update: - vport->port_base_vlan_cfg.vlan_info.vlan_tag = vlan_info->vlan_tag; - vport->port_base_vlan_cfg.vlan_info.qos = vlan_info->qos; - vport->port_base_vlan_cfg.vlan_info.vlan_proto = vlan_info->vlan_proto; + vport->port_base_vlan_cfg.vlan_info = *vlan_info; + hclge_set_vport_vlan_fltr_change(vport); return 0; } static u16 hclge_get_port_base_vlan_state(struct hclge_vport *vport, enum hnae3_port_base_vlan_state state, - u16 vlan) + u16 vlan, u8 qos) { if (state == HNAE3_PORT_BASE_VLAN_DISABLE) { - if (!vlan) - return HNAE3_PORT_BASE_VLAN_NOCHANGE; - else - return HNAE3_PORT_BASE_VLAN_ENABLE; - } else { - if (!vlan) - return HNAE3_PORT_BASE_VLAN_DISABLE; - else if (vport->port_base_vlan_cfg.vlan_info.vlan_tag == vlan) + if (!vlan && !qos) return HNAE3_PORT_BASE_VLAN_NOCHANGE; - else - return HNAE3_PORT_BASE_VLAN_MODIFY; + + return HNAE3_PORT_BASE_VLAN_ENABLE; } + + if (!vlan && !qos) + return HNAE3_PORT_BASE_VLAN_DISABLE; + + if (vport->port_base_vlan_cfg.vlan_info.vlan_tag == vlan && + vport->port_base_vlan_cfg.vlan_info.qos == qos) + return HNAE3_PORT_BASE_VLAN_NOCHANGE; + + return HNAE3_PORT_BASE_VLAN_MODIFY; } static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid, @@ -10186,7 +10426,7 @@ static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid, state = hclge_get_port_base_vlan_state(vport, vport->port_base_vlan_cfg.state, - vlan); + vlan, qos); if (state == HNAE3_PORT_BASE_VLAN_NOCHANGE) return 0; @@ -10209,8 +10449,7 @@ static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid, test_bit(HCLGE_VPORT_STATE_ALIVE, &vport->state)) hclge_push_vf_port_base_vlan_info(&hdev->vport[0], vport->vport_id, state, - vlan, qos, - ntohs(proto)); + &vlan_info); return 0; } @@ -10280,9 +10519,37 @@ int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto, */ set_bit(vlan_id, vport->vlan_del_fail_bmap); } + + hclge_set_vport_vlan_fltr_change(vport); + return ret; } +static void hclge_sync_vlan_fltr_state(struct hclge_dev *hdev) +{ + struct hclge_vport *vport; + int ret; + u16 i; + + for (i = 0; i < hdev->num_alloc_vport; i++) { + vport = &hdev->vport[i]; + if (!test_and_clear_bit(HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, + &vport->state)) + continue; + + ret = hclge_enable_vport_vlan_filter(vport, + vport->req_vlan_fltr_en); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to sync vlan filter state for vport%u, ret = %d\n", + vport->vport_id, ret); + set_bit(HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, + &vport->state); + return; + } + } +} + static void hclge_sync_vlan_filter(struct hclge_dev *hdev) { #define HCLGE_MAX_SYNC_COUNT 60 @@ -10305,6 +10572,7 @@ static void hclge_sync_vlan_filter(struct hclge_dev *hdev) clear_bit(vlan_id, vport->vlan_del_fail_bmap); hclge_rm_vport_vlan_table(vport, vlan_id, false); + hclge_set_vport_vlan_fltr_change(vport); sync_cnt++; if (sync_cnt >= HCLGE_MAX_SYNC_COUNT) @@ -10314,6 +10582,8 @@ static void hclge_sync_vlan_filter(struct hclge_dev *hdev) VLAN_N_VID); } } + + hclge_sync_vlan_fltr_state(hdev); } static int hclge_set_mac_mtu(struct hclge_dev *hdev, int new_mps) @@ -10807,6 +11077,8 @@ static void hclge_info_show(struct hclge_dev *hdev) hdev->flag & HCLGE_FLAG_DCB_ENABLE ? "enable" : "disable"); dev_info(dev, "MQPRIO %s\n", hdev->flag & HCLGE_FLAG_MQPRIO_ENABLE ? "enable" : "disable"); + dev_info(dev, "Default tx spare buffer size: %u\n", + hdev->tx_spare_buf_size); dev_info(dev, "PF info end.\n"); } @@ -11167,6 +11439,18 @@ static void hclge_clear_resetting_state(struct hclge_dev *hdev) } } +static void hclge_init_rxd_adv_layout(struct hclge_dev *hdev) +{ + if (hnae3_ae_dev_rxd_adv_layout_supported(hdev->ae_dev)) + hclge_write_dev(&hdev->hw, HCLGE_RXD_ADV_LAYOUT_EN_REG, 1); +} + +static void hclge_uninit_rxd_adv_layout(struct hclge_dev *hdev) +{ + if (hnae3_ae_dev_rxd_adv_layout_supported(hdev->ae_dev)) + hclge_write_dev(&hdev->hw, HCLGE_RXD_ADV_LAYOUT_EN_REG, 0); +} + static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) { struct pci_dev *pdev = ae_dev->pdev; @@ -11309,6 +11593,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) goto err_mdiobus_unreg; } + ret = hclge_ptp_init(hdev); + if (ret) + goto err_mdiobus_unreg; + INIT_KFIFO(hdev->mac_tnl_log); hclge_dcb_ops_set(hdev); @@ -11325,7 +11613,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) hclge_clear_resetting_state(hdev); /* Log and clear the hw errors those already occurred */ - hclge_handle_all_hns_hw_errors(ae_dev); + if (hnae3_dev_ras_imp_supported(hdev)) + hclge_handle_occurred_error(hdev); + else + hclge_handle_all_hns_hw_errors(ae_dev); /* request delayed reset for the error recovery because an immediate * global reset on a PF affecting pending initialization of other PFs @@ -11339,6 +11630,8 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) mod_timer(&hdev->reset_timer, jiffies + HCLGE_RESET_INTERVAL); } + hclge_init_rxd_adv_layout(hdev); + /* Enable MISC vector(vector0) */ hclge_enable_vector(&hdev->misc_vector, true); @@ -11471,10 +11764,7 @@ static int hclge_set_vf_trust(struct hnae3_handle *handle, int vf, bool enable) { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - struct hnae3_ae_dev *ae_dev = hdev->ae_dev; u32 new_trusted = enable ? 1 : 0; - bool en_bc_pmc; - int ret; vport = hclge_get_vf_vport(hdev, vf); if (!vport) @@ -11483,18 +11773,9 @@ static int hclge_set_vf_trust(struct hnae3_handle *handle, int vf, bool enable) if (vport->vf_info.trusted == new_trusted) return 0; - /* Disable promisc mode for VF if it is not trusted any more. */ - if (!enable && vport->vf_info.promisc_enable) { - en_bc_pmc = ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2; - ret = hclge_set_vport_promisc_mode(vport, false, false, - en_bc_pmc); - if (ret) - return ret; - vport->vf_info.promisc_enable = 0; - hclge_inform_vf_promisc_info(vport); - } - vport->vf_info.trusted = new_trusted; + set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); + hclge_task_schedule(hdev, 0); return 0; } @@ -11687,8 +11968,15 @@ static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev) return ret; } + ret = hclge_ptp_init(hdev); + if (ret) + return ret; + /* Log and clear the hw errors those already occurred */ - hclge_handle_all_hns_hw_errors(ae_dev); + if (hnae3_dev_ras_imp_supported(hdev)) + hclge_handle_occurred_error(hdev); + else + hclge_handle_all_hns_hw_errors(ae_dev); /* Re-enable the hw error interrupts because * the interrupts get disabled on global reset. @@ -11720,6 +12008,8 @@ static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev) if (ret) return ret; + hclge_init_rxd_adv_layout(hdev); + dev_info(&pdev->dev, "Reset done, %s driver initialization finished.\n", HCLGE_DRIVER_NAME); @@ -11735,6 +12025,8 @@ static void hclge_uninit_ae_dev(struct hnae3_ae_dev *ae_dev) hclge_clear_vf_vlan(hdev); hclge_misc_affinity_teardown(hdev); hclge_state_uninit(hdev); + hclge_ptp_uninit(hdev); + hclge_uninit_rxd_adv_layout(hdev); hclge_uninit_mac_table(hdev); hclge_del_all_fd_entries(hdev); @@ -12385,21 +12677,50 @@ static void hclge_sync_promisc_mode(struct hclge_dev *hdev) struct hnae3_handle *handle = &vport->nic; u8 tmp_flags; int ret; + u16 i; if (vport->last_promisc_flags != vport->overflow_promisc_flags) { - set_bit(HCLGE_STATE_PROMISC_CHANGED, &hdev->state); + set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); vport->last_promisc_flags = vport->overflow_promisc_flags; } - if (test_bit(HCLGE_STATE_PROMISC_CHANGED, &hdev->state)) { + if (test_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state)) { tmp_flags = handle->netdev_flags | vport->last_promisc_flags; ret = hclge_set_promisc_mode(handle, tmp_flags & HNAE3_UPE, tmp_flags & HNAE3_MPE); if (!ret) { - clear_bit(HCLGE_STATE_PROMISC_CHANGED, &hdev->state); - hclge_enable_vlan_filter(handle, - tmp_flags & HNAE3_VLAN_FLTR); + clear_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, + &vport->state); + set_bit(HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, + &vport->state); + } + } + + for (i = 1; i < hdev->num_alloc_vport; i++) { + bool uc_en = false; + bool mc_en = false; + bool bc_en; + + vport = &hdev->vport[i]; + + if (!test_and_clear_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, + &vport->state)) + continue; + + if (vport->vf_info.trusted) { + uc_en = vport->vf_info.request_uc_en > 0; + mc_en = vport->vf_info.request_mc_en > 0; + } + bc_en = vport->vf_info.request_bc_en > 0; + + ret = hclge_cmd_set_promisc_mode(hdev, vport->vport_id, uc_en, + mc_en, bc_en); + if (ret) { + set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, + &vport->state); + return; } + hclge_set_vport_vlan_fltr_change(vport); } } @@ -12578,7 +12899,6 @@ static const struct hnae3_ae_ops hclge_ops = { .get_fd_all_rules = hclge_get_all_rules, .enable_fd = hclge_enable_fd, .add_arfs_entry = hclge_add_fd_entry_by_arfs, - .dbg_run_cmd = hclge_dbg_run_cmd, .dbg_read_cmd = hclge_dbg_read_cmd, .handle_hw_ras_error = hclge_handle_hw_ras_error, .get_hw_reset_stat = hclge_get_hw_reset_stat, @@ -12602,6 +12922,9 @@ static const struct hnae3_ae_ops hclge_ops = { .cls_flower_active = hclge_is_cls_flower_active, .get_phy_link_ksettings = hclge_get_phy_link_ksettings, .set_phy_link_ksettings = hclge_set_phy_link_ksettings, + .set_tx_hwts_info = hclge_ptp_set_tx_info, + .get_rx_hwts = hclge_ptp_get_rx_hwts, + .get_ts_info = hclge_ptp_get_ts_info, }; static struct hnae3_ae_algo ae_algo = { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index ff1d47308c2d..3d3352491dba 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -10,6 +10,7 @@ #include <linux/kfifo.h> #include "hclge_cmd.h" +#include "hclge_ptp.h" #include "hnae3.h" #define HCLGE_MOD_VERSION "1.0" @@ -53,6 +54,7 @@ /* bar registers for common func */ #define HCLGE_VECTOR0_OTER_EN_REG 0x20600 #define HCLGE_GRO_EN_REG 0x28000 +#define HCLGE_RXD_ADV_LAYOUT_EN_REG 0x28008 /* bar registers for rcb */ #define HCLGE_RING_RX_ADDR_L_REG 0x80000 @@ -147,6 +149,8 @@ #define HCLGE_MAX_QSET_NUM 1024 +#define HCLGE_DBG_RESET_INFO_LEN 1024 + enum HLCGE_PORT_TYPE { HOST_PORT, NETWORK_PORT @@ -175,6 +179,7 @@ enum HLCGE_PORT_TYPE { #define HCLGE_FUN_RST_ING_B 0 /* Vector0 register bits define */ +#define HCLGE_VECTOR0_REG_PTP_INT_B 0 #define HCLGE_VECTOR0_GLOBALRESET_INT_B 5 #define HCLGE_VECTOR0_CORERESET_INT_B 6 #define HCLGE_VECTOR0_IMPRESET_INT_B 7 @@ -187,6 +192,7 @@ enum HLCGE_PORT_TYPE { #define HCLGE_VECTOR0_IMP_RESET_INT_B 1 #define HCLGE_VECTOR0_IMP_CMDQ_ERR_B 4U #define HCLGE_VECTOR0_IMP_RD_POISON_B 5U +#define HCLGE_VECTOR0_ALL_MSIX_ERR_B 6U #define HCLGE_MAC_DEFAULT_FRAME \ (ETH_HLEN + ETH_FCS_LEN + 2 * VLAN_HLEN + ETH_DATA_LEN) @@ -218,14 +224,16 @@ enum HCLGE_DEV_STATE { HCLGE_STATE_RST_HANDLING, HCLGE_STATE_MBX_SERVICE_SCHED, HCLGE_STATE_MBX_HANDLING, + HCLGE_STATE_ERR_SERVICE_SCHED, HCLGE_STATE_STATISTICS_UPDATING, HCLGE_STATE_CMD_DISABLE, HCLGE_STATE_LINK_UPDATING, - HCLGE_STATE_PROMISC_CHANGED, HCLGE_STATE_RST_FAIL, HCLGE_STATE_FD_TBL_CHANGED, HCLGE_STATE_FD_CLEAR_ALL, HCLGE_STATE_FD_USER_DEF_CHANGED, + HCLGE_STATE_PTP_EN, + HCLGE_STATE_PTP_TX_HANDLING, HCLGE_STATE_MAX }; @@ -233,6 +241,7 @@ enum hclge_evt_cause { HCLGE_VECTOR0_EVENT_RST, HCLGE_VECTOR0_EVENT_MBX, HCLGE_VECTOR0_EVENT_ERR, + HCLGE_VECTOR0_EVENT_PTP, HCLGE_VECTOR0_EVENT_OTHER, }; @@ -319,6 +328,22 @@ enum hclge_fc_mode { HCLGE_FC_DEFAULT }; +#define HCLGE_FILTER_TYPE_VF 0 +#define HCLGE_FILTER_TYPE_PORT 1 +#define HCLGE_FILTER_FE_EGRESS_V1_B BIT(0) +#define HCLGE_FILTER_FE_NIC_INGRESS_B BIT(0) +#define HCLGE_FILTER_FE_NIC_EGRESS_B BIT(1) +#define HCLGE_FILTER_FE_ROCE_INGRESS_B BIT(2) +#define HCLGE_FILTER_FE_ROCE_EGRESS_B BIT(3) +#define HCLGE_FILTER_FE_EGRESS (HCLGE_FILTER_FE_NIC_EGRESS_B \ + | HCLGE_FILTER_FE_ROCE_EGRESS_B) +#define HCLGE_FILTER_FE_INGRESS (HCLGE_FILTER_FE_NIC_INGRESS_B \ + | HCLGE_FILTER_FE_ROCE_INGRESS_B) + +enum hclge_vlan_fltr_cap { + HCLGE_VLAN_FLTR_DEF, + HCLGE_VLAN_FLTR_CAN_MDF, +}; enum hclge_link_fail_code { HCLGE_LF_NORMAL, HCLGE_LF_REF_CLOCK_LOST, @@ -349,6 +374,7 @@ struct hclge_tc_info { struct hclge_cfg { u8 tc_num; + u8 vlan_fliter_cap; u16 tqp_desc_num; u16 rx_buf_len; u16 vf_rss_size_max; @@ -358,6 +384,7 @@ struct hclge_cfg { u8 mac_addr[ETH_ALEN]; u8 default_speed; u32 numa_node_map; + u32 tx_spare_buf_size; u16 speed_ability; u16 umv_space; }; @@ -757,9 +784,14 @@ struct hclge_mac_tnl_stats { struct hclge_vf_vlan_cfg { u8 mbx_cmd; u8 subcode; - u8 is_kill; - u16 vlan; - u16 proto; + union { + struct { + u8 is_kill; + u16 vlan; + u16 proto; + }; + u8 enable; + }; }; #pragma pack() @@ -817,6 +849,7 @@ struct hclge_dev { u16 alloc_rss_size; /* Allocated RSS task queue */ u16 vf_rss_size_max; /* HW defined VF max RSS task queue */ u16 pf_rss_size_max; /* HW defined PF max RSS task queue */ + u32 tx_spare_buf_size; /* HW defined TX spare buffer size */ u16 fdir_pf_filter_count; /* Num of guaranteed filters for this PF */ u16 num_alloc_vport; /* Num vports this driver supports */ @@ -909,6 +942,7 @@ struct hclge_dev { /* affinity mask and notify for misc interrupt */ cpumask_t affinity_mask; struct irq_affinity_notify affinity_notify; + struct hclge_ptp *ptp; }; /* VPort level vlan tag configuration for TX direction */ @@ -949,6 +983,8 @@ struct hclge_rss_tuple_cfg { enum HCLGE_VPORT_STATE { HCLGE_VPORT_STATE_ALIVE, HCLGE_VPORT_STATE_MAC_TBL_CHANGE, + HCLGE_VPORT_STATE_PROMISC_CHANGE, + HCLGE_VPORT_STATE_VLAN_FLTR_CHANGE, HCLGE_VPORT_STATE_MAX }; @@ -969,7 +1005,9 @@ struct hclge_vf_info { u32 spoofchk; u32 max_tx_rate; u32 trusted; - u16 promisc_enable; + u8 request_uc_en; + u8 request_mc_en; + u8 request_bc_en; }; struct hclge_vport { @@ -988,6 +1026,8 @@ struct hclge_vport { u32 bw_limit; /* VSI BW Limit (0 = disabled) */ u8 dwrr; + bool req_vlan_fltr_en; + bool cur_vlan_fltr_en; unsigned long vlan_del_fail_bmap[BITS_TO_LONGS(VLAN_N_VID)]; struct hclge_port_base_vlan_config port_base_vlan_cfg; struct hclge_tx_vtag_cfg txvlan_cfg; @@ -1059,8 +1099,7 @@ int hclge_func_reset_cmd(struct hclge_dev *hdev, int func_id); int hclge_vport_start(struct hclge_vport *vport); void hclge_vport_stop(struct hclge_vport *vport); int hclge_set_vport_mtu(struct hclge_vport *vport, int new_mtu); -int hclge_dbg_run_cmd(struct hnae3_handle *handle, const char *cmd_buf); -int hclge_dbg_read_cmd(struct hnae3_handle *handle, const char *cmd_buf, +int hclge_dbg_read_cmd(struct hnae3_handle *handle, enum hnae3_dbg_cmd cmd, char *buf, int len); u16 hclge_covert_handle_qid_global(struct hnae3_handle *handle, u16 queue_id); int hclge_notify_client(struct hclge_dev *hdev, @@ -1080,14 +1119,15 @@ void hclge_restore_vport_vlan_table(struct hclge_vport *vport); int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, struct hclge_vlan_info *vlan_info); int hclge_push_vf_port_base_vlan_info(struct hclge_vport *vport, u8 vfid, - u16 state, u16 vlan_tag, u16 qos, - u16 vlan_proto); + u16 state, + struct hclge_vlan_info *vlan_info); void hclge_task_schedule(struct hclge_dev *hdev, unsigned long delay_time); int hclge_query_bd_num_cmd_send(struct hclge_dev *hdev, struct hclge_desc *desc); void hclge_report_hw_error(struct hclge_dev *hdev, enum hnae3_hw_error_type type); void hclge_inform_vf_promisc_info(struct hclge_vport *vport); -void hclge_dbg_dump_rst_info(struct hclge_dev *hdev); +int hclge_dbg_dump_rst_info(struct hclge_dev *hdev, char *buf, int len); int hclge_push_vf_link_status(struct hclge_vport *vport); +int hclge_enable_vport_vlan_filter(struct hclge_vport *vport, bool request_en); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c index 8e5f9dc8791d..e10a2c36b706 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c @@ -231,19 +231,15 @@ static int hclge_map_unmap_ring_to_vf_vector(struct hclge_vport *vport, bool en, return ret; } -static int hclge_set_vf_promisc_mode(struct hclge_vport *vport, - struct hclge_mbx_vf_to_pf_cmd *req) +static void hclge_set_vf_promisc_mode(struct hclge_vport *vport, + struct hclge_mbx_vf_to_pf_cmd *req) { - bool en_bc = req->msg.en_bc ? true : false; - bool en_uc = req->msg.en_uc ? true : false; - bool en_mc = req->msg.en_mc ? true : false; struct hnae3_handle *handle = &vport->nic; - int ret; + struct hclge_dev *hdev = vport->back; - if (!vport->vf_info.trusted) { - en_uc = false; - en_mc = false; - } + vport->vf_info.request_uc_en = req->msg.en_uc; + vport->vf_info.request_mc_en = req->msg.en_mc; + vport->vf_info.request_bc_en = req->msg.en_bc; if (req->msg.en_limit_promisc) set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags); @@ -251,22 +247,8 @@ static int hclge_set_vf_promisc_mode(struct hclge_vport *vport, clear_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->priv_flags); - ret = hclge_set_vport_promisc_mode(vport, en_uc, en_mc, en_bc); - - vport->vf_info.promisc_enable = (en_uc || en_mc) ? 1 : 0; - - return ret; -} - -void hclge_inform_vf_promisc_info(struct hclge_vport *vport) -{ - u8 dest_vfid = (u8)vport->vport_id; - u8 msg_data[2]; - - memcpy(&msg_data[0], &vport->vf_info.promisc_enable, sizeof(u16)); - - hclge_send_mbx_msg(vport, msg_data, sizeof(msg_data), - HCLGE_MBX_PUSH_PROMISC_INFO, dest_vfid); + set_bit(HCLGE_VPORT_STATE_PROMISC_CHANGE, &vport->state); + hclge_task_schedule(hdev, 0); } static int hclge_set_vf_uc_mac_addr(struct hclge_vport *vport, @@ -336,17 +318,17 @@ static int hclge_set_vf_mc_mac_addr(struct hclge_vport *vport, } int hclge_push_vf_port_base_vlan_info(struct hclge_vport *vport, u8 vfid, - u16 state, u16 vlan_tag, u16 qos, - u16 vlan_proto) + u16 state, + struct hclge_vlan_info *vlan_info) { #define MSG_DATA_SIZE 8 u8 msg_data[MSG_DATA_SIZE]; memcpy(&msg_data[0], &state, sizeof(u16)); - memcpy(&msg_data[2], &vlan_proto, sizeof(u16)); - memcpy(&msg_data[4], &qos, sizeof(u16)); - memcpy(&msg_data[6], &vlan_tag, sizeof(u16)); + memcpy(&msg_data[2], &vlan_info->vlan_proto, sizeof(u16)); + memcpy(&msg_data[4], &vlan_info->qos, sizeof(u16)); + memcpy(&msg_data[6], &vlan_info->vlan_tag, sizeof(u16)); return hclge_send_mbx_msg(vport, msg_data, sizeof(msg_data), HCLGE_MBX_PUSH_VLAN_INFO, vfid); @@ -359,49 +341,35 @@ static int hclge_set_vf_vlan_cfg(struct hclge_vport *vport, #define HCLGE_MBX_VLAN_STATE_OFFSET 0 #define HCLGE_MBX_VLAN_INFO_OFFSET 2 + struct hnae3_handle *handle = &vport->nic; + struct hclge_dev *hdev = vport->back; struct hclge_vf_vlan_cfg *msg_cmd; - int status = 0; msg_cmd = (struct hclge_vf_vlan_cfg *)&mbx_req->msg; - if (msg_cmd->subcode == HCLGE_MBX_VLAN_FILTER) { - struct hnae3_handle *handle = &vport->nic; - u16 vlan, proto; - bool is_kill; - - is_kill = !!msg_cmd->is_kill; - vlan = msg_cmd->vlan; - proto = msg_cmd->proto; - status = hclge_set_vlan_filter(handle, cpu_to_be16(proto), - vlan, is_kill); - } else if (msg_cmd->subcode == HCLGE_MBX_VLAN_RX_OFF_CFG) { - struct hnae3_handle *handle = &vport->nic; - bool en = msg_cmd->is_kill ? true : false; - - status = hclge_en_hw_strip_rxvtag(handle, en); - } else if (msg_cmd->subcode == HCLGE_MBX_PORT_BASE_VLAN_CFG) { - struct hclge_vlan_info *vlan_info; - u16 *state; - - state = (u16 *)&mbx_req->msg.data[HCLGE_MBX_VLAN_STATE_OFFSET]; - vlan_info = (struct hclge_vlan_info *) - &mbx_req->msg.data[HCLGE_MBX_VLAN_INFO_OFFSET]; - status = hclge_update_port_base_vlan_cfg(vport, *state, - vlan_info); - } else if (msg_cmd->subcode == HCLGE_MBX_GET_PORT_BASE_VLAN_STATE) { - struct hnae3_ae_dev *ae_dev = pci_get_drvdata(vport->nic.pdev); + switch (msg_cmd->subcode) { + case HCLGE_MBX_VLAN_FILTER: + return hclge_set_vlan_filter(handle, + cpu_to_be16(msg_cmd->proto), + msg_cmd->vlan, msg_cmd->is_kill); + case HCLGE_MBX_VLAN_RX_OFF_CFG: + return hclge_en_hw_strip_rxvtag(handle, msg_cmd->enable); + case HCLGE_MBX_GET_PORT_BASE_VLAN_STATE: /* vf does not need to know about the port based VLAN state * on device HNAE3_DEVICE_VERSION_V3. So always return disable * on device HNAE3_DEVICE_VERSION_V3 if vf queries the port * based VLAN state. */ resp_msg->data[0] = - ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3 ? + hdev->ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3 ? HNAE3_PORT_BASE_VLAN_DISABLE : vport->port_base_vlan_cfg.state; resp_msg->len = sizeof(u8); + return 0; + case HCLGE_MBX_ENABLE_VLAN_FILTER: + return hclge_enable_vport_vlan_filter(vport, msg_cmd->enable); + default: + return 0; } - - return status; } static int hclge_set_vf_alive(struct hclge_vport *vport, @@ -418,16 +386,23 @@ static int hclge_set_vf_alive(struct hclge_vport *vport, return ret; } -static void hclge_get_vf_tcinfo(struct hclge_vport *vport, - struct hclge_respond_to_vf_msg *resp_msg) +static void hclge_get_basic_info(struct hclge_vport *vport, + struct hclge_respond_to_vf_msg *resp_msg) { struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo; + struct hnae3_ae_dev *ae_dev = vport->back->ae_dev; + struct hclge_basic_info *basic_info; unsigned int i; + basic_info = (struct hclge_basic_info *)resp_msg->data; for (i = 0; i < kinfo->tc_info.num_tc; i++) - resp_msg->data[0] |= BIT(i); + basic_info->hw_tc_map |= BIT(i); - resp_msg->len = sizeof(u8); + if (test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps)) + hnae3_set_bit(basic_info->pf_caps, + HNAE3_PF_SUPPORT_VLAN_FLTR_MDF_B, 1); + + resp_msg->len = HCLGE_MBX_MAX_RESP_DATA_SIZE; } static void hclge_get_vf_queue_info(struct hclge_vport *vport, @@ -710,7 +685,6 @@ void hclge_mbx_handler(struct hclge_dev *hdev) unsigned int flag; int ret = 0; - memset(&resp_msg, 0, sizeof(resp_msg)); /* handle all the mailbox requests in the queue */ while (!hclge_cmd_crq_empty(&hdev->hw)) { if (test_bit(HCLGE_STATE_CMD_DISABLE, &hdev->state)) { @@ -738,6 +712,9 @@ void hclge_mbx_handler(struct hclge_dev *hdev) trace_hclge_pf_mbx_get(hdev, req); + /* clear the resp_msg before processing every mailbox message */ + memset(&resp_msg, 0, sizeof(resp_msg)); + switch (req->msg.code) { case HCLGE_MBX_MAP_RING_TO_VECTOR: ret = hclge_map_unmap_ring_to_vf_vector(vport, true, @@ -748,11 +725,7 @@ void hclge_mbx_handler(struct hclge_dev *hdev) req); break; case HCLGE_MBX_SET_PROMISC_MODE: - ret = hclge_set_vf_promisc_mode(vport, req); - if (ret) - dev_err(&hdev->pdev->dev, - "PF fail(%d) to set VF promisc mode\n", - ret); + hclge_set_vf_promisc_mode(vport, req); break; case HCLGE_MBX_SET_UNICAST: ret = hclge_set_vf_uc_mac_addr(vport, req); @@ -788,8 +761,8 @@ void hclge_mbx_handler(struct hclge_dev *hdev) case HCLGE_MBX_GET_QDEPTH: hclge_get_vf_queue_depth(vport, &resp_msg); break; - case HCLGE_MBX_GET_TCINFO: - hclge_get_vf_tcinfo(vport, &resp_msg); + case HCLGE_MBX_GET_BASIC_INFO: + hclge_get_basic_info(vport, &resp_msg); break; case HCLGE_MBX_GET_LINK_STATUS: ret = hclge_push_vf_link_status(vport); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c new file mode 100644 index 000000000000..3b1f84502e36 --- /dev/null +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright (c) 2021 Hisilicon Limited. + +#include <linux/skbuff.h> +#include "hclge_main.h" +#include "hnae3.h" + +static int hclge_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + struct hclge_dev *hdev = hclge_ptp_get_hdev(ptp); + u64 adj_val, adj_base, diff; + unsigned long flags; + bool is_neg = false; + u32 quo, numerator; + + if (ppb < 0) { + ppb = -ppb; + is_neg = true; + } + + adj_base = HCLGE_PTP_CYCLE_ADJ_BASE * HCLGE_PTP_CYCLE_ADJ_UNIT; + adj_val = adj_base * ppb; + diff = div_u64(adj_val, 1000000000ULL); + + if (is_neg) + adj_val = adj_base - diff; + else + adj_val = adj_base + diff; + + /* This clock cycle is defined by three part: quotient, numerator + * and denominator. For example, 2.5ns, the quotient is 2, + * denominator is fixed to HCLGE_PTP_CYCLE_ADJ_UNIT, and numerator + * is 0.5 * HCLGE_PTP_CYCLE_ADJ_UNIT. + */ + quo = div_u64_rem(adj_val, HCLGE_PTP_CYCLE_ADJ_UNIT, &numerator); + + spin_lock_irqsave(&hdev->ptp->lock, flags); + writel(quo, hdev->ptp->io_base + HCLGE_PTP_CYCLE_QUO_REG); + writel(numerator, hdev->ptp->io_base + HCLGE_PTP_CYCLE_NUM_REG); + writel(HCLGE_PTP_CYCLE_ADJ_UNIT, + hdev->ptp->io_base + HCLGE_PTP_CYCLE_DEN_REG); + writel(HCLGE_PTP_CYCLE_ADJ_EN, + hdev->ptp->io_base + HCLGE_PTP_CYCLE_CFG_REG); + spin_unlock_irqrestore(&hdev->ptp->lock, flags); + + return 0; +} + +bool hclge_ptp_set_tx_info(struct hnae3_handle *handle, struct sk_buff *skb) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + struct hclge_ptp *ptp = hdev->ptp; + + if (!test_bit(HCLGE_PTP_FLAG_TX_EN, &ptp->flags) || + test_and_set_bit(HCLGE_STATE_PTP_TX_HANDLING, &hdev->state)) { + ptp->tx_skipped++; + return false; + } + + ptp->tx_start = jiffies; + ptp->tx_skb = skb_get(skb); + ptp->tx_cnt++; + + return true; +} + +void hclge_ptp_clean_tx_hwts(struct hclge_dev *hdev) +{ + struct sk_buff *skb = hdev->ptp->tx_skb; + struct skb_shared_hwtstamps hwts; + u32 hi, lo; + u64 ns; + + ns = readl(hdev->ptp->io_base + HCLGE_PTP_TX_TS_NSEC_REG) & + HCLGE_PTP_TX_TS_NSEC_MASK; + lo = readl(hdev->ptp->io_base + HCLGE_PTP_TX_TS_SEC_L_REG); + hi = readl(hdev->ptp->io_base + HCLGE_PTP_TX_TS_SEC_H_REG) & + HCLGE_PTP_TX_TS_SEC_H_MASK; + hdev->ptp->last_tx_seqid = readl(hdev->ptp->io_base + + HCLGE_PTP_TX_TS_SEQID_REG); + + if (skb) { + hdev->ptp->tx_skb = NULL; + hdev->ptp->tx_cleaned++; + + ns += (((u64)hi) << 32 | lo) * NSEC_PER_SEC; + hwts.hwtstamp = ns_to_ktime(ns); + skb_tstamp_tx(skb, &hwts); + dev_kfree_skb_any(skb); + } + + clear_bit(HCLGE_STATE_PTP_TX_HANDLING, &hdev->state); +} + +void hclge_ptp_get_rx_hwts(struct hnae3_handle *handle, struct sk_buff *skb, + u32 nsec, u32 sec) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + unsigned long flags; + u64 ns = nsec; + u32 sec_h; + + if (!test_bit(HCLGE_PTP_FLAG_RX_EN, &hdev->ptp->flags)) + return; + + /* Since the BD does not have enough space for the higher 16 bits of + * second, and this part will not change frequently, so read it + * from register. + */ + spin_lock_irqsave(&hdev->ptp->lock, flags); + sec_h = readl(hdev->ptp->io_base + HCLGE_PTP_CUR_TIME_SEC_H_REG); + spin_unlock_irqrestore(&hdev->ptp->lock, flags); + + ns += (((u64)sec_h) << HCLGE_PTP_SEC_H_OFFSET | sec) * NSEC_PER_SEC; + skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); + hdev->ptp->last_rx = jiffies; + hdev->ptp->rx_cnt++; +} + +static int hclge_ptp_gettimex(struct ptp_clock_info *ptp, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct hclge_dev *hdev = hclge_ptp_get_hdev(ptp); + unsigned long flags; + u32 hi, lo; + u64 ns; + + spin_lock_irqsave(&hdev->ptp->lock, flags); + ns = readl(hdev->ptp->io_base + HCLGE_PTP_CUR_TIME_NSEC_REG); + hi = readl(hdev->ptp->io_base + HCLGE_PTP_CUR_TIME_SEC_H_REG); + lo = readl(hdev->ptp->io_base + HCLGE_PTP_CUR_TIME_SEC_L_REG); + spin_unlock_irqrestore(&hdev->ptp->lock, flags); + + ns += (((u64)hi) << HCLGE_PTP_SEC_H_OFFSET | lo) * NSEC_PER_SEC; + *ts = ns_to_timespec64(ns); + + return 0; +} + +static int hclge_ptp_settime(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct hclge_dev *hdev = hclge_ptp_get_hdev(ptp); + unsigned long flags; + + spin_lock_irqsave(&hdev->ptp->lock, flags); + writel(ts->tv_nsec, hdev->ptp->io_base + HCLGE_PTP_TIME_NSEC_REG); + writel(ts->tv_sec >> HCLGE_PTP_SEC_H_OFFSET, + hdev->ptp->io_base + HCLGE_PTP_TIME_SEC_H_REG); + writel(ts->tv_sec & HCLGE_PTP_SEC_L_MASK, + hdev->ptp->io_base + HCLGE_PTP_TIME_SEC_L_REG); + /* synchronize the time of phc */ + writel(HCLGE_PTP_TIME_SYNC_EN, + hdev->ptp->io_base + HCLGE_PTP_TIME_SYNC_REG); + spin_unlock_irqrestore(&hdev->ptp->lock, flags); + + return 0; +} + +static int hclge_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct hclge_dev *hdev = hclge_ptp_get_hdev(ptp); + unsigned long flags; + bool is_neg = false; + u32 adj_val = 0; + + if (delta < 0) { + adj_val |= HCLGE_PTP_TIME_NSEC_NEG; + delta = -delta; + is_neg = true; + } + + if (delta > HCLGE_PTP_TIME_NSEC_MASK) { + struct timespec64 ts; + s64 ns; + + hclge_ptp_gettimex(ptp, &ts, NULL); + ns = timespec64_to_ns(&ts); + ns = is_neg ? ns - delta : ns + delta; + ts = ns_to_timespec64(ns); + return hclge_ptp_settime(ptp, &ts); + } + + adj_val |= delta & HCLGE_PTP_TIME_NSEC_MASK; + + spin_lock_irqsave(&hdev->ptp->lock, flags); + writel(adj_val, hdev->ptp->io_base + HCLGE_PTP_TIME_NSEC_REG); + writel(HCLGE_PTP_TIME_ADJ_EN, + hdev->ptp->io_base + HCLGE_PTP_TIME_ADJ_REG); + spin_unlock_irqrestore(&hdev->ptp->lock, flags); + + return 0; +} + +int hclge_ptp_get_cfg(struct hclge_dev *hdev, struct ifreq *ifr) +{ + if (!test_bit(HCLGE_STATE_PTP_EN, &hdev->state)) + return -EOPNOTSUPP; + + return copy_to_user(ifr->ifr_data, &hdev->ptp->ts_cfg, + sizeof(struct hwtstamp_config)) ? -EFAULT : 0; +} + +static int hclge_ptp_int_en(struct hclge_dev *hdev, bool en) +{ + struct hclge_ptp_int_cmd *req; + struct hclge_desc desc; + int ret; + + req = (struct hclge_ptp_int_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PTP_INT_EN, false); + req->int_en = en ? 1 : 0; + + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to %s ptp interrupt, ret = %d\n", + en ? "enable" : "disable", ret); + + return ret; +} + +int hclge_ptp_cfg_qry(struct hclge_dev *hdev, u32 *cfg) +{ + struct hclge_ptp_cfg_cmd *req; + struct hclge_desc desc; + int ret; + + req = (struct hclge_ptp_cfg_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PTP_MODE_CFG, true); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to query ptp config, ret = %d\n", ret); + return ret; + } + + *cfg = le32_to_cpu(req->cfg); + + return 0; +} + +static int hclge_ptp_cfg(struct hclge_dev *hdev, u32 cfg) +{ + struct hclge_ptp_cfg_cmd *req; + struct hclge_desc desc; + int ret; + + req = (struct hclge_ptp_cfg_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_PTP_MODE_CFG, false); + req->cfg = cpu_to_le32(cfg); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to config ptp, ret = %d\n", ret); + + return ret; +} + +static int hclge_ptp_set_tx_mode(struct hwtstamp_config *cfg, + unsigned long *flags, u32 *ptp_cfg) +{ + switch (cfg->tx_type) { + case HWTSTAMP_TX_OFF: + clear_bit(HCLGE_PTP_FLAG_TX_EN, flags); + break; + case HWTSTAMP_TX_ON: + set_bit(HCLGE_PTP_FLAG_TX_EN, flags); + *ptp_cfg |= HCLGE_PTP_TX_EN_B; + break; + default: + return -ERANGE; + } + + return 0; +} + +static int hclge_ptp_set_rx_mode(struct hwtstamp_config *cfg, + unsigned long *flags, u32 *ptp_cfg) +{ + int rx_filter = cfg->rx_filter; + + switch (cfg->rx_filter) { + case HWTSTAMP_FILTER_NONE: + clear_bit(HCLGE_PTP_FLAG_RX_EN, flags); + break; + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + set_bit(HCLGE_PTP_FLAG_RX_EN, flags); + *ptp_cfg |= HCLGE_PTP_RX_EN_B; + *ptp_cfg |= HCLGE_PTP_UDP_FULL_TYPE << HCLGE_PTP_UDP_EN_SHIFT; + rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + set_bit(HCLGE_PTP_FLAG_RX_EN, flags); + *ptp_cfg |= HCLGE_PTP_RX_EN_B; + *ptp_cfg |= HCLGE_PTP_UDP_FULL_TYPE << HCLGE_PTP_UDP_EN_SHIFT; + *ptp_cfg |= HCLGE_PTP_MSG1_V2_DEFAULT << HCLGE_PTP_MSG1_SHIFT; + *ptp_cfg |= HCLGE_PTP_MSG0_V2_EVENT << HCLGE_PTP_MSG0_SHIFT; + *ptp_cfg |= HCLGE_PTP_MSG_TYPE_V2 << HCLGE_PTP_MSG_TYPE_SHIFT; + rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + break; + case HWTSTAMP_FILTER_ALL: + default: + return -ERANGE; + } + + cfg->rx_filter = rx_filter; + + return 0; +} + +static int hclge_ptp_set_ts_mode(struct hclge_dev *hdev, + struct hwtstamp_config *cfg) +{ + unsigned long flags = hdev->ptp->flags; + u32 ptp_cfg = 0; + int ret; + + if (test_bit(HCLGE_PTP_FLAG_EN, &hdev->ptp->flags)) + ptp_cfg |= HCLGE_PTP_EN_B; + + ret = hclge_ptp_set_tx_mode(cfg, &flags, &ptp_cfg); + if (ret) + return ret; + + ret = hclge_ptp_set_rx_mode(cfg, &flags, &ptp_cfg); + if (ret) + return ret; + + ret = hclge_ptp_cfg(hdev, ptp_cfg); + if (ret) + return ret; + + hdev->ptp->flags = flags; + hdev->ptp->ptp_cfg = ptp_cfg; + + return 0; +} + +int hclge_ptp_set_cfg(struct hclge_dev *hdev, struct ifreq *ifr) +{ + struct hwtstamp_config cfg; + int ret; + + if (!test_bit(HCLGE_STATE_PTP_EN, &hdev->state)) { + dev_err(&hdev->pdev->dev, "phc is unsupported\n"); + return -EOPNOTSUPP; + } + + if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) + return -EFAULT; + + ret = hclge_ptp_set_ts_mode(hdev, &cfg); + if (ret) + return ret; + + hdev->ptp->ts_cfg = cfg; + + return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; +} + +int hclge_ptp_get_ts_info(struct hnae3_handle *handle, + struct ethtool_ts_info *info) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + + if (!test_bit(HCLGE_STATE_PTP_EN, &hdev->state)) { + dev_err(&hdev->pdev->dev, "phc is unsupported\n"); + return -EOPNOTSUPP; + } + + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + if (hdev->ptp->clock) + info->phc_index = ptp_clock_index(hdev->ptp->clock); + else + info->phc_index = -1; + + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L2_SYNC) | + BIT(HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ); + + info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V1_L4_SYNC) | + BIT(HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) | + BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_SYNC) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_SYNC) | + BIT(HWTSTAMP_FILTER_PTP_V2_DELAY_REQ) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ); + + return 0; +} + +static int hclge_ptp_create_clock(struct hclge_dev *hdev) +{ + struct hclge_ptp *ptp; + + ptp = devm_kzalloc(&hdev->pdev->dev, sizeof(*ptp), GFP_KERNEL); + if (!ptp) + return -ENOMEM; + + ptp->hdev = hdev; + snprintf(ptp->info.name, sizeof(ptp->info.name), "%s", + HCLGE_DRIVER_NAME); + ptp->info.owner = THIS_MODULE; + ptp->info.max_adj = HCLGE_PTP_CYCLE_ADJ_MAX; + ptp->info.n_ext_ts = 0; + ptp->info.pps = 0; + ptp->info.adjfreq = hclge_ptp_adjfreq; + ptp->info.adjtime = hclge_ptp_adjtime; + ptp->info.gettimex64 = hclge_ptp_gettimex; + ptp->info.settime64 = hclge_ptp_settime; + + ptp->info.n_alarm = 0; + ptp->clock = ptp_clock_register(&ptp->info, &hdev->pdev->dev); + if (IS_ERR(ptp->clock)) { + dev_err(&hdev->pdev->dev, + "%d failed to register ptp clock, ret = %ld\n", + ptp->info.n_alarm, PTR_ERR(ptp->clock)); + return -ENODEV; + } else if (!ptp->clock) { + dev_err(&hdev->pdev->dev, "failed to register ptp clock\n"); + return -ENODEV; + } + + spin_lock_init(&ptp->lock); + ptp->io_base = hdev->hw.io_base + HCLGE_PTP_REG_OFFSET; + ptp->ts_cfg.rx_filter = HWTSTAMP_FILTER_NONE; + ptp->ts_cfg.tx_type = HWTSTAMP_TX_OFF; + hdev->ptp = ptp; + + return 0; +} + +static void hclge_ptp_destroy_clock(struct hclge_dev *hdev) +{ + ptp_clock_unregister(hdev->ptp->clock); + hdev->ptp->clock = NULL; + devm_kfree(&hdev->pdev->dev, hdev->ptp); + hdev->ptp = NULL; +} + +int hclge_ptp_init(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + struct timespec64 ts; + int ret; + + if (!test_bit(HNAE3_DEV_SUPPORT_PTP_B, ae_dev->caps)) + return 0; + + if (!hdev->ptp) { + ret = hclge_ptp_create_clock(hdev); + if (ret) + return ret; + } + + ret = hclge_ptp_int_en(hdev, true); + if (ret) + goto out; + + set_bit(HCLGE_PTP_FLAG_EN, &hdev->ptp->flags); + ret = hclge_ptp_adjfreq(&hdev->ptp->info, 0); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to init freq, ret = %d\n", ret); + goto out; + } + + ret = hclge_ptp_set_ts_mode(hdev, &hdev->ptp->ts_cfg); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to init ts mode, ret = %d\n", ret); + goto out; + } + + ktime_get_real_ts64(&ts); + ret = hclge_ptp_settime(&hdev->ptp->info, &ts); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to init ts time, ret = %d\n", ret); + goto out; + } + + set_bit(HCLGE_STATE_PTP_EN, &hdev->state); + dev_info(&hdev->pdev->dev, "phc initializes ok!\n"); + + return 0; + +out: + hclge_ptp_destroy_clock(hdev); + + return ret; +} + +void hclge_ptp_uninit(struct hclge_dev *hdev) +{ + struct hclge_ptp *ptp = hdev->ptp; + + if (!ptp) + return; + + hclge_ptp_int_en(hdev, false); + clear_bit(HCLGE_STATE_PTP_EN, &hdev->state); + clear_bit(HCLGE_PTP_FLAG_EN, &ptp->flags); + ptp->ts_cfg.rx_filter = HWTSTAMP_FILTER_NONE; + ptp->ts_cfg.tx_type = HWTSTAMP_TX_OFF; + + if (hclge_ptp_set_ts_mode(hdev, &ptp->ts_cfg)) + dev_err(&hdev->pdev->dev, "failed to disable phc\n"); + + if (ptp->tx_skb) { + struct sk_buff *skb = ptp->tx_skb; + + ptp->tx_skb = NULL; + dev_kfree_skb_any(skb); + } + + hclge_ptp_destroy_clock(hdev); +} diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h new file mode 100644 index 000000000000..5a202b775471 --- /dev/null +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +// Copyright (c) 2021 Hisilicon Limited. + +#ifndef __HCLGE_PTP_H +#define __HCLGE_PTP_H + +#include <linux/ptp_clock_kernel.h> +#include <linux/net_tstamp.h> +#include <linux/types.h> + +#define HCLGE_PTP_REG_OFFSET 0x29000 + +#define HCLGE_PTP_TX_TS_SEQID_REG 0x0 +#define HCLGE_PTP_TX_TS_NSEC_REG 0x4 +#define HCLGE_PTP_TX_TS_NSEC_MASK GENMASK(29, 0) +#define HCLGE_PTP_TX_TS_SEC_L_REG 0x8 +#define HCLGE_PTP_TX_TS_SEC_H_REG 0xC +#define HCLGE_PTP_TX_TS_SEC_H_MASK GENMASK(15, 0) +#define HCLGE_PTP_TX_TS_CNT_REG 0x30 + +#define HCLGE_PTP_TIME_SEC_H_REG 0x50 +#define HCLGE_PTP_TIME_SEC_H_MASK GENMASK(15, 0) +#define HCLGE_PTP_TIME_SEC_L_REG 0x54 +#define HCLGE_PTP_TIME_NSEC_REG 0x58 +#define HCLGE_PTP_TIME_NSEC_MASK GENMASK(29, 0) +#define HCLGE_PTP_TIME_NSEC_NEG BIT(31) +#define HCLGE_PTP_TIME_SYNC_REG 0x5C +#define HCLGE_PTP_TIME_SYNC_EN BIT(0) +#define HCLGE_PTP_TIME_ADJ_REG 0x60 +#define HCLGE_PTP_TIME_ADJ_EN BIT(0) +#define HCLGE_PTP_CYCLE_QUO_REG 0x64 +#define HCLGE_PTP_CYCLE_DEN_REG 0x68 +#define HCLGE_PTP_CYCLE_NUM_REG 0x6C +#define HCLGE_PTP_CYCLE_CFG_REG 0x70 +#define HCLGE_PTP_CYCLE_ADJ_EN BIT(0) +#define HCLGE_PTP_CUR_TIME_SEC_H_REG 0x74 +#define HCLGE_PTP_CUR_TIME_SEC_L_REG 0x78 +#define HCLGE_PTP_CUR_TIME_NSEC_REG 0x7C + +#define HCLGE_PTP_CYCLE_ADJ_BASE 2 +#define HCLGE_PTP_CYCLE_ADJ_MAX 500000000 +#define HCLGE_PTP_CYCLE_ADJ_UNIT 100000000 +#define HCLGE_PTP_SEC_H_OFFSET 32u +#define HCLGE_PTP_SEC_L_MASK GENMASK(31, 0) + +#define HCLGE_PTP_FLAG_EN 0 +#define HCLGE_PTP_FLAG_TX_EN 1 +#define HCLGE_PTP_FLAG_RX_EN 2 + +struct hclge_ptp { + struct hclge_dev *hdev; + struct ptp_clock *clock; + struct sk_buff *tx_skb; + unsigned long flags; + void __iomem *io_base; + struct ptp_clock_info info; + struct hwtstamp_config ts_cfg; + spinlock_t lock; /* protects ptp registers */ + u32 ptp_cfg; + u32 last_tx_seqid; + unsigned long tx_start; + unsigned long tx_cnt; + unsigned long tx_skipped; + unsigned long tx_cleaned; + unsigned long last_rx; + unsigned long rx_cnt; + unsigned long tx_timeout; +}; + +struct hclge_ptp_int_cmd { +#define HCLGE_PTP_INT_EN_B BIT(0) + + u8 int_en; + u8 rsvd[23]; +}; + +enum hclge_ptp_udp_type { + HCLGE_PTP_UDP_NOT_TYPE, + HCLGE_PTP_UDP_P13F_TYPE, + HCLGE_PTP_UDP_P140_TYPE, + HCLGE_PTP_UDP_FULL_TYPE, +}; + +enum hclge_ptp_msg_type { + HCLGE_PTP_MSG_TYPE_V2_L2, + HCLGE_PTP_MSG_TYPE_V2, + HCLGE_PTP_MSG_TYPE_V2_EVENT, +}; + +enum hclge_ptp_msg0_type { + HCLGE_PTP_MSG0_V2_DELAY_REQ = 1, + HCLGE_PTP_MSG0_V2_PDELAY_REQ, + HCLGE_PTP_MSG0_V2_DELAY_RESP, + HCLGE_PTP_MSG0_V2_EVENT = 0xF, +}; + +#define HCLGE_PTP_MSG1_V2_DEFAULT 1 + +struct hclge_ptp_cfg_cmd { +#define HCLGE_PTP_EN_B BIT(0) +#define HCLGE_PTP_TX_EN_B BIT(1) +#define HCLGE_PTP_RX_EN_B BIT(2) +#define HCLGE_PTP_UDP_EN_SHIFT 3 +#define HCLGE_PTP_UDP_EN_MASK GENMASK(4, 3) +#define HCLGE_PTP_MSG_TYPE_SHIFT 8 +#define HCLGE_PTP_MSG_TYPE_MASK GENMASK(9, 8) +#define HCLGE_PTP_MSG1_SHIFT 16 +#define HCLGE_PTP_MSG1_MASK GENMASK(19, 16) +#define HCLGE_PTP_MSG0_SHIFT 24 +#define HCLGE_PTP_MSG0_MASK GENMASK(27, 24) + + __le32 cfg; + u8 rsvd[20]; +}; + +static inline struct hclge_dev *hclge_ptp_get_hdev(struct ptp_clock_info *info) +{ + struct hclge_ptp *ptp = container_of(info, struct hclge_ptp, info); + + return ptp->hdev; +} + +bool hclge_ptp_set_tx_info(struct hnae3_handle *handle, struct sk_buff *skb); +void hclge_ptp_clean_tx_hwts(struct hclge_dev *dev); +void hclge_ptp_get_rx_hwts(struct hnae3_handle *handle, struct sk_buff *skb, + u32 nsec, u32 sec); +int hclge_ptp_get_cfg(struct hclge_dev *hdev, struct ifreq *ifr); +int hclge_ptp_set_cfg(struct hclge_dev *hdev, struct ifreq *ifr); +int hclge_ptp_init(struct hclge_dev *hdev); +void hclge_ptp_uninit(struct hclge_dev *hdev); +int hclge_ptp_get_ts_info(struct hnae3_handle *handle, + struct ethtool_ts_info *info); +int hclge_ptp_cfg_qry(struct hclge_dev *hdev, u32 *cfg); +#endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c index ebb962bad451..78d5bf1ea561 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c @@ -1733,6 +1733,36 @@ int hclge_tm_get_qset_weight(struct hclge_dev *hdev, u16 qset_id, u8 *weight) return 0; } +int hclge_tm_get_qset_shaper(struct hclge_dev *hdev, u16 qset_id, + struct hclge_tm_shaper_para *para) +{ + struct hclge_qs_shapping_cmd *shap_cfg_cmd; + struct hclge_desc desc; + u32 shapping_para; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QCN_SHAPPING_CFG, true); + shap_cfg_cmd = (struct hclge_qs_shapping_cmd *)desc.data; + shap_cfg_cmd->qs_id = cpu_to_le16(qset_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get qset %u shaper, ret = %d\n", qset_id, + ret); + return ret; + } + + shapping_para = le32_to_cpu(shap_cfg_cmd->qs_shapping_para); + para->ir_b = hclge_tm_get_field(shapping_para, IR_B); + para->ir_u = hclge_tm_get_field(shapping_para, IR_U); + para->ir_s = hclge_tm_get_field(shapping_para, IR_S); + para->bs_b = hclge_tm_get_field(shapping_para, BS_B); + para->bs_s = hclge_tm_get_field(shapping_para, BS_S); + para->flag = shap_cfg_cmd->flag; + para->rate = le32_to_cpu(shap_cfg_cmd->qs_rate); + return 0; +} + int hclge_tm_get_pri_sch_mode(struct hclge_dev *hdev, u8 pri_id, u8 *mode) { struct hclge_pri_sch_mode_cfg_cmd *pri_sch_mode; @@ -1775,7 +1805,7 @@ int hclge_tm_get_pri_weight(struct hclge_dev *hdev, u8 pri_id, u8 *weight) int hclge_tm_get_pri_shaper(struct hclge_dev *hdev, u8 pri_id, enum hclge_opcode_type cmd, - struct hclge_pri_shaper_para *para) + struct hclge_tm_shaper_para *para) { struct hclge_pri_shapping_cmd *shap_cfg_cmd; struct hclge_desc desc; @@ -1807,3 +1837,186 @@ int hclge_tm_get_pri_shaper(struct hclge_dev *hdev, u8 pri_id, para->rate = le32_to_cpu(shap_cfg_cmd->pri_rate); return 0; } + +int hclge_tm_get_q_to_qs_map(struct hclge_dev *hdev, u16 q_id, u16 *qset_id) +{ + struct hclge_nq_to_qs_link_cmd *map; + struct hclge_desc desc; + u16 qs_id_l; + u16 qs_id_h; + int ret; + + map = (struct hclge_nq_to_qs_link_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_NQ_TO_QS_LINK, true); + map->nq_id = cpu_to_le16(q_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get queue to qset map, ret = %d\n", ret); + return ret; + } + *qset_id = le16_to_cpu(map->qset_id); + + /* convert qset_id to the following format, drop the vld bit + * | qs_id_h | vld | qs_id_l | + * qset_id: | 15 ~ 11 | 10 | 9 ~ 0 | + * \ \ / / + * \ \ / / + * qset_id: | 15 | 14 ~ 10 | 9 ~ 0 | + */ + qs_id_l = hnae3_get_field(*qset_id, HCLGE_TM_QS_ID_L_MSK, + HCLGE_TM_QS_ID_L_S); + qs_id_h = hnae3_get_field(*qset_id, HCLGE_TM_QS_ID_H_EXT_MSK, + HCLGE_TM_QS_ID_H_EXT_S); + *qset_id = 0; + hnae3_set_field(*qset_id, HCLGE_TM_QS_ID_L_MSK, HCLGE_TM_QS_ID_L_S, + qs_id_l); + hnae3_set_field(*qset_id, HCLGE_TM_QS_ID_H_MSK, HCLGE_TM_QS_ID_H_S, + qs_id_h); + return 0; +} + +int hclge_tm_get_q_to_tc(struct hclge_dev *hdev, u16 q_id, u8 *tc_id) +{ +#define HCLGE_TM_TC_MASK 0x7 + + struct hclge_tqp_tx_queue_tc_cmd *tc; + struct hclge_desc desc; + int ret; + + tc = (struct hclge_tqp_tx_queue_tc_cmd *)desc.data; + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TQP_TX_QUEUE_TC, true); + tc->queue_id = cpu_to_le16(q_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get queue to tc map, ret = %d\n", ret); + return ret; + } + + *tc_id = tc->tc_id & HCLGE_TM_TC_MASK; + return 0; +} + +int hclge_tm_get_pg_to_pri_map(struct hclge_dev *hdev, u8 pg_id, + u8 *pri_bit_map) +{ + struct hclge_pg_to_pri_link_cmd *map; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PG_TO_PRI_LINK, true); + map = (struct hclge_pg_to_pri_link_cmd *)desc.data; + map->pg_id = pg_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get pg to pri map, ret = %d\n", ret); + return ret; + } + + *pri_bit_map = map->pri_bit_map; + return 0; +} + +int hclge_tm_get_pg_weight(struct hclge_dev *hdev, u8 pg_id, u8 *weight) +{ + struct hclge_pg_weight_cmd *pg_weight_cmd; + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PG_WEIGHT, true); + pg_weight_cmd = (struct hclge_pg_weight_cmd *)desc.data; + pg_weight_cmd->pg_id = pg_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get pg weight, ret = %d\n", ret); + return ret; + } + + *weight = pg_weight_cmd->dwrr; + return 0; +} + +int hclge_tm_get_pg_sch_mode(struct hclge_dev *hdev, u8 pg_id, u8 *mode) +{ + struct hclge_desc desc; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PG_SCH_MODE_CFG, true); + desc.data[0] = cpu_to_le32(pg_id); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get pg sch mode, ret = %d\n", ret); + return ret; + } + + *mode = (u8)le32_to_cpu(desc.data[1]); + return 0; +} + +int hclge_tm_get_pg_shaper(struct hclge_dev *hdev, u8 pg_id, + enum hclge_opcode_type cmd, + struct hclge_tm_shaper_para *para) +{ + struct hclge_pg_shapping_cmd *shap_cfg_cmd; + struct hclge_desc desc; + u32 shapping_para; + int ret; + + if (cmd != HCLGE_OPC_TM_PG_C_SHAPPING && + cmd != HCLGE_OPC_TM_PG_P_SHAPPING) + return -EINVAL; + + hclge_cmd_setup_basic_desc(&desc, cmd, true); + shap_cfg_cmd = (struct hclge_pg_shapping_cmd *)desc.data; + shap_cfg_cmd->pg_id = pg_id; + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get pg shaper(%#x), ret = %d\n", + cmd, ret); + return ret; + } + + shapping_para = le32_to_cpu(shap_cfg_cmd->pg_shapping_para); + para->ir_b = hclge_tm_get_field(shapping_para, IR_B); + para->ir_u = hclge_tm_get_field(shapping_para, IR_U); + para->ir_s = hclge_tm_get_field(shapping_para, IR_S); + para->bs_b = hclge_tm_get_field(shapping_para, BS_B); + para->bs_s = hclge_tm_get_field(shapping_para, BS_S); + para->flag = shap_cfg_cmd->flag; + para->rate = le32_to_cpu(shap_cfg_cmd->pg_rate); + return 0; +} + +int hclge_tm_get_port_shaper(struct hclge_dev *hdev, + struct hclge_tm_shaper_para *para) +{ + struct hclge_port_shapping_cmd *port_shap_cfg_cmd; + struct hclge_desc desc; + u32 shapping_para; + int ret; + + hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_TM_PORT_SHAPPING, true); + ret = hclge_cmd_send(&hdev->hw, &desc, 1); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to get port shaper, ret = %d\n", ret); + return ret; + } + + port_shap_cfg_cmd = (struct hclge_port_shapping_cmd *)desc.data; + shapping_para = le32_to_cpu(port_shap_cfg_cmd->port_shapping_para); + para->ir_b = hclge_tm_get_field(shapping_para, IR_B); + para->ir_u = hclge_tm_get_field(shapping_para, IR_U); + para->ir_s = hclge_tm_get_field(shapping_para, IR_S); + para->bs_b = hclge_tm_get_field(shapping_para, BS_B); + para->bs_s = hclge_tm_get_field(shapping_para, BS_S); + para->flag = port_shap_cfg_cmd->flag; + para->rate = le32_to_cpu(port_shap_cfg_cmd->port_rate); + + return 0; +} diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h index b25d76023af0..2ee9b795f71d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h @@ -199,14 +199,14 @@ struct hclge_tm_nodes_cmd { __le16 queue_num; }; -struct hclge_pri_shaper_para { +struct hclge_tm_shaper_para { + u32 rate; u8 ir_b; u8 ir_u; u8 ir_s; u8 bs_b; u8 bs_s; u8 flag; - u32 rate; }; #define hclge_tm_set_field(dest, string, val) \ @@ -237,9 +237,22 @@ int hclge_tm_get_qset_map_pri(struct hclge_dev *hdev, u16 qset_id, u8 *priority, u8 *link_vld); int hclge_tm_get_qset_sch_mode(struct hclge_dev *hdev, u16 qset_id, u8 *mode); int hclge_tm_get_qset_weight(struct hclge_dev *hdev, u16 qset_id, u8 *weight); +int hclge_tm_get_qset_shaper(struct hclge_dev *hdev, u16 qset_id, + struct hclge_tm_shaper_para *para); int hclge_tm_get_pri_sch_mode(struct hclge_dev *hdev, u8 pri_id, u8 *mode); int hclge_tm_get_pri_weight(struct hclge_dev *hdev, u8 pri_id, u8 *weight); int hclge_tm_get_pri_shaper(struct hclge_dev *hdev, u8 pri_id, enum hclge_opcode_type cmd, - struct hclge_pri_shaper_para *para); + struct hclge_tm_shaper_para *para); +int hclge_tm_get_q_to_qs_map(struct hclge_dev *hdev, u16 q_id, u16 *qset_id); +int hclge_tm_get_q_to_tc(struct hclge_dev *hdev, u16 q_id, u8 *tc_id); +int hclge_tm_get_pg_to_pri_map(struct hclge_dev *hdev, u8 pg_id, + u8 *pri_bit_map); +int hclge_tm_get_pg_weight(struct hclge_dev *hdev, u8 pg_id, u8 *weight); +int hclge_tm_get_pg_sch_mode(struct hclge_dev *hdev, u8 pg_id, u8 *mode); +int hclge_tm_get_pg_shaper(struct hclge_dev *hdev, u8 pg_id, + enum hclge_opcode_type cmd, + struct hclge_tm_shaper_para *para); +int hclge_tm_get_port_shaper(struct hclge_dev *hdev, + struct hclge_tm_shaper_para *para); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c index d8c5c5810b99..bd19a2d89f6c 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c @@ -359,6 +359,8 @@ static void hclgevf_parse_capability(struct hclgevf_dev *hdev, set_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps); if (hnae3_get_bit(caps, HCLGEVF_CAP_UDP_TUNNEL_CSUM_B)) set_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps); + if (hnae3_get_bit(caps, HCLGEVF_CAP_RXD_ADV_LAYOUT_B)) + set_bit(HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B, ae_dev->caps); } static __le32 hclgevf_build_api_caps(void) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h index c6dc11b32aa7..202feb70dba5 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h @@ -159,6 +159,7 @@ enum HCLGEVF_CAP_BITS { HCLGEVF_CAP_HW_PAD_B, HCLGEVF_CAP_STASH_B, HCLGEVF_CAP_UDP_TUNNEL_CSUM_B, + HCLGEVF_CAP_RXD_ADV_LAYOUT_B = 15, }; enum HCLGEVF_API_CAP_BITS { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 0db51ef15ef6..52eaf82b7cd7 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -243,23 +243,31 @@ static void hclgevf_build_send_msg(struct hclge_vf_to_pf_msg *msg, u8 code, } } -static int hclgevf_get_tc_info(struct hclgevf_dev *hdev) +static int hclgevf_get_basic_info(struct hclgevf_dev *hdev) { + struct hnae3_ae_dev *ae_dev = hdev->ae_dev; + u8 resp_msg[HCLGE_MBX_MAX_RESP_DATA_SIZE]; + struct hclge_basic_info *basic_info; struct hclge_vf_to_pf_msg send_msg; - u8 resp_msg; + unsigned long caps; int status; - hclgevf_build_send_msg(&send_msg, HCLGE_MBX_GET_TCINFO, 0); - status = hclgevf_send_mbx_msg(hdev, &send_msg, true, &resp_msg, + hclgevf_build_send_msg(&send_msg, HCLGE_MBX_GET_BASIC_INFO, 0); + status = hclgevf_send_mbx_msg(hdev, &send_msg, true, resp_msg, sizeof(resp_msg)); if (status) { dev_err(&hdev->pdev->dev, - "VF request to get TC info from PF failed %d", - status); + "failed to get basic info from pf, ret = %d", status); return status; } - hdev->hw_tc_map = resp_msg; + basic_info = (struct hclge_basic_info *)resp_msg; + + hdev->hw_tc_map = basic_info->hw_tc_map; + hdev->mbx_api_version = basic_info->mbx_api_version; + caps = basic_info->pf_caps; + if (test_bit(HNAE3_PF_SUPPORT_VLAN_FLTR_MDF_B, &caps)) + set_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps); return 0; } @@ -1528,8 +1536,7 @@ static void hclgevf_sync_from_add_list(struct list_head *add_list, kfree(mac_node); } else if (mac_node->state == HCLGEVF_MAC_ACTIVE) { mac_node->state = HCLGEVF_MAC_TO_DEL; - list_del(&mac_node->node); - list_add_tail(&mac_node->node, mac_list); + list_move_tail(&mac_node->node, mac_list); } else { list_del(&mac_node->node); kfree(mac_node); @@ -1554,8 +1561,7 @@ static void hclgevf_sync_from_del_list(struct list_head *del_list, list_del(&mac_node->node); kfree(mac_node); } else { - list_del(&mac_node->node); - list_add_tail(&mac_node->node, mac_list); + list_move_tail(&mac_node->node, mac_list); } } } @@ -1591,8 +1597,7 @@ static void hclgevf_sync_mac_list(struct hclgevf_dev *hdev, list_for_each_entry_safe(mac_node, tmp, list, node) { switch (mac_node->state) { case HCLGEVF_MAC_TO_DEL: - list_del(&mac_node->node); - list_add_tail(&mac_node->node, &tmp_del_list); + list_move_tail(&mac_node->node, &tmp_del_list); break; case HCLGEVF_MAC_TO_ADD: new_node = kzalloc(sizeof(*new_node), GFP_ATOMIC); @@ -1642,6 +1647,22 @@ static void hclgevf_uninit_mac_list(struct hclgevf_dev *hdev) spin_unlock_bh(&hdev->mac_table.mac_list_lock); } +static int hclgevf_enable_vlan_filter(struct hnae3_handle *handle, bool enable) +{ + struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); + struct hnae3_ae_dev *ae_dev = hdev->ae_dev; + struct hclge_vf_to_pf_msg send_msg; + + if (!test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps)) + return -EOPNOTSUPP; + + hclgevf_build_send_msg(&send_msg, HCLGE_MBX_SET_VLAN, + HCLGE_MBX_ENABLE_VLAN_FILTER); + send_msg.data[0] = enable ? 1 : 0; + + return hclgevf_send_mbx_msg(hdev, &send_msg, true, NULL, 0); +} + static int hclgevf_set_vlan_filter(struct hnae3_handle *handle, __be16 proto, u16 vlan_id, bool is_kill) @@ -2466,6 +2487,10 @@ static int hclgevf_configure(struct hclgevf_dev *hdev) { int ret; + ret = hclgevf_get_basic_info(hdev); + if (ret) + return ret; + /* get current port based vlan state from PF */ ret = hclgevf_get_port_base_vlan_filter_state(hdev); if (ret) @@ -2481,12 +2506,7 @@ static int hclgevf_configure(struct hclgevf_dev *hdev) if (ret) return ret; - ret = hclgevf_get_pf_media_type(hdev); - if (ret) - return ret; - - /* get tc configuration from PF */ - return hclgevf_get_tc_info(hdev); + return hclgevf_get_pf_media_type(hdev); } static int hclgevf_alloc_hdev(struct hnae3_ae_dev *ae_dev) @@ -3242,6 +3262,18 @@ static int hclgevf_clear_vport_list(struct hclgevf_dev *hdev) return hclgevf_send_mbx_msg(hdev, &send_msg, false, NULL, 0); } +static void hclgevf_init_rxd_adv_layout(struct hclgevf_dev *hdev) +{ + if (hnae3_ae_dev_rxd_adv_layout_supported(hdev->ae_dev)) + hclgevf_write_dev(&hdev->hw, HCLGEVF_RXD_ADV_LAYOUT_EN_REG, 1); +} + +static void hclgevf_uninit_rxd_adv_layout(struct hclgevf_dev *hdev) +{ + if (hnae3_ae_dev_rxd_adv_layout_supported(hdev->ae_dev)) + hclgevf_write_dev(&hdev->hw, HCLGEVF_RXD_ADV_LAYOUT_EN_REG, 0); +} + static int hclgevf_reset_hdev(struct hclgevf_dev *hdev) { struct pci_dev *pdev = hdev->pdev; @@ -3279,6 +3311,8 @@ static int hclgevf_reset_hdev(struct hclgevf_dev *hdev) set_bit(HCLGEVF_STATE_PROMISC_CHANGED, &hdev->state); + hclgevf_init_rxd_adv_layout(hdev); + dev_info(&hdev->pdev->dev, "Reset done\n"); return 0; @@ -3379,6 +3413,8 @@ static int hclgevf_init_hdev(struct hclgevf_dev *hdev) goto err_config; } + hclgevf_init_rxd_adv_layout(hdev); + hdev->last_reset_time = jiffies; dev_info(&hdev->pdev->dev, "finished initializing %s driver\n", HCLGEVF_DRIVER_NAME); @@ -3405,6 +3441,7 @@ static void hclgevf_uninit_hdev(struct hclgevf_dev *hdev) struct hclge_vf_to_pf_msg send_msg; hclgevf_state_uninit(hdev); + hclgevf_uninit_rxd_adv_layout(hdev); hclgevf_build_send_msg(&send_msg, HCLGE_MBX_VF_UNINIT, 0); hclgevf_send_mbx_msg(hdev, &send_msg, false, NULL, 0); @@ -3784,6 +3821,7 @@ static const struct hnae3_ae_ops hclgevf_ops = { .get_tc_size = hclgevf_get_tc_size, .get_fw_version = hclgevf_get_fw_version, .set_vlan_filter = hclgevf_set_vlan_filter, + .enable_vlan_filter = hclgevf_enable_vlan_filter, .enable_hw_strip_rxvtag = hclgevf_en_hw_strip_rxvtag, .reset_event = hclgevf_reset_event, .set_default_reset_request = hclgevf_set_def_reset_request, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h index 265c9b0b4728..d7d02848d674 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h @@ -47,6 +47,7 @@ /* bar registers for common func */ #define HCLGEVF_GRO_EN_REG 0x28000 +#define HCLGEVF_RXD_ADV_LAYOUT_EN_REG 0x28008 /* bar registers for rcb */ #define HCLGEVF_RING_RX_ADDR_L_REG 0x80000 @@ -284,6 +285,7 @@ struct hclgevf_dev { struct semaphore reset_sem; /* protect reset process */ u32 fw_version; + u16 mbx_api_version; u16 num_tqps; /* num task queue pairs of this VF */ u16 alloc_rss_size; /* allocated RSS task queue */ diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c index dc024ef521c0..162d3c330dec 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c @@ -1663,7 +1663,6 @@ static void hinic_diag_test(struct net_device *netdev, err = hinic_port_link_state(nic_dev, &link_state); if (!err && link_state == HINIC_LINK_STATE_UP) netif_carrier_on(netdev); - } static int hinic_set_phys_id(struct net_device *netdev, diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c index 5a6bbee819cd..307a6d4af993 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c @@ -223,7 +223,7 @@ static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped, saved_data = CMDQ_WQE_HEADER(wqe)->saved_data; saved_data = HINIC_SAVED_DATA_CLEAR(saved_data, ARM); - if ((cmd == CMDQ_SET_ARM_CMD) && (mod == HINIC_MOD_COMM)) + if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM) CMDQ_WQE_HEADER(wqe)->saved_data |= HINIC_SAVED_DATA_SET(1, ARM); else @@ -594,7 +594,7 @@ static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx, } /** - * cmdq_arm_ceq_handler - cmdq completion event handler for sync command + * cmdq_sync_cmd_handler - cmdq completion event handler for sync command * @cmdq: the cmdq of the command * @cons_idx: the consumer index to update the error code for * @errcode: the error code diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c index 0c74f6674634..428108eb10d2 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c @@ -48,7 +48,7 @@ enum io_status { }; /** - * get_capability - convert device capabilities to NIC capabilities + * parse_capability - convert device capabilities to NIC capabilities * @hwdev: the HW device to set and convert device capabilities for * @dev_cap: device capabilities from FW * @@ -92,7 +92,7 @@ static int parse_capability(struct hinic_hwdev *hwdev, } /** - * get_cap_from_fw - get device capabilities from FW + * get_capability - get device capabilities from FW * @pfhwdev: the PF HW device to get capabilities for * * Return 0 - Success, negative - Failure @@ -257,7 +257,7 @@ static int init_fw_ctxt(struct hinic_hwdev *hwdev) err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_FWCTXT_INIT, &fw_ctxt, sizeof(fw_ctxt), &fw_ctxt, &out_size); - if (err || (out_size != sizeof(fw_ctxt)) || fw_ctxt.status) { + if (err || out_size != sizeof(fw_ctxt) || fw_ctxt.status) { dev_err(&pdev->dev, "Failed to init FW ctxt, err: %d, status: 0x%x, out size: 0x%x\n", err, fw_ctxt.status, out_size); return -EIO; @@ -346,7 +346,7 @@ static int wait_for_db_state(struct hinic_hwdev *hwdev) } /** - * clear_io_resource - set the IO resources as not active in the NIC + * clear_io_resources - set the IO resources as not active in the NIC * @hwdev: the NIC HW device * * Return 0 - Success, negative - Failure @@ -424,7 +424,7 @@ static int get_base_qpn(struct hinic_hwdev *hwdev, u16 *base_qpn) err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_GLOBAL_QPN, &cmd_base_qpn, sizeof(cmd_base_qpn), &cmd_base_qpn, &out_size); - if (err || (out_size != sizeof(cmd_base_qpn)) || cmd_base_qpn.status) { + if (err || out_size != sizeof(cmd_base_qpn) || cmd_base_qpn.status) { dev_err(&pdev->dev, "Failed to get base qpn, err: %d, status: 0x%x, out size: 0x%x\n", err, cmd_base_qpn.status, out_size); return -EIO; @@ -605,8 +605,8 @@ static void nic_mgmt_msg_handler(void *handle, u8 cmd, void *buf_in, hwif = hwdev->hwif; pdev = hwif->pdev; - if ((cmd < HINIC_MGMT_MSG_CMD_BASE) || - (cmd >= HINIC_MGMT_MSG_CMD_MAX)) { + if (cmd < HINIC_MGMT_MSG_CMD_BASE || + cmd >= HINIC_MGMT_MSG_CMD_MAX) { dev_err(&pdev->dev, "unknown L2NIC event, cmd = %d\n", cmd); return; } @@ -619,7 +619,7 @@ static void nic_mgmt_msg_handler(void *handle, u8 cmd, void *buf_in, HINIC_CB_ENABLED, HINIC_CB_ENABLED | HINIC_CB_RUNNING); - if ((cb_state == HINIC_CB_ENABLED) && (nic_cb->handler)) + if (cb_state == HINIC_CB_ENABLED && nic_cb->handler) nic_cb->handler(nic_cb->handle, buf_in, in_size, buf_out, out_size); else @@ -1090,7 +1090,7 @@ struct hinic_sq *hinic_hwdev_get_sq(struct hinic_hwdev *hwdev, int i) } /** - * hinic_hwdev_get_sq - get RQ + * hinic_hwdev_get_rq - get RQ * @hwdev: the NIC HW device * @i: the position of the RQ * diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c index 19942fef99d9..d3fc05a07fdb 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c @@ -254,8 +254,8 @@ static void aeq_irq_handler(struct hinic_eq *eq) HINIC_EQE_ENABLED, HINIC_EQE_ENABLED | HINIC_EQE_RUNNING); - if ((eqe_state == HINIC_EQE_ENABLED) && - (hwe_cb->hwe_handler)) + if (eqe_state == HINIC_EQE_ENABLED && + hwe_cb->hwe_handler) hwe_cb->hwe_handler(hwe_cb->handle, aeqe_curr->data, size); else @@ -299,7 +299,7 @@ static void ceq_event_handler(struct hinic_ceqs *ceqs, u32 ceqe) HINIC_EQE_ENABLED, HINIC_EQE_ENABLED | HINIC_EQE_RUNNING); - if ((eqe_state == HINIC_EQE_ENABLED) && (ceq_cb->handler)) + if (eqe_state == HINIC_EQE_ENABLED && ceq_cb->handler) ceq_cb->handler(ceq_cb->handle, CEQE_DATA(ceqe)); else dev_err(&pdev->dev, "Unhandled CEQ Event %d\n", event); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c index cab38ff0713c..0428faa68e80 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c @@ -334,7 +334,7 @@ static void set_dma_attr(struct hinic_hwif *hwif, u32 entry_idx, } /** - * dma_attr_table_init - initialize the default dma attributes + * dma_attr_init - initialize the default dma attributes * @hwif: the HW interface of a pci function device **/ static void dma_attr_init(struct hinic_hwif *hwif) @@ -395,7 +395,7 @@ static void __print_selftest_reg(struct hinic_hwif *hwif) /** * hinic_init_hwif - initialize the hw interface * @hwif: the HW interface of a pci function device - * @pdev: the pci device for acessing PCI resources + * @pdev: the pci device for accessing PCI resources * * Return 0 - Success, negative - Failure **/ diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c index 4ef4008e65bd..a6e43d686293 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c @@ -137,7 +137,7 @@ static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn, err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC, IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf, &out_param); - if ((err) || (out_param != 0)) { + if (err || out_param != 0) { dev_err(&pdev->dev, "Failed to set SQ ctxts\n"); err = -EFAULT; } @@ -181,7 +181,7 @@ static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn, err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC, IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf, &out_param); - if ((err) || (out_param != 0)) { + if (err || out_param != 0) { dev_err(&pdev->dev, "Failed to set RQ ctxts\n"); err = -EFAULT; } diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c index 817173f1fbb7..ebc77771f5da 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c @@ -294,7 +294,7 @@ static int msg_to_mgmt_sync(struct hinic_pf_to_mgmt *pf_to_mgmt, goto unlock_sync_msg; } - if ((buf_out) && (recv_msg->msg_len <= MAX_PF_MGMT_BUF_SIZE)) { + if (buf_out && recv_msg->msg_len <= MAX_PF_MGMT_BUF_SIZE) { memcpy(buf_out, recv_msg->msg, recv_msg->msg_len); *out_size = recv_msg->msg_len; } @@ -411,7 +411,7 @@ static void recv_mgmt_msg_work_handler(struct work_struct *work) HINIC_MGMT_CB_ENABLED, HINIC_MGMT_CB_ENABLED | HINIC_MGMT_CB_RUNNING); - if ((cb_state == HINIC_MGMT_CB_ENABLED) && (mgmt_cb->cb)) + if (cb_state == HINIC_MGMT_CB_ENABLED && mgmt_cb->cb) mgmt_cb->cb(mgmt_cb->handle, mgmt_work->cmd, mgmt_work->msg, mgmt_work->msg_len, buf_out, &out_size); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c index dcba4d009bad..336248aa2e48 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c @@ -894,7 +894,7 @@ struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq, } /** - * hinic_put_wqe - release the ci for new wqes + * hinic_rq_put_wqe - release the ci for new wqes * @rq: recv queue * @cons_idx: consumer index of the wqe * @wqe_size: the size of the wqe diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c index 5dc3743f8091..7f0f1aa3cedd 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c @@ -89,6 +89,7 @@ static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) return (((idx) >> ((wq)->wqebbs_per_page_shift)) & ((wq)->num_q_pages - 1)); } + /** * queue_alloc_page - allocate page for Queue * @hwif: HW interface for allocating DMA diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c index 9a9b09401d01..405ee4d2d2b1 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_main.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c @@ -172,7 +172,6 @@ static int create_txqs(struct hinic_dev *nic_dev) "Failed to add SQ%d debug\n", i); goto err_add_sq_dbg; } - } return 0; @@ -233,7 +232,7 @@ static void free_txqs(struct hinic_dev *nic_dev) } /** - * create_txqs - Create the Logical Rx Queues of specific NIC device + * create_rxqs - Create the Logical Rx Queues of specific NIC device * @nic_dev: the specific NIC device * * Return 0 - Success, negative - Failure @@ -289,7 +288,7 @@ err_init_rxq: } /** - * free_txqs - Free the Logical Rx Queues of specific NIC device + * free_rxqs - Free the Logical Rx Queues of specific NIC device * @nic_dev: the specific NIC device **/ static void free_rxqs(struct hinic_dev *nic_dev) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_port.c b/drivers/net/ethernet/huawei/hinic/hinic_port.c index eb97f2d6b1ad..28ae6f1201a8 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_port.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_port.c @@ -128,7 +128,7 @@ int hinic_port_get_mac(struct hinic_dev *nic_dev, u8 *addr) err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_MAC, &port_mac_cmd, sizeof(port_mac_cmd), &port_mac_cmd, &out_size); - if (err || (out_size != sizeof(port_mac_cmd)) || port_mac_cmd.status) { + if (err || out_size != sizeof(port_mac_cmd) || port_mac_cmd.status) { dev_err(&pdev->dev, "Failed to get mac, err: %d, status: 0x%x, out size: 0x%x\n", err, port_mac_cmd.status, out_size); return -EFAULT; @@ -263,7 +263,7 @@ int hinic_port_link_state(struct hinic_dev *nic_dev, err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_LINK_STATE, &link_cmd, sizeof(link_cmd), &link_cmd, &out_size); - if (err || (out_size != sizeof(link_cmd)) || link_cmd.status) { + if (err || out_size != sizeof(link_cmd) || link_cmd.status) { dev_err(&pdev->dev, "Failed to get link state, err: %d, status: 0x%x, out size: 0x%x\n", err, link_cmd.status, out_size); return -EINVAL; @@ -297,7 +297,7 @@ int hinic_port_set_state(struct hinic_dev *nic_dev, enum hinic_port_state state) err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_SET_PORT_STATE, &port_state, sizeof(port_state), &port_state, &out_size); - if (err || (out_size != sizeof(port_state)) || port_state.status) { + if (err || out_size != sizeof(port_state) || port_state.status) { dev_err(&pdev->dev, "Failed to set port state, err: %d, status: 0x%x, out size: 0x%x\n", err, port_state.status, out_size); return -EFAULT; @@ -329,7 +329,7 @@ int hinic_port_set_func_state(struct hinic_dev *nic_dev, err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_SET_FUNC_STATE, &func_state, sizeof(func_state), &func_state, &out_size); - if (err || (out_size != sizeof(func_state)) || func_state.status) { + if (err || out_size != sizeof(func_state) || func_state.status) { dev_err(&pdev->dev, "Failed to set port func state, err: %d, status: 0x%x, out size: 0x%x\n", err, func_state.status, out_size); return -EFAULT; @@ -359,7 +359,7 @@ int hinic_port_get_cap(struct hinic_dev *nic_dev, err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_GET_CAP, port_cap, sizeof(*port_cap), port_cap, &out_size); - if (err || (out_size != sizeof(*port_cap)) || port_cap->status) { + if (err || out_size != sizeof(*port_cap) || port_cap->status) { dev_err(&pdev->dev, "Failed to get port capabilities, err: %d, status: 0x%x, out size: 0x%x\n", err, port_cap->status, out_size); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c index cce08647b9b2..fed3b6bc0d76 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_rx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c @@ -118,6 +118,7 @@ static void rx_csum(struct hinic_rxq *rxq, u32 status, skb->ip_summed = CHECKSUM_NONE; } } + /** * rx_alloc_skb - allocate skb and map it to dma address * @rxq: rx queue diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c index 710c4ff7bc0e..c5bdb0d374ef 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c @@ -660,7 +660,7 @@ static void tx_free_skb(struct hinic_dev *nic_dev, struct sk_buff *skb, } /** - * free_all_rx_skbs - free all skbs in tx queue + * free_all_tx_skbs - free all skbs in tx queue * @txq: tx queue **/ static void free_all_tx_skbs(struct hinic_txq *txq) @@ -717,7 +717,7 @@ static int free_tx_poll(struct napi_struct *napi, int budget) /* Reading a WQEBB to get real WQE size and consumer index. */ sq_wqe = hinic_sq_read_wqebb(sq, &skb, &wqe_size, &sw_ci); - if ((!sq_wqe) || + if (!sq_wqe || (((hw_ci - sw_ci) & wq->mask) * wq->wqebb_size < wqe_size)) break; diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c index ea55314b209d..d5df131b183c 100644 --- a/drivers/net/ethernet/ibm/ehea/ehea_main.c +++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c @@ -2618,10 +2618,8 @@ static int ehea_restart_qps(struct net_device *dev) u16 dummy16 = 0; cb0 = (void *)get_zeroed_page(GFP_KERNEL); - if (!cb0) { - ret = -ENOMEM; - goto out; - } + if (!cb0) + return -ENOMEM; for (i = 0; i < (port->num_def_qps); i++) { struct ehea_port_res *pr = &port->port_res[i]; @@ -2641,6 +2639,7 @@ static int ehea_restart_qps(struct net_device *dev) cb0); if (hret != H_SUCCESS) { netdev_err(dev, "query_ehea_qp failed (1)\n"); + ret = -EFAULT; goto out; } @@ -2653,6 +2652,7 @@ static int ehea_restart_qps(struct net_device *dev) &dummy64, &dummy16, &dummy16); if (hret != H_SUCCESS) { netdev_err(dev, "modify_ehea_qp failed (1)\n"); + ret = -EFAULT; goto out; } @@ -2661,6 +2661,7 @@ static int ehea_restart_qps(struct net_device *dev) cb0); if (hret != H_SUCCESS) { netdev_err(dev, "query_ehea_qp failed (2)\n"); + ret = -EFAULT; goto out; } @@ -2867,14 +2868,14 @@ out: return ret; } -static ssize_t ehea_show_port_id(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t log_port_id_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct ehea_port *port = container_of(dev, struct ehea_port, ofdev.dev); return sprintf(buf, "%d", port->logical_port_id); } -static DEVICE_ATTR(log_port_id, 0444, ehea_show_port_id, NULL); +static DEVICE_ATTR_RO(log_port_id); static void logical_port_release(struct device *dev) { @@ -3113,7 +3114,7 @@ static struct device_node *ehea_get_eth_dn(struct ehea_adapter *adapter, return NULL; } -static ssize_t ehea_probe_port(struct device *dev, +static ssize_t probe_port_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -3168,9 +3169,9 @@ static ssize_t ehea_probe_port(struct device *dev, return (ssize_t) count; } -static ssize_t ehea_remove_port(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t remove_port_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { struct ehea_adapter *adapter = dev_get_drvdata(dev); struct ehea_port *port; @@ -3203,8 +3204,8 @@ static ssize_t ehea_remove_port(struct device *dev, return (ssize_t) count; } -static DEVICE_ATTR(probe_port, 0200, NULL, ehea_probe_port); -static DEVICE_ATTR(remove_port, 0200, NULL, ehea_remove_port); +static DEVICE_ATTR_WO(probe_port); +static DEVICE_ATTR_WO(remove_port); static int ehea_create_device_sysfs(struct platform_device *dev) { diff --git a/drivers/net/ethernet/ibm/emac/emac.h b/drivers/net/ethernet/ibm/emac/emac.h index aa9f651288d5..09d3ac374b2d 100644 --- a/drivers/net/ethernet/ibm/emac/emac.h +++ b/drivers/net/ethernet/ibm/emac/emac.h @@ -77,7 +77,7 @@ struct emac_regs { struct { u32 rsvd1; u32 revid; - u32 rsvd2[2]; + u32 rsvd2[2]; u32 iaht1; /* Reset, R */ u32 iaht2; /* Reset, R */ u32 iaht3; /* Reset, R */ diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 7fea9ae60f13..737ba85e409f 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -1285,36 +1285,41 @@ static void ibmveth_rx_csum_helper(struct sk_buff *skb, iph_proto = iph6->nexthdr; } - /* In OVS environment, when a flow is not cached, specifically for a - * new TCP connection, the first packet information is passed up + /* When CSO is enabled the TCP checksum may have be set to NULL by + * the sender given that we zeroed out TCP checksum field in + * transmit path (refer ibmveth_start_xmit routine). In this case set + * up CHECKSUM_PARTIAL. If the packet is forwarded, the checksum will + * then be recalculated by the destination NIC (CSO must be enabled + * on the destination NIC). + * + * In an OVS environment, when a flow is not cached, specifically for a + * new TCP connection, the first packet information is passed up to * the user space for finding a flow. During this process, OVS computes * checksum on the first packet when CHECKSUM_PARTIAL flag is set. * - * Given that we zeroed out TCP checksum field in transmit path - * (refer ibmveth_start_xmit routine) as we set "no checksum bit", - * OVS computed checksum will be incorrect w/o TCP pseudo checksum - * in the packet. This leads to OVS dropping the packet and hence - * TCP retransmissions are seen. - * - * So, re-compute TCP pseudo header checksum. + * So, re-compute TCP pseudo header checksum when configured for + * trunk mode. */ - if (iph_proto == IPPROTO_TCP && adapter->is_active_trunk) { + if (iph_proto == IPPROTO_TCP) { struct tcphdr *tcph = (struct tcphdr *)(skb->data + iphlen); - - tcphdrlen = skb->len - iphlen; - - /* Recompute TCP pseudo header checksum */ - if (skb_proto == ETH_P_IP) - tcph->check = ~csum_tcpudp_magic(iph->saddr, + if (tcph->check == 0x0000) { + /* Recompute TCP pseudo header checksum */ + if (adapter->is_active_trunk) { + tcphdrlen = skb->len - iphlen; + if (skb_proto == ETH_P_IP) + tcph->check = + ~csum_tcpudp_magic(iph->saddr, iph->daddr, tcphdrlen, iph_proto, 0); - else if (skb_proto == ETH_P_IPV6) - tcph->check = ~csum_ipv6_magic(&iph6->saddr, + else if (skb_proto == ETH_P_IPV6) + tcph->check = + ~csum_ipv6_magic(&iph6->saddr, &iph6->daddr, tcphdrlen, iph_proto, 0); - - /* Setup SKB fields for checksum offload */ - skb_partial_csum_set(skb, iphlen, - offsetof(struct tcphdr, check)); - skb_reset_network_header(skb); + } + /* Setup SKB fields for checksum offload */ + skb_partial_csum_set(skb, iphlen, + offsetof(struct tcphdr, check)); + skb_reset_network_header(skb); + } } } @@ -1799,8 +1804,7 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr, struct ibmveth_buff_pool *pool = container_of(kobj, struct ibmveth_buff_pool, kobj); - struct net_device *netdev = dev_get_drvdata( - container_of(kobj->parent, struct device, kobj)); + struct net_device *netdev = dev_get_drvdata(kobj_to_dev(kobj->parent)); struct ibmveth_adapter *adapter = netdev_priv(netdev); long value = simple_strtol(buf, NULL, 10); long rc; diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 5788bb956d73..374a75d4faea 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -95,7 +95,7 @@ static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *, struct ibmvnic_sub_crq_queue *); static int ibmvnic_poll(struct napi_struct *napi, int data); static void send_query_map(struct ibmvnic_adapter *adapter); -static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8); +static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, u32, u8); static int send_request_unmap(struct ibmvnic_adapter *, u8); static int send_login(struct ibmvnic_adapter *adapter); static void send_query_cap(struct ibmvnic_adapter *adapter); @@ -106,6 +106,8 @@ static void release_crq_queue(struct ibmvnic_adapter *); static int __ibmvnic_set_mac(struct net_device *, u8 *); static int init_crq_queue(struct ibmvnic_adapter *adapter); static int send_query_phys_parms(struct ibmvnic_adapter *adapter); +static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter, + struct ibmvnic_sub_crq_queue *tx_scrq); struct ibmvnic_stat { char name[ETH_GSTRING_LEN]; @@ -141,6 +143,29 @@ static const struct ibmvnic_stat ibmvnic_stats[] = { {"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)}, }; +static int send_crq_init_complete(struct ibmvnic_adapter *adapter) +{ + union ibmvnic_crq crq; + + memset(&crq, 0, sizeof(crq)); + crq.generic.first = IBMVNIC_CRQ_INIT_CMD; + crq.generic.cmd = IBMVNIC_CRQ_INIT_COMPLETE; + + return ibmvnic_send_crq(adapter, &crq); +} + +static int send_version_xchg(struct ibmvnic_adapter *adapter) +{ + union ibmvnic_crq crq; + + memset(&crq, 0, sizeof(crq)); + crq.version_exchange.first = IBMVNIC_CRQ_CMD; + crq.version_exchange.cmd = VERSION_EXCHANGE; + crq.version_exchange.version = cpu_to_be16(ibmvnic_version); + + return ibmvnic_send_crq(adapter, &crq); +} + static long h_reg_sub_crq(unsigned long unit_address, unsigned long token, unsigned long length, unsigned long *number, unsigned long *irq) @@ -209,12 +234,11 @@ static int alloc_long_term_buff(struct ibmvnic_adapter *adapter, mutex_lock(&adapter->fw_lock); adapter->fw_done_rc = 0; reinit_completion(&adapter->fw_done); - rc = send_request_map(adapter, ltb->addr, - ltb->size, ltb->map_id); + + rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id); if (rc) { - dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); - mutex_unlock(&adapter->fw_lock); - return rc; + dev_err(dev, "send_request_map failed, rc = %d\n", rc); + goto out; } rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); @@ -222,20 +246,23 @@ static int alloc_long_term_buff(struct ibmvnic_adapter *adapter, dev_err(dev, "Long term map request aborted or timed out,rc = %d\n", rc); - dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); - mutex_unlock(&adapter->fw_lock); - return rc; + goto out; } if (adapter->fw_done_rc) { dev_err(dev, "Couldn't map long term buffer,rc = %d\n", adapter->fw_done_rc); + rc = -1; + goto out; + } + rc = 0; +out: + if (rc) { dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); - mutex_unlock(&adapter->fw_lock); - return -1; + ltb->buff = NULL; } mutex_unlock(&adapter->fw_lock); - return 0; + return rc; } static void free_long_term_buff(struct ibmvnic_adapter *adapter, @@ -255,14 +282,44 @@ static void free_long_term_buff(struct ibmvnic_adapter *adapter, adapter->reset_reason != VNIC_RESET_TIMEOUT) send_request_unmap(adapter, ltb->map_id); dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr); + ltb->buff = NULL; + ltb->map_id = 0; } -static int reset_long_term_buff(struct ibmvnic_long_term_buff *ltb) +static int reset_long_term_buff(struct ibmvnic_adapter *adapter, + struct ibmvnic_long_term_buff *ltb) { - if (!ltb->buff) - return -EINVAL; + struct device *dev = &adapter->vdev->dev; + int rc; memset(ltb->buff, 0, ltb->size); + + mutex_lock(&adapter->fw_lock); + adapter->fw_done_rc = 0; + + reinit_completion(&adapter->fw_done); + rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id); + if (rc) { + mutex_unlock(&adapter->fw_lock); + return rc; + } + + rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000); + if (rc) { + dev_info(dev, + "Reset failed, long term map request timed out or aborted\n"); + mutex_unlock(&adapter->fw_lock); + return rc; + } + + if (adapter->fw_done_rc) { + dev_info(dev, + "Reset failed, attempting to free and reallocate buffer\n"); + free_long_term_buff(adapter, ltb); + mutex_unlock(&adapter->fw_lock); + return alloc_long_term_buff(adapter, ltb, ltb->size); + } + mutex_unlock(&adapter->fw_lock); return 0; } @@ -298,7 +355,14 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter, rx_scrq = adapter->rx_scrq[pool->index]; ind_bufp = &rx_scrq->ind_buf; - for (i = 0; i < count; ++i) { + + /* netdev_skb_alloc() could have failed after we saved a few skbs + * in the indir_buf and we would not have sent them to VIOS yet. + * To account for them, start the loop at ind_bufp->index rather + * than 0. If we pushed all the skbs to VIOS, ind_bufp->index will + * be 0. + */ + for (i = ind_bufp->index; i < count; ++i) { skb = netdev_alloc_skb(adapter->netdev, pool->buff_size); if (!skb) { dev_err(dev, "Couldn't replenish rx buff\n"); @@ -484,7 +548,8 @@ static int reset_rx_pools(struct ibmvnic_adapter *adapter) rx_pool->size * rx_pool->buff_size); } else { - rc = reset_long_term_buff(&rx_pool->long_term_buff); + rc = reset_long_term_buff(adapter, + &rx_pool->long_term_buff); } if (rc) @@ -607,11 +672,12 @@ static int init_rx_pools(struct net_device *netdev) return 0; } -static int reset_one_tx_pool(struct ibmvnic_tx_pool *tx_pool) +static int reset_one_tx_pool(struct ibmvnic_adapter *adapter, + struct ibmvnic_tx_pool *tx_pool) { int rc, i; - rc = reset_long_term_buff(&tx_pool->long_term_buff); + rc = reset_long_term_buff(adapter, &tx_pool->long_term_buff); if (rc) return rc; @@ -638,10 +704,11 @@ static int reset_tx_pools(struct ibmvnic_adapter *adapter) tx_scrqs = adapter->num_active_tx_pools; for (i = 0; i < tx_scrqs; i++) { - rc = reset_one_tx_pool(&adapter->tso_pool[i]); + ibmvnic_tx_scrq_clean_buffer(adapter, adapter->tx_scrq[i]); + rc = reset_one_tx_pool(adapter, &adapter->tso_pool[i]); if (rc) return rc; - rc = reset_one_tx_pool(&adapter->tx_pool[i]); + rc = reset_one_tx_pool(adapter, &adapter->tx_pool[i]); if (rc) return rc; } @@ -734,8 +801,11 @@ static int init_tx_pools(struct net_device *netdev) adapter->tso_pool = kcalloc(tx_subcrqs, sizeof(struct ibmvnic_tx_pool), GFP_KERNEL); - if (!adapter->tso_pool) + if (!adapter->tso_pool) { + kfree(adapter->tx_pool); + adapter->tx_pool = NULL; return -1; + } adapter->num_active_tx_pools = tx_subcrqs; @@ -846,9 +916,10 @@ static const char *adapter_state_to_string(enum vnic_state state) return "REMOVING"; case VNIC_REMOVED: return "REMOVED"; - default: - return "UNKNOWN"; + case VNIC_DOWN: + return "DOWN"; } + return "UNKNOWN"; } static int ibmvnic_login(struct net_device *netdev) @@ -1180,6 +1251,11 @@ static int __ibmvnic_open(struct net_device *netdev) netif_tx_start_all_queues(netdev); + if (prev_state == VNIC_CLOSED) { + for (i = 0; i < adapter->req_rx_queues; i++) + napi_schedule(&adapter->napi[i]); + } + adapter->state = VNIC_OPEN; return rc; } @@ -1502,7 +1578,8 @@ static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len, /** * build_hdr_descs_arr - build a header descriptor array - * @txbuff: tx buffer + * @skb: tx socket buffer + * @indir_arr: indirect array * @num_entries: number of descriptors to be sent * @hdr_field: bit field determining which headers will be sent * @@ -1583,7 +1660,8 @@ static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter, ind_bufp->index = 0; if (atomic_sub_return(entries, &tx_scrq->used) <= (adapter->req_tx_entries_per_subcrq / 2) && - __netif_subqueue_stopped(adapter->netdev, queue_num)) { + __netif_subqueue_stopped(adapter->netdev, queue_num) && + !test_bit(0, &adapter->resetting)) { netif_wake_subqueue(adapter->netdev, queue_num); netdev_dbg(adapter->netdev, "Started queue %d\n", queue_num); @@ -1676,7 +1754,6 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev) tx_send_failed++; tx_dropped++; ret = NETDEV_TX_OK; - ibmvnic_tx_scrq_flush(adapter, tx_scrq); goto out; } @@ -1946,9 +2023,10 @@ static const char *reset_reason_to_string(enum ibmvnic_reset_reason reason) return "TIMEOUT"; case VNIC_RESET_CHANGE_PARAM: return "CHANGE_PARAM"; - default: - return "UNKNOWN"; + case VNIC_RESET_PASSIVE_INIT: + return "PASSIVE_INIT"; } + return "UNKNOWN"; } /* @@ -2085,10 +2163,10 @@ static int do_reset(struct ibmvnic_adapter *adapter, goto out; } - /* If the adapter was in PROBE state prior to the reset, + /* If the adapter was in PROBE or DOWN state prior to the reset, * exit here. */ - if (reset_state == VNIC_PROBED) { + if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN) { rc = 0; goto out; } @@ -2214,10 +2292,10 @@ static int do_hard_reset(struct ibmvnic_adapter *adapter, if (rc) goto out; - /* If the adapter was in PROBE state prior to the reset, + /* If the adapter was in PROBE or DOWN state prior to the reset, * exit here. */ - if (reset_state == VNIC_PROBED) + if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN) goto out; rc = ibmvnic_login(netdev); @@ -2270,6 +2348,76 @@ static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter) return rwi; } +/** + * do_passive_init - complete probing when partner device is detected. + * @adapter: ibmvnic_adapter struct + * + * If the ibmvnic device does not have a partner device to communicate with at boot + * and that partner device comes online at a later time, this function is called + * to complete the initialization process of ibmvnic device. + * Caller is expected to hold rtnl_lock(). + * + * Returns non-zero if sub-CRQs are not initialized properly leaving the device + * in the down state. + * Returns 0 upon success and the device is in PROBED state. + */ + +static int do_passive_init(struct ibmvnic_adapter *adapter) +{ + unsigned long timeout = msecs_to_jiffies(30000); + struct net_device *netdev = adapter->netdev; + struct device *dev = &adapter->vdev->dev; + int rc; + + netdev_dbg(netdev, "Partner device found, probing.\n"); + + adapter->state = VNIC_PROBING; + reinit_completion(&adapter->init_done); + adapter->init_done_rc = 0; + adapter->crq.active = true; + + rc = send_crq_init_complete(adapter); + if (rc) + goto out; + + rc = send_version_xchg(adapter); + if (rc) + netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc); + + if (!wait_for_completion_timeout(&adapter->init_done, timeout)) { + dev_err(dev, "Initialization sequence timed out\n"); + rc = -ETIMEDOUT; + goto out; + } + + rc = init_sub_crqs(adapter); + if (rc) { + dev_err(dev, "Initialization of sub crqs failed, rc=%d\n", rc); + goto out; + } + + rc = init_sub_crq_irqs(adapter); + if (rc) { + dev_err(dev, "Failed to initialize sub crq irqs\n, rc=%d", rc); + goto init_failed; + } + + netdev->mtu = adapter->req_mtu - ETH_HLEN; + netdev->min_mtu = adapter->min_mtu - ETH_HLEN; + netdev->max_mtu = adapter->max_mtu - ETH_HLEN; + + adapter->state = VNIC_PROBED; + netdev_dbg(netdev, "Probed successfully. Waiting for signal from partner device.\n"); + + return 0; + +init_failed: + release_sub_crqs(adapter, 1); +out: + adapter->state = VNIC_DOWN; + return rc; +} + static void __ibmvnic_reset(struct work_struct *work) { struct ibmvnic_rwi *rwi; @@ -2306,7 +2454,13 @@ static void __ibmvnic_reset(struct work_struct *work) } spin_unlock_irqrestore(&adapter->state_lock, flags); - if (adapter->force_reset_recovery) { + if (rwi->reset_reason == VNIC_RESET_PASSIVE_INIT) { + rtnl_lock(); + rc = do_passive_init(adapter); + rtnl_unlock(); + if (!rc) + netif_carrier_on(adapter->netdev); + } else if (adapter->force_reset_recovery) { /* Since we are doing a hard reset now, clear the * failover_pending flag so we don't ignore any * future MOBILITY or other resets. @@ -2402,8 +2556,7 @@ static int ibmvnic_reset(struct ibmvnic_adapter *adapter, goto err; } - list_for_each(entry, &adapter->rwi_list) { - tmp = list_entry(entry, struct ibmvnic_rwi, list); + list_for_each_entry(tmp, &adapter->rwi_list, list) { if (tmp->reset_reason == reason) { netdev_dbg(netdev, "Skipping matching reset, reason=%s\n", reset_reason_to_string(reason)); @@ -3140,6 +3293,7 @@ static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free) netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n", i); + ibmvnic_tx_scrq_clean_buffer(adapter, adapter->tx_scrq[i]); if (adapter->tx_scrq[i]->irq) { free_irq(adapter->tx_scrq[i]->irq, adapter->tx_scrq[i]); @@ -3213,7 +3367,7 @@ static int enable_scrq_irq(struct ibmvnic_adapter *adapter, /* H_EOI would fail with rc = H_FUNCTION when running * in XIVE mode which is expected, but not an error. */ - if (rc && rc != H_FUNCTION) + if (rc && (rc != H_FUNCTION)) dev_err(dev, "H_EOI FAILED irq 0x%llx. rc=%ld\n", val, rc); } @@ -3776,18 +3930,6 @@ static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter) return 0; } -static int send_version_xchg(struct ibmvnic_adapter *adapter) -{ - union ibmvnic_crq crq; - - memset(&crq, 0, sizeof(crq)); - crq.version_exchange.first = IBMVNIC_CRQ_CMD; - crq.version_exchange.cmd = VERSION_EXCHANGE; - crq.version_exchange.version = cpu_to_be16(ibmvnic_version); - - return ibmvnic_send_crq(adapter, &crq); -} - struct vnic_login_client_data { u8 type; __be16 len; @@ -3820,21 +3962,21 @@ static void vnic_add_client_data(struct ibmvnic_adapter *adapter, vlcd->type = 1; len = strlen(os_name) + 1; vlcd->len = cpu_to_be16(len); - strncpy(vlcd->name, os_name, len); + strscpy(vlcd->name, os_name, len); vlcd = (struct vnic_login_client_data *)(vlcd->name + len); /* Type 2 - LPAR name */ vlcd->type = 2; len = strlen(utsname()->nodename) + 1; vlcd->len = cpu_to_be16(len); - strncpy(vlcd->name, utsname()->nodename, len); + strscpy(vlcd->name, utsname()->nodename, len); vlcd = (struct vnic_login_client_data *)(vlcd->name + len); /* Type 3 - device name */ vlcd->type = 3; len = strlen(adapter->netdev->name) + 1; vlcd->len = cpu_to_be16(len); - strncpy(vlcd->name, adapter->netdev->name, len); + strscpy(vlcd->name, adapter->netdev->name, len); } static int send_login(struct ibmvnic_adapter *adapter) @@ -4301,7 +4443,7 @@ static void handle_vpd_rsp(union ibmvnic_crq *crq, complete: if (adapter->fw_version[0] == '\0') - strncpy((char *)adapter->fw_version, "N/A", 3 * sizeof(char)); + strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version)); complete(&adapter->fw_done); } @@ -4907,7 +5049,12 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq, complete(&adapter->init_done); adapter->init_done_rc = -EIO; } - rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER); + + if (adapter->state == VNIC_DOWN) + rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT); + else + rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER); + if (rc && rc != -EBUSY) { /* We were unable to schedule the failover * reset either because the adapter was still @@ -5330,6 +5477,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) struct ibmvnic_adapter *adapter; struct net_device *netdev; unsigned char *mac_addr_p; + bool init_success; int rc; dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n", @@ -5376,6 +5524,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) init_completion(&adapter->stats_done); clear_bit(0, &adapter->resetting); + init_success = false; do { rc = init_crq_queue(adapter); if (rc) { @@ -5385,10 +5534,16 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) } rc = ibmvnic_reset_init(adapter, false); - if (rc && rc != EAGAIN) - goto ibmvnic_init_fail; } while (rc == EAGAIN); + /* We are ignoring the error from ibmvnic_reset_init() assuming that the + * partner is not ready. CRQ is not active. When the partner becomes + * ready, we will do the passive init reset. + */ + + if (!rc) + init_success = true; + rc = init_stats_buffers(adapter); if (rc) goto ibmvnic_init_fail; @@ -5397,10 +5552,6 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) if (rc) goto ibmvnic_stats_fail; - netdev->mtu = adapter->req_mtu - ETH_HLEN; - netdev->min_mtu = adapter->min_mtu - ETH_HLEN; - netdev->max_mtu = adapter->max_mtu - ETH_HLEN; - rc = device_create_file(&dev->dev, &dev_attr_failover); if (rc) goto ibmvnic_dev_file_err; @@ -5413,7 +5564,14 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) } dev_info(&dev->dev, "ibmvnic registered\n"); - adapter->state = VNIC_PROBED; + if (init_success) { + adapter->state = VNIC_PROBED; + netdev->mtu = adapter->req_mtu - ETH_HLEN; + netdev->min_mtu = adapter->min_mtu - ETH_HLEN; + netdev->max_mtu = adapter->max_mtu - ETH_HLEN; + } else { + adapter->state = VNIC_DOWN; + } adapter->wait_for_reset = false; adapter->last_reset_time = jiffies; diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index c1d39a748546..22df602323bc 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -851,14 +851,16 @@ enum vnic_state {VNIC_PROBING = 1, VNIC_CLOSING, VNIC_CLOSED, VNIC_REMOVING, - VNIC_REMOVED}; + VNIC_REMOVED, + VNIC_DOWN}; enum ibmvnic_reset_reason {VNIC_RESET_FAILOVER = 1, VNIC_RESET_MOBILITY, VNIC_RESET_FATAL, VNIC_RESET_NON_FATAL, VNIC_RESET_TIMEOUT, - VNIC_RESET_CHANGE_PARAM}; + VNIC_RESET_CHANGE_PARAM, + VNIC_RESET_PASSIVE_INIT}; struct ibmvnic_rwi { enum ibmvnic_reset_reason reset_reason; diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index c1d155690341..82744a7501c7 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -241,6 +241,7 @@ config I40E tristate "Intel(R) Ethernet Controller XL710 Family support" imply PTP_1588_CLOCK depends on PCI + select AUXILIARY_BUS help This driver supports Intel(R) Ethernet Controller XL710 Family of devices. For more information on how to identify your adapter, go @@ -294,9 +295,11 @@ config ICE tristate "Intel(R) Ethernet Connection E800 Series Support" default n depends on PCI_MSI + select AUXILIARY_BUS select DIMLIB select NET_DEVLINK select PLDMFW + imply PTP_1588_CLOCK help This driver supports Intel(R) Ethernet Connection E800 Series of devices. For more information on how to identify your adapter, go diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c index f8d78af76d7d..1b0958bd24f6 100644 --- a/drivers/net/ethernet/intel/e100.c +++ b/drivers/net/ethernet/intel/e100.c @@ -1395,7 +1395,7 @@ static int e100_phy_check_without_mii(struct nic *nic) u8 phy_type; int without_mii; - phy_type = (nic->eeprom[eeprom_phy_iface] >> 8) & 0x0f; + phy_type = (le16_to_cpu(nic->eeprom[eeprom_phy_iface]) >> 8) & 0x0f; switch (phy_type) { case NoSuchPhy: /* Non-MII PHY; UNTESTED! */ @@ -1515,7 +1515,7 @@ static int e100_phy_init(struct nic *nic) mdio_write(netdev, nic->mii.phy_id, MII_BMCR, bmcr); } else if ((nic->mac >= mac_82550_D102) || ((nic->flags & ich) && (mdio_read(netdev, nic->mii.phy_id, MII_TPISTATUS) & 0x8000) && - (nic->eeprom[eeprom_cnfg_mdix] & eeprom_mdix_enabled))) { + (le16_to_cpu(nic->eeprom[eeprom_cnfg_mdix]) & eeprom_mdix_enabled))) { /* enable/disable MDI/MDI-X auto-switching. */ mdio_write(netdev, nic->mii.phy_id, MII_NCONFIG, nic->mii.force_media ? 0 : NCONFIG_AUTO_SWITCH); @@ -2269,9 +2269,9 @@ static int e100_asf(struct nic *nic) { /* ASF can be enabled from eeprom */ return (nic->pdev->device >= 0x1050) && (nic->pdev->device <= 0x1057) && - (nic->eeprom[eeprom_config_asf] & eeprom_asf) && - !(nic->eeprom[eeprom_config_asf] & eeprom_gcl) && - ((nic->eeprom[eeprom_smbus_addr] & 0xFF) != 0xFE); + (le16_to_cpu(nic->eeprom[eeprom_config_asf]) & eeprom_asf) && + !(le16_to_cpu(nic->eeprom[eeprom_config_asf]) & eeprom_gcl) && + ((le16_to_cpu(nic->eeprom[eeprom_smbus_addr]) & 0xFF) != 0xFE); } static int e100_up(struct nic *nic) @@ -2926,7 +2926,7 @@ static int e100_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Wol magic packet can be enabled from eeprom */ if ((nic->mac >= mac_82558_D101_A4) && - (nic->eeprom[eeprom_id] & eeprom_id_wol)) { + (le16_to_cpu(nic->eeprom[eeprom_id]) & eeprom_id_wol)) { nic->flags |= wol_magic; device_set_wakeup_enable(&pdev->dev, true); } diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index f976e9daa3d8..3c51ee94fa00 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -513,7 +513,7 @@ static int e1000_set_eeprom(struct net_device *netdev, memcpy(ptr, bytes, eeprom->len); for (i = 0; i < last_word - first_word + 1; i++) - eeprom_buff[i] = cpu_to_le16(eeprom_buff[i]); + cpu_to_le16s(&eeprom_buff[i]); ret_val = e1000_write_eeprom(hw, first_word, last_word - first_word + 1, eeprom_buff); diff --git a/drivers/net/ethernet/intel/e1000/e1000_hw.c b/drivers/net/ethernet/intel/e1000/e1000_hw.c index 19cf36360933..1042e79a1397 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_hw.c +++ b/drivers/net/ethernet/intel/e1000/e1000_hw.c @@ -2522,7 +2522,7 @@ s32 e1000_check_for_link(struct e1000_hw *hw) * turn it on. For compatibility with a TBI link * partner, we will store bad packets. Some * frames have an additional byte on the end and - * will look like CRC errors to to the hardware. + * will look like CRC errors to the hardware. */ if (!hw->tbi_compatibility_on) { hw->tbi_compatibility_on = true; @@ -2723,7 +2723,7 @@ static void e1000_shift_out_mdi_bits(struct e1000_hw *hw, u32 data, u16 count) * e1000_shift_in_mdi_bits - Shifts data bits in from the PHY * @hw: Struct containing variables accessed by shared code * - * Bits are shifted in in MSB to LSB order. + * Bits are shifted in MSB to LSB order. */ static u16 e1000_shift_in_mdi_bits(struct e1000_hw *hw) { diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 042de276e632..c2a109126c27 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -5245,7 +5245,7 @@ static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags)) pci_disable_device(pdev); - /* Request a slot slot reset. */ + /* Request a slot reset. */ return PCI_ERS_RESULT_NEED_RESET; } diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c index 590ad110d383..cf7b3887da1d 100644 --- a/drivers/net/ethernet/intel/e1000e/ich8lan.c +++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c @@ -4639,7 +4639,7 @@ static s32 e1000_id_led_init_pchlan(struct e1000_hw *hw) * @hw: pointer to the HW structure * * ICH8 use the PCI Express bus, but does not contain a PCI Express Capability - * register, so the the bus width is hard coded. + * register, so the bus width is hard coded. **/ static s32 e1000_get_bus_info_ich8lan(struct e1000_hw *hw) { diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 88e9035b75cf..d150dade06cf 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -5223,18 +5223,20 @@ static void e1000_watchdog_task(struct work_struct *work) pm_runtime_resume(netdev->dev.parent); /* Checking if MAC is in DMoff state*/ - pcim_state = er32(STATUS); - while (pcim_state & E1000_STATUS_PCIM_STATE) { - if (tries++ == dmoff_exit_timeout) { - e_dbg("Error in exiting dmoff\n"); - break; - } - usleep_range(10000, 20000); + if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) { pcim_state = er32(STATUS); - - /* Checking if MAC exited DMoff state */ - if (!(pcim_state & E1000_STATUS_PCIM_STATE)) - e1000_phy_hw_reset(&adapter->hw); + while (pcim_state & E1000_STATUS_PCIM_STATE) { + if (tries++ == dmoff_exit_timeout) { + e_dbg("Error in exiting dmoff\n"); + break; + } + usleep_range(10000, 20000); + pcim_state = er32(STATUS); + + /* Checking if MAC exited DMoff state */ + if (!(pcim_state & E1000_STATUS_PCIM_STATE)) + e1000_phy_hw_reset(&adapter->hw); + } } /* update snapshot of PHY registers on LSC */ @@ -7118,7 +7120,7 @@ static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev, pci_disable_device(pdev); - /* Request a slot slot reset. */ + /* Request a slot reset. */ return PCI_ERS_RESULT_NEED_RESET; } diff --git a/drivers/net/ethernet/intel/e1000e/phy.c b/drivers/net/ethernet/intel/e1000e/phy.c index 1db35b2c7750..0f0efee5fc8e 100644 --- a/drivers/net/ethernet/intel/e1000e/phy.c +++ b/drivers/net/ethernet/intel/e1000e/phy.c @@ -2978,7 +2978,7 @@ static u32 e1000_get_phy_addr_for_hv_page(u32 page) * @data: pointer to the data to be read or written * @read: determines if operation is read or write * - * Reads the PHY register at offset and stores the retreived information + * Reads the PHY register at offset and stores the retrieved information * in data. Assumes semaphore already acquired. Note that the procedure * to access these regs uses the address port and data port to read/write. * These accesses done with PHY address 2 and without using pages. diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c index 9e3103fae723..dbcae92bb18d 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c @@ -1370,7 +1370,6 @@ static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data) struct fm10k_hw *hw = &interface->hw; struct fm10k_mbx_info *mbx = &hw->mbx; u32 eicr; - s32 err = 0; /* unmask any set bits related to this interrupt */ eicr = fm10k_read_reg(hw, FM10K_EICR); @@ -1386,15 +1385,16 @@ static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data) /* service mailboxes */ if (fm10k_mbx_trylock(interface)) { - err = mbx->ops.process(hw, mbx); + s32 err = mbx->ops.process(hw, mbx); + + if (err == FM10K_ERR_RESET_REQUESTED) + set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); + /* handle VFLRE events */ fm10k_iov_event(interface); fm10k_mbx_unlock(interface); } - if (err == FM10K_ERR_RESET_REQUESTED) - set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); - /* if switch toggled state we should reset GLORTs */ if (eicr & FM10K_EICR_SWITCHNOTREADY) { /* force link down for at least 4 seconds */ diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 85d3dd3a3339..b9417dc0007c 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -870,6 +870,8 @@ struct i40e_netdev_priv { struct i40e_vsi *vsi; }; +extern struct ida i40e_client_ida; + /* struct that defines an interrupt vector */ struct i40e_q_vector { struct i40e_vsi *vsi; diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c index 32f3facbed1a..ea2bb0140a6e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_client.c +++ b/drivers/net/ethernet/intel/i40e/i40e_client.c @@ -8,10 +8,9 @@ #include "i40e.h" #include "i40e_prototype.h" -static const char i40e_client_interface_version_str[] = I40E_CLIENT_VERSION_STR; -static struct i40e_client *registered_client; static LIST_HEAD(i40e_devices); static DEFINE_MUTEX(i40e_device_mutex); +DEFINE_IDA(i40e_client_ida); static int i40e_client_virtchnl_send(struct i40e_info *ldev, struct i40e_client *client, @@ -275,6 +274,57 @@ void i40e_client_update_msix_info(struct i40e_pf *pf) cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector]; } +static void i40e_auxiliary_dev_release(struct device *dev) +{ + struct i40e_auxiliary_device *i40e_aux_dev = + container_of(dev, struct i40e_auxiliary_device, aux_dev.dev); + + ida_free(&i40e_client_ida, i40e_aux_dev->aux_dev.id); + kfree(i40e_aux_dev); +} + +static int i40e_register_auxiliary_dev(struct i40e_info *ldev, const char *name) +{ + struct i40e_auxiliary_device *i40e_aux_dev; + struct pci_dev *pdev = ldev->pcidev; + struct auxiliary_device *aux_dev; + int ret; + + i40e_aux_dev = kzalloc(sizeof(*i40e_aux_dev), GFP_KERNEL); + if (!i40e_aux_dev) + return -ENOMEM; + + i40e_aux_dev->ldev = ldev; + + aux_dev = &i40e_aux_dev->aux_dev; + aux_dev->name = name; + aux_dev->dev.parent = &pdev->dev; + aux_dev->dev.release = i40e_auxiliary_dev_release; + ldev->aux_dev = aux_dev; + + ret = ida_alloc(&i40e_client_ida, GFP_KERNEL); + if (ret < 0) { + kfree(i40e_aux_dev); + return ret; + } + aux_dev->id = ret; + + ret = auxiliary_device_init(aux_dev); + if (ret < 0) { + ida_free(&i40e_client_ida, aux_dev->id); + kfree(i40e_aux_dev); + return ret; + } + + ret = auxiliary_device_add(aux_dev); + if (ret) { + auxiliary_device_uninit(aux_dev); + return ret; + } + + return ret; +} + /** * i40e_client_add_instance - add a client instance struct to the instance list * @pf: pointer to the board struct @@ -286,9 +336,6 @@ static void i40e_client_add_instance(struct i40e_pf *pf) struct netdev_hw_addr *mac = NULL; struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; - if (!registered_client || pf->cinst) - return; - cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); if (!cdev) return; @@ -308,11 +355,8 @@ static void i40e_client_add_instance(struct i40e_pf *pf) cdev->lan_info.fw_build = pf->hw.aq.fw_build; set_bit(__I40E_CLIENT_INSTANCE_NONE, &cdev->state); - if (i40e_client_get_params(vsi, &cdev->lan_info.params)) { - kfree(cdev); - cdev = NULL; - return; - } + if (i40e_client_get_params(vsi, &cdev->lan_info.params)) + goto free_cdev; mac = list_first_entry(&cdev->lan_info.netdev->dev_addrs.list, struct netdev_hw_addr, list); @@ -321,10 +365,19 @@ static void i40e_client_add_instance(struct i40e_pf *pf) else dev_err(&pf->pdev->dev, "MAC address list is empty!\n"); - cdev->client = registered_client; pf->cinst = cdev; - i40e_client_update_msix_info(pf); + cdev->lan_info.msix_count = pf->num_iwarp_msix; + cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector]; + + if (i40e_register_auxiliary_dev(&cdev->lan_info, "iwarp")) + goto free_cdev; + + return; + +free_cdev: + kfree(cdev); + pf->cinst = NULL; } /** @@ -345,7 +398,7 @@ void i40e_client_del_instance(struct i40e_pf *pf) **/ void i40e_client_subtask(struct i40e_pf *pf) { - struct i40e_client *client = registered_client; + struct i40e_client *client; struct i40e_client_instance *cdev; struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; int ret = 0; @@ -359,9 +412,11 @@ void i40e_client_subtask(struct i40e_pf *pf) test_bit(__I40E_CONFIG_BUSY, pf->state)) return; - if (!client || !cdev) + if (!cdev || !cdev->client) return; + client = cdev->client; + /* Here we handle client opens. If the client is down, and * the netdev is registered, then open the client. */ @@ -423,16 +478,8 @@ int i40e_lan_add_device(struct i40e_pf *pf) pf->hw.pf_id, pf->hw.bus.bus_id, pf->hw.bus.device, pf->hw.bus.func); - /* If a client has already been registered, we need to add an instance - * of it to our new LAN device. - */ - if (registered_client) - i40e_client_add_instance(pf); + i40e_client_add_instance(pf); - /* Since in some cases register may have happened before a device gets - * added, we can schedule a subtask to go initiate the clients if - * they can be launched at probe time. - */ set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); i40e_service_event_schedule(pf); @@ -449,9 +496,13 @@ out: **/ int i40e_lan_del_device(struct i40e_pf *pf) { + struct auxiliary_device *aux_dev = pf->cinst->lan_info.aux_dev; struct i40e_device *ldev, *tmp; int ret = -ENODEV; + auxiliary_device_delete(aux_dev); + auxiliary_device_uninit(aux_dev); + /* First, remove any client instance. */ i40e_client_del_instance(pf); @@ -472,69 +523,6 @@ int i40e_lan_del_device(struct i40e_pf *pf) } /** - * i40e_client_release - release client specific resources - * @client: pointer to the registered client - * - **/ -static void i40e_client_release(struct i40e_client *client) -{ - struct i40e_client_instance *cdev; - struct i40e_device *ldev; - struct i40e_pf *pf; - - mutex_lock(&i40e_device_mutex); - list_for_each_entry(ldev, &i40e_devices, list) { - pf = ldev->pf; - cdev = pf->cinst; - if (!cdev) - continue; - - while (test_and_set_bit(__I40E_SERVICE_SCHED, - pf->state)) - usleep_range(500, 1000); - - if (test_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state)) { - if (client->ops && client->ops->close) - client->ops->close(&cdev->lan_info, client, - false); - i40e_client_release_qvlist(&cdev->lan_info); - clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state); - - dev_warn(&pf->pdev->dev, - "Client %s instance for PF id %d closed\n", - client->name, pf->hw.pf_id); - } - /* delete the client instance */ - i40e_client_del_instance(pf); - dev_info(&pf->pdev->dev, "Deleted client instance of Client %s\n", - client->name); - clear_bit(__I40E_SERVICE_SCHED, pf->state); - } - mutex_unlock(&i40e_device_mutex); -} - -/** - * i40e_client_prepare - prepare client specific resources - * @client: pointer to the registered client - * - **/ -static void i40e_client_prepare(struct i40e_client *client) -{ - struct i40e_device *ldev; - struct i40e_pf *pf; - - mutex_lock(&i40e_device_mutex); - list_for_each_entry(ldev, &i40e_devices, list) { - pf = ldev->pf; - i40e_client_add_instance(pf); - /* Start the client subtask */ - set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); - i40e_service_event_schedule(pf); - } - mutex_unlock(&i40e_device_mutex); -} - -/** * i40e_client_virtchnl_send - TBD * @ldev: pointer to L2 context * @client: Client pointer @@ -579,7 +567,7 @@ static int i40e_client_setup_qvlist(struct i40e_info *ldev, u32 v_idx, i, reg_idx, reg; ldev->qvlist_info = kzalloc(struct_size(ldev->qvlist_info, qv_info, - qvlist_info->num_vectors - 1), GFP_KERNEL); + qvlist_info->num_vectors), GFP_KERNEL); if (!ldev->qvlist_info) return -ENOMEM; ldev->qvlist_info->num_vectors = qvlist_info->num_vectors; @@ -732,81 +720,34 @@ static int i40e_client_update_vsi_ctxt(struct i40e_info *ldev, return err; } -/** - * i40e_register_client - Register a i40e client driver with the L2 driver - * @client: pointer to the i40e_client struct - * - * Returns 0 on success or non-0 on error - **/ -int i40e_register_client(struct i40e_client *client) +void i40e_client_device_register(struct i40e_info *ldev, struct i40e_client *client) { - int ret = 0; - - if (!client) { - ret = -EIO; - goto out; - } - - if (strlen(client->name) == 0) { - pr_info("i40e: Failed to register client with no name\n"); - ret = -EIO; - goto out; - } - - if (registered_client) { - pr_info("i40e: Client %s has already been registered!\n", - client->name); - ret = -EEXIST; - goto out; - } - - if ((client->version.major != I40E_CLIENT_VERSION_MAJOR) || - (client->version.minor != I40E_CLIENT_VERSION_MINOR)) { - pr_info("i40e: Failed to register client %s due to mismatched client interface version\n", - client->name); - pr_info("Client is using version: %02d.%02d.%02d while LAN driver supports %s\n", - client->version.major, client->version.minor, - client->version.build, - i40e_client_interface_version_str); - ret = -EIO; - goto out; - } - - registered_client = client; - - i40e_client_prepare(client); + struct i40e_pf *pf = ldev->pf; - pr_info("i40e: Registered client %s\n", client->name); -out: - return ret; + pf->cinst->client = client; + set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); + i40e_service_event_schedule(pf); } -EXPORT_SYMBOL(i40e_register_client); +EXPORT_SYMBOL_GPL(i40e_client_device_register); -/** - * i40e_unregister_client - Unregister a i40e client driver with the L2 driver - * @client: pointer to the i40e_client struct - * - * Returns 0 on success or non-0 on error - **/ -int i40e_unregister_client(struct i40e_client *client) +void i40e_client_device_unregister(struct i40e_info *ldev) { - int ret = 0; + struct i40e_pf *pf = ldev->pf; + struct i40e_client_instance *cdev = pf->cinst; - if (registered_client != client) { - pr_info("i40e: Client %s has not been registered\n", - client->name); - ret = -ENODEV; - goto out; + if (!cdev) + return; + + while (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) + usleep_range(500, 1000); + + if (test_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state)) { + cdev->client->ops->close(&cdev->lan_info, cdev->client, false); + clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state); + i40e_client_release_qvlist(&cdev->lan_info); } - registered_client = NULL; - /* When a unregister request comes through we would have to send - * a close for each of the client instances that were opened. - * client_release function is called to handle this. - */ - i40e_client_release(client); - pr_info("i40e: Unregistered client %s\n", client->name); -out: - return ret; + pf->cinst->client = NULL; + clear_bit(__I40E_SERVICE_SCHED, pf->state); } -EXPORT_SYMBOL(i40e_unregister_client); +EXPORT_SYMBOL_GPL(i40e_client_device_unregister); diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index 67cb0b47416a..b4d3fed0d2f2 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -552,9 +552,9 @@ i40e_status i40e_aq_set_rss_key(struct i40e_hw *hw, * ENDIF */ -/* macro to make the table lines short */ +/* macro to make the table lines short, use explicit indexing with [PTYPE] */ #define I40E_PTT(PTYPE, OUTER_IP, OUTER_IP_VER, OUTER_FRAG, T, TE, TEF, I, PL)\ - { PTYPE, \ + [PTYPE] = { \ 1, \ I40E_RX_PTYPE_OUTER_##OUTER_IP, \ I40E_RX_PTYPE_OUTER_##OUTER_IP_VER, \ @@ -565,16 +565,15 @@ i40e_status i40e_aq_set_rss_key(struct i40e_hw *hw, I40E_RX_PTYPE_INNER_PROT_##I, \ I40E_RX_PTYPE_PAYLOAD_LAYER_##PL } -#define I40E_PTT_UNUSED_ENTRY(PTYPE) \ - { PTYPE, 0, 0, 0, 0, 0, 0, 0, 0, 0 } +#define I40E_PTT_UNUSED_ENTRY(PTYPE) [PTYPE] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } /* shorter macros makes the table fit but are terse */ #define I40E_RX_PTYPE_NOF I40E_RX_PTYPE_NOT_FRAG #define I40E_RX_PTYPE_FRG I40E_RX_PTYPE_FRAG #define I40E_RX_PTYPE_INNER_PROT_TS I40E_RX_PTYPE_INNER_PROT_TIMESYNC -/* Lookup table mapping the HW PTYPE to the bit field for decoding */ -struct i40e_rx_ptype_decoded i40e_ptype_lookup[] = { +/* Lookup table mapping in the 8-bit HW PTYPE to the bit field for decoding */ +struct i40e_rx_ptype_decoded i40e_ptype_lookup[BIT(8)] = { /* L2 Packet types */ I40E_PTT_UNUSED_ENTRY(0), I40E_PTT(1, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2), @@ -780,118 +779,7 @@ struct i40e_rx_ptype_decoded i40e_ptype_lookup[] = { I40E_PTT(153, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), /* unused entries */ - I40E_PTT_UNUSED_ENTRY(154), - I40E_PTT_UNUSED_ENTRY(155), - I40E_PTT_UNUSED_ENTRY(156), - I40E_PTT_UNUSED_ENTRY(157), - I40E_PTT_UNUSED_ENTRY(158), - I40E_PTT_UNUSED_ENTRY(159), - - I40E_PTT_UNUSED_ENTRY(160), - I40E_PTT_UNUSED_ENTRY(161), - I40E_PTT_UNUSED_ENTRY(162), - I40E_PTT_UNUSED_ENTRY(163), - I40E_PTT_UNUSED_ENTRY(164), - I40E_PTT_UNUSED_ENTRY(165), - I40E_PTT_UNUSED_ENTRY(166), - I40E_PTT_UNUSED_ENTRY(167), - I40E_PTT_UNUSED_ENTRY(168), - I40E_PTT_UNUSED_ENTRY(169), - - I40E_PTT_UNUSED_ENTRY(170), - I40E_PTT_UNUSED_ENTRY(171), - I40E_PTT_UNUSED_ENTRY(172), - I40E_PTT_UNUSED_ENTRY(173), - I40E_PTT_UNUSED_ENTRY(174), - I40E_PTT_UNUSED_ENTRY(175), - I40E_PTT_UNUSED_ENTRY(176), - I40E_PTT_UNUSED_ENTRY(177), - I40E_PTT_UNUSED_ENTRY(178), - I40E_PTT_UNUSED_ENTRY(179), - - I40E_PTT_UNUSED_ENTRY(180), - I40E_PTT_UNUSED_ENTRY(181), - I40E_PTT_UNUSED_ENTRY(182), - I40E_PTT_UNUSED_ENTRY(183), - I40E_PTT_UNUSED_ENTRY(184), - I40E_PTT_UNUSED_ENTRY(185), - I40E_PTT_UNUSED_ENTRY(186), - I40E_PTT_UNUSED_ENTRY(187), - I40E_PTT_UNUSED_ENTRY(188), - I40E_PTT_UNUSED_ENTRY(189), - - I40E_PTT_UNUSED_ENTRY(190), - I40E_PTT_UNUSED_ENTRY(191), - I40E_PTT_UNUSED_ENTRY(192), - I40E_PTT_UNUSED_ENTRY(193), - I40E_PTT_UNUSED_ENTRY(194), - I40E_PTT_UNUSED_ENTRY(195), - I40E_PTT_UNUSED_ENTRY(196), - I40E_PTT_UNUSED_ENTRY(197), - I40E_PTT_UNUSED_ENTRY(198), - I40E_PTT_UNUSED_ENTRY(199), - - I40E_PTT_UNUSED_ENTRY(200), - I40E_PTT_UNUSED_ENTRY(201), - I40E_PTT_UNUSED_ENTRY(202), - I40E_PTT_UNUSED_ENTRY(203), - I40E_PTT_UNUSED_ENTRY(204), - I40E_PTT_UNUSED_ENTRY(205), - I40E_PTT_UNUSED_ENTRY(206), - I40E_PTT_UNUSED_ENTRY(207), - I40E_PTT_UNUSED_ENTRY(208), - I40E_PTT_UNUSED_ENTRY(209), - - I40E_PTT_UNUSED_ENTRY(210), - I40E_PTT_UNUSED_ENTRY(211), - I40E_PTT_UNUSED_ENTRY(212), - I40E_PTT_UNUSED_ENTRY(213), - I40E_PTT_UNUSED_ENTRY(214), - I40E_PTT_UNUSED_ENTRY(215), - I40E_PTT_UNUSED_ENTRY(216), - I40E_PTT_UNUSED_ENTRY(217), - I40E_PTT_UNUSED_ENTRY(218), - I40E_PTT_UNUSED_ENTRY(219), - - I40E_PTT_UNUSED_ENTRY(220), - I40E_PTT_UNUSED_ENTRY(221), - I40E_PTT_UNUSED_ENTRY(222), - I40E_PTT_UNUSED_ENTRY(223), - I40E_PTT_UNUSED_ENTRY(224), - I40E_PTT_UNUSED_ENTRY(225), - I40E_PTT_UNUSED_ENTRY(226), - I40E_PTT_UNUSED_ENTRY(227), - I40E_PTT_UNUSED_ENTRY(228), - I40E_PTT_UNUSED_ENTRY(229), - - I40E_PTT_UNUSED_ENTRY(230), - I40E_PTT_UNUSED_ENTRY(231), - I40E_PTT_UNUSED_ENTRY(232), - I40E_PTT_UNUSED_ENTRY(233), - I40E_PTT_UNUSED_ENTRY(234), - I40E_PTT_UNUSED_ENTRY(235), - I40E_PTT_UNUSED_ENTRY(236), - I40E_PTT_UNUSED_ENTRY(237), - I40E_PTT_UNUSED_ENTRY(238), - I40E_PTT_UNUSED_ENTRY(239), - - I40E_PTT_UNUSED_ENTRY(240), - I40E_PTT_UNUSED_ENTRY(241), - I40E_PTT_UNUSED_ENTRY(242), - I40E_PTT_UNUSED_ENTRY(243), - I40E_PTT_UNUSED_ENTRY(244), - I40E_PTT_UNUSED_ENTRY(245), - I40E_PTT_UNUSED_ENTRY(246), - I40E_PTT_UNUSED_ENTRY(247), - I40E_PTT_UNUSED_ENTRY(248), - I40E_PTT_UNUSED_ENTRY(249), - - I40E_PTT_UNUSED_ENTRY(250), - I40E_PTT_UNUSED_ENTRY(251), - I40E_PTT_UNUSED_ENTRY(252), - I40E_PTT_UNUSED_ENTRY(253), - I40E_PTT_UNUSED_ENTRY(254), - I40E_PTT_UNUSED_ENTRY(255) + [154 ... 255] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index ccd5b9486ea9..3e822bad4851 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -1262,8 +1262,7 @@ static int i40e_set_link_ksettings(struct net_device *netdev, if (ethtool_link_ksettings_test_link_mode(&safe_ks, supported, Autoneg) && - hw->phy.link_info.phy_type != - I40E_PHY_TYPE_10GBASE_T) { + hw->phy.media_type != I40E_MEDIA_TYPE_BASET) { netdev_info(netdev, "Autoneg cannot be disabled on this phy\n"); err = -EINVAL; goto done; diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 704e474879c5..861e59a350bd 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -32,7 +32,7 @@ static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); static int i40e_add_vsi(struct i40e_vsi *vsi); static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); -static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); +static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired); static int i40e_setup_misc_vector(struct i40e_pf *pf); static void i40e_determine_queue_usage(struct i40e_pf *pf); static int i40e_setup_pf_filter_control(struct i40e_pf *pf); @@ -8703,6 +8703,8 @@ int i40e_vsi_open(struct i40e_vsi *vsi) dev_driver_string(&pf->pdev->dev), dev_name(&pf->pdev->dev)); err = i40e_vsi_request_irq(vsi, int_name); + if (err) + goto err_setup_rx; } else { err = -EINVAL; @@ -10569,7 +10571,7 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) #endif /* CONFIG_I40E_DCB */ if (!lock_acquired) rtnl_lock(); - ret = i40e_setup_pf_switch(pf, reinit); + ret = i40e_setup_pf_switch(pf, reinit, true); if (ret) goto end_unlock; @@ -14627,10 +14629,11 @@ int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) * i40e_setup_pf_switch - Setup the HW switch on startup or after reset * @pf: board private structure * @reinit: if the Main VSI needs to re-initialized. + * @lock_acquired: indicates whether or not the lock has been acquired * * Returns 0 on success, negative value on failure **/ -static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) +static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired) { u16 flags = 0; int ret; @@ -14732,9 +14735,15 @@ static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) i40e_ptp_init(pf); + if (!lock_acquired) + rtnl_lock(); + /* repopulate tunnel port filters */ udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev); + if (!lock_acquired) + rtnl_unlock(); + return ret; } @@ -15528,7 +15537,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; } #endif - err = i40e_setup_pf_switch(pf, false); + err = i40e_setup_pf_switch(pf, false, false); if (err) { dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); goto err_vsis; @@ -16270,6 +16279,7 @@ static void __exit i40e_exit_module(void) { pci_unregister_driver(&i40e_driver); destroy_workqueue(i40e_wq); + ida_destroy(&i40e_client_ida); i40e_dbg_exit(); } module_exit(i40e_exit_module); diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c index f1f6fc3744e9..7b971b205d36 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c @@ -11,13 +11,14 @@ * operate with the nanosecond field directly without fear of overflow. * * Much like the 82599, the update period is dependent upon the link speed: - * At 40Gb link or no link, the period is 1.6ns. - * At 10Gb link, the period is multiplied by 2. (3.2ns) + * At 40Gb, 25Gb, or no link, the period is 1.6ns. + * At 10Gb or 5Gb link, the period is multiplied by 2. (3.2ns) * At 1Gb link, the period is multiplied by 20. (32ns) * 1588 functionality is not supported at 100Mbps. */ #define I40E_PTP_40GB_INCVAL 0x0199999999ULL #define I40E_PTP_10GB_INCVAL_MULT 2 +#define I40E_PTP_5GB_INCVAL_MULT 2 #define I40E_PTP_1GB_INCVAL_MULT 20 #define I40E_PRTTSYN_CTL1_TSYNTYPE_V1 BIT(I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT) @@ -465,6 +466,9 @@ void i40e_ptp_set_increment(struct i40e_pf *pf) case I40E_LINK_SPEED_10GB: mult = I40E_PTP_10GB_INCVAL_MULT; break; + case I40E_LINK_SPEED_5GB: + mult = I40E_PTP_5GB_INCVAL_MULT; + break; case I40E_LINK_SPEED_1GB: mult = I40E_PTP_1GB_INCVAL_MULT; break; diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index de70c16ef619..38eb8151ee9a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -2298,7 +2298,6 @@ static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp) struct bpf_prog *xdp_prog; u32 act; - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); if (!xdp_prog) @@ -2313,15 +2312,20 @@ static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp) case XDP_TX: xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); + if (result == I40E_XDP_CONSUMED) + goto out_failure; break; case XDP_REDIRECT: err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - result = !err ? I40E_XDP_REDIR : I40E_XDP_CONSUMED; + if (err) + goto out_failure; + result = I40E_XDP_REDIR; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: @@ -2329,7 +2333,6 @@ static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp) break; } xdp_out: - rcu_read_unlock(); return result; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h index c81109a63e90..36a4ca1ffb1a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_type.h +++ b/drivers/net/ethernet/intel/i40e/i40e_type.h @@ -804,7 +804,6 @@ enum i40e_rx_l2_ptype { }; struct i40e_rx_ptype_decoded { - u32 ptype:8; u32 known:1; u32 outer_ip:1; u32 outer_ip_ver:1; diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index 46d884417c63..e7e778ca074c 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -153,7 +153,6 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) struct bpf_prog *xdp_prog; u32 act; - rcu_read_lock(); /* NB! xdp_prog will always be !NULL, due to the fact that * this path is enabled by setting an XDP program. */ @@ -162,9 +161,9 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) if (likely(act == XDP_REDIRECT)) { err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - result = !err ? I40E_XDP_REDIR : I40E_XDP_CONSUMED; - rcu_read_unlock(); - return result; + if (err) + goto out_failure; + return I40E_XDP_REDIR; } switch (act) { @@ -173,18 +172,20 @@ static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) case XDP_TX: xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); + if (result == I40E_XDP_CONSUMED) + goto out_failure; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = I40E_XDP_CONSUMED; break; } - rcu_read_unlock(); return result; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_common.c b/drivers/net/ethernet/intel/iavf/iavf_common.c index 8547fc8fdfd6..e9cc7f6ddc46 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_common.c +++ b/drivers/net/ethernet/intel/iavf/iavf_common.c @@ -522,9 +522,9 @@ enum iavf_status iavf_aq_set_rss_key(struct iavf_hw *hw, u16 vsi_id, * ENDIF */ -/* macro to make the table lines short */ +/* macro to make the table lines short, use explicit indexing with [PTYPE] */ #define IAVF_PTT(PTYPE, OUTER_IP, OUTER_IP_VER, OUTER_FRAG, T, TE, TEF, I, PL)\ - { PTYPE, \ + [PTYPE] = { \ 1, \ IAVF_RX_PTYPE_OUTER_##OUTER_IP, \ IAVF_RX_PTYPE_OUTER_##OUTER_IP_VER, \ @@ -535,16 +535,15 @@ enum iavf_status iavf_aq_set_rss_key(struct iavf_hw *hw, u16 vsi_id, IAVF_RX_PTYPE_INNER_PROT_##I, \ IAVF_RX_PTYPE_PAYLOAD_LAYER_##PL } -#define IAVF_PTT_UNUSED_ENTRY(PTYPE) \ - { PTYPE, 0, 0, 0, 0, 0, 0, 0, 0, 0 } +#define IAVF_PTT_UNUSED_ENTRY(PTYPE) [PTYPE] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } /* shorter macros makes the table fit but are terse */ #define IAVF_RX_PTYPE_NOF IAVF_RX_PTYPE_NOT_FRAG #define IAVF_RX_PTYPE_FRG IAVF_RX_PTYPE_FRAG #define IAVF_RX_PTYPE_INNER_PROT_TS IAVF_RX_PTYPE_INNER_PROT_TIMESYNC -/* Lookup table mapping the HW PTYPE to the bit field for decoding */ -struct iavf_rx_ptype_decoded iavf_ptype_lookup[] = { +/* Lookup table mapping the 8-bit HW PTYPE to the bit field for decoding */ +struct iavf_rx_ptype_decoded iavf_ptype_lookup[BIT(8)] = { /* L2 Packet types */ IAVF_PTT_UNUSED_ENTRY(0), IAVF_PTT(1, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2), @@ -750,118 +749,7 @@ struct iavf_rx_ptype_decoded iavf_ptype_lookup[] = { IAVF_PTT(153, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), /* unused entries */ - IAVF_PTT_UNUSED_ENTRY(154), - IAVF_PTT_UNUSED_ENTRY(155), - IAVF_PTT_UNUSED_ENTRY(156), - IAVF_PTT_UNUSED_ENTRY(157), - IAVF_PTT_UNUSED_ENTRY(158), - IAVF_PTT_UNUSED_ENTRY(159), - - IAVF_PTT_UNUSED_ENTRY(160), - IAVF_PTT_UNUSED_ENTRY(161), - IAVF_PTT_UNUSED_ENTRY(162), - IAVF_PTT_UNUSED_ENTRY(163), - IAVF_PTT_UNUSED_ENTRY(164), - IAVF_PTT_UNUSED_ENTRY(165), - IAVF_PTT_UNUSED_ENTRY(166), - IAVF_PTT_UNUSED_ENTRY(167), - IAVF_PTT_UNUSED_ENTRY(168), - IAVF_PTT_UNUSED_ENTRY(169), - - IAVF_PTT_UNUSED_ENTRY(170), - IAVF_PTT_UNUSED_ENTRY(171), - IAVF_PTT_UNUSED_ENTRY(172), - IAVF_PTT_UNUSED_ENTRY(173), - IAVF_PTT_UNUSED_ENTRY(174), - IAVF_PTT_UNUSED_ENTRY(175), - IAVF_PTT_UNUSED_ENTRY(176), - IAVF_PTT_UNUSED_ENTRY(177), - IAVF_PTT_UNUSED_ENTRY(178), - IAVF_PTT_UNUSED_ENTRY(179), - - IAVF_PTT_UNUSED_ENTRY(180), - IAVF_PTT_UNUSED_ENTRY(181), - IAVF_PTT_UNUSED_ENTRY(182), - IAVF_PTT_UNUSED_ENTRY(183), - IAVF_PTT_UNUSED_ENTRY(184), - IAVF_PTT_UNUSED_ENTRY(185), - IAVF_PTT_UNUSED_ENTRY(186), - IAVF_PTT_UNUSED_ENTRY(187), - IAVF_PTT_UNUSED_ENTRY(188), - IAVF_PTT_UNUSED_ENTRY(189), - - IAVF_PTT_UNUSED_ENTRY(190), - IAVF_PTT_UNUSED_ENTRY(191), - IAVF_PTT_UNUSED_ENTRY(192), - IAVF_PTT_UNUSED_ENTRY(193), - IAVF_PTT_UNUSED_ENTRY(194), - IAVF_PTT_UNUSED_ENTRY(195), - IAVF_PTT_UNUSED_ENTRY(196), - IAVF_PTT_UNUSED_ENTRY(197), - IAVF_PTT_UNUSED_ENTRY(198), - IAVF_PTT_UNUSED_ENTRY(199), - - IAVF_PTT_UNUSED_ENTRY(200), - IAVF_PTT_UNUSED_ENTRY(201), - IAVF_PTT_UNUSED_ENTRY(202), - IAVF_PTT_UNUSED_ENTRY(203), - IAVF_PTT_UNUSED_ENTRY(204), - IAVF_PTT_UNUSED_ENTRY(205), - IAVF_PTT_UNUSED_ENTRY(206), - IAVF_PTT_UNUSED_ENTRY(207), - IAVF_PTT_UNUSED_ENTRY(208), - IAVF_PTT_UNUSED_ENTRY(209), - - IAVF_PTT_UNUSED_ENTRY(210), - IAVF_PTT_UNUSED_ENTRY(211), - IAVF_PTT_UNUSED_ENTRY(212), - IAVF_PTT_UNUSED_ENTRY(213), - IAVF_PTT_UNUSED_ENTRY(214), - IAVF_PTT_UNUSED_ENTRY(215), - IAVF_PTT_UNUSED_ENTRY(216), - IAVF_PTT_UNUSED_ENTRY(217), - IAVF_PTT_UNUSED_ENTRY(218), - IAVF_PTT_UNUSED_ENTRY(219), - - IAVF_PTT_UNUSED_ENTRY(220), - IAVF_PTT_UNUSED_ENTRY(221), - IAVF_PTT_UNUSED_ENTRY(222), - IAVF_PTT_UNUSED_ENTRY(223), - IAVF_PTT_UNUSED_ENTRY(224), - IAVF_PTT_UNUSED_ENTRY(225), - IAVF_PTT_UNUSED_ENTRY(226), - IAVF_PTT_UNUSED_ENTRY(227), - IAVF_PTT_UNUSED_ENTRY(228), - IAVF_PTT_UNUSED_ENTRY(229), - - IAVF_PTT_UNUSED_ENTRY(230), - IAVF_PTT_UNUSED_ENTRY(231), - IAVF_PTT_UNUSED_ENTRY(232), - IAVF_PTT_UNUSED_ENTRY(233), - IAVF_PTT_UNUSED_ENTRY(234), - IAVF_PTT_UNUSED_ENTRY(235), - IAVF_PTT_UNUSED_ENTRY(236), - IAVF_PTT_UNUSED_ENTRY(237), - IAVF_PTT_UNUSED_ENTRY(238), - IAVF_PTT_UNUSED_ENTRY(239), - - IAVF_PTT_UNUSED_ENTRY(240), - IAVF_PTT_UNUSED_ENTRY(241), - IAVF_PTT_UNUSED_ENTRY(242), - IAVF_PTT_UNUSED_ENTRY(243), - IAVF_PTT_UNUSED_ENTRY(244), - IAVF_PTT_UNUSED_ENTRY(245), - IAVF_PTT_UNUSED_ENTRY(246), - IAVF_PTT_UNUSED_ENTRY(247), - IAVF_PTT_UNUSED_ENTRY(248), - IAVF_PTT_UNUSED_ENTRY(249), - - IAVF_PTT_UNUSED_ENTRY(250), - IAVF_PTT_UNUSED_ENTRY(251), - IAVF_PTT_UNUSED_ENTRY(252), - IAVF_PTT_UNUSED_ENTRY(253), - IAVF_PTT_UNUSED_ENTRY(254), - IAVF_PTT_UNUSED_ENTRY(255) + [154 ... 255] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_type.h b/drivers/net/ethernet/intel/iavf/iavf_type.h index de9fda78b43a..9f1f523807c4 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_type.h +++ b/drivers/net/ethernet/intel/iavf/iavf_type.h @@ -370,7 +370,6 @@ enum iavf_rx_l2_ptype { }; struct iavf_rx_ptype_decoded { - u32 ptype:8; u32 known:1; u32 outer_ip:1; u32 outer_ip_ver:1; diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index 07fe857e9e3a..4f538cdf42c1 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -22,12 +22,14 @@ ice-y := ice_main.o \ ice_ethtool_fdir.o \ ice_flex_pipe.o \ ice_flow.o \ + ice_idc.o \ ice_devlink.o \ ice_fw_update.o \ ice_lag.o \ ice_ethtool.o ice-$(CONFIG_PCI_IOV) += ice_virtchnl_allowlist.o ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o ice_virtchnl_fdir.o +ice-$(CONFIG_PTP_1588_CLOCK) += ice_ptp.o ice_ptp_hw.o ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_nl.o ice_dcb_lib.o ice-$(CONFIG_RFS_ACCEL) += ice_arfs.o ice-$(CONFIG_XDP_SOCKETS) += ice_xsk.o diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index e35db3ff583b..a450343fbb92 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -34,6 +34,7 @@ #include <linux/if_bridge.h> #include <linux/ctype.h> #include <linux/bpf.h> +#include <linux/auxiliary_bus.h> #include <linux/avf/virtchnl.h> #include <linux/cpu_rmap.h> #include <linux/dim.h> @@ -55,8 +56,10 @@ #include "ice_switch.h" #include "ice_common.h" #include "ice_sched.h" +#include "ice_idc_int.h" #include "ice_virtchnl_pf.h" #include "ice_sriov.h" +#include "ice_ptp.h" #include "ice_fdir.h" #include "ice_xsk.h" #include "ice_arfs.h" @@ -72,12 +75,15 @@ #define ICE_DFLT_TRAFFIC_CLASS BIT(0) #define ICE_INT_NAME_STR_LEN (IFNAMSIZ + 16) -#define ICE_AQ_LEN 64 +#define ICE_AQ_LEN 192 #define ICE_MBXSQ_LEN 64 +#define ICE_SBQ_LEN 64 #define ICE_MIN_LAN_TXRX_MSIX 1 #define ICE_MIN_LAN_OICR_MSIX 1 #define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX) #define ICE_FDIR_MSIX 2 +#define ICE_RDMA_NUM_AEQ_MSIX 4 +#define ICE_MIN_RDMA_MSIX 2 #define ICE_NO_VSI 0xffff #define ICE_VSI_MAP_CONTIG 0 #define ICE_VSI_MAP_SCATTER 1 @@ -88,8 +94,9 @@ #define ICE_MAX_LG_RSS_QS 256 #define ICE_RES_VALID_BIT 0x8000 #define ICE_RES_MISC_VEC_ID (ICE_RES_VALID_BIT - 1) +#define ICE_RES_RDMA_VEC_ID (ICE_RES_MISC_VEC_ID - 1) /* All VF control VSIs share the same IRQ, so assign a unique ID for them */ -#define ICE_RES_VF_CTRL_VEC_ID (ICE_RES_MISC_VEC_ID - 1) +#define ICE_RES_VF_CTRL_VEC_ID (ICE_RES_RDMA_VEC_ID - 1) #define ICE_INVAL_Q_INDEX 0xffff #define ICE_INVAL_VFID 256 @@ -203,9 +210,9 @@ enum ice_pf_state { ICE_NEEDS_RESTART, ICE_PREPARED_FOR_RESET, /* set by driver when prepared */ ICE_RESET_OICR_RECV, /* set by driver after rcv reset OICR */ - ICE_PFR_REQ, /* set by driver and peers */ - ICE_CORER_REQ, /* set by driver and peers */ - ICE_GLOBR_REQ, /* set by driver and peers */ + ICE_PFR_REQ, /* set by driver */ + ICE_CORER_REQ, /* set by driver */ + ICE_GLOBR_REQ, /* set by driver */ ICE_CORER_RECV, /* set by OICR handler */ ICE_GLOBR_RECV, /* set by OICR handler */ ICE_EMPR_RECV, /* set by OICR handler */ @@ -222,6 +229,7 @@ enum ice_pf_state { ICE_STATE_NOMINAL_CHECK_BITS, ICE_ADMINQ_EVENT_PENDING, ICE_MAILBOXQ_EVENT_PENDING, + ICE_SIDEBANDQ_EVENT_PENDING, ICE_MDD_EVENT_PENDING, ICE_VFLR_EVENT_PENDING, ICE_FLTR_OVERFLOW_PROMISC, @@ -332,9 +340,11 @@ struct ice_vsi { u16 req_rxq; /* User requested Rx queues */ u16 num_rx_desc; u16 num_tx_desc; + u16 qset_handle[ICE_MAX_TRAFFIC_CLASS]; struct ice_tc_cfg tc_cfg; struct bpf_prog *xdp_prog; struct ice_ring **xdp_rings; /* XDP ring array */ + unsigned long *af_xdp_zc_qps; /* tracks AF_XDP ZC enabled qps */ u16 num_xdp_txq; /* Used XDP queues */ u8 xdp_mapping_mode; /* ICE_MAP_MODE_[CONTIG|SCATTER] */ @@ -373,17 +383,22 @@ struct ice_q_vector { enum ice_pf_flags { ICE_FLAG_FLTR_SYNC, + ICE_FLAG_RDMA_ENA, ICE_FLAG_RSS_ENA, ICE_FLAG_SRIOV_ENA, ICE_FLAG_SRIOV_CAPABLE, ICE_FLAG_DCB_CAPABLE, ICE_FLAG_DCB_ENA, ICE_FLAG_FD_ENA, + ICE_FLAG_PTP_SUPPORTED, /* PTP is supported by NVM */ + ICE_FLAG_PTP, /* PTP is enabled by software */ + ICE_FLAG_AUX_ENA, ICE_FLAG_ADV_FEATURES, ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, ICE_FLAG_NO_MEDIA, ICE_FLAG_FW_LLDP_AGENT, + ICE_FLAG_MOD_POWER_UNSUPPORTED, ICE_FLAG_ETHTOOL_CTXT, /* set when ethtool holds RTNL lock */ ICE_FLAG_LEGACY_RX, ICE_FLAG_VF_TRUE_PROMISC_ENA, @@ -439,12 +454,17 @@ struct ice_pf { struct mutex sw_mutex; /* lock for protecting VSI alloc flow */ struct mutex tc_mutex; /* lock to protect TC changes */ u32 msg_enable; + struct ice_ptp ptp; + u16 num_rdma_msix; /* Total MSIX vectors for RDMA driver */ + u16 rdma_base_vector; /* spinlock to protect the AdminQ wait list */ spinlock_t aq_wait_lock; struct hlist_head aq_wait_list; wait_queue_head_t aq_wait_queue; + wait_queue_head_t reset_wait_queue; + u32 hw_csum_rx_error; u16 oicr_idx; /* Other interrupt cause MSIX vector index */ u16 num_avail_sw_msix; /* remaining MSIX SW vectors left unclaimed */ @@ -471,6 +491,8 @@ struct ice_pf { unsigned long tx_timeout_last_recovery; u32 tx_timeout_recovery_level; char int_name[ICE_INT_NAME_STR_LEN]; + struct auxiliary_device *adev; + int aux_idx; u32 sw_int_count; __le64 nvm_phy_type_lo; /* NVM PHY type low */ @@ -547,15 +569,16 @@ static inline void ice_set_ring_xdp(struct ice_ring *ring) */ static inline struct xsk_buff_pool *ice_xsk_pool(struct ice_ring *ring) { + struct ice_vsi *vsi = ring->vsi; u16 qid = ring->q_index; if (ice_ring_is_xdp(ring)) - qid -= ring->vsi->num_xdp_txq; + qid -= vsi->num_xdp_txq; - if (!ice_is_xdp_ena_vsi(ring->vsi)) + if (!ice_is_xdp_ena_vsi(vsi) || !test_bit(qid, vsi->af_xdp_zc_qps)) return NULL; - return xsk_get_pool_from_qid(ring->vsi->netdev, qid); + return xsk_get_pool_from_qid(vsi->netdev, qid); } /** @@ -636,6 +659,9 @@ int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed); void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size); int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset); void ice_print_link_msg(struct ice_vsi *vsi, bool isup); +int ice_plug_aux_dev(struct ice_pf *pf); +void ice_unplug_aux_dev(struct ice_pf *pf); +int ice_init_rdma(struct ice_pf *pf); const char *ice_stat_str(enum ice_status stat_err); const char *ice_aq_str(enum ice_aq_err aq_err); bool ice_is_wol_supported(struct ice_hw *hw); @@ -660,4 +686,25 @@ int ice_open_internal(struct net_device *netdev); int ice_stop(struct net_device *netdev); void ice_service_task_schedule(struct ice_pf *pf); +/** + * ice_set_rdma_cap - enable RDMA support + * @pf: PF struct + */ +static inline void ice_set_rdma_cap(struct ice_pf *pf) +{ + if (pf->hw.func_caps.common_cap.rdma && pf->num_rdma_msix) { + set_bit(ICE_FLAG_RDMA_ENA, pf->flags); + ice_plug_aux_dev(pf); + } +} + +/** + * ice_clear_rdma_cap - disable RDMA support + * @pf: PF struct + */ +static inline void ice_clear_rdma_cap(struct ice_pf *pf) +{ + ice_unplug_aux_dev(pf); + clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); +} #endif /* _ICE_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 5cdfe406af84..21b4c7cd6f05 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -108,6 +108,7 @@ struct ice_aqc_list_caps_elem { #define ICE_AQC_CAPS_TXQS 0x0042 #define ICE_AQC_CAPS_MSIX 0x0043 #define ICE_AQC_CAPS_FD 0x0045 +#define ICE_AQC_CAPS_1588 0x0046 #define ICE_AQC_CAPS_MAX_MTU 0x0047 #define ICE_AQC_CAPS_NVM_VER 0x0048 #define ICE_AQC_CAPS_PENDING_NVM_VER 0x0049 @@ -115,6 +116,7 @@ struct ice_aqc_list_caps_elem { #define ICE_AQC_CAPS_PENDING_OROM_VER 0x004B #define ICE_AQC_CAPS_NET_VER 0x004C #define ICE_AQC_CAPS_PENDING_NET_VER 0x004D +#define ICE_AQC_CAPS_RDMA 0x0051 #define ICE_AQC_CAPS_NVM_MGMT 0x0080 u8 major_ver; @@ -1122,7 +1124,9 @@ struct ice_aqc_get_link_status_data { #define ICE_AQ_LINK_TOPO_UNDRUTIL_PRT BIT(5) #define ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA BIT(6) #define ICE_AQ_LINK_TOPO_UNSUPP_MEDIA BIT(7) - u8 reserved1; + u8 link_cfg_err; +#define ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED BIT(5) +#define ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT BIT(7) u8 link_info; #define ICE_AQ_LINK_UP BIT(0) /* Link Status */ #define ICE_AQ_LINK_FAULT BIT(1) @@ -1165,7 +1169,7 @@ struct ice_aqc_get_link_status_data { #define ICE_AQ_CFG_PACING_TYPE_FIXED ICE_AQ_CFG_PACING_TYPE_M /* External Device Power Ability */ u8 power_desc; -#define ICE_AQ_PWR_CLASS_M 0x3 +#define ICE_AQ_PWR_CLASS_M 0x3F #define ICE_AQ_LINK_PWR_BASET_LOW_HIGH 0 #define ICE_AQ_LINK_PWR_BASET_HIGH 1 #define ICE_AQ_LINK_PWR_QSFP_CLASS_1 0 @@ -1608,6 +1612,15 @@ struct ice_aqc_get_set_rss_lut { __le32 addr_low; }; +/* Sideband Control Interface Commands */ +/* Neighbor Device Request (indirect 0x0C00); also used for the response. */ +struct ice_aqc_neigh_dev_req { + __le16 sb_data_len; + u8 reserved[6]; + __le32 addr_high; + __le32 addr_low; +}; + /* Add Tx LAN Queues (indirect 0x0C30) */ struct ice_aqc_add_txqs { u8 num_qgrps; @@ -1684,6 +1697,36 @@ struct ice_aqc_dis_txq_item { __le16 q_id[]; } __packed; +/* Add Tx RDMA Queue Set (indirect 0x0C33) */ +struct ice_aqc_add_rdma_qset { + u8 num_qset_grps; + u8 reserved[7]; + __le32 addr_high; + __le32 addr_low; +}; + +/* This is the descriptor of each Qset entry for the Add Tx RDMA Queue Set + * command (0x0C33). Only used within struct ice_aqc_add_rdma_qset. + */ +struct ice_aqc_add_tx_rdma_qset_entry { + __le16 tx_qset_id; + u8 rsvd[2]; + __le32 qset_teid; + struct ice_aqc_txsched_elem info; +}; + +/* The format of the command buffer for Add Tx RDMA Queue Set(0x0C33) + * is an array of the following structs. Please note that the length of + * each struct ice_aqc_add_rdma_qset is variable due to the variable + * number of queues in each group! + */ +struct ice_aqc_add_rdma_qset_data { + __le32 parent_teid; + __le16 num_qsets; + u8 rsvd[2]; + struct ice_aqc_add_tx_rdma_qset_entry rdma_qsets[]; +}; + /* Configure Firmware Logging Command (indirect 0xFF09) * Logging Information Read Response (indirect 0xFF10) * Note: The 0xFF10 command has no input parameters. @@ -1810,6 +1853,30 @@ struct ice_aqc_get_pkg_info_resp { struct ice_aqc_get_pkg_info pkg_info[]; }; +/* Driver Shared Parameters (direct, 0x0C90) */ +struct ice_aqc_driver_shared_params { + u8 set_or_get_op; +#define ICE_AQC_DRIVER_PARAM_OP_MASK BIT(0) +#define ICE_AQC_DRIVER_PARAM_SET 0 +#define ICE_AQC_DRIVER_PARAM_GET 1 + u8 param_indx; +#define ICE_AQC_DRIVER_PARAM_MAX_IDX 15 + u8 rsvd[2]; + __le32 param_val; + __le32 addr_high; + __le32 addr_low; +}; + +enum ice_aqc_driver_params { + /* OS clock index for PTP timer Domain 0 */ + ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0 = 0, + /* OS clock index for PTP timer Domain 1 */ + ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1, + + /* Add new parameters above */ + ICE_AQC_DRIVER_PARAM_MAX = 16, +}; + /* Lan Queue Overflow Event (direct, 0x1001) */ struct ice_aqc_event_lan_overflow { __le32 prtdcb_ruptq; @@ -1878,13 +1945,16 @@ struct ice_aq_desc { struct ice_aqc_lldp_filter_ctrl lldp_filter_ctrl; struct ice_aqc_get_set_rss_lut get_set_rss_lut; struct ice_aqc_get_set_rss_key get_set_rss_key; + struct ice_aqc_neigh_dev_req neigh_dev; struct ice_aqc_add_txqs add_txqs; struct ice_aqc_dis_txqs dis_txqs; + struct ice_aqc_add_rdma_qset add_rdma_qset; struct ice_aqc_add_get_update_free_vsi vsi_cmd; struct ice_aqc_add_update_free_vsi_resp add_update_free_vsi_res; struct ice_aqc_fw_logging fw_logging; struct ice_aqc_get_clear_fw_log get_clear_fw_log; struct ice_aqc_download_pkg download_pkg; + struct ice_aqc_driver_shared_params drv_shared_params; struct ice_aqc_set_mac_lb set_mac_lb; struct ice_aqc_alloc_free_res_cmd sw_res_ctrl; struct ice_aqc_set_mac_cfg set_mac_cfg; @@ -2025,15 +2095,21 @@ enum ice_adminq_opc { ice_aqc_opc_get_rss_key = 0x0B04, ice_aqc_opc_get_rss_lut = 0x0B05, + /* Sideband Control Interface commands */ + ice_aqc_opc_neighbour_device_request = 0x0C00, + /* Tx queue handling commands/events */ ice_aqc_opc_add_txqs = 0x0C30, ice_aqc_opc_dis_txqs = 0x0C31, + ice_aqc_opc_add_rdma_qset = 0x0C33, /* package commands */ ice_aqc_opc_download_pkg = 0x0C40, ice_aqc_opc_update_pkg = 0x0C42, ice_aqc_opc_get_pkg_info_list = 0x0C43, + ice_aqc_opc_driver_shared_params = 0x0C90, + /* Standalone Commands/Events */ ice_aqc_opc_event_lan_overflow = 0x1001, diff --git a/drivers/net/ethernet/intel/ice/ice_arfs.h b/drivers/net/ethernet/intel/ice/ice_arfs.h index f39cd16403ed..80ed76f0cace 100644 --- a/drivers/net/ethernet/intel/ice/ice_arfs.h +++ b/drivers/net/ethernet/intel/ice/ice_arfs.h @@ -52,12 +52,12 @@ bool ice_is_arfs_using_perfect_flow(struct ice_hw *hw, enum ice_fltr_ptype flow_type); #else -#define ice_sync_arfs_fltrs(pf) do {} while (0) -#define ice_init_arfs(vsi) do {} while (0) -#define ice_clear_arfs(vsi) do {} while (0) -#define ice_remove_arfs(pf) do {} while (0) -#define ice_free_cpu_rx_rmap(vsi) do {} while (0) -#define ice_rebuild_arfs(pf) do {} while (0) +static inline void ice_clear_arfs(struct ice_vsi *vsi) { } +static inline void ice_free_cpu_rx_rmap(struct ice_vsi *vsi) { } +static inline void ice_init_arfs(struct ice_vsi *vsi) { } +static inline void ice_sync_arfs_fltrs(struct ice_pf *pf) { } +static inline void ice_remove_arfs(struct ice_pf *pf) { } +static inline void ice_rebuild_arfs(struct ice_pf *pf) { } static inline int ice_set_cpu_rx_rmap(struct ice_vsi __always_unused *vsi) { diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index 5985a7e5ca8a..c36057efc7ae 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -287,6 +287,15 @@ ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) /* make sure the context is associated with the right VSI */ tlan_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx); + /* Restrict Tx timestamps to the PF VSI */ + switch (vsi->type) { + case ICE_VSI_PF: + tlan_ctx->tsyn_ena = 1; + break; + default: + break; + } + tlan_ctx->tso_ena = ICE_TX_LEGACY; tlan_ctx->tso_qnum = pf_q; @@ -319,11 +328,9 @@ static unsigned int ice_rx_offset(struct ice_ring *rx_ring) * * Configure the Rx descriptor ring in RLAN context. */ -int ice_setup_rx_ctx(struct ice_ring *ring) +static int ice_setup_rx_ctx(struct ice_ring *ring) { - struct device *dev = ice_pf_to_dev(ring->vsi->back); int chain_len = ICE_MAX_CHAINED_RX_BUFS; - u16 num_bufs = ICE_DESC_UNUSED(ring); struct ice_vsi *vsi = ring->vsi; u32 rxdid = ICE_RXDID_FLEX_NIC; struct ice_rlan_ctx rlan_ctx; @@ -339,48 +346,6 @@ int ice_setup_rx_ctx(struct ice_ring *ring) /* clear the context structure first */ memset(&rlan_ctx, 0, sizeof(rlan_ctx)); - ring->rx_buf_len = vsi->rx_buf_len; - - if (ring->vsi->type == ICE_VSI_PF) { - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) - /* coverity[check_return] */ - xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->q_index, ring->q_vector->napi.napi_id); - - ring->xsk_pool = ice_xsk_pool(ring); - if (ring->xsk_pool) { - xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); - - ring->rx_buf_len = - xsk_pool_get_rx_frame_size(ring->xsk_pool); - /* For AF_XDP ZC, we disallow packets to span on - * multiple buffers, thus letting us skip that - * handling in the fast-path. - */ - chain_len = 1; - err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_XSK_BUFF_POOL, - NULL); - if (err) - return err; - xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); - - dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", - ring->q_index); - } else { - if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) - /* coverity[check_return] */ - xdp_rxq_info_reg(&ring->xdp_rxq, - ring->netdev, - ring->q_index, ring->q_vector->napi.napi_id); - - err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_PAGE_SHARED, - NULL); - if (err) - return err; - } - } /* Receive Queue Base Address. * Indicates the starting address of the descriptor queue defined in * 128 Byte units. @@ -415,6 +380,12 @@ int ice_setup_rx_ctx(struct ice_ring *ring) */ rlan_ctx.showiv = 0; + /* For AF_XDP ZC, we disallow packets to span on + * multiple buffers, thus letting us skip that + * handling in the fast-path. + */ + if (ring->xsk_pool) + chain_len = 1; /* Max packet size for this queue - must not be set to a larger value * than 5 x DBUF */ @@ -431,14 +402,15 @@ int ice_setup_rx_ctx(struct ice_ring *ring) * of same priority */ if (vsi->type != ICE_VSI_VF) - ice_write_qrxflxp_cntxt(hw, pf_q, rxdid, 0x3); + ice_write_qrxflxp_cntxt(hw, pf_q, rxdid, 0x3, true); else - ice_write_qrxflxp_cntxt(hw, pf_q, ICE_RXDID_LEGACY_1, 0x3); + ice_write_qrxflxp_cntxt(hw, pf_q, ICE_RXDID_LEGACY_1, 0x3, + false); /* Absolute queue number out of 2K needs to be passed */ err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q); if (err) { - dev_err(dev, "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n", + dev_err(ice_pf_to_dev(vsi->back), "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n", pf_q, err); return -EIO; } @@ -458,6 +430,66 @@ int ice_setup_rx_ctx(struct ice_ring *ring) ring->tail = hw->hw_addr + QRX_TAIL(pf_q); writel(0, ring->tail); + return 0; +} + +/** + * ice_vsi_cfg_rxq - Configure an Rx queue + * @ring: the ring being configured + * + * Return 0 on success and a negative value on error. + */ +int ice_vsi_cfg_rxq(struct ice_ring *ring) +{ + struct device *dev = ice_pf_to_dev(ring->vsi->back); + u16 num_bufs = ICE_DESC_UNUSED(ring); + int err; + + ring->rx_buf_len = ring->vsi->rx_buf_len; + + if (ring->vsi->type == ICE_VSI_PF) { + if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) + /* coverity[check_return] */ + xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->q_index, ring->q_vector->napi.napi_id); + + ring->xsk_pool = ice_xsk_pool(ring); + if (ring->xsk_pool) { + xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + + ring->rx_buf_len = + xsk_pool_get_rx_frame_size(ring->xsk_pool); + err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL); + if (err) + return err; + xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); + + dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", + ring->q_index); + } else { + if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) + /* coverity[check_return] */ + xdp_rxq_info_reg(&ring->xdp_rxq, + ring->netdev, + ring->q_index, ring->q_vector->napi.napi_id); + + err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_PAGE_SHARED, + NULL); + if (err) + return err; + } + } + + err = ice_setup_rx_ctx(ring); + if (err) { + dev_err(dev, "ice_setup_rx_ctx failed for RxQ %d, err %d\n", + ring->q_index, err); + return err; + } + if (ring->xsk_pool) { bool ok; @@ -470,9 +502,13 @@ int ice_setup_rx_ctx(struct ice_ring *ring) } ok = ice_alloc_rx_bufs_zc(ring, num_bufs); - if (!ok) + if (!ok) { + u16 pf_q = ring->vsi->rxq_map[ring->q_index]; + dev_info(dev, "Failed to allocate some buffers on XSK buffer pool enabled Rx ring %d (pf_q %d)\n", ring->q_index, pf_q); + } + return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice_base.h b/drivers/net/ethernet/intel/ice/ice_base.h index 44efdb627043..20e1c29aa68a 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.h +++ b/drivers/net/ethernet/intel/ice/ice_base.h @@ -6,7 +6,7 @@ #include "ice.h" -int ice_setup_rx_ctx(struct ice_ring *ring); +int ice_vsi_cfg_rxq(struct ice_ring *ring); int __ice_vsi_get_qs(struct ice_qs_cfg *qs_cfg); int ice_vsi_ctrl_one_rx_ring(struct ice_vsi *vsi, bool ena, u16 rxq_idx, bool wait); diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index e93b1e40f627..2fb81e359cdf 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -2,6 +2,7 @@ /* Copyright (c) 2018, Intel Corporation. */ #include "ice_common.h" +#include "ice_lib.h" #include "ice_sched.h" #include "ice_adminq_cmd.h" #include "ice_flow.h" @@ -58,6 +59,17 @@ static enum ice_status ice_set_mac_type(struct ice_hw *hw) } /** + * ice_is_e810 + * @hw: pointer to the hardware structure + * + * returns true if the device is E810 based, false if not. + */ +bool ice_is_e810(struct ice_hw *hw) +{ + return hw->mac_type == ICE_MAC_E810; +} + +/** * ice_clear_pf_cfg - Clear PF configuration * @hw: pointer to the hardware structure * @@ -424,6 +436,7 @@ ice_aq_get_link_info(struct ice_port_info *pi, bool ena_lse, li->phy_type_high = le64_to_cpu(link_data.phy_type_high); *hw_media_type = ice_get_media_type(pi); li->link_info = link_data.link_info; + li->link_cfg_err = link_data.link_cfg_err; li->an_info = link_data.an_info; li->ext_info = link_data.ext_info; li->max_frame_size = le16_to_cpu(link_data.max_frame_size); @@ -454,6 +467,7 @@ ice_aq_get_link_info(struct ice_port_info *pi, bool ena_lse, (unsigned long long)li->phy_type_high); ice_debug(hw, ICE_DBG_LINK, " media_type = 0x%x\n", *hw_media_type); ice_debug(hw, ICE_DBG_LINK, " link_info = 0x%x\n", li->link_info); + ice_debug(hw, ICE_DBG_LINK, " link_cfg_err = 0x%x\n", li->link_cfg_err); ice_debug(hw, ICE_DBG_LINK, " an_info = 0x%x\n", li->an_info); ice_debug(hw, ICE_DBG_LINK, " ext_info = 0x%x\n", li->ext_info); ice_debug(hw, ICE_DBG_LINK, " fec_info = 0x%x\n", li->fec_info); @@ -1062,7 +1076,8 @@ enum ice_status ice_check_reset(struct ice_hw *hw) GLNVM_ULD_POR_DONE_1_M |\ GLNVM_ULD_PCIER_DONE_2_M) - uld_mask = ICE_RESET_DONE_MASK; + uld_mask = ICE_RESET_DONE_MASK | (hw->func_caps.common_cap.rdma ? + GLNVM_ULD_PE_DONE_M : 0); /* Device is Active; check Global Reset processes are done */ for (cnt = 0; cnt < ICE_PF_RESET_WAIT_COUNT; cnt++) { @@ -1289,6 +1304,64 @@ const struct ice_ctx_ele ice_tlan_ctx_info[] = { { 0 } }; +/* Sideband Queue command wrappers */ + +/** + * ice_sbq_send_cmd - send Sideband Queue command to Sideband Queue + * @hw: pointer to the HW struct + * @desc: descriptor describing the command + * @buf: buffer to use for indirect commands (NULL for direct commands) + * @buf_size: size of buffer for indirect commands (0 for direct commands) + * @cd: pointer to command details structure + */ +static int +ice_sbq_send_cmd(struct ice_hw *hw, struct ice_sbq_cmd_desc *desc, + void *buf, u16 buf_size, struct ice_sq_cd *cd) +{ + return ice_status_to_errno(ice_sq_send_cmd(hw, ice_get_sbq(hw), + (struct ice_aq_desc *)desc, + buf, buf_size, cd)); +} + +/** + * ice_sbq_rw_reg - Fill Sideband Queue command + * @hw: pointer to the HW struct + * @in: message info to be filled in descriptor + */ +int ice_sbq_rw_reg(struct ice_hw *hw, struct ice_sbq_msg_input *in) +{ + struct ice_sbq_cmd_desc desc = {0}; + struct ice_sbq_msg_req msg = {0}; + u16 msg_len; + int status; + + msg_len = sizeof(msg); + + msg.dest_dev = in->dest_dev; + msg.opcode = in->opcode; + msg.flags = ICE_SBQ_MSG_FLAGS; + msg.sbe_fbe = ICE_SBQ_MSG_SBE_FBE; + msg.msg_addr_low = cpu_to_le16(in->msg_addr_low); + msg.msg_addr_high = cpu_to_le32(in->msg_addr_high); + + if (in->opcode) + msg.data = cpu_to_le32(in->data); + else + /* data read comes back in completion, so shorten the struct by + * sizeof(msg.data) + */ + msg_len -= sizeof(msg.data); + + desc.flags = cpu_to_le16(ICE_AQ_FLAG_RD); + desc.opcode = cpu_to_le16(ice_sbq_opc_neigh_dev_req); + desc.param0.cmd_len = cpu_to_le16(msg_len); + status = ice_sbq_send_cmd(hw, &desc, &msg, msg_len, NULL); + if (!status && !in->opcode) + in->data = le32_to_cpu + (((struct ice_sbq_msg_cmpl *)&msg)->data); + return status; +} + /* FW Admin Queue command wrappers */ /* Software lock/mutex that is meant to be held while the Global Config Lock @@ -1938,6 +2011,10 @@ ice_parse_common_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps, ice_debug(hw, ICE_DBG_INIT, "%s: nvm_unified_update = %d\n", prefix, caps->nvm_unified_update); break; + case ICE_AQC_CAPS_RDMA: + caps->rdma = (number == 1); + ice_debug(hw, ICE_DBG_INIT, "%s: rdma = %d\n", prefix, caps->rdma); + break; case ICE_AQC_CAPS_MAX_MTU: caps->max_mtu = number; ice_debug(hw, ICE_DBG_INIT, "%s: max_mtu = %d\n", @@ -1971,6 +2048,16 @@ ice_recalc_port_limited_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps) caps->maxtc = 4; ice_debug(hw, ICE_DBG_INIT, "reducing maxtc to %d (based on #ports)\n", caps->maxtc); + if (caps->rdma) { + ice_debug(hw, ICE_DBG_INIT, "forcing RDMA off\n"); + caps->rdma = 0; + } + + /* print message only when processing device capabilities + * during initialization. + */ + if (caps == &hw->dev_caps.common_cap) + dev_info(ice_hw_to_dev(hw), "RDMA functionality is not available with the current device configuration.\n"); } } @@ -2017,6 +2104,48 @@ ice_parse_vsi_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p, } /** + * ice_parse_1588_func_caps - Parse ICE_AQC_CAPS_1588 function caps + * @hw: pointer to the HW struct + * @func_p: pointer to function capabilities structure + * @cap: pointer to the capability element to parse + * + * Extract function capabilities for ICE_AQC_CAPS_1588. + */ +static void +ice_parse_1588_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p, + struct ice_aqc_list_caps_elem *cap) +{ + struct ice_ts_func_info *info = &func_p->ts_func_info; + u32 number = le32_to_cpu(cap->number); + + info->ena = ((number & ICE_TS_FUNC_ENA_M) != 0); + func_p->common_cap.ieee_1588 = info->ena; + + info->src_tmr_owned = ((number & ICE_TS_SRC_TMR_OWND_M) != 0); + info->tmr_ena = ((number & ICE_TS_TMR_ENA_M) != 0); + info->tmr_index_owned = ((number & ICE_TS_TMR_IDX_OWND_M) != 0); + info->tmr_index_assoc = ((number & ICE_TS_TMR_IDX_ASSOC_M) != 0); + + info->clk_freq = (number & ICE_TS_CLK_FREQ_M) >> ICE_TS_CLK_FREQ_S; + info->clk_src = ((number & ICE_TS_CLK_SRC_M) != 0); + + ice_debug(hw, ICE_DBG_INIT, "func caps: ieee_1588 = %u\n", + func_p->common_cap.ieee_1588); + ice_debug(hw, ICE_DBG_INIT, "func caps: src_tmr_owned = %u\n", + info->src_tmr_owned); + ice_debug(hw, ICE_DBG_INIT, "func caps: tmr_ena = %u\n", + info->tmr_ena); + ice_debug(hw, ICE_DBG_INIT, "func caps: tmr_index_owned = %u\n", + info->tmr_index_owned); + ice_debug(hw, ICE_DBG_INIT, "func caps: tmr_index_assoc = %u\n", + info->tmr_index_assoc); + ice_debug(hw, ICE_DBG_INIT, "func caps: clk_freq = %u\n", + info->clk_freq); + ice_debug(hw, ICE_DBG_INIT, "func caps: clk_src = %u\n", + info->clk_src); +} + +/** * ice_parse_fdir_func_caps - Parse ICE_AQC_CAPS_FD function caps * @hw: pointer to the HW struct * @func_p: pointer to function capabilities structure @@ -2082,6 +2211,9 @@ ice_parse_func_caps(struct ice_hw *hw, struct ice_hw_func_caps *func_p, case ICE_AQC_CAPS_VSI: ice_parse_vsi_func_caps(hw, func_p, &cap_resp[i]); break; + case ICE_AQC_CAPS_1588: + ice_parse_1588_func_caps(hw, func_p, &cap_resp[i]); + break; case ICE_AQC_CAPS_FD: ice_parse_fdir_func_caps(hw, func_p); break; @@ -2155,6 +2287,57 @@ ice_parse_vsi_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p, } /** + * ice_parse_1588_dev_caps - Parse ICE_AQC_CAPS_1588 device caps + * @hw: pointer to the HW struct + * @dev_p: pointer to device capabilities structure + * @cap: capability element to parse + * + * Parse ICE_AQC_CAPS_1588 for device capabilities. + */ +static void +ice_parse_1588_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p, + struct ice_aqc_list_caps_elem *cap) +{ + struct ice_ts_dev_info *info = &dev_p->ts_dev_info; + u32 logical_id = le32_to_cpu(cap->logical_id); + u32 phys_id = le32_to_cpu(cap->phys_id); + u32 number = le32_to_cpu(cap->number); + + info->ena = ((number & ICE_TS_DEV_ENA_M) != 0); + dev_p->common_cap.ieee_1588 = info->ena; + + info->tmr0_owner = number & ICE_TS_TMR0_OWNR_M; + info->tmr0_owned = ((number & ICE_TS_TMR0_OWND_M) != 0); + info->tmr0_ena = ((number & ICE_TS_TMR0_ENA_M) != 0); + + info->tmr1_owner = (number & ICE_TS_TMR1_OWNR_M) >> ICE_TS_TMR1_OWNR_S; + info->tmr1_owned = ((number & ICE_TS_TMR1_OWND_M) != 0); + info->tmr1_ena = ((number & ICE_TS_TMR1_ENA_M) != 0); + + info->ena_ports = logical_id; + info->tmr_own_map = phys_id; + + ice_debug(hw, ICE_DBG_INIT, "dev caps: ieee_1588 = %u\n", + dev_p->common_cap.ieee_1588); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr0_owner = %u\n", + info->tmr0_owner); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr0_owned = %u\n", + info->tmr0_owned); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr0_ena = %u\n", + info->tmr0_ena); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr1_owner = %u\n", + info->tmr1_owner); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr1_owned = %u\n", + info->tmr1_owned); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr1_ena = %u\n", + info->tmr1_ena); + ice_debug(hw, ICE_DBG_INIT, "dev caps: ieee_1588 ena_ports = %u\n", + info->ena_ports); + ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr_own_map = %u\n", + info->tmr_own_map); +} + +/** * ice_parse_fdir_dev_caps - Parse ICE_AQC_CAPS_FD device caps * @hw: pointer to the HW struct * @dev_p: pointer to device capabilities structure @@ -2215,6 +2398,9 @@ ice_parse_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p, case ICE_AQC_CAPS_VSI: ice_parse_vsi_dev_caps(hw, dev_p, &cap_resp[i]); break; + case ICE_AQC_CAPS_1588: + ice_parse_1588_dev_caps(hw, dev_p, &cap_resp[i]); + break; case ICE_AQC_CAPS_FD: ice_parse_fdir_dev_caps(hw, dev_p, &cap_resp[i]); break; @@ -3635,6 +3821,52 @@ do_aq: return status; } +/** + * ice_aq_add_rdma_qsets + * @hw: pointer to the hardware structure + * @num_qset_grps: Number of RDMA Qset groups + * @qset_list: list of Qset groups to be added + * @buf_size: size of buffer for indirect command + * @cd: pointer to command details structure or NULL + * + * Add Tx RDMA Qsets (0x0C33) + */ +static int +ice_aq_add_rdma_qsets(struct ice_hw *hw, u8 num_qset_grps, + struct ice_aqc_add_rdma_qset_data *qset_list, + u16 buf_size, struct ice_sq_cd *cd) +{ + struct ice_aqc_add_rdma_qset_data *list; + struct ice_aqc_add_rdma_qset *cmd; + struct ice_aq_desc desc; + u16 i, sum_size = 0; + + cmd = &desc.params.add_rdma_qset; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_add_rdma_qset); + + if (num_qset_grps > ICE_LAN_TXQ_MAX_QGRPS) + return -EINVAL; + + for (i = 0, list = qset_list; i < num_qset_grps; i++) { + u16 num_qsets = le16_to_cpu(list->num_qsets); + + sum_size += struct_size(list, rdma_qsets, num_qsets); + list = (struct ice_aqc_add_rdma_qset_data *)(list->rdma_qsets + + num_qsets); + } + + if (buf_size != sum_size) + return -EINVAL; + + desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD); + + cmd->num_qset_grps = num_qset_grps; + + return ice_status_to_errno(ice_aq_send_cmd(hw, &desc, qset_list, + buf_size, cd)); +} + /* End of FW Admin Queue command wrappers */ /** @@ -4133,6 +4365,162 @@ ice_cfg_vsi_lan(struct ice_port_info *pi, u16 vsi_handle, u8 tc_bitmap, } /** + * ice_cfg_vsi_rdma - configure the VSI RDMA queues + * @pi: port information structure + * @vsi_handle: software VSI handle + * @tc_bitmap: TC bitmap + * @max_rdmaqs: max RDMA queues array per TC + * + * This function adds/updates the VSI RDMA queues per TC. + */ +int +ice_cfg_vsi_rdma(struct ice_port_info *pi, u16 vsi_handle, u16 tc_bitmap, + u16 *max_rdmaqs) +{ + return ice_status_to_errno(ice_cfg_vsi_qs(pi, vsi_handle, tc_bitmap, + max_rdmaqs, + ICE_SCHED_NODE_OWNER_RDMA)); +} + +/** + * ice_ena_vsi_rdma_qset + * @pi: port information structure + * @vsi_handle: software VSI handle + * @tc: TC number + * @rdma_qset: pointer to RDMA Qset + * @num_qsets: number of RDMA Qsets + * @qset_teid: pointer to Qset node TEIDs + * + * This function adds RDMA Qset + */ +int +ice_ena_vsi_rdma_qset(struct ice_port_info *pi, u16 vsi_handle, u8 tc, + u16 *rdma_qset, u16 num_qsets, u32 *qset_teid) +{ + struct ice_aqc_txsched_elem_data node = { 0 }; + struct ice_aqc_add_rdma_qset_data *buf; + struct ice_sched_node *parent; + enum ice_status status; + struct ice_hw *hw; + u16 i, buf_size; + int ret; + + if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY) + return -EIO; + hw = pi->hw; + + if (!ice_is_vsi_valid(hw, vsi_handle)) + return -EINVAL; + + buf_size = struct_size(buf, rdma_qsets, num_qsets); + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + mutex_lock(&pi->sched_lock); + + parent = ice_sched_get_free_qparent(pi, vsi_handle, tc, + ICE_SCHED_NODE_OWNER_RDMA); + if (!parent) { + ret = -EINVAL; + goto rdma_error_exit; + } + buf->parent_teid = parent->info.node_teid; + node.parent_teid = parent->info.node_teid; + + buf->num_qsets = cpu_to_le16(num_qsets); + for (i = 0; i < num_qsets; i++) { + buf->rdma_qsets[i].tx_qset_id = cpu_to_le16(rdma_qset[i]); + buf->rdma_qsets[i].info.valid_sections = + ICE_AQC_ELEM_VALID_GENERIC | ICE_AQC_ELEM_VALID_CIR | + ICE_AQC_ELEM_VALID_EIR; + buf->rdma_qsets[i].info.generic = 0; + buf->rdma_qsets[i].info.cir_bw.bw_profile_idx = + cpu_to_le16(ICE_SCHED_DFLT_RL_PROF_ID); + buf->rdma_qsets[i].info.cir_bw.bw_alloc = + cpu_to_le16(ICE_SCHED_DFLT_BW_WT); + buf->rdma_qsets[i].info.eir_bw.bw_profile_idx = + cpu_to_le16(ICE_SCHED_DFLT_RL_PROF_ID); + buf->rdma_qsets[i].info.eir_bw.bw_alloc = + cpu_to_le16(ICE_SCHED_DFLT_BW_WT); + } + ret = ice_aq_add_rdma_qsets(hw, 1, buf, buf_size, NULL); + if (ret) { + ice_debug(hw, ICE_DBG_RDMA, "add RDMA qset failed\n"); + goto rdma_error_exit; + } + node.data.elem_type = ICE_AQC_ELEM_TYPE_LEAF; + for (i = 0; i < num_qsets; i++) { + node.node_teid = buf->rdma_qsets[i].qset_teid; + status = ice_sched_add_node(pi, hw->num_tx_sched_layers - 1, + &node); + if (status) { + ret = ice_status_to_errno(status); + break; + } + qset_teid[i] = le32_to_cpu(node.node_teid); + } +rdma_error_exit: + mutex_unlock(&pi->sched_lock); + kfree(buf); + return ret; +} + +/** + * ice_dis_vsi_rdma_qset - free RDMA resources + * @pi: port_info struct + * @count: number of RDMA Qsets to free + * @qset_teid: TEID of Qset node + * @q_id: list of queue IDs being disabled + */ +int +ice_dis_vsi_rdma_qset(struct ice_port_info *pi, u16 count, u32 *qset_teid, + u16 *q_id) +{ + struct ice_aqc_dis_txq_item *qg_list; + enum ice_status status = 0; + struct ice_hw *hw; + u16 qg_size; + int i; + + if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY) + return -EIO; + + hw = pi->hw; + + qg_size = struct_size(qg_list, q_id, 1); + qg_list = kzalloc(qg_size, GFP_KERNEL); + if (!qg_list) + return -ENOMEM; + + mutex_lock(&pi->sched_lock); + + for (i = 0; i < count; i++) { + struct ice_sched_node *node; + + node = ice_sched_find_node_by_teid(pi->root, qset_teid[i]); + if (!node) + continue; + + qg_list->parent_teid = node->info.parent_teid; + qg_list->num_qs = 1; + qg_list->q_id[0] = + cpu_to_le16(q_id[i] | + ICE_AQC_Q_DIS_BUF_ELEM_TYPE_RDMA_QSET); + + status = ice_aq_dis_lan_txq(hw, 1, qg_list, qg_size, + ICE_NO_RESET, 0, NULL); + if (status) + break; + + ice_free_sched_node(pi, node); + } + + mutex_unlock(&pi->sched_lock); + kfree(qg_list); + return ice_status_to_errno(status); +} + +/** * ice_replay_pre_init - replay pre initialization * @hw: pointer to the HW struct * @@ -4304,6 +4692,81 @@ ice_sched_query_elem(struct ice_hw *hw, u32 node_teid, } /** + * ice_aq_set_driver_param - Set driver parameter to share via firmware + * @hw: pointer to the HW struct + * @idx: parameter index to set + * @value: the value to set the parameter to + * @cd: pointer to command details structure or NULL + * + * Set the value of one of the software defined parameters. All PFs connected + * to this device can read the value using ice_aq_get_driver_param. + * + * Note that firmware provides no synchronization or locking, and will not + * save the parameter value during a device reset. It is expected that + * a single PF will write the parameter value, while all other PFs will only + * read it. + */ +int +ice_aq_set_driver_param(struct ice_hw *hw, enum ice_aqc_driver_params idx, + u32 value, struct ice_sq_cd *cd) +{ + struct ice_aqc_driver_shared_params *cmd; + struct ice_aq_desc desc; + + if (idx >= ICE_AQC_DRIVER_PARAM_MAX) + return -EIO; + + cmd = &desc.params.drv_shared_params; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_driver_shared_params); + + cmd->set_or_get_op = ICE_AQC_DRIVER_PARAM_SET; + cmd->param_indx = idx; + cmd->param_val = cpu_to_le32(value); + + return ice_status_to_errno(ice_aq_send_cmd(hw, &desc, NULL, 0, cd)); +} + +/** + * ice_aq_get_driver_param - Get driver parameter shared via firmware + * @hw: pointer to the HW struct + * @idx: parameter index to set + * @value: storage to return the shared parameter + * @cd: pointer to command details structure or NULL + * + * Get the value of one of the software defined parameters. + * + * Note that firmware provides no synchronization or locking. It is expected + * that only a single PF will write a given parameter. + */ +int +ice_aq_get_driver_param(struct ice_hw *hw, enum ice_aqc_driver_params idx, + u32 *value, struct ice_sq_cd *cd) +{ + struct ice_aqc_driver_shared_params *cmd; + struct ice_aq_desc desc; + enum ice_status status; + + if (idx >= ICE_AQC_DRIVER_PARAM_MAX) + return -EIO; + + cmd = &desc.params.drv_shared_params; + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_driver_shared_params); + + cmd->set_or_get_op = ICE_AQC_DRIVER_PARAM_GET; + cmd->param_indx = idx; + + status = ice_aq_send_cmd(hw, &desc, NULL, 0, cd); + if (status) + return ice_status_to_errno(status); + + *value = le32_to_cpu(cmd->param_val); + + return 0; +} + +/** * ice_fw_supports_link_override * @hw: pointer to the hardware structure * diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 7a9d2dfb21a2..fb16070f02e2 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -40,6 +40,8 @@ enum ice_status ice_aq_alloc_free_res(struct ice_hw *hw, u16 num_entries, struct ice_aqc_alloc_free_res_elem *buf, u16 buf_size, enum ice_adminq_opc opc, struct ice_sq_cd *cd); +bool ice_is_sbq_supported(struct ice_hw *hw); +struct ice_ctl_q_info *ice_get_sbq(struct ice_hw *hw); enum ice_status ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, struct ice_aq_desc *desc, void *buf, u16 buf_size, @@ -97,6 +99,7 @@ ice_update_phy_type(u64 *phy_type_low, u64 *phy_type_high, enum ice_status ice_aq_manage_mac_write(struct ice_hw *hw, const u8 *mac_addr, u8 flags, struct ice_sq_cd *cd); +bool ice_is_e810(struct ice_hw *hw); enum ice_status ice_clear_pf_cfg(struct ice_hw *hw); enum ice_status ice_aq_set_phy_cfg(struct ice_hw *hw, struct ice_port_info *pi, @@ -147,6 +150,15 @@ ice_aq_sff_eeprom(struct ice_hw *hw, u16 lport, u8 bus_addr, u16 mem_addr, u8 page, u8 set_page, u8 *data, u8 length, bool write, struct ice_sq_cd *cd); +int +ice_cfg_vsi_rdma(struct ice_port_info *pi, u16 vsi_handle, u16 tc_bitmap, + u16 *max_rdmaqs); +int +ice_ena_vsi_rdma_qset(struct ice_port_info *pi, u16 vsi_handle, u8 tc, + u16 *rdma_qset, u16 num_qsets, u32 *qset_teid); +int +ice_dis_vsi_rdma_qset(struct ice_port_info *pi, u16 count, u32 *qset_teid, + u16 *q_id); enum ice_status ice_dis_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u8 num_queues, u16 *q_handle, u16 *q_ids, u32 *q_teids, @@ -164,6 +176,7 @@ void ice_replay_post(struct ice_hw *hw); void ice_output_fw_log(struct ice_hw *hw, struct ice_aq_desc *desc, void *buf); struct ice_q_ctx * ice_get_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 q_handle); +int ice_sbq_rw_reg(struct ice_hw *hw, struct ice_sbq_msg_input *in); void ice_stat_update40(struct ice_hw *hw, u32 reg, bool prev_stat_loaded, u64 *prev_stat, u64 *cur_stat); @@ -173,6 +186,12 @@ ice_stat_update32(struct ice_hw *hw, u32 reg, bool prev_stat_loaded, enum ice_status ice_sched_query_elem(struct ice_hw *hw, u32 node_teid, struct ice_aqc_txsched_elem_data *buf); +int +ice_aq_set_driver_param(struct ice_hw *hw, enum ice_aqc_driver_params idx, + u32 value, struct ice_sq_cd *cd); +int +ice_aq_get_driver_param(struct ice_hw *hw, enum ice_aqc_driver_params idx, + u32 *value, struct ice_sq_cd *cd); enum ice_status ice_aq_set_lldp_mib(struct ice_hw *hw, u8 mib_type, void *buf, u16 buf_size, struct ice_sq_cd *cd); diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.c b/drivers/net/ethernet/intel/ice/ice_controlq.c index 87b33bdd4960..03bdb125be36 100644 --- a/drivers/net/ethernet/intel/ice/ice_controlq.c +++ b/drivers/net/ethernet/intel/ice/ice_controlq.c @@ -52,6 +52,19 @@ static void ice_mailbox_init_regs(struct ice_hw *hw) } /** + * ice_sb_init_regs - Initialize Sideband registers + * @hw: pointer to the hardware structure + * + * This assumes the alloc_sq and alloc_rq functions have already been called + */ +static void ice_sb_init_regs(struct ice_hw *hw) +{ + struct ice_ctl_q_info *cq = &hw->sbq; + + ICE_CQ_INIT_REGS(cq, PF_SB); +} + +/** * ice_check_sq_alive * @hw: pointer to the HW struct * @cq: pointer to the specific Control queue @@ -609,6 +622,10 @@ static enum ice_status ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) ice_adminq_init_regs(hw); cq = &hw->adminq; break; + case ICE_CTL_Q_SB: + ice_sb_init_regs(hw); + cq = &hw->sbq; + break; case ICE_CTL_Q_MAILBOX: ice_mailbox_init_regs(hw); cq = &hw->mailboxq; @@ -646,6 +663,32 @@ init_ctrlq_free_sq: } /** + * ice_is_sbq_supported - is the sideband queue supported + * @hw: pointer to the hardware structure + * + * Returns true if the sideband control queue interface is + * supported for the device, false otherwise + */ +bool ice_is_sbq_supported(struct ice_hw *hw) +{ + /* The device sideband queue is only supported on devices with the + * generic MAC type. + */ + return hw->mac_type == ICE_MAC_GENERIC; +} + +/** + * ice_get_sbq - returns the right control queue to use for sideband + * @hw: pointer to the hardware structure + */ +struct ice_ctl_q_info *ice_get_sbq(struct ice_hw *hw) +{ + if (ice_is_sbq_supported(hw)) + return &hw->sbq; + return &hw->adminq; +} + +/** * ice_shutdown_ctrlq - shutdown routine for any control queue * @hw: pointer to the hardware structure * @q_type: specific Control queue type @@ -662,6 +705,9 @@ static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) if (ice_check_sq_alive(hw, cq)) ice_aq_q_shutdown(hw, true); break; + case ICE_CTL_Q_SB: + cq = &hw->sbq; + break; case ICE_CTL_Q_MAILBOX: cq = &hw->mailboxq; break; @@ -685,6 +731,9 @@ void ice_shutdown_all_ctrlq(struct ice_hw *hw) { /* Shutdown FW admin queue */ ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN); + /* Shutdown PHY Sideband */ + if (ice_is_sbq_supported(hw)) + ice_shutdown_ctrlq(hw, ICE_CTL_Q_SB); /* Shutdown PF-VF Mailbox */ ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX); } @@ -724,6 +773,15 @@ enum ice_status ice_init_all_ctrlq(struct ice_hw *hw) if (status) return status; + /* sideband control queue (SBQ) interface is not supported on some + * devices. Initialize if supported, else fallback to the admin queue + * interface + */ + if (ice_is_sbq_supported(hw)) { + status = ice_init_ctrlq(hw, ICE_CTL_Q_SB); + if (status) + return status; + } /* Init Mailbox queue */ return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX); } @@ -759,6 +817,8 @@ static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq) enum ice_status ice_create_all_ctrlq(struct ice_hw *hw) { ice_init_ctrlq_locks(&hw->adminq); + if (ice_is_sbq_supported(hw)) + ice_init_ctrlq_locks(&hw->sbq); ice_init_ctrlq_locks(&hw->mailboxq); return ice_init_all_ctrlq(hw); @@ -791,6 +851,8 @@ void ice_destroy_all_ctrlq(struct ice_hw *hw) ice_shutdown_all_ctrlq(hw); ice_destroy_ctrlq_locks(&hw->adminq); + if (ice_is_sbq_supported(hw)) + ice_destroy_ctrlq_locks(&hw->sbq); ice_destroy_ctrlq_locks(&hw->mailboxq); } diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.h b/drivers/net/ethernet/intel/ice/ice_controlq.h index fe75871e48ca..c07e9cc9fc6e 100644 --- a/drivers/net/ethernet/intel/ice/ice_controlq.h +++ b/drivers/net/ethernet/intel/ice/ice_controlq.h @@ -9,6 +9,7 @@ /* Maximum buffer lengths for all control queue types */ #define ICE_AQ_MAX_BUF_LEN 4096 #define ICE_MBXQ_MAX_BUF_LEN 4096 +#define ICE_SBQ_MAX_BUF_LEN 512 #define ICE_CTL_Q_DESC(R, i) \ (&(((struct ice_aq_desc *)((R).desc_buf.va))[i])) @@ -29,6 +30,7 @@ enum ice_ctl_q { ICE_CTL_Q_UNKNOWN = 0, ICE_CTL_Q_ADMIN, ICE_CTL_Q_MAILBOX, + ICE_CTL_Q_SB, }; /* Control Queue timeout settings - max delay 1s */ diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c index df02cffdf209..926cf748c5ec 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c @@ -275,6 +275,7 @@ int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked) struct ice_dcbx_cfg *old_cfg, *curr_cfg; struct device *dev = ice_pf_to_dev(pf); int ret = ICE_DCB_NO_HW_CHG; + struct iidc_event *event; struct ice_vsi *pf_vsi; curr_cfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg; @@ -313,6 +314,17 @@ int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked) goto free_cfg; } + /* Notify AUX drivers about impending change to TCs */ + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) { + ret = -ENOMEM; + goto free_cfg; + } + + set_bit(IIDC_EVENT_BEFORE_TC_CHANGE, event->type); + ice_send_event_to_aux(pf, event); + kfree(event); + /* avoid race conditions by holding the lock while disabling and * re-enabling the VSI */ @@ -640,6 +652,7 @@ static int ice_dcb_noncontig_cfg(struct ice_pf *pf) void ice_pf_dcb_recfg(struct ice_pf *pf) { struct ice_dcbx_cfg *dcbcfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg; + struct iidc_event *event; u8 tc_map = 0; int v, ret; @@ -675,6 +688,14 @@ void ice_pf_dcb_recfg(struct ice_pf *pf) if (vsi->type == ICE_VSI_PF) ice_dcbnl_set_all(vsi); } + /* Notify the AUX drivers that TC change is finished */ + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return; + + set_bit(IIDC_EVENT_AFTER_TC_CHANGE, event->type); + ice_send_event_to_aux(pf, event); + kfree(event); } /** diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.h b/drivers/net/ethernet/intel/ice/ice_dcb_lib.h index 35c21d9ae009..261b6e2ed7bc 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.h @@ -60,7 +60,7 @@ static inline bool ice_is_dcb_active(struct ice_pf *pf) test_bit(ICE_FLAG_DCB_ENA, pf->flags)); } #else -#define ice_dcb_rebuild(pf) do {} while (0) +static inline void ice_dcb_rebuild(struct ice_pf *pf) { } static inline u8 ice_dcb_get_ena_tc(struct ice_dcbx_cfg __always_unused *dcbcfg) { @@ -113,11 +113,12 @@ ice_is_pfc_causing_hung_q(struct ice_pf __always_unused *pf, return false; } -#define ice_update_dcb_stats(pf) do {} while (0) -#define ice_pf_dcb_recfg(pf) do {} while (0) -#define ice_vsi_cfg_dcb_rings(vsi) do {} while (0) -#define ice_dcb_process_lldp_set_mib_change(pf, event) do {} while (0) -#define ice_set_cgd_num(tlan_ctx, ring) do {} while (0) -#define ice_vsi_cfg_netdev_tc(vsi, ena_tc) do {} while (0) +static inline void ice_pf_dcb_recfg(struct ice_pf *pf) { } +static inline void ice_vsi_cfg_dcb_rings(struct ice_vsi *vsi) { } +static inline void ice_update_dcb_stats(struct ice_pf *pf) { } +static inline void +ice_dcb_process_lldp_set_mib_change(struct ice_pf *pf, struct ice_rq_event_info *event) { } +static inline void ice_vsi_cfg_netdev_tc(struct ice_vsi *vsi, u8 ena_tc) { } +static inline void ice_set_cgd_num(struct ice_tlan_ctx *tlan_ctx, struct ice_ring *ring) { } #endif /* CONFIG_DCB */ #endif /* _ICE_DCB_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_nl.h b/drivers/net/ethernet/intel/ice/ice_dcb_nl.h index 6c630a362293..eac2f34bdcdd 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_nl.h +++ b/drivers/net/ethernet/intel/ice/ice_dcb_nl.h @@ -11,9 +11,10 @@ void ice_dcbnl_flush_apps(struct ice_pf *pf, struct ice_dcbx_cfg *old_cfg, struct ice_dcbx_cfg *new_cfg); #else -#define ice_dcbnl_setup(vsi) do {} while (0) -#define ice_dcbnl_set_all(vsi) do {} while (0) -#define ice_dcbnl_flush_apps(pf, old_cfg, new_cfg) do {} while (0) +static inline void ice_dcbnl_setup(struct ice_vsi *vsi) { } +static inline void ice_dcbnl_set_all(struct ice_vsi *vsi) { } +static inline void +ice_dcbnl_flush_apps(struct ice_pf *pf, struct ice_dcbx_cfg *old_cfg, + struct ice_dcbx_cfg *new_cfg) { } #endif /* CONFIG_DCB */ - #endif /* _ICE_DCB_NL_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index cf685eeea198..91b545ab8b8f 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -276,6 +276,12 @@ static int ice_devlink_info_get(struct devlink *devlink, size_t i; int err; + err = ice_wait_for_reset(pf, 10 * HZ); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Device is busy resetting"); + return err; + } + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; @@ -283,6 +289,9 @@ static int ice_devlink_info_get(struct devlink *devlink, /* discover capabilities first */ status = ice_discover_dev_caps(hw, &ctx->dev_caps); if (status) { + dev_dbg(dev, "Failed to discover device capabilities, status %s aq_err %s\n", + ice_stat_str(status), ice_aq_str(hw->adminq.sq_last_status)); + NL_SET_ERR_MSG_MOD(extack, "Unable to discover device capabilities"); err = -EIO; goto out_free_ctx; } diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index d9ddd0bcf65f..d95a5daca114 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -1773,49 +1773,6 @@ ice_phy_type_to_ethtool(struct net_device *netdev, ice_ethtool_advertise_link_mode(ICE_AQ_LINK_SPEED_100GB, 100000baseKR4_Full); } - - /* Autoneg PHY types */ - if (phy_types_low & ICE_PHY_TYPE_LOW_100BASE_TX || - phy_types_low & ICE_PHY_TYPE_LOW_1000BASE_T || - phy_types_low & ICE_PHY_TYPE_LOW_1000BASE_KX || - phy_types_low & ICE_PHY_TYPE_LOW_2500BASE_T || - phy_types_low & ICE_PHY_TYPE_LOW_2500BASE_KX || - phy_types_low & ICE_PHY_TYPE_LOW_5GBASE_T || - phy_types_low & ICE_PHY_TYPE_LOW_5GBASE_KR || - phy_types_low & ICE_PHY_TYPE_LOW_10GBASE_T || - phy_types_low & ICE_PHY_TYPE_LOW_10GBASE_KR_CR1 || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_T || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_CR || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_CR_S || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_CR1 || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_KR || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_KR_S || - phy_types_low & ICE_PHY_TYPE_LOW_25GBASE_KR1 || - phy_types_low & ICE_PHY_TYPE_LOW_40GBASE_CR4 || - phy_types_low & ICE_PHY_TYPE_LOW_40GBASE_KR4) { - ethtool_link_ksettings_add_link_mode(ks, supported, - Autoneg); - ethtool_link_ksettings_add_link_mode(ks, advertising, - Autoneg); - } - if (phy_types_low & ICE_PHY_TYPE_LOW_50GBASE_CR2 || - phy_types_low & ICE_PHY_TYPE_LOW_50GBASE_KR2 || - phy_types_low & ICE_PHY_TYPE_LOW_50GBASE_CP || - phy_types_low & ICE_PHY_TYPE_LOW_50GBASE_KR_PAM4) { - ethtool_link_ksettings_add_link_mode(ks, supported, - Autoneg); - ethtool_link_ksettings_add_link_mode(ks, advertising, - Autoneg); - } - if (phy_types_low & ICE_PHY_TYPE_LOW_100GBASE_CR4 || - phy_types_low & ICE_PHY_TYPE_LOW_100GBASE_KR4 || - phy_types_low & ICE_PHY_TYPE_LOW_100GBASE_KR_PAM4 || - phy_types_low & ICE_PHY_TYPE_LOW_100GBASE_CP2) { - ethtool_link_ksettings_add_link_mode(ks, supported, - Autoneg); - ethtool_link_ksettings_add_link_mode(ks, advertising, - Autoneg); - } } #define TEST_SET_BITS_TIMEOUT 50 @@ -1972,9 +1929,7 @@ ice_get_link_ksettings(struct net_device *netdev, ks->base.port = PORT_TP; break; case ICE_MEDIA_BACKPLANE: - ethtool_link_ksettings_add_link_mode(ks, supported, Autoneg); ethtool_link_ksettings_add_link_mode(ks, supported, Backplane); - ethtool_link_ksettings_add_link_mode(ks, advertising, Autoneg); ethtool_link_ksettings_add_link_mode(ks, advertising, Backplane); ks->base.port = PORT_NONE; @@ -2049,6 +2004,12 @@ ice_get_link_ksettings(struct net_device *netdev, if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_CLAUSE91_EN) ethtool_link_ksettings_add_link_mode(ks, supported, FEC_RS); + /* Set supported and advertised autoneg */ + if (ice_is_phy_caps_an_enabled(caps)) { + ethtool_link_ksettings_add_link_mode(ks, supported, Autoneg); + ethtool_link_ksettings_add_link_mode(ks, advertising, Autoneg); + } + done: kfree(caps); return err; @@ -3234,6 +3195,31 @@ ice_set_rxfh(struct net_device *netdev, const u32 *indir, const u8 *key, return 0; } +static int +ice_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) +{ + struct ice_pf *pf = ice_netdev_to_pf(dev); + + /* only report timestamping if PTP is enabled */ + if (!test_bit(ICE_FLAG_PTP, pf->flags)) + return ethtool_op_get_ts_info(dev, info); + + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + info->phc_index = ice_get_ptp_clock_index(pf); + + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | BIT(HWTSTAMP_FILTER_ALL); + + return 0; +} + /** * ice_get_max_txq - return the maximum number of Tx queues for in a PF * @pf: PF structure @@ -3501,13 +3487,9 @@ static int ice_get_rc_coalesce(struct ethtool_coalesce *ec, enum ice_container_type c_type, struct ice_ring_container *rc) { - struct ice_pf *pf; - if (!rc->ring) return -EINVAL; - pf = rc->ring->vsi->back; - switch (c_type) { case ICE_RX_CONTAINER: ec->use_adaptive_rx_coalesce = ITR_IS_DYNAMIC(rc); @@ -3519,7 +3501,7 @@ ice_get_rc_coalesce(struct ethtool_coalesce *ec, enum ice_container_type c_type, ec->tx_coalesce_usecs = rc->itr_setting; break; default: - dev_dbg(ice_pf_to_dev(pf), "Invalid c_type %d\n", c_type); + dev_dbg(ice_pf_to_dev(rc->ring->vsi->back), "Invalid c_type %d\n", c_type); return -EINVAL; } @@ -4029,7 +4011,7 @@ static const struct ethtool_ops ice_ethtool_ops = { .set_rxfh = ice_set_rxfh, .get_channels = ice_get_channels, .set_channels = ice_set_channels, - .get_ts_info = ethtool_op_get_ts_info, + .get_ts_info = ice_get_ts_info, .get_per_queue_coalesce = ice_get_per_q_coalesce, .set_per_queue_coalesce = ice_set_per_q_coalesce, .get_fecparam = ice_get_fecparam, diff --git a/drivers/net/ethernet/intel/ice/ice_fw_update.c b/drivers/net/ethernet/intel/ice/ice_fw_update.c index dcec0360ce55..f8601d5b0b19 100644 --- a/drivers/net/ethernet/intel/ice/ice_fw_update.c +++ b/drivers/net/ethernet/intel/ice/ice_fw_update.c @@ -702,6 +702,16 @@ int ice_flash_pldm_image(struct ice_pf *pf, const struct firmware *fw, } err = pldmfw_flash_image(&priv.context, fw); + if (err == -ENOENT) { + dev_err(dev, "Firmware image has no record matching this device\n"); + NL_SET_ERR_MSG_MOD(extack, "Firmware image has no record matching this device"); + } else if (err) { + /* Do not set a generic extended ACK message here. A more + * specific message may already have been set by one of our + * ops. + */ + dev_err(dev, "Failed to flash PLDM image, err %d", err); + } ice_release_nvm(hw); diff --git a/drivers/net/ethernet/intel/ice/ice_hw_autogen.h b/drivers/net/ethernet/intel/ice/ice_hw_autogen.h index de38a0fc9665..76021d977b60 100644 --- a/drivers/net/ethernet/intel/ice/ice_hw_autogen.h +++ b/drivers/net/ethernet/intel/ice/ice_hw_autogen.h @@ -31,6 +31,7 @@ #define PF_FW_ATQLEN_ATQOVFL_M BIT(29) #define PF_FW_ATQLEN_ATQCRIT_M BIT(30) #define VF_MBX_ARQLEN(_VF) (0x0022BC00 + ((_VF) * 4)) +#define VF_MBX_ATQLEN(_VF) (0x0022A800 + ((_VF) * 4)) #define PF_FW_ATQLEN_ATQENABLE_M BIT(31) #define PF_FW_ATQT 0x00080400 #define PF_MBX_ARQBAH 0x0022E400 @@ -51,6 +52,54 @@ #define PF_MBX_ATQLEN_ATQCRIT_M BIT(30) #define PF_MBX_ATQLEN_ATQENABLE_M BIT(31) #define PF_MBX_ATQT 0x0022E300 +#define PF_SB_ARQBAH 0x0022FF00 +#define PF_SB_ARQBAH_ARQBAH_S 0 +#define PF_SB_ARQBAH_ARQBAH_M ICE_M(0xFFFFFFFF, 0) +#define PF_SB_ARQBAL 0x0022FE80 +#define PF_SB_ARQBAL_ARQBAL_LSB_S 0 +#define PF_SB_ARQBAL_ARQBAL_LSB_M ICE_M(0x3F, 0) +#define PF_SB_ARQBAL_ARQBAL_S 6 +#define PF_SB_ARQBAL_ARQBAL_M ICE_M(0x3FFFFFF, 6) +#define PF_SB_ARQH 0x00230000 +#define PF_SB_ARQH_ARQH_S 0 +#define PF_SB_ARQH_ARQH_M ICE_M(0x3FF, 0) +#define PF_SB_ARQLEN 0x0022FF80 +#define PF_SB_ARQLEN_ARQLEN_S 0 +#define PF_SB_ARQLEN_ARQLEN_M ICE_M(0x3FF, 0) +#define PF_SB_ARQLEN_ARQVFE_S 28 +#define PF_SB_ARQLEN_ARQVFE_M BIT(28) +#define PF_SB_ARQLEN_ARQOVFL_S 29 +#define PF_SB_ARQLEN_ARQOVFL_M BIT(29) +#define PF_SB_ARQLEN_ARQCRIT_S 30 +#define PF_SB_ARQLEN_ARQCRIT_M BIT(30) +#define PF_SB_ARQLEN_ARQENABLE_S 31 +#define PF_SB_ARQLEN_ARQENABLE_M BIT(31) +#define PF_SB_ARQT 0x00230080 +#define PF_SB_ARQT_ARQT_S 0 +#define PF_SB_ARQT_ARQT_M ICE_M(0x3FF, 0) +#define PF_SB_ATQBAH 0x0022FC80 +#define PF_SB_ATQBAH_ATQBAH_S 0 +#define PF_SB_ATQBAH_ATQBAH_M ICE_M(0xFFFFFFFF, 0) +#define PF_SB_ATQBAL 0x0022FC00 +#define PF_SB_ATQBAL_ATQBAL_S 6 +#define PF_SB_ATQBAL_ATQBAL_M ICE_M(0x3FFFFFF, 6) +#define PF_SB_ATQH 0x0022FD80 +#define PF_SB_ATQH_ATQH_S 0 +#define PF_SB_ATQH_ATQH_M ICE_M(0x3FF, 0) +#define PF_SB_ATQLEN 0x0022FD00 +#define PF_SB_ATQLEN_ATQLEN_S 0 +#define PF_SB_ATQLEN_ATQLEN_M ICE_M(0x3FF, 0) +#define PF_SB_ATQLEN_ATQVFE_S 28 +#define PF_SB_ATQLEN_ATQVFE_M BIT(28) +#define PF_SB_ATQLEN_ATQOVFL_S 29 +#define PF_SB_ATQLEN_ATQOVFL_M BIT(29) +#define PF_SB_ATQLEN_ATQCRIT_S 30 +#define PF_SB_ATQLEN_ATQCRIT_M BIT(30) +#define PF_SB_ATQLEN_ATQENABLE_S 31 +#define PF_SB_ATQLEN_ATQENABLE_M BIT(31) +#define PF_SB_ATQT 0x0022FE00 +#define PF_SB_ATQT_ATQT_S 0 +#define PF_SB_ATQT_ATQT_M ICE_M(0x3FF, 0) #define PRTDCB_GENC 0x00083000 #define PRTDCB_GENC_PFCLDA_S 16 #define PRTDCB_GENC_PFCLDA_M ICE_M(0xFFFF, 16) @@ -89,6 +138,10 @@ #define GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_S 4 #define GLGEN_CLKSTAT_SRC_PSM_CLK_SRC_M ICE_M(0x3, 4) #define GLGEN_CLKSTAT_SRC 0x000B826C +#define GLGEN_GPIO_CTL(_i) (0x000880C8 + ((_i) * 4)) +#define GLGEN_GPIO_CTL_PIN_DIR_M BIT(4) +#define GLGEN_GPIO_CTL_PIN_FUNC_S 8 +#define GLGEN_GPIO_CTL_PIN_FUNC_M ICE_M(0xF, 8) #define GLGEN_RSTAT 0x000B8188 #define GLGEN_RSTAT_DEVSTATE_M ICE_M(0x3, 0) #define GLGEN_RSTCTL 0x000B8180 @@ -110,8 +163,6 @@ #define VPGEN_VFRSTAT_VFRD_M BIT(0) #define VPGEN_VFRTRIG(_VF) (0x00090000 + ((_VF) * 4)) #define VPGEN_VFRTRIG_VFSWR_M BIT(0) -#define PFHMC_ERRORDATA 0x00520500 -#define PFHMC_ERRORINFO 0x00520400 #define GLINT_CTL 0x0016CC54 #define GLINT_CTL_DIS_AUTOMASK_M BIT(0) #define GLINT_CTL_ITR_GRAN_200_S 16 @@ -155,11 +206,14 @@ #define PFINT_MBX_CTL_ITR_INDX_M ICE_M(0x3, 11) #define PFINT_MBX_CTL_CAUSE_ENA_M BIT(30) #define PFINT_OICR 0x0016CA00 +#define PFINT_OICR_TSYN_TX_M BIT(11) +#define PFINT_OICR_TSYN_EVNT_M BIT(12) #define PFINT_OICR_ECC_ERR_M BIT(16) #define PFINT_OICR_MAL_DETECT_M BIT(19) #define PFINT_OICR_GRST_M BIT(20) #define PFINT_OICR_PCI_EXCEPTION_M BIT(21) #define PFINT_OICR_HMC_ERR_M BIT(26) +#define PFINT_OICR_PE_PUSH_M BIT(27) #define PFINT_OICR_PE_CRITERR_M BIT(28) #define PFINT_OICR_VFLR_M BIT(29) #define PFINT_OICR_SWINT_M BIT(31) @@ -169,6 +223,9 @@ #define PFINT_OICR_CTL_ITR_INDX_M ICE_M(0x3, 11) #define PFINT_OICR_CTL_CAUSE_ENA_M BIT(30) #define PFINT_OICR_ENA 0x0016C900 +#define PFINT_SB_CTL 0x0016B600 +#define PFINT_SB_CTL_MSIX_INDX_M ICE_M(0x7FF, 0) +#define PFINT_SB_CTL_CAUSE_ENA_M BIT(30) #define QINT_RQCTL(_QRX) (0x00150000 + ((_QRX) * 4)) #define QINT_RQCTL_MSIX_INDX_S 0 #define QINT_RQCTL_MSIX_INDX_M ICE_M(0x7FF, 0) @@ -382,6 +439,36 @@ #define GLV_UPRCL(_i) (0x003B2000 + ((_i) * 8)) #define GLV_UPTCL(_i) (0x0030A000 + ((_i) * 8)) #define PRTRPB_RDPC 0x000AC260 +#define GLTSYN_AUX_IN_0(_i) (0x000889D8 + ((_i) * 4)) +#define GLTSYN_AUX_IN_0_INT_ENA_M BIT(4) +#define GLTSYN_AUX_OUT_0(_i) (0x00088998 + ((_i) * 4)) +#define GLTSYN_AUX_OUT_0_OUT_ENA_M BIT(0) +#define GLTSYN_AUX_OUT_0_OUTMOD_M ICE_M(0x3, 1) +#define GLTSYN_CLKO_0(_i) (0x000889B8 + ((_i) * 4)) +#define GLTSYN_CMD 0x00088810 +#define GLTSYN_CMD_SYNC 0x00088814 +#define GLTSYN_ENA(_i) (0x00088808 + ((_i) * 4)) +#define GLTSYN_ENA_TSYN_ENA_M BIT(0) +#define GLTSYN_EVNT_H_0(_i) (0x00088970 + ((_i) * 4)) +#define GLTSYN_EVNT_L_0(_i) (0x00088968 + ((_i) * 4)) +#define GLTSYN_INCVAL_H(_i) (0x00088920 + ((_i) * 4)) +#define GLTSYN_INCVAL_L(_i) (0x00088918 + ((_i) * 4)) +#define GLTSYN_SHADJ_H(_i) (0x00088910 + ((_i) * 4)) +#define GLTSYN_SHADJ_L(_i) (0x00088908 + ((_i) * 4)) +#define GLTSYN_SHTIME_0(_i) (0x000888E0 + ((_i) * 4)) +#define GLTSYN_SHTIME_H(_i) (0x000888F0 + ((_i) * 4)) +#define GLTSYN_SHTIME_L(_i) (0x000888E8 + ((_i) * 4)) +#define GLTSYN_STAT(_i) (0x000888C0 + ((_i) * 4)) +#define GLTSYN_STAT_EVENT0_M BIT(0) +#define GLTSYN_STAT_EVENT1_M BIT(1) +#define GLTSYN_STAT_EVENT2_M BIT(2) +#define GLTSYN_SYNC_DLAY 0x00088818 +#define GLTSYN_TGT_H_0(_i) (0x00088930 + ((_i) * 4)) +#define GLTSYN_TGT_L_0(_i) (0x00088928 + ((_i) * 4)) +#define GLTSYN_TIME_H(_i) (0x000888D8 + ((_i) * 4)) +#define GLTSYN_TIME_L(_i) (0x000888D0 + ((_i) * 4)) +#define PFTSYN_SEM 0x00088880 +#define PFTSYN_SEM_BUSY_M BIT(0) #define VSIQF_FD_CNT(_VSI) (0x00464000 + ((_VSI) * 4)) #define VSIQF_FD_CNT_FD_GCNT_S 0 #define VSIQF_FD_CNT_FD_GCNT_M ICE_M(0x3FFF, 0) diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c new file mode 100644 index 000000000000..1f2afdf6cd48 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_idc.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021, Intel Corporation. */ + +/* Inter-Driver Communication */ +#include "ice.h" +#include "ice_lib.h" +#include "ice_dcb_lib.h" + +/** + * ice_get_auxiliary_drv - retrieve iidc_auxiliary_drv struct + * @pf: pointer to PF struct + * + * This function has to be called with a device_lock on the + * pf->adev.dev to avoid race conditions. + */ +static struct iidc_auxiliary_drv *ice_get_auxiliary_drv(struct ice_pf *pf) +{ + struct auxiliary_device *adev; + + adev = pf->adev; + if (!adev || !adev->dev.driver) + return NULL; + + return container_of(adev->dev.driver, struct iidc_auxiliary_drv, + adrv.driver); +} + +/** + * ice_send_event_to_aux - send event to RDMA AUX driver + * @pf: pointer to PF struct + * @event: event struct + */ +void ice_send_event_to_aux(struct ice_pf *pf, struct iidc_event *event) +{ + struct iidc_auxiliary_drv *iadrv; + + if (!pf->adev) + return; + + device_lock(&pf->adev->dev); + iadrv = ice_get_auxiliary_drv(pf); + if (iadrv && iadrv->event_handler) + iadrv->event_handler(pf, event); + device_unlock(&pf->adev->dev); +} + +/** + * ice_find_vsi - Find the VSI from VSI ID + * @pf: The PF pointer to search in + * @vsi_num: The VSI ID to search for + */ +static struct ice_vsi *ice_find_vsi(struct ice_pf *pf, u16 vsi_num) +{ + int i; + + ice_for_each_vsi(pf, i) + if (pf->vsi[i] && pf->vsi[i]->vsi_num == vsi_num) + return pf->vsi[i]; + return NULL; +} + +/** + * ice_add_rdma_qset - Add Leaf Node for RDMA Qset + * @pf: PF struct + * @qset: Resource to be allocated + */ +int ice_add_rdma_qset(struct ice_pf *pf, struct iidc_rdma_qset_params *qset) +{ + u16 max_rdmaqs[ICE_MAX_TRAFFIC_CLASS]; + struct ice_vsi *vsi; + struct device *dev; + u32 qset_teid; + u16 qs_handle; + int status; + int i; + + if (WARN_ON(!pf || !qset)) + return -EINVAL; + + dev = ice_pf_to_dev(pf); + + if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) + return -EINVAL; + + vsi = ice_get_main_vsi(pf); + if (!vsi) { + dev_err(dev, "RDMA QSet invalid VSI\n"); + return -EINVAL; + } + + ice_for_each_traffic_class(i) + max_rdmaqs[i] = 0; + + max_rdmaqs[qset->tc]++; + qs_handle = qset->qs_handle; + + status = ice_cfg_vsi_rdma(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, + max_rdmaqs); + if (status) { + dev_err(dev, "Failed VSI RDMA Qset config\n"); + return status; + } + + status = ice_ena_vsi_rdma_qset(vsi->port_info, vsi->idx, qset->tc, + &qs_handle, 1, &qset_teid); + if (status) { + dev_err(dev, "Failed VSI RDMA Qset enable\n"); + return status; + } + vsi->qset_handle[qset->tc] = qset->qs_handle; + qset->teid = qset_teid; + + return 0; +} +EXPORT_SYMBOL_GPL(ice_add_rdma_qset); + +/** + * ice_del_rdma_qset - Delete leaf node for RDMA Qset + * @pf: PF struct + * @qset: Resource to be freed + */ +int ice_del_rdma_qset(struct ice_pf *pf, struct iidc_rdma_qset_params *qset) +{ + struct ice_vsi *vsi; + u32 teid; + u16 q_id; + + if (WARN_ON(!pf || !qset)) + return -EINVAL; + + vsi = ice_find_vsi(pf, qset->vport_id); + if (!vsi) { + dev_err(ice_pf_to_dev(pf), "RDMA Invalid VSI\n"); + return -EINVAL; + } + + q_id = qset->qs_handle; + teid = qset->teid; + + vsi->qset_handle[qset->tc] = 0; + + return ice_dis_vsi_rdma_qset(vsi->port_info, 1, &teid, &q_id); +} +EXPORT_SYMBOL_GPL(ice_del_rdma_qset); + +/** + * ice_rdma_request_reset - accept request from RDMA to perform a reset + * @pf: struct for PF + * @reset_type: type of reset + */ +int ice_rdma_request_reset(struct ice_pf *pf, enum iidc_reset_type reset_type) +{ + enum ice_reset_req reset; + + if (WARN_ON(!pf)) + return -EINVAL; + + switch (reset_type) { + case IIDC_PFR: + reset = ICE_RESET_PFR; + break; + case IIDC_CORER: + reset = ICE_RESET_CORER; + break; + case IIDC_GLOBR: + reset = ICE_RESET_GLOBR; + break; + default: + dev_err(ice_pf_to_dev(pf), "incorrect reset request\n"); + return -EINVAL; + } + + return ice_schedule_reset(pf, reset); +} +EXPORT_SYMBOL_GPL(ice_rdma_request_reset); + +/** + * ice_rdma_update_vsi_filter - update main VSI filters for RDMA + * @pf: pointer to struct for PF + * @vsi_id: VSI HW idx to update filter on + * @enable: bool whether to enable or disable filters + */ +int ice_rdma_update_vsi_filter(struct ice_pf *pf, u16 vsi_id, bool enable) +{ + struct ice_vsi *vsi; + int status; + + if (WARN_ON(!pf)) + return -EINVAL; + + vsi = ice_find_vsi(pf, vsi_id); + if (!vsi) + return -EINVAL; + + status = ice_cfg_rdma_fltr(&pf->hw, vsi->idx, enable); + if (status) { + dev_err(ice_pf_to_dev(pf), "Failed to %sable RDMA filtering\n", + enable ? "en" : "dis"); + } else { + if (enable) + vsi->info.q_opt_flags |= ICE_AQ_VSI_Q_OPT_PE_FLTR_EN; + else + vsi->info.q_opt_flags &= ~ICE_AQ_VSI_Q_OPT_PE_FLTR_EN; + } + + return status; +} +EXPORT_SYMBOL_GPL(ice_rdma_update_vsi_filter); + +/** + * ice_get_qos_params - parse QoS params for RDMA consumption + * @pf: pointer to PF struct + * @qos: set of QoS values + */ +void ice_get_qos_params(struct ice_pf *pf, struct iidc_qos_params *qos) +{ + struct ice_dcbx_cfg *dcbx_cfg; + unsigned int i; + u32 up2tc; + + dcbx_cfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg; + up2tc = rd32(&pf->hw, PRTDCB_TUP2TC); + + qos->num_tc = ice_dcb_get_num_tc(dcbx_cfg); + for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++) + qos->up2tc[i] = (up2tc >> (i * 3)) & 0x7; + + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) + qos->tc_info[i].rel_bw = dcbx_cfg->etscfg.tcbwtable[i]; +} +EXPORT_SYMBOL_GPL(ice_get_qos_params); + +/** + * ice_reserve_rdma_qvector - Reserve vector resources for RDMA driver + * @pf: board private structure to initialize + */ +static int ice_reserve_rdma_qvector(struct ice_pf *pf) +{ + if (test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) { + int index; + + index = ice_get_res(pf, pf->irq_tracker, pf->num_rdma_msix, + ICE_RES_RDMA_VEC_ID); + if (index < 0) + return index; + pf->num_avail_sw_msix -= pf->num_rdma_msix; + pf->rdma_base_vector = (u16)index; + } + return 0; +} + +/** + * ice_adev_release - function to be mapped to AUX dev's release op + * @dev: pointer to device to free + */ +static void ice_adev_release(struct device *dev) +{ + struct iidc_auxiliary_dev *iadev; + + iadev = container_of(dev, struct iidc_auxiliary_dev, adev.dev); + kfree(iadev); +} + +/** + * ice_plug_aux_dev - allocate and register AUX device + * @pf: pointer to pf struct + */ +int ice_plug_aux_dev(struct ice_pf *pf) +{ + struct iidc_auxiliary_dev *iadev; + struct auxiliary_device *adev; + int ret; + + iadev = kzalloc(sizeof(*iadev), GFP_KERNEL); + if (!iadev) + return -ENOMEM; + + adev = &iadev->adev; + pf->adev = adev; + iadev->pf = pf; + + adev->id = pf->aux_idx; + adev->dev.release = ice_adev_release; + adev->dev.parent = &pf->pdev->dev; + adev->name = IIDC_RDMA_ROCE_NAME; + + ret = auxiliary_device_init(adev); + if (ret) { + pf->adev = NULL; + kfree(iadev); + return ret; + } + + ret = auxiliary_device_add(adev); + if (ret) { + pf->adev = NULL; + auxiliary_device_uninit(adev); + return ret; + } + + return 0; +} + +/* ice_unplug_aux_dev - unregister and free AUX device + * @pf: pointer to pf struct + */ +void ice_unplug_aux_dev(struct ice_pf *pf) +{ + if (!pf->adev) + return; + + auxiliary_device_delete(pf->adev); + auxiliary_device_uninit(pf->adev); + pf->adev = NULL; +} + +/** + * ice_init_rdma - initializes PF for RDMA use + * @pf: ptr to ice_pf + */ +int ice_init_rdma(struct ice_pf *pf) +{ + struct device *dev = &pf->pdev->dev; + int ret; + + /* Reserve vector resources */ + ret = ice_reserve_rdma_qvector(pf); + if (ret < 0) { + dev_err(dev, "failed to reserve vectors for RDMA\n"); + return ret; + } + + return ice_plug_aux_dev(pf); +} diff --git a/drivers/net/ethernet/intel/ice/ice_idc_int.h b/drivers/net/ethernet/intel/ice/ice_idc_int.h new file mode 100644 index 000000000000..b7796b8aecbd --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_idc_int.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2021, Intel Corporation. */ + +#ifndef _ICE_IDC_INT_H_ +#define _ICE_IDC_INT_H_ + +#include <linux/net/intel/iidc.h> +#include "ice.h" + +struct ice_pf; + +void ice_send_event_to_aux(struct ice_pf *pf, struct iidc_event *event); + +#endif /* !_ICE_IDC_INT_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index 4599fc3b4ed8..37c18c66b5c7 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -172,6 +172,7 @@ ice_lag_link(struct ice_lag *lag, struct netdev_notifier_changeupper_info *info) } ice_clear_sriov_cap(pf); + ice_clear_rdma_cap(pf); lag->bonded = true; lag->role = ICE_LAG_UNSET; @@ -222,6 +223,7 @@ ice_lag_unlink(struct ice_lag *lag, } ice_set_sriov_cap(pf); + ice_set_rdma_cap(pf); lag->bonded = false; lag->role = ICE_LAG_NONE; } diff --git a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h index 21329ed3087e..80736e0ec0dc 100644 --- a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h +++ b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h @@ -161,7 +161,6 @@ struct ice_fltr_desc { #define ICE_FXD_FLTR_WB_QW1_FAIL_PROF_YES 0x1ULL struct ice_rx_ptype_decoded { - u32 ptype:10; u32 known:1; u32 outer_ip:1; u32 outer_ip_ver:2; @@ -606,9 +605,32 @@ struct ice_tlan_ctx { u8 int_q_state; /* width not needed - internal - DO NOT WRITE!!! */ }; -/* macro to make the table lines short */ +/* The ice_ptype_lkup table is used to convert from the 10-bit ptype in the + * hardware to a bit-field that can be used by SW to more easily determine the + * packet type. + * + * Macros are used to shorten the table lines and make this table human + * readable. + * + * We store the PTYPE in the top byte of the bit field - this is just so that + * we can check that the table doesn't have a row missing, as the index into + * the table should be the PTYPE. + * + * Typical work flow: + * + * IF NOT ice_ptype_lkup[ptype].known + * THEN + * Packet is unknown + * ELSE IF ice_ptype_lkup[ptype].outer_ip == ICE_RX_PTYPE_OUTER_IP + * Use the rest of the fields to look at the tunnels, inner protocols, etc + * ELSE + * Use the enum ice_rx_l2_ptype to decode the packet type + * ENDIF + */ + +/* macro to make the table lines short, use explicit indexing with [PTYPE] */ #define ICE_PTT(PTYPE, OUTER_IP, OUTER_IP_VER, OUTER_FRAG, T, TE, TEF, I, PL)\ - { PTYPE, \ + [PTYPE] = { \ 1, \ ICE_RX_PTYPE_OUTER_##OUTER_IP, \ ICE_RX_PTYPE_OUTER_##OUTER_IP_VER, \ @@ -619,18 +641,18 @@ struct ice_tlan_ctx { ICE_RX_PTYPE_INNER_PROT_##I, \ ICE_RX_PTYPE_PAYLOAD_LAYER_##PL } -#define ICE_PTT_UNUSED_ENTRY(PTYPE) { PTYPE, 0, 0, 0, 0, 0, 0, 0, 0, 0 } +#define ICE_PTT_UNUSED_ENTRY(PTYPE) [PTYPE] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } /* shorter macros makes the table fit but are terse */ #define ICE_RX_PTYPE_NOF ICE_RX_PTYPE_NOT_FRAG #define ICE_RX_PTYPE_FRG ICE_RX_PTYPE_FRAG -/* Lookup table mapping the HW PTYPE to the bit field for decoding */ -static const struct ice_rx_ptype_decoded ice_ptype_lkup[] = { +/* Lookup table mapping in the 10-bit HW PTYPE to the bit field for decoding */ +static const struct ice_rx_ptype_decoded ice_ptype_lkup[BIT(10)] = { /* L2 Packet types */ ICE_PTT_UNUSED_ENTRY(0), ICE_PTT(1, L2, NONE, NOF, NONE, NONE, NOF, NONE, PAY2), - ICE_PTT(2, L2, NONE, NOF, NONE, NONE, NOF, NONE, NONE), + ICE_PTT_UNUSED_ENTRY(2), ICE_PTT_UNUSED_ENTRY(3), ICE_PTT_UNUSED_ENTRY(4), ICE_PTT_UNUSED_ENTRY(5), @@ -744,7 +766,7 @@ static const struct ice_rx_ptype_decoded ice_ptype_lkup[] = { /* Non Tunneled IPv6 */ ICE_PTT(88, IP, IPV6, FRG, NONE, NONE, NOF, NONE, PAY3), ICE_PTT(89, IP, IPV6, NOF, NONE, NONE, NOF, NONE, PAY3), - ICE_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY3), + ICE_PTT(90, IP, IPV6, NOF, NONE, NONE, NOF, UDP, PAY4), ICE_PTT_UNUSED_ENTRY(91), ICE_PTT(92, IP, IPV6, NOF, NONE, NONE, NOF, TCP, PAY4), ICE_PTT(93, IP, IPV6, NOF, NONE, NONE, NOF, SCTP, PAY4), @@ -832,118 +854,7 @@ static const struct ice_rx_ptype_decoded ice_ptype_lkup[] = { ICE_PTT(153, IP, IPV6, NOF, IP_GRENAT_MAC_VLAN, IPV6, NOF, ICMP, PAY4), /* unused entries */ - ICE_PTT_UNUSED_ENTRY(154), - ICE_PTT_UNUSED_ENTRY(155), - ICE_PTT_UNUSED_ENTRY(156), - ICE_PTT_UNUSED_ENTRY(157), - ICE_PTT_UNUSED_ENTRY(158), - ICE_PTT_UNUSED_ENTRY(159), - - ICE_PTT_UNUSED_ENTRY(160), - ICE_PTT_UNUSED_ENTRY(161), - ICE_PTT_UNUSED_ENTRY(162), - ICE_PTT_UNUSED_ENTRY(163), - ICE_PTT_UNUSED_ENTRY(164), - ICE_PTT_UNUSED_ENTRY(165), - ICE_PTT_UNUSED_ENTRY(166), - ICE_PTT_UNUSED_ENTRY(167), - ICE_PTT_UNUSED_ENTRY(168), - ICE_PTT_UNUSED_ENTRY(169), - - ICE_PTT_UNUSED_ENTRY(170), - ICE_PTT_UNUSED_ENTRY(171), - ICE_PTT_UNUSED_ENTRY(172), - ICE_PTT_UNUSED_ENTRY(173), - ICE_PTT_UNUSED_ENTRY(174), - ICE_PTT_UNUSED_ENTRY(175), - ICE_PTT_UNUSED_ENTRY(176), - ICE_PTT_UNUSED_ENTRY(177), - ICE_PTT_UNUSED_ENTRY(178), - ICE_PTT_UNUSED_ENTRY(179), - - ICE_PTT_UNUSED_ENTRY(180), - ICE_PTT_UNUSED_ENTRY(181), - ICE_PTT_UNUSED_ENTRY(182), - ICE_PTT_UNUSED_ENTRY(183), - ICE_PTT_UNUSED_ENTRY(184), - ICE_PTT_UNUSED_ENTRY(185), - ICE_PTT_UNUSED_ENTRY(186), - ICE_PTT_UNUSED_ENTRY(187), - ICE_PTT_UNUSED_ENTRY(188), - ICE_PTT_UNUSED_ENTRY(189), - - ICE_PTT_UNUSED_ENTRY(190), - ICE_PTT_UNUSED_ENTRY(191), - ICE_PTT_UNUSED_ENTRY(192), - ICE_PTT_UNUSED_ENTRY(193), - ICE_PTT_UNUSED_ENTRY(194), - ICE_PTT_UNUSED_ENTRY(195), - ICE_PTT_UNUSED_ENTRY(196), - ICE_PTT_UNUSED_ENTRY(197), - ICE_PTT_UNUSED_ENTRY(198), - ICE_PTT_UNUSED_ENTRY(199), - - ICE_PTT_UNUSED_ENTRY(200), - ICE_PTT_UNUSED_ENTRY(201), - ICE_PTT_UNUSED_ENTRY(202), - ICE_PTT_UNUSED_ENTRY(203), - ICE_PTT_UNUSED_ENTRY(204), - ICE_PTT_UNUSED_ENTRY(205), - ICE_PTT_UNUSED_ENTRY(206), - ICE_PTT_UNUSED_ENTRY(207), - ICE_PTT_UNUSED_ENTRY(208), - ICE_PTT_UNUSED_ENTRY(209), - - ICE_PTT_UNUSED_ENTRY(210), - ICE_PTT_UNUSED_ENTRY(211), - ICE_PTT_UNUSED_ENTRY(212), - ICE_PTT_UNUSED_ENTRY(213), - ICE_PTT_UNUSED_ENTRY(214), - ICE_PTT_UNUSED_ENTRY(215), - ICE_PTT_UNUSED_ENTRY(216), - ICE_PTT_UNUSED_ENTRY(217), - ICE_PTT_UNUSED_ENTRY(218), - ICE_PTT_UNUSED_ENTRY(219), - - ICE_PTT_UNUSED_ENTRY(220), - ICE_PTT_UNUSED_ENTRY(221), - ICE_PTT_UNUSED_ENTRY(222), - ICE_PTT_UNUSED_ENTRY(223), - ICE_PTT_UNUSED_ENTRY(224), - ICE_PTT_UNUSED_ENTRY(225), - ICE_PTT_UNUSED_ENTRY(226), - ICE_PTT_UNUSED_ENTRY(227), - ICE_PTT_UNUSED_ENTRY(228), - ICE_PTT_UNUSED_ENTRY(229), - - ICE_PTT_UNUSED_ENTRY(230), - ICE_PTT_UNUSED_ENTRY(231), - ICE_PTT_UNUSED_ENTRY(232), - ICE_PTT_UNUSED_ENTRY(233), - ICE_PTT_UNUSED_ENTRY(234), - ICE_PTT_UNUSED_ENTRY(235), - ICE_PTT_UNUSED_ENTRY(236), - ICE_PTT_UNUSED_ENTRY(237), - ICE_PTT_UNUSED_ENTRY(238), - ICE_PTT_UNUSED_ENTRY(239), - - ICE_PTT_UNUSED_ENTRY(240), - ICE_PTT_UNUSED_ENTRY(241), - ICE_PTT_UNUSED_ENTRY(242), - ICE_PTT_UNUSED_ENTRY(243), - ICE_PTT_UNUSED_ENTRY(244), - ICE_PTT_UNUSED_ENTRY(245), - ICE_PTT_UNUSED_ENTRY(246), - ICE_PTT_UNUSED_ENTRY(247), - ICE_PTT_UNUSED_ENTRY(248), - ICE_PTT_UNUSED_ENTRY(249), - - ICE_PTT_UNUSED_ENTRY(250), - ICE_PTT_UNUSED_ENTRY(251), - ICE_PTT_UNUSED_ENTRY(252), - ICE_PTT_UNUSED_ENTRY(253), - ICE_PTT_UNUSED_ENTRY(254), - ICE_PTT_UNUSED_ENTRY(255), + [154 ... 1023] = { 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; static inline struct ice_rx_ptype_decoded ice_decode_rx_desc_ptype(u16 ptype) diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 82e2ce23df3d..dde9802c6c72 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -105,8 +105,14 @@ static int ice_vsi_alloc_arrays(struct ice_vsi *vsi) if (!vsi->q_vectors) goto err_vectors; + vsi->af_xdp_zc_qps = bitmap_zalloc(max_t(int, vsi->alloc_txq, vsi->alloc_rxq), GFP_KERNEL); + if (!vsi->af_xdp_zc_qps) + goto err_zc_qps; + return 0; +err_zc_qps: + devm_kfree(dev, vsi->q_vectors); err_vectors: devm_kfree(dev, vsi->rxq_map); err_rxq_map: @@ -163,12 +169,13 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id) switch (vsi->type) { case ICE_VSI_PF: - vsi->alloc_txq = min3(pf->num_lan_msix, - ice_get_avail_txq_count(pf), - (u16)num_online_cpus()); if (vsi->req_txq) { vsi->alloc_txq = vsi->req_txq; vsi->num_txq = vsi->req_txq; + } else { + vsi->alloc_txq = min3(pf->num_lan_msix, + ice_get_avail_txq_count(pf), + (u16)num_online_cpus()); } pf->num_lan_tx = vsi->alloc_txq; @@ -177,12 +184,13 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id) if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { vsi->alloc_rxq = 1; } else { - vsi->alloc_rxq = min3(pf->num_lan_msix, - ice_get_avail_rxq_count(pf), - (u16)num_online_cpus()); if (vsi->req_rxq) { vsi->alloc_rxq = vsi->req_rxq; vsi->num_rxq = vsi->req_rxq; + } else { + vsi->alloc_rxq = min3(pf->num_lan_msix, + ice_get_avail_rxq_count(pf), + (u16)num_online_cpus()); } } @@ -194,6 +202,8 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi, u16 vf_id) break; case ICE_VSI_VF: vf = &pf->vf[vsi->vf_id]; + if (vf->num_req_qs) + vf->num_vf_qs = vf->num_req_qs; vsi->alloc_txq = vf->num_vf_qs; vsi->alloc_rxq = vf->num_vf_qs; /* pf->num_msix_per_vf includes (VF miscellaneous vector + @@ -288,6 +298,10 @@ static void ice_vsi_free_arrays(struct ice_vsi *vsi) dev = ice_pf_to_dev(pf); + if (vsi->af_xdp_zc_qps) { + bitmap_free(vsi->af_xdp_zc_qps); + vsi->af_xdp_zc_qps = NULL; + } /* free the ring and vector containers */ if (vsi->q_vectors) { devm_kfree(dev, vsi->q_vectors); @@ -617,6 +631,17 @@ bool ice_is_safe_mode(struct ice_pf *pf) } /** + * ice_is_aux_ena + * @pf: pointer to the PF struct + * + * returns true if AUX devices/drivers are supported, false otherwise + */ +bool ice_is_aux_ena(struct ice_pf *pf) +{ + return test_bit(ICE_FLAG_AUX_ENA, pf->flags); +} + +/** * ice_vsi_clean_rss_flow_fld - Delete RSS configuration * @vsi: the VSI being cleaned up * @@ -1180,11 +1205,11 @@ static int ice_vsi_setup_vector_base(struct ice_vsi *vsi) num_q_vectors = vsi->num_q_vectors; /* reserve slots from OS requested IRQs */ if (vsi->type == ICE_VSI_CTRL && vsi->vf_id != ICE_INVAL_VFID) { - struct ice_vf *vf; int i; ice_for_each_vf(pf, i) { - vf = &pf->vf[i]; + struct ice_vf *vf = &pf->vf[i]; + if (i != vsi->vf_id && vf->ctrl_vsi_idx != ICE_NO_VSI) { base = pf->vsi[vf->ctrl_vsi_idx]->base_vector; break; @@ -1273,6 +1298,7 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) ring->reg_idx = vsi->txq_map[i]; ring->ring_active = false; ring->vsi = vsi; + ring->tx_tstamps = &pf->ptp.port.tx; ring->dev = dev; ring->count = vsi->num_tx_desc; WRITE_ONCE(vsi->tx_rings[i], ring); @@ -1650,9 +1676,11 @@ void ice_vsi_cfg_frame_size(struct ice_vsi *vsi) * @pf_q: index of the Rx queue in the PF's queue space * @rxdid: flexible descriptor RXDID * @prio: priority for the RXDID for this queue + * @ena_ts: true to enable timestamp and false to disable timestamp */ void -ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio) +ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio, + bool ena_ts) { int regval = rd32(hw, QRXFLXP_CNTXT(pf_q)); @@ -1667,9 +1695,40 @@ ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio) regval |= (prio << QRXFLXP_CNTXT_RXDID_PRIO_S) & QRXFLXP_CNTXT_RXDID_PRIO_M; + if (ena_ts) + /* Enable TimeSync on this queue */ + regval |= QRXFLXP_CNTXT_TS_M; + wr32(hw, QRXFLXP_CNTXT(pf_q), regval); } +int ice_vsi_cfg_single_rxq(struct ice_vsi *vsi, u16 q_idx) +{ + if (q_idx >= vsi->num_rxq) + return -EINVAL; + + return ice_vsi_cfg_rxq(vsi->rx_rings[q_idx]); +} + +int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_ring **tx_rings, u16 q_idx) +{ + struct ice_aqc_add_tx_qgrp *qg_buf; + int err; + + if (q_idx >= vsi->alloc_txq || !tx_rings || !tx_rings[q_idx]) + return -EINVAL; + + qg_buf = kzalloc(struct_size(qg_buf, txqs, 1), GFP_KERNEL); + if (!qg_buf) + return -ENOMEM; + + qg_buf->num_txqs = 1; + + err = ice_vsi_cfg_txq(vsi, tx_rings[q_idx], qg_buf); + kfree(qg_buf); + return err; +} + /** * ice_vsi_cfg_rxqs - Configure the VSI for Rx * @vsi: the VSI being configured @@ -1687,15 +1746,11 @@ int ice_vsi_cfg_rxqs(struct ice_vsi *vsi) ice_vsi_cfg_frame_size(vsi); setup_rings: /* set up individual rings */ - for (i = 0; i < vsi->num_rxq; i++) { - int err; + ice_for_each_rxq(vsi, i) { + int err = ice_vsi_cfg_rxq(vsi->rx_rings[i]); - err = ice_setup_rx_ctx(vsi->rx_rings[i]); - if (err) { - dev_err(ice_pf_to_dev(vsi->back), "ice_setup_rx_ctx failed for RxQ %d, err %d\n", - i, err); + if (err) return err; - } } return 0; @@ -1705,12 +1760,13 @@ setup_rings: * ice_vsi_cfg_txqs - Configure the VSI for Tx * @vsi: the VSI being configured * @rings: Tx ring array to be configured + * @count: number of Tx ring array elements * * Return 0 on success and a negative value on error * Configure the Tx VSI for operation. */ static int -ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_ring **rings) +ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_ring **rings, u16 count) { struct ice_aqc_add_tx_qgrp *qg_buf; u16 q_idx = 0; @@ -1722,7 +1778,7 @@ ice_vsi_cfg_txqs(struct ice_vsi *vsi, struct ice_ring **rings) qg_buf->num_txqs = 1; - for (q_idx = 0; q_idx < vsi->num_txq; q_idx++) { + for (q_idx = 0; q_idx < count; q_idx++) { err = ice_vsi_cfg_txq(vsi, rings[q_idx], qg_buf); if (err) goto err_cfg_txqs; @@ -1742,7 +1798,7 @@ err_cfg_txqs: */ int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi) { - return ice_vsi_cfg_txqs(vsi, vsi->tx_rings); + return ice_vsi_cfg_txqs(vsi, vsi->tx_rings, vsi->num_txq); } /** @@ -1757,7 +1813,7 @@ int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi) int ret; int i; - ret = ice_vsi_cfg_txqs(vsi, vsi->xdp_rings); + ret = ice_vsi_cfg_txqs(vsi, vsi->xdp_rings, vsi->num_xdp_txq); if (ret) return ret; @@ -1997,17 +2053,18 @@ int ice_vsi_stop_all_rx_rings(struct ice_vsi *vsi) * @rst_src: reset source * @rel_vmvf_num: Relative ID of VF/VM * @rings: Tx ring array to be stopped + * @count: number of Tx ring array elements */ static int ice_vsi_stop_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, - u16 rel_vmvf_num, struct ice_ring **rings) + u16 rel_vmvf_num, struct ice_ring **rings, u16 count) { u16 q_idx; if (vsi->num_txq > ICE_LAN_TXQ_MAX_QDIS) return -EINVAL; - for (q_idx = 0; q_idx < vsi->num_txq; q_idx++) { + for (q_idx = 0; q_idx < count; q_idx++) { struct ice_txq_meta txq_meta = { }; int status; @@ -2035,7 +2092,7 @@ int ice_vsi_stop_lan_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, u16 rel_vmvf_num) { - return ice_vsi_stop_tx_rings(vsi, rst_src, rel_vmvf_num, vsi->tx_rings); + return ice_vsi_stop_tx_rings(vsi, rst_src, rel_vmvf_num, vsi->tx_rings, vsi->num_txq); } /** @@ -2044,7 +2101,7 @@ ice_vsi_stop_lan_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, */ int ice_vsi_stop_xdp_tx_rings(struct ice_vsi *vsi) { - return ice_vsi_stop_tx_rings(vsi, ICE_NO_RESET, 0, vsi->xdp_rings); + return ice_vsi_stop_tx_rings(vsi, ICE_NO_RESET, 0, vsi->xdp_rings, vsi->num_xdp_txq); } /** @@ -2203,7 +2260,7 @@ void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create) } if (status) - dev_err(dev, "Fail %s %s LLDP rule on VSI %i error: %s\n", + dev_dbg(dev, "Fail %s %s LLDP rule on VSI %i error: %s\n", create ? "adding" : "removing", tx ? "TX" : "RX", vsi->vsi_num, ice_stat_str(status)); } @@ -2818,11 +2875,11 @@ int ice_vsi_release(struct ice_vsi *vsi) * cleared in the same manner. */ if (vsi->type == ICE_VSI_CTRL && vsi->vf_id != ICE_INVAL_VFID) { - struct ice_vf *vf; int i; ice_for_each_vf(pf, i) { - vf = &pf->vf[i]; + struct ice_vf *vf = &pf->vf[i]; + if (i != vsi->vf_id && vf->ctrl_vsi_idx != ICE_NO_VSI) break; } @@ -3182,6 +3239,34 @@ bool ice_is_reset_in_progress(unsigned long *state) test_bit(ICE_GLOBR_REQ, state); } +/** + * ice_wait_for_reset - Wait for driver to finish reset and rebuild + * @pf: pointer to the PF structure + * @timeout: length of time to wait, in jiffies + * + * Wait (sleep) for a short time until the driver finishes cleaning up from + * a device reset. The caller must be able to sleep. Use this to delay + * operations that could fail while the driver is cleaning up after a device + * reset. + * + * Returns 0 on success, -EBUSY if the reset is not finished within the + * timeout, and -ERESTARTSYS if the thread was interrupted. + */ +int ice_wait_for_reset(struct ice_pf *pf, unsigned long timeout) +{ + long ret; + + ret = wait_event_interruptible_timeout(pf->reset_wait_queue, + !ice_is_reset_in_progress(pf->state), + timeout); + if (ret < 0) + return ret; + else if (!ret) + return -EBUSY; + else + return 0; +} + #ifdef CONFIG_DCB /** * ice_vsi_update_q_map - update our copy of the VSI info with new queue map @@ -3316,13 +3401,22 @@ int ice_status_to_errno(enum ice_status err) case ICE_ERR_DOES_NOT_EXIST: return -ENOENT; case ICE_ERR_OUT_OF_RANGE: - return -ENOTTY; + case ICE_ERR_AQ_ERROR: + case ICE_ERR_AQ_TIMEOUT: + case ICE_ERR_AQ_EMPTY: + case ICE_ERR_AQ_FW_CRITICAL: + return -EIO; case ICE_ERR_PARAM: + case ICE_ERR_INVAL_SIZE: return -EINVAL; case ICE_ERR_NO_MEMORY: return -ENOMEM; case ICE_ERR_MAX_LIMIT: return -EAGAIN; + case ICE_ERR_RESET_ONGOING: + return -EBUSY; + case ICE_ERR_AQ_FULL: + return -ENOSPC; default: return -EINVAL; } diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h index 511c2316c40c..d5a28bf0fc2c 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_lib.h @@ -12,6 +12,10 @@ bool ice_pf_state_is_nominal(struct ice_pf *pf); void ice_update_eth_stats(struct ice_vsi *vsi); +int ice_vsi_cfg_single_rxq(struct ice_vsi *vsi, u16 q_idx); + +int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_ring **tx_rings, u16 q_idx); + int ice_vsi_cfg_rxqs(struct ice_vsi *vsi); int ice_vsi_cfg_lan_txqs(struct ice_vsi *vsi); @@ -73,9 +77,11 @@ ice_get_res(struct ice_pf *pf, struct ice_res_tracker *res, u16 needed, u16 id); int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi); bool ice_is_reset_in_progress(unsigned long *state); +int ice_wait_for_reset(struct ice_pf *pf, unsigned long timeout); void -ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio); +ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio, + bool ena_ts); void ice_vsi_dis_irq(struct ice_vsi *vsi); @@ -102,7 +108,7 @@ enum ice_status ice_vsi_cfg_mac_fltr(struct ice_vsi *vsi, const u8 *macaddr, bool set); bool ice_is_safe_mode(struct ice_pf *pf); - +bool ice_is_aux_ena(struct ice_pf *pf); bool ice_is_dflt_vsi_in_use(struct ice_sw *sw); bool ice_is_vsi_dflt_vsi(struct ice_sw *sw, struct ice_vsi *vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 4ee85a217c6f..ef8d1815af56 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -13,6 +13,12 @@ #include "ice_dcb_lib.h" #include "ice_dcb_nl.h" #include "ice_devlink.h" +/* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the + * ice tracepoint functions. This must be done exactly once across the + * ice driver. + */ +#define CREATE_TRACE_POINTS +#include "ice_trace.h" #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" static const char ice_driver_string[] = DRV_SUMMARY; @@ -35,6 +41,8 @@ MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXX MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); #endif /* !CONFIG_DYNAMIC_DEBUG */ +static DEFINE_IDA(ice_aux_ida); + static struct workqueue_struct *ice_wq; static const struct net_device_ops ice_netdev_safe_mode_ops; static const struct net_device_ops ice_netdev_ops; @@ -454,6 +462,8 @@ ice_prepare_for_reset(struct ice_pf *pf) if (test_bit(ICE_PREPARED_FOR_RESET, pf->state)) return; + ice_unplug_aux_dev(pf); + /* Notify VFs of impending reset */ if (ice_check_sq_alive(hw, &hw->mailboxq)) ice_vc_notify_reset(pf); @@ -467,6 +477,9 @@ ice_prepare_for_reset(struct ice_pf *pf) /* disable the VSIs and their queues that are not already DOWN */ ice_pf_dis_all_vsi(pf, false); + if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) + ice_ptp_release(pf); + if (hw->port_info) ice_sched_clear_port(hw->port_info); @@ -499,6 +512,7 @@ static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) clear_bit(ICE_PFR_REQ, pf->state); clear_bit(ICE_CORER_REQ, pf->state); clear_bit(ICE_GLOBR_REQ, pf->state); + wake_up(&pf->reset_wait_queue); return; } @@ -511,6 +525,7 @@ static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) ice_rebuild(pf, reset_type); clear_bit(ICE_PREPARED_FOR_RESET, pf->state); clear_bit(ICE_PFR_REQ, pf->state); + wake_up(&pf->reset_wait_queue); ice_reset_all_vfs(pf, true); } } @@ -561,6 +576,7 @@ static void ice_reset_subtask(struct ice_pf *pf) clear_bit(ICE_PFR_REQ, pf->state); clear_bit(ICE_CORER_REQ, pf->state); clear_bit(ICE_GLOBR_REQ, pf->state); + wake_up(&pf->reset_wait_queue); ice_reset_all_vfs(pf, true); } @@ -858,6 +874,38 @@ static void ice_set_dflt_mib(struct ice_pf *pf) } /** + * ice_check_module_power + * @pf: pointer to PF struct + * @link_cfg_err: bitmap from the link info structure + * + * check module power level returned by a previous call to aq_get_link_info + * and print error messages if module power level is not supported + */ +static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err) +{ + /* if module power level is supported, clear the flag */ + if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT | + ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) { + clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); + return; + } + + /* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the + * above block didn't clear this bit, there's nothing to do + */ + if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags)) + return; + + if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) { + dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n"); + set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); + } else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) { + dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n"); + set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); + } +} + +/** * ice_link_event - process the link event * @pf: PF that the link event is associated with * @pi: port_info for the port that the link event is associated with @@ -892,6 +940,8 @@ ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, pi->lport, ice_stat_str(status), ice_aq_str(pi->hw->adminq.sq_last_status)); + ice_check_module_power(pf, pi->phy.link_info.link_cfg_err); + /* Check if the link state is up after updating link info, and treat * this event as an UP event since the link is actually UP now. */ @@ -1190,6 +1240,10 @@ static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) cq = &hw->adminq; qtype = "Admin"; break; + case ICE_CTL_Q_SB: + cq = &hw->sbq; + qtype = "Sideband"; + break; case ICE_CTL_Q_MAILBOX: cq = &hw->mailboxq; qtype = "Mailbox"; @@ -1364,6 +1418,34 @@ static void ice_clean_mailboxq_subtask(struct ice_pf *pf) } /** + * ice_clean_sbq_subtask - clean the Sideband Queue rings + * @pf: board private structure + */ +static void ice_clean_sbq_subtask(struct ice_pf *pf) +{ + struct ice_hw *hw = &pf->hw; + + /* Nothing to do here if sideband queue is not supported */ + if (!ice_is_sbq_supported(hw)) { + clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); + return; + } + + if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state)) + return; + + if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB)) + return; + + clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); + + if (ice_ctrlq_pending(hw, &hw->sbq)) + __ice_clean_ctrlq(pf, ICE_CTL_Q_SB); + + ice_flush(hw); +} + +/** * ice_service_task_schedule - schedule the service task to wake up * @pf: board private structure * @@ -2006,6 +2088,8 @@ static void ice_check_media_subtask(struct ice_pf *pf) if (err) return; + ice_check_module_power(pf, pi->phy.link_info.link_cfg_err); + if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) ice_init_phy_user_cfg(pi); @@ -2063,6 +2147,7 @@ static void ice_service_task(struct work_struct *work) ice_process_vflr_event(pf); ice_clean_mailboxq_subtask(pf); + ice_clean_sbq_subtask(pf); ice_sync_arfs_fltrs(pf); ice_flush_fdir_ctx(pf); @@ -2078,6 +2163,7 @@ static void ice_service_task(struct work_struct *work) test_bit(ICE_VFLR_EVENT_PENDING, pf->state) || test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) || test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) || + test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) || test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) mod_timer(&pf->serv_tmr, jiffies); } @@ -2096,6 +2182,10 @@ static void ice_set_ctrlq_len(struct ice_hw *hw) hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; + hw->sbq.num_rq_entries = ICE_SBQ_LEN; + hw->sbq.num_sq_entries = ICE_SBQ_LEN; + hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN; + hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN; } /** @@ -2118,6 +2208,8 @@ int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) return -EBUSY; } + ice_unplug_aux_dev(pf); + switch (reset) { case ICE_RESET_PFR: set_bit(ICE_PFR_REQ, pf->state); @@ -2556,6 +2648,20 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, } /** + * ice_xdp_safe_mode - XDP handler for safe mode + * @dev: netdevice + * @xdp: XDP command + */ +static int ice_xdp_safe_mode(struct net_device __always_unused *dev, + struct netdev_bpf *xdp) +{ + NL_SET_ERR_MSG_MOD(xdp->extack, + "Please provide working DDP firmware package in order to use XDP\n" + "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst"); + return -EOPNOTSUPP; +} + +/** * ice_xdp - implements XDP handler * @dev: netdevice * @xdp: XDP command @@ -2608,6 +2714,7 @@ static void ice_ena_misc_vector(struct ice_pf *pf) PFINT_OICR_PCI_EXCEPTION_M | PFINT_OICR_VFLR_M | PFINT_OICR_HMC_ERR_M | + PFINT_OICR_PE_PUSH_M | PFINT_OICR_PE_CRITERR_M); wr32(hw, PFINT_OICR_ENA, val); @@ -2633,6 +2740,7 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) dev = ice_pf_to_dev(pf); set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); + set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); oicr = rd32(hw, PFINT_OICR); ena_mask = rd32(hw, PFINT_OICR_ENA); @@ -2678,8 +2786,6 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) /* If a reset cycle isn't already in progress, we set a bit in * pf->state so that the service task can start a reset/rebuild. - * We also make note of which reset happened so that peer - * devices/drivers can be informed. */ if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) { if (reset == ICE_RESET_CORER) @@ -2706,11 +2812,36 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) } } - if (oicr & PFINT_OICR_HMC_ERR_M) { - ena_mask &= ~PFINT_OICR_HMC_ERR_M; - dev_dbg(dev, "HMC Error interrupt - info 0x%x, data 0x%x\n", - rd32(hw, PFHMC_ERRORINFO), - rd32(hw, PFHMC_ERRORDATA)); + if (oicr & PFINT_OICR_TSYN_TX_M) { + ena_mask &= ~PFINT_OICR_TSYN_TX_M; + ice_ptp_process_ts(pf); + } + + if (oicr & PFINT_OICR_TSYN_EVNT_M) { + u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx)); + + /* Save EVENTs from GTSYN register */ + pf->ptp.ext_ts_irq |= gltsyn_stat & (GLTSYN_STAT_EVENT0_M | + GLTSYN_STAT_EVENT1_M | + GLTSYN_STAT_EVENT2_M); + ena_mask &= ~PFINT_OICR_TSYN_EVNT_M; + kthread_queue_work(pf->ptp.kworker, &pf->ptp.extts_work); + } + +#define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M) + if (oicr & ICE_AUX_CRIT_ERR) { + struct iidc_event *event; + + ena_mask &= ~ICE_AUX_CRIT_ERR; + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (event) { + set_bit(IIDC_EVENT_CRIT_ERR, event->type); + /* report the entire OICR value to AUX driver */ + event->reg = oicr; + ice_send_event_to_aux(pf, event); + kfree(event); + } } /* Report any remaining unexpected interrupts */ @@ -2720,8 +2851,7 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) /* If a critical error is pending there is no choice but to * reset the device. */ - if (oicr & (PFINT_OICR_PE_CRITERR_M | - PFINT_OICR_PCI_EXCEPTION_M | + if (oicr & (PFINT_OICR_PCI_EXCEPTION_M | PFINT_OICR_ECC_ERR_M)) { set_bit(ICE_PFR_REQ, pf->state); ice_service_task_schedule(pf); @@ -2749,6 +2879,9 @@ static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) wr32(hw, PFINT_MBX_CTL, rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); + wr32(hw, PFINT_SB_CTL, + rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M); + /* disable Control queue Interrupt causes */ wr32(hw, PFINT_OICR_CTL, rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); @@ -2803,6 +2936,11 @@ static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) PFINT_MBX_CTL_CAUSE_ENA_M); wr32(hw, PFINT_MBX_CTL, val); + /* This enables Sideband queue Interrupt causes */ + val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) | + PFINT_SB_CTL_CAUSE_ENA_M); + wr32(hw, PFINT_SB_CTL, val); + ice_flush(hw); } @@ -2972,7 +3110,6 @@ static void ice_set_netdev_features(struct net_device *netdev) */ static int ice_cfg_netdev(struct ice_vsi *vsi) { - struct ice_pf *pf = vsi->back; struct ice_netdev_priv *np; struct net_device *netdev; u8 mac_addr[ETH_ALEN]; @@ -2992,7 +3129,7 @@ static int ice_cfg_netdev(struct ice_vsi *vsi) ice_set_ops(netdev); if (vsi->type == ICE_VSI_PF) { - SET_NETDEV_DEV(netdev, ice_pf_to_dev(pf)); + SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back)); ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); ether_addr_copy(netdev->dev_addr, mac_addr); ether_addr_copy(netdev->perm_addr, mac_addr); @@ -3266,6 +3403,9 @@ static void ice_deinit_pf(struct ice_pf *pf) bitmap_free(pf->avail_rxqs); pf->avail_rxqs = NULL; } + + if (pf->ptp.clock) + ptp_clock_unregister(pf->ptp.clock); } /** @@ -3276,6 +3416,12 @@ static void ice_set_pf_caps(struct ice_pf *pf) { struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; + clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); + clear_bit(ICE_FLAG_AUX_ENA, pf->flags); + if (func_caps->common_cap.rdma) { + set_bit(ICE_FLAG_RDMA_ENA, pf->flags); + set_bit(ICE_FLAG_AUX_ENA, pf->flags); + } clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); if (func_caps->common_cap.dcb) set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); @@ -3306,6 +3452,10 @@ static void ice_set_pf_caps(struct ice_pf *pf) func_caps->fd_fltr_best_effort); } + clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); + if (func_caps->common_cap.ieee_1588) + set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); + pf->max_pf_txqs = func_caps->common_cap.num_txq; pf->max_pf_rxqs = func_caps->common_cap.num_rxq; } @@ -3325,6 +3475,8 @@ static int ice_init_pf(struct ice_pf *pf) spin_lock_init(&pf->aq_wait_lock); init_waitqueue_head(&pf->aq_wait_queue); + init_waitqueue_head(&pf->reset_wait_queue); + /* setup service timer and periodic service task */ timer_setup(&pf->serv_tmr, ice_service_timer, 0); pf->serv_tmr_period = HZ; @@ -3355,11 +3507,12 @@ static int ice_init_pf(struct ice_pf *pf) */ static int ice_ena_msix_range(struct ice_pf *pf) { - int v_left, v_actual, v_other, v_budget = 0; + int num_cpus, v_left, v_actual, v_other, v_budget = 0; struct device *dev = ice_pf_to_dev(pf); int needed, err, i; v_left = pf->hw.func_caps.common_cap.num_msix_vectors; + num_cpus = num_online_cpus(); /* reserve for LAN miscellaneous handler */ needed = ICE_MIN_LAN_OICR_MSIX; @@ -3381,13 +3534,23 @@ static int ice_ena_msix_range(struct ice_pf *pf) v_other = v_budget; /* reserve vectors for LAN traffic */ - needed = min_t(int, num_online_cpus(), v_left); + needed = num_cpus; if (v_left < needed) goto no_hw_vecs_left_err; pf->num_lan_msix = needed; v_budget += needed; v_left -= needed; + /* reserve vectors for RDMA auxiliary driver */ + if (test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) { + needed = num_cpus + ICE_RDMA_NUM_AEQ_MSIX; + if (v_left < needed) + goto no_hw_vecs_left_err; + pf->num_rdma_msix = needed; + v_budget += needed; + v_left -= needed; + } + pf->msix_entries = devm_kcalloc(dev, v_budget, sizeof(*pf->msix_entries), GFP_KERNEL); if (!pf->msix_entries) { @@ -3417,16 +3580,46 @@ static int ice_ena_msix_range(struct ice_pf *pf) err = -ERANGE; goto msix_err; } else { - int v_traffic = v_actual - v_other; + int v_remain = v_actual - v_other; + int v_rdma = 0, v_min_rdma = 0; + + if (test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) { + /* Need at least 1 interrupt in addition to + * AEQ MSIX + */ + v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1; + v_min_rdma = ICE_MIN_RDMA_MSIX; + } if (v_actual == ICE_MIN_MSIX || - v_traffic < ICE_MIN_LAN_TXRX_MSIX) + v_remain < ICE_MIN_LAN_TXRX_MSIX + v_min_rdma) { + dev_warn(dev, "Not enough MSI-X vectors to support RDMA.\n"); + clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); + + pf->num_rdma_msix = 0; pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX; - else - pf->num_lan_msix = v_traffic; + } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) || + (v_remain - v_rdma < v_rdma)) { + /* Support minimum RDMA and give remaining + * vectors to LAN MSIX + */ + pf->num_rdma_msix = v_min_rdma; + pf->num_lan_msix = v_remain - v_min_rdma; + } else { + /* Split remaining MSIX with RDMA after + * accounting for AEQ MSIX + */ + pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 + + ICE_RDMA_NUM_AEQ_MSIX; + pf->num_lan_msix = v_remain - pf->num_rdma_msix; + } dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n", pf->num_lan_msix); + + if (test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) + dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n", + pf->num_rdma_msix); } } @@ -3441,6 +3634,7 @@ no_hw_vecs_left_err: needed, v_left); err = -ERANGE; exit_err: + pf->num_rdma_msix = 0; pf->num_lan_msix = 0; return err; } @@ -4204,6 +4398,8 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) ice_init_link_dflt_override(pf->hw.port_info); + ice_check_module_power(pf, pf->hw.port_info->phy.link_info.link_cfg_err); + /* if media available, initialize PHY settings */ if (pf->hw.port_info->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { @@ -4242,6 +4438,8 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) } /* initialize DDP driven features */ + if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) + ice_ptp_init(pf); /* Note: Flow director init failure is non-fatal to load */ if (ice_init_fdir(pf)) @@ -4268,8 +4466,29 @@ probe_done: /* ready to go, so clear down state bit */ clear_bit(ICE_DOWN, pf->state); + if (ice_is_aux_ena(pf)) { + pf->aux_idx = ida_alloc(&ice_aux_ida, GFP_KERNEL); + if (pf->aux_idx < 0) { + dev_err(dev, "Failed to allocate device ID for AUX driver\n"); + err = -ENOMEM; + goto err_netdev_reg; + } + + err = ice_init_rdma(pf); + if (err) { + dev_err(dev, "Failed to initialize RDMA: %d\n", err); + err = -EIO; + goto err_init_aux_unroll; + } + } else { + dev_warn(dev, "RDMA is not supported on this device\n"); + } + return 0; +err_init_aux_unroll: + pf->adev = NULL; + ida_free(&ice_aux_ida, pf->aux_idx); err_netdev_reg: err_send_version_unroll: ice_vsi_release_all(pf); @@ -4379,13 +4598,17 @@ static void ice_remove(struct pci_dev *pdev) ice_free_vfs(pf); } - set_bit(ICE_DOWN, pf->state); ice_service_task_stop(pf); ice_aq_cancel_waiting_tasks(pf); + ice_unplug_aux_dev(pf); + ida_free(&ice_aux_ida, pf->aux_idx); + set_bit(ICE_DOWN, pf->state); mutex_destroy(&(&pf->hw)->fdir_fltr_lock); ice_deinit_lag(pf); + if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) + ice_ptp_release(pf); if (!ice_is_safe_mode(pf)) ice_remove_arfs(pf); ice_setup_mc_magic_wake(pf); @@ -4538,6 +4761,8 @@ static int __maybe_unused ice_suspend(struct device *dev) */ disabled = ice_service_task_stop(pf); + ice_unplug_aux_dev(pf); + /* Already suspended?, then there is nothing to do */ if (test_and_set_bit(ICE_SUSPENDED, pf->state)) { if (!disabled) @@ -5270,6 +5495,7 @@ static void ice_tx_dim_work(struct work_struct *work) itr = tx_profile[dim->profile_ix].itr; intrl = tx_profile[dim->profile_ix].intrl; + ice_trace(tx_dim_work, q_vector, dim); ice_write_itr(rc, itr); ice_write_intrl(q_vector, intrl); @@ -5294,6 +5520,7 @@ static void ice_rx_dim_work(struct work_struct *work) itr = rx_profile[dim->profile_ix].itr; intrl = rx_profile[dim->profile_ix].intrl; + ice_trace(rx_dim_work, q_vector, dim); ice_write_itr(rc, itr); ice_write_intrl(q_vector, intrl); @@ -5437,7 +5664,6 @@ ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, struct ice_ring **rings, static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) { struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; - struct ice_ring *ring; u64 pkts, bytes; int i; @@ -5461,7 +5687,8 @@ static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) /* update Rx rings counters */ ice_for_each_rxq(vsi, i) { - ring = READ_ONCE(vsi->rx_rings[i]); + struct ice_ring *ring = READ_ONCE(vsi->rx_rings[i]); + ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); vsi_stats->rx_packets += pkts; vsi_stats->rx_bytes += bytes; @@ -6128,6 +6355,12 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ice_clear_pxe_mode(hw); + ret = ice_init_nvm(hw); + if (ret) { + dev_err(dev, "ice_init_nvm failed %s\n", ice_stat_str(ret)); + goto err_init_ctrlq; + } + ret = ice_get_caps(hw); if (ret) { dev_err(dev, "ice_get_caps failed %s\n", ice_stat_str(ret)); @@ -6169,6 +6402,13 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) ice_dcb_rebuild(pf); + /* If the PF previously had enabled PTP, PTP init needs to happen before + * the VSI rebuild. If not, this causes the PTP link status events to + * fail. + */ + if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) + ice_ptp_init(pf); + /* rebuild PF VSI */ err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); if (err) { @@ -6208,6 +6448,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) /* if we get here, reset flow is successful */ clear_bit(ICE_RESET_FAILED, pf->state); + + ice_plug_aux_dev(pf); return; err_vsi_rebuild: @@ -6246,7 +6488,9 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu) struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; struct ice_pf *pf = vsi->back; + struct iidc_event *event; u8 count = 0; + int err = 0; if (new_mtu == (int)netdev->mtu) { netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); @@ -6279,27 +6523,59 @@ static int ice_change_mtu(struct net_device *netdev, int new_mtu) return -EBUSY; } + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return -ENOMEM; + + set_bit(IIDC_EVENT_BEFORE_MTU_CHANGE, event->type); + ice_send_event_to_aux(pf, event); + clear_bit(IIDC_EVENT_BEFORE_MTU_CHANGE, event->type); + netdev->mtu = (unsigned int)new_mtu; /* if VSI is up, bring it down and then back up */ if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { - int err; - err = ice_down(vsi); if (err) { netdev_err(netdev, "change MTU if_down err %d\n", err); - return err; + goto event_after; } err = ice_up(vsi); if (err) { netdev_err(netdev, "change MTU if_up err %d\n", err); - return err; + goto event_after; } } netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); - return 0; +event_after: + set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type); + ice_send_event_to_aux(pf, event); + kfree(event); + + return err; +} + +/** + * ice_do_ioctl - Access the hwtstamp interface + * @netdev: network interface device structure + * @ifr: interface request data + * @cmd: ioctl command + */ +static int ice_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_pf *pf = np->vsi->back; + + switch (cmd) { + case SIOCGHWTSTAMP: + return ice_ptp_get_ts_config(pf, ifr); + case SIOCSHWTSTAMP: + return ice_ptp_set_ts_config(pf, ifr); + default: + return -EOPNOTSUPP; + } } /** @@ -6818,6 +7094,8 @@ int ice_open_internal(struct net_device *netdev) return -EIO; } + ice_check_module_power(pf, pi->phy.link_info.link_cfg_err); + /* Set PHY if there is media, otherwise, turn off PHY */ if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); @@ -6937,6 +7215,7 @@ static const struct net_device_ops ice_netdev_safe_mode_ops = { .ndo_change_mtu = ice_change_mtu, .ndo_get_stats64 = ice_get_stats64, .ndo_tx_timeout = ice_tx_timeout, + .ndo_bpf = ice_xdp_safe_mode, }; static const struct net_device_ops ice_netdev_ops = { @@ -6950,6 +7229,7 @@ static const struct net_device_ops ice_netdev_ops = { .ndo_change_mtu = ice_change_mtu, .ndo_get_stats64 = ice_get_stats64, .ndo_set_tx_maxrate = ice_set_tx_maxrate, + .ndo_do_ioctl = ice_do_ioctl, .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, .ndo_set_vf_mac = ice_set_vf_mac, .ndo_get_vf_config = ice_get_vf_cfg, diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c new file mode 100644 index 000000000000..5d5207b56ca9 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -0,0 +1,1558 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021, Intel Corporation. */ + +#include "ice.h" +#include "ice_lib.h" + +#define E810_OUT_PROP_DELAY_NS 1 + +/** + * ice_set_tx_tstamp - Enable or disable Tx timestamping + * @pf: The PF pointer to search in + * @on: bool value for whether timestamps are enabled or disabled + */ +static void ice_set_tx_tstamp(struct ice_pf *pf, bool on) +{ + struct ice_vsi *vsi; + u32 val; + u16 i; + + vsi = ice_get_main_vsi(pf); + if (!vsi) + return; + + /* Set the timestamp enable flag for all the Tx rings */ + ice_for_each_rxq(vsi, i) { + if (!vsi->tx_rings[i]) + continue; + vsi->tx_rings[i]->ptp_tx = on; + } + + /* Configure the Tx timestamp interrupt */ + val = rd32(&pf->hw, PFINT_OICR_ENA); + if (on) + val |= PFINT_OICR_TSYN_TX_M; + else + val &= ~PFINT_OICR_TSYN_TX_M; + wr32(&pf->hw, PFINT_OICR_ENA, val); +} + +/** + * ice_set_rx_tstamp - Enable or disable Rx timestamping + * @pf: The PF pointer to search in + * @on: bool value for whether timestamps are enabled or disabled + */ +static void ice_set_rx_tstamp(struct ice_pf *pf, bool on) +{ + struct ice_vsi *vsi; + u16 i; + + vsi = ice_get_main_vsi(pf); + if (!vsi) + return; + + /* Set the timestamp flag for all the Rx rings */ + ice_for_each_rxq(vsi, i) { + if (!vsi->rx_rings[i]) + continue; + vsi->rx_rings[i]->ptp_rx = on; + } +} + +/** + * ice_ptp_cfg_timestamp - Configure timestamp for init/deinit + * @pf: Board private structure + * @ena: bool value to enable or disable time stamp + * + * This function will configure timestamping during PTP initialization + * and deinitialization + */ +static void ice_ptp_cfg_timestamp(struct ice_pf *pf, bool ena) +{ + ice_set_tx_tstamp(pf, ena); + ice_set_rx_tstamp(pf, ena); + + if (ena) { + pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_ALL; + pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_ON; + } else { + pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; + pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_OFF; + } +} + +/** + * ice_get_ptp_clock_index - Get the PTP clock index + * @pf: the PF pointer + * + * Determine the clock index of the PTP clock associated with this device. If + * this is the PF controlling the clock, just use the local access to the + * clock device pointer. + * + * Otherwise, read from the driver shared parameters to determine the clock + * index value. + * + * Returns: the index of the PTP clock associated with this device, or -1 if + * there is no associated clock. + */ +int ice_get_ptp_clock_index(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + enum ice_aqc_driver_params param_idx; + struct ice_hw *hw = &pf->hw; + u8 tmr_idx; + u32 value; + int err; + + /* Use the ptp_clock structure if we're the main PF */ + if (pf->ptp.clock) + return ptp_clock_index(pf->ptp.clock); + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc; + if (!tmr_idx) + param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0; + else + param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1; + + err = ice_aq_get_driver_param(hw, param_idx, &value, NULL); + if (err) { + dev_err(dev, "Failed to read PTP clock index parameter, err %d aq_err %s\n", + err, ice_aq_str(hw->adminq.sq_last_status)); + return -1; + } + + /* The PTP clock index is an integer, and will be between 0 and + * INT_MAX. The highest bit of the driver shared parameter is used to + * indicate whether or not the currently stored clock index is valid. + */ + if (!(value & PTP_SHARED_CLK_IDX_VALID)) + return -1; + + return value & ~PTP_SHARED_CLK_IDX_VALID; +} + +/** + * ice_set_ptp_clock_index - Set the PTP clock index + * @pf: the PF pointer + * + * Set the PTP clock index for this device into the shared driver parameters, + * so that other PFs associated with this device can read it. + * + * If the PF is unable to store the clock index, it will log an error, but + * will continue operating PTP. + */ +static void ice_set_ptp_clock_index(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + enum ice_aqc_driver_params param_idx; + struct ice_hw *hw = &pf->hw; + u8 tmr_idx; + u32 value; + int err; + + if (!pf->ptp.clock) + return; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc; + if (!tmr_idx) + param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0; + else + param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1; + + value = (u32)ptp_clock_index(pf->ptp.clock); + if (value > INT_MAX) { + dev_err(dev, "PTP Clock index is too large to store\n"); + return; + } + value |= PTP_SHARED_CLK_IDX_VALID; + + err = ice_aq_set_driver_param(hw, param_idx, value, NULL); + if (err) { + dev_err(dev, "Failed to set PTP clock index parameter, err %d aq_err %s\n", + err, ice_aq_str(hw->adminq.sq_last_status)); + } +} + +/** + * ice_clear_ptp_clock_index - Clear the PTP clock index + * @pf: the PF pointer + * + * Clear the PTP clock index for this device. Must be called when + * unregistering the PTP clock, in order to ensure other PFs stop reporting + * a clock object that no longer exists. + */ +static void ice_clear_ptp_clock_index(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + enum ice_aqc_driver_params param_idx; + struct ice_hw *hw = &pf->hw; + u8 tmr_idx; + int err; + + /* Do not clear the index if we don't own the timer */ + if (!hw->func_caps.ts_func_info.src_tmr_owned) + return; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc; + if (!tmr_idx) + param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR0; + else + param_idx = ICE_AQC_DRIVER_PARAM_CLK_IDX_TMR1; + + err = ice_aq_set_driver_param(hw, param_idx, 0, NULL); + if (err) { + dev_dbg(dev, "Failed to clear PTP clock index parameter, err %d aq_err %s\n", + err, ice_aq_str(hw->adminq.sq_last_status)); + } +} + +/** + * ice_ptp_read_src_clk_reg - Read the source clock register + * @pf: Board private structure + * @sts: Optional parameter for holding a pair of system timestamps from + * the system clock. Will be ignored if NULL is given. + */ +static u64 +ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts) +{ + struct ice_hw *hw = &pf->hw; + u32 hi, lo, lo2; + u8 tmr_idx; + + tmr_idx = ice_get_ptp_src_clock_index(hw); + /* Read the system timestamp pre PHC read */ + ptp_read_system_prets(sts); + + lo = rd32(hw, GLTSYN_TIME_L(tmr_idx)); + + /* Read the system timestamp post PHC read */ + ptp_read_system_postts(sts); + + hi = rd32(hw, GLTSYN_TIME_H(tmr_idx)); + lo2 = rd32(hw, GLTSYN_TIME_L(tmr_idx)); + + if (lo2 < lo) { + /* if TIME_L rolled over read TIME_L again and update + * system timestamps + */ + ptp_read_system_prets(sts); + lo = rd32(hw, GLTSYN_TIME_L(tmr_idx)); + ptp_read_system_postts(sts); + hi = rd32(hw, GLTSYN_TIME_H(tmr_idx)); + } + + return ((u64)hi << 32) | lo; +} + +/** + * ice_ptp_update_cached_phctime - Update the cached PHC time values + * @pf: Board specific private structure + * + * This function updates the system time values which are cached in the PF + * structure and the Rx rings. + * + * This function must be called periodically to ensure that the cached value + * is never more than 2 seconds old. It must also be called whenever the PHC + * time has been changed. + */ +static void ice_ptp_update_cached_phctime(struct ice_pf *pf) +{ + u64 systime; + int i; + + /* Read the current PHC time */ + systime = ice_ptp_read_src_clk_reg(pf, NULL); + + /* Update the cached PHC time stored in the PF structure */ + WRITE_ONCE(pf->ptp.cached_phc_time, systime); + + ice_for_each_vsi(pf, i) { + struct ice_vsi *vsi = pf->vsi[i]; + int j; + + if (!vsi) + continue; + + if (vsi->type != ICE_VSI_PF) + continue; + + ice_for_each_rxq(vsi, j) { + if (!vsi->rx_rings[j]) + continue; + WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime); + } + } +} + +/** + * ice_ptp_extend_32b_ts - Convert a 32b nanoseconds timestamp to 64b + * @cached_phc_time: recently cached copy of PHC time + * @in_tstamp: Ingress/egress 32b nanoseconds timestamp value + * + * Hardware captures timestamps which contain only 32 bits of nominal + * nanoseconds, as opposed to the 64bit timestamps that the stack expects. + * Note that the captured timestamp values may be 40 bits, but the lower + * 8 bits are sub-nanoseconds and generally discarded. + * + * Extend the 32bit nanosecond timestamp using the following algorithm and + * assumptions: + * + * 1) have a recently cached copy of the PHC time + * 2) assume that the in_tstamp was captured 2^31 nanoseconds (~2.1 + * seconds) before or after the PHC time was captured. + * 3) calculate the delta between the cached time and the timestamp + * 4) if the delta is smaller than 2^31 nanoseconds, then the timestamp was + * captured after the PHC time. In this case, the full timestamp is just + * the cached PHC time plus the delta. + * 5) otherwise, if the delta is larger than 2^31 nanoseconds, then the + * timestamp was captured *before* the PHC time, i.e. because the PHC + * cache was updated after the timestamp was captured by hardware. In this + * case, the full timestamp is the cached time minus the inverse delta. + * + * This algorithm works even if the PHC time was updated after a Tx timestamp + * was requested, but before the Tx timestamp event was reported from + * hardware. + * + * This calculation primarily relies on keeping the cached PHC time up to + * date. If the timestamp was captured more than 2^31 nanoseconds after the + * PHC time, it is possible that the lower 32bits of PHC time have + * overflowed more than once, and we might generate an incorrect timestamp. + * + * This is prevented by (a) periodically updating the cached PHC time once + * a second, and (b) discarding any Tx timestamp packet if it has waited for + * a timestamp for more than one second. + */ +static u64 ice_ptp_extend_32b_ts(u64 cached_phc_time, u32 in_tstamp) +{ + u32 delta, phc_time_lo; + u64 ns; + + /* Extract the lower 32 bits of the PHC time */ + phc_time_lo = (u32)cached_phc_time; + + /* Calculate the delta between the lower 32bits of the cached PHC + * time and the in_tstamp value + */ + delta = (in_tstamp - phc_time_lo); + + /* Do not assume that the in_tstamp is always more recent than the + * cached PHC time. If the delta is large, it indicates that the + * in_tstamp was taken in the past, and should be converted + * forward. + */ + if (delta > (U32_MAX / 2)) { + /* reverse the delta calculation here */ + delta = (phc_time_lo - in_tstamp); + ns = cached_phc_time - delta; + } else { + ns = cached_phc_time + delta; + } + + return ns; +} + +/** + * ice_ptp_extend_40b_ts - Convert a 40b timestamp to 64b nanoseconds + * @pf: Board private structure + * @in_tstamp: Ingress/egress 40b timestamp value + * + * The Tx and Rx timestamps are 40 bits wide, including 32 bits of nominal + * nanoseconds, 7 bits of sub-nanoseconds, and a valid bit. + * + * *--------------------------------------------------------------* + * | 32 bits of nanoseconds | 7 high bits of sub ns underflow | v | + * *--------------------------------------------------------------* + * + * The low bit is an indicator of whether the timestamp is valid. The next + * 7 bits are a capture of the upper 7 bits of the sub-nanosecond underflow, + * and the remaining 32 bits are the lower 32 bits of the PHC timer. + * + * It is assumed that the caller verifies the timestamp is valid prior to + * calling this function. + * + * Extract the 32bit nominal nanoseconds and extend them. Use the cached PHC + * time stored in the device private PTP structure as the basis for timestamp + * extension. + * + * See ice_ptp_extend_32b_ts for a detailed explanation of the extension + * algorithm. + */ +static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp) +{ + const u64 mask = GENMASK_ULL(31, 0); + + return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time, + (in_tstamp >> 8) & mask); +} + +/** + * ice_ptp_read_time - Read the time from the device + * @pf: Board private structure + * @ts: timespec structure to hold the current time value + * @sts: Optional parameter for holding a pair of system timestamps from + * the system clock. Will be ignored if NULL is given. + * + * This function reads the source clock registers and stores them in a timespec. + * However, since the registers are 64 bits of nanoseconds, we must convert the + * result to a timespec before we can return. + */ +static void +ice_ptp_read_time(struct ice_pf *pf, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + u64 time_ns = ice_ptp_read_src_clk_reg(pf, sts); + + *ts = ns_to_timespec64(time_ns); +} + +/** + * ice_ptp_write_init - Set PHC time to provided value + * @pf: Board private structure + * @ts: timespec structure that holds the new time value + * + * Set the PHC time to the specified time provided in the timespec. + */ +static int ice_ptp_write_init(struct ice_pf *pf, struct timespec64 *ts) +{ + u64 ns = timespec64_to_ns(ts); + struct ice_hw *hw = &pf->hw; + + return ice_ptp_init_time(hw, ns); +} + +/** + * ice_ptp_write_adj - Adjust PHC clock time atomically + * @pf: Board private structure + * @adj: Adjustment in nanoseconds + * + * Perform an atomic adjustment of the PHC time by the specified number of + * nanoseconds. + */ +static int ice_ptp_write_adj(struct ice_pf *pf, s32 adj) +{ + struct ice_hw *hw = &pf->hw; + + return ice_ptp_adj_clock(hw, adj); +} + +/** + * ice_ptp_adjfine - Adjust clock increment rate + * @info: the driver's PTP info structure + * @scaled_ppm: Parts per million with 16-bit fractional field + * + * Adjust the frequency of the clock by the indicated scaled ppm from the + * base frequency. + */ +static int ice_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm) +{ + struct ice_pf *pf = ptp_info_to_pf(info); + u64 freq, divisor = 1000000ULL; + struct ice_hw *hw = &pf->hw; + s64 incval, diff; + int neg_adj = 0; + int err; + + incval = ICE_PTP_NOMINAL_INCVAL_E810; + + if (scaled_ppm < 0) { + neg_adj = 1; + scaled_ppm = -scaled_ppm; + } + + while ((u64)scaled_ppm > div_u64(U64_MAX, incval)) { + /* handle overflow by scaling down the scaled_ppm and + * the divisor, losing some precision + */ + scaled_ppm >>= 2; + divisor >>= 2; + } + + freq = (incval * (u64)scaled_ppm) >> 16; + diff = div_u64(freq, divisor); + + if (neg_adj) + incval -= diff; + else + incval += diff; + + err = ice_ptp_write_incval_locked(hw, incval); + if (err) { + dev_err(ice_pf_to_dev(pf), "PTP failed to set incval, err %d\n", + err); + return -EIO; + } + + return 0; +} + +/** + * ice_ptp_extts_work - Workqueue task function + * @work: external timestamp work structure + * + * Service for PTP external clock event + */ +static void ice_ptp_extts_work(struct kthread_work *work) +{ + struct ice_ptp *ptp = container_of(work, struct ice_ptp, extts_work); + struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp); + struct ptp_clock_event event; + struct ice_hw *hw = &pf->hw; + u8 chan, tmr_idx; + u32 hi, lo; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + /* Event time is captured by one of the two matched registers + * GLTSYN_EVNT_L: 32 LSB of sampled time event + * GLTSYN_EVNT_H: 32 MSB of sampled time event + * Event is defined in GLTSYN_EVNT_0 register + */ + for (chan = 0; chan < GLTSYN_EVNT_H_IDX_MAX; chan++) { + /* Check if channel is enabled */ + if (pf->ptp.ext_ts_irq & (1 << chan)) { + lo = rd32(hw, GLTSYN_EVNT_L(chan, tmr_idx)); + hi = rd32(hw, GLTSYN_EVNT_H(chan, tmr_idx)); + event.timestamp = (((u64)hi) << 32) | lo; + event.type = PTP_CLOCK_EXTTS; + event.index = chan; + + /* Fire event */ + ptp_clock_event(pf->ptp.clock, &event); + pf->ptp.ext_ts_irq &= ~(1 << chan); + } + } +} + +/** + * ice_ptp_cfg_extts - Configure EXTTS pin and channel + * @pf: Board private structure + * @ena: true to enable; false to disable + * @chan: GPIO channel (0-3) + * @gpio_pin: GPIO pin + * @extts_flags: request flags from the ptp_extts_request.flags + */ +static int +ice_ptp_cfg_extts(struct ice_pf *pf, bool ena, unsigned int chan, u32 gpio_pin, + unsigned int extts_flags) +{ + u32 func, aux_reg, gpio_reg, irq_reg; + struct ice_hw *hw = &pf->hw; + u8 tmr_idx; + + if (chan > (unsigned int)pf->ptp.info.n_ext_ts) + return -EINVAL; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + irq_reg = rd32(hw, PFINT_OICR_ENA); + + if (ena) { + /* Enable the interrupt */ + irq_reg |= PFINT_OICR_TSYN_EVNT_M; + aux_reg = GLTSYN_AUX_IN_0_INT_ENA_M; + +#define GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE BIT(0) +#define GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE BIT(1) + + /* set event level to requested edge */ + if (extts_flags & PTP_FALLING_EDGE) + aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE; + if (extts_flags & PTP_RISING_EDGE) + aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE; + + /* Write GPIO CTL reg. + * 0x1 is input sampled by EVENT register(channel) + * + num_in_channels * tmr_idx + */ + func = 1 + chan + (tmr_idx * 3); + gpio_reg = ((func << GLGEN_GPIO_CTL_PIN_FUNC_S) & + GLGEN_GPIO_CTL_PIN_FUNC_M); + pf->ptp.ext_ts_chan |= (1 << chan); + } else { + /* clear the values we set to reset defaults */ + aux_reg = 0; + gpio_reg = 0; + pf->ptp.ext_ts_chan &= ~(1 << chan); + if (!pf->ptp.ext_ts_chan) + irq_reg &= ~PFINT_OICR_TSYN_EVNT_M; + } + + wr32(hw, PFINT_OICR_ENA, irq_reg); + wr32(hw, GLTSYN_AUX_IN(chan, tmr_idx), aux_reg); + wr32(hw, GLGEN_GPIO_CTL(gpio_pin), gpio_reg); + + return 0; +} + +/** + * ice_ptp_cfg_clkout - Configure clock to generate periodic wave + * @pf: Board private structure + * @chan: GPIO channel (0-3) + * @config: desired periodic clk configuration. NULL will disable channel + * @store: If set to true the values will be stored + * + * Configure the internal clock generator modules to generate the clock wave of + * specified period. + */ +static int ice_ptp_cfg_clkout(struct ice_pf *pf, unsigned int chan, + struct ice_perout_channel *config, bool store) +{ + u64 current_time, period, start_time, phase; + struct ice_hw *hw = &pf->hw; + u32 func, val, gpio_pin; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + /* 0. Reset mode & out_en in AUX_OUT */ + wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), 0); + + /* If we're disabling the output, clear out CLKO and TGT and keep + * output level low + */ + if (!config || !config->ena) { + wr32(hw, GLTSYN_CLKO(chan, tmr_idx), 0); + wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), 0); + wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), 0); + + val = GLGEN_GPIO_CTL_PIN_DIR_M; + gpio_pin = pf->ptp.perout_channels[chan].gpio_pin; + wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val); + + /* Store the value if requested */ + if (store) + memset(&pf->ptp.perout_channels[chan], 0, + sizeof(struct ice_perout_channel)); + + return 0; + } + period = config->period; + start_time = config->start_time; + div64_u64_rem(start_time, period, &phase); + gpio_pin = config->gpio_pin; + + /* 1. Write clkout with half of required period value */ + if (period & 0x1) { + dev_err(ice_pf_to_dev(pf), "CLK Period must be an even value\n"); + goto err; + } + + period >>= 1; + + /* For proper operation, the GLTSYN_CLKO must be larger than clock tick + */ +#define MIN_PULSE 3 + if (period <= MIN_PULSE || period > U32_MAX) { + dev_err(ice_pf_to_dev(pf), "CLK Period must be > %d && < 2^33", + MIN_PULSE * 2); + goto err; + } + + wr32(hw, GLTSYN_CLKO(chan, tmr_idx), lower_32_bits(period)); + + /* Allow time for programming before start_time is hit */ + current_time = ice_ptp_read_src_clk_reg(pf, NULL); + + /* if start time is in the past start the timer at the nearest second + * maintaining phase + */ + if (start_time < current_time) + start_time = div64_u64(current_time + NSEC_PER_MSEC - 1, + NSEC_PER_SEC) * NSEC_PER_SEC + phase; + + start_time -= E810_OUT_PROP_DELAY_NS; + + /* 2. Write TARGET time */ + wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), lower_32_bits(start_time)); + wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), upper_32_bits(start_time)); + + /* 3. Write AUX_OUT register */ + val = GLTSYN_AUX_OUT_0_OUT_ENA_M | GLTSYN_AUX_OUT_0_OUTMOD_M; + wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), val); + + /* 4. write GPIO CTL reg */ + func = 8 + chan + (tmr_idx * 4); + val = GLGEN_GPIO_CTL_PIN_DIR_M | + ((func << GLGEN_GPIO_CTL_PIN_FUNC_S) & GLGEN_GPIO_CTL_PIN_FUNC_M); + wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val); + + /* Store the value if requested */ + if (store) { + memcpy(&pf->ptp.perout_channels[chan], config, + sizeof(struct ice_perout_channel)); + pf->ptp.perout_channels[chan].start_time = phase; + } + + return 0; +err: + dev_err(ice_pf_to_dev(pf), "PTP failed to cfg per_clk\n"); + return -EFAULT; +} + +/** + * ice_ptp_gpio_enable_e810 - Enable/disable ancillary features of PHC + * @info: the driver's PTP info structure + * @rq: The requested feature to change + * @on: Enable/disable flag + */ +static int +ice_ptp_gpio_enable_e810(struct ptp_clock_info *info, + struct ptp_clock_request *rq, int on) +{ + struct ice_pf *pf = ptp_info_to_pf(info); + struct ice_perout_channel clk_cfg = {0}; + unsigned int chan; + u32 gpio_pin; + int err; + + switch (rq->type) { + case PTP_CLK_REQ_PEROUT: + chan = rq->perout.index; + if (chan == PPS_CLK_GEN_CHAN) + clk_cfg.gpio_pin = PPS_PIN_INDEX; + else + clk_cfg.gpio_pin = chan; + + clk_cfg.period = ((rq->perout.period.sec * NSEC_PER_SEC) + + rq->perout.period.nsec); + clk_cfg.start_time = ((rq->perout.start.sec * NSEC_PER_SEC) + + rq->perout.start.nsec); + clk_cfg.ena = !!on; + + err = ice_ptp_cfg_clkout(pf, chan, &clk_cfg, true); + break; + case PTP_CLK_REQ_EXTTS: + chan = rq->extts.index; + gpio_pin = chan; + + err = ice_ptp_cfg_extts(pf, !!on, chan, gpio_pin, + rq->extts.flags); + break; + default: + return -EOPNOTSUPP; + } + + return err; +} + +/** + * ice_ptp_gettimex64 - Get the time of the clock + * @info: the driver's PTP info structure + * @ts: timespec64 structure to hold the current time value + * @sts: Optional parameter for holding a pair of system timestamps from + * the system clock. Will be ignored if NULL is given. + * + * Read the device clock and return the correct value on ns, after converting it + * into a timespec struct. + */ +static int +ice_ptp_gettimex64(struct ptp_clock_info *info, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct ice_pf *pf = ptp_info_to_pf(info); + struct ice_hw *hw = &pf->hw; + + if (!ice_ptp_lock(hw)) { + dev_err(ice_pf_to_dev(pf), "PTP failed to get time\n"); + return -EBUSY; + } + + ice_ptp_read_time(pf, ts, sts); + ice_ptp_unlock(hw); + + return 0; +} + +/** + * ice_ptp_settime64 - Set the time of the clock + * @info: the driver's PTP info structure + * @ts: timespec64 structure that holds the new time value + * + * Set the device clock to the user input value. The conversion from timespec + * to ns happens in the write function. + */ +static int +ice_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts) +{ + struct ice_pf *pf = ptp_info_to_pf(info); + struct timespec64 ts64 = *ts; + struct ice_hw *hw = &pf->hw; + int err; + + if (!ice_ptp_lock(hw)) { + err = -EBUSY; + goto exit; + } + + err = ice_ptp_write_init(pf, &ts64); + ice_ptp_unlock(hw); + + if (!err) + ice_ptp_update_cached_phctime(pf); + +exit: + if (err) { + dev_err(ice_pf_to_dev(pf), "PTP failed to set time %d\n", err); + return err; + } + + return 0; +} + +/** + * ice_ptp_adjtime_nonatomic - Do a non-atomic clock adjustment + * @info: the driver's PTP info structure + * @delta: Offset in nanoseconds to adjust the time by + */ +static int ice_ptp_adjtime_nonatomic(struct ptp_clock_info *info, s64 delta) +{ + struct timespec64 now, then; + + then = ns_to_timespec64(delta); + ice_ptp_gettimex64(info, &now, NULL); + now = timespec64_add(now, then); + + return ice_ptp_settime64(info, (const struct timespec64 *)&now); +} + +/** + * ice_ptp_adjtime - Adjust the time of the clock by the indicated delta + * @info: the driver's PTP info structure + * @delta: Offset in nanoseconds to adjust the time by + */ +static int ice_ptp_adjtime(struct ptp_clock_info *info, s64 delta) +{ + struct ice_pf *pf = ptp_info_to_pf(info); + struct ice_hw *hw = &pf->hw; + struct device *dev; + int err; + + dev = ice_pf_to_dev(pf); + + /* Hardware only supports atomic adjustments using signed 32-bit + * integers. For any adjustment outside this range, perform + * a non-atomic get->adjust->set flow. + */ + if (delta > S32_MAX || delta < S32_MIN) { + dev_dbg(dev, "delta = %lld, adjtime non-atomic\n", delta); + return ice_ptp_adjtime_nonatomic(info, delta); + } + + if (!ice_ptp_lock(hw)) { + dev_err(dev, "PTP failed to acquire semaphore in adjtime\n"); + return -EBUSY; + } + + err = ice_ptp_write_adj(pf, delta); + + ice_ptp_unlock(hw); + + if (err) { + dev_err(dev, "PTP failed to adjust time, err %d\n", err); + return err; + } + + ice_ptp_update_cached_phctime(pf); + + return 0; +} + +/** + * ice_ptp_get_ts_config - ioctl interface to read the timestamping config + * @pf: Board private structure + * @ifr: ioctl data + * + * Copy the timestamping config to user buffer + */ +int ice_ptp_get_ts_config(struct ice_pf *pf, struct ifreq *ifr) +{ + struct hwtstamp_config *config; + + if (!test_bit(ICE_FLAG_PTP, pf->flags)) + return -EIO; + + config = &pf->ptp.tstamp_config; + + return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ? + -EFAULT : 0; +} + +/** + * ice_ptp_set_timestamp_mode - Setup driver for requested timestamp mode + * @pf: Board private structure + * @config: hwtstamp settings requested or saved + */ +static int +ice_ptp_set_timestamp_mode(struct ice_pf *pf, struct hwtstamp_config *config) +{ + /* Reserved for future extensions. */ + if (config->flags) + return -EINVAL; + + switch (config->tx_type) { + case HWTSTAMP_TX_OFF: + ice_set_tx_tstamp(pf, false); + break; + case HWTSTAMP_TX_ON: + ice_set_tx_tstamp(pf, true); + break; + default: + return -ERANGE; + } + + switch (config->rx_filter) { + case HWTSTAMP_FILTER_NONE: + ice_set_rx_tstamp(pf, false); + break; + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + case HWTSTAMP_FILTER_NTP_ALL: + case HWTSTAMP_FILTER_ALL: + config->rx_filter = HWTSTAMP_FILTER_ALL; + ice_set_rx_tstamp(pf, true); + break; + default: + return -ERANGE; + } + + return 0; +} + +/** + * ice_ptp_set_ts_config - ioctl interface to control the timestamping + * @pf: Board private structure + * @ifr: ioctl data + * + * Get the user config and store it + */ +int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr) +{ + struct hwtstamp_config config; + int err; + + if (!test_bit(ICE_FLAG_PTP, pf->flags)) + return -EAGAIN; + + if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) + return -EFAULT; + + err = ice_ptp_set_timestamp_mode(pf, &config); + if (err) + return err; + + /* Save these settings for future reference */ + pf->ptp.tstamp_config = config; + + return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? + -EFAULT : 0; +} + +/** + * ice_ptp_rx_hwtstamp - Check for an Rx timestamp + * @rx_ring: Ring to get the VSI info + * @rx_desc: Receive descriptor + * @skb: Particular skb to send timestamp with + * + * The driver receives a notification in the receive descriptor with timestamp. + * The timestamp is in ns, so we must convert the result first. + */ +void +ice_ptp_rx_hwtstamp(struct ice_ring *rx_ring, + union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) +{ + u32 ts_high; + u64 ts_ns; + + /* Populate timesync data into skb */ + if (rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID) { + struct skb_shared_hwtstamps *hwtstamps; + + /* Use ice_ptp_extend_32b_ts directly, using the ring-specific + * cached PHC value, rather than accessing the PF. This also + * allows us to simply pass the upper 32bits of nanoseconds + * directly. Calling ice_ptp_extend_40b_ts is unnecessary as + * it would just discard these bits itself. + */ + ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high); + ts_ns = ice_ptp_extend_32b_ts(rx_ring->cached_phctime, ts_high); + + hwtstamps = skb_hwtstamps(skb); + memset(hwtstamps, 0, sizeof(*hwtstamps)); + hwtstamps->hwtstamp = ns_to_ktime(ts_ns); + } +} + +/** + * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs + * @info: PTP clock capabilities + */ +static void ice_ptp_setup_pins_e810(struct ptp_clock_info *info) +{ + info->n_per_out = E810_N_PER_OUT; + info->n_ext_ts = E810_N_EXT_TS; +} + +/** + * ice_ptp_set_funcs_e810 - Set specialized functions for E810 support + * @pf: Board private structure + * @info: PTP info to fill + * + * Assign functions to the PTP capabiltiies structure for E810 devices. + * Functions which operate across all device families should be set directly + * in ice_ptp_set_caps. Only add functions here which are distinct for e810 + * devices. + */ +static void +ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info) +{ + info->enable = ice_ptp_gpio_enable_e810; + + ice_ptp_setup_pins_e810(info); +} + +/** + * ice_ptp_set_caps - Set PTP capabilities + * @pf: Board private structure + */ +static void ice_ptp_set_caps(struct ice_pf *pf) +{ + struct ptp_clock_info *info = &pf->ptp.info; + struct device *dev = ice_pf_to_dev(pf); + + snprintf(info->name, sizeof(info->name) - 1, "%s-%s-clk", + dev_driver_string(dev), dev_name(dev)); + info->owner = THIS_MODULE; + info->max_adj = 999999999; + info->adjtime = ice_ptp_adjtime; + info->adjfine = ice_ptp_adjfine; + info->gettimex64 = ice_ptp_gettimex64; + info->settime64 = ice_ptp_settime64; + + ice_ptp_set_funcs_e810(pf, info); +} + +/** + * ice_ptp_create_clock - Create PTP clock device for userspace + * @pf: Board private structure + * + * This function creates a new PTP clock device. It only creates one if we + * don't already have one. Will return error if it can't create one, but success + * if we already have a device. Should be used by ice_ptp_init to create clock + * initially, and prevent global resets from creating new clock devices. + */ +static long ice_ptp_create_clock(struct ice_pf *pf) +{ + struct ptp_clock_info *info; + struct ptp_clock *clock; + struct device *dev; + + /* No need to create a clock device if we already have one */ + if (pf->ptp.clock) + return 0; + + ice_ptp_set_caps(pf); + + info = &pf->ptp.info; + dev = ice_pf_to_dev(pf); + + /* Allocate memory for kernel pins interface */ + if (info->n_pins) { + info->pin_config = devm_kcalloc(dev, info->n_pins, + sizeof(*info->pin_config), + GFP_KERNEL); + if (!info->pin_config) { + info->n_pins = 0; + return -ENOMEM; + } + } + + /* Attempt to register the clock before enabling the hardware. */ + clock = ptp_clock_register(info, dev); + if (IS_ERR(clock)) + return PTR_ERR(clock); + + pf->ptp.clock = clock; + + return 0; +} + +/** + * ice_ptp_tx_tstamp_work - Process Tx timestamps for a port + * @work: pointer to the kthread_work struct + * + * Process timestamps captured by the PHY associated with this port. To do + * this, loop over each index with a waiting skb. + * + * If a given index has a valid timestamp, perform the following steps: + * + * 1) copy the timestamp out of the PHY register + * 4) clear the timestamp valid bit in the PHY register + * 5) unlock the index by clearing the associated in_use bit. + * 2) extend the 40b timestamp value to get a 64bit timestamp + * 3) send that timestamp to the stack + * + * After looping, if we still have waiting SKBs, then re-queue the work. This + * may cause us effectively poll even when not strictly necessary. We do this + * because it's possible a new timestamp was requested around the same time as + * the interrupt. In some cases hardware might not interrupt us again when the + * timestamp is captured. + * + * Note that we only take the tracking lock when clearing the bit and when + * checking if we need to re-queue this task. The only place where bits can be + * set is the hard xmit routine where an SKB has a request flag set. The only + * places where we clear bits are this work function, or the periodic cleanup + * thread. If the cleanup thread clears a bit we're processing we catch it + * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread + * starts a new timestamp, we might not begin processing it right away but we + * will notice it at the end when we re-queue the work item. If a Tx thread + * starts a new timestamp just after this function exits without re-queuing, + * the interrupt when the timestamp finishes should trigger. Avoiding holding + * the lock for the entire function is important in order to ensure that Tx + * threads do not get blocked while waiting for the lock. + */ +static void ice_ptp_tx_tstamp_work(struct kthread_work *work) +{ + struct ice_ptp_port *ptp_port; + struct ice_ptp_tx *tx; + struct ice_pf *pf; + struct ice_hw *hw; + u8 idx; + + tx = container_of(work, struct ice_ptp_tx, work); + if (!tx->init) + return; + + ptp_port = container_of(tx, struct ice_ptp_port, tx); + pf = ptp_port_to_pf(ptp_port); + hw = &pf->hw; + + for_each_set_bit(idx, tx->in_use, tx->len) { + struct skb_shared_hwtstamps shhwtstamps = {}; + u8 phy_idx = idx + tx->quad_offset; + u64 raw_tstamp, tstamp; + struct sk_buff *skb; + int err; + + err = ice_read_phy_tstamp(hw, tx->quad, phy_idx, + &raw_tstamp); + if (err) + continue; + + /* Check if the timestamp is valid */ + if (!(raw_tstamp & ICE_PTP_TS_VALID)) + continue; + + /* clear the timestamp register, so that it won't show valid + * again when re-used. + */ + ice_clear_phy_tstamp(hw, tx->quad, phy_idx); + + /* The timestamp is valid, so we'll go ahead and clear this + * index and then send the timestamp up to the stack. + */ + spin_lock(&tx->lock); + clear_bit(idx, tx->in_use); + skb = tx->tstamps[idx].skb; + tx->tstamps[idx].skb = NULL; + spin_unlock(&tx->lock); + + /* it's (unlikely but) possible we raced with the cleanup + * thread for discarding old timestamp requests. + */ + if (!skb) + continue; + + /* Extend the timestamp using cached PHC time */ + tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp); + shhwtstamps.hwtstamp = ns_to_ktime(tstamp); + + skb_tstamp_tx(skb, &shhwtstamps); + dev_kfree_skb_any(skb); + } + + /* Check if we still have work to do. If so, re-queue this task to + * poll for remaining timestamps. + */ + spin_lock(&tx->lock); + if (!bitmap_empty(tx->in_use, tx->len)) + kthread_queue_work(pf->ptp.kworker, &tx->work); + spin_unlock(&tx->lock); +} + +/** + * ice_ptp_request_ts - Request an available Tx timestamp index + * @tx: the PTP Tx timestamp tracker to request from + * @skb: the SKB to associate with this timestamp request + */ +s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) +{ + u8 idx; + + /* Check if this tracker is initialized */ + if (!tx->init) + return -1; + + spin_lock(&tx->lock); + /* Find and set the first available index */ + idx = find_first_zero_bit(tx->in_use, tx->len); + if (idx < tx->len) { + /* We got a valid index that no other thread could have set. Store + * a reference to the skb and the start time to allow discarding old + * requests. + */ + set_bit(idx, tx->in_use); + tx->tstamps[idx].start = jiffies; + tx->tstamps[idx].skb = skb_get(skb); + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + } + + spin_unlock(&tx->lock); + + /* return the appropriate PHY timestamp register index, -1 if no + * indexes were available. + */ + if (idx >= tx->len) + return -1; + else + return idx + tx->quad_offset; +} + +/** + * ice_ptp_process_ts - Spawn kthread work to handle timestamps + * @pf: Board private structure + * + * Queue work required to process the PTP Tx timestamps outside of interrupt + * context. + */ +void ice_ptp_process_ts(struct ice_pf *pf) +{ + if (pf->ptp.port.tx.init) + kthread_queue_work(pf->ptp.kworker, &pf->ptp.port.tx.work); +} + +/** + * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps + * @tx: Tx tracking structure to initialize + * + * Assumes that the length has already been initialized. Do not call directly, + * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead. + */ +static int +ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx) +{ + tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL); + if (!tx->tstamps) + return -ENOMEM; + + tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL); + if (!tx->in_use) { + kfree(tx->tstamps); + tx->tstamps = NULL; + return -ENOMEM; + } + + spin_lock_init(&tx->lock); + kthread_init_work(&tx->work, ice_ptp_tx_tstamp_work); + + tx->init = 1; + + return 0; +} + +/** + * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker + * @pf: Board private structure + * @tx: the tracker to flush + */ +static void +ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + u8 idx; + + for (idx = 0; idx < tx->len; idx++) { + u8 phy_idx = idx + tx->quad_offset; + + /* Clear any potential residual timestamp in the PHY block */ + if (!pf->hw.reset_ongoing) + ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx); + + if (tx->tstamps[idx].skb) { + dev_kfree_skb_any(tx->tstamps[idx].skb); + tx->tstamps[idx].skb = NULL; + } + } +} + +/** + * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker + * @pf: Board private structure + * @tx: Tx tracking structure to release + * + * Free memory associated with the Tx timestamp tracker. + */ +static void +ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + tx->init = 0; + + kthread_cancel_work_sync(&tx->work); + + ice_ptp_flush_tx_tracker(pf, tx); + + kfree(tx->tstamps); + tx->tstamps = NULL; + + kfree(tx->in_use); + tx->in_use = NULL; + + tx->len = 0; +} + +/** + * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps + * @pf: Board private structure + * @tx: the Tx tracking structure to initialize + * + * Initialize the Tx timestamp tracker for this PF. For E810 devices, each + * port has its own block of timestamps, independent of the other ports. + */ +static int +ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + tx->quad = pf->hw.port_info->lport; + tx->quad_offset = 0; + tx->len = INDEX_PER_QUAD; + + return ice_ptp_alloc_tx_tracker(tx); +} + +/** + * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped + * @tx: PTP Tx tracker to clean up + * + * Loop through the Tx timestamp requests and see if any of them have been + * waiting for a long time. Discard any SKBs that have been waiting for more + * than 2 seconds. This is long enough to be reasonably sure that the + * timestamp will never be captured. This might happen if the packet gets + * discarded before it reaches the PHY timestamping block. + */ +static void ice_ptp_tx_tstamp_cleanup(struct ice_ptp_tx *tx) +{ + u8 idx; + + if (!tx->init) + return; + + for_each_set_bit(idx, tx->in_use, tx->len) { + struct sk_buff *skb; + + /* Check if this SKB has been waiting for too long */ + if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ)) + continue; + + spin_lock(&tx->lock); + skb = tx->tstamps[idx].skb; + tx->tstamps[idx].skb = NULL; + clear_bit(idx, tx->in_use); + spin_unlock(&tx->lock); + + /* Free the SKB after we've cleared the bit */ + dev_kfree_skb_any(skb); + } +} + +static void ice_ptp_periodic_work(struct kthread_work *work) +{ + struct ice_ptp *ptp = container_of(work, struct ice_ptp, work.work); + struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp); + + if (!test_bit(ICE_FLAG_PTP, pf->flags)) + return; + + ice_ptp_update_cached_phctime(pf); + + ice_ptp_tx_tstamp_cleanup(&pf->ptp.port.tx); + + /* Run twice a second */ + kthread_queue_delayed_work(ptp->kworker, &ptp->work, + msecs_to_jiffies(500)); +} + +/** + * ice_ptp_init_owner - Initialize PTP_1588_CLOCK device + * @pf: Board private structure + * + * Setup and initialize a PTP clock device that represents the device hardware + * clock. Save the clock index for other functions connected to the same + * hardware resource. + */ +static int ice_ptp_init_owner(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + struct ice_hw *hw = &pf->hw; + struct timespec64 ts; + u8 src_idx; + int err; + + wr32(hw, GLTSYN_SYNC_DLAY, 0); + + /* Clear some HW residue and enable source clock */ + src_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + /* Enable source clocks */ + wr32(hw, GLTSYN_ENA(src_idx), GLTSYN_ENA_TSYN_ENA_M); + + /* Enable PHY time sync */ + err = ice_ptp_init_phy_e810(hw); + if (err) + goto err_exit; + + /* Clear event status indications for auxiliary pins */ + (void)rd32(hw, GLTSYN_STAT(src_idx)); + + /* Acquire the global hardware lock */ + if (!ice_ptp_lock(hw)) { + err = -EBUSY; + goto err_exit; + } + + /* Write the increment time value to PHY and LAN */ + err = ice_ptp_write_incval(hw, ICE_PTP_NOMINAL_INCVAL_E810); + if (err) { + ice_ptp_unlock(hw); + goto err_exit; + } + + ts = ktime_to_timespec64(ktime_get_real()); + /* Write the initial Time value to PHY and LAN */ + err = ice_ptp_write_init(pf, &ts); + if (err) { + ice_ptp_unlock(hw); + goto err_exit; + } + + /* Release the global hardware lock */ + ice_ptp_unlock(hw); + + /* Ensure we have a clock device */ + err = ice_ptp_create_clock(pf); + if (err) + goto err_clk; + + /* Store the PTP clock index for other PFs */ + ice_set_ptp_clock_index(pf); + + return 0; + +err_clk: + pf->ptp.clock = NULL; +err_exit: + dev_err(dev, "PTP failed to register clock, err %d\n", err); + + return err; +} + +/** + * ice_ptp_init - Initialize the PTP support after device probe or reset + * @pf: Board private structure + * + * This function sets device up for PTP support. The first time it is run, it + * will create a clock device. It does not create a clock device if one + * already exists. It also reconfigures the device after a reset. + */ +void ice_ptp_init(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + struct kthread_worker *kworker; + struct ice_hw *hw = &pf->hw; + int err; + + /* PTP is currently only supported on E810 devices */ + if (!ice_is_e810(hw)) + return; + + /* Check if this PF owns the source timer */ + if (hw->func_caps.ts_func_info.src_tmr_owned) { + err = ice_ptp_init_owner(pf); + if (err) + return; + } + + /* Disable timestamping for both Tx and Rx */ + ice_ptp_cfg_timestamp(pf, false); + + /* Initialize the PTP port Tx timestamp tracker */ + ice_ptp_init_tx_e810(pf, &pf->ptp.port.tx); + + /* Initialize work functions */ + kthread_init_delayed_work(&pf->ptp.work, ice_ptp_periodic_work); + kthread_init_work(&pf->ptp.extts_work, ice_ptp_extts_work); + + /* Allocate a kworker for handling work required for the ports + * connected to the PTP hardware clock. + */ + kworker = kthread_create_worker(0, "ice-ptp-%s", dev_name(dev)); + if (IS_ERR(kworker)) { + err = PTR_ERR(kworker); + goto err_kworker; + } + pf->ptp.kworker = kworker; + + set_bit(ICE_FLAG_PTP, pf->flags); + + /* Start periodic work going */ + kthread_queue_delayed_work(pf->ptp.kworker, &pf->ptp.work, 0); + + dev_info(dev, "PTP init successful\n"); + return; + +err_kworker: + /* If we registered a PTP clock, release it */ + if (pf->ptp.clock) { + ptp_clock_unregister(pf->ptp.clock); + pf->ptp.clock = NULL; + } + dev_err(dev, "PTP failed %d\n", err); +} + +/** + * ice_ptp_release - Disable the driver/HW support and unregister the clock + * @pf: Board private structure + * + * This function handles the cleanup work required from the initialization by + * clearing out the important information and unregistering the clock + */ +void ice_ptp_release(struct ice_pf *pf) +{ + /* Disable timestamping for both Tx and Rx */ + ice_ptp_cfg_timestamp(pf, false); + + ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx); + + clear_bit(ICE_FLAG_PTP, pf->flags); + + kthread_cancel_delayed_work_sync(&pf->ptp.work); + + if (pf->ptp.kworker) { + kthread_destroy_worker(pf->ptp.kworker); + pf->ptp.kworker = NULL; + } + + if (!pf->ptp.clock) + return; + + ice_clear_ptp_clock_index(pf); + ptp_clock_unregister(pf->ptp.clock); + pf->ptp.clock = NULL; + + dev_info(ice_pf_to_dev(pf), "Removed PTP clock\n"); +} diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h new file mode 100644 index 000000000000..e1c787bd5b96 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_ptp.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2021, Intel Corporation. */ + +#ifndef _ICE_PTP_H_ +#define _ICE_PTP_H_ + +#include <linux/ptp_clock_kernel.h> +#include <linux/kthread.h> + +#include "ice_ptp_hw.h" + +enum ice_ptp_pin { + GPIO_20 = 0, + GPIO_21, + GPIO_22, + GPIO_23, + NUM_ICE_PTP_PIN +}; + +struct ice_perout_channel { + bool ena; + u32 gpio_pin; + u64 period; + u64 start_time; +}; + +/* The ice hardware captures Tx hardware timestamps in the PHY. The timestamp + * is stored in a buffer of registers. Depending on the specific hardware, + * this buffer might be shared across multiple PHY ports. + * + * On transmit of a packet to be timestamped, software is responsible for + * selecting an open index. Hardware makes no attempt to lock or prevent + * re-use of an index for multiple packets. + * + * To handle this, timestamp indexes must be tracked by software to ensure + * that an index is not re-used for multiple transmitted packets. The + * structures and functions declared in this file track the available Tx + * register indexes, as well as provide storage for the SKB pointers. + * + * To allow multiple ports to access the shared register block independently, + * the blocks are split up so that indexes are assigned to each port based on + * hardware logical port number. + */ + +/** + * struct ice_tx_tstamp - Tracking for a single Tx timestamp + * @skb: pointer to the SKB for this timestamp request + * @start: jiffies when the timestamp was first requested + * + * This structure tracks a single timestamp request. The SKB pointer is + * provided when initiating a request. The start time is used to ensure that + * we discard old requests that were not fulfilled within a 2 second time + * window. + */ +struct ice_tx_tstamp { + struct sk_buff *skb; + unsigned long start; +}; + +/** + * struct ice_ptp_tx - Tracking structure for all Tx timestamp requests on a port + * @work: work function to handle processing of Tx timestamps + * @lock: lock to prevent concurrent write to in_use bitmap + * @tstamps: array of len to store outstanding requests + * @in_use: bitmap of len to indicate which slots are in use + * @quad: which quad the timestamps are captured in + * @quad_offset: offset into timestamp block of the quad to get the real index + * @len: length of the tstamps and in_use fields. + * @init: if true, the tracker is initialized; + */ +struct ice_ptp_tx { + struct kthread_work work; + spinlock_t lock; /* lock protecting in_use bitmap */ + struct ice_tx_tstamp *tstamps; + unsigned long *in_use; + u8 quad; + u8 quad_offset; + u8 len; + u8 init; +}; + +/* Quad and port information for initializing timestamp blocks */ +#define INDEX_PER_QUAD 64 +#define INDEX_PER_PORT (INDEX_PER_QUAD / ICE_PORTS_PER_QUAD) + +/** + * struct ice_ptp_port - data used to initialize an external port for PTP + * + * This structure contains PTP data related to the external ports. Currently + * it is used for tracking the Tx timestamps of a port. In the future this + * structure will also hold information for the E822 port initialization + * logic. + * + * @tx: Tx timestamp tracking for this port + */ +struct ice_ptp_port { + struct ice_ptp_tx tx; +}; + +#define GLTSYN_TGT_H_IDX_MAX 4 + +/** + * struct ice_ptp - data used for integrating with CONFIG_PTP_1588_CLOCK + * @port: data for the PHY port initialization procedure + * @work: delayed work function for periodic tasks + * @extts_work: work function for handling external Tx timestamps + * @cached_phc_time: a cached copy of the PHC time for timestamp extension + * @ext_ts_chan: the external timestamp channel in use + * @ext_ts_irq: the external timestamp IRQ in use + * @kworker: kwork thread for handling periodic work + * @perout_channels: periodic output data + * @info: structure defining PTP hardware capabilities + * @clock: pointer to registered PTP clock device + * @tstamp_config: hardware timestamping configuration + */ +struct ice_ptp { + struct ice_ptp_port port; + struct kthread_delayed_work work; + struct kthread_work extts_work; + u64 cached_phc_time; + u8 ext_ts_chan; + u8 ext_ts_irq; + struct kthread_worker *kworker; + struct ice_perout_channel perout_channels[GLTSYN_TGT_H_IDX_MAX]; + struct ptp_clock_info info; + struct ptp_clock *clock; + struct hwtstamp_config tstamp_config; +}; + +#define __ptp_port_to_ptp(p) \ + container_of((p), struct ice_ptp, port) +#define ptp_port_to_pf(p) \ + container_of(__ptp_port_to_ptp((p)), struct ice_pf, ptp) + +#define __ptp_info_to_ptp(i) \ + container_of((i), struct ice_ptp, info) +#define ptp_info_to_pf(i) \ + container_of(__ptp_info_to_ptp((i)), struct ice_pf, ptp) + +#define PTP_SHARED_CLK_IDX_VALID BIT(31) +#define ICE_PTP_TS_VALID BIT(0) + +/* Per-channel register definitions */ +#define GLTSYN_AUX_OUT(_chan, _idx) (GLTSYN_AUX_OUT_0(_idx) + ((_chan) * 8)) +#define GLTSYN_AUX_IN(_chan, _idx) (GLTSYN_AUX_IN_0(_idx) + ((_chan) * 8)) +#define GLTSYN_CLKO(_chan, _idx) (GLTSYN_CLKO_0(_idx) + ((_chan) * 8)) +#define GLTSYN_TGT_L(_chan, _idx) (GLTSYN_TGT_L_0(_idx) + ((_chan) * 16)) +#define GLTSYN_TGT_H(_chan, _idx) (GLTSYN_TGT_H_0(_idx) + ((_chan) * 16)) +#define GLTSYN_EVNT_L(_chan, _idx) (GLTSYN_EVNT_L_0(_idx) + ((_chan) * 16)) +#define GLTSYN_EVNT_H(_chan, _idx) (GLTSYN_EVNT_H_0(_idx) + ((_chan) * 16)) +#define GLTSYN_EVNT_H_IDX_MAX 3 + +/* Pin definitions for PTP PPS out */ +#define PPS_CLK_GEN_CHAN 3 +#define PPS_CLK_SRC_CHAN 2 +#define PPS_PIN_INDEX 5 +#define TIME_SYNC_PIN_INDEX 4 +#define E810_N_EXT_TS 3 +#define E810_N_PER_OUT 4 + +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) +struct ice_pf; +int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr); +int ice_ptp_get_ts_config(struct ice_pf *pf, struct ifreq *ifr); +int ice_get_ptp_clock_index(struct ice_pf *pf); + +s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb); +void ice_ptp_process_ts(struct ice_pf *pf); + +void +ice_ptp_rx_hwtstamp(struct ice_ring *rx_ring, + union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb); +void ice_ptp_init(struct ice_pf *pf); +void ice_ptp_release(struct ice_pf *pf); +#else /* IS_ENABLED(CONFIG_PTP_1588_CLOCK) */ +static inline int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr) +{ + return -EOPNOTSUPP; +} + +static inline int ice_ptp_get_ts_config(struct ice_pf *pf, struct ifreq *ifr) +{ + return -EOPNOTSUPP; +} + +static inline int ice_get_ptp_clock_index(struct ice_pf *pf) +{ + return -1; +} + +static inline s8 +ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) +{ + return -1; +} + +static inline void ice_ptp_process_ts(struct ice_pf *pf) { } +static inline void +ice_ptp_rx_hwtstamp(struct ice_ring *rx_ring, + union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) { } +static inline void ice_ptp_init(struct ice_pf *pf) { } +static inline void ice_ptp_release(struct ice_pf *pf) { } +#endif /* IS_ENABLED(CONFIG_PTP_1588_CLOCK) */ +#endif /* _ICE_PTP_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c new file mode 100644 index 000000000000..3eca0e4eab0b --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -0,0 +1,651 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021, Intel Corporation. */ + +#include "ice_common.h" +#include "ice_ptp_hw.h" + +/* Low level functions for interacting with and managing the device clock used + * for the Precision Time Protocol. + * + * The ice hardware represents the current time using three registers: + * + * GLTSYN_TIME_H GLTSYN_TIME_L GLTSYN_TIME_R + * +---------------+ +---------------+ +---------------+ + * | 32 bits | | 32 bits | | 32 bits | + * +---------------+ +---------------+ +---------------+ + * + * The registers are incremented every clock tick using a 40bit increment + * value defined over two registers: + * + * GLTSYN_INCVAL_H GLTSYN_INCVAL_L + * +---------------+ +---------------+ + * | 8 bit s | | 32 bits | + * +---------------+ +---------------+ + * + * The increment value is added to the GLSTYN_TIME_R and GLSTYN_TIME_L + * registers every clock source tick. Depending on the specific device + * configuration, the clock source frequency could be one of a number of + * values. + * + * For E810 devices, the increment frequency is 812.5 MHz + * + * The hardware captures timestamps in the PHY for incoming packets, and for + * outgoing packets on request. To support this, the PHY maintains a timer + * that matches the lower 64 bits of the global source timer. + * + * In order to ensure that the PHY timers and the source timer are equivalent, + * shadow registers are used to prepare the desired initial values. A special + * sync command is issued to trigger copying from the shadow registers into + * the appropriate source and PHY registers simultaneously. + */ + +/** + * ice_get_ptp_src_clock_index - determine source clock index + * @hw: pointer to HW struct + * + * Determine the source clock index currently in use, based on device + * capabilities reported during initialization. + */ +u8 ice_get_ptp_src_clock_index(struct ice_hw *hw) +{ + return hw->func_caps.ts_func_info.tmr_index_assoc; +} + +/* E810 functions + * + * The following functions operate on the E810 series devices which use + * a separate external PHY. + */ + +/** + * ice_read_phy_reg_e810 - Read register from external PHY on E810 + * @hw: pointer to the HW struct + * @addr: the address to read from + * @val: On return, the value read from the PHY + * + * Read a register from the external PHY on the E810 device. + */ +static int ice_read_phy_reg_e810(struct ice_hw *hw, u32 addr, u32 *val) +{ + struct ice_sbq_msg_input msg = {0}; + int status; + + msg.msg_addr_low = lower_16_bits(addr); + msg.msg_addr_high = upper_16_bits(addr); + msg.opcode = ice_sbq_msg_rd; + msg.dest_dev = rmn_0; + + status = ice_sbq_rw_reg(hw, &msg); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to send message to PHY, status %d\n", + status); + return status; + } + + *val = msg.data; + + return 0; +} + +/** + * ice_write_phy_reg_e810 - Write register on external PHY on E810 + * @hw: pointer to the HW struct + * @addr: the address to writem to + * @val: the value to write to the PHY + * + * Write a value to a register of the external PHY on the E810 device. + */ +static int ice_write_phy_reg_e810(struct ice_hw *hw, u32 addr, u32 val) +{ + struct ice_sbq_msg_input msg = {0}; + int status; + + msg.msg_addr_low = lower_16_bits(addr); + msg.msg_addr_high = upper_16_bits(addr); + msg.opcode = ice_sbq_msg_wr; + msg.dest_dev = rmn_0; + msg.data = val; + + status = ice_sbq_rw_reg(hw, &msg); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to send message to PHY, status %d\n", + status); + return status; + } + + return 0; +} + +/** + * ice_read_phy_tstamp_e810 - Read a PHY timestamp out of the external PHY + * @hw: pointer to the HW struct + * @lport: the lport to read from + * @idx: the timestamp index to read + * @tstamp: on return, the 40bit timestamp value + * + * Read a 40bit timestamp value out of the timestamp block of the external PHY + * on the E810 device. + */ +static int +ice_read_phy_tstamp_e810(struct ice_hw *hw, u8 lport, u8 idx, u64 *tstamp) +{ + u32 lo_addr, hi_addr, lo, hi; + int status; + + lo_addr = TS_EXT(LOW_TX_MEMORY_BANK_START, lport, idx); + hi_addr = TS_EXT(HIGH_TX_MEMORY_BANK_START, lport, idx); + + status = ice_read_phy_reg_e810(hw, lo_addr, &lo); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read low PTP timestamp register, status %d\n", + status); + return status; + } + + status = ice_read_phy_reg_e810(hw, hi_addr, &hi); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read high PTP timestamp register, status %d\n", + status); + return status; + } + + /* For E810 devices, the timestamp is reported with the lower 32 bits + * in the low register, and the upper 8 bits in the high register. + */ + *tstamp = ((u64)hi) << TS_HIGH_S | ((u64)lo & TS_LOW_M); + + return 0; +} + +/** + * ice_clear_phy_tstamp_e810 - Clear a timestamp from the external PHY + * @hw: pointer to the HW struct + * @lport: the lport to read from + * @idx: the timestamp index to reset + * + * Clear a timestamp, resetting its valid bit, from the timestamp block of the + * external PHY on the E810 device. + */ +static int ice_clear_phy_tstamp_e810(struct ice_hw *hw, u8 lport, u8 idx) +{ + u32 lo_addr, hi_addr; + int status; + + lo_addr = TS_EXT(LOW_TX_MEMORY_BANK_START, lport, idx); + hi_addr = TS_EXT(HIGH_TX_MEMORY_BANK_START, lport, idx); + + status = ice_write_phy_reg_e810(hw, lo_addr, 0); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to clear low PTP timestamp register, status %d\n", + status); + return status; + } + + status = ice_write_phy_reg_e810(hw, hi_addr, 0); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to clear high PTP timestamp register, status %d\n", + status); + return status; + } + + return 0; +} + +/** + * ice_ptp_init_phy_e810 - Enable PTP function on the external PHY + * @hw: pointer to HW struct + * + * Enable the timesync PTP functionality for the external PHY connected to + * this function. + */ +int ice_ptp_init_phy_e810(struct ice_hw *hw) +{ + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_ENA(tmr_idx), + GLTSYN_ENA_TSYN_ENA_M); + if (status) + ice_debug(hw, ICE_DBG_PTP, "PTP failed in ena_phy_time_syn %d\n", + status); + + return status; +} + +/** + * ice_ptp_prep_phy_time_e810 - Prepare PHY port with initial time + * @hw: Board private structure + * @time: Time to initialize the PHY port clock to + * + * Program the PHY port ETH_GLTSYN_SHTIME registers in preparation setting the + * initial clock time. The time will not actually be programmed until the + * driver issues an INIT_TIME command. + * + * The time value is the upper 32 bits of the PHY timer, usually in units of + * nominal nanoseconds. + */ +static int ice_ptp_prep_phy_time_e810(struct ice_hw *hw, u32 time) +{ + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_SHTIME_0(tmr_idx), 0); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write SHTIME_0, status %d\n", + status); + return status; + } + + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_SHTIME_L(tmr_idx), time); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write SHTIME_L, status %d\n", + status); + return status; + } + + return 0; +} + +/** + * ice_ptp_prep_phy_adj_e810 - Prep PHY port for a time adjustment + * @hw: pointer to HW struct + * @adj: adjustment value to program + * + * Prepare the PHY port for an atomic adjustment by programming the PHY + * ETH_GLTSYN_SHADJ_L and ETH_GLTSYN_SHADJ_H registers. The actual adjustment + * is completed by issuing an ADJ_TIME sync command. + * + * The adjustment value only contains the portion used for the upper 32bits of + * the PHY timer, usually in units of nominal nanoseconds. Negative + * adjustments are supported using 2s complement arithmetic. + */ +static int ice_ptp_prep_phy_adj_e810(struct ice_hw *hw, s32 adj) +{ + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + /* Adjustments are represented as signed 2's complement values in + * nanoseconds. Sub-nanosecond adjustment is not supported. + */ + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_SHADJ_L(tmr_idx), 0); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write adj to PHY SHADJ_L, status %d\n", + status); + return status; + } + + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_SHADJ_H(tmr_idx), adj); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write adj to PHY SHADJ_H, status %d\n", + status); + return status; + } + + return 0; +} + +/** + * ice_ptp_prep_phy_incval_e810 - Prep PHY port increment value change + * @hw: pointer to HW struct + * @incval: The new 40bit increment value to prepare + * + * Prepare the PHY port for a new increment value by programming the PHY + * ETH_GLTSYN_SHADJ_L and ETH_GLTSYN_SHADJ_H registers. The actual change is + * completed by issuing an INIT_INCVAL command. + */ +static int ice_ptp_prep_phy_incval_e810(struct ice_hw *hw, u64 incval) +{ + u32 high, low; + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + low = lower_32_bits(incval); + high = upper_32_bits(incval); + + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_SHADJ_L(tmr_idx), low); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write incval to PHY SHADJ_L, status %d\n", + status); + return status; + } + + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_SHADJ_H(tmr_idx), high); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write incval PHY SHADJ_H, status %d\n", + status); + return status; + } + + return 0; +} + +/** + * ice_ptp_port_cmd_e810 - Prepare all external PHYs for a timer command + * @hw: pointer to HW struct + * @cmd: Command to be sent to the port + * + * Prepare the external PHYs connected to this device for a timer sync + * command. + */ +static int ice_ptp_port_cmd_e810(struct ice_hw *hw, enum ice_ptp_tmr_cmd cmd) +{ + u32 cmd_val, val; + int status; + + switch (cmd) { + case INIT_TIME: + cmd_val = GLTSYN_CMD_INIT_TIME; + break; + case INIT_INCVAL: + cmd_val = GLTSYN_CMD_INIT_INCVAL; + break; + case ADJ_TIME: + cmd_val = GLTSYN_CMD_ADJ_TIME; + break; + case READ_TIME: + cmd_val = GLTSYN_CMD_READ_TIME; + break; + case ADJ_TIME_AT_TIME: + cmd_val = GLTSYN_CMD_ADJ_INIT_TIME; + break; + } + + /* Read, modify, write */ + status = ice_read_phy_reg_e810(hw, ETH_GLTSYN_CMD, &val); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read GLTSYN_CMD, status %d\n", status); + return status; + } + + /* Modify necessary bits only and perform write */ + val &= ~TS_CMD_MASK_E810; + val |= cmd_val; + + status = ice_write_phy_reg_e810(hw, ETH_GLTSYN_CMD, val); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write back GLTSYN_CMD, status %d\n", status); + return status; + } + + return 0; +} + +/* Device agnostic functions + * + * The following functions implement useful behavior to hide the differences + * between E810 and other devices. They call the device-specific + * implementations where necessary. + * + * Currently, the driver only supports E810, but future work will enable + * support for E822-based devices. + */ + +/** + * ice_ptp_lock - Acquire PTP global semaphore register lock + * @hw: pointer to the HW struct + * + * Acquire the global PTP hardware semaphore lock. Returns true if the lock + * was acquired, false otherwise. + * + * The PFTSYN_SEM register sets the busy bit on read, returning the previous + * value. If software sees the busy bit cleared, this means that this function + * acquired the lock (and the busy bit is now set). If software sees the busy + * bit set, it means that another function acquired the lock. + * + * Software must clear the busy bit with a write to release the lock for other + * functions when done. + */ +bool ice_ptp_lock(struct ice_hw *hw) +{ + u32 hw_lock; + int i; + +#define MAX_TRIES 5 + + for (i = 0; i < MAX_TRIES; i++) { + hw_lock = rd32(hw, PFTSYN_SEM + (PFTSYN_SEM_BYTES * hw->pf_id)); + hw_lock = hw_lock & PFTSYN_SEM_BUSY_M; + if (!hw_lock) + break; + + /* Somebody is holding the lock */ + usleep_range(10000, 20000); + } + + return !hw_lock; +} + +/** + * ice_ptp_unlock - Release PTP global semaphore register lock + * @hw: pointer to the HW struct + * + * Release the global PTP hardware semaphore lock. This is done by writing to + * the PFTSYN_SEM register. + */ +void ice_ptp_unlock(struct ice_hw *hw) +{ + wr32(hw, PFTSYN_SEM + (PFTSYN_SEM_BYTES * hw->pf_id), 0); +} + +/** + * ice_ptp_src_cmd - Prepare source timer for a timer command + * @hw: pointer to HW structure + * @cmd: Timer command + * + * Prepare the source timer for an upcoming timer sync command. + */ +static void ice_ptp_src_cmd(struct ice_hw *hw, enum ice_ptp_tmr_cmd cmd) +{ + u32 cmd_val; + u8 tmr_idx; + + tmr_idx = ice_get_ptp_src_clock_index(hw); + cmd_val = tmr_idx << SEL_CPK_SRC; + + switch (cmd) { + case INIT_TIME: + cmd_val |= GLTSYN_CMD_INIT_TIME; + break; + case INIT_INCVAL: + cmd_val |= GLTSYN_CMD_INIT_INCVAL; + break; + case ADJ_TIME: + cmd_val |= GLTSYN_CMD_ADJ_TIME; + break; + case ADJ_TIME_AT_TIME: + cmd_val |= GLTSYN_CMD_ADJ_INIT_TIME; + break; + case READ_TIME: + cmd_val |= GLTSYN_CMD_READ_TIME; + break; + } + + wr32(hw, GLTSYN_CMD, cmd_val); +} + +/** + * ice_ptp_tmr_cmd - Prepare and trigger a timer sync command + * @hw: pointer to HW struct + * @cmd: the command to issue + * + * Prepare the source timer and PHY timers and then trigger the requested + * command. This causes the shadow registers previously written in preparation + * for the command to be synchronously applied to both the source and PHY + * timers. + */ +static int ice_ptp_tmr_cmd(struct ice_hw *hw, enum ice_ptp_tmr_cmd cmd) +{ + int status; + + /* First, prepare the source timer */ + ice_ptp_src_cmd(hw, cmd); + + /* Next, prepare the ports */ + status = ice_ptp_port_cmd_e810(hw, cmd); + if (status) { + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY ports for timer command %u, status %d\n", + cmd, status); + return status; + } + + /* Write the sync command register to drive both source and PHY timer commands + * synchronously + */ + wr32(hw, GLTSYN_CMD_SYNC, SYNC_EXEC_CMD); + + return 0; +} + +/** + * ice_ptp_init_time - Initialize device time to provided value + * @hw: pointer to HW struct + * @time: 64bits of time (GLTSYN_TIME_L and GLTSYN_TIME_H) + * + * Initialize the device to the specified time provided. This requires a three + * step process: + * + * 1) write the new init time to the source timer shadow registers + * 2) write the new init time to the PHY timer shadow registers + * 3) issue an init_time timer command to synchronously switch both the source + * and port timers to the new init time value at the next clock cycle. + */ +int ice_ptp_init_time(struct ice_hw *hw, u64 time) +{ + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + /* Source timers */ + wr32(hw, GLTSYN_SHTIME_L(tmr_idx), lower_32_bits(time)); + wr32(hw, GLTSYN_SHTIME_H(tmr_idx), upper_32_bits(time)); + wr32(hw, GLTSYN_SHTIME_0(tmr_idx), 0); + + /* PHY timers */ + /* Fill Rx and Tx ports and send msg to PHY */ + status = ice_ptp_prep_phy_time_e810(hw, time & 0xFFFFFFFF); + if (status) + return status; + + return ice_ptp_tmr_cmd(hw, INIT_TIME); +} + +/** + * ice_ptp_write_incval - Program PHC with new increment value + * @hw: pointer to HW struct + * @incval: Source timer increment value per clock cycle + * + * Program the PHC with a new increment value. This requires a three-step + * process: + * + * 1) Write the increment value to the source timer shadow registers + * 2) Write the increment value to the PHY timer shadow registers + * 3) Issue an INIT_INCVAL timer command to synchronously switch both the + * source and port timers to the new increment value at the next clock + * cycle. + */ +int ice_ptp_write_incval(struct ice_hw *hw, u64 incval) +{ + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + /* Shadow Adjust */ + wr32(hw, GLTSYN_SHADJ_L(tmr_idx), lower_32_bits(incval)); + wr32(hw, GLTSYN_SHADJ_H(tmr_idx), upper_32_bits(incval)); + + status = ice_ptp_prep_phy_incval_e810(hw, incval); + if (status) + return status; + + return ice_ptp_tmr_cmd(hw, INIT_INCVAL); +} + +/** + * ice_ptp_write_incval_locked - Program new incval while holding semaphore + * @hw: pointer to HW struct + * @incval: Source timer increment value per clock cycle + * + * Program a new PHC incval while holding the PTP semaphore. + */ +int ice_ptp_write_incval_locked(struct ice_hw *hw, u64 incval) +{ + int status; + + if (!ice_ptp_lock(hw)) + return -EBUSY; + + status = ice_ptp_write_incval(hw, incval); + + ice_ptp_unlock(hw); + + return status; +} + +/** + * ice_ptp_adj_clock - Adjust PHC clock time atomically + * @hw: pointer to HW struct + * @adj: Adjustment in nanoseconds + * + * Perform an atomic adjustment of the PHC time by the specified number of + * nanoseconds. This requires a three-step process: + * + * 1) Write the adjustment to the source timer shadow registers + * 2) Write the adjustment to the PHY timer shadow registers + * 3) Issue an ADJ_TIME timer command to synchronously apply the adjustment to + * both the source and port timers at the next clock cycle. + */ +int ice_ptp_adj_clock(struct ice_hw *hw, s32 adj) +{ + int status; + u8 tmr_idx; + + tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; + + /* Write the desired clock adjustment into the GLTSYN_SHADJ register. + * For an ADJ_TIME command, this set of registers represents the value + * to add to the clock time. It supports subtraction by interpreting + * the value as a 2's complement integer. + */ + wr32(hw, GLTSYN_SHADJ_L(tmr_idx), 0); + wr32(hw, GLTSYN_SHADJ_H(tmr_idx), adj); + + status = ice_ptp_prep_phy_adj_e810(hw, adj); + if (status) + return status; + + return ice_ptp_tmr_cmd(hw, ADJ_TIME); +} + +/** + * ice_read_phy_tstamp - Read a PHY timestamp from the timestamo block + * @hw: pointer to the HW struct + * @block: the block to read from + * @idx: the timestamp index to read + * @tstamp: on return, the 40bit timestamp value + * + * Read a 40bit timestamp value out of the timestamp block. + */ +int ice_read_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx, u64 *tstamp) +{ + return ice_read_phy_tstamp_e810(hw, block, idx, tstamp); +} + +/** + * ice_clear_phy_tstamp - Clear a timestamp from the timestamp block + * @hw: pointer to the HW struct + * @block: the block to read from + * @idx: the timestamp index to reset + * + * Clear a timestamp, resetting its valid bit, from the timestamp block. + */ +int ice_clear_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx) +{ + return ice_clear_phy_tstamp_e810(hw, block, idx); +} diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h new file mode 100644 index 000000000000..55a414e87018 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2021, Intel Corporation. */ + +#ifndef _ICE_PTP_HW_H_ +#define _ICE_PTP_HW_H_ + +enum ice_ptp_tmr_cmd { + INIT_TIME, + INIT_INCVAL, + ADJ_TIME, + ADJ_TIME_AT_TIME, + READ_TIME +}; + +/* Increment value to generate nanoseconds in the GLTSYN_TIME_L register for + * the E810 devices. Based off of a PLL with an 812.5 MHz frequency. + */ +#define ICE_PTP_NOMINAL_INCVAL_E810 0x13b13b13bULL + +/* Device agnostic functions */ +u8 ice_get_ptp_src_clock_index(struct ice_hw *hw); +bool ice_ptp_lock(struct ice_hw *hw); +void ice_ptp_unlock(struct ice_hw *hw); +int ice_ptp_init_time(struct ice_hw *hw, u64 time); +int ice_ptp_write_incval(struct ice_hw *hw, u64 incval); +int ice_ptp_write_incval_locked(struct ice_hw *hw, u64 incval); +int ice_ptp_adj_clock(struct ice_hw *hw, s32 adj); +int ice_read_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx, u64 *tstamp); +int ice_clear_phy_tstamp(struct ice_hw *hw, u8 block, u8 idx); + +/* E810 family functions */ +int ice_ptp_init_phy_e810(struct ice_hw *hw); + +#define PFTSYN_SEM_BYTES 4 + +/* PHY timer commands */ +#define SEL_CPK_SRC 8 + +/* Time Sync command Definitions */ +#define GLTSYN_CMD_INIT_TIME BIT(0) +#define GLTSYN_CMD_INIT_INCVAL BIT(1) +#define GLTSYN_CMD_ADJ_TIME BIT(2) +#define GLTSYN_CMD_ADJ_INIT_TIME (BIT(2) | BIT(3)) +#define GLTSYN_CMD_READ_TIME BIT(7) + +#define TS_CMD_MASK_E810 0xFF +#define SYNC_EXEC_CMD 0x3 + +/* E810 timesync enable register */ +#define ETH_GLTSYN_ENA(_i) (0x03000348 + ((_i) * 4)) + +/* E810 shadow init time registers */ +#define ETH_GLTSYN_SHTIME_0(i) (0x03000368 + ((i) * 32)) +#define ETH_GLTSYN_SHTIME_L(i) (0x0300036C + ((i) * 32)) + +/* E810 shadow time adjust registers */ +#define ETH_GLTSYN_SHADJ_L(_i) (0x03000378 + ((_i) * 32)) +#define ETH_GLTSYN_SHADJ_H(_i) (0x0300037C + ((_i) * 32)) + +/* E810 timer command register */ +#define ETH_GLTSYN_CMD 0x03000344 + +/* Source timer incval macros */ +#define INCVAL_HIGH_M 0xFF + +/* Timestamp block macros */ +#define TS_LOW_M 0xFFFFFFFF +#define TS_HIGH_S 32 + +#define BYTES_PER_IDX_ADDR_L_U 8 + +/* External PHY timestamp address */ +#define TS_EXT(a, port, idx) ((a) + (0x1000 * (port)) + \ + ((idx) * BYTES_PER_IDX_ADDR_L_U)) + +#define LOW_TX_MEMORY_BANK_START 0x03090000 +#define HIGH_TX_MEMORY_BANK_START 0x03090004 + +#endif /* _ICE_PTP_HW_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_sbq_cmd.h b/drivers/net/ethernet/intel/ice/ice_sbq_cmd.h new file mode 100644 index 000000000000..ead75fe2bcda --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_sbq_cmd.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2021, Intel Corporation. */ + +#ifndef _ICE_SBQ_CMD_H_ +#define _ICE_SBQ_CMD_H_ + +/* This header file defines the Sideband Queue commands, error codes and + * descriptor format. It is shared between Firmware and Software. + */ + +/* Sideband Queue command structure and opcodes */ +enum ice_sbq_opc { + /* Sideband Queue commands */ + ice_sbq_opc_neigh_dev_req = 0x0C00, + ice_sbq_opc_neigh_dev_ev = 0x0C01 +}; + +/* Sideband Queue descriptor. Indirect command + * and non posted + */ +struct ice_sbq_cmd_desc { + __le16 flags; + __le16 opcode; + __le16 datalen; + __le16 cmd_retval; + + /* Opaque message data */ + __le32 cookie_high; + __le32 cookie_low; + + union { + __le16 cmd_len; + __le16 cmpl_len; + } param0; + + u8 reserved[6]; + __le32 addr_high; + __le32 addr_low; +}; + +struct ice_sbq_evt_desc { + __le16 flags; + __le16 opcode; + __le16 datalen; + __le16 cmd_retval; + u8 data[24]; +}; + +enum ice_sbq_msg_dev { + rmn_0 = 0x02, + rmn_1 = 0x03, + rmn_2 = 0x04, + cgu = 0x06 +}; + +enum ice_sbq_msg_opcode { + ice_sbq_msg_rd = 0x00, + ice_sbq_msg_wr = 0x01 +}; + +#define ICE_SBQ_MSG_FLAGS 0x40 +#define ICE_SBQ_MSG_SBE_FBE 0x0F + +struct ice_sbq_msg_req { + u8 dest_dev; + u8 src_dev; + u8 opcode; + u8 flags; + u8 sbe_fbe; + u8 func_id; + __le16 msg_addr_low; + __le32 msg_addr_high; + __le32 data; +}; + +struct ice_sbq_msg_cmpl { + u8 dest_dev; + u8 src_dev; + u8 opcode; + u8 flags; + __le32 data; +}; + +/* Internal struct */ +struct ice_sbq_msg_input { + u8 dest_dev; + u8 opcode; + u16 msg_addr_low; + u32 msg_addr_high; + u32 data; +}; +#endif /* _ICE_SBQ_CMD_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index 2f097637e405..9f07b6641705 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -596,6 +596,50 @@ ice_alloc_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs) } /** + * ice_alloc_rdma_q_ctx - allocate RDMA queue contexts for the given VSI and TC + * @hw: pointer to the HW struct + * @vsi_handle: VSI handle + * @tc: TC number + * @new_numqs: number of queues + */ +static enum ice_status +ice_alloc_rdma_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 new_numqs) +{ + struct ice_vsi_ctx *vsi_ctx; + struct ice_q_ctx *q_ctx; + + vsi_ctx = ice_get_vsi_ctx(hw, vsi_handle); + if (!vsi_ctx) + return ICE_ERR_PARAM; + /* allocate RDMA queue contexts */ + if (!vsi_ctx->rdma_q_ctx[tc]) { + vsi_ctx->rdma_q_ctx[tc] = devm_kcalloc(ice_hw_to_dev(hw), + new_numqs, + sizeof(*q_ctx), + GFP_KERNEL); + if (!vsi_ctx->rdma_q_ctx[tc]) + return ICE_ERR_NO_MEMORY; + vsi_ctx->num_rdma_q_entries[tc] = new_numqs; + return 0; + } + /* num queues are increased, update the queue contexts */ + if (new_numqs > vsi_ctx->num_rdma_q_entries[tc]) { + u16 prev_num = vsi_ctx->num_rdma_q_entries[tc]; + + q_ctx = devm_kcalloc(ice_hw_to_dev(hw), new_numqs, + sizeof(*q_ctx), GFP_KERNEL); + if (!q_ctx) + return ICE_ERR_NO_MEMORY; + memcpy(q_ctx, vsi_ctx->rdma_q_ctx[tc], + prev_num * sizeof(*q_ctx)); + devm_kfree(ice_hw_to_dev(hw), vsi_ctx->rdma_q_ctx[tc]); + vsi_ctx->rdma_q_ctx[tc] = q_ctx; + vsi_ctx->num_rdma_q_entries[tc] = new_numqs; + } + return 0; +} + +/** * ice_aq_rl_profile - performs a rate limiting task * @hw: pointer to the HW struct * @opcode: opcode for add, query, or remove profile(s) @@ -1774,13 +1818,22 @@ ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle, if (!vsi_ctx) return ICE_ERR_PARAM; - prev_numqs = vsi_ctx->sched.max_lanq[tc]; + if (owner == ICE_SCHED_NODE_OWNER_LAN) + prev_numqs = vsi_ctx->sched.max_lanq[tc]; + else + prev_numqs = vsi_ctx->sched.max_rdmaq[tc]; /* num queues are not changed or less than the previous number */ if (new_numqs <= prev_numqs) return status; - status = ice_alloc_lan_q_ctx(hw, vsi_handle, tc, new_numqs); - if (status) - return status; + if (owner == ICE_SCHED_NODE_OWNER_LAN) { + status = ice_alloc_lan_q_ctx(hw, vsi_handle, tc, new_numqs); + if (status) + return status; + } else { + status = ice_alloc_rdma_q_ctx(hw, vsi_handle, tc, new_numqs); + if (status) + return status; + } if (new_numqs) ice_sched_calc_vsi_child_nodes(hw, new_numqs, new_num_nodes); @@ -1795,7 +1848,10 @@ ice_sched_update_vsi_child_nodes(struct ice_port_info *pi, u16 vsi_handle, new_num_nodes, owner); if (status) return status; - vsi_ctx->sched.max_lanq[tc] = new_numqs; + if (owner == ICE_SCHED_NODE_OWNER_LAN) + vsi_ctx->sched.max_lanq[tc] = new_numqs; + else + vsi_ctx->sched.max_rdmaq[tc] = new_numqs; return 0; } @@ -1861,6 +1917,7 @@ ice_sched_cfg_vsi(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u16 maxqs, * recreate the child nodes all the time in these cases. */ vsi_ctx->sched.max_lanq[tc] = 0; + vsi_ctx->sched.max_rdmaq[tc] = 0; } /* update the VSI child nodes */ @@ -1990,6 +2047,8 @@ ice_sched_rm_vsi_cfg(struct ice_port_info *pi, u16 vsi_handle, u8 owner) } if (owner == ICE_SCHED_NODE_OWNER_LAN) vsi_ctx->sched.max_lanq[i] = 0; + else + vsi_ctx->sched.max_rdmaq[i] = 0; } status = 0; @@ -2686,8 +2745,8 @@ static enum ice_status ice_sched_assoc_vsi_to_agg(struct ice_port_info *pi, u32 agg_id, u16 vsi_handle, unsigned long *tc_bitmap) { - struct ice_sched_agg_vsi_info *agg_vsi_info; - struct ice_sched_agg_info *agg_info; + struct ice_sched_agg_vsi_info *agg_vsi_info, *old_agg_vsi_info = NULL; + struct ice_sched_agg_info *agg_info, *old_agg_info; enum ice_status status = 0; struct ice_hw *hw = pi->hw; u8 tc; @@ -2697,6 +2756,20 @@ ice_sched_assoc_vsi_to_agg(struct ice_port_info *pi, u32 agg_id, agg_info = ice_get_agg_info(hw, agg_id); if (!agg_info) return ICE_ERR_PARAM; + /* If the VSI is already part of another aggregator then update + * its VSI info list + */ + old_agg_info = ice_get_vsi_agg_info(hw, vsi_handle); + if (old_agg_info && old_agg_info != agg_info) { + struct ice_sched_agg_vsi_info *vtmp; + + list_for_each_entry_safe(old_agg_vsi_info, vtmp, + &old_agg_info->agg_vsi_list, + list_entry) + if (old_agg_vsi_info->vsi_handle == vsi_handle) + break; + } + /* check if entry already exist */ agg_vsi_info = ice_get_agg_vsi_info(agg_info, vsi_handle); if (!agg_vsi_info) { @@ -2721,6 +2794,12 @@ ice_sched_assoc_vsi_to_agg(struct ice_port_info *pi, u32 agg_id, break; set_bit(tc, agg_vsi_info->tc_bitmap); + if (old_agg_vsi_info) + clear_bit(tc, old_agg_vsi_info->tc_bitmap); + } + if (old_agg_vsi_info && !old_agg_vsi_info->tc_bitmap[0]) { + list_del(&old_agg_vsi_info->list_entry); + devm_kfree(ice_hw_to_dev(pi->hw), old_agg_vsi_info); } return status; } diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 357d3073d814..3b6c1420aa7b 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2018, Intel Corporation. */ +#include "ice_lib.h" #include "ice_switch.h" #define ICE_ETH_DA_OFFSET 0 @@ -302,6 +303,10 @@ static void ice_clear_vsi_q_ctx(struct ice_hw *hw, u16 vsi_handle) devm_kfree(ice_hw_to_dev(hw), vsi->lan_q_ctx[i]); vsi->lan_q_ctx[i] = NULL; } + if (vsi->rdma_q_ctx[i]) { + devm_kfree(ice_hw_to_dev(hw), vsi->rdma_q_ctx[i]); + vsi->rdma_q_ctx[i] = NULL; + } } } @@ -423,6 +428,29 @@ ice_update_vsi(struct ice_hw *hw, u16 vsi_handle, struct ice_vsi_ctx *vsi_ctx, } /** + * ice_cfg_rdma_fltr - enable/disable RDMA filtering on VSI + * @hw: pointer to HW struct + * @vsi_handle: VSI SW index + * @enable: boolean for enable/disable + */ +int +ice_cfg_rdma_fltr(struct ice_hw *hw, u16 vsi_handle, bool enable) +{ + struct ice_vsi_ctx *ctx; + + ctx = ice_get_vsi_ctx(hw, vsi_handle); + if (!ctx) + return -EIO; + + if (enable) + ctx->info.q_opt_flags |= ICE_AQ_VSI_Q_OPT_PE_FLTR_EN; + else + ctx->info.q_opt_flags &= ~ICE_AQ_VSI_Q_OPT_PE_FLTR_EN; + + return ice_status_to_errno(ice_update_vsi(hw, vsi_handle, ctx, NULL)); +} + +/** * ice_aq_alloc_free_vsi_list * @hw: pointer to the HW struct * @vsi_list_id: VSI list ID returned or used for lookup diff --git a/drivers/net/ethernet/intel/ice/ice_switch.h b/drivers/net/ethernet/intel/ice/ice_switch.h index 8b4f9d35c860..c5db8d56133f 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.h +++ b/drivers/net/ethernet/intel/ice/ice_switch.h @@ -26,6 +26,8 @@ struct ice_vsi_ctx { u8 vf_num; u16 num_lan_q_entries[ICE_MAX_TRAFFIC_CLASS]; struct ice_q_ctx *lan_q_ctx[ICE_MAX_TRAFFIC_CLASS]; + u16 num_rdma_q_entries[ICE_MAX_TRAFFIC_CLASS]; + struct ice_q_ctx *rdma_q_ctx[ICE_MAX_TRAFFIC_CLASS]; }; enum ice_sw_fwd_act_type { @@ -223,6 +225,8 @@ enum ice_status ice_add_eth_mac(struct ice_hw *hw, struct list_head *em_list); enum ice_status ice_remove_eth_mac(struct ice_hw *hw, struct list_head *em_list); +int +ice_cfg_rdma_fltr(struct ice_hw *hw, u16 vsi_handle, bool enable); void ice_remove_vsi_fltr(struct ice_hw *hw, u16 vsi_handle); enum ice_status ice_add_vlan(struct ice_hw *hw, struct list_head *m_list); @@ -243,7 +247,6 @@ ice_set_vlan_vsi_promisc(struct ice_hw *hw, u16 vsi_handle, u8 promisc_mask, enum ice_status ice_init_def_sw_recp(struct ice_hw *hw); u16 ice_get_hw_vsi_num(struct ice_hw *hw, u16 vsi_handle); -bool ice_is_vsi_valid(struct ice_hw *hw, u16 vsi_handle); enum ice_status ice_replay_vsi_all_fltr(struct ice_hw *hw, u16 vsi_handle); void ice_rm_all_sw_replay_rule_info(struct ice_hw *hw); diff --git a/drivers/net/ethernet/intel/ice/ice_trace.h b/drivers/net/ethernet/intel/ice/ice_trace.h new file mode 100644 index 000000000000..9bc0b8fdfc77 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_trace.h @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2021 Intel Corporation. */ + +/* Modeled on trace-events-sample.h */ + +/* The trace subsystem name for ice will be "ice". + * + * This file is named ice_trace.h. + * + * Since this include file's name is different from the trace + * subsystem name, we'll have to define TRACE_INCLUDE_FILE at the end + * of this file. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ice + +/* See trace-events-sample.h for a detailed description of why this + * guard clause is different from most normal include files. + */ +#if !defined(_ICE_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _ICE_TRACE_H_ + +#include <linux/tracepoint.h> + +/* ice_trace() macro enables shared code to refer to trace points + * like: + * + * trace_ice_example(args...) + * + * ... as: + * + * ice_trace(example, args...) + * + * ... to resolve to the PF version of the tracepoint without + * ifdefs, and to allow tracepoints to be disabled entirely at build + * time. + * + * Trace point should always be referred to in the driver via this + * macro. + * + * Similarly, ice_trace_enabled(trace_name) wraps references to + * trace_ice_<trace_name>_enabled() functions. + * @trace_name: name of tracepoint + */ +#define _ICE_TRACE_NAME(trace_name) (trace_##ice##_##trace_name) +#define ICE_TRACE_NAME(trace_name) _ICE_TRACE_NAME(trace_name) + +#define ice_trace(trace_name, args...) ICE_TRACE_NAME(trace_name)(args) + +#define ice_trace_enabled(trace_name) ICE_TRACE_NAME(trace_name##_enabled)() + +/* This is for events common to PF. Corresponding versions will be named + * trace_ice_*. The ice_trace() macro above will select the right trace point + * name for the driver. + */ + +/* Begin tracepoints */ + +/* Global tracepoints */ + +/* Events related to DIM, q_vectors and ring containers */ +DECLARE_EVENT_CLASS(ice_rx_dim_template, + TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), + TP_ARGS(q_vector, dim), + TP_STRUCT__entry(__field(struct ice_q_vector *, q_vector) + __field(struct dim *, dim) + __string(devname, q_vector->rx.ring->netdev->name)), + + TP_fast_assign(__entry->q_vector = q_vector; + __entry->dim = dim; + __assign_str(devname, q_vector->rx.ring->netdev->name);), + + TP_printk("netdev: %s Rx-Q: %d dim-state: %d dim-profile: %d dim-tune: %d dim-st-right: %d dim-st-left: %d dim-tired: %d", + __get_str(devname), + __entry->q_vector->rx.ring->q_index, + __entry->dim->state, + __entry->dim->profile_ix, + __entry->dim->tune_state, + __entry->dim->steps_right, + __entry->dim->steps_left, + __entry->dim->tired) +); + +DEFINE_EVENT(ice_rx_dim_template, ice_rx_dim_work, + TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), + TP_ARGS(q_vector, dim) +); + +DECLARE_EVENT_CLASS(ice_tx_dim_template, + TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), + TP_ARGS(q_vector, dim), + TP_STRUCT__entry(__field(struct ice_q_vector *, q_vector) + __field(struct dim *, dim) + __string(devname, q_vector->tx.ring->netdev->name)), + + TP_fast_assign(__entry->q_vector = q_vector; + __entry->dim = dim; + __assign_str(devname, q_vector->tx.ring->netdev->name);), + + TP_printk("netdev: %s Tx-Q: %d dim-state: %d dim-profile: %d dim-tune: %d dim-st-right: %d dim-st-left: %d dim-tired: %d", + __get_str(devname), + __entry->q_vector->tx.ring->q_index, + __entry->dim->state, + __entry->dim->profile_ix, + __entry->dim->tune_state, + __entry->dim->steps_right, + __entry->dim->steps_left, + __entry->dim->tired) +); + +DEFINE_EVENT(ice_tx_dim_template, ice_tx_dim_work, + TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), + TP_ARGS(q_vector, dim) +); + +/* Events related to a vsi & ring */ +DECLARE_EVENT_CLASS(ice_tx_template, + TP_PROTO(struct ice_ring *ring, struct ice_tx_desc *desc, + struct ice_tx_buf *buf), + + TP_ARGS(ring, desc, buf), + TP_STRUCT__entry(__field(void *, ring) + __field(void *, desc) + __field(void *, buf) + __string(devname, ring->netdev->name)), + + TP_fast_assign(__entry->ring = ring; + __entry->desc = desc; + __entry->buf = buf; + __assign_str(devname, ring->netdev->name);), + + TP_printk("netdev: %s ring: %pK desc: %pK buf %pK", __get_str(devname), + __entry->ring, __entry->desc, __entry->buf) +); + +#define DEFINE_TX_TEMPLATE_OP_EVENT(name) \ +DEFINE_EVENT(ice_tx_template, name, \ + TP_PROTO(struct ice_ring *ring, \ + struct ice_tx_desc *desc, \ + struct ice_tx_buf *buf), \ + TP_ARGS(ring, desc, buf)) + +DEFINE_TX_TEMPLATE_OP_EVENT(ice_clean_tx_irq); +DEFINE_TX_TEMPLATE_OP_EVENT(ice_clean_tx_irq_unmap); +DEFINE_TX_TEMPLATE_OP_EVENT(ice_clean_tx_irq_unmap_eop); + +DECLARE_EVENT_CLASS(ice_rx_template, + TP_PROTO(struct ice_ring *ring, union ice_32b_rx_flex_desc *desc), + + TP_ARGS(ring, desc), + + TP_STRUCT__entry(__field(void *, ring) + __field(void *, desc) + __string(devname, ring->netdev->name)), + + TP_fast_assign(__entry->ring = ring; + __entry->desc = desc; + __assign_str(devname, ring->netdev->name);), + + TP_printk("netdev: %s ring: %pK desc: %pK", __get_str(devname), + __entry->ring, __entry->desc) +); +DEFINE_EVENT(ice_rx_template, ice_clean_rx_irq, + TP_PROTO(struct ice_ring *ring, union ice_32b_rx_flex_desc *desc), + TP_ARGS(ring, desc) +); + +DECLARE_EVENT_CLASS(ice_rx_indicate_template, + TP_PROTO(struct ice_ring *ring, union ice_32b_rx_flex_desc *desc, + struct sk_buff *skb), + + TP_ARGS(ring, desc, skb), + + TP_STRUCT__entry(__field(void *, ring) + __field(void *, desc) + __field(void *, skb) + __string(devname, ring->netdev->name)), + + TP_fast_assign(__entry->ring = ring; + __entry->desc = desc; + __entry->skb = skb; + __assign_str(devname, ring->netdev->name);), + + TP_printk("netdev: %s ring: %pK desc: %pK skb %pK", __get_str(devname), + __entry->ring, __entry->desc, __entry->skb) +); + +DEFINE_EVENT(ice_rx_indicate_template, ice_clean_rx_irq_indicate, + TP_PROTO(struct ice_ring *ring, union ice_32b_rx_flex_desc *desc, + struct sk_buff *skb), + TP_ARGS(ring, desc, skb) +); + +DECLARE_EVENT_CLASS(ice_xmit_template, + TP_PROTO(struct ice_ring *ring, struct sk_buff *skb), + + TP_ARGS(ring, skb), + + TP_STRUCT__entry(__field(void *, ring) + __field(void *, skb) + __string(devname, ring->netdev->name)), + + TP_fast_assign(__entry->ring = ring; + __entry->skb = skb; + __assign_str(devname, ring->netdev->name);), + + TP_printk("netdev: %s skb: %pK ring: %pK", __get_str(devname), + __entry->skb, __entry->ring) +); + +#define DEFINE_XMIT_TEMPLATE_OP_EVENT(name) \ +DEFINE_EVENT(ice_xmit_template, name, \ + TP_PROTO(struct ice_ring *ring, struct sk_buff *skb), \ + TP_ARGS(ring, skb)) + +DEFINE_XMIT_TEMPLATE_OP_EVENT(ice_xmit_frame_ring); +DEFINE_XMIT_TEMPLATE_OP_EVENT(ice_xmit_frame_ring_drop); + +/* End tracepoints */ + +#endif /* _ICE_TRACE_H_ */ +/* This must be outside ifdef _ICE_TRACE_H */ + +/* This trace include file is not located in the .../include/trace + * with the kernel tracepoint definitions, because we're a loadable + * module. + */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE ../../drivers/net/ethernet/intel/ice/ice_trace +#include <trace/define_trace.h> diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index e2b4b29ea207..6ee8e0032d52 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -10,6 +10,7 @@ #include "ice_txrx_lib.h" #include "ice_lib.h" #include "ice.h" +#include "ice_trace.h" #include "ice_dcb_lib.h" #include "ice_xsk.h" @@ -224,6 +225,7 @@ static bool ice_clean_tx_irq(struct ice_ring *tx_ring, int napi_budget) smp_rmb(); /* prevent any other reads prior to eop_desc */ + ice_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf); /* if the descriptor isn't done, no work yet to do */ if (!(eop_desc->cmd_type_offset_bsz & cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) @@ -254,6 +256,7 @@ static bool ice_clean_tx_irq(struct ice_ring *tx_ring, int napi_budget) /* unmap remaining buffers */ while (tx_desc != eop_desc) { + ice_trace(clean_tx_irq_unmap, tx_ring, tx_desc, tx_buf); tx_buf++; tx_desc++; i++; @@ -272,6 +275,7 @@ static bool ice_clean_tx_irq(struct ice_ring *tx_ring, int napi_budget) dma_unmap_len_set(tx_buf, len, 0); } } + ice_trace(clean_tx_irq_unmap_eop, tx_ring, tx_desc, tx_buf); /* move us one more past the eop_desc for start of next pkt */ tx_buf++; @@ -523,7 +527,7 @@ ice_run_xdp(struct ice_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { struct ice_ring *xdp_ring; - int err; + int err, result; u32 act; act = bpf_prog_run_xdp(xdp_prog, xdp); @@ -532,14 +536,20 @@ ice_run_xdp(struct ice_ring *rx_ring, struct xdp_buff *xdp, return ICE_XDP_PASS; case XDP_TX: xdp_ring = rx_ring->vsi->xdp_rings[smp_processor_id()]; - return ice_xmit_xdp_buff(xdp, xdp_ring); + result = ice_xmit_xdp_buff(xdp, xdp_ring); + if (result == ICE_XDP_CONSUMED) + goto out_failure; + return result; case XDP_REDIRECT: err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - return !err ? ICE_XDP_REDIR : ICE_XDP_CONSUMED; + if (err) + goto out_failure; + return ICE_XDP_REDIR; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_DROP: @@ -1076,7 +1086,7 @@ int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) u16 stat_err_bits; int rx_buf_pgcnt; u16 vlan_tag = 0; - u8 rx_ptype; + u16 rx_ptype; /* get the Rx desc from Rx ring based on 'next_to_clean' */ rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); @@ -1096,6 +1106,7 @@ int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) */ dma_rmb(); + ice_trace(clean_rx_irq, rx_ring, rx_desc); if (rx_desc->wb.rxdid == FDIR_DESC_RXDID || !rx_ring->netdev) { struct ice_vsi *ctrl_vsi = rx_ring->vsi; @@ -1129,15 +1140,11 @@ int ice_clean_rx_irq(struct ice_ring *rx_ring, int budget) xdp.frame_sz = ice_rx_frame_truesize(rx_ring, size); #endif - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); - if (!xdp_prog) { - rcu_read_unlock(); + if (!xdp_prog) goto construct_skb; - } xdp_res = ice_run_xdp(rx_ring, &xdp, xdp_prog); - rcu_read_unlock(); if (!xdp_res) goto construct_skb; if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) { @@ -1201,6 +1208,7 @@ construct_skb: ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); + ice_trace(clean_rx_irq_indicate, rx_ring, rx_desc, skb); /* send completed skb up the stack */ ice_receive_skb(rx_ring, skb, vlan_tag); skb = NULL; @@ -2131,6 +2139,41 @@ static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count) } /** + * ice_tstamp - set up context descriptor for hardware timestamp + * @tx_ring: pointer to the Tx ring to send buffer on + * @skb: pointer to the SKB we're sending + * @first: Tx buffer + * @off: Tx offload parameters + */ +static void +ice_tstamp(struct ice_ring *tx_ring, struct sk_buff *skb, + struct ice_tx_buf *first, struct ice_tx_offload_params *off) +{ + s8 idx; + + /* only timestamp the outbound packet if the user has requested it */ + if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) + return; + + if (!tx_ring->ptp_tx) + return; + + /* Tx timestamps cannot be sampled when doing TSO */ + if (first->tx_flags & ICE_TX_FLAGS_TSO) + return; + + /* Grab an open timestamp slot */ + idx = ice_ptp_request_ts(tx_ring->tx_tstamps, skb); + if (idx < 0) + return; + + off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX | + (ICE_TX_CTX_DESC_TSYN << ICE_TXD_CTX_QW1_CMD_S) | + ((u64)idx << ICE_TXD_CTX_QW1_TSO_LEN_S)); + first->tx_flags |= ICE_TX_FLAGS_TSYN; +} + +/** * ice_xmit_frame_ring - Sends buffer on Tx ring * @skb: send buffer * @tx_ring: ring to send buffer on @@ -2143,9 +2186,12 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) struct ice_tx_offload_params offload = { 0 }; struct ice_vsi *vsi = tx_ring->vsi; struct ice_tx_buf *first; + struct ethhdr *eth; unsigned int count; int tso, csum; + ice_trace(xmit_frame_ring, tx_ring, skb); + count = ice_xmit_desc_count(skb); if (ice_chk_linearize(skb, count)) { if (__skb_linearize(skb)) @@ -2189,13 +2235,17 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) goto out_drop; /* allow CONTROL frames egress from main VSI if FW LLDP disabled */ - if (unlikely(skb->priority == TC_PRIO_CONTROL && + eth = (struct ethhdr *)skb_mac_header(skb); + if (unlikely((skb->priority == TC_PRIO_CONTROL || + eth->h_proto == htons(ETH_P_LLDP)) && vsi->type == ICE_VSI_PF && vsi->port_info->qos_cfg.is_sw_lldp)) offload.cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX | ICE_TX_CTX_DESC_SWTCH_UPLINK << ICE_TXD_CTX_QW1_CMD_S); + ice_tstamp(tx_ring, skb, first, &offload); + if (offload.cd_qw1 & ICE_TX_DESC_DTYPE_CTX) { struct ice_tx_ctx_desc *cdesc; u16 i = tx_ring->next_to_use; @@ -2216,6 +2266,7 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_ring *tx_ring) return NETDEV_TX_OK; out_drop: + ice_trace(xmit_frame_ring_drop, tx_ring, skb); dev_kfree_skb_any(skb); return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index c5a92ac787d6..1e46e80f3d6f 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -118,6 +118,7 @@ static inline int ice_skb_pad(void) * freed instead of returned like skb packets. */ #define ICE_TX_FLAGS_DUMMY_PKT BIT(3) +#define ICE_TX_FLAGS_TSYN BIT(4) #define ICE_TX_FLAGS_IPV4 BIT(5) #define ICE_TX_FLAGS_IPV6 BIT(6) #define ICE_TX_FLAGS_TUNNEL BIT(7) @@ -311,6 +312,10 @@ struct ice_ring { u32 txq_teid; /* Added Tx queue TEID */ u16 rx_buf_len; u8 dcb_tc; /* Traffic class of ring */ + struct ice_ptp_tx *tx_tstamps; + u64 cached_phctime; + u8 ptp_rx:1; + u8 ptp_tx:1; } ____cacheline_internodealigned_in_smp; static inline bool ice_ring_uses_build_skb(struct ice_ring *ring) diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c index 207f6ee3a7f6..171397dcf00a 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c @@ -38,10 +38,23 @@ void ice_release_rx_desc(struct ice_ring *rx_ring, u16 val) * ice_ptype_to_htype - get a hash type * @ptype: the ptype value from the descriptor * - * Returns a hash type to be used by skb_set_hash + * Returns appropriate hash type (such as PKT_HASH_TYPE_L2/L3/L4) to be used by + * skb_set_hash based on PTYPE as parsed by HW Rx pipeline and is part of + * Rx desc. */ -static enum pkt_hash_types ice_ptype_to_htype(u8 __always_unused ptype) +static enum pkt_hash_types ice_ptype_to_htype(u16 ptype) { + struct ice_rx_ptype_decoded decoded = ice_decode_rx_desc_ptype(ptype); + + if (!decoded.known) + return PKT_HASH_TYPE_NONE; + if (decoded.payload_layer == ICE_RX_PTYPE_PAYLOAD_LAYER_PAY4) + return PKT_HASH_TYPE_L4; + if (decoded.payload_layer == ICE_RX_PTYPE_PAYLOAD_LAYER_PAY3) + return PKT_HASH_TYPE_L3; + if (decoded.outer_ip == ICE_RX_PTYPE_OUTER_L2) + return PKT_HASH_TYPE_L2; + return PKT_HASH_TYPE_NONE; } @@ -54,7 +67,7 @@ static enum pkt_hash_types ice_ptype_to_htype(u8 __always_unused ptype) */ static void ice_rx_hash(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, - struct sk_buff *skb, u8 rx_ptype) + struct sk_buff *skb, u16 rx_ptype) { struct ice_32b_rx_flex_desc_nic *nic_mdid; u32 hash; @@ -81,7 +94,7 @@ ice_rx_hash(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, */ static void ice_rx_csum(struct ice_ring *ring, struct sk_buff *skb, - union ice_32b_rx_flex_desc *rx_desc, u8 ptype) + union ice_32b_rx_flex_desc *rx_desc, u16 ptype) { struct ice_rx_ptype_decoded decoded; u16 rx_status0, rx_status1; @@ -167,7 +180,7 @@ checksum_fail: void ice_process_skb_fields(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, - struct sk_buff *skb, u8 ptype) + struct sk_buff *skb, u16 ptype) { ice_rx_hash(rx_ring, rx_desc, skb, ptype); @@ -175,6 +188,9 @@ ice_process_skb_fields(struct ice_ring *rx_ring, skb->protocol = eth_type_trans(skb, rx_ring->netdev); ice_rx_csum(rx_ring, skb, rx_desc, ptype); + + if (rx_ring->ptp_rx) + ice_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); } /** diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index 58ff58f0f972..05ac30752902 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -53,7 +53,7 @@ void ice_release_rx_desc(struct ice_ring *rx_ring, u16 val); void ice_process_skb_fields(struct ice_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, - struct sk_buff *skb, u8 ptype); + struct sk_buff *skb, u16 ptype); void ice_receive_skb(struct ice_ring *rx_ring, struct sk_buff *skb, u16 vlan_tag); #endif /* !_ICE_TXRX_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 4474dd6a7ba1..d33d1906103c 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -14,6 +14,7 @@ #include "ice_lan_tx_rx.h" #include "ice_flex_type.h" #include "ice_protocol_type.h" +#include "ice_sbq_cmd.h" static inline bool ice_is_tc_ena(unsigned long bitmap, u8 tc) { @@ -45,8 +46,10 @@ static inline u32 ice_round_to_num(u32 N, u32 R) #define ICE_DBG_FLOW BIT_ULL(9) #define ICE_DBG_SW BIT_ULL(13) #define ICE_DBG_SCHED BIT_ULL(14) +#define ICE_DBG_RDMA BIT_ULL(15) #define ICE_DBG_PKG BIT_ULL(16) #define ICE_DBG_RES BIT_ULL(17) +#define ICE_DBG_PTP BIT_ULL(19) #define ICE_DBG_AQ_MSG BIT_ULL(24) #define ICE_DBG_AQ_DESC BIT_ULL(25) #define ICE_DBG_AQ_DESC_BUF BIT_ULL(26) @@ -63,7 +66,7 @@ enum ice_aq_res_ids { /* FW update timeout definitions are in milliseconds */ #define ICE_NVM_TIMEOUT 180000 #define ICE_CHANGE_LOCK_TIMEOUT 1000 -#define ICE_GLOBAL_CFG_LOCK_TIMEOUT 3000 +#define ICE_GLOBAL_CFG_LOCK_TIMEOUT 5000 enum ice_aq_res_access_type { ICE_RES_READ = 1, @@ -146,6 +149,7 @@ struct ice_link_status { u16 max_frame_size; u16 link_speed; u16 req_speeds; + u8 link_cfg_err; u8 lse_ena; /* Link Status Event notification */ u8 link_info; u8 an_info; @@ -262,6 +266,8 @@ struct ice_hw_common_caps { u8 rss_table_entry_width; /* RSS Entry width in bits */ u8 dcb; + u8 ieee_1588; + u8 rdma; bool nvm_update_pending_nvm; bool nvm_update_pending_orom; @@ -273,6 +279,54 @@ struct ice_hw_common_caps { #define ICE_NVM_MGMT_UNIFIED_UPD_SUPPORT BIT(3) }; +/* IEEE 1588 TIME_SYNC specific info */ +/* Function specific definitions */ +#define ICE_TS_FUNC_ENA_M BIT(0) +#define ICE_TS_SRC_TMR_OWND_M BIT(1) +#define ICE_TS_TMR_ENA_M BIT(2) +#define ICE_TS_TMR_IDX_OWND_S 4 +#define ICE_TS_TMR_IDX_OWND_M BIT(4) +#define ICE_TS_CLK_FREQ_S 16 +#define ICE_TS_CLK_FREQ_M ICE_M(0x7, ICE_TS_CLK_FREQ_S) +#define ICE_TS_CLK_SRC_S 20 +#define ICE_TS_CLK_SRC_M BIT(20) +#define ICE_TS_TMR_IDX_ASSOC_S 24 +#define ICE_TS_TMR_IDX_ASSOC_M BIT(24) + +struct ice_ts_func_info { + /* Function specific info */ + u32 clk_freq; + u8 clk_src; + u8 tmr_index_assoc; + u8 ena; + u8 tmr_index_owned; + u8 src_tmr_owned; + u8 tmr_ena; +}; + +/* Device specific definitions */ +#define ICE_TS_TMR0_OWNR_M 0x7 +#define ICE_TS_TMR0_OWND_M BIT(3) +#define ICE_TS_TMR1_OWNR_S 4 +#define ICE_TS_TMR1_OWNR_M ICE_M(0x7, ICE_TS_TMR1_OWNR_S) +#define ICE_TS_TMR1_OWND_M BIT(7) +#define ICE_TS_DEV_ENA_M BIT(24) +#define ICE_TS_TMR0_ENA_M BIT(25) +#define ICE_TS_TMR1_ENA_M BIT(26) + +struct ice_ts_dev_info { + /* Device specific info */ + u32 ena_ports; + u32 tmr_own_map; + u32 tmr0_owner; + u32 tmr1_owner; + u8 tmr0_owned; + u8 tmr1_owned; + u8 ena; + u8 tmr0_ena; + u8 tmr1_ena; +}; + /* Function specific capabilities */ struct ice_hw_func_caps { struct ice_hw_common_caps common_cap; @@ -281,6 +335,7 @@ struct ice_hw_func_caps { u32 guar_num_vsi; u32 fd_fltr_guar; /* Number of filters guaranteed */ u32 fd_fltr_best_effort; /* Number of best effort filters */ + struct ice_ts_func_info ts_func_info; }; /* Device wide capabilities */ @@ -289,6 +344,7 @@ struct ice_hw_dev_caps { u32 num_vfs_exposed; /* Total number of VFs exposed */ u32 num_vsi_allocd_to_host; /* Excluding EMP VSI */ u32 num_flow_director_fltr; /* Number of FD filters available */ + struct ice_ts_dev_info ts_dev_info; u32 num_funcs; }; @@ -440,6 +496,7 @@ struct ice_sched_node { u8 tc_num; u8 owner; #define ICE_SCHED_NODE_OWNER_LAN 0 +#define ICE_SCHED_NODE_OWNER_RDMA 2 }; /* Access Macros for Tx Sched Elements data */ @@ -511,6 +568,7 @@ struct ice_sched_vsi_info { struct ice_sched_node *ag_node[ICE_MAX_TRAFFIC_CLASS]; struct list_head list_entry; u16 max_lanq[ICE_MAX_TRAFFIC_CLASS]; + u16 max_rdmaq[ICE_MAX_TRAFFIC_CLASS]; }; /* driver defines the policy */ @@ -749,6 +807,7 @@ struct ice_hw { /* Control Queue info */ struct ice_ctl_q_info adminq; + struct ice_ctl_q_info sbq; struct ice_ctl_q_info mailboxq; u8 api_branch; /* API branch version */ @@ -784,6 +843,14 @@ struct ice_hw { u8 ucast_shared; /* true if VSIs can share unicast addr */ +#define ICE_PHY_PER_NAC 1 +#define ICE_MAX_QUAD 2 +#define ICE_NUM_QUAD_TYPE 2 +#define ICE_PORTS_PER_QUAD 4 +#define ICE_PHY_0_LAST_QUAD 1 +#define ICE_PORTS_PER_PHY 8 +#define ICE_NUM_EXTERNAL_PORTS ICE_PORTS_PER_PHY + /* Active package version (currently active) */ struct ice_pkg_ver active_pkg_ver; u32 active_track_id; diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c index a1d22d2aa0bd..2826570dab51 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c @@ -713,13 +713,15 @@ static void ice_trigger_vf_reset(struct ice_vf *vf, bool is_vflr, bool is_pfr) */ clear_bit(ICE_VF_STATE_INIT, vf->vf_states); - /* VF_MBX_ARQLEN is cleared by PFR, so the driver needs to clear it - * in the case of VFR. If this is done for PFR, it can mess up VF - * resets because the VF driver may already have started cleanup - * by the time we get here. + /* VF_MBX_ARQLEN and VF_MBX_ATQLEN are cleared by PFR, so the driver + * needs to clear them in the case of VFR/VFLR. If this is done for + * PFR, it can mess up VF resets because the VF driver may already + * have started cleanup by the time we get here. */ - if (!is_pfr) + if (!is_pfr) { wr32(hw, VF_MBX_ARQLEN(vf->vf_id), 0); + wr32(hw, VF_MBX_ATQLEN(vf->vf_id), 0); + } /* In the case of a VFLR, the HW has already reset the VF and we * just need to clean up, so don't hit the VFRTRIG register. @@ -937,16 +939,18 @@ static int ice_vf_rebuild_host_mac_cfg(struct ice_vf *vf) vf->num_mac++; - if (is_valid_ether_addr(vf->dflt_lan_addr.addr)) { - status = ice_fltr_add_mac(vsi, vf->dflt_lan_addr.addr, + if (is_valid_ether_addr(vf->hw_lan_addr.addr)) { + status = ice_fltr_add_mac(vsi, vf->hw_lan_addr.addr, ICE_FWD_TO_VSI); if (status) { dev_err(dev, "failed to add default unicast MAC filter %pM for VF %u, error %s\n", - &vf->dflt_lan_addr.addr[0], vf->vf_id, + &vf->hw_lan_addr.addr[0], vf->vf_id, ice_stat_str(status)); return ice_status_to_errno(status); } vf->num_mac++; + + ether_addr_copy(vf->dev_lan_addr.addr, vf->hw_lan_addr.addr); } return 0; @@ -1685,7 +1689,6 @@ bool ice_reset_vf(struct ice_vf *vf, bool is_vflr) else promisc_m = ICE_UCAST_PROMISC_BITS; - vsi = ice_get_vf_vsi(vf); if (ice_vf_set_vsi_promisc(vf, vsi, promisc_m, true)) dev_err(dev, "disabling promiscuous mode failed\n"); } @@ -1698,7 +1701,12 @@ bool ice_reset_vf(struct ice_vf *vf, bool is_vflr) ice_vf_ctrl_vsi_release(vf); ice_vf_pre_vsi_rebuild(vf); - ice_vf_rebuild_vsi_with_release(vf); + + if (ice_vf_rebuild_vsi_with_release(vf)) { + dev_err(dev, "Failed to release and setup the VF%u's VSI\n", vf->vf_id); + return false; + } + ice_vf_post_vsi_rebuild(vf); /* if the VF has been reset allow it to come up again */ @@ -2379,7 +2387,7 @@ static int ice_vc_get_vf_res_msg(struct ice_vf *vf, u8 *msg) vfres->vsi_res[0].vsi_type = VIRTCHNL_VSI_SRIOV; vfres->vsi_res[0].num_queue_pairs = vsi->num_txq; ether_addr_copy(vfres->vsi_res[0].default_mac_addr, - vf->dflt_lan_addr.addr); + vf->hw_lan_addr.addr); /* match guest capabilities */ vf->driver_caps = vfres->vf_cap_flags; @@ -3535,10 +3543,9 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) struct virtchnl_vsi_queue_config_info *qci = (struct virtchnl_vsi_queue_config_info *)msg; struct virtchnl_queue_pair_info *qpi; - u16 num_rxq = 0, num_txq = 0; struct ice_pf *pf = vf->pf; struct ice_vsi *vsi; - int i; + int i, q_idx; if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { v_ret = VIRTCHNL_STATUS_ERR_PARAM; @@ -3576,18 +3583,31 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) v_ret = VIRTCHNL_STATUS_ERR_PARAM; goto error_param; } + + q_idx = qpi->rxq.queue_id; + + /* make sure selected "q_idx" is in valid range of queues + * for selected "vsi" + */ + if (q_idx >= vsi->alloc_txq || q_idx >= vsi->alloc_rxq) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + /* copy Tx queue info from VF into VSI */ if (qpi->txq.ring_len > 0) { - num_txq++; vsi->tx_rings[i]->dma = qpi->txq.dma_ring_addr; vsi->tx_rings[i]->count = qpi->txq.ring_len; + if (ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } } /* copy Rx queue info from VF into VSI */ if (qpi->rxq.ring_len > 0) { u16 max_frame_size = ice_vc_get_max_frame_size(vf); - num_rxq++; vsi->rx_rings[i]->dma = qpi->rxq.dma_ring_addr; vsi->rx_rings[i]->count = qpi->rxq.ring_len; @@ -3604,27 +3624,20 @@ static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) v_ret = VIRTCHNL_STATUS_ERR_PARAM; goto error_param; } - } - vsi->max_frame = qpi->rxq.max_pkt_size; - /* add space for the port VLAN since the VF driver is not - * expected to account for it in the MTU calculation - */ - if (vf->port_vlan_info) - vsi->max_frame += VLAN_HLEN; - } - - /* VF can request to configure less than allocated queues or default - * allocated queues. So update the VSI with new number - */ - vsi->num_txq = num_txq; - vsi->num_rxq = num_rxq; - /* All queues of VF VSI are in TC 0 */ - vsi->tc_cfg.tc_info[0].qcount_tx = num_txq; - vsi->tc_cfg.tc_info[0].qcount_rx = num_rxq; + vsi->max_frame = qpi->rxq.max_pkt_size; + /* add space for the port VLAN since the VF driver is not + * expected to account for it in the MTU calculation + */ + if (vf->port_vlan_info) + vsi->max_frame += VLAN_HLEN; - if (ice_vsi_cfg_lan_txqs(vsi) || ice_vsi_cfg_rxqs(vsi)) - v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; + if (ice_vsi_cfg_single_rxq(vsi, q_idx)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + } + } error_param: /* send the response to the VF */ @@ -3660,19 +3673,95 @@ static bool ice_can_vf_change_mac(struct ice_vf *vf) } /** + * ice_vc_ether_addr_type - get type of virtchnl_ether_addr + * @vc_ether_addr: used to extract the type + */ +static u8 +ice_vc_ether_addr_type(struct virtchnl_ether_addr *vc_ether_addr) +{ + return (vc_ether_addr->type & VIRTCHNL_ETHER_ADDR_TYPE_MASK); +} + +/** + * ice_is_vc_addr_legacy - check if the MAC address is from an older VF + * @vc_ether_addr: VIRTCHNL structure that contains MAC and type + */ +static bool +ice_is_vc_addr_legacy(struct virtchnl_ether_addr *vc_ether_addr) +{ + u8 type = ice_vc_ether_addr_type(vc_ether_addr); + + return (type == VIRTCHNL_ETHER_ADDR_LEGACY); +} + +/** + * ice_is_vc_addr_primary - check if the MAC address is the VF's primary MAC + * @vc_ether_addr: VIRTCHNL structure that contains MAC and type + * + * This function should only be called when the MAC address in + * virtchnl_ether_addr is a valid unicast MAC + */ +static bool +ice_is_vc_addr_primary(struct virtchnl_ether_addr __maybe_unused *vc_ether_addr) +{ + u8 type = ice_vc_ether_addr_type(vc_ether_addr); + + return (type == VIRTCHNL_ETHER_ADDR_PRIMARY); +} + +/** + * ice_vfhw_mac_add - update the VF's cached hardware MAC if allowed + * @vf: VF to update + * @vc_ether_addr: structure from VIRTCHNL with MAC to add + */ +static void +ice_vfhw_mac_add(struct ice_vf *vf, struct virtchnl_ether_addr *vc_ether_addr) +{ + u8 *mac_addr = vc_ether_addr->addr; + + if (!is_valid_ether_addr(mac_addr)) + return; + + /* only allow legacy VF drivers to set the device and hardware MAC if it + * is zero and allow new VF drivers to set the hardware MAC if the type + * was correctly specified over VIRTCHNL + */ + if ((ice_is_vc_addr_legacy(vc_ether_addr) && + is_zero_ether_addr(vf->hw_lan_addr.addr)) || + ice_is_vc_addr_primary(vc_ether_addr)) { + ether_addr_copy(vf->dev_lan_addr.addr, mac_addr); + ether_addr_copy(vf->hw_lan_addr.addr, mac_addr); + } + + /* hardware and device MACs are already set, but its possible that the + * VF driver sent the VIRTCHNL_OP_ADD_ETH_ADDR message before the + * VIRTCHNL_OP_DEL_ETH_ADDR when trying to update its MAC, so save it + * away for the legacy VF driver case as it will be updated in the + * delete flow for this case + */ + if (ice_is_vc_addr_legacy(vc_ether_addr)) { + ether_addr_copy(vf->legacy_last_added_umac.addr, + mac_addr); + vf->legacy_last_added_umac.time_modified = jiffies; + } +} + +/** * ice_vc_add_mac_addr - attempt to add the MAC address passed in * @vf: pointer to the VF info * @vsi: pointer to the VF's VSI - * @mac_addr: MAC address to add + * @vc_ether_addr: VIRTCHNL MAC address structure used to add MAC */ static int -ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) +ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, + struct virtchnl_ether_addr *vc_ether_addr) { struct device *dev = ice_pf_to_dev(vf->pf); + u8 *mac_addr = vc_ether_addr->addr; enum ice_status status; - /* default unicast MAC already added */ - if (ether_addr_equal(mac_addr, vf->dflt_lan_addr.addr)) + /* device MAC already added */ + if (ether_addr_equal(mac_addr, vf->dev_lan_addr.addr)) return 0; if (is_unicast_ether_addr(mac_addr) && !ice_can_vf_change_mac(vf)) { @@ -3691,12 +3780,7 @@ ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) return -EIO; } - /* Set the default LAN address to the latest unicast MAC address added - * by the VF. The default LAN address is reported by the PF via - * ndo_get_vf_config. - */ - if (is_unicast_ether_addr(mac_addr)) - ether_addr_copy(vf->dflt_lan_addr.addr, mac_addr); + ice_vfhw_mac_add(vf, vc_ether_addr); vf->num_mac++; @@ -3704,19 +3788,65 @@ ice_vc_add_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) } /** + * ice_is_legacy_umac_expired - check if last added legacy unicast MAC expired + * @last_added_umac: structure used to check expiration + */ +static bool ice_is_legacy_umac_expired(struct ice_time_mac *last_added_umac) +{ +#define ICE_LEGACY_VF_MAC_CHANGE_EXPIRE_TIME msecs_to_jiffies(3000) + return time_is_before_jiffies(last_added_umac->time_modified + + ICE_LEGACY_VF_MAC_CHANGE_EXPIRE_TIME); +} + +/** + * ice_vfhw_mac_del - update the VF's cached hardware MAC if allowed + * @vf: VF to update + * @vc_ether_addr: structure from VIRTCHNL with MAC to delete + */ +static void +ice_vfhw_mac_del(struct ice_vf *vf, struct virtchnl_ether_addr *vc_ether_addr) +{ + u8 *mac_addr = vc_ether_addr->addr; + + if (!is_valid_ether_addr(mac_addr) || + !ether_addr_equal(vf->dev_lan_addr.addr, mac_addr)) + return; + + /* allow the device MAC to be repopulated in the add flow and don't + * clear the hardware MAC (i.e. hw_lan_addr.addr) here as that is meant + * to be persistent on VM reboot and across driver unload/load, which + * won't work if we clear the hardware MAC here + */ + eth_zero_addr(vf->dev_lan_addr.addr); + + /* only update cached hardware MAC for legacy VF drivers on delete + * because we cannot guarantee order/type of MAC from the VF driver + */ + if (ice_is_vc_addr_legacy(vc_ether_addr) && + !ice_is_legacy_umac_expired(&vf->legacy_last_added_umac)) { + ether_addr_copy(vf->dev_lan_addr.addr, + vf->legacy_last_added_umac.addr); + ether_addr_copy(vf->hw_lan_addr.addr, + vf->legacy_last_added_umac.addr); + } +} + +/** * ice_vc_del_mac_addr - attempt to delete the MAC address passed in * @vf: pointer to the VF info * @vsi: pointer to the VF's VSI - * @mac_addr: MAC address to delete + * @vc_ether_addr: VIRTCHNL MAC address structure used to delete MAC */ static int -ice_vc_del_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) +ice_vc_del_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, + struct virtchnl_ether_addr *vc_ether_addr) { struct device *dev = ice_pf_to_dev(vf->pf); + u8 *mac_addr = vc_ether_addr->addr; enum ice_status status; if (!ice_can_vf_change_mac(vf) && - ether_addr_equal(mac_addr, vf->dflt_lan_addr.addr)) + ether_addr_equal(vf->dev_lan_addr.addr, mac_addr)) return 0; status = ice_fltr_remove_mac(vsi, mac_addr, ICE_FWD_TO_VSI); @@ -3730,8 +3860,7 @@ ice_vc_del_mac_addr(struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr) return -EIO; } - if (ether_addr_equal(mac_addr, vf->dflt_lan_addr.addr)) - eth_zero_addr(vf->dflt_lan_addr.addr); + ice_vfhw_mac_del(vf, vc_ether_addr); vf->num_mac--; @@ -3750,7 +3879,8 @@ static int ice_vc_handle_mac_addr_msg(struct ice_vf *vf, u8 *msg, bool set) { int (*ice_vc_cfg_mac) - (struct ice_vf *vf, struct ice_vsi *vsi, u8 *mac_addr); + (struct ice_vf *vf, struct ice_vsi *vsi, + struct virtchnl_ether_addr *virtchnl_ether_addr); enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; struct virtchnl_ether_addr_list *al = (struct virtchnl_ether_addr_list *)msg; @@ -3799,7 +3929,7 @@ ice_vc_handle_mac_addr_msg(struct ice_vf *vf, u8 *msg, bool set) is_zero_ether_addr(mac_addr)) continue; - result = ice_vc_cfg_mac(vf, vsi, mac_addr); + result = ice_vc_cfg_mac(vf, vsi, &al->list[i]); if (result == -EEXIST || result == -ENOENT) { continue; } else if (result) { @@ -4437,7 +4567,7 @@ ice_get_vf_cfg(struct net_device *netdev, int vf_id, struct ifla_vf_info *ivi) return -EBUSY; ivi->vf = vf_id; - ether_addr_copy(ivi->mac, vf->dflt_lan_addr.addr); + ether_addr_copy(ivi->mac, vf->hw_lan_addr.addr); /* VF configuration for VLAN and applicable QoS */ ivi->vlan = vf->port_vlan_info & VLAN_VID_MASK; @@ -4513,7 +4643,8 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) vf = &pf->vf[vf_id]; /* nothing left to do, unicast MAC already set */ - if (ether_addr_equal(vf->dflt_lan_addr.addr, mac)) + if (ether_addr_equal(vf->dev_lan_addr.addr, mac) && + ether_addr_equal(vf->hw_lan_addr.addr, mac)) return 0; ret = ice_check_vf_ready_for_cfg(vf); @@ -4529,7 +4660,8 @@ int ice_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) /* VF is notified of its new MAC via the PF's response to the * VIRTCHNL_OP_GET_VF_RESOURCES message after the VF has been reset */ - ether_addr_copy(vf->dflt_lan_addr.addr, mac); + ether_addr_copy(vf->dev_lan_addr.addr, mac); + ether_addr_copy(vf->hw_lan_addr.addr, mac); if (is_zero_ether_addr(mac)) { /* VF will send VIRTCHNL_OP_ADD_ETH_ADDR message with its MAC */ vf->pf_set_mac = false; @@ -4682,7 +4814,7 @@ void ice_print_vf_rx_mdd_event(struct ice_vf *vf) dev_info(dev, "%d Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pM. mdd-auto-reset-vfs=%s\n", vf->mdd_rx_events.count, pf->hw.pf_id, vf->vf_id, - vf->dflt_lan_addr.addr, + vf->dev_lan_addr.addr, test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags) ? "on" : "off"); } @@ -4726,7 +4858,7 @@ void ice_print_vfs_mdd_events(struct ice_pf *pf) dev_info(dev, "%d Tx Malicious Driver Detection events detected on PF %d VF %d MAC %pM.\n", vf->mdd_tx_events.count, hw->pf_id, i, - vf->dflt_lan_addr.addr); + vf->dev_lan_addr.addr); } } } @@ -4816,7 +4948,7 @@ ice_is_malicious_vf(struct ice_pf *pf, struct ice_rq_event_info *event, if (pf_vsi) dev_warn(dev, "VF MAC %pM on PF MAC %pM is generating asynchronous messages and may be overflowing the PF message queue. Please see the Adapter User Guide for more information\n", - &vf->dflt_lan_addr.addr[0], + &vf->dev_lan_addr.addr[0], pf_vsi->netdev->dev_addr); } diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h index d800ed83d6c3..842cb077df86 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_pf.h @@ -58,6 +58,11 @@ enum ice_virtchnl_cap { ICE_VIRTCHNL_VF_CAP_PRIVILEGE, }; +struct ice_time_mac { + unsigned long time_modified; + u8 addr[ETH_ALEN]; +}; + /* VF MDD events print structure */ struct ice_mdd_vf_events { u16 count; /* total count of Rx|Tx events */ @@ -78,7 +83,9 @@ struct ice_vf { struct ice_sw *vf_sw_id; /* switch ID the VF VSIs connect to */ struct virtchnl_version_info vf_ver; u32 driver_caps; /* reported by VF driver */ - struct virtchnl_ether_addr dflt_lan_addr; + struct virtchnl_ether_addr dev_lan_addr; + struct virtchnl_ether_addr hw_lan_addr; + struct ice_time_mac legacy_last_added_umac; DECLARE_BITMAP(txq_ena, ICE_MAX_RSS_QS_PER_VF); DECLARE_BITMAP(rxq_ena, ICE_MAX_RSS_QS_PER_VF); u16 port_vlan_info; /* Port VLAN ID and QoS */ @@ -151,16 +158,18 @@ ice_vc_send_msg_to_vf(struct ice_vf *vf, u32 v_opcode, enum virtchnl_status_code v_retval, u8 *msg, u16 msglen); bool ice_vc_isvalid_vsi_id(struct ice_vf *vf, u16 vsi_id); #else /* CONFIG_PCI_IOV */ -#define ice_process_vflr_event(pf) do {} while (0) -#define ice_free_vfs(pf) do {} while (0) -#define ice_vc_process_vf_msg(pf, event) do {} while (0) -#define ice_vc_notify_link_state(pf) do {} while (0) -#define ice_vc_notify_reset(pf) do {} while (0) -#define ice_set_vf_state_qs_dis(vf) do {} while (0) -#define ice_vf_lan_overflow_event(pf, event) do {} while (0) -#define ice_print_vfs_mdd_events(pf) do {} while (0) -#define ice_print_vf_rx_mdd_event(vf) do {} while (0) -#define ice_restore_all_vfs_msi_state(pdev) do {} while (0) +static inline void ice_process_vflr_event(struct ice_pf *pf) { } +static inline void ice_free_vfs(struct ice_pf *pf) { } +static inline +void ice_vc_process_vf_msg(struct ice_pf *pf, struct ice_rq_event_info *event) { } +static inline void ice_vc_notify_link_state(struct ice_pf *pf) { } +static inline void ice_vc_notify_reset(struct ice_pf *pf) { } +static inline void ice_set_vf_state_qs_dis(struct ice_vf *vf) { } +static inline +void ice_vf_lan_overflow_event(struct ice_pf *pf, struct ice_rq_event_info *event) { } +static inline void ice_print_vfs_mdd_events(struct ice_pf *pf) { } +static inline void ice_print_vf_rx_mdd_event(struct ice_vf *vf) { } +static inline void ice_restore_all_vfs_msi_state(struct pci_dev *pdev) { } static inline bool ice_is_malicious_vf(struct ice_pf __always_unused *pf, diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index faa7b8d96adb..5a9f61deeb38 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -236,7 +236,7 @@ static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) xdp_ring->xsk_pool = ice_xsk_pool(xdp_ring); } - err = ice_setup_rx_ctx(rx_ring); + err = ice_vsi_cfg_rxq(rx_ring); if (err) goto free_buf; @@ -270,6 +270,7 @@ static int ice_xsk_pool_disable(struct ice_vsi *vsi, u16 qid) if (!pool) return -EINVAL; + clear_bit(qid, vsi->af_xdp_zc_qps); xsk_pool_dma_unmap(pool, ICE_RX_DMA_ATTR); return 0; @@ -300,6 +301,8 @@ ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) if (err) return err; + set_bit(qid, vsi->af_xdp_zc_qps); + return 0; } @@ -463,7 +466,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp) struct ice_ring *xdp_ring; u32 act; - rcu_read_lock(); /* ZC patch is enabled only when XDP program is set, * so here it can not be NULL */ @@ -473,9 +475,9 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp) if (likely(act == XDP_REDIRECT)) { err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - result = !err ? ICE_XDP_REDIR : ICE_XDP_CONSUMED; - rcu_read_unlock(); - return result; + if (err) + goto out_failure; + return ICE_XDP_REDIR; } switch (act) { @@ -484,11 +486,14 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp) case XDP_TX: xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->q_index]; result = ice_xmit_xdp_buff(xdp, xdp_ring); + if (result == ICE_XDP_CONSUMED) + goto out_failure; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_DROP: @@ -496,7 +501,6 @@ ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp) break; } - rcu_read_unlock(); return result; } @@ -521,7 +525,7 @@ int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget) struct sk_buff *skb; u16 stat_err_bits; u16 vlan_tag = 0; - u8 rx_ptype; + u16 rx_ptype; rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean); diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h index fad783690134..ea208808623a 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.h +++ b/drivers/net/ethernet/intel/ice/ice_xsk.h @@ -60,7 +60,7 @@ ice_xsk_wakeup(struct net_device __always_unused *netdev, return -EOPNOTSUPP; } -#define ice_xsk_clean_rx_ring(rx_ring) do {} while (0) -#define ice_xsk_clean_xdp_ring(xdp_ring) do {} while (0) +static inline void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring) { } +static inline void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring) { } #endif /* CONFIG_XDP_SOCKETS */ #endif /* !_ICE_XSK_H_ */ diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.c b/drivers/net/ethernet/intel/igb/e1000_82575.c index 50863fd87d53..cbe92fd23a70 100644 --- a/drivers/net/ethernet/intel/igb/e1000_82575.c +++ b/drivers/net/ethernet/intel/igb/e1000_82575.c @@ -2756,6 +2756,7 @@ out: return ret_val; } +#ifdef CONFIG_IGB_HWMON static const u8 e1000_emc_temp_data[4] = { E1000_EMC_INTERNAL_DATA, E1000_EMC_DIODE1_DATA, @@ -2769,7 +2770,6 @@ static const u8 e1000_emc_therm_limit[4] = { E1000_EMC_DIODE3_THERM_LIMIT }; -#ifdef CONFIG_IGB_HWMON /** * igb_get_thermal_sensor_data_generic - Gathers thermal sensor data * @hw: pointer to hardware structure diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h index 7bda8c5edea5..2d3daf022651 100644 --- a/drivers/net/ethernet/intel/igb/igb.h +++ b/drivers/net/ethernet/intel/igb/igb.h @@ -749,7 +749,7 @@ void igb_ptp_rx_hang(struct igb_adapter *adapter); void igb_ptp_tx_hang(struct igb_adapter *adapter); void igb_ptp_rx_rgtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb); int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va, - struct sk_buff *skb); + ktime_t *timestamp); int igb_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr); int igb_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr); void igb_set_flag_queue_pairs(struct igb_adapter *, const u32); diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 7545da216d8b..636a1b1fb7e1 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -831,7 +831,7 @@ static int igb_set_eeprom(struct net_device *netdev, memcpy(ptr, bytes, eeprom->len); for (i = 0; i < last_word - first_word + 1; i++) - eeprom_buff[i] = cpu_to_le16(eeprom_buff[i]); + cpu_to_le16s(&eeprom_buff[i]); ret_val = hw->nvm.ops.write(hw, first_word, last_word - first_word + 1, eeprom_buff); diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 038a9fd1af44..7e6435dc7e80 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -356,7 +356,7 @@ static void igb_dump(struct igb_adapter *adapter) struct igb_reg_info *reginfo; struct igb_ring *tx_ring; union e1000_adv_tx_desc *tx_desc; - struct my_u0 { u64 a; u64 b; } *u0; + struct my_u0 { __le64 a; __le64 b; } *u0; struct igb_ring *rx_ring; union e1000_adv_rx_desc *rx_desc; u32 staterr; @@ -2643,7 +2643,8 @@ static int igb_parse_cls_flower(struct igb_adapter *adapter, } input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI; - input->filter.vlan_tci = match.key->vlan_priority; + input->filter.vlan_tci = + (__force __be16)match.key->vlan_priority; } } @@ -6275,12 +6276,12 @@ int igb_xmit_xdp_ring(struct igb_adapter *adapter, cmd_type |= len | IGB_TXD_DCMD; tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - olinfo_status = cpu_to_le32(len << E1000_ADVTXD_PAYLEN_SHIFT); + olinfo_status = len << E1000_ADVTXD_PAYLEN_SHIFT; /* 82575 requires a unique index per ring */ if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) olinfo_status |= tx_ring->reg_idx << 4; - tx_desc->read.olinfo_status = olinfo_status; + tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); netdev_tx_sent_queue(txring_txq(tx_ring), tx_buffer->bytecount); @@ -8280,7 +8281,7 @@ static void igb_add_rx_frag(struct igb_ring *rx_ring, static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, struct igb_rx_buffer *rx_buffer, struct xdp_buff *xdp, - union e1000_adv_rx_desc *rx_desc) + ktime_t timestamp) { #if (PAGE_SIZE < 8192) unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; @@ -8300,12 +8301,8 @@ static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, if (unlikely(!skb)) return NULL; - if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) { - if (!igb_ptp_rx_pktstamp(rx_ring->q_vector, xdp->data, skb)) { - xdp->data += IGB_TS_HDR_LEN; - size -= IGB_TS_HDR_LEN; - } - } + if (timestamp) + skb_hwtstamps(skb)->hwtstamp = timestamp; /* Determine available headroom for copy */ headlen = size; @@ -8336,7 +8333,7 @@ static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, struct igb_rx_buffer *rx_buffer, struct xdp_buff *xdp, - union e1000_adv_rx_desc *rx_desc) + ktime_t timestamp) { #if (PAGE_SIZE < 8192) unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; @@ -8363,11 +8360,8 @@ static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, if (metasize) skb_metadata_set(skb, metasize); - /* pull timestamp out of packet data */ - if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { - if (!igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb)) - __skb_pull(skb, IGB_TS_HDR_LEN); - } + if (timestamp) + skb_hwtstamps(skb)->hwtstamp = timestamp; /* update buffer offset */ #if (PAGE_SIZE < 8192) @@ -8387,7 +8381,6 @@ static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, struct bpf_prog *xdp_prog; u32 act; - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); if (!xdp_prog) @@ -8401,18 +8394,20 @@ static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, break; case XDP_TX: result = igb_xdp_xmit_back(adapter, xdp); + if (result == IGB_XDP_CONSUMED) + goto out_failure; break; case XDP_REDIRECT: err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); - if (!err) - result = IGB_XDP_REDIR; - else - result = IGB_XDP_CONSUMED; + if (err) + goto out_failure; + result = IGB_XDP_REDIR; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; case XDP_DROP: @@ -8420,7 +8415,6 @@ static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, break; } xdp_out: - rcu_read_unlock(); return ERR_PTR(-result); } @@ -8597,7 +8591,7 @@ static void igb_process_skb_fields(struct igb_ring *rx_ring, if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) && test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) - vid = be16_to_cpu(rx_desc->wb.upper.vlan); + vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); else vid = le16_to_cpu(rx_desc->wb.upper.vlan); @@ -8682,7 +8676,10 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) while (likely(total_packets < budget)) { union e1000_adv_rx_desc *rx_desc; struct igb_rx_buffer *rx_buffer; + ktime_t timestamp = 0; + int pkt_offset = 0; unsigned int size; + void *pktbuf; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IGB_RX_BUFFER_WRITE) { @@ -8702,14 +8699,24 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) dma_rmb(); rx_buffer = igb_get_rx_buffer(rx_ring, size, &rx_buf_pgcnt); + pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; + + /* pull rx packet timestamp if available and valid */ + if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { + int ts_hdr_len; + + ts_hdr_len = igb_ptp_rx_pktstamp(rx_ring->q_vector, + pktbuf, ×tamp); + + pkt_offset += ts_hdr_len; + size -= ts_hdr_len; + } /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = igb_rx_offset(rx_ring); - unsigned char *hard_start; + unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring); + unsigned int offset = pkt_offset + igb_rx_offset(rx_ring); - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); #if (PAGE_SIZE > 4096) /* At larger PAGE_SIZE, frame_sz depend on len size */ @@ -8732,10 +8739,11 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) } else if (skb) igb_add_rx_frag(rx_ring, rx_buffer, skb, size); else if (ring_uses_build_skb(rx_ring)) - skb = igb_build_skb(rx_ring, rx_buffer, &xdp, rx_desc); + skb = igb_build_skb(rx_ring, rx_buffer, &xdp, + timestamp); else skb = igb_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + &xdp, timestamp); /* exit if we failed to retrieve a buffer */ if (!skb) { diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c index ba61fe9bfaf4..0011b15e678c 100644 --- a/drivers/net/ethernet/intel/igb/igb_ptp.c +++ b/drivers/net/ethernet/intel/igb/igb_ptp.c @@ -856,30 +856,28 @@ static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter) dev_kfree_skb_any(skb); } -#define IGB_RET_PTP_DISABLED 1 -#define IGB_RET_PTP_INVALID 2 - /** * igb_ptp_rx_pktstamp - retrieve Rx per packet timestamp * @q_vector: Pointer to interrupt specific structure * @va: Pointer to address containing Rx buffer - * @skb: Buffer containing timestamp and packet + * @timestamp: Pointer where timestamp will be stored * * This function is meant to retrieve a timestamp from the first buffer of an * incoming frame. The value is stored in little endian format starting on * byte 8 * - * Returns: 0 if success, nonzero if failure + * Returns: The timestamp header length or 0 if not available **/ int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va, - struct sk_buff *skb) + ktime_t *timestamp) { struct igb_adapter *adapter = q_vector->adapter; + struct skb_shared_hwtstamps ts; __le64 *regval = (__le64 *)va; int adjust = 0; if (!(adapter->ptp_flags & IGB_PTP_ENABLED)) - return IGB_RET_PTP_DISABLED; + return 0; /* The timestamp is recorded in little endian format. * DWORD: 0 1 2 3 @@ -888,10 +886,9 @@ int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va, /* check reserved dwords are zero, be/le doesn't matter for zero */ if (regval[0]) - return IGB_RET_PTP_INVALID; + return 0; - igb_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), - le64_to_cpu(regval[1])); + igb_ptp_systim_to_hwtstamp(adapter, &ts, le64_to_cpu(regval[1])); /* adjust timestamp for the RX latency based on link speed */ if (adapter->hw.mac.type == e1000_i210) { @@ -907,10 +904,10 @@ int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va, break; } } - skb_hwtstamps(skb)->hwtstamp = - ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust); - return 0; + *timestamp = ktime_sub_ns(ts.hwtstamp, adjust); + + return IGB_TS_HDR_LEN; } /** @@ -1134,12 +1131,12 @@ static int igb_ptp_set_timestamp_mode(struct igb_adapter *adapter, | E1000_FTQF_MASK); /* mask all inputs */ ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */ - wr32(E1000_IMIR(3), htons(PTP_EV_PORT)); + wr32(E1000_IMIR(3), (__force unsigned int)htons(PTP_EV_PORT)); wr32(E1000_IMIREXT(3), (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP)); if (hw->mac.type == e1000_82576) { /* enable source port check */ - wr32(E1000_SPQF(3), htons(PTP_EV_PORT)); + wr32(E1000_SPQF(3), (__force unsigned int)htons(PTP_EV_PORT)); ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP; } wr32(E1000_FTQF(3), ftqf); diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c index fb3fbcb13331..1bbe9862a758 100644 --- a/drivers/net/ethernet/intel/igbvf/netdev.c +++ b/drivers/net/ethernet/intel/igbvf/netdev.c @@ -83,14 +83,14 @@ static int igbvf_desc_unused(struct igbvf_ring *ring) static void igbvf_receive_skb(struct igbvf_adapter *adapter, struct net_device *netdev, struct sk_buff *skb, - u32 status, u16 vlan) + u32 status, __le16 vlan) { u16 vid; if (status & E1000_RXD_STAT_VP) { if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) && (status & E1000_RXDEXT_STATERR_LB)) - vid = be16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK; + vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK; else vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK; if (test_bit(vid, adapter->active_vlans)) @@ -2056,7 +2056,7 @@ static int igbvf_tso(struct igbvf_ring *tx_ring, /* remove payload length from inner checksum */ paylen = skb->len - l4_offset; - csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); + csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); /* MSS L4LEN IDX */ mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT; diff --git a/drivers/net/ethernet/intel/igbvf/vf.h b/drivers/net/ethernet/intel/igbvf/vf.h index c71b0d7dbcee..ba9bb3132d5d 100644 --- a/drivers/net/ethernet/intel/igbvf/vf.h +++ b/drivers/net/ethernet/intel/igbvf/vf.h @@ -35,31 +35,31 @@ struct e1000_hw; /* Receive Descriptor - Advanced */ union e1000_adv_rx_desc { struct { - u64 pkt_addr; /* Packet buffer address */ - u64 hdr_addr; /* Header buffer address */ + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ } read; struct { struct { union { - u32 data; + __le32 data; struct { - u16 pkt_info; /* RSS/Packet type */ + __le16 pkt_info; /* RSS/Packet type */ /* Split Header, hdr buffer length */ - u16 hdr_info; + __le16 hdr_info; } hs_rss; } lo_dword; union { - u32 rss; /* RSS Hash */ + __le32 rss; /* RSS Hash */ struct { - u16 ip_id; /* IP id */ - u16 csum; /* Packet Checksum */ + __le16 ip_id; /* IP id */ + __le16 csum; /* Packet Checksum */ } csum_ip; } hi_dword; } lower; struct { - u32 status_error; /* ext status/error */ - u16 length; /* Packet length */ - u16 vlan; /* VLAN tag */ + __le32 status_error; /* ext status/error */ + __le16 length; /* Packet length */ + __le16 vlan; /* VLAN tag */ } upper; } wb; /* writeback */ }; @@ -70,14 +70,14 @@ union e1000_adv_rx_desc { /* Transmit Descriptor - Advanced */ union e1000_adv_tx_desc { struct { - u64 buffer_addr; /* Address of descriptor's data buf */ - u32 cmd_type_len; - u32 olinfo_status; + __le64 buffer_addr; /* Address of descriptor's data buf */ + __le32 cmd_type_len; + __le32 olinfo_status; } read; struct { - u64 rsvd; /* Reserved */ - u32 nxtseq_seed; - u32 status; + __le64 rsvd; /* Reserved */ + __le32 nxtseq_seed; + __le32 status; } wb; }; @@ -94,10 +94,10 @@ union e1000_adv_tx_desc { /* Context descriptors */ struct e1000_adv_tx_context_desc { - u32 vlan_macip_lens; - u32 seqnum_seed; - u32 type_tucmd_mlhl; - u32 mss_l4len_idx; + __le32 vlan_macip_lens; + __le32 seqnum_seed; + __le32 type_tucmd_mlhl; + __le32 mss_l4len_idx; }; #define E1000_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 25871351730b..9e0bbb2e55e3 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -118,6 +118,7 @@ struct igc_ring { }; struct xdp_rxq_info xdp_rxq; + struct xsk_buff_pool *xsk_pool; } ____cacheline_internodealigned_in_smp; /* Board specific private data structure */ @@ -255,6 +256,11 @@ bool igc_has_link(struct igc_adapter *adapter); void igc_reset(struct igc_adapter *adapter); int igc_set_spd_dplx(struct igc_adapter *adapter, u32 spd, u8 dplx); void igc_update_stats(struct igc_adapter *adapter); +void igc_disable_rx_ring(struct igc_ring *ring); +void igc_enable_rx_ring(struct igc_ring *ring); +void igc_disable_tx_ring(struct igc_ring *ring); +void igc_enable_tx_ring(struct igc_ring *ring); +int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); /* igc_dump declarations */ void igc_rings_dump(struct igc_adapter *adapter); @@ -366,6 +372,7 @@ extern char igc_driver_name[]; /* VLAN info */ #define IGC_TX_FLAGS_VLAN_MASK 0xffff0000 +#define IGC_TX_FLAGS_VLAN_SHIFT 16 /* igc_test_staterr - tests bits within Rx descriptor status and error fields */ static inline __le32 igc_test_staterr(union igc_adv_rx_desc *rx_desc, @@ -390,8 +397,6 @@ enum igc_tx_flags { /* olinfo flags */ IGC_TX_FLAGS_IPV4 = 0x10, IGC_TX_FLAGS_CSUM = 0x20, - - IGC_TX_FLAGS_XDP = 0x100, }; enum igc_boards { @@ -408,12 +413,19 @@ enum igc_boards { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IGC_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) +enum igc_tx_buffer_type { + IGC_TX_BUFFER_TYPE_SKB, + IGC_TX_BUFFER_TYPE_XDP, + IGC_TX_BUFFER_TYPE_XSK, +}; + /* wrapper around a pointer to a socket buffer, * so a DMA handle can be stored along with the buffer */ struct igc_tx_buffer { union igc_adv_tx_desc *next_to_watch; unsigned long time_stamp; + enum igc_tx_buffer_type type; union { struct sk_buff *skb; struct xdp_frame *xdpf; @@ -428,14 +440,19 @@ struct igc_tx_buffer { }; struct igc_rx_buffer { - dma_addr_t dma; - struct page *page; + union { + struct { + dma_addr_t dma; + struct page *page; #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) - __u32 page_offset; + __u32 page_offset; #else - __u16 page_offset; + __u16 page_offset; #endif - __u16 pagecnt_bias; + __u16 pagecnt_bias; + }; + struct xdp_buff *xdp; + }; }; struct igc_q_vector { @@ -521,7 +538,8 @@ enum igc_ring_flags_t { IGC_RING_FLAG_RX_SCTP_CSUM, IGC_RING_FLAG_RX_LB_VLAN_BSWAP, IGC_RING_FLAG_TX_CTX_IDX, - IGC_RING_FLAG_TX_DETECT_HANG + IGC_RING_FLAG_TX_DETECT_HANG, + IGC_RING_FLAG_AF_XDP_ZC, }; #define ring_uses_large_buffer(ring) \ diff --git a/drivers/net/ethernet/intel/igc/igc_base.h b/drivers/net/ethernet/intel/igc/igc_base.h index ea627ce52525..ce530f5fd7bd 100644 --- a/drivers/net/ethernet/intel/igc/igc_base.h +++ b/drivers/net/ethernet/intel/igc/igc_base.h @@ -78,9 +78,11 @@ union igc_adv_rx_desc { /* Additional Transmit Descriptor Control definitions */ #define IGC_TXDCTL_QUEUE_ENABLE 0x02000000 /* Ena specific Tx Queue */ +#define IGC_TXDCTL_SWFLUSH 0x04000000 /* Transmit Software Flush */ /* Additional Receive Descriptor Control definitions */ #define IGC_RXDCTL_QUEUE_ENABLE 0x02000000 /* Ena specific Rx Queue */ +#define IGC_RXDCTL_SWFLUSH 0x04000000 /* Receive Software Flush */ /* SRRCTL bit definitions */ #define IGC_SRRCTL_BSIZEPKT_SHIFT 10 /* Shift _right_ */ diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 0103dda32f39..c3a5a5518790 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -94,12 +94,13 @@ #define IGC_CTRL_SLU 0x00000040 /* Set link up (Force Link) */ #define IGC_CTRL_FRCSPD 0x00000800 /* Force Speed */ #define IGC_CTRL_FRCDPX 0x00001000 /* Force Duplex */ +#define IGC_CTRL_VME 0x40000000 /* IEEE VLAN mode enable */ #define IGC_CTRL_RFCE 0x08000000 /* Receive Flow Control enable */ #define IGC_CTRL_TFCE 0x10000000 /* Transmit flow control enable */ -#define IGC_CTRL_SDP0_DIR 0x00400000 /* SDP0 Data direction */ -#define IGC_CTRL_SDP1_DIR 0x00800000 /* SDP1 Data direction */ +#define IGC_CTRL_SDP0_DIR 0x00400000 /* SDP0 Data direction */ +#define IGC_CTRL_SDP1_DIR 0x00800000 /* SDP1 Data direction */ /* As per the EAS the maximum supported size is 9.5KB (9728 bytes) */ #define MAX_JUMBO_FRAME_SIZE 0x2600 @@ -128,7 +129,6 @@ #define NWAY_LPAR_ASM_DIR 0x0800 /* LP Asymmetric Pause Direction bit */ /* 1000BASE-T Control Register */ -#define CR_1000T_ASYM_PAUSE 0x0080 /* Advertise asymmetric pause bit */ #define CR_1000T_HD_CAPS 0x0100 /* Advertise 1000T HD capability */ #define CR_1000T_FD_CAPS 0x0200 /* Advertise 1000T FD capability */ @@ -323,6 +323,9 @@ #define IGC_RXD_STAT_IXSM 0x04 /* Ignore checksum */ #define IGC_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ #define IGC_RXD_STAT_TCPCS 0x20 /* TCP xsum calculated */ +#define IGC_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ + +#define IGC_RXDEXT_STATERR_LB 0x00040000 /* Advanced Receive Descriptor bit definitions */ #define IGC_RXDADV_STAT_TSIP 0x08000 /* timestamp in packet */ diff --git a/drivers/net/ethernet/intel/igc/igc_dump.c b/drivers/net/ethernet/intel/igc/igc_dump.c index 495bed47ed0a..c09c95cc5f70 100644 --- a/drivers/net/ethernet/intel/igc/igc_dump.c +++ b/drivers/net/ethernet/intel/igc/igc_dump.c @@ -112,7 +112,7 @@ static void igc_regdump(struct igc_hw *hw, struct igc_reg_info *reginfo) void igc_rings_dump(struct igc_adapter *adapter) { struct net_device *netdev = adapter->netdev; - struct my_u0 { u64 a; u64 b; } *u0; + struct my_u0 { __le64 a; __le64 b; } *u0; union igc_adv_tx_desc *tx_desc; union igc_adv_rx_desc *rx_desc; struct igc_ring *tx_ring; diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index 9722449d7633..fa4171860623 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -554,7 +554,7 @@ static int igc_ethtool_set_eeprom(struct net_device *netdev, memcpy(ptr, bytes, eeprom->len); for (i = 0; i < last_word - first_word + 1; i++) - eeprom_buff[i] = cpu_to_le16(eeprom_buff[i]); + cpu_to_le16s(&eeprom_buff[i]); ret_val = hw->nvm.ops.write(hw, first_word, last_word - first_word + 1, eeprom_buff); @@ -765,35 +765,22 @@ static void igc_ethtool_get_strings(struct net_device *netdev, u32 stringset, IGC_TEST_LEN * ETH_GSTRING_LEN); break; case ETH_SS_STATS: - for (i = 0; i < IGC_GLOBAL_STATS_LEN; i++) { - memcpy(p, igc_gstrings_stats[i].stat_string, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - for (i = 0; i < IGC_NETDEV_STATS_LEN; i++) { - memcpy(p, igc_gstrings_net_stats[i].stat_string, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } + for (i = 0; i < IGC_GLOBAL_STATS_LEN; i++) + ethtool_sprintf(&p, igc_gstrings_stats[i].stat_string); + for (i = 0; i < IGC_NETDEV_STATS_LEN; i++) + ethtool_sprintf(&p, + igc_gstrings_net_stats[i].stat_string); for (i = 0; i < adapter->num_tx_queues; i++) { - sprintf(p, "tx_queue_%u_packets", i); - p += ETH_GSTRING_LEN; - sprintf(p, "tx_queue_%u_bytes", i); - p += ETH_GSTRING_LEN; - sprintf(p, "tx_queue_%u_restart", i); - p += ETH_GSTRING_LEN; + ethtool_sprintf(&p, "tx_queue_%u_packets", i); + ethtool_sprintf(&p, "tx_queue_%u_bytes", i); + ethtool_sprintf(&p, "tx_queue_%u_restart", i); } for (i = 0; i < adapter->num_rx_queues; i++) { - sprintf(p, "rx_queue_%u_packets", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_bytes", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_drops", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_csum_err", i); - p += ETH_GSTRING_LEN; - sprintf(p, "rx_queue_%u_alloc_failed", i); - p += ETH_GSTRING_LEN; + ethtool_sprintf(&p, "rx_queue_%u_packets", i); + ethtool_sprintf(&p, "rx_queue_%u_bytes", i); + ethtool_sprintf(&p, "rx_queue_%u_drops", i); + ethtool_sprintf(&p, "rx_queue_%u_csum_err", i); + ethtool_sprintf(&p, "rx_queue_%u_alloc_failed", i); } /* BUG_ON(p - data != IGC_STATS_LEN * ETH_GSTRING_LEN); */ break; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 069471b7ffb0..95323095094d 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -11,7 +11,7 @@ #include <linux/pm_runtime.h> #include <net/pkt_sched.h> #include <linux/bpf_trace.h> - +#include <net/xdp_sock_drv.h> #include <net/ipv6.h> #include "igc.h" @@ -111,6 +111,9 @@ void igc_reset(struct igc_adapter *adapter) if (!netif_running(adapter->netdev)) igc_power_down_phy_copper_base(&adapter->hw); + /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */ + wr32(IGC_VET, ETH_P_8021Q); + /* Re-enable PTP, where applicable. */ igc_ptp_reset(adapter); @@ -171,6 +174,14 @@ static void igc_get_hw_control(struct igc_adapter *adapter) ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); } +static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf) +{ + dma_unmap_single(dev, dma_unmap_addr(buf, dma), + dma_unmap_len(buf, len), DMA_TO_DEVICE); + + dma_unmap_len_set(buf, len, 0); +} + /** * igc_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned @@ -179,20 +190,27 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) { u16 i = tx_ring->next_to_clean; struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; + u32 xsk_frames = 0; while (i != tx_ring->next_to_use) { union igc_adv_tx_desc *eop_desc, *tx_desc; - if (tx_buffer->tx_flags & IGC_TX_FLAGS_XDP) + switch (tx_buffer->type) { + case IGC_TX_BUFFER_TYPE_XSK: + xsk_frames++; + break; + case IGC_TX_BUFFER_TYPE_XDP: xdp_return_frame(tx_buffer->xdpf); - else + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + case IGC_TX_BUFFER_TYPE_SKB: dev_kfree_skb_any(tx_buffer->skb); - - /* unmap skb header data */ - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + default: + netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); + break; + } /* check for eop_desc to determine the end of the packet */ eop_desc = tx_buffer->next_to_watch; @@ -211,10 +229,7 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); } /* move us one more past the eop_desc for start of next pkt */ @@ -226,6 +241,9 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) } } + if (tx_ring->xsk_pool && xsk_frames) + xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); + /* reset BQL for queue */ netdev_tx_reset_queue(txring_txq(tx_ring)); @@ -346,11 +364,7 @@ static int igc_setup_all_tx_resources(struct igc_adapter *adapter) return err; } -/** - * igc_clean_rx_ring - Free Rx Buffers per Queue - * @rx_ring: ring to free buffers from - */ -static void igc_clean_rx_ring(struct igc_ring *rx_ring) +static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring) { u16 i = rx_ring->next_to_clean; @@ -383,12 +397,39 @@ static void igc_clean_rx_ring(struct igc_ring *rx_ring) if (i == rx_ring->count) i = 0; } +} + +static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring) +{ + struct igc_rx_buffer *bi; + u16 i; + + for (i = 0; i < ring->count; i++) { + bi = &ring->rx_buffer_info[i]; + if (!bi->xdp) + continue; + + xsk_buff_free(bi->xdp); + bi->xdp = NULL; + } +} - clear_ring_uses_large_buffer(rx_ring); +/** + * igc_clean_rx_ring - Free Rx Buffers per Queue + * @ring: ring to free buffers from + */ +static void igc_clean_rx_ring(struct igc_ring *ring) +{ + if (ring->xsk_pool) + igc_clean_rx_ring_xsk_pool(ring); + else + igc_clean_rx_ring_page_shared(ring); - rx_ring->next_to_alloc = 0; - rx_ring->next_to_clean = 0; - rx_ring->next_to_use = 0; + clear_ring_uses_large_buffer(ring); + + ring->next_to_alloc = 0; + ring->next_to_clean = 0; + ring->next_to_use = 0; } /** @@ -414,7 +455,7 @@ void igc_free_rx_resources(struct igc_ring *rx_ring) { igc_clean_rx_ring(rx_ring); - igc_xdp_unregister_rxq_info(rx_ring); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; @@ -453,11 +494,16 @@ int igc_setup_rx_resources(struct igc_ring *rx_ring) { struct net_device *ndev = rx_ring->netdev; struct device *dev = rx_ring->dev; + u8 index = rx_ring->queue_index; int size, desc_len, res; - res = igc_xdp_register_rxq_info(rx_ring); - if (res < 0) + res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index, + rx_ring->q_vector->napi.napi_id); + if (res < 0) { + netdev_err(ndev, "Failed to register xdp_rxq index %u\n", + index); return res; + } size = sizeof(struct igc_rx_buffer) * rx_ring->count; rx_ring->rx_buffer_info = vzalloc(size); @@ -483,7 +529,7 @@ int igc_setup_rx_resources(struct igc_ring *rx_ring) return 0; err: - igc_xdp_unregister_rxq_info(rx_ring); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n"); @@ -515,9 +561,14 @@ static int igc_setup_all_rx_resources(struct igc_adapter *adapter) return err; } -static bool igc_xdp_is_enabled(struct igc_adapter *adapter) +static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter, + struct igc_ring *ring) { - return !!adapter->xdp_prog; + if (!igc_xdp_is_enabled(adapter) || + !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags)) + return NULL; + + return xsk_get_pool_from_qid(ring->netdev, ring->queue_index); } /** @@ -535,6 +586,20 @@ static void igc_configure_rx_ring(struct igc_adapter *adapter, int reg_idx = ring->reg_idx; u32 srrctl = 0, rxdctl = 0; u64 rdba = ring->dma; + u32 buf_size; + + xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + ring->xsk_pool = igc_get_xsk_pool(adapter, ring); + if (ring->xsk_pool) { + WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL)); + xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); + } else { + WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, + MEM_TYPE_PAGE_SHARED, + NULL)); + } if (igc_xdp_is_enabled(adapter)) set_ring_uses_large_buffer(ring); @@ -558,12 +623,15 @@ static void igc_configure_rx_ring(struct igc_adapter *adapter, ring->next_to_clean = 0; ring->next_to_use = 0; - /* set descriptor configuration */ - srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring_uses_large_buffer(ring)) - srrctl |= IGC_RXBUFFER_3072 >> IGC_SRRCTL_BSIZEPKT_SHIFT; + if (ring->xsk_pool) + buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool); + else if (ring_uses_large_buffer(ring)) + buf_size = IGC_RXBUFFER_3072; else - srrctl |= IGC_RXBUFFER_2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; + buf_size = IGC_RXBUFFER_2048; + + srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT; + srrctl |= buf_size >> IGC_SRRCTL_BSIZEPKT_SHIFT; srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; wr32(IGC_SRRCTL(reg_idx), srrctl); @@ -618,6 +686,8 @@ static void igc_configure_tx_ring(struct igc_adapter *adapter, u64 tdba = ring->dma; u32 txdctl = 0; + ring->xsk_pool = igc_get_xsk_pool(adapter, ring); + /* disable the queue */ wr32(IGC_TXDCTL(reg_idx), 0); wrfl(); @@ -1055,13 +1125,17 @@ static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) -static u32 igc_tx_cmd_type(u32 tx_flags) +static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ u32 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DCMD_IFCS; + /* set HW vlan bit if vlan is present */ + cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN, + IGC_ADVTXD_DCMD_VLE); + /* set segmentation bits for TSO */ cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, (IGC_ADVTXD_DCMD_TSE)); @@ -1070,6 +1144,9 @@ static u32 igc_tx_cmd_type(u32 tx_flags) cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, (IGC_ADVTXD_MAC_TSTAMP)); + /* insert frame checksum */ + cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS); + return cmd_type; } @@ -1104,8 +1181,9 @@ static int igc_tx_map(struct igc_ring *tx_ring, u16 i = tx_ring->next_to_use; unsigned int data_len, size; dma_addr_t dma; - u32 cmd_type = igc_tx_cmd_type(tx_flags); + u32 cmd_type; + cmd_type = igc_tx_cmd_type(skb, tx_flags); tx_desc = IGC_TX_DESC(tx_ring, i); igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len); @@ -1211,11 +1289,7 @@ dma_error: /* clear dma mappings for failed tx_buffer_info map */ while (tx_buffer != first) { if (dma_unmap_len(tx_buffer, len)) - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); if (i-- == 0) i += tx_ring->count; @@ -1223,11 +1297,7 @@ dma_error: } if (dma_unmap_len(tx_buffer, len)) - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); dev_kfree_skb_any(tx_buffer->skb); tx_buffer->skb = NULL; @@ -1359,6 +1429,7 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, /* record the location of the first descriptor for this packet */ first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first->type = IGC_TX_BUFFER_TYPE_SKB; first->skb = skb; first->bytecount = skb->len; first->gso_segs = 1; @@ -1383,6 +1454,11 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, } } + if (skb_vlan_tag_present(skb)) { + tx_flags |= IGC_TX_FLAGS_VLAN; + tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT); + } + /* record initial flags and protocol */ first->tx_flags = tx_flags; first->protocol = protocol; @@ -1482,6 +1558,25 @@ static inline void igc_rx_hash(struct igc_ring *ring, PKT_HASH_TYPE_L3); } +static void igc_rx_vlan(struct igc_ring *rx_ring, + union igc_adv_rx_desc *rx_desc, + struct sk_buff *skb) +{ + struct net_device *dev = rx_ring->netdev; + u16 vid; + + if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && + igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) { + if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) && + test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) + vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); + else + vid = le16_to_cpu(rx_desc->wb.upper.vlan); + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + } +} + /** * igc_process_skb_fields - Populate skb header fields from Rx descriptor * @rx_ring: rx descriptor ring packet is being transacted on @@ -1500,11 +1595,37 @@ static void igc_process_skb_fields(struct igc_ring *rx_ring, igc_rx_checksum(rx_ring, rx_desc, skb); + igc_rx_vlan(rx_ring, rx_desc, skb); + skb_record_rx_queue(skb, rx_ring->queue_index); skb->protocol = eth_type_trans(skb, rx_ring->netdev); } +static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features) +{ + bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + u32 ctrl; + + ctrl = rd32(IGC_CTRL); + + if (enable) { + /* enable VLAN tag insert/strip */ + ctrl |= IGC_CTRL_VME; + } else { + /* disable VLAN tag insert/strip */ + ctrl &= ~IGC_CTRL_VME; + } + wr32(IGC_CTRL, ctrl); +} + +static void igc_restore_vlan(struct igc_adapter *adapter) +{ + igc_vlan_mode(adapter->netdev, adapter->netdev->features); +} + static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, const unsigned int size, int *rx_buffer_pgcnt) @@ -1930,6 +2051,63 @@ static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) } } +static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) +{ + union igc_adv_rx_desc *desc; + u16 i = ring->next_to_use; + struct igc_rx_buffer *bi; + dma_addr_t dma; + bool ok = true; + + if (!count) + return ok; + + desc = IGC_RX_DESC(ring, i); + bi = &ring->rx_buffer_info[i]; + i -= ring->count; + + do { + bi->xdp = xsk_buff_alloc(ring->xsk_pool); + if (!bi->xdp) { + ok = false; + break; + } + + dma = xsk_buff_xdp_get_dma(bi->xdp); + desc->read.pkt_addr = cpu_to_le64(dma); + + desc++; + bi++; + i++; + if (unlikely(!i)) { + desc = IGC_RX_DESC(ring, 0); + bi = ring->rx_buffer_info; + i -= ring->count; + } + + /* Clear the length for the next_to_use descriptor. */ + desc->wb.upper.length = 0; + + count--; + } while (count); + + i += ring->count; + + if (ring->next_to_use != i) { + ring->next_to_use = i; + + /* Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. (Only + * applicable for weak-ordered memory model archs, + * such as IA-64). + */ + wmb(); + writel(i, ring->tail); + } + + return ok; +} + static int igc_xdp_init_tx_buffer(struct igc_tx_buffer *buffer, struct xdp_frame *xdpf, struct igc_ring *ring) @@ -1942,8 +2120,8 @@ static int igc_xdp_init_tx_buffer(struct igc_tx_buffer *buffer, return -ENOMEM; } + buffer->type = IGC_TX_BUFFER_TYPE_XDP; buffer->xdpf = xdpf; - buffer->tx_flags = IGC_TX_FLAGS_XDP; buffer->protocol = 0; buffer->bytecount = xdpf->len; buffer->gso_segs = 1; @@ -2025,51 +2203,52 @@ static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp) return res; } -static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter, - struct xdp_buff *xdp) +/* This function assumes rcu_read_lock() is held by the caller. */ +static int __igc_xdp_run_prog(struct igc_adapter *adapter, + struct bpf_prog *prog, + struct xdp_buff *xdp) { - struct bpf_prog *prog; - int res; - u32 act; + u32 act = bpf_prog_run_xdp(prog, xdp); - rcu_read_lock(); - - prog = READ_ONCE(adapter->xdp_prog); - if (!prog) { - res = IGC_XDP_PASS; - goto unlock; - } - - act = bpf_prog_run_xdp(prog, xdp); switch (act) { case XDP_PASS: - res = IGC_XDP_PASS; - break; + return IGC_XDP_PASS; case XDP_TX: if (igc_xdp_xmit_back(adapter, xdp) < 0) - res = IGC_XDP_CONSUMED; - else - res = IGC_XDP_TX; - break; + goto out_failure; + return IGC_XDP_TX; case XDP_REDIRECT: if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0) - res = IGC_XDP_CONSUMED; - else - res = IGC_XDP_REDIRECT; + goto out_failure; + return IGC_XDP_REDIRECT; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(adapter->netdev, prog, act); fallthrough; case XDP_DROP: - res = IGC_XDP_CONSUMED; - break; + return IGC_XDP_CONSUMED; } +} -unlock: - rcu_read_unlock(); +static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter, + struct xdp_buff *xdp) +{ + struct bpf_prog *prog; + int res; + + prog = READ_ONCE(adapter->xdp_prog); + if (!prog) { + res = IGC_XDP_PASS; + goto out; + } + + res = __igc_xdp_run_prog(adapter, prog, xdp); + +out: return ERR_PTR(-res); } @@ -2103,6 +2282,20 @@ static void igc_finalize_xdp(struct igc_adapter *adapter, int status) xdp_do_flush(); } +static void igc_update_rx_stats(struct igc_q_vector *q_vector, + unsigned int packets, unsigned int bytes) +{ + struct igc_ring *ring = q_vector->rx.ring; + + u64_stats_update_begin(&ring->rx_syncp); + ring->rx_stats.packets += packets; + ring->rx_stats.bytes += bytes; + u64_stats_update_end(&ring->rx_syncp); + + q_vector->rx.total_packets += packets; + q_vector->rx.total_bytes += bytes; +} + static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) { unsigned int total_bytes = 0, total_packets = 0; @@ -2151,12 +2344,9 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) } if (!skb) { - xdp.data = pktbuf + pkt_offset; - xdp.data_end = xdp.data + size; - xdp.data_hard_start = pktbuf - igc_rx_offset(rx_ring); - xdp_set_data_meta_invalid(&xdp); - xdp.frame_sz = truesize; - xdp.rxq = &rx_ring->xdp_rxq; + xdp_init_buff(&xdp, truesize, &rx_ring->xdp_rxq); + xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring), + igc_rx_offset(rx_ring) + pkt_offset, size, false); skb = igc_xdp_run_prog(adapter, &xdp); } @@ -2226,12 +2416,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) /* place incomplete frames back on ring for completion */ rx_ring->skb = skb; - u64_stats_update_begin(&rx_ring->rx_syncp); - rx_ring->rx_stats.packets += total_packets; - rx_ring->rx_stats.bytes += total_bytes; - u64_stats_update_end(&rx_ring->rx_syncp); - q_vector->rx.total_packets += total_packets; - q_vector->rx.total_bytes += total_bytes; + igc_update_rx_stats(q_vector, total_packets, total_bytes); if (cleaned_count) igc_alloc_rx_buffers(rx_ring, cleaned_count); @@ -2239,6 +2424,221 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) return total_packets; } +static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, + struct xdp_buff *xdp) +{ + unsigned int metasize = xdp->data - xdp->data_meta; + unsigned int datasize = xdp->data_end - xdp->data; + unsigned int totalsize = metasize + datasize; + struct sk_buff *skb; + + skb = __napi_alloc_skb(&ring->q_vector->napi, + xdp->data_end - xdp->data_hard_start, + GFP_ATOMIC | __GFP_NOWARN); + if (unlikely(!skb)) + return NULL; + + skb_reserve(skb, xdp->data_meta - xdp->data_hard_start); + memcpy(__skb_put(skb, totalsize), xdp->data_meta, totalsize); + if (metasize) + skb_metadata_set(skb, metasize); + + return skb; +} + +static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, + union igc_adv_rx_desc *desc, + struct xdp_buff *xdp, + ktime_t timestamp) +{ + struct igc_ring *ring = q_vector->rx.ring; + struct sk_buff *skb; + + skb = igc_construct_skb_zc(ring, xdp); + if (!skb) { + ring->rx_stats.alloc_failed++; + return; + } + + if (timestamp) + skb_hwtstamps(skb)->hwtstamp = timestamp; + + if (igc_cleanup_headers(ring, desc, skb)) + return; + + igc_process_skb_fields(ring, desc, skb); + napi_gro_receive(&q_vector->napi, skb); +} + +static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) +{ + struct igc_adapter *adapter = q_vector->adapter; + struct igc_ring *ring = q_vector->rx.ring; + u16 cleaned_count = igc_desc_unused(ring); + int total_bytes = 0, total_packets = 0; + u16 ntc = ring->next_to_clean; + struct bpf_prog *prog; + bool failure = false; + int xdp_status = 0; + + rcu_read_lock(); + + prog = READ_ONCE(adapter->xdp_prog); + + while (likely(total_packets < budget)) { + union igc_adv_rx_desc *desc; + struct igc_rx_buffer *bi; + ktime_t timestamp = 0; + unsigned int size; + int res; + + desc = IGC_RX_DESC(ring, ntc); + size = le16_to_cpu(desc->wb.upper.length); + if (!size) + break; + + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * descriptor has been written back + */ + dma_rmb(); + + bi = &ring->rx_buffer_info[ntc]; + + if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) { + timestamp = igc_ptp_rx_pktstamp(q_vector->adapter, + bi->xdp->data); + + bi->xdp->data += IGC_TS_HDR_LEN; + + /* HW timestamp has been copied into local variable. Metadata + * length when XDP program is called should be 0. + */ + bi->xdp->data_meta += IGC_TS_HDR_LEN; + size -= IGC_TS_HDR_LEN; + } + + bi->xdp->data_end = bi->xdp->data + size; + xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool); + + res = __igc_xdp_run_prog(adapter, prog, bi->xdp); + switch (res) { + case IGC_XDP_PASS: + igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp); + fallthrough; + case IGC_XDP_CONSUMED: + xsk_buff_free(bi->xdp); + break; + case IGC_XDP_TX: + case IGC_XDP_REDIRECT: + xdp_status |= res; + break; + } + + bi->xdp = NULL; + total_bytes += size; + total_packets++; + cleaned_count++; + ntc++; + if (ntc == ring->count) + ntc = 0; + } + + ring->next_to_clean = ntc; + rcu_read_unlock(); + + if (cleaned_count >= IGC_RX_BUFFER_WRITE) + failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count); + + if (xdp_status) + igc_finalize_xdp(adapter, xdp_status); + + igc_update_rx_stats(q_vector, total_packets, total_bytes); + + if (xsk_uses_need_wakeup(ring->xsk_pool)) { + if (failure || ring->next_to_clean == ring->next_to_use) + xsk_set_rx_need_wakeup(ring->xsk_pool); + else + xsk_clear_rx_need_wakeup(ring->xsk_pool); + return total_packets; + } + + return failure ? budget : total_packets; +} + +static void igc_update_tx_stats(struct igc_q_vector *q_vector, + unsigned int packets, unsigned int bytes) +{ + struct igc_ring *ring = q_vector->tx.ring; + + u64_stats_update_begin(&ring->tx_syncp); + ring->tx_stats.bytes += bytes; + ring->tx_stats.packets += packets; + u64_stats_update_end(&ring->tx_syncp); + + q_vector->tx.total_bytes += bytes; + q_vector->tx.total_packets += packets; +} + +static void igc_xdp_xmit_zc(struct igc_ring *ring) +{ + struct xsk_buff_pool *pool = ring->xsk_pool; + struct netdev_queue *nq = txring_txq(ring); + union igc_adv_tx_desc *tx_desc = NULL; + int cpu = smp_processor_id(); + u16 ntu = ring->next_to_use; + struct xdp_desc xdp_desc; + u16 budget; + + if (!netif_carrier_ok(ring->netdev)) + return; + + __netif_tx_lock(nq, cpu); + + budget = igc_desc_unused(ring); + + while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) { + u32 cmd_type, olinfo_status; + struct igc_tx_buffer *bi; + dma_addr_t dma; + + cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | + IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | + xdp_desc.len; + olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; + + dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); + xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); + + tx_desc = IGC_TX_DESC(ring, ntu); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); + tx_desc->read.buffer_addr = cpu_to_le64(dma); + + bi = &ring->tx_buffer_info[ntu]; + bi->type = IGC_TX_BUFFER_TYPE_XSK; + bi->protocol = 0; + bi->bytecount = xdp_desc.len; + bi->gso_segs = 1; + bi->time_stamp = jiffies; + bi->next_to_watch = tx_desc; + + netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len); + + ntu++; + if (ntu == ring->count) + ntu = 0; + } + + ring->next_to_use = ntu; + if (tx_desc) { + igc_flush_tx_descriptors(ring); + xsk_tx_release(pool); + } + + __netif_tx_unlock(nq); +} + /** * igc_clean_tx_irq - Reclaim resources after transmit completes * @q_vector: pointer to q_vector containing needed info @@ -2255,6 +2655,7 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) unsigned int i = tx_ring->next_to_clean; struct igc_tx_buffer *tx_buffer; union igc_adv_tx_desc *tx_desc; + u32 xsk_frames = 0; if (test_bit(__IGC_DOWN, &adapter->state)) return true; @@ -2284,19 +2685,22 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) total_bytes += tx_buffer->bytecount; total_packets += tx_buffer->gso_segs; - if (tx_buffer->tx_flags & IGC_TX_FLAGS_XDP) + switch (tx_buffer->type) { + case IGC_TX_BUFFER_TYPE_XSK: + xsk_frames++; + break; + case IGC_TX_BUFFER_TYPE_XDP: xdp_return_frame(tx_buffer->xdpf); - else + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + case IGC_TX_BUFFER_TYPE_SKB: napi_consume_skb(tx_buffer->skb, napi_budget); - - /* unmap skb header data */ - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - - /* clear tx_buffer data */ - dma_unmap_len_set(tx_buffer, len, 0); + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); + break; + default: + netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n"); + break; + } /* clear last DMA location and unmap remaining buffers */ while (tx_desc != eop_desc) { @@ -2310,13 +2714,8 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) } /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buffer, len)) { - dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buffer, len, 0); - } + if (dma_unmap_len(tx_buffer, len)) + igc_unmap_tx_buffer(tx_ring->dev, tx_buffer); } /* move us one more past the eop_desc for start of next pkt */ @@ -2341,12 +2740,16 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) i += tx_ring->count; tx_ring->next_to_clean = i; - u64_stats_update_begin(&tx_ring->tx_syncp); - tx_ring->tx_stats.bytes += total_bytes; - tx_ring->tx_stats.packets += total_packets; - u64_stats_update_end(&tx_ring->tx_syncp); - q_vector->tx.total_bytes += total_bytes; - q_vector->tx.total_packets += total_packets; + + igc_update_tx_stats(q_vector, total_packets, total_bytes); + + if (tx_ring->xsk_pool) { + if (xsk_frames) + xsk_tx_completed(tx_ring->xsk_pool, xsk_frames); + if (xsk_uses_need_wakeup(tx_ring->xsk_pool)) + xsk_set_tx_need_wakeup(tx_ring->xsk_pool); + igc_xdp_xmit_zc(tx_ring); + } if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { struct igc_hw *hw = &adapter->hw; @@ -2907,6 +3310,8 @@ static void igc_configure(struct igc_adapter *adapter) igc_get_hw_control(adapter); igc_set_rx_mode(netdev); + igc_restore_vlan(adapter); + igc_setup_tctl(adapter); igc_setup_mrqc(adapter); igc_setup_rctl(adapter); @@ -2926,7 +3331,10 @@ static void igc_configure(struct igc_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { struct igc_ring *ring = adapter->rx_ring[i]; - igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); + if (ring->xsk_pool) + igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); + else + igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); } } @@ -3541,14 +3949,17 @@ static int igc_poll(struct napi_struct *napi, int budget) struct igc_q_vector *q_vector = container_of(napi, struct igc_q_vector, napi); + struct igc_ring *rx_ring = q_vector->rx.ring; bool clean_complete = true; int work_done = 0; if (q_vector->tx.ring) clean_complete = igc_clean_tx_irq(q_vector, budget); - if (q_vector->rx.ring) { - int cleaned = igc_clean_rx_irq(q_vector, budget); + if (rx_ring) { + int cleaned = rx_ring->xsk_pool ? + igc_clean_rx_irq_zc(q_vector, budget) : + igc_clean_rx_irq(q_vector, budget); work_done += cleaned; if (cleaned >= budget) @@ -4200,6 +4611,9 @@ static int igc_set_features(struct net_device *netdev, netdev_features_t changed = netdev->features ^ features; struct igc_adapter *adapter = netdev_priv(netdev); + if (changed & NETIF_F_HW_VLAN_CTAG_RX) + igc_vlan_mode(netdev, features); + /* Add VLAN support */ if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) return 0; @@ -5186,6 +5600,9 @@ static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf) switch (bpf->command) { case XDP_SETUP_PROG: return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack); + case XDP_SETUP_XSK_POOL: + return igc_xdp_setup_pool(adapter, bpf->xsk.pool, + bpf->xsk.queue_id); default: return -EOPNOTSUPP; } @@ -5231,6 +5648,43 @@ static int igc_xdp_xmit(struct net_device *dev, int num_frames, return num_frames - drops; } +static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter, + struct igc_q_vector *q_vector) +{ + struct igc_hw *hw = &adapter->hw; + u32 eics = 0; + + eics |= q_vector->eims_value; + wr32(IGC_EICS, eics); +} + +int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) +{ + struct igc_adapter *adapter = netdev_priv(dev); + struct igc_q_vector *q_vector; + struct igc_ring *ring; + + if (test_bit(__IGC_DOWN, &adapter->state)) + return -ENETDOWN; + + if (!igc_xdp_is_enabled(adapter)) + return -ENXIO; + + if (queue_id >= adapter->num_rx_queues) + return -EINVAL; + + ring = adapter->rx_ring[queue_id]; + + if (!ring->xsk_pool) + return -ENXIO; + + q_vector = adapter->q_vector[queue_id]; + if (!napi_if_scheduled_mark_missed(&q_vector->napi)) + igc_trigger_rxtxq_interrupt(adapter, q_vector); + + return 0; +} + static const struct net_device_ops igc_netdev_ops = { .ndo_open = igc_open, .ndo_stop = igc_close, @@ -5246,6 +5700,7 @@ static const struct net_device_ops igc_netdev_ops = { .ndo_setup_tc = igc_setup_tc, .ndo_bpf = igc_bpf, .ndo_xdp_xmit = igc_xdp_xmit, + .ndo_xsk_wakeup = igc_xsk_wakeup, }; /* PCIe configuration access */ @@ -5485,11 +5940,15 @@ static int igc_probe(struct pci_dev *pdev, /* copy netdev features into list of user selectable features */ netdev->hw_features |= NETIF_F_NTUPLE; + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; + netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; netdev->hw_features |= netdev->features; if (pci_using_dac) netdev->features |= NETIF_F_HIGHDMA; + netdev->vlan_features |= netdev->features; + /* MTU range: 68 - 9216 */ netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; @@ -5998,6 +6457,61 @@ struct net_device *igc_get_hw_dev(struct igc_hw *hw) return adapter->netdev; } +static void igc_disable_rx_ring_hw(struct igc_ring *ring) +{ + struct igc_hw *hw = &ring->q_vector->adapter->hw; + u8 idx = ring->reg_idx; + u32 rxdctl; + + rxdctl = rd32(IGC_RXDCTL(idx)); + rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE; + rxdctl |= IGC_RXDCTL_SWFLUSH; + wr32(IGC_RXDCTL(idx), rxdctl); +} + +void igc_disable_rx_ring(struct igc_ring *ring) +{ + igc_disable_rx_ring_hw(ring); + igc_clean_rx_ring(ring); +} + +void igc_enable_rx_ring(struct igc_ring *ring) +{ + struct igc_adapter *adapter = ring->q_vector->adapter; + + igc_configure_rx_ring(adapter, ring); + + if (ring->xsk_pool) + igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring)); + else + igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); +} + +static void igc_disable_tx_ring_hw(struct igc_ring *ring) +{ + struct igc_hw *hw = &ring->q_vector->adapter->hw; + u8 idx = ring->reg_idx; + u32 txdctl; + + txdctl = rd32(IGC_TXDCTL(idx)); + txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE; + txdctl |= IGC_TXDCTL_SWFLUSH; + wr32(IGC_TXDCTL(idx), txdctl); +} + +void igc_disable_tx_ring(struct igc_ring *ring) +{ + igc_disable_tx_ring_hw(ring); + igc_clean_tx_ring(ring); +} + +void igc_enable_tx_ring(struct igc_ring *ring) +{ + struct igc_adapter *adapter = ring->q_vector->adapter; + + igc_configure_tx_ring(adapter, ring); +} + /** * igc_init_module - Driver Registration Routine * diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index cc174853554b..0f82990567d9 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -10,8 +10,8 @@ #define IGC_EECD 0x00010 /* EEPROM/Flash Control - RW */ #define IGC_CTRL_EXT 0x00018 /* Extended Device Control - RW */ #define IGC_MDIC 0x00020 /* MDI Control - RW */ -#define IGC_MDICNFG 0x00E04 /* MDC/MDIO Configuration - RW */ #define IGC_CONNSW 0x00034 /* Copper/Fiber switch control - RW */ +#define IGC_VET 0x00038 /* VLAN Ether Type - RW */ #define IGC_I225_PHPM 0x00E14 /* I225 PHY Power Management */ #define IGC_GPHY_VERSION 0x0001E /* I225 gPHY Firmware Version */ diff --git a/drivers/net/ethernet/intel/igc/igc_xdp.c b/drivers/net/ethernet/intel/igc/igc_xdp.c index 11133c4619bb..a8cf5374be47 100644 --- a/drivers/net/ethernet/intel/igc/igc_xdp.c +++ b/drivers/net/ethernet/intel/igc/igc_xdp.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2020, Intel Corporation. */ +#include <net/xdp_sock_drv.h> + #include "igc.h" #include "igc_xdp.h" @@ -32,29 +34,112 @@ int igc_xdp_set_prog(struct igc_adapter *adapter, struct bpf_prog *prog, return 0; } -int igc_xdp_register_rxq_info(struct igc_ring *ring) +static int igc_xdp_enable_pool(struct igc_adapter *adapter, + struct xsk_buff_pool *pool, u16 queue_id) { - struct net_device *dev = ring->netdev; + struct net_device *ndev = adapter->netdev; + struct device *dev = &adapter->pdev->dev; + struct igc_ring *rx_ring, *tx_ring; + struct napi_struct *napi; + bool needs_reset; + u32 frame_size; int err; - err = xdp_rxq_info_reg(&ring->xdp_rxq, dev, ring->queue_index, 0); - if (err) { - netdev_err(dev, "Failed to register xdp rxq info\n"); - return err; + if (queue_id >= adapter->num_rx_queues || + queue_id >= adapter->num_tx_queues) + return -EINVAL; + + frame_size = xsk_pool_get_rx_frame_size(pool); + if (frame_size < ETH_FRAME_LEN + VLAN_HLEN * 2) { + /* When XDP is enabled, the driver doesn't support frames that + * span over multiple buffers. To avoid that, we check if xsk + * frame size is big enough to fit the max ethernet frame size + * + vlan double tagging. + */ + return -EOPNOTSUPP; } - err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, - NULL); + err = xsk_pool_dma_map(pool, dev, IGC_RX_DMA_ATTR); if (err) { - netdev_err(dev, "Failed to register xdp rxq mem model\n"); - xdp_rxq_info_unreg(&ring->xdp_rxq); + netdev_err(ndev, "Failed to map xsk pool\n"); return err; } + needs_reset = netif_running(adapter->netdev) && igc_xdp_is_enabled(adapter); + + rx_ring = adapter->rx_ring[queue_id]; + tx_ring = adapter->tx_ring[queue_id]; + /* Rx and Tx rings share the same napi context. */ + napi = &rx_ring->q_vector->napi; + + if (needs_reset) { + igc_disable_rx_ring(rx_ring); + igc_disable_tx_ring(tx_ring); + napi_disable(napi); + } + + set_bit(IGC_RING_FLAG_AF_XDP_ZC, &rx_ring->flags); + set_bit(IGC_RING_FLAG_AF_XDP_ZC, &tx_ring->flags); + + if (needs_reset) { + napi_enable(napi); + igc_enable_rx_ring(rx_ring); + igc_enable_tx_ring(tx_ring); + + err = igc_xsk_wakeup(ndev, queue_id, XDP_WAKEUP_RX); + if (err) { + xsk_pool_dma_unmap(pool, IGC_RX_DMA_ATTR); + return err; + } + } + + return 0; +} + +static int igc_xdp_disable_pool(struct igc_adapter *adapter, u16 queue_id) +{ + struct igc_ring *rx_ring, *tx_ring; + struct xsk_buff_pool *pool; + struct napi_struct *napi; + bool needs_reset; + + if (queue_id >= adapter->num_rx_queues || + queue_id >= adapter->num_tx_queues) + return -EINVAL; + + pool = xsk_get_pool_from_qid(adapter->netdev, queue_id); + if (!pool) + return -EINVAL; + + needs_reset = netif_running(adapter->netdev) && igc_xdp_is_enabled(adapter); + + rx_ring = adapter->rx_ring[queue_id]; + tx_ring = adapter->tx_ring[queue_id]; + /* Rx and Tx rings share the same napi context. */ + napi = &rx_ring->q_vector->napi; + + if (needs_reset) { + igc_disable_rx_ring(rx_ring); + igc_disable_tx_ring(tx_ring); + napi_disable(napi); + } + + xsk_pool_dma_unmap(pool, IGC_RX_DMA_ATTR); + clear_bit(IGC_RING_FLAG_AF_XDP_ZC, &rx_ring->flags); + clear_bit(IGC_RING_FLAG_AF_XDP_ZC, &tx_ring->flags); + + if (needs_reset) { + napi_enable(napi); + igc_enable_rx_ring(rx_ring); + igc_enable_tx_ring(tx_ring); + } + return 0; } -void igc_xdp_unregister_rxq_info(struct igc_ring *ring) +int igc_xdp_setup_pool(struct igc_adapter *adapter, struct xsk_buff_pool *pool, + u16 queue_id) { - xdp_rxq_info_unreg(&ring->xdp_rxq); + return pool ? igc_xdp_enable_pool(adapter, pool, queue_id) : + igc_xdp_disable_pool(adapter, queue_id); } diff --git a/drivers/net/ethernet/intel/igc/igc_xdp.h b/drivers/net/ethernet/intel/igc/igc_xdp.h index cfecb515b718..a74e5487d199 100644 --- a/drivers/net/ethernet/intel/igc/igc_xdp.h +++ b/drivers/net/ethernet/intel/igc/igc_xdp.h @@ -6,8 +6,12 @@ int igc_xdp_set_prog(struct igc_adapter *adapter, struct bpf_prog *prog, struct netlink_ext_ack *extack); +int igc_xdp_setup_pool(struct igc_adapter *adapter, struct xsk_buff_pool *pool, + u16 queue_id); -int igc_xdp_register_rxq_info(struct igc_ring *ring); -void igc_xdp_unregister_rxq_info(struct igc_ring *ring); +static inline bool igc_xdp_is_enabled(struct igc_adapter *adapter) +{ + return !!adapter->xdp_prog; +} #endif /* _IGC_XDP_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c index e324e42fab2d..58ea959a4482 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c @@ -1514,8 +1514,7 @@ static u32 ixgbe_get_fdirtcpm_82599(union ixgbe_atr_input *input_mask) #define IXGBE_WRITE_REG_BE32(a, reg, value) \ IXGBE_WRITE_REG((a), (reg), IXGBE_STORE_AS_BE32(ntohl(value))) -#define IXGBE_STORE_AS_BE16(_value) \ - ntohs(((u16)(_value) >> 8) | ((u16)(_value) << 8)) +#define IXGBE_STORE_AS_BE16(_value) __swab16(ntohs((_value))) s32 ixgbe_fdir_set_input_mask_82599(struct ixgbe_hw *hw, union ixgbe_atr_input *input_mask) @@ -1651,13 +1650,13 @@ s32 ixgbe_fdir_write_perfect_filter_82599(struct ixgbe_hw *hw, IXGBE_WRITE_REG_BE32(hw, IXGBE_FDIRIPDA, input->formatted.dst_ip[0]); /* record source and destination port (little-endian)*/ - fdirport = ntohs(input->formatted.dst_port); + fdirport = be16_to_cpu(input->formatted.dst_port); fdirport <<= IXGBE_FDIRPORT_DESTINATION_SHIFT; - fdirport |= ntohs(input->formatted.src_port); + fdirport |= be16_to_cpu(input->formatted.src_port); IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, fdirport); /* record vlan (little-endian) and flex_bytes(big-endian) */ - fdirvlan = IXGBE_STORE_AS_BE16((__force u16)input->formatted.flex_bytes); + fdirvlan = IXGBE_STORE_AS_BE16(input->formatted.flex_bytes); fdirvlan <<= IXGBE_FDIRVLAN_FLEX_SHIFT; fdirvlan |= ntohs(input->formatted.vlan_id); IXGBE_WRITE_REG(hw, IXGBE_FDIRVLAN, fdirvlan); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c index 03ccbe6b66d2..e90b5047e695 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c @@ -3678,10 +3678,8 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, void *buffer, bool return_data) { u32 hdr_size = sizeof(struct ixgbe_hic_hdr); - union { - struct ixgbe_hic_hdr hdr; - u32 u32arr[1]; - } *bp = buffer; + struct ixgbe_hic_hdr *hdr = buffer; + u32 *u32arr = buffer; u16 buf_len, dword_len; s32 status; u32 bi; @@ -3707,12 +3705,12 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, void *buffer, /* first pull in the header so we know the buffer length */ for (bi = 0; bi < dword_len; bi++) { - bp->u32arr[bi] = IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, bi); - le32_to_cpus(&bp->u32arr[bi]); + u32arr[bi] = IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, bi); + le32_to_cpus(&u32arr[bi]); } /* If there is any thing in data position pull it in */ - buf_len = bp->hdr.buf_len; + buf_len = hdr->buf_len; if (!buf_len) goto rel_out; @@ -3727,8 +3725,8 @@ s32 ixgbe_host_interface_command(struct ixgbe_hw *hw, void *buffer, /* Pull in the rest of the buffer (bi is where we left off) */ for (; bi <= dword_len; bi++) { - bp->u32arr[bi] = IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, bi); - le32_to_cpus(&bp->u32arr[bi]); + u32arr[bi] = IXGBE_READ_REG_ARRAY(hw, IXGBE_FLEX_MNG, bi); + le32_to_cpus(&u32arr[bi]); } rel_out: diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c index 54d47265a7ac..e596e1a9fc75 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c @@ -511,14 +511,14 @@ static int ixgbe_ipsec_check_mgmt_ip(struct xfrm_state *xs) continue; reg = IXGBE_READ_REG(hw, MIPAF_ARR(3, i)); - if (reg == xs->id.daddr.a4) + if (reg == (__force u32)xs->id.daddr.a4) return 1; } } if ((bmcipval & BMCIP_MASK) == BMCIP_V4) { reg = IXGBE_READ_REG(hw, IXGBE_BMCIP(3)); - if (reg == xs->id.daddr.a4) + if (reg == (__force u32)xs->id.daddr.a4) return 1; } @@ -533,7 +533,7 @@ static int ixgbe_ipsec_check_mgmt_ip(struct xfrm_state *xs) for (j = 0; j < 4; j++) { reg = IXGBE_READ_REG(hw, MIPAF_ARR(i, j)); - if (reg != xs->id.daddr.a6[j]) + if (reg != (__force u32)xs->id.daddr.a6[j]) break; } if (j == 4) /* did we match all 4 words? */ @@ -543,7 +543,7 @@ static int ixgbe_ipsec_check_mgmt_ip(struct xfrm_state *xs) if ((bmcipval & BMCIP_MASK) == BMCIP_V6) { for (j = 0; j < 4; j++) { reg = IXGBE_READ_REG(hw, IXGBE_BMCIP(j)); - if (reg != xs->id.daddr.a6[j]) + if (reg != (__force u32)xs->id.daddr.a6[j]) break; } if (j == 4) /* did we match all 4 words? */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c5ec17d19c59..ffff69efd78a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -2199,7 +2199,6 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct xdp_frame *xdpf; u32 act; - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); if (!xdp_prog) @@ -2213,23 +2212,23 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter, break; case XDP_TX: xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) { - result = IXGBE_XDP_CONSUMED; - break; - } + if (unlikely(!xdpf)) + goto out_failure; result = ixgbe_xmit_xdp_ring(adapter, xdpf); + if (result == IXGBE_XDP_CONSUMED) + goto out_failure; break; case XDP_REDIRECT: err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); - if (!err) - result = IXGBE_XDP_REDIR; - else - result = IXGBE_XDP_CONSUMED; + if (err) + goto out_failure; + result = IXGBE_XDP_REDIR; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: @@ -2237,7 +2236,6 @@ static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter, break; } xdp_out: - rcu_read_unlock(); return ERR_PTR(-result); } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c index 988db46bff0e..214a38de3f41 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c @@ -467,12 +467,16 @@ static int ixgbe_set_vf_vlan(struct ixgbe_adapter *adapter, int add, int vid, return err; } -static s32 ixgbe_set_vf_lpe(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf) +static int ixgbe_set_vf_lpe(struct ixgbe_adapter *adapter, u32 max_frame, u32 vf) { struct ixgbe_hw *hw = &adapter->hw; - int max_frame = msgbuf[1]; u32 max_frs; + if (max_frame < ETH_MIN_MTU || max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE) { + e_err(drv, "VF max_frame %d out of range\n", max_frame); + return -EINVAL; + } + /* * For 82599EB we have to keep all PFs and VFs operating with * the same max_frame value in order to avoid sending an oversize @@ -533,12 +537,6 @@ static s32 ixgbe_set_vf_lpe(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf) } } - /* MTU < 68 is an error and causes problems on some kernels */ - if (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE) { - e_err(drv, "VF max_frame %d out of range\n", max_frame); - return -EINVAL; - } - /* pull current max frame size from hardware */ max_frs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); max_frs &= IXGBE_MHADD_MFS_MASK; @@ -1249,7 +1247,7 @@ static int ixgbe_rcv_msg_from_vf(struct ixgbe_adapter *adapter, u32 vf) retval = ixgbe_set_vf_vlan_msg(adapter, msgbuf, vf); break; case IXGBE_VF_SET_LPE: - retval = ixgbe_set_vf_lpe(adapter, msgbuf, vf); + retval = ixgbe_set_vf_lpe(adapter, msgbuf[1], vf); break; case IXGBE_VF_SET_MACVLAN: retval = ixgbe_set_vf_macvlan_msg(adapter, msgbuf, vf); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 91ad5b902673..96dd1a4f956a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -100,15 +100,14 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, struct xdp_frame *xdpf; u32 act; - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); act = bpf_prog_run_xdp(xdp_prog, xdp); if (likely(act == XDP_REDIRECT)) { err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - result = !err ? IXGBE_XDP_REDIR : IXGBE_XDP_CONSUMED; - rcu_read_unlock(); - return result; + if (err) + goto out_failure; + return IXGBE_XDP_REDIR; } switch (act) { @@ -116,23 +115,23 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, break; case XDP_TX: xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) { - result = IXGBE_XDP_CONSUMED; - break; - } + if (unlikely(!xdpf)) + goto out_failure; result = ixgbe_xmit_xdp_ring(adapter, xdpf); + if (result == IXGBE_XDP_CONSUMED) + goto out_failure; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBE_XDP_CONSUMED; break; } - rcu_read_unlock(); return result; } diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index ba2ed8a43d2d..c714e1ecd308 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1054,7 +1054,6 @@ static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, struct bpf_prog *xdp_prog; u32 act; - rcu_read_lock(); xdp_prog = READ_ONCE(rx_ring->xdp_prog); if (!xdp_prog) @@ -1067,11 +1066,14 @@ static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, case XDP_TX: xdp_ring = adapter->xdp_ring[rx_ring->queue_index]; result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp); + if (result == IXGBEVF_XDP_CONSUMED) + goto out_failure; break; default: bpf_warn_invalid_xdp_action(act); fallthrough; case XDP_ABORTED: +out_failure: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: @@ -1079,7 +1081,6 @@ static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter, break; } xdp_out: - rcu_read_unlock(); return ERR_PTR(-result); } @@ -3814,7 +3815,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, /* remove payload length from inner checksum */ paylen = skb->len - l4_offset; - csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); + csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); /* update gso size and bytecount with header size */ first->gso_segs = skb_shinfo(skb)->gso_segs; diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c index 6f987a7ffcb3..b30a45725374 100644 --- a/drivers/net/ethernet/korina.c +++ b/drivers/net/ethernet/korina.c @@ -1315,23 +1315,23 @@ static int korina_probe(struct platform_device *pdev) lp->tx_irq = platform_get_irq_byname(pdev, "tx"); p = devm_platform_ioremap_resource_byname(pdev, "emac"); - if (!p) { + if (IS_ERR(p)) { printk(KERN_ERR DRV_NAME ": cannot remap registers\n"); - return -ENOMEM; + return PTR_ERR(p); } lp->eth_regs = p; p = devm_platform_ioremap_resource_byname(pdev, "dma_rx"); - if (!p) { + if (IS_ERR(p)) { printk(KERN_ERR DRV_NAME ": cannot remap Rx DMA registers\n"); - return -ENOMEM; + return PTR_ERR(p); } lp->rx_dma_regs = p; p = devm_platform_ioremap_resource_byname(pdev, "dma_tx"); - if (!p) { + if (IS_ERR(p)) { printk(KERN_ERR DRV_NAME ": cannot remap Tx DMA registers\n"); - return -ENOMEM; + return PTR_ERR(p); } lp->tx_dma_regs = p; diff --git a/drivers/net/ethernet/lantiq_xrx200.c b/drivers/net/ethernet/lantiq_xrx200.c index 41c2ad210bc9..fb78f17d734f 100644 --- a/drivers/net/ethernet/lantiq_xrx200.c +++ b/drivers/net/ethernet/lantiq_xrx200.c @@ -154,6 +154,8 @@ static int xrx200_close(struct net_device *net_dev) static int xrx200_alloc_skb(struct xrx200_chan *ch) { + struct sk_buff *skb = ch->skb[ch->dma.desc]; + dma_addr_t mapping; int ret = 0; ch->skb[ch->dma.desc] = netdev_alloc_skb_ip_align(ch->priv->net_dev, @@ -163,16 +165,18 @@ static int xrx200_alloc_skb(struct xrx200_chan *ch) goto skip; } - ch->dma.desc_base[ch->dma.desc].addr = dma_map_single(ch->priv->dev, - ch->skb[ch->dma.desc]->data, XRX200_DMA_DATA_LEN, - DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(ch->priv->dev, - ch->dma.desc_base[ch->dma.desc].addr))) { + mapping = dma_map_single(ch->priv->dev, ch->skb[ch->dma.desc]->data, + XRX200_DMA_DATA_LEN, DMA_FROM_DEVICE); + if (unlikely(dma_mapping_error(ch->priv->dev, mapping))) { dev_kfree_skb_any(ch->skb[ch->dma.desc]); + ch->skb[ch->dma.desc] = skb; ret = -ENOMEM; goto skip; } + ch->dma.desc_base[ch->dma.desc].addr = mapping; + /* Make sure the address is written before we give it to HW */ + wmb(); skip: ch->dma.desc_base[ch->dma.desc].ctl = LTQ_DMA_OWN | LTQ_DMA_RX_OFFSET(NET_IP_ALIGN) | @@ -196,6 +200,7 @@ static int xrx200_hw_receive(struct xrx200_chan *ch) ch->dma.desc %= LTQ_DESC_NUM; if (ret) { + net_dev->stats.rx_dropped++; netdev_err(net_dev, "failed to allocate new rx buffer\n"); return ret; } @@ -348,8 +353,8 @@ static irqreturn_t xrx200_dma_irq(int irq, void *ptr) struct xrx200_chan *ch = ptr; if (napi_schedule_prep(&ch->napi)) { - __napi_schedule(&ch->napi); ltq_dma_disable_irq(&ch->dma); + __napi_schedule(&ch->napi); } ltq_dma_ack_irq(&ch->dma); @@ -432,7 +437,6 @@ static int xrx200_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; - struct resource *res; struct xrx200_priv *priv; struct net_device *net_dev; int err; @@ -452,13 +456,7 @@ static int xrx200_probe(struct platform_device *pdev) net_dev->max_mtu = XRX200_DMA_DATA_LEN; /* load the memory ranges */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(dev, "failed to get resources\n"); - return -ENOENT; - } - - priv->pmac_reg = devm_ioremap_resource(dev, res); + priv->pmac_reg = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); if (IS_ERR(priv->pmac_reg)) return PTR_ERR(priv->pmac_reg); diff --git a/drivers/net/ethernet/marvell/mvmdio.c b/drivers/net/ethernet/marvell/mvmdio.c index d14762d93640..62a97c46fba0 100644 --- a/drivers/net/ethernet/marvell/mvmdio.c +++ b/drivers/net/ethernet/marvell/mvmdio.c @@ -17,6 +17,8 @@ * warranty of any kind, whether express or implied. */ +#include <linux/acpi.h> +#include <linux/acpi_mdio.h> #include <linux/clk.h> #include <linux/delay.h> #include <linux/interrupt.h> @@ -281,7 +283,7 @@ static int orion_mdio_probe(struct platform_device *pdev) struct orion_mdio_dev *dev; int i, ret; - type = (enum orion_mdio_bus_type)of_device_get_match_data(&pdev->dev); + type = (enum orion_mdio_bus_type)device_get_match_data(&pdev->dev); r = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!r) { @@ -369,7 +371,13 @@ static int orion_mdio_probe(struct platform_device *pdev) goto out_mdio; } - ret = of_mdiobus_register(bus, pdev->dev.of_node); + /* For the platforms not supporting DT/ACPI fall-back + * to mdiobus_register via of_mdiobus_register. + */ + if (is_acpi_node(pdev->dev.fwnode)) + ret = acpi_mdiobus_register(bus, pdev->dev.fwnode); + else + ret = of_mdiobus_register(bus, pdev->dev.of_node); if (ret < 0) { dev_err(&pdev->dev, "Cannot register MDIO bus (%d)\n", ret); goto out_mdio; @@ -421,12 +429,20 @@ static const struct of_device_id orion_mdio_match[] = { }; MODULE_DEVICE_TABLE(of, orion_mdio_match); +static const struct acpi_device_id orion_mdio_acpi_match[] = { + { "MRVL0100", BUS_TYPE_SMI }, + { "MRVL0101", BUS_TYPE_XSMI }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, orion_mdio_acpi_match); + static struct platform_driver orion_mdio_driver = { .probe = orion_mdio_probe, .remove = orion_mdio_remove, .driver = { .name = "orion-mdio", .of_match_table = orion_mdio_match, + .acpi_match_table = ACPI_PTR(orion_mdio_acpi_match), }, }; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 7d5cd9bc6c99..361bc4fbe20b 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -1805,18 +1805,14 @@ static void mvneta_rx_error(struct mvneta_port *pp, } /* Handle RX checksum offload based on the descriptor's status */ -static void mvneta_rx_csum(struct mvneta_port *pp, u32 status, - struct sk_buff *skb) +static int mvneta_rx_csum(struct mvneta_port *pp, u32 status) { if ((pp->dev->features & NETIF_F_RXCSUM) && (status & MVNETA_RXD_L3_IP4) && - (status & MVNETA_RXD_L4_CSUM_OK)) { - skb->csum = 0; - skb->ip_summed = CHECKSUM_UNNECESSARY; - return; - } + (status & MVNETA_RXD_L4_CSUM_OK)) + return CHECKSUM_UNNECESSARY; - skb->ip_summed = CHECKSUM_NONE; + return CHECKSUM_NONE; } /* Return tx queue pointer (find last set bit) according to <cause> returned @@ -2320,7 +2316,7 @@ mvneta_swbm_add_rx_fragment(struct mvneta_port *pp, } static struct sk_buff * -mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, +mvneta_swbm_build_skb(struct mvneta_port *pp, struct page_pool *pool, struct xdp_buff *xdp, u32 desc_status) { struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); @@ -2331,11 +2327,11 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, if (!skb) return ERR_PTR(-ENOMEM); - page_pool_release_page(rxq->page_pool, virt_to_page(xdp->data)); + skb_mark_for_recycle(skb, virt_to_page(xdp->data), pool); skb_reserve(skb, xdp->data - xdp->data_hard_start); skb_put(skb, xdp->data_end - xdp->data); - mvneta_rx_csum(pp, desc_status, skb); + skb->ip_summed = mvneta_rx_csum(pp, desc_status); for (i = 0; i < num_frags; i++) { skb_frag_t *frag = &sinfo->frags[i]; @@ -2343,7 +2339,10 @@ mvneta_swbm_build_skb(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, skb_frag_page(frag), skb_frag_off(frag), skb_frag_size(frag), PAGE_SIZE); - page_pool_release_page(rxq->page_pool, skb_frag_page(frag)); + /* We don't need to reset pp_recycle here. It's already set, so + * just mark fragments for recycling. + */ + page_pool_store_mem_info(skb_frag_page(frag), pool); } return skb; @@ -2370,7 +2369,6 @@ static int mvneta_rx_swbm(struct napi_struct *napi, /* Get number of received packets */ rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); - rcu_read_lock(); xdp_prog = READ_ONCE(pp->xdp_prog); /* Fairness NAPI loop */ @@ -2425,7 +2423,7 @@ static int mvneta_rx_swbm(struct napi_struct *napi, mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps)) goto next; - skb = mvneta_swbm_build_skb(pp, rxq, &xdp_buf, desc_status); + skb = mvneta_swbm_build_skb(pp, rxq->page_pool, &xdp_buf, desc_status); if (IS_ERR(skb)) { struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); @@ -2448,7 +2446,6 @@ next: xdp_buf.data_hard_start = NULL; sinfo.nr_frags = 0; } - rcu_read_unlock(); if (xdp_buf.data_hard_start) mvneta_xdp_put_buff(pp, rxq, &xdp_buf, &sinfo, -1); @@ -2532,7 +2529,7 @@ err_drop_frame: rx_bytes); skb->protocol = eth_type_trans(skb, dev); - mvneta_rx_csum(pp, rx_status, skb); + skb->ip_summed = mvneta_rx_csum(pp, rx_status); napi_gro_receive(napi, skb); rcvd_pkts++; @@ -2581,8 +2578,7 @@ err_drop_frame: skb_put(skb, rx_bytes); skb->protocol = eth_type_trans(skb, dev); - - mvneta_rx_csum(pp, rx_status, skb); + skb->ip_summed = mvneta_rx_csum(pp, rx_status); napi_gro_receive(napi, skb); } diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h index 8edba5ea90f0..b9fbc9f000f2 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h @@ -993,6 +993,14 @@ enum mvpp22_ptp_packet_format { #define MVPP2_DESC_DMA_MASK DMA_BIT_MASK(40) +/* Buffer header info bits */ +#define MVPP2_B_HDR_INFO_MC_ID_MASK 0xfff +#define MVPP2_B_HDR_INFO_MC_ID(info) ((info) & MVPP2_B_HDR_INFO_MC_ID_MASK) +#define MVPP2_B_HDR_INFO_LAST_OFFS 12 +#define MVPP2_B_HDR_INFO_LAST_MASK BIT(12) +#define MVPP2_B_HDR_INFO_IS_LAST(info) \ + (((info) & MVPP2_B_HDR_INFO_LAST_MASK) >> MVPP2_B_HDR_INFO_LAST_OFFS) + struct mvpp2_tai; /* Definitions */ @@ -1002,6 +1010,20 @@ struct mvpp2_rss_table { u32 indir[MVPP22_RSS_TABLE_ENTRIES]; }; +struct mvpp2_buff_hdr { + __le32 next_phys_addr; + __le32 next_dma_addr; + __le16 byte_count; + __le16 info; + __le16 reserved1; /* bm_qset (for future use, BM) */ + u8 next_phys_addr_high; + u8 next_dma_addr_high; + __le16 reserved2; + __le16 reserved3; + __le16 reserved4; + __le16 reserved5; +}; + /* Shared Packet Processor resources */ struct mvpp2 { /* Shared registers' base addresses */ @@ -1175,9 +1197,6 @@ struct mvpp2_port { /* Firmware node associated to the port */ struct fwnode_handle *fwnode; - /* Is a PHY always connected to the port */ - bool has_phy; - /* Per-port registers' base address */ void __iomem *base; void __iomem *stats_base; diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index ec706d614cac..3229bafa2a2c 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -3543,21 +3543,17 @@ static void mvpp2_rx_error(struct mvpp2_port *port, } /* Handle RX checksum offload */ -static void mvpp2_rx_csum(struct mvpp2_port *port, u32 status, - struct sk_buff *skb) +static int mvpp2_rx_csum(struct mvpp2_port *port, u32 status) { if (((status & MVPP2_RXD_L3_IP4) && !(status & MVPP2_RXD_IP4_HEADER_ERR)) || (status & MVPP2_RXD_L3_IP6)) if (((status & MVPP2_RXD_L4_UDP) || (status & MVPP2_RXD_L4_TCP)) && - (status & MVPP2_RXD_L4_CSUM_OK)) { - skb->csum = 0; - skb->ip_summed = CHECKSUM_UNNECESSARY; - return; - } + (status & MVPP2_RXD_L4_CSUM_OK)) + return CHECKSUM_UNNECESSARY; - skb->ip_summed = CHECKSUM_NONE; + return CHECKSUM_NONE; } /* Allocate a new skb and add it to BM pool */ @@ -3784,9 +3780,9 @@ mvpp2_xdp_xmit(struct net_device *dev, int num_frame, } static int -mvpp2_run_xdp(struct mvpp2_port *port, struct mvpp2_rx_queue *rxq, - struct bpf_prog *prog, struct xdp_buff *xdp, - struct page_pool *pp, struct mvpp2_pcpu_stats *stats) +mvpp2_run_xdp(struct mvpp2_port *port, struct bpf_prog *prog, + struct xdp_buff *xdp, struct page_pool *pp, + struct mvpp2_pcpu_stats *stats) { unsigned int len, sync, err; struct page *page; @@ -3839,6 +3835,35 @@ mvpp2_run_xdp(struct mvpp2_port *port, struct mvpp2_rx_queue *rxq, return ret; } +static void mvpp2_buff_hdr_pool_put(struct mvpp2_port *port, struct mvpp2_rx_desc *rx_desc, + int pool, u32 rx_status) +{ + phys_addr_t phys_addr, phys_addr_next; + dma_addr_t dma_addr, dma_addr_next; + struct mvpp2_buff_hdr *buff_hdr; + + phys_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc); + dma_addr = mvpp2_rxdesc_cookie_get(port, rx_desc); + + do { + buff_hdr = (struct mvpp2_buff_hdr *)phys_to_virt(phys_addr); + + phys_addr_next = le32_to_cpu(buff_hdr->next_phys_addr); + dma_addr_next = le32_to_cpu(buff_hdr->next_dma_addr); + + if (port->priv->hw_version >= MVPP22) { + phys_addr_next |= ((u64)buff_hdr->next_phys_addr_high << 32); + dma_addr_next |= ((u64)buff_hdr->next_dma_addr_high << 32); + } + + mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr); + + phys_addr = phys_addr_next; + dma_addr = dma_addr_next; + + } while (!MVPP2_B_HDR_INFO_IS_LAST(le16_to_cpu(buff_hdr->info))); +} + /* Main rx processing */ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, int rx_todo, struct mvpp2_rx_queue *rxq) @@ -3852,8 +3877,6 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, int rx_done = 0; u32 xdp_ret = 0; - rcu_read_lock(); - xdp_prog = READ_ONCE(port->xdp_prog); /* Get number of received packets and clamp the to-do */ @@ -3871,28 +3894,24 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, phys_addr_t phys_addr; u32 rx_status, timestamp; int pool, rx_bytes, err, ret; + struct page *page; void *data; + phys_addr = mvpp2_rxdesc_cookie_get(port, rx_desc); + data = (void *)phys_to_virt(phys_addr); + page = virt_to_page(data); + prefetch(page); + rx_done++; rx_status = mvpp2_rxdesc_status_get(port, rx_desc); rx_bytes = mvpp2_rxdesc_size_get(port, rx_desc); rx_bytes -= MVPP2_MH_SIZE; dma_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc); - phys_addr = mvpp2_rxdesc_cookie_get(port, rx_desc); - data = (void *)phys_to_virt(phys_addr); pool = (rx_status & MVPP2_RXD_BM_POOL_ID_MASK) >> MVPP2_RXD_BM_POOL_ID_OFFS; bm_pool = &port->priv->bm_pools[pool]; - /* In case of an error, release the requested buffer pointer - * to the Buffer Manager. This request process is controlled - * by the hardware, and the information about the buffer is - * comprised by the RX descriptor. - */ - if (rx_status & MVPP2_RXD_ERR_SUMMARY) - goto err_drop_frame; - if (port->priv->percpu_pools) { pp = port->priv->page_pool[pool]; dma_dir = page_pool_get_dma_dir(pp); @@ -3904,8 +3923,20 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, rx_bytes + MVPP2_MH_SIZE, dma_dir); + /* Buffer header not supported */ + if (rx_status & MVPP2_RXD_BUF_HDR) + goto err_drop_frame; + + /* In case of an error, release the requested buffer pointer + * to the Buffer Manager. This request process is controlled + * by the hardware, and the information about the buffer is + * comprised by the RX descriptor. + */ + if (rx_status & MVPP2_RXD_ERR_SUMMARY) + goto err_drop_frame; + /* Prefetch header */ - prefetch(data); + prefetch(data + MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM); if (bm_pool->frag_size > PAGE_SIZE) frag_size = 0; @@ -3925,7 +3956,7 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM, rx_bytes, false); - ret = mvpp2_run_xdp(port, rxq, xdp_prog, &xdp, pp, &ps); + ret = mvpp2_run_xdp(port, xdp_prog, &xdp, pp, &ps); if (ret) { xdp_ret |= ret; @@ -3964,7 +3995,7 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, } if (pp) - page_pool_release_page(pp, virt_to_page(data)); + skb_mark_for_recycle(skb, page, pp); else dma_unmap_single_attrs(dev->dev.parent, dma_addr, bm_pool->buf_size, DMA_FROM_DEVICE, @@ -3975,8 +4006,8 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, skb_reserve(skb, MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM); skb_put(skb, rx_bytes); + skb->ip_summed = mvpp2_rx_csum(port, rx_status); skb->protocol = eth_type_trans(skb, dev); - mvpp2_rx_csum(port, rx_status, skb); napi_gro_receive(napi, skb); continue; @@ -3985,11 +4016,12 @@ err_drop_frame: dev->stats.rx_errors++; mvpp2_rx_error(port, rx_desc); /* Return the buffer to the pool */ - mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr); + if (rx_status & MVPP2_RXD_BUF_HDR) + mvpp2_buff_hdr_pool_put(port, rx_desc, pool, rx_status); + else + mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr); } - rcu_read_unlock(); - if (xdp_ret & MVPP2_XDP_REDIR) xdp_do_flush_map(); @@ -4753,9 +4785,8 @@ static int mvpp2_open(struct net_device *dev) goto err_cleanup_txqs; } - /* Phylink isn't supported yet in ACPI mode */ - if (port->of_node) { - err = phylink_of_phy_connect(port->phylink, port->of_node, 0); + if (port->phylink) { + err = phylink_fwnode_phy_connect(port->phylink, port->fwnode, 0); if (err) { netdev_err(port->dev, "could not attach PHY (%d)\n", err); @@ -6663,6 +6694,19 @@ static void mvpp2_acpi_start(struct mvpp2_port *port) SPEED_UNKNOWN, DUPLEX_UNKNOWN, false, false); } +/* In order to ensure backward compatibility for ACPI, check if the port + * firmware node comprises the necessary description allowing to use phylink. + */ +static bool mvpp2_use_acpi_compat_mode(struct fwnode_handle *port_fwnode) +{ + if (!is_acpi_node(port_fwnode)) + return false; + + return (!fwnode_property_present(port_fwnode, "phy-handle") && + !fwnode_property_present(port_fwnode, "managed") && + !fwnode_get_named_child_node(port_fwnode, "fixed-link")); +} + /* Ports initialization */ static int mvpp2_port_probe(struct platform_device *pdev, struct fwnode_handle *port_fwnode, @@ -6738,7 +6782,6 @@ static int mvpp2_port_probe(struct platform_device *pdev, port = netdev_priv(dev); port->dev = dev; port->fwnode = port_fwnode; - port->has_phy = !!of_find_property(port_node, "phy", NULL); port->ntxqs = ntxqs; port->nrxqs = nrxqs; port->priv = priv; @@ -6881,8 +6924,7 @@ static int mvpp2_port_probe(struct platform_device *pdev, dev->max_mtu = MVPP2_BM_JUMBO_PKT_SIZE; dev->dev.of_node = port_node; - /* Phylink isn't used w/ ACPI as of now */ - if (port_node) { + if (!mvpp2_use_acpi_compat_mode(port_fwnode)) { port->phylink_config.dev = &dev->dev; port->phylink_config.type = PHYLINK_NETDEV; @@ -6894,6 +6936,7 @@ static int mvpp2_port_probe(struct platform_device *pdev, } port->phylink = phylink; } else { + dev_warn(&pdev->dev, "Use link irqs for port#%d. FW update required\n", port->id); port->phylink = NULL; } @@ -7311,7 +7354,6 @@ static int mvpp2_get_sram(struct platform_device *pdev, static int mvpp2_probe(struct platform_device *pdev) { - const struct acpi_device_id *acpi_id; struct fwnode_handle *fwnode = pdev->dev.fwnode; struct fwnode_handle *port_fwnode; struct mvpp2 *priv; @@ -7324,16 +7366,7 @@ static int mvpp2_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; - if (has_acpi_companion(&pdev->dev)) { - acpi_id = acpi_match_device(pdev->dev.driver->acpi_match_table, - &pdev->dev); - if (!acpi_id) - return -EINVAL; - priv->hw_version = (unsigned long)acpi_id->driver_data; - } else { - priv->hw_version = - (unsigned long)of_device_get_match_data(&pdev->dev); - } + priv->hw_version = (unsigned long)device_get_match_data(&pdev->dev); /* multi queue mode isn't supported on PPV2.1, fallback to single * mode @@ -7351,6 +7384,10 @@ static int mvpp2_probe(struct platform_device *pdev) return PTR_ERR(priv->lms_base); } else { res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) { + dev_err(&pdev->dev, "Invalid resource\n"); + return -EINVAL; + } if (has_acpi_companion(&pdev->dev)) { /* In case the MDIO memory region is declared in * the ACPI, it can already appear as 'in-use' @@ -7445,34 +7482,35 @@ static int mvpp2_probe(struct platform_device *pdev) if (err < 0) goto err_gop_clk; - priv->mg_core_clk = devm_clk_get(&pdev->dev, "mg_core_clk"); + priv->mg_core_clk = devm_clk_get_optional(&pdev->dev, "mg_core_clk"); if (IS_ERR(priv->mg_core_clk)) { - priv->mg_core_clk = NULL; - } else { - err = clk_prepare_enable(priv->mg_core_clk); - if (err < 0) - goto err_mg_clk; + err = PTR_ERR(priv->mg_core_clk); + goto err_mg_clk; } + + err = clk_prepare_enable(priv->mg_core_clk); + if (err < 0) + goto err_mg_clk; } - priv->axi_clk = devm_clk_get(&pdev->dev, "axi_clk"); + priv->axi_clk = devm_clk_get_optional(&pdev->dev, "axi_clk"); if (IS_ERR(priv->axi_clk)) { err = PTR_ERR(priv->axi_clk); - if (err == -EPROBE_DEFER) - goto err_mg_core_clk; - priv->axi_clk = NULL; - } else { - err = clk_prepare_enable(priv->axi_clk); - if (err < 0) - goto err_mg_core_clk; + goto err_mg_core_clk; } + err = clk_prepare_enable(priv->axi_clk); + if (err < 0) + goto err_mg_core_clk; + /* Get system's tclk rate */ priv->tclk = clk_get_rate(priv->pp_clk); - } else if (device_property_read_u32(&pdev->dev, "clock-frequency", - &priv->tclk)) { - dev_err(&pdev->dev, "missing clock-frequency value\n"); - return -EINVAL; + } else { + err = device_property_read_u32(&pdev->dev, "clock-frequency", &priv->tclk); + if (err) { + dev_err(&pdev->dev, "missing clock-frequency value\n"); + return err; + } } if (priv->hw_version >= MVPP22) { @@ -7552,6 +7590,8 @@ static int mvpp2_probe(struct platform_device *pdev) return 0; err_port_probe: + fwnode_handle_put(port_fwnode); + i = 0; fwnode_for_each_available_child_node(fwnode, port_fwnode) { if (priv->port_list[i]) @@ -7560,13 +7600,10 @@ err_port_probe: } err_axi_clk: clk_disable_unprepare(priv->axi_clk); - err_mg_core_clk: - if (priv->hw_version >= MVPP22) - clk_disable_unprepare(priv->mg_core_clk); + clk_disable_unprepare(priv->mg_core_clk); err_mg_clk: - if (priv->hw_version >= MVPP22) - clk_disable_unprepare(priv->mg_clk); + clk_disable_unprepare(priv->mg_clk); err_gop_clk: clk_disable_unprepare(priv->gop_clk); err_pp_clk: diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c index 7cc7d72d761e..93575800ca92 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_prs.c @@ -394,9 +394,6 @@ static int mvpp2_prs_tcam_first_free(struct mvpp2 *priv, unsigned char start, if (start > end) swap(start, end); - if (end >= MVPP2_PRS_TCAM_SRAM_SIZE) - end = MVPP2_PRS_TCAM_SRAM_SIZE - 1; - for (tid = start; tid <= end; tid++) { if (!priv->prs_shadow[tid].valid) return tid; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h index e66109367487..47f5ed006a93 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h @@ -197,6 +197,11 @@ enum nix_scheduler { #define SDP_CHANNELS 256 +/* The mask is to extract lower 10-bits of channel number + * which CPT will pass to X2P. + */ +#define NIX_CHAN_CPT_X2P_MASK (0x3ffull) + /* NIX LSO format indices. * As of now TSO is the only one using, so statically assigning indices. */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index cedb2616c509..770d86262838 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -134,6 +134,7 @@ M(MSIX_OFFSET, 0x005, msix_offset, msg_req, msix_offset_rsp) \ M(VF_FLR, 0x006, vf_flr, msg_req, msg_rsp) \ M(PTP_OP, 0x007, ptp_op, ptp_req, ptp_rsp) \ M(GET_HW_CAP, 0x008, get_hw_cap, msg_req, get_hw_cap_rsp) \ +M(SET_VF_PERM, 0x00b, set_vf_perm, set_vf_perm, msg_rsp) \ /* CGX mbox IDs (range 0x200 - 0x3FF) */ \ M(CGX_START_RXTX, 0x200, cgx_start_rxtx, msg_req, msg_rsp) \ M(CGX_STOP_RXTX, 0x201, cgx_stop_rxtx, msg_req, msg_rsp) \ @@ -259,7 +260,11 @@ M(NIX_BP_DISABLE, 0x8017, nix_bp_disable, nix_bp_cfg_req, msg_rsp) \ M(NIX_GET_MAC_ADDR, 0x8018, nix_get_mac_addr, msg_req, nix_get_mac_addr_rsp) \ M(NIX_CN10K_AQ_ENQ, 0x8019, nix_cn10k_aq_enq, nix_cn10k_aq_enq_req, \ nix_cn10k_aq_enq_rsp) \ -M(NIX_GET_HW_INFO, 0x801a, nix_get_hw_info, msg_req, nix_hw_info) +M(NIX_GET_HW_INFO, 0x801c, nix_get_hw_info, msg_req, nix_hw_info) \ +M(NIX_BANDPROF_ALLOC, 0x801d, nix_bandprof_alloc, nix_bandprof_alloc_req, \ + nix_bandprof_alloc_rsp) \ +M(NIX_BANDPROF_FREE, 0x801e, nix_bandprof_free, nix_bandprof_free_req, \ + msg_rsp) /* Messages initiated by AF (range 0xC00 - 0xDFF) */ #define MBOX_UP_CGX_MESSAGES \ @@ -611,7 +616,12 @@ enum nix_af_status { NIX_AF_INVAL_SSO_PF_FUNC = -420, NIX_AF_ERR_TX_VTAG_NOSPC = -421, NIX_AF_ERR_RX_VTAG_INUSE = -422, - NIX_AF_ERR_NPC_KEY_NOT_SUPP = -423, + NIX_AF_ERR_PTP_CONFIG_FAIL = -423, + NIX_AF_ERR_NPC_KEY_NOT_SUPP = -424, + NIX_AF_ERR_INVALID_NIXBLK = -425, + NIX_AF_ERR_INVALID_BANDPROF = -426, + NIX_AF_ERR_IPOLICER_NOTSUPP = -427, + NIX_AF_ERR_BANDPROF_INVAL_REQ = -428, }; /* For NIX RX vtag action */ @@ -680,6 +690,7 @@ struct nix_cn10k_aq_enq_req { struct nix_cq_ctx_s cq; struct nix_rsse_s rss; struct nix_rx_mce_s mce; + struct nix_bandprof_s prof; }; union { struct nix_cn10k_rq_ctx_s rq_mask; @@ -687,6 +698,7 @@ struct nix_cn10k_aq_enq_req { struct nix_cq_ctx_s cq_mask; struct nix_rsse_s rss_mask; struct nix_rx_mce_s mce_mask; + struct nix_bandprof_s prof_mask; }; }; @@ -698,6 +710,7 @@ struct nix_cn10k_aq_enq_rsp { struct nix_cq_ctx_s cq; struct nix_rsse_s rss; struct nix_rx_mce_s mce; + struct nix_bandprof_s prof; }; }; @@ -713,6 +726,7 @@ struct nix_aq_enq_req { struct nix_cq_ctx_s cq; struct nix_rsse_s rss; struct nix_rx_mce_s mce; + u64 prof; }; union { struct nix_rq_ctx_s rq_mask; @@ -720,6 +734,7 @@ struct nix_aq_enq_req { struct nix_cq_ctx_s cq_mask; struct nix_rsse_s rss_mask; struct nix_rx_mce_s mce_mask; + u64 prof_mask; }; }; @@ -731,6 +746,7 @@ struct nix_aq_enq_rsp { struct nix_cq_ctx_s cq; struct nix_rsse_s rss; struct nix_rx_mce_s mce; + struct nix_bandprof_s prof; }; }; @@ -913,6 +929,7 @@ struct nix_rx_mode { #define NIX_RX_MODE_UCAST BIT(0) #define NIX_RX_MODE_PROMISC BIT(1) #define NIX_RX_MODE_ALLMULTI BIT(2) +#define NIX_RX_MODE_USE_MCE BIT(3) u16 mode; }; @@ -971,6 +988,31 @@ struct nix_hw_info { u16 min_mtu; }; +struct nix_bandprof_alloc_req { + struct mbox_msghdr hdr; + /* Count of profiles needed per layer */ + u16 prof_count[BAND_PROF_NUM_LAYERS]; +}; + +struct nix_bandprof_alloc_rsp { + struct mbox_msghdr hdr; + u16 prof_count[BAND_PROF_NUM_LAYERS]; + + /* There is no need to allocate morethan 1 bandwidth profile + * per RQ of a PF_FUNC's NIXLF. So limit the maximum + * profiles to 64 per PF_FUNC. + */ +#define MAX_BANDPROF_PER_PFFUNC 64 + u16 prof_idx[BAND_PROF_NUM_LAYERS][MAX_BANDPROF_PER_PFFUNC]; +}; + +struct nix_bandprof_free_req { + struct mbox_msghdr hdr; + u8 free_all; + u16 prof_count[BAND_PROF_NUM_LAYERS]; + u16 prof_idx[BAND_PROF_NUM_LAYERS][MAX_BANDPROF_PER_PFFUNC]; +}; + /* NPC mbox message structs */ #define NPC_MCAM_ENTRY_INVALID 0xFFFF @@ -1228,6 +1270,14 @@ struct ptp_rsp { u64 clk; }; +struct set_vf_perm { + struct mbox_msghdr hdr; + u16 vf; +#define RESET_VF_PERM BIT_ULL(0) +#define VF_TRUSTED BIT_ULL(1) + u64 flags; +}; + /* CPT mailbox error codes * Range 901 - 1000. */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h index 1e012e787260..19bad9a59c8f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h @@ -33,6 +33,10 @@ enum npc_kpu_la_ltype { NPC_LT_LA_IH_2_ETHER, NPC_LT_LA_HIGIG2_ETHER, NPC_LT_LA_IH_NIX_HIGIG2_ETHER, + NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_LT_LA_CH_LEN_90B_ETHER, + NPC_LT_LA_CPT_HDR, + NPC_LT_LA_CUSTOM_L2_24B_ETHER, NPC_LT_LA_CUSTOM0 = 0xE, NPC_LT_LA_CUSTOM1 = 0xF, }; @@ -42,7 +46,7 @@ enum npc_kpu_lb_ltype { NPC_LT_LB_CTAG, NPC_LT_LB_STAG_QINQ, NPC_LT_LB_BTAG, - NPC_LT_LB_ITAG, + NPC_LT_LB_PPPOE, NPC_LT_LB_DSA, NPC_LT_LB_DSA_VLAN, NPC_LT_LB_EDSA, @@ -50,6 +54,7 @@ enum npc_kpu_lb_ltype { NPC_LT_LB_EXDSA, NPC_LT_LB_EXDSA_VLAN, NPC_LT_LB_FDSA, + NPC_LT_LB_VLAN_EXDSA, NPC_LT_LB_CUSTOM0 = 0xE, NPC_LT_LB_CUSTOM1 = 0xF, }; @@ -65,6 +70,7 @@ enum npc_kpu_lc_ltype { NPC_LT_LC_NSH, NPC_LT_LC_PTP, NPC_LT_LC_FCOE, + NPC_LT_LC_NGIO, NPC_LT_LC_CUSTOM0 = 0xE, NPC_LT_LC_CUSTOM1 = 0xF, }; @@ -146,7 +152,14 @@ enum npc_kpu_lh_ltype { * Ethernet interfaces, LBK interfaces, etc. */ enum npc_pkind_type { - NPC_TX_DEF_PKIND = 63ULL, /* NIX-TX PKIND */ + NPC_RX_VLAN_EXDSA_PKIND = 56ULL, + NPC_RX_CHLEN24B_PKIND = 57ULL, + NPC_RX_CPT_HDR_PKIND, + NPC_RX_CHLEN90B_PKIND, + NPC_TX_HIGIG_PKIND, + NPC_RX_HIGIG_PKIND, + NPC_RX_EDSA_PKIND, + NPC_TX_DEF_PKIND, /* NIX-TX PKIND */ }; /* list of known and supported fields in packet header and @@ -213,7 +226,7 @@ struct npc_kpu_profile_cam { u16 dp1_mask; u16 dp2; u16 dp2_mask; -}; +} __packed; struct npc_kpu_profile_action { u8 errlev; @@ -233,13 +246,13 @@ struct npc_kpu_profile_action { u8 mask; u8 right; u8 shift; -}; +} __packed; struct npc_kpu_profile { int cam_entries; int action_entries; - const struct npc_kpu_profile_cam *cam; - const struct npc_kpu_profile_action *action; + struct npc_kpu_profile_cam *cam; + struct npc_kpu_profile_action *action; }; /* NPC KPU register formats */ @@ -425,7 +438,19 @@ struct nix_tx_action { /* NPC MCAM reserved entry index per nixlf */ #define NIXLF_UCAST_ENTRY 0 #define NIXLF_BCAST_ENTRY 1 -#define NIXLF_PROMISC_ENTRY 2 +#define NIXLF_ALLMULTI_ENTRY 2 +#define NIXLF_PROMISC_ENTRY 3 + +struct npc_coalesced_kpu_prfl { +#define NPC_SIGN 0x00666f727063706e +#define NPC_PRFL_NAME "npc_prfls_array" +#define NPC_NAME_LEN 32 + __le64 signature; /* "npcprof\0" (8 bytes/ASCII characters) */ + u8 name[NPC_NAME_LEN]; /* KPU Profile name */ + u64 version; /* KPU firmware/profile version */ + u8 num_prfl; /* No of NPC profiles. */ + u16 prfl_sz[0]; +}; struct npc_mcam_kex { /* MKEX Profle Header */ @@ -445,6 +470,15 @@ struct npc_mcam_kex { u64 intf_ld_flags[NPC_MAX_INTF][NPC_MAX_LD][NPC_MAX_LFL]; } __packed; +struct npc_kpu_fwdata { + int entries; + /* What follows is: + * struct npc_kpu_profile_cam[entries]; + * struct npc_kpu_profile_action[entries]; + */ + u8 data[0]; +} __packed; + struct npc_lt_def { u8 ltype_mask; u8 ltype_match; @@ -459,6 +493,29 @@ struct npc_lt_def_ipsec { u8 spi_nz; }; +struct npc_lt_def_apad { + u8 ltype_mask; + u8 ltype_match; + u8 lid; + u8 valid; +} __packed; + +struct npc_lt_def_color { + u8 ltype_mask; + u8 ltype_match; + u8 lid; + u8 noffset; + u8 offset; +} __packed; + +struct npc_lt_def_et { + u8 ltype_mask; + u8 ltype_match; + u8 lid; + u8 valid; + u8 offset; +} __packed; + struct npc_lt_def_cfg { struct npc_lt_def rx_ol2; struct npc_lt_def rx_oip4; @@ -476,7 +533,41 @@ struct npc_lt_def_cfg { struct npc_lt_def pck_oip4; struct npc_lt_def pck_oip6; struct npc_lt_def pck_iip4; -}; + struct npc_lt_def_apad rx_apad0; + struct npc_lt_def_apad rx_apad1; + struct npc_lt_def_color ovlan; + struct npc_lt_def_color ivlan; + struct npc_lt_def_color rx_gen0_color; + struct npc_lt_def_color rx_gen1_color; + struct npc_lt_def_et rx_et[2]; +} __packed; + +/* Loadable KPU profile firmware data */ +struct npc_kpu_profile_fwdata { +#define KPU_SIGN 0x00666f727075706b +#define KPU_NAME_LEN 32 +/** Maximum number of custom KPU entries supported by the built-in profile. */ +#define KPU_MAX_CST_ENT 2 + /* KPU Profle Header */ + __le64 signature; /* "kpuprof\0" (8 bytes/ASCII characters) */ + u8 name[KPU_NAME_LEN]; /* KPU Profile name */ + __le64 version; /* KPU profile version */ + u8 kpus; + u8 reserved[7]; + + /* Default MKEX profile to be used with this KPU profile. May be + * overridden with mkex_profile module parameter. Format is same as for + * the MKEX profile to streamline processing. + */ + struct npc_mcam_kex mkex; + /* LTYPE values for specific HW offloaded protocols. */ + struct npc_lt_def_cfg lt_def; + /* Dynamically sized data: + * Custom KPU CAM and ACTION configuration entries. + * struct npc_kpu_fwdata kpu[kpus]; + */ + u8 data[0]; +} __packed; struct rvu_npc_mcam_rule { struct flow_msg packet; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h index 5c372d2c24a1..fee655cc7523 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h @@ -11,7 +11,10 @@ #ifndef NPC_PROFILE_H #define NPC_PROFILE_H -#define NPC_KPU_PROFILE_VER 0x0000000100050000 +#define NPC_KPU_PROFILE_VER 0x0000000100060000 +#define NPC_KPU_VER_MAJ(ver) ((u16)(((ver) >> 32) & 0xFFFF)) +#define NPC_KPU_VER_MIN(ver) ((u16)(((ver) >> 16) & 0xFFFF)) +#define NPC_KPU_VER_PATCH(ver) ((u16)((ver) & 0xFFFF)) #define NPC_IH_W 0x8000 #define NPC_IH_UTAG 0x2000 @@ -20,6 +23,7 @@ #define NPC_ETYPE_IP6 0x86dd #define NPC_ETYPE_ARP 0x0806 #define NPC_ETYPE_RARP 0x8035 +#define NPC_ETYPE_NGIO 0x8842 #define NPC_ETYPE_MPLSU 0x8847 #define NPC_ETYPE_MPLSM 0x8848 #define NPC_ETYPE_ETAG 0x893f @@ -33,6 +37,10 @@ #define NPC_ETYPE_PPP 0x880b #define NPC_ETYPE_NSH 0x894f #define NPC_ETYPE_DSA 0xdada +#define NPC_ETYPE_PPPOE 0x8864 + +#define NPC_PPP_IP 0x0021 +#define NPC_PPP_IP6 0x0057 #define NPC_IPNH_HOP 0 #define NPC_IPNH_ICMP 1 @@ -142,14 +150,15 @@ #define NPC_DSA_EDSA 0x8000 #define NPC_DSA_FDSA 0xc000 -#define NPC_KEXOF_DMAC 8 -#define MKEX_SIGN 0x19bbfdbd15f /* strtoull of "mkexprof" with base:36 */ +#define NPC_KEXOF_DMAC 9 +#define MKEX_SIGN 0x19bbfdbd15f #define KEX_LD_CFG(bytesm1, hdr_ofs, ena, flags_ena, key_ofs) \ (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ ((flags_ena) << 6) | ((key_ofs) & 0x3F)) /* Rx parse key extract nibble enable */ #define NPC_PARSE_NIBBLE_INTF_RX (NPC_PARSE_NIBBLE_CHAN | \ + NPC_PARSE_NIBBLE_ERRCODE | \ NPC_PARSE_NIBBLE_LA_LTYPE | \ NPC_PARSE_NIBBLE_LB_LTYPE | \ NPC_PARSE_NIBBLE_LC_LTYPE | \ @@ -170,25 +179,31 @@ enum npc_kpu_parser_state { NPC_S_KPU1_EXDSA, NPC_S_KPU1_HIGIG2, NPC_S_KPU1_IH_NIX_HIGIG2, + NPC_S_KPU1_CUSTOM_L2_90B, + NPC_S_KPU1_CPT_HDR, + NPC_S_KPU1_CUSTOM_L2_24B, + NPC_S_KPU1_VLAN_EXDSA, NPC_S_KPU2_CTAG, NPC_S_KPU2_CTAG2, NPC_S_KPU2_SBTAG, NPC_S_KPU2_QINQ, NPC_S_KPU2_ETAG, - NPC_S_KPU2_ITAG, NPC_S_KPU2_PREHEADER, NPC_S_KPU2_EXDSA, + NPC_S_KPU2_NGIO, NPC_S_KPU3_CTAG, NPC_S_KPU3_STAG, NPC_S_KPU3_QINQ, - NPC_S_KPU3_ITAG, NPC_S_KPU3_CTAG_C, NPC_S_KPU3_STAG_C, NPC_S_KPU3_QINQ_C, NPC_S_KPU3_DSA, + NPC_S_KPU3_VLAN_EXDSA, NPC_S_KPU4_MPLS, NPC_S_KPU4_NSH, NPC_S_KPU4_FDSA, + NPC_S_KPU4_VLAN_EXDSA, + NPC_S_KPU4_PPPOE, NPC_S_KPU5_IP, NPC_S_KPU5_IP6, NPC_S_KPU5_ARP, @@ -198,13 +213,19 @@ enum npc_kpu_parser_state { NPC_S_KPU5_MPLS, NPC_S_KPU5_MPLS_PL, NPC_S_KPU5_NSH, + NPC_S_KPU5_CPT_IP, + NPC_S_KPU5_CPT_IP6, NPC_S_KPU6_IP6_EXT, NPC_S_KPU6_IP6_HOP_DEST, NPC_S_KPU6_IP6_ROUT, NPC_S_KPU6_IP6_FRAG, + NPC_S_KPU6_IP6_CPT_FRAG, + NPC_S_KPU6_IP6_CPT_HOP_DEST, + NPC_S_KPU6_IP6_CPT_ROUT, NPC_S_KPU7_IP6_EXT, NPC_S_KPU7_IP6_ROUT, NPC_S_KPU7_IP6_FRAG, + NPC_S_KPU7_CPT_IP6_FRAG, NPC_S_KPU8_TCP, NPC_S_KPU8_UDP, NPC_S_KPU8_SCTP, @@ -265,7 +286,6 @@ enum npc_kpu_la_lflag { NPC_F_LA_L_UNK_ETYPE = 1, NPC_F_LA_L_WITH_VLAN, NPC_F_LA_L_WITH_ETAG, - NPC_F_LA_L_WITH_ITAG, NPC_F_LA_L_WITH_MPLS, NPC_F_LA_L_WITH_NSH, }; @@ -442,7 +462,28 @@ enum NPC_ERRLEV_E { NPC_ERRLEV_ENUM_LAST = 16, }; -static const struct npc_kpu_profile_action ikpu_action_entries[] = { +#define NPC_KPU_NOP_CAM \ + { \ + NPC_S_NA, 0xff, \ + 0x0000, \ + 0x0000, \ + 0x0000, \ + 0x0000, \ + 0x0000, \ + 0x0000, \ + } + +#define NPC_KPU_NOP_ACTION \ + { \ + NPC_ERRLEV_RE, NPC_EC_NOERR, \ + 0, 0, 0, 0, 0, \ + NPC_S_NA, 0, 0, \ + NPC_LID_LA, NPC_LT_NA, \ + 0, \ + 0, 0, 0, 0, \ + } + +static struct npc_kpu_profile_action ikpu_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 12, 16, 20, 0, 0, @@ -950,7 +991,7 @@ static const struct npc_kpu_profile_action ikpu_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 12, 16, 20, 0, 0, - NPC_S_KPU1_ETHER, 0, 0, + NPC_S_KPU1_VLAN_EXDSA, 0, 0, NPC_LID_LA, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -958,8 +999,8 @@ static const struct npc_kpu_profile_action ikpu_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 12, 16, 20, 0, 0, - NPC_S_KPU1_ETHER, 0, 0, + 36, 40, 44, 0, 0, + NPC_S_KPU1_CUSTOM_L2_24B, 0, 0, NPC_LID_LA, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -967,8 +1008,8 @@ static const struct npc_kpu_profile_action ikpu_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 12, 16, 20, 0, 0, - NPC_S_KPU1_ETHER, 0, 0, + 40, 54, 58, 0, 0, + NPC_S_KPU1_CPT_HDR, 0, 0, NPC_LID_LA, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -976,8 +1017,8 @@ static const struct npc_kpu_profile_action ikpu_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 12, 16, 20, 0, 0, - NPC_S_KPU1_ETHER, 0, 0, + 102, 106, 110, 0, 0, + NPC_S_KPU1_CUSTOM_L2_90B, 0, 0, NPC_LID_LA, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -1021,7 +1062,9 @@ static const struct npc_kpu_profile_action ikpu_action_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { +static struct npc_kpu_profile_cam kpu1_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU1_ETHER, 0xff, NPC_ETYPE_IP, @@ -1080,6 +1123,15 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { NPC_S_KPU1_ETHER, 0xff, NPC_ETYPE_CTAG, 0xffff, + NPC_ETYPE_NGIO, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_ETHER, 0xff, + NPC_ETYPE_CTAG, + 0xffff, NPC_ETYPE_CTAG, 0xffff, 0x0000, @@ -1123,7 +1175,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_ETHER, 0xff, - NPC_ETYPE_ITAG, + NPC_ETYPE_MPLSU, 0xffff, 0x0000, 0x0000, @@ -1132,7 +1184,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_ETHER, 0xff, - NPC_ETYPE_MPLSU, + NPC_ETYPE_MPLSM, 0xffff, 0x0000, 0x0000, @@ -1141,7 +1193,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_ETHER, 0xff, - NPC_ETYPE_MPLSM, + NPC_ETYPE_NSH, 0xffff, 0x0000, 0x0000, @@ -1150,7 +1202,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_ETHER, 0xff, - NPC_ETYPE_NSH, + NPC_ETYPE_DSA, 0xffff, 0x0000, 0x0000, @@ -1159,7 +1211,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_ETHER, 0xff, - NPC_ETYPE_DSA, + NPC_ETYPE_PPPOE, 0xffff, 0x0000, 0x0000, @@ -1294,15 +1346,6 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_IH_NIX, 0xff, - NPC_ETYPE_ITAG, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU1_IH_NIX, 0xff, NPC_ETYPE_MPLSU, 0xffff, 0x0000, @@ -1339,8 +1382,8 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_IH, 0xff, - NPC_IH_W|NPC_IH_UTAG, - NPC_IH_W|NPC_IH_UTAG, + NPC_IH_W | NPC_IH_UTAG, + NPC_IH_W | NPC_IH_UTAG, 0x0000, 0x0000, 0x0000, @@ -1349,7 +1392,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { { NPC_S_KPU1_IH, 0xff, NPC_IH_W, - NPC_IH_W|NPC_IH_UTAG, + NPC_IH_W | NPC_IH_UTAG, 0x0000, 0x0000, 0x0000, @@ -1358,7 +1401,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { { NPC_S_KPU1_IH, 0xff, 0x0000, - NPC_IH_W|NPC_IH_UTAG, + NPC_IH_W | NPC_IH_UTAG, 0x0000, 0x0000, 0x0000, @@ -1501,15 +1544,6 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_HIGIG2, 0xff, - NPC_ETYPE_ITAG, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU1_HIGIG2, 0xff, NPC_ETYPE_MPLSU, 0xffff, 0x0000, @@ -1645,7 +1679,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_IH_NIX_HIGIG2, 0xff, - NPC_ETYPE_ITAG, + NPC_ETYPE_MPLSU, 0xffff, 0x0000, 0x0000, @@ -1654,7 +1688,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_IH_NIX_HIGIG2, 0xff, - NPC_ETYPE_MPLSU, + NPC_ETYPE_MPLSM, 0xffff, 0x0000, 0x0000, @@ -1663,7 +1697,7 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_IH_NIX_HIGIG2, 0xff, - NPC_ETYPE_MPLSM, + NPC_ETYPE_NSH, 0xffff, 0x0000, 0x0000, @@ -1672,6 +1706,132 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, { NPC_S_KPU1_IH_NIX_HIGIG2, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_IP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_IP6, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_ARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_RARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_PTP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_FCOE, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_CTAG, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_SBTAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_QINQ, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_ETAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_MPLSU, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + NPC_ETYPE_MPLSM, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, NPC_ETYPE_NSH, 0xffff, 0x0000, @@ -1680,7 +1840,88 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { 0x0000, }, { - NPC_S_KPU1_IH_NIX_HIGIG2, 0xff, + NPC_S_KPU1_CUSTOM_L2_90B, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + NPC_ETYPE_IP, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + NPC_ETYPE_IP6, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + NPC_ETYPE_CTAG, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + NPC_ETYPE_QINQ, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + 0x0000, + 0x0000, + NPC_ETYPE_IP, + 0xffff, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + 0x0000, + 0x0000, + NPC_ETYPE_IP6, + 0xffff, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + 0x0000, + 0x0000, + NPC_ETYPE_CTAG, + 0xffff, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, + 0x0000, + 0xffff, + 0x0000, + 0x0000, + NPC_ETYPE_QINQ, + 0xffff, + }, + { + NPC_S_KPU1_CPT_HDR, 0xff, 0x0000, 0x0000, 0x0000, @@ -1689,6 +1930,150 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { 0x0000, }, { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_IP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_IP6, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_ARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_RARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_PTP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_FCOE, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_CTAG, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_SBTAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_QINQ, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_ETAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_MPLSU, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_MPLSM, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + NPC_ETYPE_NSH, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_CUSTOM_L2_24B, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU1_VLAN_EXDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -1699,7 +2084,9 @@ static const struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu2_cam_entries[] = { +static struct npc_kpu_profile_cam kpu2_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU2_CTAG, 0xff, NPC_ETYPE_IP, @@ -1783,6 +2170,24 @@ static const struct npc_kpu_profile_cam kpu2_cam_entries[] = { }, { NPC_S_KPU2_CTAG, 0xff, + NPC_ETYPE_PPPOE, + 0xffff, + 0x0000, + 0x0000, + NPC_PPP_IP, + 0xffff, + }, + { + NPC_S_KPU2_CTAG, 0xff, + NPC_ETYPE_PPPOE, + 0xffff, + 0x0000, + 0x0000, + NPC_PPP_IP6, + 0xffff, + }, + { + NPC_S_KPU2_CTAG, 0xff, 0x0000, 0x0000, 0x0000, @@ -2226,15 +2631,6 @@ static const struct npc_kpu_profile_cam kpu2_cam_entries[] = { NPC_S_KPU2_ETAG, 0xff, NPC_ETYPE_SBTAG, 0xffff, - NPC_ETYPE_ITAG, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ETAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, 0x0000, 0x0000, 0x0000, @@ -2313,159 +2709,6 @@ static const struct npc_kpu_profile_cam kpu2_cam_entries[] = { 0x0000, }, { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_IP, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_IP6, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_ARP, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_RARP, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP, - 0xffff, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP6, - 0xffff, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_ARP, - 0xffff, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_IP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_IP6, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_ARP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP6, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_ARP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU2_ITAG, 0xff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { NPC_S_KPU2_CTAG2, 0xff, NPC_ETYPE_IP, 0xffff, @@ -2817,6 +3060,15 @@ static const struct npc_kpu_profile_cam kpu2_cam_entries[] = { 0x0000, }, { + NPC_S_KPU2_NGIO, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -2827,7 +3079,9 @@ static const struct npc_kpu_profile_cam kpu2_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu3_cam_entries[] = { +static struct npc_kpu_profile_cam kpu3_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU3_CTAG, 0xff, NPC_ETYPE_IP, @@ -3243,159 +3497,6 @@ static const struct npc_kpu_profile_cam kpu3_cam_entries[] = { 0x0000, }, { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_IP, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_IP6, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_ARP, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_RARP, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP, - 0xffff, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP6, - 0xffff, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_ARP, - 0xffff, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_IP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_IP6, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_ARP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - NPC_ETYPE_CTAG, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_SBTAG, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_IP6, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - NPC_ETYPE_ARP, - 0xffff, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - NPC_ETYPE_CTAG, - 0xffff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { - NPC_S_KPU3_ITAG, 0xff, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - 0x0000, - }, - { NPC_S_KPU3_CTAG_C, 0xff, NPC_ETYPE_IP, 0xffff, @@ -3936,6 +4037,15 @@ static const struct npc_kpu_profile_cam kpu3_cam_entries[] = { 0x0000, }, { + NPC_S_KPU3_VLAN_EXDSA, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -3946,7 +4056,9 @@ static const struct npc_kpu_profile_cam kpu3_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu4_cam_entries[] = { +static struct npc_kpu_profile_cam kpu4_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU4_MPLS, 0xff, NPC_MPLS_S, @@ -4084,6 +4196,78 @@ static const struct npc_kpu_profile_cam kpu4_cam_entries[] = { }, { NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_IP, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_IP6, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_ARP, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_RARP, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_PTP, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_CTAG, + 0xffff, + NPC_ETYPE_FCOE, + 0xffff, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_PPPOE, + 0xffff, + 0x0000, + 0x0000, + NPC_PPP_IP, + 0xffff, + }, + { + NPC_S_KPU4_FDSA, 0xff, + NPC_ETYPE_PPPOE, + 0xffff, + 0x0000, + 0x0000, + NPC_PPP_IP6, + 0xffff, + }, + { + NPC_S_KPU4_FDSA, 0xff, 0x0000, NPC_DSA_FDSA, 0x0000, @@ -4092,6 +4276,87 @@ static const struct npc_kpu_profile_cam kpu4_cam_entries[] = { 0x0000, }, { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + NPC_ETYPE_IP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + NPC_ETYPE_IP6, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + NPC_ETYPE_ARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + NPC_ETYPE_RARP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + NPC_ETYPE_PTP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + NPC_ETYPE_FCOE, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_VLAN_EXDSA, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_PPPOE, 0xff, + NPC_PPP_IP, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU4_PPPOE, 0xff, + NPC_PPP_IP6, + 0xffff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -4102,7 +4367,9 @@ static const struct npc_kpu_profile_cam kpu4_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { +static struct npc_kpu_profile_cam kpu5_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU5_IP, 0xff, 0x0000, @@ -4125,116 +4392,116 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_S_KPU5_IP, 0xff, NPC_IPNH_TCP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_UDP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_SCTP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_ICMP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_IGMP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_ESP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_AH, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_GRE, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_IP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_IP6, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, NPC_IPNH_MPLS, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, 0x0000, 0x0000, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, 0x0000, 0x0000, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -4245,7 +4512,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4254,7 +4521,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4263,7 +4530,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4272,7 +4539,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4281,7 +4548,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4290,7 +4557,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4299,7 +4566,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4308,7 +4575,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4317,7 +4584,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4326,7 +4593,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4335,7 +4602,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4344,7 +4611,7 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { NPC_IP_VER_4, NPC_IP_VER_MASK, 0x0000, - NPC_IP_HDR_MF|NPC_IP_HDR_FRAGOFF, + NPC_IP_HDR_MF | NPC_IP_HDR_FRAGOFF, }, { NPC_S_KPU5_IP, 0xff, @@ -4662,6 +4929,429 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { 0x0000, }, { + NPC_S_KPU5_CPT_IP, 0xff, + 0x0000, + NPC_IP_TTL_MASK, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0001, + NPC_IP_HDR_FRAGOFF, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_TCP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_UDP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_SCTP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_ICMP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_IGMP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_ESP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_AH, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_GRE, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_IP, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_IP6, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_MPLS, + 0x00ff, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + 0x0000, + 0x0000, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_TCP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_UDP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_SCTP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_ICMP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_IGMP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_ESP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_AH, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_GRE, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_IP, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_IP6, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + NPC_IPNH_MPLS, + 0x00ff, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + 0x0000, + 0x0000, + NPC_IP_VER_4, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + 0x0000, + NPC_IP6_HOP_MASK, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_TCP << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_UDP << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_SCTP << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_ICMP << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_ICMP6 << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_GRE << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_IP6 << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_MPLS << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_HOP << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_DEST << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_ROUT << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_FRAG << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_ESP << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_AH << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_MOBILITY << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_HOSTID << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + NPC_IPNH_SHIM6 << 8, + 0xff00, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + 0x0000, + 0x0000, + NPC_IP_VER_6, + NPC_IP_VER_MASK, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU5_CPT_IP6, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -4672,7 +5362,9 @@ static const struct npc_kpu_profile_cam kpu5_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu6_cam_entries[] = { +static struct npc_kpu_profile_cam kpu6_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU6_IP6_EXT, 0xff, 0x0000, @@ -5007,6 +5699,330 @@ static const struct npc_kpu_profile_cam kpu6_cam_entries[] = { 0x0000, }, { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_TCP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_UDP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_SCTP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_ICMP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_ICMP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_ESP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_AH << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_GRE << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_IP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + NPC_IPNH_MPLS << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_FRAG, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_TCP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_UDP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_SCTP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_ICMP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_ICMP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_ESP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_AH << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_GRE << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_IP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_MPLS << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_ROUT << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + NPC_IPNH_FRAG << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_HOP_DEST, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_TCP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_UDP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_SCTP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_ICMP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_ICMP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_ESP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_AH << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_GRE << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_IP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_MPLS << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + NPC_IPNH_FRAG << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU6_IP6_CPT_ROUT, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -5017,7 +6033,9 @@ static const struct npc_kpu_profile_cam kpu6_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu7_cam_entries[] = { +static struct npc_kpu_profile_cam kpu7_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU7_IP6_EXT, 0xff, 0x0000, @@ -5226,6 +6244,105 @@ static const struct npc_kpu_profile_cam kpu7_cam_entries[] = { 0x0000, }, { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_TCP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_UDP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_SCTP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_ICMP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_ICMP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_ESP << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_AH << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_GRE << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_IP6 << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + NPC_IPNH_MPLS << 8, + 0xff00, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { + NPC_S_KPU7_CPT_IP6_FRAG, 0xff, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + 0x0000, + }, + { NPC_S_NA, 0X00, 0x0000, 0x0000, @@ -5236,7 +6353,9 @@ static const struct npc_kpu_profile_cam kpu7_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { +static struct npc_kpu_profile_cam kpu8_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU8_TCP, 0xff, 0x0000, @@ -5259,8 +6378,8 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_FIN, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_FIN, 0x0000, 0x0000, }, @@ -5268,8 +6387,8 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_URG|NPC_TCP_FLAGS_SYN, - NPC_TCP_FLAGS_URG|NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_URG | NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_URG | NPC_TCP_FLAGS_SYN, 0x0000, 0x0000, }, @@ -5277,8 +6396,8 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_SYN, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_SYN, 0x0000, 0x0000, }, @@ -5286,8 +6405,8 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_SYN|NPC_TCP_FLAGS_FIN, - NPC_TCP_FLAGS_SYN|NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_SYN | NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_SYN | NPC_TCP_FLAGS_FIN, 0x0000, 0x0000, }, @@ -5565,7 +6684,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSU, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY, 0xffff, 0x0000, 0x0000, @@ -5574,7 +6693,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSU, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5583,7 +6702,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSU, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5592,7 +6711,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSU, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5637,7 +6756,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSM, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY, 0xffff, 0x0000, 0x0000, @@ -5646,7 +6765,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSM, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5655,7 +6774,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSM, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5664,7 +6783,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_MPLSM, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5709,7 +6828,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_NSH, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY, 0xffff, 0x0000, 0x0000, @@ -5718,7 +6837,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_NSH, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5727,7 +6846,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_NSH, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5736,7 +6855,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_NSH, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5781,7 +6900,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY, 0xffff, 0x0000, 0x0000, @@ -5790,7 +6909,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5799,7 +6918,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5808,7 +6927,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5853,7 +6972,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP6, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY, 0xffff, 0x0000, 0x0000, @@ -5862,7 +6981,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP6, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5871,7 +6990,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP6, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5880,7 +6999,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_IP6, 0xffff, - NPC_GRE_F_CSUM|NPC_GRE_F_KEY|NPC_GRE_F_SEQ, + NPC_GRE_F_CSUM | NPC_GRE_F_KEY | NPC_GRE_F_SEQ, 0xffff, 0x0000, 0x0000, @@ -5916,7 +7035,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_PPP, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_VER_1, + NPC_GRE_F_KEY | NPC_GRE_VER_1, 0xffff, 0x0000, 0x0000, @@ -5925,7 +7044,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_PPP, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ|NPC_GRE_VER_1, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ | NPC_GRE_VER_1, 0xffff, 0x0000, 0x0000, @@ -5934,7 +7053,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_PPP, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_ACK|NPC_GRE_VER_1, + NPC_GRE_F_KEY | NPC_GRE_F_ACK | NPC_GRE_VER_1, 0xffff, 0x0000, 0x0000, @@ -5943,7 +7062,7 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { NPC_S_KPU8_GRE, 0xff, NPC_ETYPE_PPP, 0xffff, - NPC_GRE_F_KEY|NPC_GRE_F_SEQ|NPC_GRE_F_ACK|NPC_GRE_VER_1, + NPC_GRE_F_KEY | NPC_GRE_F_SEQ | NPC_GRE_F_ACK | NPC_GRE_VER_1, 0xffff, 0x0000, 0x0000, @@ -5977,7 +7096,9 @@ static const struct npc_kpu_profile_cam kpu8_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu9_cam_entries[] = { +static struct npc_kpu_profile_cam kpu9_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU9_TU_MPLS_IN_GRE, 0xff, NPC_MPLS_S, @@ -6387,8 +7508,8 @@ static const struct npc_kpu_profile_cam kpu9_cam_entries[] = { NPC_S_KPU9_GTPU, 0xff, 0x0000, 0x0000, - 0x0000, - 0x0000, + NPC_GTP_PT_GTP | NPC_GTP_VER1, + NPC_GTP_PT_MASK | NPC_GTP_VER_MASK, 0x0000, 0x0000, }, @@ -6448,7 +7569,9 @@ static const struct npc_kpu_profile_cam kpu9_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu10_cam_entries[] = { +static struct npc_kpu_profile_cam kpu10_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU10_TU_MPLS, 0xff, NPC_MPLS_S, @@ -6613,7 +7736,9 @@ static const struct npc_kpu_profile_cam kpu10_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu11_cam_entries[] = { +static struct npc_kpu_profile_cam kpu11_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU11_TU_ETHER, 0xff, NPC_ETYPE_IP, @@ -6922,13 +8047,15 @@ static const struct npc_kpu_profile_cam kpu11_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { +static struct npc_kpu_profile_cam kpu12_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_TCP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6936,8 +8063,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_UDP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6945,8 +8072,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_SCTP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6954,8 +8081,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_ICMP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6963,8 +8090,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_IGMP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6972,8 +8099,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_ESP, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6981,8 +8108,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, NPC_IPNH_AH, 0x00ff, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -6990,8 +8117,8 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { NPC_S_KPU12_TU_IP, 0xff, 0x0000, 0x0000, - NPC_IP_VER_4|NPC_IP_HDR_LEN_5, - NPC_IP_VER_MASK|NPC_IP_HDR_LEN_MASK, + NPC_IP_VER_4 | NPC_IP_HDR_LEN_5, + NPC_IP_VER_MASK | NPC_IP_HDR_LEN_MASK, 0x0000, 0x0000, }, @@ -7177,7 +8304,9 @@ static const struct npc_kpu_profile_cam kpu12_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu13_cam_entries[] = { +static struct npc_kpu_profile_cam kpu13_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU13_TU_IP6_EXT, 0xff, 0x0000, @@ -7189,7 +8318,9 @@ static const struct npc_kpu_profile_cam kpu13_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu14_cam_entries[] = { +static struct npc_kpu_profile_cam kpu14_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU14_TU_IP6_EXT, 0xff, 0x0000, @@ -7201,7 +8332,9 @@ static const struct npc_kpu_profile_cam kpu14_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu15_cam_entries[] = { +static struct npc_kpu_profile_cam kpu15_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU15_TU_TCP, 0xff, 0x0000, @@ -7224,8 +8357,8 @@ static const struct npc_kpu_profile_cam kpu15_cam_entries[] = { NPC_S_KPU15_TU_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_FIN, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_FIN, 0x0000, 0x0000, }, @@ -7233,8 +8366,8 @@ static const struct npc_kpu_profile_cam kpu15_cam_entries[] = { NPC_S_KPU15_TU_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_URG|NPC_TCP_FLAGS_SYN, - NPC_TCP_FLAGS_URG|NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_URG | NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_URG | NPC_TCP_FLAGS_SYN, 0x0000, 0x0000, }, @@ -7242,8 +8375,8 @@ static const struct npc_kpu_profile_cam kpu15_cam_entries[] = { NPC_S_KPU15_TU_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_SYN, - NPC_TCP_FLAGS_RST|NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_SYN, + NPC_TCP_FLAGS_RST | NPC_TCP_FLAGS_SYN, 0x0000, 0x0000, }, @@ -7251,8 +8384,8 @@ static const struct npc_kpu_profile_cam kpu15_cam_entries[] = { NPC_S_KPU15_TU_TCP, 0xff, 0x0000, 0x0000, - NPC_TCP_FLAGS_SYN|NPC_TCP_FLAGS_FIN, - NPC_TCP_FLAGS_SYN|NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_SYN | NPC_TCP_FLAGS_FIN, + NPC_TCP_FLAGS_SYN | NPC_TCP_FLAGS_FIN, 0x0000, 0x0000, }, @@ -7402,7 +8535,9 @@ static const struct npc_kpu_profile_cam kpu15_cam_entries[] = { }, }; -static const struct npc_kpu_profile_cam kpu16_cam_entries[] = { +static struct npc_kpu_profile_cam kpu16_cam_entries[] = { + NPC_KPU_NOP_CAM, + NPC_KPU_NOP_CAM, { NPC_S_KPU16_TCP_DATA, 0xff, 0x0000, @@ -7459,7 +8594,9 @@ static const struct npc_kpu_profile_cam kpu16_cam_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu1_action_entries[] = { +static struct npc_kpu_profile_action kpu1_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 3, 0, @@ -7511,6 +8648,14 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 12, 0, 0, 0, + NPC_S_KPU2_NGIO, 12, 1, + NPC_LID_LA, NPC_LT_LA_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 12, 0, 0, 0, NPC_S_KPU2_CTAG2, 12, 1, NPC_LID_LA, NPC_LT_LA_ETHER, NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, @@ -7518,7 +8663,7 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 4, 8, 0, 0, 0, + 4, 8, 12, 0, 0, NPC_S_KPU2_CTAG, 12, 1, NPC_LID_LA, NPC_LT_LA_ETHER, NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, @@ -7550,14 +8695,6 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 18, 22, 26, 0, 0, - NPC_S_KPU2_ITAG, 12, 1, - NPC_LID_LA, NPC_LT_LA_ETHER, - NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_ITAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 6, 10, 2, 0, NPC_S_KPU4_MPLS, 14, 1, NPC_LID_LA, NPC_LT_LA_ETHER, @@ -7590,6 +8727,14 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 12, 0, 2, 0, + NPC_S_KPU4_PPPOE, 12, 1, + NPC_LID_LA, NPC_LT_LA_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, NPC_LID_LA, NPC_LT_LA_8023, @@ -7707,15 +8852,6 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 18, 22, 26, 0, 0, - NPC_S_KPU2_ITAG, 20, 1, - NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, - NPC_F_LA_U_HAS_IH_NIX | NPC_F_LA_U_HAS_TAG - | NPC_F_LA_L_WITH_ITAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 6, 10, 2, 0, NPC_S_KPU4_MPLS, 22, 1, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, @@ -7788,7 +8924,7 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 4, 8, 16, 2, 0, + 4, 8, 12, 2, 0, NPC_S_KPU4_FDSA, 12, 1, NPC_LID_LA, NPC_LT_LA_ETHER, 0, @@ -7897,15 +9033,6 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 18, 22, 26, 0, 0, - NPC_S_KPU2_ITAG, 28, 1, - NPC_LID_LA, NPC_LT_LA_HIGIG2_ETHER, - NPC_F_LA_U_HAS_HIGIG2 | NPC_F_LA_U_HAS_TAG - | NPC_F_LA_L_WITH_ITAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 6, 10, 2, 0, NPC_S_KPU4_MPLS, 30, 1, NPC_LID_LA, NPC_LT_LA_HIGIG2_ETHER, @@ -8031,15 +9158,6 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 18, 22, 26, 0, 0, - NPC_S_KPU2_ITAG, 36, 1, - NPC_LID_LA, NPC_LT_LA_IH_NIX_HIGIG2_ETHER, - NPC_F_LA_U_HAS_IH_NIX | NPC_F_LA_U_HAS_HIGIG2 - | NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_ITAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 6, 10, 2, 0, NPC_S_KPU4_MPLS, 38, 1, NPC_LID_LA, NPC_LT_LA_IH_NIX_HIGIG2_ETHER, @@ -8075,6 +9193,326 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 3, 0, + NPC_S_KPU5_IP, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 3, 0, + NPC_S_KPU5_IP6, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_ARP, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_RARP, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_PTP, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_FCOE, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 12, 0, 0, 0, + NPC_S_KPU2_CTAG2, 102, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_CTAG, 102, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 22, 0, 0, + NPC_S_KPU2_SBTAG, 102, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_QINQ, 102, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 12, 26, 0, 0, + NPC_S_KPU2_ETAG, 102, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_ETAG, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU4_MPLS, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_L_WITH_MPLS, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU4_MPLS, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_L_WITH_MPLS, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 0, 0, 2, 0, + NPC_S_KPU4_NSH, 104, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_L_WITH_NSH, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_90B_ETHER, + NPC_F_LA_L_UNK_ETYPE, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 3, 0, + NPC_S_KPU5_CPT_IP, 56, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 3, 0, + NPC_S_KPU5_CPT_IP6, 56, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_CTAG, 54, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_QINQ, 54, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 3, 0, + NPC_S_KPU5_CPT_IP, 60, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 3, 0, + NPC_S_KPU5_CPT_IP6, 60, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_CTAG, 58, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_QINQ, 58, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LA, NPC_LT_LA_CPT_HDR, + NPC_F_LA_L_UNK_ETYPE, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 3, 0, + NPC_S_KPU5_IP, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 3, 0, + NPC_S_KPU5_IP6, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_ARP, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_RARP, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_PTP, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU5_FCOE, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 12, 0, 0, 0, + NPC_S_KPU2_CTAG2, 36, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_CTAG, 36, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 22, 0, 0, + NPC_S_KPU2_SBTAG, 36, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 4, 8, 0, 0, 0, + NPC_S_KPU2_QINQ, 36, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_VLAN, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 12, 26, 0, 0, + NPC_S_KPU2_ETAG, 36, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_U_HAS_TAG | NPC_F_LA_L_WITH_ETAG, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU4_MPLS, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_L_WITH_MPLS, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU4_MPLS, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_L_WITH_MPLS, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 0, 0, 2, 0, + NPC_S_KPU4_NSH, 38, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_L_WITH_NSH, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LA, NPC_LT_LA_CUSTOM_L2_24B_ETHER, + NPC_F_LA_L_UNK_ETYPE, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 12, 0, 0, 1, 0, + NPC_S_KPU3_VLAN_EXDSA, 12, 1, + NPC_LID_LA, NPC_LT_LA_ETHER, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LA, NPC_EC_L2_K1, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -8084,7 +9522,9 @@ static const struct npc_kpu_profile_action kpu1_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu2_action_entries[] = { +static struct npc_kpu_profile_action kpu2_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 2, 0, @@ -8159,6 +9599,22 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 2, 0, + NPC_S_KPU5_IP, 14, 1, + NPC_LID_LB, NPC_LT_LB_PPPOE, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 2, 0, + NPC_S_KPU5_IP6, 14, 1, + NPC_LID_LB, NPC_LT_LB_PPPOE, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, NPC_LID_LB, NPC_LT_LB_CTAG, @@ -8170,7 +9626,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 8, 0, 6, 2, 0, NPC_S_KPU5_IP, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8178,7 +9634,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 6, 0, 0, 2, 0, NPC_S_KPU5_IP6, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8186,7 +9642,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_ARP, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8194,7 +9650,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_RARP, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8202,7 +9658,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_PTP, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8210,7 +9666,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_FCOE, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8218,7 +9674,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 10, 1, 0, NPC_S_KPU4_MPLS, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8226,7 +9682,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 10, 1, 0, NPC_S_KPU4_MPLS, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8234,7 +9690,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 1, 0, NPC_S_KPU4_NSH, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8242,7 +9698,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG_UNK, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG_UNK, 0, 0, 0, 0, }, { @@ -8250,7 +9706,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 0, 0, 0, NPC_S_KPU3_CTAG, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_STAG_CTAG, 0, 0, 0, 0, }, { @@ -8258,7 +9714,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 0, 0, 0, NPC_S_KPU3_STAG, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG_STAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_STAG_STAG, 0, 0, 0, 0, }, { @@ -8266,7 +9722,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 8, 0, 6, 2, 0, NPC_S_KPU5_IP, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8274,7 +9730,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 6, 0, 0, 2, 0, NPC_S_KPU5_IP6, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8282,7 +9738,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_ARP, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8290,7 +9746,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_RARP, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8298,7 +9754,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_PTP, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8306,7 +9762,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_FCOE, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8314,7 +9770,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 10, 1, 0, NPC_S_KPU4_MPLS, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8322,7 +9778,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 10, 1, 0, NPC_S_KPU4_MPLS, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8330,7 +9786,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 1, 0, NPC_S_KPU4_NSH, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8338,7 +9794,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 0, 0, NPC_S_KPU3_STAG, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG_STAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG_STAG, 0, 0, 0, 0, }, { @@ -8346,7 +9802,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 0, 0, NPC_S_KPU3_CTAG, 24, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG_CTAG, 0, 0, 0, 0, }, { @@ -8354,7 +9810,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, NPC_LID_LB, NPC_LT_LB_BTAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG_UNK, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG_UNK, 0, 0, 0, 0, }, { @@ -8546,15 +10002,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 0, 0, NPC_S_KPU3_CTAG, 10, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 16, 20, 24, 0, 0, - NPC_S_KPU3_ITAG, 14, 1, - NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_BTAG_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_CTAG, 0, 0, 0, 0, }, { @@ -8562,7 +10010,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 0, 0, 0, NPC_S_KPU3_STAG, 10, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_STAG, 0, 0, 0, 0, }, { @@ -8570,7 +10018,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 6, 0, 0, 0, NPC_S_KPU3_QINQ, 10, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_QINQ, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_QINQ, 0, 0, 0, 0, }, { @@ -8578,7 +10026,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 8, 0, 6, 2, 0, NPC_S_KPU5_IP, 28, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8586,7 +10034,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 6, 0, 0, 2, 0, NPC_S_KPU5_IP6, 28, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8594,7 +10042,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 2, 0, NPC_S_KPU5_ARP, 28, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG, 0, 0, 0, 0, }, { @@ -8602,7 +10050,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 0, 0, NPC_S_KPU3_STAG, 28, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG_STAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG_STAG, 0, 0, 0, 0, }, { @@ -8610,7 +10058,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 2, 0, 0, 0, 0, NPC_S_KPU3_CTAG, 28, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG_CTAG, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG_CTAG, 0, 0, 0, 0, }, { @@ -8618,7 +10066,7 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, NPC_LID_LB, NPC_LT_LB_ETAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_ITAG_UNK, + NPC_F_LB_U_MORE_TAG | NPC_F_LB_L_WITH_ITAG_UNK, 0, 0, 0, 0, }, { @@ -8632,142 +10080,6 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 2, 0, - NPC_S_KPU5_IP, 20, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 2, 0, - NPC_S_KPU5_IP6, 20, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_ARP, 20, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_RARP, 20, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 2, 0, - NPC_S_KPU5_IP, 28, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 2, 0, - NPC_S_KPU5_IP6, 28, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_ARP, 28, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 2, 0, - NPC_S_KPU5_IP, 24, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 2, 0, - NPC_S_KPU5_IP6, 24, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_ARP, 24, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_STAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 2, 0, - NPC_S_KPU5_IP, 24, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 2, 0, - NPC_S_KPU5_IP6, 24, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_ARP, 24, 1, - NPC_LID_LB, NPC_LT_LB_ITAG, - NPC_F_LB_U_MORE_TAG|NPC_F_LB_L_WITH_CTAG, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 2, 0, NPC_S_KPU5_IP, 10, 1, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 0, @@ -9078,6 +10390,14 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_NGIO, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LB, NPC_EC_L2_K3, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -9087,11 +10407,13 @@ static const struct npc_kpu_profile_action kpu2_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu3_action_entries[] = { +static struct npc_kpu_profile_action kpu3_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 1, 0, - NPC_S_KPU5_IP, 4, 0, + NPC_S_KPU5_IP, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9099,7 +10421,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 6, 0, 0, 1, 0, - NPC_S_KPU5_IP6, 4, 0, + NPC_S_KPU5_IP6, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9107,7 +10429,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 1, 0, - NPC_S_KPU5_ARP, 4, 0, + NPC_S_KPU5_ARP, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9115,7 +10437,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 1, 0, - NPC_S_KPU5_RARP, 4, 0, + NPC_S_KPU5_RARP, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9123,7 +10445,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 1, 0, - NPC_S_KPU5_PTP, 4, 0, + NPC_S_KPU5_PTP, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9131,7 +10453,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 1, 0, - NPC_S_KPU5_FCOE, 4, 0, + NPC_S_KPU5_FCOE, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9139,7 +10461,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 6, 10, 0, 0, - NPC_S_KPU4_MPLS, 4, 0, + NPC_S_KPU4_MPLS, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9147,7 +10469,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 6, 10, 0, 0, - NPC_S_KPU4_MPLS, 4, 0, + NPC_S_KPU4_MPLS, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9155,7 +10477,7 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 0, 0, 0, 0, - NPC_S_KPU4_NSH, 4, 0, + NPC_S_KPU4_NSH, 6, 0, NPC_LID_LB, NPC_LT_NA, 0, 0, 0, 0, 0, @@ -9458,142 +10780,6 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 2, 0, - NPC_S_KPU5_IP, 18, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 2, 0, - NPC_S_KPU5_IP6, 18, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_ARP, 18, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 2, 0, - NPC_S_KPU5_RARP, 18, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 1, 0, - NPC_S_KPU5_IP, 26, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 1, 0, - NPC_S_KPU5_IP6, 26, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU5_ARP, 26, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 1, 0, - NPC_S_KPU5_IP, 22, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 1, 0, - NPC_S_KPU5_IP6, 22, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU5_ARP, 22, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 8, 0, 6, 1, 0, - NPC_S_KPU5_IP, 22, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 6, 0, 0, 1, 0, - NPC_S_KPU5_IP6, 22, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 1, 0, - NPC_S_KPU5_ARP, 22, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_LB, NPC_EC_L2_K3_ETYPE_UNK, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 0, - NPC_LID_LB, NPC_LT_NA, - 0, - 0, 0, 0, 0, - }, - { - NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 1, 0, NPC_S_KPU5_IP, 4, 1, NPC_LID_LB, NPC_LT_LB_CTAG, @@ -10073,6 +11259,14 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU4_VLAN_EXDSA, 12, 1, + NPC_LID_LB, NPC_LT_LB_VLAN_EXDSA, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LB, NPC_EC_L2_K3, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -10082,7 +11276,9 @@ static const struct npc_kpu_profile_action kpu3_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu4_action_entries[] = { +static struct npc_kpu_profile_action kpu4_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 0, @@ -10205,6 +11401,70 @@ static const struct npc_kpu_profile_action kpu4_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_IP, 10, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_IP6, 10, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU5_ARP, 10, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_RARP, 10, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_PTP, 10, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU5_FCOE, 10, 1, + NPC_LID_LB, NPC_LT_LB_FDSA, + NPC_F_LB_L_FDSA, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_IP, 14, 1, + NPC_LID_LB, NPC_LT_LB_PPPOE, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_IP6, 14, 1, + NPC_LID_LB, NPC_LT_LB_PPPOE, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, NPC_S_NA, 0, 1, NPC_LID_LB, NPC_LT_LB_FDSA, @@ -10212,6 +11472,78 @@ static const struct npc_kpu_profile_action kpu4_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_IP, 2, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_IP6, 2, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU5_ARP, 2, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_RARP, 2, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_PTP, 2, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU5_FCOE, 2, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 0, 0, + NPC_S_KPU5_IP, 10, 0, + NPC_LID_LB, NPC_LT_LB_PPPOE, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 0, 0, + NPC_S_KPU5_IP6, 10, 0, + NPC_LID_LB, NPC_LT_LB_PPPOE, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LB, NPC_EC_L2_K4, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -10221,7 +11553,9 @@ static const struct npc_kpu_profile_action kpu4_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu5_action_entries[] = { +static struct npc_kpu_profile_action kpu5_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_LC, NPC_EC_IP_TTL_0, 0, 0, 0, 0, 1, @@ -10719,6 +12053,382 @@ static const struct npc_kpu_profile_action kpu5_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_LC, NPC_EC_IP_TTL_0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_LC, NPC_EC_IP_FRAG_OFFSET_1, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP, + NPC_F_LC_U_IP_FRAG, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 2, 0, + NPC_S_KPU8_TCP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 0, 0, 2, 0, + NPC_S_KPU8_UDP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_SCTP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_ICMP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_IGMP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU9_ESP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_AH, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 0, 0, 2, 0, + NPC_S_KPU8_GRE, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 6, 0, + NPC_S_KPU12_TU_IP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + NPC_F_LC_L_IP_IN_IP, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 6, 0, + NPC_S_KPU12_TU_IP6, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + NPC_F_LC_L_6TO4, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 3, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP, + NPC_F_LC_L_MPLS_IN_IP, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP, + NPC_F_LC_U_UNK_PROTO, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 2, 0, + NPC_S_KPU8_TCP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 8, 10, 2, 0, + NPC_S_KPU8_UDP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_SCTP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_ICMP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_IGMP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU9_ESP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_AH, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 0, 0, 2, 0, + NPC_S_KPU8_GRE, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + 0, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 8, 0, 6, 6, 0, + NPC_S_KPU12_TU_IP, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + NPC_F_LC_L_IP_IN_IP, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 6, 0, + NPC_S_KPU12_TU_IP6, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + NPC_F_LC_L_6TO4, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 3, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 20, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + NPC_F_LC_L_MPLS_IN_IP, + 0, 0xf, 0, 2, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP_OPT, + NPC_F_LC_U_UNK_PROTO, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_LC, NPC_EC_IP_VER, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_LC, NPC_EC_IP6_HOP_0, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 2, 0, + NPC_S_KPU8_TCP, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 0, 0, 2, 0, + NPC_S_KPU8_UDP, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_SCTP, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_ICMP, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_ICMP6, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_GRE, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 6, 0, + NPC_S_KPU12_TU_IP6, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + NPC_F_LC_L_IP6_TUN_IP6, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 3, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + NPC_F_LC_L_IP6_MPLS_IN_IP, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU6_IP6_CPT_HOP_DEST, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_L_EXT_HOP, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU6_IP6_CPT_HOP_DEST, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_L_EXT_DEST, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU6_IP6_CPT_ROUT, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_L_EXT_ROUT, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 2, 0, 0, 0, + NPC_S_KPU6_IP6_CPT_FRAG, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_U_IP6_FRAG, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 3, 0, + NPC_S_KPU9_ESP, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU8_AH, 40, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_L_EXT_MOBILITY, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_L_EXT_HOSTID, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP6_EXT, + NPC_F_LC_L_EXT_SHIM6, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + NPC_F_LC_U_UNK_PROTO, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_LC, NPC_EC_IP6_VER, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 1, + NPC_LID_LC, NPC_LT_LC_IP6, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LC, NPC_EC_UNK, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -10728,7 +12438,9 @@ static const struct npc_kpu_profile_action kpu5_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu6_action_entries[] = { +static struct npc_kpu_profile_action kpu6_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, @@ -11026,6 +12738,294 @@ static const struct npc_kpu_profile_action kpu6_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 1, 0, + NPC_S_KPU8_TCP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 8, 10, 1, 0, + NPC_S_KPU8_UDP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_SCTP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_ICMP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_ICMP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU9_ESP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_AH, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_GRE, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 5, 0, + NPC_S_KPU12_TU_IP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 1, 0, + NPC_S_KPU8_TCP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 8, 10, 1, 0, + NPC_S_KPU8_UDP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_SCTP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_ICMP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_ICMP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU9_ESP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_AH, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_GRE, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 5, 0, + NPC_S_KPU12_TU_IP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU7_IP6_ROUT, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 2, 0, 0, 0, + NPC_S_KPU7_CPT_IP6_FRAG, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 1, 0, + NPC_S_KPU8_TCP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 8, 10, 1, 0, + NPC_S_KPU8_UDP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_SCTP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_ICMP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_ICMP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 2, 0, + NPC_S_KPU9_ESP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_AH, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU8_GRE, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 5, 0, + NPC_S_KPU12_TU_IP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 2, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 2, 0, 0, 0, + NPC_S_KPU7_CPT_IP6_FRAG, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 1, 0xff, 0, 3, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LC, NPC_EC_UNK, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -11035,7 +13035,9 @@ static const struct npc_kpu_profile_action kpu6_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu7_action_entries[] = { +static struct npc_kpu_profile_action kpu7_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, @@ -11221,6 +13223,94 @@ static const struct npc_kpu_profile_action kpu7_action_entries[] = { 0, 0, 0, 0, }, { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 12, 0, 0, 0, + NPC_S_KPU8_TCP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 8, 10, 0, 0, + NPC_S_KPU8_UDP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU8_SCTP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU8_ICMP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU8_ICMP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 1, 0, + NPC_S_KPU9_ESP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU8_AH, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 0, + NPC_S_KPU8_GRE, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 6, 0, 0, 4, 0, + NPC_S_KPU12_TU_IP6, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 2, 6, 10, 1, 0, + NPC_S_KPU9_TU_MPLS_IN_IP, 8, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { + NPC_ERRLEV_RE, NPC_EC_NOERR, + 0, 0, 0, 0, 1, + NPC_S_NA, 0, 0, + NPC_LID_LC, NPC_LT_NA, + 0, + 0, 0, 0, 0, + }, + { NPC_ERRLEV_LC, NPC_EC_UNK, 0, 0, 0, 0, 1, NPC_S_NA, 0, 0, @@ -11230,7 +13320,9 @@ static const struct npc_kpu_profile_action kpu7_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu8_action_entries[] = { +static struct npc_kpu_profile_action kpu8_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_LD, NPC_EC_TCP_FLAGS_FIN_ONLY, 0, 0, 0, 0, 1, @@ -11889,7 +13981,9 @@ static const struct npc_kpu_profile_action kpu8_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu9_action_entries[] = { +static struct npc_kpu_profile_action kpu9_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 0, @@ -12252,10 +14346,10 @@ static const struct npc_kpu_profile_action kpu9_action_entries[] = { }, { NPC_ERRLEV_RE, NPC_EC_NOERR, - 0, 0, 0, 0, 1, - NPC_S_NA, 0, 1, + 8, 0, 6, 2, 0, + NPC_S_KPU12_TU_IP, 8, 1, NPC_LID_LE, NPC_LT_LE_GTPU, - NPC_F_LE_L_GTPU_UNK, + 0, 0, 0, 0, 0, }, { @@ -12308,7 +14402,9 @@ static const struct npc_kpu_profile_action kpu9_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu10_action_entries[] = { +static struct npc_kpu_profile_action kpu10_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 1, 0, @@ -12455,7 +14551,9 @@ static const struct npc_kpu_profile_action kpu10_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu11_action_entries[] = { +static struct npc_kpu_profile_action kpu11_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 8, 0, 6, 0, 0, @@ -12730,7 +14828,9 @@ static const struct npc_kpu_profile_action kpu11_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu12_action_entries[] = { +static struct npc_kpu_profile_action kpu12_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 2, 12, 0, 2, 0, @@ -12957,7 +15057,9 @@ static const struct npc_kpu_profile_action kpu12_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu13_action_entries[] = { +static struct npc_kpu_profile_action kpu13_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, @@ -12968,7 +15070,9 @@ static const struct npc_kpu_profile_action kpu13_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu14_action_entries[] = { +static struct npc_kpu_profile_action kpu14_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, @@ -12979,7 +15083,9 @@ static const struct npc_kpu_profile_action kpu14_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu15_action_entries[] = { +static struct npc_kpu_profile_action kpu15_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_LG, NPC_EC_TCP_FLAGS_FIN_ONLY, 0, 0, 0, 0, 1, @@ -13158,7 +15264,9 @@ static const struct npc_kpu_profile_action kpu15_action_entries[] = { }, }; -static const struct npc_kpu_profile_action kpu16_action_entries[] = { +static struct npc_kpu_profile_action kpu16_action_entries[] = { + NPC_KPU_NOP_ACTION, + NPC_KPU_NOP_ACTION, { NPC_ERRLEV_RE, NPC_EC_NOERR, 0, 0, 0, 0, 1, @@ -13209,7 +15317,7 @@ static const struct npc_kpu_profile_action kpu16_action_entries[] = { }, }; -static const struct npc_kpu_profile npc_kpu_profiles[] = { +static struct npc_kpu_profile npc_kpu_profiles[] = { { ARRAY_SIZE(kpu1_cam_entries), ARRAY_SIZE(kpu1_action_entries), @@ -13308,12 +15416,22 @@ static const struct npc_kpu_profile npc_kpu_profiles[] = { }, }; -static const struct npc_lt_def_cfg npc_lt_defaults = { +static struct npc_lt_def_cfg npc_lt_defaults = { .rx_ol2 = { .lid = NPC_LID_LA, .ltype_match = NPC_LT_LA_ETHER, .ltype_mask = 0x0F, }, + .ovlan = { + .lid = NPC_LID_LB, + .ltype_match = NPC_LT_LB_CTAG, + .ltype_mask = 0x0F, + }, + .ivlan = { + .lid = NPC_LID_LB, + .ltype_match = NPC_LT_LB_STAG_QINQ, + .ltype_mask = 0x0F, + }, .rx_oip4 = { .lid = NPC_LID_LC, .ltype_match = NPC_LT_LC_IP, @@ -13392,6 +15510,30 @@ static const struct npc_lt_def_cfg npc_lt_defaults = { .ltype_match = NPC_LT_LG_TU_IP, .ltype_mask = 0x0F, }, + .rx_apad0 = { + .valid = 0, + .lid = NPC_LID_LC, + .ltype_match = NPC_LT_LC_IP6, + .ltype_mask = 0x0F, + }, + .rx_apad1 = { + .valid = 0, + .lid = NPC_LID_LC, + .ltype_match = NPC_LT_LC_IP6, + .ltype_mask = 0x0F, + }, + .rx_et = { + { + .lid = NPC_LID_LB, + .ltype_match = NPC_LT_NA, + .ltype_mask = 0x0, + }, + { + .lid = NPC_LID_LB, + .ltype_match = NPC_LT_NA, + .ltype_mask = 0x0, + }, + }, }; static struct npc_mcam_kex npc_mkex_default = { @@ -13399,7 +15541,7 @@ static struct npc_mcam_kex npc_mkex_default = { .name = "default", .kpu_version = NPC_KPU_PROFILE_VER, .keyx_cfg = { - /* nibble: LA..LE (ltype only) + channel */ + /* nibble: LA..LE (ltype only) + Error code + Channel */ [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_X2 << 32) | NPC_PARSE_NIBBLE_INTF_RX, /* nibble: LA..LE (ltype only) */ [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | NPC_PARSE_NIBBLE_INTF_TX, @@ -13410,30 +15552,40 @@ static struct npc_mcam_kex npc_mkex_default = { [NPC_LID_LA] = { /* Layer A: Ethernet: */ [NPC_LT_LA_ETHER] = { - /* DMAC: 6 bytes, KW1[47:0] */ + /* DMAC: 6 bytes, KW1[55:8] */ KEX_LD_CFG(0x05, 0x0, 0x1, 0x0, NPC_KEXOF_DMAC), - /* Ethertype: 2 bytes, KW0[47:32] */ - KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, 0x4), + /* Ethertype: 2 bytes, KW0[55:40] */ + KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, 0x5), + }, + /* Layer A: HiGig2: */ + [NPC_LT_LA_HIGIG2_ETHER] = { + /* Classification: 2 bytes, KW1[23:8] */ + KEX_LD_CFG(0x01, 0x8, 0x1, 0x0, NPC_KEXOF_DMAC), + /* VID: 2 bytes, KW1[39:24] */ + KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, + NPC_KEXOF_DMAC + 2), }, }, [NPC_LID_LB] = { /* Layer B: Single VLAN (CTAG) */ - /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ [NPC_LT_LB_CTAG] = { - KEX_LD_CFG(0x03, 0x2, 0x1, 0x0, 0x4), + /* CTAG VLAN: 2 bytes, KW1[7:0], KW0[63:56] */ + KEX_LD_CFG(0x01, 0x2, 0x1, 0x0, 0x7), + /* Ethertype: 2 bytes, KW0[55:40] */ + KEX_LD_CFG(0x01, 0x4, 0x1, 0x0, 0x5), }, /* Layer B: Stacked VLAN (STAG|QinQ) */ [NPC_LT_LB_STAG_QINQ] = { - /* Outer VLAN: 2 bytes, KW0[63:48] */ - KEX_LD_CFG(0x01, 0x2, 0x1, 0x0, 0x6), - /* Ethertype: 2 bytes, KW0[47:32] */ - KEX_LD_CFG(0x01, 0x8, 0x1, 0x0, 0x4), + /* Outer VLAN: 2 bytes, KW1[7:0], KW0[63:56] */ + KEX_LD_CFG(0x01, 0x2, 0x1, 0x0, 0x7), + /* Ethertype: 2 bytes, KW0[55:40] */ + KEX_LD_CFG(0x01, 0x8, 0x1, 0x0, 0x5), }, [NPC_LT_LB_FDSA] = { - /* SWITCH PORT: 1 byte, KW0[63:48] */ - KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0x6), - /* Ethertype: 2 bytes, KW0[47:32] */ - KEX_LD_CFG(0x01, 0x4, 0x1, 0x0, 0x4), + /* SWITCH PORT: 1 byte, KW0[63:56] */ + KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0x7), + /* Ethertype: 2 bytes, KW0[55:40] */ + KEX_LD_CFG(0x01, 0x4, 0x1, 0x0, 0x5), }, }, [NPC_LID_LC] = { @@ -13477,6 +15629,13 @@ static struct npc_mcam_kex npc_mkex_default = { /* DMAC: 6 bytes, KW1[63:16] */ KEX_LD_CFG(0x05, 0x8, 0x1, 0x0, 0xa), }, + /* Layer A: HiGig2: */ + [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = { + /* PF_FUNC: 2B , KW0 [47:32] */ + KEX_LD_CFG(0x01, 0x0, 0x1, 0x0, 0x4), + /* VID: 2 bytes, KW1[31:16] */ + KEX_LD_CFG(0x01, 0x10, 0x1, 0x0, 0xa), + }, }, [NPC_LID_LB] = { /* Layer B: Single VLAN (CTAG) */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index ab24a5e8ee8a..0b092949d7ac 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -57,6 +57,10 @@ static char *mkex_profile; /* MKEX profile name */ module_param(mkex_profile, charp, 0000); MODULE_PARM_DESC(mkex_profile, "MKEX profile name string"); +static char *kpu_profile; /* KPU profile name */ +module_param(kpu_profile, charp, 0000); +MODULE_PARM_DESC(kpu_profile, "KPU profile name string"); + static void rvu_setup_hw_capabilities(struct rvu *rvu) { struct rvu_hwinfo *hw = rvu->hw; @@ -180,6 +184,14 @@ int rvu_rsrc_free_count(struct rsrc_bmap *rsrc) return (rsrc->max - used); } +bool is_rsrc_free(struct rsrc_bmap *rsrc, int id) +{ + if (!rsrc->bmap) + return false; + + return !test_bit(id, rsrc->bmap); +} + int rvu_alloc_bitmap(struct rsrc_bmap *rsrc) { rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max), @@ -1754,6 +1766,48 @@ int rvu_mbox_handler_get_hw_cap(struct rvu *rvu, struct msg_req *req, return 0; } +int rvu_mbox_handler_set_vf_perm(struct rvu *rvu, struct set_vf_perm *req, + struct msg_rsp *rsp) +{ + struct rvu_hwinfo *hw = rvu->hw; + u16 pcifunc = req->hdr.pcifunc; + struct rvu_pfvf *pfvf; + int blkaddr, nixlf; + u16 target; + + /* Only PF can add VF permissions */ + if ((pcifunc & RVU_PFVF_FUNC_MASK) || is_afvf(pcifunc)) + return -EOPNOTSUPP; + + target = (pcifunc & ~RVU_PFVF_FUNC_MASK) | (req->vf + 1); + pfvf = rvu_get_pfvf(rvu, target); + + if (req->flags & RESET_VF_PERM) { + pfvf->flags &= RVU_CLEAR_VF_PERM; + } else if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) ^ + (req->flags & VF_TRUSTED)) { + change_bit(PF_SET_VF_TRUSTED, &pfvf->flags); + /* disable multicast and promisc entries */ + if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags)) { + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, target); + if (blkaddr < 0) + return 0; + nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], + target, 0); + if (nixlf < 0) + return 0; + npc_enadis_default_mce_entry(rvu, target, nixlf, + NIXLF_ALLMULTI_ENTRY, + false); + npc_enadis_default_mce_entry(rvu, target, nixlf, + NIXLF_PROMISC_ENTRY, + false); + } + } + + return 0; +} + static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid, struct mbox_msghdr *req) { @@ -2842,6 +2896,8 @@ static void rvu_update_module_params(struct rvu *rvu) strscpy(rvu->mkex_pfl_name, mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN); + strscpy(rvu->kpu_pfl_name, + kpu_profile ? kpu_profile : default_pfl_name, KPU_NAME_LEN); } static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index c2cc4806d13c..9e5d9ba6f01e 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -223,13 +223,17 @@ struct rvu_pfvf { u16 maxlen; u16 minlen; - u8 pf_set_vf_cfg; u8 mac_addr[ETH_ALEN]; /* MAC address of this PF/VF */ u8 default_mac[ETH_ALEN]; /* MAC address from FWdata */ - /* Broadcast pkt replication info */ + /* Broadcast/Multicast/Promisc pkt replication info */ u16 bcast_mce_idx; + u16 mcast_mce_idx; + u16 promisc_mce_idx; struct nix_mce_list bcast_mce_list; + struct nix_mce_list mcast_mce_list; + struct nix_mce_list promisc_mce_list; + bool use_mce_list; struct rvu_npc_mcam_rule *def_ucast_rule; @@ -239,8 +243,18 @@ struct rvu_pfvf { u8 nix_blkaddr; /* BLKADDR_NIX0/1 assigned to this PF */ u8 nix_rx_intf; /* NIX0_RX/NIX1_RX interface to NPC */ u8 nix_tx_intf; /* NIX0_TX/NIX1_TX interface to NPC */ + unsigned long flags; }; +enum rvu_pfvf_flags { + NIXLF_INITIALIZED = 0, + PF_SET_VF_MAC, + PF_SET_VF_CFG, + PF_SET_VF_TRUSTED, +}; + +#define RVU_CLEAR_VF_PERM ~GENMASK(PF_SET_VF_TRUSTED, PF_SET_VF_MAC) + struct nix_txsch { struct rsrc_bmap schq; u8 lvl; @@ -282,6 +296,13 @@ struct nix_txvlan { struct mutex rsrc_lock; /* Serialize resource alloc/free */ }; +struct nix_ipolicer { + struct rsrc_bmap band_prof; + u16 *pfvf_map; + u16 *match_id; + u16 *ref_count; +}; + struct nix_hw { int blkaddr; struct rvu *rvu; @@ -291,6 +312,7 @@ struct nix_hw { struct nix_mark_format mark_format; struct nix_lso lso; struct nix_txvlan txvlan; + struct nix_ipolicer *ipolicer; }; /* RVU block's capabilities or functionality, @@ -308,6 +330,7 @@ struct hw_cap { bool nix_rx_multicast; /* Rx packet replication support */ bool per_pf_mbox_regs; /* PF mbox specified in per PF registers ? */ bool programmable_chans; /* Channels programmable ? */ + bool ipolicer; }; struct rvu_hwinfo { @@ -386,6 +409,7 @@ struct npc_kpu_profile_adapter { const struct npc_kpu_profile_action *ikpu; /* array[pkinds] */ const struct npc_kpu_profile *kpu; /* array[kpus] */ struct npc_mcam_kex *mkex; + bool custom; size_t pkinds; size_t kpus; }; @@ -435,9 +459,13 @@ struct rvu { struct mutex cgx_cfg_lock; /* serialize cgx configuration */ char mkex_pfl_name[MKEX_NAME_LEN]; /* Configured MKEX profile name */ + char kpu_pfl_name[KPU_NAME_LEN]; /* Configured KPU profile name */ /* Firmware data */ struct rvu_fwdata *fwdata; + void *kpu_fwdata; + size_t kpu_fwdata_sz; + void __iomem *kpu_prfl_addr; /* NPC KPU data */ struct npc_kpu_profile_adapter kpu; @@ -543,11 +571,16 @@ static inline u16 rvu_nix_chan_cpt(struct rvu *rvu, u8 chan) /* Function Prototypes * RVU */ -static inline int is_afvf(u16 pcifunc) +static inline bool is_afvf(u16 pcifunc) { return !(pcifunc & ~RVU_PFVF_FUNC_MASK); } +static inline bool is_vf(u16 pcifunc) +{ + return !!(pcifunc & RVU_PFVF_FUNC_MASK); +} + /* check if PF_FUNC is AF */ static inline bool is_pffunc_af(u16 pcifunc) { @@ -563,6 +596,7 @@ static inline bool is_rvu_fwdata_valid(struct rvu *rvu) int rvu_alloc_bitmap(struct rsrc_bmap *rsrc); int rvu_alloc_rsrc(struct rsrc_bmap *rsrc); void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id); +bool is_rsrc_free(struct rsrc_bmap *rsrc, int id); int rvu_rsrc_free_count(struct rsrc_bmap *rsrc); int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc); bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc); @@ -603,6 +637,12 @@ static inline void rvu_get_cgx_lmac_id(u8 map, u8 *cgx_id, u8 *lmac_id) *lmac_id = (map & 0xF); } +static inline bool is_cgx_vf(struct rvu *rvu, u16 pcifunc) +{ + return ((pcifunc & RVU_PFVF_FUNC_MASK) && + is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))); +} + #define M(_name, _id, fn_name, req, rsp) \ int rvu_mbox_handler_ ## fn_name(struct rvu *, struct req *, struct rsp *); MBOX_MESSAGES @@ -632,10 +672,22 @@ void rvu_nix_freemem(struct rvu *rvu); int rvu_get_nixlf_count(struct rvu *rvu); void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int npalf); int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr); -int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add); +int nix_update_mce_list(struct rvu *rvu, u16 pcifunc, + struct nix_mce_list *mce_list, + int mce_idx, int mcam_index, bool add); +void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type, + struct nix_mce_list **mce_list, int *mce_idx); struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr); int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr); void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc); +int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc, + struct nix_hw **nix_hw, int *blkaddr); +int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc, + u16 rq_idx, u16 match_id); +int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw, + struct nix_cn10k_aq_enq_req *aq_req, + struct nix_cn10k_aq_enq_rsp *aq_rsp, + u16 pcifunc, u8 ctype, u32 qidx); /* NPC APIs */ int rvu_npc_init(struct rvu *rvu); @@ -646,13 +698,19 @@ int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool en); void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, u64 chan, u8 *mac_addr); void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, - int nixlf, u64 chan, u8 chan_cnt, - bool allmulti); -void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf); -void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf); + int nixlf, u64 chan, u8 chan_cnt); +void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + bool enable); void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, int nixlf, u64 chan); -void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable); +void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + bool enable); +void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + u64 chan); +void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + bool enable); +void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, + int nixlf, int type, bool enable); void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf); void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf); void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 9bf8eaabf9ab..3cc3c6fd1d84 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -1632,6 +1632,165 @@ static int rvu_dbg_nix_qsize_display(struct seq_file *filp, void *unused) RVU_DEBUG_SEQ_FOPS(nix_qsize, nix_qsize_display, nix_qsize_write); +static void print_band_prof_ctx(struct seq_file *m, + struct nix_bandprof_s *prof) +{ + char *str; + + switch (prof->pc_mode) { + case NIX_RX_PC_MODE_VLAN: + str = "VLAN"; + break; + case NIX_RX_PC_MODE_DSCP: + str = "DSCP"; + break; + case NIX_RX_PC_MODE_GEN: + str = "Generic"; + break; + case NIX_RX_PC_MODE_RSVD: + str = "Reserved"; + break; + } + seq_printf(m, "W0: pc_mode\t\t%s\n", str); + str = (prof->icolor == 3) ? "Color blind" : + (prof->icolor == 0) ? "Green" : + (prof->icolor == 1) ? "Yellow" : "Red"; + seq_printf(m, "W0: icolor\t\t%s\n", str); + seq_printf(m, "W0: tnl_ena\t\t%d\n", prof->tnl_ena); + seq_printf(m, "W0: peir_exponent\t%d\n", prof->peir_exponent); + seq_printf(m, "W0: pebs_exponent\t%d\n", prof->pebs_exponent); + seq_printf(m, "W0: cir_exponent\t%d\n", prof->cir_exponent); + seq_printf(m, "W0: cbs_exponent\t%d\n", prof->cbs_exponent); + seq_printf(m, "W0: peir_mantissa\t%d\n", prof->peir_mantissa); + seq_printf(m, "W0: pebs_mantissa\t%d\n", prof->pebs_mantissa); + seq_printf(m, "W0: cir_mantissa\t%d\n", prof->cir_mantissa); + + seq_printf(m, "W1: cbs_mantissa\t%d\n", prof->cbs_mantissa); + str = (prof->lmode == 0) ? "byte" : "packet"; + seq_printf(m, "W1: lmode\t\t%s\n", str); + seq_printf(m, "W1: l_select\t\t%d\n", prof->l_sellect); + seq_printf(m, "W1: rdiv\t\t%d\n", prof->rdiv); + seq_printf(m, "W1: adjust_exponent\t%d\n", prof->adjust_exponent); + seq_printf(m, "W1: adjust_mantissa\t%d\n", prof->adjust_mantissa); + str = (prof->gc_action == 0) ? "PASS" : + (prof->gc_action == 1) ? "DROP" : "RED"; + seq_printf(m, "W1: gc_action\t\t%s\n", str); + str = (prof->yc_action == 0) ? "PASS" : + (prof->yc_action == 1) ? "DROP" : "RED"; + seq_printf(m, "W1: yc_action\t\t%s\n", str); + str = (prof->rc_action == 0) ? "PASS" : + (prof->rc_action == 1) ? "DROP" : "RED"; + seq_printf(m, "W1: rc_action\t\t%s\n", str); + seq_printf(m, "W1: meter_algo\t\t%d\n", prof->meter_algo); + seq_printf(m, "W1: band_prof_id\t%d\n", prof->band_prof_id); + seq_printf(m, "W1: hl_en\t\t%d\n", prof->hl_en); + + seq_printf(m, "W2: ts\t\t\t%lld\n", (u64)prof->ts); + seq_printf(m, "W3: pe_accum\t\t%d\n", prof->pe_accum); + seq_printf(m, "W3: c_accum\t\t%d\n", prof->c_accum); + seq_printf(m, "W4: green_pkt_pass\t%lld\n", + (u64)prof->green_pkt_pass); + seq_printf(m, "W5: yellow_pkt_pass\t%lld\n", + (u64)prof->yellow_pkt_pass); + seq_printf(m, "W6: red_pkt_pass\t%lld\n", (u64)prof->red_pkt_pass); + seq_printf(m, "W7: green_octs_pass\t%lld\n", + (u64)prof->green_octs_pass); + seq_printf(m, "W8: yellow_octs_pass\t%lld\n", + (u64)prof->yellow_octs_pass); + seq_printf(m, "W9: red_octs_pass\t%lld\n", (u64)prof->red_octs_pass); + seq_printf(m, "W10: green_pkt_drop\t%lld\n", + (u64)prof->green_pkt_drop); + seq_printf(m, "W11: yellow_pkt_drop\t%lld\n", + (u64)prof->yellow_pkt_drop); + seq_printf(m, "W12: red_pkt_drop\t%lld\n", (u64)prof->red_pkt_drop); + seq_printf(m, "W13: green_octs_drop\t%lld\n", + (u64)prof->green_octs_drop); + seq_printf(m, "W14: yellow_octs_drop\t%lld\n", + (u64)prof->yellow_octs_drop); + seq_printf(m, "W15: red_octs_drop\t%lld\n", (u64)prof->red_octs_drop); + seq_puts(m, "==============================\n"); +} + +static int rvu_dbg_nix_band_prof_ctx_display(struct seq_file *m, void *unused) +{ + struct nix_hw *nix_hw = m->private; + struct nix_cn10k_aq_enq_req aq_req; + struct nix_cn10k_aq_enq_rsp aq_rsp; + struct rvu *rvu = nix_hw->rvu; + struct nix_ipolicer *ipolicer; + int layer, prof_idx, idx, rc; + u16 pcifunc; + char *str; + + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + if (layer == BAND_PROF_INVAL_LAYER) + continue; + str = (layer == BAND_PROF_LEAF_LAYER) ? "Leaf" : + (layer == BAND_PROF_MID_LAYER) ? "Mid" : "Top"; + + seq_printf(m, "\n%s bandwidth profiles\n", str); + seq_puts(m, "=======================\n"); + + ipolicer = &nix_hw->ipolicer[layer]; + + for (idx = 0; idx < ipolicer->band_prof.max; idx++) { + if (is_rsrc_free(&ipolicer->band_prof, idx)) + continue; + + prof_idx = (idx & 0x3FFF) | (layer << 14); + rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, + 0x00, NIX_AQ_CTYPE_BANDPROF, + prof_idx); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to fetch context of %s profile %d, err %d\n", + __func__, str, idx, rc); + return 0; + } + seq_printf(m, "\n%s bandwidth profile:: %d\n", str, idx); + pcifunc = ipolicer->pfvf_map[idx]; + if (!(pcifunc & RVU_PFVF_FUNC_MASK)) + seq_printf(m, "Allocated to :: PF %d\n", + rvu_get_pf(pcifunc)); + else + seq_printf(m, "Allocated to :: PF %d VF %d\n", + rvu_get_pf(pcifunc), + (pcifunc & RVU_PFVF_FUNC_MASK) - 1); + print_band_prof_ctx(m, &aq_rsp.prof); + } + } + return 0; +} + +RVU_DEBUG_SEQ_FOPS(nix_band_prof_ctx, nix_band_prof_ctx_display, NULL); + +static int rvu_dbg_nix_band_prof_rsrc_display(struct seq_file *m, void *unused) +{ + struct nix_hw *nix_hw = m->private; + struct nix_ipolicer *ipolicer; + int layer; + char *str; + + seq_puts(m, "\nBandwidth profile resource free count\n"); + seq_puts(m, "=====================================\n"); + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + if (layer == BAND_PROF_INVAL_LAYER) + continue; + str = (layer == BAND_PROF_LEAF_LAYER) ? "Leaf" : + (layer == BAND_PROF_MID_LAYER) ? "Mid " : "Top "; + + ipolicer = &nix_hw->ipolicer[layer]; + seq_printf(m, "%s :: Max: %4d Free: %4d\n", str, + ipolicer->band_prof.max, + rvu_rsrc_free_count(&ipolicer->band_prof)); + } + seq_puts(m, "=====================================\n"); + + return 0; +} + +RVU_DEBUG_SEQ_FOPS(nix_band_prof_rsrc, nix_band_prof_rsrc_display, NULL); + static void rvu_dbg_nix_init(struct rvu *rvu, int blkaddr) { struct nix_hw *nix_hw; @@ -1664,6 +1823,10 @@ static void rvu_dbg_nix_init(struct rvu *rvu, int blkaddr) &rvu_dbg_nix_ndc_rx_hits_miss_fops); debugfs_create_file("qsize", 0600, rvu->rvu_dbg.nix, rvu, &rvu_dbg_nix_qsize_fops); + debugfs_create_file("ingress_policer_ctx", 0600, rvu->rvu_dbg.nix, nix_hw, + &rvu_dbg_nix_band_prof_ctx_fops); + debugfs_create_file("ingress_policer_rsrc", 0600, rvu->rvu_dbg.nix, nix_hw, + &rvu_dbg_nix_band_prof_rsrc_fops); } static void rvu_dbg_npa_init(struct rvu *rvu) @@ -2132,6 +2295,7 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) struct rvu *rvu = s->private; struct npc_mcam *mcam; int pf, vf = -1; + bool enabled; int blkaddr; u16 target; u64 hits; @@ -2173,7 +2337,9 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) } rvu_dbg_npc_mcam_show_action(s, iter); - seq_printf(s, "\tenabled: %s\n", iter->enable ? "yes" : "no"); + + enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, iter->entry); + seq_printf(s, "\tenabled: %s\n", enabled ? "yes" : "no"); if (!iter->has_cntr) continue; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 0a8bd667cb11..d6f8210652c5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -21,6 +21,16 @@ static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc); static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, int type, int chan_id); +static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, + int type, bool add); +static int nix_setup_ipolicers(struct rvu *rvu, + struct nix_hw *nix_hw, int blkaddr); +static void nix_ipolicer_freemem(struct nix_hw *nix_hw); +static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, + struct nix_hw *nix_hw, u16 pcifunc); +static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc); +static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, + u32 leaf_prof); enum mc_tbl_sz { MC_TBL_SZ_256, @@ -132,6 +142,22 @@ int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr) return 0; } +int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc, + struct nix_hw **nix_hw, int *blkaddr) +{ + struct rvu_pfvf *pfvf; + + pfvf = rvu_get_pfvf(rvu, pcifunc); + *blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); + if (!pfvf->nixlf || *blkaddr < 0) + return NIX_AF_ERR_AF_LF_INVALID; + + *nix_hw = get_nix_hw(rvu->hw, *blkaddr); + if (!*nix_hw) + return NIX_AF_ERR_INVALID_NIXBLK; + return 0; +} + static void nix_mce_list_init(struct nix_mce_list *list, int max) { INIT_HLIST_HEAD(&list->head); @@ -274,7 +300,7 @@ static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf) pfvf->tx_chan_cnt = 1; rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, pfvf->rx_chan_base, - pfvf->rx_chan_cnt, false); + pfvf->rx_chan_cnt); break; } @@ -285,16 +311,17 @@ static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf) pfvf->rx_chan_base, pfvf->mac_addr); /* Add this PF_FUNC to bcast pkt replication list */ - err = nix_update_bcast_mce_list(rvu, pcifunc, true); + err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, true); if (err) { dev_err(rvu->dev, "Bcast list, failed to enable PF_FUNC 0x%x\n", pcifunc); return err; } - + /* Install MCAM rule matching Ethernet broadcast mac address */ rvu_npc_install_bcast_match_entry(rvu, pcifunc, nixlf, pfvf->rx_chan_base); + pfvf->maxlen = NIC_HW_MIN_FRS; pfvf->minlen = NIC_HW_MIN_FRS; @@ -310,7 +337,7 @@ static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf) pfvf->minlen = 0; /* Remove this PF_FUNC from bcast pkt replication list */ - err = nix_update_bcast_mce_list(rvu, pcifunc, false); + err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, false); if (err) { dev_err(rvu->dev, "Bcast list, failed to disable PF_FUNC 0x%x\n", @@ -680,8 +707,11 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, pfvf = rvu_get_pfvf(rvu, pcifunc); nixlf = rvu_get_lf(rvu, block, pcifunc, 0); - /* Skip NIXLF check for broadcast MCE entry init */ - if (!(!rsp && req->ctype == NIX_AQ_CTYPE_MCE)) { + /* Skip NIXLF check for broadcast MCE entry and bandwidth profile + * operations done by AF itself. + */ + if (!((!rsp && req->ctype == NIX_AQ_CTYPE_MCE) || + (req->ctype == NIX_AQ_CTYPE_BANDPROF && !pcifunc))) { if (!pfvf->nixlf || nixlf < 0) return NIX_AF_ERR_AF_LF_INVALID; } @@ -721,6 +751,11 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, if (rsp) rc = NIX_AF_ERR_AQ_ENQUEUE; break; + case NIX_AQ_CTYPE_BANDPROF: + if (nix_verify_bandprof((struct nix_cn10k_aq_enq_req *)req, + nix_hw, pcifunc)) + rc = NIX_AF_ERR_INVALID_BANDPROF; + break; default: rc = NIX_AF_ERR_AQ_ENQUEUE; } @@ -777,6 +812,9 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, else if (req->ctype == NIX_AQ_CTYPE_MCE) memcpy(mask, &req->mce_mask, sizeof(struct nix_rx_mce_s)); + else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) + memcpy(mask, &req->prof_mask, + sizeof(struct nix_bandprof_s)); fallthrough; case NIX_AQ_INSTOP_INIT: if (req->ctype == NIX_AQ_CTYPE_RQ) @@ -789,6 +827,8 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, memcpy(ctx, &req->rss, sizeof(struct nix_rsse_s)); else if (req->ctype == NIX_AQ_CTYPE_MCE) memcpy(ctx, &req->mce, sizeof(struct nix_rx_mce_s)); + else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) + memcpy(ctx, &req->prof, sizeof(struct nix_bandprof_s)); break; case NIX_AQ_INSTOP_NOP: case NIX_AQ_INSTOP_READ: @@ -866,6 +906,9 @@ static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, else if (req->ctype == NIX_AQ_CTYPE_MCE) memcpy(&rsp->mce, ctx, sizeof(struct nix_rx_mce_s)); + else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) + memcpy(&rsp->prof, ctx, + sizeof(struct nix_bandprof_s)); } } @@ -2203,8 +2246,8 @@ static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw, aq_req.op = op; aq_req.qidx = mce; - /* Forward bcast pkts to RQ0, RSS not needed */ - aq_req.mce.op = 0; + /* Use RSS with RSS index 0 */ + aq_req.mce.op = 1; aq_req.mce.index = 0; aq_req.mce.eol = eol; aq_req.mce.pf_func = pcifunc; @@ -2222,8 +2265,8 @@ static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw, return 0; } -static int nix_update_mce_list(struct nix_mce_list *mce_list, - u16 pcifunc, bool add) +static int nix_update_mce_list_entry(struct nix_mce_list *mce_list, + u16 pcifunc, bool add) { struct mce *mce, *tail = NULL; bool delete = false; @@ -2234,6 +2277,9 @@ static int nix_update_mce_list(struct nix_mce_list *mce_list, if (mce->pcifunc == pcifunc && !add) { delete = true; break; + } else if (mce->pcifunc == pcifunc && add) { + /* entry already exists */ + return 0; } tail = mce; } @@ -2261,36 +2307,23 @@ static int nix_update_mce_list(struct nix_mce_list *mce_list, return 0; } -int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add) +int nix_update_mce_list(struct rvu *rvu, u16 pcifunc, + struct nix_mce_list *mce_list, + int mce_idx, int mcam_index, bool add) { - int err = 0, idx, next_idx, last_idx; - struct nix_mce_list *mce_list; + int err = 0, idx, next_idx, last_idx, blkaddr, npc_blkaddr; + struct npc_mcam *mcam = &rvu->hw->mcam; struct nix_mcast *mcast; struct nix_hw *nix_hw; - struct rvu_pfvf *pfvf; struct mce *mce; - int blkaddr; - - /* Broadcast pkt replication is not needed for AF's VFs, hence skip */ - if (is_afvf(pcifunc)) - return 0; - - blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); - if (blkaddr < 0) - return 0; - nix_hw = get_nix_hw(rvu->hw, blkaddr); - if (!nix_hw) - return 0; - - mcast = &nix_hw->mcast; + if (!mce_list) + return -EINVAL; /* Get this PF/VF func's MCE index */ - pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); - idx = pfvf->bcast_mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK); + idx = mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK); - mce_list = &pfvf->bcast_mce_list; - if (idx > (pfvf->bcast_mce_idx + mce_list->max)) { + if (idx > (mce_idx + mce_list->max)) { dev_err(rvu->dev, "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n", __func__, idx, mce_list->max, @@ -2298,20 +2331,26 @@ int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add) return -EINVAL; } + err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); + if (err) + return err; + + mcast = &nix_hw->mcast; mutex_lock(&mcast->mce_lock); - err = nix_update_mce_list(mce_list, pcifunc, add); + err = nix_update_mce_list_entry(mce_list, pcifunc, add); if (err) goto end; /* Disable MCAM entry in NPC */ if (!mce_list->count) { - rvu_npc_enable_bcast_entry(rvu, pcifunc, false); + npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, mcam_index, false); goto end; } /* Dump the updated list to HW */ - idx = pfvf->bcast_mce_idx; + idx = mce_idx; last_idx = idx + mce_list->count - 1; hlist_for_each_entry(mce, &mce_list->head, node) { if (idx > last_idx) @@ -2332,7 +2371,71 @@ end: return err; } -static int nix_setup_bcast_tables(struct rvu *rvu, struct nix_hw *nix_hw) +void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type, + struct nix_mce_list **mce_list, int *mce_idx) +{ + struct rvu_hwinfo *hw = rvu->hw; + struct rvu_pfvf *pfvf; + + if (!hw->cap.nix_rx_multicast || + !is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc & ~RVU_PFVF_FUNC_MASK))) { + *mce_list = NULL; + *mce_idx = 0; + return; + } + + /* Get this PF/VF func's MCE index */ + pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); + + if (type == NIXLF_BCAST_ENTRY) { + *mce_list = &pfvf->bcast_mce_list; + *mce_idx = pfvf->bcast_mce_idx; + } else if (type == NIXLF_ALLMULTI_ENTRY) { + *mce_list = &pfvf->mcast_mce_list; + *mce_idx = pfvf->mcast_mce_idx; + } else if (type == NIXLF_PROMISC_ENTRY) { + *mce_list = &pfvf->promisc_mce_list; + *mce_idx = pfvf->promisc_mce_idx; + } else { + *mce_list = NULL; + *mce_idx = 0; + } +} + +static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, + int type, bool add) +{ + int err = 0, nixlf, blkaddr, mcam_index, mce_idx; + struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_hwinfo *hw = rvu->hw; + struct nix_mce_list *mce_list; + + /* skip multicast pkt replication for AF's VFs */ + if (is_afvf(pcifunc)) + return 0; + + if (!hw->cap.nix_rx_multicast) + return 0; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); + if (blkaddr < 0) + return -EINVAL; + + nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); + if (nixlf < 0) + return -EINVAL; + + nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); + + mcam_index = npc_get_nixlf_mcam_index(mcam, + pcifunc & ~RVU_PFVF_FUNC_MASK, + nixlf, type); + err = nix_update_mce_list(rvu, pcifunc, mce_list, + mce_idx, mcam_index, add); + return err; +} + +static int nix_setup_mce_tables(struct rvu *rvu, struct nix_hw *nix_hw) { struct nix_mcast *mcast = &nix_hw->mcast; int err, pf, numvfs, idx; @@ -2355,11 +2458,18 @@ static int nix_setup_bcast_tables(struct rvu *rvu, struct nix_hw *nix_hw) if (pfvf->nix_blkaddr != nix_hw->blkaddr) continue; - /* Save the start MCE */ + /* save start idx of broadcast mce list */ pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1); - nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1); + /* save start idx of multicast mce list */ + pfvf->mcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1); + nix_mce_list_init(&pfvf->mcast_mce_list, numvfs + 1); + + /* save the start idx of promisc mce list */ + pfvf->promisc_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1); + nix_mce_list_init(&pfvf->promisc_mce_list, numvfs + 1); + for (idx = 0; idx < (numvfs + 1); idx++) { /* idx-0 is for PF, followed by VFs */ pcifunc = (pf << RVU_PFVF_PF_SHIFT); @@ -2375,6 +2485,22 @@ static int nix_setup_bcast_tables(struct rvu *rvu, struct nix_hw *nix_hw) pcifunc, 0, true); if (err) return err; + + /* add dummy entries to multicast mce list */ + err = nix_blk_setup_mce(rvu, nix_hw, + pfvf->mcast_mce_idx + idx, + NIX_AQ_INSTOP_INIT, + pcifunc, 0, true); + if (err) + return err; + + /* add dummy entries to promisc mce list */ + err = nix_blk_setup_mce(rvu, nix_hw, + pfvf->promisc_mce_idx + idx, + NIX_AQ_INSTOP_INIT, + pcifunc, 0, true); + if (err) + return err; } } return 0; @@ -2421,7 +2547,7 @@ static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) mutex_init(&mcast->mce_lock); - return nix_setup_bcast_tables(rvu, nix_hw); + return nix_setup_mce_tables(rvu, nix_hw); } static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw) @@ -3035,15 +3161,22 @@ int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu, pfvf = rvu_get_pfvf(rvu, pcifunc); - /* VF can't overwrite admin(PF) changes */ - if (from_vf && pfvf->pf_set_vf_cfg) + /* untrusted VF can't overwrite admin(PF) changes */ + if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && + (from_vf && test_bit(PF_SET_VF_MAC, &pfvf->flags))) { + dev_warn(rvu->dev, + "MAC address set by admin(PF) cannot be overwritten by untrusted VF"); return -EPERM; + } ether_addr_copy(pfvf->mac_addr, req->mac_addr); rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, pfvf->rx_chan_base, req->mac_addr); + if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && from_vf) + ether_addr_copy(pfvf->default_mac, req->mac_addr); + return 0; } @@ -3067,30 +3200,75 @@ int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu, int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, struct msg_rsp *rsp) { - bool allmulti = false, disable_promisc = false; + bool allmulti, promisc, nix_rx_multicast; u16 pcifunc = req->hdr.pcifunc; - int blkaddr, nixlf, err; struct rvu_pfvf *pfvf; + int nixlf, err; - err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); + pfvf = rvu_get_pfvf(rvu, pcifunc); + promisc = req->mode & NIX_RX_MODE_PROMISC ? true : false; + allmulti = req->mode & NIX_RX_MODE_ALLMULTI ? true : false; + pfvf->use_mce_list = req->mode & NIX_RX_MODE_USE_MCE ? true : false; + + nix_rx_multicast = rvu->hw->cap.nix_rx_multicast & pfvf->use_mce_list; + + if (is_vf(pcifunc) && !nix_rx_multicast && + (promisc || allmulti)) { + dev_warn_ratelimited(rvu->dev, + "VF promisc/multicast not supported\n"); + return 0; + } + + /* untrusted VF can't configure promisc/allmulti */ + if (is_vf(pcifunc) && !test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && + (promisc || allmulti)) + return 0; + + err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); if (err) return err; - pfvf = rvu_get_pfvf(rvu, pcifunc); + if (nix_rx_multicast) { + /* add/del this PF_FUNC to/from mcast pkt replication list */ + err = nix_update_mce_rule(rvu, pcifunc, NIXLF_ALLMULTI_ENTRY, + allmulti); + if (err) { + dev_err(rvu->dev, + "Failed to update pcifunc 0x%x to multicast list\n", + pcifunc); + return err; + } - if (req->mode & NIX_RX_MODE_PROMISC) - allmulti = false; - else if (req->mode & NIX_RX_MODE_ALLMULTI) - allmulti = true; - else - disable_promisc = true; + /* add/del this PF_FUNC to/from promisc pkt replication list */ + err = nix_update_mce_rule(rvu, pcifunc, NIXLF_PROMISC_ENTRY, + promisc); + if (err) { + dev_err(rvu->dev, + "Failed to update pcifunc 0x%x to promisc list\n", + pcifunc); + return err; + } + } - if (disable_promisc) - rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); - else + /* install/uninstall allmulti entry */ + if (allmulti) { + rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf, + pfvf->rx_chan_base); + } else { + if (!nix_rx_multicast) + rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false); + } + + /* install/uninstall promisc entry */ + if (promisc) { rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, pfvf->rx_chan_base, - pfvf->rx_chan_cnt, allmulti); + pfvf->rx_chan_cnt); + } else { + if (!nix_rx_multicast) + rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false); + } + return 0; } @@ -3470,6 +3648,10 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) if (err) return err; + err = nix_setup_ipolicers(rvu, nix_hw, blkaddr); + if (err) + return err; + err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr); if (err) return err; @@ -3523,6 +3705,40 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) | ltdefs->rx_isctp.ltype_mask); + if (!is_rvu_otx2(rvu)) { + /* Enable APAD calculation for other protocols + * matching APAD0 and APAD1 lt def registers. + */ + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD0, + (ltdefs->rx_apad0.valid << 11) | + (ltdefs->rx_apad0.lid << 8) | + (ltdefs->rx_apad0.ltype_match << 4) | + ltdefs->rx_apad0.ltype_mask); + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD1, + (ltdefs->rx_apad1.valid << 11) | + (ltdefs->rx_apad1.lid << 8) | + (ltdefs->rx_apad1.ltype_match << 4) | + ltdefs->rx_apad1.ltype_mask); + + /* Receive ethertype defination register defines layer + * information in NPC_RESULT_S to identify the Ethertype + * location in L2 header. Used for Ethertype overwriting + * in inline IPsec flow. + */ + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(0), + (ltdefs->rx_et[0].offset << 12) | + (ltdefs->rx_et[0].valid << 11) | + (ltdefs->rx_et[0].lid << 8) | + (ltdefs->rx_et[0].ltype_match << 4) | + ltdefs->rx_et[0].ltype_mask); + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(1), + (ltdefs->rx_et[1].offset << 12) | + (ltdefs->rx_et[1].valid << 11) | + (ltdefs->rx_et[1].lid << 8) | + (ltdefs->rx_et[1].ltype_match << 4) | + ltdefs->rx_et[1].ltype_mask); + } + err = nix_rx_flowkey_alg_cfg(rvu, blkaddr); if (err) return err; @@ -3584,6 +3800,8 @@ static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr, kfree(txsch->schq.bmap); } + nix_ipolicer_freemem(nix_hw); + vlan = &nix_hw->txvlan; kfree(vlan->rsrc.bmap); mutex_destroy(&vlan->rsrc_lock); @@ -3614,6 +3832,7 @@ int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req, struct msg_rsp *rsp) { u16 pcifunc = req->hdr.pcifunc; + struct rvu_pfvf *pfvf; int nixlf, err; err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); @@ -3624,6 +3843,9 @@ int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req, npc_mcam_enable_flows(rvu, pcifunc); + pfvf = rvu_get_pfvf(rvu, pcifunc); + set_bit(NIXLF_INITIALIZED, &pfvf->flags); + return rvu_cgx_start_stop_io(rvu, pcifunc, true); } @@ -3631,6 +3853,7 @@ int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, struct msg_rsp *rsp) { u16 pcifunc = req->hdr.pcifunc; + struct rvu_pfvf *pfvf; int nixlf, err; err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); @@ -3639,6 +3862,9 @@ int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); + pfvf = rvu_get_pfvf(rvu, pcifunc); + clear_bit(NIXLF_INITIALIZED, &pfvf->flags); + return rvu_cgx_start_stop_io(rvu, pcifunc, false); } @@ -3657,6 +3883,8 @@ void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf) nix_rx_sync(rvu, blkaddr); nix_txschq_free(rvu, pcifunc); + clear_bit(NIXLF_INITIALIZED, &pfvf->flags); + rvu_cgx_start_stop_io(rvu, pcifunc, false); if (pfvf->sq_ctx) { @@ -3681,6 +3909,8 @@ void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf) } nix_ctx_free(rvu, pfvf); + + nix_free_all_bandprof(rvu, pcifunc); } #define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32) @@ -3789,3 +4019,586 @@ void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc) if (from_vf) ether_addr_copy(pfvf->mac_addr, pfvf->default_mac); } + +/* NIX ingress policers or bandwidth profiles APIs */ +static void nix_config_rx_pkt_policer_precolor(struct rvu *rvu, int blkaddr) +{ + struct npc_lt_def_cfg defs, *ltdefs; + + ltdefs = &defs; + memcpy(ltdefs, rvu->kpu.lt_def, sizeof(struct npc_lt_def_cfg)); + + /* Extract PCP and DEI fields from outer VLAN from byte offset + * 2 from the start of LB_PTR (ie TAG). + * VLAN0 is Outer VLAN and VLAN1 is Inner VLAN. Inner VLAN + * fields are considered when 'Tunnel enable' is set in profile. + */ + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN0_PCP_DEI, + (2UL << 12) | (ltdefs->ovlan.lid << 8) | + (ltdefs->ovlan.ltype_match << 4) | + ltdefs->ovlan.ltype_mask); + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN1_PCP_DEI, + (2UL << 12) | (ltdefs->ivlan.lid << 8) | + (ltdefs->ivlan.ltype_match << 4) | + ltdefs->ivlan.ltype_mask); + + /* DSCP field in outer and tunneled IPv4 packets */ + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4_DSCP, + (1UL << 12) | (ltdefs->rx_oip4.lid << 8) | + (ltdefs->rx_oip4.ltype_match << 4) | + ltdefs->rx_oip4.ltype_mask); + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4_DSCP, + (1UL << 12) | (ltdefs->rx_iip4.lid << 8) | + (ltdefs->rx_iip4.ltype_match << 4) | + ltdefs->rx_iip4.ltype_mask); + + /* DSCP field (traffic class) in outer and tunneled IPv6 packets */ + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6_DSCP, + (1UL << 11) | (ltdefs->rx_oip6.lid << 8) | + (ltdefs->rx_oip6.ltype_match << 4) | + ltdefs->rx_oip6.ltype_mask); + rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6_DSCP, + (1UL << 11) | (ltdefs->rx_iip6.lid << 8) | + (ltdefs->rx_iip6.ltype_match << 4) | + ltdefs->rx_iip6.ltype_mask); +} + +static int nix_init_policer_context(struct rvu *rvu, struct nix_hw *nix_hw, + int layer, int prof_idx) +{ + struct nix_cn10k_aq_enq_req aq_req; + int rc; + + memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); + + aq_req.qidx = (prof_idx & 0x3FFF) | (layer << 14); + aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; + aq_req.op = NIX_AQ_INSTOP_INIT; + + /* Context is all zeros, submit to AQ */ + rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, + (struct nix_aq_enq_req *)&aq_req, NULL); + if (rc) + dev_err(rvu->dev, "Failed to INIT bandwidth profile layer %d profile %d\n", + layer, prof_idx); + return rc; +} + +static int nix_setup_ipolicers(struct rvu *rvu, + struct nix_hw *nix_hw, int blkaddr) +{ + struct rvu_hwinfo *hw = rvu->hw; + struct nix_ipolicer *ipolicer; + int err, layer, prof_idx; + u64 cfg; + + cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST); + if (!(cfg & BIT_ULL(61))) { + hw->cap.ipolicer = false; + return 0; + } + + hw->cap.ipolicer = true; + nix_hw->ipolicer = devm_kcalloc(rvu->dev, BAND_PROF_NUM_LAYERS, + sizeof(*ipolicer), GFP_KERNEL); + if (!nix_hw->ipolicer) + return -ENOMEM; + + cfg = rvu_read64(rvu, blkaddr, NIX_AF_PL_CONST); + + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + ipolicer = &nix_hw->ipolicer[layer]; + switch (layer) { + case BAND_PROF_LEAF_LAYER: + ipolicer->band_prof.max = cfg & 0XFFFF; + break; + case BAND_PROF_MID_LAYER: + ipolicer->band_prof.max = (cfg >> 16) & 0XFFFF; + break; + case BAND_PROF_TOP_LAYER: + ipolicer->band_prof.max = (cfg >> 32) & 0XFFFF; + break; + } + + if (!ipolicer->band_prof.max) + continue; + + err = rvu_alloc_bitmap(&ipolicer->band_prof); + if (err) + return err; + + ipolicer->pfvf_map = devm_kcalloc(rvu->dev, + ipolicer->band_prof.max, + sizeof(u16), GFP_KERNEL); + if (!ipolicer->pfvf_map) + return -ENOMEM; + + ipolicer->match_id = devm_kcalloc(rvu->dev, + ipolicer->band_prof.max, + sizeof(u16), GFP_KERNEL); + if (!ipolicer->match_id) + return -ENOMEM; + + for (prof_idx = 0; + prof_idx < ipolicer->band_prof.max; prof_idx++) { + /* Set AF as current owner for INIT ops to succeed */ + ipolicer->pfvf_map[prof_idx] = 0x00; + + /* There is no enable bit in the profile context, + * so no context disable. So let's INIT them here + * so that PF/VF later on have to just do WRITE to + * setup policer rates and config. + */ + err = nix_init_policer_context(rvu, nix_hw, + layer, prof_idx); + if (err) + return err; + } + + /* Allocate memory for maintaining ref_counts for MID level + * profiles, this will be needed for leaf layer profiles' + * aggregation. + */ + if (layer != BAND_PROF_MID_LAYER) + continue; + + ipolicer->ref_count = devm_kcalloc(rvu->dev, + ipolicer->band_prof.max, + sizeof(u16), GFP_KERNEL); + } + + /* Set policer timeunit to 2us ie (19 + 1) * 100 nsec = 2us */ + rvu_write64(rvu, blkaddr, NIX_AF_PL_TS, 19); + + nix_config_rx_pkt_policer_precolor(rvu, blkaddr); + + return 0; +} + +static void nix_ipolicer_freemem(struct nix_hw *nix_hw) +{ + struct nix_ipolicer *ipolicer; + int layer; + + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + ipolicer = &nix_hw->ipolicer[layer]; + + if (!ipolicer->band_prof.max) + continue; + + kfree(ipolicer->band_prof.bmap); + } +} + +static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, + struct nix_hw *nix_hw, u16 pcifunc) +{ + struct nix_ipolicer *ipolicer; + int layer, hi_layer, prof_idx; + + /* Bits [15:14] in profile index represent layer */ + layer = (req->qidx >> 14) & 0x03; + prof_idx = req->qidx & 0x3FFF; + + ipolicer = &nix_hw->ipolicer[layer]; + if (prof_idx >= ipolicer->band_prof.max) + return -EINVAL; + + /* Check if the profile is allocated to the requesting PCIFUNC or not + * with the exception of AF. AF is allowed to read and update contexts. + */ + if (pcifunc && ipolicer->pfvf_map[prof_idx] != pcifunc) + return -EINVAL; + + /* If this profile is linked to higher layer profile then check + * if that profile is also allocated to the requesting PCIFUNC + * or not. + */ + if (!req->prof.hl_en) + return 0; + + /* Leaf layer profile can link only to mid layer and + * mid layer to top layer. + */ + if (layer == BAND_PROF_LEAF_LAYER) + hi_layer = BAND_PROF_MID_LAYER; + else if (layer == BAND_PROF_MID_LAYER) + hi_layer = BAND_PROF_TOP_LAYER; + else + return -EINVAL; + + ipolicer = &nix_hw->ipolicer[hi_layer]; + prof_idx = req->prof.band_prof_id; + if (prof_idx >= ipolicer->band_prof.max || + ipolicer->pfvf_map[prof_idx] != pcifunc) + return -EINVAL; + + return 0; +} + +int rvu_mbox_handler_nix_bandprof_alloc(struct rvu *rvu, + struct nix_bandprof_alloc_req *req, + struct nix_bandprof_alloc_rsp *rsp) +{ + int blkaddr, layer, prof, idx, err; + u16 pcifunc = req->hdr.pcifunc; + struct nix_ipolicer *ipolicer; + struct nix_hw *nix_hw; + + if (!rvu->hw->cap.ipolicer) + return NIX_AF_ERR_IPOLICER_NOTSUPP; + + err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); + if (err) + return err; + + mutex_lock(&rvu->rsrc_lock); + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + if (layer == BAND_PROF_INVAL_LAYER) + continue; + if (!req->prof_count[layer]) + continue; + + ipolicer = &nix_hw->ipolicer[layer]; + for (idx = 0; idx < req->prof_count[layer]; idx++) { + /* Allocate a max of 'MAX_BANDPROF_PER_PFFUNC' profiles */ + if (idx == MAX_BANDPROF_PER_PFFUNC) + break; + + prof = rvu_alloc_rsrc(&ipolicer->band_prof); + if (prof < 0) + break; + rsp->prof_count[layer]++; + rsp->prof_idx[layer][idx] = prof; + ipolicer->pfvf_map[prof] = pcifunc; + } + } + mutex_unlock(&rvu->rsrc_lock); + return 0; +} + +static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc) +{ + int blkaddr, layer, prof_idx, err; + struct nix_ipolicer *ipolicer; + struct nix_hw *nix_hw; + + if (!rvu->hw->cap.ipolicer) + return NIX_AF_ERR_IPOLICER_NOTSUPP; + + err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); + if (err) + return err; + + mutex_lock(&rvu->rsrc_lock); + /* Free all the profiles allocated to the PCIFUNC */ + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + if (layer == BAND_PROF_INVAL_LAYER) + continue; + ipolicer = &nix_hw->ipolicer[layer]; + + for (prof_idx = 0; prof_idx < ipolicer->band_prof.max; prof_idx++) { + if (ipolicer->pfvf_map[prof_idx] != pcifunc) + continue; + + /* Clear ratelimit aggregation, if any */ + if (layer == BAND_PROF_LEAF_LAYER && + ipolicer->match_id[prof_idx]) + nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); + + ipolicer->pfvf_map[prof_idx] = 0x00; + ipolicer->match_id[prof_idx] = 0; + rvu_free_rsrc(&ipolicer->band_prof, prof_idx); + } + } + mutex_unlock(&rvu->rsrc_lock); + return 0; +} + +int rvu_mbox_handler_nix_bandprof_free(struct rvu *rvu, + struct nix_bandprof_free_req *req, + struct msg_rsp *rsp) +{ + int blkaddr, layer, prof_idx, idx, err; + u16 pcifunc = req->hdr.pcifunc; + struct nix_ipolicer *ipolicer; + struct nix_hw *nix_hw; + + if (req->free_all) + return nix_free_all_bandprof(rvu, pcifunc); + + if (!rvu->hw->cap.ipolicer) + return NIX_AF_ERR_IPOLICER_NOTSUPP; + + err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); + if (err) + return err; + + mutex_lock(&rvu->rsrc_lock); + /* Free the requested profile indices */ + for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { + if (layer == BAND_PROF_INVAL_LAYER) + continue; + if (!req->prof_count[layer]) + continue; + + ipolicer = &nix_hw->ipolicer[layer]; + for (idx = 0; idx < req->prof_count[layer]; idx++) { + prof_idx = req->prof_idx[layer][idx]; + if (prof_idx >= ipolicer->band_prof.max || + ipolicer->pfvf_map[prof_idx] != pcifunc) + continue; + + /* Clear ratelimit aggregation, if any */ + if (layer == BAND_PROF_LEAF_LAYER && + ipolicer->match_id[prof_idx]) + nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); + + ipolicer->pfvf_map[prof_idx] = 0x00; + ipolicer->match_id[prof_idx] = 0; + rvu_free_rsrc(&ipolicer->band_prof, prof_idx); + if (idx == MAX_BANDPROF_PER_PFFUNC) + break; + } + } + mutex_unlock(&rvu->rsrc_lock); + return 0; +} + +int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw, + struct nix_cn10k_aq_enq_req *aq_req, + struct nix_cn10k_aq_enq_rsp *aq_rsp, + u16 pcifunc, u8 ctype, u32 qidx) +{ + memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); + aq_req->hdr.pcifunc = pcifunc; + aq_req->ctype = ctype; + aq_req->op = NIX_AQ_INSTOP_READ; + aq_req->qidx = qidx; + + return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, + (struct nix_aq_enq_req *)aq_req, + (struct nix_aq_enq_rsp *)aq_rsp); +} + +static int nix_ipolicer_map_leaf_midprofs(struct rvu *rvu, + struct nix_hw *nix_hw, + struct nix_cn10k_aq_enq_req *aq_req, + struct nix_cn10k_aq_enq_rsp *aq_rsp, + u32 leaf_prof, u16 mid_prof) +{ + memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); + aq_req->hdr.pcifunc = 0x00; + aq_req->ctype = NIX_AQ_CTYPE_BANDPROF; + aq_req->op = NIX_AQ_INSTOP_WRITE; + aq_req->qidx = leaf_prof; + + aq_req->prof.band_prof_id = mid_prof; + aq_req->prof_mask.band_prof_id = GENMASK(6, 0); + aq_req->prof.hl_en = 1; + aq_req->prof_mask.hl_en = 1; + + return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, + (struct nix_aq_enq_req *)aq_req, + (struct nix_aq_enq_rsp *)aq_rsp); +} + +int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc, + u16 rq_idx, u16 match_id) +{ + int leaf_prof, mid_prof, leaf_match; + struct nix_cn10k_aq_enq_req aq_req; + struct nix_cn10k_aq_enq_rsp aq_rsp; + struct nix_ipolicer *ipolicer; + struct nix_hw *nix_hw; + int blkaddr, idx, rc; + + if (!rvu->hw->cap.ipolicer) + return 0; + + rc = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); + if (rc) + return rc; + + /* Fetch the RQ's context to see if policing is enabled */ + rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, pcifunc, + NIX_AQ_CTYPE_RQ, rq_idx); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to fetch RQ%d context of PFFUNC 0x%x\n", + __func__, rq_idx, pcifunc); + return rc; + } + + if (!aq_rsp.rq.policer_ena) + return 0; + + /* Get the bandwidth profile ID mapped to this RQ */ + leaf_prof = aq_rsp.rq.band_prof_id; + + ipolicer = &nix_hw->ipolicer[BAND_PROF_LEAF_LAYER]; + ipolicer->match_id[leaf_prof] = match_id; + + /* Check if any other leaf profile is marked with same match_id */ + for (idx = 0; idx < ipolicer->band_prof.max; idx++) { + if (idx == leaf_prof) + continue; + if (ipolicer->match_id[idx] != match_id) + continue; + + leaf_match = idx; + break; + } + + if (idx == ipolicer->band_prof.max) + return 0; + + /* Fetch the matching profile's context to check if it's already + * mapped to a mid level profile. + */ + rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, + NIX_AQ_CTYPE_BANDPROF, leaf_match); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to fetch context of leaf profile %d\n", + __func__, leaf_match); + return rc; + } + + ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; + if (aq_rsp.prof.hl_en) { + /* Get Mid layer prof index and map leaf_prof index + * also such that flows that are being steered + * to different RQs and marked with same match_id + * are rate limited in a aggregate fashion + */ + mid_prof = aq_rsp.prof.band_prof_id; + rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, + &aq_req, &aq_rsp, + leaf_prof, mid_prof); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to map leaf(%d) and mid(%d) profiles\n", + __func__, leaf_prof, mid_prof); + goto exit; + } + + mutex_lock(&rvu->rsrc_lock); + ipolicer->ref_count[mid_prof]++; + mutex_unlock(&rvu->rsrc_lock); + goto exit; + } + + /* Allocate a mid layer profile and + * map both 'leaf_prof' and 'leaf_match' profiles to it. + */ + mutex_lock(&rvu->rsrc_lock); + mid_prof = rvu_alloc_rsrc(&ipolicer->band_prof); + if (mid_prof < 0) { + dev_err(rvu->dev, + "%s: Unable to allocate mid layer profile\n", __func__); + mutex_unlock(&rvu->rsrc_lock); + goto exit; + } + mutex_unlock(&rvu->rsrc_lock); + ipolicer->pfvf_map[mid_prof] = 0x00; + ipolicer->ref_count[mid_prof] = 0; + + /* Initialize mid layer profile same as 'leaf_prof' */ + rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, + NIX_AQ_CTYPE_BANDPROF, leaf_prof); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to fetch context of leaf profile %d\n", + __func__, leaf_prof); + goto exit; + } + + memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); + aq_req.hdr.pcifunc = 0x00; + aq_req.qidx = (mid_prof & 0x3FFF) | (BAND_PROF_MID_LAYER << 14); + aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; + aq_req.op = NIX_AQ_INSTOP_WRITE; + memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s)); + /* Clear higher layer enable bit in the mid profile, just in case */ + aq_req.prof.hl_en = 0; + aq_req.prof_mask.hl_en = 1; + + rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, + (struct nix_aq_enq_req *)&aq_req, NULL); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to INIT context of mid layer profile %d\n", + __func__, mid_prof); + goto exit; + } + + /* Map both leaf profiles to this mid layer profile */ + rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, + &aq_req, &aq_rsp, + leaf_prof, mid_prof); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to map leaf(%d) and mid(%d) profiles\n", + __func__, leaf_prof, mid_prof); + goto exit; + } + + mutex_lock(&rvu->rsrc_lock); + ipolicer->ref_count[mid_prof]++; + mutex_unlock(&rvu->rsrc_lock); + + rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, + &aq_req, &aq_rsp, + leaf_match, mid_prof); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to map leaf(%d) and mid(%d) profiles\n", + __func__, leaf_match, mid_prof); + ipolicer->ref_count[mid_prof]--; + goto exit; + } + + mutex_lock(&rvu->rsrc_lock); + ipolicer->ref_count[mid_prof]++; + mutex_unlock(&rvu->rsrc_lock); + +exit: + return rc; +} + +/* Called with mutex rsrc_lock */ +static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, + u32 leaf_prof) +{ + struct nix_cn10k_aq_enq_req aq_req; + struct nix_cn10k_aq_enq_rsp aq_rsp; + struct nix_ipolicer *ipolicer; + u16 mid_prof; + int rc; + + mutex_unlock(&rvu->rsrc_lock); + + rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, + NIX_AQ_CTYPE_BANDPROF, leaf_prof); + + mutex_lock(&rvu->rsrc_lock); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to fetch context of leaf profile %d\n", + __func__, leaf_prof); + return; + } + + if (!aq_rsp.prof.hl_en) + return; + + mid_prof = aq_rsp.prof.band_prof_id; + ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; + ipolicer->ref_count[mid_prof]--; + /* If ref_count is zero, free mid layer profile */ + if (!ipolicer->ref_count[mid_prof]) { + ipolicer->pfvf_map[mid_prof] = 0x00; + rvu_free_rsrc(&ipolicer->band_prof, mid_prof); + } +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index 0bc4529691ec..3612e0a2cab3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -19,7 +19,7 @@ #include "cgx.h" #include "npc_profile.h" -#define RSVD_MCAM_ENTRIES_PER_PF 2 /* Bcast & Promisc */ +#define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ #define NPC_PARSE_RESULT_DMAC_OFFSET 8 @@ -27,6 +27,8 @@ #define NPC_KEX_CHAN_MASK 0xFFFULL #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL +#define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) + static const char def_pfl_name[] = "default"; static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, @@ -212,8 +214,10 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, */ if (type == NIXLF_BCAST_ENTRY) return index; - else if (type == NIXLF_PROMISC_ENTRY) + else if (type == NIXLF_ALLMULTI_ENTRY) return index + 1; + else if (type == NIXLF_PROMISC_ENTRY) + return index + 2; } return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); @@ -411,37 +415,49 @@ static void npc_fill_entryword(struct mcam_entry *entry, int idx, } } -static void npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, - int blkaddr, int index, - struct mcam_entry *entry) +static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, + int blkaddr, u16 pf_func) +{ + int bank, nixlf, index; + + /* get ucast entry rule entry index */ + nix_get_nixlf(rvu, pf_func, &nixlf, NULL); + index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, + NIXLF_UCAST_ENTRY); + bank = npc_get_bank(mcam, index); + index &= (mcam->banksize - 1); + + return rvu_read64(rvu, blkaddr, + NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); +} + +static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, + int blkaddr, int index, struct mcam_entry *entry, + bool *enable) { u16 owner, target_func; struct rvu_pfvf *pfvf; - int bank, nixlf; u64 rx_action; owner = mcam->entry2pfvf_map[index]; target_func = (entry->action >> 4) & 0xffff; - /* return incase target is PF or LBK or rule owner is not PF */ + /* do nothing when target is LBK/PF or owner is not PF */ if (is_afvf(target_func) || (owner & RVU_PFVF_FUNC_MASK) || !(target_func & RVU_PFVF_FUNC_MASK)) return; + /* save entry2target_pffunc */ pfvf = rvu_get_pfvf(rvu, target_func); mcam->entry2target_pffunc[index] = target_func; - /* return if nixlf is not attached or initialized */ - if (!is_nixlf_attached(rvu, target_func) || !pfvf->def_ucast_rule) - return; - /* get VF ucast entry rule */ - nix_get_nixlf(rvu, target_func, &nixlf, NULL); - index = npc_get_nixlf_mcam_index(mcam, target_func, - nixlf, NIXLF_UCAST_ENTRY); - bank = npc_get_bank(mcam, index); - index &= (mcam->banksize - 1); + /* don't enable rule when nixlf not attached or initialized */ + if (!(is_nixlf_attached(rvu, target_func) && + test_bit(NIXLF_INITIALIZED, &pfvf->flags))) + *enable = false; - rx_action = rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + /* copy VF default entry action to the VF mcam entry */ + rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, + target_func); if (rx_action) entry->action = rx_action; } @@ -493,10 +509,9 @@ static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); } - /* copy VF default entry action to the VF mcam entry */ + /* PF installing VF rule */ if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries) - npc_get_default_entry_action(rvu, mcam, blkaddr, actindex, - entry); + npc_fixup_vf_rule(rvu, mcam, blkaddr, index, entry, &enable); /* Set 'action' */ rvu_write64(rvu, blkaddr, @@ -647,30 +662,32 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, } void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, - int nixlf, u64 chan, u8 chan_cnt, - bool allmulti) + int nixlf, u64 chan, u8 chan_cnt) { struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); struct npc_install_flow_req req = { 0 }; struct npc_install_flow_rsp rsp = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_hwinfo *hw = rvu->hw; int blkaddr, ucast_idx, index; - u8 mac_addr[ETH_ALEN] = { 0 }; struct nix_rx_action action; u64 relaxed_mask; - /* Only PF or AF VF can add a promiscuous entry */ - if ((pcifunc & RVU_PFVF_FUNC_MASK) && !is_afvf(pcifunc)) + if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) return; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return; - *(u64 *)&action = 0x00; index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, NIXLF_PROMISC_ENTRY); + if (is_cgx_vf(rvu, pcifunc)) + index = npc_get_nixlf_mcam_index(mcam, + pcifunc & ~RVU_PFVF_FUNC_MASK, + nixlf, NIXLF_PROMISC_ENTRY); + /* If the corresponding PF's ucast action is RSS, * use the same action for promisc also */ @@ -678,19 +695,20 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, nixlf, NIXLF_UCAST_ENTRY); if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) *(u64 *)&action = npc_get_mcam_action(rvu, mcam, - blkaddr, ucast_idx); + blkaddr, ucast_idx); if (action.op != NIX_RX_ACTIONOP_RSS) { *(u64 *)&action = 0x00; action.op = NIX_RX_ACTIONOP_UCAST; - action.pf_func = pcifunc; } - if (allmulti) { - mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ - ether_addr_copy(req.packet.dmac, mac_addr); - ether_addr_copy(req.mask.dmac, mac_addr); - req.features = BIT_ULL(NPC_DMAC); + /* RX_ACTION set to MCAST for CGX PF's */ + if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && + is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { + *(u64 *)&action = 0x00; + action.op = NIX_RX_ACTIONOP_MCAST; + pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); + action.index = pfvf->promisc_mce_idx; } req.chan_mask = 0xFFFU; @@ -718,8 +736,8 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } -static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, - int nixlf, bool enable) +void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, + int nixlf, bool enable) { struct npc_mcam *mcam = &rvu->hw->mcam; int blkaddr, index; @@ -728,25 +746,14 @@ static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, if (blkaddr < 0) return; - /* Only PF's have a promiscuous entry */ - if (pcifunc & RVU_PFVF_FUNC_MASK) - return; + /* Get 'pcifunc' of PF device */ + pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, NIXLF_PROMISC_ENTRY); npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); } -void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) -{ - npc_enadis_promisc_entry(rvu, pcifunc, nixlf, false); -} - -void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) -{ - npc_enadis_promisc_entry(rvu, pcifunc, nixlf, true); -} - void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, int nixlf, u64 chan) { @@ -756,8 +763,6 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; int blkaddr, index; - u32 req_index = 0; - u8 op; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) @@ -770,7 +775,7 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, /* If pkt replication is not supported, * then only PF is allowed to add a bcast match entry. */ - if (!hw->cap.nix_rx_multicast && pcifunc & RVU_PFVF_FUNC_MASK) + if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) return; /* Get 'pcifunc' of PF device */ @@ -784,10 +789,10 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, * so install entry with UCAST action, so that PF * receives all broadcast packets. */ - op = NIX_RX_ACTIONOP_UCAST; + req.op = NIX_RX_ACTIONOP_UCAST; } else { - op = NIX_RX_ACTIONOP_MCAST; - req_index = pfvf->bcast_mce_idx; + req.op = NIX_RX_ACTIONOP_MCAST; + req.index = pfvf->bcast_mce_idx; } eth_broadcast_addr((u8 *)&req.packet.dmac); @@ -796,15 +801,14 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, req.channel = chan; req.intf = pfvf->nix_rx_intf; req.entry = index; - req.op = op; req.hdr.pcifunc = 0; /* AF is requester */ req.vf = pcifunc; - req.index = req_index; rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } -void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable) +void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + bool enable) { struct npc_mcam *mcam = &rvu->hw->mcam; int blkaddr, index; @@ -816,7 +820,104 @@ void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable) /* Get 'pcifunc' of PF device */ pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; - index = npc_get_nixlf_mcam_index(mcam, pcifunc, 0, NIXLF_BCAST_ENTRY); + index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, + NIXLF_BCAST_ENTRY); + npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); +} + +void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + u64 chan) +{ + struct npc_install_flow_req req = { 0 }; + struct npc_install_flow_rsp rsp = { 0 }; + struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_hwinfo *hw = rvu->hw; + int blkaddr, ucast_idx, index; + u8 mac_addr[ETH_ALEN] = { 0 }; + struct nix_rx_action action; + struct rvu_pfvf *pfvf; + u16 vf_func; + + /* Only CGX PF/VF can add allmulticast entry */ + if (is_afvf(pcifunc)) + return; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return; + + /* Get 'pcifunc' of PF device */ + vf_func = pcifunc & RVU_PFVF_FUNC_MASK; + pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; + pfvf = rvu_get_pfvf(rvu, pcifunc); + index = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_ALLMULTI_ENTRY); + + /* If the corresponding PF's ucast action is RSS, + * use the same action for multicast entry also + */ + ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, + nixlf, NIXLF_UCAST_ENTRY); + if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) + *(u64 *)&action = npc_get_mcam_action(rvu, mcam, + blkaddr, ucast_idx); + + if (action.op != NIX_RX_ACTIONOP_RSS) { + *(u64 *)&action = 0x00; + action.op = NIX_RX_ACTIONOP_UCAST; + action.pf_func = pcifunc; + } + + /* RX_ACTION set to MCAST for CGX PF's */ + if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { + *(u64 *)&action = 0x00; + action.op = NIX_RX_ACTIONOP_MCAST; + action.index = pfvf->mcast_mce_idx; + } + + mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ + ether_addr_copy(req.packet.dmac, mac_addr); + ether_addr_copy(req.mask.dmac, mac_addr); + req.features = BIT_ULL(NPC_DMAC); + + /* For cn10k the upper two bits of the channel number are + * cpt channel number. with masking out these bits in the + * mcam entry, same entry used for NIX will allow packets + * received from cpt for parsing. + */ + if (!is_rvu_otx2(rvu)) + req.chan_mask = NIX_CHAN_CPT_X2P_MASK; + else + req.chan_mask = 0xFFFU; + + req.channel = chan; + req.intf = pfvf->nix_rx_intf; + req.entry = index; + req.op = action.op; + req.hdr.pcifunc = 0; /* AF is requester */ + req.vf = pcifunc | vf_func; + req.index = action.index; + req.match_id = action.match_id; + req.flow_key_alg = action.flow_key_alg; + + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); +} + +void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, + bool enable) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int blkaddr, index; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return; + + /* Get 'pcifunc' of PF device */ + pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; + + index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, + NIXLF_ALLMULTI_ENTRY); npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); } @@ -858,6 +959,7 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, int group, int alg_idx, int mcam_index) { struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_hwinfo *hw = rvu->hw; struct nix_rx_action action; int blkaddr, index, bank; struct rvu_pfvf *pfvf; @@ -913,7 +1015,8 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, /* If PF's promiscuous entry is enabled, * Set RSS action for that entry as well */ - if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { + if ((!hw->cap.nix_rx_multicast || !pfvf->use_mce_list) && + is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { bank = npc_get_bank(mcam, index); index &= (mcam->banksize - 1); @@ -923,12 +1026,47 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, } } +void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, + int nixlf, int type, bool enable) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + struct rvu_hwinfo *hw = rvu->hw; + struct nix_mce_list *mce_list; + int index, blkaddr, mce_idx; + struct rvu_pfvf *pfvf; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return; + + index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, + nixlf, type); + + /* disable MCAM entry when packet replication is not supported by hw */ + if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { + npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); + return; + } + + /* return incase mce list is not enabled */ + pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); + if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && + type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) + return; + + nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); + + nix_update_mce_list(rvu, pcifunc, mce_list, + mce_idx, index, enable); + if (enable) + npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); +} + static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf, bool enable) { struct npc_mcam *mcam = &rvu->hw->mcam; - struct nix_rx_action action; - int index, bank, blkaddr; + int index, blkaddr; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) @@ -939,48 +1077,33 @@ static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, nixlf, NIXLF_UCAST_ENTRY); npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); - /* For PF, ena/dis promisc and bcast MCAM match entries. - * For VFs add/delete from bcast list when RX multicast - * feature is present. + /* Nothing to do for VFs, on platforms where pkt replication + * is not supported */ - if (pcifunc & RVU_PFVF_FUNC_MASK && !rvu->hw->cap.nix_rx_multicast) + if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) return; - /* For bcast, enable/disable only if it's action is not - * packet replication, incase if action is replication - * then this PF/VF's nixlf is removed from bcast replication - * list. - */ - index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, - nixlf, NIXLF_BCAST_ENTRY); - bank = npc_get_bank(mcam, index); - *(u64 *)&action = rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank)); - - /* VFs will not have BCAST entry */ - if (action.op != NIX_RX_ACTIONOP_MCAST && - !(pcifunc & RVU_PFVF_FUNC_MASK)) { - npc_enable_mcam_entry(rvu, mcam, - blkaddr, index, enable); - } else { - nix_update_bcast_mce_list(rvu, pcifunc, enable); - /* Enable PF's BCAST entry for packet replication */ - rvu_npc_enable_bcast_entry(rvu, pcifunc, enable); - } - - if (enable) - rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf); - else - rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); + /* add/delete pf_func to broadcast MCE list */ + npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, + NIXLF_BCAST_ENTRY, enable); } void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) { npc_enadis_default_entries(rvu, pcifunc, nixlf, false); + + /* Delete multicast and promisc MCAM entries */ + npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, + NIXLF_ALLMULTI_ENTRY, false); + npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, + NIXLF_PROMISC_ENTRY, false); } void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) { + /* Enables only broadcast match entry. Promisc/Allmulti are enabled + * in set_rx_mode mbox handler. + */ npc_enadis_default_entries(rvu, pcifunc, nixlf, true); } @@ -1000,7 +1123,8 @@ void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) /* Disable MCAM entries directing traffic to this 'pcifunc' */ list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { if (is_npc_intf_rx(rule->intf) && - rule->rx_action.pf_func == pcifunc) { + rule->rx_action.pf_func == pcifunc && + rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { npc_enable_mcam_entry(rvu, mcam, blkaddr, rule->entry, false); rule->enable = false; @@ -1134,6 +1258,30 @@ static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, } } +static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, + u64 *size) +{ + u64 prfl_addr, prfl_sz; + + if (!rvu->fwdata) + return -EINVAL; + + prfl_addr = rvu->fwdata->mcam_addr; + prfl_sz = rvu->fwdata->mcam_sz; + + if (!prfl_addr || !prfl_sz) + return -EINVAL; + + *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); + if (!(*prfl_img_addr)) + return -ENOMEM; + + *size = prfl_sz; + + return 0; +} + +/* strtoull of "mkexprof" with base:36 */ #define MKEX_END_SIGN 0xdeadbeef static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, @@ -1141,26 +1289,21 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, { struct device *dev = &rvu->pdev->dev; struct npc_mcam_kex *mcam_kex; - void *mkex_prfl_addr = NULL; - u64 prfl_addr, prfl_sz; + void __iomem *mkex_prfl_addr = NULL; + u64 prfl_sz; + int ret; /* If user not selected mkex profile */ - if (!strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) - goto program_mkex; - - if (!rvu->fwdata) - goto program_mkex; - prfl_addr = rvu->fwdata->mcam_addr; - prfl_sz = rvu->fwdata->mcam_sz; - - if (!prfl_addr || !prfl_sz) + if (rvu->kpu_fwdata_sz || + !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) goto program_mkex; - mkex_prfl_addr = memremap(prfl_addr, prfl_sz, MEMREMAP_WC); - if (!mkex_prfl_addr) + /* Setting up the mapping for mkex profile image */ + ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); + if (ret < 0) goto program_mkex; - mcam_kex = (struct npc_mcam_kex *)mkex_prfl_addr; + mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { /* Compare with mkex mod_param name string */ @@ -1186,7 +1329,7 @@ program_mkex: /* Program selected mkex profile */ npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); if (mkex_prfl_addr) - memunmap(mkex_prfl_addr); + iounmap(mkex_prfl_addr); } static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, @@ -1263,6 +1406,7 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, const struct npc_kpu_profile *profile) { int entry, num_entries, max_entries; + u64 entry_mask; if (profile->cam_entries != profile->action_entries) { dev_err(rvu->dev, @@ -1286,8 +1430,12 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, /* Enable all programmed entries */ num_entries = min_t(int, profile->action_entries, profile->cam_entries); + entry_mask = enable_mask(num_entries); + /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ + if (!rvu->kpu.custom) + entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); rvu_write64(rvu, blkaddr, - NPC_AF_KPUX_ENTRY_DISX(kpu, 0), enable_mask(num_entries)); + NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); if (num_entries > 64) { rvu_write64(rvu, blkaddr, NPC_AF_KPUX_ENTRY_DISX(kpu, 1), @@ -1300,6 +1448,7 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) { + profile->custom = 0; profile->name = def_pfl_name; profile->version = NPC_KPU_PROFILE_VER; profile->ikpu = ikpu_action_entries; @@ -1312,10 +1461,245 @@ static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) return 0; } +static int npc_apply_custom_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile) +{ + size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; + struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; + struct npc_kpu_profile_action *action; + struct npc_kpu_profile_cam *cam; + struct npc_kpu_fwdata *fw_kpu; + int entries; + u16 kpu, entry; + + if (rvu->kpu_fwdata_sz < hdr_sz) { + dev_warn(rvu->dev, "Invalid KPU profile size\n"); + return -EINVAL; + } + if (le64_to_cpu(fw->signature) != KPU_SIGN) { + dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", + fw->signature); + return -EINVAL; + } + /* Verify if the using known profile structure */ + if (NPC_KPU_VER_MAJ(profile->version) > + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { + dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", + NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); + return -EINVAL; + } + /* Verify if profile is aligned with the required kernel changes */ + if (NPC_KPU_VER_MIN(profile->version) < + NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { + dev_warn(rvu->dev, + "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", + NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MIN(profile->version), + NPC_KPU_VER_PATCH(profile->version), + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), + NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), + NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); + return -EINVAL; + } + /* Verify if profile fits the HW */ + if (fw->kpus > profile->kpus) { + dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, + profile->kpus); + return -EINVAL; + } + + profile->custom = 1; + profile->name = fw->name; + profile->version = le64_to_cpu(fw->version); + profile->mkex = &fw->mkex; + profile->lt_def = &fw->lt_def; + + for (kpu = 0; kpu < fw->kpus; kpu++) { + fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); + if (fw_kpu->entries > KPU_MAX_CST_ENT) + dev_warn(rvu->dev, + "Too many custom entries on KPU%d: %d > %d\n", + kpu, fw_kpu->entries, KPU_MAX_CST_ENT); + entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); + cam = (struct npc_kpu_profile_cam *)fw_kpu->data; + offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); + action = (struct npc_kpu_profile_action *)(fw->data + offset); + offset += fw_kpu->entries * sizeof(*action); + if (rvu->kpu_fwdata_sz < hdr_sz + offset) { + dev_warn(rvu->dev, + "Profile size mismatch on KPU%i parsing.\n", + kpu + 1); + return -EINVAL; + } + for (entry = 0; entry < entries; entry++) { + profile->kpu[kpu].cam[entry] = cam[entry]; + profile->kpu[kpu].action[entry] = action[entry]; + } + } + + return 0; +} + +static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, + u64 prfl_sz, const char *kpu_profile) +{ + struct npc_kpu_profile_fwdata *kpu_data = NULL; + int rc = -EINVAL; + + kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; + if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && + !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { + dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", + kpu_profile); + rvu->kpu_fwdata = kpu_data; + rvu->kpu_fwdata_sz = prfl_sz; + rvu->kpu_prfl_addr = prfl_addr; + rc = 0; + } + + return rc; +} + +static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, + const char *kpu_profile) +{ + struct npc_coalesced_kpu_prfl *img_data = NULL; + int i = 0, rc = -EINVAL; + void __iomem *kpu_prfl_addr; + u16 offset; + + img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; + if (le64_to_cpu(img_data->signature) == KPU_SIGN && + !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { + /* Loaded profile is a single KPU profile. */ + rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, + prfl_sz, kpu_profile); + goto done; + } + + /* Loaded profile is coalesced image, offset of first KPU profile.*/ + offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + + (img_data->num_prfl * sizeof(uint16_t)); + /* Check if mapped image is coalesced image. */ + while (i < img_data->num_prfl) { + /* Profile image offsets are rounded up to next 8 multiple.*/ + offset = ALIGN_8B_CEIL(offset); + kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + + offset); + rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, + img_data->prfl_sz[i], kpu_profile); + if (!rc) + break; + /* Calculating offset of profile image based on profile size.*/ + offset += img_data->prfl_sz[i]; + i++; + } +done: + return rc; +} + +static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) +{ + int ret = -EINVAL; + u64 prfl_sz; + + /* Setting up the mapping for NPC profile image */ + ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); + if (ret < 0) + goto done; + + /* Detect if profile is coalesced or single KPU profile and load */ + ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); + if (ret == 0) + goto done; + + /* Cleaning up if KPU profile image from fwdata is not valid. */ + if (rvu->kpu_prfl_addr) { + iounmap(rvu->kpu_prfl_addr); + rvu->kpu_prfl_addr = NULL; + rvu->kpu_fwdata_sz = 0; + rvu->kpu_fwdata = NULL; + } + +done: + return ret; +} + static void npc_load_kpu_profile(struct rvu *rvu) { struct npc_kpu_profile_adapter *profile = &rvu->kpu; + const char *kpu_profile = rvu->kpu_pfl_name; + const struct firmware *fw = NULL; + bool retry_fwdb = false; + + /* If user not specified profile customization */ + if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) + goto revert_to_default; + /* First prepare default KPU, then we'll customize top entries. */ + npc_prepare_default_kpu(profile); + /* Order of preceedence for load loading NPC profile (high to low) + * Firmware binary in filesystem. + * Firmware database method. + * Default KPU profile. + */ + if (!request_firmware(&fw, kpu_profile, rvu->dev)) { + dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", + kpu_profile); + rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); + if (rvu->kpu_fwdata) { + memcpy(rvu->kpu_fwdata, fw->data, fw->size); + rvu->kpu_fwdata_sz = fw->size; + } + release_firmware(fw); + retry_fwdb = true; + goto program_kpu; + } + +load_image_fwdb: + /* Loading the KPU profile using firmware database */ + if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) + goto revert_to_default; + +program_kpu: + /* Apply profile customization if firmware was loaded. */ + if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { + /* If image from firmware filesystem fails to load or invalid + * retry with firmware database method. + */ + if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { + /* Loading image from firmware database failed. */ + if (rvu->kpu_prfl_addr) { + iounmap(rvu->kpu_prfl_addr); + rvu->kpu_prfl_addr = NULL; + } else { + kfree(rvu->kpu_fwdata); + } + rvu->kpu_fwdata = NULL; + rvu->kpu_fwdata_sz = 0; + if (retry_fwdb) { + retry_fwdb = false; + goto load_image_fwdb; + } + } + + dev_warn(rvu->dev, + "Can't load KPU profile %s. Using default.\n", + kpu_profile); + kfree(rvu->kpu_fwdata); + rvu->kpu_fwdata = NULL; + goto revert_to_default; + } + + dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", + profile->name, NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MIN(profile->version), + NPC_KPU_VER_PATCH(profile->version)); + + return; + +revert_to_default: npc_prepare_default_kpu(profile); } @@ -1654,6 +2038,10 @@ void rvu_npc_freemem(struct rvu *rvu) kfree(pkind->rsrc.bmap); kfree(mcam->counters.bmap); + if (rvu->kpu_prfl_addr) + iounmap(rvu->kpu_prfl_addr); + else + kfree(rvu->kpu_fwdata); mutex_destroy(&mcam->lock); } @@ -2149,8 +2537,11 @@ int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, rsp->free_count = 0; /* Check if ref_entry is within range */ - if (req->priority && req->ref_entry >= mcam->bmap_entries) + if (req->priority && req->ref_entry >= mcam->bmap_entries) { + dev_err(rvu->dev, "%s: reference entry %d is out of range\n", + __func__, req->ref_entry); return NPC_MCAM_INVALID_REQ; + } /* ref_entry can't be '0' if requested priority is high. * Can't be last entry if requested priority is low. @@ -2163,8 +2554,12 @@ int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, /* Since list of allocated indices needs to be sent to requester, * max number of non-contiguous entries per mbox msg is limited. */ - if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) + if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { + dev_err(rvu->dev, + "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", + __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); return NPC_MCAM_INVALID_REQ; + } /* Alloc request from PFFUNC with no NIXLF attached should be denied */ if (!is_nixlf_attached(rvu, pcifunc)) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index 7f35b62eea13..68633145a8b8 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -123,11 +123,8 @@ static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf) static bool npc_is_same(struct npc_key_field *input, struct npc_key_field *field) { - int ret; - - ret = memcmp(&input->layer_mdata, &field->layer_mdata, - sizeof(struct npc_layer_mdata)); - return ret == 0; + return memcmp(&input->layer_mdata, &field->layer_mdata, + sizeof(struct npc_layer_mdata)) == 0; } static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, @@ -1103,11 +1100,18 @@ find_rule: if (pf_set_vfs_mac) { ether_addr_copy(pfvf->default_mac, req->packet.dmac); ether_addr_copy(pfvf->mac_addr, req->packet.dmac); + set_bit(PF_SET_VF_MAC, &pfvf->flags); } - if (pfvf->pf_set_vf_cfg && req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7) + if (test_bit(PF_SET_VF_CFG, &pfvf->flags) && + req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7) rule->vfvlan_cfg = true; + if (is_npc_intf_rx(req->intf) && req->match_id && + (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS)) + return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc, + req->index, req->match_id); + return 0; } @@ -1167,7 +1171,7 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, /* PF installing for its VF */ if (req->hdr.pcifunc && !from_vf && req->vf) - pfvf->pf_set_vf_cfg = 1; + set_bit(PF_SET_VF_CFG, &pfvf->flags); /* update req destination mac addr */ if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) && @@ -1177,9 +1181,12 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, } err = nix_get_nixlf(rvu, target, &nixlf, NULL); + if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) + return -EINVAL; - /* If interface is uninitialized then do not enable entry */ - if (err || (!req->default_rule && !pfvf->def_ucast_rule)) + /* don't enable rule when nixlf not attached or initialized */ + if (!(is_nixlf_attached(rvu, target) && + test_bit(NIXLF_INITIALIZED, &pfvf->flags))) enable = false; /* Packets reaching NPC in Tx path implies that a @@ -1193,6 +1200,14 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, if (from_vf && !enable) return -EINVAL; + /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ + if (pf_set_vfs_mac && !enable) { + ether_addr_copy(pfvf->default_mac, req->packet.dmac); + ether_addr_copy(pfvf->mac_addr, req->packet.dmac); + set_bit(PF_SET_VF_MAC, &pfvf->flags); + return 0; + } + /* If message is from VF then its flow should not overlap with * reserved unicast flow. */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h index ac71c0f2f960..76837d5e19c6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h @@ -171,6 +171,7 @@ #define NIX_AF_SQ_CONST (0x0040) #define NIX_AF_CQ_CONST (0x0048) #define NIX_AF_RQ_CONST (0x0050) +#define NIX_AF_PL_CONST (0x0058) #define NIX_AF_PSE_CONST (0x0060) #define NIX_AF_TL1_CONST (0x0070) #define NIX_AF_TL2_CONST (0x0078) @@ -181,6 +182,7 @@ #define NIX_AF_LSO_CFG (0x00A8) #define NIX_AF_BLK_RST (0x00B0) #define NIX_AF_TX_TSTMP_CFG (0x00C0) +#define NIX_AF_PL_TS (0x00C8) #define NIX_AF_RX_CFG (0x00D0) #define NIX_AF_AVG_DELAY (0x00E0) #define NIX_AF_CINT_DELAY (0x00F0) @@ -208,19 +210,27 @@ #define NIX_AF_RVU_INT_ENA_W1S (0x01D0) #define NIX_AF_RVU_INT_ENA_W1C (0x01D8) #define NIX_AF_TCP_TIMER (0x01E0) -#define NIX_AF_RX_WQE_TAG_CTL (0x01F0) +#define NIX_AF_RX_DEF_ET(a) (0x01F0ull | (uint64_t)(a) << 3) #define NIX_AF_RX_DEF_OL2 (0x0200) #define NIX_AF_RX_DEF_OIP4 (0x0210) #define NIX_AF_RX_DEF_IIP4 (0x0220) +#define NIX_AF_RX_DEF_VLAN0_PCP_DEI (0x0228) #define NIX_AF_RX_DEF_OIP6 (0x0230) +#define NIX_AF_RX_DEF_VLAN1_PCP_DEI (0x0238) #define NIX_AF_RX_DEF_IIP6 (0x0240) #define NIX_AF_RX_DEF_OTCP (0x0250) #define NIX_AF_RX_DEF_ITCP (0x0260) #define NIX_AF_RX_DEF_OUDP (0x0270) #define NIX_AF_RX_DEF_IUDP (0x0280) #define NIX_AF_RX_DEF_OSCTP (0x0290) +#define NIX_AF_RX_DEF_CST_APAD0 (0x0298) #define NIX_AF_RX_DEF_ISCTP (0x02A0) #define NIX_AF_RX_DEF_IPSECX (0x02B0) +#define NIX_AF_RX_DEF_CST_APAD1 (0x02A8) +#define NIX_AF_RX_DEF_IIP4_DSCP (0x02E0) +#define NIX_AF_RX_DEF_OIP4_DSCP (0x02E8) +#define NIX_AF_RX_DEF_IIP6_DSCP (0x02F0) +#define NIX_AF_RX_DEF_OIP6_DSCP (0x02F8) #define NIX_AF_RX_IPSEC_GEN_CFG (0x0300) #define NIX_AF_RX_CPTX_INST_ADDR (0x0310) #define NIX_AF_NDC_TX_SYNC (0x03F0) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_struct.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_struct.h index 5e5f45c7eab0..14aa8e37ea41 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_struct.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_struct.h @@ -286,7 +286,7 @@ enum nix_aq_ctype { NIX_AQ_CTYPE_MCE = 0x3, NIX_AQ_CTYPE_RSS = 0x4, NIX_AQ_CTYPE_DYNO = 0x5, - NIX_AQ_CTYPE_BAND_PROF = 0x6, + NIX_AQ_CTYPE_BANDPROF = 0x6, }; /* NIX admin queue instruction opcodes */ @@ -665,6 +665,89 @@ struct nix_rx_mce_s { uint64_t next : 16; }; +enum nix_band_prof_layers { + BAND_PROF_LEAF_LAYER = 0, + BAND_PROF_INVAL_LAYER = 1, + BAND_PROF_MID_LAYER = 2, + BAND_PROF_TOP_LAYER = 3, + BAND_PROF_NUM_LAYERS = 4, +}; + +enum NIX_RX_BAND_PROF_ACTIONRESULT_E { + NIX_RX_BAND_PROF_ACTIONRESULT_PASS = 0x0, + NIX_RX_BAND_PROF_ACTIONRESULT_DROP = 0x1, + NIX_RX_BAND_PROF_ACTIONRESULT_RED = 0x2, +}; + +enum nix_band_prof_pc_mode { + NIX_RX_PC_MODE_VLAN = 0, + NIX_RX_PC_MODE_DSCP = 1, + NIX_RX_PC_MODE_GEN = 2, + NIX_RX_PC_MODE_RSVD = 3, +}; + +/* NIX ingress policer bandwidth profile structure */ +struct nix_bandprof_s { + uint64_t pc_mode : 2; /* W0 */ + uint64_t icolor : 2; + uint64_t tnl_ena : 1; + uint64_t reserved_5_7 : 3; + uint64_t peir_exponent : 5; + uint64_t reserved_13_15 : 3; + uint64_t pebs_exponent : 5; + uint64_t reserved_21_23 : 3; + uint64_t cir_exponent : 5; + uint64_t reserved_29_31 : 3; + uint64_t cbs_exponent : 5; + uint64_t reserved_37_39 : 3; + uint64_t peir_mantissa : 8; + uint64_t pebs_mantissa : 8; + uint64_t cir_mantissa : 8; + uint64_t cbs_mantissa : 8; /* W1 */ + uint64_t lmode : 1; + uint64_t l_sellect : 3; + uint64_t rdiv : 4; + uint64_t adjust_exponent : 5; + uint64_t reserved_85_86 : 2; + uint64_t adjust_mantissa : 9; + uint64_t gc_action : 2; + uint64_t yc_action : 2; + uint64_t rc_action : 2; + uint64_t meter_algo : 2; + uint64_t band_prof_id : 7; + uint64_t reserved_111_118 : 8; + uint64_t hl_en : 1; + uint64_t reserved_120_127 : 8; + uint64_t ts : 48; /* W2 */ + uint64_t reserved_176_191 : 16; + uint64_t pe_accum : 32; /* W3 */ + uint64_t c_accum : 32; + uint64_t green_pkt_pass : 48; /* W4 */ + uint64_t reserved_304_319 : 16; + uint64_t yellow_pkt_pass : 48; /* W5 */ + uint64_t reserved_368_383 : 16; + uint64_t red_pkt_pass : 48; /* W6 */ + uint64_t reserved_432_447 : 16; + uint64_t green_octs_pass : 48; /* W7 */ + uint64_t reserved_496_511 : 16; + uint64_t yellow_octs_pass : 48; /* W8 */ + uint64_t reserved_560_575 : 16; + uint64_t red_octs_pass : 48; /* W9 */ + uint64_t reserved_624_639 : 16; + uint64_t green_pkt_drop : 48; /* W10 */ + uint64_t reserved_688_703 : 16; + uint64_t yellow_pkt_drop : 48; /* W11 */ + uint64_t reserved_752_767 : 16; + uint64_t red_pkt_drop : 48; /* W12 */ + uint64_t reserved_816_831 : 16; + uint64_t green_octs_drop : 48; /* W13 */ + uint64_t reserved_880_895 : 16; + uint64_t yellow_octs_drop : 48; /* W14 */ + uint64_t reserved_944_959 : 16; + uint64_t red_octs_drop : 48; /* W15 */ + uint64_t reserved_1008_1023 : 16; +}; + enum nix_lsoalg { NIX_LSOALG_NOP, NIX_LSOALG_ADD_SEGNUM, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c index 9ec0313f13fc..1b08896b46d2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -179,3 +179,326 @@ void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) sq->head++; sq->head &= (sq->sqe_cnt - 1); } + +int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) +{ + struct nix_bandprof_free_req *req; + int rc; + + if (is_dev_otx2(pfvf->pdev)) + return 0; + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); + if (!req) { + rc = -ENOMEM; + goto out; + } + + /* Free all bandwidth profiles allocated */ + req->free_all = true; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); +out: + mutex_unlock(&pfvf->mbox.lock); + return rc; +} + +int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) +{ + struct nix_bandprof_alloc_req *req; + struct nix_bandprof_alloc_rsp *rsp; + int rc; + + req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox); + if (!req) + return -ENOMEM; + + req->prof_count[BAND_PROF_LEAF_LAYER] = 1; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + if (rc) + goto out; + + rsp = (struct nix_bandprof_alloc_rsp *) + otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); + if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { + rc = -EIO; + goto out; + } + + *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; +out: + if (rc) { + dev_warn(pfvf->dev, + "Failed to allocate ingress bandwidth policer\n"); + } + + return rc; +} + +int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) +{ + struct otx2_hw *hw = &pfvf->hw; + int ret; + + mutex_lock(&pfvf->mbox.lock); + + ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer); + + mutex_unlock(&pfvf->mbox.lock); + + return ret; +} + +#define POLICER_TIMESTAMP 1 /* 1 second */ +#define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ + +static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, + u32 *burst_mantissa) +{ + int tmp; + + /* Burst is calculated as + * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] + * This is the upper limit on number tokens (bytes) that + * can be accumulated in the bucket. + */ + *burst_exp = ilog2(burst); + if (burst < 256) { + /* No float: can't express mantissa in this case */ + *burst_mantissa = 0; + return; + } + + if (*burst_exp > MAX_RATE_EXP) + *burst_exp = MAX_RATE_EXP; + + /* Calculate mantissa + * Find remaining bytes 'burst - 2^burst_exp' + * mantissa = (remaining bytes) / 2^ (burst_exp - 8) + */ + tmp = burst - rounddown_pow_of_two(burst); + *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); +} + +static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, + u32 *rate_mantissa, u32 *rdiv) +{ + u32 div = 0; + u32 exp = 0; + u64 tmp; + + /* Figure out mantissa, exponent and divider from given max pkt rate + * + * To achieve desired rate HW adds + * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every + * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. + * Here policer timeunit is 2 usecs and rate is in bits per sec. + * Since floating point cannot be used below algorithm uses 1000000 + * scale factor to support rates upto 100Gbps. + */ + tmp = rate * 32 * 2; + if (tmp < 256000000) { + while (tmp < 256000000) { + tmp = tmp * 2; + div++; + } + } else { + for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) + tmp = tmp / 2; + + if (exp > MAX_RATE_EXP) + exp = MAX_RATE_EXP; + } + + *rate_mantissa = (tmp - 256000000) / 1000000; + *rate_exp = exp; + *rdiv = div; +} + +int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, + u16 policer, bool map) +{ + struct nix_cn10k_aq_enq_req *aq; + + aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!aq) + return -ENOMEM; + + /* Enable policing and set the bandwidth profile (policer) index */ + if (map) + aq->rq.policer_ena = 1; + else + aq->rq.policer_ena = 0; + aq->rq_mask.policer_ena = 1; + + aq->rq.band_prof_id = policer; + aq->rq_mask.band_prof_id = GENMASK(9, 0); + + /* Fill AQ info */ + aq->qidx = rq_idx; + aq->ctype = NIX_AQ_CTYPE_RQ; + aq->op = NIX_AQ_INSTOP_WRITE; + + return otx2_sync_mbox_msg(&pfvf->mbox); +} + +int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) +{ + struct nix_bandprof_free_req *req; + + req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); + if (!req) + return -ENOMEM; + + req->prof_count[BAND_PROF_LEAF_LAYER] = 1; + req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; + + return otx2_sync_mbox_msg(&pfvf->mbox); +} + +int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) +{ + struct otx2_hw *hw = &pfvf->hw; + int qidx, rc; + + mutex_lock(&pfvf->mbox.lock); + + /* Remove RQ's policer mapping */ + for (qidx = 0; qidx < hw->rx_queues; qidx++) + cn10k_map_unmap_rq_policer(pfvf, qidx, + hw->matchall_ipolicer, false); + + rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); + + mutex_unlock(&pfvf->mbox.lock); + return rc; +} + +int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, + u32 burst, u64 rate, bool pps) +{ + struct nix_cn10k_aq_enq_req *aq; + u32 burst_exp, burst_mantissa; + u32 rate_exp, rate_mantissa; + u32 rdiv; + + /* Get exponent and mantissa values for the desired rate */ + cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa); + cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv); + + /* Init bandwidth profile */ + aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!aq) + return -ENOMEM; + + /* Set initial color mode to blind */ + aq->prof.icolor = 0x03; + aq->prof_mask.icolor = 0x03; + + /* Set rate and burst values */ + aq->prof.cir_exponent = rate_exp; + aq->prof_mask.cir_exponent = 0x1F; + + aq->prof.cir_mantissa = rate_mantissa; + aq->prof_mask.cir_mantissa = 0xFF; + + aq->prof.cbs_exponent = burst_exp; + aq->prof_mask.cbs_exponent = 0x1F; + + aq->prof.cbs_mantissa = burst_mantissa; + aq->prof_mask.cbs_mantissa = 0xFF; + + aq->prof.rdiv = rdiv; + aq->prof_mask.rdiv = 0xF; + + if (pps) { + /* The amount of decremented tokens is calculated according to + * the following equation: + * max([ LMODE ? 0 : (packet_length - LXPTR)] + + * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], + * 1/256) + * if LMODE is 1 then rate limiting will be based on + * PPS otherwise bps. + * The aim of the ADJUST value is to specify a token cost per + * packet in contrary to the packet length that specifies a + * cost per byte. To rate limit based on PPS adjust mantissa + * is set as 384 and exponent as 1 so that number of tokens + * decremented becomes 1 i.e, 1 token per packeet. + */ + aq->prof.adjust_exponent = 1; + aq->prof_mask.adjust_exponent = 0x1F; + + aq->prof.adjust_mantissa = 384; + aq->prof_mask.adjust_mantissa = 0x1FF; + + aq->prof.lmode = 0x1; + aq->prof_mask.lmode = 0x1; + } + + /* Two rate three color marker + * With PEIR/EIR set to zero, color will be either green or red + */ + aq->prof.meter_algo = 2; + aq->prof_mask.meter_algo = 0x3; + + aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; + aq->prof_mask.rc_action = 0x3; + + aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; + aq->prof_mask.yc_action = 0x3; + + aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; + aq->prof_mask.gc_action = 0x3; + + /* Setting exponent value as 24 and mantissa as 0 configures + * the bucket with zero values making bucket unused. Peak + * information rate and Excess information rate buckets are + * unused here. + */ + aq->prof.peir_exponent = 24; + aq->prof_mask.peir_exponent = 0x1F; + + aq->prof.peir_mantissa = 0; + aq->prof_mask.peir_mantissa = 0xFF; + + aq->prof.pebs_exponent = 24; + aq->prof_mask.pebs_exponent = 0x1F; + + aq->prof.pebs_mantissa = 0; + aq->prof_mask.pebs_mantissa = 0xFF; + + /* Fill AQ info */ + aq->qidx = profile; + aq->ctype = NIX_AQ_CTYPE_BANDPROF; + aq->op = NIX_AQ_INSTOP_WRITE; + + return otx2_sync_mbox_msg(&pfvf->mbox); +} + +int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, + u32 burst, u64 rate) +{ + struct otx2_hw *hw = &pfvf->hw; + int qidx, rc; + + mutex_lock(&pfvf->mbox.lock); + + rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst, + rate, false); + if (rc) + goto out; + + for (qidx = 0; qidx < hw->rx_queues; qidx++) { + rc = cn10k_map_unmap_rq_policer(pfvf, qidx, + hw->matchall_ipolicer, true); + if (rc) + break; + } + +out: + mutex_unlock(&pfvf->mbox.lock); + return rc; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h index e0bc595cbb78..71292a4cf1f3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.h @@ -14,4 +14,15 @@ void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx); int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura); int cn10k_pf_lmtst_init(struct otx2_nic *pf); int cn10k_vf_lmtst_init(struct otx2_nic *vf); +int cn10k_free_all_ipolicers(struct otx2_nic *pfvf); +int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf); +int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf); +int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, + u32 burst, u64 rate); +int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, + u16 policer, bool map); +int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf); +int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, + u32 burst, u64 rate, bool pps); +int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf); #endif /* CN10K_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 45730d0d92f2..234b330f3183 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -180,6 +180,7 @@ struct otx2_hw { /* NIX */ u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + u16 matchall_ipolicer; /* HW settings, coalescing etc */ u16 rx_chan_base; @@ -223,6 +224,11 @@ struct otx2_hw { u64 *nix_lmt_base; }; +enum vfperm { + OTX2_RESET_VF_PERM, + OTX2_TRUSTED_VF, +}; + struct otx2_vf_config { struct otx2_nic *pf; struct delayed_work link_event_work; @@ -230,6 +236,7 @@ struct otx2_vf_config { u8 mac[ETH_ALEN]; u16 vlan; int tx_vtag_idx; + bool trusted; }; struct flr_work { @@ -261,24 +268,26 @@ struct otx2_mac_table { struct otx2_flow_config { u16 entry[NPC_MAX_NONCONTIG_ENTRIES]; - u32 nr_flows; -#define OTX2_MAX_NTUPLE_FLOWS 32 -#define OTX2_MAX_UNICAST_FLOWS 8 -#define OTX2_MAX_VLAN_FLOWS 1 -#define OTX2_MAX_TC_FLOWS OTX2_MAX_NTUPLE_FLOWS -#define OTX2_MCAM_COUNT (OTX2_MAX_NTUPLE_FLOWS + \ + u16 *flow_ent; + u16 *def_ent; + u16 nr_flows; +#define OTX2_DEFAULT_FLOWCOUNT 16 +#define OTX2_MAX_UNICAST_FLOWS 8 +#define OTX2_MAX_VLAN_FLOWS 1 +#define OTX2_MAX_TC_FLOWS OTX2_DEFAULT_FLOWCOUNT +#define OTX2_MCAM_COUNT (OTX2_DEFAULT_FLOWCOUNT + \ OTX2_MAX_UNICAST_FLOWS + \ OTX2_MAX_VLAN_FLOWS) - u32 ntuple_offset; - u32 unicast_offset; - u32 rx_vlan_offset; - u32 vf_vlan_offset; -#define OTX2_PER_VF_VLAN_FLOWS 2 /* rx+tx per VF */ + u16 ntuple_offset; + u16 unicast_offset; + u16 rx_vlan_offset; + u16 vf_vlan_offset; +#define OTX2_PER_VF_VLAN_FLOWS 2 /* Rx + Tx per VF */ #define OTX2_VF_VLAN_RX_INDEX 0 #define OTX2_VF_VLAN_TX_INDEX 1 - u32 tc_flower_offset; - u32 ntuple_max_flows; - u32 tc_max_flows; + u16 tc_flower_offset; + u16 ntuple_max_flows; + u16 tc_max_flows; struct list_head flow_list; }; @@ -319,6 +328,7 @@ struct otx2_nic { #define OTX2_FLAG_TX_PAUSE_ENABLED BIT_ULL(10) #define OTX2_FLAG_TC_FLOWER_SUPPORT BIT_ULL(11) #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12) +#define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13) u64 flags; struct otx2_qset qset; @@ -362,6 +372,7 @@ struct otx2_nic { struct otx2_flow_config *flow_cfg; struct otx2_tc_info tc_info; + unsigned long rq_bmap; }; static inline bool is_otx2_lbkvf(struct pci_dev *pdev) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index f4962a97a075..8df748e0677b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -286,6 +286,12 @@ static int otx2_set_channels(struct net_device *dev, if (!channel->rx_count || !channel->tx_count) return -EINVAL; + if (bitmap_weight(&pfvf->rq_bmap, pfvf->hw.rx_queues) > 1) { + netdev_err(dev, + "Receive queues are in use by TC police action\n"); + return -EINVAL; + } + if (if_up) dev->netdev_ops->ndo_stop(dev); @@ -786,6 +792,10 @@ static int otx2_set_rxfh_context(struct net_device *dev, const u32 *indir, if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; + if (*rss_context != ETH_RXFH_CONTEXT_ALLOC && + *rss_context >= MAX_RSS_GROUPS) + return -EINVAL; + rss = &pfvf->hw.rss_info; if (!rss->enable) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c index 0b4fa92ba821..8c97106bdd1c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c @@ -20,13 +20,125 @@ struct otx2_flow { int vf; }; +static void otx2_clear_ntuple_flow_info(struct otx2_nic *pfvf, struct otx2_flow_config *flow_cfg) +{ + devm_kfree(pfvf->dev, flow_cfg->flow_ent); + flow_cfg->flow_ent = NULL; + flow_cfg->ntuple_max_flows = 0; + flow_cfg->tc_max_flows = 0; +} + +static int otx2_free_ntuple_mcam_entries(struct otx2_nic *pfvf) +{ + struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; + struct npc_mcam_free_entry_req *req; + int ent, err; + + if (!flow_cfg->ntuple_max_flows) + return 0; + + mutex_lock(&pfvf->mbox.lock); + for (ent = 0; ent < flow_cfg->ntuple_max_flows; ent++) { + req = otx2_mbox_alloc_msg_npc_mcam_free_entry(&pfvf->mbox); + if (!req) + break; + + req->entry = flow_cfg->flow_ent[ent]; + + /* Send message to AF to free MCAM entries */ + err = otx2_sync_mbox_msg(&pfvf->mbox); + if (err) + break; + } + mutex_unlock(&pfvf->mbox.lock); + otx2_clear_ntuple_flow_info(pfvf, flow_cfg); + return 0; +} + +static int otx2_alloc_ntuple_mcam_entries(struct otx2_nic *pfvf, u16 count) +{ + struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; + struct npc_mcam_alloc_entry_req *req; + struct npc_mcam_alloc_entry_rsp *rsp; + int ent, allocated = 0; + + /* Free current ones and allocate new ones with requested count */ + otx2_free_ntuple_mcam_entries(pfvf); + + if (!count) + return 0; + + flow_cfg->flow_ent = devm_kmalloc_array(pfvf->dev, count, + sizeof(u16), GFP_KERNEL); + if (!flow_cfg->flow_ent) + return -ENOMEM; + + mutex_lock(&pfvf->mbox.lock); + + /* In a single request a max of NPC_MAX_NONCONTIG_ENTRIES MCAM entries + * can only be allocated. + */ + while (allocated < count) { + req = otx2_mbox_alloc_msg_npc_mcam_alloc_entry(&pfvf->mbox); + if (!req) + goto exit; + + req->contig = false; + req->count = (count - allocated) > NPC_MAX_NONCONTIG_ENTRIES ? + NPC_MAX_NONCONTIG_ENTRIES : count - allocated; + req->priority = NPC_MCAM_HIGHER_PRIO; + req->ref_entry = flow_cfg->def_ent[0]; + + /* Send message to AF */ + if (otx2_sync_mbox_msg(&pfvf->mbox)) + goto exit; + + rsp = (struct npc_mcam_alloc_entry_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + + for (ent = 0; ent < rsp->count; ent++) + flow_cfg->flow_ent[ent + allocated] = rsp->entry_list[ent]; + + allocated += rsp->count; + + /* If this request is not fulfilled, no need to send + * further requests. + */ + if (rsp->count != req->count) + break; + } + +exit: + mutex_unlock(&pfvf->mbox.lock); + + flow_cfg->ntuple_offset = 0; + flow_cfg->ntuple_max_flows = allocated; + flow_cfg->tc_max_flows = allocated; + + if (allocated != count) + netdev_info(pfvf->netdev, + "Unable to allocate %d MCAM entries for ntuple, got %d\n", + count, allocated); + + return allocated; +} + int otx2_alloc_mcam_entries(struct otx2_nic *pfvf) { struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; struct npc_mcam_alloc_entry_req *req; struct npc_mcam_alloc_entry_rsp *rsp; int vf_vlan_max_flows; - int i; + int ent, count; + + vf_vlan_max_flows = pfvf->total_vfs * OTX2_PER_VF_VLAN_FLOWS; + count = OTX2_MAX_UNICAST_FLOWS + + OTX2_MAX_VLAN_FLOWS + vf_vlan_max_flows; + + flow_cfg->def_ent = devm_kmalloc_array(pfvf->dev, count, + sizeof(u16), GFP_KERNEL); + if (!flow_cfg->def_ent) + return -ENOMEM; mutex_lock(&pfvf->mbox.lock); @@ -36,9 +148,8 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf) return -ENOMEM; } - vf_vlan_max_flows = pfvf->total_vfs * OTX2_PER_VF_VLAN_FLOWS; req->contig = false; - req->count = OTX2_MCAM_COUNT + vf_vlan_max_flows; + req->count = count; /* Send message to AF */ if (otx2_sync_mbox_msg(&pfvf->mbox)) { @@ -51,37 +162,36 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf) if (rsp->count != req->count) { netdev_info(pfvf->netdev, - "Unable to allocate %d MCAM entries, got %d\n", - req->count, rsp->count); - /* support only ntuples here */ - flow_cfg->ntuple_max_flows = rsp->count; - flow_cfg->ntuple_offset = 0; - pfvf->flags |= OTX2_FLAG_NTUPLE_SUPPORT; - flow_cfg->tc_max_flows = flow_cfg->ntuple_max_flows; - pfvf->flags |= OTX2_FLAG_TC_FLOWER_SUPPORT; - } else { - flow_cfg->vf_vlan_offset = 0; - flow_cfg->ntuple_offset = flow_cfg->vf_vlan_offset + - vf_vlan_max_flows; - flow_cfg->tc_flower_offset = flow_cfg->ntuple_offset; - flow_cfg->unicast_offset = flow_cfg->ntuple_offset + - OTX2_MAX_NTUPLE_FLOWS; - flow_cfg->rx_vlan_offset = flow_cfg->unicast_offset + - OTX2_MAX_UNICAST_FLOWS; - pfvf->flags |= OTX2_FLAG_NTUPLE_SUPPORT; - pfvf->flags |= OTX2_FLAG_UCAST_FLTR_SUPPORT; - pfvf->flags |= OTX2_FLAG_RX_VLAN_SUPPORT; - pfvf->flags |= OTX2_FLAG_VF_VLAN_SUPPORT; - pfvf->flags |= OTX2_FLAG_TC_FLOWER_SUPPORT; - } - - for (i = 0; i < rsp->count; i++) - flow_cfg->entry[i] = rsp->entry_list[i]; + "Unable to allocate MCAM entries for ucast, vlan and vf_vlan\n"); + mutex_unlock(&pfvf->mbox.lock); + devm_kfree(pfvf->dev, flow_cfg->def_ent); + return 0; + } - pfvf->flags |= OTX2_FLAG_MCAM_ENTRIES_ALLOC; + for (ent = 0; ent < rsp->count; ent++) + flow_cfg->def_ent[ent] = rsp->entry_list[ent]; + + flow_cfg->vf_vlan_offset = 0; + flow_cfg->unicast_offset = vf_vlan_max_flows; + flow_cfg->rx_vlan_offset = flow_cfg->unicast_offset + + OTX2_MAX_UNICAST_FLOWS; + pfvf->flags |= OTX2_FLAG_UCAST_FLTR_SUPPORT; + pfvf->flags |= OTX2_FLAG_RX_VLAN_SUPPORT; + pfvf->flags |= OTX2_FLAG_VF_VLAN_SUPPORT; + pfvf->flags |= OTX2_FLAG_MCAM_ENTRIES_ALLOC; mutex_unlock(&pfvf->mbox.lock); + /* Allocate entries for Ntuple filters */ + count = otx2_alloc_ntuple_mcam_entries(pfvf, OTX2_DEFAULT_FLOWCOUNT); + if (count <= 0) { + otx2_clear_ntuple_flow_info(pfvf, flow_cfg); + return 0; + } + + pfvf->flags |= OTX2_FLAG_NTUPLE_SUPPORT; + pfvf->flags |= OTX2_FLAG_TC_FLOWER_SUPPORT; + return 0; } @@ -96,13 +206,14 @@ int otx2_mcam_flow_init(struct otx2_nic *pf) INIT_LIST_HEAD(&pf->flow_cfg->flow_list); - pf->flow_cfg->ntuple_max_flows = OTX2_MAX_NTUPLE_FLOWS; - pf->flow_cfg->tc_max_flows = pf->flow_cfg->ntuple_max_flows; - err = otx2_alloc_mcam_entries(pf); if (err) return err; + /* Check if MCAM entries are allocate or not */ + if (!(pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT)) + return 0; + pf->mac_table = devm_kzalloc(pf->dev, sizeof(struct otx2_mac_table) * OTX2_MAX_UNICAST_FLOWS, GFP_KERNEL); if (!pf->mac_table) @@ -146,7 +257,7 @@ static int otx2_do_add_macfilter(struct otx2_nic *pf, const u8 *mac) ether_addr_copy(pf->mac_table[i].addr, mac); pf->mac_table[i].inuse = true; pf->mac_table[i].mcam_entry = - flow_cfg->entry[i + flow_cfg->unicast_offset]; + flow_cfg->def_ent[i + flow_cfg->unicast_offset]; req->entry = pf->mac_table[i].mcam_entry; break; } @@ -551,6 +662,7 @@ static int otx2_prepare_ipv6_flow(struct ethtool_rx_flow_spec *fsp, req->features |= BIT_ULL(NPC_IPPROTO_AH); else req->features |= BIT_ULL(NPC_IPPROTO_ESP); + break; default: break; } @@ -731,8 +843,7 @@ int otx2_add_flow(struct otx2_nic *pfvf, struct ethtool_rxnfc *nfc) if (!flow) return -ENOMEM; flow->location = fsp->location; - flow->entry = flow_cfg->entry[flow_cfg->ntuple_offset + - flow->location]; + flow->entry = flow_cfg->flow_ent[flow->location]; new = true; } /* struct copy */ @@ -836,9 +947,8 @@ int otx2_destroy_ntuple_flows(struct otx2_nic *pfvf) return -ENOMEM; } - req->start = flow_cfg->entry[flow_cfg->ntuple_offset]; - req->end = flow_cfg->entry[flow_cfg->ntuple_offset + - flow_cfg->ntuple_max_flows - 1]; + req->start = flow_cfg->flow_ent[0]; + req->end = flow_cfg->flow_ent[flow_cfg->ntuple_max_flows - 1]; err = otx2_sync_mbox_msg(&pfvf->mbox); mutex_unlock(&pfvf->mbox.lock); @@ -905,7 +1015,7 @@ int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf) return -ENOMEM; } - req->entry = flow_cfg->entry[flow_cfg->rx_vlan_offset]; + req->entry = flow_cfg->def_ent[flow_cfg->rx_vlan_offset]; req->intf = NIX_INTF_RX; ether_addr_copy(req->packet.dmac, pfvf->netdev->dev_addr); eth_broadcast_addr((u8 *)&req->mask.dmac); @@ -934,7 +1044,7 @@ static int otx2_delete_rxvlan_offload_flow(struct otx2_nic *pfvf) return -ENOMEM; } - req->entry = flow_cfg->entry[flow_cfg->rx_vlan_offset]; + req->entry = flow_cfg->def_ent[flow_cfg->rx_vlan_offset]; /* Send message to AF */ err = otx2_sync_mbox_msg(&pfvf->mbox); mutex_unlock(&pfvf->mbox.lock); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 03004fdac0c6..59912f73417b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -39,6 +39,8 @@ MODULE_DESCRIPTION(DRV_STRING); MODULE_LICENSE("GPL v2"); MODULE_DEVICE_TABLE(pci, otx2_pf_id_table); +static void otx2_vf_link_event_task(struct work_struct *work); + enum { TYPE_PFAF, TYPE_PFVF, @@ -1459,6 +1461,9 @@ static void otx2_free_hw_resources(struct otx2_nic *pf) otx2_free_cq_res(pf); + /* Free all ingress bandwidth profiles allocated */ + cn10k_free_all_ipolicers(pf); + mutex_lock(&mbox->lock); /* Reset NIX LF */ free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox); @@ -1820,9 +1825,11 @@ static void otx2_do_set_rx_mode(struct work_struct *work) if (promisc) req->mode |= NIX_RX_MODE_PROMISC; - else if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST)) + if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST)) req->mode |= NIX_RX_MODE_ALLMULTI; + req->mode |= NIX_RX_MODE_USE_MCE; + otx2_sync_mbox_msg(&pf->mbox); mutex_unlock(&pf->mbox.lock); } @@ -2044,7 +2051,7 @@ static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) if (!netif_running(netdev)) return -EAGAIN; - if (vf >= pci_num_vf(pdev)) + if (vf >= pf->total_vfs) return -EINVAL; if (!is_valid_ether_addr(mac)) @@ -2055,7 +2062,8 @@ static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) ret = otx2_do_set_vf_mac(pf, vf, mac); if (ret == 0) - dev_info(&pdev->dev, "Reload VF driver to apply the changes\n"); + dev_info(&pdev->dev, + "Load/Reload VF driver\n"); return ret; } @@ -2104,7 +2112,7 @@ static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos, } idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX); del_req->entry = - flow_cfg->entry[flow_cfg->vf_vlan_offset + idx]; + flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; err = otx2_sync_mbox_msg(&pf->mbox); if (err) goto out; @@ -2117,7 +2125,7 @@ static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos, } idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX); del_req->entry = - flow_cfg->entry[flow_cfg->vf_vlan_offset + idx]; + flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; err = otx2_sync_mbox_msg(&pf->mbox); goto out; @@ -2131,7 +2139,7 @@ static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos, } idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX); - req->entry = flow_cfg->entry[flow_cfg->vf_vlan_offset + idx]; + req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; req->packet.vlan_tci = htons(vlan); req->mask.vlan_tci = htons(VLAN_VID_MASK); /* af fills the destination mac addr */ @@ -2182,7 +2190,7 @@ static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos, eth_zero_addr((u8 *)&req->mask.dmac); idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX); - req->entry = flow_cfg->entry[flow_cfg->vf_vlan_offset + idx]; + req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx]; req->features = BIT_ULL(NPC_DMAC); req->channel = pf->hw.tx_chan_base; req->intf = NIX_INTF_TX; @@ -2241,10 +2249,63 @@ static int otx2_get_vf_config(struct net_device *netdev, int vf, ivi->vf = vf; ether_addr_copy(ivi->mac, config->mac); ivi->vlan = config->vlan; + ivi->trusted = config->trusted; return 0; } +static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf, + int req_perm) +{ + struct set_vf_perm *req; + int rc; + + mutex_lock(&pf->mbox.lock); + req = otx2_mbox_alloc_msg_set_vf_perm(&pf->mbox); + if (!req) { + rc = -ENOMEM; + goto out; + } + + /* Let AF reset VF permissions as sriov is disabled */ + if (req_perm == OTX2_RESET_VF_PERM) { + req->flags |= RESET_VF_PERM; + } else if (req_perm == OTX2_TRUSTED_VF) { + if (pf->vf_configs[vf].trusted) + req->flags |= VF_TRUSTED; + } + + req->vf = vf; + rc = otx2_sync_mbox_msg(&pf->mbox); +out: + mutex_unlock(&pf->mbox.lock); + return rc; +} + +static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf, + bool enable) +{ + struct otx2_nic *pf = netdev_priv(netdev); + struct pci_dev *pdev = pf->pdev; + int rc; + + if (vf >= pci_num_vf(pdev)) + return -EINVAL; + + if (pf->vf_configs[vf].trusted == enable) + return 0; + + pf->vf_configs[vf].trusted = enable; + rc = otx2_set_vf_permissions(pf, vf, OTX2_TRUSTED_VF); + + if (rc) + pf->vf_configs[vf].trusted = !enable; + else + netdev_info(pf->netdev, "VF %d is %strusted\n", + vf, enable ? "" : "not "); + return rc; +} + static const struct net_device_ops otx2_netdev_ops = { .ndo_open = otx2_open, .ndo_stop = otx2_stop, @@ -2261,6 +2322,7 @@ static const struct net_device_ops otx2_netdev_ops = { .ndo_set_vf_vlan = otx2_set_vf_vlan, .ndo_get_vf_config = otx2_get_vf_config, .ndo_setup_tc = otx2_setup_tc, + .ndo_set_vf_trust = otx2_ndo_set_vf_trust, }; static int otx2_wq_init(struct otx2_nic *pf) @@ -2315,6 +2377,40 @@ static int otx2_realloc_msix_vectors(struct otx2_nic *pf) return otx2_register_mbox_intr(pf, false); } +static int otx2_sriov_vfcfg_init(struct otx2_nic *pf) +{ + int i; + + pf->vf_configs = devm_kcalloc(pf->dev, pf->total_vfs, + sizeof(struct otx2_vf_config), + GFP_KERNEL); + if (!pf->vf_configs) + return -ENOMEM; + + for (i = 0; i < pf->total_vfs; i++) { + pf->vf_configs[i].pf = pf; + pf->vf_configs[i].intf_down = true; + pf->vf_configs[i].trusted = false; + INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work, + otx2_vf_link_event_task); + } + + return 0; +} + +static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf) +{ + int i; + + if (!pf->vf_configs) + return; + + for (i = 0; i < pf->total_vfs; i++) { + cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work); + otx2_set_vf_permissions(pf, i, OTX2_RESET_VF_PERM); + } +} + static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; @@ -2509,6 +2605,11 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_mcam_flow_del; + /* Initialize SR-IOV resources */ + err = otx2_sriov_vfcfg_init(pf); + if (err) + goto err_pf_sriov_init; + /* Enable link notifications */ otx2_cgx_config_linkevents(pf, true); @@ -2518,6 +2619,8 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; +err_pf_sriov_init: + otx2_shutdown_tc(pf); err_mcam_flow_del: otx2_mcam_flow_del(pf); err_unreg_netdev: @@ -2576,7 +2679,7 @@ static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs) { struct net_device *netdev = pci_get_drvdata(pdev); struct otx2_nic *pf = netdev_priv(netdev); - int ret, i; + int ret; /* Init PF <=> VF mailbox stuff */ ret = otx2_pfvf_mbox_init(pf, numvfs); @@ -2587,23 +2690,9 @@ static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs) if (ret) goto free_mbox; - pf->vf_configs = kcalloc(numvfs, sizeof(struct otx2_vf_config), - GFP_KERNEL); - if (!pf->vf_configs) { - ret = -ENOMEM; - goto free_intr; - } - - for (i = 0; i < numvfs; i++) { - pf->vf_configs[i].pf = pf; - pf->vf_configs[i].intf_down = true; - INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work, - otx2_vf_link_event_task); - } - ret = otx2_pf_flr_init(pf, numvfs); if (ret) - goto free_configs; + goto free_intr; ret = otx2_register_flr_me_intr(pf, numvfs); if (ret) @@ -2618,8 +2707,6 @@ free_flr_intr: otx2_disable_flr_me_intr(pf); free_flr: otx2_flr_wq_destroy(pf); -free_configs: - kfree(pf->vf_configs); free_intr: otx2_disable_pfvf_mbox_intr(pf, numvfs); free_mbox: @@ -2632,17 +2719,12 @@ static int otx2_sriov_disable(struct pci_dev *pdev) struct net_device *netdev = pci_get_drvdata(pdev); struct otx2_nic *pf = netdev_priv(netdev); int numvfs = pci_num_vf(pdev); - int i; if (!numvfs) return 0; pci_disable_sriov(pdev); - for (i = 0; i < pci_num_vf(pdev); i++) - cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work); - kfree(pf->vf_configs); - otx2_disable_flr_me_intr(pf); otx2_flr_wq_destroy(pf); otx2_disable_pfvf_mbox_intr(pf, numvfs); @@ -2682,6 +2764,7 @@ static void otx2_remove(struct pci_dev *pdev) unregister_netdev(netdev); otx2_sriov_disable(pf->pdev); + otx2_sriov_vfcfg_cleanup(pf); if (pf->otx2_wq) destroy_workqueue(pf->otx2_wq); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 51157b283f6f..905fc02a7dfe 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -15,6 +15,7 @@ #include <net/tc_act/tc_vlan.h> #include <net/ipv6.h> +#include "cn10k.h" #include "otx2_common.h" /* Egress rate limiting definitions */ @@ -41,11 +42,14 @@ struct otx2_tc_flow_stats { struct otx2_tc_flow { struct rhash_head node; unsigned long cookie; - u16 entry; unsigned int bitpos; struct rcu_head rcu; struct otx2_tc_flow_stats stats; spinlock_t lock; /* lock for stats */ + u16 rq; + u16 entry; + u16 leaf_profile; + bool is_act_police; }; static void otx2_get_egress_burst_cfg(u32 burst, u32 *burst_exp, @@ -220,17 +224,76 @@ static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic, return err; } +static int otx2_tc_act_set_police(struct otx2_nic *nic, + struct otx2_tc_flow *node, + struct flow_cls_offload *f, + u64 rate, u32 burst, u32 mark, + struct npc_install_flow_req *req, bool pps) +{ + struct netlink_ext_ack *extack = f->common.extack; + struct otx2_hw *hw = &nic->hw; + int rq_idx, rc; + + rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues); + if (rq_idx >= hw->rx_queues) { + NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded"); + return -EINVAL; + } + + mutex_lock(&nic->mbox.lock); + + rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile); + if (rc) { + mutex_unlock(&nic->mbox.lock); + return rc; + } + + rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile, burst, rate, pps); + if (rc) + goto free_leaf; + + rc = cn10k_map_unmap_rq_policer(nic, rq_idx, node->leaf_profile, true); + if (rc) + goto free_leaf; + + mutex_unlock(&nic->mbox.lock); + + req->match_id = mark & 0xFFFFULL; + req->index = rq_idx; + req->op = NIX_RX_ACTIONOP_UCAST; + set_bit(rq_idx, &nic->rq_bmap); + node->is_act_police = true; + node->rq = rq_idx; + + return 0; + +free_leaf: + if (cn10k_free_leaf_profile(nic, node->leaf_profile)) + netdev_err(nic->netdev, + "Unable to free leaf bandwidth profile(%d)\n", + node->leaf_profile); + mutex_unlock(&nic->mbox.lock); + return rc; +} + static int otx2_tc_parse_actions(struct otx2_nic *nic, struct flow_action *flow_action, - struct npc_install_flow_req *req) + struct npc_install_flow_req *req, + struct flow_cls_offload *f, + struct otx2_tc_flow *node) { + struct netlink_ext_ack *extack = f->common.extack; struct flow_action_entry *act; struct net_device *target; struct otx2_nic *priv; + u32 burst, mark = 0; + u8 nr_police = 0; + bool pps; + u64 rate; int i; if (!flow_action_has_entries(flow_action)) { - netdev_info(nic->netdev, "no tc actions specified"); + NL_SET_ERR_MSG_MOD(extack, "no tc actions specified"); return -EINVAL; } @@ -247,8 +310,8 @@ static int otx2_tc_parse_actions(struct otx2_nic *nic, priv = netdev_priv(target); /* npc_install_flow_req doesn't support passing a target pcifunc */ if (rvu_get_pf(nic->pcifunc) != rvu_get_pf(priv->pcifunc)) { - netdev_info(nic->netdev, - "can't redirect to other pf/vf\n"); + NL_SET_ERR_MSG_MOD(extack, + "can't redirect to other pf/vf"); return -EOPNOTSUPP; } req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK; @@ -259,18 +322,55 @@ static int otx2_tc_parse_actions(struct otx2_nic *nic, /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */ req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7; break; + case FLOW_ACTION_POLICE: + /* Ingress ratelimiting is not supported on OcteonTx2 */ + if (is_dev_otx2(nic->pdev)) { + NL_SET_ERR_MSG_MOD(extack, + "Ingress policing not supported on this platform"); + return -EOPNOTSUPP; + } + + if (act->police.rate_bytes_ps > 0) { + rate = act->police.rate_bytes_ps * 8; + burst = act->police.burst; + } else if (act->police.rate_pkt_ps > 0) { + /* The algorithm used to calculate rate + * mantissa, exponent values for a given token + * rate (token can be byte or packet) requires + * token rate to be mutiplied by 8. + */ + rate = act->police.rate_pkt_ps * 8; + burst = act->police.burst_pkt; + pps = true; + } + nr_police++; + break; + case FLOW_ACTION_MARK: + mark = act->mark; + break; default: return -EOPNOTSUPP; } } + if (nr_police > 1) { + NL_SET_ERR_MSG_MOD(extack, + "rate limit police offload requires a single action"); + return -EOPNOTSUPP; + } + + if (nr_police) + return otx2_tc_act_set_police(nic, node, f, rate, burst, + mark, req, pps); + return 0; } -static int otx2_tc_prepare_flow(struct otx2_nic *nic, +static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node, struct flow_cls_offload *f, struct npc_install_flow_req *req) { + struct netlink_ext_ack *extack = f->common.extack; struct flow_msg *flow_spec = &req->packet; struct flow_msg *flow_mask = &req->mask; struct flow_dissector *dissector; @@ -335,7 +435,7 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, flow_rule_match_eth_addrs(rule, &match); if (!is_zero_ether_addr(match.mask->src)) { - netdev_err(nic->netdev, "src mac match not supported\n"); + NL_SET_ERR_MSG_MOD(extack, "src mac match not supported"); return -EOPNOTSUPP; } @@ -353,11 +453,11 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, flow_rule_match_ip(rule, &match); if ((ntohs(flow_spec->etype) != ETH_P_IP) && match.mask->tos) { - netdev_err(nic->netdev, "tos not supported\n"); + NL_SET_ERR_MSG_MOD(extack, "tos not supported"); return -EOPNOTSUPP; } if (match.mask->ttl) { - netdev_err(nic->netdev, "ttl not supported\n"); + NL_SET_ERR_MSG_MOD(extack, "ttl not supported"); return -EOPNOTSUPP; } flow_spec->tos = match.key->tos; @@ -413,8 +513,8 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, if (ipv6_addr_loopback(&match.key->dst) || ipv6_addr_loopback(&match.key->src)) { - netdev_err(nic->netdev, - "Flow matching on IPv6 loopback addr is not supported\n"); + NL_SET_ERR_MSG_MOD(extack, + "Flow matching IPv6 loopback addr not supported"); return -EOPNOTSUPP; } @@ -463,7 +563,7 @@ static int otx2_tc_prepare_flow(struct otx2_nic *nic, req->features |= BIT_ULL(NPC_SPORT_SCTP); } - return otx2_tc_parse_actions(nic, &rule->action, req); + return otx2_tc_parse_actions(nic, &rule->action, req, f, node); } static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry) @@ -498,6 +598,7 @@ static int otx2_tc_del_flow(struct otx2_nic *nic, { struct otx2_tc_info *tc_info = &nic->tc_info; struct otx2_tc_flow *flow_node; + int err; flow_node = rhashtable_lookup_fast(&tc_info->flow_table, &tc_flow_cmd->cookie, @@ -508,6 +609,27 @@ static int otx2_tc_del_flow(struct otx2_nic *nic, return -EINVAL; } + if (flow_node->is_act_police) { + mutex_lock(&nic->mbox.lock); + + err = cn10k_map_unmap_rq_policer(nic, flow_node->rq, + flow_node->leaf_profile, false); + if (err) + netdev_err(nic->netdev, + "Unmapping RQ %d & profile %d failed\n", + flow_node->rq, flow_node->leaf_profile); + + err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile); + if (err) + netdev_err(nic->netdev, + "Unable to free leaf bandwidth profile(%d)\n", + flow_node->leaf_profile); + + __clear_bit(flow_node->rq, &nic->rq_bmap); + + mutex_unlock(&nic->mbox.lock); + } + otx2_del_mcam_flow_entry(nic, flow_node->entry); WARN_ON(rhashtable_remove_fast(&nic->tc_info.flow_table, @@ -524,14 +646,21 @@ static int otx2_tc_del_flow(struct otx2_nic *nic, static int otx2_tc_add_flow(struct otx2_nic *nic, struct flow_cls_offload *tc_flow_cmd) { + struct netlink_ext_ack *extack = tc_flow_cmd->common.extack; struct otx2_tc_info *tc_info = &nic->tc_info; struct otx2_tc_flow *new_node, *old_node; - struct npc_install_flow_req *req; - int rc; + struct npc_install_flow_req *req, dummy; + int rc, err; if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) return -ENOMEM; + if (bitmap_full(tc_info->tc_entries_bitmap, nic->flow_cfg->tc_max_flows)) { + NL_SET_ERR_MSG_MOD(extack, + "Not enough MCAM space to add the flow"); + return -ENOMEM; + } + /* allocate memory for the new flow and it's node */ new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); if (!new_node) @@ -539,17 +668,11 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, spin_lock_init(&new_node->lock); new_node->cookie = tc_flow_cmd->cookie; - mutex_lock(&nic->mbox.lock); - req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); - if (!req) { - mutex_unlock(&nic->mbox.lock); - return -ENOMEM; - } + memset(&dummy, 0, sizeof(struct npc_install_flow_req)); - rc = otx2_tc_prepare_flow(nic, tc_flow_cmd, req); + rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy); if (rc) { - otx2_mbox_reset(&nic->mbox.mbox, 0); - mutex_unlock(&nic->mbox.lock); + kfree_rcu(new_node, rcu); return rc; } @@ -560,18 +683,22 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, if (old_node) otx2_tc_del_flow(nic, tc_flow_cmd); - if (bitmap_full(tc_info->tc_entries_bitmap, nic->flow_cfg->tc_max_flows)) { - netdev_err(nic->netdev, "Not enough MCAM space to add the flow\n"); - otx2_mbox_reset(&nic->mbox.mbox, 0); + mutex_lock(&nic->mbox.lock); + req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); + if (!req) { mutex_unlock(&nic->mbox.lock); - return -ENOMEM; + rc = -ENOMEM; + goto free_leaf; } + memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); + memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); + new_node->bitpos = find_first_zero_bit(tc_info->tc_entries_bitmap, nic->flow_cfg->tc_max_flows); req->channel = nic->hw.rx_chan_base; - req->entry = nic->flow_cfg->entry[nic->flow_cfg->tc_flower_offset + - nic->flow_cfg->tc_max_flows - new_node->bitpos]; + req->entry = nic->flow_cfg->flow_ent[nic->flow_cfg->tc_flower_offset + + nic->flow_cfg->tc_max_flows - new_node->bitpos]; req->intf = NIX_INTF_RX; req->set_cntr = 1; new_node->entry = req->entry; @@ -579,9 +706,10 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, /* Send message to AF */ rc = otx2_sync_mbox_msg(&nic->mbox); if (rc) { - netdev_err(nic->netdev, "Failed to install MCAM flow entry\n"); + NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry"); mutex_unlock(&nic->mbox.lock); - goto out; + kfree_rcu(new_node, rcu); + goto free_leaf; } mutex_unlock(&nic->mbox.lock); @@ -591,12 +719,35 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, if (rc) { otx2_del_mcam_flow_entry(nic, req->entry); kfree_rcu(new_node, rcu); - goto out; + goto free_leaf; } set_bit(new_node->bitpos, tc_info->tc_entries_bitmap); tc_info->num_entries++; -out: + + return 0; + +free_leaf: + if (new_node->is_act_police) { + mutex_lock(&nic->mbox.lock); + + err = cn10k_map_unmap_rq_policer(nic, new_node->rq, + new_node->leaf_profile, false); + if (err) + netdev_err(nic->netdev, + "Unmapping RQ %d & profile %d failed\n", + new_node->rq, new_node->leaf_profile); + err = cn10k_free_leaf_profile(nic, new_node->leaf_profile); + if (err) + netdev_err(nic->netdev, + "Unable to free leaf bandwidth profile(%d)\n", + new_node->leaf_profile); + + __clear_bit(new_node->rq, &nic->rq_bmap); + + mutex_unlock(&nic->mbox.lock); + } + return rc; } @@ -675,6 +826,87 @@ static int otx2_setup_tc_cls_flower(struct otx2_nic *nic, } } +static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic, + struct tc_cls_matchall_offload *cls) +{ + struct netlink_ext_ack *extack = cls->common.extack; + struct flow_action *actions = &cls->rule->action; + struct flow_action_entry *entry; + u64 rate; + int err; + + err = otx2_tc_validate_flow(nic, actions, extack); + if (err) + return err; + + if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) { + NL_SET_ERR_MSG_MOD(extack, + "Only one ingress MATCHALL ratelimitter can be offloaded"); + return -ENOMEM; + } + + entry = &cls->rule->action.entries[0]; + switch (entry->id) { + case FLOW_ACTION_POLICE: + /* Ingress ratelimiting is not supported on OcteonTx2 */ + if (is_dev_otx2(nic->pdev)) { + NL_SET_ERR_MSG_MOD(extack, + "Ingress policing not supported on this platform"); + return -EOPNOTSUPP; + } + + err = cn10k_alloc_matchall_ipolicer(nic); + if (err) + return err; + + /* Convert to bits per second */ + rate = entry->police.rate_bytes_ps * 8; + err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate); + if (err) + return err; + nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; + break; + default: + NL_SET_ERR_MSG_MOD(extack, + "Only police action supported with Ingress MATCHALL offload"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic, + struct tc_cls_matchall_offload *cls) +{ + struct netlink_ext_ack *extack = cls->common.extack; + int err; + + if (nic->flags & OTX2_FLAG_INTF_DOWN) { + NL_SET_ERR_MSG_MOD(extack, "Interface not initialized"); + return -EINVAL; + } + + err = cn10k_free_matchall_ipolicer(nic); + nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED; + return err; +} + +static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic, + struct tc_cls_matchall_offload *cls_matchall) +{ + switch (cls_matchall->command) { + case TC_CLSMATCHALL_REPLACE: + return otx2_tc_ingress_matchall_install(nic, cls_matchall); + case TC_CLSMATCHALL_DESTROY: + return otx2_tc_ingress_matchall_delete(nic, cls_matchall); + case TC_CLSMATCHALL_STATS: + default: + break; + } + + return -EOPNOTSUPP; +} + static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type, void *type_data, void *cb_priv) { @@ -686,6 +918,8 @@ static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type, switch (type) { case TC_SETUP_CLSFLOWER: return otx2_setup_tc_cls_flower(nic, type_data); + case TC_SETUP_CLSMATCHALL: + return otx2_setup_tc_ingress_matchall(nic, type_data); default: break; } @@ -775,6 +1009,9 @@ int otx2_init_tc(struct otx2_nic *nic) { struct otx2_tc_info *tc = &nic->tc_info; + /* Exclude receive queue 0 being used for police action */ + set_bit(0, &nic->rq_bmap); + tc->flow_ht_params = tc_flow_ht_params; return rhashtable_init(&tc->flow_table, &tc->flow_ht_params); } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index 085be90a03eb..13a908f75ba0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -395,6 +395,42 @@ static netdev_tx_t otx2vf_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +static void otx2vf_set_rx_mode(struct net_device *netdev) +{ + struct otx2_nic *vf = netdev_priv(netdev); + + queue_work(vf->otx2_wq, &vf->rx_mode_work); +} + +static void otx2vf_do_set_rx_mode(struct work_struct *work) +{ + struct otx2_nic *vf = container_of(work, struct otx2_nic, rx_mode_work); + struct net_device *netdev = vf->netdev; + unsigned int flags = netdev->flags; + struct nix_rx_mode *req; + + mutex_lock(&vf->mbox.lock); + + req = otx2_mbox_alloc_msg_nix_set_rx_mode(&vf->mbox); + if (!req) { + mutex_unlock(&vf->mbox.lock); + return; + } + + req->mode = NIX_RX_MODE_UCAST; + + if (flags & IFF_PROMISC) + req->mode |= NIX_RX_MODE_PROMISC; + if (flags & (IFF_ALLMULTI | IFF_MULTICAST)) + req->mode |= NIX_RX_MODE_ALLMULTI; + + req->mode |= NIX_RX_MODE_USE_MCE; + + otx2_sync_mbox_msg(&vf->mbox); + + mutex_unlock(&vf->mbox.lock); +} + static int otx2vf_change_mtu(struct net_device *netdev, int new_mtu) { bool if_up = netif_running(netdev); @@ -432,12 +468,24 @@ static const struct net_device_ops otx2vf_netdev_ops = { .ndo_open = otx2vf_open, .ndo_stop = otx2vf_stop, .ndo_start_xmit = otx2vf_xmit, + .ndo_set_rx_mode = otx2vf_set_rx_mode, .ndo_set_mac_address = otx2_set_mac_address, .ndo_change_mtu = otx2vf_change_mtu, .ndo_get_stats64 = otx2_get_stats64, .ndo_tx_timeout = otx2_tx_timeout, }; +static int otx2_wq_init(struct otx2_nic *vf) +{ + vf->otx2_wq = create_singlethread_workqueue("otx2vf_wq"); + if (!vf->otx2_wq) + return -ENOMEM; + + INIT_WORK(&vf->rx_mode_work, otx2vf_do_set_rx_mode); + INIT_WORK(&vf->reset_task, otx2vf_reset_task); + return 0; +} + static int otx2vf_realloc_msix_vectors(struct otx2_nic *vf) { struct otx2_hw *hw = &vf->hw; @@ -588,8 +636,6 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->min_mtu = OTX2_MIN_MTU; netdev->max_mtu = otx2_get_max_mtu(vf); - INIT_WORK(&vf->reset_task, otx2vf_reset_task); - /* To distinguish, for LBK VFs set netdev name explicitly */ if (is_otx2_lbkvf(vf->pdev)) { int n; @@ -606,6 +652,10 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_detach_rsrc; } + err = otx2_wq_init(vf); + if (err) + goto err_unreg_netdev; + otx2vf_set_ethtool_ops(netdev); /* Enable pause frames by default */ @@ -614,6 +664,8 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; +err_unreg_netdev: + unregister_netdev(netdev); err_detach_rsrc: if (hw->lmt_base) iounmap(hw->lmt_base); @@ -644,6 +696,8 @@ static void otx2vf_remove(struct pci_dev *pdev) cancel_work_sync(&vf->reset_task); unregister_netdev(netdev); + if (vf->otx2_wq) + destroy_workqueue(vf->otx2_wq); otx2vf_disable_mbox_intr(vf); otx2_detach_resources(&vf->mbox); diff --git a/drivers/net/ethernet/marvell/prestera/Makefile b/drivers/net/ethernet/marvell/prestera/Makefile index 93129e32ebc5..0609df8b913d 100644 --- a/drivers/net/ethernet/marvell/prestera/Makefile +++ b/drivers/net/ethernet/marvell/prestera/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_PRESTERA) += prestera.o prestera-objs := prestera_main.o prestera_hw.o prestera_dsa.o \ prestera_rxtx.o prestera_devlink.o prestera_ethtool.o \ - prestera_switchdev.o + prestera_switchdev.o prestera_acl.o prestera_flow.o \ + prestera_flower.o prestera_span.o obj-$(CONFIG_PRESTERA_PCI) += prestera_pci.o diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h index 55aa4bf8a27c..f18fe664b373 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera.h +++ b/drivers/net/ethernet/marvell/prestera/prestera.h @@ -60,10 +60,22 @@ struct prestera_port_caps { u8 transceiver; }; +struct prestera_lag { + struct net_device *dev; + struct list_head members; + u16 member_count; + u16 lag_id; +}; + +struct prestera_flow_block; + struct prestera_port { struct net_device *dev; struct prestera_switch *sw; + struct prestera_flow_block *flow_block; struct devlink_port dl_port; + struct list_head lag_member; + struct prestera_lag *lag; u32 id; u32 hw_id; u32 dev_id; @@ -127,6 +139,12 @@ struct prestera_port_event { } data; }; +enum prestera_fdb_entry_type { + PRESTERA_FDB_ENTRY_TYPE_REG_PORT, + PRESTERA_FDB_ENTRY_TYPE_LAG, + PRESTERA_FDB_ENTRY_TYPE_MAX +}; + enum prestera_fdb_event_id { PRESTERA_FDB_EVENT_UNSPEC, PRESTERA_FDB_EVENT_LEARNED, @@ -134,7 +152,11 @@ enum prestera_fdb_event_id { }; struct prestera_fdb_event { - u32 port_id; + enum prestera_fdb_entry_type type; + union { + u32 port_id; + u16 lag_id; + } dest; u32 vid; union { u8 mac[ETH_ALEN]; @@ -150,14 +172,20 @@ struct prestera_event { }; struct prestera_switchdev; +struct prestera_span; struct prestera_rxtx; +struct prestera_trap_data; +struct prestera_acl; struct prestera_switch { struct prestera_device *dev; struct prestera_switchdev *swdev; struct prestera_rxtx *rxtx; + struct prestera_acl *acl; + struct prestera_span *span; struct list_head event_handlers; struct notifier_block netdev_nb; + struct prestera_trap_data *trap_data; char base_mac[ETH_ALEN]; struct list_head port_list; rwlock_t port_list_lock; @@ -165,6 +193,9 @@ struct prestera_switch { u32 mtu_min; u32 mtu_max; u8 id; + struct prestera_lag *lags; + u8 lag_member_max; + u8 lag_max; }; struct prestera_rxtx_params { @@ -203,4 +234,10 @@ int prestera_port_pvid_set(struct prestera_port *port, u16 vid); bool prestera_netdev_check(const struct net_device *dev); +bool prestera_port_is_lag_member(const struct prestera_port *port); + +struct prestera_lag *prestera_lag_by_id(struct prestera_switch *sw, u16 id); + +u16 prestera_port_lag_id(const struct prestera_port *port); + #endif /* _PRESTERA_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c new file mode 100644 index 000000000000..83c75ffb1a1c --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved */ + +#include <linux/rhashtable.h> + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_acl.h" +#include "prestera_span.h" + +struct prestera_acl { + struct prestera_switch *sw; + struct list_head rules; +}; + +struct prestera_acl_ruleset { + struct rhashtable rule_ht; + struct prestera_switch *sw; + u16 id; +}; + +struct prestera_acl_rule { + struct rhash_head ht_node; + struct list_head list; + struct list_head match_list; + struct list_head action_list; + struct prestera_flow_block *block; + unsigned long cookie; + u32 priority; + u8 n_actions; + u8 n_matches; + u32 id; +}; + +static const struct rhashtable_params prestera_acl_rule_ht_params = { + .key_len = sizeof(unsigned long), + .key_offset = offsetof(struct prestera_acl_rule, cookie), + .head_offset = offsetof(struct prestera_acl_rule, ht_node), + .automatic_shrinking = true, +}; + +static struct prestera_acl_ruleset * +prestera_acl_ruleset_create(struct prestera_switch *sw) +{ + struct prestera_acl_ruleset *ruleset; + int err; + + ruleset = kzalloc(sizeof(*ruleset), GFP_KERNEL); + if (!ruleset) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(&ruleset->rule_ht, &prestera_acl_rule_ht_params); + if (err) + goto err_rhashtable_init; + + err = prestera_hw_acl_ruleset_create(sw, &ruleset->id); + if (err) + goto err_ruleset_create; + + ruleset->sw = sw; + + return ruleset; + +err_ruleset_create: + rhashtable_destroy(&ruleset->rule_ht); +err_rhashtable_init: + kfree(ruleset); + return ERR_PTR(err); +} + +static void prestera_acl_ruleset_destroy(struct prestera_acl_ruleset *ruleset) +{ + prestera_hw_acl_ruleset_del(ruleset->sw, ruleset->id); + rhashtable_destroy(&ruleset->rule_ht); + kfree(ruleset); +} + +struct prestera_flow_block * +prestera_acl_block_create(struct prestera_switch *sw, struct net *net) +{ + struct prestera_flow_block *block; + + block = kzalloc(sizeof(*block), GFP_KERNEL); + if (!block) + return NULL; + INIT_LIST_HEAD(&block->binding_list); + block->net = net; + block->sw = sw; + + block->ruleset = prestera_acl_ruleset_create(sw); + if (IS_ERR(block->ruleset)) { + kfree(block); + return NULL; + } + + return block; +} + +void prestera_acl_block_destroy(struct prestera_flow_block *block) +{ + prestera_acl_ruleset_destroy(block->ruleset); + WARN_ON(!list_empty(&block->binding_list)); + kfree(block); +} + +static struct prestera_flow_block_binding * +prestera_acl_block_lookup(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + if (binding->port == port) + return binding; + + return NULL; +} + +int prestera_acl_block_bind(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + int err; + + if (WARN_ON(prestera_acl_block_lookup(block, port))) + return -EEXIST; + + binding = kzalloc(sizeof(*binding), GFP_KERNEL); + if (!binding) + return -ENOMEM; + binding->span_id = PRESTERA_SPAN_INVALID_ID; + binding->port = port; + + err = prestera_hw_acl_port_bind(port, block->ruleset->id); + if (err) + goto err_rules_bind; + + list_add(&binding->list, &block->binding_list); + return 0; + +err_rules_bind: + kfree(binding); + return err; +} + +int prestera_acl_block_unbind(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + + binding = prestera_acl_block_lookup(block, port); + if (!binding) + return -ENOENT; + + list_del(&binding->list); + + prestera_hw_acl_port_unbind(port, block->ruleset->id); + + kfree(binding); + return 0; +} + +struct prestera_acl_ruleset * +prestera_acl_block_ruleset_get(struct prestera_flow_block *block) +{ + return block->ruleset; +} + +u16 prestera_acl_rule_ruleset_id_get(const struct prestera_acl_rule *rule) +{ + return rule->block->ruleset->id; +} + +struct net *prestera_acl_block_net(struct prestera_flow_block *block) +{ + return block->net; +} + +struct prestera_switch *prestera_acl_block_sw(struct prestera_flow_block *block) +{ + return block->sw; +} + +struct prestera_acl_rule * +prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, + unsigned long cookie) +{ + return rhashtable_lookup_fast(&ruleset->rule_ht, &cookie, + prestera_acl_rule_ht_params); +} + +struct prestera_acl_rule * +prestera_acl_rule_create(struct prestera_flow_block *block, + unsigned long cookie) +{ + struct prestera_acl_rule *rule; + + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&rule->match_list); + INIT_LIST_HEAD(&rule->action_list); + rule->cookie = cookie; + rule->block = block; + + return rule; +} + +struct list_head * +prestera_acl_rule_match_list_get(struct prestera_acl_rule *rule) +{ + return &rule->match_list; +} + +struct list_head * +prestera_acl_rule_action_list_get(struct prestera_acl_rule *rule) +{ + return &rule->action_list; +} + +int prestera_acl_rule_action_add(struct prestera_acl_rule *rule, + struct prestera_acl_rule_action_entry *entry) +{ + struct prestera_acl_rule_action_entry *a_entry; + + a_entry = kmalloc(sizeof(*a_entry), GFP_KERNEL); + if (!a_entry) + return -ENOMEM; + + memcpy(a_entry, entry, sizeof(*entry)); + list_add(&a_entry->list, &rule->action_list); + + rule->n_actions++; + return 0; +} + +u8 prestera_acl_rule_action_len(struct prestera_acl_rule *rule) +{ + return rule->n_actions; +} + +u32 prestera_acl_rule_priority_get(struct prestera_acl_rule *rule) +{ + return rule->priority; +} + +void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, + u32 priority) +{ + rule->priority = priority; +} + +int prestera_acl_rule_match_add(struct prestera_acl_rule *rule, + struct prestera_acl_rule_match_entry *entry) +{ + struct prestera_acl_rule_match_entry *m_entry; + + m_entry = kmalloc(sizeof(*m_entry), GFP_KERNEL); + if (!m_entry) + return -ENOMEM; + + memcpy(m_entry, entry, sizeof(*entry)); + list_add(&m_entry->list, &rule->match_list); + + rule->n_matches++; + return 0; +} + +u8 prestera_acl_rule_match_len(struct prestera_acl_rule *rule) +{ + return rule->n_matches; +} + +void prestera_acl_rule_destroy(struct prestera_acl_rule *rule) +{ + struct prestera_acl_rule_action_entry *a_entry; + struct prestera_acl_rule_match_entry *m_entry; + struct list_head *pos, *n; + + list_for_each_safe(pos, n, &rule->match_list) { + m_entry = list_entry(pos, typeof(*m_entry), list); + list_del(pos); + kfree(m_entry); + } + + list_for_each_safe(pos, n, &rule->action_list) { + a_entry = list_entry(pos, typeof(*a_entry), list); + list_del(pos); + kfree(a_entry); + } + + kfree(rule); +} + +int prestera_acl_rule_add(struct prestera_switch *sw, + struct prestera_acl_rule *rule) +{ + u32 rule_id; + int err; + + /* try to add rule to hash table first */ + err = rhashtable_insert_fast(&rule->block->ruleset->rule_ht, + &rule->ht_node, + prestera_acl_rule_ht_params); + if (err) + return err; + + /* add rule to hw */ + err = prestera_hw_acl_rule_add(sw, rule, &rule_id); + if (err) + goto err_rule_add; + + rule->id = rule_id; + + list_add_tail(&rule->list, &sw->acl->rules); + + return 0; + +err_rule_add: + rhashtable_remove_fast(&rule->block->ruleset->rule_ht, &rule->ht_node, + prestera_acl_rule_ht_params); + return err; +} + +void prestera_acl_rule_del(struct prestera_switch *sw, + struct prestera_acl_rule *rule) +{ + rhashtable_remove_fast(&rule->block->ruleset->rule_ht, &rule->ht_node, + prestera_acl_rule_ht_params); + list_del(&rule->list); + prestera_hw_acl_rule_del(sw, rule->id); +} + +int prestera_acl_rule_get_stats(struct prestera_switch *sw, + struct prestera_acl_rule *rule, + u64 *packets, u64 *bytes, u64 *last_use) +{ + u64 current_packets; + u64 current_bytes; + int err; + + err = prestera_hw_acl_rule_stats_get(sw, rule->id, ¤t_packets, + ¤t_bytes); + if (err) + return err; + + *packets = current_packets; + *bytes = current_bytes; + *last_use = jiffies; + + return 0; +} + +int prestera_acl_init(struct prestera_switch *sw) +{ + struct prestera_acl *acl; + + acl = kzalloc(sizeof(*acl), GFP_KERNEL); + if (!acl) + return -ENOMEM; + + INIT_LIST_HEAD(&acl->rules); + sw->acl = acl; + acl->sw = sw; + + return 0; +} + +void prestera_acl_fini(struct prestera_switch *sw) +{ + struct prestera_acl *acl = sw->acl; + + WARN_ON(!list_empty(&acl->rules)); + kfree(acl); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.h b/drivers/net/ethernet/marvell/prestera/prestera_acl.h new file mode 100644 index 000000000000..39b7869be659 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_ACL_H_ +#define _PRESTERA_ACL_H_ + +enum prestera_acl_rule_match_entry_type { + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE = 1, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_SRC, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_DST, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE, + PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE +}; + +enum prestera_acl_rule_action { + PRESTERA_ACL_RULE_ACTION_ACCEPT, + PRESTERA_ACL_RULE_ACTION_DROP, + PRESTERA_ACL_RULE_ACTION_TRAP +}; + +struct prestera_switch; +struct prestera_port; +struct prestera_acl_rule; +struct prestera_acl_ruleset; + +struct prestera_flow_block_binding { + struct list_head list; + struct prestera_port *port; + int span_id; +}; + +struct prestera_flow_block { + struct list_head binding_list; + struct prestera_switch *sw; + struct net *net; + struct prestera_acl_ruleset *ruleset; + struct flow_block_cb *block_cb; +}; + +struct prestera_acl_rule_action_entry { + struct list_head list; + enum prestera_acl_rule_action id; +}; + +struct prestera_acl_rule_match_entry { + struct list_head list; + enum prestera_acl_rule_match_entry_type type; + union { + struct { + u8 key; + u8 mask; + } u8; + struct { + u16 key; + u16 mask; + } u16; + struct { + u32 key; + u32 mask; + } u32; + struct { + u64 key; + u64 mask; + } u64; + struct { + u8 key[ETH_ALEN]; + u8 mask[ETH_ALEN]; + } mac; + } keymask; +}; + +int prestera_acl_init(struct prestera_switch *sw); +void prestera_acl_fini(struct prestera_switch *sw); +struct prestera_flow_block * +prestera_acl_block_create(struct prestera_switch *sw, struct net *net); +void prestera_acl_block_destroy(struct prestera_flow_block *block); +struct net *prestera_acl_block_net(struct prestera_flow_block *block); +struct prestera_switch *prestera_acl_block_sw(struct prestera_flow_block *block); +int prestera_acl_block_bind(struct prestera_flow_block *block, + struct prestera_port *port); +int prestera_acl_block_unbind(struct prestera_flow_block *block, + struct prestera_port *port); +struct prestera_acl_ruleset * +prestera_acl_block_ruleset_get(struct prestera_flow_block *block); +struct prestera_acl_rule * +prestera_acl_rule_create(struct prestera_flow_block *block, + unsigned long cookie); +u32 prestera_acl_rule_priority_get(struct prestera_acl_rule *rule); +void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, + u32 priority); +u16 prestera_acl_rule_ruleset_id_get(const struct prestera_acl_rule *rule); +struct list_head * +prestera_acl_rule_action_list_get(struct prestera_acl_rule *rule); +u8 prestera_acl_rule_action_len(struct prestera_acl_rule *rule); +u8 prestera_acl_rule_match_len(struct prestera_acl_rule *rule); +int prestera_acl_rule_action_add(struct prestera_acl_rule *rule, + struct prestera_acl_rule_action_entry *entry); +struct list_head * +prestera_acl_rule_match_list_get(struct prestera_acl_rule *rule); +int prestera_acl_rule_match_add(struct prestera_acl_rule *rule, + struct prestera_acl_rule_match_entry *entry); +void prestera_acl_rule_destroy(struct prestera_acl_rule *rule); +struct prestera_acl_rule * +prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, + unsigned long cookie); +int prestera_acl_rule_add(struct prestera_switch *sw, + struct prestera_acl_rule *rule); +void prestera_acl_rule_del(struct prestera_switch *sw, + struct prestera_acl_rule *rule); +int prestera_acl_rule_get_stats(struct prestera_switch *sw, + struct prestera_acl_rule *rule, + u64 *packets, u64 *bytes, u64 *last_use); + +#endif /* _PRESTERA_ACL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_devlink.c b/drivers/net/ethernet/marvell/prestera/prestera_devlink.c index 94c185a0e2b8..d12e21db9fd6 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_devlink.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_devlink.c @@ -4,6 +4,352 @@ #include <net/devlink.h> #include "prestera_devlink.h" +#include "prestera_hw.h" + +/* All driver-specific traps must be documented in + * Documentation/networking/devlink/prestera.rst + */ +enum { + DEVLINK_PRESTERA_TRAP_ID_BASE = DEVLINK_TRAP_GENERIC_ID_MAX, + DEVLINK_PRESTERA_TRAP_ID_ARP_BC, + DEVLINK_PRESTERA_TRAP_ID_IS_IS, + DEVLINK_PRESTERA_TRAP_ID_OSPF, + DEVLINK_PRESTERA_TRAP_ID_IP_BC_MAC, + DEVLINK_PRESTERA_TRAP_ID_ROUTER_MC, + DEVLINK_PRESTERA_TRAP_ID_VRRP, + DEVLINK_PRESTERA_TRAP_ID_DHCP, + DEVLINK_PRESTERA_TRAP_ID_MAC_TO_ME, + DEVLINK_PRESTERA_TRAP_ID_IPV4_OPTIONS, + DEVLINK_PRESTERA_TRAP_ID_IP_DEFAULT_ROUTE, + DEVLINK_PRESTERA_TRAP_ID_IP_TO_ME, + DEVLINK_PRESTERA_TRAP_ID_IPV4_ICMP_REDIRECT, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_0, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_1, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_2, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_3, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_4, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_5, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_6, + DEVLINK_PRESTERA_TRAP_ID_ACL_CODE_7, + DEVLINK_PRESTERA_TRAP_ID_BGP, + DEVLINK_PRESTERA_TRAP_ID_SSH, + DEVLINK_PRESTERA_TRAP_ID_TELNET, + DEVLINK_PRESTERA_TRAP_ID_ICMP, + DEVLINK_PRESTERA_TRAP_ID_MET_RED, + DEVLINK_PRESTERA_TRAP_ID_IP_SIP_IS_ZERO, + DEVLINK_PRESTERA_TRAP_ID_IP_UC_DIP_DA_MISMATCH, + DEVLINK_PRESTERA_TRAP_ID_ILLEGAL_IPV4_HDR, + DEVLINK_PRESTERA_TRAP_ID_ILLEGAL_IP_ADDR, + DEVLINK_PRESTERA_TRAP_ID_INVALID_SA, + DEVLINK_PRESTERA_TRAP_ID_LOCAL_PORT, + DEVLINK_PRESTERA_TRAP_ID_PORT_NO_VLAN, + DEVLINK_PRESTERA_TRAP_ID_RXDMA_DROP, +}; + +#define DEVLINK_PRESTERA_TRAP_NAME_ARP_BC \ + "arp_bc" +#define DEVLINK_PRESTERA_TRAP_NAME_IS_IS \ + "is_is" +#define DEVLINK_PRESTERA_TRAP_NAME_OSPF \ + "ospf" +#define DEVLINK_PRESTERA_TRAP_NAME_IP_BC_MAC \ + "ip_bc_mac" +#define DEVLINK_PRESTERA_TRAP_NAME_ROUTER_MC \ + "router_mc" +#define DEVLINK_PRESTERA_TRAP_NAME_VRRP \ + "vrrp" +#define DEVLINK_PRESTERA_TRAP_NAME_DHCP \ + "dhcp" +#define DEVLINK_PRESTERA_TRAP_NAME_MAC_TO_ME \ + "mac_to_me" +#define DEVLINK_PRESTERA_TRAP_NAME_IPV4_OPTIONS \ + "ipv4_options" +#define DEVLINK_PRESTERA_TRAP_NAME_IP_DEFAULT_ROUTE \ + "ip_default_route" +#define DEVLINK_PRESTERA_TRAP_NAME_IP_TO_ME \ + "ip_to_me" +#define DEVLINK_PRESTERA_TRAP_NAME_IPV4_ICMP_REDIRECT \ + "ipv4_icmp_redirect" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_0 \ + "acl_code_0" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_1 \ + "acl_code_1" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_2 \ + "acl_code_2" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_3 \ + "acl_code_3" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_4 \ + "acl_code_4" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_5 \ + "acl_code_5" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_6 \ + "acl_code_6" +#define DEVLINK_PRESTERA_TRAP_NAME_ACL_CODE_7 \ + "acl_code_7" +#define DEVLINK_PRESTERA_TRAP_NAME_BGP \ + "bgp" +#define DEVLINK_PRESTERA_TRAP_NAME_SSH \ + "ssh" +#define DEVLINK_PRESTERA_TRAP_NAME_TELNET \ + "telnet" +#define DEVLINK_PRESTERA_TRAP_NAME_ICMP \ + "icmp" +#define DEVLINK_PRESTERA_TRAP_NAME_RXDMA_DROP \ + "rxdma_drop" +#define DEVLINK_PRESTERA_TRAP_NAME_PORT_NO_VLAN \ + "port_no_vlan" +#define DEVLINK_PRESTERA_TRAP_NAME_LOCAL_PORT \ + "local_port" +#define DEVLINK_PRESTERA_TRAP_NAME_INVALID_SA \ + "invalid_sa" +#define DEVLINK_PRESTERA_TRAP_NAME_ILLEGAL_IP_ADDR \ + "illegal_ip_addr" +#define DEVLINK_PRESTERA_TRAP_NAME_ILLEGAL_IPV4_HDR \ + "illegal_ipv4_hdr" +#define DEVLINK_PRESTERA_TRAP_NAME_IP_UC_DIP_DA_MISMATCH \ + "ip_uc_dip_da_mismatch" +#define DEVLINK_PRESTERA_TRAP_NAME_IP_SIP_IS_ZERO \ + "ip_sip_is_zero" +#define DEVLINK_PRESTERA_TRAP_NAME_MET_RED \ + "met_red" + +struct prestera_trap { + struct devlink_trap trap; + u8 cpu_code; +}; + +struct prestera_trap_item { + enum devlink_trap_action action; + void *trap_ctx; +}; + +struct prestera_trap_data { + struct prestera_switch *sw; + struct prestera_trap_item *trap_items_arr; + u32 traps_count; +}; + +#define PRESTERA_TRAP_METADATA DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT + +#define PRESTERA_TRAP_CONTROL(_id, _group_id, _action) \ + DEVLINK_TRAP_GENERIC(CONTROL, _action, _id, \ + DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \ + PRESTERA_TRAP_METADATA) + +#define PRESTERA_TRAP_DRIVER_CONTROL(_id, _group_id) \ + DEVLINK_TRAP_DRIVER(CONTROL, TRAP, DEVLINK_PRESTERA_TRAP_ID_##_id, \ + DEVLINK_PRESTERA_TRAP_NAME_##_id, \ + DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \ + PRESTERA_TRAP_METADATA) + +#define PRESTERA_TRAP_EXCEPTION(_id, _group_id) \ + DEVLINK_TRAP_GENERIC(EXCEPTION, TRAP, _id, \ + DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \ + PRESTERA_TRAP_METADATA) + +#define PRESTERA_TRAP_DRIVER_EXCEPTION(_id, _group_id) \ + DEVLINK_TRAP_DRIVER(EXCEPTION, TRAP, DEVLINK_PRESTERA_TRAP_ID_##_id, \ + DEVLINK_PRESTERA_TRAP_NAME_##_id, \ + DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \ + PRESTERA_TRAP_METADATA) + +#define PRESTERA_TRAP_DRIVER_DROP(_id, _group_id) \ + DEVLINK_TRAP_DRIVER(DROP, DROP, DEVLINK_PRESTERA_TRAP_ID_##_id, \ + DEVLINK_PRESTERA_TRAP_NAME_##_id, \ + DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \ + PRESTERA_TRAP_METADATA) + +static const struct devlink_trap_group prestera_trap_groups_arr[] = { + /* No policer is associated with following groups (policerid == 0)*/ + DEVLINK_TRAP_GROUP_GENERIC(L2_DROPS, 0), + DEVLINK_TRAP_GROUP_GENERIC(L3_DROPS, 0), + DEVLINK_TRAP_GROUP_GENERIC(L3_EXCEPTIONS, 0), + DEVLINK_TRAP_GROUP_GENERIC(NEIGH_DISCOVERY, 0), + DEVLINK_TRAP_GROUP_GENERIC(ACL_TRAP, 0), + DEVLINK_TRAP_GROUP_GENERIC(ACL_DROPS, 0), + DEVLINK_TRAP_GROUP_GENERIC(ACL_SAMPLE, 0), + DEVLINK_TRAP_GROUP_GENERIC(OSPF, 0), + DEVLINK_TRAP_GROUP_GENERIC(STP, 0), + DEVLINK_TRAP_GROUP_GENERIC(LACP, 0), + DEVLINK_TRAP_GROUP_GENERIC(LLDP, 0), + DEVLINK_TRAP_GROUP_GENERIC(VRRP, 0), + DEVLINK_TRAP_GROUP_GENERIC(DHCP, 0), + DEVLINK_TRAP_GROUP_GENERIC(BGP, 0), + DEVLINK_TRAP_GROUP_GENERIC(LOCAL_DELIVERY, 0), + DEVLINK_TRAP_GROUP_GENERIC(BUFFER_DROPS, 0), +}; + +/* Initialize trap list, as well as associate CPU code with them. */ +static struct prestera_trap prestera_trap_items_arr[] = { + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ARP_BC, NEIGH_DISCOVERY), + .cpu_code = 5, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(IS_IS, LOCAL_DELIVERY), + .cpu_code = 13, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(OSPF, OSPF), + .cpu_code = 16, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(IP_BC_MAC, LOCAL_DELIVERY), + .cpu_code = 19, + }, + { + .trap = PRESTERA_TRAP_CONTROL(STP, STP, TRAP), + .cpu_code = 26, + }, + { + .trap = PRESTERA_TRAP_CONTROL(LACP, LACP, TRAP), + .cpu_code = 27, + }, + { + .trap = PRESTERA_TRAP_CONTROL(LLDP, LLDP, TRAP), + .cpu_code = 28, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ROUTER_MC, LOCAL_DELIVERY), + .cpu_code = 29, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(VRRP, VRRP), + .cpu_code = 30, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(DHCP, DHCP), + .cpu_code = 33, + }, + { + .trap = PRESTERA_TRAP_EXCEPTION(MTU_ERROR, L3_EXCEPTIONS), + .cpu_code = 63, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(MAC_TO_ME, LOCAL_DELIVERY), + .cpu_code = 65, + }, + { + .trap = PRESTERA_TRAP_EXCEPTION(TTL_ERROR, L3_EXCEPTIONS), + .cpu_code = 133, + }, + { + .trap = PRESTERA_TRAP_DRIVER_EXCEPTION(IPV4_OPTIONS, + L3_EXCEPTIONS), + .cpu_code = 141, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(IP_DEFAULT_ROUTE, + LOCAL_DELIVERY), + .cpu_code = 160, + }, + { + .trap = PRESTERA_TRAP_CONTROL(LOCAL_ROUTE, LOCAL_DELIVERY, + TRAP), + .cpu_code = 161, + }, + { + .trap = PRESTERA_TRAP_DRIVER_EXCEPTION(IPV4_ICMP_REDIRECT, + L3_EXCEPTIONS), + .cpu_code = 180, + }, + { + .trap = PRESTERA_TRAP_CONTROL(ARP_RESPONSE, NEIGH_DISCOVERY, + TRAP), + .cpu_code = 188, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_0, ACL_TRAP), + .cpu_code = 192, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_1, ACL_TRAP), + .cpu_code = 193, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_2, ACL_TRAP), + .cpu_code = 194, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_3, ACL_TRAP), + .cpu_code = 195, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_4, ACL_TRAP), + .cpu_code = 196, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_5, ACL_TRAP), + .cpu_code = 197, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_6, ACL_TRAP), + .cpu_code = 198, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ACL_CODE_7, ACL_TRAP), + .cpu_code = 199, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(BGP, BGP), + .cpu_code = 206, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(SSH, LOCAL_DELIVERY), + .cpu_code = 207, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(TELNET, LOCAL_DELIVERY), + .cpu_code = 208, + }, + { + .trap = PRESTERA_TRAP_DRIVER_CONTROL(ICMP, LOCAL_DELIVERY), + .cpu_code = 209, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(RXDMA_DROP, BUFFER_DROPS), + .cpu_code = 37, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(PORT_NO_VLAN, L2_DROPS), + .cpu_code = 39, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(LOCAL_PORT, L2_DROPS), + .cpu_code = 56, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(INVALID_SA, L2_DROPS), + .cpu_code = 60, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(ILLEGAL_IP_ADDR, L3_DROPS), + .cpu_code = 136, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(ILLEGAL_IPV4_HDR, L3_DROPS), + .cpu_code = 137, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(IP_UC_DIP_DA_MISMATCH, + L3_DROPS), + .cpu_code = 138, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(IP_SIP_IS_ZERO, L3_DROPS), + .cpu_code = 145, + }, + { + .trap = PRESTERA_TRAP_DRIVER_DROP(MET_RED, BUFFER_DROPS), + .cpu_code = 185, + }, +}; + +static void prestera_devlink_traps_fini(struct prestera_switch *sw); + +static int prestera_drop_counter_get(struct devlink *devlink, + const struct devlink_trap *trap, + u64 *p_drops); static int prestera_dl_info_get(struct devlink *dl, struct devlink_info_req *req, @@ -27,8 +373,21 @@ static int prestera_dl_info_get(struct devlink *dl, buf); } +static int prestera_trap_init(struct devlink *devlink, + const struct devlink_trap *trap, void *trap_ctx); + +static int prestera_trap_action_set(struct devlink *devlink, + const struct devlink_trap *trap, + enum devlink_trap_action action, + struct netlink_ext_ack *extack); + +static int prestera_devlink_traps_register(struct prestera_switch *sw); + static const struct devlink_ops prestera_dl_ops = { .info_get = prestera_dl_info_get, + .trap_init = prestera_trap_init, + .trap_action_set = prestera_trap_action_set, + .trap_drop_counter_get = prestera_drop_counter_get, }; struct prestera_switch *prestera_devlink_alloc(void) @@ -53,17 +412,32 @@ int prestera_devlink_register(struct prestera_switch *sw) int err; err = devlink_register(dl, sw->dev->dev); - if (err) + if (err) { dev_err(prestera_dev(sw), "devlink_register failed: %d\n", err); + return err; + } - return err; + err = prestera_devlink_traps_register(sw); + if (err) { + devlink_unregister(dl); + dev_err(sw->dev->dev, "devlink_traps_register failed: %d\n", + err); + return err; + } + + return 0; } void prestera_devlink_unregister(struct prestera_switch *sw) { + struct prestera_trap_data *trap_data = sw->trap_data; struct devlink *dl = priv_to_devlink(sw); + prestera_devlink_traps_fini(sw); devlink_unregister(dl); + + kfree(trap_data->trap_items_arr); + kfree(trap_data); } int prestera_devlink_port_register(struct prestera_port *port) @@ -110,3 +484,155 @@ struct devlink_port *prestera_devlink_get_port(struct net_device *dev) return &port->dl_port; } + +static int prestera_devlink_traps_register(struct prestera_switch *sw) +{ + const u32 groups_count = ARRAY_SIZE(prestera_trap_groups_arr); + const u32 traps_count = ARRAY_SIZE(prestera_trap_items_arr); + struct devlink *devlink = priv_to_devlink(sw); + struct prestera_trap_data *trap_data; + struct prestera_trap *prestera_trap; + int err, i; + + trap_data = kzalloc(sizeof(*trap_data), GFP_KERNEL); + if (!trap_data) + return -ENOMEM; + + trap_data->trap_items_arr = kcalloc(traps_count, + sizeof(struct prestera_trap_item), + GFP_KERNEL); + if (!trap_data->trap_items_arr) { + err = -ENOMEM; + goto err_trap_items_alloc; + } + + trap_data->sw = sw; + trap_data->traps_count = traps_count; + sw->trap_data = trap_data; + + err = devlink_trap_groups_register(devlink, prestera_trap_groups_arr, + groups_count); + if (err) + goto err_groups_register; + + for (i = 0; i < traps_count; i++) { + prestera_trap = &prestera_trap_items_arr[i]; + err = devlink_traps_register(devlink, &prestera_trap->trap, 1, + sw); + if (err) + goto err_trap_register; + } + + return 0; + +err_trap_register: + for (i--; i >= 0; i--) { + prestera_trap = &prestera_trap_items_arr[i]; + devlink_traps_unregister(devlink, &prestera_trap->trap, 1); + } +err_groups_register: + kfree(trap_data->trap_items_arr); +err_trap_items_alloc: + kfree(trap_data); + return err; +} + +static struct prestera_trap_item * +prestera_get_trap_item_by_cpu_code(struct prestera_switch *sw, u8 cpu_code) +{ + struct prestera_trap_data *trap_data = sw->trap_data; + struct prestera_trap *prestera_trap; + int i; + + for (i = 0; i < trap_data->traps_count; i++) { + prestera_trap = &prestera_trap_items_arr[i]; + if (cpu_code == prestera_trap->cpu_code) + return &trap_data->trap_items_arr[i]; + } + + return NULL; +} + +void prestera_devlink_trap_report(struct prestera_port *port, + struct sk_buff *skb, u8 cpu_code) +{ + struct prestera_trap_item *trap_item; + struct devlink *devlink; + + devlink = port->dl_port.devlink; + + trap_item = prestera_get_trap_item_by_cpu_code(port->sw, cpu_code); + if (unlikely(!trap_item)) + return; + + devlink_trap_report(devlink, skb, trap_item->trap_ctx, + &port->dl_port, NULL); +} + +static struct prestera_trap_item * +prestera_devlink_trap_item_lookup(struct prestera_switch *sw, u16 trap_id) +{ + struct prestera_trap_data *trap_data = sw->trap_data; + int i; + + for (i = 0; i < ARRAY_SIZE(prestera_trap_items_arr); i++) { + if (prestera_trap_items_arr[i].trap.id == trap_id) + return &trap_data->trap_items_arr[i]; + } + + return NULL; +} + +static int prestera_trap_init(struct devlink *devlink, + const struct devlink_trap *trap, void *trap_ctx) +{ + struct prestera_switch *sw = devlink_priv(devlink); + struct prestera_trap_item *trap_item; + + trap_item = prestera_devlink_trap_item_lookup(sw, trap->id); + if (WARN_ON(!trap_item)) + return -EINVAL; + + trap_item->trap_ctx = trap_ctx; + trap_item->action = trap->init_action; + + return 0; +} + +static int prestera_trap_action_set(struct devlink *devlink, + const struct devlink_trap *trap, + enum devlink_trap_action action, + struct netlink_ext_ack *extack) +{ + /* Currently, driver does not support trap action altering */ + return -EOPNOTSUPP; +} + +static int prestera_drop_counter_get(struct devlink *devlink, + const struct devlink_trap *trap, + u64 *p_drops) +{ + struct prestera_switch *sw = devlink_priv(devlink); + enum prestera_hw_cpu_code_cnt_t cpu_code_type = + PRESTERA_HW_CPU_CODE_CNT_TYPE_DROP; + struct prestera_trap *prestera_trap = + container_of(trap, struct prestera_trap, trap); + + return prestera_hw_cpu_code_counters_get(sw, prestera_trap->cpu_code, + cpu_code_type, p_drops); +} + +static void prestera_devlink_traps_fini(struct prestera_switch *sw) +{ + struct devlink *dl = priv_to_devlink(sw); + const struct devlink_trap *trap; + int i; + + for (i = 0; i < ARRAY_SIZE(prestera_trap_items_arr); ++i) { + trap = &prestera_trap_items_arr[i].trap; + devlink_traps_unregister(dl, trap, 1); + } + + devlink_trap_groups_unregister(dl, prestera_trap_groups_arr, + ARRAY_SIZE(prestera_trap_groups_arr)); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_devlink.h b/drivers/net/ethernet/marvell/prestera/prestera_devlink.h index 51bee9f75415..5d73aa9db897 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_devlink.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_devlink.h @@ -20,4 +20,7 @@ void prestera_devlink_port_clear(struct prestera_port *port); struct devlink_port *prestera_devlink_get_port(struct net_device *dev); +void prestera_devlink_trap_report(struct prestera_port *port, + struct sk_buff *skb, u8 cpu_code); + #endif /* _PRESTERA_DEVLINK_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_dsa.c b/drivers/net/ethernet/marvell/prestera/prestera_dsa.c index a5e01c7a307b..b7e89c0ca5c0 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_dsa.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_dsa.c @@ -19,6 +19,7 @@ #define PRESTERA_DSA_W1_EXT_BIT BIT(31) #define PRESTERA_DSA_W1_CFI_BIT BIT(30) #define PRESTERA_DSA_W1_PORT_NUM GENMASK(11, 10) +#define PRESTERA_DSA_W1_MASK_CPU_CODE GENMASK(7, 0) #define PRESTERA_DSA_W2_EXT_BIT BIT(31) #define PRESTERA_DSA_W2_PORT_NUM BIT(20) @@ -74,6 +75,8 @@ int prestera_dsa_parse(struct prestera_dsa *dsa, const u8 *dsa_buf) (FIELD_GET(PRESTERA_DSA_W1_PORT_NUM, words[1]) << 5) | (FIELD_GET(PRESTERA_DSA_W2_PORT_NUM, words[2]) << 7); + dsa->cpu_code = FIELD_GET(PRESTERA_DSA_W1_MASK_CPU_CODE, words[1]); + return 0; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_dsa.h b/drivers/net/ethernet/marvell/prestera/prestera_dsa.h index 67018629bdd2..c99342f475cf 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_dsa.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_dsa.h @@ -27,6 +27,7 @@ struct prestera_dsa { struct prestera_dsa_vlan vlan; u32 hw_dev_num; u32 port_num; + u8 cpu_code; }; int prestera_dsa_parse(struct prestera_dsa *dsa, const u8 *dsa_buf); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.c b/drivers/net/ethernet/marvell/prestera/prestera_flow.c new file mode 100644 index 000000000000..c9891e968259 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved */ + +#include <linux/kernel.h> +#include <linux/list.h> + +#include "prestera.h" +#include "prestera_acl.h" +#include "prestera_flow.h" +#include "prestera_span.h" +#include "prestera_flower.h" + +static LIST_HEAD(prestera_block_cb_list); + +static int prestera_flow_block_mall_cb(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + switch (f->command) { + case TC_CLSMATCHALL_REPLACE: + return prestera_span_replace(block, f); + case TC_CLSMATCHALL_DESTROY: + prestera_span_destroy(block); + return 0; + default: + return -EOPNOTSUPP; + } +} + +static int prestera_flow_block_flower_cb(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + if (f->common.chain_index != 0) + return -EOPNOTSUPP; + + switch (f->command) { + case FLOW_CLS_REPLACE: + return prestera_flower_replace(block, f); + case FLOW_CLS_DESTROY: + prestera_flower_destroy(block, f); + return 0; + case FLOW_CLS_STATS: + return prestera_flower_stats(block, f); + default: + return -EOPNOTSUPP; + } +} + +static int prestera_flow_block_cb(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + struct prestera_flow_block *block = cb_priv; + + switch (type) { + case TC_SETUP_CLSFLOWER: + return prestera_flow_block_flower_cb(block, type_data); + case TC_SETUP_CLSMATCHALL: + return prestera_flow_block_mall_cb(block, type_data); + default: + return -EOPNOTSUPP; + } +} + +static void prestera_flow_block_release(void *cb_priv) +{ + struct prestera_flow_block *block = cb_priv; + + prestera_acl_block_destroy(block); +} + +static struct prestera_flow_block * +prestera_flow_block_get(struct prestera_switch *sw, + struct flow_block_offload *f, + bool *register_block) +{ + struct prestera_flow_block *block; + struct flow_block_cb *block_cb; + + block_cb = flow_block_cb_lookup(f->block, + prestera_flow_block_cb, sw); + if (!block_cb) { + block = prestera_acl_block_create(sw, f->net); + if (!block) + return ERR_PTR(-ENOMEM); + + block_cb = flow_block_cb_alloc(prestera_flow_block_cb, + sw, block, + prestera_flow_block_release); + if (IS_ERR(block_cb)) { + prestera_acl_block_destroy(block); + return ERR_CAST(block_cb); + } + + block->block_cb = block_cb; + *register_block = true; + } else { + block = flow_block_cb_priv(block_cb); + *register_block = false; + } + + flow_block_cb_incref(block_cb); + + return block; +} + +static void prestera_flow_block_put(struct prestera_flow_block *block) +{ + struct flow_block_cb *block_cb = block->block_cb; + + if (flow_block_cb_decref(block_cb)) + return; + + flow_block_cb_free(block_cb); + prestera_acl_block_destroy(block); +} + +static int prestera_setup_flow_block_bind(struct prestera_port *port, + struct flow_block_offload *f) +{ + struct prestera_switch *sw = port->sw; + struct prestera_flow_block *block; + struct flow_block_cb *block_cb; + bool register_block; + int err; + + block = prestera_flow_block_get(sw, f, ®ister_block); + if (IS_ERR(block)) + return PTR_ERR(block); + + block_cb = block->block_cb; + + err = prestera_acl_block_bind(block, port); + if (err) + goto err_block_bind; + + if (register_block) { + flow_block_cb_add(block_cb, f); + list_add_tail(&block_cb->driver_list, &prestera_block_cb_list); + } + + port->flow_block = block; + return 0; + +err_block_bind: + prestera_flow_block_put(block); + + return err; +} + +static void prestera_setup_flow_block_unbind(struct prestera_port *port, + struct flow_block_offload *f) +{ + struct prestera_switch *sw = port->sw; + struct prestera_flow_block *block; + struct flow_block_cb *block_cb; + int err; + + block_cb = flow_block_cb_lookup(f->block, prestera_flow_block_cb, sw); + if (!block_cb) + return; + + block = flow_block_cb_priv(block_cb); + + prestera_span_destroy(block); + + err = prestera_acl_block_unbind(block, port); + if (err) + goto error; + + if (!flow_block_cb_decref(block_cb)) { + flow_block_cb_remove(block_cb, f); + list_del(&block_cb->driver_list); + } +error: + port->flow_block = NULL; +} + +int prestera_flow_block_setup(struct prestera_port *port, + struct flow_block_offload *f) +{ + if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) + return -EOPNOTSUPP; + + f->driver_block_list = &prestera_block_cb_list; + + switch (f->command) { + case FLOW_BLOCK_BIND: + return prestera_setup_flow_block_bind(port, f); + case FLOW_BLOCK_UNBIND: + prestera_setup_flow_block_unbind(port, f); + return 0; + default: + return -EOPNOTSUPP; + } +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.h b/drivers/net/ethernet/marvell/prestera/prestera_flow.h new file mode 100644 index 000000000000..467c7038cace --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_FLOW_H_ +#define _PRESTERA_FLOW_H_ + +#include <net/flow_offload.h> + +struct prestera_port; + +int prestera_flow_block_setup(struct prestera_port *port, + struct flow_block_offload *f); + +#endif /* _PRESTERA_FLOW_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c new file mode 100644 index 000000000000..e571ba09ec08 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved */ + +#include "prestera.h" +#include "prestera_acl.h" +#include "prestera_flower.h" + +static int prestera_flower_parse_actions(struct prestera_flow_block *block, + struct prestera_acl_rule *rule, + struct flow_action *flow_action, + struct netlink_ext_ack *extack) +{ + struct prestera_acl_rule_action_entry a_entry; + const struct flow_action_entry *act; + int err, i; + + if (!flow_action_has_entries(flow_action)) + return 0; + + flow_action_for_each(i, act, flow_action) { + memset(&a_entry, 0, sizeof(a_entry)); + + switch (act->id) { + case FLOW_ACTION_ACCEPT: + a_entry.id = PRESTERA_ACL_RULE_ACTION_ACCEPT; + break; + case FLOW_ACTION_DROP: + a_entry.id = PRESTERA_ACL_RULE_ACTION_DROP; + break; + case FLOW_ACTION_TRAP: + a_entry.id = PRESTERA_ACL_RULE_ACTION_TRAP; + break; + default: + NL_SET_ERR_MSG_MOD(extack, "Unsupported action"); + pr_err("Unsupported action\n"); + return -EOPNOTSUPP; + } + + err = prestera_acl_rule_action_add(rule, &a_entry); + if (err) + return err; + } + + return 0; +} + +static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, + struct flow_cls_offload *f, + struct prestera_flow_block *block) +{ + struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); + struct prestera_acl_rule_match_entry m_entry = {0}; + struct net_device *ingress_dev; + struct flow_match_meta match; + struct prestera_port *port; + + flow_rule_match_meta(f_rule, &match); + if (match.mask->ingress_ifindex != 0xFFFFFFFF) { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Unsupported ingress ifindex mask"); + return -EINVAL; + } + + ingress_dev = __dev_get_by_index(prestera_acl_block_net(block), + match.key->ingress_ifindex); + if (!ingress_dev) { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Can't find specified ingress port to match on"); + return -EINVAL; + } + + if (!prestera_netdev_check(ingress_dev)) { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Can't match on switchdev ingress port"); + return -EINVAL; + } + port = netdev_priv(ingress_dev); + + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT; + m_entry.keymask.u64.key = port->hw_id | ((u64)port->dev_id << 32); + m_entry.keymask.u64.mask = ~(u64)0; + + return prestera_acl_rule_match_add(rule, &m_entry); +} + +static int prestera_flower_parse(struct prestera_flow_block *block, + struct prestera_acl_rule *rule, + struct flow_cls_offload *f) +{ + struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); + struct flow_dissector *dissector = f_rule->match.dissector; + struct prestera_acl_rule_match_entry m_entry; + u16 n_proto_mask = 0; + u16 n_proto_key = 0; + u16 addr_type = 0; + u8 ip_proto = 0; + int err; + + if (dissector->used_keys & + ~(BIT(FLOW_DISSECTOR_KEY_META) | + BIT(FLOW_DISSECTOR_KEY_CONTROL) | + BIT(FLOW_DISSECTOR_KEY_BASIC) | + BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | + BIT(FLOW_DISSECTOR_KEY_ICMP) | + BIT(FLOW_DISSECTOR_KEY_PORTS) | + BIT(FLOW_DISSECTOR_KEY_VLAN))) { + NL_SET_ERR_MSG_MOD(f->common.extack, "Unsupported key"); + return -EOPNOTSUPP; + } + + prestera_acl_rule_priority_set(rule, f->common.prio); + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_META)) { + err = prestera_flower_parse_meta(rule, f, block); + if (err) + return err; + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_CONTROL)) { + struct flow_match_control match; + + flow_rule_match_control(f_rule, &match); + addr_type = match.key->addr_type; + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_BASIC)) { + struct flow_match_basic match; + + flow_rule_match_basic(f_rule, &match); + n_proto_key = ntohs(match.key->n_proto); + n_proto_mask = ntohs(match.mask->n_proto); + + if (n_proto_key == ETH_P_ALL) { + n_proto_key = 0; + n_proto_mask = 0; + } + + /* add eth type key,mask */ + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE; + m_entry.keymask.u16.key = n_proto_key; + m_entry.keymask.u16.mask = n_proto_mask; + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + + /* add ip proto key,mask */ + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO; + m_entry.keymask.u8.key = match.key->ip_proto; + m_entry.keymask.u8.mask = match.mask->ip_proto; + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + + ip_proto = match.key->ip_proto; + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs match; + + flow_rule_match_eth_addrs(f_rule, &match); + + /* add ethernet dst key,mask */ + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC; + memcpy(&m_entry.keymask.mac.key, + &match.key->dst, sizeof(match.key->dst)); + memcpy(&m_entry.keymask.mac.mask, + &match.mask->dst, sizeof(match.mask->dst)); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + + /* add ethernet src key,mask */ + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC; + memcpy(&m_entry.keymask.mac.key, + &match.key->src, sizeof(match.key->src)); + memcpy(&m_entry.keymask.mac.mask, + &match.mask->src, sizeof(match.mask->src)); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + } + + if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { + struct flow_match_ipv4_addrs match; + + flow_rule_match_ipv4_addrs(f_rule, &match); + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC; + memcpy(&m_entry.keymask.u32.key, + &match.key->src, sizeof(match.key->src)); + memcpy(&m_entry.keymask.u32.mask, + &match.mask->src, sizeof(match.mask->src)); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST; + memcpy(&m_entry.keymask.u32.key, + &match.key->dst, sizeof(match.key->dst)); + memcpy(&m_entry.keymask.u32.mask, + &match.mask->dst, sizeof(match.mask->dst)); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_PORTS)) { + struct flow_match_ports match; + + if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP) { + NL_SET_ERR_MSG_MOD + (f->common.extack, + "Only UDP and TCP keys are supported"); + return -EINVAL; + } + + flow_rule_match_ports(f_rule, &match); + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC; + m_entry.keymask.u16.key = ntohs(match.key->src); + m_entry.keymask.u16.mask = ntohs(match.mask->src); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST; + m_entry.keymask.u16.key = ntohs(match.key->dst); + m_entry.keymask.u16.mask = ntohs(match.mask->dst); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan match; + + flow_rule_match_vlan(f_rule, &match); + + if (match.mask->vlan_id != 0) { + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID; + m_entry.keymask.u16.key = match.key->vlan_id; + m_entry.keymask.u16.mask = match.mask->vlan_id; + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + } + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID; + m_entry.keymask.u16.key = ntohs(match.key->vlan_tpid); + m_entry.keymask.u16.mask = ntohs(match.mask->vlan_tpid); + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + } + + if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_ICMP)) { + struct flow_match_icmp match; + + flow_rule_match_icmp(f_rule, &match); + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE; + m_entry.keymask.u8.key = match.key->type; + m_entry.keymask.u8.mask = match.mask->type; + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + + memset(&m_entry, 0, sizeof(m_entry)); + m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE; + m_entry.keymask.u8.key = match.key->code; + m_entry.keymask.u8.mask = match.mask->code; + err = prestera_acl_rule_match_add(rule, &m_entry); + if (err) + return err; + } + + return prestera_flower_parse_actions(block, rule, + &f->rule->action, + f->common.extack); +} + +int prestera_flower_replace(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + struct prestera_switch *sw = prestera_acl_block_sw(block); + struct prestera_acl_rule *rule; + int err; + + rule = prestera_acl_rule_create(block, f->cookie); + if (IS_ERR(rule)) + return PTR_ERR(rule); + + err = prestera_flower_parse(block, rule, f); + if (err) + goto err_flower_parse; + + err = prestera_acl_rule_add(sw, rule); + if (err) + goto err_rule_add; + + return 0; + +err_rule_add: +err_flower_parse: + prestera_acl_rule_destroy(rule); + return err; +} + +void prestera_flower_destroy(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + struct prestera_acl_rule *rule; + struct prestera_switch *sw; + + rule = prestera_acl_rule_lookup(prestera_acl_block_ruleset_get(block), + f->cookie); + if (rule) { + sw = prestera_acl_block_sw(block); + prestera_acl_rule_del(sw, rule); + prestera_acl_rule_destroy(rule); + } +} + +int prestera_flower_stats(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + struct prestera_switch *sw = prestera_acl_block_sw(block); + struct prestera_acl_rule *rule; + u64 packets; + u64 lastuse; + u64 bytes; + int err; + + rule = prestera_acl_rule_lookup(prestera_acl_block_ruleset_get(block), + f->cookie); + if (!rule) + return -EINVAL; + + err = prestera_acl_rule_get_stats(sw, rule, &packets, &bytes, &lastuse); + if (err) + return err; + + flow_stats_update(&f->stats, bytes, packets, 0, lastuse, + FLOW_ACTION_HW_STATS_IMMEDIATE); + return 0; +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.h b/drivers/net/ethernet/marvell/prestera/prestera_flower.h new file mode 100644 index 000000000000..91e045eec58b --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_FLOWER_H_ +#define _PRESTERA_FLOWER_H_ + +#include <net/pkt_cls.h> + +struct prestera_flow_block; + +int prestera_flower_replace(struct prestera_flow_block *block, + struct flow_cls_offload *f); +void prestera_flower_destroy(struct prestera_flow_block *block, + struct flow_cls_offload *f); +int prestera_flower_stats(struct prestera_flow_block *block, + struct flow_cls_offload *f); + +#endif /* _PRESTERA_FLOWER_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_hw.c index 0424718d5998..c1297859e471 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.c @@ -2,11 +2,13 @@ /* Copyright (c) 2019-2020 Marvell International Ltd. All rights reserved */ #include <linux/etherdevice.h> +#include <linux/if_bridge.h> #include <linux/ethtool.h> #include <linux/list.h> #include "prestera.h" #include "prestera_hw.h" +#include "prestera_acl.h" #define PRESTERA_SWITCH_INIT_TIMEOUT_MS (30 * 1000) @@ -36,11 +38,31 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_BRIDGE_PORT_ADD = 0x402, PRESTERA_CMD_TYPE_BRIDGE_PORT_DELETE = 0x403, + PRESTERA_CMD_TYPE_ACL_RULE_ADD = 0x500, + PRESTERA_CMD_TYPE_ACL_RULE_DELETE = 0x501, + PRESTERA_CMD_TYPE_ACL_RULE_STATS_GET = 0x510, + PRESTERA_CMD_TYPE_ACL_RULESET_CREATE = 0x520, + PRESTERA_CMD_TYPE_ACL_RULESET_DELETE = 0x521, + PRESTERA_CMD_TYPE_ACL_PORT_BIND = 0x530, + PRESTERA_CMD_TYPE_ACL_PORT_UNBIND = 0x531, + PRESTERA_CMD_TYPE_RXTX_INIT = 0x800, PRESTERA_CMD_TYPE_RXTX_PORT_INIT = 0x801, + PRESTERA_CMD_TYPE_LAG_MEMBER_ADD = 0x900, + PRESTERA_CMD_TYPE_LAG_MEMBER_DELETE = 0x901, + PRESTERA_CMD_TYPE_LAG_MEMBER_ENABLE = 0x902, + PRESTERA_CMD_TYPE_LAG_MEMBER_DISABLE = 0x903, + PRESTERA_CMD_TYPE_STP_PORT_SET = 0x1000, + PRESTERA_CMD_TYPE_SPAN_GET = 0x1100, + PRESTERA_CMD_TYPE_SPAN_BIND = 0x1101, + PRESTERA_CMD_TYPE_SPAN_UNBIND = 0x1102, + PRESTERA_CMD_TYPE_SPAN_RELEASE = 0x1103, + + PRESTERA_CMD_TYPE_CPU_CODE_COUNTERS_GET = 0x2000, + PRESTERA_CMD_TYPE_ACK = 0x10000, PRESTERA_CMD_TYPE_MAX }; @@ -86,6 +108,11 @@ enum { }; enum { + PRESTERA_PORT_FLOOD_TYPE_UC = 0, + PRESTERA_PORT_FLOOD_TYPE_MC = 1, +}; + +enum { PRESTERA_PORT_GOOD_OCTETS_RCV_CNT, PRESTERA_PORT_BAD_OCTETS_RCV_CNT, PRESTERA_PORT_MAC_TRANSMIT_ERR_CNT, @@ -127,6 +154,12 @@ enum { PRESTERA_FC_SYMM_ASYMM, }; +enum { + PRESTERA_HW_FDB_ENTRY_TYPE_REG_PORT = 0, + PRESTERA_HW_FDB_ENTRY_TYPE_LAG = 1, + PRESTERA_HW_FDB_ENTRY_TYPE_MAX = 2, +}; + struct prestera_fw_event_handler { struct list_head list; struct rcu_head rcu; @@ -168,6 +201,8 @@ struct prestera_msg_switch_init_resp { u32 port_count; u32 mtu_max; u8 switch_id; + u8 lag_max; + u8 lag_member_max; }; struct prestera_msg_port_autoneg_param { @@ -188,6 +223,11 @@ struct prestera_msg_port_mdix_param { u8 admin_mode; }; +struct prestera_msg_port_flood_param { + u8 type; + u8 enable; +}; + union prestera_msg_port_param { u8 admin_state; u8 oper_state; @@ -205,6 +245,7 @@ union prestera_msg_port_param { struct prestera_msg_port_mdix_param mdix; struct prestera_msg_port_autoneg_param autoneg; struct prestera_msg_port_cap_param cap; + struct prestera_msg_port_flood_param flood_ext; }; struct prestera_msg_port_attr_req { @@ -249,8 +290,13 @@ struct prestera_msg_vlan_req { struct prestera_msg_fdb_req { struct prestera_msg_cmd cmd; u8 dest_type; - u32 port; - u32 dev; + union { + struct { + u32 port; + u32 dev; + }; + u16 lag_id; + } dest; u8 mac[ETH_ALEN]; u16 vid; u8 dynamic; @@ -269,6 +315,85 @@ struct prestera_msg_bridge_resp { u16 bridge; }; +struct prestera_msg_acl_action { + u32 id; +}; + +struct prestera_msg_acl_match { + u32 type; + union { + struct { + u8 key; + u8 mask; + } u8; + struct { + u16 key; + u16 mask; + } u16; + struct { + u32 key; + u32 mask; + } u32; + struct { + u64 key; + u64 mask; + } u64; + struct { + u8 key[ETH_ALEN]; + u8 mask[ETH_ALEN]; + } mac; + } __packed keymask; +}; + +struct prestera_msg_acl_rule_req { + struct prestera_msg_cmd cmd; + u32 id; + u32 priority; + u16 ruleset_id; + u8 n_actions; + u8 n_matches; +}; + +struct prestera_msg_acl_rule_resp { + struct prestera_msg_ret ret; + u32 id; +}; + +struct prestera_msg_acl_rule_stats_resp { + struct prestera_msg_ret ret; + u64 packets; + u64 bytes; +}; + +struct prestera_msg_acl_ruleset_bind_req { + struct prestera_msg_cmd cmd; + u32 port; + u32 dev; + u16 ruleset_id; +}; + +struct prestera_msg_acl_ruleset_req { + struct prestera_msg_cmd cmd; + u16 id; +}; + +struct prestera_msg_acl_ruleset_resp { + struct prestera_msg_ret ret; + u16 id; +}; + +struct prestera_msg_span_req { + struct prestera_msg_cmd cmd; + u32 port; + u32 dev; + u8 id; +} __packed __aligned(4); + +struct prestera_msg_span_resp { + struct prestera_msg_ret ret; + u8 id; +} __packed __aligned(4); + struct prestera_msg_stp_req { struct prestera_msg_cmd cmd; u32 port; @@ -293,6 +418,24 @@ struct prestera_msg_rxtx_port_req { u32 dev; }; +struct prestera_msg_lag_req { + struct prestera_msg_cmd cmd; + u32 port; + u32 dev; + u16 lag_id; +}; + +struct prestera_msg_cpu_code_counter_req { + struct prestera_msg_cmd cmd; + u8 counter_type; + u8 code; +}; + +struct mvsw_msg_cpu_code_counter_ret { + struct prestera_msg_ret ret; + u64 packet_count; +}; + struct prestera_msg_event { u16 type; u16 id; @@ -315,7 +458,10 @@ union prestera_msg_event_fdb_param { struct prestera_msg_event_fdb { struct prestera_msg_event id; u8 dest_type; - u32 port_id; + union { + u32 port_id; + u16 lag_id; + } dest; u32 vid; union prestera_msg_event_fdb_param param; }; @@ -386,7 +532,19 @@ static int prestera_fw_parse_fdb_evt(void *msg, struct prestera_event *evt) { struct prestera_msg_event_fdb *hw_evt = msg; - evt->fdb_evt.port_id = hw_evt->port_id; + switch (hw_evt->dest_type) { + case PRESTERA_HW_FDB_ENTRY_TYPE_REG_PORT: + evt->fdb_evt.type = PRESTERA_FDB_ENTRY_TYPE_REG_PORT; + evt->fdb_evt.dest.port_id = hw_evt->dest.port_id; + break; + case PRESTERA_HW_FDB_ENTRY_TYPE_LAG: + evt->fdb_evt.type = PRESTERA_FDB_ENTRY_TYPE_LAG; + evt->fdb_evt.dest.lag_id = hw_evt->dest.lag_id; + break; + default: + return -EINVAL; + } + evt->fdb_evt.vid = hw_evt->vid; ether_addr_copy(evt->fdb_evt.data.mac, hw_evt->param.mac); @@ -531,6 +689,8 @@ int prestera_hw_switch_init(struct prestera_switch *sw) sw->mtu_min = PRESTERA_MIN_MTU; sw->mtu_max = resp.mtu_max; sw->id = resp.switch_id; + sw->lag_member_max = resp.lag_member_max; + sw->lag_max = resp.lag_max; return 0; } @@ -696,6 +856,274 @@ int prestera_hw_port_remote_fc_get(const struct prestera_port *port, return 0; } +int prestera_hw_acl_ruleset_create(struct prestera_switch *sw, u16 *ruleset_id) +{ + struct prestera_msg_acl_ruleset_resp resp; + struct prestera_msg_acl_ruleset_req req; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULESET_CREATE, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *ruleset_id = resp.id; + + return 0; +} + +int prestera_hw_acl_ruleset_del(struct prestera_switch *sw, u16 ruleset_id) +{ + struct prestera_msg_acl_ruleset_req req = { + .id = ruleset_id, + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ACL_RULESET_DELETE, + &req.cmd, sizeof(req)); +} + +static int prestera_hw_acl_actions_put(struct prestera_msg_acl_action *action, + struct prestera_acl_rule *rule) +{ + struct list_head *a_list = prestera_acl_rule_action_list_get(rule); + struct prestera_acl_rule_action_entry *a_entry; + int i = 0; + + list_for_each_entry(a_entry, a_list, list) { + action[i].id = a_entry->id; + + switch (a_entry->id) { + case PRESTERA_ACL_RULE_ACTION_ACCEPT: + case PRESTERA_ACL_RULE_ACTION_DROP: + case PRESTERA_ACL_RULE_ACTION_TRAP: + /* just rule action id, no specific data */ + break; + default: + return -EINVAL; + } + + i++; + } + + return 0; +} + +static int prestera_hw_acl_matches_put(struct prestera_msg_acl_match *match, + struct prestera_acl_rule *rule) +{ + struct list_head *m_list = prestera_acl_rule_match_list_get(rule); + struct prestera_acl_rule_match_entry *m_entry; + int i = 0; + + list_for_each_entry(m_entry, m_list, list) { + match[i].type = m_entry->type; + + switch (m_entry->type) { + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID: + match[i].keymask.u16.key = m_entry->keymask.u16.key; + match[i].keymask.u16.mask = m_entry->keymask.u16.mask; + break; + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO: + match[i].keymask.u8.key = m_entry->keymask.u8.key; + match[i].keymask.u8.mask = m_entry->keymask.u8.mask; + break; + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC: + memcpy(match[i].keymask.mac.key, + m_entry->keymask.mac.key, + sizeof(match[i].keymask.mac.key)); + memcpy(match[i].keymask.mac.mask, + m_entry->keymask.mac.mask, + sizeof(match[i].keymask.mac.mask)); + break; + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_SRC: + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_DST: + match[i].keymask.u32.key = m_entry->keymask.u32.key; + match[i].keymask.u32.mask = m_entry->keymask.u32.mask; + break; + case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT: + match[i].keymask.u64.key = m_entry->keymask.u64.key; + match[i].keymask.u64.mask = m_entry->keymask.u64.mask; + break; + default: + return -EINVAL; + } + + i++; + } + + return 0; +} + +int prestera_hw_acl_rule_add(struct prestera_switch *sw, + struct prestera_acl_rule *rule, + u32 *rule_id) +{ + struct prestera_msg_acl_action *actions; + struct prestera_msg_acl_match *matches; + struct prestera_msg_acl_rule_resp resp; + struct prestera_msg_acl_rule_req *req; + u8 n_actions; + u8 n_matches; + void *buff; + u32 size; + int err; + + n_actions = prestera_acl_rule_action_len(rule); + n_matches = prestera_acl_rule_match_len(rule); + + size = sizeof(*req) + sizeof(*actions) * n_actions + + sizeof(*matches) * n_matches; + + buff = kzalloc(size, GFP_KERNEL); + if (!buff) + return -ENOMEM; + + req = buff; + actions = buff + sizeof(*req); + matches = buff + sizeof(*req) + sizeof(*actions) * n_actions; + + /* put acl actions into the message */ + err = prestera_hw_acl_actions_put(actions, rule); + if (err) + goto free_buff; + + /* put acl matches into the message */ + err = prestera_hw_acl_matches_put(matches, rule); + if (err) + goto free_buff; + + req->ruleset_id = prestera_acl_rule_ruleset_id_get(rule); + req->priority = prestera_acl_rule_priority_get(rule); + req->n_actions = prestera_acl_rule_action_len(rule); + req->n_matches = prestera_acl_rule_match_len(rule); + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULE_ADD, + &req->cmd, size, &resp.ret, sizeof(resp)); + if (err) + goto free_buff; + + *rule_id = resp.id; +free_buff: + kfree(buff); + return err; +} + +int prestera_hw_acl_rule_del(struct prestera_switch *sw, u32 rule_id) +{ + struct prestera_msg_acl_rule_req req = { + .id = rule_id + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ACL_RULE_DELETE, + &req.cmd, sizeof(req)); +} + +int prestera_hw_acl_rule_stats_get(struct prestera_switch *sw, u32 rule_id, + u64 *packets, u64 *bytes) +{ + struct prestera_msg_acl_rule_stats_resp resp; + struct prestera_msg_acl_rule_req req = { + .id = rule_id + }; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULE_STATS_GET, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *packets = resp.packets; + *bytes = resp.bytes; + + return 0; +} + +int prestera_hw_acl_port_bind(const struct prestera_port *port, u16 ruleset_id) +{ + struct prestera_msg_acl_ruleset_bind_req req = { + .port = port->hw_id, + .dev = port->dev_id, + .ruleset_id = ruleset_id, + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_ACL_PORT_BIND, + &req.cmd, sizeof(req)); +} + +int prestera_hw_acl_port_unbind(const struct prestera_port *port, + u16 ruleset_id) +{ + struct prestera_msg_acl_ruleset_bind_req req = { + .port = port->hw_id, + .dev = port->dev_id, + .ruleset_id = ruleset_id, + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_ACL_PORT_UNBIND, + &req.cmd, sizeof(req)); +} + +int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id) +{ + struct prestera_msg_span_resp resp; + struct prestera_msg_span_req req = { + .port = port->hw_id, + .dev = port->dev_id, + }; + int err; + + err = prestera_cmd_ret(port->sw, PRESTERA_CMD_TYPE_SPAN_GET, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *span_id = resp.id; + + return 0; +} + +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id) +{ + struct prestera_msg_span_req req = { + .port = port->hw_id, + .dev = port->dev_id, + .id = span_id, + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_BIND, + &req.cmd, sizeof(req)); +} + +int prestera_hw_span_unbind(const struct prestera_port *port) +{ + struct prestera_msg_span_req req = { + .port = port->hw_id, + .dev = port->dev_id, + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_UNBIND, + &req.cmd, sizeof(req)); +} + +int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id) +{ + struct prestera_msg_span_req req = { + .id = span_id + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_SPAN_RELEASE, + &req.cmd, sizeof(req)); +} + int prestera_hw_port_type_get(const struct prestera_port *port, u8 *type) { struct prestera_msg_port_attr_req req = { @@ -988,7 +1416,43 @@ int prestera_hw_port_learning_set(struct prestera_port *port, bool enable) &req.cmd, sizeof(req)); } -int prestera_hw_port_flood_set(struct prestera_port *port, bool flood) +static int prestera_hw_port_uc_flood_set(struct prestera_port *port, bool flood) +{ + struct prestera_msg_port_attr_req req = { + .attr = PRESTERA_CMD_PORT_ATTR_FLOOD, + .port = port->hw_id, + .dev = port->dev_id, + .param = { + .flood_ext = { + .type = PRESTERA_PORT_FLOOD_TYPE_UC, + .enable = flood, + } + } + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_PORT_ATTR_SET, + &req.cmd, sizeof(req)); +} + +static int prestera_hw_port_mc_flood_set(struct prestera_port *port, bool flood) +{ + struct prestera_msg_port_attr_req req = { + .attr = PRESTERA_CMD_PORT_ATTR_FLOOD, + .port = port->hw_id, + .dev = port->dev_id, + .param = { + .flood_ext = { + .type = PRESTERA_PORT_FLOOD_TYPE_MC, + .enable = flood, + } + } + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_PORT_ATTR_SET, + &req.cmd, sizeof(req)); +} + +static int prestera_hw_port_flood_set_v2(struct prestera_port *port, bool flood) { struct prestera_msg_port_attr_req req = { .attr = PRESTERA_CMD_PORT_ATTR_FLOOD, @@ -1003,6 +1467,41 @@ int prestera_hw_port_flood_set(struct prestera_port *port, bool flood) &req.cmd, sizeof(req)); } +int prestera_hw_port_flood_set(struct prestera_port *port, unsigned long mask, + unsigned long val) +{ + int err; + + if (port->sw->dev->fw_rev.maj <= 2) { + if (!(mask & BR_FLOOD)) + return 0; + + return prestera_hw_port_flood_set_v2(port, val & BR_FLOOD); + } + + if (mask & BR_FLOOD) { + err = prestera_hw_port_uc_flood_set(port, val & BR_FLOOD); + if (err) + goto err_uc_flood; + } + + if (mask & BR_MCAST_FLOOD) { + err = prestera_hw_port_mc_flood_set(port, val & BR_MCAST_FLOOD); + if (err) + goto err_mc_flood; + } + + return 0; + +err_mc_flood: + prestera_hw_port_mc_flood_set(port, 0); +err_uc_flood: + if (mask & BR_FLOOD) + prestera_hw_port_uc_flood_set(port, 0); + + return err; +} + int prestera_hw_vlan_create(struct prestera_switch *sw, u16 vid) { struct prestera_msg_vlan_req req = { @@ -1067,8 +1566,10 @@ int prestera_hw_fdb_add(struct prestera_port *port, const unsigned char *mac, u16 vid, bool dynamic) { struct prestera_msg_fdb_req req = { - .port = port->hw_id, - .dev = port->dev_id, + .dest = { + .dev = port->dev_id, + .port = port->hw_id, + }, .vid = vid, .dynamic = dynamic, }; @@ -1083,8 +1584,10 @@ int prestera_hw_fdb_del(struct prestera_port *port, const unsigned char *mac, u16 vid) { struct prestera_msg_fdb_req req = { - .port = port->hw_id, - .dev = port->dev_id, + .dest = { + .dev = port->dev_id, + .port = port->hw_id, + }, .vid = vid, }; @@ -1094,11 +1597,48 @@ int prestera_hw_fdb_del(struct prestera_port *port, const unsigned char *mac, &req.cmd, sizeof(req)); } +int prestera_hw_lag_fdb_add(struct prestera_switch *sw, u16 lag_id, + const unsigned char *mac, u16 vid, bool dynamic) +{ + struct prestera_msg_fdb_req req = { + .dest_type = PRESTERA_HW_FDB_ENTRY_TYPE_LAG, + .dest = { + .lag_id = lag_id, + }, + .vid = vid, + .dynamic = dynamic, + }; + + ether_addr_copy(req.mac, mac); + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_FDB_ADD, + &req.cmd, sizeof(req)); +} + +int prestera_hw_lag_fdb_del(struct prestera_switch *sw, u16 lag_id, + const unsigned char *mac, u16 vid) +{ + struct prestera_msg_fdb_req req = { + .dest_type = PRESTERA_HW_FDB_ENTRY_TYPE_LAG, + .dest = { + .lag_id = lag_id, + }, + .vid = vid, + }; + + ether_addr_copy(req.mac, mac); + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_FDB_DELETE, + &req.cmd, sizeof(req)); +} + int prestera_hw_fdb_flush_port(struct prestera_port *port, u32 mode) { struct prestera_msg_fdb_req req = { - .port = port->hw_id, - .dev = port->dev_id, + .dest = { + .dev = port->dev_id, + .port = port->hw_id, + }, .flush_mode = mode, }; @@ -1121,8 +1661,10 @@ int prestera_hw_fdb_flush_port_vlan(struct prestera_port *port, u16 vid, u32 mode) { struct prestera_msg_fdb_req req = { - .port = port->hw_id, - .dev = port->dev_id, + .dest = { + .dev = port->dev_id, + .port = port->hw_id, + }, .vid = vid, .flush_mode = mode, }; @@ -1131,6 +1673,37 @@ int prestera_hw_fdb_flush_port_vlan(struct prestera_port *port, u16 vid, &req.cmd, sizeof(req)); } +int prestera_hw_fdb_flush_lag(struct prestera_switch *sw, u16 lag_id, + u32 mode) +{ + struct prestera_msg_fdb_req req = { + .dest_type = PRESTERA_HW_FDB_ENTRY_TYPE_LAG, + .dest = { + .lag_id = lag_id, + }, + .flush_mode = mode, + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_FDB_FLUSH_PORT, + &req.cmd, sizeof(req)); +} + +int prestera_hw_fdb_flush_lag_vlan(struct prestera_switch *sw, + u16 lag_id, u16 vid, u32 mode) +{ + struct prestera_msg_fdb_req req = { + .dest_type = PRESTERA_HW_FDB_ENTRY_TYPE_LAG, + .dest = { + .lag_id = lag_id, + }, + .vid = vid, + .flush_mode = mode, + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_FDB_FLUSH_PORT_VLAN, + &req.cmd, sizeof(req)); +} + int prestera_hw_bridge_create(struct prestera_switch *sw, u16 *bridge_id) { struct prestera_msg_bridge_resp resp; @@ -1212,6 +1785,68 @@ int prestera_hw_rxtx_port_init(struct prestera_port *port) &req.cmd, sizeof(req)); } +int prestera_hw_lag_member_add(struct prestera_port *port, u16 lag_id) +{ + struct prestera_msg_lag_req req = { + .port = port->hw_id, + .dev = port->dev_id, + .lag_id = lag_id, + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_LAG_MEMBER_ADD, + &req.cmd, sizeof(req)); +} + +int prestera_hw_lag_member_del(struct prestera_port *port, u16 lag_id) +{ + struct prestera_msg_lag_req req = { + .port = port->hw_id, + .dev = port->dev_id, + .lag_id = lag_id, + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_LAG_MEMBER_DELETE, + &req.cmd, sizeof(req)); +} + +int prestera_hw_lag_member_enable(struct prestera_port *port, u16 lag_id, + bool enable) +{ + struct prestera_msg_lag_req req = { + .port = port->hw_id, + .dev = port->dev_id, + .lag_id = lag_id, + }; + u32 cmd; + + cmd = enable ? PRESTERA_CMD_TYPE_LAG_MEMBER_ENABLE : + PRESTERA_CMD_TYPE_LAG_MEMBER_DISABLE; + + return prestera_cmd(port->sw, cmd, &req.cmd, sizeof(req)); +} + +int +prestera_hw_cpu_code_counters_get(struct prestera_switch *sw, u8 code, + enum prestera_hw_cpu_code_cnt_t counter_type, + u64 *packet_count) +{ + struct prestera_msg_cpu_code_counter_req req = { + .counter_type = counter_type, + .code = code, + }; + struct mvsw_msg_cpu_code_counter_ret resp; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_CPU_CODE_COUNTERS_GET, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *packet_count = resp.packet_count; + + return 0; +} + int prestera_hw_event_handler_register(struct prestera_switch *sw, enum prestera_event_type type, prestera_event_cb_t fn, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_hw.h index b2b5ac95b4e3..546d5fd8240d 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.h @@ -89,12 +89,18 @@ enum { PRESTERA_STP_FORWARD, }; +enum prestera_hw_cpu_code_cnt_t { + PRESTERA_HW_CPU_CODE_CNT_TYPE_DROP = 0, + PRESTERA_HW_CPU_CODE_CNT_TYPE_TRAP = 1, +}; + struct prestera_switch; struct prestera_port; struct prestera_port_stats; struct prestera_port_caps; enum prestera_event_type; struct prestera_event; +struct prestera_acl_rule; typedef void (*prestera_event_cb_t) (struct prestera_switch *sw, struct prestera_event *evt, void *arg); @@ -138,7 +144,8 @@ int prestera_hw_port_mdix_get(const struct prestera_port *port, u8 *status, int prestera_hw_port_mdix_set(const struct prestera_port *port, u8 mode); int prestera_hw_port_speed_get(const struct prestera_port *port, u32 *speed); int prestera_hw_port_learning_set(struct prestera_port *port, bool enable); -int prestera_hw_port_flood_set(struct prestera_port *port, bool flood); +int prestera_hw_port_flood_set(struct prestera_port *port, unsigned long mask, + unsigned long val); int prestera_hw_port_accept_frm_type(struct prestera_port *port, enum prestera_accept_frm_type type); /* Vlan API */ @@ -165,6 +172,28 @@ int prestera_hw_bridge_delete(struct prestera_switch *sw, u16 bridge_id); int prestera_hw_bridge_port_add(struct prestera_port *port, u16 bridge_id); int prestera_hw_bridge_port_delete(struct prestera_port *port, u16 bridge_id); +/* ACL API */ +int prestera_hw_acl_ruleset_create(struct prestera_switch *sw, + u16 *ruleset_id); +int prestera_hw_acl_ruleset_del(struct prestera_switch *sw, + u16 ruleset_id); +int prestera_hw_acl_rule_add(struct prestera_switch *sw, + struct prestera_acl_rule *rule, + u32 *rule_id); +int prestera_hw_acl_rule_del(struct prestera_switch *sw, u32 rule_id); +int prestera_hw_acl_rule_stats_get(struct prestera_switch *sw, + u32 rule_id, u64 *packets, u64 *bytes); +int prestera_hw_acl_port_bind(const struct prestera_port *port, + u16 ruleset_id); +int prestera_hw_acl_port_unbind(const struct prestera_port *port, + u16 ruleset_id); + +/* SPAN API */ +int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id); +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id); +int prestera_hw_span_unbind(const struct prestera_port *port); +int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id); + /* Event handlers */ int prestera_hw_event_handler_register(struct prestera_switch *sw, enum prestera_event_type type, @@ -179,4 +208,24 @@ int prestera_hw_rxtx_init(struct prestera_switch *sw, struct prestera_rxtx_params *params); int prestera_hw_rxtx_port_init(struct prestera_port *port); +/* LAG API */ +int prestera_hw_lag_member_add(struct prestera_port *port, u16 lag_id); +int prestera_hw_lag_member_del(struct prestera_port *port, u16 lag_id); +int prestera_hw_lag_member_enable(struct prestera_port *port, u16 lag_id, + bool enable); +int prestera_hw_lag_fdb_add(struct prestera_switch *sw, u16 lag_id, + const unsigned char *mac, u16 vid, bool dynamic); +int prestera_hw_lag_fdb_del(struct prestera_switch *sw, u16 lag_id, + const unsigned char *mac, u16 vid); +int prestera_hw_fdb_flush_lag(struct prestera_switch *sw, u16 lag_id, + u32 mode); +int prestera_hw_fdb_flush_lag_vlan(struct prestera_switch *sw, + u16 lag_id, u16 vid, u32 mode); + +/* HW trap/drop counters API */ +int +prestera_hw_cpu_code_counters_get(struct prestera_switch *sw, u8 code, + enum prestera_hw_cpu_code_cnt_t counter_type, + u64 *packet_count); + #endif /* _PRESTERA_HW_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index 2768c78528a5..226f4ff29f6e 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -8,9 +8,13 @@ #include <linux/netdev_features.h> #include <linux/of.h> #include <linux/of_net.h> +#include <linux/if_vlan.h> #include "prestera.h" #include "prestera_hw.h" +#include "prestera_acl.h" +#include "prestera_flow.h" +#include "prestera_span.h" #include "prestera_rxtx.h" #include "prestera_devlink.h" #include "prestera_ethtool.h" @@ -199,10 +203,25 @@ static void prestera_port_stats_update(struct work_struct *work) msecs_to_jiffies(PRESTERA_STATS_DELAY_MS)); } +static int prestera_port_setup_tc(struct net_device *dev, + enum tc_setup_type type, + void *type_data) +{ + struct prestera_port *port = netdev_priv(dev); + + switch (type) { + case TC_SETUP_BLOCK: + return prestera_flow_block_setup(port, type_data); + default: + return -EOPNOTSUPP; + } +} + static const struct net_device_ops prestera_netdev_ops = { .ndo_open = prestera_port_open, .ndo_stop = prestera_port_close, .ndo_start_xmit = prestera_port_xmit, + .ndo_setup_tc = prestera_port_setup_tc, .ndo_change_mtu = prestera_port_change_mtu, .ndo_get_stats64 = prestera_port_get_stats64, .ndo_set_mac_address = prestera_port_set_mac_address, @@ -281,6 +300,7 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) INIT_LIST_HEAD(&port->vlans_list); port->pvid = PRESTERA_DEFAULT_VID; + port->lag = NULL; port->dev = dev; port->id = id; port->sw = sw; @@ -296,7 +316,7 @@ static int prestera_port_create(struct prestera_switch *sw, u32 id) if (err) goto err_dl_port_register; - dev->features |= NETIF_F_NETNS_LOCAL; + dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_HW_TC; dev->netdev_ops = &prestera_netdev_ops; dev->ethtool_ops = &prestera_ethtool_ops; @@ -472,6 +492,149 @@ static int prestera_switch_set_base_mac_addr(struct prestera_switch *sw) return prestera_hw_switch_mac_set(sw, sw->base_mac); } +struct prestera_lag *prestera_lag_by_id(struct prestera_switch *sw, u16 id) +{ + return id < sw->lag_max ? &sw->lags[id] : NULL; +} + +static struct prestera_lag *prestera_lag_by_dev(struct prestera_switch *sw, + struct net_device *dev) +{ + struct prestera_lag *lag; + u16 id; + + for (id = 0; id < sw->lag_max; id++) { + lag = &sw->lags[id]; + if (lag->dev == dev) + return lag; + } + + return NULL; +} + +static struct prestera_lag *prestera_lag_create(struct prestera_switch *sw, + struct net_device *lag_dev) +{ + struct prestera_lag *lag = NULL; + u16 id; + + for (id = 0; id < sw->lag_max; id++) { + lag = &sw->lags[id]; + if (!lag->dev) + break; + } + if (lag) { + INIT_LIST_HEAD(&lag->members); + lag->dev = lag_dev; + } + + return lag; +} + +static void prestera_lag_destroy(struct prestera_switch *sw, + struct prestera_lag *lag) +{ + WARN_ON(!list_empty(&lag->members)); + lag->member_count = 0; + lag->dev = NULL; +} + +static int prestera_lag_port_add(struct prestera_port *port, + struct net_device *lag_dev) +{ + struct prestera_switch *sw = port->sw; + struct prestera_lag *lag; + int err; + + lag = prestera_lag_by_dev(sw, lag_dev); + if (!lag) { + lag = prestera_lag_create(sw, lag_dev); + if (!lag) + return -ENOSPC; + } + + if (lag->member_count >= sw->lag_member_max) + return -ENOSPC; + + err = prestera_hw_lag_member_add(port, lag->lag_id); + if (err) { + if (!lag->member_count) + prestera_lag_destroy(sw, lag); + return err; + } + + list_add(&port->lag_member, &lag->members); + lag->member_count++; + port->lag = lag; + + return 0; +} + +static int prestera_lag_port_del(struct prestera_port *port) +{ + struct prestera_switch *sw = port->sw; + struct prestera_lag *lag = port->lag; + int err; + + if (!lag || !lag->member_count) + return -EINVAL; + + err = prestera_hw_lag_member_del(port, lag->lag_id); + if (err) + return err; + + list_del(&port->lag_member); + lag->member_count--; + port->lag = NULL; + + if (netif_is_bridge_port(lag->dev)) { + struct net_device *br_dev; + + br_dev = netdev_master_upper_dev_get(lag->dev); + + prestera_bridge_port_leave(br_dev, port); + } + + if (!lag->member_count) + prestera_lag_destroy(sw, lag); + + return 0; +} + +bool prestera_port_is_lag_member(const struct prestera_port *port) +{ + return !!port->lag; +} + +u16 prestera_port_lag_id(const struct prestera_port *port) +{ + return port->lag->lag_id; +} + +static int prestera_lag_init(struct prestera_switch *sw) +{ + u16 id; + + sw->lags = kcalloc(sw->lag_max, sizeof(*sw->lags), GFP_KERNEL); + if (!sw->lags) + return -ENOMEM; + + for (id = 0; id < sw->lag_max; id++) + sw->lags[id].lag_id = id; + + return 0; +} + +static void prestera_lag_fini(struct prestera_switch *sw) +{ + u8 idx; + + for (idx = 0; idx < sw->lag_max; idx++) + WARN_ON(sw->lags[idx].member_count); + + kfree(sw->lags); +} + bool prestera_netdev_check(const struct net_device *dev) { return dev->netdev_ops == &prestera_netdev_ops; @@ -505,16 +668,119 @@ struct prestera_port *prestera_port_dev_lower_find(struct net_device *dev) return port; } -static int prestera_netdev_port_event(struct net_device *dev, +static int prestera_netdev_port_lower_event(struct net_device *dev, + unsigned long event, void *ptr) +{ + struct netdev_notifier_changelowerstate_info *info = ptr; + struct netdev_lag_lower_state_info *lower_state_info; + struct prestera_port *port = netdev_priv(dev); + bool enabled; + + if (!netif_is_lag_port(dev)) + return 0; + if (!prestera_port_is_lag_member(port)) + return 0; + + lower_state_info = info->lower_state_info; + enabled = lower_state_info->link_up && lower_state_info->tx_enabled; + + return prestera_hw_lag_member_enable(port, port->lag->lag_id, enabled); +} + +static bool prestera_lag_master_check(struct net_device *lag_dev, + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *ext_ack) +{ + if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) { + NL_SET_ERR_MSG_MOD(ext_ack, "Unsupported LAG Tx type"); + return false; + } + + return true; +} + +static int prestera_netdev_port_event(struct net_device *lower, + struct net_device *dev, unsigned long event, void *ptr) { + struct netdev_notifier_changeupper_info *info = ptr; + struct prestera_port *port = netdev_priv(dev); + struct netlink_ext_ack *extack; + struct net_device *upper; + + extack = netdev_notifier_info_to_extack(&info->info); + upper = info->upper_dev; + switch (event) { case NETDEV_PRECHANGEUPPER: + if (!netif_is_bridge_master(upper) && + !netif_is_lag_master(upper)) { + NL_SET_ERR_MSG_MOD(extack, "Unknown upper device type"); + return -EINVAL; + } + + if (!info->linking) + break; + + if (netdev_has_any_upper_dev(upper)) { + NL_SET_ERR_MSG_MOD(extack, "Upper device is already enslaved"); + return -EINVAL; + } + + if (netif_is_lag_master(upper) && + !prestera_lag_master_check(upper, info->upper_info, extack)) + return -EOPNOTSUPP; + if (netif_is_lag_master(upper) && vlan_uses_dev(dev)) { + NL_SET_ERR_MSG_MOD(extack, + "Master device is a LAG master and port has a VLAN"); + return -EINVAL; + } + if (netif_is_lag_port(dev) && is_vlan_dev(upper) && + !netif_is_lag_master(vlan_dev_real_dev(upper))) { + NL_SET_ERR_MSG_MOD(extack, + "Can not put a VLAN on a LAG port"); + return -EINVAL; + } + break; + case NETDEV_CHANGEUPPER: - return prestera_bridge_port_event(dev, event, ptr); - default: - return 0; + if (netif_is_bridge_master(upper)) { + if (info->linking) + return prestera_bridge_port_join(upper, port); + else + prestera_bridge_port_leave(upper, port); + } else if (netif_is_lag_master(upper)) { + if (info->linking) + return prestera_lag_port_add(port, upper); + else + prestera_lag_port_del(port); + } + break; + + case NETDEV_CHANGELOWERSTATE: + return prestera_netdev_port_lower_event(dev, event, ptr); } + + return 0; +} + +static int prestera_netdevice_lag_event(struct net_device *lag_dev, + unsigned long event, void *ptr) +{ + struct net_device *dev; + struct list_head *iter; + int err; + + netdev_for_each_lower_dev(lag_dev, dev, iter) { + if (prestera_netdev_check(dev)) { + err = prestera_netdev_port_event(lag_dev, dev, event, + ptr); + if (err) + return err; + } + } + + return 0; } static int prestera_netdev_event_handler(struct notifier_block *nb, @@ -524,7 +790,9 @@ static int prestera_netdev_event_handler(struct notifier_block *nb, int err = 0; if (prestera_netdev_check(dev)) - err = prestera_netdev_port_event(dev, event, ptr); + err = prestera_netdev_port_event(dev, dev, event, ptr); + else if (netif_is_lag_master(dev)) + err = prestera_netdevice_lag_event(dev, event, ptr); return notifier_from_errno(err); } @@ -574,10 +842,22 @@ static int prestera_switch_init(struct prestera_switch *sw) if (err) goto err_handlers_register; + err = prestera_acl_init(sw); + if (err) + goto err_acl_init; + + err = prestera_span_init(sw); + if (err) + goto err_span_init; + err = prestera_devlink_register(sw); if (err) goto err_dl_register; + err = prestera_lag_init(sw); + if (err) + goto err_lag_init; + err = prestera_create_ports(sw); if (err) goto err_ports_create; @@ -585,8 +865,14 @@ static int prestera_switch_init(struct prestera_switch *sw) return 0; err_ports_create: + prestera_lag_fini(sw); +err_lag_init: prestera_devlink_unregister(sw); err_dl_register: + prestera_span_fini(sw); +err_span_init: + prestera_acl_fini(sw); +err_acl_init: prestera_event_handlers_unregister(sw); err_handlers_register: prestera_rxtx_switch_fini(sw); @@ -602,7 +888,10 @@ err_swdev_register: static void prestera_switch_fini(struct prestera_switch *sw) { prestera_destroy_ports(sw); + prestera_lag_fini(sw); prestera_devlink_unregister(sw); + prestera_span_fini(sw); + prestera_acl_fini(sw); prestera_event_handlers_unregister(sw); prestera_rxtx_switch_fini(sw); prestera_switchdev_fini(sw); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_pci.c b/drivers/net/ethernet/marvell/prestera/prestera_pci.c index 298110119272..a250d394da38 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_pci.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_pci.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 /* Copyright (c) 2019-2020 Marvell International Ltd. All rights reserved */ +#include <linux/bitfield.h> #include <linux/circ_buf.h> #include <linux/device.h> #include <linux/firmware.h> @@ -13,9 +14,12 @@ #define PRESTERA_MSG_MAX_SIZE 1500 -#define PRESTERA_SUPP_FW_MAJ_VER 2 +#define PRESTERA_SUPP_FW_MAJ_VER 3 #define PRESTERA_SUPP_FW_MIN_VER 0 +#define PRESTERA_PREV_FW_MAJ_VER 2 +#define PRESTERA_PREV_FW_MIN_VER 0 + #define PRESTERA_FW_PATH_FMT "mrvl/prestera/mvsw_prestera_fw-v%u.%u.img" #define PRESTERA_FW_HDR_MAGIC 0x351D9D06 @@ -144,6 +148,11 @@ struct prestera_fw_regs { /* PRESTERA_CMD_RCV_CTL_REG flags */ #define PRESTERA_CMD_F_REPL_SENT BIT(0) +#define PRESTERA_FW_EVT_CTL_STATUS_MASK GENMASK(1, 0) + +#define PRESTERA_FW_EVT_CTL_STATUS_ON 0 +#define PRESTERA_FW_EVT_CTL_STATUS_OFF 1 + #define PRESTERA_EVTQ_REG_OFFSET(q, f) \ (PRESTERA_FW_REG_OFFSET(evtq_list) + \ (q) * sizeof(struct prestera_fw_evtq_regs) + \ @@ -166,6 +175,8 @@ struct prestera_fw_evtq { }; struct prestera_fw { + struct prestera_fw_rev rev_supp; + const struct firmware *bin; struct workqueue_struct *wq; struct prestera_device dev; u8 __iomem *ldr_regs; @@ -260,6 +271,15 @@ static u8 prestera_fw_evtq_pick(struct prestera_fw *fw) return PRESTERA_EVT_QNUM_MAX; } +static void prestera_fw_evt_ctl_status_set(struct prestera_fw *fw, u32 val) +{ + u32 status = prestera_fw_read(fw, PRESTERA_FW_STATUS_REG); + + u32p_replace_bits(&status, val, PRESTERA_FW_EVT_CTL_STATUS_MASK); + + prestera_fw_write(fw, PRESTERA_FW_STATUS_REG, status); +} + static void prestera_fw_evt_work_fn(struct work_struct *work) { struct prestera_fw *fw; @@ -269,6 +289,8 @@ static void prestera_fw_evt_work_fn(struct work_struct *work) fw = container_of(work, struct prestera_fw, evt_work); msg = fw->evt_msg; + prestera_fw_evt_ctl_status_set(fw, PRESTERA_FW_EVT_CTL_STATUS_OFF); + while ((qid = prestera_fw_evtq_pick(fw)) < PRESTERA_EVT_QNUM_MAX) { u32 idx; u32 len; @@ -288,6 +310,8 @@ static void prestera_fw_evt_work_fn(struct work_struct *work) if (fw->dev.recv_msg) fw->dev.recv_msg(&fw->dev, msg, len); } + + prestera_fw_evt_ctl_status_set(fw, PRESTERA_FW_EVT_CTL_STATUS_ON); } static int prestera_fw_wait_reg32(struct prestera_fw *fw, u32 reg, u32 cmp, @@ -576,25 +600,24 @@ static void prestera_fw_rev_parse(const struct prestera_fw_header *hdr, static int prestera_fw_rev_check(struct prestera_fw *fw) { struct prestera_fw_rev *rev = &fw->dev.fw_rev; - u16 maj_supp = PRESTERA_SUPP_FW_MAJ_VER; - u16 min_supp = PRESTERA_SUPP_FW_MIN_VER; - if (rev->maj == maj_supp && rev->min >= min_supp) + if (rev->maj == fw->rev_supp.maj && rev->min >= fw->rev_supp.min) return 0; dev_err(fw->dev.dev, "Driver supports FW version only '%u.%u.x'", - PRESTERA_SUPP_FW_MAJ_VER, PRESTERA_SUPP_FW_MIN_VER); + fw->rev_supp.maj, fw->rev_supp.min); return -EINVAL; } -static int prestera_fw_hdr_parse(struct prestera_fw *fw, - const struct firmware *img) +static int prestera_fw_hdr_parse(struct prestera_fw *fw) { - struct prestera_fw_header *hdr = (struct prestera_fw_header *)img->data; struct prestera_fw_rev *rev = &fw->dev.fw_rev; + struct prestera_fw_header *hdr; u32 magic; + hdr = (struct prestera_fw_header *)fw->bin->data; + magic = be32_to_cpu(hdr->magic_number); if (magic != PRESTERA_FW_HDR_MAGIC) { dev_err(fw->dev.dev, "FW img hdr magic is invalid"); @@ -609,11 +632,52 @@ static int prestera_fw_hdr_parse(struct prestera_fw *fw, return prestera_fw_rev_check(fw); } +static int prestera_fw_get(struct prestera_fw *fw) +{ + int ver_maj = PRESTERA_SUPP_FW_MAJ_VER; + int ver_min = PRESTERA_SUPP_FW_MIN_VER; + char fw_path[128]; + int err; + +pick_fw_ver: + snprintf(fw_path, sizeof(fw_path), PRESTERA_FW_PATH_FMT, + ver_maj, ver_min); + + err = request_firmware_direct(&fw->bin, fw_path, fw->dev.dev); + if (err) { + if (ver_maj == PRESTERA_SUPP_FW_MAJ_VER) { + ver_maj = PRESTERA_PREV_FW_MAJ_VER; + ver_min = PRESTERA_PREV_FW_MIN_VER; + + dev_warn(fw->dev.dev, + "missing latest %s firmware, fall-back to previous %u.%u version\n", + fw_path, ver_maj, ver_min); + + goto pick_fw_ver; + } else { + dev_err(fw->dev.dev, "failed to request previous firmware: %s\n", + fw_path); + return err; + } + } + + dev_info(fw->dev.dev, "Loading %s ...", fw_path); + + fw->rev_supp.maj = ver_maj; + fw->rev_supp.min = ver_min; + fw->rev_supp.sub = 0; + + return 0; +} + +static void prestera_fw_put(struct prestera_fw *fw) +{ + release_firmware(fw->bin); +} + static int prestera_fw_load(struct prestera_fw *fw) { size_t hlen = sizeof(struct prestera_fw_header); - const struct firmware *f; - char fw_path[128]; int err; err = prestera_ldr_wait_reg32(fw, PRESTERA_LDR_READY_REG, @@ -632,30 +696,24 @@ static int prestera_fw_load(struct prestera_fw *fw) fw->ldr_wr_idx = 0; - snprintf(fw_path, sizeof(fw_path), PRESTERA_FW_PATH_FMT, - PRESTERA_SUPP_FW_MAJ_VER, PRESTERA_SUPP_FW_MIN_VER); - - err = request_firmware_direct(&f, fw_path, fw->dev.dev); - if (err) { - dev_err(fw->dev.dev, "failed to request firmware file\n"); + err = prestera_fw_get(fw); + if (err) return err; - } - err = prestera_fw_hdr_parse(fw, f); + err = prestera_fw_hdr_parse(fw); if (err) { dev_err(fw->dev.dev, "FW image header is invalid\n"); goto out_release; } - prestera_ldr_write(fw, PRESTERA_LDR_IMG_SIZE_REG, f->size - hlen); + prestera_ldr_write(fw, PRESTERA_LDR_IMG_SIZE_REG, fw->bin->size - hlen); prestera_ldr_write(fw, PRESTERA_LDR_CTL_REG, PRESTERA_LDR_CTL_DL_START); - dev_info(fw->dev.dev, "Loading %s ...", fw_path); - - err = prestera_ldr_fw_send(fw, f->data + hlen, f->size - hlen); + err = prestera_ldr_fw_send(fw, fw->bin->data + hlen, + fw->bin->size - hlen); out_release: - release_firmware(f); + prestera_fw_put(fw); return err; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c index 2a13c318048c..73d2eba5262f 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c @@ -14,6 +14,7 @@ #include "prestera.h" #include "prestera_hw.h" #include "prestera_rxtx.h" +#include "prestera_devlink.h" #define PRESTERA_SDMA_WAIT_MUL 10 @@ -214,9 +215,10 @@ static struct sk_buff *prestera_sdma_rx_skb_get(struct prestera_sdma *sdma, static int prestera_rxtx_process_skb(struct prestera_sdma *sdma, struct sk_buff *skb) { - const struct prestera_port *port; + struct prestera_port *port; struct prestera_dsa dsa; u32 hw_port, dev_id; + u8 cpu_code; int err; skb_pull(skb, ETH_HLEN); @@ -259,6 +261,9 @@ static int prestera_rxtx_process_skb(struct prestera_sdma *sdma, __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tci); } + cpu_code = dsa.cpu_code; + prestera_devlink_trap_report(port, skb, cpu_code); + return 0; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.c b/drivers/net/ethernet/marvell/prestera/prestera_span.c new file mode 100644 index 000000000000..3cafca827bb7 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2020 Marvell International Ltd. All rights reserved */ + +#include <linux/kernel.h> +#include <linux/list.h> + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_acl.h" +#include "prestera_span.h" + +struct prestera_span_entry { + struct list_head list; + struct prestera_port *port; + refcount_t ref_count; + u8 id; +}; + +struct prestera_span { + struct prestera_switch *sw; + struct list_head entries; +}; + +static struct prestera_span_entry * +prestera_span_entry_create(struct prestera_port *port, u8 span_id) +{ + struct prestera_span_entry *entry; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return ERR_PTR(-ENOMEM); + + refcount_set(&entry->ref_count, 1); + entry->port = port; + entry->id = span_id; + list_add_tail(&entry->list, &port->sw->span->entries); + + return entry; +} + +static void prestera_span_entry_del(struct prestera_span_entry *entry) +{ + list_del(&entry->list); + kfree(entry); +} + +static struct prestera_span_entry * +prestera_span_entry_find_by_id(struct prestera_span *span, u8 span_id) +{ + struct prestera_span_entry *entry; + + list_for_each_entry(entry, &span->entries, list) { + if (entry->id == span_id) + return entry; + } + + return NULL; +} + +static struct prestera_span_entry * +prestera_span_entry_find_by_port(struct prestera_span *span, + struct prestera_port *port) +{ + struct prestera_span_entry *entry; + + list_for_each_entry(entry, &span->entries, list) { + if (entry->port == port) + return entry; + } + + return NULL; +} + +static int prestera_span_get(struct prestera_port *port, u8 *span_id) +{ + u8 new_span_id; + struct prestera_switch *sw = port->sw; + struct prestera_span_entry *entry; + int err; + + entry = prestera_span_entry_find_by_port(sw->span, port); + if (entry) { + refcount_inc(&entry->ref_count); + *span_id = entry->id; + return 0; + } + + err = prestera_hw_span_get(port, &new_span_id); + if (err) + return err; + + entry = prestera_span_entry_create(port, new_span_id); + if (IS_ERR(entry)) { + prestera_hw_span_release(sw, new_span_id); + return PTR_ERR(entry); + } + + *span_id = new_span_id; + return 0; +} + +static int prestera_span_put(struct prestera_switch *sw, u8 span_id) +{ + struct prestera_span_entry *entry; + int err; + + entry = prestera_span_entry_find_by_id(sw->span, span_id); + if (!entry) + return false; + + if (!refcount_dec_and_test(&entry->ref_count)) + return 0; + + err = prestera_hw_span_release(sw, span_id); + if (err) + return err; + + prestera_span_entry_del(entry); + return 0; +} + +static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port) +{ + struct prestera_switch *sw = binding->port->sw; + u8 span_id; + int err; + + if (binding->span_id != PRESTERA_SPAN_INVALID_ID) + /* port already in mirroring */ + return -EEXIST; + + err = prestera_span_get(to_port, &span_id); + if (err) + return err; + + err = prestera_hw_span_bind(binding->port, span_id); + if (err) { + prestera_span_put(sw, span_id); + return err; + } + + binding->span_id = span_id; + return 0; +} + +static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) +{ + int err; + + err = prestera_hw_span_unbind(binding->port); + if (err) + return err; + + err = prestera_span_put(binding->port->sw, binding->span_id); + if (err) + return err; + + binding->span_id = PRESTERA_SPAN_INVALID_ID; + return 0; +} + +int prestera_span_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + struct prestera_flow_block_binding *binding; + __be16 protocol = f->common.protocol; + struct flow_action_entry *act; + struct prestera_port *port; + int err; + + if (!flow_offload_has_one_action(&f->rule->action)) { + NL_SET_ERR_MSG(f->common.extack, + "Only singular actions are supported"); + return -EOPNOTSUPP; + } + + act = &f->rule->action.entries[0]; + + if (!prestera_netdev_check(act->dev)) { + NL_SET_ERR_MSG(f->common.extack, + "Only Marvell Prestera port is supported"); + return -EINVAL; + } + if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) + return -EOPNOTSUPP; + if (act->id != FLOW_ACTION_MIRRED) + return -EOPNOTSUPP; + if (protocol != htons(ETH_P_ALL)) + return -EOPNOTSUPP; + + port = netdev_priv(act->dev); + + list_for_each_entry(binding, &block->binding_list, list) { + err = prestera_span_rule_add(binding, port); + if (err) + goto rollback; + } + + return 0; + +rollback: + list_for_each_entry_continue_reverse(binding, + &block->binding_list, list) + prestera_span_rule_del(binding); + return err; +} + +void prestera_span_destroy(struct prestera_flow_block *block) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + prestera_span_rule_del(binding); +} + +int prestera_span_init(struct prestera_switch *sw) +{ + struct prestera_span *span; + + span = kzalloc(sizeof(*span), GFP_KERNEL); + if (!span) + return -ENOMEM; + + INIT_LIST_HEAD(&span->entries); + + sw->span = span; + span->sw = sw; + + return 0; +} + +void prestera_span_fini(struct prestera_switch *sw) +{ + struct prestera_span *span = sw->span; + + WARN_ON(!list_empty(&span->entries)); + kfree(span); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.h b/drivers/net/ethernet/marvell/prestera/prestera_span.h new file mode 100644 index 000000000000..f0644521f78a --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2019-2020 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_SPAN_H_ +#define _PRESTERA_SPAN_H_ + +#include <net/pkt_cls.h> + +#define PRESTERA_SPAN_INVALID_ID -1 + +struct prestera_switch; +struct prestera_flow_block; + +int prestera_span_init(struct prestera_switch *sw); +void prestera_span_fini(struct prestera_switch *sw); +int prestera_span_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f); +void prestera_span_destroy(struct prestera_flow_block *block); + +#endif /* _PRESTERA_SPAN_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c index cb564890a3dc..0b3e8f2db294 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c @@ -180,6 +180,45 @@ err_port_vlan_alloc: return ERR_PTR(err); } +static int prestera_fdb_add(struct prestera_port *port, + const unsigned char *mac, u16 vid, bool dynamic) +{ + if (prestera_port_is_lag_member(port)) + return prestera_hw_lag_fdb_add(port->sw, prestera_port_lag_id(port), + mac, vid, dynamic); + + return prestera_hw_fdb_add(port, mac, vid, dynamic); +} + +static int prestera_fdb_del(struct prestera_port *port, + const unsigned char *mac, u16 vid) +{ + if (prestera_port_is_lag_member(port)) + return prestera_hw_lag_fdb_del(port->sw, prestera_port_lag_id(port), + mac, vid); + else + return prestera_hw_fdb_del(port, mac, vid); +} + +static int prestera_fdb_flush_port_vlan(struct prestera_port *port, u16 vid, + u32 mode) +{ + if (prestera_port_is_lag_member(port)) + return prestera_hw_fdb_flush_lag_vlan(port->sw, prestera_port_lag_id(port), + vid, mode); + else + return prestera_hw_fdb_flush_port_vlan(port, vid, mode); +} + +static int prestera_fdb_flush_port(struct prestera_port *port, u32 mode) +{ + if (prestera_port_is_lag_member(port)) + return prestera_hw_fdb_flush_lag(port->sw, prestera_port_lag_id(port), + mode); + else + return prestera_hw_fdb_flush_port(port, mode); +} + static void prestera_port_vlan_bridge_leave(struct prestera_port_vlan *port_vlan) { @@ -199,11 +238,11 @@ prestera_port_vlan_bridge_leave(struct prestera_port_vlan *port_vlan) last_port = port_count == 1; if (last_vlan) - prestera_hw_fdb_flush_port(port, fdb_flush_mode); + prestera_fdb_flush_port(port, fdb_flush_mode); else if (last_port) prestera_hw_fdb_flush_vlan(port->sw, vid, fdb_flush_mode); else - prestera_hw_fdb_flush_port_vlan(port, vid, fdb_flush_mode); + prestera_fdb_flush_port_vlan(port, vid, fdb_flush_mode); list_del(&port_vlan->br_vlan_head); prestera_bridge_vlan_put(br_vlan); @@ -312,11 +351,29 @@ __prestera_bridge_port_by_dev(struct prestera_bridge *bridge, return NULL; } +static int prestera_match_upper_bridge_dev(struct net_device *dev, + struct netdev_nested_priv *priv) +{ + if (netif_is_bridge_master(dev)) + priv->data = dev; + + return 0; +} + +static struct net_device *prestera_get_upper_bridge_dev(struct net_device *dev) +{ + struct netdev_nested_priv priv = { }; + + netdev_walk_all_upper_dev_rcu(dev, prestera_match_upper_bridge_dev, + &priv); + return priv.data; +} + static struct prestera_bridge_port * prestera_bridge_port_by_dev(struct prestera_switchdev *swdev, struct net_device *dev) { - struct net_device *br_dev = netdev_master_upper_dev_get(dev); + struct net_device *br_dev = prestera_get_upper_bridge_dev(dev); struct prestera_bridge *bridge; if (!br_dev) @@ -404,7 +461,8 @@ prestera_bridge_1d_port_join(struct prestera_bridge_port *br_port) if (err) return err; - err = prestera_hw_port_flood_set(port, br_port->flags & BR_FLOOD); + err = prestera_hw_port_flood_set(port, BR_FLOOD | BR_MCAST_FLOOD, + br_port->flags); if (err) goto err_port_flood_set; @@ -415,24 +473,23 @@ prestera_bridge_1d_port_join(struct prestera_bridge_port *br_port) return 0; err_port_learning_set: - prestera_hw_port_flood_set(port, false); err_port_flood_set: prestera_hw_bridge_port_delete(port, bridge->bridge_id); return err; } -static int prestera_port_bridge_join(struct prestera_port *port, - struct net_device *upper) +int prestera_bridge_port_join(struct net_device *br_dev, + struct prestera_port *port) { struct prestera_switchdev *swdev = port->sw->swdev; struct prestera_bridge_port *br_port; struct prestera_bridge *bridge; int err; - bridge = prestera_bridge_by_dev(swdev, upper); + bridge = prestera_bridge_by_dev(swdev, br_dev); if (!bridge) { - bridge = prestera_bridge_create(swdev, upper); + bridge = prestera_bridge_create(swdev, br_dev); if (IS_ERR(bridge)) return PTR_ERR(bridge); } @@ -505,14 +562,14 @@ static int prestera_port_vid_stp_set(struct prestera_port *port, u16 vid, return prestera_hw_vlan_port_stp_set(port, vid, hw_state); } -static void prestera_port_bridge_leave(struct prestera_port *port, - struct net_device *upper) +void prestera_bridge_port_leave(struct net_device *br_dev, + struct prestera_port *port) { struct prestera_switchdev *swdev = port->sw->swdev; struct prestera_bridge_port *br_port; struct prestera_bridge *bridge; - bridge = prestera_bridge_by_dev(swdev, upper); + bridge = prestera_bridge_by_dev(swdev, br_dev); if (!bridge) return; @@ -528,57 +585,11 @@ static void prestera_port_bridge_leave(struct prestera_port *port, prestera_bridge_1d_port_leave(br_port); prestera_hw_port_learning_set(port, false); - prestera_hw_port_flood_set(port, false); + prestera_hw_port_flood_set(port, BR_FLOOD | BR_MCAST_FLOOD, 0); prestera_port_vid_stp_set(port, PRESTERA_VID_ALL, BR_STATE_FORWARDING); prestera_bridge_port_put(br_port); } -int prestera_bridge_port_event(struct net_device *dev, unsigned long event, - void *ptr) -{ - struct netdev_notifier_changeupper_info *info = ptr; - struct netlink_ext_ack *extack; - struct prestera_port *port; - struct net_device *upper; - int err; - - extack = netdev_notifier_info_to_extack(&info->info); - port = netdev_priv(dev); - upper = info->upper_dev; - - switch (event) { - case NETDEV_PRECHANGEUPPER: - if (!netif_is_bridge_master(upper)) { - NL_SET_ERR_MSG_MOD(extack, "Unknown upper device type"); - return -EINVAL; - } - - if (!info->linking) - break; - - if (netdev_has_any_upper_dev(upper)) { - NL_SET_ERR_MSG_MOD(extack, "Upper device is already enslaved"); - return -EINVAL; - } - break; - - case NETDEV_CHANGEUPPER: - if (!netif_is_bridge_master(upper)) - break; - - if (info->linking) { - err = prestera_port_bridge_join(port, upper); - if (err) - return err; - } else { - prestera_port_bridge_leave(port, upper); - } - break; - } - - return 0; -} - static int prestera_port_attr_br_flags_set(struct prestera_port *port, struct net_device *dev, struct switchdev_brport_flags flags) @@ -590,11 +601,9 @@ static int prestera_port_attr_br_flags_set(struct prestera_port *port, if (!br_port) return 0; - if (flags.mask & BR_FLOOD) { - err = prestera_hw_port_flood_set(port, flags.val & BR_FLOOD); - if (err) - return err; - } + err = prestera_hw_port_flood_set(port, flags.mask, flags.val); + if (err) + return err; if (flags.mask & BR_LEARNING) { err = prestera_hw_port_learning_set(port, @@ -699,7 +708,7 @@ err_port_stp_set: return err; } -static int prestera_port_obj_attr_set(struct net_device *dev, +static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) { @@ -771,9 +780,9 @@ static int prestera_port_fdb_set(struct prestera_port *port, vid = bridge->bridge_id; if (adding) - err = prestera_hw_fdb_add(port, fdb_info->addr, vid, false); + err = prestera_fdb_add(port, fdb_info->addr, vid, false); else - err = prestera_hw_fdb_del(port, fdb_info->addr, vid); + err = prestera_fdb_del(port, fdb_info->addr, vid); return err; } @@ -901,7 +910,8 @@ prestera_port_vlan_bridge_join(struct prestera_port_vlan *port_vlan, if (port_vlan->br_port) return 0; - err = prestera_hw_port_flood_set(port, br_port->flags & BR_FLOOD); + err = prestera_hw_port_flood_set(port, BR_FLOOD | BR_MCAST_FLOOD, + br_port->flags); if (err) return err; @@ -1009,15 +1019,15 @@ static int prestera_port_vlans_add(struct prestera_port *port, { bool flag_untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED; bool flag_pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID; - struct net_device *dev = vlan->obj.orig_dev; + struct net_device *orig_dev = vlan->obj.orig_dev; struct prestera_bridge_port *br_port; struct prestera_switch *sw = port->sw; struct prestera_bridge *bridge; - if (netif_is_bridge_master(dev)) + if (netif_is_bridge_master(orig_dev)) return 0; - br_port = prestera_bridge_port_by_dev(sw->swdev, dev); + br_port = prestera_bridge_port_by_dev(sw->swdev, port->dev); if (WARN_ON(!br_port)) return -EINVAL; @@ -1030,7 +1040,7 @@ static int prestera_port_vlans_add(struct prestera_port *port, flag_pvid, extack); } -static int prestera_port_obj_add(struct net_device *dev, +static int prestera_port_obj_add(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) { @@ -1049,14 +1059,14 @@ static int prestera_port_obj_add(struct net_device *dev, static int prestera_port_vlans_del(struct prestera_port *port, const struct switchdev_obj_port_vlan *vlan) { - struct net_device *dev = vlan->obj.orig_dev; + struct net_device *orig_dev = vlan->obj.orig_dev; struct prestera_bridge_port *br_port; struct prestera_switch *sw = port->sw; - if (netif_is_bridge_master(dev)) + if (netif_is_bridge_master(orig_dev)) return -EOPNOTSUPP; - br_port = prestera_bridge_port_by_dev(sw->swdev, dev); + br_port = prestera_bridge_port_by_dev(sw->swdev, port->dev); if (WARN_ON(!br_port)) return -EINVAL; @@ -1068,7 +1078,7 @@ static int prestera_port_vlans_del(struct prestera_port *port, return 0; } -static int prestera_port_obj_del(struct net_device *dev, +static int prestera_port_obj_del(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj) { struct prestera_port *port = netdev_priv(dev); @@ -1114,10 +1124,26 @@ static void prestera_fdb_event(struct prestera_switch *sw, struct prestera_event *evt, void *arg) { struct switchdev_notifier_fdb_info info; + struct net_device *dev = NULL; struct prestera_port *port; + struct prestera_lag *lag; - port = prestera_find_port(sw, evt->fdb_evt.port_id); - if (!port) + switch (evt->fdb_evt.type) { + case PRESTERA_FDB_ENTRY_TYPE_REG_PORT: + port = prestera_find_port(sw, evt->fdb_evt.dest.port_id); + if (port) + dev = port->dev; + break; + case PRESTERA_FDB_ENTRY_TYPE_LAG: + lag = prestera_lag_by_id(sw, evt->fdb_evt.dest.lag_id); + if (lag) + dev = lag->dev; + break; + default: + return; + } + + if (!dev) return; info.addr = evt->fdb_evt.data.mac; @@ -1129,11 +1155,11 @@ static void prestera_fdb_event(struct prestera_switch *sw, switch (evt->id) { case PRESTERA_FDB_EVENT_LEARNED: call_switchdev_notifiers(SWITCHDEV_FDB_ADD_TO_BRIDGE, - port->dev, &info.info, NULL); + dev, &info.info, NULL); break; case PRESTERA_FDB_EVENT_AGED: call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, - port->dev, &info.info, NULL); + dev, &info.info, NULL); break; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.h b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.h index 606e21d2355b..a91bc35d235f 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.h @@ -7,7 +7,10 @@ int prestera_switchdev_init(struct prestera_switch *sw); void prestera_switchdev_fini(struct prestera_switch *sw); -int prestera_bridge_port_event(struct net_device *dev, unsigned long event, - void *ptr); +int prestera_bridge_port_join(struct net_device *br_dev, + struct prestera_port *port); + +void prestera_bridge_port_leave(struct net_device *br_dev, + struct prestera_port *port); #endif /* _PRESTERA_SWITCHDEV_H_ */ diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c index e967867828d8..9b48ae4bac39 100644 --- a/drivers/net/ethernet/marvell/pxa168_eth.c +++ b/drivers/net/ethernet/marvell/pxa168_eth.c @@ -1528,6 +1528,7 @@ static int pxa168_eth_remove(struct platform_device *pdev) struct net_device *dev = platform_get_drvdata(pdev); struct pxa168_eth_private *pep = netdev_priv(dev); + cancel_work_sync(&pep->tx_timeout_task); if (pep->htpr) { dma_free_coherent(pep->dev->dev.parent, HASH_ADDR_TABLE_SIZE, pep->htpr, pep->htpr_dma); @@ -1539,7 +1540,6 @@ static int pxa168_eth_remove(struct platform_device *pdev) clk_disable_unprepare(pep->clk); mdiobus_unregister(pep->smi_bus); mdiobus_free(pep->smi_bus); - cancel_work_sync(&pep->tx_timeout_task); unregister_netdev(dev); free_netdev(dev); return 0; diff --git a/drivers/net/ethernet/marvell/skge.h b/drivers/net/ethernet/marvell/skge.h index 6928abcec0a3..f72217348eb4 100644 --- a/drivers/net/ethernet/marvell/skge.h +++ b/drivers/net/ethernet/marvell/skge.h @@ -263,7 +263,7 @@ enum { CHIP_ID_YUKON_LP = 0xb2, /* Chip ID for YUKON-LP */ CHIP_ID_YUKON_XL = 0xb3, /* Chip ID for YUKON-2 XL */ CHIP_ID_YUKON_EC = 0xb6, /* Chip ID for YUKON-2 EC */ - CHIP_ID_YUKON_FE = 0xb7, /* Chip ID for YUKON-2 FE */ + CHIP_ID_YUKON_FE = 0xb7, /* Chip ID for YUKON-2 FE */ CHIP_REV_YU_LITE_A1 = 3, /* Chip Rev. for YUKON-Lite A1,A2 */ CHIP_REV_YU_LITE_A3 = 7, /* Chip Rev. for YUKON-Lite A3 */ diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 222c32367b2c..8b8bff59c8fe 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -471,7 +471,7 @@ static void sky2_phy_init(struct sky2_hw *hw, unsigned port) adv |= fiber_fc_adv[sky2->flow_mode]; } else { reg |= GM_GPCR_AU_FCT_DIS; - reg |= gm_fc_disable[sky2->flow_mode]; + reg |= gm_fc_disable[sky2->flow_mode]; /* Forward pause packets to GMAC? */ if (sky2->flow_mode & FC_RX) @@ -1656,16 +1656,16 @@ static void sky2_hw_up(struct sky2_port *sky2) tx_init(sky2); /* - * On dual port PCI-X card, there is an problem where status + * On dual port PCI-X card, there is an problem where status * can be received out of order due to split transactions */ if (otherdev && netif_running(otherdev) && - (cap = pci_find_capability(hw->pdev, PCI_CAP_ID_PCIX))) { - u16 cmd; + (cap = pci_find_capability(hw->pdev, PCI_CAP_ID_PCIX))) { + u16 cmd; cmd = sky2_pci_read16(hw, cap + PCI_X_CMD); - cmd &= ~PCI_X_CMD_MAX_SPLIT; - sky2_pci_write16(hw, cap + PCI_X_CMD, cmd); + cmd &= ~PCI_X_CMD_MAX_SPLIT; + sky2_pci_write16(hw, cap + PCI_X_CMD, cmd); } sky2_mac_init(hw, port); @@ -1836,8 +1836,8 @@ static netdev_tx_t sky2_xmit_frame(struct sk_buff *skb, u16 mss; u8 ctrl; - if (unlikely(tx_avail(sky2) < tx_le_req(skb))) - return NETDEV_TX_BUSY; + if (unlikely(tx_avail(sky2) < tx_le_req(skb))) + return NETDEV_TX_BUSY; len = skb_headlen(skb); mapping = dma_map_single(&hw->pdev->dev, skb->data, len, @@ -1866,9 +1866,9 @@ static netdev_tx_t sky2_xmit_frame(struct sk_buff *skb, if (!(hw->flags & SKY2_HW_NEW_LE)) mss += ETH_HLEN + ip_hdrlen(skb) + tcp_hdrlen(skb); - if (mss != sky2->tx_last_mss) { + if (mss != sky2->tx_last_mss) { le = get_tx_le(sky2, &slot); - le->addr = cpu_to_le32(mss); + le->addr = cpu_to_le32(mss); if (hw->flags & SKY2_HW_NEW_LE) le->opcode = OP_MSS | HW_OWNER; @@ -1895,8 +1895,8 @@ static netdev_tx_t sky2_xmit_frame(struct sk_buff *skb, /* Handle TCP checksum offload */ if (skb->ip_summed == CHECKSUM_PARTIAL) { /* On Yukon EX (some versions) encoding change. */ - if (hw->flags & SKY2_HW_AUTO_TX_SUM) - ctrl |= CALSUM; /* auto checksum */ + if (hw->flags & SKY2_HW_AUTO_TX_SUM) + ctrl |= CALSUM; /* auto checksum */ else { const unsigned offset = skb_transport_offset(skb); u32 tcpsum; @@ -2503,7 +2503,7 @@ static void skb_put_frags(struct sk_buff *skb, unsigned int hdr_space, if (length == 0) { /* don't need this page */ - __skb_frag_unref(frag); + __skb_frag_unref(frag, false); --skb_shinfo(skb)->nr_frags; } else { size = min(length, (unsigned) PAGE_SIZE); @@ -2557,7 +2557,7 @@ nobuf: static struct sk_buff *sky2_receive(struct net_device *dev, u16 length, u32 status) { - struct sky2_port *sky2 = netdev_priv(dev); + struct sky2_port *sky2 = netdev_priv(dev); struct rx_ring_info *re = sky2->rx_ring + sky2->rx_next; struct sk_buff *skb = NULL; u16 count = (status & GMR_FS_LEN) >> 16; @@ -5063,11 +5063,11 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!disable_msi && pci_enable_msi(pdev) == 0) { err = sky2_test_msi(hw); if (err) { - pci_disable_msi(pdev); + pci_disable_msi(pdev); if (err != -EOPNOTSUPP) goto err_out_free_netdev; } - } + } netif_napi_add(dev, &hw->napi, sky2_poll, NAPI_WEIGHT); diff --git a/drivers/net/ethernet/marvell/sky2.h b/drivers/net/ethernet/marvell/sky2.h index b2dddd8a246c..ddec1627f1a7 100644 --- a/drivers/net/ethernet/marvell/sky2.h +++ b/drivers/net/ethernet/marvell/sky2.h @@ -538,8 +538,8 @@ enum { CHIP_ID_YUKON_EC_U = 0xb4, /* YUKON-2 EC Ultra */ CHIP_ID_YUKON_EX = 0xb5, /* YUKON-2 Extreme */ CHIP_ID_YUKON_EC = 0xb6, /* YUKON-2 EC */ - CHIP_ID_YUKON_FE = 0xb7, /* YUKON-2 FE */ - CHIP_ID_YUKON_FE_P = 0xb8, /* YUKON-2 FE+ */ + CHIP_ID_YUKON_FE = 0xb7, /* YUKON-2 FE */ + CHIP_ID_YUKON_FE_P = 0xb8, /* YUKON-2 FE+ */ CHIP_ID_YUKON_SUPR = 0xb9, /* YUKON-2 Supreme */ CHIP_ID_YUKON_UL_2 = 0xba, /* YUKON-2 Ultra 2 */ CHIP_ID_YUKON_OPT = 0xbc, /* YUKON-2 Optima */ @@ -2262,8 +2262,8 @@ struct sky2_port { #define SKY2_FLAG_AUTO_SPEED 0x0002 #define SKY2_FLAG_AUTO_PAUSE 0x0004 - enum flow_control flow_mode; - enum flow_control flow_status; + enum flow_control flow_mode; + enum flow_control flow_status; #ifdef CONFIG_SKY2_DEBUG struct dentry *debugfs; diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index ed4eacef17ce..64adfd24e134 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -681,32 +681,53 @@ static int mtk_set_mac_address(struct net_device *dev, void *p) void mtk_stats_update_mac(struct mtk_mac *mac) { struct mtk_hw_stats *hw_stats = mac->hw_stats; - unsigned int base = MTK_GDM1_TX_GBCNT; - u64 stats; - - base += hw_stats->reg_offset; + struct mtk_eth *eth = mac->hw; u64_stats_update_begin(&hw_stats->syncp); - hw_stats->rx_bytes += mtk_r32(mac->hw, base); - stats = mtk_r32(mac->hw, base + 0x04); - if (stats) - hw_stats->rx_bytes += (stats << 32); - hw_stats->rx_packets += mtk_r32(mac->hw, base + 0x08); - hw_stats->rx_overflow += mtk_r32(mac->hw, base + 0x10); - hw_stats->rx_fcs_errors += mtk_r32(mac->hw, base + 0x14); - hw_stats->rx_short_errors += mtk_r32(mac->hw, base + 0x18); - hw_stats->rx_long_errors += mtk_r32(mac->hw, base + 0x1c); - hw_stats->rx_checksum_errors += mtk_r32(mac->hw, base + 0x20); - hw_stats->rx_flow_control_packets += - mtk_r32(mac->hw, base + 0x24); - hw_stats->tx_skip += mtk_r32(mac->hw, base + 0x28); - hw_stats->tx_collisions += mtk_r32(mac->hw, base + 0x2c); - hw_stats->tx_bytes += mtk_r32(mac->hw, base + 0x30); - stats = mtk_r32(mac->hw, base + 0x34); - if (stats) - hw_stats->tx_bytes += (stats << 32); - hw_stats->tx_packets += mtk_r32(mac->hw, base + 0x38); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) { + hw_stats->tx_packets += mtk_r32(mac->hw, MT7628_SDM_TPCNT); + hw_stats->tx_bytes += mtk_r32(mac->hw, MT7628_SDM_TBCNT); + hw_stats->rx_packets += mtk_r32(mac->hw, MT7628_SDM_RPCNT); + hw_stats->rx_bytes += mtk_r32(mac->hw, MT7628_SDM_RBCNT); + hw_stats->rx_checksum_errors += + mtk_r32(mac->hw, MT7628_SDM_CS_ERR); + } else { + unsigned int offs = hw_stats->reg_offset; + u64 stats; + + hw_stats->rx_bytes += mtk_r32(mac->hw, + MTK_GDM1_RX_GBCNT_L + offs); + stats = mtk_r32(mac->hw, MTK_GDM1_RX_GBCNT_H + offs); + if (stats) + hw_stats->rx_bytes += (stats << 32); + hw_stats->rx_packets += + mtk_r32(mac->hw, MTK_GDM1_RX_GPCNT + offs); + hw_stats->rx_overflow += + mtk_r32(mac->hw, MTK_GDM1_RX_OERCNT + offs); + hw_stats->rx_fcs_errors += + mtk_r32(mac->hw, MTK_GDM1_RX_FERCNT + offs); + hw_stats->rx_short_errors += + mtk_r32(mac->hw, MTK_GDM1_RX_SERCNT + offs); + hw_stats->rx_long_errors += + mtk_r32(mac->hw, MTK_GDM1_RX_LENCNT + offs); + hw_stats->rx_checksum_errors += + mtk_r32(mac->hw, MTK_GDM1_RX_CERCNT + offs); + hw_stats->rx_flow_control_packets += + mtk_r32(mac->hw, MTK_GDM1_RX_FCCNT + offs); + hw_stats->tx_skip += + mtk_r32(mac->hw, MTK_GDM1_TX_SKIPCNT + offs); + hw_stats->tx_collisions += + mtk_r32(mac->hw, MTK_GDM1_TX_COLCNT + offs); + hw_stats->tx_bytes += + mtk_r32(mac->hw, MTK_GDM1_TX_GBCNT_L + offs); + stats = mtk_r32(mac->hw, MTK_GDM1_TX_GBCNT_H + offs); + if (stats) + hw_stats->tx_bytes += (stats << 32); + hw_stats->tx_packets += + mtk_r32(mac->hw, MTK_GDM1_TX_GPCNT + offs); + } + u64_stats_update_end(&hw_stats->syncp); } @@ -2423,7 +2444,8 @@ static void mtk_dim_rx(struct work_struct *work) val |= cur << MTK_PDMA_DELAY_RX_PINT_SHIFT; mtk_w32(eth, val, MTK_PDMA_DELAY_INT); - mtk_w32(eth, val, MTK_QDMA_DELAY_INT); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) + mtk_w32(eth, val, MTK_QDMA_DELAY_INT); spin_unlock_bh(ð->dim_lock); @@ -2452,7 +2474,8 @@ static void mtk_dim_tx(struct work_struct *work) val |= cur << MTK_PDMA_DELAY_TX_PINT_SHIFT; mtk_w32(eth, val, MTK_PDMA_DELAY_INT); - mtk_w32(eth, val, MTK_QDMA_DELAY_INT); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) + mtk_w32(eth, val, MTK_QDMA_DELAY_INT); spin_unlock_bh(ð->dim_lock); @@ -2480,6 +2503,10 @@ static int mtk_hw_init(struct mtk_eth *eth) goto err_disable_pm; } + /* set interrupt delays based on current Net DIM sample */ + mtk_dim_rx(ð->rx_dim.work); + mtk_dim_tx(ð->tx_dim.work); + /* disable delay and normal interrupt */ mtk_tx_irq_disable(eth, ~0); mtk_rx_irq_disable(eth, ~0); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index 11331b44ba07..5ef70dd8b49c 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -278,8 +278,21 @@ /* QDMA FQ Free Page Buffer Length Register */ #define MTK_QDMA_FQ_BLEN 0x1B2C -/* GMA1 Received Good Byte Count Register */ -#define MTK_GDM1_TX_GBCNT 0x2400 +/* GMA1 counter / statics register */ +#define MTK_GDM1_RX_GBCNT_L 0x2400 +#define MTK_GDM1_RX_GBCNT_H 0x2404 +#define MTK_GDM1_RX_GPCNT 0x2408 +#define MTK_GDM1_RX_OERCNT 0x2410 +#define MTK_GDM1_RX_FERCNT 0x2414 +#define MTK_GDM1_RX_SERCNT 0x2418 +#define MTK_GDM1_RX_LENCNT 0x241c +#define MTK_GDM1_RX_CERCNT 0x2420 +#define MTK_GDM1_RX_FCCNT 0x2424 +#define MTK_GDM1_TX_SKIPCNT 0x2428 +#define MTK_GDM1_TX_COLCNT 0x242c +#define MTK_GDM1_TX_GBCNT_L 0x2430 +#define MTK_GDM1_TX_GBCNT_H 0x2434 +#define MTK_GDM1_TX_GPCNT 0x2438 #define MTK_STAT_OFFSET 0x40 /* QDMA descriptor txd4 */ @@ -502,6 +515,13 @@ #define MT7628_SDM_MAC_ADRL (MT7628_SDM_OFFSET + 0x0c) #define MT7628_SDM_MAC_ADRH (MT7628_SDM_OFFSET + 0x10) +/* Counter / stat register */ +#define MT7628_SDM_TPCNT (MT7628_SDM_OFFSET + 0x100) +#define MT7628_SDM_TBCNT (MT7628_SDM_OFFSET + 0x104) +#define MT7628_SDM_RPCNT (MT7628_SDM_OFFSET + 0x108) +#define MT7628_SDM_RBCNT (MT7628_SDM_OFFSET + 0x10c) +#define MT7628_SDM_CS_ERR (MT7628_SDM_OFFSET + 0x110) + struct mtk_rx_dma { unsigned int rxd1; unsigned int rxd2; diff --git a/drivers/net/ethernet/mellanox/Kconfig b/drivers/net/ethernet/mellanox/Kconfig index ff6613a5cdd3..b4f66eb9ddb9 100644 --- a/drivers/net/ethernet/mellanox/Kconfig +++ b/drivers/net/ethernet/mellanox/Kconfig @@ -22,5 +22,6 @@ source "drivers/net/ethernet/mellanox/mlx4/Kconfig" source "drivers/net/ethernet/mellanox/mlx5/core/Kconfig" source "drivers/net/ethernet/mellanox/mlxsw/Kconfig" source "drivers/net/ethernet/mellanox/mlxfw/Kconfig" +source "drivers/net/ethernet/mellanox/mlxbf_gige/Kconfig" endif # NET_VENDOR_MELLANOX diff --git a/drivers/net/ethernet/mellanox/Makefile b/drivers/net/ethernet/mellanox/Makefile index 79773ac331ee..d4b5f547a727 100644 --- a/drivers/net/ethernet/mellanox/Makefile +++ b/drivers/net/ethernet/mellanox/Makefile @@ -7,3 +7,4 @@ obj-$(CONFIG_MLX4_CORE) += mlx4/ obj-$(CONFIG_MLX5_CORE) += mlx5/core/ obj-$(CONFIG_MLXSW_CORE) += mlxsw/ obj-$(CONFIG_MLXFW) += mlxfw/ +obj-$(CONFIG_MLXBF_GIGE) += mlxbf_gige/ diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 1434df66fcf2..3616b77caa0a 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -2027,8 +2027,6 @@ static int mlx4_en_set_tunable(struct net_device *dev, return ret; } -#define MLX4_EEPROM_PAGE_LEN 256 - static int mlx4_en_get_module_info(struct net_device *dev, struct ethtool_modinfo *modinfo) { @@ -2063,7 +2061,7 @@ static int mlx4_en_get_module_info(struct net_device *dev, break; case MLX4_MODULE_ID_SFP: modinfo->type = ETH_MODULE_SFF_8472; - modinfo->eeprom_len = MLX4_EEPROM_PAGE_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN; break; default: return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index e35e4d7ef4d1..442991d91c15 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -526,7 +526,7 @@ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv, fail: while (nr > 0) { nr--; - __skb_frag_unref(skb_shinfo(skb)->frags + nr); + __skb_frag_unref(skb_shinfo(skb)->frags + nr, false); } return 0; } @@ -679,9 +679,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud ring = priv->rx_ring[cq_ring]; - /* Protect accesses to: ring->xdp_prog, priv->mac_hash list */ - rcu_read_lock(); - xdp_prog = rcu_dereference(ring->xdp_prog); + xdp_prog = rcu_dereference_bh(ring->xdp_prog); xdp_init_buff(&xdp, priv->frag_info[0].frag_stride, &ring->xdp_rxq); doorbell_pending = false; @@ -744,7 +742,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud /* Drop the packet, since HW loopback-ed it */ mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX]; bucket = &priv->mac_hash[mac_hash]; - hlist_for_each_entry_rcu(entry, bucket, hlist) { + hlist_for_each_entry_rcu_bh(entry, bucket, hlist) { if (ether_addr_equal_64bits(entry->mac, ethh->h_source)) goto next; @@ -899,8 +897,6 @@ next: break; } - rcu_read_unlock(); - if (likely(polled)) { if (doorbell_pending) { priv->tx_cq[TX_XDP][cq_ring]->xdp_busy = true; diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c index f6cfec81ccc3..dc4ac1a2b6b6 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw.c +++ b/drivers/net/ethernet/mellanox/mlx4/fw.c @@ -823,6 +823,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) #define QUERY_DEV_CAP_MAD_DEMUX_OFFSET 0xb0 #define QUERY_DEV_CAP_DMFS_HIGH_RATE_QPN_BASE_OFFSET 0xa8 #define QUERY_DEV_CAP_DMFS_HIGH_RATE_QPN_RANGE_OFFSET 0xac +#define QUERY_DEV_CAP_MAP_CLOCK_TO_USER 0xc1 #define QUERY_DEV_CAP_QP_RATE_LIMIT_NUM_OFFSET 0xcc #define QUERY_DEV_CAP_QP_RATE_LIMIT_MAX_OFFSET 0xd0 #define QUERY_DEV_CAP_QP_RATE_LIMIT_MIN_OFFSET 0xd2 @@ -841,6 +842,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) if (mlx4_is_mfunc(dev)) disable_unsupported_roce_caps(outbox); + MLX4_GET(field, outbox, QUERY_DEV_CAP_MAP_CLOCK_TO_USER); + dev_cap->map_clock_to_user = field & 0x80; MLX4_GET(field, outbox, QUERY_DEV_CAP_RSVD_QP_OFFSET); dev_cap->reserved_qps = 1 << (field & 0xf); MLX4_GET(field, outbox, QUERY_DEV_CAP_MAX_QP_OFFSET); diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.h b/drivers/net/ethernet/mellanox/mlx4/fw.h index 8f020f26ebf5..cf64e54eecb0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw.h +++ b/drivers/net/ethernet/mellanox/mlx4/fw.h @@ -131,6 +131,7 @@ struct mlx4_dev_cap { u32 health_buffer_addrs; struct mlx4_port_cap port_cap[MLX4_MAX_PORTS + 1]; bool wol_port[MLX4_MAX_PORTS + 1]; + bool map_clock_to_user; }; struct mlx4_func_cap { diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index c326b434734e..00c84656b2e7 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -498,6 +498,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) } } + dev->caps.map_clock_to_user = dev_cap->map_clock_to_user; dev->caps.uar_page_size = PAGE_SIZE; dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE; dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay; @@ -1948,6 +1949,11 @@ int mlx4_get_internal_clock_params(struct mlx4_dev *dev, if (mlx4_is_slave(dev)) return -EOPNOTSUPP; + if (!dev->caps.map_clock_to_user) { + mlx4_dbg(dev, "Map clock to user is not supported.\n"); + return -EOPNOTSUPP; + } + if (!params) return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c index ba6ac31a339d..256a06b3c096 100644 --- a/drivers/net/ethernet/mellanox/mlx4/port.c +++ b/drivers/net/ethernet/mellanox/mlx4/port.c @@ -1973,6 +1973,7 @@ EXPORT_SYMBOL(mlx4_get_roce_gid_from_slave); #define I2C_ADDR_LOW 0x50 #define I2C_ADDR_HIGH 0x51 #define I2C_PAGE_SIZE 256 +#define I2C_HIGH_PAGE_SIZE 128 /* Module Info Data */ struct mlx4_cable_info { @@ -2026,6 +2027,88 @@ static inline const char *cable_info_mad_err_str(u16 mad_status) return "Unknown Error"; } +static int mlx4_get_module_id(struct mlx4_dev *dev, u8 port, u8 *module_id) +{ + struct mlx4_cmd_mailbox *inbox, *outbox; + struct mlx4_mad_ifc *inmad, *outmad; + struct mlx4_cable_info *cable_info; + int ret; + + inbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(inbox)) + return PTR_ERR(inbox); + + outbox = mlx4_alloc_cmd_mailbox(dev); + if (IS_ERR(outbox)) { + mlx4_free_cmd_mailbox(dev, inbox); + return PTR_ERR(outbox); + } + + inmad = (struct mlx4_mad_ifc *)(inbox->buf); + outmad = (struct mlx4_mad_ifc *)(outbox->buf); + + inmad->method = 0x1; /* Get */ + inmad->class_version = 0x1; + inmad->mgmt_class = 0x1; + inmad->base_version = 0x1; + inmad->attr_id = cpu_to_be16(0xFF60); /* Module Info */ + + cable_info = (struct mlx4_cable_info *)inmad->data; + cable_info->dev_mem_address = 0; + cable_info->page_num = 0; + cable_info->i2c_addr = I2C_ADDR_LOW; + cable_info->size = cpu_to_be16(1); + + ret = mlx4_cmd_box(dev, inbox->dma, outbox->dma, port, 3, + MLX4_CMD_MAD_IFC, MLX4_CMD_TIME_CLASS_C, + MLX4_CMD_NATIVE); + if (ret) + goto out; + + if (be16_to_cpu(outmad->status)) { + /* Mad returned with bad status */ + ret = be16_to_cpu(outmad->status); + mlx4_warn(dev, + "MLX4_CMD_MAD_IFC Get Module ID attr(%x) port(%d) i2c_addr(%x) offset(%d) size(%d): Response Mad Status(%x) - %s\n", + 0xFF60, port, I2C_ADDR_LOW, 0, 1, ret, + cable_info_mad_err_str(ret)); + ret = -ret; + goto out; + } + cable_info = (struct mlx4_cable_info *)outmad->data; + *module_id = cable_info->data[0]; +out: + mlx4_free_cmd_mailbox(dev, inbox); + mlx4_free_cmd_mailbox(dev, outbox); + return ret; +} + +static void mlx4_sfp_eeprom_params_set(u8 *i2c_addr, u8 *page_num, u16 *offset) +{ + *i2c_addr = I2C_ADDR_LOW; + *page_num = 0; + + if (*offset < I2C_PAGE_SIZE) + return; + + *i2c_addr = I2C_ADDR_HIGH; + *offset -= I2C_PAGE_SIZE; +} + +static void mlx4_qsfp_eeprom_params_set(u8 *i2c_addr, u8 *page_num, u16 *offset) +{ + /* Offsets 0-255 belong to page 0. + * Offsets 256-639 belong to pages 01, 02, 03. + * For example, offset 400 is page 02: 1 + (400 - 256) / 128 = 2 + */ + if (*offset < I2C_PAGE_SIZE) + *page_num = 0; + else + *page_num = 1 + (*offset - I2C_PAGE_SIZE) / I2C_HIGH_PAGE_SIZE; + *i2c_addr = I2C_ADDR_LOW; + *offset -= *page_num * I2C_HIGH_PAGE_SIZE; +} + /** * mlx4_get_module_info - Read cable module eeprom data * @dev: mlx4_dev. @@ -2045,12 +2128,30 @@ int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, struct mlx4_cmd_mailbox *inbox, *outbox; struct mlx4_mad_ifc *inmad, *outmad; struct mlx4_cable_info *cable_info; - u16 i2c_addr; + u8 module_id, i2c_addr, page_num; int ret; if (size > MODULE_INFO_MAX_READ) size = MODULE_INFO_MAX_READ; + ret = mlx4_get_module_id(dev, port, &module_id); + if (ret) + return ret; + + switch (module_id) { + case MLX4_MODULE_ID_SFP: + mlx4_sfp_eeprom_params_set(&i2c_addr, &page_num, &offset); + break; + case MLX4_MODULE_ID_QSFP: + case MLX4_MODULE_ID_QSFP_PLUS: + case MLX4_MODULE_ID_QSFP28: + mlx4_qsfp_eeprom_params_set(&i2c_addr, &page_num, &offset); + break; + default: + mlx4_err(dev, "Module ID not recognized: %#x\n", module_id); + return -EINVAL; + } + inbox = mlx4_alloc_cmd_mailbox(dev); if (IS_ERR(inbox)) return PTR_ERR(inbox); @@ -2076,11 +2177,9 @@ int mlx4_get_module_info(struct mlx4_dev *dev, u8 port, */ size -= offset + size - I2C_PAGE_SIZE; - i2c_addr = I2C_ADDR_LOW; - cable_info = (struct mlx4_cable_info *)inmad->data; cable_info->dev_mem_address = cpu_to_be16(offset); - cable_info->page_num = 0; + cable_info->page_num = page_num; cable_info->i2c_addr = i2c_addr; cable_info->size = cpu_to_be16(size); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 461a43f338e6..e1a5a79e27c7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -12,7 +12,6 @@ config MLX5_CORE depends on MLXFW || !MLXFW depends on PTP_1588_CLOCK || !PTP_1588_CLOCK depends on PCI_HYPERV_INTERFACE || !PCI_HYPERV_INTERFACE - default n help Core driver for low level functionality of the ConnectX-4 and Connect-IB cards by Mellanox Technologies. @@ -36,7 +35,6 @@ config MLX5_CORE_EN depends on NETDEVICES && ETHERNET && INET && PCI && MLX5_CORE select PAGE_POOL select DIMLIB - default n help Ethernet support in Mellanox Technologies ConnectX-4 NIC. @@ -79,6 +77,16 @@ config MLX5_ESWITCH Legacy SRIOV mode (L2 mac vlan steering based). Switchdev mode (eswitch offloads). +config MLX5_BRIDGE + bool + depends on MLX5_ESWITCH && BRIDGE + default y + help + mlx5 ConnectX offloads support for Ethernet Bridging (BRIDGE). + Enable adding representors of mlx5 uplink and VF ports to Bridge and + offloading rules for traffic between such ports. Supports VLANs (trunk and + access modes). + config MLX5_CLS_ACT bool "MLX5 TC classifier action support" depends on MLX5_ESWITCH && NET_CLS_ACT @@ -131,7 +139,6 @@ config MLX5_CORE_EN_DCB config MLX5_CORE_IPOIB bool "Mellanox 5th generation network adapters (connectX series) IPoIB offloads support" depends on MLX5_CORE_EN - default n help MLX5 IPoIB offloads & acceleration support. @@ -139,7 +146,6 @@ config MLX5_FPGA_IPSEC bool "Mellanox Technologies IPsec Innova support" depends on MLX5_CORE depends on MLX5_FPGA - default n help Build IPsec support for the Innova family of network cards by Mellanox Technologies. Innova network cards are comprised of a ConnectX chip @@ -153,7 +159,6 @@ config MLX5_IPSEC depends on XFRM_OFFLOAD depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD select MLX5_ACCEL - default n help Build IPsec support for the Connect-X family of network cards by Mellanox Technologies. @@ -166,7 +171,6 @@ config MLX5_EN_IPSEC depends on XFRM_OFFLOAD depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD depends on MLX5_FPGA_IPSEC || MLX5_IPSEC - default n help Build support for IPsec cryptography-offload acceleration in the NIC. Note: Support for hardware with this capability needs to be selected @@ -179,7 +183,6 @@ config MLX5_FPGA_TLS depends on MLX5_CORE_EN depends on MLX5_FPGA select MLX5_EN_TLS - default n help Build TLS support for the Innova family of network cards by Mellanox Technologies. Innova network cards are comprised of a ConnectX chip @@ -194,7 +197,6 @@ config MLX5_TLS depends on MLX5_CORE_EN select MLX5_ACCEL select MLX5_EN_TLS - default n help Build TLS support for the Connect-X family of network cards by Mellanox Technologies. @@ -217,7 +219,6 @@ config MLX5_SW_STEERING config MLX5_SF bool "Mellanox Technologies subfunction device support using auxiliary device" depends on MLX5_CORE && MLX5_CORE_EN - default n help Build support for subfuction device in the NIC. A Mellanox subfunction device can support RDMA, netdevice and vdpa device. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index a1223e904190..b5072a3a2585 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -14,7 +14,7 @@ obj-$(CONFIG_MLX5_CORE) += mlx5_core.o mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ health.o mcg.o cq.o alloc.o port.o mr.o pd.o \ transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \ - fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \ + fs_counters.o fs_ft_pool.o rl.o lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \ fw_reset.o qos.o @@ -56,6 +56,7 @@ mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \ esw/acl/ingress_lgcy.o esw/acl/ingress_ofld.o \ esw/devlink_port.o esw/vporttbl.o mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += esw/sample.o +mlx5_core-$(CONFIG_MLX5_BRIDGE) += esw/bridge.o en/rep/bridge.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o mlx5_core-$(CONFIG_VXLAN) += lib/vxlan.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index a9166cd85013..ceebfc20f65e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -303,6 +303,7 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) int ret = 0, i; mutex_lock(&mlx5_intf_mutex); + priv->flags &= ~MLX5_PRIV_FLAGS_DETACH; for (i = 0; i < ARRAY_SIZE(mlx5_adev_devices); i++) { if (!priv->adev[i]) { bool is_supported = false; @@ -320,6 +321,16 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) } } else { adev = &priv->adev[i]->adev; + + /* Pay attention that this is not PCI driver that + * mlx5_core_dev is connected, but auxiliary driver. + * + * Here we can race of module unload with devlink + * reload, but we don't need to take extra lock because + * we are holding global mlx5_intf_mutex. + */ + if (!adev->dev.driver) + continue; adrv = to_auxiliary_drv(adev->dev.driver); if (adrv->resume) @@ -350,6 +361,10 @@ void mlx5_detach_device(struct mlx5_core_dev *dev) continue; adev = &priv->adev[i]->adev; + /* Auxiliary driver was unbind manually through sysfs */ + if (!adev->dev.driver) + goto skip_suspend; + adrv = to_auxiliary_drv(adev->dev.driver); if (adrv->suspend) { @@ -357,9 +372,11 @@ void mlx5_detach_device(struct mlx5_core_dev *dev) continue; } +skip_suspend: del_adev(&priv->adev[i]->adev); priv->adev[i] = NULL; } + priv->flags |= MLX5_PRIV_FLAGS_DETACH; mutex_unlock(&mlx5_intf_mutex); } @@ -448,6 +465,8 @@ int mlx5_rescan_drivers_locked(struct mlx5_core_dev *dev) struct mlx5_priv *priv = &dev->priv; lockdep_assert_held(&mlx5_intf_mutex); + if (priv->flags & MLX5_PRIV_FLAGS_DETACH) + return 0; delete_drivers(dev); if (priv->flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 44c458443428..d791d351b489 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -63,6 +63,11 @@ mlx5_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, err = devlink_info_version_running_put(req, "fw.version", version_str); if (err) return err; + err = devlink_info_version_running_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW, + version_str); + if (err) + return err; /* no pending version, return running (stored) version */ if (stored_fw == 0) @@ -74,8 +79,9 @@ mlx5_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, err = devlink_info_version_stored_put(req, "fw.version", version_str); if (err) return err; - - return 0; + return devlink_info_version_stored_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW, + version_str); } static int mlx5_devlink_reload_fw_activate(struct devlink *devlink, struct netlink_ext_ack *extack) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index b636d63358d2..b1b51bbba054 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -974,7 +974,6 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, int node, struct mlx5e_rq *rq); int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); -void mlx5e_deactivate_rq(struct mlx5e_rq *rq); void mlx5e_close_rq(struct mlx5e_rq *rq); int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param); void mlx5e_destroy_rq(struct mlx5e_rq *rq); @@ -1163,6 +1162,13 @@ mlx5e_calc_max_nch(struct mlx5e_priv *priv, const struct mlx5e_profile *profile) return priv->netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1); } +static inline bool +mlx5e_tx_mpwqe_supported(struct mlx5_core_dev *mdev) +{ + return !is_kdump_kernel() && + MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe); +} + int mlx5e_priv_init(struct mlx5e_priv *priv, struct net_device *netdev, struct mlx5_core_dev *mdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c index 0dd7615e5931..bc33eaada3b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c @@ -64,6 +64,8 @@ struct devlink_port *mlx5e_get_devlink_port(struct net_device *dev) struct mlx5e_priv *priv = netdev_priv(dev); struct devlink_port *port; + if (!netif_device_present(dev)) + return NULL; port = mlx5e_devlink_get_dl_port(priv); if (port->registered) return port; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index f410c1268422..150c8e82c738 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -201,7 +201,7 @@ int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *param static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) { - struct dim_cq_moder moder; + struct dim_cq_moder moder = {}; moder.cq_period_mode = cq_period_mode; moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; @@ -214,7 +214,7 @@ static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) { - struct dim_cq_moder moder; + struct dim_cq_moder moder = {}; moder.cq_period_mode = cq_period_mode; moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; @@ -614,7 +614,7 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, static u8 mlx5e_build_async_icosq_log_wq_sz(struct mlx5_core_dev *mdev) { - if (mlx5_accel_is_ktls_rx(mdev)) + if (mlx5e_accel_is_ktls_rx(mdev)) return MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; @@ -643,7 +643,7 @@ static void mlx5e_build_async_icosq_param(struct mlx5_core_dev *mdev, mlx5e_build_sq_param_common(mdev, param); param->stop_room = mlx5e_stop_room_for_wqe(1); /* for XSK NOP */ - param->is_tls = mlx5_accel_is_ktls_rx(mdev); + param->is_tls = mlx5e_accel_is_ktls_rx(mdev); if (param->is_tls) param->stop_room += mlx5e_stop_room_for_wqe(1); /* for TLS RX resync NOP */ MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(mdev, reg_umr_sq)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index d907c1acd4d5..778e229310a9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2020 Mellanox Technologies -#include <linux/ptp_classify.h> #include "en/ptp.h" #include "en/txrx.h" #include "en/params.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index ab935cce952b..c96668bd701c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -6,6 +6,7 @@ #include "en.h" #include "en_stats.h" +#include <linux/ptp_classify.h> struct mlx5e_ptpsq { struct mlx5e_txqsq txqsq; @@ -43,6 +44,27 @@ struct mlx5e_ptp { DECLARE_BITMAP(state, MLX5E_PTP_STATE_NUM_STATES); }; +static inline bool mlx5e_use_ptpsq(struct sk_buff *skb) +{ + struct flow_keys fk; + + if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + return false; + + if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) + return false; + + if (fk.basic.n_proto == htons(ETH_P_1588)) + return true; + + if (fk.basic.n_proto != htons(ETH_P_IP) && + fk.basic.n_proto != htons(ETH_P_IPV6)) + return false; + + return (fk.basic.ip_proto == IPPROTO_UDP && + fk.ports.dst == htons(PTP_EV_PORT)); +} + int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params, u8 lag_port, struct mlx5e_ptp **cp); void mlx5e_ptp_close(struct mlx5e_ptp *c); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c index 95f2b26a3ee3..9c076aa20306 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bond.c @@ -223,6 +223,8 @@ static void mlx5e_rep_changelowerstate_event(struct net_device *netdev, void *pt rpriv = priv->ppriv; fwd_vport_num = rpriv->rep->vport; lag_dev = netdev_master_upper_dev_get(netdev); + if (!lag_dev) + return; netdev_dbg(netdev, "lag_dev(%s)'s slave vport(%d) is txable(%d)\n", lag_dev->name, fwd_vport_num, net_lag_port_dev_txable(netdev)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c new file mode 100644 index 000000000000..3c0032c9647c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021 Mellanox Technologies. */ + +#include <linux/netdevice.h> +#include <linux/if_bridge.h> +#include <net/netevent.h> +#include <net/switchdev.h> +#include "bridge.h" +#include "esw/bridge.h" +#include "en_rep.h" + +#define MLX5_ESW_BRIDGE_UPDATE_INTERVAL 1000 + +struct mlx5_bridge_switchdev_fdb_work { + struct work_struct work; + struct switchdev_notifier_fdb_info fdb_info; + struct net_device *dev; + bool add; +}; + +static int mlx5_esw_bridge_port_changeupper(struct notifier_block *nb, void *ptr) +{ + struct mlx5_esw_bridge_offloads *br_offloads = container_of(nb, + struct mlx5_esw_bridge_offloads, + netdev_nb); + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct netdev_notifier_changeupper_info *info = ptr; + struct netlink_ext_ack *extack; + struct mlx5e_rep_priv *rpriv; + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + struct net_device *upper; + struct mlx5e_priv *priv; + u16 vport_num; + + if (!mlx5e_eswitch_rep(dev)) + return 0; + + upper = info->upper_dev; + if (!netif_is_bridge_master(upper)) + return 0; + + esw = br_offloads->esw; + priv = netdev_priv(dev); + if (esw != priv->mdev->priv.eswitch) + return 0; + + rpriv = priv->ppriv; + vport_num = rpriv->rep->vport; + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return PTR_ERR(vport); + + extack = netdev_notifier_info_to_extack(&info->info); + + return info->linking ? + mlx5_esw_bridge_vport_link(upper->ifindex, br_offloads, vport, extack) : + mlx5_esw_bridge_vport_unlink(upper->ifindex, br_offloads, vport, extack); +} + +static int mlx5_esw_bridge_switchdev_port_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + int err = 0; + + switch (event) { + case NETDEV_PRECHANGEUPPER: + break; + + case NETDEV_CHANGEUPPER: + err = mlx5_esw_bridge_port_changeupper(nb, ptr); + break; + } + + return notifier_from_errno(err); +} + +static int mlx5_esw_bridge_port_obj_add(struct net_device *dev, + const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack) +{ + const struct switchdev_obj_port_vlan *vlan; + struct mlx5e_rep_priv *rpriv; + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + struct mlx5e_priv *priv; + u16 vport_num; + int err = 0; + + priv = netdev_priv(dev); + rpriv = priv->ppriv; + vport_num = rpriv->rep->vport; + esw = priv->mdev->priv.eswitch; + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return PTR_ERR(vport); + + switch (obj->id) { + case SWITCHDEV_OBJ_ID_PORT_VLAN: + vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); + err = mlx5_esw_bridge_port_vlan_add(vlan->vid, vlan->flags, esw, vport, extack); + break; + default: + return -EOPNOTSUPP; + } + return err; +} + +static int mlx5_esw_bridge_port_obj_del(struct net_device *dev, + const void *ctx, + const struct switchdev_obj *obj) +{ + const struct switchdev_obj_port_vlan *vlan; + struct mlx5e_rep_priv *rpriv; + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + struct mlx5e_priv *priv; + u16 vport_num; + + priv = netdev_priv(dev); + rpriv = priv->ppriv; + vport_num = rpriv->rep->vport; + esw = priv->mdev->priv.eswitch; + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return PTR_ERR(vport); + + switch (obj->id) { + case SWITCHDEV_OBJ_ID_PORT_VLAN: + vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); + mlx5_esw_bridge_port_vlan_del(vlan->vid, esw, vport); + break; + default: + return -EOPNOTSUPP; + } + return 0; +} + +static int mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev, + const void *ctx, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack) +{ + struct mlx5e_rep_priv *rpriv; + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + struct mlx5e_priv *priv; + u16 vport_num; + int err = 0; + + priv = netdev_priv(dev); + rpriv = priv->ppriv; + vport_num = rpriv->rep->vport; + esw = priv->mdev->priv.eswitch; + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + return PTR_ERR(vport); + + switch (attr->id) { + case SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS: + if (attr->u.brport_flags.mask & ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD)) { + NL_SET_ERR_MSG_MOD(extack, "Flag is not supported"); + err = -EINVAL; + } + break; + case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS: + break; + case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME: + err = mlx5_esw_bridge_ageing_time_set(attr->u.ageing_time, esw, vport); + break; + case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING: + err = mlx5_esw_bridge_vlan_filtering_set(attr->u.vlan_filtering, esw, vport); + break; + default: + err = -EOPNOTSUPP; + } + + return err; +} + +static int mlx5_esw_bridge_event_blocking(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + int err; + + switch (event) { + case SWITCHDEV_PORT_OBJ_ADD: + err = switchdev_handle_port_obj_add(dev, ptr, + mlx5e_eswitch_rep, + mlx5_esw_bridge_port_obj_add); + break; + case SWITCHDEV_PORT_OBJ_DEL: + err = switchdev_handle_port_obj_del(dev, ptr, + mlx5e_eswitch_rep, + mlx5_esw_bridge_port_obj_del); + break; + case SWITCHDEV_PORT_ATTR_SET: + err = switchdev_handle_port_attr_set(dev, ptr, + mlx5e_eswitch_rep, + mlx5_esw_bridge_port_obj_attr_set); + break; + default: + err = 0; + } + + return notifier_from_errno(err); +} + +static void +mlx5_esw_bridge_cleanup_switchdev_fdb_work(struct mlx5_bridge_switchdev_fdb_work *fdb_work) +{ + dev_put(fdb_work->dev); + kfree(fdb_work->fdb_info.addr); + kfree(fdb_work); +} + +static void mlx5_esw_bridge_switchdev_fdb_event_work(struct work_struct *work) +{ + struct mlx5_bridge_switchdev_fdb_work *fdb_work = + container_of(work, struct mlx5_bridge_switchdev_fdb_work, work); + struct switchdev_notifier_fdb_info *fdb_info = + &fdb_work->fdb_info; + struct net_device *dev = fdb_work->dev; + struct mlx5e_rep_priv *rpriv; + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + struct mlx5e_priv *priv; + u16 vport_num; + + rtnl_lock(); + + priv = netdev_priv(dev); + rpriv = priv->ppriv; + vport_num = rpriv->rep->vport; + esw = priv->mdev->priv.eswitch; + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) + goto out; + + if (fdb_work->add) + mlx5_esw_bridge_fdb_create(dev, esw, vport, fdb_info); + else + mlx5_esw_bridge_fdb_remove(dev, esw, vport, fdb_info); + +out: + rtnl_unlock(); + mlx5_esw_bridge_cleanup_switchdev_fdb_work(fdb_work); +} + +static struct mlx5_bridge_switchdev_fdb_work * +mlx5_esw_bridge_init_switchdev_fdb_work(struct net_device *dev, bool add, + struct switchdev_notifier_fdb_info *fdb_info) +{ + struct mlx5_bridge_switchdev_fdb_work *work; + u8 *addr; + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return ERR_PTR(-ENOMEM); + + INIT_WORK(&work->work, mlx5_esw_bridge_switchdev_fdb_event_work); + memcpy(&work->fdb_info, fdb_info, sizeof(work->fdb_info)); + + addr = kzalloc(ETH_ALEN, GFP_ATOMIC); + if (!addr) { + kfree(work); + return ERR_PTR(-ENOMEM); + } + ether_addr_copy(addr, fdb_info->addr); + work->fdb_info.addr = addr; + + dev_hold(dev); + work->dev = dev; + work->add = add; + return work; +} + +static int mlx5_esw_bridge_switchdev_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct mlx5_esw_bridge_offloads *br_offloads = container_of(nb, + struct mlx5_esw_bridge_offloads, + nb); + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + struct switchdev_notifier_fdb_info *fdb_info; + struct mlx5_bridge_switchdev_fdb_work *work; + struct switchdev_notifier_info *info = ptr; + struct net_device *upper; + struct mlx5e_priv *priv; + + if (!mlx5e_eswitch_rep(dev)) + return NOTIFY_DONE; + priv = netdev_priv(dev); + if (priv->mdev->priv.eswitch != br_offloads->esw) + return NOTIFY_DONE; + + if (event == SWITCHDEV_PORT_ATTR_SET) { + int err = switchdev_handle_port_attr_set(dev, ptr, + mlx5e_eswitch_rep, + mlx5_esw_bridge_port_obj_attr_set); + return notifier_from_errno(err); + } + + upper = netdev_master_upper_dev_get_rcu(dev); + if (!upper) + return NOTIFY_DONE; + if (!netif_is_bridge_master(upper)) + return NOTIFY_DONE; + + switch (event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + case SWITCHDEV_FDB_DEL_TO_DEVICE: + fdb_info = container_of(info, + struct switchdev_notifier_fdb_info, + info); + + work = mlx5_esw_bridge_init_switchdev_fdb_work(dev, + event == SWITCHDEV_FDB_ADD_TO_DEVICE, + fdb_info); + if (IS_ERR(work)) { + WARN_ONCE(1, "Failed to init switchdev work, err=%ld", + PTR_ERR(work)); + return notifier_from_errno(PTR_ERR(work)); + } + + queue_work(br_offloads->wq, &work->work); + break; + default: + break; + } + return NOTIFY_DONE; +} + +static void mlx5_esw_bridge_update_work(struct work_struct *work) +{ + struct mlx5_esw_bridge_offloads *br_offloads = container_of(work, + struct mlx5_esw_bridge_offloads, + update_work.work); + + rtnl_lock(); + mlx5_esw_bridge_update(br_offloads); + rtnl_unlock(); + + queue_delayed_work(br_offloads->wq, &br_offloads->update_work, + msecs_to_jiffies(MLX5_ESW_BRIDGE_UPDATE_INTERVAL)); +} + +void mlx5e_rep_bridge_init(struct mlx5e_priv *priv) +{ + struct mlx5_esw_bridge_offloads *br_offloads; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_eswitch *esw = + mdev->priv.eswitch; + int err; + + rtnl_lock(); + br_offloads = mlx5_esw_bridge_init(esw); + rtnl_unlock(); + if (IS_ERR(br_offloads)) { + esw_warn(mdev, "Failed to init esw bridge (err=%ld)\n", PTR_ERR(br_offloads)); + return; + } + + br_offloads->wq = alloc_ordered_workqueue("mlx5_bridge_wq", 0); + if (!br_offloads->wq) { + esw_warn(mdev, "Failed to allocate bridge offloads workqueue\n"); + goto err_alloc_wq; + } + INIT_DELAYED_WORK(&br_offloads->update_work, mlx5_esw_bridge_update_work); + queue_delayed_work(br_offloads->wq, &br_offloads->update_work, + msecs_to_jiffies(MLX5_ESW_BRIDGE_UPDATE_INTERVAL)); + + br_offloads->nb.notifier_call = mlx5_esw_bridge_switchdev_event; + err = register_switchdev_notifier(&br_offloads->nb); + if (err) { + esw_warn(mdev, "Failed to register switchdev notifier (err=%d)\n", err); + goto err_register_swdev; + } + + br_offloads->nb_blk.notifier_call = mlx5_esw_bridge_event_blocking; + err = register_switchdev_blocking_notifier(&br_offloads->nb_blk); + if (err) { + esw_warn(mdev, "Failed to register blocking switchdev notifier (err=%d)\n", err); + goto err_register_swdev_blk; + } + + br_offloads->netdev_nb.notifier_call = mlx5_esw_bridge_switchdev_port_event; + err = register_netdevice_notifier(&br_offloads->netdev_nb); + if (err) { + esw_warn(mdev, "Failed to register bridge offloads netdevice notifier (err=%d)\n", + err); + goto err_register_netdev; + } + return; + +err_register_netdev: + unregister_switchdev_blocking_notifier(&br_offloads->nb_blk); +err_register_swdev_blk: + unregister_switchdev_notifier(&br_offloads->nb); +err_register_swdev: + destroy_workqueue(br_offloads->wq); +err_alloc_wq: + mlx5_esw_bridge_cleanup(esw); +} + +void mlx5e_rep_bridge_cleanup(struct mlx5e_priv *priv) +{ + struct mlx5_esw_bridge_offloads *br_offloads; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_eswitch *esw = + mdev->priv.eswitch; + + br_offloads = esw->br_offloads; + if (!br_offloads) + return; + + unregister_netdevice_notifier(&br_offloads->netdev_nb); + unregister_switchdev_blocking_notifier(&br_offloads->nb_blk); + unregister_switchdev_notifier(&br_offloads->nb); + cancel_delayed_work(&br_offloads->update_work); + destroy_workqueue(br_offloads->wq); + rtnl_lock(); + mlx5_esw_bridge_cleanup(esw); + rtnl_unlock(); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.h new file mode 100644 index 000000000000..fbeb64242831 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#ifndef __MLX5_EN_REP_BRIDGE__ +#define __MLX5_EN_REP_BRIDGE__ + +#include "en.h" + +#if IS_ENABLED(CONFIG_MLX5_BRIDGE) + +void mlx5e_rep_bridge_init(struct mlx5e_priv *priv); +void mlx5e_rep_bridge_cleanup(struct mlx5e_priv *priv); + +#else /* CONFIG_MLX5_BRIDGE */ + +static inline void mlx5e_rep_bridge_init(struct mlx5e_priv *priv) {} +static inline void mlx5e_rep_bridge_cleanup(struct mlx5e_priv *priv) {} + +#endif /* CONFIG_MLX5_BRIDGE */ + +#endif /* __MLX5_EN_REP_BRIDGE__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c index be0ee03de721..2e9bee4e5209 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c @@ -129,10 +129,9 @@ static void mlx5e_rep_neigh_update(struct work_struct *work) work); struct mlx5e_neigh_hash_entry *nhe = update_work->nhe; struct neighbour *n = update_work->n; + struct mlx5e_encap_entry *e = NULL; bool neigh_connected, same_dev; - struct mlx5e_encap_entry *e; unsigned char ha[ETH_ALEN]; - struct mlx5e_priv *priv; u8 nud_state, dead; rtnl_lock(); @@ -156,14 +155,12 @@ static void mlx5e_rep_neigh_update(struct work_struct *work) if (!same_dev) goto out; - list_for_each_entry(e, &nhe->encap_list, encap_list) { - if (!mlx5e_encap_take(e)) - continue; + /* mlx5e_get_next_init_encap() releases previous encap before returning + * the next one. + */ + while ((e = mlx5e_get_next_init_encap(nhe, e)) != NULL) + mlx5e_rep_update_flows(netdev_priv(e->out_dev), e, neigh_connected, ha); - priv = netdev_priv(e->out_dev); - mlx5e_rep_update_flows(priv, e, neigh_connected, ha); - mlx5e_encap_put(priv, e); - } out: rtnl_unlock(); mlx5e_release_neigh_update_work(update_work); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index 6cdc52d50a48..059799e4f483 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -94,13 +94,9 @@ void mlx5e_rep_update_flows(struct mlx5e_priv *priv, ASSERT_RTNL(); - /* wait for encap to be fully initialized */ - wait_for_completion(&e->res_ready); - mutex_lock(&esw->offloads.encap_tbl_lock); encap_connected = !!(e->flags & MLX5_ENCAP_ENTRY_VALID); - if (e->compl_result < 0 || (encap_connected == neigh_connected && - ether_addr_equal(e->h_dest, ha))) + if (encap_connected == neigh_connected && ether_addr_equal(e->h_dest, ha)) goto unlock; mlx5e_take_all_encap_flows(e, &flow_list); @@ -617,7 +613,7 @@ static bool mlx5e_restore_skb(struct sk_buff *skb, u32 chain, u32 reg_c1, struct mlx5e_tc_update_priv *tc_priv) { struct mlx5e_priv *priv = netdev_priv(skb->dev); - u32 tunnel_id = reg_c1 >> ESW_TUN_OFFSET; + u32 tunnel_id = (reg_c1 >> ESW_TUN_OFFSET) & TUNNEL_ID_MASK; if (chain) { struct mlx5_rep_uplink_priv *uplink_priv; @@ -626,7 +622,7 @@ static bool mlx5e_restore_skb(struct sk_buff *skb, u32 chain, u32 reg_c1, struct mlx5_eswitch *esw; u32 zone_restore_id; - tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT); + tc_skb_ext = tc_skb_ext_alloc(skb); if (!tc_skb_ext) { WARN_ON(1); return false; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 5da5e5323a44..91e7a01e32be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -23,7 +23,7 @@ #include "en_tc.h" #include "en_rep.h" -#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen * 8) +#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen) #define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0) #define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1) #define MLX5_CT_STATE_TRK_BIT BIT(2) @@ -32,11 +32,11 @@ #define MLX5_CT_STATE_RELATED_BIT BIT(5) #define MLX5_CT_STATE_INVALID_BIT BIT(6) -#define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen * 8) +#define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen) #define MLX5_FTE_ID_MAX GENMASK(MLX5_FTE_ID_BITS - 1, 0) #define MLX5_FTE_ID_MASK MLX5_FTE_ID_MAX -#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen * 8) +#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen) #define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0) #define ct_dbg(fmt, args...)\ @@ -150,6 +150,11 @@ struct mlx5_ct_entry { unsigned long flags; }; +static void +mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, + struct mlx5_flow_attr *attr, + struct mlx5e_mod_hdr_handle *mh); + static const struct rhashtable_params cts_ht_params = { .head_offset = offsetof(struct mlx5_ct_entry, node), .key_offset = offsetof(struct mlx5_ct_entry, cookie), @@ -458,8 +463,7 @@ mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv, ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone); mlx5_tc_rule_delete(netdev_priv(ct_priv->netdev), zone_rule->rule, attr); - mlx5e_mod_hdr_detach(ct_priv->dev, - ct_priv->mod_hdr_tbl, zone_rule->mh); + mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh); mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); kfree(attr); } @@ -686,15 +690,27 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, if (err) goto err_mapping; - *mh = mlx5e_mod_hdr_attach(ct_priv->dev, - ct_priv->mod_hdr_tbl, - ct_priv->ns_type, - &mod_acts); - if (IS_ERR(*mh)) { - err = PTR_ERR(*mh); - goto err_mapping; + if (nat) { + attr->modify_hdr = mlx5_modify_header_alloc(ct_priv->dev, ct_priv->ns_type, + mod_acts.num_actions, + mod_acts.actions); + if (IS_ERR(attr->modify_hdr)) { + err = PTR_ERR(attr->modify_hdr); + goto err_mapping; + } + + *mh = NULL; + } else { + *mh = mlx5e_mod_hdr_attach(ct_priv->dev, + ct_priv->mod_hdr_tbl, + ct_priv->ns_type, + &mod_acts); + if (IS_ERR(*mh)) { + err = PTR_ERR(*mh); + goto err_mapping; + } + attr->modify_hdr = mlx5e_mod_hdr_get(*mh); } - attr->modify_hdr = mlx5e_mod_hdr_get(*mh); dealloc_mod_hdr_actions(&mod_acts); return 0; @@ -705,6 +721,17 @@ err_mapping: return err; } +static void +mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, + struct mlx5_flow_attr *attr, + struct mlx5e_mod_hdr_handle *mh) +{ + if (mh) + mlx5e_mod_hdr_detach(ct_priv->dev, ct_priv->mod_hdr_tbl, mh); + else + mlx5_modify_header_dealloc(ct_priv->dev, attr->modify_hdr); +} + static int mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, struct flow_rule *flow_rule, @@ -767,8 +794,7 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, return 0; err_rule: - mlx5e_mod_hdr_detach(ct_priv->dev, - ct_priv->mod_hdr_tbl, zone_rule->mh); + mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh); mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); err_mod_hdr: kfree(attr); @@ -918,7 +944,7 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, } if (rev_entry && refcount_inc_not_zero(&rev_entry->counter->refcount)) { - ct_dbg("Using shared counter entry=0x%p rev=0x%p\n", entry, rev_entry); + ct_dbg("Using shared counter entry=0x%p rev=0x%p", entry, rev_entry); shared_counter = rev_entry->counter; spin_unlock_bh(&ct_priv->ht_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h index 69e618d17071..644cf1641cde 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h @@ -33,15 +33,15 @@ struct mlx5_ct_attr { #define zone_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_2,\ .moffset = 0,\ - .mlen = 2,\ + .mlen = 16,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ - misc_parameters_2.metadata_reg_c_2) + 2,\ + misc_parameters_2.metadata_reg_c_2),\ } #define ctstate_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_2,\ - .moffset = 2,\ - .mlen = 2,\ + .moffset = 16,\ + .mlen = 16,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_2),\ } @@ -49,7 +49,7 @@ struct mlx5_ct_attr { #define mark_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_3,\ .moffset = 0,\ - .mlen = 4,\ + .mlen = 32,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_3),\ } @@ -57,7 +57,7 @@ struct mlx5_ct_attr { #define labels_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_4,\ .moffset = 0,\ - .mlen = 4,\ + .mlen = 32,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_4),\ } @@ -65,7 +65,7 @@ struct mlx5_ct_attr { #define fteid_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_5,\ .moffset = 0,\ - .mlen = 4,\ + .mlen = 32,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_5),\ } @@ -73,20 +73,19 @@ struct mlx5_ct_attr { #define zone_restore_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,\ .moffset = 0,\ - .mlen = (ESW_ZONE_ID_BITS / 8),\ + .mlen = ESW_ZONE_ID_BITS,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ - misc_parameters_2.metadata_reg_c_1) + 3,\ + misc_parameters_2.metadata_reg_c_1),\ } #define nic_zone_restore_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_B,\ - .moffset = 2,\ - .mlen = (ESW_ZONE_ID_BITS / 8),\ + .moffset = 16,\ + .mlen = ESW_ZONE_ID_BITS,\ } #define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen) #define REG_MAPPING_MOFFSET(reg) (mlx5e_tc_attr_to_reg_mappings[reg].moffset) -#define REG_MAPPING_SHIFT(reg) (REG_MAPPING_MOFFSET(reg) * 8) #if IS_ENABLED(CONFIG_MLX5_TC_CT) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c index 172e0474f2e6..8f79f04eccd6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c @@ -212,6 +212,7 @@ int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv, { int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); const struct ip_tunnel_key *tun_key = &e->tun_info->key; + struct mlx5_pkt_reformat_params reformat_params; struct mlx5e_neigh m_neigh = {}; TC_TUN_ROUTE_ATTR_INIT(attr); int ipv4_encap_size; @@ -295,9 +296,12 @@ int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv, */ goto release_neigh; } - e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, - e->reformat_type, - ipv4_encap_size, encap_header, + + memset(&reformat_params, 0, sizeof(reformat_params)); + reformat_params.type = e->reformat_type; + reformat_params.size = ipv4_encap_size; + reformat_params.data = encap_header; + e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(e->pkt_reformat)) { err = PTR_ERR(e->pkt_reformat); @@ -324,6 +328,7 @@ int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv, { int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); const struct ip_tunnel_key *tun_key = &e->tun_info->key; + struct mlx5_pkt_reformat_params reformat_params; TC_TUN_ROUTE_ATTR_INIT(attr); int ipv4_encap_size; char *encap_header; @@ -396,9 +401,12 @@ int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv, */ goto release_neigh; } - e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, - e->reformat_type, - ipv4_encap_size, encap_header, + + memset(&reformat_params, 0, sizeof(reformat_params)); + reformat_params.type = e->reformat_type; + reformat_params.size = ipv4_encap_size; + reformat_params.data = encap_header; + e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(e->pkt_reformat)) { err = PTR_ERR(e->pkt_reformat); @@ -471,6 +479,7 @@ int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv, { int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); const struct ip_tunnel_key *tun_key = &e->tun_info->key; + struct mlx5_pkt_reformat_params reformat_params; struct mlx5e_neigh m_neigh = {}; TC_TUN_ROUTE_ATTR_INIT(attr); struct ipv6hdr *ip6h; @@ -553,9 +562,11 @@ int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv, goto release_neigh; } - e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, - e->reformat_type, - ipv6_encap_size, encap_header, + memset(&reformat_params, 0, sizeof(reformat_params)); + reformat_params.type = e->reformat_type; + reformat_params.size = ipv6_encap_size; + reformat_params.data = encap_header; + e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(e->pkt_reformat)) { err = PTR_ERR(e->pkt_reformat); @@ -582,6 +593,7 @@ int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv, { int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size); const struct ip_tunnel_key *tun_key = &e->tun_info->key; + struct mlx5_pkt_reformat_params reformat_params; TC_TUN_ROUTE_ATTR_INIT(attr); struct ipv6hdr *ip6h; int ipv6_encap_size; @@ -654,9 +666,11 @@ int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv, goto release_neigh; } - e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, - e->reformat_type, - ipv6_encap_size, encap_header, + memset(&reformat_params, 0, sizeof(reformat_params)); + reformat_params.type = e->reformat_type; + reformat_params.size = ipv6_encap_size; + reformat_params.data = encap_header; + e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(e->pkt_reformat)) { err = PTR_ERR(e->pkt_reformat); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c index 593503bc4d07..2e846b741280 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c @@ -120,6 +120,7 @@ void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv, struct list_head *flow_list) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + struct mlx5_pkt_reformat_params reformat_params; struct mlx5_esw_flow_attr *esw_attr; struct mlx5_flow_handle *rule; struct mlx5_flow_attr *attr; @@ -130,9 +131,12 @@ void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv, if (e->flags & MLX5_ENCAP_ENTRY_NO_ROUTE) return; + memset(&reformat_params, 0, sizeof(reformat_params)); + reformat_params.type = e->reformat_type; + reformat_params.size = e->encap_size; + reformat_params.data = e->encap_header; e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, - e->reformat_type, - e->encap_size, e->encap_header, + &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(e->pkt_reformat)) { mlx5_core_warn(priv->mdev, "Failed to offload cached encapsulation header, %lu\n", @@ -251,9 +255,12 @@ static void mlx5e_take_all_route_decap_flows(struct mlx5e_route_entry *r, mlx5e_take_tmp_flow(flow, flow_list, 0); } +typedef bool (match_cb)(struct mlx5e_encap_entry *); + static struct mlx5e_encap_entry * -mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry *nhe, - struct mlx5e_encap_entry *e) +mlx5e_get_next_matching_encap(struct mlx5e_neigh_hash_entry *nhe, + struct mlx5e_encap_entry *e, + match_cb match) { struct mlx5e_encap_entry *next = NULL; @@ -288,7 +295,7 @@ retry: /* wait for encap to be fully initialized */ wait_for_completion(&next->res_ready); /* continue searching if encap entry is not in valid state after completion */ - if (!(next->flags & MLX5_ENCAP_ENTRY_VALID)) { + if (!match(next)) { e = next; goto retry; } @@ -296,6 +303,30 @@ retry: return next; } +static bool mlx5e_encap_valid(struct mlx5e_encap_entry *e) +{ + return e->flags & MLX5_ENCAP_ENTRY_VALID; +} + +static struct mlx5e_encap_entry * +mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry *nhe, + struct mlx5e_encap_entry *e) +{ + return mlx5e_get_next_matching_encap(nhe, e, mlx5e_encap_valid); +} + +static bool mlx5e_encap_initialized(struct mlx5e_encap_entry *e) +{ + return e->compl_result >= 0; +} + +struct mlx5e_encap_entry * +mlx5e_get_next_init_encap(struct mlx5e_neigh_hash_entry *nhe, + struct mlx5e_encap_entry *e) +{ + return mlx5e_get_next_matching_encap(nhe, e, mlx5e_encap_initialized); +} + void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe) { struct mlx5e_neigh *m_neigh = &nhe->m_neigh; @@ -812,6 +843,7 @@ int mlx5e_attach_decap(struct mlx5e_priv *priv, { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr; + struct mlx5_pkt_reformat_params reformat_params; struct mlx5e_tc_flow_parse_attr *parse_attr; struct mlx5e_decap_entry *d; struct mlx5e_decap_key key; @@ -853,10 +885,12 @@ int mlx5e_attach_decap(struct mlx5e_priv *priv, hash_add_rcu(esw->offloads.decap_tbl, &d->hlist, hash_key); mutex_unlock(&esw->offloads.decap_tbl_lock); + memset(&reformat_params, 0, sizeof(reformat_params)); + reformat_params.type = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2; + reformat_params.size = sizeof(parse_attr->eth); + reformat_params.data = &parse_attr->eth; d->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, - MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2, - sizeof(parse_attr->eth), - &parse_attr->eth, + &reformat_params, MLX5_FLOW_NAMESPACE_FDB); if (IS_ERR(d->pkt_reformat)) { err = PTR_ERR(d->pkt_reformat); @@ -1505,7 +1539,7 @@ mlx5e_init_fib_work_ipv4(struct mlx5e_priv *priv, fen_info = container_of(info, struct fib_entry_notifier_info, info); fib_dev = fib_info_nh(fen_info->fi, 0)->fib_nh_dev; - if (fib_dev->netdev_ops != &mlx5e_netdev_ops || + if (!fib_dev || fib_dev->netdev_ops != &mlx5e_netdev_ops || fen_info->dst_len != 32) return NULL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 00af0b831a28..d964665eaa63 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -162,7 +162,7 @@ static inline unsigned int mlx5e_accel_tx_ids_len(struct mlx5e_txqsq *sq, /* Part of the eseg touched by TX offloads */ #define MLX5E_ACCEL_ESEG_LEN offsetof(struct mlx5_wqe_eth_seg, mss) -static inline bool mlx5e_accel_tx_eseg(struct mlx5e_priv *priv, +static inline void mlx5e_accel_tx_eseg(struct mlx5e_priv *priv, struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { @@ -175,8 +175,6 @@ static inline bool mlx5e_accel_tx_eseg(struct mlx5e_priv *priv, if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) mlx5e_tx_tunnel_accel(skb, eseg, ihs); #endif - - return true; } static inline void mlx5e_accel_tx_finish(struct mlx5e_txqsq *sq, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 3d45341e2216..7cab08a2f715 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -428,7 +428,6 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv) spin_lock_init(&ipsec->sadb_rx_lock); ida_init(&ipsec->halloc); ipsec->en_priv = priv; - ipsec->en_priv->ipsec = ipsec; ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) & MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER); ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0, @@ -438,6 +437,7 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv) return -ENOMEM; } + priv->ipsec = ipsec; mlx5e_accel_ipsec_fs_init(priv); netdev_dbg(priv->netdev, "IPSec attached to netdevice\n"); return 0; @@ -532,9 +532,6 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) struct mlx5_core_dev *mdev = priv->mdev; struct net_device *netdev = priv->netdev; - if (!priv->ipsec) - return; - if (!(mlx5_accel_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) || !MLX5_CAP_ETH(mdev, swp)) { mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n"); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c index a97e8d205094..33de8f0092a6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c @@ -136,8 +136,6 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, u8 mode, struct xfrm_offload *xo) { - struct mlx5e_swp_spec swp_spec = {}; - /* Tunnel Mode: * SWP: OutL3 InL3 InL4 * Pkt: MAC IP ESP IP L4 @@ -146,23 +144,58 @@ static void mlx5e_ipsec_set_swp(struct sk_buff *skb, * SWP: OutL3 InL4 * InL3 * Pkt: MAC IP ESP L4 + * + * Tunnel(VXLAN TCP/UDP) over Transport Mode + * SWP: OutL3 InL3 InL4 + * Pkt: MAC IP ESP UDP VXLAN IP L4 */ - swp_spec.l3_proto = skb->protocol; - swp_spec.is_tun = mode == XFRM_MODE_TUNNEL; - if (swp_spec.is_tun) { - if (xo->proto == IPPROTO_IPV6) { - swp_spec.tun_l3_proto = htons(ETH_P_IPV6); - swp_spec.tun_l4_proto = inner_ipv6_hdr(skb)->nexthdr; - } else { - swp_spec.tun_l3_proto = htons(ETH_P_IP); - swp_spec.tun_l4_proto = inner_ip_hdr(skb)->protocol; - } - } else { - swp_spec.tun_l3_proto = skb->protocol; - swp_spec.tun_l4_proto = xo->proto; + + /* Shared settings */ + eseg->swp_outer_l3_offset = skb_network_offset(skb) / 2; + if (skb->protocol == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_OUTER_L3_IPV6; + + /* Tunnel mode */ + if (mode == XFRM_MODE_TUNNEL) { + eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; + eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; + if (xo->proto == IPPROTO_IPV6) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; + if (inner_ip_hdr(skb)->protocol == IPPROTO_UDP) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; + return; + } + + /* Transport mode */ + if (mode != XFRM_MODE_TRANSPORT) + return; + + if (!xo->inner_ipproto) { + eseg->swp_inner_l3_offset = skb_network_offset(skb) / 2; + eseg->swp_inner_l4_offset = skb_inner_transport_offset(skb) / 2; + if (skb->protocol == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; + if (xo->proto == IPPROTO_UDP) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; + return; + } + + /* Tunnel(VXLAN TCP/UDP) over Transport Mode */ + switch (xo->inner_ipproto) { + case IPPROTO_UDP: + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L4_UDP; + fallthrough; + case IPPROTO_TCP: + eseg->swp_inner_l3_offset = skb_inner_network_offset(skb) / 2; + eseg->swp_inner_l4_offset = (skb->csum_start + skb->head - skb->data) / 2; + if (skb->protocol == htons(ETH_P_IPV6)) + eseg->swp_flags |= MLX5_ETH_WQE_SWP_INNER_L3_IPV6; + break; + default: + break; } - mlx5e_set_eseg_swp(skb, eseg, &swp_spec); + return; } void mlx5e_ipsec_set_iv_esn(struct sk_buff *skb, struct xfrm_state *x, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index 3e80742a3caf..5120a59361e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -93,18 +93,38 @@ static inline bool mlx5e_ipsec_eseg_meta(struct mlx5_wqe_eth_seg *eseg) void mlx5e_ipsec_tx_build_eseg(struct mlx5e_priv *priv, struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg); -static inline bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev, - netdev_features_t features) +static inline netdev_features_t +mlx5e_ipsec_feature_check(struct sk_buff *skb, netdev_features_t features) { + struct xfrm_offload *xo = xfrm_offload(skb); struct sec_path *sp = skb_sec_path(skb); - if (sp && sp->len) { + if (sp && sp->len && xo) { struct xfrm_state *x = sp->xvec[0]; - if (x && x->xso.offload_handle) - return true; + if (!x || !x->xso.offload_handle) + goto out_disable; + + if (xo->inner_ipproto) { + /* Cannot support tunnel packet over IPsec tunnel mode + * because we cannot offload three IP header csum + */ + if (x->props.mode == XFRM_MODE_TUNNEL) + goto out_disable; + + /* Only support UDP or TCP L4 checksum */ + if (xo->inner_ipproto != IPPROTO_UDP && + xo->inner_ipproto != IPPROTO_TCP) + goto out_disable; + } + + return features; + } - return false; + + /* Disable CSUM and GSO for software IPsec */ +out_disable: + return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); } #else @@ -120,8 +140,9 @@ static inline bool mlx5e_ipsec_eseg_meta(struct mlx5_wqe_eth_seg *eseg) } static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false; } -static inline bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev, - netdev_features_t features) { return false; } +static inline netdev_features_t +mlx5e_ipsec_feature_check(struct sk_buff *skb, netdev_features_t features) +{ return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); } #endif /* CONFIG_MLX5_EN_IPSEC */ #endif /* __MLX5E_IPSEC_RXTX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 95293ee0d38d..d93aadbf10da 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -59,12 +59,15 @@ void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; - if (mlx5_accel_is_ktls_tx(mdev)) { + if (!mlx5e_accel_is_ktls_tx(mdev) && !mlx5e_accel_is_ktls_rx(mdev)) + return; + + if (mlx5e_accel_is_ktls_tx(mdev)) { netdev->hw_features |= NETIF_F_HW_TLS_TX; netdev->features |= NETIF_F_HW_TLS_TX; } - if (mlx5_accel_is_ktls_rx(mdev)) + if (mlx5e_accel_is_ktls_rx(mdev)) netdev->hw_features |= NETIF_F_HW_TLS_RX; netdev->tlsdev_ops = &mlx5e_ktls_ops; @@ -89,7 +92,7 @@ int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) { int err; - if (!mlx5_accel_is_ktls_rx(priv->mdev)) + if (!mlx5e_accel_is_ktls_rx(priv->mdev)) return 0; priv->tls->rx_wq = create_singlethread_workqueue("mlx5e_tls_rx"); @@ -109,7 +112,7 @@ int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv) { - if (!mlx5_accel_is_ktls_rx(priv->mdev)) + if (!mlx5e_accel_is_ktls_rx(priv->mdev)) return; if (priv->netdev->features & NETIF_F_HW_TLS_RX) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index aaa579bf9a39..5833deb2354c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -15,6 +15,25 @@ int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable); struct mlx5e_ktls_resync_resp * mlx5e_ktls_rx_resync_create_resp_list(void); void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_list); + +static inline bool mlx5e_accel_is_ktls_tx(struct mlx5_core_dev *mdev) +{ + return !is_kdump_kernel() && + mlx5_accel_is_ktls_tx(mdev); +} + +static inline bool mlx5e_accel_is_ktls_rx(struct mlx5_core_dev *mdev) +{ + return !is_kdump_kernel() && + mlx5_accel_is_ktls_rx(mdev); +} + +static inline bool mlx5e_accel_is_ktls_device(struct mlx5_core_dev *mdev) +{ + return !is_kdump_kernel() && + mlx5_accel_is_ktls_device(mdev); +} + #else static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) @@ -44,6 +63,11 @@ mlx5e_ktls_rx_resync_create_resp_list(void) static inline void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_list) {} + +static inline bool mlx5e_accel_is_ktls_tx(struct mlx5_core_dev *mdev) { return false; } +static inline bool mlx5e_accel_is_ktls_rx(struct mlx5_core_dev *mdev) { return false; } +static inline bool mlx5e_accel_is_ktls_device(struct mlx5_core_dev *mdev) { return false; } + #endif #endif /* __MLX5E_TLS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 51bdf71073f3..9ad3459fb63a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -23,10 +23,13 @@ mlx5e_ktls_dumps_num_wqes(struct mlx5e_params *params, unsigned int nfrags, return nfrags + DIV_ROUND_UP(sync_len, MLX5E_SW2HW_MTU(params, params->sw_mtu)); } -u16 mlx5e_ktls_get_stop_room(struct mlx5e_params *params) +u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { u16 num_dumps, stop_room = 0; + if (!mlx5e_accel_is_ktls_tx(mdev)) + return 0; + num_dumps = mlx5e_ktls_dumps_num_wqes(params, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); @@ -135,6 +138,7 @@ void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) priv = netdev_priv(netdev); mdev = priv->mdev; + atomic64_inc(&priv_tx->sw_stats->tx_tls_del); mlx5e_destroy_tis(mdev, priv_tx->tisn); mlx5_ktls_destroy_key(mdev, priv_tx->key_id); kfree(priv_tx); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h index 8f79335057dc..08c9d5134479 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h @@ -14,7 +14,7 @@ struct mlx5e_accel_tx_tls_state { u32 tls_tisn; }; -u16 mlx5e_ktls_get_stop_room(struct mlx5e_params *params); +u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params); bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq, struct sk_buff *skb, int datalen, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c index d6b21b899dbc..b8fc863aa68d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.c @@ -192,13 +192,13 @@ void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) struct net_device *netdev = priv->netdev; u32 caps; - if (mlx5_accel_is_ktls_device(priv->mdev)) { + if (mlx5e_accel_is_ktls_device(priv->mdev)) { mlx5e_ktls_build_netdev(priv); return; } /* FPGA */ - if (!mlx5_accel_is_tls_device(priv->mdev)) + if (!mlx5e_accel_is_tls_device(priv->mdev)) return; caps = mlx5_accel_tls_device_caps(priv->mdev); @@ -224,7 +224,7 @@ int mlx5e_tls_init(struct mlx5e_priv *priv) { struct mlx5e_tls *tls; - if (!mlx5_accel_is_tls_device(priv->mdev)) + if (!mlx5e_accel_is_tls_device(priv->mdev)) return 0; tls = kzalloc(sizeof(*tls), GFP_KERNEL); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h index 4c9274d390da..62ecf14bf86a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls.h @@ -42,6 +42,7 @@ struct mlx5e_tls_sw_stats { atomic64_t tx_tls_ctx; + atomic64_t tx_tls_del; atomic64_t tx_tls_drop_metadata; atomic64_t tx_tls_drop_resync_alloc; atomic64_t tx_tls_drop_no_sync_data; @@ -103,11 +104,18 @@ int mlx5e_tls_get_count(struct mlx5e_priv *priv); int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data); int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data); +static inline bool mlx5e_accel_is_tls_device(struct mlx5_core_dev *mdev) +{ + return !is_kdump_kernel() && + mlx5_accel_is_tls_device(mdev); +} + #else static inline void mlx5e_tls_build_netdev(struct mlx5e_priv *priv) { - if (mlx5_accel_is_ktls_device(priv->mdev)) + if (!is_kdump_kernel() && + mlx5_accel_is_ktls_device(priv->mdev)) mlx5e_ktls_build_netdev(priv); } @@ -117,6 +125,7 @@ static inline void mlx5e_tls_cleanup(struct mlx5e_priv *priv) { } static inline int mlx5e_tls_get_count(struct mlx5e_priv *priv) { return 0; } static inline int mlx5e_tls_get_strings(struct mlx5e_priv *priv, uint8_t *data) { return 0; } static inline int mlx5e_tls_get_stats(struct mlx5e_priv *priv, u64 *data) { return 0; } +static inline bool mlx5e_accel_is_tls_device(struct mlx5_core_dev *mdev) { return false; } #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c index 82dc09aaa7fc..7a700f913582 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_rxtx.c @@ -273,7 +273,7 @@ bool mlx5e_tls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, if (WARN_ON_ONCE(tls_ctx->netdev != netdev)) goto err_out; - if (mlx5_accel_is_ktls_tx(sq->mdev)) + if (mlx5e_accel_is_ktls_tx(sq->mdev)) return mlx5e_ktls_handle_tx_skb(tls_ctx, sq, skb, datalen, state); /* FPGA */ @@ -378,11 +378,11 @@ void mlx5e_tls_handle_rx_skb_metadata(struct mlx5e_rq *rq, struct sk_buff *skb, u16 mlx5e_tls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { - if (!mlx5_accel_is_tls_device(mdev)) + if (!mlx5e_accel_is_tls_device(mdev)) return 0; - if (mlx5_accel_is_ktls_device(mdev)) - return mlx5e_ktls_get_stop_room(params); + if (mlx5e_accel_is_ktls_device(mdev)) + return mlx5e_ktls_get_stop_room(mdev, params); /* FPGA */ /* Resync SKB. */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c index 29463bdb7715..56e7b2aee85f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/tls_stats.c @@ -47,6 +47,7 @@ static const struct counter_desc mlx5e_tls_sw_stats_desc[] = { static const struct counter_desc mlx5e_ktls_sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_ctx) }, + { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_del) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, rx_tls_ctx) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, rx_tls_del) }, }; @@ -58,7 +59,7 @@ static const struct counter_desc *get_tls_atomic_stats(struct mlx5e_priv *priv) { if (!priv->tls) return NULL; - if (mlx5_accel_is_ktls_device(priv->mdev)) + if (mlx5e_accel_is_ktls_device(priv->mdev)) return mlx5e_ktls_sw_stats_desc; return mlx5e_tls_sw_stats_desc; } @@ -67,7 +68,7 @@ int mlx5e_tls_get_count(struct mlx5e_priv *priv) { if (!priv->tls) return 0; - if (mlx5_accel_is_ktls_device(priv->mdev)) + if (mlx5e_accel_is_ktls_device(priv->mdev)) return ARRAY_SIZE(mlx5e_ktls_sw_stats_desc); return ARRAY_SIZE(mlx5e_tls_sw_stats_desc); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index 5cd466ec6492..25403af32859 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -356,7 +356,7 @@ err: int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) { - int err = 0; + int err = -ENOMEM; int i; if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 8360289813f0..bd72572e03d1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -1624,12 +1624,13 @@ static int mlx5e_set_fecparam(struct net_device *netdev, { struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5_core_dev *mdev = priv->mdev; + unsigned long fec_bitmap; u16 fec_policy = 0; int mode; int err; - if (bitmap_weight((unsigned long *)&fecparam->fec, - ETHTOOL_FEC_LLRS_BIT + 1) > 1) + bitmap_from_arr32(&fec_bitmap, &fecparam->fec, sizeof(fecparam->fec) * BITS_PER_BYTE); + if (bitmap_weight(&fec_bitmap, ETHTOOL_FEC_LLRS_BIT + 1) > 1) return -EOPNOTSUPP; for (mode = 0; mode < ARRAY_SIZE(pplm_fec_2_ethtool); mode++) { @@ -1893,6 +1894,13 @@ int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool new_val if (curr_val == new_val) return 0; + if (new_val && !priv->profile->rx_ptp_support && + priv->tstamp.rx_filter != HWTSTAMP_FILTER_NONE) { + netdev_err(priv->netdev, + "Profile doesn't support enabling of CQE compression while hardware time-stamping is enabled.\n"); + return -EINVAL; + } + new_params = priv->channels.params; MLX5E_SET_PFLAG(&new_params, MLX5E_PFLAG_RX_CQE_COMPRESS, new_val); if (priv->tstamp.rx_filter != HWTSTAMP_FILTER_NONE) @@ -1984,7 +1992,7 @@ static int set_pflag_tx_mpwqe_common(struct net_device *netdev, u32 flag, bool e struct mlx5_core_dev *mdev = priv->mdev; struct mlx5e_params new_params; - if (enable && !MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe)) + if (enable && !mlx5e_tx_mpwqe_supported(mdev)) return -EOPNOTSUPP; new_params = priv->channels.params; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 0d571a0c76d9..0b75fab41ae8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -35,6 +35,7 @@ #include <linux/ipv6.h> #include <linux/tcp.h> #include <linux/mlx5/fs.h> +#include <linux/mlx5/mpfs.h> #include "en.h" #include "en_rep.h" #include "lib/mpfs.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index bca832cdc4cb..d09e65557e75 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -91,12 +91,16 @@ void mlx5e_update_carrier(struct mlx5e_priv *priv) { struct mlx5_core_dev *mdev = priv->mdev; u8 port_state; + bool up; port_state = mlx5_query_vport_state(mdev, MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 0); - if (port_state == VPORT_STATE_UP) { + up = port_state == VPORT_STATE_UP; + if (up == netif_carrier_ok(priv->netdev)) + netif_carrier_event(priv->netdev); + if (up) { netdev_info(priv->netdev, "Link up\n"); netif_carrier_on(priv->netdev); } else { @@ -644,8 +648,8 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) return -ENOMEM; ts_format = mlx5_is_real_time_rq(mdev) ? - MLX5_RQC_TIMESTAMP_FORMAT_REAL_TIME : - MLX5_RQC_TIMESTAMP_FORMAT_FREE_RUNNING; + MLX5_TIMESTAMP_FORMAT_REAL_TIME : + MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); wq = MLX5_ADDR_OF(rqc, rqc, wq); @@ -853,7 +857,7 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, if (err) goto err_destroy_rq; - if (mlx5e_is_tls_on(rq->priv) && !mlx5_accel_is_ktls_device(mdev)) + if (mlx5e_is_tls_on(rq->priv) && !mlx5e_accel_is_ktls_device(mdev)) __set_bit(MLX5E_RQ_STATE_FPGA_TLS, &rq->state); /* must be FPGA */ if (MLX5_CAP_ETH(mdev, cqe_checksum_full)) @@ -889,10 +893,13 @@ err_free_rq: void mlx5e_activate_rq(struct mlx5e_rq *rq) { set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); - if (rq->icosq) + if (rq->icosq) { mlx5e_trigger_irq(rq->icosq); - else + } else { + local_bh_disable(); napi_schedule(rq->cq.napi); + local_bh_enable(); + } } void mlx5e_deactivate_rq(struct mlx5e_rq *rq) @@ -1185,8 +1192,8 @@ static int mlx5e_create_sq(struct mlx5_core_dev *mdev, return -ENOMEM; ts_format = mlx5_is_real_time_sq(mdev) ? - MLX5_SQC_TIMESTAMP_FORMAT_REAL_TIME : - MLX5_SQC_TIMESTAMP_FORMAT_FREE_RUNNING; + MLX5_TIMESTAMP_FORMAT_REAL_TIME : + MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); wq = MLX5_ADDR_OF(sqc, sqc, wq); @@ -2697,13 +2704,11 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) int err; old_num_txqs = netdev->real_num_tx_queues; - old_ntc = netdev->num_tc; + old_ntc = netdev->num_tc ? : 1; nch = priv->channels.params.num_channels; ntc = priv->channels.params.num_tc; num_rxqs = nch * priv->profile->rq_groups; - if (priv->channels.params.ptp_rx) - num_rxqs++; mlx5e_netdev_set_tcs(netdev, nch, ntc); @@ -3855,6 +3860,16 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); } + if (mlx5e_is_uplink_rep(priv)) { + features &= ~NETIF_F_HW_TLS_RX; + if (netdev->features & NETIF_F_HW_TLS_RX) + netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n"); + + features &= ~NETIF_F_HW_TLS_TX; + if (netdev->features & NETIF_F_HW_TLS_TX) + netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n"); + } + mutex_unlock(&priv->state_lock); return features; @@ -3971,11 +3986,45 @@ int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx) return mlx5e_ptp_rx_manage_fs(priv, set); } -int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) +static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter) +{ + bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; + int err; + + if (!rx_filter) + /* Reset CQE compression to Admin default */ + return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def); + + if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) + return 0; + + /* Disable CQE compression */ + netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); + err = mlx5e_modify_rx_cqe_compression_locked(priv, false); + if (err) + netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); + + return err; +} + +static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx) { struct mlx5e_params new_params; + + if (ptp_rx == priv->channels.params.ptp_rx) + return 0; + + new_params = priv->channels.params; + new_params.ptp_rx = ptp_rx; + return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, + &new_params.ptp_rx, true); +} + +int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) +{ struct hwtstamp_config config; bool rx_cqe_compress_def; + bool ptp_rx; int err; if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || @@ -3995,13 +4044,12 @@ int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) } mutex_lock(&priv->state_lock); - new_params = priv->channels.params; rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; /* RX HW timestamp */ switch (config.rx_filter) { case HWTSTAMP_FILTER_NONE: - new_params.ptp_rx = false; + ptp_rx = false; break; case HWTSTAMP_FILTER_ALL: case HWTSTAMP_FILTER_SOME: @@ -4018,24 +4066,25 @@ int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: case HWTSTAMP_FILTER_NTP_ALL: - new_params.ptp_rx = rx_cqe_compress_def; config.rx_filter = HWTSTAMP_FILTER_ALL; + /* ptp_rx is set if both HW TS is set and CQE + * compression is set + */ + ptp_rx = rx_cqe_compress_def; break; default: - mutex_unlock(&priv->state_lock); - return -ERANGE; + err = -ERANGE; + goto err_unlock; } - if (new_params.ptp_rx == priv->channels.params.ptp_rx) - goto out; + if (!priv->profile->rx_ptp_support) + err = mlx5e_hwstamp_config_no_ptp_rx(priv, + config.rx_filter != HWTSTAMP_FILTER_NONE); + else + err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx); + if (err) + goto err_unlock; - err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, - &new_params.ptp_rx, true); - if (err) { - mutex_unlock(&priv->state_lock); - return err; - } -out: memcpy(&priv->tstamp, &config, sizeof(config)); mutex_unlock(&priv->state_lock); @@ -4044,6 +4093,9 @@ out: return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; +err_unlock: + mutex_unlock(&priv->state_lock); + return err; } int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr) @@ -4279,6 +4331,11 @@ static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) return features; #endif + break; +#ifdef CONFIG_MLX5_EN_IPSEC + case IPPROTO_ESP: + return mlx5e_ipsec_feature_check(skb, features); +#endif } out: @@ -4295,9 +4352,6 @@ netdev_features_t mlx5e_features_check(struct sk_buff *skb, features = vlan_features_check(skb, features); features = vxlan_features_check(skb, features); - if (mlx5e_ipsec_feature_check(skb, netdev, features)) - return features; - /* Validate if the tunneled packet is being offloaded by HW */ if (skb->encapsulation && (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) @@ -4613,12 +4667,10 @@ void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 params->log_sq_size = is_kdump_kernel() ? MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; - MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, - MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe)); + MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); /* XDP SQ */ - MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, - MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe)); + MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); /* set CQE compression */ params->rx_cqe_compress_def = false; @@ -4774,22 +4826,15 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) } if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { - netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM; - netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM; - netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; - netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | - NETIF_F_GSO_UDP_TUNNEL_CSUM; + netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; + netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL; + netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL; } if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) { - netdev->hw_features |= NETIF_F_GSO_GRE | - NETIF_F_GSO_GRE_CSUM; - netdev->hw_enc_features |= NETIF_F_GSO_GRE | - NETIF_F_GSO_GRE_CSUM; - netdev->gso_partial_features |= NETIF_F_GSO_GRE | - NETIF_F_GSO_GRE_CSUM; + netdev->hw_features |= NETIF_F_GSO_GRE; + netdev->hw_enc_features |= NETIF_F_GSO_GRE; + netdev->gso_partial_features |= NETIF_F_GSO_GRE; } if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) { @@ -5062,7 +5107,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) mlx5e_set_netdev_mtu_boundaries(priv); mlx5e_set_dev_port_mtu(priv); - mlx5_lag_add(mdev, netdev); + mlx5_lag_add_netdev(mdev, netdev); mlx5e_enable_async_events(priv); mlx5e_enable_blocking_events(priv); @@ -5110,7 +5155,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv) priv->en_trap = NULL; } mlx5e_disable_async_events(priv); - mlx5_lag_remove(mdev); + mlx5_lag_remove_netdev(mdev, priv->netdev); mlx5_vxlan_reset_to_default(mdev->vxlan); } @@ -5229,6 +5274,11 @@ static void mlx5e_update_features(struct net_device *netdev) rtnl_unlock(); } +static void mlx5e_reset_channels(struct net_device *netdev) +{ + netdev_reset_tc(netdev); +} + int mlx5e_attach_netdev(struct mlx5e_priv *priv) { const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED; @@ -5283,6 +5333,7 @@ err_cleanup_tx: profile->cleanup_tx(priv); out: + mlx5e_reset_channels(priv->netdev); set_bit(MLX5E_STATE_DESTROYING, &priv->state); cancel_work_sync(&priv->update_stats_work); return err; @@ -5300,6 +5351,7 @@ void mlx5e_detach_netdev(struct mlx5e_priv *priv) profile->cleanup_rx(priv); profile->cleanup_tx(priv); + mlx5e_reset_channels(priv->netdev); cancel_work_sync(&priv->update_stats_work); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 34eb1118670f..bf94bcb6fa5d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -45,11 +45,13 @@ #include "en_tc.h" #include "en/rep/tc.h" #include "en/rep/neigh.h" +#include "en/rep/bridge.h" #include "en/devlink.h" #include "fs_core.h" #include "lib/mlx5.h" #define CREATE_TRACE_POINTS #include "diag/en_rep_tracepoint.h" +#include "en_accel/ipsec.h" #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \ max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE) @@ -536,13 +538,13 @@ static const struct net_device_ops mlx5e_netdev_ops_rep = { .ndo_change_carrier = mlx5e_rep_change_carrier, }; -bool mlx5e_eswitch_uplink_rep(struct net_device *netdev) +bool mlx5e_eswitch_uplink_rep(const struct net_device *netdev) { return netdev->netdev_ops == &mlx5e_netdev_ops && mlx5e_is_uplink_rep(netdev_priv(netdev)); } -bool mlx5e_eswitch_vf_rep(struct net_device *netdev) +bool mlx5e_eswitch_vf_rep(const struct net_device *netdev) { return netdev->netdev_ops == &mlx5e_netdev_ops_rep; } @@ -629,6 +631,11 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, struct net_device *netdev) { struct mlx5e_priv *priv = netdev_priv(netdev); + int err; + + err = mlx5e_ipsec_init(priv); + if (err) + mlx5_core_err(mdev, "Uplink rep IPsec initialization failed, %d\n", err); mlx5e_vxlan_set_netdev_info(priv); return mlx5e_init_rep(mdev, netdev); @@ -636,6 +643,7 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, static void mlx5e_cleanup_rep(struct mlx5e_priv *priv) { + mlx5e_ipsec_cleanup(priv); } static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) @@ -975,12 +983,13 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) if (MLX5_CAP_GEN(mdev, uplink_follow)) mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); - mlx5_lag_add(mdev, netdev); + mlx5_lag_add_netdev(mdev, netdev); priv->events_nb.notifier_call = uplink_rep_async_event; mlx5_notifier_register(mdev, &priv->events_nb); mlx5e_dcbnl_initialize(priv); mlx5e_dcbnl_init_app(priv); mlx5e_rep_neigh_init(rpriv); + mlx5e_rep_bridge_init(priv); netdev->wanted_features |= NETIF_F_HW_TC; @@ -1002,11 +1011,12 @@ static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv) netif_device_detach(priv->netdev); rtnl_unlock(); + mlx5e_rep_bridge_cleanup(priv); mlx5e_rep_neigh_cleanup(rpriv); mlx5e_dcbnl_delete_app(priv); mlx5_notifier_unregister(mdev, &priv->events_nb); mlx5e_rep_tc_disable(priv); - mlx5_lag_remove(mdev); + mlx5_lag_remove_netdev(mdev, priv->netdev); } static MLX5E_DEFINE_STATS_GRP(sw_rep, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index 22585015c7a7..47a2dfb7792a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -231,9 +231,9 @@ void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv); void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv); -bool mlx5e_eswitch_vf_rep(struct net_device *netdev); -bool mlx5e_eswitch_uplink_rep(struct net_device *netdev); -static inline bool mlx5e_eswitch_rep(struct net_device *netdev) +bool mlx5e_eswitch_vf_rep(const struct net_device *netdev); +bool mlx5e_eswitch_uplink_rep(const struct net_device *netdev); +static inline bool mlx5e_eswitch_rep(const struct net_device *netdev) { return mlx5e_eswitch_vf_rep(netdev) || mlx5e_eswitch_uplink_rep(netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index f90894eea9e0..3c65fd0bcf31 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -579,6 +579,9 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) if (mlx5_wq_cyc_missing(wq) < wqe_bulk) return false; + if (rq->page_pool) + page_pool_nid_changed(rq->page_pool, numa_mem_id()); + do { u16 head = mlx5_wq_cyc_get_head(wq); @@ -734,6 +737,9 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq) if (likely(missing < UMR_WQE_BULK)) return false; + if (rq->page_pool) + page_pool_nid_changed(rq->page_pool, numa_mem_id()); + head = rq->mpwqe.actual_wq_head; i = missing; do { @@ -1310,7 +1316,8 @@ static void mlx5e_handle_rx_cqe_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) if (rep->vlan && skb_vlan_tag_present(skb)) skb_vlan_pop(skb); - if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv)) { + if (unlikely(!mlx5_ipsec_is_rx_flow(cqe) && + !mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))) { dev_kfree_skb_any(skb); goto free_wqe; } @@ -1367,7 +1374,8 @@ static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb); - if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv)) { + if (unlikely(!mlx5_ipsec_is_rx_flow(cqe) && + !mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))) { dev_kfree_skb_any(skb); goto mpwrq_cqe_out; } @@ -1553,12 +1561,9 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) return 0; - if (rq->page_pool) - page_pool_nid_changed(rq->page_pool, numa_mem_id()); - if (rq->cqd.left) { work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget); - if (rq->cqd.left || work_done >= budget) + if (work_done >= budget) goto out; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 47a9c49b25fd..629a61e8022f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -83,17 +83,17 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { [CHAIN_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0, .moffset = 0, - .mlen = 2, + .mlen = 16, }, [VPORT_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0, - .moffset = 2, - .mlen = 2, + .moffset = 16, + .mlen = 16, }, [TUNNEL_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1, - .moffset = 1, - .mlen = ((ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS) / 8), + .moffset = 8, + .mlen = ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS, .soffset = MLX5_BYTE_OFF(fte_match_param, misc_parameters_2.metadata_reg_c_1), }, @@ -110,7 +110,7 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { [NIC_CHAIN_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_B, .moffset = 0, - .mlen = 2, + .mlen = 16, }, [NIC_ZONE_RESTORE_TO_REG] = nic_zone_restore_to_reg_ct, }; @@ -128,23 +128,46 @@ static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow); void mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec, enum mlx5e_tc_attr_to_reg type, - u32 data, + u32 val, u32 mask) { + void *headers_c = spec->match_criteria, *headers_v = spec->match_value, *fmask, *fval; int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset; + int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset; int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen; - void *headers_c = spec->match_criteria; - void *headers_v = spec->match_value; - void *fmask, *fval; + u32 max_mask = GENMASK(match_len - 1, 0); + __be32 curr_mask_be, curr_val_be; + u32 curr_mask, curr_val; fmask = headers_c + soffset; fval = headers_v + soffset; - mask = (__force u32)(cpu_to_be32(mask)) >> (32 - (match_len * 8)); - data = (__force u32)(cpu_to_be32(data)) >> (32 - (match_len * 8)); + memcpy(&curr_mask_be, fmask, 4); + memcpy(&curr_val_be, fval, 4); + + curr_mask = be32_to_cpu(curr_mask_be); + curr_val = be32_to_cpu(curr_val_be); + + //move to correct offset + WARN_ON(mask > max_mask); + mask <<= moffset; + val <<= moffset; + max_mask <<= moffset; + + //zero val and mask + curr_mask &= ~max_mask; + curr_val &= ~max_mask; + + //add current to mask + curr_mask |= mask; + curr_val |= val; + + //back to be32 and write + curr_mask_be = cpu_to_be32(curr_mask); + curr_val_be = cpu_to_be32(curr_val); - memcpy(fmask, &mask, match_len); - memcpy(fval, &data, match_len); + memcpy(fmask, &curr_mask_be, 4); + memcpy(fval, &curr_val_be, 4); spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; } @@ -152,23 +175,28 @@ mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec, void mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec, enum mlx5e_tc_attr_to_reg type, - u32 *data, + u32 *val, u32 *mask) { + void *headers_c = spec->match_criteria, *headers_v = spec->match_value, *fmask, *fval; int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset; + int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset; int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen; - void *headers_c = spec->match_criteria; - void *headers_v = spec->match_value; - void *fmask, *fval; + u32 max_mask = GENMASK(match_len - 1, 0); + __be32 curr_mask_be, curr_val_be; + u32 curr_mask, curr_val; fmask = headers_c + soffset; fval = headers_v + soffset; - memcpy(mask, fmask, match_len); - memcpy(data, fval, match_len); + memcpy(&curr_mask_be, fmask, 4); + memcpy(&curr_val_be, fval, 4); + + curr_mask = be32_to_cpu(curr_mask_be); + curr_val = be32_to_cpu(curr_val_be); - *mask = be32_to_cpu((__force __be32)(*mask << (32 - (match_len * 8)))); - *data = be32_to_cpu((__force __be32)(*data << (32 - (match_len * 8)))); + *mask = (curr_mask >> moffset) & max_mask; + *val = (curr_val >> moffset) & max_mask; } int @@ -192,13 +220,13 @@ mlx5e_tc_match_to_reg_set_and_get_id(struct mlx5_core_dev *mdev, (mod_hdr_acts->num_actions * MLX5_MH_ACT_SZ); /* Firmware has 5bit length field and 0 means 32bits */ - if (mlen == 4) + if (mlen == 32) mlen = 0; MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, modact, field, mfield); - MLX5_SET(set_action_in, modact, offset, moffset * 8); - MLX5_SET(set_action_in, modact, length, mlen * 8); + MLX5_SET(set_action_in, modact, offset, moffset); + MLX5_SET(set_action_in, modact, length, mlen); MLX5_SET(set_action_in, modact, data, data); err = mod_hdr_acts->num_actions; mod_hdr_acts->num_actions++; @@ -296,13 +324,13 @@ void mlx5e_tc_match_to_reg_mod_hdr_change(struct mlx5_core_dev *mdev, modact = mod_hdr_acts->actions + (act_id * MLX5_MH_ACT_SZ); /* Firmware has 5bit length field and 0 means 32bits */ - if (mlen == 4) + if (mlen == 32) mlen = 0; MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, modact, field, mfield); - MLX5_SET(set_action_in, modact, offset, moffset * 8); - MLX5_SET(set_action_in, modact, length, mlen * 8); + MLX5_SET(set_action_in, modact, offset, moffset); + MLX5_SET(set_action_in, modact, length, mlen); MLX5_SET(set_action_in, modact, data, data); } @@ -818,7 +846,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, hash_hairpin_info(peer_id, match_prio)); mutex_unlock(&priv->fs.tc.hairpin_tbl_lock); - params.log_data_size = 15; + params.log_data_size = 16; params.log_data_size = min_t(u8, params.log_data_size, MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); params.log_data_size = max_t(u8, params.log_data_size, @@ -1322,10 +1350,10 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, struct netlink_ext_ack *extack) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; - struct net_device *out_dev, *encap_dev = NULL; struct mlx5e_tc_flow_parse_attr *parse_attr; struct mlx5_flow_attr *attr = flow->attr; bool vf_tun = false, encap_valid = true; + struct net_device *encap_dev = NULL; struct mlx5_esw_flow_attr *esw_attr; struct mlx5_fc *counter = NULL; struct mlx5e_rep_priv *rpriv; @@ -1371,16 +1399,22 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, esw_attr = attr->esw_attr; for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++) { + struct net_device *out_dev; int mirred_ifindex; if (!(esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP)) continue; mirred_ifindex = parse_attr->mirred_ifindex[out_index]; - out_dev = __dev_get_by_index(dev_net(priv->netdev), - mirred_ifindex); + out_dev = dev_get_by_index(dev_net(priv->netdev), mirred_ifindex); + if (!out_dev) { + NL_SET_ERR_MSG_MOD(extack, "Requested mirred device not found"); + err = -ENODEV; + goto err_out; + } err = mlx5e_attach_encap(priv, flow, out_dev, out_index, extack, &encap_dev, &encap_valid); + dev_put(out_dev); if (err) goto err_out; @@ -1393,6 +1427,12 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, esw_attr->dests[out_index].mdev = out_priv->mdev; } + if (vf_tun && esw_attr->out_count > 1) { + NL_SET_ERR_MSG_MOD(extack, "VF tunnel encap with mirroring is not supported"); + err = -EOPNOTSUPP; + goto err_out; + } + err = mlx5_eswitch_add_vlan_action(esw, attr); if (err) goto err_out; @@ -2003,11 +2043,13 @@ static int __parse_cls_flower(struct mlx5e_priv *priv, misc_parameters_3); struct flow_rule *rule = flow_cls_offload_flow_rule(f); struct flow_dissector *dissector = rule->match.dissector; + enum fs_flow_table_type fs_type; u16 addr_type = 0; u8 ip_proto = 0; u8 *match_level; int err; + fs_type = mlx5e_is_eswitch_flow(flow) ? FS_FT_FDB : FS_FT_NIC_RX; match_level = outer_match_level; if (dissector->used_keys & @@ -2133,6 +2175,13 @@ static int __parse_cls_flower(struct mlx5e_priv *priv, if (match.mask->vlan_id || match.mask->vlan_priority || match.mask->vlan_tpid) { + if (!MLX5_CAP_FLOWTABLE_TYPE(priv->mdev, ft_field_support.outer_second_vid, + fs_type)) { + NL_SET_ERR_MSG_MOD(extack, + "Matching on CVLAN is not supported"); + return -EOPNOTSUPP; + } + if (match.key->vlan_tpid == htons(ETH_P_8021AD)) { MLX5_SET(fte_match_set_misc, misc_c, outer_second_svlan_tag, 1); @@ -3526,8 +3575,12 @@ static int add_vlan_push_action(struct mlx5e_priv *priv, if (err) return err; - *out_dev = dev_get_by_index_rcu(dev_net(vlan_dev), - dev_get_iflink(vlan_dev)); + rcu_read_lock(); + *out_dev = dev_get_by_index_rcu(dev_net(vlan_dev), dev_get_iflink(vlan_dev)); + rcu_read_unlock(); + if (!*out_dev) + return -ENODEV; + if (is_vlan_dev(*out_dev)) err = add_vlan_push_action(priv, attr, out_dev, action); @@ -4740,7 +4793,7 @@ static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv, list_for_each_entry_safe(hpe, tmp, &init_wait_list, dead_peer_wait_list) { wait_for_completion(&hpe->res_ready); if (!IS_ERR_OR_NULL(hpe->hp) && hpe->peer_vhca_id == peer_vhca_id) - hpe->hp->pair->peer_gone = true; + mlx5_core_hairpin_clear_dead_peer(hpe->hp->pair); mlx5e_hairpin_put(priv, hpe); } @@ -5074,13 +5127,13 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, if (mapped_obj.type == MLX5_MAPPED_OBJ_CHAIN) { chain = mapped_obj.chain; - tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT); + tc_skb_ext = tc_skb_ext_alloc(skb); if (WARN_ON(!tc_skb_ext)) return false; tc_skb_ext->chain = chain; - zone_restore_id = (reg_b >> REG_MAPPING_SHIFT(NIC_ZONE_RESTORE_TO_REG)) & + zone_restore_id = (reg_b >> REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) & ESW_ZONE_ID_MASK; if (!mlx5e_tc_ct_restore_flow(tc->ct, skb, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index 25c091795bcd..f7cbeb0b66d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -129,7 +129,7 @@ struct tunnel_match_enc_opts { */ #define TUNNEL_INFO_BITS 12 #define TUNNEL_INFO_BITS_MASK GENMASK(TUNNEL_INFO_BITS - 1, 0) -#define ENC_OPTS_BITS 12 +#define ENC_OPTS_BITS 11 #define ENC_OPTS_BITS_MASK GENMASK(ENC_OPTS_BITS - 1, 0) #define TUNNEL_ID_BITS (TUNNEL_INFO_BITS + ENC_OPTS_BITS) #define TUNNEL_ID_MASK GENMASK(TUNNEL_ID_BITS - 1, 0) @@ -178,6 +178,9 @@ void mlx5e_take_all_encap_flows(struct mlx5e_encap_entry *e, struct list_head *f void mlx5e_put_flow_list(struct mlx5e_priv *priv, struct list_head *flow_list); struct mlx5e_neigh_hash_entry; +struct mlx5e_encap_entry * +mlx5e_get_next_init_encap(struct mlx5e_neigh_hash_entry *nhe, + struct mlx5e_encap_entry *e); void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe); void mlx5e_tc_reoffload_flows_work(struct work_struct *work); @@ -198,10 +201,10 @@ enum mlx5e_tc_attr_to_reg { struct mlx5e_tc_attr_to_reg_mapping { int mfield; /* rewrite field */ - int moffset; /* offset of mfield */ - int mlen; /* bytes to rewrite/match */ + int moffset; /* bit offset of mfield */ + int mlen; /* bits to rewrite/match */ - int soffset; /* offset of spec for match */ + int soffset; /* byte offset of spec for match */ }; extern struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[]; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 8ba62671f5f1..c63d78eda606 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -32,7 +32,6 @@ #include <linux/tcp.h> #include <linux/if_vlan.h> -#include <linux/ptp_classify.h> #include <net/geneve.h> #include <net/dsfield.h> #include "en.h" @@ -67,24 +66,6 @@ static inline int mlx5e_get_dscp_up(struct mlx5e_priv *priv, struct sk_buff *skb } #endif -static bool mlx5e_use_ptpsq(struct sk_buff *skb) -{ - struct flow_keys fk; - - if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) - return false; - - if (fk.basic.n_proto == htons(ETH_P_1588)) - return true; - - if (fk.basic.n_proto != htons(ETH_P_IP) && - fk.basic.n_proto != htons(ETH_P_IPV6)) - return false; - - return (fk.basic.ip_proto == IPPROTO_UDP && - fk.ports.dst == htons(PTP_EV_PORT)); -} - static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -145,9 +126,9 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, } ptp_channel = READ_ONCE(priv->channels.ptp); - if (unlikely(ptp_channel) && - test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state) && - mlx5e_use_ptpsq(skb)) + if (unlikely(ptp_channel && + test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state) && + mlx5e_use_ptpsq(skb))) return mlx5e_select_ptpsq(dev, skb); txq_ix = netdev_pick_tx(dev, skb, NULL); @@ -706,16 +687,12 @@ void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) mlx5e_tx_mpwqe_session_complete(sq); } -static bool mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, +static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { - if (unlikely(!mlx5e_accel_tx_eseg(priv, skb, eseg, ihs))) - return false; - + mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); - - return true; } netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) @@ -744,10 +721,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) if (mlx5e_tx_skb_supports_mpwqe(skb, &attr)) { struct mlx5_wqe_eth_seg eseg = {}; - if (unlikely(!mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &eseg, - attr.ihs))) - return NETDEV_TX_OK; - + mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &eseg, attr.ihs); mlx5e_sq_xmit_mpwqe(sq, skb, &eseg, netdev_xmit_more()); return NETDEV_TX_OK; } @@ -762,9 +736,7 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) /* May update the WQE, but may not post other WQEs. */ mlx5e_accel_tx_finish(sq, wqe, &accel, (struct mlx5_wqe_inline_seg *)(wqe->data + wqe_attr.ds_cnt_inl)); - if (unlikely(!mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &wqe->eth, attr.ihs))) - return NETDEV_TX_OK; - + mlx5e_txwqe_build_eseg(priv, sq, skb, &accel, &wqe->eth, attr.ihs); mlx5e_sq_xmit_wqe(sq, skb, &attr, &wqe_attr, wqe, pi, netdev_xmit_more()); return NETDEV_TX_OK; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 77c0ca655975..6e074cc457de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -1,33 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* - * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. + * Copyright (c) 2013-2021, Mellanox Technologies inc. All rights reserved. */ #include <linux/interrupt.h> @@ -45,6 +18,7 @@ #include "eswitch.h" #include "lib/clock.h" #include "diag/fw_tracer.h" +#include "mlx5_irq.h" enum { MLX5_EQE_OWNER_INIT_VAL = 0x1, @@ -84,6 +58,9 @@ struct mlx5_eq_table { struct mutex lock; /* sync async eqs creations */ int num_comp_eqs; struct mlx5_irq_table *irq_table; +#ifdef CONFIG_RFS_ACCEL + struct cpu_rmap *rmap; +#endif }; #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \ @@ -136,7 +113,7 @@ static int mlx5_eq_comp_int(struct notifier_block *nb, eqe = next_eqe_sw(eq); if (!eqe) - return 0; + goto out; do { struct mlx5_core_cq *cq; @@ -161,6 +138,8 @@ static int mlx5_eq_comp_int(struct notifier_block *nb, ++eq->cons_index; } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq))); + +out: eq_update_ci(eq, 1); if (cqn != -1) @@ -248,9 +227,9 @@ static int mlx5_eq_async_int(struct notifier_block *nb, ++eq->cons_index; } while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq))); - eq_update_ci(eq, 1); out: + eq_update_ci(eq, 1); mlx5_eq_async_int_unlock(eq_async, recovery, &flags); return unlikely(recovery) ? num_eqes : 0; @@ -286,7 +265,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0}; u8 log_eq_stride = ilog2(MLX5_EQE_SIZE); struct mlx5_priv *priv = &dev->priv; - u8 vecidx = param->irq_index; + u16 vecidx = param->irq_index; __be64 *pas; void *eqc; int inlen; @@ -309,13 +288,20 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc); init_eq_buf(eq); + eq->irq = mlx5_irq_request(dev, vecidx, param->affinity); + if (IS_ERR(eq->irq)) { + err = PTR_ERR(eq->irq); + goto err_buf; + } + + vecidx = mlx5_irq_get_index(eq->irq); inlen = MLX5_ST_SZ_BYTES(create_eq_in) + MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages; in = kvzalloc(inlen, GFP_KERNEL); if (!in) { err = -ENOMEM; - goto err_buf; + goto err_irq; } pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas); @@ -359,6 +345,8 @@ err_eq: err_in: kvfree(in); +err_irq: + mlx5_irq_release(eq->irq); err_buf: mlx5_frag_buf_free(dev, &eq->frag_buf); return err; @@ -377,10 +365,9 @@ err_buf: int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq, struct notifier_block *nb) { - struct mlx5_eq_table *eq_table = dev->priv.eq_table; int err; - err = mlx5_irq_attach_nb(eq_table->irq_table, eq->vecidx, nb); + err = mlx5_irq_attach_nb(eq->irq, nb); if (!err) eq_update_ci(eq, 1); @@ -399,9 +386,7 @@ EXPORT_SYMBOL(mlx5_eq_enable); void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq, struct notifier_block *nb) { - struct mlx5_eq_table *eq_table = dev->priv.eq_table; - - mlx5_irq_detach_nb(eq_table->irq_table, eq->vecidx, nb); + mlx5_irq_detach_nb(eq->irq, nb); } EXPORT_SYMBOL(mlx5_eq_disable); @@ -415,10 +400,9 @@ static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) if (err) mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n", eq->eqn); - synchronize_irq(eq->irqn); + mlx5_irq_release(eq->irq); mlx5_frag_buf_free(dev, &eq->frag_buf); - return err; } @@ -490,14 +474,7 @@ static int create_async_eq(struct mlx5_core_dev *dev, int err; mutex_lock(&eq_table->lock); - /* Async EQs must share irq index 0 */ - if (param->irq_index != 0) { - err = -EINVAL; - goto unlock; - } - err = create_map_eq(dev, eq, param); -unlock: mutex_unlock(&eq_table->lock); return err; } @@ -616,8 +593,11 @@ setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, eq->irq_nb.notifier_call = mlx5_eq_async_int; spin_lock_init(&eq->lock); + if (!zalloc_cpumask_var(¶m->affinity, GFP_KERNEL)) + return -ENOMEM; err = create_async_eq(dev, &eq->core, param); + free_cpumask_var(param->affinity); if (err) { mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err); return err; @@ -652,7 +632,6 @@ static int create_async_eqs(struct mlx5_core_dev *dev) mlx5_eq_notifier_register(dev, &table->cq_err_nb); param = (struct mlx5_eq_param) { - .irq_index = 0, .nent = MLX5_NUM_CMD_EQE, .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD, }; @@ -665,7 +644,6 @@ static int create_async_eqs(struct mlx5_core_dev *dev) mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL); param = (struct mlx5_eq_param) { - .irq_index = 0, .nent = MLX5_NUM_ASYNC_EQE, }; @@ -675,7 +653,6 @@ static int create_async_eqs(struct mlx5_core_dev *dev) goto err2; param = (struct mlx5_eq_param) { - .irq_index = 0, .nent = /* TODO: sriov max_vf + */ 1, .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST, }; @@ -735,6 +712,9 @@ mlx5_eq_create_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL); int err; + if (!cpumask_available(param->affinity)) + return ERR_PTR(-EINVAL); + if (!eq) return ERR_PTR(-ENOMEM); @@ -845,16 +825,21 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) .irq_index = vecidx, .nent = nent, }; - err = create_map_eq(dev, &eq->core, ¶m); - if (err) { - kfree(eq); - goto clean; + + if (!zalloc_cpumask_var(¶m.affinity, GFP_KERNEL)) { + err = -ENOMEM; + goto clean_eq; } + cpumask_set_cpu(cpumask_local_spread(i, dev->priv.numa_node), + param.affinity); + err = create_map_eq(dev, &eq->core, ¶m); + free_cpumask_var(param.affinity); + if (err) + goto clean_eq; err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb); if (err) { destroy_unmap_eq(dev, &eq->core); - kfree(eq); - goto clean; + goto clean_eq; } mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn); @@ -863,7 +848,8 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) } return 0; - +clean_eq: + kfree(eq); clean: destroy_comp_eqs(dev); return err; @@ -899,17 +885,23 @@ EXPORT_SYMBOL(mlx5_comp_vectors_count); struct cpumask * mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector) { - int vecidx = vector + MLX5_IRQ_VEC_COMP_BASE; + struct mlx5_eq_table *table = dev->priv.eq_table; + struct mlx5_eq_comp *eq, *n; + int i = 0; + + list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { + if (i++ == vector) + break; + } - return mlx5_irq_get_affinity_mask(dev->priv.eq_table->irq_table, - vecidx); + return mlx5_irq_get_affinity_mask(eq->core.irq); } EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask); #ifdef CONFIG_RFS_ACCEL struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev) { - return mlx5_irq_get_rmap(dev->priv.eq_table->irq_table); + return dev->priv.eq_table->rmap; } #endif @@ -926,12 +918,57 @@ struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn) return ERR_PTR(-ENOENT); } +static void clear_rmap(struct mlx5_core_dev *dev) +{ +#ifdef CONFIG_RFS_ACCEL + struct mlx5_eq_table *eq_table = dev->priv.eq_table; + + free_irq_cpu_rmap(eq_table->rmap); +#endif +} + +static int set_rmap(struct mlx5_core_dev *mdev) +{ + int err = 0; +#ifdef CONFIG_RFS_ACCEL + struct mlx5_eq_table *eq_table = mdev->priv.eq_table; + int vecidx; + + eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs); + if (!eq_table->rmap) { + err = -ENOMEM; + mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err); + goto err_out; + } + + vecidx = MLX5_IRQ_VEC_COMP_BASE; + for (; vecidx < eq_table->num_comp_eqs + MLX5_IRQ_VEC_COMP_BASE; + vecidx++) { + err = irq_cpu_rmap_add(eq_table->rmap, + pci_irq_vector(mdev->pdev, vecidx)); + if (err) { + mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d", + err); + goto err_irq_cpu_rmap_add; + } + } + return 0; + +err_irq_cpu_rmap_add: + clear_rmap(mdev); +err_out: +#endif + return err; +} + /* This function should only be called after mlx5_cmd_force_teardown_hca */ void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev) { struct mlx5_eq_table *table = dev->priv.eq_table; mutex_lock(&table->lock); /* sync with create/destroy_async_eq */ + if (!mlx5_core_is_sf(dev)) + clear_rmap(dev); mlx5_irq_table_destroy(dev); mutex_unlock(&table->lock); } @@ -948,12 +985,19 @@ int mlx5_eq_table_create(struct mlx5_core_dev *dev) int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ? MLX5_CAP_GEN(dev, max_num_eqs) : 1 << MLX5_CAP_GEN(dev, log_max_eq); + int max_eqs_sf; int err; eq_table->num_comp_eqs = min_t(int, - mlx5_irq_get_num_comp(eq_table->irq_table), + mlx5_irq_table_get_num_comp(eq_table->irq_table), num_eqs - MLX5_MAX_ASYNC_EQS); + if (mlx5_core_is_sf(dev)) { + max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF, + mlx5_irq_table_get_sfs_vec(eq_table->irq_table)); + eq_table->num_comp_eqs = min_t(int, eq_table->num_comp_eqs, + max_eqs_sf); + } err = create_async_eqs(dev); if (err) { @@ -961,6 +1005,18 @@ int mlx5_eq_table_create(struct mlx5_core_dev *dev) goto err_async_eqs; } + if (!mlx5_core_is_sf(dev)) { + /* rmap is a mapping between irq number and queue number. + * each irq can be assign only to a single rmap. + * since SFs share IRQs, rmap mapping cannot function correctly + * for irqs that are shared for different core/netdev RX rings. + * Hence we don't allow netdev rmap for SFs + */ + err = set_rmap(dev); + if (err) + goto err_rmap; + } + err = create_comp_eqs(dev); if (err) { mlx5_core_err(dev, "Failed to create completion EQs\n"); @@ -969,6 +1025,9 @@ int mlx5_eq_table_create(struct mlx5_core_dev *dev) return 0; err_comp_eqs: + if (!mlx5_core_is_sf(dev)) + clear_rmap(dev); +err_rmap: destroy_async_eqs(dev); err_async_eqs: return err; @@ -976,6 +1035,8 @@ err_async_eqs: void mlx5_eq_table_destroy(struct mlx5_core_dev *dev) { + if (!mlx5_core_is_sf(dev)) + clear_rmap(dev); destroy_comp_eqs(dev); destroy_async_eqs(dev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c new file mode 100644 index 000000000000..a6e1d4f78268 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -0,0 +1,1299 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021 Mellanox Technologies. */ + +#include <linux/list.h> +#include <linux/notifier.h> +#include <net/netevent.h> +#include <net/switchdev.h> +#include "bridge.h" +#include "eswitch.h" +#include "bridge_priv.h" +#define CREATE_TRACE_POINTS +#include "diag/bridge_tracepoint.h" + +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE 64000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM 0 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE / 4 - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE / 2 - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE - 1) + +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE 64000 +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM 0 +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE / 2 - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE - 1) + +#define MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE 0 + +enum { + MLX5_ESW_BRIDGE_LEVEL_INGRESS_TABLE, + MLX5_ESW_BRIDGE_LEVEL_EGRESS_TABLE, + MLX5_ESW_BRIDGE_LEVEL_SKIP_TABLE, +}; + +static const struct rhashtable_params fdb_ht_params = { + .key_offset = offsetof(struct mlx5_esw_bridge_fdb_entry, key), + .key_len = sizeof(struct mlx5_esw_bridge_fdb_key), + .head_offset = offsetof(struct mlx5_esw_bridge_fdb_entry, ht_node), + .automatic_shrinking = true, +}; + +enum { + MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG = BIT(0), +}; + +struct mlx5_esw_bridge { + int ifindex; + int refcnt; + struct list_head list; + struct mlx5_esw_bridge_offloads *br_offloads; + + struct list_head fdb_list; + struct rhashtable fdb_ht; + struct xarray vports; + + struct mlx5_flow_table *egress_ft; + struct mlx5_flow_group *egress_vlan_fg; + struct mlx5_flow_group *egress_mac_fg; + unsigned long ageing_time; + u32 flags; +}; + +static void +mlx5_esw_bridge_fdb_offload_notify(struct net_device *dev, const unsigned char *addr, u16 vid, + unsigned long val) +{ + struct switchdev_notifier_fdb_info send_info; + + send_info.addr = addr; + send_info.vid = vid; + send_info.offloaded = true; + call_switchdev_notifiers(val, dev, &send_info.info, NULL); +} + +static struct mlx5_flow_table * +mlx5_esw_bridge_table_create(int max_fte, u32 level, struct mlx5_eswitch *esw) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_core_dev *dev = esw->dev; + struct mlx5_flow_namespace *ns; + struct mlx5_flow_table *fdb; + + ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); + if (!ns) { + esw_warn(dev, "Failed to get FDB namespace\n"); + return ERR_PTR(-ENOENT); + } + + ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; + ft_attr.max_fte = max_fte; + ft_attr.level = level; + ft_attr.prio = FDB_BR_OFFLOAD; + fdb = mlx5_create_flow_table(ns, &ft_attr); + if (IS_ERR(fdb)) + esw_warn(dev, "Failed to create bridge FDB Table (err=%ld)\n", PTR_ERR(fdb)); + + return fdb; +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + u32 *in, *match; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return ERR_PTR(-ENOMEM); + + MLX5_SET(create_flow_group_in, in, match_criteria_enable, + MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2); + match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid); + + MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_mask()); + + MLX5_SET(create_flow_group_in, in, start_flow_index, + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM); + MLX5_SET(create_flow_group_in, in, end_flow_index, + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO); + + fg = mlx5_create_flow_group(ingress_ft, in); + kvfree(in); + if (IS_ERR(fg)) + esw_warn(esw->dev, + "Failed to create VLAN flow group for bridge ingress table (err=%ld)\n", + PTR_ERR(fg)); + + return fg; +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_filter_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + u32 *in, *match; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return ERR_PTR(-ENOMEM); + + MLX5_SET(create_flow_group_in, in, match_criteria_enable, + MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2); + match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + + MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_mask()); + + MLX5_SET(create_flow_group_in, in, start_flow_index, + MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_FROM); + MLX5_SET(create_flow_group_in, in, end_flow_index, + MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO); + + fg = mlx5_create_flow_group(ingress_ft, in); + if (IS_ERR(fg)) + esw_warn(esw->dev, + "Failed to create bridge ingress table VLAN filter flow group (err=%ld)\n", + PTR_ERR(fg)); + + kvfree(in); + return fg; +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + u32 *in, *match; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return ERR_PTR(-ENOMEM); + + MLX5_SET(create_flow_group_in, in, match_criteria_enable, + MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2); + match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); + + MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_mask()); + + MLX5_SET(create_flow_group_in, in, start_flow_index, + MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM); + MLX5_SET(create_flow_group_in, in, end_flow_index, + MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO); + + fg = mlx5_create_flow_group(ingress_ft, in); + if (IS_ERR(fg)) + esw_warn(esw->dev, + "Failed to create MAC flow group for bridge ingress table (err=%ld)\n", + PTR_ERR(fg)); + + kvfree(in); + return fg; +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + u32 *in, *match; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return ERR_PTR(-ENOMEM); + + MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_47_16); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_15_0); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid); + + MLX5_SET(create_flow_group_in, in, start_flow_index, + MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM); + MLX5_SET(create_flow_group_in, in, end_flow_index, + MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO); + + fg = mlx5_create_flow_group(egress_ft, in); + if (IS_ERR(fg)) + esw_warn(esw->dev, + "Failed to create VLAN flow group for bridge egress table (err=%ld)\n", + PTR_ERR(fg)); + kvfree(in); + return fg; +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_egress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + u32 *in, *match; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return ERR_PTR(-ENOMEM); + + MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_47_16); + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_15_0); + + MLX5_SET(create_flow_group_in, in, start_flow_index, + MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM); + MLX5_SET(create_flow_group_in, in, end_flow_index, + MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO); + + fg = mlx5_create_flow_group(egress_ft, in); + if (IS_ERR(fg)) + esw_warn(esw->dev, + "Failed to create bridge egress table MAC flow group (err=%ld)\n", + PTR_ERR(fg)); + kvfree(in); + return fg; +} + +static int +mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_flow_group *mac_fg, *filter_fg, *vlan_fg; + struct mlx5_flow_table *ingress_ft, *skip_ft; + int err; + + if (!mlx5_eswitch_vport_match_metadata_enabled(br_offloads->esw)) + return -EOPNOTSUPP; + + ingress_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE, + MLX5_ESW_BRIDGE_LEVEL_INGRESS_TABLE, + br_offloads->esw); + if (IS_ERR(ingress_ft)) + return PTR_ERR(ingress_ft); + + skip_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE, + MLX5_ESW_BRIDGE_LEVEL_SKIP_TABLE, + br_offloads->esw); + if (IS_ERR(skip_ft)) { + err = PTR_ERR(skip_ft); + goto err_skip_tbl; + } + + vlan_fg = mlx5_esw_bridge_ingress_vlan_fg_create(br_offloads->esw, ingress_ft); + if (IS_ERR(vlan_fg)) { + err = PTR_ERR(vlan_fg); + goto err_vlan_fg; + } + + filter_fg = mlx5_esw_bridge_ingress_filter_fg_create(br_offloads->esw, ingress_ft); + if (IS_ERR(filter_fg)) { + err = PTR_ERR(filter_fg); + goto err_filter_fg; + } + + mac_fg = mlx5_esw_bridge_ingress_mac_fg_create(br_offloads->esw, ingress_ft); + if (IS_ERR(mac_fg)) { + err = PTR_ERR(mac_fg); + goto err_mac_fg; + } + + br_offloads->ingress_ft = ingress_ft; + br_offloads->skip_ft = skip_ft; + br_offloads->ingress_vlan_fg = vlan_fg; + br_offloads->ingress_filter_fg = filter_fg; + br_offloads->ingress_mac_fg = mac_fg; + return 0; + +err_mac_fg: + mlx5_destroy_flow_group(filter_fg); +err_filter_fg: + mlx5_destroy_flow_group(vlan_fg); +err_vlan_fg: + mlx5_destroy_flow_table(skip_ft); +err_skip_tbl: + mlx5_destroy_flow_table(ingress_ft); + return err; +} + +static void +mlx5_esw_bridge_ingress_table_cleanup(struct mlx5_esw_bridge_offloads *br_offloads) +{ + mlx5_destroy_flow_group(br_offloads->ingress_mac_fg); + br_offloads->ingress_mac_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_filter_fg); + br_offloads->ingress_filter_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_vlan_fg); + br_offloads->ingress_vlan_fg = NULL; + mlx5_destroy_flow_table(br_offloads->skip_ft); + br_offloads->skip_ft = NULL; + mlx5_destroy_flow_table(br_offloads->ingress_ft); + br_offloads->ingress_ft = NULL; +} + +static int +mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_flow_group *mac_fg, *vlan_fg; + struct mlx5_flow_table *egress_ft; + int err; + + egress_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE, + MLX5_ESW_BRIDGE_LEVEL_EGRESS_TABLE, + br_offloads->esw); + if (IS_ERR(egress_ft)) + return PTR_ERR(egress_ft); + + vlan_fg = mlx5_esw_bridge_egress_vlan_fg_create(br_offloads->esw, egress_ft); + if (IS_ERR(vlan_fg)) { + err = PTR_ERR(vlan_fg); + goto err_vlan_fg; + } + + mac_fg = mlx5_esw_bridge_egress_mac_fg_create(br_offloads->esw, egress_ft); + if (IS_ERR(mac_fg)) { + err = PTR_ERR(mac_fg); + goto err_mac_fg; + } + + bridge->egress_ft = egress_ft; + bridge->egress_vlan_fg = vlan_fg; + bridge->egress_mac_fg = mac_fg; + return 0; + +err_mac_fg: + mlx5_destroy_flow_group(vlan_fg); +err_vlan_fg: + mlx5_destroy_flow_table(egress_ft); + return err; +} + +static void +mlx5_esw_bridge_egress_table_cleanup(struct mlx5_esw_bridge *bridge) +{ + mlx5_destroy_flow_group(bridge->egress_mac_fg); + mlx5_destroy_flow_group(bridge->egress_vlan_fg); + mlx5_destroy_flow_table(bridge->egress_ft); +} + +static struct mlx5_flow_handle * +mlx5_esw_bridge_ingress_flow_create(u16 vport_num, const unsigned char *addr, + struct mlx5_esw_bridge_vlan *vlan, u32 counter_id, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads; + struct mlx5_flow_act flow_act = { + .action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT, + .flags = FLOW_ACT_NO_APPEND, + }; + struct mlx5_flow_destination dests[2] = {}; + struct mlx5_flow_spec *rule_spec; + struct mlx5_flow_handle *handle; + u8 *smac_v, *smac_c; + + rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL); + if (!rule_spec) + return ERR_PTR(-ENOMEM); + + rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2; + + smac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value, + outer_headers.smac_47_16); + ether_addr_copy(smac_v, addr); + smac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria, + outer_headers.smac_47_16); + eth_broadcast_addr(smac_c); + + MLX5_SET(fte_match_param, rule_spec->match_criteria, + misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); + MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_for_match(br_offloads->esw, vport_num)); + + if (vlan && vlan->pkt_reformat_push) { + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + flow_act.pkt_reformat = vlan->pkt_reformat_push; + } else if (vlan) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.first_vid); + MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, + vlan->vid); + } + + dests[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dests[0].ft = bridge->egress_ft; + dests[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dests[1].counter_id = counter_id; + + handle = mlx5_add_flow_rules(br_offloads->ingress_ft, rule_spec, &flow_act, dests, + ARRAY_SIZE(dests)); + + kvfree(rule_spec); + return handle; +} + +static struct mlx5_flow_handle * +mlx5_esw_bridge_ingress_filter_flow_create(u16 vport_num, const unsigned char *addr, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads; + struct mlx5_flow_destination dest = { + .type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE, + .ft = br_offloads->skip_ft, + }; + struct mlx5_flow_act flow_act = { + .action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, + .flags = FLOW_ACT_NO_APPEND, + }; + struct mlx5_flow_spec *rule_spec; + struct mlx5_flow_handle *handle; + u8 *smac_v, *smac_c; + + rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL); + if (!rule_spec) + return ERR_PTR(-ENOMEM); + + rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2; + + smac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value, + outer_headers.smac_47_16); + ether_addr_copy(smac_v, addr); + smac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria, + outer_headers.smac_47_16); + eth_broadcast_addr(smac_c); + + MLX5_SET(fte_match_param, rule_spec->match_criteria, + misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); + MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_for_match(br_offloads->esw, vport_num)); + + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + + handle = mlx5_add_flow_rules(br_offloads->ingress_ft, rule_spec, &flow_act, &dest, 1); + + kvfree(rule_spec); + return handle; +} + +static struct mlx5_flow_handle * +mlx5_esw_bridge_egress_flow_create(u16 vport_num, const unsigned char *addr, + struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_flow_destination dest = { + .type = MLX5_FLOW_DESTINATION_TYPE_VPORT, + .vport.num = vport_num, + }; + struct mlx5_flow_act flow_act = { + .action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, + .flags = FLOW_ACT_NO_APPEND, + }; + struct mlx5_flow_spec *rule_spec; + struct mlx5_flow_handle *handle; + u8 *dmac_v, *dmac_c; + + rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL); + if (!rule_spec) + return ERR_PTR(-ENOMEM); + + rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + + dmac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value, + outer_headers.dmac_47_16); + ether_addr_copy(dmac_v, addr); + dmac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria, + outer_headers.dmac_47_16); + eth_broadcast_addr(dmac_c); + + if (vlan) { + if (vlan->pkt_reformat_pop) { + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + flow_act.pkt_reformat = vlan->pkt_reformat_pop; + } + + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.first_vid); + MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, + vlan->vid); + } + + handle = mlx5_add_flow_rules(bridge->egress_ft, rule_spec, &flow_act, &dest, 1); + + kvfree(rule_spec); + return handle; +} + +static struct mlx5_esw_bridge *mlx5_esw_bridge_create(int ifindex, + struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_esw_bridge *bridge; + int err; + + bridge = kvzalloc(sizeof(*bridge), GFP_KERNEL); + if (!bridge) + return ERR_PTR(-ENOMEM); + + bridge->br_offloads = br_offloads; + err = mlx5_esw_bridge_egress_table_init(br_offloads, bridge); + if (err) + goto err_egress_tbl; + + err = rhashtable_init(&bridge->fdb_ht, &fdb_ht_params); + if (err) + goto err_fdb_ht; + + INIT_LIST_HEAD(&bridge->fdb_list); + xa_init(&bridge->vports); + bridge->ifindex = ifindex; + bridge->refcnt = 1; + bridge->ageing_time = BR_DEFAULT_AGEING_TIME; + list_add(&bridge->list, &br_offloads->bridges); + + return bridge; + +err_fdb_ht: + mlx5_esw_bridge_egress_table_cleanup(bridge); +err_egress_tbl: + kvfree(bridge); + return ERR_PTR(err); +} + +static void mlx5_esw_bridge_get(struct mlx5_esw_bridge *bridge) +{ + bridge->refcnt++; +} + +static void mlx5_esw_bridge_put(struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_esw_bridge *bridge) +{ + if (--bridge->refcnt) + return; + + mlx5_esw_bridge_egress_table_cleanup(bridge); + WARN_ON(!xa_empty(&bridge->vports)); + list_del(&bridge->list); + rhashtable_destroy(&bridge->fdb_ht); + kvfree(bridge); + + if (list_empty(&br_offloads->bridges)) + mlx5_esw_bridge_ingress_table_cleanup(br_offloads); +} + +static struct mlx5_esw_bridge * +mlx5_esw_bridge_lookup(int ifindex, struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_esw_bridge *bridge; + + ASSERT_RTNL(); + + list_for_each_entry(bridge, &br_offloads->bridges, list) { + if (bridge->ifindex == ifindex) { + mlx5_esw_bridge_get(bridge); + return bridge; + } + } + + if (!br_offloads->ingress_ft) { + int err = mlx5_esw_bridge_ingress_table_init(br_offloads); + + if (err) + return ERR_PTR(err); + } + + bridge = mlx5_esw_bridge_create(ifindex, br_offloads); + if (IS_ERR(bridge) && list_empty(&br_offloads->bridges)) + mlx5_esw_bridge_ingress_table_cleanup(br_offloads); + return bridge; +} + +static int mlx5_esw_bridge_port_insert(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge *bridge) +{ + return xa_insert(&bridge->vports, port->vport_num, port, GFP_KERNEL); +} + +static struct mlx5_esw_bridge_port * +mlx5_esw_bridge_port_lookup(u16 vport_num, struct mlx5_esw_bridge *bridge) +{ + return xa_load(&bridge->vports, vport_num); +} + +static void mlx5_esw_bridge_port_erase(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge *bridge) +{ + xa_erase(&bridge->vports, port->vport_num); +} + +static void mlx5_esw_bridge_fdb_entry_refresh(unsigned long lastuse, + struct mlx5_esw_bridge_fdb_entry *entry) +{ + trace_mlx5_esw_bridge_fdb_entry_refresh(entry); + + entry->lastuse = lastuse; + mlx5_esw_bridge_fdb_offload_notify(entry->dev, entry->key.addr, + entry->key.vid, + SWITCHDEV_FDB_ADD_TO_BRIDGE); +} + +static void +mlx5_esw_bridge_fdb_entry_cleanup(struct mlx5_esw_bridge_fdb_entry *entry, + struct mlx5_esw_bridge *bridge) +{ + trace_mlx5_esw_bridge_fdb_entry_cleanup(entry); + + rhashtable_remove_fast(&bridge->fdb_ht, &entry->ht_node, fdb_ht_params); + mlx5_del_flow_rules(entry->egress_handle); + if (entry->filter_handle) + mlx5_del_flow_rules(entry->filter_handle); + mlx5_del_flow_rules(entry->ingress_handle); + mlx5_fc_destroy(bridge->br_offloads->esw->dev, entry->ingress_counter); + list_del(&entry->vlan_list); + list_del(&entry->list); + kvfree(entry); +} + +static void mlx5_esw_bridge_fdb_flush(struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_fdb_entry *entry, *tmp; + + list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list) { + if (!(entry->flags & MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER)) + mlx5_esw_bridge_fdb_offload_notify(entry->dev, entry->key.addr, + entry->key.vid, + SWITCHDEV_FDB_DEL_TO_BRIDGE); + mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge); + } +} + +static struct mlx5_esw_bridge_vlan * +mlx5_esw_bridge_vlan_lookup(u16 vid, struct mlx5_esw_bridge_port *port) +{ + return xa_load(&port->vlans, vid); +} + +static int +mlx5_esw_bridge_vlan_push_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw) +{ + struct { + __be16 h_vlan_proto; + __be16 h_vlan_TCI; + } vlan_hdr = { htons(ETH_P_8021Q), htons(vlan->vid) }; + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5_pkt_reformat *pkt_reformat; + + if (!BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat_insert)) || + MLX5_CAP_GEN_2(esw->dev, max_reformat_insert_size) < sizeof(vlan_hdr) || + MLX5_CAP_GEN_2(esw->dev, max_reformat_insert_offset) < + offsetof(struct vlan_ethhdr, h_vlan_proto)) { + esw_warn(esw->dev, "Packet reformat INSERT_HEADER is not supported\n"); + return -EOPNOTSUPP; + } + + reformat_params.type = MLX5_REFORMAT_TYPE_INSERT_HDR; + reformat_params.param_0 = MLX5_REFORMAT_CONTEXT_ANCHOR_MAC_START; + reformat_params.param_1 = offsetof(struct vlan_ethhdr, h_vlan_proto); + reformat_params.size = sizeof(vlan_hdr); + reformat_params.data = &vlan_hdr; + pkt_reformat = mlx5_packet_reformat_alloc(esw->dev, + &reformat_params, + MLX5_FLOW_NAMESPACE_FDB); + if (IS_ERR(pkt_reformat)) { + esw_warn(esw->dev, "Failed to alloc packet reformat INSERT_HEADER (err=%ld)\n", + PTR_ERR(pkt_reformat)); + return PTR_ERR(pkt_reformat); + } + + vlan->pkt_reformat_push = pkt_reformat; + return 0; +} + +static void +mlx5_esw_bridge_vlan_push_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw) +{ + mlx5_packet_reformat_dealloc(esw->dev, vlan->pkt_reformat_push); + vlan->pkt_reformat_push = NULL; +} + +static int +mlx5_esw_bridge_vlan_pop_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw) +{ + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5_pkt_reformat *pkt_reformat; + + if (!BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat_remove)) || + MLX5_CAP_GEN_2(esw->dev, max_reformat_remove_size) < sizeof(struct vlan_hdr) || + MLX5_CAP_GEN_2(esw->dev, max_reformat_remove_offset) < + offsetof(struct vlan_ethhdr, h_vlan_proto)) { + esw_warn(esw->dev, "Packet reformat REMOVE_HEADER is not supported\n"); + return -EOPNOTSUPP; + } + + reformat_params.type = MLX5_REFORMAT_TYPE_REMOVE_HDR; + reformat_params.param_0 = MLX5_REFORMAT_CONTEXT_ANCHOR_MAC_START; + reformat_params.param_1 = offsetof(struct vlan_ethhdr, h_vlan_proto); + reformat_params.size = sizeof(struct vlan_hdr); + pkt_reformat = mlx5_packet_reformat_alloc(esw->dev, + &reformat_params, + MLX5_FLOW_NAMESPACE_FDB); + if (IS_ERR(pkt_reformat)) { + esw_warn(esw->dev, "Failed to alloc packet reformat REMOVE_HEADER (err=%ld)\n", + PTR_ERR(pkt_reformat)); + return PTR_ERR(pkt_reformat); + } + + vlan->pkt_reformat_pop = pkt_reformat; + return 0; +} + +static void +mlx5_esw_bridge_vlan_pop_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw) +{ + mlx5_packet_reformat_dealloc(esw->dev, vlan->pkt_reformat_pop); + vlan->pkt_reformat_pop = NULL; +} + +static struct mlx5_esw_bridge_vlan * +mlx5_esw_bridge_vlan_create(u16 vid, u16 flags, struct mlx5_esw_bridge_port *port, + struct mlx5_eswitch *esw) +{ + struct mlx5_esw_bridge_vlan *vlan; + int err; + + vlan = kvzalloc(sizeof(*vlan), GFP_KERNEL); + if (!vlan) + return ERR_PTR(-ENOMEM); + + vlan->vid = vid; + vlan->flags = flags; + INIT_LIST_HEAD(&vlan->fdb_list); + + if (flags & BRIDGE_VLAN_INFO_PVID) { + err = mlx5_esw_bridge_vlan_push_create(vlan, esw); + if (err) + goto err_vlan_push; + } + if (flags & BRIDGE_VLAN_INFO_UNTAGGED) { + err = mlx5_esw_bridge_vlan_pop_create(vlan, esw); + if (err) + goto err_vlan_pop; + } + + err = xa_insert(&port->vlans, vid, vlan, GFP_KERNEL); + if (err) + goto err_xa_insert; + + trace_mlx5_esw_bridge_vlan_create(vlan); + return vlan; + +err_xa_insert: + if (vlan->pkt_reformat_pop) + mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw); +err_vlan_pop: + if (vlan->pkt_reformat_push) + mlx5_esw_bridge_vlan_push_cleanup(vlan, esw); +err_vlan_push: + kvfree(vlan); + return ERR_PTR(err); +} + +static void mlx5_esw_bridge_vlan_erase(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge_vlan *vlan) +{ + xa_erase(&port->vlans, vlan->vid); +} + +static void mlx5_esw_bridge_vlan_flush(struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_fdb_entry *entry, *tmp; + + list_for_each_entry_safe(entry, tmp, &vlan->fdb_list, vlan_list) { + if (!(entry->flags & MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER)) + mlx5_esw_bridge_fdb_offload_notify(entry->dev, entry->key.addr, + entry->key.vid, + SWITCHDEV_FDB_DEL_TO_BRIDGE); + mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge); + } + + if (vlan->pkt_reformat_pop) + mlx5_esw_bridge_vlan_pop_cleanup(vlan, bridge->br_offloads->esw); + if (vlan->pkt_reformat_push) + mlx5_esw_bridge_vlan_push_cleanup(vlan, bridge->br_offloads->esw); +} + +static void mlx5_esw_bridge_vlan_cleanup(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_esw_bridge *bridge) +{ + trace_mlx5_esw_bridge_vlan_cleanup(vlan); + mlx5_esw_bridge_vlan_flush(vlan, bridge); + mlx5_esw_bridge_vlan_erase(port, vlan); + kvfree(vlan); +} + +static void mlx5_esw_bridge_port_vlans_flush(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_vlan *vlan; + unsigned long index; + + xa_for_each(&port->vlans, index, vlan) + mlx5_esw_bridge_vlan_cleanup(port, vlan, bridge); +} + +static struct mlx5_esw_bridge_vlan * +mlx5_esw_bridge_port_vlan_lookup(u16 vid, u16 vport_num, struct mlx5_esw_bridge *bridge, + struct mlx5_eswitch *esw) +{ + struct mlx5_esw_bridge_port *port; + struct mlx5_esw_bridge_vlan *vlan; + + port = mlx5_esw_bridge_port_lookup(vport_num, bridge); + if (!port) { + /* FDB is added asynchronously on wq while port might have been deleted + * concurrently. Report on 'info' logging level and skip the FDB offload. + */ + esw_info(esw->dev, "Failed to lookup bridge port (vport=%u)\n", vport_num); + return ERR_PTR(-EINVAL); + } + + vlan = mlx5_esw_bridge_vlan_lookup(vid, port); + if (!vlan) { + /* FDB is added asynchronously on wq while vlan might have been deleted + * concurrently. Report on 'info' logging level and skip the FDB offload. + */ + esw_info(esw->dev, "Failed to lookup bridge port vlan metadata (vport=%u)\n", + vport_num); + return ERR_PTR(-EINVAL); + } + + return vlan; +} + +static struct mlx5_esw_bridge_fdb_entry * +mlx5_esw_bridge_fdb_entry_init(struct net_device *dev, u16 vport_num, const unsigned char *addr, + u16 vid, bool added_by_user, struct mlx5_eswitch *esw, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_vlan *vlan = NULL; + struct mlx5_esw_bridge_fdb_entry *entry; + struct mlx5_flow_handle *handle; + struct mlx5_fc *counter; + struct mlx5e_priv *priv; + int err; + + if (bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG && vid) { + vlan = mlx5_esw_bridge_port_vlan_lookup(vid, vport_num, bridge, esw); + if (IS_ERR(vlan)) + return ERR_CAST(vlan); + } + + priv = netdev_priv(dev); + entry = kvzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return ERR_PTR(-ENOMEM); + + ether_addr_copy(entry->key.addr, addr); + entry->key.vid = vid; + entry->dev = dev; + entry->vport_num = vport_num; + entry->lastuse = jiffies; + if (added_by_user) + entry->flags |= MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER; + + counter = mlx5_fc_create(priv->mdev, true); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_ingress_fc_create; + } + entry->ingress_counter = counter; + + handle = mlx5_esw_bridge_ingress_flow_create(vport_num, addr, vlan, mlx5_fc_id(counter), + bridge); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + esw_warn(esw->dev, "Failed to create ingress flow(vport=%u,err=%d)\n", + vport_num, err); + goto err_ingress_flow_create; + } + entry->ingress_handle = handle; + + if (bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG) { + handle = mlx5_esw_bridge_ingress_filter_flow_create(vport_num, addr, bridge); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + esw_warn(esw->dev, "Failed to create ingress filter(vport=%u,err=%d)\n", + vport_num, err); + goto err_ingress_filter_flow_create; + } + entry->filter_handle = handle; + } + + handle = mlx5_esw_bridge_egress_flow_create(vport_num, addr, vlan, bridge); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + esw_warn(esw->dev, "Failed to create egress flow(vport=%u,err=%d)\n", + vport_num, err); + goto err_egress_flow_create; + } + entry->egress_handle = handle; + + err = rhashtable_insert_fast(&bridge->fdb_ht, &entry->ht_node, fdb_ht_params); + if (err) { + esw_warn(esw->dev, "Failed to insert FDB flow(vport=%u,err=%d)\n", vport_num, err); + goto err_ht_init; + } + + if (vlan) + list_add(&entry->vlan_list, &vlan->fdb_list); + else + INIT_LIST_HEAD(&entry->vlan_list); + list_add(&entry->list, &bridge->fdb_list); + + trace_mlx5_esw_bridge_fdb_entry_init(entry); + return entry; + +err_ht_init: + mlx5_del_flow_rules(entry->egress_handle); +err_egress_flow_create: + if (entry->filter_handle) + mlx5_del_flow_rules(entry->filter_handle); +err_ingress_filter_flow_create: + mlx5_del_flow_rules(entry->ingress_handle); +err_ingress_flow_create: + mlx5_fc_destroy(priv->mdev, entry->ingress_counter); +err_ingress_fc_create: + kvfree(entry); + return ERR_PTR(err); +} + +int mlx5_esw_bridge_ageing_time_set(unsigned long ageing_time, struct mlx5_eswitch *esw, + struct mlx5_vport *vport) +{ + if (!vport->bridge) + return -EINVAL; + + vport->bridge->ageing_time = ageing_time; + return 0; +} + +int mlx5_esw_bridge_vlan_filtering_set(bool enable, struct mlx5_eswitch *esw, + struct mlx5_vport *vport) +{ + struct mlx5_esw_bridge *bridge; + bool filtering; + + if (!vport->bridge) + return -EINVAL; + + bridge = vport->bridge; + filtering = bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG; + if (filtering == enable) + return 0; + + mlx5_esw_bridge_fdb_flush(bridge); + if (enable) + bridge->flags |= MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG; + else + bridge->flags &= ~MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG; + + return 0; +} + +static int mlx5_esw_bridge_vport_init(struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_esw_bridge *bridge, + struct mlx5_vport *vport) +{ + struct mlx5_eswitch *esw = br_offloads->esw; + struct mlx5_esw_bridge_port *port; + int err; + + port = kvzalloc(sizeof(*port), GFP_KERNEL); + if (!port) { + err = -ENOMEM; + goto err_port_alloc; + } + + port->vport_num = vport->vport; + xa_init(&port->vlans); + err = mlx5_esw_bridge_port_insert(port, bridge); + if (err) { + esw_warn(esw->dev, "Failed to insert port metadata (vport=%u,err=%d)\n", + vport->vport, err); + goto err_port_insert; + } + trace_mlx5_esw_bridge_vport_init(port); + + vport->bridge = bridge; + return 0; + +err_port_insert: + kvfree(port); +err_port_alloc: + mlx5_esw_bridge_put(br_offloads, bridge); + return err; +} + +static int mlx5_esw_bridge_vport_cleanup(struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_vport *vport) +{ + struct mlx5_esw_bridge *bridge = vport->bridge; + struct mlx5_esw_bridge_fdb_entry *entry, *tmp; + struct mlx5_esw_bridge_port *port; + + list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list) + if (entry->vport_num == vport->vport) + mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge); + + port = mlx5_esw_bridge_port_lookup(vport->vport, bridge); + if (!port) { + WARN(1, "Vport %u metadata not found on bridge", vport->vport); + return -EINVAL; + } + + trace_mlx5_esw_bridge_vport_cleanup(port); + mlx5_esw_bridge_port_vlans_flush(port, bridge); + mlx5_esw_bridge_port_erase(port, bridge); + kvfree(port); + mlx5_esw_bridge_put(br_offloads, bridge); + vport->bridge = NULL; + return 0; +} + +int mlx5_esw_bridge_vport_link(int ifindex, struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_vport *vport, struct netlink_ext_ack *extack) +{ + struct mlx5_esw_bridge *bridge; + int err; + + WARN_ON(vport->bridge); + + bridge = mlx5_esw_bridge_lookup(ifindex, br_offloads); + if (IS_ERR(bridge)) { + NL_SET_ERR_MSG_MOD(extack, "Error checking for existing bridge with same ifindex"); + return PTR_ERR(bridge); + } + + err = mlx5_esw_bridge_vport_init(br_offloads, bridge, vport); + if (err) + NL_SET_ERR_MSG_MOD(extack, "Error initializing port"); + return err; +} + +int mlx5_esw_bridge_vport_unlink(int ifindex, struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_vport *vport, struct netlink_ext_ack *extack) +{ + struct mlx5_esw_bridge *bridge = vport->bridge; + int err; + + if (!bridge) { + NL_SET_ERR_MSG_MOD(extack, "Port is not attached to any bridge"); + return -EINVAL; + } + if (bridge->ifindex != ifindex) { + NL_SET_ERR_MSG_MOD(extack, "Port is attached to another bridge"); + return -EINVAL; + } + + err = mlx5_esw_bridge_vport_cleanup(br_offloads, vport); + if (err) + NL_SET_ERR_MSG_MOD(extack, "Port cleanup failed"); + return err; +} + +int mlx5_esw_bridge_port_vlan_add(u16 vid, u16 flags, struct mlx5_eswitch *esw, + struct mlx5_vport *vport, struct netlink_ext_ack *extack) +{ + struct mlx5_esw_bridge_port *port; + struct mlx5_esw_bridge_vlan *vlan; + + port = mlx5_esw_bridge_port_lookup(vport->vport, vport->bridge); + if (!port) + return -EINVAL; + + vlan = mlx5_esw_bridge_vlan_lookup(vid, port); + if (vlan) { + if (vlan->flags == flags) + return 0; + mlx5_esw_bridge_vlan_cleanup(port, vlan, vport->bridge); + } + + vlan = mlx5_esw_bridge_vlan_create(vid, flags, port, esw); + if (IS_ERR(vlan)) { + NL_SET_ERR_MSG_MOD(extack, "Failed to create VLAN entry"); + return PTR_ERR(vlan); + } + return 0; +} + +void mlx5_esw_bridge_port_vlan_del(u16 vid, struct mlx5_eswitch *esw, struct mlx5_vport *vport) +{ + struct mlx5_esw_bridge_port *port; + struct mlx5_esw_bridge_vlan *vlan; + + port = mlx5_esw_bridge_port_lookup(vport->vport, vport->bridge); + if (!port) + return; + + vlan = mlx5_esw_bridge_vlan_lookup(vid, port); + if (!vlan) + return; + mlx5_esw_bridge_vlan_cleanup(port, vlan, vport->bridge); +} + +void mlx5_esw_bridge_fdb_create(struct net_device *dev, struct mlx5_eswitch *esw, + struct mlx5_vport *vport, + struct switchdev_notifier_fdb_info *fdb_info) +{ + struct mlx5_esw_bridge *bridge = vport->bridge; + struct mlx5_esw_bridge_fdb_entry *entry; + u16 vport_num = vport->vport; + + if (!bridge) { + esw_info(esw->dev, "Vport is not assigned to bridge (vport=%u)\n", vport_num); + return; + } + + entry = mlx5_esw_bridge_fdb_entry_init(dev, vport_num, fdb_info->addr, fdb_info->vid, + fdb_info->added_by_user, esw, bridge); + if (IS_ERR(entry)) + return; + + if (entry->flags & MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER) + mlx5_esw_bridge_fdb_offload_notify(dev, entry->key.addr, entry->key.vid, + SWITCHDEV_FDB_OFFLOADED); + else + /* Take over dynamic entries to prevent kernel bridge from aging them out. */ + mlx5_esw_bridge_fdb_offload_notify(dev, entry->key.addr, entry->key.vid, + SWITCHDEV_FDB_ADD_TO_BRIDGE); +} + +void mlx5_esw_bridge_fdb_remove(struct net_device *dev, struct mlx5_eswitch *esw, + struct mlx5_vport *vport, + struct switchdev_notifier_fdb_info *fdb_info) +{ + struct mlx5_esw_bridge *bridge = vport->bridge; + struct mlx5_esw_bridge_fdb_entry *entry; + struct mlx5_esw_bridge_fdb_key key; + u16 vport_num = vport->vport; + + if (!bridge) { + esw_warn(esw->dev, "Vport is not assigned to bridge (vport=%u)\n", vport_num); + return; + } + + ether_addr_copy(key.addr, fdb_info->addr); + key.vid = fdb_info->vid; + entry = rhashtable_lookup_fast(&bridge->fdb_ht, &key, fdb_ht_params); + if (!entry) { + esw_warn(esw->dev, + "FDB entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n", + key.addr, key.vid, vport_num); + return; + } + + if (!(entry->flags & MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER)) + mlx5_esw_bridge_fdb_offload_notify(dev, entry->key.addr, entry->key.vid, + SWITCHDEV_FDB_DEL_TO_BRIDGE); + mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge); +} + +void mlx5_esw_bridge_update(struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_esw_bridge_fdb_entry *entry, *tmp; + struct mlx5_esw_bridge *bridge; + + list_for_each_entry(bridge, &br_offloads->bridges, list) { + list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list) { + unsigned long lastuse = + (unsigned long)mlx5_fc_query_lastuse(entry->ingress_counter); + + if (entry->flags & MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER) + continue; + + if (time_after(lastuse, entry->lastuse)) { + mlx5_esw_bridge_fdb_entry_refresh(lastuse, entry); + } else if (time_is_before_jiffies(entry->lastuse + bridge->ageing_time)) { + mlx5_esw_bridge_fdb_offload_notify(entry->dev, entry->key.addr, + entry->key.vid, + SWITCHDEV_FDB_DEL_TO_BRIDGE); + mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge); + } + } + } +} + +static void mlx5_esw_bridge_flush(struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_eswitch *esw = br_offloads->esw; + struct mlx5_vport *vport; + unsigned long i; + + mlx5_esw_for_each_vport(esw, i, vport) + if (vport->bridge) + mlx5_esw_bridge_vport_cleanup(br_offloads, vport); + + WARN_ONCE(!list_empty(&br_offloads->bridges), + "Cleaning up bridge offloads while still having bridges attached\n"); +} + +struct mlx5_esw_bridge_offloads *mlx5_esw_bridge_init(struct mlx5_eswitch *esw) +{ + struct mlx5_esw_bridge_offloads *br_offloads; + + br_offloads = kvzalloc(sizeof(*br_offloads), GFP_KERNEL); + if (!br_offloads) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&br_offloads->bridges); + br_offloads->esw = esw; + esw->br_offloads = br_offloads; + + return br_offloads; +} + +void mlx5_esw_bridge_cleanup(struct mlx5_eswitch *esw) +{ + struct mlx5_esw_bridge_offloads *br_offloads = esw->br_offloads; + + if (!br_offloads) + return; + + mlx5_esw_bridge_flush(br_offloads); + + esw->br_offloads = NULL; + kvfree(br_offloads); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h new file mode 100644 index 000000000000..d826942b27fc --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#ifndef __MLX5_ESW_BRIDGE_H__ +#define __MLX5_ESW_BRIDGE_H__ + +#include <linux/notifier.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include "eswitch.h" + +struct mlx5_flow_table; +struct mlx5_flow_group; + +struct mlx5_esw_bridge_offloads { + struct mlx5_eswitch *esw; + struct list_head bridges; + struct notifier_block netdev_nb; + struct notifier_block nb_blk; + struct notifier_block nb; + struct workqueue_struct *wq; + struct delayed_work update_work; + + struct mlx5_flow_table *ingress_ft; + struct mlx5_flow_group *ingress_vlan_fg; + struct mlx5_flow_group *ingress_filter_fg; + struct mlx5_flow_group *ingress_mac_fg; + + struct mlx5_flow_table *skip_ft; +}; + +struct mlx5_esw_bridge_offloads *mlx5_esw_bridge_init(struct mlx5_eswitch *esw); +void mlx5_esw_bridge_cleanup(struct mlx5_eswitch *esw); +int mlx5_esw_bridge_vport_link(int ifindex, struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_vport *vport, struct netlink_ext_ack *extack); +int mlx5_esw_bridge_vport_unlink(int ifindex, struct mlx5_esw_bridge_offloads *br_offloads, + struct mlx5_vport *vport, struct netlink_ext_ack *extack); +void mlx5_esw_bridge_fdb_create(struct net_device *dev, struct mlx5_eswitch *esw, + struct mlx5_vport *vport, + struct switchdev_notifier_fdb_info *fdb_info); +void mlx5_esw_bridge_fdb_remove(struct net_device *dev, struct mlx5_eswitch *esw, + struct mlx5_vport *vport, + struct switchdev_notifier_fdb_info *fdb_info); +void mlx5_esw_bridge_update(struct mlx5_esw_bridge_offloads *br_offloads); +int mlx5_esw_bridge_ageing_time_set(unsigned long ageing_time, struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +int mlx5_esw_bridge_vlan_filtering_set(bool enable, struct mlx5_eswitch *esw, + struct mlx5_vport *vport); +int mlx5_esw_bridge_port_vlan_add(u16 vid, u16 flags, struct mlx5_eswitch *esw, + struct mlx5_vport *vport, struct netlink_ext_ack *extack); +void mlx5_esw_bridge_port_vlan_del(u16 vid, struct mlx5_eswitch *esw, struct mlx5_vport *vport); + +#endif /* __MLX5_ESW_BRIDGE_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h new file mode 100644 index 000000000000..d9ab2e8bc2cb --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge_priv.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#ifndef _MLX5_ESW_BRIDGE_PRIVATE_ +#define _MLX5_ESW_BRIDGE_PRIVATE_ + +#include <linux/netdevice.h> +#include <linux/if_bridge.h> +#include <linux/if_vlan.h> +#include <linux/if_ether.h> +#include <linux/rhashtable.h> +#include <linux/xarray.h> +#include "fs_core.h" + +struct mlx5_esw_bridge_fdb_key { + unsigned char addr[ETH_ALEN]; + u16 vid; +}; + +enum { + MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER = BIT(0), +}; + +struct mlx5_esw_bridge_fdb_entry { + struct mlx5_esw_bridge_fdb_key key; + struct rhash_head ht_node; + struct net_device *dev; + struct list_head list; + struct list_head vlan_list; + u16 vport_num; + u16 flags; + + struct mlx5_flow_handle *ingress_handle; + struct mlx5_fc *ingress_counter; + unsigned long lastuse; + struct mlx5_flow_handle *egress_handle; + struct mlx5_flow_handle *filter_handle; +}; + +struct mlx5_esw_bridge_vlan { + u16 vid; + u16 flags; + struct list_head fdb_list; + struct mlx5_pkt_reformat *pkt_reformat_push; + struct mlx5_pkt_reformat *pkt_reformat_pop; +}; + +struct mlx5_esw_bridge_port { + u16 vport_num; + struct xarray vlans; +}; + +#endif /* _MLX5_ESW_BRIDGE_PRIVATE_ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h new file mode 100644 index 000000000000..227964b7d3b9 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/diag/bridge_tracepoint.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mlx5 + +#if !defined(_MLX5_ESW_BRIDGE_TRACEPOINT_) || defined(TRACE_HEADER_MULTI_READ) +#define _MLX5_ESW_BRIDGE_TRACEPOINT_ + +#include <linux/tracepoint.h> +#include "../bridge_priv.h" + +DECLARE_EVENT_CLASS(mlx5_esw_bridge_fdb_template, + TP_PROTO(const struct mlx5_esw_bridge_fdb_entry *fdb), + TP_ARGS(fdb), + TP_STRUCT__entry( + __array(char, dev_name, IFNAMSIZ) + __array(unsigned char, addr, ETH_ALEN) + __field(u16, vid) + __field(u16, flags) + __field(unsigned int, used) + ), + TP_fast_assign( + strncpy(__entry->dev_name, + netdev_name(fdb->dev), + IFNAMSIZ); + memcpy(__entry->addr, fdb->key.addr, ETH_ALEN); + __entry->vid = fdb->key.vid; + __entry->flags = fdb->flags; + __entry->used = jiffies_to_msecs(jiffies - fdb->lastuse) + ), + TP_printk("net_device=%s addr=%pM vid=%hu flags=%hx used=%u", + __entry->dev_name, + __entry->addr, + __entry->vid, + __entry->flags, + __entry->used / 1000) + ); + +DEFINE_EVENT(mlx5_esw_bridge_fdb_template, + mlx5_esw_bridge_fdb_entry_init, + TP_PROTO(const struct mlx5_esw_bridge_fdb_entry *fdb), + TP_ARGS(fdb) + ); +DEFINE_EVENT(mlx5_esw_bridge_fdb_template, + mlx5_esw_bridge_fdb_entry_refresh, + TP_PROTO(const struct mlx5_esw_bridge_fdb_entry *fdb), + TP_ARGS(fdb) + ); +DEFINE_EVENT(mlx5_esw_bridge_fdb_template, + mlx5_esw_bridge_fdb_entry_cleanup, + TP_PROTO(const struct mlx5_esw_bridge_fdb_entry *fdb), + TP_ARGS(fdb) + ); + +DECLARE_EVENT_CLASS(mlx5_esw_bridge_vlan_template, + TP_PROTO(const struct mlx5_esw_bridge_vlan *vlan), + TP_ARGS(vlan), + TP_STRUCT__entry( + __field(u16, vid) + __field(u16, flags) + ), + TP_fast_assign( + __entry->vid = vlan->vid; + __entry->flags = vlan->flags; + ), + TP_printk("vid=%hu flags=%hx", + __entry->vid, + __entry->flags) + ); + +DEFINE_EVENT(mlx5_esw_bridge_vlan_template, + mlx5_esw_bridge_vlan_create, + TP_PROTO(const struct mlx5_esw_bridge_vlan *vlan), + TP_ARGS(vlan) + ); +DEFINE_EVENT(mlx5_esw_bridge_vlan_template, + mlx5_esw_bridge_vlan_cleanup, + TP_PROTO(const struct mlx5_esw_bridge_vlan *vlan), + TP_ARGS(vlan) + ); + +DECLARE_EVENT_CLASS(mlx5_esw_bridge_port_template, + TP_PROTO(const struct mlx5_esw_bridge_port *port), + TP_ARGS(port), + TP_STRUCT__entry( + __field(u16, vport_num) + ), + TP_fast_assign( + __entry->vport_num = port->vport_num; + ), + TP_printk("vport_num=%hu", __entry->vport_num) + ); + +DEFINE_EVENT(mlx5_esw_bridge_port_template, + mlx5_esw_bridge_vport_init, + TP_PROTO(const struct mlx5_esw_bridge_port *port), + TP_ARGS(port) + ); +DEFINE_EVENT(mlx5_esw_bridge_port_template, + mlx5_esw_bridge_vport_cleanup, + TP_PROTO(const struct mlx5_esw_bridge_port *port), + TP_ARGS(port) + ); + +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH esw/diag +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE bridge_tracepoint +#include <trace/define_trace.h> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 570f2280823c..97e6cb6f13c1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -35,6 +35,7 @@ #include <linux/mlx5/mlx5_ifc.h> #include <linux/mlx5/vport.h> #include <linux/mlx5/fs.h> +#include <linux/mlx5/mpfs.h> #include "esw/acl/lgcy.h" #include "esw/legacy.h" #include "mlx5_core.h" @@ -1053,6 +1054,12 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, u16 vport_num, goto err_vhca_mapping; } + /* External controller host PF has factory programmed MAC. + * Read it from the device. + */ + if (mlx5_core_is_ecpf(esw->dev) && vport_num == MLX5_VPORT_PF) + mlx5_query_nic_vport_mac_address(esw->dev, vport_num, true, vport->info.mac); + esw_vport_change_handle_locked(vport); esw->enabled_vports++; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 64ccb2bc0b58..48cac5bf606d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -150,6 +150,8 @@ enum mlx5_eswitch_vport_event { MLX5_VPORT_PROMISC_CHANGE = BIT(3), }; +struct mlx5_esw_bridge; + struct mlx5_vport { struct mlx5_core_dev *dev; struct hlist_head uc_list[MLX5_L2_ADDR_HASH_SIZE]; @@ -178,6 +180,7 @@ struct mlx5_vport { enum mlx5_eswitch_vport_event enabled_events; int index; struct devlink_port *dl_port; + struct mlx5_esw_bridge *bridge; }; struct mlx5_esw_indir_table; @@ -196,6 +199,7 @@ struct mlx5_eswitch_fdb { struct offloads_fdb { struct mlx5_flow_namespace *ns; + struct mlx5_flow_table *tc_miss_table; struct mlx5_flow_table *slow_fdb; struct mlx5_flow_group *send_to_vport_grp; struct mlx5_flow_group *send_to_vport_meta_grp; @@ -270,6 +274,8 @@ enum { MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED = BIT(1), }; +struct mlx5_esw_bridge_offloads; + struct mlx5_eswitch { struct mlx5_core_dev *dev; struct mlx5_nb nb; @@ -299,6 +305,7 @@ struct mlx5_eswitch { u32 root_tsar_id; } qos; + struct mlx5_esw_bridge_offloads *br_offloads; struct mlx5_esw_offload offloads; int mode; u16 manager_vport; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index db1e74280e57..7579f3402776 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -219,7 +219,8 @@ esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_fs_chains *chains, int i) { - flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; + if (mlx5_chains_ignore_flow_level_supported(chains)) + flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest[i].ft = mlx5_chains_get_tc_end_ft(chains); } @@ -1633,7 +1634,21 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) } esw->fdb_table.offloads.slow_fdb = fdb; - err = esw_chains_create(esw, fdb); + /* Create empty TC-miss managed table. This allows plugging in following + * priorities without directly exposing their level 0 table to + * eswitch_offloads and passing it as miss_fdb to following call to + * esw_chains_create(). + */ + memset(&ft_attr, 0, sizeof(ft_attr)); + ft_attr.prio = FDB_TC_MISS; + esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); + if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { + err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); + esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); + goto tc_miss_table_err; + } + + err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); if (err) { esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); goto fdb_chains_err; @@ -1778,6 +1793,8 @@ send_vport_meta_err: send_vport_err: esw_chains_destroy(esw, esw_chains(esw)); fdb_chains_err: + mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); +tc_miss_table_err: mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); slow_fdb_err: /* Holds true only as long as DMFS is the default */ @@ -1805,6 +1822,7 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) esw_chains_destroy(esw, esw_chains(esw)); + mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); /* Holds true only as long as DMFS is the default */ mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c index a81ece94f599..b45954905845 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c @@ -65,7 +65,7 @@ mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev, { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_namespace *root_ns; - int err; + int err, err2; root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); if (!root_ns) { @@ -76,33 +76,34 @@ mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev, /* As this is the terminating action then the termination table is the * same prio as the slow path */ - ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | + ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED | MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; - ft_attr.prio = FDB_SLOW_PATH; + ft_attr.prio = FDB_TC_OFFLOAD; ft_attr.max_fte = 1; + ft_attr.level = 1; ft_attr.autogroup.max_num_groups = 1; tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); if (IS_ERR(tt->termtbl)) { - esw_warn(dev, "Failed to create termination table (error %d)\n", - IS_ERR(tt->termtbl)); - return -EOPNOTSUPP; + err = PTR_ERR(tt->termtbl); + esw_warn(dev, "Failed to create termination table, err %pe\n", tt->termtbl); + return err; } tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act, &tt->dest, 1); if (IS_ERR(tt->rule)) { - esw_warn(dev, "Failed to create termination table rule (error %d)\n", - IS_ERR(tt->rule)); + err = PTR_ERR(tt->rule); + esw_warn(dev, "Failed to create termination table rule, err %pe\n", tt->rule); goto add_flow_err; } return 0; add_flow_err: - err = mlx5_destroy_flow_table(tt->termtbl); - if (err) - esw_warn(dev, "Failed to destroy termination table\n"); + err2 = mlx5_destroy_flow_table(tt->termtbl); + if (err2) + esw_warn(dev, "Failed to destroy termination table, err %d\n", err2); - return -EOPNOTSUPP; + return err; } static struct mlx5_termtbl_handle * @@ -172,19 +173,6 @@ mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw, } } -static bool mlx5_eswitch_termtbl_is_encap_reformat(struct mlx5_pkt_reformat *rt) -{ - switch (rt->reformat_type) { - case MLX5_REFORMAT_TYPE_L2_TO_VXLAN: - case MLX5_REFORMAT_TYPE_L2_TO_NVGRE: - case MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL: - case MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL: - return true; - default: - return false; - } -} - static void mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src, struct mlx5_flow_act *dst) @@ -202,14 +190,6 @@ mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src, memset(&src->vlan[1], 0, sizeof(src->vlan[1])); } } - - if (src->action & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT && - mlx5_eswitch_termtbl_is_encap_reformat(src->pkt_reformat)) { - src->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; - dst->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; - dst->pkt_reformat = src->pkt_reformat; - src->pkt_reformat = NULL; - } } static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw, @@ -238,6 +218,7 @@ mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw, int i; if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) || + !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) || attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH || !mlx5_eswitch_offload_is_uplink_port(esw, spec)) return false; @@ -279,12 +260,19 @@ mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw, if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT) continue; + if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) { + term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat; + } else { + term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + term_tbl_act.pkt_reformat = NULL; + } + /* get the terminating table for the action list */ tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act, &dest[i], attr); if (IS_ERR(tt)) { - esw_warn(esw->dev, "Failed to get termination table (error %d)\n", - IS_ERR(tt)); + esw_warn(esw->dev, "Failed to get termination table, err %pe\n", tt); goto revert_changes; } attr->dests[num_vport_dests].termtbl = tt; @@ -301,6 +289,9 @@ mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw, goto revert_changes; /* create the FTE */ + flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + flow_act->pkt_reformat = NULL; + flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest); if (IS_ERR(rule)) goto revert_changes; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 8e06731d3cb3..896a6c3dbdb7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -36,6 +36,7 @@ #include "fs_core.h" #include "fs_cmd.h" +#include "fs_ft_pool.h" #include "mlx5_core.h" #include "eswitch.h" @@ -49,9 +50,11 @@ static int mlx5_cmd_stub_update_root_ft(struct mlx5_flow_root_namespace *ns, static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int log_size, + unsigned int size, struct mlx5_flow_table *next_ft) { + ft->max_fte = size ? roundup_pow_of_two(size) : 1; + return 0; } @@ -108,9 +111,7 @@ static int mlx5_cmd_stub_delete_fte(struct mlx5_flow_root_namespace *ns, } static int mlx5_cmd_stub_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, - int reformat_type, - size_t size, - void *reformat_data, + struct mlx5_pkt_reformat_params *params, enum mlx5_flow_namespace_type namespace, struct mlx5_pkt_reformat *pkt_reformat) { @@ -181,7 +182,7 @@ static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns, static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int log_size, + unsigned int size, struct mlx5_flow_table *next_ft) { int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT); @@ -192,12 +193,18 @@ static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_core_dev *dev = ns->dev; int err; + if (size != POOL_NEXT_SIZE) + size = roundup_pow_of_two(size); + size = mlx5_ft_pool_get_avail_sz(dev, ft->type, size); + if (!size) + return -ENOSPC; + MLX5_SET(create_flow_table_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_TABLE); MLX5_SET(create_flow_table_in, in, table_type, ft->type); MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level); - MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, log_size); + MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0); MLX5_SET(create_flow_table_in, in, vport_number, ft->vport); MLX5_SET(create_flow_table_in, in, other_vport, !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); @@ -234,9 +241,14 @@ static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns, } err = mlx5_cmd_exec_inout(dev, create_flow_table, in, out); - if (!err) + if (!err) { ft->id = MLX5_GET(create_flow_table_out, out, table_id); + ft->max_fte = size; + } else { + mlx5_ft_pool_put_sz(ns->dev, size); + } + return err; } @@ -245,6 +257,7 @@ static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns, { u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {}; struct mlx5_core_dev *dev = ns->dev; + int err; MLX5_SET(destroy_flow_table_in, in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE); @@ -254,7 +267,11 @@ static int mlx5_cmd_destroy_flow_table(struct mlx5_flow_root_namespace *ns, MLX5_SET(destroy_flow_table_in, in, other_vport, !!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT)); - return mlx5_cmd_exec_in(dev, destroy_flow_table, in); + err = mlx5_cmd_exec_in(dev, destroy_flow_table, in); + if (!err) + mlx5_ft_pool_put_sz(ns->dev, ft->max_fte); + + return err; } static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns, @@ -682,9 +699,7 @@ int mlx5_cmd_fc_bulk_query(struct mlx5_core_dev *dev, u32 base_id, int bulk_len, } static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, - int reformat_type, - size_t size, - void *reformat_data, + struct mlx5_pkt_reformat_params *params, enum mlx5_flow_namespace_type namespace, struct mlx5_pkt_reformat *pkt_reformat) { @@ -702,14 +717,14 @@ static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, else max_encap_size = MLX5_CAP_FLOWTABLE(dev, max_encap_header_size); - if (size > max_encap_size) { + if (params->size > max_encap_size) { mlx5_core_warn(dev, "encap size %zd too big, max supported is %d\n", - size, max_encap_size); + params->size, max_encap_size); return -EINVAL; } - in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) + size, - GFP_KERNEL); + in = kzalloc(MLX5_ST_SZ_BYTES(alloc_packet_reformat_context_in) + + params->size, GFP_KERNEL); if (!in) return -ENOMEM; @@ -718,15 +733,20 @@ static int mlx5_cmd_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, reformat = MLX5_ADDR_OF(packet_reformat_context_in, packet_reformat_context_in, reformat_data); - inlen = reformat - (void *)in + size; + inlen = reformat - (void *)in + params->size; MLX5_SET(alloc_packet_reformat_context_in, in, opcode, MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT); MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, - reformat_data_size, size); + reformat_data_size, params->size); + MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, + reformat_type, params->type); + MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, + reformat_param_0, params->param_0); MLX5_SET(packet_reformat_context_in, packet_reformat_context_in, - reformat_type, reformat_type); - memcpy(reformat, reformat_data, size); + reformat_param_1, params->param_1); + if (params->data && params->size) + memcpy(reformat, params->data, params->size); err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h index d62de642eca9..5ecd33cdc087 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.h @@ -38,7 +38,7 @@ struct mlx5_flow_cmds { int (*create_flow_table)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int log_size, + unsigned int size, struct mlx5_flow_table *next_ft); int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft); @@ -77,9 +77,7 @@ struct mlx5_flow_cmds { bool disconnect); int (*packet_reformat_alloc)(struct mlx5_flow_root_namespace *ns, - int reformat_type, - size_t size, - void *reformat_data, + struct mlx5_pkt_reformat_params *params, enum mlx5_flow_namespace_type namespace, struct mlx5_pkt_reformat *pkt_reformat); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index f74d2c834037..d7bf0a3e4a52 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -38,6 +38,7 @@ #include "mlx5_core.h" #include "fs_core.h" #include "fs_cmd.h" +#include "fs_ft_pool.h" #include "diag/fs_tracepoint.h" #include "accel/ipsec.h" #include "fpga/ipsec.h" @@ -752,7 +753,7 @@ static struct mlx5_flow_group *alloc_insert_flow_group(struct mlx5_flow_table *f return fg; } -static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_fte, +static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, enum fs_flow_table_type table_type, enum fs_flow_table_op_mod op_mod, u32 flags) @@ -775,7 +776,6 @@ static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_ft ft->op_mod = op_mod; ft->type = table_type; ft->vport = vport; - ft->max_fte = max_fte; ft->flags = flags; INIT_LIST_HEAD(&ft->fwd_rules); mutex_init(&ft->lock); @@ -1070,7 +1070,6 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa struct mlx5_flow_table *next_ft; struct fs_prio *fs_prio = NULL; struct mlx5_flow_table *ft; - int log_table_sz; int err; if (!root) { @@ -1101,7 +1100,6 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa */ ft = alloc_flow_table(ft_attr->level, vport, - ft_attr->max_fte ? roundup_pow_of_two(ft_attr->max_fte) : 0, root->table_type, op_mod, ft_attr->flags); if (IS_ERR(ft)) { @@ -1110,12 +1108,11 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa } tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table); - log_table_sz = ft->max_fte ? ilog2(ft->max_fte) : 0; next_ft = unmanaged ? ft_attr->next_ft : find_next_chained_ft(fs_prio); ft->def_miss_action = ns->def_miss_action; ft->ns = ns; - err = root->cmds->create_flow_table(root, ft, log_table_sz, next_ft); + err = root->cmds->create_flow_table(root, ft, ft_attr->max_fte, next_ft); if (err) goto free_ft; @@ -1170,28 +1167,36 @@ mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns, ft_attr.level = level; ft_attr.prio = prio; + ft_attr.max_fte = 1; + return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0); } EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table); +#define MAX_FLOW_GROUP_SIZE BIT(24) struct mlx5_flow_table* mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, struct mlx5_flow_table_attr *ft_attr) { int num_reserved_entries = ft_attr->autogroup.num_reserved_entries; - int autogroups_max_fte = ft_attr->max_fte - num_reserved_entries; int max_num_groups = ft_attr->autogroup.max_num_groups; struct mlx5_flow_table *ft; - - if (max_num_groups > autogroups_max_fte) - return ERR_PTR(-EINVAL); - if (num_reserved_entries > ft_attr->max_fte) - return ERR_PTR(-EINVAL); + int autogroups_max_fte; ft = mlx5_create_flow_table(ns, ft_attr); if (IS_ERR(ft)) return ft; + autogroups_max_fte = ft->max_fte - num_reserved_entries; + if (max_num_groups > autogroups_max_fte) + goto err_validate; + if (num_reserved_entries > ft->max_fte) + goto err_validate; + + /* Align the number of groups according to the largest group size */ + if (autogroups_max_fte / (max_num_groups + 1) > MAX_FLOW_GROUP_SIZE) + max_num_groups = (autogroups_max_fte / MAX_FLOW_GROUP_SIZE) - 1; + ft->autogroup.active = true; ft->autogroup.required_groups = max_num_groups; ft->autogroup.max_fte = autogroups_max_fte; @@ -1199,6 +1204,10 @@ mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, ft->autogroup.group_size = autogroups_max_fte / (max_num_groups + 1); return ft; + +err_validate: + mlx5_destroy_flow_table(ft); + return ERR_PTR(-ENOSPC); } EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table); @@ -1495,7 +1504,9 @@ static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1, (d1->type == MLX5_FLOW_DESTINATION_TYPE_TIR && d1->tir_num == d2->tir_num) || (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM && - d1->ft_num == d2->ft_num)) + d1->ft_num == d2->ft_num) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER && + d1->sampler_id == d2->sampler_id)) return true; } @@ -2592,6 +2603,7 @@ void mlx5_cleanup_fs(struct mlx5_core_dev *dev) mlx5_cleanup_fc_stats(dev); kmem_cache_destroy(steering->ftes_cache); kmem_cache_destroy(steering->fgs_cache); + mlx5_ft_pool_destroy(dev); kfree(steering); } @@ -2770,6 +2782,18 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) if (err) goto out_err; + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_TC_MISS, 1); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_BR_OFFLOAD, 3); + if (IS_ERR(maj_prio)) { + err = PTR_ERR(maj_prio); + goto out_err; + } + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_SLOW_PATH, 1); if (IS_ERR(maj_prio)) { err = PTR_ERR(maj_prio); @@ -2942,9 +2966,16 @@ int mlx5_init_fs(struct mlx5_core_dev *dev) if (err) return err; + err = mlx5_ft_pool_init(dev); + if (err) + return err; + steering = kzalloc(sizeof(*steering), GFP_KERNEL); - if (!steering) - return -ENOMEM; + if (!steering) { + err = -ENOMEM; + goto err; + } + steering->dev = dev; dev->priv.steering = steering; @@ -3151,9 +3182,7 @@ void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev, EXPORT_SYMBOL(mlx5_modify_header_dealloc); struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev, - int reformat_type, - size_t size, - void *reformat_data, + struct mlx5_pkt_reformat_params *params, enum mlx5_flow_namespace_type ns_type) { struct mlx5_pkt_reformat *pkt_reformat; @@ -3169,9 +3198,8 @@ struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev, return ERR_PTR(-ENOMEM); pkt_reformat->ns_type = ns_type; - pkt_reformat->reformat_type = reformat_type; - err = root->cmds->packet_reformat_alloc(root, reformat_type, size, - reformat_data, ns_type, + pkt_reformat->reformat_type = params->type; + err = root->cmds->packet_reformat_alloc(root, params, ns_type, pkt_reformat); if (err) { kfree(pkt_reformat); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h index e577a2c424af..7317cdeab661 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h @@ -331,6 +331,7 @@ void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev); #define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \ (type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \ + (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \ (type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \ (type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \ (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c new file mode 100644 index 000000000000..c14590acc772 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021 Mellanox Technologies. */ + +#include "fs_ft_pool.h" + +/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS), + * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated + * for each flow table pool. We can allocate up to 16M of each pool, + * and we keep track of how much we used via mlx5_ft_pool_get_avail_sz. + * Firmware doesn't report any of this for now. + * ESW_POOL is expected to be sorted from large to small and match firmware + * pools. + */ +#define FT_SIZE (16 * 1024 * 1024) +static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024, + 1 * 1024 * 1024, + 64 * 1024, + 128, + 1 /* size for termination tables */ }; +struct mlx5_ft_pool { + int ft_left[ARRAY_SIZE(FT_POOLS)]; +}; + +int mlx5_ft_pool_init(struct mlx5_core_dev *dev) +{ + struct mlx5_ft_pool *ft_pool; + int i; + + ft_pool = kzalloc(sizeof(*ft_pool), GFP_KERNEL); + if (!ft_pool) + return -ENOMEM; + + for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) + ft_pool->ft_left[i] = FT_SIZE / FT_POOLS[i]; + + dev->priv.ft_pool = ft_pool; + return 0; +} + +void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev) +{ + kfree(dev->priv.ft_pool); +} + +int +mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type, + int desired_size) +{ + u32 max_ft_size = 1 << MLX5_CAP_FLOWTABLE_TYPE(dev, log_max_ft_size, table_type); + int i, found_i = -1; + + for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { + if (dev->priv.ft_pool->ft_left[i] && FT_POOLS[i] >= desired_size && + FT_POOLS[i] <= max_ft_size) { + found_i = i; + if (desired_size != POOL_NEXT_SIZE) + break; + } + } + + if (found_i != -1) { + --dev->priv.ft_pool->ft_left[found_i]; + return FT_POOLS[found_i]; + } + + return 0; +} + +void +mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz) +{ + int i; + + if (!sz) + return; + + for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { + if (sz == FT_POOLS[i]) { + ++dev->priv.ft_pool->ft_left[i]; + return; + } + } + + WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h new file mode 100644 index 000000000000..25f4274b372b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_ft_pool.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#ifndef __MLX5_FS_FT_POOL_H__ +#define __MLX5_FS_FT_POOL_H__ + +#include <linux/mlx5/driver.h> +#include "fs_core.h" + +#define POOL_NEXT_SIZE 0 + +int mlx5_ft_pool_init(struct mlx5_core_dev *dev); +void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev); + +int +mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type, + int desired_size); +void +mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz); + +#endif /* __MLX5_FS_FT_POOL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index 02558ac2ace6..016d26f809a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -148,6 +148,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) if (err) return err; + if (MLX5_CAP_GEN(dev, hca_cap_2)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2); + if (err) + return err; + } + if (MLX5_CAP_GEN(dev, eth_net_offloads)) { err = mlx5_core_get_caps(dev, MLX5_CAP_ETHERNET_OFFLOADS); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index d5d57630015f..106b50e42b46 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -349,6 +349,9 @@ static void mlx5_sync_reset_abort_event(struct work_struct *work) reset_abort_work); struct mlx5_core_dev *dev = fw_reset->dev; + if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) + return; + mlx5_sync_reset_clear_reset_requested(dev, true); mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n"); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index 97d96fc38a65..0e487ec57d5c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -150,6 +150,7 @@ enum mlx5_ptys_rate { MLX5_PTYS_RATE_FDR = 1 << 4, MLX5_PTYS_RATE_EDR = 1 << 5, MLX5_PTYS_RATE_HDR = 1 << 6, + MLX5_PTYS_RATE_NDR = 1 << 7, }; static inline int mlx5_ptys_rate_enum_to_int(enum mlx5_ptys_rate rate) @@ -162,6 +163,7 @@ static inline int mlx5_ptys_rate_enum_to_int(enum mlx5_ptys_rate rate) case MLX5_PTYS_RATE_FDR: return 14000; case MLX5_PTYS_RATE_EDR: return 25000; case MLX5_PTYS_RATE_HDR: return 50000; + case MLX5_PTYS_RATE_NDR: return 100000; default: return -1; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag.c index b8748390335f..5c043c5cc403 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag.c @@ -93,6 +93,64 @@ int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev) } EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag); +static int mlx5_lag_netdev_event(struct notifier_block *this, + unsigned long event, void *ptr); +static void mlx5_do_bond_work(struct work_struct *work); + +static void mlx5_ldev_free(struct kref *ref) +{ + struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref); + + if (ldev->nb.notifier_call) + unregister_netdevice_notifier_net(&init_net, &ldev->nb); + mlx5_lag_mp_cleanup(ldev); + cancel_delayed_work_sync(&ldev->bond_work); + destroy_workqueue(ldev->wq); + kfree(ldev); +} + +static void mlx5_ldev_put(struct mlx5_lag *ldev) +{ + kref_put(&ldev->ref, mlx5_ldev_free); +} + +static void mlx5_ldev_get(struct mlx5_lag *ldev) +{ + kref_get(&ldev->ref); +} + +static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev) +{ + struct mlx5_lag *ldev; + int err; + + ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); + if (!ldev) + return NULL; + + ldev->wq = create_singlethread_workqueue("mlx5_lag"); + if (!ldev->wq) { + kfree(ldev); + return NULL; + } + + kref_init(&ldev->ref); + INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work); + + ldev->nb.notifier_call = mlx5_lag_netdev_event; + if (register_netdevice_notifier_net(&init_net, &ldev->nb)) { + ldev->nb.notifier_call = NULL; + mlx5_core_err(dev, "Failed to register LAG netdev notifier\n"); + } + + err = mlx5_lag_mp_init(ldev); + if (err) + mlx5_core_err(dev, "Failed to init multipath lag err=%d\n", + err); + + return ldev; +} + int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev, struct net_device *ndev) { @@ -118,17 +176,24 @@ static bool __mlx5_lag_is_sriov(struct mlx5_lag *ldev) static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker, u8 *port1, u8 *port2) { + bool p1en; + bool p2en; + + p1en = tracker->netdev_state[MLX5_LAG_P1].tx_enabled && + tracker->netdev_state[MLX5_LAG_P1].link_up; + + p2en = tracker->netdev_state[MLX5_LAG_P2].tx_enabled && + tracker->netdev_state[MLX5_LAG_P2].link_up; + *port1 = 1; *port2 = 2; - if (!tracker->netdev_state[MLX5_LAG_P1].tx_enabled || - !tracker->netdev_state[MLX5_LAG_P1].link_up) { - *port1 = 2; + if ((!p1en && !p2en) || (p1en && p2en)) return; - } - if (!tracker->netdev_state[MLX5_LAG_P2].tx_enabled || - !tracker->netdev_state[MLX5_LAG_P2].link_up) + if (p1en) *port2 = 1; + else + *port1 = 2; } void mlx5_modify_lag(struct mlx5_lag *ldev, @@ -251,6 +316,10 @@ static void mlx5_lag_add_devices(struct mlx5_lag *ldev) if (!ldev->pf[i].dev) continue; + if (ldev->pf[i].dev->priv.flags & + MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) + continue; + ldev->pf[i].dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; mlx5_rescan_drivers_locked(ldev->pf[i].dev); } @@ -269,6 +338,31 @@ static void mlx5_lag_remove_devices(struct mlx5_lag *ldev) } } +static void mlx5_disable_lag(struct mlx5_lag *ldev) +{ + struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev; + struct mlx5_core_dev *dev1 = ldev->pf[MLX5_LAG_P2].dev; + bool roce_lag; + int err; + + roce_lag = __mlx5_lag_is_roce(ldev); + + if (roce_lag) { + if (!(dev0->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) { + dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + mlx5_rescan_drivers_locked(dev0); + } + mlx5_nic_vport_disable_roce(dev1); + } + + err = mlx5_deactivate_lag(ldev); + if (err) + return; + + if (roce_lag) + mlx5_lag_add_devices(ldev); +} + static void mlx5_do_bond(struct mlx5_lag *ldev) { struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev; @@ -280,9 +374,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) if (!mlx5_lag_is_ready(ldev)) return; - spin_lock(&lag_lock); tracker = ldev->tracker; - spin_unlock(&lag_lock); do_bond = tracker.is_bonded && mlx5_lag_check_prereq(ldev); @@ -291,8 +383,9 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) !mlx5_sriov_is_enabled(dev1); #ifdef CONFIG_MLX5_ESWITCH - roce_lag &= dev0->priv.eswitch->mode == MLX5_ESWITCH_NONE && - dev1->priv.eswitch->mode == MLX5_ESWITCH_NONE; + roce_lag = roce_lag && + dev0->priv.eswitch->mode == MLX5_ESWITCH_NONE && + dev1->priv.eswitch->mode == MLX5_ESWITCH_NONE; #endif if (roce_lag) @@ -316,20 +409,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) } else if (do_bond && __mlx5_lag_is_active(ldev)) { mlx5_modify_lag(ldev, &tracker); } else if (!do_bond && __mlx5_lag_is_active(ldev)) { - roce_lag = __mlx5_lag_is_roce(ldev); - - if (roce_lag) { - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - mlx5_nic_vport_disable_roce(dev1); - } - - err = mlx5_deactivate_lag(ldev); - if (err) - return; - - if (roce_lag) - mlx5_lag_add_devices(ldev); + mlx5_disable_lag(ldev); } } @@ -481,9 +561,7 @@ static int mlx5_lag_netdev_event(struct notifier_block *this, break; } - spin_lock(&lag_lock); ldev->tracker = tracker; - spin_unlock(&lag_lock); if (changed) mlx5_queue_bond_work(ldev, 0); @@ -491,55 +569,52 @@ static int mlx5_lag_netdev_event(struct notifier_block *this, return NOTIFY_DONE; } -static struct mlx5_lag *mlx5_lag_dev_alloc(void) +static void mlx5_ldev_add_netdev(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev, + struct net_device *netdev) { - struct mlx5_lag *ldev; - - ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); - if (!ldev) - return NULL; - - ldev->wq = create_singlethread_workqueue("mlx5_lag"); - if (!ldev->wq) { - kfree(ldev); - return NULL; - } + unsigned int fn = PCI_FUNC(dev->pdev->devfn); - INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work); + if (fn >= MLX5_MAX_PORTS) + return; - return ldev; + spin_lock(&lag_lock); + ldev->pf[fn].netdev = netdev; + ldev->tracker.netdev_state[fn].link_up = 0; + ldev->tracker.netdev_state[fn].tx_enabled = 0; + spin_unlock(&lag_lock); } -static void mlx5_lag_dev_free(struct mlx5_lag *ldev) +static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev, + struct net_device *netdev) { - destroy_workqueue(ldev->wq); - kfree(ldev); + int i; + + spin_lock(&lag_lock); + for (i = 0; i < MLX5_MAX_PORTS; i++) { + if (ldev->pf[i].netdev == netdev) { + ldev->pf[i].netdev = NULL; + break; + } + } + spin_unlock(&lag_lock); } -static int mlx5_lag_dev_add_pf(struct mlx5_lag *ldev, - struct mlx5_core_dev *dev, - struct net_device *netdev) +static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev) { unsigned int fn = PCI_FUNC(dev->pdev->devfn); if (fn >= MLX5_MAX_PORTS) - return -EPERM; - - spin_lock(&lag_lock); - ldev->pf[fn].dev = dev; - ldev->pf[fn].netdev = netdev; - ldev->tracker.netdev_state[fn].link_up = 0; - ldev->tracker.netdev_state[fn].tx_enabled = 0; + return; + ldev->pf[fn].dev = dev; dev->priv.lag = ldev; - - spin_unlock(&lag_lock); - - return fn; } -static void mlx5_lag_dev_remove_pf(struct mlx5_lag *ldev, - struct mlx5_core_dev *dev) +/* Must be called with intf_mutex held */ +static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev) { int i; @@ -550,19 +625,15 @@ static void mlx5_lag_dev_remove_pf(struct mlx5_lag *ldev, if (i == MLX5_MAX_PORTS) return; - spin_lock(&lag_lock); - memset(&ldev->pf[i], 0, sizeof(*ldev->pf)); - + ldev->pf[i].dev = NULL; dev->priv.lag = NULL; - spin_unlock(&lag_lock); } /* Must be called with intf_mutex held */ -void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev) +static void __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev = NULL; struct mlx5_core_dev *tmp_dev; - int i, err; if (!MLX5_CAP_GEN(dev, vport_group_manager) || !MLX5_CAP_GEN(dev, lag_master) || @@ -574,67 +645,77 @@ void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev) ldev = tmp_dev->priv.lag; if (!ldev) { - ldev = mlx5_lag_dev_alloc(); + ldev = mlx5_lag_dev_alloc(dev); if (!ldev) { mlx5_core_err(dev, "Failed to alloc lag dev\n"); return; } + } else { + mlx5_ldev_get(ldev); } - if (mlx5_lag_dev_add_pf(ldev, dev, netdev) < 0) - return; + mlx5_ldev_add_mdev(ldev, dev); - for (i = 0; i < MLX5_MAX_PORTS; i++) - if (!ldev->pf[i].dev) - break; + return; +} - if (i >= MLX5_MAX_PORTS) - ldev->flags |= MLX5_LAG_FLAG_READY; +void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev) +{ + struct mlx5_lag *ldev; - if (!ldev->nb.notifier_call) { - ldev->nb.notifier_call = mlx5_lag_netdev_event; - if (register_netdevice_notifier_net(&init_net, &ldev->nb)) { - ldev->nb.notifier_call = NULL; - mlx5_core_err(dev, "Failed to register LAG netdev notifier\n"); - } - } + ldev = mlx5_lag_dev(dev); + if (!ldev) + return; - err = mlx5_lag_mp_init(ldev); - if (err) - mlx5_core_err(dev, "Failed to init multipath lag err=%d\n", - err); + mlx5_dev_list_lock(); + mlx5_ldev_remove_mdev(ldev, dev); + mlx5_dev_list_unlock(); + mlx5_ldev_put(ldev); +} + +void mlx5_lag_add_mdev(struct mlx5_core_dev *dev) +{ + mlx5_dev_list_lock(); + __mlx5_lag_dev_add_mdev(dev); + mlx5_dev_list_unlock(); } /* Must be called with intf_mutex held */ -void mlx5_lag_remove(struct mlx5_core_dev *dev) +void mlx5_lag_remove_netdev(struct mlx5_core_dev *dev, + struct net_device *netdev) { struct mlx5_lag *ldev; - int i; - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); if (!ldev) return; if (__mlx5_lag_is_active(ldev)) - mlx5_deactivate_lag(ldev); - - mlx5_lag_dev_remove_pf(ldev, dev); + mlx5_disable_lag(ldev); + mlx5_ldev_remove_netdev(ldev, netdev); ldev->flags &= ~MLX5_LAG_FLAG_READY; +} + +/* Must be called with intf_mutex held */ +void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, + struct net_device *netdev) +{ + struct mlx5_lag *ldev; + int i; + + ldev = mlx5_lag_dev(dev); + if (!ldev) + return; + + mlx5_ldev_add_netdev(ldev, dev, netdev); for (i = 0; i < MLX5_MAX_PORTS; i++) - if (ldev->pf[i].dev) + if (!ldev->pf[i].dev) break; - if (i == MLX5_MAX_PORTS) { - if (ldev->nb.notifier_call) { - unregister_netdevice_notifier_net(&init_net, &ldev->nb); - ldev->nb.notifier_call = NULL; - } - mlx5_lag_mp_cleanup(ldev); - cancel_delayed_work_sync(&ldev->bond_work); - mlx5_lag_dev_free(ldev); - } + if (i >= MLX5_MAX_PORTS) + ldev->flags |= MLX5_LAG_FLAG_READY; } bool mlx5_lag_is_roce(struct mlx5_core_dev *dev) @@ -643,7 +724,7 @@ bool mlx5_lag_is_roce(struct mlx5_core_dev *dev) bool res; spin_lock(&lag_lock); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_roce(ldev); spin_unlock(&lag_lock); @@ -657,7 +738,7 @@ bool mlx5_lag_is_active(struct mlx5_core_dev *dev) bool res; spin_lock(&lag_lock); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_active(ldev); spin_unlock(&lag_lock); @@ -671,7 +752,7 @@ bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev) bool res; spin_lock(&lag_lock); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_sriov(ldev); spin_unlock(&lag_lock); @@ -684,7 +765,7 @@ void mlx5_lag_update(struct mlx5_core_dev *dev) struct mlx5_lag *ldev; mlx5_dev_list_lock(); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); if (!ldev) goto unlock; @@ -700,7 +781,7 @@ struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev) struct mlx5_lag *ldev; spin_lock(&lag_lock); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); if (!(ldev && __mlx5_lag_is_roce(ldev))) goto unlock; @@ -729,7 +810,7 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev, u8 port = 0; spin_lock(&lag_lock); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); if (!(ldev && __mlx5_lag_is_roce(ldev))) goto unlock; @@ -765,7 +846,7 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, memset(values, 0, sizeof(*values) * num_counters); spin_lock(&lag_lock); - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); if (ldev && __mlx5_lag_is_active(ldev)) { num_ports = MLX5_MAX_PORTS; mdev[MLX5_LAG_P1] = ldev->pf[MLX5_LAG_P1].dev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag.h index 8d8cf2d0bc6d..191392c37558 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag.h @@ -40,6 +40,7 @@ struct lag_tracker { struct mlx5_lag { u8 flags; u8 v2p_map[MLX5_MAX_PORTS]; + struct kref ref; struct lag_func pf[MLX5_MAX_PORTS]; struct lag_tracker tracker; struct workqueue_struct *wq; @@ -49,7 +50,7 @@ struct mlx5_lag { }; static inline struct mlx5_lag * -mlx5_lag_dev_get(struct mlx5_core_dev *dev) +mlx5_lag_dev(struct mlx5_core_dev *dev) { return dev->priv.lag; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c index 2c41a6920264..c4bf8b679541 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag_mp.c @@ -28,7 +28,7 @@ bool mlx5_lag_is_multipath(struct mlx5_core_dev *dev) struct mlx5_lag *ldev; bool res; - ldev = mlx5_lag_dev_get(dev); + ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_multipath(ldev); return res; @@ -307,6 +307,11 @@ int mlx5_lag_mp_init(struct mlx5_lag *ldev) struct lag_mp *mp = &ldev->lag_mp; int err; + /* always clear mfi, as it might become stale when a route delete event + * has been missed + */ + mp->mfi = NULL; + if (mp->fib_nb.notifier_call) return 0; @@ -335,4 +340,5 @@ void mlx5_lag_mp_cleanup(struct mlx5_lag *ldev) unregister_fib_notifier(&init_net, &mp->fib_nb); destroy_workqueue(mp->wq); mp->fib_nb.notifier_call = NULL; + mp->mfi = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h index ceae6bc378e0..bd95b9f8d143 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.h @@ -37,16 +37,18 @@ static inline bool mlx5_is_real_time_rq(struct mlx5_core_dev *mdev) { u8 rq_ts_format_cap = MLX5_CAP_GEN(mdev, rq_ts_format); - return (rq_ts_format_cap == MLX5_RQ_TIMESTAMP_FORMAT_CAP_REAL_TIME || - rq_ts_format_cap == MLX5_RQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME); + return (rq_ts_format_cap == MLX5_TIMESTAMP_FORMAT_CAP_REAL_TIME || + rq_ts_format_cap == + MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME); } static inline bool mlx5_is_real_time_sq(struct mlx5_core_dev *mdev) { u8 sq_ts_format_cap = MLX5_CAP_GEN(mdev, sq_ts_format); - return (sq_ts_format_cap == MLX5_SQ_TIMESTAMP_FORMAT_CAP_REAL_TIME || - sq_ts_format_cap == MLX5_SQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME); + return (sq_ts_format_cap == MLX5_TIMESTAMP_FORMAT_CAP_REAL_TIME || + sq_ts_format_cap == + MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME); } typedef ktime_t (*cqe_ts_to_ns)(struct mlx5_clock *, u64); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h index f607a3858ef5..624cedebb510 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* Copyright (c) 2018 Mellanox Technologies */ +/* Copyright (c) 2018-2021, Mellanox Technologies inc. All rights reserved. */ #ifndef __LIB_MLX5_EQ_H__ #define __LIB_MLX5_EQ_H__ @@ -32,6 +32,7 @@ struct mlx5_eq { unsigned int irqn; u8 eqn; struct mlx5_rsc_debug *dbg; + struct mlx5_irq *irq; }; struct mlx5_eq_async { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c index 00ef10a1a9f8..97e5845b4cfd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.c @@ -6,6 +6,7 @@ #include <linux/mlx5/fs.h> #include "lib/fs_chains.h" +#include "fs_ft_pool.h" #include "en/mapping.h" #include "fs_core.h" #include "en_tc.h" @@ -13,25 +14,10 @@ #define chains_lock(chains) ((chains)->lock) #define chains_ht(chains) ((chains)->chains_ht) #define prios_ht(chains) ((chains)->prios_ht) -#define ft_pool_left(chains) ((chains)->ft_left) #define tc_default_ft(chains) ((chains)->tc_default_ft) #define tc_end_ft(chains) ((chains)->tc_end_ft) #define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \ FDB_TC_OFFLOAD : MLX5E_TC_PRIO) - -/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS), - * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated - * for each flow table pool. We can allocate up to 16M of each pool, - * and we keep track of how much we used via get_next_avail_sz_from_pool. - * Firmware doesn't report any of this for now. - * ESW_POOL is expected to be sorted from large to small and match firmware - * pools. - */ -#define FT_SIZE (16 * 1024 * 1024) -static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024, - 1 * 1024 * 1024, - 64 * 1024, - 128 }; #define FT_TBL_SZ (64 * 1024) struct mlx5_fs_chains { @@ -49,8 +35,6 @@ struct mlx5_fs_chains { enum mlx5_flow_namespace_type ns; u32 group_num; u32 flags; - - int ft_left[ARRAY_SIZE(FT_POOLS)]; }; struct fs_chain { @@ -107,7 +91,7 @@ bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains) return chains->flags & MLX5_CHAINS_AND_PRIOS_SUPPORTED; } -static bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains) +bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains) { return chains->flags & MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; } @@ -160,54 +144,6 @@ mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains, tc_end_ft(chains) = ft; } -#define POOL_NEXT_SIZE 0 -static int -mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains *chains, - int desired_size) -{ - int i, found_i = -1; - - for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { - if (ft_pool_left(chains)[i] && FT_POOLS[i] > desired_size) { - found_i = i; - if (desired_size != POOL_NEXT_SIZE) - break; - } - } - - if (found_i != -1) { - --ft_pool_left(chains)[found_i]; - return FT_POOLS[found_i]; - } - - return 0; -} - -static void -mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains *chains, int sz) -{ - int i; - - for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) { - if (sz == FT_POOLS[i]) { - ++ft_pool_left(chains)[i]; - return; - } - } - - WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz); -} - -static void -mlx5_chains_init_sz_pool(struct mlx5_fs_chains *chains, u32 ft_max) -{ - int i; - - for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) - ft_pool_left(chains)[i] = - FT_POOLS[i] <= ft_max ? FT_SIZE / FT_POOLS[i] : 0; -} - static struct mlx5_flow_table * mlx5_chains_create_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) @@ -221,11 +157,7 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains, ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); - sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? - mlx5_chains_get_avail_sz_from_pool(chains, FT_TBL_SZ) : - mlx5_chains_get_avail_sz_from_pool(chains, POOL_NEXT_SIZE); - if (!sz) - return ERR_PTR(-ENOSPC); + sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE; ft_attr.max_fte = sz; /* We use tc_default_ft(chains) as the table's next_ft till @@ -266,21 +198,12 @@ mlx5_chains_create_table(struct mlx5_fs_chains *chains, if (IS_ERR(ft)) { mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n", (int)PTR_ERR(ft), chain, prio, level, sz); - mlx5_chains_put_sz_to_pool(chains, sz); return ft; } return ft; } -static void -mlx5_chains_destroy_table(struct mlx5_fs_chains *chains, - struct mlx5_flow_table *ft) -{ - mlx5_chains_put_sz_to_pool(chains, ft->max_fte); - mlx5_destroy_flow_table(ft); -} - static int create_chain_restore(struct fs_chain *chain) { @@ -336,9 +259,10 @@ create_chain_restore(struct fs_chain *chain) MLX5_SET(set_action_in, modact, field, mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield); MLX5_SET(set_action_in, modact, offset, - mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset * 8); + mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset); MLX5_SET(set_action_in, modact, length, - mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen * 8); + mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen == 32 ? + 0 : mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen); MLX5_SET(set_action_in, modact, data, chain->id); mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns, 1, modact); @@ -636,7 +560,7 @@ err_insert: err_miss_rule: mlx5_destroy_flow_group(miss_group); err_group: - mlx5_chains_destroy_table(chains, ft); + mlx5_destroy_flow_table(ft); err_create: err_alloc: kvfree(prio_s); @@ -659,7 +583,7 @@ mlx5_chains_destroy_prio(struct mlx5_fs_chains *chains, prio_params); mlx5_del_flow_rules(prio->miss_rule); mlx5_destroy_flow_group(prio->miss_group); - mlx5_chains_destroy_table(chains, prio->ft); + mlx5_destroy_flow_table(prio->ft); mlx5_chains_put_chain(chain); kvfree(prio); } @@ -784,7 +708,7 @@ void mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains, struct mlx5_flow_table *ft) { - mlx5_chains_destroy_table(chains, ft); + mlx5_destroy_flow_table(ft); } static struct mlx5_fs_chains * @@ -816,8 +740,6 @@ mlx5_chains_init(struct mlx5_core_dev *dev, struct mlx5_chains_attr *attr) mlx5_chains_get_chain_range(chains_priv), mlx5_chains_get_prio_range(chains_priv)); - mlx5_chains_init_sz_pool(chains_priv, attr->max_ft_sz); - err = rhashtable_init(&chains_ht(chains_priv), &chain_params); if (err) goto init_chains_ht_err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h index e96f345e7dae..d50bdb226cef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_chains.h @@ -28,6 +28,7 @@ struct mlx5_chains_attr { bool mlx5_chains_prios_supported(struct mlx5_fs_chains *chains); +bool mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains); bool mlx5_chains_backwards_supported(struct mlx5_fs_chains *chains); u32 @@ -70,6 +71,10 @@ mlx5_chains_set_end_ft(struct mlx5_fs_chains *chains, #else /* CONFIG_MLX5_CLS_ACT */ +static inline bool +mlx5_chains_ignore_flow_level_supported(struct mlx5_fs_chains *chains) +{ return false; } + static inline struct mlx5_flow_table * mlx5_chains_get_table(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) { return ERR_PTR(-EOPNOTSUPP); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c index fd8449ff9e17..839a01da110f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c @@ -33,6 +33,7 @@ #include <linux/etherdevice.h> #include <linux/mlx5/driver.h> #include <linux/mlx5/mlx5_ifc.h> +#include <linux/mlx5/mpfs.h> #include <linux/mlx5/eswitch.h> #include "mlx5_core.h" #include "lib/mpfs.h" @@ -175,6 +176,7 @@ out: mutex_unlock(&mpfs->lock); return err; } +EXPORT_SYMBOL(mlx5_mpfs_add_mac); int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac) { @@ -206,3 +208,4 @@ unlock: mutex_unlock(&mpfs->lock); return err; } +EXPORT_SYMBOL(mlx5_mpfs_del_mac); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h index 4a7b2c3203a7..4a293542a7aa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h @@ -84,12 +84,9 @@ struct l2addr_node { #ifdef CONFIG_MLX5_MPFS int mlx5_mpfs_init(struct mlx5_core_dev *dev); void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev); -int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac); -int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac); #else /* #ifndef CONFIG_MLX5_MPFS */ static inline int mlx5_mpfs_init(struct mlx5_core_dev *dev) { return 0; } static inline void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev) {} -static inline int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac) { return 0; } -static inline int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac) { return 0; } #endif + #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sf.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/sf.h new file mode 100644 index 000000000000..84e5683861be --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sf.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies Ltd */ + +#ifndef __LIB_MLX5_SF_H__ +#define __LIB_MLX5_SF_H__ + +#include <linux/mlx5/driver.h> + +static inline u16 mlx5_sf_start_function_id(const struct mlx5_core_dev *dev) +{ + return MLX5_CAP_GEN(dev, sf_base_id); +} + +#ifdef CONFIG_MLX5_SF + +static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev) +{ + return MLX5_CAP_GEN(dev, sf); +} + +static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) +{ + if (!mlx5_sf_supported(dev)) + return 0; + if (MLX5_CAP_GEN(dev, max_num_sf)) + return MLX5_CAP_GEN(dev, max_num_sf); + else + return 1 << MLX5_CAP_GEN(dev, log_max_sf); +} + +#else + +static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev) +{ + return false; +} + +static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) +{ + return 0; +} + +#endif + +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index c114365eb126..eb1b316560a8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -76,6 +76,7 @@ #include "sf/vhca_event.h" #include "sf/dev/dev.h" #include "sf/sf.h" +#include "mlx5_irq.h" MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver"); @@ -503,7 +504,7 @@ static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx) static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) { - struct mlx5_profile *prof = dev->profile; + struct mlx5_profile *prof = &dev->profile; void *set_hca_cap; int err; @@ -524,11 +525,11 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) to_fw_pkey_sz(dev, 128)); /* Check log_max_qp from HCA caps to set in current profile */ - if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < profile[prof_sel].log_max_qp) { + if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) { mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n", - profile[prof_sel].log_max_qp, + prof->log_max_qp, MLX5_CAP_GEN_MAX(dev, log_max_qp)); - profile[prof_sel].log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); + prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp); } if (prof->mask & MLX5_PROF_MASK_QP_SIZE) MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, @@ -1161,7 +1162,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) err = mlx5_core_set_hca_defaults(dev); if (err) { mlx5_core_err(dev, "Failed to set hca defaults\n"); - goto err_sriov; + goto err_set_hca; } mlx5_vhca_event_start(dev); @@ -1185,6 +1186,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) } mlx5_sf_dev_table_create(dev); + mlx5_lag_add_mdev(dev); return 0; @@ -1194,6 +1196,7 @@ err_ec: mlx5_sf_hw_table_destroy(dev); err_vhca: mlx5_vhca_event_stop(dev); +err_set_hca: mlx5_cleanup_fs(dev); err_fs: mlx5_accel_tls_cleanup(dev); @@ -1219,6 +1222,7 @@ err_irq_table: static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_lag_remove_mdev(dev); mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); mlx5_ec_cleanup(dev); @@ -1381,8 +1385,7 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) struct mlx5_priv *priv = &dev->priv; int err; - dev->profile = &profile[profile_idx]; - + memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile)); INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); mutex_init(&dev->intf_state_mutex); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index a22b706eebd3..343807ac2036 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -164,27 +164,10 @@ int mlx5_query_mcam_reg(struct mlx5_core_dev *dev, u32 *mcap, u8 feature_group, int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u32 *qcam, u8 feature_group, u8 access_reg_group); -void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev); -void mlx5_lag_remove(struct mlx5_core_dev *dev); - -int mlx5_irq_table_init(struct mlx5_core_dev *dev); -void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev); -int mlx5_irq_table_create(struct mlx5_core_dev *dev); -void mlx5_irq_table_destroy(struct mlx5_core_dev *dev); -int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx, - struct notifier_block *nb); -int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx, - struct notifier_block *nb); - -int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int devfn, - int msix_vec_count); -int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs); - -struct cpumask * -mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx); -struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *table); -int mlx5_irq_get_num_comp(struct mlx5_irq_table *table); -struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev); +void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, struct net_device *netdev); +void mlx5_lag_remove_netdev(struct mlx5_core_dev *dev, struct net_device *netdev); +void mlx5_lag_add_mdev(struct mlx5_core_dev *dev); +void mlx5_lag_remove_mdev(struct mlx5_core_dev *dev); int mlx5_events_init(struct mlx5_core_dev *dev); void mlx5_events_cleanup(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h new file mode 100644 index 000000000000..abd024173c42 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021 Mellanox Technologies. */ + +#ifndef __MLX5_IRQ_H__ +#define __MLX5_IRQ_H__ + +#include <linux/mlx5/driver.h> + +#define MLX5_COMP_EQS_PER_SF 8 + +#define MLX5_IRQ_EQ_CTRL (0) + +struct mlx5_irq; + +int mlx5_irq_table_init(struct mlx5_core_dev *dev); +void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev); +int mlx5_irq_table_create(struct mlx5_core_dev *dev); +void mlx5_irq_table_destroy(struct mlx5_core_dev *dev); +int mlx5_irq_table_get_num_comp(struct mlx5_irq_table *table); +int mlx5_irq_table_get_sfs_vec(struct mlx5_irq_table *table); +struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev); + +int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int devfn, + int msix_vec_count); +int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs); + +struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx, + struct cpumask *affinity); +void mlx5_irq_release(struct mlx5_irq *irq); +int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb); +int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb); +struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq); +int mlx5_irq_get_index(struct mlx5_irq *irq); + +#endif /* __MLX5_IRQ_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mr.c b/drivers/net/ethernet/mellanox/mlx5/core/mr.c index 50af84e76fb6..174f71ed5280 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/mr.c @@ -54,7 +54,7 @@ int mlx5_core_create_mkey(struct mlx5_core_dev *dev, mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index); mkey->iova = MLX5_GET64(mkc, mkc, start_addr); mkey->size = MLX5_GET64(mkc, mkc, len); - mkey->key |= mlx5_idx_to_mkey(mkey_index); + mkey->key = (u32)mlx5_mkey_variant(mkey->key) | mlx5_idx_to_mkey(mkey_index); mkey->pd = MLX5_GET(mkc, mkc, pd); init_waitqueue_head(&mkey->wait); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 1f907df5b3a2..b25f764daa08 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -6,60 +6,52 @@ #include <linux/module.h> #include <linux/mlx5/driver.h> #include "mlx5_core.h" +#include "mlx5_irq.h" +#include "lib/sf.h" #ifdef CONFIG_RFS_ACCEL #include <linux/cpu_rmap.h> #endif #define MLX5_MAX_IRQ_NAME (32) +/* max irq_index is 255. three chars */ +#define MLX5_MAX_IRQ_IDX_CHARS (3) + +#define MLX5_SFS_PER_CTRL_IRQ 64 +#define MLX5_IRQ_CTRL_SF_MAX 8 +/* min num of vectores for SFs to be enabled */ +#define MLX5_IRQ_VEC_COMP_BASE_SF 2 + +#define MLX5_EQ_SHARE_IRQ_MAX_COMP (8) +#define MLX5_EQ_SHARE_IRQ_MAX_CTRL (UINT_MAX) +#define MLX5_EQ_SHARE_IRQ_MIN_COMP (1) +#define MLX5_EQ_SHARE_IRQ_MIN_CTRL (4) +#define MLX5_EQ_REFS_PER_IRQ (2) struct mlx5_irq { + u32 index; struct atomic_notifier_head nh; cpumask_var_t mask; char name[MLX5_MAX_IRQ_NAME]; + struct kref kref; + int irqn; + struct mlx5_irq_pool *pool; }; -struct mlx5_irq_table { - struct mlx5_irq *irq; - int nvec; -#ifdef CONFIG_RFS_ACCEL - struct cpu_rmap *rmap; -#endif +struct mlx5_irq_pool { + char name[MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS]; + struct xa_limit xa_num_irqs; + struct mutex lock; /* sync IRQs creations */ + struct xarray irqs; + u32 max_threshold; + u32 min_threshold; + struct mlx5_core_dev *dev; }; -int mlx5_irq_table_init(struct mlx5_core_dev *dev) -{ - struct mlx5_irq_table *irq_table; - - if (mlx5_core_is_sf(dev)) - return 0; - - irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL); - if (!irq_table) - return -ENOMEM; - - dev->priv.irq_table = irq_table; - return 0; -} - -void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev) -{ - if (mlx5_core_is_sf(dev)) - return; - - kvfree(dev->priv.irq_table); -} - -int mlx5_irq_get_num_comp(struct mlx5_irq_table *table) -{ - return table->nvec - MLX5_IRQ_VEC_COMP_BASE; -} - -static struct mlx5_irq *mlx5_irq_get(struct mlx5_core_dev *dev, int vecidx) -{ - struct mlx5_irq_table *irq_table = dev->priv.irq_table; - - return &irq_table->irq[vecidx]; -} +struct mlx5_irq_table { + struct mlx5_irq_pool *pf_pool; + struct mlx5_irq_pool *sf_ctrl_pool; + struct mlx5_irq_pool *sf_comp_pool; +}; /** * mlx5_get_default_msix_vec_count - Get the default number of MSI-X vectors @@ -95,9 +87,10 @@ int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs) int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id, int msix_vec_count) { - int sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); + int query_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); + void *hca_cap = NULL, *query_cap = NULL, *cap; int num_vf_msix, min_msix, max_msix; - void *hca_cap, *cap; int ret; num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix); @@ -116,11 +109,20 @@ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id, if (msix_vec_count > max_msix) return -EOVERFLOW; - hca_cap = kzalloc(sz, GFP_KERNEL); - if (!hca_cap) - return -ENOMEM; + query_cap = kzalloc(query_sz, GFP_KERNEL); + hca_cap = kzalloc(set_sz, GFP_KERNEL); + if (!hca_cap || !query_cap) { + ret = -ENOMEM; + goto out; + } + + ret = mlx5_vport_get_other_func_cap(dev, function_id, query_cap); + if (ret) + goto out; cap = MLX5_ADDR_OF(set_hca_cap_in, hca_cap, capability); + memcpy(cap, MLX5_ADDR_OF(query_hca_cap_out, query_cap, capability), + MLX5_UN_SZ_BYTES(hca_cap_union)); MLX5_SET(cmd_hca_cap, cap, dynamic_msix_table_size, msix_vec_count); MLX5_SET(set_hca_cap_in, hca_cap, opcode, MLX5_CMD_OP_SET_HCA_CAP); @@ -130,38 +132,52 @@ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id, MLX5_SET(set_hca_cap_in, hca_cap, op_mod, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1); ret = mlx5_cmd_exec_in(dev, set_hca_cap, hca_cap); +out: kfree(hca_cap); + kfree(query_cap); return ret; } -int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx, - struct notifier_block *nb) +static void irq_release(struct kref *kref) { - struct mlx5_irq *irq; + struct mlx5_irq *irq = container_of(kref, struct mlx5_irq, kref); + struct mlx5_irq_pool *pool = irq->pool; - irq = &irq_table->irq[vecidx]; - return atomic_notifier_chain_register(&irq->nh, nb); + xa_erase(&pool->irqs, irq->index); + /* free_irq requires that affinity and rmap will be cleared + * before calling it. This is why there is asymmetry with set_rmap + * which should be called after alloc_irq but before request_irq. + */ + irq_set_affinity_hint(irq->irqn, NULL); + free_cpumask_var(irq->mask); + free_irq(irq->irqn, &irq->nh); + kfree(irq); } -int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx, - struct notifier_block *nb) +static void irq_put(struct mlx5_irq *irq) { - struct mlx5_irq *irq; + struct mlx5_irq_pool *pool = irq->pool; - irq = &irq_table->irq[vecidx]; - return atomic_notifier_chain_unregister(&irq->nh, nb); + mutex_lock(&pool->lock); + kref_put(&irq->kref, irq_release); + mutex_unlock(&pool->lock); } -static irqreturn_t mlx5_irq_int_handler(int irq, void *nh) +static irqreturn_t irq_int_handler(int irq, void *nh) { atomic_notifier_call_chain(nh, 0, NULL); return IRQ_HANDLED; } +static void irq_sf_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx) +{ + snprintf(name, MLX5_MAX_IRQ_NAME, "%s%d", pool->name, vecidx); +} + static void irq_set_name(char *name, int vecidx) { if (vecidx == 0) { - snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async"); + snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async%d", vecidx); return; } @@ -169,251 +185,431 @@ static void irq_set_name(char *name, int vecidx) vecidx - MLX5_IRQ_VEC_COMP_BASE); } -static int request_irqs(struct mlx5_core_dev *dev, int nvec) +static struct mlx5_irq *irq_request(struct mlx5_irq_pool *pool, int i) { + struct mlx5_core_dev *dev = pool->dev; char name[MLX5_MAX_IRQ_NAME]; + struct mlx5_irq *irq; int err; - int i; - - for (i = 0; i < nvec; i++) { - struct mlx5_irq *irq = mlx5_irq_get(dev, i); - int irqn = pci_irq_vector(dev->pdev, i); + irq = kzalloc(sizeof(*irq), GFP_KERNEL); + if (!irq) + return ERR_PTR(-ENOMEM); + irq->irqn = pci_irq_vector(dev->pdev, i); + if (!pool->name[0]) irq_set_name(name, i); - ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh); - snprintf(irq->name, MLX5_MAX_IRQ_NAME, - "%s@pci:%s", name, pci_name(dev->pdev)); - err = request_irq(irqn, mlx5_irq_int_handler, 0, irq->name, - &irq->nh); - if (err) { - mlx5_core_err(dev, "Failed to request irq\n"); - goto err_request_irq; - } + else + irq_sf_set_name(pool, name, i); + ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh); + snprintf(irq->name, MLX5_MAX_IRQ_NAME, + "%s@pci:%s", name, pci_name(dev->pdev)); + err = request_irq(irq->irqn, irq_int_handler, 0, irq->name, + &irq->nh); + if (err) { + mlx5_core_err(dev, "Failed to request irq. err = %d\n", err); + goto err_req_irq; } - return 0; + if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) { + mlx5_core_warn(dev, "zalloc_cpumask_var failed\n"); + err = -ENOMEM; + goto err_cpumask; + } + kref_init(&irq->kref); + irq->index = i; + err = xa_err(xa_store(&pool->irqs, irq->index, irq, GFP_KERNEL)); + if (err) { + mlx5_core_err(dev, "Failed to alloc xa entry for irq(%u). err = %d\n", + irq->index, err); + goto err_xa; + } + irq->pool = pool; + return irq; +err_xa: + free_cpumask_var(irq->mask); +err_cpumask: + free_irq(irq->irqn, &irq->nh); +err_req_irq: + kfree(irq); + return ERR_PTR(err); +} -err_request_irq: - while (i--) { - struct mlx5_irq *irq = mlx5_irq_get(dev, i); - int irqn = pci_irq_vector(dev->pdev, i); +int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb) +{ + int err; - free_irq(irqn, &irq->nh); - } - return err; + err = kref_get_unless_zero(&irq->kref); + if (WARN_ON_ONCE(!err)) + /* Something very bad happens here, we are enabling EQ + * on non-existing IRQ. + */ + return -ENOENT; + err = atomic_notifier_chain_register(&irq->nh, nb); + if (err) + irq_put(irq); + return err; } -static void irq_clear_rmap(struct mlx5_core_dev *dev) +int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb) { -#ifdef CONFIG_RFS_ACCEL - struct mlx5_irq_table *irq_table = dev->priv.irq_table; + irq_put(irq); + return atomic_notifier_chain_unregister(&irq->nh, nb); +} - free_irq_cpu_rmap(irq_table->rmap); -#endif +struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq) +{ + return irq->mask; } -static int irq_set_rmap(struct mlx5_core_dev *mdev) +int mlx5_irq_get_index(struct mlx5_irq *irq) { - int err = 0; -#ifdef CONFIG_RFS_ACCEL - struct mlx5_irq_table *irq_table = mdev->priv.irq_table; - int num_affinity_vec; - int vecidx; + return irq->index; +} - num_affinity_vec = mlx5_irq_get_num_comp(irq_table); - irq_table->rmap = alloc_irq_cpu_rmap(num_affinity_vec); - if (!irq_table->rmap) { - err = -ENOMEM; - mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err); - goto err_out; +/* irq_pool API */ + +/* creating an irq from irq_pool */ +static struct mlx5_irq *irq_pool_create_irq(struct mlx5_irq_pool *pool, + struct cpumask *affinity) +{ + struct mlx5_irq *irq; + u32 irq_index; + int err; + + err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, + GFP_KERNEL); + if (err) + return ERR_PTR(err); + irq = irq_request(pool, irq_index); + if (IS_ERR(irq)) + return irq; + cpumask_copy(irq->mask, affinity); + irq_set_affinity_hint(irq->irqn, irq->mask); + return irq; +} + +/* looking for the irq with the smallest refcount and the same affinity */ +static struct mlx5_irq *irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, + struct cpumask *affinity) +{ + int start = pool->xa_num_irqs.min; + int end = pool->xa_num_irqs.max; + struct mlx5_irq *irq = NULL; + struct mlx5_irq *iter; + unsigned long index; + + lockdep_assert_held(&pool->lock); + xa_for_each_range(&pool->irqs, index, iter, start, end) { + if (!cpumask_equal(iter->mask, affinity)) + continue; + if (kref_read(&iter->kref) < pool->min_threshold) + return iter; + if (!irq || kref_read(&iter->kref) < + kref_read(&irq->kref)) + irq = iter; } + return irq; +} - vecidx = MLX5_IRQ_VEC_COMP_BASE; - for (; vecidx < irq_table->nvec; vecidx++) { - err = irq_cpu_rmap_add(irq_table->rmap, - pci_irq_vector(mdev->pdev, vecidx)); - if (err) { - mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d", - err); - goto err_irq_cpu_rmap_add; +/* requesting an irq from a given pool according to given affinity */ +static struct mlx5_irq *irq_pool_request_affinity(struct mlx5_irq_pool *pool, + struct cpumask *affinity) +{ + struct mlx5_irq *least_loaded_irq, *new_irq; + + mutex_lock(&pool->lock); + least_loaded_irq = irq_pool_find_least_loaded(pool, affinity); + if (least_loaded_irq && + kref_read(&least_loaded_irq->kref) < pool->min_threshold) + goto out; + new_irq = irq_pool_create_irq(pool, affinity); + if (IS_ERR(new_irq)) { + if (!least_loaded_irq) { + mlx5_core_err(pool->dev, "Didn't find IRQ for cpu = %u\n", + cpumask_first(affinity)); + mutex_unlock(&pool->lock); + return new_irq; } + /* We failed to create a new IRQ for the requested affinity, + * sharing existing IRQ. + */ + goto out; } - return 0; + least_loaded_irq = new_irq; + goto unlock; +out: + kref_get(&least_loaded_irq->kref); + if (kref_read(&least_loaded_irq->kref) > pool->max_threshold) + mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n", + least_loaded_irq->irqn, pool->name, + kref_read(&least_loaded_irq->kref) / MLX5_EQ_REFS_PER_IRQ); +unlock: + mutex_unlock(&pool->lock); + return least_loaded_irq; +} -err_irq_cpu_rmap_add: - irq_clear_rmap(mdev); -err_out: -#endif - return err; +/* requesting an irq from a given pool according to given index */ +static struct mlx5_irq * +irq_pool_request_vector(struct mlx5_irq_pool *pool, int vecidx, + struct cpumask *affinity) +{ + struct mlx5_irq *irq; + + mutex_lock(&pool->lock); + irq = xa_load(&pool->irqs, vecidx); + if (irq) { + kref_get(&irq->kref); + goto unlock; + } + irq = irq_request(pool, vecidx); + if (IS_ERR(irq) || !affinity) + goto unlock; + cpumask_copy(irq->mask, affinity); + irq_set_affinity_hint(irq->irqn, irq->mask); +unlock: + mutex_unlock(&pool->lock); + return irq; } -/* Completion IRQ vectors */ +static struct mlx5_irq_pool *find_sf_irq_pool(struct mlx5_irq_table *irq_table, + int i, struct cpumask *affinity) +{ + if (cpumask_empty(affinity) && i == MLX5_IRQ_EQ_CTRL) + return irq_table->sf_ctrl_pool; + return irq_table->sf_comp_pool; +} -static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i) +/** + * mlx5_irq_release - release an IRQ back to the system. + * @irq: irq to be released. + */ +void mlx5_irq_release(struct mlx5_irq *irq) { - int vecidx = MLX5_IRQ_VEC_COMP_BASE + i; + synchronize_irq(irq->irqn); + irq_put(irq); +} + +/** + * mlx5_irq_request - request an IRQ for mlx5 device. + * @dev: mlx5 device that requesting the IRQ. + * @vecidx: vector index of the IRQ. This argument is ignore if affinity is + * provided. + * @affinity: cpumask requested for this IRQ. + * + * This function returns a pointer to IRQ, or ERR_PTR in case of error. + */ +struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx, + struct cpumask *affinity) +{ + struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev); + struct mlx5_irq_pool *pool; struct mlx5_irq *irq; - int irqn; - irq = mlx5_irq_get(mdev, vecidx); - irqn = pci_irq_vector(mdev->pdev, vecidx); - if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) { - mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); - return -ENOMEM; + if (mlx5_core_is_sf(dev)) { + pool = find_sf_irq_pool(irq_table, vecidx, affinity); + if (!pool) + /* we don't have IRQs for SFs, using the PF IRQs */ + goto pf_irq; + if (cpumask_empty(affinity) && !strcmp(pool->name, "mlx5_sf_comp")) + /* In case an SF user request IRQ with vecidx */ + irq = irq_pool_request_vector(pool, vecidx, NULL); + else + irq = irq_pool_request_affinity(pool, affinity); + goto out; } +pf_irq: + pool = irq_table->pf_pool; + irq = irq_pool_request_vector(pool, vecidx, affinity); +out: + if (IS_ERR(irq)) + return irq; + mlx5_core_dbg(dev, "irq %u mapped to cpu %*pbl, %u EQs on this irq\n", + irq->irqn, cpumask_pr_args(affinity), + kref_read(&irq->kref) / MLX5_EQ_REFS_PER_IRQ); + return irq; +} - cpumask_set_cpu(cpumask_local_spread(i, mdev->priv.numa_node), - irq->mask); - if (IS_ENABLED(CONFIG_SMP) && - irq_set_affinity_hint(irqn, irq->mask)) - mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x", - irqn); - - return 0; +static struct mlx5_irq_pool * +irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name, + u32 min_threshold, u32 max_threshold) +{ + struct mlx5_irq_pool *pool = kvzalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return ERR_PTR(-ENOMEM); + pool->dev = dev; + xa_init_flags(&pool->irqs, XA_FLAGS_ALLOC); + pool->xa_num_irqs.min = start; + pool->xa_num_irqs.max = start + size - 1; + if (name) + snprintf(pool->name, MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS, + name); + pool->min_threshold = min_threshold * MLX5_EQ_REFS_PER_IRQ; + pool->max_threshold = max_threshold * MLX5_EQ_REFS_PER_IRQ; + mutex_init(&pool->lock); + mlx5_core_dbg(dev, "pool->name = %s, pool->size = %d, pool->start = %d", + name, size, start); + return pool; } -static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i) +static void irq_pool_free(struct mlx5_irq_pool *pool) { - int vecidx = MLX5_IRQ_VEC_COMP_BASE + i; struct mlx5_irq *irq; - int irqn; + unsigned long index; - irq = mlx5_irq_get(mdev, vecidx); - irqn = pci_irq_vector(mdev->pdev, vecidx); - irq_set_affinity_hint(irqn, NULL); - free_cpumask_var(irq->mask); + xa_for_each(&pool->irqs, index, irq) + irq_release(&irq->kref); + xa_destroy(&pool->irqs); + kvfree(pool); } -static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev) +static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec) { - int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table); + struct mlx5_irq_table *table = dev->priv.irq_table; + int num_sf_ctrl_by_msix; + int num_sf_ctrl_by_sfs; + int num_sf_ctrl; int err; - int i; - for (i = 0; i < nvec; i++) { - err = set_comp_irq_affinity_hint(mdev, i); - if (err) - goto err_out; + /* init pf_pool */ + table->pf_pool = irq_pool_alloc(dev, 0, pf_vec, NULL, + MLX5_EQ_SHARE_IRQ_MIN_COMP, + MLX5_EQ_SHARE_IRQ_MAX_COMP); + if (IS_ERR(table->pf_pool)) + return PTR_ERR(table->pf_pool); + if (!mlx5_sf_max_functions(dev)) + return 0; + if (sf_vec < MLX5_IRQ_VEC_COMP_BASE_SF) { + mlx5_core_err(dev, "Not enough IRQs for SFs. SF may run at lower performance\n"); + return 0; } + /* init sf_ctrl_pool */ + num_sf_ctrl_by_msix = DIV_ROUND_UP(sf_vec, MLX5_COMP_EQS_PER_SF); + num_sf_ctrl_by_sfs = DIV_ROUND_UP(mlx5_sf_max_functions(dev), + MLX5_SFS_PER_CTRL_IRQ); + num_sf_ctrl = min_t(int, num_sf_ctrl_by_msix, num_sf_ctrl_by_sfs); + num_sf_ctrl = min_t(int, MLX5_IRQ_CTRL_SF_MAX, num_sf_ctrl); + table->sf_ctrl_pool = irq_pool_alloc(dev, pf_vec, num_sf_ctrl, + "mlx5_sf_ctrl", + MLX5_EQ_SHARE_IRQ_MIN_CTRL, + MLX5_EQ_SHARE_IRQ_MAX_CTRL); + if (IS_ERR(table->sf_ctrl_pool)) { + err = PTR_ERR(table->sf_ctrl_pool); + goto err_pf; + } + /* init sf_comp_pool */ + table->sf_comp_pool = irq_pool_alloc(dev, pf_vec + num_sf_ctrl, + sf_vec - num_sf_ctrl, "mlx5_sf_comp", + MLX5_EQ_SHARE_IRQ_MIN_COMP, + MLX5_EQ_SHARE_IRQ_MAX_COMP); + if (IS_ERR(table->sf_comp_pool)) { + err = PTR_ERR(table->sf_comp_pool); + goto err_sf_ctrl; + } return 0; - -err_out: - for (i--; i >= 0; i--) - clear_comp_irq_affinity_hint(mdev, i); - +err_sf_ctrl: + irq_pool_free(table->sf_ctrl_pool); +err_pf: + irq_pool_free(table->pf_pool); return err; } -static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev) +static void irq_pools_destroy(struct mlx5_irq_table *table) { - int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table); - int i; - - for (i = 0; i < nvec; i++) - clear_comp_irq_affinity_hint(mdev, i); + if (table->sf_ctrl_pool) { + irq_pool_free(table->sf_comp_pool); + irq_pool_free(table->sf_ctrl_pool); + } + irq_pool_free(table->pf_pool); } -struct cpumask * -mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx) +/* irq_table API */ + +int mlx5_irq_table_init(struct mlx5_core_dev *dev) { - return irq_table->irq[vecidx].mask; + struct mlx5_irq_table *irq_table; + + if (mlx5_core_is_sf(dev)) + return 0; + + irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL); + if (!irq_table) + return -ENOMEM; + + dev->priv.irq_table = irq_table; + return 0; } -#ifdef CONFIG_RFS_ACCEL -struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *irq_table) +void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev) { - return irq_table->rmap; + if (mlx5_core_is_sf(dev)) + return; + + kvfree(dev->priv.irq_table); } -#endif -static void unrequest_irqs(struct mlx5_core_dev *dev) +int mlx5_irq_table_get_num_comp(struct mlx5_irq_table *table) { - struct mlx5_irq_table *table = dev->priv.irq_table; - int i; - - for (i = 0; i < table->nvec; i++) - free_irq(pci_irq_vector(dev->pdev, i), - &mlx5_irq_get(dev, i)->nh); + return table->pf_pool->xa_num_irqs.max - table->pf_pool->xa_num_irqs.min; } int mlx5_irq_table_create(struct mlx5_core_dev *dev) { - struct mlx5_priv *priv = &dev->priv; - struct mlx5_irq_table *table = priv->irq_table; int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ? MLX5_CAP_GEN(dev, max_num_eqs) : 1 << MLX5_CAP_GEN(dev, log_max_eq); - int nvec; + int total_vec; + int pf_vec; int err; if (mlx5_core_is_sf(dev)) return 0; - nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + - MLX5_IRQ_VEC_COMP_BASE; - nvec = min_t(int, nvec, num_eqs); - if (nvec <= MLX5_IRQ_VEC_COMP_BASE) - return -ENOMEM; - - table->irq = kcalloc(nvec, sizeof(*table->irq), GFP_KERNEL); - if (!table->irq) + pf_vec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + + MLX5_IRQ_VEC_COMP_BASE; + pf_vec = min_t(int, pf_vec, num_eqs); + if (pf_vec <= MLX5_IRQ_VEC_COMP_BASE) return -ENOMEM; - nvec = pci_alloc_irq_vectors(dev->pdev, MLX5_IRQ_VEC_COMP_BASE + 1, - nvec, PCI_IRQ_MSIX); - if (nvec < 0) { - err = nvec; - goto err_free_irq; - } - - table->nvec = nvec; + total_vec = pf_vec; + if (mlx5_sf_max_functions(dev)) + total_vec += MLX5_IRQ_CTRL_SF_MAX + + MLX5_COMP_EQS_PER_SF * mlx5_sf_max_functions(dev); - err = irq_set_rmap(dev); - if (err) - goto err_set_rmap; + total_vec = pci_alloc_irq_vectors(dev->pdev, MLX5_IRQ_VEC_COMP_BASE + 1, + total_vec, PCI_IRQ_MSIX); + if (total_vec < 0) + return total_vec; + pf_vec = min(pf_vec, total_vec); - err = request_irqs(dev, nvec); + err = irq_pools_init(dev, total_vec - pf_vec, pf_vec); if (err) - goto err_request_irqs; - - err = set_comp_irq_affinity_hints(dev); - if (err) { - mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n"); - goto err_set_affinity; - } - - return 0; + pci_free_irq_vectors(dev->pdev); -err_set_affinity: - unrequest_irqs(dev); -err_request_irqs: - irq_clear_rmap(dev); -err_set_rmap: - pci_free_irq_vectors(dev->pdev); -err_free_irq: - kfree(table->irq); return err; } void mlx5_irq_table_destroy(struct mlx5_core_dev *dev) { struct mlx5_irq_table *table = dev->priv.irq_table; - int i; if (mlx5_core_is_sf(dev)) return; - /* free_irq requires that affinity and rmap will be cleared - * before calling it. This is why there is asymmetry with set_rmap - * which should be called after alloc_irq but before request_irq. + /* There are cases where IRQs still will be in used when we reaching + * to here. Hence, making sure all the irqs are realeased. */ - irq_clear_rmap(dev); - clear_comp_irqs_affinity_hints(dev); - for (i = 0; i < table->nvec; i++) - free_irq(pci_irq_vector(dev->pdev, i), - &mlx5_irq_get(dev, i)->nh); + irq_pools_destroy(table); pci_free_irq_vectors(dev->pdev); - kfree(table->irq); +} + +int mlx5_irq_table_get_sfs_vec(struct mlx5_irq_table *table) +{ + if (table->sf_comp_pool) + return table->sf_comp_pool->xa_num_irqs.max - + table->sf_comp_pool->xa_num_irqs.min + 1; + else + return mlx5_irq_table_get_num_comp(table); } struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c index 441b5453acae..540cf05f6373 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/rdma.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/rdma.c @@ -156,6 +156,9 @@ void mlx5_rdma_enable_roce(struct mlx5_core_dev *dev) { int err; + if (!MLX5_CAP_GEN(dev, roce)) + return; + err = mlx5_nic_vport_enable_roce(dev); if (err) { mlx5_core_err(dev, "Failed to enable RoCE: %d\n", err); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c index 6a0c6f965ad1..fa0288afc0dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c @@ -163,6 +163,7 @@ mlx5_sf_dev_state_change_handler(struct notifier_block *nb, unsigned long event_ sf_index = event->function_id - base_id; sf_dev = xa_load(&table->devices, sf_index); switch (event->new_vhca_state) { + case MLX5_VHCA_STATE_INVALID: case MLX5_VHCA_STATE_ALLOCATED: if (sf_dev) mlx5_sf_dev_del(table->dev, sf_dev, sf_index); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index a8e73c9ed1ea..1be048769309 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -136,10 +136,10 @@ static enum devlink_port_fn_state mlx5_sf_to_devlink_state(u8 hw_state) switch (hw_state) { case MLX5_VHCA_STATE_ACTIVE: case MLX5_VHCA_STATE_IN_USE: - case MLX5_VHCA_STATE_TEARDOWN_REQUEST: return DEVLINK_PORT_FN_STATE_ACTIVE; case MLX5_VHCA_STATE_INVALID: case MLX5_VHCA_STATE_ALLOCATED: + case MLX5_VHCA_STATE_TEARDOWN_REQUEST: default: return DEVLINK_PORT_FN_STATE_INACTIVE; } @@ -192,14 +192,17 @@ sf_err: return err; } -static int mlx5_sf_activate(struct mlx5_core_dev *dev, struct mlx5_sf *sf) +static int mlx5_sf_activate(struct mlx5_core_dev *dev, struct mlx5_sf *sf, + struct netlink_ext_ack *extack) { int err; if (mlx5_sf_is_active(sf)) return 0; - if (sf->hw_state != MLX5_VHCA_STATE_ALLOCATED) - return -EINVAL; + if (sf->hw_state != MLX5_VHCA_STATE_ALLOCATED) { + NL_SET_ERR_MSG_MOD(extack, "SF is inactivated but it is still attached"); + return -EBUSY; + } err = mlx5_cmd_sf_enable_hca(dev, sf->hw_fn_id); if (err) @@ -226,7 +229,8 @@ static int mlx5_sf_deactivate(struct mlx5_core_dev *dev, struct mlx5_sf *sf) static int mlx5_sf_state_set(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, struct mlx5_sf *sf, - enum devlink_port_fn_state state) + enum devlink_port_fn_state state, + struct netlink_ext_ack *extack) { int err = 0; @@ -234,7 +238,7 @@ static int mlx5_sf_state_set(struct mlx5_core_dev *dev, struct mlx5_sf_table *ta if (state == mlx5_sf_to_devlink_state(sf->hw_state)) goto out; if (state == DEVLINK_PORT_FN_STATE_ACTIVE) - err = mlx5_sf_activate(dev, sf); + err = mlx5_sf_activate(dev, sf, extack); else if (state == DEVLINK_PORT_FN_STATE_INACTIVE) err = mlx5_sf_deactivate(dev, sf); else @@ -265,7 +269,7 @@ int mlx5_devlink_sf_port_fn_state_set(struct devlink *devlink, struct devlink_po goto out; } - err = mlx5_sf_state_set(dev, table, sf, state); + err = mlx5_sf_state_set(dev, table, sf, state, extack); out: mlx5_sf_table_put(table); return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c index ef5f892aafad..d9c69123c1ab 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c @@ -6,7 +6,6 @@ #include "sf.h" #include "mlx5_ifc_vhca_event.h" #include "ecpf.h" -#include "vhca_event.h" #include "mlx5_core.h" #include "eswitch.h" @@ -74,26 +73,29 @@ static int mlx5_sf_hw_table_id_alloc(struct mlx5_sf_hw_table *table, u32 control u32 usr_sfnum) { struct mlx5_sf_hwc_table *hwc; + int free_idx = -1; int i; hwc = mlx5_sf_controller_to_hwc(table->dev, controller); if (!hwc->sfs) return -ENOSPC; - /* Check if sf with same sfnum already exists or not. */ for (i = 0; i < hwc->max_fn; i++) { + if (!hwc->sfs[i].allocated && free_idx == -1) { + free_idx = i; + continue; + } + if (hwc->sfs[i].allocated && hwc->sfs[i].usr_sfnum == usr_sfnum) return -EEXIST; } - /* Find the free entry and allocate the entry from the array */ - for (i = 0; i < hwc->max_fn; i++) { - if (!hwc->sfs[i].allocated) { - hwc->sfs[i].usr_sfnum = usr_sfnum; - hwc->sfs[i].allocated = true; - return i; - } - } - return -ENOSPC; + + if (free_idx == -1) + return -ENOSPC; + + hwc->sfs[free_idx].usr_sfnum = usr_sfnum; + hwc->sfs[free_idx].allocated = true; + return free_idx; } static void mlx5_sf_hw_table_id_free(struct mlx5_sf_hw_table *table, u32 controller, int id) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h index 0b6aea1e6a94..81ce13b19ee8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h @@ -5,42 +5,7 @@ #define __MLX5_SF_H__ #include <linux/mlx5/driver.h> - -static inline u16 mlx5_sf_start_function_id(const struct mlx5_core_dev *dev) -{ - return MLX5_CAP_GEN(dev, sf_base_id); -} - -#ifdef CONFIG_MLX5_SF - -static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev) -{ - return MLX5_CAP_GEN(dev, sf); -} - -static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) -{ - if (!mlx5_sf_supported(dev)) - return 0; - if (MLX5_CAP_GEN(dev, max_num_sf)) - return MLX5_CAP_GEN(dev, max_num_sf); - else - return 1 << MLX5_CAP_GEN(dev, log_max_sf); -} - -#else - -static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev) -{ - return false; -} - -static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev) -{ - return 0; -} - -#endif +#include "lib/sf.h" #ifdef CONFIG_MLX5_SF_MANAGER diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index 2338989d4403..e8185b69ac6c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -34,6 +34,7 @@ #include <linux/mlx5/driver.h> #include <linux/mlx5/vport.h> #include "mlx5_core.h" +#include "mlx5_irq.h" #include "eswitch.h" static int sriov_restore_guids(struct mlx5_core_dev *dev, int vf) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c index 949879cf2092..6475ba35cf6b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c @@ -2,6 +2,7 @@ /* Copyright (c) 2019 Mellanox Technologies. */ #include "dr_types.h" +#include "dr_ste.h" enum dr_action_domain { DR_ACTION_DOMAIN_NIC_INGRESS, @@ -14,7 +15,8 @@ enum dr_action_domain { enum dr_action_valid_state { DR_ACTION_STATE_ERR, DR_ACTION_STATE_NO_ACTION, - DR_ACTION_STATE_REFORMAT, + DR_ACTION_STATE_ENCAP, + DR_ACTION_STATE_DECAP, DR_ACTION_STATE_MODIFY_HDR, DR_ACTION_STATE_MODIFY_VLAN, DR_ACTION_STATE_NON_TERM, @@ -29,46 +31,74 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, - [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, }, - [DR_ACTION_STATE_REFORMAT] = { + [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, - [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, }, + [DR_ACTION_STATE_ENCAP] = { + [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, + }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_MODIFY_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, - [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, }, @@ -80,39 +110,48 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NO_ACTION] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, }, - [DR_ACTION_STATE_REFORMAT] = { + [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, - [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, }, [DR_ACTION_STATE_MODIFY_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, }, @@ -124,41 +163,69 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NO_ACTION] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, - [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, - [DR_ACTION_STATE_REFORMAT] = { + [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, - [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, + }, + [DR_ACTION_STATE_ENCAP] = { + [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_MODIFY_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, - [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, @@ -171,44 +238,53 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_STATE_NO_ACTION] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, - [DR_ACTION_STATE_REFORMAT] = { + [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, - [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_MODIFY_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_VLAN, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, - [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, - [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_MODIFY_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, @@ -235,6 +311,9 @@ dr_action_reformat_to_action_type(enum mlx5dr_action_reformat_type reformat_type case DR_ACTION_REFORMAT_TYP_L2_TO_TNL_L3: *action_type = DR_ACTION_TYP_L2_TO_TNL_L3; break; + case DR_ACTION_REFORMAT_TYP_INSERT_HDR: + *action_type = DR_ACTION_TYP_INSERT_HDR; + break; default: return -EINVAL; } @@ -454,8 +533,17 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, break; case DR_ACTION_TYP_L2_TO_TNL_L2: case DR_ACTION_TYP_L2_TO_TNL_L3: - attr.reformat_size = action->reformat->reformat_size; - attr.reformat_id = action->reformat->reformat_id; + if (rx_rule && + !(dmn->ste_ctx->actions_caps & DR_STE_CTX_ACTION_CAP_RX_ENCAP)) { + mlx5dr_info(dmn, "Device doesn't support Encap on RX\n"); + goto out_invalid_arg; + } + attr.reformat.size = action->reformat->size; + attr.reformat.id = action->reformat->id; + break; + case DR_ACTION_TYP_SAMPLER: + attr.final_icm_addr = rx_rule ? action->sampler->rx_icm_addr : + action->sampler->tx_icm_addr; break; case DR_ACTION_TYP_VPORT: attr.hit_gvmi = action->vport->caps->vhca_gvmi; @@ -481,6 +569,12 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, attr.vlans.headers[attr.vlans.count++] = action->push_vlan->vlan_hdr; break; + case DR_ACTION_TYP_INSERT_HDR: + attr.reformat.size = action->reformat->size; + attr.reformat.id = action->reformat->id; + attr.reformat.param_0 = action->reformat->param_0; + attr.reformat.param_1 = action->reformat->param_1; + break; default: goto out_invalid_arg; } @@ -543,6 +637,8 @@ static unsigned int action_size[DR_ACTION_TYP_MAX] = { [DR_ACTION_TYP_MODIFY_HDR] = sizeof(struct mlx5dr_action_rewrite), [DR_ACTION_TYP_VPORT] = sizeof(struct mlx5dr_action_vport), [DR_ACTION_TYP_PUSH_VLAN] = sizeof(struct mlx5dr_action_push_vlan), + [DR_ACTION_TYP_INSERT_HDR] = sizeof(struct mlx5dr_action_reformat), + [DR_ACTION_TYP_SAMPLER] = sizeof(struct mlx5dr_action_sampler), }; static struct mlx5dr_action * @@ -651,7 +747,7 @@ mlx5dr_action_create_mult_dest_tbl(struct mlx5dr_domain *dmn, if (reformat_action) { reformat_req = true; hw_dests[i].vport.reformat_id = - reformat_action->reformat->reformat_id; + reformat_action->reformat->id; ref_actions[num_of_ref++] = reformat_action; hw_dests[i].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; } @@ -755,14 +851,43 @@ struct mlx5dr_action *mlx5dr_action_create_tag(u32 tag_value) return action; } +struct mlx5dr_action * +mlx5dr_action_create_flow_sampler(struct mlx5dr_domain *dmn, u32 sampler_id) +{ + struct mlx5dr_action *action; + u64 icm_rx, icm_tx; + int ret; + + ret = mlx5dr_cmd_query_flow_sampler(dmn->mdev, sampler_id, + &icm_rx, &icm_tx); + if (ret) + return NULL; + + action = dr_action_create_generic(DR_ACTION_TYP_SAMPLER); + if (!action) + return NULL; + + action->sampler->dmn = dmn; + action->sampler->sampler_id = sampler_id; + action->sampler->rx_icm_addr = icm_rx; + action->sampler->tx_icm_addr = icm_tx; + + refcount_inc(&dmn->refcount); + return action; +} + static int dr_action_verify_reformat_params(enum mlx5dr_action_type reformat_type, struct mlx5dr_domain *dmn, + u8 reformat_param_0, + u8 reformat_param_1, size_t data_sz, void *data) { - if ((!data && data_sz) || (data && !data_sz) || reformat_type > - DR_ACTION_TYP_L2_TO_TNL_L3) { + if ((!data && data_sz) || (data && !data_sz) || + ((reformat_param_0 || reformat_param_1) && + reformat_type != DR_ACTION_TYP_INSERT_HDR) || + reformat_type > DR_ACTION_TYP_INSERT_HDR) { mlx5dr_dbg(dmn, "Invalid reformat parameter!\n"); goto out_err; } @@ -794,6 +919,7 @@ out_err: static int dr_action_create_reformat_action(struct mlx5dr_domain *dmn, + u8 reformat_param_0, u8 reformat_param_1, size_t data_sz, void *data, struct mlx5dr_action *action) { @@ -811,13 +937,14 @@ dr_action_create_reformat_action(struct mlx5dr_domain *dmn, else rt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL; - ret = mlx5dr_cmd_create_reformat_ctx(dmn->mdev, rt, data_sz, data, + ret = mlx5dr_cmd_create_reformat_ctx(dmn->mdev, rt, 0, 0, + data_sz, data, &reformat_id); if (ret) return ret; - action->reformat->reformat_id = reformat_id; - action->reformat->reformat_size = data_sz; + action->reformat->id = reformat_id; + action->reformat->size = data_sz; return 0; } case DR_ACTION_TYP_TNL_L2_TO_L2: @@ -859,6 +986,23 @@ dr_action_create_reformat_action(struct mlx5dr_domain *dmn, } return 0; } + case DR_ACTION_TYP_INSERT_HDR: + { + ret = mlx5dr_cmd_create_reformat_ctx(dmn->mdev, + MLX5_REFORMAT_TYPE_INSERT_HDR, + reformat_param_0, + reformat_param_1, + data_sz, data, + &reformat_id); + if (ret) + return ret; + + action->reformat->id = reformat_id; + action->reformat->size = data_sz; + action->reformat->param_0 = reformat_param_0; + action->reformat->param_1 = reformat_param_1; + return 0; + } default: mlx5dr_info(dmn, "Reformat type is not supported %d\n", action->action_type); return -EINVAL; @@ -896,6 +1040,8 @@ struct mlx5dr_action *mlx5dr_action_create_push_vlan(struct mlx5dr_domain *dmn, struct mlx5dr_action * mlx5dr_action_create_packet_reformat(struct mlx5dr_domain *dmn, enum mlx5dr_action_reformat_type reformat_type, + u8 reformat_param_0, + u8 reformat_param_1, size_t data_sz, void *data) { @@ -912,7 +1058,9 @@ mlx5dr_action_create_packet_reformat(struct mlx5dr_domain *dmn, goto dec_ref; } - ret = dr_action_verify_reformat_params(action_type, dmn, data_sz, data); + ret = dr_action_verify_reformat_params(action_type, dmn, + reformat_param_0, reformat_param_1, + data_sz, data); if (ret) goto dec_ref; @@ -923,6 +1071,8 @@ mlx5dr_action_create_packet_reformat(struct mlx5dr_domain *dmn, action->reformat->dmn = dmn; ret = dr_action_create_reformat_action(dmn, + reformat_param_0, + reformat_param_1, data_sz, data, action); @@ -1516,8 +1666,9 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action) break; case DR_ACTION_TYP_L2_TO_TNL_L2: case DR_ACTION_TYP_L2_TO_TNL_L3: + case DR_ACTION_TYP_INSERT_HDR: mlx5dr_cmd_destroy_reformat_ctx((action->reformat->dmn)->mdev, - action->reformat->reformat_id); + action->reformat->id); refcount_dec(&action->reformat->dmn->refcount); break; case DR_ACTION_TYP_MODIFY_HDR: @@ -1525,6 +1676,9 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action) kfree(action->rewrite->data); refcount_dec(&action->rewrite->dmn->refcount); break; + case DR_ACTION_TYP_SAMPLER: + refcount_dec(&action->sampler->dmn->refcount); + break; default: break; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c index 5970cb8fc0c0..54e1f5438bbe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_cmd.c @@ -228,6 +228,36 @@ int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev, return 0; } +int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev, + u32 sampler_id, + u64 *rx_icm_addr, + u64 *tx_icm_addr) +{ + u32 out[MLX5_ST_SZ_DW(query_sampler_obj_out)] = {}; + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + void *attr; + int ret; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_QUERY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_SAMPLER); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sampler_id); + + ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (ret) + return ret; + + attr = MLX5_ADDR_OF(query_sampler_obj_out, out, sampler_object); + + *rx_icm_addr = MLX5_GET64(sampler_obj, attr, + sw_steering_icm_address_rx); + *tx_icm_addr = MLX5_GET64(sampler_obj, attr, + sw_steering_icm_address_tx); + + return 0; +} + int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev) { u32 in[MLX5_ST_SZ_DW(sync_steering_in)] = {}; @@ -460,6 +490,8 @@ int mlx5dr_cmd_destroy_flow_table(struct mlx5_core_dev *mdev, int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, enum mlx5_reformat_ctx_type rt, + u8 reformat_param_0, + u8 reformat_param_1, size_t reformat_size, void *reformat_data, u32 *reformat_id) @@ -486,8 +518,11 @@ int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, pdata = MLX5_ADDR_OF(packet_reformat_context_in, prctx, reformat_data); MLX5_SET(packet_reformat_context_in, prctx, reformat_type, rt); + MLX5_SET(packet_reformat_context_in, prctx, reformat_param_0, reformat_param_0); + MLX5_SET(packet_reformat_context_in, prctx, reformat_param_1, reformat_param_1); MLX5_SET(packet_reformat_context_in, prctx, reformat_data_size, reformat_size); - memcpy(pdata, reformat_data, reformat_size); + if (reformat_data && reformat_size) + memcpy(pdata, reformat_data, reformat_size); err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); if (err) @@ -706,6 +741,9 @@ int mlx5dr_cmd_set_fte(struct mlx5_core_dev *dev, fte->dest_arr[i].vport.reformat_id); } break; + case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER: + id = fte->dest_arr[i].sampler_id; + break; default: id = fte->dest_arr[i].tir_num; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c index 1fbcd012bb85..7ccfd40586ce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_fw.c @@ -112,7 +112,8 @@ int mlx5dr_fw_create_md_tbl(struct mlx5dr_domain *dmn, int ret; ft_attr.table_type = MLX5_FLOW_TABLE_TYPE_FDB; - ft_attr.level = dmn->info.caps.max_ft_level - 2; + ft_attr.level = min_t(int, dmn->info.caps.max_ft_level - 2, + MLX5_FT_MAX_MULTIPATH_LEVEL); ft_attr.reformat_en = reformat_req; ft_attr.decap_en = reformat_req; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h index 992b591bf0c5..12a8bbbf944b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste.h @@ -156,6 +156,7 @@ struct mlx5dr_ste_ctx { u16 (*get_byte_mask)(u8 *hw_ste_p); /* Actions */ + u32 actions_caps; void (*set_actions_rx)(struct mlx5dr_domain *dmn, u8 *action_type_set, u8 *hw_ste_arr, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v0.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v0.c index 0757a4e8540e..f1950e4968da 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v0.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v0.c @@ -437,8 +437,8 @@ dr_ste_v0_set_actions_tx(struct mlx5dr_domain *dmn, attr->gvmi); dr_ste_v0_set_tx_encap(last_ste, - attr->reformat_id, - attr->reformat_size, + attr->reformat.id, + attr->reformat.size, action_type_set[DR_ACTION_TYP_L2_TO_TNL_L3]); /* Whenever prio_tag_required enabled, we can be sure that the * previous table (ACL) already push vlan to our packet, @@ -1893,6 +1893,7 @@ struct mlx5dr_ste_ctx ste_ctx_v0 = { .get_byte_mask = &dr_ste_v0_get_byte_mask, /* Actions */ + .actions_caps = DR_STE_CTX_ACTION_CAP_NONE, .set_actions_rx = &dr_ste_v0_set_actions_rx, .set_actions_tx = &dr_ste_v0_set_actions_tx, .modify_field_arr_sz = ARRAY_SIZE(dr_ste_v0_action_modify_field_arr), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c index 054c2e2b6554..4aaca8eb7597 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c @@ -116,6 +116,8 @@ enum { DR_STE_V1_ACTION_MDFY_FLD_IPV6_SRC_OUT_3 = 0x4f, DR_STE_V1_ACTION_MDFY_FLD_TCP_MISC_0 = 0x5e, DR_STE_V1_ACTION_MDFY_FLD_TCP_MISC_1 = 0x5f, + DR_STE_V1_ACTION_MDFY_FLD_CFG_HDR_0_0 = 0x6f, + DR_STE_V1_ACTION_MDFY_FLD_CFG_HDR_0_1 = 0x70, DR_STE_V1_ACTION_MDFY_FLD_METADATA_2_CQE = 0x7b, DR_STE_V1_ACTION_MDFY_FLD_GNRL_PURPOSE = 0x7c, DR_STE_V1_ACTION_MDFY_FLD_REGISTER_2 = 0x8c, @@ -246,6 +248,12 @@ static const struct mlx5dr_ste_action_modify_field dr_ste_v1_action_modify_field [MLX5_ACTION_IN_FIELD_OUT_FIRST_VID] = { .hw_field = DR_STE_V1_ACTION_MDFY_FLD_L2_OUT_2, .start = 0, .end = 15, }, + [MLX5_ACTION_IN_FIELD_OUT_EMD_31_0] = { + .hw_field = DR_STE_V1_ACTION_MDFY_FLD_CFG_HDR_0_1, .start = 0, .end = 31, + }, + [MLX5_ACTION_IN_FIELD_OUT_EMD_47_32] = { + .hw_field = DR_STE_V1_ACTION_MDFY_FLD_CFG_HDR_0_0, .start = 0, .end = 15, + }, }; static void dr_ste_v1_set_entry_type(u8 *hw_ste_p, u8 entry_type) @@ -361,8 +369,8 @@ static void dr_ste_v1_set_reparse(u8 *hw_ste_p) MLX5_SET(ste_match_bwc_v1, hw_ste_p, reparse, 1); } -static void dr_ste_v1_set_tx_encap(u8 *hw_ste_p, u8 *d_action, - u32 reformat_id, int size) +static void dr_ste_v1_set_encap(u8 *hw_ste_p, u8 *d_action, + u32 reformat_id, int size) { MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, action_id, DR_STE_V1_ACTION_ID_INSERT_POINTER); @@ -374,6 +382,26 @@ static void dr_ste_v1_set_tx_encap(u8 *hw_ste_p, u8 *d_action, dr_ste_v1_set_reparse(hw_ste_p); } +static void dr_ste_v1_set_insert_hdr(u8 *hw_ste_p, u8 *d_action, + u32 reformat_id, + u8 anchor, u8 offset, + int size) +{ + MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, + action_id, DR_STE_V1_ACTION_ID_INSERT_POINTER); + MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, start_anchor, anchor); + + /* The hardware expects here size and offset in words (2 byte) */ + MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, size, size / 2); + MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, start_offset, offset / 2); + + MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, pointer, reformat_id); + MLX5_SET(ste_double_action_insert_with_ptr_v1, d_action, attributes, + DR_STE_V1_ACTION_INSERT_PTR_ATTR_NONE); + + dr_ste_v1_set_reparse(hw_ste_p); +} + static void dr_ste_v1_set_tx_push_vlan(u8 *hw_ste_p, u8 *d_action, u32 vlan_hdr) { @@ -401,11 +429,11 @@ static void dr_ste_v1_set_rx_pop_vlan(u8 *hw_ste_p, u8 *s_action, u8 vlans_num) dr_ste_v1_set_reparse(hw_ste_p); } -static void dr_ste_v1_set_tx_encap_l3(u8 *hw_ste_p, - u8 *frst_s_action, - u8 *scnd_d_action, - u32 reformat_id, - int size) +static void dr_ste_v1_set_encap_l3(u8 *hw_ste_p, + u8 *frst_s_action, + u8 *scnd_d_action, + u32 reformat_id, + int size) { /* Remove L2 headers */ MLX5_SET(ste_single_action_remove_header_v1, frst_s_action, action_id, @@ -519,9 +547,9 @@ static void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn, action_sz = DR_STE_ACTION_TRIPLE_SZ; allow_encap = true; } - dr_ste_v1_set_tx_encap(last_ste, action, - attr->reformat_id, - attr->reformat_size); + dr_ste_v1_set_encap(last_ste, action, + attr->reformat.id, + attr->reformat.size); action_sz -= DR_STE_ACTION_DOUBLE_SZ; action += DR_STE_ACTION_DOUBLE_SZ; } else if (action_type_set[DR_ACTION_TYP_L2_TO_TNL_L3]) { @@ -532,12 +560,25 @@ static void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn, action_sz = DR_STE_ACTION_TRIPLE_SZ; d_action = action + DR_STE_ACTION_SINGLE_SZ; - dr_ste_v1_set_tx_encap_l3(last_ste, - action, d_action, - attr->reformat_id, - attr->reformat_size); + dr_ste_v1_set_encap_l3(last_ste, + action, d_action, + attr->reformat.id, + attr->reformat.size); action_sz -= DR_STE_ACTION_TRIPLE_SZ; action += DR_STE_ACTION_TRIPLE_SZ; + } else if (action_type_set[DR_ACTION_TYP_INSERT_HDR]) { + if (!allow_encap || action_sz < DR_STE_ACTION_DOUBLE_SZ) { + dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); + action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); + action_sz = DR_STE_ACTION_TRIPLE_SZ; + } + dr_ste_v1_set_insert_hdr(last_ste, action, + attr->reformat.id, + attr->reformat.param_0, + attr->reformat.param_1, + attr->reformat.size); + action_sz -= DR_STE_ACTION_DOUBLE_SZ; + action += DR_STE_ACTION_DOUBLE_SZ; } dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi); @@ -616,7 +657,9 @@ static void dr_ste_v1_set_actions_rx(struct mlx5dr_domain *dmn, } if (action_type_set[DR_ACTION_TYP_CTR]) { - /* Counter action set after decap to exclude decaped header */ + /* Counter action set after decap and before insert_hdr + * to exclude decaped / encaped header respectively. + */ if (!allow_ctr) { dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); @@ -627,6 +670,52 @@ static void dr_ste_v1_set_actions_rx(struct mlx5dr_domain *dmn, dr_ste_v1_set_counter_id(last_ste, attr->ctr_id); } + if (action_type_set[DR_ACTION_TYP_L2_TO_TNL_L2]) { + if (action_sz < DR_STE_ACTION_DOUBLE_SZ) { + dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); + action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); + action_sz = DR_STE_ACTION_TRIPLE_SZ; + } + dr_ste_v1_set_encap(last_ste, action, + attr->reformat.id, + attr->reformat.size); + action_sz -= DR_STE_ACTION_DOUBLE_SZ; + action += DR_STE_ACTION_DOUBLE_SZ; + allow_modify_hdr = false; + } else if (action_type_set[DR_ACTION_TYP_L2_TO_TNL_L3]) { + u8 *d_action; + + if (action_sz < DR_STE_ACTION_TRIPLE_SZ) { + dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); + action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); + action_sz = DR_STE_ACTION_TRIPLE_SZ; + } + + d_action = action + DR_STE_ACTION_SINGLE_SZ; + + dr_ste_v1_set_encap_l3(last_ste, + action, d_action, + attr->reformat.id, + attr->reformat.size); + action_sz -= DR_STE_ACTION_TRIPLE_SZ; + allow_modify_hdr = false; + } else if (action_type_set[DR_ACTION_TYP_INSERT_HDR]) { + /* Modify header, decap, and encap must use different STEs */ + if (!allow_modify_hdr || action_sz < DR_STE_ACTION_DOUBLE_SZ) { + dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); + action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); + action_sz = DR_STE_ACTION_TRIPLE_SZ; + } + dr_ste_v1_set_insert_hdr(last_ste, action, + attr->reformat.id, + attr->reformat.param_0, + attr->reformat.param_1, + attr->reformat.size); + action_sz -= DR_STE_ACTION_DOUBLE_SZ; + action += DR_STE_ACTION_DOUBLE_SZ; + allow_modify_hdr = false; + } + dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi); dr_ste_v1_set_hit_addr(last_ste, attr->final_icm_addr, 1); } @@ -694,7 +783,11 @@ static int dr_ste_v1_set_action_decap_l3_list(void *data, if (hw_action_sz / DR_STE_ACTION_DOUBLE_SZ < DR_STE_DECAP_L3_ACTION_NUM) return -EINVAL; - memcpy(padded_data, data, data_sz); + inline_data_sz = + MLX5_FLD_SZ_BYTES(ste_double_action_insert_with_inline_v1, inline_data); + + /* Add an alignment padding */ + memcpy(padded_data + data_sz % inline_data_sz, data, data_sz); /* Remove L2L3 outer headers */ MLX5_SET(ste_single_action_remove_header_v1, hw_action, action_id, @@ -706,32 +799,34 @@ static int dr_ste_v1_set_action_decap_l3_list(void *data, hw_action += DR_STE_ACTION_DOUBLE_SZ; used_actions++; /* Remove and NOP are a single double action */ - inline_data_sz = - MLX5_FLD_SZ_BYTES(ste_double_action_insert_with_inline_v1, inline_data); + /* Point to the last dword of the header */ + data_ptr += (data_sz / inline_data_sz) * inline_data_sz; - /* Add the new header inline + 2 extra bytes */ + /* Add the new header using inline action 4Byte at a time, the header + * is added in reversed order to the beginning of the packet to avoid + * incorrect parsing by the HW. Since header is 14B or 18B an extra + * two bytes are padded and later removed. + */ for (i = 0; i < data_sz / inline_data_sz + 1; i++) { void *addr_inline; MLX5_SET(ste_double_action_insert_with_inline_v1, hw_action, action_id, DR_STE_V1_ACTION_ID_INSERT_INLINE); /* The hardware expects here offset to words (2 bytes) */ - MLX5_SET(ste_double_action_insert_with_inline_v1, hw_action, start_offset, - i * 2); + MLX5_SET(ste_double_action_insert_with_inline_v1, hw_action, start_offset, 0); /* Copy bytes one by one to avoid endianness problem */ addr_inline = MLX5_ADDR_OF(ste_double_action_insert_with_inline_v1, hw_action, inline_data); - memcpy(addr_inline, data_ptr, inline_data_sz); + memcpy(addr_inline, data_ptr - i * inline_data_sz, inline_data_sz); hw_action += DR_STE_ACTION_DOUBLE_SZ; - data_ptr += inline_data_sz; used_actions++; } - /* Remove 2 extra bytes */ + /* Remove first 2 extra bytes */ MLX5_SET(ste_single_action_remove_header_size_v1, hw_action, action_id, DR_STE_V1_ACTION_ID_REMOVE_BY_SIZE); - MLX5_SET(ste_single_action_remove_header_size_v1, hw_action, start_offset, data_sz / 2); + MLX5_SET(ste_single_action_remove_header_size_v1, hw_action, start_offset, 0); /* The hardware expects here size in words (2 bytes) */ MLX5_SET(ste_single_action_remove_header_size_v1, hw_action, remove_size, 1); used_actions++; @@ -1865,6 +1960,7 @@ struct mlx5dr_ste_ctx ste_ctx_v1 = { .set_byte_mask = &dr_ste_v1_set_byte_mask, .get_byte_mask = &dr_ste_v1_get_byte_mask, /* Actions */ + .actions_caps = DR_STE_CTX_ACTION_CAP_RX_ENCAP, .set_actions_rx = &dr_ste_v1_set_actions_rx, .set_actions_tx = &dr_ste_v1_set_actions_tx, .modify_field_arr_sz = ARRAY_SIZE(dr_ste_v1_action_modify_field_arr), diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 67460c42a99b..f5e93fa87aff 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -89,6 +89,11 @@ enum { DR_STE_SIZE_REDUCED = DR_STE_SIZE - DR_STE_SIZE_MASK, }; +enum mlx5dr_ste_ctx_action_cap { + DR_STE_CTX_ACTION_CAP_NONE = 0, + DR_STE_CTX_ACTION_CAP_RX_ENCAP = 1 << 0, +}; + enum { DR_MODIFY_ACTION_SIZE = 8, }; @@ -118,6 +123,8 @@ enum mlx5dr_action_type { DR_ACTION_TYP_VPORT, DR_ACTION_TYP_POP_VLAN, DR_ACTION_TYP_PUSH_VLAN, + DR_ACTION_TYP_INSERT_HDR, + DR_ACTION_TYP_SAMPLER, DR_ACTION_TYP_MAX, }; @@ -261,8 +268,12 @@ struct mlx5dr_ste_actions_attr { u32 ctr_id; u16 gvmi; u16 hit_gvmi; - u32 reformat_id; - u32 reformat_size; + struct { + u32 id; + u32 size; + u8 param_0; + u8 param_1; + } reformat; struct { int count; u32 headers[MLX5DR_MAX_VLANS]; @@ -903,8 +914,17 @@ struct mlx5dr_action_rewrite { struct mlx5dr_action_reformat { struct mlx5dr_domain *dmn; - u32 reformat_id; - u32 reformat_size; + u32 id; + u32 size; + u8 param_0; + u8 param_1; +}; + +struct mlx5dr_action_sampler { + struct mlx5dr_domain *dmn; + u64 rx_icm_addr; + u64 tx_icm_addr; + u32 sampler_id; }; struct mlx5dr_action_dest_tbl { @@ -950,6 +970,7 @@ struct mlx5dr_action { void *data; struct mlx5dr_action_rewrite *rewrite; struct mlx5dr_action_reformat *reformat; + struct mlx5dr_action_sampler *sampler; struct mlx5dr_action_dest_tbl *dest_tbl; struct mlx5dr_action_ctr *ctr; struct mlx5dr_action_vport *vport; @@ -1104,6 +1125,10 @@ int mlx5dr_cmd_query_gvmi(struct mlx5_core_dev *mdev, bool other_vport, u16 vport_number, u16 *gvmi); int mlx5dr_cmd_query_esw_caps(struct mlx5_core_dev *mdev, struct mlx5dr_esw_caps *caps); +int mlx5dr_cmd_query_flow_sampler(struct mlx5_core_dev *dev, + u32 sampler_id, + u64 *rx_icm_addr, + u64 *tx_icm_addr); int mlx5dr_cmd_sync_steering(struct mlx5_core_dev *mdev); int mlx5dr_cmd_set_fte_modify_and_vport(struct mlx5_core_dev *mdev, u32 table_type, @@ -1142,6 +1167,8 @@ int mlx5dr_cmd_query_flow_table(struct mlx5_core_dev *dev, struct mlx5dr_cmd_query_flow_table_details *output); int mlx5dr_cmd_create_reformat_ctx(struct mlx5_core_dev *mdev, enum mlx5_reformat_ctx_type rt, + u8 reformat_param_0, + u8 reformat_param_1, size_t reformat_size, void *reformat_data, u32 *reformat_id); @@ -1252,7 +1279,6 @@ struct mlx5dr_send_ring { u32 tx_head; void *buf; u32 buf_size; - struct ib_wc wc[MAX_SEND_CQE]; u8 sync_buff[MIN_READ_SYNC]; struct mlx5dr_mr *sync_mr; spinlock_t lock; /* Protect the data path of the send ring */ @@ -1290,6 +1316,7 @@ struct mlx5dr_cmd_flow_destination_hw_info { u32 ft_num; u32 ft_id; u32 counter_id; + u32 sampler_id; struct { u16 num; u16 vhca_id; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c index 96c39a17d026..d5926dd7e972 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c @@ -62,7 +62,7 @@ static int set_miss_action(struct mlx5_flow_root_namespace *ns, static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, struct mlx5_flow_table *ft, - unsigned int log_size, + unsigned int size, struct mlx5_flow_table *next_ft) { struct mlx5dr_table *tbl; @@ -71,7 +71,7 @@ static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, if (mlx5_dr_is_fw_table(ft->flags)) return mlx5_fs_cmd_get_fw_cmds()->create_flow_table(ns, ft, - log_size, + size, next_ft); flags = ft->flags; /* turn off encap/decap if not supported for sw-str by fw */ @@ -97,6 +97,8 @@ static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns, } } + ft->max_fte = INT_MAX; + return 0; } @@ -287,7 +289,8 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, DR_ACTION_REFORMAT_TYP_TNL_L2_TO_L2; tmp_action = mlx5dr_action_create_packet_reformat(domain, - decap_type, 0, + decap_type, + 0, 0, 0, NULL); if (!tmp_action) { err = -ENOMEM; @@ -384,7 +387,7 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { list_for_each_entry(dst, &fte->node.children, node.list) { enum mlx5_flow_destination_type type = dst->dest_attr.type; - u32 ft_id; + u32 id; if (num_actions == MLX5_FLOW_CONTEXT_ACTION_MAX || num_term_actions >= MLX5_FLOW_CONTEXT_ACTION_MAX) { @@ -422,9 +425,20 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, num_term_actions++; break; case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM: - ft_id = dst->dest_attr.ft_num; + id = dst->dest_attr.ft_num; tmp_action = mlx5dr_action_create_dest_table_num(domain, - ft_id); + id); + if (!tmp_action) { + err = -ENOMEM; + goto free_actions; + } + fs_dr_actions[fs_dr_num_actions++] = tmp_action; + term_actions[num_term_actions++].dest = tmp_action; + break; + case MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER: + id = dst->dest_attr.sampler_id; + tmp_action = mlx5dr_action_create_flow_sampler(domain, + id); if (!tmp_action) { err = -ENOMEM; goto free_actions; @@ -520,9 +534,7 @@ out_err: } static int mlx5_cmd_dr_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns, - int reformat_type, - size_t size, - void *reformat_data, + struct mlx5_pkt_reformat_params *params, enum mlx5_flow_namespace_type namespace, struct mlx5_pkt_reformat *pkt_reformat) { @@ -530,7 +542,7 @@ static int mlx5_cmd_dr_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns struct mlx5dr_action *action; int dr_reformat; - switch (reformat_type) { + switch (params->type) { case MLX5_REFORMAT_TYPE_L2_TO_VXLAN: case MLX5_REFORMAT_TYPE_L2_TO_NVGRE: case MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL: @@ -542,16 +554,21 @@ static int mlx5_cmd_dr_packet_reformat_alloc(struct mlx5_flow_root_namespace *ns case MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL: dr_reformat = DR_ACTION_REFORMAT_TYP_L2_TO_TNL_L3; break; + case MLX5_REFORMAT_TYPE_INSERT_HDR: + dr_reformat = DR_ACTION_REFORMAT_TYP_INSERT_HDR; + break; default: mlx5_core_err(ns->dev, "Packet-reformat not supported(%d)\n", - reformat_type); + params->type); return -EOPNOTSUPP; } action = mlx5dr_action_create_packet_reformat(dr_domain, dr_reformat, - size, - reformat_data); + params->param_0, + params->param_1, + params->size, + params->data); if (!action) { mlx5_core_err(ns->dev, "Failed allocating packet-reformat action\n"); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h index 612b0ac31db2..bbfe101d4e57 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h @@ -26,6 +26,7 @@ enum mlx5dr_action_reformat_type { DR_ACTION_REFORMAT_TYP_L2_TO_TNL_L2, DR_ACTION_REFORMAT_TYP_TNL_L3_TO_L2, DR_ACTION_REFORMAT_TYP_L2_TO_TNL_L3, + DR_ACTION_REFORMAT_TYP_INSERT_HDR, }; struct mlx5dr_match_parameters { @@ -100,11 +101,16 @@ struct mlx5dr_action *mlx5dr_action_create_drop(void); struct mlx5dr_action *mlx5dr_action_create_tag(u32 tag_value); struct mlx5dr_action * +mlx5dr_action_create_flow_sampler(struct mlx5dr_domain *dmn, u32 sampler_id); + +struct mlx5dr_action * mlx5dr_action_create_flow_counter(u32 counter_id); struct mlx5dr_action * mlx5dr_action_create_packet_reformat(struct mlx5dr_domain *dmn, enum mlx5dr_action_reformat_type reformat_type, + u8 reformat_param_0, + u8 reformat_param_1, size_t data_sz, void *data); @@ -124,10 +130,11 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action); static inline bool mlx5dr_is_supported(struct mlx5_core_dev *dev) { - return MLX5_CAP_ESW_FLOWTABLE_FDB(dev, sw_owner) || - (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, sw_owner_v2) && - (MLX5_CAP_GEN(dev, steering_format_version) <= - MLX5_STEERING_FORMAT_CONNECTX_6DX)); + return MLX5_CAP_GEN(dev, roce) && + (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, sw_owner) || + (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, sw_owner_v2) && + (MLX5_CAP_GEN(dev, steering_format_version) <= + MLX5_STEERING_FORMAT_CONNECTX_6DX))); } /* buddy functions & structure */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c index 01cc00ad8acf..b6931bbe52d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c @@ -424,6 +424,15 @@ err_modify_sq: return err; } +static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp) +{ + int i; + + for (i = 0; i < hp->num_channels; i++) + mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY, + MLX5_SQC_STATE_RST, 0, 0); +} + static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp) { int i; @@ -432,13 +441,9 @@ static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp) for (i = 0; i < hp->num_channels; i++) mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_RST, 0, 0); - /* unset peer SQs */ - if (hp->peer_gone) - return; - for (i = 0; i < hp->num_channels; i++) - mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY, - MLX5_SQC_STATE_RST, 0, 0); + if (!hp->peer_gone) + mlx5_hairpin_unpair_peer_sq(hp); } struct mlx5_hairpin * @@ -485,3 +490,16 @@ void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp) mlx5_hairpin_destroy_queues(hp); kfree(hp); } + +void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp) +{ + int i; + + mlx5_hairpin_unpair_peer_sq(hp); + + /* destroy peer SQ */ + for (i = 0; i < hp->num_channels; i++) + mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]); + + hp->peer_gone = true; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 457ad42eaa2a..4c1440a95ad7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -465,8 +465,6 @@ int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev, void *in; int err; - if (!vport) - return -EINVAL; if (!MLX5_CAP_GEN(mdev, vport_group_manager)) return -EACCES; diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/Kconfig b/drivers/net/ethernet/mellanox/mlxbf_gige/Kconfig new file mode 100644 index 000000000000..4cdebafaf222 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +# +# Mellanox GigE driver configuration +# + +config MLXBF_GIGE + tristate "Mellanox Technologies BlueField Gigabit Ethernet support" + depends on (ARM64 && ACPI) || COMPILE_TEST + select PHYLIB + help + The second generation BlueField SoC from Mellanox Technologies + supports an out-of-band Gigabit Ethernet management port to the + Arm subsystem. diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/Makefile b/drivers/net/ethernet/mellanox/mlxbf_gige/Makefile new file mode 100644 index 000000000000..e57c1375f236 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +obj-$(CONFIG_MLXBF_GIGE) += mlxbf_gige.o + +mlxbf_gige-y := mlxbf_gige_ethtool.o \ + mlxbf_gige_gpio.o \ + mlxbf_gige_intr.o \ + mlxbf_gige_main.o \ + mlxbf_gige_mdio.o \ + mlxbf_gige_rx.o \ + mlxbf_gige_tx.o diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h new file mode 100644 index 000000000000..e3509e69ed1c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */ + +/* Header file for Gigabit Ethernet driver for Mellanox BlueField SoC + * - this file contains software data structures and any chip-specific + * data structures (e.g. TX WQE format) that are memory resident. + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#ifndef __MLXBF_GIGE_H__ +#define __MLXBF_GIGE_H__ + +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/irqreturn.h> +#include <linux/netdevice.h> +#include <linux/irq.h> + +/* The silicon design supports a maximum RX ring size of + * 32K entries. Based on current testing this maximum size + * is not required to be supported. Instead the RX ring + * will be capped at a realistic value of 1024 entries. + */ +#define MLXBF_GIGE_MIN_RXQ_SZ 32 +#define MLXBF_GIGE_MAX_RXQ_SZ 1024 +#define MLXBF_GIGE_DEFAULT_RXQ_SZ 128 + +#define MLXBF_GIGE_MIN_TXQ_SZ 4 +#define MLXBF_GIGE_MAX_TXQ_SZ 256 +#define MLXBF_GIGE_DEFAULT_TXQ_SZ 128 + +#define MLXBF_GIGE_DEFAULT_BUF_SZ 2048 + +#define MLXBF_GIGE_DMA_PAGE_SZ 4096 +#define MLXBF_GIGE_DMA_PAGE_SHIFT 12 + +/* There are four individual MAC RX filters. Currently + * two of them are being used: one for the broadcast MAC + * (index 0) and one for local MAC (index 1) + */ +#define MLXBF_GIGE_BCAST_MAC_FILTER_IDX 0 +#define MLXBF_GIGE_LOCAL_MAC_FILTER_IDX 1 + +/* Define for broadcast MAC literal */ +#define BCAST_MAC_ADDR 0xFFFFFFFFFFFF + +/* There are three individual interrupts: + * 1) Errors, "OOB" interrupt line + * 2) Receive Packet, "OOB_LLU" interrupt line + * 3) LLU and PLU Events, "OOB_PLU" interrupt line + */ +#define MLXBF_GIGE_ERROR_INTR_IDX 0 +#define MLXBF_GIGE_RECEIVE_PKT_INTR_IDX 1 +#define MLXBF_GIGE_LLU_PLU_INTR_IDX 2 +#define MLXBF_GIGE_PHY_INT_N 3 + +#define MLXBF_GIGE_MDIO_DEFAULT_PHY_ADDR 0x3 + +#define MLXBF_GIGE_DEFAULT_PHY_INT_GPIO 12 + +struct mlxbf_gige_stats { + u64 hw_access_errors; + u64 tx_invalid_checksums; + u64 tx_small_frames; + u64 tx_index_errors; + u64 sw_config_errors; + u64 sw_access_errors; + u64 rx_truncate_errors; + u64 rx_mac_errors; + u64 rx_din_dropped_pkts; + u64 tx_fifo_full; + u64 rx_filter_passed_pkts; + u64 rx_filter_discard_pkts; +}; + +struct mlxbf_gige { + void __iomem *base; + void __iomem *llu_base; + void __iomem *plu_base; + struct device *dev; + struct net_device *netdev; + struct platform_device *pdev; + void __iomem *mdio_io; + struct mii_bus *mdiobus; + void __iomem *gpio_io; + struct irq_domain *irqdomain; + u32 phy_int_gpio_mask; + spinlock_t lock; /* for packet processing indices */ + spinlock_t gpio_lock; /* for GPIO bus access */ + u16 rx_q_entries; + u16 tx_q_entries; + u64 *tx_wqe_base; + dma_addr_t tx_wqe_base_dma; + u64 *tx_wqe_next; + u64 *tx_cc; + dma_addr_t tx_cc_dma; + dma_addr_t *rx_wqe_base; + dma_addr_t rx_wqe_base_dma; + u64 *rx_cqe_base; + dma_addr_t rx_cqe_base_dma; + u16 tx_pi; + u16 prev_tx_ci; + u64 error_intr_count; + u64 rx_intr_count; + u64 llu_plu_intr_count; + struct sk_buff *rx_skb[MLXBF_GIGE_MAX_RXQ_SZ]; + struct sk_buff *tx_skb[MLXBF_GIGE_MAX_TXQ_SZ]; + int error_irq; + int rx_irq; + int llu_plu_irq; + int phy_irq; + int hw_phy_irq; + bool promisc_enabled; + u8 valid_polarity; + struct napi_struct napi; + struct mlxbf_gige_stats stats; +}; + +/* Rx Work Queue Element definitions */ +#define MLXBF_GIGE_RX_WQE_SZ 8 + +/* Rx Completion Queue Element definitions */ +#define MLXBF_GIGE_RX_CQE_SZ 8 +#define MLXBF_GIGE_RX_CQE_PKT_LEN_MASK GENMASK(10, 0) +#define MLXBF_GIGE_RX_CQE_VALID_MASK GENMASK(11, 11) +#define MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK GENMASK(15, 12) +#define MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR GENMASK(12, 12) +#define MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED GENMASK(13, 13) +#define MLXBF_GIGE_RX_CQE_CHKSUM_MASK GENMASK(31, 16) + +/* Tx Work Queue Element definitions */ +#define MLXBF_GIGE_TX_WQE_SZ_QWORDS 2 +#define MLXBF_GIGE_TX_WQE_SZ 16 +#define MLXBF_GIGE_TX_WQE_PKT_LEN_MASK GENMASK(10, 0) +#define MLXBF_GIGE_TX_WQE_UPDATE_MASK GENMASK(31, 31) +#define MLXBF_GIGE_TX_WQE_CHKSUM_LEN_MASK GENMASK(42, 32) +#define MLXBF_GIGE_TX_WQE_CHKSUM_START_MASK GENMASK(55, 48) +#define MLXBF_GIGE_TX_WQE_CHKSUM_OFFSET_MASK GENMASK(63, 56) + +/* Macro to return packet length of specified TX WQE */ +#define MLXBF_GIGE_TX_WQE_PKT_LEN(tx_wqe_addr) \ + (*((tx_wqe_addr) + 1) & MLXBF_GIGE_TX_WQE_PKT_LEN_MASK) + +/* Tx Completion Count */ +#define MLXBF_GIGE_TX_CC_SZ 8 + +/* List of resources in ACPI table */ +enum mlxbf_gige_res { + MLXBF_GIGE_RES_MAC, + MLXBF_GIGE_RES_MDIO9, + MLXBF_GIGE_RES_GPIO0, + MLXBF_GIGE_RES_LLU, + MLXBF_GIGE_RES_PLU +}; + +/* Version of register data returned by mlxbf_gige_get_regs() */ +#define MLXBF_GIGE_REGS_VERSION 1 + +int mlxbf_gige_mdio_probe(struct platform_device *pdev, + struct mlxbf_gige *priv); +void mlxbf_gige_mdio_remove(struct mlxbf_gige *priv); +irqreturn_t mlxbf_gige_mdio_handle_phy_interrupt(int irq, void *dev_id); +void mlxbf_gige_mdio_enable_phy_int(struct mlxbf_gige *priv); + +void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv, + unsigned int index, u64 dmac); +void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv, + unsigned int index, u64 *dmac); +void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv); +void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv); +int mlxbf_gige_rx_init(struct mlxbf_gige *priv); +void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv); +int mlxbf_gige_tx_init(struct mlxbf_gige *priv); +void mlxbf_gige_tx_deinit(struct mlxbf_gige *priv); +bool mlxbf_gige_handle_tx_complete(struct mlxbf_gige *priv); +netdev_tx_t mlxbf_gige_start_xmit(struct sk_buff *skb, + struct net_device *netdev); +struct sk_buff *mlxbf_gige_alloc_skb(struct mlxbf_gige *priv, + unsigned int map_len, + dma_addr_t *buf_dma, + enum dma_data_direction dir); +int mlxbf_gige_request_irqs(struct mlxbf_gige *priv); +void mlxbf_gige_free_irqs(struct mlxbf_gige *priv); +int mlxbf_gige_poll(struct napi_struct *napi, int budget); +extern const struct ethtool_ops mlxbf_gige_ethtool_ops; +void mlxbf_gige_update_tx_wqe_next(struct mlxbf_gige *priv); + +int mlxbf_gige_gpio_init(struct platform_device *pdev, struct mlxbf_gige *priv); +void mlxbf_gige_gpio_free(struct mlxbf_gige *priv); + +#endif /* !defined(__MLXBF_GIGE_H__) */ diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c new file mode 100644 index 000000000000..92b798f8e73a --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* Ethtool support for Mellanox Gigabit Ethernet driver + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/phy.h> + +#include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" + +/* Start of struct ethtool_ops functions */ +static int mlxbf_gige_get_regs_len(struct net_device *netdev) +{ + return MLXBF_GIGE_MMIO_REG_SZ; +} + +static void mlxbf_gige_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, void *p) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + + regs->version = MLXBF_GIGE_REGS_VERSION; + + /* Read entire MMIO register space and store results + * into the provided buffer. Each 64-bit word is converted + * to big-endian to make the output more readable. + * + * NOTE: by design, a read to an offset without an existing + * register will be acknowledged and return zero. + */ + memcpy_fromio(p, priv->base, MLXBF_GIGE_MMIO_REG_SZ); +} + +static void mlxbf_gige_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + + ering->rx_max_pending = MLXBF_GIGE_MAX_RXQ_SZ; + ering->tx_max_pending = MLXBF_GIGE_MAX_TXQ_SZ; + ering->rx_pending = priv->rx_q_entries; + ering->tx_pending = priv->tx_q_entries; +} + +static const struct { + const char string[ETH_GSTRING_LEN]; +} mlxbf_gige_ethtool_stats_keys[] = { + { "hw_access_errors" }, + { "tx_invalid_checksums" }, + { "tx_small_frames" }, + { "tx_index_errors" }, + { "sw_config_errors" }, + { "sw_access_errors" }, + { "rx_truncate_errors" }, + { "rx_mac_errors" }, + { "rx_din_dropped_pkts" }, + { "tx_fifo_full" }, + { "rx_filter_passed_pkts" }, + { "rx_filter_discard_pkts" }, +}; + +static int mlxbf_gige_get_sset_count(struct net_device *netdev, int stringset) +{ + if (stringset != ETH_SS_STATS) + return -EOPNOTSUPP; + return ARRAY_SIZE(mlxbf_gige_ethtool_stats_keys); +} + +static void mlxbf_gige_get_strings(struct net_device *netdev, u32 stringset, + u8 *buf) +{ + if (stringset != ETH_SS_STATS) + return; + memcpy(buf, &mlxbf_gige_ethtool_stats_keys, + sizeof(mlxbf_gige_ethtool_stats_keys)); +} + +static void mlxbf_gige_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *estats, + u64 *data) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + + /* Fill data array with interface statistics + * + * NOTE: the data writes must be in + * sync with the strings shown in + * the mlxbf_gige_ethtool_stats_keys[] array + * + * NOTE2: certain statistics below are zeroed upon + * port disable, so the calculation below + * must include the "cached" value of the stat + * plus the value read directly from hardware. + * Cached statistics are currently: + * rx_din_dropped_pkts + * rx_filter_passed_pkts + * rx_filter_discard_pkts + */ + *data++ = priv->stats.hw_access_errors; + *data++ = priv->stats.tx_invalid_checksums; + *data++ = priv->stats.tx_small_frames; + *data++ = priv->stats.tx_index_errors; + *data++ = priv->stats.sw_config_errors; + *data++ = priv->stats.sw_access_errors; + *data++ = priv->stats.rx_truncate_errors; + *data++ = priv->stats.rx_mac_errors; + *data++ = (priv->stats.rx_din_dropped_pkts + + readq(priv->base + MLXBF_GIGE_RX_DIN_DROP_COUNTER)); + *data++ = priv->stats.tx_fifo_full; + *data++ = (priv->stats.rx_filter_passed_pkts + + readq(priv->base + MLXBF_GIGE_RX_PASS_COUNTER_ALL)); + *data++ = (priv->stats.rx_filter_discard_pkts + + readq(priv->base + MLXBF_GIGE_RX_DISC_COUNTER_ALL)); +} + +static void mlxbf_gige_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) +{ + pause->autoneg = AUTONEG_DISABLE; + pause->rx_pause = 1; + pause->tx_pause = 1; +} + +const struct ethtool_ops mlxbf_gige_ethtool_ops = { + .get_link = ethtool_op_get_link, + .get_ringparam = mlxbf_gige_get_ringparam, + .get_regs_len = mlxbf_gige_get_regs_len, + .get_regs = mlxbf_gige_get_regs, + .get_strings = mlxbf_gige_get_strings, + .get_sset_count = mlxbf_gige_get_sset_count, + .get_ethtool_stats = mlxbf_gige_get_ethtool_stats, + .nway_reset = phy_ethtool_nway_reset, + .get_pauseparam = mlxbf_gige_get_pauseparam, + .get_link_ksettings = phy_ethtool_get_link_ksettings, +}; diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_gpio.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_gpio.c new file mode 100644 index 000000000000..a8d966db5715 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_gpio.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* Initialize and handle GPIO interrupt triggered by INT_N PHY signal. + * This GPIO interrupt triggers the PHY state machine to bring the link + * up/down. + * + * Copyright (C) 2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/acpi.h> +#include <linux/bitfield.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/gpio/driver.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/irqreturn.h> +#include <linux/platform_device.h> +#include <linux/property.h> + +#include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" + +#define MLXBF_GIGE_GPIO_CAUSE_FALL_EN 0x48 +#define MLXBF_GIGE_GPIO_CAUSE_OR_CAUSE_EVTEN0 0x80 +#define MLXBF_GIGE_GPIO_CAUSE_OR_EVTEN0 0x94 +#define MLXBF_GIGE_GPIO_CAUSE_OR_CLRCAUSE 0x98 + +static void mlxbf_gige_gpio_enable(struct mlxbf_gige *priv) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&priv->gpio_lock, flags); + val = readl(priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_CLRCAUSE); + val |= priv->phy_int_gpio_mask; + writel(val, priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_CLRCAUSE); + + /* The INT_N interrupt level is active low. + * So enable cause fall bit to detect when GPIO + * state goes low. + */ + val = readl(priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_FALL_EN); + val |= priv->phy_int_gpio_mask; + writel(val, priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_FALL_EN); + + /* Enable PHY interrupt by setting the priority level */ + val = readl(priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_EVTEN0); + val |= priv->phy_int_gpio_mask; + writel(val, priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_EVTEN0); + spin_unlock_irqrestore(&priv->gpio_lock, flags); +} + +static void mlxbf_gige_gpio_disable(struct mlxbf_gige *priv) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&priv->gpio_lock, flags); + val = readl(priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_EVTEN0); + val &= ~priv->phy_int_gpio_mask; + writel(val, priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_EVTEN0); + spin_unlock_irqrestore(&priv->gpio_lock, flags); +} + +static irqreturn_t mlxbf_gige_gpio_handler(int irq, void *ptr) +{ + struct mlxbf_gige *priv; + u32 val; + + priv = ptr; + + /* Check if this interrupt is from PHY device. + * Return if it is not. + */ + val = readl(priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_CAUSE_EVTEN0); + if (!(val & priv->phy_int_gpio_mask)) + return IRQ_NONE; + + /* Clear interrupt when done, otherwise, no further interrupt + * will be triggered. + */ + val = readl(priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_CLRCAUSE); + val |= priv->phy_int_gpio_mask; + writel(val, priv->gpio_io + MLXBF_GIGE_GPIO_CAUSE_OR_CLRCAUSE); + + generic_handle_irq(priv->phy_irq); + + return IRQ_HANDLED; +} + +static void mlxbf_gige_gpio_mask(struct irq_data *irqd) +{ + struct mlxbf_gige *priv = irq_data_get_irq_chip_data(irqd); + + mlxbf_gige_gpio_disable(priv); +} + +static void mlxbf_gige_gpio_unmask(struct irq_data *irqd) +{ + struct mlxbf_gige *priv = irq_data_get_irq_chip_data(irqd); + + mlxbf_gige_gpio_enable(priv); +} + +static struct irq_chip mlxbf_gige_gpio_chip = { + .name = "mlxbf_gige_phy", + .irq_mask = mlxbf_gige_gpio_mask, + .irq_unmask = mlxbf_gige_gpio_unmask, +}; + +static int mlxbf_gige_gpio_domain_map(struct irq_domain *d, + unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_data(irq, d->host_data); + irq_set_chip_and_handler(irq, &mlxbf_gige_gpio_chip, handle_simple_irq); + irq_set_noprobe(irq); + + return 0; +} + +static const struct irq_domain_ops mlxbf_gige_gpio_domain_ops = { + .map = mlxbf_gige_gpio_domain_map, + .xlate = irq_domain_xlate_twocell, +}; + +#ifdef CONFIG_ACPI +static int mlxbf_gige_gpio_resources(struct acpi_resource *ares, + void *data) +{ + struct acpi_resource_gpio *gpio; + u32 *phy_int_gpio = data; + + if (ares->type == ACPI_RESOURCE_TYPE_GPIO) { + gpio = &ares->data.gpio; + *phy_int_gpio = gpio->pin_table[0]; + } + + return 1; +} +#endif + +void mlxbf_gige_gpio_free(struct mlxbf_gige *priv) +{ + irq_dispose_mapping(priv->phy_irq); + irq_domain_remove(priv->irqdomain); +} + +int mlxbf_gige_gpio_init(struct platform_device *pdev, + struct mlxbf_gige *priv) +{ + struct device *dev = &pdev->dev; + struct resource *res; + u32 phy_int_gpio = 0; + int ret; + + LIST_HEAD(resources); + + res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_GPIO0); + if (!res) + return -ENODEV; + + priv->gpio_io = devm_ioremap(dev, res->start, resource_size(res)); + if (!priv->gpio_io) + return -ENOMEM; + +#ifdef CONFIG_ACPI + ret = acpi_dev_get_resources(ACPI_COMPANION(dev), + &resources, mlxbf_gige_gpio_resources, + &phy_int_gpio); + acpi_dev_free_resource_list(&resources); + if (ret < 0 || !phy_int_gpio) { + dev_err(dev, "Error retrieving the gpio phy pin"); + return -EINVAL; + } +#endif + + priv->phy_int_gpio_mask = BIT(phy_int_gpio); + + mlxbf_gige_gpio_disable(priv); + + priv->hw_phy_irq = platform_get_irq(pdev, MLXBF_GIGE_PHY_INT_N); + + priv->irqdomain = irq_domain_add_simple(NULL, 1, 0, + &mlxbf_gige_gpio_domain_ops, + priv); + if (!priv->irqdomain) { + dev_err(dev, "Failed to add IRQ domain\n"); + return -ENOMEM; + } + + priv->phy_irq = irq_create_mapping(priv->irqdomain, 0); + if (!priv->phy_irq) { + irq_domain_remove(priv->irqdomain); + priv->irqdomain = NULL; + dev_err(dev, "Error mapping PHY IRQ\n"); + return -EINVAL; + } + + ret = devm_request_irq(dev, priv->hw_phy_irq, mlxbf_gige_gpio_handler, + IRQF_ONESHOT | IRQF_SHARED, "mlxbf_gige_phy", priv); + if (ret) { + dev_err(dev, "Failed to request PHY IRQ"); + mlxbf_gige_gpio_free(priv); + return ret; + } + + return ret; +} diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_intr.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_intr.c new file mode 100644 index 000000000000..c38795be04a2 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_intr.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* Interrupt related logic for Mellanox Gigabit Ethernet driver + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/interrupt.h> + +#include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" + +static irqreturn_t mlxbf_gige_error_intr(int irq, void *dev_id) +{ + struct mlxbf_gige *priv; + u64 int_status; + + priv = dev_id; + + priv->error_intr_count++; + + int_status = readq(priv->base + MLXBF_GIGE_INT_STATUS); + + if (int_status & MLXBF_GIGE_INT_STATUS_HW_ACCESS_ERROR) + priv->stats.hw_access_errors++; + + if (int_status & MLXBF_GIGE_INT_STATUS_TX_CHECKSUM_INPUTS) { + priv->stats.tx_invalid_checksums++; + /* This error condition is latched into MLXBF_GIGE_INT_STATUS + * when the GigE silicon operates on the offending + * TX WQE. The write to MLXBF_GIGE_INT_STATUS at the bottom + * of this routine clears this error condition. + */ + } + + if (int_status & MLXBF_GIGE_INT_STATUS_TX_SMALL_FRAME_SIZE) { + priv->stats.tx_small_frames++; + /* This condition happens when the networking stack invokes + * this driver's "start_xmit()" method with a packet whose + * size < 60 bytes. The GigE silicon will automatically pad + * this small frame up to a minimum-sized frame before it is + * sent. The "tx_small_frame" condition is latched into the + * MLXBF_GIGE_INT_STATUS register when the GigE silicon + * operates on the offending TX WQE. The write to + * MLXBF_GIGE_INT_STATUS at the bottom of this routine + * clears this condition. + */ + } + + if (int_status & MLXBF_GIGE_INT_STATUS_TX_PI_CI_EXCEED_WQ_SIZE) + priv->stats.tx_index_errors++; + + if (int_status & MLXBF_GIGE_INT_STATUS_SW_CONFIG_ERROR) + priv->stats.sw_config_errors++; + + if (int_status & MLXBF_GIGE_INT_STATUS_SW_ACCESS_ERROR) + priv->stats.sw_access_errors++; + + /* Clear all error interrupts by writing '1' back to + * all the asserted bits in INT_STATUS. Do not write + * '1' back to 'receive packet' bit, since that is + * managed separately. + */ + + int_status &= ~MLXBF_GIGE_INT_STATUS_RX_RECEIVE_PACKET; + + writeq(int_status, priv->base + MLXBF_GIGE_INT_STATUS); + + return IRQ_HANDLED; +} + +static irqreturn_t mlxbf_gige_rx_intr(int irq, void *dev_id) +{ + struct mlxbf_gige *priv; + + priv = dev_id; + + priv->rx_intr_count++; + + /* NOTE: GigE silicon automatically disables "packet rx" interrupt by + * setting MLXBF_GIGE_INT_MASK bit0 upon triggering the interrupt + * to the ARM cores. Software needs to re-enable "packet rx" + * interrupts by clearing MLXBF_GIGE_INT_MASK bit0. + */ + + napi_schedule(&priv->napi); + + return IRQ_HANDLED; +} + +static irqreturn_t mlxbf_gige_llu_plu_intr(int irq, void *dev_id) +{ + struct mlxbf_gige *priv; + + priv = dev_id; + priv->llu_plu_intr_count++; + + return IRQ_HANDLED; +} + +int mlxbf_gige_request_irqs(struct mlxbf_gige *priv) +{ + int err; + + err = request_irq(priv->error_irq, mlxbf_gige_error_intr, 0, + "mlxbf_gige_error", priv); + if (err) { + dev_err(priv->dev, "Request error_irq failure\n"); + return err; + } + + err = request_irq(priv->rx_irq, mlxbf_gige_rx_intr, 0, + "mlxbf_gige_rx", priv); + if (err) { + dev_err(priv->dev, "Request rx_irq failure\n"); + goto free_error_irq; + } + + err = request_irq(priv->llu_plu_irq, mlxbf_gige_llu_plu_intr, 0, + "mlxbf_gige_llu_plu", priv); + if (err) { + dev_err(priv->dev, "Request llu_plu_irq failure\n"); + goto free_rx_irq; + } + + return 0; + +free_rx_irq: + free_irq(priv->rx_irq, priv); + +free_error_irq: + free_irq(priv->error_irq, priv); + + return err; +} + +void mlxbf_gige_free_irqs(struct mlxbf_gige *priv) +{ + free_irq(priv->error_irq, priv); + free_irq(priv->rx_irq, priv); + free_irq(priv->llu_plu_irq, priv); +} diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c new file mode 100644 index 000000000000..a0a059e0154f --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* Gigabit Ethernet driver for Mellanox BlueField SoC + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/acpi.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/etherdevice.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/phy.h> +#include <linux/platform_device.h> +#include <linux/skbuff.h> + +#include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" + +#define DRV_NAME "mlxbf_gige" + +/* Allocate SKB whose payload pointer aligns with the Bluefield + * hardware DMA limitation, i.e. DMA operation can't cross + * a 4KB boundary. A maximum packet size of 2KB is assumed in the + * alignment formula. The alignment logic overallocates an SKB, + * and then adjusts the headroom so that the SKB data pointer is + * naturally aligned to a 2KB boundary. + */ +struct sk_buff *mlxbf_gige_alloc_skb(struct mlxbf_gige *priv, + unsigned int map_len, + dma_addr_t *buf_dma, + enum dma_data_direction dir) +{ + struct sk_buff *skb; + u64 addr, offset; + + /* Overallocate the SKB so that any headroom adjustment (to + * provide 2KB natural alignment) does not exceed payload area + */ + skb = netdev_alloc_skb(priv->netdev, MLXBF_GIGE_DEFAULT_BUF_SZ * 2); + if (!skb) + return NULL; + + /* Adjust the headroom so that skb->data is naturally aligned to + * a 2KB boundary, which is the maximum packet size supported. + */ + addr = (long)skb->data; + offset = (addr + MLXBF_GIGE_DEFAULT_BUF_SZ - 1) & + ~(MLXBF_GIGE_DEFAULT_BUF_SZ - 1); + offset -= addr; + if (offset) + skb_reserve(skb, offset); + + /* Return streaming DMA mapping to caller */ + *buf_dma = dma_map_single(priv->dev, skb->data, map_len, dir); + if (dma_mapping_error(priv->dev, *buf_dma)) { + dev_kfree_skb(skb); + *buf_dma = (dma_addr_t)0; + return NULL; + } + + return skb; +} + +static void mlxbf_gige_initial_mac(struct mlxbf_gige *priv) +{ + u8 mac[ETH_ALEN]; + u64 local_mac; + + memset(mac, 0, ETH_ALEN); + mlxbf_gige_get_mac_rx_filter(priv, MLXBF_GIGE_LOCAL_MAC_FILTER_IDX, + &local_mac); + u64_to_ether_addr(local_mac, mac); + + if (is_valid_ether_addr(mac)) { + ether_addr_copy(priv->netdev->dev_addr, mac); + } else { + /* Provide a random MAC if for some reason the device has + * not been configured with a valid MAC address already. + */ + eth_hw_addr_random(priv->netdev); + } + + local_mac = ether_addr_to_u64(priv->netdev->dev_addr); + mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_LOCAL_MAC_FILTER_IDX, + local_mac); +} + +static void mlxbf_gige_cache_stats(struct mlxbf_gige *priv) +{ + struct mlxbf_gige_stats *p; + + /* Cache stats that will be cleared by clean port operation */ + p = &priv->stats; + p->rx_din_dropped_pkts += readq(priv->base + + MLXBF_GIGE_RX_DIN_DROP_COUNTER); + p->rx_filter_passed_pkts += readq(priv->base + + MLXBF_GIGE_RX_PASS_COUNTER_ALL); + p->rx_filter_discard_pkts += readq(priv->base + + MLXBF_GIGE_RX_DISC_COUNTER_ALL); +} + +static int mlxbf_gige_clean_port(struct mlxbf_gige *priv) +{ + u64 control; + u64 temp; + int err; + + /* Set the CLEAN_PORT_EN bit to trigger SW reset */ + control = readq(priv->base + MLXBF_GIGE_CONTROL); + control |= MLXBF_GIGE_CONTROL_CLEAN_PORT_EN; + writeq(control, priv->base + MLXBF_GIGE_CONTROL); + + /* Ensure completion of "clean port" write before polling status */ + mb(); + + err = readq_poll_timeout_atomic(priv->base + MLXBF_GIGE_STATUS, temp, + (temp & MLXBF_GIGE_STATUS_READY), + 100, 100000); + + /* Clear the CLEAN_PORT_EN bit at end of this loop */ + control = readq(priv->base + MLXBF_GIGE_CONTROL); + control &= ~MLXBF_GIGE_CONTROL_CLEAN_PORT_EN; + writeq(control, priv->base + MLXBF_GIGE_CONTROL); + + return err; +} + +static int mlxbf_gige_open(struct net_device *netdev) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + struct phy_device *phydev = netdev->phydev; + u64 int_en; + int err; + + err = mlxbf_gige_request_irqs(priv); + if (err) + return err; + mlxbf_gige_cache_stats(priv); + err = mlxbf_gige_clean_port(priv); + if (err) + goto free_irqs; + err = mlxbf_gige_rx_init(priv); + if (err) + goto free_irqs; + err = mlxbf_gige_tx_init(priv); + if (err) + goto rx_deinit; + + phy_start(phydev); + + netif_napi_add(netdev, &priv->napi, mlxbf_gige_poll, NAPI_POLL_WEIGHT); + napi_enable(&priv->napi); + netif_start_queue(netdev); + + /* Set bits in INT_EN that we care about */ + int_en = MLXBF_GIGE_INT_EN_HW_ACCESS_ERROR | + MLXBF_GIGE_INT_EN_TX_CHECKSUM_INPUTS | + MLXBF_GIGE_INT_EN_TX_SMALL_FRAME_SIZE | + MLXBF_GIGE_INT_EN_TX_PI_CI_EXCEED_WQ_SIZE | + MLXBF_GIGE_INT_EN_SW_CONFIG_ERROR | + MLXBF_GIGE_INT_EN_SW_ACCESS_ERROR | + MLXBF_GIGE_INT_EN_RX_RECEIVE_PACKET; + + /* Ensure completion of all initialization before enabling interrupts */ + mb(); + + writeq(int_en, priv->base + MLXBF_GIGE_INT_EN); + + return 0; + +rx_deinit: + mlxbf_gige_rx_deinit(priv); + +free_irqs: + mlxbf_gige_free_irqs(priv); + return err; +} + +static int mlxbf_gige_stop(struct net_device *netdev) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + + writeq(0, priv->base + MLXBF_GIGE_INT_EN); + netif_stop_queue(netdev); + napi_disable(&priv->napi); + netif_napi_del(&priv->napi); + mlxbf_gige_free_irqs(priv); + + phy_stop(netdev->phydev); + + mlxbf_gige_rx_deinit(priv); + mlxbf_gige_tx_deinit(priv); + mlxbf_gige_cache_stats(priv); + mlxbf_gige_clean_port(priv); + + return 0; +} + +static int mlxbf_gige_do_ioctl(struct net_device *netdev, + struct ifreq *ifr, int cmd) +{ + if (!(netif_running(netdev))) + return -EINVAL; + + return phy_mii_ioctl(netdev->phydev, ifr, cmd); +} + +static void mlxbf_gige_set_rx_mode(struct net_device *netdev) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + bool new_promisc_enabled; + + new_promisc_enabled = netdev->flags & IFF_PROMISC; + + /* Only write to the hardware registers if the new setting + * of promiscuous mode is different from the current one. + */ + if (new_promisc_enabled != priv->promisc_enabled) { + priv->promisc_enabled = new_promisc_enabled; + + if (new_promisc_enabled) + mlxbf_gige_enable_promisc(priv); + else + mlxbf_gige_disable_promisc(priv); + } +} + +static void mlxbf_gige_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + + netdev_stats_to_stats64(stats, &netdev->stats); + + stats->rx_length_errors = priv->stats.rx_truncate_errors; + stats->rx_fifo_errors = priv->stats.rx_din_dropped_pkts + + readq(priv->base + MLXBF_GIGE_RX_DIN_DROP_COUNTER); + stats->rx_crc_errors = priv->stats.rx_mac_errors; + stats->rx_errors = stats->rx_length_errors + + stats->rx_fifo_errors + + stats->rx_crc_errors; + + stats->tx_fifo_errors = priv->stats.tx_fifo_full; + stats->tx_errors = stats->tx_fifo_errors; +} + +static const struct net_device_ops mlxbf_gige_netdev_ops = { + .ndo_open = mlxbf_gige_open, + .ndo_stop = mlxbf_gige_stop, + .ndo_start_xmit = mlxbf_gige_start_xmit, + .ndo_set_mac_address = eth_mac_addr, + .ndo_validate_addr = eth_validate_addr, + .ndo_do_ioctl = mlxbf_gige_do_ioctl, + .ndo_set_rx_mode = mlxbf_gige_set_rx_mode, + .ndo_get_stats64 = mlxbf_gige_get_stats64, +}; + +static void mlxbf_gige_adjust_link(struct net_device *netdev) +{ + struct phy_device *phydev = netdev->phydev; + + phy_print_status(phydev); +} + +static int mlxbf_gige_probe(struct platform_device *pdev) +{ + struct phy_device *phydev; + struct net_device *netdev; + struct resource *mac_res; + struct resource *llu_res; + struct resource *plu_res; + struct mlxbf_gige *priv; + void __iomem *llu_base; + void __iomem *plu_base; + void __iomem *base; + u64 control; + int addr; + int err; + + mac_res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_MAC); + if (!mac_res) + return -ENXIO; + + base = devm_ioremap_resource(&pdev->dev, mac_res); + if (IS_ERR(base)) + return PTR_ERR(base); + + llu_res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_LLU); + if (!llu_res) + return -ENXIO; + + llu_base = devm_ioremap_resource(&pdev->dev, llu_res); + if (IS_ERR(llu_base)) + return PTR_ERR(llu_base); + + plu_res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_PLU); + if (!plu_res) + return -ENXIO; + + plu_base = devm_ioremap_resource(&pdev->dev, plu_res); + if (IS_ERR(plu_base)) + return PTR_ERR(plu_base); + + /* Perform general init of GigE block */ + control = readq(base + MLXBF_GIGE_CONTROL); + control |= MLXBF_GIGE_CONTROL_PORT_EN; + writeq(control, base + MLXBF_GIGE_CONTROL); + + netdev = devm_alloc_etherdev(&pdev->dev, sizeof(*priv)); + if (!netdev) + return -ENOMEM; + + SET_NETDEV_DEV(netdev, &pdev->dev); + netdev->netdev_ops = &mlxbf_gige_netdev_ops; + netdev->ethtool_ops = &mlxbf_gige_ethtool_ops; + priv = netdev_priv(netdev); + priv->netdev = netdev; + + platform_set_drvdata(pdev, priv); + priv->dev = &pdev->dev; + priv->pdev = pdev; + + spin_lock_init(&priv->lock); + spin_lock_init(&priv->gpio_lock); + + /* Attach MDIO device */ + err = mlxbf_gige_mdio_probe(pdev, priv); + if (err) + return err; + + err = mlxbf_gige_gpio_init(pdev, priv); + if (err) { + dev_err(&pdev->dev, "PHY IRQ initialization failed\n"); + mlxbf_gige_mdio_remove(priv); + return -ENODEV; + } + + priv->base = base; + priv->llu_base = llu_base; + priv->plu_base = plu_base; + + priv->rx_q_entries = MLXBF_GIGE_DEFAULT_RXQ_SZ; + priv->tx_q_entries = MLXBF_GIGE_DEFAULT_TXQ_SZ; + + /* Write initial MAC address to hardware */ + mlxbf_gige_initial_mac(priv); + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err); + goto out; + } + + priv->error_irq = platform_get_irq(pdev, MLXBF_GIGE_ERROR_INTR_IDX); + priv->rx_irq = platform_get_irq(pdev, MLXBF_GIGE_RECEIVE_PKT_INTR_IDX); + priv->llu_plu_irq = platform_get_irq(pdev, MLXBF_GIGE_LLU_PLU_INTR_IDX); + + phydev = phy_find_first(priv->mdiobus); + if (!phydev) { + err = -ENODEV; + goto out; + } + + addr = phydev->mdio.addr; + priv->mdiobus->irq[addr] = priv->phy_irq; + phydev->irq = priv->phy_irq; + + err = phy_connect_direct(netdev, phydev, + mlxbf_gige_adjust_link, + PHY_INTERFACE_MODE_GMII); + if (err) { + dev_err(&pdev->dev, "Could not attach to PHY\n"); + goto out; + } + + /* MAC only supports 1000T full duplex mode */ + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Full_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); + + /* Only symmetric pause with flow control enabled is supported so no + * need to negotiate pause. + */ + linkmode_clear_bit(ETHTOOL_LINK_MODE_Pause_BIT, phydev->advertising); + linkmode_clear_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, phydev->advertising); + + /* Display information about attached PHY device */ + phy_attached_info(phydev); + + err = register_netdev(netdev); + if (err) { + dev_err(&pdev->dev, "Failed to register netdev\n"); + phy_disconnect(phydev); + goto out; + } + + return 0; + +out: + mlxbf_gige_gpio_free(priv); + mlxbf_gige_mdio_remove(priv); + return err; +} + +static int mlxbf_gige_remove(struct platform_device *pdev) +{ + struct mlxbf_gige *priv = platform_get_drvdata(pdev); + + unregister_netdev(priv->netdev); + phy_disconnect(priv->netdev->phydev); + mlxbf_gige_gpio_free(priv); + mlxbf_gige_mdio_remove(priv); + platform_set_drvdata(pdev, NULL); + + return 0; +} + +static void mlxbf_gige_shutdown(struct platform_device *pdev) +{ + struct mlxbf_gige *priv = platform_get_drvdata(pdev); + + writeq(0, priv->base + MLXBF_GIGE_INT_EN); + mlxbf_gige_clean_port(priv); +} + +static const struct acpi_device_id __maybe_unused mlxbf_gige_acpi_match[] = { + { "MLNXBF17", 0 }, + {}, +}; +MODULE_DEVICE_TABLE(acpi, mlxbf_gige_acpi_match); + +static struct platform_driver mlxbf_gige_driver = { + .probe = mlxbf_gige_probe, + .remove = mlxbf_gige_remove, + .shutdown = mlxbf_gige_shutdown, + .driver = { + .name = DRV_NAME, + .acpi_match_table = ACPI_PTR(mlxbf_gige_acpi_match), + }, +}; + +module_platform_driver(mlxbf_gige_driver); + +MODULE_DESCRIPTION("Mellanox BlueField SoC Gigabit Ethernet Driver"); +MODULE_AUTHOR("David Thompson <davthompson@nvidia.com>"); +MODULE_AUTHOR("Asmaa Mnebhi <asmaa@nvidia.com>"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c new file mode 100644 index 000000000000..e32dd34fdcc0 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* MDIO support for Mellanox Gigabit Ethernet driver + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/acpi.h> +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/ioport.h> +#include <linux/irqreturn.h> +#include <linux/jiffies.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/phy.h> +#include <linux/platform_device.h> +#include <linux/property.h> + +#include "mlxbf_gige.h" + +#define MLXBF_GIGE_MDIO_GW_OFFSET 0x0 +#define MLXBF_GIGE_MDIO_CFG_OFFSET 0x4 + +/* Support clause 22 */ +#define MLXBF_GIGE_MDIO_CL22_ST1 0x1 +#define MLXBF_GIGE_MDIO_CL22_WRITE 0x1 +#define MLXBF_GIGE_MDIO_CL22_READ 0x2 + +/* Busy bit is set by software and cleared by hardware */ +#define MLXBF_GIGE_MDIO_SET_BUSY 0x1 + +/* MDIO GW register bits */ +#define MLXBF_GIGE_MDIO_GW_AD_MASK GENMASK(15, 0) +#define MLXBF_GIGE_MDIO_GW_DEVAD_MASK GENMASK(20, 16) +#define MLXBF_GIGE_MDIO_GW_PARTAD_MASK GENMASK(25, 21) +#define MLXBF_GIGE_MDIO_GW_OPCODE_MASK GENMASK(27, 26) +#define MLXBF_GIGE_MDIO_GW_ST1_MASK GENMASK(28, 28) +#define MLXBF_GIGE_MDIO_GW_BUSY_MASK GENMASK(30, 30) + +/* MDIO config register bits */ +#define MLXBF_GIGE_MDIO_CFG_MDIO_MODE_MASK GENMASK(1, 0) +#define MLXBF_GIGE_MDIO_CFG_MDIO3_3_MASK GENMASK(2, 2) +#define MLXBF_GIGE_MDIO_CFG_MDIO_FULL_DRIVE_MASK GENMASK(4, 4) +#define MLXBF_GIGE_MDIO_CFG_MDC_PERIOD_MASK GENMASK(15, 8) +#define MLXBF_GIGE_MDIO_CFG_MDIO_IN_SAMP_MASK GENMASK(23, 16) +#define MLXBF_GIGE_MDIO_CFG_MDIO_OUT_SAMP_MASK GENMASK(31, 24) + +/* Formula for encoding the MDIO period. The encoded value is + * passed to the MDIO config register. + * + * mdc_clk = 2*(val + 1)*i1clk + * + * 400 ns = 2*(val + 1)*(((1/430)*1000) ns) + * + * val = (((400 * 430 / 1000) / 2) - 1) + */ +#define MLXBF_GIGE_I1CLK_MHZ 430 +#define MLXBF_GIGE_MDC_CLK_NS 400 + +#define MLXBF_GIGE_MDIO_PERIOD (((MLXBF_GIGE_MDC_CLK_NS * MLXBF_GIGE_I1CLK_MHZ / 1000) / 2) - 1) + +#define MLXBF_GIGE_MDIO_CFG_VAL (FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_MODE_MASK, 1) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO3_3_MASK, 1) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_FULL_DRIVE_MASK, 1) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDC_PERIOD_MASK, \ + MLXBF_GIGE_MDIO_PERIOD) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_IN_SAMP_MASK, 6) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_OUT_SAMP_MASK, 13)) + +static u32 mlxbf_gige_mdio_create_cmd(u16 data, int phy_add, + int phy_reg, u32 opcode) +{ + u32 gw_reg = 0; + + gw_reg |= FIELD_PREP(MLXBF_GIGE_MDIO_GW_AD_MASK, data); + gw_reg |= FIELD_PREP(MLXBF_GIGE_MDIO_GW_DEVAD_MASK, phy_reg); + gw_reg |= FIELD_PREP(MLXBF_GIGE_MDIO_GW_PARTAD_MASK, phy_add); + gw_reg |= FIELD_PREP(MLXBF_GIGE_MDIO_GW_OPCODE_MASK, opcode); + gw_reg |= FIELD_PREP(MLXBF_GIGE_MDIO_GW_ST1_MASK, + MLXBF_GIGE_MDIO_CL22_ST1); + gw_reg |= FIELD_PREP(MLXBF_GIGE_MDIO_GW_BUSY_MASK, + MLXBF_GIGE_MDIO_SET_BUSY); + + return gw_reg; +} + +static int mlxbf_gige_mdio_read(struct mii_bus *bus, int phy_add, int phy_reg) +{ + struct mlxbf_gige *priv = bus->priv; + u32 cmd; + int ret; + u32 val; + + if (phy_reg & MII_ADDR_C45) + return -EOPNOTSUPP; + + /* Send mdio read request */ + cmd = mlxbf_gige_mdio_create_cmd(0, phy_add, phy_reg, MLXBF_GIGE_MDIO_CL22_READ); + + writel(cmd, priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET); + + ret = readl_poll_timeout_atomic(priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET, + val, !(val & MLXBF_GIGE_MDIO_GW_BUSY_MASK), 100, 1000000); + + if (ret) { + writel(0, priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET); + return ret; + } + + ret = readl(priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET); + /* Only return ad bits of the gw register */ + ret &= MLXBF_GIGE_MDIO_GW_AD_MASK; + + return ret; +} + +static int mlxbf_gige_mdio_write(struct mii_bus *bus, int phy_add, + int phy_reg, u16 val) +{ + struct mlxbf_gige *priv = bus->priv; + u32 cmd; + int ret; + u32 temp; + + if (phy_reg & MII_ADDR_C45) + return -EOPNOTSUPP; + + /* Send mdio write request */ + cmd = mlxbf_gige_mdio_create_cmd(val, phy_add, phy_reg, + MLXBF_GIGE_MDIO_CL22_WRITE); + writel(cmd, priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET); + + /* If the poll timed out, drop the request */ + ret = readl_poll_timeout_atomic(priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET, + temp, !(temp & MLXBF_GIGE_MDIO_GW_BUSY_MASK), 100, 1000000); + + return ret; +} + +int mlxbf_gige_mdio_probe(struct platform_device *pdev, struct mlxbf_gige *priv) +{ + struct device *dev = &pdev->dev; + struct resource *res; + int ret; + + res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_MDIO9); + if (!res) + return -ENODEV; + + priv->mdio_io = devm_ioremap_resource(dev, res); + if (IS_ERR(priv->mdio_io)) + return PTR_ERR(priv->mdio_io); + + /* Configure mdio parameters */ + writel(MLXBF_GIGE_MDIO_CFG_VAL, + priv->mdio_io + MLXBF_GIGE_MDIO_CFG_OFFSET); + + priv->mdiobus = devm_mdiobus_alloc(dev); + if (!priv->mdiobus) { + dev_err(dev, "Failed to alloc MDIO bus\n"); + return -ENOMEM; + } + + priv->mdiobus->name = "mlxbf-mdio"; + priv->mdiobus->read = mlxbf_gige_mdio_read; + priv->mdiobus->write = mlxbf_gige_mdio_write; + priv->mdiobus->parent = dev; + priv->mdiobus->priv = priv; + snprintf(priv->mdiobus->id, MII_BUS_ID_SIZE, "%s", + dev_name(dev)); + + ret = mdiobus_register(priv->mdiobus); + if (ret) + dev_err(dev, "Failed to register MDIO bus\n"); + + return ret; +} + +void mlxbf_gige_mdio_remove(struct mlxbf_gige *priv) +{ + mdiobus_unregister(priv->mdiobus); +} diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h new file mode 100644 index 000000000000..5fb33c9294bf --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */ + +/* Header file for Mellanox BlueField GigE register defines + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#ifndef __MLXBF_GIGE_REGS_H__ +#define __MLXBF_GIGE_REGS_H__ + +#define MLXBF_GIGE_STATUS 0x0010 +#define MLXBF_GIGE_STATUS_READY BIT(0) +#define MLXBF_GIGE_INT_STATUS 0x0028 +#define MLXBF_GIGE_INT_STATUS_RX_RECEIVE_PACKET BIT(0) +#define MLXBF_GIGE_INT_STATUS_RX_MAC_ERROR BIT(1) +#define MLXBF_GIGE_INT_STATUS_RX_TRN_ERROR BIT(2) +#define MLXBF_GIGE_INT_STATUS_SW_ACCESS_ERROR BIT(3) +#define MLXBF_GIGE_INT_STATUS_SW_CONFIG_ERROR BIT(4) +#define MLXBF_GIGE_INT_STATUS_TX_PI_CI_EXCEED_WQ_SIZE BIT(5) +#define MLXBF_GIGE_INT_STATUS_TX_SMALL_FRAME_SIZE BIT(6) +#define MLXBF_GIGE_INT_STATUS_TX_CHECKSUM_INPUTS BIT(7) +#define MLXBF_GIGE_INT_STATUS_HW_ACCESS_ERROR BIT(8) +#define MLXBF_GIGE_INT_EN 0x0030 +#define MLXBF_GIGE_INT_EN_RX_RECEIVE_PACKET BIT(0) +#define MLXBF_GIGE_INT_EN_RX_MAC_ERROR BIT(1) +#define MLXBF_GIGE_INT_EN_RX_TRN_ERROR BIT(2) +#define MLXBF_GIGE_INT_EN_SW_ACCESS_ERROR BIT(3) +#define MLXBF_GIGE_INT_EN_SW_CONFIG_ERROR BIT(4) +#define MLXBF_GIGE_INT_EN_TX_PI_CI_EXCEED_WQ_SIZE BIT(5) +#define MLXBF_GIGE_INT_EN_TX_SMALL_FRAME_SIZE BIT(6) +#define MLXBF_GIGE_INT_EN_TX_CHECKSUM_INPUTS BIT(7) +#define MLXBF_GIGE_INT_EN_HW_ACCESS_ERROR BIT(8) +#define MLXBF_GIGE_INT_MASK 0x0038 +#define MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET BIT(0) +#define MLXBF_GIGE_CONTROL 0x0040 +#define MLXBF_GIGE_CONTROL_PORT_EN BIT(0) +#define MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN BIT(1) +#define MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC BIT(4) +#define MLXBF_GIGE_CONTROL_CLEAN_PORT_EN BIT(31) +#define MLXBF_GIGE_RX_WQ_BASE 0x0200 +#define MLXBF_GIGE_RX_WQE_SIZE_LOG2 0x0208 +#define MLXBF_GIGE_RX_WQE_SIZE_LOG2_RESET_VAL 7 +#define MLXBF_GIGE_RX_CQ_BASE 0x0210 +#define MLXBF_GIGE_TX_WQ_BASE 0x0218 +#define MLXBF_GIGE_TX_WQ_SIZE_LOG2 0x0220 +#define MLXBF_GIGE_TX_WQ_SIZE_LOG2_RESET_VAL 7 +#define MLXBF_GIGE_TX_CI_UPDATE_ADDRESS 0x0228 +#define MLXBF_GIGE_RX_WQE_PI 0x0230 +#define MLXBF_GIGE_TX_PRODUCER_INDEX 0x0238 +#define MLXBF_GIGE_RX_MAC_FILTER 0x0240 +#define MLXBF_GIGE_RX_MAC_FILTER_STRIDE 0x0008 +#define MLXBF_GIGE_RX_DIN_DROP_COUNTER 0x0260 +#define MLXBF_GIGE_TX_CONSUMER_INDEX 0x0310 +#define MLXBF_GIGE_TX_CONTROL 0x0318 +#define MLXBF_GIGE_TX_CONTROL_GRACEFUL_STOP BIT(0) +#define MLXBF_GIGE_TX_STATUS 0x0388 +#define MLXBF_GIGE_TX_STATUS_DATA_FIFO_FULL BIT(1) +#define MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START 0x0520 +#define MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END 0x0528 +#define MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC 0x0540 +#define MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN BIT(0) +#define MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS 0x0548 +#define MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN BIT(0) +#define MLXBF_GIGE_RX_PASS_COUNTER_ALL 0x0550 +#define MLXBF_GIGE_RX_DISC_COUNTER_ALL 0x0560 +#define MLXBF_GIGE_RX 0x0578 +#define MLXBF_GIGE_RX_STRIP_CRC_EN BIT(1) +#define MLXBF_GIGE_RX_DMA 0x0580 +#define MLXBF_GIGE_RX_DMA_EN BIT(0) +#define MLXBF_GIGE_RX_CQE_PACKET_CI 0x05b0 +#define MLXBF_GIGE_MAC_CFG 0x05e8 + +/* NOTE: MLXBF_GIGE_MAC_CFG is the last defined register offset, + * so use that plus size of single register to derive total size + */ +#define MLXBF_GIGE_MMIO_REG_SZ (MLXBF_GIGE_MAC_CFG + 8) + +#endif /* !defined(__MLXBF_GIGE_REGS_H__) */ diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_rx.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_rx.c new file mode 100644 index 000000000000..afa3b92a6905 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_rx.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* Packet receive logic for Mellanox Gigabit Ethernet driver + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/etherdevice.h> +#include <linux/skbuff.h> + +#include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" + +void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv, + unsigned int index, u64 dmac) +{ + void __iomem *base = priv->base; + u64 control; + + /* Write destination MAC to specified MAC RX filter */ + writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER + + (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE)); + + /* Enable MAC receive filter mask for specified index */ + control = readq(base + MLXBF_GIGE_CONTROL); + control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index); + writeq(control, base + MLXBF_GIGE_CONTROL); +} + +void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv, + unsigned int index, u64 *dmac) +{ + void __iomem *base = priv->base; + + /* Read destination MAC from specified MAC RX filter */ + *dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER + + (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE)); +} + +void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv) +{ + void __iomem *base = priv->base; + u64 control; + u64 end_mac; + + /* Enable MAC_ID_RANGE match functionality */ + control = readq(base + MLXBF_GIGE_CONTROL); + control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN; + writeq(control, base + MLXBF_GIGE_CONTROL); + + /* Set start of destination MAC range check to 0 */ + writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START); + + /* Set end of destination MAC range check to all FFs */ + end_mac = BCAST_MAC_ADDR; + writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END); +} + +void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv) +{ + void __iomem *base = priv->base; + u64 control; + + /* Disable MAC_ID_RANGE match functionality */ + control = readq(base + MLXBF_GIGE_CONTROL); + control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN; + writeq(control, base + MLXBF_GIGE_CONTROL); + + /* NOTE: no need to change DMAC_RANGE_START or END; + * those values are ignored since MAC_ID_RANGE_EN=0 + */ +} + +/* Receive Initialization + * 1) Configures RX MAC filters via MMIO registers + * 2) Allocates RX WQE array using coherent DMA mapping + * 3) Initializes each element of RX WQE array with a receive + * buffer pointer (also using coherent DMA mapping) + * 4) Allocates RX CQE array using coherent DMA mapping + * 5) Completes other misc receive initialization + */ +int mlxbf_gige_rx_init(struct mlxbf_gige *priv) +{ + size_t wq_size, cq_size; + dma_addr_t *rx_wqe_ptr; + dma_addr_t rx_buf_dma; + u64 data; + int i, j; + + /* Configure MAC RX filter #0 to allow RX of broadcast pkts */ + mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX, + BCAST_MAC_ADDR); + + wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries; + priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size, + &priv->rx_wqe_base_dma, + GFP_KERNEL); + if (!priv->rx_wqe_base) + return -ENOMEM; + + /* Initialize 'rx_wqe_ptr' to point to first RX WQE in array + * Each RX WQE is simply a receive buffer pointer, so walk + * the entire array, allocating a 2KB buffer for each element + */ + rx_wqe_ptr = priv->rx_wqe_base; + + for (i = 0; i < priv->rx_q_entries; i++) { + priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ, + &rx_buf_dma, DMA_FROM_DEVICE); + if (!priv->rx_skb[i]) + goto free_wqe_and_skb; + *rx_wqe_ptr++ = rx_buf_dma; + } + + /* Write RX WQE base address into MMIO reg */ + writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE); + + cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries; + priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size, + &priv->rx_cqe_base_dma, + GFP_KERNEL); + if (!priv->rx_cqe_base) + goto free_wqe_and_skb; + + for (i = 0; i < priv->rx_q_entries; i++) + priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK; + + /* Write RX CQE base address into MMIO reg */ + writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE); + + /* Write RX_WQE_PI with current number of replenished buffers */ + writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI); + + /* Enable removal of CRC during RX */ + data = readq(priv->base + MLXBF_GIGE_RX); + data |= MLXBF_GIGE_RX_STRIP_CRC_EN; + writeq(data, priv->base + MLXBF_GIGE_RX); + + /* Enable RX MAC filter pass and discard counters */ + writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN, + priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC); + writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN, + priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS); + + /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to + * indicate readiness to receive interrupts + */ + data = readq(priv->base + MLXBF_GIGE_INT_MASK); + data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET; + writeq(data, priv->base + MLXBF_GIGE_INT_MASK); + + /* Enable RX DMA to write new packets to memory */ + data = readq(priv->base + MLXBF_GIGE_RX_DMA); + data |= MLXBF_GIGE_RX_DMA_EN; + writeq(data, priv->base + MLXBF_GIGE_RX_DMA); + + writeq(ilog2(priv->rx_q_entries), + priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2); + + return 0; + +free_wqe_and_skb: + rx_wqe_ptr = priv->rx_wqe_base; + for (j = 0; j < i; j++) { + dma_unmap_single(priv->dev, *rx_wqe_ptr, + MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE); + dev_kfree_skb(priv->rx_skb[j]); + rx_wqe_ptr++; + } + dma_free_coherent(priv->dev, wq_size, + priv->rx_wqe_base, priv->rx_wqe_base_dma); + return -ENOMEM; +} + +/* Receive Deinitialization + * This routine will free allocations done by mlxbf_gige_rx_init(), + * namely the RX WQE and RX CQE arrays, as well as all RX buffers + */ +void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv) +{ + dma_addr_t *rx_wqe_ptr; + size_t size; + u64 data; + int i; + + /* Disable RX DMA to prevent packet transfers to memory */ + data = readq(priv->base + MLXBF_GIGE_RX_DMA); + data &= ~MLXBF_GIGE_RX_DMA_EN; + writeq(data, priv->base + MLXBF_GIGE_RX_DMA); + + rx_wqe_ptr = priv->rx_wqe_base; + + for (i = 0; i < priv->rx_q_entries; i++) { + dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ, + DMA_FROM_DEVICE); + dev_kfree_skb(priv->rx_skb[i]); + rx_wqe_ptr++; + } + + size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries; + dma_free_coherent(priv->dev, size, + priv->rx_wqe_base, priv->rx_wqe_base_dma); + + size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries; + dma_free_coherent(priv->dev, size, + priv->rx_cqe_base, priv->rx_cqe_base_dma); + + priv->rx_wqe_base = NULL; + priv->rx_wqe_base_dma = 0; + priv->rx_cqe_base = NULL; + priv->rx_cqe_base_dma = 0; + writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE); + writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE); +} + +static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts) +{ + struct net_device *netdev = priv->netdev; + struct sk_buff *skb = NULL, *rx_skb; + u16 rx_pi_rem, rx_ci_rem; + dma_addr_t *rx_wqe_addr; + dma_addr_t rx_buf_dma; + u64 *rx_cqe_addr; + u64 datalen; + u64 rx_cqe; + u16 rx_ci; + u16 rx_pi; + + /* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */ + rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI); + rx_pi_rem = rx_pi % priv->rx_q_entries; + + rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem; + rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem; + rx_cqe = *rx_cqe_addr; + + if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity) + return false; + + if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) { + /* Packet is OK, increment stats */ + datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK; + netdev->stats.rx_packets++; + netdev->stats.rx_bytes += datalen; + + skb = priv->rx_skb[rx_pi_rem]; + + skb_put(skb, datalen); + + skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */ + + skb->protocol = eth_type_trans(skb, netdev); + + /* Alloc another RX SKB for this same index */ + rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ, + &rx_buf_dma, DMA_FROM_DEVICE); + if (!rx_skb) + return false; + priv->rx_skb[rx_pi_rem] = rx_skb; + dma_unmap_single(priv->dev, *rx_wqe_addr, + MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE); + *rx_wqe_addr = rx_buf_dma; + } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) { + priv->stats.rx_mac_errors++; + } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) { + priv->stats.rx_truncate_errors++; + } + + /* Let hardware know we've replenished one buffer */ + rx_pi++; + + /* Ensure completion of all writes before notifying HW of replenish */ + wmb(); + writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI); + + (*rx_pkts)++; + + rx_pi_rem = rx_pi % priv->rx_q_entries; + if (rx_pi_rem == 0) + priv->valid_polarity ^= 1; + rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI); + rx_ci_rem = rx_ci % priv->rx_q_entries; + + if (skb) + netif_receive_skb(skb); + + return rx_pi_rem != rx_ci_rem; +} + +/* Driver poll() function called by NAPI infrastructure */ +int mlxbf_gige_poll(struct napi_struct *napi, int budget) +{ + struct mlxbf_gige *priv; + bool remaining_pkts; + int work_done = 0; + u64 data; + + priv = container_of(napi, struct mlxbf_gige, napi); + + mlxbf_gige_handle_tx_complete(priv); + + do { + remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done); + } while (remaining_pkts && work_done < budget); + + /* If amount of work done < budget, turn off NAPI polling + * via napi_complete_done(napi, work_done) and then + * re-enable interrupts. + */ + if (work_done < budget && napi_complete_done(napi, work_done)) { + /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to + * indicate receive readiness + */ + data = readq(priv->base + MLXBF_GIGE_INT_MASK); + data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET; + writeq(data, priv->base + MLXBF_GIGE_INT_MASK); + } + + return work_done; +} diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_tx.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_tx.c new file mode 100644 index 000000000000..04982e888c63 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_tx.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + +/* Packet transmit logic for Mellanox Gigabit Ethernet driver + * + * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES + */ + +#include <linux/skbuff.h> + +#include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" + +/* Transmit Initialization + * 1) Allocates TX WQE array using coherent DMA mapping + * 2) Allocates TX completion counter using coherent DMA mapping + */ +int mlxbf_gige_tx_init(struct mlxbf_gige *priv) +{ + size_t size; + + size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries; + priv->tx_wqe_base = dma_alloc_coherent(priv->dev, size, + &priv->tx_wqe_base_dma, + GFP_KERNEL); + if (!priv->tx_wqe_base) + return -ENOMEM; + + priv->tx_wqe_next = priv->tx_wqe_base; + + /* Write TX WQE base address into MMIO reg */ + writeq(priv->tx_wqe_base_dma, priv->base + MLXBF_GIGE_TX_WQ_BASE); + + /* Allocate address for TX completion count */ + priv->tx_cc = dma_alloc_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ, + &priv->tx_cc_dma, GFP_KERNEL); + if (!priv->tx_cc) { + dma_free_coherent(priv->dev, size, + priv->tx_wqe_base, priv->tx_wqe_base_dma); + return -ENOMEM; + } + + /* Write TX CC base address into MMIO reg */ + writeq(priv->tx_cc_dma, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS); + + writeq(ilog2(priv->tx_q_entries), + priv->base + MLXBF_GIGE_TX_WQ_SIZE_LOG2); + + priv->prev_tx_ci = 0; + priv->tx_pi = 0; + + return 0; +} + +/* Transmit Deinitialization + * This routine will free allocations done by mlxbf_gige_tx_init(), + * namely the TX WQE array and the TX completion counter + */ +void mlxbf_gige_tx_deinit(struct mlxbf_gige *priv) +{ + u64 *tx_wqe_addr; + size_t size; + int i; + + tx_wqe_addr = priv->tx_wqe_base; + + for (i = 0; i < priv->tx_q_entries; i++) { + if (priv->tx_skb[i]) { + dma_unmap_single(priv->dev, *tx_wqe_addr, + priv->tx_skb[i]->len, DMA_TO_DEVICE); + dev_kfree_skb(priv->tx_skb[i]); + priv->tx_skb[i] = NULL; + } + tx_wqe_addr += 2; + } + + size = MLXBF_GIGE_TX_WQE_SZ * priv->tx_q_entries; + dma_free_coherent(priv->dev, size, + priv->tx_wqe_base, priv->tx_wqe_base_dma); + + dma_free_coherent(priv->dev, MLXBF_GIGE_TX_CC_SZ, + priv->tx_cc, priv->tx_cc_dma); + + priv->tx_wqe_base = NULL; + priv->tx_wqe_base_dma = 0; + priv->tx_cc = NULL; + priv->tx_cc_dma = 0; + priv->tx_wqe_next = NULL; + writeq(0, priv->base + MLXBF_GIGE_TX_WQ_BASE); + writeq(0, priv->base + MLXBF_GIGE_TX_CI_UPDATE_ADDRESS); +} + +/* Function that returns status of TX ring: + * 0: TX ring is full, i.e. there are no + * available un-used entries in TX ring. + * non-null: TX ring is not full, i.e. there are + * some available entries in TX ring. + * The non-null value is a measure of + * how many TX entries are available, but + * it is not the exact number of available + * entries (see below). + * + * The algorithm makes the assumption that if + * (prev_tx_ci == tx_pi) then the TX ring is empty. + * An empty ring actually has (tx_q_entries-1) + * entries, which allows the algorithm to differentiate + * the case of an empty ring vs. a full ring. + */ +static u16 mlxbf_gige_tx_buffs_avail(struct mlxbf_gige *priv) +{ + unsigned long flags; + u16 avail; + + spin_lock_irqsave(&priv->lock, flags); + + if (priv->prev_tx_ci == priv->tx_pi) + avail = priv->tx_q_entries - 1; + else + avail = ((priv->tx_q_entries + priv->prev_tx_ci - priv->tx_pi) + % priv->tx_q_entries) - 1; + + spin_unlock_irqrestore(&priv->lock, flags); + + return avail; +} + +bool mlxbf_gige_handle_tx_complete(struct mlxbf_gige *priv) +{ + struct net_device_stats *stats; + u16 tx_wqe_index; + u64 *tx_wqe_addr; + u64 tx_status; + u16 tx_ci; + + tx_status = readq(priv->base + MLXBF_GIGE_TX_STATUS); + if (tx_status & MLXBF_GIGE_TX_STATUS_DATA_FIFO_FULL) + priv->stats.tx_fifo_full++; + tx_ci = readq(priv->base + MLXBF_GIGE_TX_CONSUMER_INDEX); + stats = &priv->netdev->stats; + + /* Transmit completion logic needs to loop until the completion + * index (in SW) equals TX consumer index (from HW). These + * parameters are unsigned 16-bit values and the wrap case needs + * to be supported, that is TX consumer index wrapped from 0xFFFF + * to 0 while TX completion index is still < 0xFFFF. + */ + for (; priv->prev_tx_ci != tx_ci; priv->prev_tx_ci++) { + tx_wqe_index = priv->prev_tx_ci % priv->tx_q_entries; + /* Each TX WQE is 16 bytes. The 8 MSB store the 2KB TX + * buffer address and the 8 LSB contain information + * about the TX WQE. + */ + tx_wqe_addr = priv->tx_wqe_base + + (tx_wqe_index * MLXBF_GIGE_TX_WQE_SZ_QWORDS); + + stats->tx_packets++; + stats->tx_bytes += MLXBF_GIGE_TX_WQE_PKT_LEN(tx_wqe_addr); + + dma_unmap_single(priv->dev, *tx_wqe_addr, + priv->tx_skb[tx_wqe_index]->len, DMA_TO_DEVICE); + dev_consume_skb_any(priv->tx_skb[tx_wqe_index]); + priv->tx_skb[tx_wqe_index] = NULL; + + /* Ensure completion of updates across all cores */ + mb(); + } + + /* Since the TX ring was likely just drained, check if TX queue + * had previously been stopped and now that there are TX buffers + * available the TX queue can be awakened. + */ + if (netif_queue_stopped(priv->netdev) && + mlxbf_gige_tx_buffs_avail(priv)) + netif_wake_queue(priv->netdev); + + return true; +} + +/* Function to advance the tx_wqe_next pointer to next TX WQE */ +void mlxbf_gige_update_tx_wqe_next(struct mlxbf_gige *priv) +{ + /* Advance tx_wqe_next pointer */ + priv->tx_wqe_next += MLXBF_GIGE_TX_WQE_SZ_QWORDS; + + /* Check if 'next' pointer is beyond end of TX ring */ + /* If so, set 'next' back to 'base' pointer of ring */ + if (priv->tx_wqe_next == (priv->tx_wqe_base + + (priv->tx_q_entries * MLXBF_GIGE_TX_WQE_SZ_QWORDS))) + priv->tx_wqe_next = priv->tx_wqe_base; +} + +netdev_tx_t mlxbf_gige_start_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + struct mlxbf_gige *priv = netdev_priv(netdev); + long buff_addr, start_dma_page, end_dma_page; + struct sk_buff *tx_skb; + dma_addr_t tx_buf_dma; + unsigned long flags; + u64 *tx_wqe_addr; + u64 word2; + + /* If needed, linearize TX SKB as hardware DMA expects this */ + if (skb->len > MLXBF_GIGE_DEFAULT_BUF_SZ || skb_linearize(skb)) { + dev_kfree_skb(skb); + netdev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + + buff_addr = (long)skb->data; + start_dma_page = buff_addr >> MLXBF_GIGE_DMA_PAGE_SHIFT; + end_dma_page = (buff_addr + skb->len - 1) >> MLXBF_GIGE_DMA_PAGE_SHIFT; + + /* Verify that payload pointer and data length of SKB to be + * transmitted does not violate the hardware DMA limitation. + */ + if (start_dma_page != end_dma_page) { + /* DMA operation would fail as-is, alloc new aligned SKB */ + tx_skb = mlxbf_gige_alloc_skb(priv, skb->len, + &tx_buf_dma, DMA_TO_DEVICE); + if (!tx_skb) { + /* Free original skb, could not alloc new aligned SKB */ + dev_kfree_skb(skb); + netdev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + + skb_put_data(tx_skb, skb->data, skb->len); + + /* Free the original SKB */ + dev_kfree_skb(skb); + } else { + tx_skb = skb; + tx_buf_dma = dma_map_single(priv->dev, skb->data, + skb->len, DMA_TO_DEVICE); + if (dma_mapping_error(priv->dev, tx_buf_dma)) { + dev_kfree_skb(skb); + netdev->stats.tx_dropped++; + return NETDEV_TX_OK; + } + } + + /* Get address of TX WQE */ + tx_wqe_addr = priv->tx_wqe_next; + + mlxbf_gige_update_tx_wqe_next(priv); + + /* Put PA of buffer address into first 64-bit word of TX WQE */ + *tx_wqe_addr = tx_buf_dma; + + /* Set TX WQE pkt_len appropriately + * NOTE: GigE silicon will automatically pad up to + * minimum packet length if needed. + */ + word2 = tx_skb->len & MLXBF_GIGE_TX_WQE_PKT_LEN_MASK; + + /* Write entire 2nd word of TX WQE */ + *(tx_wqe_addr + 1) = word2; + + spin_lock_irqsave(&priv->lock, flags); + priv->tx_skb[priv->tx_pi % priv->tx_q_entries] = tx_skb; + priv->tx_pi++; + spin_unlock_irqrestore(&priv->lock, flags); + + if (!netdev_xmit_more()) { + /* Create memory barrier before write to TX PI */ + wmb(); + writeq(priv->tx_pi, priv->base + MLXBF_GIGE_TX_PRODUCER_INDEX); + } + + /* Check if the last TX entry was just used */ + if (!mlxbf_gige_tx_buffs_avail(priv)) { + /* TX ring is full, inform stack */ + netif_stop_queue(netdev); + + /* Since there is no separate "TX complete" interrupt, need + * to explicitly schedule NAPI poll. This will trigger logic + * which processes TX completions, and will hopefully drain + * the TX ring allowing the TX queue to be awakened. + */ + napi_schedule(&priv->napi); + } + + return NETDEV_TX_OK; +} diff --git a/drivers/net/ethernet/mellanox/mlxsw/Kconfig b/drivers/net/ethernet/mellanox/mlxsw/Kconfig index a619d90559f7..12871c8dc7c1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Kconfig +++ b/drivers/net/ethernet/mellanox/mlxsw/Kconfig @@ -49,28 +49,6 @@ config MLXSW_I2C To compile this driver as a module, choose M here: the module will be called mlxsw_i2c. -config MLXSW_SWITCHIB - tristate "Mellanox Technologies SwitchIB and SwitchIB-2 support" - depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV - default m - help - This driver supports Mellanox Technologies SwitchIB and SwitchIB-2 - Infiniband Switch ASICs. - - To compile this driver as a module, choose M here: the - module will be called mlxsw_switchib. - -config MLXSW_SWITCHX2 - tristate "Mellanox Technologies SwitchX-2 support" - depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV - default m - help - This driver supports Mellanox Technologies SwitchX-2 Ethernet - Switch ASICs. - - To compile this driver as a module, choose M here: the - module will be called mlxsw_switchx2. - config MLXSW_SPECTRUM tristate "Mellanox Technologies Spectrum family support" depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV && VLAN_8021Q diff --git a/drivers/net/ethernet/mellanox/mlxsw/Makefile b/drivers/net/ethernet/mellanox/mlxsw/Makefile index f545fd2c5896..196adeb33495 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Makefile +++ b/drivers/net/ethernet/mellanox/mlxsw/Makefile @@ -8,10 +8,6 @@ obj-$(CONFIG_MLXSW_PCI) += mlxsw_pci.o mlxsw_pci-objs := pci.o obj-$(CONFIG_MLXSW_I2C) += mlxsw_i2c.o mlxsw_i2c-objs := i2c.o -obj-$(CONFIG_MLXSW_SWITCHIB) += mlxsw_switchib.o -mlxsw_switchib-objs := switchib.o -obj-$(CONFIG_MLXSW_SWITCHX2) += mlxsw_switchx2.o -mlxsw_switchx2-objs := switchx2.o obj-$(CONFIG_MLXSW_SPECTRUM) += mlxsw_spectrum.o mlxsw_spectrum-objs := spectrum.o spectrum_buffers.o \ spectrum_switchdev.o spectrum_router.o \ diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 7e9a7cb31720..e775f08fb464 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -630,7 +630,7 @@ static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb; int err; - skb = skb_copy(trans->tx_skb, GFP_KERNEL); + skb = skb_clone(trans->tx_skb, GFP_KERNEL); if (!skb) return -ENOMEM; @@ -1444,7 +1444,9 @@ mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, if (err) return err; - err = devlink_info_version_fixed_put(req, "fw.psid", fw_info_psid); + err = devlink_info_version_fixed_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW_PSID, + fw_info_psid); if (err) return err; @@ -1453,7 +1455,9 @@ mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, if (err) return err; - return 0; + return devlink_info_version_running_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW, + buf); } static int diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index dd26865bd587..3713c45cfa1e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -3,6 +3,7 @@ #include <linux/kernel.h> #include <linux/err.h> +#include <linux/ethtool.h> #include <linux/sfp.h> #include "core.h" @@ -25,8 +26,8 @@ struct mlxsw_env { static int mlxsw_env_validate_cable_ident(struct mlxsw_core *core, int id, bool *qsfp, bool *cmis) { - char eeprom_tmp[MLXSW_REG_MCIA_EEPROM_SIZE]; char mcia_pl[MLXSW_REG_MCIA_LEN]; + char *eeprom_tmp; u8 ident; int err; @@ -35,7 +36,7 @@ static int mlxsw_env_validate_cable_ident(struct mlxsw_core *core, int id, err = mlxsw_reg_query(core, MLXSW_REG(mcia), mcia_pl); if (err) return err; - mlxsw_reg_mcia_eeprom_memcpy_from(mcia_pl, eeprom_tmp); + eeprom_tmp = mlxsw_reg_mcia_eeprom_data(mcia_pl); ident = eeprom_tmp[0]; *cmis = false; switch (ident) { @@ -63,8 +64,8 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, int module, u16 offset, u16 size, void *data, bool qsfp, unsigned int *p_read_size) { - char eeprom_tmp[MLXSW_REG_MCIA_EEPROM_SIZE]; char mcia_pl[MLXSW_REG_MCIA_LEN]; + char *eeprom_tmp; u16 i2c_addr; u8 page = 0; int status; @@ -115,7 +116,7 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, int module, if (status) return -EIO; - mlxsw_reg_mcia_eeprom_memcpy_from(mcia_pl, eeprom_tmp); + eeprom_tmp = mlxsw_reg_mcia_eeprom_data(mcia_pl); memcpy(data, eeprom_tmp, size); *p_read_size = size; @@ -125,14 +126,14 @@ mlxsw_env_query_module_eeprom(struct mlxsw_core *mlxsw_core, int module, int mlxsw_env_module_temp_thresholds_get(struct mlxsw_core *core, int module, int off, int *temp) { - char eeprom_tmp[MLXSW_REG_MCIA_EEPROM_SIZE]; + unsigned int module_temp, module_crit, module_emerg; union { u8 buf[MLXSW_REG_MCIA_TH_ITEM_SIZE]; u16 temp; } temp_thresh; char mcia_pl[MLXSW_REG_MCIA_LEN] = {0}; char mtmp_pl[MLXSW_REG_MTMP_LEN]; - unsigned int module_temp; + char *eeprom_tmp; bool qsfp, cmis; int page; int err; @@ -142,12 +143,21 @@ int mlxsw_env_module_temp_thresholds_get(struct mlxsw_core *core, int module, err = mlxsw_reg_query(core, MLXSW_REG(mtmp), mtmp_pl); if (err) return err; - mlxsw_reg_mtmp_unpack(mtmp_pl, &module_temp, NULL, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, &module_temp, NULL, &module_crit, + &module_emerg, NULL); if (!module_temp) { *temp = 0; return 0; } + /* Validate if threshold reading is available through MTMP register, + * otherwise fallback to read through MCIA. + */ + if (module_emerg) { + *temp = off == SFP_TEMP_HIGH_WARN ? module_crit : module_emerg; + return 0; + } + /* Read Free Side Device Temperature Thresholds from page 03h * (MSB at lower byte address). * Bytes: @@ -185,7 +195,7 @@ int mlxsw_env_module_temp_thresholds_get(struct mlxsw_core *core, int module, if (err) return err; - mlxsw_reg_mcia_eeprom_memcpy_from(mcia_pl, eeprom_tmp); + eeprom_tmp = mlxsw_reg_mcia_eeprom_data(mcia_pl); memcpy(temp_thresh.buf, eeprom_tmp, MLXSW_REG_MCIA_TH_ITEM_SIZE); *temp = temp_thresh.temp * 1000; @@ -306,6 +316,79 @@ int mlxsw_env_get_module_eeprom(struct net_device *netdev, } EXPORT_SYMBOL(mlxsw_env_get_module_eeprom); +static int mlxsw_env_mcia_status_process(const char *mcia_pl, + struct netlink_ext_ack *extack) +{ + u8 status = mlxsw_reg_mcia_status_get(mcia_pl); + + switch (status) { + case MLXSW_REG_MCIA_STATUS_GOOD: + return 0; + case MLXSW_REG_MCIA_STATUS_NO_EEPROM_MODULE: + NL_SET_ERR_MSG_MOD(extack, "No response from module's EEPROM"); + return -EIO; + case MLXSW_REG_MCIA_STATUS_MODULE_NOT_SUPPORTED: + NL_SET_ERR_MSG_MOD(extack, "Module type not supported by the device"); + return -EOPNOTSUPP; + case MLXSW_REG_MCIA_STATUS_MODULE_NOT_CONNECTED: + NL_SET_ERR_MSG_MOD(extack, "No module present indication"); + return -EIO; + case MLXSW_REG_MCIA_STATUS_I2C_ERROR: + NL_SET_ERR_MSG_MOD(extack, "Error occurred while trying to access module's EEPROM using I2C"); + return -EIO; + case MLXSW_REG_MCIA_STATUS_MODULE_DISABLED: + NL_SET_ERR_MSG_MOD(extack, "Module is disabled"); + return -EIO; + default: + NL_SET_ERR_MSG_MOD(extack, "Unknown error"); + return -EIO; + } +} + +int +mlxsw_env_get_module_eeprom_by_page(struct mlxsw_core *mlxsw_core, u8 module, + const struct ethtool_module_eeprom *page, + struct netlink_ext_ack *extack) +{ + u32 bytes_read = 0; + u16 device_addr; + + /* Offset cannot be larger than 2 * ETH_MODULE_EEPROM_PAGE_LEN */ + device_addr = page->offset; + + while (bytes_read < page->length) { + char mcia_pl[MLXSW_REG_MCIA_LEN]; + char *eeprom_tmp; + u8 size; + int err; + + size = min_t(u8, page->length - bytes_read, + MLXSW_REG_MCIA_EEPROM_SIZE); + + mlxsw_reg_mcia_pack(mcia_pl, module, 0, page->page, + device_addr + bytes_read, size, + page->i2c_address); + mlxsw_reg_mcia_bank_number_set(mcia_pl, page->bank); + + err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mcia), mcia_pl); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to access module's EEPROM"); + return err; + } + + err = mlxsw_env_mcia_status_process(mcia_pl, extack); + if (err) + return err; + + eeprom_tmp = mlxsw_reg_mcia_eeprom_data(mcia_pl); + memcpy(page->data + bytes_read, eeprom_tmp, size); + bytes_read += size; + } + + return bytes_read; +} +EXPORT_SYMBOL(mlxsw_env_get_module_eeprom_by_page); + static int mlxsw_env_module_has_temp_sensor(struct mlxsw_core *mlxsw_core, u8 module, bool *p_has_temp_sensor) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.h b/drivers/net/ethernet/mellanox/mlxsw/core_env.h index 2b23f8a87862..0bf5bd0f8a7e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.h @@ -4,6 +4,8 @@ #ifndef _MLXSW_CORE_ENV_H #define _MLXSW_CORE_ENV_H +#include <linux/ethtool.h> + struct ethtool_modinfo; struct ethtool_eeprom; @@ -18,6 +20,11 @@ int mlxsw_env_get_module_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee, u8 *data); int +mlxsw_env_get_module_eeprom_by_page(struct mlxsw_core *mlxsw_core, u8 module, + const struct ethtool_module_eeprom *page, + struct netlink_ext_ack *extack); + +int mlxsw_env_module_overheat_counter_get(struct mlxsw_core *mlxsw_core, u8 module, u64 *p_counter); int mlxsw_env_init(struct mlxsw_core *core, struct mlxsw_env **p_env); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c b/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c index 2196c946698a..d41afdfbd085 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_hwmon.c @@ -72,7 +72,7 @@ static ssize_t mlxsw_hwmon_temp_show(struct device *dev, dev_err(mlxsw_hwmon->bus_info->dev, "Failed to query temp sensor\n"); return err; } - mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL); return sprintf(buf, "%d\n", temp); } @@ -95,7 +95,7 @@ static ssize_t mlxsw_hwmon_temp_max_show(struct device *dev, dev_err(mlxsw_hwmon->bus_info->dev, "Failed to query temp sensor\n"); return err; } - mlxsw_reg_mtmp_unpack(mtmp_pl, NULL, &temp_max, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, NULL, &temp_max, NULL, NULL, NULL); return sprintf(buf, "%d\n", temp_max); } @@ -239,7 +239,7 @@ static int mlxsw_hwmon_module_temp_get(struct device *dev, dev_err(dev, "Failed to query module temperature\n"); return err; } - mlxsw_reg_mtmp_unpack(mtmp_pl, p_temp, NULL, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, p_temp, NULL, NULL, NULL, NULL); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index dfea14399607..0998dcc9cac0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -149,22 +149,27 @@ mlxsw_thermal_module_trips_reset(struct mlxsw_thermal_module *tz) static int mlxsw_thermal_module_trips_update(struct device *dev, struct mlxsw_core *core, - struct mlxsw_thermal_module *tz) + struct mlxsw_thermal_module *tz, + int crit_temp, int emerg_temp) { - int crit_temp, emerg_temp; int err; - err = mlxsw_env_module_temp_thresholds_get(core, tz->module, - SFP_TEMP_HIGH_WARN, - &crit_temp); - if (err) - return err; + /* Do not try to query temperature thresholds directly from the module's + * EEPROM if we got valid thresholds from MTMP. + */ + if (!emerg_temp || !crit_temp) { + err = mlxsw_env_module_temp_thresholds_get(core, tz->module, + SFP_TEMP_HIGH_WARN, + &crit_temp); + if (err) + return err; - err = mlxsw_env_module_temp_thresholds_get(core, tz->module, - SFP_TEMP_HIGH_ALARM, - &emerg_temp); - if (err) - return err; + err = mlxsw_env_module_temp_thresholds_get(core, tz->module, + SFP_TEMP_HIGH_ALARM, + &emerg_temp); + if (err) + return err; + } if (crit_temp > emerg_temp) { dev_warn(dev, "%s : Critical threshold %d is above emergency threshold %d\n", @@ -281,7 +286,7 @@ static int mlxsw_thermal_get_temp(struct thermal_zone_device *tzdev, dev_err(dev, "Failed to query temp sensor\n"); return err; } - mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL); if (temp > 0) mlxsw_thermal_tz_score_update(thermal, tzdev, thermal->trips, temp); @@ -420,36 +425,57 @@ static int mlxsw_thermal_module_unbind(struct thermal_zone_device *tzdev, return err; } -static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev, - int *p_temp) +static void +mlxsw_thermal_module_temp_and_thresholds_get(struct mlxsw_core *core, + u16 sensor_index, int *p_temp, + int *p_crit_temp, + int *p_emerg_temp) { - struct mlxsw_thermal_module *tz = tzdev->devdata; - struct mlxsw_thermal *thermal = tz->parent; - struct device *dev = thermal->bus_info->dev; char mtmp_pl[MLXSW_REG_MTMP_LEN]; - int temp; int err; - /* Read module temperature. */ - mlxsw_reg_mtmp_pack(mtmp_pl, MLXSW_REG_MTMP_MODULE_INDEX_MIN + - tz->module, false, false); - err = mlxsw_reg_query(thermal->core, MLXSW_REG(mtmp), mtmp_pl); + /* Read module temperature and thresholds. */ + mlxsw_reg_mtmp_pack(mtmp_pl, sensor_index, false, false); + err = mlxsw_reg_query(core, MLXSW_REG(mtmp), mtmp_pl); if (err) { - /* Do not return error - in case of broken module's sensor - * it will cause error message flooding. + /* Set temperature and thresholds to zero to avoid passing + * uninitialized data back to the caller. */ - temp = 0; - *p_temp = (int) temp; - return 0; + *p_temp = 0; + *p_crit_temp = 0; + *p_emerg_temp = 0; + + return; } - mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, p_temp, NULL, p_crit_temp, p_emerg_temp, + NULL); +} + +static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev, + int *p_temp) +{ + struct mlxsw_thermal_module *tz = tzdev->devdata; + struct mlxsw_thermal *thermal = tz->parent; + int temp, crit_temp, emerg_temp; + struct device *dev; + u16 sensor_index; + int err; + + dev = thermal->bus_info->dev; + sensor_index = MLXSW_REG_MTMP_MODULE_INDEX_MIN + tz->module; + + /* Read module temperature and thresholds. */ + mlxsw_thermal_module_temp_and_thresholds_get(thermal->core, + sensor_index, &temp, + &crit_temp, &emerg_temp); *p_temp = temp; if (!temp) return 0; /* Update trip points. */ - err = mlxsw_thermal_module_trips_update(dev, thermal->core, tz); + err = mlxsw_thermal_module_trips_update(dev, thermal->core, tz, + crit_temp, emerg_temp); if (!err && temp > 0) mlxsw_thermal_tz_score_update(thermal, tzdev, tz->trips, temp); @@ -560,7 +586,7 @@ static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev, if (err) return err; - mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL); + mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL); if (temp > 0) mlxsw_thermal_tz_score_update(thermal, tzdev, tz->trips, temp); @@ -693,7 +719,8 @@ mlxsw_thermal_module_tz_init(struct mlxsw_thermal_module *module_tz) MLXSW_THERMAL_TRIP_MASK, module_tz, &mlxsw_thermal_module_ops, - NULL, 0, 0); + NULL, 0, + module_tz->parent->polling_delay); if (IS_ERR(module_tz->tzdev)) { err = PTR_ERR(module_tz->tzdev); return err; @@ -716,7 +743,10 @@ mlxsw_thermal_module_init(struct device *dev, struct mlxsw_core *core, struct mlxsw_thermal *thermal, u8 module) { struct mlxsw_thermal_module *module_tz; + int dummy_temp, crit_temp, emerg_temp; + u16 sensor_index; + sensor_index = MLXSW_REG_MTMP_MODULE_INDEX_MIN + module; module_tz = &thermal->tz_module_arr[module]; /* Skip if parent is already set (case of port split). */ if (module_tz->parent) @@ -727,8 +757,12 @@ mlxsw_thermal_module_init(struct device *dev, struct mlxsw_core *core, sizeof(thermal->trips)); /* Initialize all trip point. */ mlxsw_thermal_module_trips_reset(module_tz); + /* Read module temperature and thresholds. */ + mlxsw_thermal_module_temp_and_thresholds_get(core, sensor_index, &dummy_temp, + &crit_temp, &emerg_temp); /* Update trip point according to the module data. */ - return mlxsw_thermal_module_trips_update(dev, core, module_tz); + return mlxsw_thermal_module_trips_update(dev, core, module_tz, + crit_temp, emerg_temp); } static void mlxsw_thermal_module_fini(struct mlxsw_thermal_module *module_tz) @@ -815,7 +849,8 @@ mlxsw_thermal_gearbox_tz_init(struct mlxsw_thermal_module *gearbox_tz) MLXSW_THERMAL_TRIP_MASK, gearbox_tz, &mlxsw_thermal_gearbox_ops, - NULL, 0, 0); + NULL, 0, + gearbox_tz->parent->polling_delay); if (IS_ERR(gearbox_tz->tzdev)) return PTR_ERR(gearbox_tz->tzdev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/ib.h b/drivers/net/ethernet/mellanox/mlxsw/ib.h deleted file mode 100644 index 2d0cb0f5eb85..000000000000 --- a/drivers/net/ethernet/mellanox/mlxsw/ib.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ -/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ - -#ifndef _MLXSW_IB_H -#define _MLXSW_IB_H - -#define MLXSW_IB_DEFAULT_MTU 4096 - -#endif /* _MLXSW_IB_H */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index b34c44723f8b..d9d56c44e994 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -112,10 +112,23 @@ mlxsw_m_get_module_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee, ee, data); } +static int +mlxsw_m_get_module_eeprom_by_page(struct net_device *netdev, + const struct ethtool_module_eeprom *page, + struct netlink_ext_ack *extack) +{ + struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); + struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; + + return mlxsw_env_get_module_eeprom_by_page(core, mlxsw_m_port->module, + page, extack); +} + static const struct ethtool_ops mlxsw_m_port_ethtool_ops = { .get_drvinfo = mlxsw_m_module_get_drvinfo, .get_module_info = mlxsw_m_get_module_info, .get_module_eeprom = mlxsw_m_get_module_eeprom, + .get_module_eeprom_by_page = mlxsw_m_get_module_eeprom_by_page, }; static int @@ -234,6 +247,7 @@ static void mlxsw_m_port_remove(struct mlxsw_m *mlxsw_m, u8 local_port) static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u8 local_port, u8 *last_module) { + unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core); u8 module, width; int err; @@ -249,6 +263,9 @@ static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u8 local_port, if (module == *last_module) return 0; *last_module = module; + + if (WARN_ON_ONCE(module >= max_ports)) + return -EINVAL; mlxsw_m->module_to_port[module] = ++mlxsw_m->max_ports; return 0; diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 8e8456811384..13b0259f7ea6 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1426,11 +1426,6 @@ static int mlxsw_pci_sys_ready_wait(struct mlxsw_pci *mlxsw_pci, unsigned long end; u32 val; - if (id->device == PCI_DEVICE_ID_MELLANOX_SWITCHX2) { - msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); - return 0; - } - /* We must wait for the HW to become responsive. */ msleep(MLXSW_PCI_SW_RESET_WAIT_MSECS); diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.h b/drivers/net/ethernet/mellanox/mlxsw/pci.h index 5b1323645a5d..9899c1a2ea8f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.h @@ -6,12 +6,9 @@ #include <linux/pci.h> -#define PCI_DEVICE_ID_MELLANOX_SWITCHX2 0xc738 #define PCI_DEVICE_ID_MELLANOX_SPECTRUM 0xcb84 #define PCI_DEVICE_ID_MELLANOX_SPECTRUM2 0xcf6c #define PCI_DEVICE_ID_MELLANOX_SPECTRUM3 0xcf70 -#define PCI_DEVICE_ID_MELLANOX_SWITCHIB 0xcb20 -#define PCI_DEVICE_ID_MELLANOX_SWITCHIB2 0xcf08 #if IS_ENABLED(CONFIG_MLXSW_PCI) diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 900b4bf5bb5b..6fbda6ebd590 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -3907,7 +3907,7 @@ MLXSW_ITEM32(reg, qeec, max_shaper_bs, 0x1C, 0, 6); #define MLXSW_REG_QEEC_HIGHEST_SHAPER_BS 25 #define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP1 5 #define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP2 11 -#define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP3 5 +#define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP3 11 static inline void mlxsw_reg_qeec_pack(char *payload, u8 local_port, enum mlxsw_reg_qeec_hr hr, u8 index, @@ -8305,6 +8305,8 @@ enum { MLXSW_REG_RECR2_TCP_UDP_EN_IPV4 = 7, /* Enable TCP/UDP header fields if packet is IPv6 */ MLXSW_REG_RECR2_TCP_UDP_EN_IPV6 = 8, + + __MLXSW_REG_RECR2_HEADER_CNT, }; /* reg_recr2_outer_header_enables @@ -8339,6 +8341,8 @@ enum { MLXSW_REG_RECR2_TCP_UDP_SPORT = 74, /* TCP/UDP Destination Port */ MLXSW_REG_RECR2_TCP_UDP_DPORT = 75, + + __MLXSW_REG_RECR2_FIELD_CNT, }; /* reg_recr2_outer_header_fields_enable @@ -8347,47 +8351,47 @@ enum { */ MLXSW_ITEM_BIT_ARRAY(reg, recr2, outer_header_fields_enable, 0x14, 0x14, 1); -static inline void mlxsw_reg_recr2_ipv4_sip_enable(char *payload) -{ - int i; - - for (i = MLXSW_REG_RECR2_IPV4_SIP0; i <= MLXSW_REG_RECR2_IPV4_SIP3; i++) - mlxsw_reg_recr2_outer_header_fields_enable_set(payload, i, - true); -} - -static inline void mlxsw_reg_recr2_ipv4_dip_enable(char *payload) -{ - int i; - - for (i = MLXSW_REG_RECR2_IPV4_DIP0; i <= MLXSW_REG_RECR2_IPV4_DIP3; i++) - mlxsw_reg_recr2_outer_header_fields_enable_set(payload, i, - true); -} - -static inline void mlxsw_reg_recr2_ipv6_sip_enable(char *payload) -{ - int i = MLXSW_REG_RECR2_IPV6_SIP0_7; - - mlxsw_reg_recr2_outer_header_fields_enable_set(payload, i, true); - - i = MLXSW_REG_RECR2_IPV6_SIP8; - for (; i <= MLXSW_REG_RECR2_IPV6_SIP15; i++) - mlxsw_reg_recr2_outer_header_fields_enable_set(payload, i, - true); -} - -static inline void mlxsw_reg_recr2_ipv6_dip_enable(char *payload) -{ - int i = MLXSW_REG_RECR2_IPV6_DIP0_7; - - mlxsw_reg_recr2_outer_header_fields_enable_set(payload, i, true); +/* reg_recr2_inner_header_enables + * Bit mask where each bit enables a specific inner layer to be included in the + * hash calculation. Same values as reg_recr2_outer_header_enables. + * Access: RW + */ +MLXSW_ITEM_BIT_ARRAY(reg, recr2, inner_header_enables, 0x2C, 0x04, 1); - i = MLXSW_REG_RECR2_IPV6_DIP8; - for (; i <= MLXSW_REG_RECR2_IPV6_DIP15; i++) - mlxsw_reg_recr2_outer_header_fields_enable_set(payload, i, - true); -} +enum { + /* Inner IPv4 Source IP */ + MLXSW_REG_RECR2_INNER_IPV4_SIP0 = 3, + MLXSW_REG_RECR2_INNER_IPV4_SIP3 = 6, + /* Inner IPv4 Destination IP */ + MLXSW_REG_RECR2_INNER_IPV4_DIP0 = 7, + MLXSW_REG_RECR2_INNER_IPV4_DIP3 = 10, + /* Inner IP Protocol */ + MLXSW_REG_RECR2_INNER_IPV4_PROTOCOL = 11, + /* Inner IPv6 Source IP */ + MLXSW_REG_RECR2_INNER_IPV6_SIP0_7 = 12, + MLXSW_REG_RECR2_INNER_IPV6_SIP8 = 20, + MLXSW_REG_RECR2_INNER_IPV6_SIP15 = 27, + /* Inner IPv6 Destination IP */ + MLXSW_REG_RECR2_INNER_IPV6_DIP0_7 = 28, + MLXSW_REG_RECR2_INNER_IPV6_DIP8 = 36, + MLXSW_REG_RECR2_INNER_IPV6_DIP15 = 43, + /* Inner IPv6 Next Header */ + MLXSW_REG_RECR2_INNER_IPV6_NEXT_HEADER = 44, + /* Inner IPv6 Flow Label */ + MLXSW_REG_RECR2_INNER_IPV6_FLOW_LABEL = 45, + /* Inner TCP/UDP Source Port */ + MLXSW_REG_RECR2_INNER_TCP_UDP_SPORT = 46, + /* Inner TCP/UDP Destination Port */ + MLXSW_REG_RECR2_INNER_TCP_UDP_DPORT = 47, + + __MLXSW_REG_RECR2_INNER_FIELD_CNT, +}; + +/* reg_recr2_inner_header_fields_enable + * Inner packet fields to enable for ECMP hash subject to inner_header_enables. + * Access: RW + */ +MLXSW_ITEM_BIT_ARRAY(reg, recr2, inner_header_fields_enable, 0x30, 0x08, 1); static inline void mlxsw_reg_recr2_pack(char *payload, u32 seed) { @@ -9459,6 +9463,14 @@ MLXSW_ITEM32(reg, mtmp, sensor_index, 0x00, 0, 12); ((s16)((GENMASK(15, 0) + (v_) + 1) \ * 125)); }) +/* reg_mtmp_max_operational_temperature + * The highest temperature in the nominal operational range. Reading is in + * 0.125 Celsius degrees units. + * In case of module this is SFF critical temperature threshold. + * Access: RO + */ +MLXSW_ITEM32(reg, mtmp, max_operational_temperature, 0x04, 16, 16); + /* reg_mtmp_temperature * Temperature reading from the sensor. Reading is in 0.125 Celsius * degrees units. @@ -9537,7 +9549,9 @@ static inline void mlxsw_reg_mtmp_pack(char *payload, u16 sensor_index, } static inline void mlxsw_reg_mtmp_unpack(char *payload, int *p_temp, - int *p_max_temp, char *sensor_name) + int *p_max_temp, int *p_temp_hi, + int *p_max_oper_temp, + char *sensor_name) { s16 temp; @@ -9549,6 +9563,14 @@ static inline void mlxsw_reg_mtmp_unpack(char *payload, int *p_temp, temp = mlxsw_reg_mtmp_max_temperature_get(payload); *p_max_temp = MLXSW_REG_MTMP_TEMP_TO_MC(temp); } + if (p_temp_hi) { + temp = mlxsw_reg_mtmp_temperature_threshold_hi_get(payload); + *p_temp_hi = MLXSW_REG_MTMP_TEMP_TO_MC(temp); + } + if (p_max_oper_temp) { + temp = mlxsw_reg_mtmp_max_operational_temperature_get(payload); + *p_max_oper_temp = MLXSW_REG_MTMP_TEMP_TO_MC(temp); + } if (sensor_name) mlxsw_reg_mtmp_sensor_name_memcpy_from(payload, sensor_name); } @@ -9668,6 +9690,20 @@ MLXSW_ITEM32(reg, mcia, l, 0x00, 31, 1); */ MLXSW_ITEM32(reg, mcia, module, 0x00, 16, 8); +enum { + MLXSW_REG_MCIA_STATUS_GOOD = 0, + /* No response from module's EEPROM. */ + MLXSW_REG_MCIA_STATUS_NO_EEPROM_MODULE = 1, + /* Module type not supported by the device. */ + MLXSW_REG_MCIA_STATUS_MODULE_NOT_SUPPORTED = 2, + /* No module present indication. */ + MLXSW_REG_MCIA_STATUS_MODULE_NOT_CONNECTED = 3, + /* Error occurred while trying to access module's EEPROM using I2C. */ + MLXSW_REG_MCIA_STATUS_I2C_ERROR = 9, + /* Module is disabled. */ + MLXSW_REG_MCIA_STATUS_MODULE_DISABLED = 16, +}; + /* reg_mcia_status * Module status. * Access: RO @@ -9692,6 +9728,12 @@ MLXSW_ITEM32(reg, mcia, page_number, 0x04, 16, 8); */ MLXSW_ITEM32(reg, mcia, device_address, 0x04, 0, 16); +/* reg_mcia_bank_number + * Bank number. + * Access: Index + */ +MLXSW_ITEM32(reg, mcia, bank_number, 0x08, 16, 8); + /* reg_mcia_size * Number of bytes to read/write (up to 48 bytes). * Access: RW diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index bca0354482cb..88699e678544 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -2125,9 +2125,14 @@ static void mlxsw_sp_pude_event_func(const struct mlxsw_reg_info *reg, struct mlxsw_sp *mlxsw_sp = priv; struct mlxsw_sp_port *mlxsw_sp_port; enum mlxsw_reg_pude_oper_status status; + unsigned int max_ports; u8 local_port; + max_ports = mlxsw_core_max_ports(mlxsw_sp->core); local_port = mlxsw_reg_pude_local_port_get(pude_pl); + + if (WARN_ON_ONCE(local_port >= max_ports)) + return; mlxsw_sp_port = mlxsw_sp->ports[local_port]; if (!mlxsw_sp_port) return; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c index 37ff29a1686e..9de160e740b2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c @@ -364,7 +364,7 @@ static u16 mlxsw_sp_hdroom_buf_delay_get(const struct mlxsw_sp *mlxsw_sp, static u32 mlxsw_sp_hdroom_int_buf_size_get(struct mlxsw_sp *mlxsw_sp, int mtu, u32 speed) { - u32 buffsize = mlxsw_sp->sb_ops->int_buf_size_get(speed, mtu); + u32 buffsize = mlxsw_sp->sb_ops->int_buf_size_get(mtu, speed); return mlxsw_sp_bytes_cells(mlxsw_sp, buffsize) + 1; } @@ -388,8 +388,8 @@ void mlxsw_sp_hdroom_bufs_reset_sizes(struct mlxsw_sp_port *mlxsw_sp_port, int i; /* Internal buffer. */ - reserve_cells = mlxsw_sp_hdroom_int_buf_size_get(mlxsw_sp, mlxsw_sp_port->max_speed, - mlxsw_sp_port->max_mtu); + reserve_cells = mlxsw_sp_hdroom_int_buf_size_get(mlxsw_sp, mlxsw_sp_port->max_mtu, + mlxsw_sp_port->max_speed); reserve_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, reserve_cells); hdroom->int_buf.reserve_cells = reserve_cells; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c index c8061beed6db..267590a0eee7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c @@ -1051,6 +1051,19 @@ static int mlxsw_sp_get_module_eeprom(struct net_device *netdev, } static int +mlxsw_sp_get_module_eeprom_by_page(struct net_device *dev, + const struct ethtool_module_eeprom *page, + struct netlink_ext_ack *extack) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + u8 module = mlxsw_sp_port->mapping.module; + + return mlxsw_env_get_module_eeprom_by_page(mlxsw_sp->core, module, page, + extack); +} + +static int mlxsw_sp_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *info) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(netdev); @@ -1199,6 +1212,7 @@ const struct ethtool_ops mlxsw_sp_port_ethtool_ops = { .set_link_ksettings = mlxsw_sp_port_set_link_ksettings, .get_module_info = mlxsw_sp_get_module_info, .get_module_eeprom = mlxsw_sp_get_module_eeprom, + .get_module_eeprom_by_page = mlxsw_sp_get_module_eeprom_by_page, .get_ts_info = mlxsw_sp_get_ts_info, .get_eth_phy_stats = mlxsw_sp_get_eth_phy_stats, .get_eth_mac_stats = mlxsw_sp_get_eth_mac_stats, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c index d6e9ecb14681..bfef65d1587c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c @@ -568,10 +568,13 @@ void mlxsw_sp1_ptp_got_timestamp(struct mlxsw_sp *mlxsw_sp, bool ingress, u8 domain_number, u16 sequence_id, u64 timestamp) { + unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core); struct mlxsw_sp_port *mlxsw_sp_port; struct mlxsw_sp1_ptp_key key; u8 types; + if (WARN_ON_ONCE(local_port >= max_ports)) + return; mlxsw_sp_port = mlxsw_sp->ports[local_port]; if (!mlxsw_sp_port) return; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c index 04672eb5c7f3..9958d503bf0e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c @@ -1332,6 +1332,7 @@ __mlxsw_sp_qdisc_ets_graft(struct mlxsw_sp_port *mlxsw_sp_port, u8 band, u32 child_handle) { struct mlxsw_sp_qdisc *old_qdisc; + u32 parent; if (band < mlxsw_sp_qdisc->num_classes && mlxsw_sp_qdisc->qdiscs[band].handle == child_handle) @@ -1352,7 +1353,9 @@ __mlxsw_sp_qdisc_ets_graft(struct mlxsw_sp_port *mlxsw_sp_port, if (old_qdisc) mlxsw_sp_qdisc_destroy(mlxsw_sp_port, old_qdisc); - mlxsw_sp_qdisc = mlxsw_sp_qdisc->ops->find_class(mlxsw_sp_qdisc, band); + parent = TC_H_MAKE(mlxsw_sp_qdisc->handle, band + 1); + mlxsw_sp_qdisc = mlxsw_sp_qdisc->ops->find_class(mlxsw_sp_qdisc, + parent); if (!WARN_ON(!mlxsw_sp_qdisc)) mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 41259c0004d1..7e221ef01437 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -2282,6 +2282,7 @@ static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, char *rauhtd_pl, int ent_index) { + u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); struct net_device *dev; struct neighbour *n; __be32 dipn; @@ -2290,6 +2291,8 @@ static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); + if (WARN_ON_ONCE(rif >= max_rifs)) + return; if (!mlxsw_sp->router->rifs[rif]) { dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); return; @@ -3841,8 +3844,8 @@ mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, bool offload_change = false; u32 adj_index; bool old_adj_index_valid; - int i, err2, err = 0; u32 old_adj_index; + int i, err2, err; if (!nhgi->gateway) return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); @@ -3872,11 +3875,13 @@ mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, return 0; } mlxsw_sp_nexthop_group_normalize(nhgi); - if (!nhgi->sum_norm_weight) + if (!nhgi->sum_norm_weight) { /* No neigh of this group is connected so we just set * the trap and let everthing flow through kernel. */ + err = 0; goto set_trap; + } ecmp_size = nhgi->sum_norm_weight; err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); @@ -4307,9 +4312,6 @@ static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_nexthop_key key; struct mlxsw_sp_nexthop *nh; - if (mlxsw_sp->router->aborted) - return; - key.fib_nh = fib_nh; nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); if (!nh) @@ -5405,7 +5407,6 @@ mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, &rt->fib6_nh->fib_nh_gw6)) return nh; - continue; } return NULL; @@ -6417,9 +6418,6 @@ mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib_node *fib_node; int err; - if (mlxsw_sp->router->aborted) - return 0; - if (fen_info->fi->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) return 0; @@ -6480,9 +6478,6 @@ static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib_node *fib_node; int err; - if (mlxsw_sp->router->aborted) - return 0; - fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); if (!fib4_entry) return 0; @@ -7065,9 +7060,6 @@ static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, struct fib6_info *rt = rt_arr[0]; int err; - if (mlxsw_sp->router->aborted) - return 0; - if (rt->fib6_src.plen) return -EINVAL; @@ -7131,9 +7123,6 @@ static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, struct fib6_info *rt = rt_arr[0]; int err; - if (mlxsw_sp->router->aborted) - return 0; - if (rt->fib6_src.plen) return -EINVAL; @@ -7175,9 +7164,6 @@ static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, struct fib6_info *rt = rt_arr[0]; int err; - if (mlxsw_sp->router->aborted) - return 0; - if (mlxsw_sp_fib6_rt_should_ignore(rt)) return 0; @@ -7206,55 +7192,6 @@ static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, return err; } -static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, - enum mlxsw_sp_l3proto proto, - u8 tree_id) -{ - const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; - enum mlxsw_reg_ralxx_protocol ralxx_proto = - (enum mlxsw_reg_ralxx_protocol) proto; - struct mlxsw_sp_fib_entry_priv *priv; - char xralta_pl[MLXSW_REG_XRALTA_LEN]; - char xralst_pl[MLXSW_REG_XRALST_LEN]; - int i, err; - - mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id); - err = ll_ops->ralta_write(mlxsw_sp, xralta_pl); - if (err) - return err; - - mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id); - err = ll_ops->ralst_write(mlxsw_sp, xralst_pl); - if (err) - return err; - - for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { - struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; - struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; - char xraltb_pl[MLXSW_REG_XRALTB_LEN]; - - mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); - mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id); - err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl); - if (err) - return err; - - priv = mlxsw_sp_fib_entry_priv_create(ll_ops); - if (IS_ERR(priv)) - return PTR_ERR(priv); - - ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE, - vr->id, 0, NULL, priv); - ll_ops->fib_entry_act_ip2me_pack(op_ctx); - err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL); - mlxsw_sp_fib_entry_priv_put(priv); - if (err) - return err; - } - - return 0; -} - static struct mlxsw_sp_mr_table * mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) { @@ -7271,9 +7208,6 @@ static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_mr_table *mrt; struct mlxsw_sp_vr *vr; - if (mlxsw_sp->router->aborted) - return 0; - vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); if (IS_ERR(vr)) return PTR_ERR(vr); @@ -7288,9 +7222,6 @@ static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_mr_table *mrt; struct mlxsw_sp_vr *vr; - if (mlxsw_sp->router->aborted) - return; - vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); if (WARN_ON(!vr)) return; @@ -7308,9 +7239,6 @@ mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_rif *rif; struct mlxsw_sp_vr *vr; - if (mlxsw_sp->router->aborted) - return 0; - vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); if (IS_ERR(vr)) return PTR_ERR(vr); @@ -7329,9 +7257,6 @@ mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_mr_table *mrt; struct mlxsw_sp_vr *vr; - if (mlxsw_sp->router->aborted) - return; - vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); if (WARN_ON(!vr)) return; @@ -7341,25 +7266,6 @@ mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, mlxsw_sp_vr_put(mlxsw_sp, vr); } -static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) -{ - enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4; - int err; - - err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, - MLXSW_SP_LPM_TREE_MIN); - if (err) - return err; - - /* The multicast router code does not need an abort trap as by default, - * packets that don't match any routes are trapped to the CPU. - */ - - proto = MLXSW_SP_L3_PROTO_IPV6; - return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, - MLXSW_SP_LPM_TREE_MIN + 1); -} - static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib_node *fib_node) { @@ -7446,20 +7352,6 @@ static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) mlxsw_sp->router->adj_discard_index_valid = false; } -static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) -{ - int err; - - if (mlxsw_sp->router->aborted) - return; - dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); - mlxsw_sp_router_fib_flush(mlxsw_sp); - mlxsw_sp->router->aborted = true; - err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); - if (err) - dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); -} - struct mlxsw_sp_fib6_event { struct fib6_info **rt_arr; unsigned int nrt6; @@ -7541,7 +7433,7 @@ static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); if (err) { mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); - mlxsw_sp_router_fib_abort(mlxsw_sp); + dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, &fib_event->fen_info); } @@ -7576,7 +7468,7 @@ static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, fib_event->fib6_event.nrt6); if (err) { mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); - mlxsw_sp_router_fib_abort(mlxsw_sp); + dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, fib6_event->rt_arr, fib6_event->nrt6); @@ -7588,7 +7480,7 @@ static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, fib_event->fib6_event.nrt6); if (err) { mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); - mlxsw_sp_router_fib_abort(mlxsw_sp); + dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, fib6_event->rt_arr, fib6_event->nrt6); @@ -7620,7 +7512,7 @@ static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); if (err) - mlxsw_sp_router_fib_abort(mlxsw_sp); + dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); mr_cache_put(fib_event->men_info.mfc); break; case FIB_EVENT_ENTRY_DEL: @@ -7631,7 +7523,7 @@ static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, &fib_event->ven_info); if (err) - mlxsw_sp_router_fib_abort(mlxsw_sp); + dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); dev_put(fib_event->ven_info.dev); break; case FIB_EVENT_VIF_DEL: @@ -7795,9 +7687,6 @@ static int mlxsw_sp_router_fib_rule_event(unsigned long event, if (event == FIB_EVENT_RULE_DEL) return 0; - if (mlxsw_sp->router->aborted) - return 0; - fr_info = container_of(info, struct fib_rule_notifier_info, info); rule = fr_info->rule; @@ -7855,10 +7744,6 @@ static int mlxsw_sp_router_fib_event(struct notifier_block *nb, case FIB_EVENT_ENTRY_ADD: case FIB_EVENT_ENTRY_REPLACE: case FIB_EVENT_ENTRY_APPEND: - if (router->aborted) { - NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); - return notifier_from_errno(-EINVAL); - } if (info->family == AF_INET) { struct fib_entry_notifier_info *fen_info = ptr; @@ -9594,66 +9479,229 @@ static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) } #ifdef CONFIG_IP_ROUTE_MULTIPATH -static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header) +struct mlxsw_sp_mp_hash_config { + DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); + DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); + DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); + DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); +}; + +#define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ + bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) + +#define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ + bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) + +#define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ + bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) + +static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) { - mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true); + unsigned long *inner_headers = config->inner_headers; + unsigned long *inner_fields = config->inner_fields; + + /* IPv4 inner */ + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); + /* IPv6 inner */ + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); } -static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field) +static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) { - mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true); + unsigned long *headers = config->headers; + unsigned long *fields = config->fields; + + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); } -static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) +static void +mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, + u32 hash_fields) +{ + unsigned long *inner_headers = config->inner_headers; + unsigned long *inner_fields = config->inner_fields; + + /* IPv4 Inner */ + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); + /* IPv6 inner */ + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); + } + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); + } + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); + /* L4 inner */ + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); + MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) + MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); +} + +static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mp_hash_config *config) { struct net *net = mlxsw_sp_net(mlxsw_sp); - bool only_l3 = !net->ipv4.sysctl_fib_multipath_hash_policy; - - mlxsw_sp_mp_hash_header_set(recr2_pl, - MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP); - mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP); - mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl); - mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl); - if (only_l3) - return; - mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4); - mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL); - mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT); - mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT); + unsigned long *headers = config->headers; + unsigned long *fields = config->fields; + u32 hash_fields; + + switch (net->ipv4.sysctl_fib_multipath_hash_policy) { + case 0: + mlxsw_sp_mp4_hash_outer_addr(config); + break; + case 1: + mlxsw_sp_mp4_hash_outer_addr(config); + MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); + break; + case 2: + /* Outer */ + mlxsw_sp_mp4_hash_outer_addr(config); + /* Inner */ + mlxsw_sp_mp_hash_inner_l3(config); + break; + case 3: + hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; + /* Outer */ + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); + /* Inner */ + mlxsw_sp_mp_hash_inner_custom(config, hash_fields); + break; + } } -static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, char *recr2_pl) +static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) { - bool only_l3 = !ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp)); + unsigned long *headers = config->headers; + unsigned long *fields = config->fields; - mlxsw_sp_mp_hash_header_set(recr2_pl, - MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP); - mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP); - mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl); - mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl); - mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER); - if (only_l3) { - mlxsw_sp_mp_hash_field_set(recr2_pl, - MLXSW_REG_RECR2_IPV6_FLOW_LABEL); - } else { - mlxsw_sp_mp_hash_header_set(recr2_pl, - MLXSW_REG_RECR2_TCP_UDP_EN_IPV6); - mlxsw_sp_mp_hash_field_set(recr2_pl, - MLXSW_REG_RECR2_TCP_UDP_SPORT); - mlxsw_sp_mp_hash_field_set(recr2_pl, - MLXSW_REG_RECR2_TCP_UDP_DPORT); + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); +} + +static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_mp_hash_config *config) +{ + u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); + unsigned long *headers = config->headers; + unsigned long *fields = config->fields; + + switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { + case 0: + mlxsw_sp_mp6_hash_outer_addr(config); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); + break; + case 1: + mlxsw_sp_mp6_hash_outer_addr(config); + MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); + break; + case 2: + /* Outer */ + mlxsw_sp_mp6_hash_outer_addr(config); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); + /* Inner */ + mlxsw_sp_mp_hash_inner_l3(config); + break; + case 3: + /* Outer */ + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); + MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); + } + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); + MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); + } + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) + MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); + if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) + MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); + /* Inner */ + mlxsw_sp_mp_hash_inner_custom(config, hash_fields); + break; } } static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) { + struct mlxsw_sp_mp_hash_config config = {}; char recr2_pl[MLXSW_REG_RECR2_LEN]; + unsigned long bit; u32 seed; seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); mlxsw_reg_recr2_pack(recr2_pl, seed); - mlxsw_sp_mp4_hash_init(mlxsw_sp, recr2_pl); - mlxsw_sp_mp6_hash_init(mlxsw_sp, recr2_pl); + mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); + mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); + + for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) + mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); + for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) + mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); + for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) + mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); + for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) + mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h index be7708a375e1..c5d7007f9173 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.h @@ -58,7 +58,6 @@ struct mlxsw_sp_router { #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */ struct list_head nexthop_neighs_list; struct list_head ipip_list; - bool aborted; struct notifier_block nexthop_nb; struct notifier_block fib_nb; struct notifier_block netevent_nb; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c index eeccd586e781..c5ef9aa64efe 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c @@ -898,7 +898,7 @@ mlxsw_sp_port_attr_br_mrouter_set(struct mlxsw_sp_port *mlxsw_sp_port, return 0; } -static int mlxsw_sp_port_attr_set(struct net_device *dev, +static int mlxsw_sp_port_attr_set(struct net_device *dev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) { @@ -1766,7 +1766,7 @@ mlxsw_sp_port_mrouter_update_mdb(struct mlxsw_sp_port *mlxsw_sp_port, } } -static int mlxsw_sp_port_obj_add(struct net_device *dev, +static int mlxsw_sp_port_obj_add(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) { @@ -1916,7 +1916,7 @@ mlxsw_sp_bridge_port_mdb_flush(struct mlxsw_sp_port *mlxsw_sp_port, } } -static int mlxsw_sp_port_obj_del(struct net_device *dev, +static int mlxsw_sp_port_obj_del(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); @@ -2520,6 +2520,7 @@ static void mlxsw_sp_fdb_notify_mac_process(struct mlxsw_sp *mlxsw_sp, char *sfn_pl, int rec_index, bool adding) { + unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core); struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; struct mlxsw_sp_bridge_device *bridge_device; struct mlxsw_sp_bridge_port *bridge_port; @@ -2532,6 +2533,9 @@ static void mlxsw_sp_fdb_notify_mac_process(struct mlxsw_sp *mlxsw_sp, int err; mlxsw_reg_sfn_mac_unpack(sfn_pl, rec_index, mac, &fid, &local_port); + + if (WARN_ON_ONCE(local_port >= max_ports)) + return; mlxsw_sp_port = mlxsw_sp->ports[local_port]; if (!mlxsw_sp_port) { dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect local port in FDB notification\n"); diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchib.c b/drivers/net/ethernet/mellanox/mlxsw/switchib.c deleted file mode 100644 index 1e561132eb1e..000000000000 --- a/drivers/net/ethernet/mellanox/mlxsw/switchib.c +++ /dev/null @@ -1,595 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 -/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/pci.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/slab.h> -#include <linux/device.h> -#include <linux/skbuff.h> -#include <linux/if_vlan.h> -#include <net/switchdev.h> - -#include "pci.h" -#include "core.h" -#include "reg.h" -#include "port.h" -#include "trap.h" -#include "txheader.h" -#include "ib.h" - -static const char mlxsw_sib_driver_name[] = "mlxsw_switchib"; -static const char mlxsw_sib2_driver_name[] = "mlxsw_switchib2"; - -struct mlxsw_sib_port; - -struct mlxsw_sib { - struct mlxsw_sib_port **ports; - struct mlxsw_core *core; - const struct mlxsw_bus_info *bus_info; - u8 hw_id[ETH_ALEN]; -}; - -struct mlxsw_sib_port { - struct mlxsw_sib *mlxsw_sib; - u8 local_port; - struct { - u8 module; - } mapping; -}; - -/* tx_v1_hdr_version - * Tx header version. - * Must be set to 1. - */ -MLXSW_ITEM32(tx_v1, hdr, version, 0x00, 28, 4); - -/* tx_v1_hdr_ctl - * Packet control type. - * 0 - Ethernet control (e.g. EMADs, LACP) - * 1 - Ethernet data - */ -MLXSW_ITEM32(tx_v1, hdr, ctl, 0x00, 26, 2); - -/* tx_v1_hdr_proto - * Packet protocol type. Must be set to 1 (Ethernet). - */ -MLXSW_ITEM32(tx_v1, hdr, proto, 0x00, 21, 3); - -/* tx_v1_hdr_swid - * Switch partition ID. Must be set to 0. - */ -MLXSW_ITEM32(tx_v1, hdr, swid, 0x00, 12, 3); - -/* tx_v1_hdr_control_tclass - * Indicates if the packet should use the control TClass and not one - * of the data TClasses. - */ -MLXSW_ITEM32(tx_v1, hdr, control_tclass, 0x00, 6, 1); - -/* tx_v1_hdr_port_mid - * Destination local port for unicast packets. - * Destination multicast ID for multicast packets. - * - * Control packets are directed to a specific egress port, while data - * packets are transmitted through the CPU port (0) into the switch partition, - * where forwarding rules are applied. - */ -MLXSW_ITEM32(tx_v1, hdr, port_mid, 0x04, 16, 16); - -/* tx_v1_hdr_type - * 0 - Data packets - * 6 - Control packets - */ -MLXSW_ITEM32(tx_v1, hdr, type, 0x0C, 0, 4); - -static void -mlxsw_sib_tx_v1_hdr_construct(struct sk_buff *skb, - const struct mlxsw_tx_info *tx_info) -{ - char *txhdr = skb_push(skb, MLXSW_TXHDR_LEN); - - memset(txhdr, 0, MLXSW_TXHDR_LEN); - - mlxsw_tx_v1_hdr_version_set(txhdr, MLXSW_TXHDR_VERSION_1); - mlxsw_tx_v1_hdr_ctl_set(txhdr, MLXSW_TXHDR_ETH_CTL); - mlxsw_tx_v1_hdr_proto_set(txhdr, MLXSW_TXHDR_PROTO_ETH); - mlxsw_tx_v1_hdr_swid_set(txhdr, 0); - mlxsw_tx_v1_hdr_control_tclass_set(txhdr, 1); - mlxsw_tx_v1_hdr_port_mid_set(txhdr, tx_info->local_port); - mlxsw_tx_v1_hdr_type_set(txhdr, MLXSW_TXHDR_TYPE_CONTROL); -} - -static int mlxsw_sib_hw_id_get(struct mlxsw_sib *mlxsw_sib) -{ - char spad_pl[MLXSW_REG_SPAD_LEN] = {0}; - int err; - - err = mlxsw_reg_query(mlxsw_sib->core, MLXSW_REG(spad), spad_pl); - if (err) - return err; - mlxsw_reg_spad_base_mac_memcpy_from(spad_pl, mlxsw_sib->hw_id); - return 0; -} - -static int -mlxsw_sib_port_admin_status_set(struct mlxsw_sib_port *mlxsw_sib_port, - bool is_up) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_sib_port->mlxsw_sib; - char paos_pl[MLXSW_REG_PAOS_LEN]; - - mlxsw_reg_paos_pack(paos_pl, mlxsw_sib_port->local_port, - is_up ? MLXSW_PORT_ADMIN_STATUS_UP : - MLXSW_PORT_ADMIN_STATUS_DOWN); - return mlxsw_reg_write(mlxsw_sib->core, MLXSW_REG(paos), paos_pl); -} - -static int mlxsw_sib_port_mtu_set(struct mlxsw_sib_port *mlxsw_sib_port, - u16 mtu) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_sib_port->mlxsw_sib; - char pmtu_pl[MLXSW_REG_PMTU_LEN]; - int max_mtu; - int err; - - mlxsw_reg_pmtu_pack(pmtu_pl, mlxsw_sib_port->local_port, 0); - err = mlxsw_reg_query(mlxsw_sib->core, MLXSW_REG(pmtu), pmtu_pl); - if (err) - return err; - max_mtu = mlxsw_reg_pmtu_max_mtu_get(pmtu_pl); - - if (mtu > max_mtu) - return -EINVAL; - - mlxsw_reg_pmtu_pack(pmtu_pl, mlxsw_sib_port->local_port, mtu); - return mlxsw_reg_write(mlxsw_sib->core, MLXSW_REG(pmtu), pmtu_pl); -} - -static int mlxsw_sib_port_set(struct mlxsw_sib_port *mlxsw_sib_port, u8 port) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_sib_port->mlxsw_sib; - char plib_pl[MLXSW_REG_PLIB_LEN] = {0}; - int err; - - mlxsw_reg_plib_local_port_set(plib_pl, mlxsw_sib_port->local_port); - mlxsw_reg_plib_ib_port_set(plib_pl, port); - err = mlxsw_reg_write(mlxsw_sib->core, MLXSW_REG(plib), plib_pl); - return err; -} - -static int mlxsw_sib_port_swid_set(struct mlxsw_sib_port *mlxsw_sib_port, - u8 swid) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_sib_port->mlxsw_sib; - char pspa_pl[MLXSW_REG_PSPA_LEN]; - - mlxsw_reg_pspa_pack(pspa_pl, swid, mlxsw_sib_port->local_port); - return mlxsw_reg_write(mlxsw_sib->core, MLXSW_REG(pspa), pspa_pl); -} - -static int mlxsw_sib_port_module_info_get(struct mlxsw_sib *mlxsw_sib, - u8 local_port, u8 *p_module, - u8 *p_width) -{ - char pmlp_pl[MLXSW_REG_PMLP_LEN]; - int err; - - mlxsw_reg_pmlp_pack(pmlp_pl, local_port); - err = mlxsw_reg_query(mlxsw_sib->core, MLXSW_REG(pmlp), pmlp_pl); - if (err) - return err; - *p_module = mlxsw_reg_pmlp_module_get(pmlp_pl, 0); - *p_width = mlxsw_reg_pmlp_width_get(pmlp_pl); - return 0; -} - -static int mlxsw_sib_port_speed_set(struct mlxsw_sib_port *mlxsw_sib_port, - u16 speed, u16 width) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_sib_port->mlxsw_sib; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - - mlxsw_reg_ptys_ib_pack(ptys_pl, mlxsw_sib_port->local_port, speed, - width); - return mlxsw_reg_write(mlxsw_sib->core, MLXSW_REG(ptys), ptys_pl); -} - -static bool mlxsw_sib_port_created(struct mlxsw_sib *mlxsw_sib, u8 local_port) -{ - return mlxsw_sib->ports[local_port] != NULL; -} - -static int __mlxsw_sib_port_create(struct mlxsw_sib *mlxsw_sib, u8 local_port, - u8 module, u8 width) -{ - struct mlxsw_sib_port *mlxsw_sib_port; - int err; - - mlxsw_sib_port = kzalloc(sizeof(*mlxsw_sib_port), GFP_KERNEL); - if (!mlxsw_sib_port) - return -ENOMEM; - mlxsw_sib_port->mlxsw_sib = mlxsw_sib; - mlxsw_sib_port->local_port = local_port; - mlxsw_sib_port->mapping.module = module; - - err = mlxsw_sib_port_swid_set(mlxsw_sib_port, 0); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to set SWID\n", - mlxsw_sib_port->local_port); - goto err_port_swid_set; - } - - /* Expose the IB port number as it's front panel name */ - err = mlxsw_sib_port_set(mlxsw_sib_port, module + 1); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to set IB port\n", - mlxsw_sib_port->local_port); - goto err_port_ib_set; - } - - /* Supports all speeds from SDR to FDR (bitmask) and support bus width - * of 1x, 2x and 4x (3 bits bitmask) - */ - err = mlxsw_sib_port_speed_set(mlxsw_sib_port, - MLXSW_REG_PTYS_IB_SPEED_EDR - 1, - BIT(3) - 1); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to set speed\n", - mlxsw_sib_port->local_port); - goto err_port_speed_set; - } - - /* Change to the maximum MTU the device supports, the SMA will take - * care of the active MTU - */ - err = mlxsw_sib_port_mtu_set(mlxsw_sib_port, MLXSW_IB_DEFAULT_MTU); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to set MTU\n", - mlxsw_sib_port->local_port); - goto err_port_mtu_set; - } - - err = mlxsw_sib_port_admin_status_set(mlxsw_sib_port, true); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to change admin state to UP\n", - mlxsw_sib_port->local_port); - goto err_port_admin_set; - } - - mlxsw_core_port_ib_set(mlxsw_sib->core, mlxsw_sib_port->local_port, - mlxsw_sib_port); - mlxsw_sib->ports[local_port] = mlxsw_sib_port; - return 0; - -err_port_admin_set: -err_port_mtu_set: -err_port_speed_set: -err_port_ib_set: - mlxsw_sib_port_swid_set(mlxsw_sib_port, MLXSW_PORT_SWID_DISABLED_PORT); -err_port_swid_set: - kfree(mlxsw_sib_port); - return err; -} - -static int mlxsw_sib_port_create(struct mlxsw_sib *mlxsw_sib, u8 local_port, - u8 module, u8 width) -{ - int err; - - err = mlxsw_core_port_init(mlxsw_sib->core, local_port, - module + 1, false, 0, false, 0, - mlxsw_sib->hw_id, sizeof(mlxsw_sib->hw_id)); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Port %d: Failed to init core port\n", - local_port); - return err; - } - err = __mlxsw_sib_port_create(mlxsw_sib, local_port, module, width); - if (err) - goto err_port_create; - - return 0; - -err_port_create: - mlxsw_core_port_fini(mlxsw_sib->core, local_port); - return err; -} - -static void __mlxsw_sib_port_remove(struct mlxsw_sib *mlxsw_sib, u8 local_port) -{ - struct mlxsw_sib_port *mlxsw_sib_port = mlxsw_sib->ports[local_port]; - - mlxsw_core_port_clear(mlxsw_sib->core, local_port, mlxsw_sib); - mlxsw_sib->ports[local_port] = NULL; - mlxsw_sib_port_admin_status_set(mlxsw_sib_port, false); - mlxsw_sib_port_swid_set(mlxsw_sib_port, MLXSW_PORT_SWID_DISABLED_PORT); - kfree(mlxsw_sib_port); -} - -static void mlxsw_sib_port_remove(struct mlxsw_sib *mlxsw_sib, u8 local_port) -{ - __mlxsw_sib_port_remove(mlxsw_sib, local_port); - mlxsw_core_port_fini(mlxsw_sib->core, local_port); -} - -static void mlxsw_sib_ports_remove(struct mlxsw_sib *mlxsw_sib) -{ - int i; - - for (i = 1; i < MLXSW_PORT_MAX_IB_PORTS; i++) - if (mlxsw_sib_port_created(mlxsw_sib, i)) - mlxsw_sib_port_remove(mlxsw_sib, i); - kfree(mlxsw_sib->ports); -} - -static int mlxsw_sib_ports_create(struct mlxsw_sib *mlxsw_sib) -{ - size_t alloc_size; - u8 module, width; - int i; - int err; - - alloc_size = sizeof(struct mlxsw_sib_port *) * MLXSW_PORT_MAX_IB_PORTS; - mlxsw_sib->ports = kzalloc(alloc_size, GFP_KERNEL); - if (!mlxsw_sib->ports) - return -ENOMEM; - - for (i = 1; i < MLXSW_PORT_MAX_IB_PORTS; i++) { - err = mlxsw_sib_port_module_info_get(mlxsw_sib, i, &module, - &width); - if (err) - goto err_port_module_info_get; - if (!width) - continue; - err = mlxsw_sib_port_create(mlxsw_sib, i, module, width); - if (err) - goto err_port_create; - } - return 0; - -err_port_create: -err_port_module_info_get: - for (i--; i >= 1; i--) - if (mlxsw_sib_port_created(mlxsw_sib, i)) - mlxsw_sib_port_remove(mlxsw_sib, i); - kfree(mlxsw_sib->ports); - return err; -} - -static void -mlxsw_sib_pude_ib_event_func(struct mlxsw_sib_port *mlxsw_sib_port, - enum mlxsw_reg_pude_oper_status status) -{ - if (status == MLXSW_PORT_OPER_STATUS_UP) - pr_info("ib link for port %d - up\n", - mlxsw_sib_port->mapping.module + 1); - else - pr_info("ib link for port %d - down\n", - mlxsw_sib_port->mapping.module + 1); -} - -static void mlxsw_sib_pude_event_func(const struct mlxsw_reg_info *reg, - char *pude_pl, void *priv) -{ - struct mlxsw_sib *mlxsw_sib = priv; - struct mlxsw_sib_port *mlxsw_sib_port; - enum mlxsw_reg_pude_oper_status status; - u8 local_port; - - local_port = mlxsw_reg_pude_local_port_get(pude_pl); - mlxsw_sib_port = mlxsw_sib->ports[local_port]; - if (!mlxsw_sib_port) { - dev_warn(mlxsw_sib->bus_info->dev, "Port %d: Link event received for non-existent port\n", - local_port); - return; - } - - status = mlxsw_reg_pude_oper_status_get(pude_pl); - mlxsw_sib_pude_ib_event_func(mlxsw_sib_port, status); -} - -static const struct mlxsw_listener mlxsw_sib_listener[] = { - MLXSW_EVENTL(mlxsw_sib_pude_event_func, PUDE, EMAD), -}; - -static int mlxsw_sib_taps_init(struct mlxsw_sib *mlxsw_sib) -{ - int i; - int err; - - for (i = 0; i < ARRAY_SIZE(mlxsw_sib_listener); i++) { - err = mlxsw_core_trap_register(mlxsw_sib->core, - &mlxsw_sib_listener[i], - mlxsw_sib); - if (err) - goto err_rx_listener_register; - } - - return 0; - -err_rx_listener_register: - for (i--; i >= 0; i--) { - mlxsw_core_trap_unregister(mlxsw_sib->core, - &mlxsw_sib_listener[i], - mlxsw_sib); - } - - return err; -} - -static void mlxsw_sib_traps_fini(struct mlxsw_sib *mlxsw_sib) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(mlxsw_sib_listener); i++) { - mlxsw_core_trap_unregister(mlxsw_sib->core, - &mlxsw_sib_listener[i], mlxsw_sib); - } -} - -static int mlxsw_sib_basic_trap_groups_set(struct mlxsw_core *mlxsw_core) -{ - char htgt_pl[MLXSW_REG_HTGT_LEN]; - - mlxsw_reg_htgt_pack(htgt_pl, MLXSW_REG_HTGT_TRAP_GROUP_EMAD, - MLXSW_REG_HTGT_INVALID_POLICER, - MLXSW_REG_HTGT_DEFAULT_PRIORITY, - MLXSW_REG_HTGT_DEFAULT_TC); - mlxsw_reg_htgt_swid_set(htgt_pl, MLXSW_PORT_SWID_ALL_SWIDS); - mlxsw_reg_htgt_local_path_rdq_set(htgt_pl, - MLXSW_REG_HTGT_LOCAL_PATH_RDQ_SIB_EMAD); - return mlxsw_reg_write(mlxsw_core, MLXSW_REG(htgt), htgt_pl); -} - -static int mlxsw_sib_init(struct mlxsw_core *mlxsw_core, - const struct mlxsw_bus_info *mlxsw_bus_info, - struct netlink_ext_ack *extack) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_core_driver_priv(mlxsw_core); - int err; - - mlxsw_sib->core = mlxsw_core; - mlxsw_sib->bus_info = mlxsw_bus_info; - - err = mlxsw_sib_hw_id_get(mlxsw_sib); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Failed to get switch HW ID\n"); - return err; - } - - err = mlxsw_sib_ports_create(mlxsw_sib); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Failed to create ports\n"); - return err; - } - - err = mlxsw_sib_taps_init(mlxsw_sib); - if (err) { - dev_err(mlxsw_sib->bus_info->dev, "Failed to set traps\n"); - goto err_traps_init_err; - } - - return 0; - -err_traps_init_err: - mlxsw_sib_ports_remove(mlxsw_sib); - return err; -} - -static void mlxsw_sib_fini(struct mlxsw_core *mlxsw_core) -{ - struct mlxsw_sib *mlxsw_sib = mlxsw_core_driver_priv(mlxsw_core); - - mlxsw_sib_traps_fini(mlxsw_sib); - mlxsw_sib_ports_remove(mlxsw_sib); -} - -static const struct mlxsw_config_profile mlxsw_sib_config_profile = { - .used_max_system_port = 1, - .max_system_port = 48000, - .used_max_ib_mc = 1, - .max_ib_mc = 27, - .used_max_pkey = 1, - .max_pkey = 32, - .swid_config = { - { - .used_type = 1, - .type = MLXSW_PORT_SWID_TYPE_IB, - } - }, -}; - -static struct mlxsw_driver mlxsw_sib_driver = { - .kind = mlxsw_sib_driver_name, - .priv_size = sizeof(struct mlxsw_sib), - .init = mlxsw_sib_init, - .fini = mlxsw_sib_fini, - .basic_trap_groups_set = mlxsw_sib_basic_trap_groups_set, - .txhdr_construct = mlxsw_sib_tx_v1_hdr_construct, - .txhdr_len = MLXSW_TXHDR_LEN, - .profile = &mlxsw_sib_config_profile, -}; - -static struct mlxsw_driver mlxsw_sib2_driver = { - .kind = mlxsw_sib2_driver_name, - .priv_size = sizeof(struct mlxsw_sib), - .init = mlxsw_sib_init, - .fini = mlxsw_sib_fini, - .basic_trap_groups_set = mlxsw_sib_basic_trap_groups_set, - .txhdr_construct = mlxsw_sib_tx_v1_hdr_construct, - .txhdr_len = MLXSW_TXHDR_LEN, - .profile = &mlxsw_sib_config_profile, -}; - -static const struct pci_device_id mlxsw_sib_pci_id_table[] = { - {PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SWITCHIB), 0}, - {0, }, -}; - -static struct pci_driver mlxsw_sib_pci_driver = { - .name = mlxsw_sib_driver_name, - .id_table = mlxsw_sib_pci_id_table, -}; - -static const struct pci_device_id mlxsw_sib2_pci_id_table[] = { - {PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SWITCHIB2), 0}, - {0, }, -}; - -static struct pci_driver mlxsw_sib2_pci_driver = { - .name = mlxsw_sib2_driver_name, - .id_table = mlxsw_sib2_pci_id_table, -}; - -static int __init mlxsw_sib_module_init(void) -{ - int err; - - err = mlxsw_core_driver_register(&mlxsw_sib_driver); - if (err) - return err; - - err = mlxsw_core_driver_register(&mlxsw_sib2_driver); - if (err) - goto err_sib2_driver_register; - - err = mlxsw_pci_driver_register(&mlxsw_sib_pci_driver); - if (err) - goto err_sib_pci_driver_register; - - err = mlxsw_pci_driver_register(&mlxsw_sib2_pci_driver); - if (err) - goto err_sib2_pci_driver_register; - - return 0; - -err_sib2_pci_driver_register: - mlxsw_pci_driver_unregister(&mlxsw_sib_pci_driver); -err_sib_pci_driver_register: - mlxsw_core_driver_unregister(&mlxsw_sib2_driver); -err_sib2_driver_register: - mlxsw_core_driver_unregister(&mlxsw_sib_driver); - return err; -} - -static void __exit mlxsw_sib_module_exit(void) -{ - mlxsw_pci_driver_unregister(&mlxsw_sib2_pci_driver); - mlxsw_pci_driver_unregister(&mlxsw_sib_pci_driver); - mlxsw_core_driver_unregister(&mlxsw_sib2_driver); - mlxsw_core_driver_unregister(&mlxsw_sib_driver); -} - -module_init(mlxsw_sib_module_init); -module_exit(mlxsw_sib_module_exit); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Elad Raz <eladr@@mellanox.com>"); -MODULE_DESCRIPTION("Mellanox SwitchIB and SwitchIB-2 driver"); -MODULE_ALIAS("mlxsw_switchib2"); -MODULE_DEVICE_TABLE(pci, mlxsw_sib_pci_id_table); -MODULE_DEVICE_TABLE(pci, mlxsw_sib2_pci_id_table); diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c deleted file mode 100644 index 131b2a53d261..000000000000 --- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c +++ /dev/null @@ -1,1691 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 -/* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/types.h> -#include <linux/pci.h> -#include <linux/netdevice.h> -#include <linux/ethtool.h> -#include <linux/etherdevice.h> -#include <linux/slab.h> -#include <linux/device.h> -#include <linux/skbuff.h> -#include <linux/if_vlan.h> - -#include "pci.h" -#include "core.h" -#include "reg.h" -#include "port.h" -#include "trap.h" -#include "txheader.h" -#include "ib.h" - -static const char mlxsw_sx_driver_name[] = "mlxsw_switchx2"; -static const char mlxsw_sx_driver_version[] = "1.0"; - -struct mlxsw_sx_port; - -struct mlxsw_sx { - struct mlxsw_sx_port **ports; - struct mlxsw_core *core; - const struct mlxsw_bus_info *bus_info; - u8 hw_id[ETH_ALEN]; -}; - -struct mlxsw_sx_port_pcpu_stats { - u64 rx_packets; - u64 rx_bytes; - u64 tx_packets; - u64 tx_bytes; - struct u64_stats_sync syncp; - u32 tx_dropped; -}; - -struct mlxsw_sx_port { - struct net_device *dev; - struct mlxsw_sx_port_pcpu_stats __percpu *pcpu_stats; - struct mlxsw_sx *mlxsw_sx; - u8 local_port; - struct { - u8 module; - } mapping; -}; - -/* tx_hdr_version - * Tx header version. - * Must be set to 0. - */ -MLXSW_ITEM32(tx, hdr, version, 0x00, 28, 4); - -/* tx_hdr_ctl - * Packet control type. - * 0 - Ethernet control (e.g. EMADs, LACP) - * 1 - Ethernet data - */ -MLXSW_ITEM32(tx, hdr, ctl, 0x00, 26, 2); - -/* tx_hdr_proto - * Packet protocol type. Must be set to 1 (Ethernet). - */ -MLXSW_ITEM32(tx, hdr, proto, 0x00, 21, 3); - -/* tx_hdr_etclass - * Egress TClass to be used on the egress device on the egress port. - * The MSB is specified in the 'ctclass3' field. - * Range is 0-15, where 15 is the highest priority. - */ -MLXSW_ITEM32(tx, hdr, etclass, 0x00, 18, 3); - -/* tx_hdr_swid - * Switch partition ID. - */ -MLXSW_ITEM32(tx, hdr, swid, 0x00, 12, 3); - -/* tx_hdr_port_mid - * Destination local port for unicast packets. - * Destination multicast ID for multicast packets. - * - * Control packets are directed to a specific egress port, while data - * packets are transmitted through the CPU port (0) into the switch partition, - * where forwarding rules are applied. - */ -MLXSW_ITEM32(tx, hdr, port_mid, 0x04, 16, 16); - -/* tx_hdr_ctclass3 - * See field 'etclass'. - */ -MLXSW_ITEM32(tx, hdr, ctclass3, 0x04, 14, 1); - -/* tx_hdr_rdq - * RDQ for control packets sent to remote CPU. - * Must be set to 0x1F for EMADs, otherwise 0. - */ -MLXSW_ITEM32(tx, hdr, rdq, 0x04, 9, 5); - -/* tx_hdr_cpu_sig - * Signature control for packets going to CPU. Must be set to 0. - */ -MLXSW_ITEM32(tx, hdr, cpu_sig, 0x04, 0, 9); - -/* tx_hdr_sig - * Stacking protocl signature. Must be set to 0xE0E0. - */ -MLXSW_ITEM32(tx, hdr, sig, 0x0C, 16, 16); - -/* tx_hdr_stclass - * Stacking TClass. - */ -MLXSW_ITEM32(tx, hdr, stclass, 0x0C, 13, 3); - -/* tx_hdr_emad - * EMAD bit. Must be set for EMADs. - */ -MLXSW_ITEM32(tx, hdr, emad, 0x0C, 5, 1); - -/* tx_hdr_type - * 0 - Data packets - * 6 - Control packets - */ -MLXSW_ITEM32(tx, hdr, type, 0x0C, 0, 4); - -static void mlxsw_sx_txhdr_construct(struct sk_buff *skb, - const struct mlxsw_tx_info *tx_info) -{ - char *txhdr = skb_push(skb, MLXSW_TXHDR_LEN); - bool is_emad = tx_info->is_emad; - - memset(txhdr, 0, MLXSW_TXHDR_LEN); - - /* We currently set default values for the egress tclass (QoS). */ - mlxsw_tx_hdr_version_set(txhdr, MLXSW_TXHDR_VERSION_0); - mlxsw_tx_hdr_ctl_set(txhdr, MLXSW_TXHDR_ETH_CTL); - mlxsw_tx_hdr_proto_set(txhdr, MLXSW_TXHDR_PROTO_ETH); - mlxsw_tx_hdr_etclass_set(txhdr, is_emad ? MLXSW_TXHDR_ETCLASS_6 : - MLXSW_TXHDR_ETCLASS_5); - mlxsw_tx_hdr_swid_set(txhdr, 0); - mlxsw_tx_hdr_port_mid_set(txhdr, tx_info->local_port); - mlxsw_tx_hdr_ctclass3_set(txhdr, MLXSW_TXHDR_CTCLASS3); - mlxsw_tx_hdr_rdq_set(txhdr, is_emad ? MLXSW_TXHDR_RDQ_EMAD : - MLXSW_TXHDR_RDQ_OTHER); - mlxsw_tx_hdr_cpu_sig_set(txhdr, MLXSW_TXHDR_CPU_SIG); - mlxsw_tx_hdr_sig_set(txhdr, MLXSW_TXHDR_SIG); - mlxsw_tx_hdr_stclass_set(txhdr, MLXSW_TXHDR_STCLASS_NONE); - mlxsw_tx_hdr_emad_set(txhdr, is_emad ? MLXSW_TXHDR_EMAD : - MLXSW_TXHDR_NOT_EMAD); - mlxsw_tx_hdr_type_set(txhdr, MLXSW_TXHDR_TYPE_CONTROL); -} - -static int mlxsw_sx_port_admin_status_set(struct mlxsw_sx_port *mlxsw_sx_port, - bool is_up) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char paos_pl[MLXSW_REG_PAOS_LEN]; - - mlxsw_reg_paos_pack(paos_pl, mlxsw_sx_port->local_port, - is_up ? MLXSW_PORT_ADMIN_STATUS_UP : - MLXSW_PORT_ADMIN_STATUS_DOWN); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(paos), paos_pl); -} - -static int mlxsw_sx_port_oper_status_get(struct mlxsw_sx_port *mlxsw_sx_port, - bool *p_is_up) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char paos_pl[MLXSW_REG_PAOS_LEN]; - u8 oper_status; - int err; - - mlxsw_reg_paos_pack(paos_pl, mlxsw_sx_port->local_port, 0); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(paos), paos_pl); - if (err) - return err; - oper_status = mlxsw_reg_paos_oper_status_get(paos_pl); - *p_is_up = oper_status == MLXSW_PORT_ADMIN_STATUS_UP; - return 0; -} - -static int __mlxsw_sx_port_mtu_set(struct mlxsw_sx_port *mlxsw_sx_port, - u16 mtu) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char pmtu_pl[MLXSW_REG_PMTU_LEN]; - int max_mtu; - int err; - - mlxsw_reg_pmtu_pack(pmtu_pl, mlxsw_sx_port->local_port, 0); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(pmtu), pmtu_pl); - if (err) - return err; - max_mtu = mlxsw_reg_pmtu_max_mtu_get(pmtu_pl); - - if (mtu > max_mtu) - return -EINVAL; - - mlxsw_reg_pmtu_pack(pmtu_pl, mlxsw_sx_port->local_port, mtu); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(pmtu), pmtu_pl); -} - -static int mlxsw_sx_port_mtu_eth_set(struct mlxsw_sx_port *mlxsw_sx_port, - u16 mtu) -{ - mtu += MLXSW_TXHDR_LEN + ETH_HLEN; - return __mlxsw_sx_port_mtu_set(mlxsw_sx_port, mtu); -} - -static int mlxsw_sx_port_mtu_ib_set(struct mlxsw_sx_port *mlxsw_sx_port, - u16 mtu) -{ - return __mlxsw_sx_port_mtu_set(mlxsw_sx_port, mtu); -} - -static int mlxsw_sx_port_ib_port_set(struct mlxsw_sx_port *mlxsw_sx_port, - u8 ib_port) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char plib_pl[MLXSW_REG_PLIB_LEN] = {0}; - int err; - - mlxsw_reg_plib_local_port_set(plib_pl, mlxsw_sx_port->local_port); - mlxsw_reg_plib_ib_port_set(plib_pl, ib_port); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(plib), plib_pl); - return err; -} - -static int mlxsw_sx_port_swid_set(struct mlxsw_sx_port *mlxsw_sx_port, u8 swid) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char pspa_pl[MLXSW_REG_PSPA_LEN]; - - mlxsw_reg_pspa_pack(pspa_pl, swid, mlxsw_sx_port->local_port); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(pspa), pspa_pl); -} - -static int -mlxsw_sx_port_system_port_mapping_set(struct mlxsw_sx_port *mlxsw_sx_port) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char sspr_pl[MLXSW_REG_SSPR_LEN]; - - mlxsw_reg_sspr_pack(sspr_pl, mlxsw_sx_port->local_port); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sspr), sspr_pl); -} - -static int mlxsw_sx_port_module_info_get(struct mlxsw_sx *mlxsw_sx, - u8 local_port, u8 *p_module, - u8 *p_width) -{ - char pmlp_pl[MLXSW_REG_PMLP_LEN]; - int err; - - mlxsw_reg_pmlp_pack(pmlp_pl, local_port); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(pmlp), pmlp_pl); - if (err) - return err; - *p_module = mlxsw_reg_pmlp_module_get(pmlp_pl, 0); - *p_width = mlxsw_reg_pmlp_width_get(pmlp_pl); - return 0; -} - -static int mlxsw_sx_port_open(struct net_device *dev) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - int err; - - err = mlxsw_sx_port_admin_status_set(mlxsw_sx_port, true); - if (err) - return err; - netif_start_queue(dev); - return 0; -} - -static int mlxsw_sx_port_stop(struct net_device *dev) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - - netif_stop_queue(dev); - return mlxsw_sx_port_admin_status_set(mlxsw_sx_port, false); -} - -static netdev_tx_t mlxsw_sx_port_xmit(struct sk_buff *skb, - struct net_device *dev) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - struct mlxsw_sx_port_pcpu_stats *pcpu_stats; - const struct mlxsw_tx_info tx_info = { - .local_port = mlxsw_sx_port->local_port, - .is_emad = false, - }; - u64 len; - int err; - - if (skb_cow_head(skb, MLXSW_TXHDR_LEN)) { - this_cpu_inc(mlxsw_sx_port->pcpu_stats->tx_dropped); - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } - - memset(skb->cb, 0, sizeof(struct mlxsw_skb_cb)); - - if (mlxsw_core_skb_transmit_busy(mlxsw_sx->core, &tx_info)) - return NETDEV_TX_BUSY; - - mlxsw_sx_txhdr_construct(skb, &tx_info); - /* TX header is consumed by HW on the way so we shouldn't count its - * bytes as being sent. - */ - len = skb->len - MLXSW_TXHDR_LEN; - /* Due to a race we might fail here because of a full queue. In that - * unlikely case we simply drop the packet. - */ - err = mlxsw_core_skb_transmit(mlxsw_sx->core, skb, &tx_info); - - if (!err) { - pcpu_stats = this_cpu_ptr(mlxsw_sx_port->pcpu_stats); - u64_stats_update_begin(&pcpu_stats->syncp); - pcpu_stats->tx_packets++; - pcpu_stats->tx_bytes += len; - u64_stats_update_end(&pcpu_stats->syncp); - } else { - this_cpu_inc(mlxsw_sx_port->pcpu_stats->tx_dropped); - dev_kfree_skb_any(skb); - } - return NETDEV_TX_OK; -} - -static int mlxsw_sx_port_change_mtu(struct net_device *dev, int mtu) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - int err; - - err = mlxsw_sx_port_mtu_eth_set(mlxsw_sx_port, mtu); - if (err) - return err; - dev->mtu = mtu; - return 0; -} - -static void -mlxsw_sx_port_get_stats64(struct net_device *dev, - struct rtnl_link_stats64 *stats) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx_port_pcpu_stats *p; - u64 rx_packets, rx_bytes, tx_packets, tx_bytes; - u32 tx_dropped = 0; - unsigned int start; - int i; - - for_each_possible_cpu(i) { - p = per_cpu_ptr(mlxsw_sx_port->pcpu_stats, i); - do { - start = u64_stats_fetch_begin_irq(&p->syncp); - rx_packets = p->rx_packets; - rx_bytes = p->rx_bytes; - tx_packets = p->tx_packets; - tx_bytes = p->tx_bytes; - } while (u64_stats_fetch_retry_irq(&p->syncp, start)); - - stats->rx_packets += rx_packets; - stats->rx_bytes += rx_bytes; - stats->tx_packets += tx_packets; - stats->tx_bytes += tx_bytes; - /* tx_dropped is u32, updated without syncp protection. */ - tx_dropped += p->tx_dropped; - } - stats->tx_dropped = tx_dropped; -} - -static struct devlink_port * -mlxsw_sx_port_get_devlink_port(struct net_device *dev) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - - return mlxsw_core_port_devlink_port_get(mlxsw_sx->core, - mlxsw_sx_port->local_port); -} - -static const struct net_device_ops mlxsw_sx_port_netdev_ops = { - .ndo_open = mlxsw_sx_port_open, - .ndo_stop = mlxsw_sx_port_stop, - .ndo_start_xmit = mlxsw_sx_port_xmit, - .ndo_change_mtu = mlxsw_sx_port_change_mtu, - .ndo_get_stats64 = mlxsw_sx_port_get_stats64, - .ndo_get_devlink_port = mlxsw_sx_port_get_devlink_port, -}; - -static void mlxsw_sx_port_get_drvinfo(struct net_device *dev, - struct ethtool_drvinfo *drvinfo) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - - strlcpy(drvinfo->driver, mlxsw_sx_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, mlxsw_sx_driver_version, - sizeof(drvinfo->version)); - snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), - "%d.%d.%d", - mlxsw_sx->bus_info->fw_rev.major, - mlxsw_sx->bus_info->fw_rev.minor, - mlxsw_sx->bus_info->fw_rev.subminor); - strlcpy(drvinfo->bus_info, mlxsw_sx->bus_info->device_name, - sizeof(drvinfo->bus_info)); -} - -struct mlxsw_sx_port_hw_stats { - char str[ETH_GSTRING_LEN]; - u64 (*getter)(const char *payload); -}; - -static const struct mlxsw_sx_port_hw_stats mlxsw_sx_port_hw_stats[] = { - { - .str = "a_frames_transmitted_ok", - .getter = mlxsw_reg_ppcnt_a_frames_transmitted_ok_get, - }, - { - .str = "a_frames_received_ok", - .getter = mlxsw_reg_ppcnt_a_frames_received_ok_get, - }, - { - .str = "a_frame_check_sequence_errors", - .getter = mlxsw_reg_ppcnt_a_frame_check_sequence_errors_get, - }, - { - .str = "a_alignment_errors", - .getter = mlxsw_reg_ppcnt_a_alignment_errors_get, - }, - { - .str = "a_octets_transmitted_ok", - .getter = mlxsw_reg_ppcnt_a_octets_transmitted_ok_get, - }, - { - .str = "a_octets_received_ok", - .getter = mlxsw_reg_ppcnt_a_octets_received_ok_get, - }, - { - .str = "a_multicast_frames_xmitted_ok", - .getter = mlxsw_reg_ppcnt_a_multicast_frames_xmitted_ok_get, - }, - { - .str = "a_broadcast_frames_xmitted_ok", - .getter = mlxsw_reg_ppcnt_a_broadcast_frames_xmitted_ok_get, - }, - { - .str = "a_multicast_frames_received_ok", - .getter = mlxsw_reg_ppcnt_a_multicast_frames_received_ok_get, - }, - { - .str = "a_broadcast_frames_received_ok", - .getter = mlxsw_reg_ppcnt_a_broadcast_frames_received_ok_get, - }, - { - .str = "a_in_range_length_errors", - .getter = mlxsw_reg_ppcnt_a_in_range_length_errors_get, - }, - { - .str = "a_out_of_range_length_field", - .getter = mlxsw_reg_ppcnt_a_out_of_range_length_field_get, - }, - { - .str = "a_frame_too_long_errors", - .getter = mlxsw_reg_ppcnt_a_frame_too_long_errors_get, - }, - { - .str = "a_symbol_error_during_carrier", - .getter = mlxsw_reg_ppcnt_a_symbol_error_during_carrier_get, - }, - { - .str = "a_mac_control_frames_transmitted", - .getter = mlxsw_reg_ppcnt_a_mac_control_frames_transmitted_get, - }, - { - .str = "a_mac_control_frames_received", - .getter = mlxsw_reg_ppcnt_a_mac_control_frames_received_get, - }, - { - .str = "a_unsupported_opcodes_received", - .getter = mlxsw_reg_ppcnt_a_unsupported_opcodes_received_get, - }, - { - .str = "a_pause_mac_ctrl_frames_received", - .getter = mlxsw_reg_ppcnt_a_pause_mac_ctrl_frames_received_get, - }, - { - .str = "a_pause_mac_ctrl_frames_xmitted", - .getter = mlxsw_reg_ppcnt_a_pause_mac_ctrl_frames_transmitted_get, - }, -}; - -#define MLXSW_SX_PORT_HW_STATS_LEN ARRAY_SIZE(mlxsw_sx_port_hw_stats) - -static void mlxsw_sx_port_get_strings(struct net_device *dev, - u32 stringset, u8 *data) -{ - u8 *p = data; - int i; - - switch (stringset) { - case ETH_SS_STATS: - for (i = 0; i < MLXSW_SX_PORT_HW_STATS_LEN; i++) { - memcpy(p, mlxsw_sx_port_hw_stats[i].str, - ETH_GSTRING_LEN); - p += ETH_GSTRING_LEN; - } - break; - } -} - -static void mlxsw_sx_port_get_stats(struct net_device *dev, - struct ethtool_stats *stats, u64 *data) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char ppcnt_pl[MLXSW_REG_PPCNT_LEN]; - int i; - int err; - - mlxsw_reg_ppcnt_pack(ppcnt_pl, mlxsw_sx_port->local_port, - MLXSW_REG_PPCNT_IEEE_8023_CNT, 0); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(ppcnt), ppcnt_pl); - for (i = 0; i < MLXSW_SX_PORT_HW_STATS_LEN; i++) - data[i] = !err ? mlxsw_sx_port_hw_stats[i].getter(ppcnt_pl) : 0; -} - -static int mlxsw_sx_port_get_sset_count(struct net_device *dev, int sset) -{ - switch (sset) { - case ETH_SS_STATS: - return MLXSW_SX_PORT_HW_STATS_LEN; - default: - return -EOPNOTSUPP; - } -} - -struct mlxsw_sx_port_link_mode { - u32 mask; - u32 supported; - u32 advertised; - u32 speed; -}; - -static const struct mlxsw_sx_port_link_mode mlxsw_sx_port_link_mode[] = { - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_SGMII | - MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX, - .supported = SUPPORTED_1000baseKX_Full, - .advertised = ADVERTISED_1000baseKX_Full, - .speed = 1000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CX4 | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4, - .supported = SUPPORTED_10000baseKX4_Full, - .advertised = ADVERTISED_10000baseKX4_Full, - .speed = 10000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_ER_LR, - .supported = SUPPORTED_10000baseKR_Full, - .advertised = ADVERTISED_10000baseKR_Full, - .speed = 10000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4, - .supported = SUPPORTED_40000baseCR4_Full, - .advertised = ADVERTISED_40000baseCR4_Full, - .speed = 40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4, - .supported = SUPPORTED_40000baseKR4_Full, - .advertised = ADVERTISED_40000baseKR4_Full, - .speed = 40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4, - .supported = SUPPORTED_40000baseSR4_Full, - .advertised = ADVERTISED_40000baseSR4_Full, - .speed = 40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_40GBASE_LR4_ER4, - .supported = SUPPORTED_40000baseLR4_Full, - .advertised = ADVERTISED_40000baseLR4_Full, - .speed = 40000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_25GBASE_CR | - MLXSW_REG_PTYS_ETH_SPEED_25GBASE_KR | - MLXSW_REG_PTYS_ETH_SPEED_25GBASE_SR, - .speed = 25000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_50GBASE_CR2 | - MLXSW_REG_PTYS_ETH_SPEED_50GBASE_KR2, - .speed = 50000, - }, - { - .mask = MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_LR4_ER4, - .speed = 100000, - }, -}; - -#define MLXSW_SX_PORT_LINK_MODE_LEN ARRAY_SIZE(mlxsw_sx_port_link_mode) -#define MLXSW_SX_PORT_BASE_SPEED 10000 /* Mb/s */ - -static u32 mlxsw_sx_from_ptys_supported_port(u32 ptys_eth_proto) -{ - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4 | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_SGMII)) - return SUPPORTED_FIBRE; - - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4 | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_1000BASE_KX)) - return SUPPORTED_Backplane; - return 0; -} - -static u32 mlxsw_sx_from_ptys_supported_link(u32 ptys_eth_proto) -{ - u32 modes = 0; - int i; - - for (i = 0; i < MLXSW_SX_PORT_LINK_MODE_LEN; i++) { - if (ptys_eth_proto & mlxsw_sx_port_link_mode[i].mask) - modes |= mlxsw_sx_port_link_mode[i].supported; - } - return modes; -} - -static u32 mlxsw_sx_from_ptys_advert_link(u32 ptys_eth_proto) -{ - u32 modes = 0; - int i; - - for (i = 0; i < MLXSW_SX_PORT_LINK_MODE_LEN; i++) { - if (ptys_eth_proto & mlxsw_sx_port_link_mode[i].mask) - modes |= mlxsw_sx_port_link_mode[i].advertised; - } - return modes; -} - -static void mlxsw_sx_from_ptys_speed_duplex(bool carrier_ok, u32 ptys_eth_proto, - struct ethtool_link_ksettings *cmd) -{ - u32 speed = SPEED_UNKNOWN; - u8 duplex = DUPLEX_UNKNOWN; - int i; - - if (!carrier_ok) - goto out; - - for (i = 0; i < MLXSW_SX_PORT_LINK_MODE_LEN; i++) { - if (ptys_eth_proto & mlxsw_sx_port_link_mode[i].mask) { - speed = mlxsw_sx_port_link_mode[i].speed; - duplex = DUPLEX_FULL; - break; - } - } -out: - cmd->base.speed = speed; - cmd->base.duplex = duplex; -} - -static u8 mlxsw_sx_port_connector_port(u32 ptys_eth_proto) -{ - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_SR | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_SR4 | - MLXSW_REG_PTYS_ETH_SPEED_SGMII)) - return PORT_FIBRE; - - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_CR | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_CR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_CR4)) - return PORT_DA; - - if (ptys_eth_proto & (MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KR | - MLXSW_REG_PTYS_ETH_SPEED_10GBASE_KX4 | - MLXSW_REG_PTYS_ETH_SPEED_40GBASE_KR4 | - MLXSW_REG_PTYS_ETH_SPEED_100GBASE_KR4)) - return PORT_NONE; - - return PORT_OTHER; -} - -static int -mlxsw_sx_port_get_link_ksettings(struct net_device *dev, - struct ethtool_link_ksettings *cmd) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - u32 eth_proto_cap; - u32 eth_proto_admin; - u32 eth_proto_oper; - u32 supported, advertising, lp_advertising; - int err; - - mlxsw_reg_ptys_eth_pack(ptys_pl, mlxsw_sx_port->local_port, 0, false); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(ptys), ptys_pl); - if (err) { - netdev_err(dev, "Failed to get proto"); - return err; - } - mlxsw_reg_ptys_eth_unpack(ptys_pl, ð_proto_cap, - ð_proto_admin, ð_proto_oper); - - supported = mlxsw_sx_from_ptys_supported_port(eth_proto_cap) | - mlxsw_sx_from_ptys_supported_link(eth_proto_cap) | - SUPPORTED_Pause | SUPPORTED_Asym_Pause; - advertising = mlxsw_sx_from_ptys_advert_link(eth_proto_admin); - mlxsw_sx_from_ptys_speed_duplex(netif_carrier_ok(dev), - eth_proto_oper, cmd); - - eth_proto_oper = eth_proto_oper ? eth_proto_oper : eth_proto_cap; - cmd->base.port = mlxsw_sx_port_connector_port(eth_proto_oper); - lp_advertising = mlxsw_sx_from_ptys_advert_link(eth_proto_oper); - - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, - supported); - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, - advertising); - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.lp_advertising, - lp_advertising); - - return 0; -} - -static u32 mlxsw_sx_to_ptys_advert_link(u32 advertising) -{ - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SX_PORT_LINK_MODE_LEN; i++) { - if (advertising & mlxsw_sx_port_link_mode[i].advertised) - ptys_proto |= mlxsw_sx_port_link_mode[i].mask; - } - return ptys_proto; -} - -static u32 mlxsw_sx_to_ptys_speed(u32 speed) -{ - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SX_PORT_LINK_MODE_LEN; i++) { - if (speed == mlxsw_sx_port_link_mode[i].speed) - ptys_proto |= mlxsw_sx_port_link_mode[i].mask; - } - return ptys_proto; -} - -static u32 mlxsw_sx_to_ptys_upper_speed(u32 upper_speed) -{ - u32 ptys_proto = 0; - int i; - - for (i = 0; i < MLXSW_SX_PORT_LINK_MODE_LEN; i++) { - if (mlxsw_sx_port_link_mode[i].speed <= upper_speed) - ptys_proto |= mlxsw_sx_port_link_mode[i].mask; - } - return ptys_proto; -} - -static int -mlxsw_sx_port_set_link_ksettings(struct net_device *dev, - const struct ethtool_link_ksettings *cmd) -{ - struct mlxsw_sx_port *mlxsw_sx_port = netdev_priv(dev); - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - u32 speed; - u32 eth_proto_new; - u32 eth_proto_cap; - u32 eth_proto_admin; - u32 advertising; - bool is_up; - int err; - - speed = cmd->base.speed; - - ethtool_convert_link_mode_to_legacy_u32(&advertising, - cmd->link_modes.advertising); - - eth_proto_new = cmd->base.autoneg == AUTONEG_ENABLE ? - mlxsw_sx_to_ptys_advert_link(advertising) : - mlxsw_sx_to_ptys_speed(speed); - - mlxsw_reg_ptys_eth_pack(ptys_pl, mlxsw_sx_port->local_port, 0, false); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(ptys), ptys_pl); - if (err) { - netdev_err(dev, "Failed to get proto"); - return err; - } - mlxsw_reg_ptys_eth_unpack(ptys_pl, ð_proto_cap, ð_proto_admin, - NULL); - - eth_proto_new = eth_proto_new & eth_proto_cap; - if (!eth_proto_new) { - netdev_err(dev, "Not supported proto admin requested"); - return -EINVAL; - } - if (eth_proto_new == eth_proto_admin) - return 0; - - mlxsw_reg_ptys_eth_pack(ptys_pl, mlxsw_sx_port->local_port, - eth_proto_new, true); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(ptys), ptys_pl); - if (err) { - netdev_err(dev, "Failed to set proto admin"); - return err; - } - - err = mlxsw_sx_port_oper_status_get(mlxsw_sx_port, &is_up); - if (err) { - netdev_err(dev, "Failed to get oper status"); - return err; - } - if (!is_up) - return 0; - - err = mlxsw_sx_port_admin_status_set(mlxsw_sx_port, false); - if (err) { - netdev_err(dev, "Failed to set admin status"); - return err; - } - - err = mlxsw_sx_port_admin_status_set(mlxsw_sx_port, true); - if (err) { - netdev_err(dev, "Failed to set admin status"); - return err; - } - - return 0; -} - -static const struct ethtool_ops mlxsw_sx_port_ethtool_ops = { - .get_drvinfo = mlxsw_sx_port_get_drvinfo, - .get_link = ethtool_op_get_link, - .get_strings = mlxsw_sx_port_get_strings, - .get_ethtool_stats = mlxsw_sx_port_get_stats, - .get_sset_count = mlxsw_sx_port_get_sset_count, - .get_link_ksettings = mlxsw_sx_port_get_link_ksettings, - .set_link_ksettings = mlxsw_sx_port_set_link_ksettings, -}; - -static int mlxsw_sx_hw_id_get(struct mlxsw_sx *mlxsw_sx) -{ - char spad_pl[MLXSW_REG_SPAD_LEN] = {0}; - int err; - - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(spad), spad_pl); - if (err) - return err; - mlxsw_reg_spad_base_mac_memcpy_from(spad_pl, mlxsw_sx->hw_id); - return 0; -} - -static int mlxsw_sx_port_dev_addr_get(struct mlxsw_sx_port *mlxsw_sx_port) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - struct net_device *dev = mlxsw_sx_port->dev; - char ppad_pl[MLXSW_REG_PPAD_LEN]; - int err; - - mlxsw_reg_ppad_pack(ppad_pl, false, 0); - err = mlxsw_reg_query(mlxsw_sx->core, MLXSW_REG(ppad), ppad_pl); - if (err) - return err; - mlxsw_reg_ppad_mac_memcpy_from(ppad_pl, dev->dev_addr); - /* The last byte value in base mac address is guaranteed - * to be such it does not overflow when adding local_port - * value. - */ - dev->dev_addr[ETH_ALEN - 1] += mlxsw_sx_port->local_port; - return 0; -} - -static int mlxsw_sx_port_stp_state_set(struct mlxsw_sx_port *mlxsw_sx_port, - u16 vid, enum mlxsw_reg_spms_state state) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char *spms_pl; - int err; - - spms_pl = kmalloc(MLXSW_REG_SPMS_LEN, GFP_KERNEL); - if (!spms_pl) - return -ENOMEM; - mlxsw_reg_spms_pack(spms_pl, mlxsw_sx_port->local_port); - mlxsw_reg_spms_vid_pack(spms_pl, vid, state); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(spms), spms_pl); - kfree(spms_pl); - return err; -} - -static int mlxsw_sx_port_ib_speed_set(struct mlxsw_sx_port *mlxsw_sx_port, - u16 speed, u16 width) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - - mlxsw_reg_ptys_ib_pack(ptys_pl, mlxsw_sx_port->local_port, speed, - width); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(ptys), ptys_pl); -} - -static int -mlxsw_sx_port_speed_by_width_set(struct mlxsw_sx_port *mlxsw_sx_port, u8 width) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - u32 upper_speed = MLXSW_SX_PORT_BASE_SPEED * width; - char ptys_pl[MLXSW_REG_PTYS_LEN]; - u32 eth_proto_admin; - - eth_proto_admin = mlxsw_sx_to_ptys_upper_speed(upper_speed); - mlxsw_reg_ptys_eth_pack(ptys_pl, mlxsw_sx_port->local_port, - eth_proto_admin, true); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(ptys), ptys_pl); -} - -static int -mlxsw_sx_port_mac_learning_mode_set(struct mlxsw_sx_port *mlxsw_sx_port, - enum mlxsw_reg_spmlr_learn_mode mode) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_sx_port->mlxsw_sx; - char spmlr_pl[MLXSW_REG_SPMLR_LEN]; - - mlxsw_reg_spmlr_pack(spmlr_pl, mlxsw_sx_port->local_port, mode); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(spmlr), spmlr_pl); -} - -static int __mlxsw_sx_port_eth_create(struct mlxsw_sx *mlxsw_sx, u8 local_port, - u8 module, u8 width) -{ - struct mlxsw_sx_port *mlxsw_sx_port; - struct net_device *dev; - int err; - - dev = alloc_etherdev(sizeof(struct mlxsw_sx_port)); - if (!dev) - return -ENOMEM; - SET_NETDEV_DEV(dev, mlxsw_sx->bus_info->dev); - dev_net_set(dev, mlxsw_core_net(mlxsw_sx->core)); - mlxsw_sx_port = netdev_priv(dev); - mlxsw_sx_port->dev = dev; - mlxsw_sx_port->mlxsw_sx = mlxsw_sx; - mlxsw_sx_port->local_port = local_port; - mlxsw_sx_port->mapping.module = module; - - mlxsw_sx_port->pcpu_stats = - netdev_alloc_pcpu_stats(struct mlxsw_sx_port_pcpu_stats); - if (!mlxsw_sx_port->pcpu_stats) { - err = -ENOMEM; - goto err_alloc_stats; - } - - dev->netdev_ops = &mlxsw_sx_port_netdev_ops; - dev->ethtool_ops = &mlxsw_sx_port_ethtool_ops; - - err = mlxsw_sx_port_dev_addr_get(mlxsw_sx_port); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Unable to get port mac address\n", - mlxsw_sx_port->local_port); - goto err_dev_addr_get; - } - - netif_carrier_off(dev); - - dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_LLTX | NETIF_F_SG | - NETIF_F_VLAN_CHALLENGED; - - dev->min_mtu = 0; - dev->max_mtu = ETH_MAX_MTU; - - /* Each packet needs to have a Tx header (metadata) on top all other - * headers. - */ - dev->needed_headroom = MLXSW_TXHDR_LEN; - - err = mlxsw_sx_port_system_port_mapping_set(mlxsw_sx_port); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set system port mapping\n", - mlxsw_sx_port->local_port); - goto err_port_system_port_mapping_set; - } - - err = mlxsw_sx_port_swid_set(mlxsw_sx_port, 0); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set SWID\n", - mlxsw_sx_port->local_port); - goto err_port_swid_set; - } - - err = mlxsw_sx_port_speed_by_width_set(mlxsw_sx_port, width); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set speed\n", - mlxsw_sx_port->local_port); - goto err_port_speed_set; - } - - err = mlxsw_sx_port_mtu_eth_set(mlxsw_sx_port, ETH_DATA_LEN); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set MTU\n", - mlxsw_sx_port->local_port); - goto err_port_mtu_set; - } - - err = mlxsw_sx_port_admin_status_set(mlxsw_sx_port, false); - if (err) - goto err_port_admin_status_set; - - err = mlxsw_sx_port_stp_state_set(mlxsw_sx_port, - MLXSW_PORT_DEFAULT_VID, - MLXSW_REG_SPMS_STATE_FORWARDING); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set STP state\n", - mlxsw_sx_port->local_port); - goto err_port_stp_state_set; - } - - err = mlxsw_sx_port_mac_learning_mode_set(mlxsw_sx_port, - MLXSW_REG_SPMLR_LEARN_MODE_DISABLE); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set MAC learning mode\n", - mlxsw_sx_port->local_port); - goto err_port_mac_learning_mode_set; - } - - err = register_netdev(dev); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to register netdev\n", - mlxsw_sx_port->local_port); - goto err_register_netdev; - } - - mlxsw_core_port_eth_set(mlxsw_sx->core, mlxsw_sx_port->local_port, - mlxsw_sx_port, dev); - mlxsw_sx->ports[local_port] = mlxsw_sx_port; - return 0; - -err_register_netdev: -err_port_mac_learning_mode_set: -err_port_stp_state_set: -err_port_admin_status_set: -err_port_mtu_set: -err_port_speed_set: - mlxsw_sx_port_swid_set(mlxsw_sx_port, MLXSW_PORT_SWID_DISABLED_PORT); -err_port_swid_set: -err_port_system_port_mapping_set: -err_dev_addr_get: - free_percpu(mlxsw_sx_port->pcpu_stats); -err_alloc_stats: - free_netdev(dev); - return err; -} - -static int mlxsw_sx_port_eth_create(struct mlxsw_sx *mlxsw_sx, u8 local_port, - u8 module, u8 width) -{ - int err; - - err = mlxsw_core_port_init(mlxsw_sx->core, local_port, - module + 1, false, 0, false, 0, - mlxsw_sx->hw_id, sizeof(mlxsw_sx->hw_id)); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to init core port\n", - local_port); - return err; - } - err = __mlxsw_sx_port_eth_create(mlxsw_sx, local_port, module, width); - if (err) - goto err_port_create; - - return 0; - -err_port_create: - mlxsw_core_port_fini(mlxsw_sx->core, local_port); - return err; -} - -static void __mlxsw_sx_port_eth_remove(struct mlxsw_sx *mlxsw_sx, u8 local_port) -{ - struct mlxsw_sx_port *mlxsw_sx_port = mlxsw_sx->ports[local_port]; - - mlxsw_core_port_clear(mlxsw_sx->core, local_port, mlxsw_sx); - unregister_netdev(mlxsw_sx_port->dev); /* This calls ndo_stop */ - mlxsw_sx->ports[local_port] = NULL; - mlxsw_sx_port_swid_set(mlxsw_sx_port, MLXSW_PORT_SWID_DISABLED_PORT); - free_percpu(mlxsw_sx_port->pcpu_stats); - free_netdev(mlxsw_sx_port->dev); -} - -static bool mlxsw_sx_port_created(struct mlxsw_sx *mlxsw_sx, u8 local_port) -{ - return mlxsw_sx->ports[local_port] != NULL; -} - -static int __mlxsw_sx_port_ib_create(struct mlxsw_sx *mlxsw_sx, u8 local_port, - u8 module, u8 width) -{ - struct mlxsw_sx_port *mlxsw_sx_port; - int err; - - mlxsw_sx_port = kzalloc(sizeof(*mlxsw_sx_port), GFP_KERNEL); - if (!mlxsw_sx_port) - return -ENOMEM; - mlxsw_sx_port->mlxsw_sx = mlxsw_sx; - mlxsw_sx_port->local_port = local_port; - mlxsw_sx_port->mapping.module = module; - - err = mlxsw_sx_port_system_port_mapping_set(mlxsw_sx_port); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set system port mapping\n", - mlxsw_sx_port->local_port); - goto err_port_system_port_mapping_set; - } - - /* Adding port to Infiniband swid (1) */ - err = mlxsw_sx_port_swid_set(mlxsw_sx_port, 1); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set SWID\n", - mlxsw_sx_port->local_port); - goto err_port_swid_set; - } - - /* Expose the IB port number as it's front panel name */ - err = mlxsw_sx_port_ib_port_set(mlxsw_sx_port, module + 1); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set IB port\n", - mlxsw_sx_port->local_port); - goto err_port_ib_set; - } - - /* Supports all speeds from SDR to FDR (bitmask) and support bus width - * of 1x, 2x and 4x (3 bits bitmask) - */ - err = mlxsw_sx_port_ib_speed_set(mlxsw_sx_port, - MLXSW_REG_PTYS_IB_SPEED_EDR - 1, - BIT(3) - 1); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set speed\n", - mlxsw_sx_port->local_port); - goto err_port_speed_set; - } - - /* Change to the maximum MTU the device supports, the SMA will take - * care of the active MTU - */ - err = mlxsw_sx_port_mtu_ib_set(mlxsw_sx_port, MLXSW_IB_DEFAULT_MTU); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to set MTU\n", - mlxsw_sx_port->local_port); - goto err_port_mtu_set; - } - - err = mlxsw_sx_port_admin_status_set(mlxsw_sx_port, true); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Port %d: Failed to change admin state to UP\n", - mlxsw_sx_port->local_port); - goto err_port_admin_set; - } - - mlxsw_core_port_ib_set(mlxsw_sx->core, mlxsw_sx_port->local_port, - mlxsw_sx_port); - mlxsw_sx->ports[local_port] = mlxsw_sx_port; - return 0; - -err_port_admin_set: -err_port_mtu_set: -err_port_speed_set: -err_port_ib_set: - mlxsw_sx_port_swid_set(mlxsw_sx_port, MLXSW_PORT_SWID_DISABLED_PORT); -err_port_swid_set: -err_port_system_port_mapping_set: - kfree(mlxsw_sx_port); - return err; -} - -static void __mlxsw_sx_port_ib_remove(struct mlxsw_sx *mlxsw_sx, u8 local_port) -{ - struct mlxsw_sx_port *mlxsw_sx_port = mlxsw_sx->ports[local_port]; - - mlxsw_core_port_clear(mlxsw_sx->core, local_port, mlxsw_sx); - mlxsw_sx->ports[local_port] = NULL; - mlxsw_sx_port_admin_status_set(mlxsw_sx_port, false); - mlxsw_sx_port_swid_set(mlxsw_sx_port, MLXSW_PORT_SWID_DISABLED_PORT); - kfree(mlxsw_sx_port); -} - -static void __mlxsw_sx_port_remove(struct mlxsw_sx *mlxsw_sx, u8 local_port) -{ - enum devlink_port_type port_type = - mlxsw_core_port_type_get(mlxsw_sx->core, local_port); - - if (port_type == DEVLINK_PORT_TYPE_ETH) - __mlxsw_sx_port_eth_remove(mlxsw_sx, local_port); - else if (port_type == DEVLINK_PORT_TYPE_IB) - __mlxsw_sx_port_ib_remove(mlxsw_sx, local_port); -} - -static void mlxsw_sx_port_remove(struct mlxsw_sx *mlxsw_sx, u8 local_port) -{ - __mlxsw_sx_port_remove(mlxsw_sx, local_port); - mlxsw_core_port_fini(mlxsw_sx->core, local_port); -} - -static void mlxsw_sx_ports_remove(struct mlxsw_sx *mlxsw_sx) -{ - int i; - - for (i = 1; i < mlxsw_core_max_ports(mlxsw_sx->core); i++) - if (mlxsw_sx_port_created(mlxsw_sx, i)) - mlxsw_sx_port_remove(mlxsw_sx, i); - kfree(mlxsw_sx->ports); - mlxsw_sx->ports = NULL; -} - -static int mlxsw_sx_ports_create(struct mlxsw_sx *mlxsw_sx) -{ - unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sx->core); - size_t alloc_size; - u8 module, width; - int i; - int err; - - alloc_size = sizeof(struct mlxsw_sx_port *) * max_ports; - mlxsw_sx->ports = kzalloc(alloc_size, GFP_KERNEL); - if (!mlxsw_sx->ports) - return -ENOMEM; - - for (i = 1; i < max_ports; i++) { - err = mlxsw_sx_port_module_info_get(mlxsw_sx, i, &module, - &width); - if (err) - goto err_port_module_info_get; - if (!width) - continue; - err = mlxsw_sx_port_eth_create(mlxsw_sx, i, module, width); - if (err) - goto err_port_create; - } - return 0; - -err_port_create: -err_port_module_info_get: - for (i--; i >= 1; i--) - if (mlxsw_sx_port_created(mlxsw_sx, i)) - mlxsw_sx_port_remove(mlxsw_sx, i); - kfree(mlxsw_sx->ports); - mlxsw_sx->ports = NULL; - return err; -} - -static void mlxsw_sx_pude_eth_event_func(struct mlxsw_sx_port *mlxsw_sx_port, - enum mlxsw_reg_pude_oper_status status) -{ - if (status == MLXSW_PORT_OPER_STATUS_UP) { - netdev_info(mlxsw_sx_port->dev, "link up\n"); - netif_carrier_on(mlxsw_sx_port->dev); - } else { - netdev_info(mlxsw_sx_port->dev, "link down\n"); - netif_carrier_off(mlxsw_sx_port->dev); - } -} - -static void mlxsw_sx_pude_ib_event_func(struct mlxsw_sx_port *mlxsw_sx_port, - enum mlxsw_reg_pude_oper_status status) -{ - if (status == MLXSW_PORT_OPER_STATUS_UP) - pr_info("ib link for port %d - up\n", - mlxsw_sx_port->mapping.module + 1); - else - pr_info("ib link for port %d - down\n", - mlxsw_sx_port->mapping.module + 1); -} - -static void mlxsw_sx_pude_event_func(const struct mlxsw_reg_info *reg, - char *pude_pl, void *priv) -{ - struct mlxsw_sx *mlxsw_sx = priv; - struct mlxsw_sx_port *mlxsw_sx_port; - enum mlxsw_reg_pude_oper_status status; - enum devlink_port_type port_type; - u8 local_port; - - local_port = mlxsw_reg_pude_local_port_get(pude_pl); - mlxsw_sx_port = mlxsw_sx->ports[local_port]; - if (!mlxsw_sx_port) { - dev_warn(mlxsw_sx->bus_info->dev, "Port %d: Link event received for non-existent port\n", - local_port); - return; - } - - status = mlxsw_reg_pude_oper_status_get(pude_pl); - port_type = mlxsw_core_port_type_get(mlxsw_sx->core, local_port); - if (port_type == DEVLINK_PORT_TYPE_ETH) - mlxsw_sx_pude_eth_event_func(mlxsw_sx_port, status); - else if (port_type == DEVLINK_PORT_TYPE_IB) - mlxsw_sx_pude_ib_event_func(mlxsw_sx_port, status); -} - -static void mlxsw_sx_rx_listener_func(struct sk_buff *skb, u8 local_port, - void *priv) -{ - struct mlxsw_sx *mlxsw_sx = priv; - struct mlxsw_sx_port *mlxsw_sx_port = mlxsw_sx->ports[local_port]; - struct mlxsw_sx_port_pcpu_stats *pcpu_stats; - - if (unlikely(!mlxsw_sx_port)) { - dev_warn_ratelimited(mlxsw_sx->bus_info->dev, "Port %d: skb received for non-existent port\n", - local_port); - return; - } - - skb->dev = mlxsw_sx_port->dev; - - pcpu_stats = this_cpu_ptr(mlxsw_sx_port->pcpu_stats); - u64_stats_update_begin(&pcpu_stats->syncp); - pcpu_stats->rx_packets++; - pcpu_stats->rx_bytes += skb->len; - u64_stats_update_end(&pcpu_stats->syncp); - - skb->protocol = eth_type_trans(skb, skb->dev); - netif_receive_skb(skb); -} - -static int mlxsw_sx_port_type_set(struct mlxsw_core *mlxsw_core, u8 local_port, - enum devlink_port_type new_type) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_core_driver_priv(mlxsw_core); - u8 module, width; - int err; - - if (!mlxsw_sx->ports || !mlxsw_sx->ports[local_port]) { - dev_err(mlxsw_sx->bus_info->dev, "Port number \"%d\" does not exist\n", - local_port); - return -EINVAL; - } - - if (new_type == DEVLINK_PORT_TYPE_AUTO) - return -EOPNOTSUPP; - - __mlxsw_sx_port_remove(mlxsw_sx, local_port); - err = mlxsw_sx_port_module_info_get(mlxsw_sx, local_port, &module, - &width); - if (err) - goto err_port_module_info_get; - - if (new_type == DEVLINK_PORT_TYPE_ETH) - err = __mlxsw_sx_port_eth_create(mlxsw_sx, local_port, module, - width); - else if (new_type == DEVLINK_PORT_TYPE_IB) - err = __mlxsw_sx_port_ib_create(mlxsw_sx, local_port, module, - width); - -err_port_module_info_get: - return err; -} - -enum { - MLXSW_REG_HTGT_TRAP_GROUP_SX2_RX = 1, - MLXSW_REG_HTGT_TRAP_GROUP_SX2_CTRL = 2, -}; - -#define MLXSW_SX_RXL(_trap_id) \ - MLXSW_RXL(mlxsw_sx_rx_listener_func, _trap_id, TRAP_TO_CPU, \ - false, SX2_RX, FORWARD) - -static const struct mlxsw_listener mlxsw_sx_listener[] = { - MLXSW_EVENTL(mlxsw_sx_pude_event_func, PUDE, EMAD), - MLXSW_SX_RXL(FDB_MC), - MLXSW_SX_RXL(STP), - MLXSW_SX_RXL(LACP), - MLXSW_SX_RXL(EAPOL), - MLXSW_SX_RXL(LLDP), - MLXSW_SX_RXL(MMRP), - MLXSW_SX_RXL(MVRP), - MLXSW_SX_RXL(RPVST), - MLXSW_SX_RXL(DHCP), - MLXSW_SX_RXL(IGMP_QUERY), - MLXSW_SX_RXL(IGMP_V1_REPORT), - MLXSW_SX_RXL(IGMP_V2_REPORT), - MLXSW_SX_RXL(IGMP_V2_LEAVE), - MLXSW_SX_RXL(IGMP_V3_REPORT), -}; - -static int mlxsw_sx_traps_init(struct mlxsw_sx *mlxsw_sx) -{ - char htgt_pl[MLXSW_REG_HTGT_LEN]; - int i; - int err; - - mlxsw_reg_htgt_pack(htgt_pl, MLXSW_REG_HTGT_TRAP_GROUP_SX2_RX, - MLXSW_REG_HTGT_INVALID_POLICER, - MLXSW_REG_HTGT_DEFAULT_PRIORITY, - MLXSW_REG_HTGT_DEFAULT_TC); - mlxsw_reg_htgt_local_path_rdq_set(htgt_pl, - MLXSW_REG_HTGT_LOCAL_PATH_RDQ_SX2_RX); - - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(htgt), htgt_pl); - if (err) - return err; - - mlxsw_reg_htgt_pack(htgt_pl, MLXSW_REG_HTGT_TRAP_GROUP_SX2_CTRL, - MLXSW_REG_HTGT_INVALID_POLICER, - MLXSW_REG_HTGT_DEFAULT_PRIORITY, - MLXSW_REG_HTGT_DEFAULT_TC); - mlxsw_reg_htgt_local_path_rdq_set(htgt_pl, - MLXSW_REG_HTGT_LOCAL_PATH_RDQ_SX2_CTRL); - - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(htgt), htgt_pl); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(mlxsw_sx_listener); i++) { - err = mlxsw_core_trap_register(mlxsw_sx->core, - &mlxsw_sx_listener[i], - mlxsw_sx); - if (err) - goto err_listener_register; - - } - return 0; - -err_listener_register: - for (i--; i >= 0; i--) { - mlxsw_core_trap_unregister(mlxsw_sx->core, - &mlxsw_sx_listener[i], - mlxsw_sx); - } - return err; -} - -static void mlxsw_sx_traps_fini(struct mlxsw_sx *mlxsw_sx) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(mlxsw_sx_listener); i++) { - mlxsw_core_trap_unregister(mlxsw_sx->core, - &mlxsw_sx_listener[i], - mlxsw_sx); - } -} - -static int mlxsw_sx_flood_init(struct mlxsw_sx *mlxsw_sx) -{ - char sfgc_pl[MLXSW_REG_SFGC_LEN]; - char sgcr_pl[MLXSW_REG_SGCR_LEN]; - char *sftr_pl; - int err; - - /* Configure a flooding table, which includes only CPU port. */ - sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL); - if (!sftr_pl) - return -ENOMEM; - mlxsw_reg_sftr_pack(sftr_pl, 0, 0, MLXSW_REG_SFGC_TABLE_TYPE_SINGLE, 0, - MLXSW_PORT_CPU_PORT, true); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sftr), sftr_pl); - kfree(sftr_pl); - if (err) - return err; - - /* Flood different packet types using the flooding table. */ - mlxsw_reg_sfgc_pack(sfgc_pl, - MLXSW_REG_SFGC_TYPE_UNKNOWN_UNICAST, - MLXSW_REG_SFGC_BRIDGE_TYPE_1Q_FID, - MLXSW_REG_SFGC_TABLE_TYPE_SINGLE, - 0); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sfgc), sfgc_pl); - if (err) - return err; - - mlxsw_reg_sfgc_pack(sfgc_pl, - MLXSW_REG_SFGC_TYPE_BROADCAST, - MLXSW_REG_SFGC_BRIDGE_TYPE_1Q_FID, - MLXSW_REG_SFGC_TABLE_TYPE_SINGLE, - 0); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sfgc), sfgc_pl); - if (err) - return err; - - mlxsw_reg_sfgc_pack(sfgc_pl, - MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_NON_IP, - MLXSW_REG_SFGC_BRIDGE_TYPE_1Q_FID, - MLXSW_REG_SFGC_TABLE_TYPE_SINGLE, - 0); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sfgc), sfgc_pl); - if (err) - return err; - - mlxsw_reg_sfgc_pack(sfgc_pl, - MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV6, - MLXSW_REG_SFGC_BRIDGE_TYPE_1Q_FID, - MLXSW_REG_SFGC_TABLE_TYPE_SINGLE, - 0); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sfgc), sfgc_pl); - if (err) - return err; - - mlxsw_reg_sfgc_pack(sfgc_pl, - MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV4, - MLXSW_REG_SFGC_BRIDGE_TYPE_1Q_FID, - MLXSW_REG_SFGC_TABLE_TYPE_SINGLE, - 0); - err = mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sfgc), sfgc_pl); - if (err) - return err; - - mlxsw_reg_sgcr_pack(sgcr_pl, true); - return mlxsw_reg_write(mlxsw_sx->core, MLXSW_REG(sgcr), sgcr_pl); -} - -static int mlxsw_sx_basic_trap_groups_set(struct mlxsw_core *mlxsw_core) -{ - char htgt_pl[MLXSW_REG_HTGT_LEN]; - - mlxsw_reg_htgt_pack(htgt_pl, MLXSW_REG_HTGT_TRAP_GROUP_EMAD, - MLXSW_REG_HTGT_INVALID_POLICER, - MLXSW_REG_HTGT_DEFAULT_PRIORITY, - MLXSW_REG_HTGT_DEFAULT_TC); - mlxsw_reg_htgt_swid_set(htgt_pl, MLXSW_PORT_SWID_ALL_SWIDS); - mlxsw_reg_htgt_local_path_rdq_set(htgt_pl, - MLXSW_REG_HTGT_LOCAL_PATH_RDQ_SX2_EMAD); - return mlxsw_reg_write(mlxsw_core, MLXSW_REG(htgt), htgt_pl); -} - -static int mlxsw_sx_init(struct mlxsw_core *mlxsw_core, - const struct mlxsw_bus_info *mlxsw_bus_info, - struct netlink_ext_ack *extack) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_core_driver_priv(mlxsw_core); - int err; - - mlxsw_sx->core = mlxsw_core; - mlxsw_sx->bus_info = mlxsw_bus_info; - - err = mlxsw_sx_hw_id_get(mlxsw_sx); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Failed to get switch HW ID\n"); - return err; - } - - err = mlxsw_sx_ports_create(mlxsw_sx); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Failed to create ports\n"); - return err; - } - - err = mlxsw_sx_traps_init(mlxsw_sx); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Failed to set traps\n"); - goto err_listener_register; - } - - err = mlxsw_sx_flood_init(mlxsw_sx); - if (err) { - dev_err(mlxsw_sx->bus_info->dev, "Failed to initialize flood tables\n"); - goto err_flood_init; - } - - return 0; - -err_flood_init: - mlxsw_sx_traps_fini(mlxsw_sx); -err_listener_register: - mlxsw_sx_ports_remove(mlxsw_sx); - return err; -} - -static void mlxsw_sx_fini(struct mlxsw_core *mlxsw_core) -{ - struct mlxsw_sx *mlxsw_sx = mlxsw_core_driver_priv(mlxsw_core); - - mlxsw_sx_traps_fini(mlxsw_sx); - mlxsw_sx_ports_remove(mlxsw_sx); -} - -static const struct mlxsw_config_profile mlxsw_sx_config_profile = { - .used_max_vepa_channels = 1, - .max_vepa_channels = 0, - .used_max_mid = 1, - .max_mid = 7000, - .used_max_pgt = 1, - .max_pgt = 0, - .used_max_system_port = 1, - .max_system_port = 48000, - .used_max_vlan_groups = 1, - .max_vlan_groups = 127, - .used_max_regions = 1, - .max_regions = 400, - .used_flood_tables = 1, - .max_flood_tables = 2, - .max_vid_flood_tables = 1, - .used_flood_mode = 1, - .flood_mode = 3, - .used_max_ib_mc = 1, - .max_ib_mc = 6, - .used_max_pkey = 1, - .max_pkey = 0, - .swid_config = { - { - .used_type = 1, - .type = MLXSW_PORT_SWID_TYPE_ETH, - }, - { - .used_type = 1, - .type = MLXSW_PORT_SWID_TYPE_IB, - } - }, -}; - -static struct mlxsw_driver mlxsw_sx_driver = { - .kind = mlxsw_sx_driver_name, - .priv_size = sizeof(struct mlxsw_sx), - .init = mlxsw_sx_init, - .fini = mlxsw_sx_fini, - .basic_trap_groups_set = mlxsw_sx_basic_trap_groups_set, - .txhdr_construct = mlxsw_sx_txhdr_construct, - .txhdr_len = MLXSW_TXHDR_LEN, - .profile = &mlxsw_sx_config_profile, - .port_type_set = mlxsw_sx_port_type_set, -}; - -static const struct pci_device_id mlxsw_sx_pci_id_table[] = { - {PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SWITCHX2), 0}, - {0, }, -}; - -static struct pci_driver mlxsw_sx_pci_driver = { - .name = mlxsw_sx_driver_name, - .id_table = mlxsw_sx_pci_id_table, -}; - -static int __init mlxsw_sx_module_init(void) -{ - int err; - - err = mlxsw_core_driver_register(&mlxsw_sx_driver); - if (err) - return err; - - err = mlxsw_pci_driver_register(&mlxsw_sx_pci_driver); - if (err) - goto err_pci_driver_register; - - return 0; - -err_pci_driver_register: - mlxsw_core_driver_unregister(&mlxsw_sx_driver); - return err; -} - -static void __exit mlxsw_sx_module_exit(void) -{ - mlxsw_pci_driver_unregister(&mlxsw_sx_pci_driver); - mlxsw_core_driver_unregister(&mlxsw_sx_driver); -} - -module_init(mlxsw_sx_module_init); -module_exit(mlxsw_sx_module_exit); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>"); -MODULE_DESCRIPTION("Mellanox SwitchX-2 driver"); -MODULE_DEVICE_TABLE(pci, mlxsw_sx_pci_id_table); diff --git a/drivers/net/ethernet/micrel/ks8842.c b/drivers/net/ethernet/micrel/ks8842.c index caa251d0e381..b27713906d3a 100644 --- a/drivers/net/ethernet/micrel/ks8842.c +++ b/drivers/net/ethernet/micrel/ks8842.c @@ -1135,6 +1135,10 @@ static int ks8842_probe(struct platform_device *pdev) unsigned i; iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!iomem) { + dev_err(&pdev->dev, "Invalid resource\n"); + return -EINVAL; + } if (!request_mem_region(iomem->start, resource_size(iomem), DRV_NAME)) goto err_mem_region; diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 13eef6e9bd2d..831518466de2 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -1022,30 +1022,23 @@ static int ks8851_mdio_write(struct mii_bus *bus, int phy_id, int reg, u16 val) * * Read and check the TX/RX memory selftest information. */ -static int ks8851_read_selftest(struct ks8851_net *ks) +static void ks8851_read_selftest(struct ks8851_net *ks) { unsigned both_done = MBIR_TXMBF | MBIR_RXMBF; - int ret = 0; unsigned rd; rd = ks8851_rdreg16(ks, KS_MBIR); if ((rd & both_done) != both_done) { netdev_warn(ks->netdev, "Memory selftest not finished\n"); - return 0; + return; } - if (rd & MBIR_TXMBFA) { + if (rd & MBIR_TXMBFA) netdev_err(ks->netdev, "TX memory selftest fail\n"); - ret |= 1; - } - if (rd & MBIR_RXMBFA) { + if (rd & MBIR_RXMBFA) netdev_err(ks->netdev, "RX memory selftest fail\n"); - ret |= 2; - } - - return 0; } /* driver bus management functions */ diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c index 9ed264ed7070..7945eb5e2fe8 100644 --- a/drivers/net/ethernet/micrel/ksz884x.c +++ b/drivers/net/ethernet/micrel/ksz884x.c @@ -25,6 +25,7 @@ #include <linux/crc32.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/micrel_phy.h> /* DMA Registers */ @@ -271,84 +272,15 @@ #define KS884X_PHY_CTRL_OFFSET 0x00 -/* Mode Control Register */ -#define PHY_REG_CTRL 0 - -#define PHY_RESET 0x8000 -#define PHY_LOOPBACK 0x4000 -#define PHY_SPEED_100MBIT 0x2000 -#define PHY_AUTO_NEG_ENABLE 0x1000 -#define PHY_POWER_DOWN 0x0800 -#define PHY_MII_DISABLE 0x0400 -#define PHY_AUTO_NEG_RESTART 0x0200 -#define PHY_FULL_DUPLEX 0x0100 -#define PHY_COLLISION_TEST 0x0080 -#define PHY_HP_MDIX 0x0020 -#define PHY_FORCE_MDIX 0x0010 -#define PHY_AUTO_MDIX_DISABLE 0x0008 -#define PHY_REMOTE_FAULT_DISABLE 0x0004 -#define PHY_TRANSMIT_DISABLE 0x0002 -#define PHY_LED_DISABLE 0x0001 - #define KS884X_PHY_STATUS_OFFSET 0x02 -/* Mode Status Register */ -#define PHY_REG_STATUS 1 - -#define PHY_100BT4_CAPABLE 0x8000 -#define PHY_100BTX_FD_CAPABLE 0x4000 -#define PHY_100BTX_CAPABLE 0x2000 -#define PHY_10BT_FD_CAPABLE 0x1000 -#define PHY_10BT_CAPABLE 0x0800 -#define PHY_MII_SUPPRESS_CAPABLE 0x0040 -#define PHY_AUTO_NEG_ACKNOWLEDGE 0x0020 -#define PHY_REMOTE_FAULT 0x0010 -#define PHY_AUTO_NEG_CAPABLE 0x0008 -#define PHY_LINK_STATUS 0x0004 -#define PHY_JABBER_DETECT 0x0002 -#define PHY_EXTENDED_CAPABILITY 0x0001 - #define KS884X_PHY_ID_1_OFFSET 0x04 #define KS884X_PHY_ID_2_OFFSET 0x06 -/* PHY Identifier Registers */ -#define PHY_REG_ID_1 2 -#define PHY_REG_ID_2 3 - #define KS884X_PHY_AUTO_NEG_OFFSET 0x08 -/* Auto-Negotiation Advertisement Register */ -#define PHY_REG_AUTO_NEGOTIATION 4 - -#define PHY_AUTO_NEG_NEXT_PAGE 0x8000 -#define PHY_AUTO_NEG_REMOTE_FAULT 0x2000 -/* Not supported. */ -#define PHY_AUTO_NEG_ASYM_PAUSE 0x0800 -#define PHY_AUTO_NEG_SYM_PAUSE 0x0400 -#define PHY_AUTO_NEG_100BT4 0x0200 -#define PHY_AUTO_NEG_100BTX_FD 0x0100 -#define PHY_AUTO_NEG_100BTX 0x0080 -#define PHY_AUTO_NEG_10BT_FD 0x0040 -#define PHY_AUTO_NEG_10BT 0x0020 -#define PHY_AUTO_NEG_SELECTOR 0x001F -#define PHY_AUTO_NEG_802_3 0x0001 - -#define PHY_AUTO_NEG_PAUSE (PHY_AUTO_NEG_SYM_PAUSE | PHY_AUTO_NEG_ASYM_PAUSE) - #define KS884X_PHY_REMOTE_CAP_OFFSET 0x0A -/* Auto-Negotiation Link Partner Ability Register */ -#define PHY_REG_REMOTE_CAPABILITY 5 - -#define PHY_REMOTE_NEXT_PAGE 0x8000 -#define PHY_REMOTE_ACKNOWLEDGE 0x4000 -#define PHY_REMOTE_REMOTE_FAULT 0x2000 -#define PHY_REMOTE_SYM_PAUSE 0x0400 -#define PHY_REMOTE_100BTX_FD 0x0100 -#define PHY_REMOTE_100BTX 0x0080 -#define PHY_REMOTE_10BT_FD 0x0040 -#define PHY_REMOTE_10BT 0x0020 - /* P1VCT */ #define KS884X_P1VCT_P 0x04F0 #define KS884X_P1PHYCTRL_P 0x04F2 @@ -2153,7 +2085,7 @@ static void sw_cfg_broad_storm(struct ksz_hw *hw, u8 percent) } /** - * sw_get_board_storm - get broadcast storm threshold + * sw_get_broad_storm - get broadcast storm threshold * @hw: The hardware instance. * @percent: Buffer to store the broadcast storm threshold percentage. * @@ -2886,15 +2818,6 @@ static void sw_block_addr(struct ksz_hw *hw) } } -#define PHY_LINK_SUPPORT \ - (PHY_AUTO_NEG_ASYM_PAUSE | \ - PHY_AUTO_NEG_SYM_PAUSE | \ - PHY_AUTO_NEG_100BT4 | \ - PHY_AUTO_NEG_100BTX_FD | \ - PHY_AUTO_NEG_100BTX | \ - PHY_AUTO_NEG_10BT_FD | \ - PHY_AUTO_NEG_10BT) - static inline void hw_r_phy_ctrl(struct ksz_hw *hw, int phy, u16 *data) { *data = readw(hw->io + phy + KS884X_PHY_CTRL_OFFSET); @@ -2973,7 +2896,7 @@ static void hw_r_phy(struct ksz_hw *hw, int port, u16 reg, u16 *val) } /** - * port_w_phy - write data to PHY register + * hw_w_phy - write data to PHY register * @hw: The hardware instance. * @port: Port to write. * @reg: PHY register to write. @@ -3238,16 +3161,18 @@ static void determine_flow_ctrl(struct ksz_hw *hw, struct ksz_port *port, rx = tx = 0; if (port->force_link) rx = tx = 1; - if (remote & PHY_AUTO_NEG_SYM_PAUSE) { - if (local & PHY_AUTO_NEG_SYM_PAUSE) { + if (remote & LPA_PAUSE_CAP) { + if (local & ADVERTISE_PAUSE_CAP) { rx = tx = 1; - } else if ((remote & PHY_AUTO_NEG_ASYM_PAUSE) && - (local & PHY_AUTO_NEG_PAUSE) == - PHY_AUTO_NEG_ASYM_PAUSE) { + } else if ((remote & LPA_PAUSE_ASYM) && + (local & + (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM)) == + ADVERTISE_PAUSE_ASYM) { tx = 1; } - } else if (remote & PHY_AUTO_NEG_ASYM_PAUSE) { - if ((local & PHY_AUTO_NEG_PAUSE) == PHY_AUTO_NEG_PAUSE) + } else if (remote & LPA_PAUSE_ASYM) { + if ((local & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM)) + == (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM)) rx = 1; } if (!hw->ksz_switch) @@ -3428,16 +3353,16 @@ static void port_force_link_speed(struct ksz_port *port) phy = KS884X_PHY_1_CTRL_OFFSET + p * PHY_CTRL_INTERVAL; hw_r_phy_ctrl(hw, phy, &data); - data &= ~PHY_AUTO_NEG_ENABLE; + data &= ~BMCR_ANENABLE; if (10 == port->speed) - data &= ~PHY_SPEED_100MBIT; + data &= ~BMCR_SPEED100; else if (100 == port->speed) - data |= PHY_SPEED_100MBIT; + data |= BMCR_SPEED100; if (1 == port->duplex) - data &= ~PHY_FULL_DUPLEX; + data &= ~BMCR_FULLDPLX; else if (2 == port->duplex) - data |= PHY_FULL_DUPLEX; + data |= BMCR_FULLDPLX; hw_w_phy_ctrl(hw, phy, data); } } @@ -4782,7 +4707,7 @@ static void transmit_cleanup(struct dev_info *hw_priv, int normal) } /** - * transmit_done - transmit done processing + * tx_done - transmit done processing * @hw_priv: Network device. * * This routine is called when the transmit interrupt is triggered, indicating diff --git a/drivers/net/ethernet/microchip/Kconfig b/drivers/net/ethernet/microchip/Kconfig index d0f6dfe0dcf3..d54aa164c4e9 100644 --- a/drivers/net/ethernet/microchip/Kconfig +++ b/drivers/net/ethernet/microchip/Kconfig @@ -54,4 +54,6 @@ config LAN743X To compile this driver as a module, choose M here. The module will be called lan743x. +source "drivers/net/ethernet/microchip/sparx5/Kconfig" + endif # NET_VENDOR_MICROCHIP diff --git a/drivers/net/ethernet/microchip/Makefile b/drivers/net/ethernet/microchip/Makefile index da603540ca57..c77dc0379bfd 100644 --- a/drivers/net/ethernet/microchip/Makefile +++ b/drivers/net/ethernet/microchip/Makefile @@ -8,3 +8,5 @@ obj-$(CONFIG_ENCX24J600) += encx24j600.o encx24j600-regmap.o obj-$(CONFIG_LAN743X) += lan743x.o lan743x-objs := lan743x_main.o lan743x_ethtool.o lan743x_ptp.o + +obj-$(CONFIG_SPARX5_SWITCH) += sparx5/ diff --git a/drivers/net/ethernet/microchip/encx24j600.c b/drivers/net/ethernet/microchip/encx24j600.c index 3658c4ae3c37..ee921a99e439 100644 --- a/drivers/net/ethernet/microchip/encx24j600.c +++ b/drivers/net/ethernet/microchip/encx24j600.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/** +/* * Microchip ENCX24J600 ethernet driver * * Copyright (C) 2015 Gridpoint diff --git a/drivers/net/ethernet/microchip/encx24j600_hw.h b/drivers/net/ethernet/microchip/encx24j600_hw.h index f604a260ede7..fac61a8fbd02 100644 --- a/drivers/net/ethernet/microchip/encx24j600_hw.h +++ b/drivers/net/ethernet/microchip/encx24j600_hw.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/** +/* * encx24j600_hw.h: Register definitions * */ diff --git a/drivers/net/ethernet/microchip/sparx5/Kconfig b/drivers/net/ethernet/microchip/sparx5/Kconfig new file mode 100644 index 000000000000..a80419d8d4b5 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/Kconfig @@ -0,0 +1,9 @@ +config SPARX5_SWITCH + tristate "Sparx5 switch driver" + depends on NET_SWITCHDEV + depends on HAS_IOMEM + select PHYLINK + select PHY_SPARX5_SERDES + select RESET_CONTROLLER + help + This driver supports the Sparx5 network switch device. diff --git a/drivers/net/ethernet/microchip/sparx5/Makefile b/drivers/net/ethernet/microchip/sparx5/Makefile new file mode 100644 index 000000000000..faa8f07a6b75 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Microchip Sparx5 network device drivers. +# + +obj-$(CONFIG_SPARX5_SWITCH) += sparx5-switch.o + +sparx5-switch-objs := sparx5_main.o sparx5_packet.o \ + sparx5_netdev.o sparx5_phylink.o sparx5_port.o sparx5_mactable.o sparx5_vlan.o \ + sparx5_switchdev.o sparx5_calendar.o sparx5_ethtool.o diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c b/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c new file mode 100644 index 000000000000..76a8bb596aec --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include <linux/module.h> +#include <linux/device.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" + +/* QSYS calendar information */ +#define SPX5_PORTS_PER_CALREG 10 /* Ports mapped in a calendar register */ +#define SPX5_CALBITS_PER_PORT 3 /* Bit per port in calendar register */ + +/* DSM calendar information */ +#define SPX5_DSM_CAL_LEN 64 +#define SPX5_DSM_CAL_EMPTY 0xFFFF +#define SPX5_DSM_CAL_MAX_DEVS_PER_TAXI 13 +#define SPX5_DSM_CAL_TAXIS 8 +#define SPX5_DSM_CAL_BW_LOSS 553 + +#define SPX5_TAXI_PORT_MAX 70 + +#define SPEED_12500 12500 + +/* Maps from taxis to port numbers */ +static u32 sparx5_taxi_ports[SPX5_DSM_CAL_TAXIS][SPX5_DSM_CAL_MAX_DEVS_PER_TAXI] = { + {57, 12, 0, 1, 2, 16, 17, 18, 19, 20, 21, 22, 23}, + {58, 13, 3, 4, 5, 24, 25, 26, 27, 28, 29, 30, 31}, + {59, 14, 6, 7, 8, 32, 33, 34, 35, 36, 37, 38, 39}, + {60, 15, 9, 10, 11, 40, 41, 42, 43, 44, 45, 46, 47}, + {61, 48, 49, 50, 99, 99, 99, 99, 99, 99, 99, 99, 99}, + {62, 51, 52, 53, 99, 99, 99, 99, 99, 99, 99, 99, 99}, + {56, 63, 54, 55, 99, 99, 99, 99, 99, 99, 99, 99, 99}, + {64, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99}, +}; + +struct sparx5_calendar_data { + u32 schedule[SPX5_DSM_CAL_LEN]; + u32 avg_dist[SPX5_DSM_CAL_MAX_DEVS_PER_TAXI]; + u32 taxi_ports[SPX5_DSM_CAL_MAX_DEVS_PER_TAXI]; + u32 taxi_speeds[SPX5_DSM_CAL_MAX_DEVS_PER_TAXI]; + u32 dev_slots[SPX5_DSM_CAL_MAX_DEVS_PER_TAXI]; + u32 new_slots[SPX5_DSM_CAL_LEN]; + u32 temp_sched[SPX5_DSM_CAL_LEN]; + u32 indices[SPX5_DSM_CAL_LEN]; + u32 short_list[SPX5_DSM_CAL_LEN]; + u32 long_list[SPX5_DSM_CAL_LEN]; +}; + +static u32 sparx5_target_bandwidth(struct sparx5 *sparx5) +{ + switch (sparx5->target_ct) { + case SPX5_TARGET_CT_7546: + case SPX5_TARGET_CT_7546TSN: + return 65000; + case SPX5_TARGET_CT_7549: + case SPX5_TARGET_CT_7549TSN: + return 91000; + case SPX5_TARGET_CT_7552: + case SPX5_TARGET_CT_7552TSN: + return 129000; + case SPX5_TARGET_CT_7556: + case SPX5_TARGET_CT_7556TSN: + return 161000; + case SPX5_TARGET_CT_7558: + case SPX5_TARGET_CT_7558TSN: + return 201000; + default: + return 0; + } +} + +/* This is used in calendar configuration */ +enum sparx5_cal_bw { + SPX5_CAL_SPEED_NONE = 0, + SPX5_CAL_SPEED_1G = 1, + SPX5_CAL_SPEED_2G5 = 2, + SPX5_CAL_SPEED_5G = 3, + SPX5_CAL_SPEED_10G = 4, + SPX5_CAL_SPEED_25G = 5, + SPX5_CAL_SPEED_0G5 = 6, + SPX5_CAL_SPEED_12G5 = 7 +}; + +static u32 sparx5_clk_to_bandwidth(enum sparx5_core_clockfreq cclock) +{ + switch (cclock) { + case SPX5_CORE_CLOCK_250MHZ: return 83000; /* 250000 / 3 */ + case SPX5_CORE_CLOCK_500MHZ: return 166000; /* 500000 / 3 */ + case SPX5_CORE_CLOCK_625MHZ: return 208000; /* 625000 / 3 */ + default: return 0; + } + return 0; +} + +static u32 sparx5_cal_speed_to_value(enum sparx5_cal_bw speed) +{ + switch (speed) { + case SPX5_CAL_SPEED_1G: return 1000; + case SPX5_CAL_SPEED_2G5: return 2500; + case SPX5_CAL_SPEED_5G: return 5000; + case SPX5_CAL_SPEED_10G: return 10000; + case SPX5_CAL_SPEED_25G: return 25000; + case SPX5_CAL_SPEED_0G5: return 500; + case SPX5_CAL_SPEED_12G5: return 12500; + default: return 0; + } +} + +static u32 sparx5_bandwidth_to_calendar(u32 bw) +{ + switch (bw) { + case SPEED_10: return SPX5_CAL_SPEED_0G5; + case SPEED_100: return SPX5_CAL_SPEED_0G5; + case SPEED_1000: return SPX5_CAL_SPEED_1G; + case SPEED_2500: return SPX5_CAL_SPEED_2G5; + case SPEED_5000: return SPX5_CAL_SPEED_5G; + case SPEED_10000: return SPX5_CAL_SPEED_10G; + case SPEED_12500: return SPX5_CAL_SPEED_12G5; + case SPEED_25000: return SPX5_CAL_SPEED_25G; + case SPEED_UNKNOWN: return SPX5_CAL_SPEED_1G; + default: return SPX5_CAL_SPEED_NONE; + } +} + +static enum sparx5_cal_bw sparx5_get_port_cal_speed(struct sparx5 *sparx5, + u32 portno) +{ + struct sparx5_port *port; + + if (portno >= SPX5_PORTS) { + /* Internal ports */ + if (portno == SPX5_PORT_CPU_0 || portno == SPX5_PORT_CPU_1) { + /* Equals 1.25G */ + return SPX5_CAL_SPEED_2G5; + } else if (portno == SPX5_PORT_VD0) { + /* IPMC only idle BW */ + return SPX5_CAL_SPEED_NONE; + } else if (portno == SPX5_PORT_VD1) { + /* OAM only idle BW */ + return SPX5_CAL_SPEED_NONE; + } else if (portno == SPX5_PORT_VD2) { + /* IPinIP gets only idle BW */ + return SPX5_CAL_SPEED_NONE; + } + /* not in port map */ + return SPX5_CAL_SPEED_NONE; + } + /* Front ports - may be used */ + port = sparx5->ports[portno]; + if (!port) + return SPX5_CAL_SPEED_NONE; + return sparx5_bandwidth_to_calendar(port->conf.bandwidth); +} + +/* Auto configure the QSYS calendar based on port configuration */ +int sparx5_config_auto_calendar(struct sparx5 *sparx5) +{ + u32 cal[7], value, idx, portno; + u32 max_core_bw; + u32 total_bw = 0, used_port_bw = 0; + int err = 0; + enum sparx5_cal_bw spd; + + memset(cal, 0, sizeof(cal)); + + max_core_bw = sparx5_clk_to_bandwidth(sparx5->coreclock); + if (max_core_bw == 0) { + dev_err(sparx5->dev, "Core clock not supported"); + return -EINVAL; + } + + /* Setup the calendar with the bandwidth to each port */ + for (portno = 0; portno < SPX5_PORTS_ALL; portno++) { + u64 reg, offset, this_bw; + + spd = sparx5_get_port_cal_speed(sparx5, portno); + if (spd == SPX5_CAL_SPEED_NONE) + continue; + + this_bw = sparx5_cal_speed_to_value(spd); + if (portno < SPX5_PORTS) + used_port_bw += this_bw; + else + /* Internal ports are granted half the value */ + this_bw = this_bw / 2; + total_bw += this_bw; + reg = portno; + offset = do_div(reg, SPX5_PORTS_PER_CALREG); + cal[reg] |= spd << (offset * SPX5_CALBITS_PER_PORT); + } + + if (used_port_bw > sparx5_target_bandwidth(sparx5)) { + dev_err(sparx5->dev, + "Port BW %u above target BW %u\n", + used_port_bw, sparx5_target_bandwidth(sparx5)); + return -EINVAL; + } + + if (total_bw > max_core_bw) { + dev_err(sparx5->dev, + "Total BW %u above switch core BW %u\n", + total_bw, max_core_bw); + return -EINVAL; + } + + /* Halt the calendar while changing it */ + spx5_rmw(QSYS_CAL_CTRL_CAL_MODE_SET(10), + QSYS_CAL_CTRL_CAL_MODE, + sparx5, QSYS_CAL_CTRL); + + /* Assign port bandwidth to auto calendar */ + for (idx = 0; idx < ARRAY_SIZE(cal); idx++) + spx5_wr(cal[idx], sparx5, QSYS_CAL_AUTO(idx)); + + /* Increase grant rate of all ports to account for + * core clock ppm deviations + */ + spx5_rmw(QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE_SET(671), /* 672->671 */ + QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE, + sparx5, + QSYS_CAL_CTRL); + + /* Grant idle usage to VD 0-2 */ + for (idx = 2; idx < 5; idx++) + spx5_wr(HSCH_OUTB_SHARE_ENA_OUTB_SHARE_ENA_SET(12), + sparx5, + HSCH_OUTB_SHARE_ENA(idx)); + + /* Enable Auto mode */ + spx5_rmw(QSYS_CAL_CTRL_CAL_MODE_SET(8), + QSYS_CAL_CTRL_CAL_MODE, + sparx5, QSYS_CAL_CTRL); + + /* Verify successful calendar config */ + value = spx5_rd(sparx5, QSYS_CAL_CTRL); + if (QSYS_CAL_CTRL_CAL_AUTO_ERROR_GET(value)) { + dev_err(sparx5->dev, "QSYS calendar error\n"); + err = -EINVAL; + } + return err; +} + +static u32 sparx5_dsm_exb_gcd(u32 a, u32 b) +{ + if (b == 0) + return a; + return sparx5_dsm_exb_gcd(b, a % b); +} + +static u32 sparx5_dsm_cal_len(u32 *cal) +{ + u32 idx = 0, len = 0; + + while (idx < SPX5_DSM_CAL_LEN) { + if (cal[idx] != SPX5_DSM_CAL_EMPTY) + len++; + idx++; + } + return len; +} + +static u32 sparx5_dsm_cp_cal(u32 *sched) +{ + u32 idx = 0, tmp; + + while (idx < SPX5_DSM_CAL_LEN) { + if (sched[idx] != SPX5_DSM_CAL_EMPTY) { + tmp = sched[idx]; + sched[idx] = SPX5_DSM_CAL_EMPTY; + return tmp; + } + idx++; + } + return SPX5_DSM_CAL_EMPTY; +} + +static int sparx5_dsm_calendar_calc(struct sparx5 *sparx5, u32 taxi, + struct sparx5_calendar_data *data) +{ + bool slow_mode; + u32 gcd, idx, sum, min, factor; + u32 num_of_slots, slot_spd, empty_slots; + u32 taxi_bw, clk_period_ps; + + clk_period_ps = sparx5_clk_period(sparx5->coreclock); + taxi_bw = 128 * 1000000 / clk_period_ps; + slow_mode = !!(clk_period_ps > 2000); + memcpy(data->taxi_ports, &sparx5_taxi_ports[taxi], + sizeof(data->taxi_ports)); + + for (idx = 0; idx < SPX5_DSM_CAL_LEN; idx++) { + data->new_slots[idx] = SPX5_DSM_CAL_EMPTY; + data->schedule[idx] = SPX5_DSM_CAL_EMPTY; + data->temp_sched[idx] = SPX5_DSM_CAL_EMPTY; + } + /* Default empty calendar */ + data->schedule[0] = SPX5_DSM_CAL_MAX_DEVS_PER_TAXI; + + /* Map ports to taxi positions */ + for (idx = 0; idx < SPX5_DSM_CAL_MAX_DEVS_PER_TAXI; idx++) { + u32 portno = data->taxi_ports[idx]; + + if (portno < SPX5_TAXI_PORT_MAX) { + data->taxi_speeds[idx] = sparx5_cal_speed_to_value + (sparx5_get_port_cal_speed(sparx5, portno)); + } else { + data->taxi_speeds[idx] = 0; + } + } + + sum = 0; + min = 25000; + for (idx = 0; idx < ARRAY_SIZE(data->taxi_speeds); idx++) { + u32 jdx; + + sum += data->taxi_speeds[idx]; + if (data->taxi_speeds[idx] && data->taxi_speeds[idx] < min) + min = data->taxi_speeds[idx]; + gcd = min; + for (jdx = 0; jdx < ARRAY_SIZE(data->taxi_speeds); jdx++) + gcd = sparx5_dsm_exb_gcd(gcd, data->taxi_speeds[jdx]); + } + if (sum == 0) /* Empty calendar */ + return 0; + /* Make room for overhead traffic */ + factor = 100 * 100 * 1000 / (100 * 100 - SPX5_DSM_CAL_BW_LOSS); + + if (sum * factor > (taxi_bw * 1000)) { + dev_err(sparx5->dev, + "Taxi %u, Requested BW %u above available BW %u\n", + taxi, sum, taxi_bw); + return -EINVAL; + } + for (idx = 0; idx < 4; idx++) { + u32 raw_spd; + + if (idx == 0) + raw_spd = gcd / 5; + else if (idx == 1) + raw_spd = gcd / 2; + else if (idx == 2) + raw_spd = gcd; + else + raw_spd = min; + slot_spd = raw_spd * factor / 1000; + num_of_slots = taxi_bw / slot_spd; + if (num_of_slots <= 64) + break; + } + + num_of_slots = num_of_slots > 64 ? 64 : num_of_slots; + slot_spd = taxi_bw / num_of_slots; + + sum = 0; + for (idx = 0; idx < ARRAY_SIZE(data->taxi_speeds); idx++) { + u32 spd = data->taxi_speeds[idx]; + u32 adjusted_speed = data->taxi_speeds[idx] * factor / 1000; + + if (adjusted_speed > 0) { + data->avg_dist[idx] = (128 * 1000000 * 10) / + (adjusted_speed * clk_period_ps); + } else { + data->avg_dist[idx] = -1; + } + data->dev_slots[idx] = ((spd * factor / slot_spd) + 999) / 1000; + if (spd != 25000 && (spd != 10000 || !slow_mode)) { + if (num_of_slots < (5 * data->dev_slots[idx])) { + dev_err(sparx5->dev, + "Taxi %u, speed %u, Low slot sep.\n", + taxi, spd); + return -EINVAL; + } + } + sum += data->dev_slots[idx]; + if (sum > num_of_slots) { + dev_err(sparx5->dev, + "Taxi %u with overhead factor %u\n", + taxi, factor); + return -EINVAL; + } + } + + empty_slots = num_of_slots - sum; + + for (idx = 0; idx < empty_slots; idx++) + data->schedule[idx] = SPX5_DSM_CAL_MAX_DEVS_PER_TAXI; + + for (idx = 1; idx < num_of_slots; idx++) { + u32 indices_len = 0; + u32 slot, jdx, kdx, ts; + s32 cnt; + u32 num_of_old_slots, num_of_new_slots, tgt_score; + + for (slot = 0; slot < ARRAY_SIZE(data->dev_slots); slot++) { + if (data->dev_slots[slot] == idx) { + data->indices[indices_len] = slot; + indices_len++; + } + } + if (indices_len == 0) + continue; + kdx = 0; + for (slot = 0; slot < idx; slot++) { + for (jdx = 0; jdx < indices_len; jdx++, kdx++) + data->new_slots[kdx] = data->indices[jdx]; + } + + for (slot = 0; slot < SPX5_DSM_CAL_LEN; slot++) { + if (data->schedule[slot] == SPX5_DSM_CAL_EMPTY) + break; + } + + num_of_old_slots = slot; + num_of_new_slots = kdx; + cnt = 0; + ts = 0; + + if (num_of_new_slots > num_of_old_slots) { + memcpy(data->short_list, data->schedule, + sizeof(data->short_list)); + memcpy(data->long_list, data->new_slots, + sizeof(data->long_list)); + tgt_score = 100000 * num_of_old_slots / + num_of_new_slots; + } else { + memcpy(data->short_list, data->new_slots, + sizeof(data->short_list)); + memcpy(data->long_list, data->schedule, + sizeof(data->long_list)); + tgt_score = 100000 * num_of_new_slots / + num_of_old_slots; + } + + while (sparx5_dsm_cal_len(data->short_list) > 0 || + sparx5_dsm_cal_len(data->long_list) > 0) { + u32 act = 0; + + if (sparx5_dsm_cal_len(data->short_list) > 0) { + data->temp_sched[ts] = + sparx5_dsm_cp_cal(data->short_list); + ts++; + cnt += 100000; + act = 1; + } + while (sparx5_dsm_cal_len(data->long_list) > 0 && + cnt > 0) { + data->temp_sched[ts] = + sparx5_dsm_cp_cal(data->long_list); + ts++; + cnt -= tgt_score; + act = 1; + } + if (act == 0) { + dev_err(sparx5->dev, + "Error in DSM calendar calculation\n"); + return -EINVAL; + } + } + + for (slot = 0; slot < SPX5_DSM_CAL_LEN; slot++) { + if (data->temp_sched[slot] == SPX5_DSM_CAL_EMPTY) + break; + } + for (slot = 0; slot < SPX5_DSM_CAL_LEN; slot++) { + data->schedule[slot] = data->temp_sched[slot]; + data->temp_sched[slot] = SPX5_DSM_CAL_EMPTY; + data->new_slots[slot] = SPX5_DSM_CAL_EMPTY; + } + } + return 0; +} + +static int sparx5_dsm_calendar_check(struct sparx5 *sparx5, + struct sparx5_calendar_data *data) +{ + u32 num_of_slots, idx, port; + int cnt, max_dist; + u32 slot_indices[SPX5_DSM_CAL_LEN], distances[SPX5_DSM_CAL_LEN]; + u32 cal_length = sparx5_dsm_cal_len(data->schedule); + + for (port = 0; port < SPX5_DSM_CAL_MAX_DEVS_PER_TAXI; port++) { + num_of_slots = 0; + max_dist = data->avg_dist[port]; + for (idx = 0; idx < SPX5_DSM_CAL_LEN; idx++) { + slot_indices[idx] = SPX5_DSM_CAL_EMPTY; + distances[idx] = SPX5_DSM_CAL_EMPTY; + } + + for (idx = 0; idx < cal_length; idx++) { + if (data->schedule[idx] == port) { + slot_indices[num_of_slots] = idx; + num_of_slots++; + } + } + + slot_indices[num_of_slots] = slot_indices[0] + cal_length; + + for (idx = 0; idx < num_of_slots; idx++) { + distances[idx] = (slot_indices[idx + 1] - + slot_indices[idx]) * 10; + } + + for (idx = 0; idx < num_of_slots; idx++) { + u32 jdx, kdx; + + cnt = distances[idx] - max_dist; + if (cnt < 0) + cnt = -cnt; + kdx = 0; + for (jdx = (idx + 1) % num_of_slots; + jdx != idx; + jdx = (jdx + 1) % num_of_slots, kdx++) { + cnt = cnt + distances[jdx] - max_dist; + if (cnt < 0) + cnt = -cnt; + if (cnt > max_dist) + goto check_err; + } + } + } + return 0; +check_err: + dev_err(sparx5->dev, + "Port %u: distance %u above limit %d\n", + port, cnt, max_dist); + return -EINVAL; +} + +static int sparx5_dsm_calendar_update(struct sparx5 *sparx5, u32 taxi, + struct sparx5_calendar_data *data) +{ + u32 idx; + u32 cal_len = sparx5_dsm_cal_len(data->schedule), len; + + spx5_wr(DSM_TAXI_CAL_CFG_CAL_PGM_ENA_SET(1), + sparx5, + DSM_TAXI_CAL_CFG(taxi)); + for (idx = 0; idx < cal_len; idx++) { + spx5_rmw(DSM_TAXI_CAL_CFG_CAL_IDX_SET(idx), + DSM_TAXI_CAL_CFG_CAL_IDX, + sparx5, + DSM_TAXI_CAL_CFG(taxi)); + spx5_rmw(DSM_TAXI_CAL_CFG_CAL_PGM_VAL_SET(data->schedule[idx]), + DSM_TAXI_CAL_CFG_CAL_PGM_VAL, + sparx5, + DSM_TAXI_CAL_CFG(taxi)); + } + spx5_wr(DSM_TAXI_CAL_CFG_CAL_PGM_ENA_SET(0), + sparx5, + DSM_TAXI_CAL_CFG(taxi)); + len = DSM_TAXI_CAL_CFG_CAL_CUR_LEN_GET(spx5_rd(sparx5, + DSM_TAXI_CAL_CFG(taxi))); + if (len != cal_len - 1) + goto update_err; + return 0; +update_err: + dev_err(sparx5->dev, "Incorrect calendar length: %u\n", len); + return -EINVAL; +} + +/* Configure the DSM calendar based on port configuration */ +int sparx5_config_dsm_calendar(struct sparx5 *sparx5) +{ + int taxi; + struct sparx5_calendar_data *data; + int err = 0; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + for (taxi = 0; taxi < SPX5_DSM_CAL_TAXIS; ++taxi) { + err = sparx5_dsm_calendar_calc(sparx5, taxi, data); + if (err) { + dev_err(sparx5->dev, "DSM calendar calculation failed\n"); + goto cal_out; + } + err = sparx5_dsm_calendar_check(sparx5, data); + if (err) { + dev_err(sparx5->dev, "DSM calendar check failed\n"); + goto cal_out; + } + err = sparx5_dsm_calendar_update(sparx5, taxi, data); + if (err) { + dev_err(sparx5->dev, "DSM calendar update failed\n"); + goto cal_out; + } + } +cal_out: + kfree(data); + return err; +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c new file mode 100644 index 000000000000..59783fc46a7b --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c @@ -0,0 +1,1227 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include <linux/ethtool.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" +#include "sparx5_port.h" + +/* Index of ANA_AC port counters */ +#define SPX5_PORT_POLICER_DROPS 0 + +/* Add a potentially wrapping 32 bit value to a 64 bit counter */ +static void sparx5_update_counter(u64 *cnt, u32 val) +{ + if (val < (*cnt & U32_MAX)) + *cnt += (u64)1 << 32; /* value has wrapped */ + *cnt = (*cnt & ~(u64)U32_MAX) + val; +} + +enum sparx5_stats_entry { + spx5_stats_rx_symbol_err_cnt = 0, + spx5_stats_pmac_rx_symbol_err_cnt = 1, + spx5_stats_tx_uc_cnt = 2, + spx5_stats_pmac_tx_uc_cnt = 3, + spx5_stats_tx_mc_cnt = 4, + spx5_stats_tx_bc_cnt = 5, + spx5_stats_tx_backoff1_cnt = 6, + spx5_stats_tx_multi_coll_cnt = 7, + spx5_stats_rx_uc_cnt = 8, + spx5_stats_pmac_rx_uc_cnt = 9, + spx5_stats_rx_mc_cnt = 10, + spx5_stats_rx_bc_cnt = 11, + spx5_stats_rx_crc_err_cnt = 12, + spx5_stats_pmac_rx_crc_err_cnt = 13, + spx5_stats_rx_alignment_lost_cnt = 14, + spx5_stats_pmac_rx_alignment_lost_cnt = 15, + spx5_stats_tx_ok_bytes_cnt = 16, + spx5_stats_pmac_tx_ok_bytes_cnt = 17, + spx5_stats_tx_defer_cnt = 18, + spx5_stats_tx_late_coll_cnt = 19, + spx5_stats_tx_xcoll_cnt = 20, + spx5_stats_tx_csense_cnt = 21, + spx5_stats_rx_ok_bytes_cnt = 22, + spx5_stats_pmac_rx_ok_bytes_cnt = 23, + spx5_stats_pmac_tx_mc_cnt = 24, + spx5_stats_pmac_tx_bc_cnt = 25, + spx5_stats_tx_xdefer_cnt = 26, + spx5_stats_pmac_rx_mc_cnt = 27, + spx5_stats_pmac_rx_bc_cnt = 28, + spx5_stats_rx_in_range_len_err_cnt = 29, + spx5_stats_pmac_rx_in_range_len_err_cnt = 30, + spx5_stats_rx_out_of_range_len_err_cnt = 31, + spx5_stats_pmac_rx_out_of_range_len_err_cnt = 32, + spx5_stats_rx_oversize_cnt = 33, + spx5_stats_pmac_rx_oversize_cnt = 34, + spx5_stats_tx_pause_cnt = 35, + spx5_stats_pmac_tx_pause_cnt = 36, + spx5_stats_rx_pause_cnt = 37, + spx5_stats_pmac_rx_pause_cnt = 38, + spx5_stats_rx_unsup_opcode_cnt = 39, + spx5_stats_pmac_rx_unsup_opcode_cnt = 40, + spx5_stats_rx_undersize_cnt = 41, + spx5_stats_pmac_rx_undersize_cnt = 42, + spx5_stats_rx_fragments_cnt = 43, + spx5_stats_pmac_rx_fragments_cnt = 44, + spx5_stats_rx_jabbers_cnt = 45, + spx5_stats_pmac_rx_jabbers_cnt = 46, + spx5_stats_rx_size64_cnt = 47, + spx5_stats_pmac_rx_size64_cnt = 48, + spx5_stats_rx_size65to127_cnt = 49, + spx5_stats_pmac_rx_size65to127_cnt = 50, + spx5_stats_rx_size128to255_cnt = 51, + spx5_stats_pmac_rx_size128to255_cnt = 52, + spx5_stats_rx_size256to511_cnt = 53, + spx5_stats_pmac_rx_size256to511_cnt = 54, + spx5_stats_rx_size512to1023_cnt = 55, + spx5_stats_pmac_rx_size512to1023_cnt = 56, + spx5_stats_rx_size1024to1518_cnt = 57, + spx5_stats_pmac_rx_size1024to1518_cnt = 58, + spx5_stats_rx_size1519tomax_cnt = 59, + spx5_stats_pmac_rx_size1519tomax_cnt = 60, + spx5_stats_tx_size64_cnt = 61, + spx5_stats_pmac_tx_size64_cnt = 62, + spx5_stats_tx_size65to127_cnt = 63, + spx5_stats_pmac_tx_size65to127_cnt = 64, + spx5_stats_tx_size128to255_cnt = 65, + spx5_stats_pmac_tx_size128to255_cnt = 66, + spx5_stats_tx_size256to511_cnt = 67, + spx5_stats_pmac_tx_size256to511_cnt = 68, + spx5_stats_tx_size512to1023_cnt = 69, + spx5_stats_pmac_tx_size512to1023_cnt = 70, + spx5_stats_tx_size1024to1518_cnt = 71, + spx5_stats_pmac_tx_size1024to1518_cnt = 72, + spx5_stats_tx_size1519tomax_cnt = 73, + spx5_stats_pmac_tx_size1519tomax_cnt = 74, + spx5_stats_mm_rx_assembly_err_cnt = 75, + spx5_stats_mm_rx_assembly_ok_cnt = 76, + spx5_stats_mm_rx_merge_frag_cnt = 77, + spx5_stats_mm_rx_smd_err_cnt = 78, + spx5_stats_mm_tx_pfragment_cnt = 79, + spx5_stats_rx_bad_bytes_cnt = 80, + spx5_stats_pmac_rx_bad_bytes_cnt = 81, + spx5_stats_rx_in_bytes_cnt = 82, + spx5_stats_rx_ipg_shrink_cnt = 83, + spx5_stats_rx_sync_lost_err_cnt = 84, + spx5_stats_rx_tagged_frms_cnt = 85, + spx5_stats_rx_untagged_frms_cnt = 86, + spx5_stats_tx_out_bytes_cnt = 87, + spx5_stats_tx_tagged_frms_cnt = 88, + spx5_stats_tx_untagged_frms_cnt = 89, + spx5_stats_rx_hih_cksm_err_cnt = 90, + spx5_stats_pmac_rx_hih_cksm_err_cnt = 91, + spx5_stats_rx_xgmii_prot_err_cnt = 92, + spx5_stats_pmac_rx_xgmii_prot_err_cnt = 93, + spx5_stats_ana_ac_port_stat_lsb_cnt = 94, + spx5_stats_green_p0_rx_fwd = 95, + spx5_stats_green_p0_rx_port_drop = 111, + spx5_stats_green_p0_tx_port = 127, + spx5_stats_rx_local_drop = 143, + spx5_stats_tx_local_drop = 144, + spx5_stats_count = 145, +}; + +static const char *const sparx5_stats_layout[] = { + "mm_rx_assembly_err_cnt", + "mm_rx_assembly_ok_cnt", + "mm_rx_merge_frag_cnt", + "mm_rx_smd_err_cnt", + "mm_tx_pfragment_cnt", + "rx_bad_bytes_cnt", + "pmac_rx_bad_bytes_cnt", + "rx_in_bytes_cnt", + "rx_ipg_shrink_cnt", + "rx_sync_lost_err_cnt", + "rx_tagged_frms_cnt", + "rx_untagged_frms_cnt", + "tx_out_bytes_cnt", + "tx_tagged_frms_cnt", + "tx_untagged_frms_cnt", + "rx_hih_cksm_err_cnt", + "pmac_rx_hih_cksm_err_cnt", + "rx_xgmii_prot_err_cnt", + "pmac_rx_xgmii_prot_err_cnt", + "rx_port_policer_drop", + "rx_fwd_green_p0", + "rx_fwd_green_p1", + "rx_fwd_green_p2", + "rx_fwd_green_p3", + "rx_fwd_green_p4", + "rx_fwd_green_p5", + "rx_fwd_green_p6", + "rx_fwd_green_p7", + "rx_fwd_yellow_p0", + "rx_fwd_yellow_p1", + "rx_fwd_yellow_p2", + "rx_fwd_yellow_p3", + "rx_fwd_yellow_p4", + "rx_fwd_yellow_p5", + "rx_fwd_yellow_p6", + "rx_fwd_yellow_p7", + "rx_port_drop_green_p0", + "rx_port_drop_green_p1", + "rx_port_drop_green_p2", + "rx_port_drop_green_p3", + "rx_port_drop_green_p4", + "rx_port_drop_green_p5", + "rx_port_drop_green_p6", + "rx_port_drop_green_p7", + "rx_port_drop_yellow_p0", + "rx_port_drop_yellow_p1", + "rx_port_drop_yellow_p2", + "rx_port_drop_yellow_p3", + "rx_port_drop_yellow_p4", + "rx_port_drop_yellow_p5", + "rx_port_drop_yellow_p6", + "rx_port_drop_yellow_p7", + "tx_port_green_p0", + "tx_port_green_p1", + "tx_port_green_p2", + "tx_port_green_p3", + "tx_port_green_p4", + "tx_port_green_p5", + "tx_port_green_p6", + "tx_port_green_p7", + "tx_port_yellow_p0", + "tx_port_yellow_p1", + "tx_port_yellow_p2", + "tx_port_yellow_p3", + "tx_port_yellow_p4", + "tx_port_yellow_p5", + "tx_port_yellow_p6", + "tx_port_yellow_p7", + "rx_local_drop", + "tx_local_drop", +}; + +static void sparx5_get_queue_sys_stats(struct sparx5 *sparx5, int portno) +{ + u64 *portstats; + u64 *stats; + u32 addr; + int idx; + + portstats = &sparx5->stats[portno * sparx5->num_stats]; + mutex_lock(&sparx5->queue_stats_lock); + spx5_wr(XQS_STAT_CFG_STAT_VIEW_SET(portno), sparx5, XQS_STAT_CFG); + addr = 0; + stats = &portstats[spx5_stats_green_p0_rx_fwd]; + for (idx = 0; idx < 2 * SPX5_PRIOS; ++idx, ++addr, ++stats) + sparx5_update_counter(stats, spx5_rd(sparx5, XQS_CNT(addr))); + addr = 16; + stats = &portstats[spx5_stats_green_p0_rx_port_drop]; + for (idx = 0; idx < 2 * SPX5_PRIOS; ++idx, ++addr, ++stats) + sparx5_update_counter(stats, spx5_rd(sparx5, XQS_CNT(addr))); + addr = 256; + stats = &portstats[spx5_stats_green_p0_tx_port]; + for (idx = 0; idx < 2 * SPX5_PRIOS; ++idx, ++addr, ++stats) + sparx5_update_counter(stats, spx5_rd(sparx5, XQS_CNT(addr))); + sparx5_update_counter(&portstats[spx5_stats_rx_local_drop], + spx5_rd(sparx5, XQS_CNT(32))); + sparx5_update_counter(&portstats[spx5_stats_tx_local_drop], + spx5_rd(sparx5, XQS_CNT(272))); + mutex_unlock(&sparx5->queue_stats_lock); +} + +static void sparx5_get_ana_ac_stats_stats(struct sparx5 *sparx5, int portno) +{ + u64 *portstats = &sparx5->stats[portno * sparx5->num_stats]; + + sparx5_update_counter(&portstats[spx5_stats_ana_ac_port_stat_lsb_cnt], + spx5_rd(sparx5, ANA_AC_PORT_STAT_LSB_CNT(portno, + SPX5_PORT_POLICER_DROPS))); +} + +static void sparx5_get_dev_phy_stats(u64 *portstats, void __iomem *inst, u32 + tinst) +{ + sparx5_update_counter(&portstats[spx5_stats_rx_symbol_err_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SYMBOL_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_symbol_err_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SYMBOL_ERR_CNT(tinst))); +} + +static void sparx5_get_dev_mac_stats(u64 *portstats, void __iomem *inst, u32 + tinst) +{ + sparx5_update_counter(&portstats[spx5_stats_tx_uc_cnt], + spx5_inst_rd(inst, DEV5G_TX_UC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_uc_cnt], + spx5_inst_rd(inst, DEV5G_PMAC_TX_UC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_mc_cnt], + spx5_inst_rd(inst, DEV5G_TX_MC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_bc_cnt], + spx5_inst_rd(inst, DEV5G_TX_BC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_uc_cnt], + spx5_inst_rd(inst, DEV5G_RX_UC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_uc_cnt], + spx5_inst_rd(inst, DEV5G_PMAC_RX_UC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_mc_cnt], + spx5_inst_rd(inst, DEV5G_RX_MC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_bc_cnt], + spx5_inst_rd(inst, DEV5G_RX_BC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_crc_err_cnt], + spx5_inst_rd(inst, DEV5G_RX_CRC_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_crc_err_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_CRC_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_alignment_lost_cnt], + spx5_inst_rd(inst, + DEV5G_RX_ALIGNMENT_LOST_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_alignment_lost_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_ALIGNMENT_LOST_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_ok_bytes_cnt], + spx5_inst_rd(inst, DEV5G_TX_OK_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_ok_bytes_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_OK_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_ok_bytes_cnt], + spx5_inst_rd(inst, DEV5G_RX_OK_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_ok_bytes_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_OK_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_mc_cnt], + spx5_inst_rd(inst, DEV5G_PMAC_TX_MC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_bc_cnt], + spx5_inst_rd(inst, DEV5G_PMAC_TX_BC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_mc_cnt], + spx5_inst_rd(inst, DEV5G_PMAC_RX_MC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_bc_cnt], + spx5_inst_rd(inst, DEV5G_PMAC_RX_BC_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_in_range_len_err_cnt], + spx5_inst_rd(inst, + DEV5G_RX_IN_RANGE_LEN_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_in_range_len_err_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_IN_RANGE_LEN_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_out_of_range_len_err_cnt], + spx5_inst_rd(inst, + DEV5G_RX_OUT_OF_RANGE_LEN_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_out_of_range_len_err_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_OUT_OF_RANGE_LEN_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_oversize_cnt], + spx5_inst_rd(inst, DEV5G_RX_OVERSIZE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_oversize_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_OVERSIZE_CNT(tinst))); +} + +static void sparx5_get_dev_mac_ctrl_stats(u64 *portstats, void __iomem *inst, + u32 tinst) +{ + sparx5_update_counter(&portstats[spx5_stats_tx_pause_cnt], + spx5_inst_rd(inst, DEV5G_TX_PAUSE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_pause_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_PAUSE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_pause_cnt], + spx5_inst_rd(inst, DEV5G_RX_PAUSE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_pause_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_PAUSE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_unsup_opcode_cnt], + spx5_inst_rd(inst, + DEV5G_RX_UNSUP_OPCODE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_unsup_opcode_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_UNSUP_OPCODE_CNT(tinst))); +} + +static void sparx5_get_dev_rmon_stats(u64 *portstats, void __iomem *inst, u32 + tinst) +{ + sparx5_update_counter(&portstats[spx5_stats_rx_undersize_cnt], + spx5_inst_rd(inst, + DEV5G_RX_UNDERSIZE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_undersize_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_UNDERSIZE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_oversize_cnt], + spx5_inst_rd(inst, DEV5G_RX_OVERSIZE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_oversize_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_OVERSIZE_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_fragments_cnt], + spx5_inst_rd(inst, + DEV5G_RX_FRAGMENTS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_fragments_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_FRAGMENTS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_jabbers_cnt], + spx5_inst_rd(inst, DEV5G_RX_JABBERS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_jabbers_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_JABBERS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size64_cnt], + spx5_inst_rd(inst, DEV5G_RX_SIZE64_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size64_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE64_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size65to127_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SIZE65TO127_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size65to127_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE65TO127_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size128to255_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SIZE128TO255_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size128to255_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE128TO255_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size256to511_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SIZE256TO511_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size256to511_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE256TO511_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size512to1023_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SIZE512TO1023_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size512to1023_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE512TO1023_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size1024to1518_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SIZE1024TO1518_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size1024to1518_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE1024TO1518_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_size1519tomax_cnt], + spx5_inst_rd(inst, + DEV5G_RX_SIZE1519TOMAX_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size1519tomax_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_SIZE1519TOMAX_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size64_cnt], + spx5_inst_rd(inst, DEV5G_TX_SIZE64_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size64_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE64_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size65to127_cnt], + spx5_inst_rd(inst, + DEV5G_TX_SIZE65TO127_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size65to127_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE65TO127_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size128to255_cnt], + spx5_inst_rd(inst, + DEV5G_TX_SIZE128TO255_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size128to255_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE128TO255_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size256to511_cnt], + spx5_inst_rd(inst, + DEV5G_TX_SIZE256TO511_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size256to511_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE256TO511_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size512to1023_cnt], + spx5_inst_rd(inst, + DEV5G_TX_SIZE512TO1023_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size512to1023_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE512TO1023_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size1024to1518_cnt], + spx5_inst_rd(inst, + DEV5G_TX_SIZE1024TO1518_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size1024to1518_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE1024TO1518_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_size1519tomax_cnt], + spx5_inst_rd(inst, + DEV5G_TX_SIZE1519TOMAX_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size1519tomax_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_TX_SIZE1519TOMAX_CNT(tinst))); +} + +static void sparx5_get_dev_misc_stats(u64 *portstats, void __iomem *inst, u32 + tinst) +{ + sparx5_update_counter(&portstats[spx5_stats_mm_rx_assembly_err_cnt], + spx5_inst_rd(inst, + DEV5G_MM_RX_ASSEMBLY_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_mm_rx_assembly_ok_cnt], + spx5_inst_rd(inst, + DEV5G_MM_RX_ASSEMBLY_OK_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_mm_rx_merge_frag_cnt], + spx5_inst_rd(inst, + DEV5G_MM_RX_MERGE_FRAG_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_mm_rx_smd_err_cnt], + spx5_inst_rd(inst, + DEV5G_MM_RX_SMD_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_mm_tx_pfragment_cnt], + spx5_inst_rd(inst, + DEV5G_MM_TX_PFRAGMENT_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_bad_bytes_cnt], + spx5_inst_rd(inst, + DEV5G_RX_BAD_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_bad_bytes_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_BAD_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_in_bytes_cnt], + spx5_inst_rd(inst, DEV5G_RX_IN_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_ipg_shrink_cnt], + spx5_inst_rd(inst, + DEV5G_RX_IPG_SHRINK_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_tagged_frms_cnt], + spx5_inst_rd(inst, + DEV5G_RX_TAGGED_FRMS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_untagged_frms_cnt], + spx5_inst_rd(inst, + DEV5G_RX_UNTAGGED_FRMS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_out_bytes_cnt], + spx5_inst_rd(inst, + DEV5G_TX_OUT_BYTES_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_tagged_frms_cnt], + spx5_inst_rd(inst, + DEV5G_TX_TAGGED_FRMS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_tx_untagged_frms_cnt], + spx5_inst_rd(inst, + DEV5G_TX_UNTAGGED_FRMS_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_hih_cksm_err_cnt], + spx5_inst_rd(inst, + DEV5G_RX_HIH_CKSM_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_hih_cksm_err_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_HIH_CKSM_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_rx_xgmii_prot_err_cnt], + spx5_inst_rd(inst, + DEV5G_RX_XGMII_PROT_ERR_CNT(tinst))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_xgmii_prot_err_cnt], + spx5_inst_rd(inst, + DEV5G_PMAC_RX_XGMII_PROT_ERR_CNT(tinst))); +} + +static void sparx5_get_device_stats(struct sparx5 *sparx5, int portno) +{ + u64 *portstats = &sparx5->stats[portno * sparx5->num_stats]; + u32 tinst = sparx5_port_dev_index(portno); + u32 dev = sparx5_to_high_dev(portno); + void __iomem *inst; + + inst = spx5_inst_get(sparx5, dev, tinst); + sparx5_get_dev_phy_stats(portstats, inst, tinst); + sparx5_get_dev_mac_stats(portstats, inst, tinst); + sparx5_get_dev_mac_ctrl_stats(portstats, inst, tinst); + sparx5_get_dev_rmon_stats(portstats, inst, tinst); + sparx5_get_dev_misc_stats(portstats, inst, tinst); +} + +static void sparx5_get_asm_phy_stats(u64 *portstats, void __iomem *inst, int + portno) +{ + sparx5_update_counter(&portstats[spx5_stats_rx_symbol_err_cnt], + spx5_inst_rd(inst, + ASM_RX_SYMBOL_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_symbol_err_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SYMBOL_ERR_CNT(portno))); +} + +static void sparx5_get_asm_mac_stats(u64 *portstats, void __iomem *inst, int + portno) +{ + sparx5_update_counter(&portstats[spx5_stats_tx_uc_cnt], + spx5_inst_rd(inst, ASM_TX_UC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_uc_cnt], + spx5_inst_rd(inst, ASM_PMAC_TX_UC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_mc_cnt], + spx5_inst_rd(inst, ASM_TX_MC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_bc_cnt], + spx5_inst_rd(inst, ASM_TX_BC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_backoff1_cnt], + spx5_inst_rd(inst, ASM_TX_BACKOFF1_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_multi_coll_cnt], + spx5_inst_rd(inst, + ASM_TX_MULTI_COLL_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_uc_cnt], + spx5_inst_rd(inst, ASM_RX_UC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_uc_cnt], + spx5_inst_rd(inst, ASM_PMAC_RX_UC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_mc_cnt], + spx5_inst_rd(inst, ASM_RX_MC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_bc_cnt], + spx5_inst_rd(inst, ASM_RX_BC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_crc_err_cnt], + spx5_inst_rd(inst, ASM_RX_CRC_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_crc_err_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_CRC_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_alignment_lost_cnt], + spx5_inst_rd(inst, + ASM_RX_ALIGNMENT_LOST_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_alignment_lost_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_ALIGNMENT_LOST_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_ok_bytes_cnt], + spx5_inst_rd(inst, ASM_TX_OK_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_ok_bytes_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_OK_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_defer_cnt], + spx5_inst_rd(inst, ASM_TX_DEFER_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_late_coll_cnt], + spx5_inst_rd(inst, ASM_TX_LATE_COLL_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_xcoll_cnt], + spx5_inst_rd(inst, ASM_TX_XCOLL_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_csense_cnt], + spx5_inst_rd(inst, ASM_TX_CSENSE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_ok_bytes_cnt], + spx5_inst_rd(inst, ASM_RX_OK_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_ok_bytes_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_OK_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_mc_cnt], + spx5_inst_rd(inst, ASM_PMAC_TX_MC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_bc_cnt], + spx5_inst_rd(inst, ASM_PMAC_TX_BC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_xdefer_cnt], + spx5_inst_rd(inst, ASM_TX_XDEFER_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_mc_cnt], + spx5_inst_rd(inst, ASM_PMAC_RX_MC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_bc_cnt], + spx5_inst_rd(inst, ASM_PMAC_RX_BC_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_in_range_len_err_cnt], + spx5_inst_rd(inst, + ASM_RX_IN_RANGE_LEN_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_in_range_len_err_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_IN_RANGE_LEN_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_out_of_range_len_err_cnt], + spx5_inst_rd(inst, + ASM_RX_OUT_OF_RANGE_LEN_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_out_of_range_len_err_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_OUT_OF_RANGE_LEN_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_oversize_cnt], + spx5_inst_rd(inst, ASM_RX_OVERSIZE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_oversize_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_OVERSIZE_CNT(portno))); +} + +static void sparx5_get_asm_mac_ctrl_stats(u64 *portstats, void __iomem *inst, + int portno) +{ + sparx5_update_counter(&portstats[spx5_stats_tx_pause_cnt], + spx5_inst_rd(inst, ASM_TX_PAUSE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_pause_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_PAUSE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_pause_cnt], + spx5_inst_rd(inst, ASM_RX_PAUSE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_pause_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_PAUSE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_unsup_opcode_cnt], + spx5_inst_rd(inst, + ASM_RX_UNSUP_OPCODE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_unsup_opcode_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_UNSUP_OPCODE_CNT(portno))); +} + +static void sparx5_get_asm_rmon_stats(u64 *portstats, void __iomem *inst, int + portno) +{ + sparx5_update_counter(&portstats[spx5_stats_rx_undersize_cnt], + spx5_inst_rd(inst, ASM_RX_UNDERSIZE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_undersize_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_UNDERSIZE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_oversize_cnt], + spx5_inst_rd(inst, ASM_RX_OVERSIZE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_oversize_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_OVERSIZE_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_fragments_cnt], + spx5_inst_rd(inst, ASM_RX_FRAGMENTS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_fragments_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_FRAGMENTS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_jabbers_cnt], + spx5_inst_rd(inst, ASM_RX_JABBERS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_jabbers_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_JABBERS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size64_cnt], + spx5_inst_rd(inst, ASM_RX_SIZE64_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size64_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE64_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size65to127_cnt], + spx5_inst_rd(inst, + ASM_RX_SIZE65TO127_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size65to127_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE65TO127_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size128to255_cnt], + spx5_inst_rd(inst, + ASM_RX_SIZE128TO255_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size128to255_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE128TO255_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size256to511_cnt], + spx5_inst_rd(inst, + ASM_RX_SIZE256TO511_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size256to511_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE256TO511_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size512to1023_cnt], + spx5_inst_rd(inst, + ASM_RX_SIZE512TO1023_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size512to1023_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE512TO1023_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size1024to1518_cnt], + spx5_inst_rd(inst, + ASM_RX_SIZE1024TO1518_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size1024to1518_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE1024TO1518_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_size1519tomax_cnt], + spx5_inst_rd(inst, + ASM_RX_SIZE1519TOMAX_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_size1519tomax_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_SIZE1519TOMAX_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size64_cnt], + spx5_inst_rd(inst, ASM_TX_SIZE64_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size64_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE64_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size65to127_cnt], + spx5_inst_rd(inst, + ASM_TX_SIZE65TO127_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size65to127_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE65TO127_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size128to255_cnt], + spx5_inst_rd(inst, + ASM_TX_SIZE128TO255_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size128to255_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE128TO255_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size256to511_cnt], + spx5_inst_rd(inst, + ASM_TX_SIZE256TO511_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size256to511_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE256TO511_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size512to1023_cnt], + spx5_inst_rd(inst, + ASM_TX_SIZE512TO1023_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size512to1023_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE512TO1023_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size1024to1518_cnt], + spx5_inst_rd(inst, + ASM_TX_SIZE1024TO1518_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size1024to1518_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE1024TO1518_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_size1519tomax_cnt], + spx5_inst_rd(inst, + ASM_TX_SIZE1519TOMAX_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_tx_size1519tomax_cnt], + spx5_inst_rd(inst, + ASM_PMAC_TX_SIZE1519TOMAX_CNT(portno))); +} + +static void sparx5_get_asm_misc_stats(u64 *portstats, void __iomem *inst, int + portno) +{ + sparx5_update_counter(&portstats[spx5_stats_mm_rx_assembly_err_cnt], + spx5_inst_rd(inst, + ASM_MM_RX_ASSEMBLY_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_mm_rx_assembly_ok_cnt], + spx5_inst_rd(inst, + ASM_MM_RX_ASSEMBLY_OK_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_mm_rx_merge_frag_cnt], + spx5_inst_rd(inst, + ASM_MM_RX_MERGE_FRAG_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_mm_rx_smd_err_cnt], + spx5_inst_rd(inst, + ASM_MM_RX_SMD_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_mm_tx_pfragment_cnt], + spx5_inst_rd(inst, + ASM_MM_TX_PFRAGMENT_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_bad_bytes_cnt], + spx5_inst_rd(inst, ASM_RX_BAD_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_pmac_rx_bad_bytes_cnt], + spx5_inst_rd(inst, + ASM_PMAC_RX_BAD_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_in_bytes_cnt], + spx5_inst_rd(inst, ASM_RX_IN_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_ipg_shrink_cnt], + spx5_inst_rd(inst, + ASM_RX_IPG_SHRINK_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_sync_lost_err_cnt], + spx5_inst_rd(inst, + ASM_RX_SYNC_LOST_ERR_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_tagged_frms_cnt], + spx5_inst_rd(inst, + ASM_RX_TAGGED_FRMS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_rx_untagged_frms_cnt], + spx5_inst_rd(inst, + ASM_RX_UNTAGGED_FRMS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_out_bytes_cnt], + spx5_inst_rd(inst, ASM_TX_OUT_BYTES_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_tagged_frms_cnt], + spx5_inst_rd(inst, + ASM_TX_TAGGED_FRMS_CNT(portno))); + sparx5_update_counter(&portstats[spx5_stats_tx_untagged_frms_cnt], + spx5_inst_rd(inst, + ASM_TX_UNTAGGED_FRMS_CNT(portno))); +} + +static void sparx5_get_asm_stats(struct sparx5 *sparx5, int portno) +{ + u64 *portstats = &sparx5->stats[portno * sparx5->num_stats]; + void __iomem *inst = spx5_inst_get(sparx5, TARGET_ASM, 0); + + sparx5_get_asm_phy_stats(portstats, inst, portno); + sparx5_get_asm_mac_stats(portstats, inst, portno); + sparx5_get_asm_mac_ctrl_stats(portstats, inst, portno); + sparx5_get_asm_rmon_stats(portstats, inst, portno); + sparx5_get_asm_misc_stats(portstats, inst, portno); +} + +static const struct ethtool_rmon_hist_range sparx5_rmon_ranges[] = { + { 0, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, 10239 }, + {} +}; + +static void sparx5_get_eth_phy_stats(struct net_device *ndev, + struct ethtool_eth_phy_stats *phy_stats) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + int portno = port->portno; + void __iomem *inst; + u64 *portstats; + + portstats = &sparx5->stats[portno * sparx5->num_stats]; + if (sparx5_is_baser(port->conf.portmode)) { + u32 tinst = sparx5_port_dev_index(portno); + u32 dev = sparx5_to_high_dev(portno); + + inst = spx5_inst_get(sparx5, dev, tinst); + sparx5_get_dev_phy_stats(portstats, inst, tinst); + } else { + inst = spx5_inst_get(sparx5, TARGET_ASM, 0); + sparx5_get_asm_phy_stats(portstats, inst, portno); + } + phy_stats->SymbolErrorDuringCarrier = + portstats[spx5_stats_rx_symbol_err_cnt] + + portstats[spx5_stats_pmac_rx_symbol_err_cnt]; +} + +static void sparx5_get_eth_mac_stats(struct net_device *ndev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + int portno = port->portno; + void __iomem *inst; + u64 *portstats; + + portstats = &sparx5->stats[portno * sparx5->num_stats]; + if (sparx5_is_baser(port->conf.portmode)) { + u32 tinst = sparx5_port_dev_index(portno); + u32 dev = sparx5_to_high_dev(portno); + + inst = spx5_inst_get(sparx5, dev, tinst); + sparx5_get_dev_mac_stats(portstats, inst, tinst); + } else { + inst = spx5_inst_get(sparx5, TARGET_ASM, 0); + sparx5_get_asm_mac_stats(portstats, inst, portno); + } + mac_stats->FramesTransmittedOK = portstats[spx5_stats_tx_uc_cnt] + + portstats[spx5_stats_pmac_tx_uc_cnt] + + portstats[spx5_stats_tx_mc_cnt] + + portstats[spx5_stats_tx_bc_cnt]; + mac_stats->SingleCollisionFrames = + portstats[spx5_stats_tx_backoff1_cnt]; + mac_stats->MultipleCollisionFrames = + portstats[spx5_stats_tx_multi_coll_cnt]; + mac_stats->FramesReceivedOK = portstats[spx5_stats_rx_uc_cnt] + + portstats[spx5_stats_pmac_rx_uc_cnt] + + portstats[spx5_stats_rx_mc_cnt] + + portstats[spx5_stats_rx_bc_cnt]; + mac_stats->FrameCheckSequenceErrors = + portstats[spx5_stats_rx_crc_err_cnt] + + portstats[spx5_stats_pmac_rx_crc_err_cnt]; + mac_stats->AlignmentErrors = portstats[spx5_stats_rx_alignment_lost_cnt] + + portstats[spx5_stats_pmac_rx_alignment_lost_cnt]; + mac_stats->OctetsTransmittedOK = portstats[spx5_stats_tx_ok_bytes_cnt] + + portstats[spx5_stats_pmac_tx_ok_bytes_cnt]; + mac_stats->FramesWithDeferredXmissions = + portstats[spx5_stats_tx_defer_cnt]; + mac_stats->LateCollisions = + portstats[spx5_stats_tx_late_coll_cnt]; + mac_stats->FramesAbortedDueToXSColls = + portstats[spx5_stats_tx_xcoll_cnt]; + mac_stats->CarrierSenseErrors = portstats[spx5_stats_tx_csense_cnt]; + mac_stats->OctetsReceivedOK = portstats[spx5_stats_rx_ok_bytes_cnt] + + portstats[spx5_stats_pmac_rx_ok_bytes_cnt]; + mac_stats->MulticastFramesXmittedOK = portstats[spx5_stats_tx_mc_cnt] + + portstats[spx5_stats_pmac_tx_mc_cnt]; + mac_stats->BroadcastFramesXmittedOK = portstats[spx5_stats_tx_bc_cnt] + + portstats[spx5_stats_pmac_tx_bc_cnt]; + mac_stats->FramesWithExcessiveDeferral = + portstats[spx5_stats_tx_xdefer_cnt]; + mac_stats->MulticastFramesReceivedOK = portstats[spx5_stats_rx_mc_cnt] + + portstats[spx5_stats_pmac_rx_mc_cnt]; + mac_stats->BroadcastFramesReceivedOK = portstats[spx5_stats_rx_bc_cnt] + + portstats[spx5_stats_pmac_rx_bc_cnt]; + mac_stats->InRangeLengthErrors = + portstats[spx5_stats_rx_in_range_len_err_cnt] + + portstats[spx5_stats_pmac_rx_in_range_len_err_cnt]; + mac_stats->OutOfRangeLengthField = + portstats[spx5_stats_rx_out_of_range_len_err_cnt] + + portstats[spx5_stats_pmac_rx_out_of_range_len_err_cnt]; + mac_stats->FrameTooLongErrors = portstats[spx5_stats_rx_oversize_cnt] + + portstats[spx5_stats_pmac_rx_oversize_cnt]; +} + +static void sparx5_get_eth_mac_ctrl_stats(struct net_device *ndev, + struct ethtool_eth_ctrl_stats *mac_ctrl_stats) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + int portno = port->portno; + void __iomem *inst; + u64 *portstats; + + portstats = &sparx5->stats[portno * sparx5->num_stats]; + if (sparx5_is_baser(port->conf.portmode)) { + u32 tinst = sparx5_port_dev_index(portno); + u32 dev = sparx5_to_high_dev(portno); + + inst = spx5_inst_get(sparx5, dev, tinst); + sparx5_get_dev_mac_ctrl_stats(portstats, inst, tinst); + } else { + inst = spx5_inst_get(sparx5, TARGET_ASM, 0); + sparx5_get_asm_mac_ctrl_stats(portstats, inst, portno); + } + mac_ctrl_stats->MACControlFramesTransmitted = + portstats[spx5_stats_tx_pause_cnt] + + portstats[spx5_stats_pmac_tx_pause_cnt]; + mac_ctrl_stats->MACControlFramesReceived = + portstats[spx5_stats_rx_pause_cnt] + + portstats[spx5_stats_pmac_rx_pause_cnt]; + mac_ctrl_stats->UnsupportedOpcodesReceived = + portstats[spx5_stats_rx_unsup_opcode_cnt] + + portstats[spx5_stats_pmac_rx_unsup_opcode_cnt]; +} + +static void sparx5_get_eth_rmon_stats(struct net_device *ndev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + int portno = port->portno; + void __iomem *inst; + u64 *portstats; + + portstats = &sparx5->stats[portno * sparx5->num_stats]; + if (sparx5_is_baser(port->conf.portmode)) { + u32 tinst = sparx5_port_dev_index(portno); + u32 dev = sparx5_to_high_dev(portno); + + inst = spx5_inst_get(sparx5, dev, tinst); + sparx5_get_dev_rmon_stats(portstats, inst, tinst); + } else { + inst = spx5_inst_get(sparx5, TARGET_ASM, 0); + sparx5_get_asm_rmon_stats(portstats, inst, portno); + } + rmon_stats->undersize_pkts = portstats[spx5_stats_rx_undersize_cnt] + + portstats[spx5_stats_pmac_rx_undersize_cnt]; + rmon_stats->oversize_pkts = portstats[spx5_stats_rx_oversize_cnt] + + portstats[spx5_stats_pmac_rx_oversize_cnt]; + rmon_stats->fragments = portstats[spx5_stats_rx_fragments_cnt] + + portstats[spx5_stats_pmac_rx_fragments_cnt]; + rmon_stats->jabbers = portstats[spx5_stats_rx_jabbers_cnt] + + portstats[spx5_stats_pmac_rx_jabbers_cnt]; + rmon_stats->hist[0] = portstats[spx5_stats_rx_size64_cnt] + + portstats[spx5_stats_pmac_rx_size64_cnt]; + rmon_stats->hist[1] = portstats[spx5_stats_rx_size65to127_cnt] + + portstats[spx5_stats_pmac_rx_size65to127_cnt]; + rmon_stats->hist[2] = portstats[spx5_stats_rx_size128to255_cnt] + + portstats[spx5_stats_pmac_rx_size128to255_cnt]; + rmon_stats->hist[3] = portstats[spx5_stats_rx_size256to511_cnt] + + portstats[spx5_stats_pmac_rx_size256to511_cnt]; + rmon_stats->hist[4] = portstats[spx5_stats_rx_size512to1023_cnt] + + portstats[spx5_stats_pmac_rx_size512to1023_cnt]; + rmon_stats->hist[5] = portstats[spx5_stats_rx_size1024to1518_cnt] + + portstats[spx5_stats_pmac_rx_size1024to1518_cnt]; + rmon_stats->hist[6] = portstats[spx5_stats_rx_size1519tomax_cnt] + + portstats[spx5_stats_pmac_rx_size1519tomax_cnt]; + rmon_stats->hist_tx[0] = portstats[spx5_stats_tx_size64_cnt] + + portstats[spx5_stats_pmac_tx_size64_cnt]; + rmon_stats->hist_tx[1] = portstats[spx5_stats_tx_size65to127_cnt] + + portstats[spx5_stats_pmac_tx_size65to127_cnt]; + rmon_stats->hist_tx[2] = portstats[spx5_stats_tx_size128to255_cnt] + + portstats[spx5_stats_pmac_tx_size128to255_cnt]; + rmon_stats->hist_tx[3] = portstats[spx5_stats_tx_size256to511_cnt] + + portstats[spx5_stats_pmac_tx_size256to511_cnt]; + rmon_stats->hist_tx[4] = portstats[spx5_stats_tx_size512to1023_cnt] + + portstats[spx5_stats_pmac_tx_size512to1023_cnt]; + rmon_stats->hist_tx[5] = portstats[spx5_stats_tx_size1024to1518_cnt] + + portstats[spx5_stats_pmac_tx_size1024to1518_cnt]; + rmon_stats->hist_tx[6] = portstats[spx5_stats_tx_size1519tomax_cnt] + + portstats[spx5_stats_pmac_tx_size1519tomax_cnt]; + *ranges = sparx5_rmon_ranges; +} + +static int sparx5_get_sset_count(struct net_device *ndev, int sset) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + + if (sset != ETH_SS_STATS) + return -EOPNOTSUPP; + return sparx5->num_ethtool_stats; +} + +static void sparx5_get_sset_strings(struct net_device *ndev, u32 sset, u8 *data) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + int idx; + + if (sset != ETH_SS_STATS) + return; + + for (idx = 0; idx < sparx5->num_ethtool_stats; idx++) + strncpy(data + idx * ETH_GSTRING_LEN, + sparx5->stats_layout[idx], ETH_GSTRING_LEN); +} + +static void sparx5_get_sset_data(struct net_device *ndev, + struct ethtool_stats *stats, u64 *data) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + int portno = port->portno; + void __iomem *inst; + u64 *portstats; + int idx; + + portstats = &sparx5->stats[portno * sparx5->num_stats]; + if (sparx5_is_baser(port->conf.portmode)) { + u32 tinst = sparx5_port_dev_index(portno); + u32 dev = sparx5_to_high_dev(portno); + + inst = spx5_inst_get(sparx5, dev, tinst); + sparx5_get_dev_misc_stats(portstats, inst, tinst); + } else { + inst = spx5_inst_get(sparx5, TARGET_ASM, 0); + sparx5_get_asm_misc_stats(portstats, inst, portno); + } + sparx5_get_ana_ac_stats_stats(sparx5, portno); + sparx5_get_queue_sys_stats(sparx5, portno); + /* Copy port counters to the ethtool buffer */ + for (idx = spx5_stats_mm_rx_assembly_err_cnt; + idx < spx5_stats_mm_rx_assembly_err_cnt + + sparx5->num_ethtool_stats; idx++) + *data++ = portstats[idx]; +} + +void sparx5_get_stats64(struct net_device *ndev, + struct rtnl_link_stats64 *stats) +{ + struct sparx5_port *port = netdev_priv(ndev); + struct sparx5 *sparx5 = port->sparx5; + u64 *portstats; + int idx; + + if (!sparx5->stats) + return; /* Not initialized yet */ + + portstats = &sparx5->stats[port->portno * sparx5->num_stats]; + + stats->rx_packets = portstats[spx5_stats_rx_uc_cnt] + + portstats[spx5_stats_pmac_rx_uc_cnt] + + portstats[spx5_stats_rx_mc_cnt] + + portstats[spx5_stats_rx_bc_cnt]; + stats->tx_packets = portstats[spx5_stats_tx_uc_cnt] + + portstats[spx5_stats_pmac_tx_uc_cnt] + + portstats[spx5_stats_tx_mc_cnt] + + portstats[spx5_stats_tx_bc_cnt]; + stats->rx_bytes = portstats[spx5_stats_rx_ok_bytes_cnt] + + portstats[spx5_stats_pmac_rx_ok_bytes_cnt]; + stats->tx_bytes = portstats[spx5_stats_tx_ok_bytes_cnt] + + portstats[spx5_stats_pmac_tx_ok_bytes_cnt]; + stats->rx_errors = portstats[spx5_stats_rx_in_range_len_err_cnt] + + portstats[spx5_stats_pmac_rx_in_range_len_err_cnt] + + portstats[spx5_stats_rx_out_of_range_len_err_cnt] + + portstats[spx5_stats_pmac_rx_out_of_range_len_err_cnt] + + portstats[spx5_stats_rx_oversize_cnt] + + portstats[spx5_stats_pmac_rx_oversize_cnt] + + portstats[spx5_stats_rx_crc_err_cnt] + + portstats[spx5_stats_pmac_rx_crc_err_cnt] + + portstats[spx5_stats_rx_alignment_lost_cnt] + + portstats[spx5_stats_pmac_rx_alignment_lost_cnt]; + stats->tx_errors = portstats[spx5_stats_tx_xcoll_cnt] + + portstats[spx5_stats_tx_csense_cnt] + + portstats[spx5_stats_tx_late_coll_cnt]; + stats->multicast = portstats[spx5_stats_rx_mc_cnt] + + portstats[spx5_stats_pmac_rx_mc_cnt]; + stats->collisions = portstats[spx5_stats_tx_late_coll_cnt] + + portstats[spx5_stats_tx_xcoll_cnt] + + portstats[spx5_stats_tx_backoff1_cnt]; + stats->rx_length_errors = portstats[spx5_stats_rx_in_range_len_err_cnt] + + portstats[spx5_stats_pmac_rx_in_range_len_err_cnt] + + portstats[spx5_stats_rx_out_of_range_len_err_cnt] + + portstats[spx5_stats_pmac_rx_out_of_range_len_err_cnt] + + portstats[spx5_stats_rx_oversize_cnt] + + portstats[spx5_stats_pmac_rx_oversize_cnt]; + stats->rx_crc_errors = portstats[spx5_stats_rx_crc_err_cnt] + + portstats[spx5_stats_pmac_rx_crc_err_cnt]; + stats->rx_frame_errors = portstats[spx5_stats_rx_alignment_lost_cnt] + + portstats[spx5_stats_pmac_rx_alignment_lost_cnt]; + stats->tx_aborted_errors = portstats[spx5_stats_tx_xcoll_cnt]; + stats->tx_carrier_errors = portstats[spx5_stats_tx_csense_cnt]; + stats->tx_window_errors = portstats[spx5_stats_tx_late_coll_cnt]; + stats->rx_dropped = portstats[spx5_stats_ana_ac_port_stat_lsb_cnt]; + for (idx = 0; idx < 2 * SPX5_PRIOS; ++idx, ++stats) + stats->rx_dropped += portstats[spx5_stats_green_p0_rx_port_drop + + idx]; + stats->tx_dropped = portstats[spx5_stats_tx_local_drop]; +} + +static void sparx5_update_port_stats(struct sparx5 *sparx5, int portno) +{ + if (sparx5_is_baser(sparx5->ports[portno]->conf.portmode)) + sparx5_get_device_stats(sparx5, portno); + else + sparx5_get_asm_stats(sparx5, portno); + sparx5_get_ana_ac_stats_stats(sparx5, portno); + sparx5_get_queue_sys_stats(sparx5, portno); +} + +static void sparx5_update_stats(struct sparx5 *sparx5) +{ + int idx; + + for (idx = 0; idx < SPX5_PORTS; idx++) + if (sparx5->ports[idx]) + sparx5_update_port_stats(sparx5, idx); +} + +static void sparx5_check_stats_work(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct sparx5 *sparx5 = container_of(dwork, + struct sparx5, + stats_work); + + sparx5_update_stats(sparx5); + + queue_delayed_work(sparx5->stats_queue, &sparx5->stats_work, + SPX5_STATS_CHECK_DELAY); +} + +static int sparx5_get_link_settings(struct net_device *ndev, + struct ethtool_link_ksettings *cmd) +{ + struct sparx5_port *port = netdev_priv(ndev); + + return phylink_ethtool_ksettings_get(port->phylink, cmd); +} + +static int sparx5_set_link_settings(struct net_device *ndev, + const struct ethtool_link_ksettings *cmd) +{ + struct sparx5_port *port = netdev_priv(ndev); + + return phylink_ethtool_ksettings_set(port->phylink, cmd); +} + +static void sparx5_config_stats(struct sparx5 *sparx5) +{ + /* Enable global events for port policer drops */ + spx5_rmw(ANA_AC_PORT_SGE_CFG_MASK_SET(0xf0f0), + ANA_AC_PORT_SGE_CFG_MASK, + sparx5, + ANA_AC_PORT_SGE_CFG(SPX5_PORT_POLICER_DROPS)); +} + +static void sparx5_config_port_stats(struct sparx5 *sparx5, int portno) +{ + /* Clear Queue System counters */ + spx5_wr(XQS_STAT_CFG_STAT_VIEW_SET(portno) | + XQS_STAT_CFG_STAT_CLEAR_SHOT_SET(3), sparx5, + XQS_STAT_CFG); + + /* Use counter for port policer drop count */ + spx5_rmw(ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE_SET(1) | + ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE_SET(0) | + ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK_SET(0xff), + ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE | + ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE | + ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK, + sparx5, ANA_AC_PORT_STAT_CFG(portno, SPX5_PORT_POLICER_DROPS)); +} + +const struct ethtool_ops sparx5_ethtool_ops = { + .get_sset_count = sparx5_get_sset_count, + .get_strings = sparx5_get_sset_strings, + .get_ethtool_stats = sparx5_get_sset_data, + .get_link_ksettings = sparx5_get_link_settings, + .set_link_ksettings = sparx5_set_link_settings, + .get_link = ethtool_op_get_link, + .get_eth_phy_stats = sparx5_get_eth_phy_stats, + .get_eth_mac_stats = sparx5_get_eth_mac_stats, + .get_eth_ctrl_stats = sparx5_get_eth_mac_ctrl_stats, + .get_rmon_stats = sparx5_get_eth_rmon_stats, +}; + +int sparx_stats_init(struct sparx5 *sparx5) +{ + char queue_name[32]; + int portno; + + sparx5->stats_layout = sparx5_stats_layout; + sparx5->num_stats = spx5_stats_count; + sparx5->num_ethtool_stats = ARRAY_SIZE(sparx5_stats_layout); + sparx5->stats = devm_kcalloc(sparx5->dev, + SPX5_PORTS_ALL * sparx5->num_stats, + sizeof(u64), GFP_KERNEL); + if (!sparx5->stats) + return -ENOMEM; + + mutex_init(&sparx5->queue_stats_lock); + sparx5_config_stats(sparx5); + for (portno = 0; portno < SPX5_PORTS; portno++) + if (sparx5->ports[portno]) + sparx5_config_port_stats(sparx5, portno); + + snprintf(queue_name, sizeof(queue_name), "%s-stats", + dev_name(sparx5->dev)); + sparx5->stats_queue = create_singlethread_workqueue(queue_name); + INIT_DELAYED_WORK(&sparx5->stats_work, sparx5_check_stats_work); + queue_delayed_work(sparx5->stats_queue, &sparx5->stats_work, + SPX5_STATS_CHECK_DELAY); + + return 0; +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c new file mode 100644 index 000000000000..0443f66b5550 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include <net/switchdev.h> +#include <linux/if_bridge.h> +#include <linux/iopoll.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" + +/* Commands for Mac Table Command register */ +#define MAC_CMD_LEARN 0 /* Insert (Learn) 1 entry */ +#define MAC_CMD_UNLEARN 1 /* Unlearn (Forget) 1 entry */ +#define MAC_CMD_LOOKUP 2 /* Look up 1 entry */ +#define MAC_CMD_READ 3 /* Read entry at Mac Table Index */ +#define MAC_CMD_WRITE 4 /* Write entry at Mac Table Index */ +#define MAC_CMD_SCAN 5 /* Scan (Age or find next) */ +#define MAC_CMD_FIND_SMALLEST 6 /* Get next entry */ +#define MAC_CMD_CLEAR_ALL 7 /* Delete all entries in table */ + +/* Commands for MAC_ENTRY_ADDR_TYPE */ +#define MAC_ENTRY_ADDR_TYPE_UPSID_PN 0 +#define MAC_ENTRY_ADDR_TYPE_UPSID_CPU_OR_INT 1 +#define MAC_ENTRY_ADDR_TYPE_GLAG 2 +#define MAC_ENTRY_ADDR_TYPE_MC_IDX 3 + +#define TABLE_UPDATE_SLEEP_US 10 +#define TABLE_UPDATE_TIMEOUT_US 100000 + +struct sparx5_mact_entry { + struct list_head list; + unsigned char mac[ETH_ALEN]; + u32 flags; +#define MAC_ENT_ALIVE BIT(0) +#define MAC_ENT_MOVED BIT(1) +#define MAC_ENT_LOCK BIT(2) + u16 vid; + u16 port; +}; + +static int sparx5_mact_get_status(struct sparx5 *sparx5) +{ + return spx5_rd(sparx5, LRN_COMMON_ACCESS_CTRL); +} + +static int sparx5_mact_wait_for_completion(struct sparx5 *sparx5) +{ + u32 val; + + return readx_poll_timeout(sparx5_mact_get_status, + sparx5, val, + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_GET(val) == 0, + TABLE_UPDATE_SLEEP_US, TABLE_UPDATE_TIMEOUT_US); +} + +static void sparx5_mact_select(struct sparx5 *sparx5, + const unsigned char mac[ETH_ALEN], + u16 vid) +{ + u32 macl = 0, mach = 0; + + /* Set the MAC address to handle and the vlan associated in a format + * understood by the hardware. + */ + mach |= vid << 16; + mach |= mac[0] << 8; + mach |= mac[1] << 0; + macl |= mac[2] << 24; + macl |= mac[3] << 16; + macl |= mac[4] << 8; + macl |= mac[5] << 0; + + spx5_wr(mach, sparx5, LRN_MAC_ACCESS_CFG_0); + spx5_wr(macl, sparx5, LRN_MAC_ACCESS_CFG_1); +} + +int sparx5_mact_learn(struct sparx5 *sparx5, int pgid, + const unsigned char mac[ETH_ALEN], u16 vid) +{ + int addr, type, ret; + + if (pgid < SPX5_PORTS) { + type = MAC_ENTRY_ADDR_TYPE_UPSID_PN; + addr = pgid % 32; + addr += (pgid / 32) << 5; /* Add upsid */ + } else { + type = MAC_ENTRY_ADDR_TYPE_MC_IDX; + addr = pgid - SPX5_PORTS; + } + + mutex_lock(&sparx5->lock); + + sparx5_mact_select(sparx5, mac, vid); + + /* MAC entry properties */ + spx5_wr(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_SET(addr) | + LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE_SET(type) | + LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD_SET(1) | + LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_LOCKED_SET(1), + sparx5, LRN_MAC_ACCESS_CFG_2); + spx5_wr(0, sparx5, LRN_MAC_ACCESS_CFG_3); + + /* Insert/learn new entry */ + spx5_wr(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET(MAC_CMD_LEARN) | + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(1), + sparx5, LRN_COMMON_ACCESS_CTRL); + + ret = sparx5_mact_wait_for_completion(sparx5); + + mutex_unlock(&sparx5->lock); + + return ret; +} + +int sparx5_mc_unsync(struct net_device *dev, const unsigned char *addr) +{ + struct sparx5_port *port = netdev_priv(dev); + struct sparx5 *sparx5 = port->sparx5; + + return sparx5_mact_forget(sparx5, addr, port->pvid); +} + +int sparx5_mc_sync(struct net_device *dev, const unsigned char *addr) +{ + struct sparx5_port *port = netdev_priv(dev); + struct sparx5 *sparx5 = port->sparx5; + + return sparx5_mact_learn(sparx5, PGID_CPU, addr, port->pvid); +} + +static int sparx5_mact_get(struct sparx5 *sparx5, + unsigned char mac[ETH_ALEN], + u16 *vid, u32 *pcfg2) +{ + u32 mach, macl, cfg2; + int ret = -ENOENT; + + cfg2 = spx5_rd(sparx5, LRN_MAC_ACCESS_CFG_2); + if (LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD_GET(cfg2)) { + mach = spx5_rd(sparx5, LRN_MAC_ACCESS_CFG_0); + macl = spx5_rd(sparx5, LRN_MAC_ACCESS_CFG_1); + mac[0] = ((mach >> 8) & 0xff); + mac[1] = ((mach >> 0) & 0xff); + mac[2] = ((macl >> 24) & 0xff); + mac[3] = ((macl >> 16) & 0xff); + mac[4] = ((macl >> 8) & 0xff); + mac[5] = ((macl >> 0) & 0xff); + *vid = mach >> 16; + *pcfg2 = cfg2; + ret = 0; + } + + return ret; +} + +bool sparx5_mact_getnext(struct sparx5 *sparx5, + unsigned char mac[ETH_ALEN], u16 *vid, u32 *pcfg2) +{ + u32 cfg2; + int ret; + + mutex_lock(&sparx5->lock); + + sparx5_mact_select(sparx5, mac, *vid); + + spx5_wr(LRN_SCAN_NEXT_CFG_SCAN_NEXT_IGNORE_LOCKED_ENA_SET(1) | + LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA_SET(1), + sparx5, LRN_SCAN_NEXT_CFG); + spx5_wr(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET + (MAC_CMD_FIND_SMALLEST) | + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(1), + sparx5, LRN_COMMON_ACCESS_CTRL); + + ret = sparx5_mact_wait_for_completion(sparx5); + if (ret == 0) { + ret = sparx5_mact_get(sparx5, mac, vid, &cfg2); + if (ret == 0) + *pcfg2 = cfg2; + } + + mutex_unlock(&sparx5->lock); + + return ret == 0; +} + +static int sparx5_mact_lookup(struct sparx5 *sparx5, + const unsigned char mac[ETH_ALEN], + u16 vid) +{ + int ret; + + mutex_lock(&sparx5->lock); + + sparx5_mact_select(sparx5, mac, vid); + + /* Issue a lookup command */ + spx5_wr(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET(MAC_CMD_LOOKUP) | + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(1), + sparx5, LRN_COMMON_ACCESS_CTRL); + + ret = sparx5_mact_wait_for_completion(sparx5); + if (ret) + goto out; + + ret = LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD_GET + (spx5_rd(sparx5, LRN_MAC_ACCESS_CFG_2)); + +out: + mutex_unlock(&sparx5->lock); + + return ret; +} + +int sparx5_mact_forget(struct sparx5 *sparx5, + const unsigned char mac[ETH_ALEN], u16 vid) +{ + int ret; + + mutex_lock(&sparx5->lock); + + sparx5_mact_select(sparx5, mac, vid); + + /* Issue an unlearn command */ + spx5_wr(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET(MAC_CMD_UNLEARN) | + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(1), + sparx5, LRN_COMMON_ACCESS_CTRL); + + ret = sparx5_mact_wait_for_completion(sparx5); + + mutex_unlock(&sparx5->lock); + + return ret; +} + +static struct sparx5_mact_entry *alloc_mact_entry(struct sparx5 *sparx5, + const unsigned char *mac, + u16 vid, u16 port_index) +{ + struct sparx5_mact_entry *mact_entry; + + mact_entry = devm_kzalloc(sparx5->dev, + sizeof(*mact_entry), GFP_ATOMIC); + if (!mact_entry) + return NULL; + + memcpy(mact_entry->mac, mac, ETH_ALEN); + mact_entry->vid = vid; + mact_entry->port = port_index; + return mact_entry; +} + +static struct sparx5_mact_entry *find_mact_entry(struct sparx5 *sparx5, + const unsigned char *mac, + u16 vid, u16 port_index) +{ + struct sparx5_mact_entry *mact_entry; + struct sparx5_mact_entry *res = NULL; + + mutex_lock(&sparx5->mact_lock); + list_for_each_entry(mact_entry, &sparx5->mact_entries, list) { + if (mact_entry->vid == vid && + ether_addr_equal(mac, mact_entry->mac) && + mact_entry->port == port_index) { + res = mact_entry; + break; + } + } + mutex_unlock(&sparx5->mact_lock); + + return res; +} + +static void sparx5_fdb_call_notifiers(enum switchdev_notifier_type type, + const char *mac, u16 vid, + struct net_device *dev, bool offloaded) +{ + struct switchdev_notifier_fdb_info info; + + info.addr = mac; + info.vid = vid; + info.offloaded = offloaded; + call_switchdev_notifiers(type, dev, &info.info, NULL); +} + +int sparx5_add_mact_entry(struct sparx5 *sparx5, + struct sparx5_port *port, + const unsigned char *addr, u16 vid) +{ + struct sparx5_mact_entry *mact_entry; + int ret; + + ret = sparx5_mact_lookup(sparx5, addr, vid); + if (ret) + return 0; + + /* In case the entry already exists, don't add it again to SW, + * just update HW, but we need to look in the actual HW because + * it is possible for an entry to be learn by HW and before the + * mact thread to start the frame will reach CPU and the CPU will + * add the entry but without the extern_learn flag. + */ + mact_entry = find_mact_entry(sparx5, addr, vid, port->portno); + if (mact_entry) + goto update_hw; + + /* Add the entry in SW MAC table not to get the notification when + * SW is pulling again + */ + mact_entry = alloc_mact_entry(sparx5, addr, vid, port->portno); + if (!mact_entry) + return -ENOMEM; + + mutex_lock(&sparx5->mact_lock); + list_add_tail(&mact_entry->list, &sparx5->mact_entries); + mutex_unlock(&sparx5->mact_lock); + +update_hw: + ret = sparx5_mact_learn(sparx5, port->portno, addr, vid); + + /* New entry? */ + if (mact_entry->flags == 0) { + mact_entry->flags |= MAC_ENT_LOCK; /* Don't age this */ + sparx5_fdb_call_notifiers(SWITCHDEV_FDB_ADD_TO_BRIDGE, addr, vid, + port->ndev, true); + } + + return ret; +} + +int sparx5_del_mact_entry(struct sparx5 *sparx5, + const unsigned char *addr, + u16 vid) +{ + struct sparx5_mact_entry *mact_entry, *tmp; + + /* Delete the entry in SW MAC table not to get the notification when + * SW is pulling again + */ + mutex_lock(&sparx5->mact_lock); + list_for_each_entry_safe(mact_entry, tmp, &sparx5->mact_entries, + list) { + if ((vid == 0 || mact_entry->vid == vid) && + ether_addr_equal(addr, mact_entry->mac)) { + list_del(&mact_entry->list); + devm_kfree(sparx5->dev, mact_entry); + + sparx5_mact_forget(sparx5, addr, mact_entry->vid); + } + } + mutex_unlock(&sparx5->mact_lock); + + return 0; +} + +static void sparx5_mact_handle_entry(struct sparx5 *sparx5, + unsigned char mac[ETH_ALEN], + u16 vid, u32 cfg2) +{ + struct sparx5_mact_entry *mact_entry; + bool found = false; + u16 port; + + if (LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE_GET(cfg2) != + MAC_ENTRY_ADDR_TYPE_UPSID_PN) + return; + + port = LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_GET(cfg2); + if (port >= SPX5_PORTS) + return; + + if (!test_bit(port, sparx5->bridge_mask)) + return; + + mutex_lock(&sparx5->mact_lock); + list_for_each_entry(mact_entry, &sparx5->mact_entries, list) { + if (mact_entry->vid == vid && + ether_addr_equal(mac, mact_entry->mac)) { + found = true; + mact_entry->flags |= MAC_ENT_ALIVE; + if (mact_entry->port != port) { + dev_warn(sparx5->dev, "Entry move: %d -> %d\n", + mact_entry->port, port); + mact_entry->port = port; + mact_entry->flags |= MAC_ENT_MOVED; + } + /* Entry handled */ + break; + } + } + mutex_unlock(&sparx5->mact_lock); + + if (found && !(mact_entry->flags & MAC_ENT_MOVED)) + /* Present, not moved */ + return; + + if (!found) { + /* Entry not found - now add */ + mact_entry = alloc_mact_entry(sparx5, mac, vid, port); + if (!mact_entry) + return; + + mact_entry->flags |= MAC_ENT_ALIVE; + mutex_lock(&sparx5->mact_lock); + list_add_tail(&mact_entry->list, &sparx5->mact_entries); + mutex_unlock(&sparx5->mact_lock); + } + + /* New or moved entry - notify bridge */ + sparx5_fdb_call_notifiers(SWITCHDEV_FDB_ADD_TO_BRIDGE, + mac, vid, sparx5->ports[port]->ndev, + true); +} + +void sparx5_mact_pull_work(struct work_struct *work) +{ + struct delayed_work *del_work = to_delayed_work(work); + struct sparx5 *sparx5 = container_of(del_work, struct sparx5, + mact_work); + struct sparx5_mact_entry *mact_entry, *tmp; + unsigned char mac[ETH_ALEN]; + u32 cfg2; + u16 vid; + int ret; + + /* Reset MAC entry flags */ + mutex_lock(&sparx5->mact_lock); + list_for_each_entry(mact_entry, &sparx5->mact_entries, list) + mact_entry->flags &= MAC_ENT_LOCK; + mutex_unlock(&sparx5->mact_lock); + + /* MAIN mac address processing loop */ + vid = 0; + memset(mac, 0, sizeof(mac)); + do { + mutex_lock(&sparx5->lock); + sparx5_mact_select(sparx5, mac, vid); + spx5_wr(LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA_SET(1), + sparx5, LRN_SCAN_NEXT_CFG); + spx5_wr(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET + (MAC_CMD_FIND_SMALLEST) | + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(1), + sparx5, LRN_COMMON_ACCESS_CTRL); + ret = sparx5_mact_wait_for_completion(sparx5); + if (ret == 0) + ret = sparx5_mact_get(sparx5, mac, &vid, &cfg2); + mutex_unlock(&sparx5->lock); + if (ret == 0) + sparx5_mact_handle_entry(sparx5, mac, vid, cfg2); + } while (ret == 0); + + mutex_lock(&sparx5->mact_lock); + list_for_each_entry_safe(mact_entry, tmp, &sparx5->mact_entries, + list) { + /* If the entry is in HW or permanent, then skip */ + if (mact_entry->flags & (MAC_ENT_ALIVE | MAC_ENT_LOCK)) + continue; + + sparx5_fdb_call_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, + mact_entry->mac, mact_entry->vid, + sparx5->ports[mact_entry->port]->ndev, + true); + + list_del(&mact_entry->list); + devm_kfree(sparx5->dev, mact_entry); + } + mutex_unlock(&sparx5->mact_lock); + + queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work, + SPX5_MACT_PULL_DELAY); +} + +void sparx5_set_ageing(struct sparx5 *sparx5, int msecs) +{ + int value = max(1, msecs / 10); /* unit 10 ms */ + + spx5_rmw(LRN_AUTOAGE_CFG_UNIT_SIZE_SET(2) | /* 10 ms */ + LRN_AUTOAGE_CFG_PERIOD_VAL_SET(value / 2), /* one bit ageing */ + LRN_AUTOAGE_CFG_UNIT_SIZE | + LRN_AUTOAGE_CFG_PERIOD_VAL, + sparx5, + LRN_AUTOAGE_CFG(0)); +} + +void sparx5_mact_init(struct sparx5 *sparx5) +{ + mutex_init(&sparx5->lock); + + /* Flush MAC table */ + spx5_wr(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET(MAC_CMD_CLEAR_ALL) | + LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(1), + sparx5, LRN_COMMON_ACCESS_CTRL); + + if (sparx5_mact_wait_for_completion(sparx5) != 0) + dev_warn(sparx5->dev, "MAC flush error\n"); + + sparx5_set_ageing(sparx5, BR_DEFAULT_AGEING_TIME / HZ * 1000); +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c new file mode 100644 index 000000000000..f666133a15de --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c @@ -0,0 +1,853 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + * + * The Sparx5 Chip Register Model can be browsed at this location: + * https://github.com/microchip-ung/sparx-5_reginfo + */ +#include <linux/module.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/platform_device.h> +#include <linux/interrupt.h> +#include <linux/of.h> +#include <linux/of_net.h> +#include <linux/of_mdio.h> +#include <net/switchdev.h> +#include <linux/etherdevice.h> +#include <linux/io.h> +#include <linux/printk.h> +#include <linux/iopoll.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> +#include <linux/types.h> +#include <linux/reset.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" +#include "sparx5_port.h" + +#define QLIM_WM(fraction) \ + ((SPX5_BUFFER_MEMORY / SPX5_BUFFER_CELL_SZ - 100) * (fraction) / 100) +#define IO_RANGES 3 + +struct initial_port_config { + u32 portno; + struct device_node *node; + struct sparx5_port_config conf; + struct phy *serdes; +}; + +struct sparx5_ram_config { + void __iomem *init_reg; + u32 init_val; +}; + +struct sparx5_main_io_resource { + enum sparx5_target id; + phys_addr_t offset; + int range; +}; + +static const struct sparx5_main_io_resource sparx5_main_iomap[] = { + { TARGET_CPU, 0, 0 }, /* 0x600000000 */ + { TARGET_FDMA, 0x80000, 0 }, /* 0x600080000 */ + { TARGET_PCEP, 0x400000, 0 }, /* 0x600400000 */ + { TARGET_DEV2G5, 0x10004000, 1 }, /* 0x610004000 */ + { TARGET_DEV5G, 0x10008000, 1 }, /* 0x610008000 */ + { TARGET_PCS5G_BR, 0x1000c000, 1 }, /* 0x61000c000 */ + { TARGET_DEV2G5 + 1, 0x10010000, 1 }, /* 0x610010000 */ + { TARGET_DEV5G + 1, 0x10014000, 1 }, /* 0x610014000 */ + { TARGET_PCS5G_BR + 1, 0x10018000, 1 }, /* 0x610018000 */ + { TARGET_DEV2G5 + 2, 0x1001c000, 1 }, /* 0x61001c000 */ + { TARGET_DEV5G + 2, 0x10020000, 1 }, /* 0x610020000 */ + { TARGET_PCS5G_BR + 2, 0x10024000, 1 }, /* 0x610024000 */ + { TARGET_DEV2G5 + 6, 0x10028000, 1 }, /* 0x610028000 */ + { TARGET_DEV5G + 6, 0x1002c000, 1 }, /* 0x61002c000 */ + { TARGET_PCS5G_BR + 6, 0x10030000, 1 }, /* 0x610030000 */ + { TARGET_DEV2G5 + 7, 0x10034000, 1 }, /* 0x610034000 */ + { TARGET_DEV5G + 7, 0x10038000, 1 }, /* 0x610038000 */ + { TARGET_PCS5G_BR + 7, 0x1003c000, 1 }, /* 0x61003c000 */ + { TARGET_DEV2G5 + 8, 0x10040000, 1 }, /* 0x610040000 */ + { TARGET_DEV5G + 8, 0x10044000, 1 }, /* 0x610044000 */ + { TARGET_PCS5G_BR + 8, 0x10048000, 1 }, /* 0x610048000 */ + { TARGET_DEV2G5 + 9, 0x1004c000, 1 }, /* 0x61004c000 */ + { TARGET_DEV5G + 9, 0x10050000, 1 }, /* 0x610050000 */ + { TARGET_PCS5G_BR + 9, 0x10054000, 1 }, /* 0x610054000 */ + { TARGET_DEV2G5 + 10, 0x10058000, 1 }, /* 0x610058000 */ + { TARGET_DEV5G + 10, 0x1005c000, 1 }, /* 0x61005c000 */ + { TARGET_PCS5G_BR + 10, 0x10060000, 1 }, /* 0x610060000 */ + { TARGET_DEV2G5 + 11, 0x10064000, 1 }, /* 0x610064000 */ + { TARGET_DEV5G + 11, 0x10068000, 1 }, /* 0x610068000 */ + { TARGET_PCS5G_BR + 11, 0x1006c000, 1 }, /* 0x61006c000 */ + { TARGET_DEV2G5 + 12, 0x10070000, 1 }, /* 0x610070000 */ + { TARGET_DEV10G, 0x10074000, 1 }, /* 0x610074000 */ + { TARGET_PCS10G_BR, 0x10078000, 1 }, /* 0x610078000 */ + { TARGET_DEV2G5 + 14, 0x1007c000, 1 }, /* 0x61007c000 */ + { TARGET_DEV10G + 2, 0x10080000, 1 }, /* 0x610080000 */ + { TARGET_PCS10G_BR + 2, 0x10084000, 1 }, /* 0x610084000 */ + { TARGET_DEV2G5 + 15, 0x10088000, 1 }, /* 0x610088000 */ + { TARGET_DEV10G + 3, 0x1008c000, 1 }, /* 0x61008c000 */ + { TARGET_PCS10G_BR + 3, 0x10090000, 1 }, /* 0x610090000 */ + { TARGET_DEV2G5 + 16, 0x10094000, 1 }, /* 0x610094000 */ + { TARGET_DEV2G5 + 17, 0x10098000, 1 }, /* 0x610098000 */ + { TARGET_DEV2G5 + 18, 0x1009c000, 1 }, /* 0x61009c000 */ + { TARGET_DEV2G5 + 19, 0x100a0000, 1 }, /* 0x6100a0000 */ + { TARGET_DEV2G5 + 20, 0x100a4000, 1 }, /* 0x6100a4000 */ + { TARGET_DEV2G5 + 21, 0x100a8000, 1 }, /* 0x6100a8000 */ + { TARGET_DEV2G5 + 22, 0x100ac000, 1 }, /* 0x6100ac000 */ + { TARGET_DEV2G5 + 23, 0x100b0000, 1 }, /* 0x6100b0000 */ + { TARGET_DEV2G5 + 32, 0x100b4000, 1 }, /* 0x6100b4000 */ + { TARGET_DEV2G5 + 33, 0x100b8000, 1 }, /* 0x6100b8000 */ + { TARGET_DEV2G5 + 34, 0x100bc000, 1 }, /* 0x6100bc000 */ + { TARGET_DEV2G5 + 35, 0x100c0000, 1 }, /* 0x6100c0000 */ + { TARGET_DEV2G5 + 36, 0x100c4000, 1 }, /* 0x6100c4000 */ + { TARGET_DEV2G5 + 37, 0x100c8000, 1 }, /* 0x6100c8000 */ + { TARGET_DEV2G5 + 38, 0x100cc000, 1 }, /* 0x6100cc000 */ + { TARGET_DEV2G5 + 39, 0x100d0000, 1 }, /* 0x6100d0000 */ + { TARGET_DEV2G5 + 40, 0x100d4000, 1 }, /* 0x6100d4000 */ + { TARGET_DEV2G5 + 41, 0x100d8000, 1 }, /* 0x6100d8000 */ + { TARGET_DEV2G5 + 42, 0x100dc000, 1 }, /* 0x6100dc000 */ + { TARGET_DEV2G5 + 43, 0x100e0000, 1 }, /* 0x6100e0000 */ + { TARGET_DEV2G5 + 44, 0x100e4000, 1 }, /* 0x6100e4000 */ + { TARGET_DEV2G5 + 45, 0x100e8000, 1 }, /* 0x6100e8000 */ + { TARGET_DEV2G5 + 46, 0x100ec000, 1 }, /* 0x6100ec000 */ + { TARGET_DEV2G5 + 47, 0x100f0000, 1 }, /* 0x6100f0000 */ + { TARGET_DEV2G5 + 57, 0x100f4000, 1 }, /* 0x6100f4000 */ + { TARGET_DEV25G + 1, 0x100f8000, 1 }, /* 0x6100f8000 */ + { TARGET_PCS25G_BR + 1, 0x100fc000, 1 }, /* 0x6100fc000 */ + { TARGET_DEV2G5 + 59, 0x10104000, 1 }, /* 0x610104000 */ + { TARGET_DEV25G + 3, 0x10108000, 1 }, /* 0x610108000 */ + { TARGET_PCS25G_BR + 3, 0x1010c000, 1 }, /* 0x61010c000 */ + { TARGET_DEV2G5 + 60, 0x10114000, 1 }, /* 0x610114000 */ + { TARGET_DEV25G + 4, 0x10118000, 1 }, /* 0x610118000 */ + { TARGET_PCS25G_BR + 4, 0x1011c000, 1 }, /* 0x61011c000 */ + { TARGET_DEV2G5 + 64, 0x10124000, 1 }, /* 0x610124000 */ + { TARGET_DEV5G + 12, 0x10128000, 1 }, /* 0x610128000 */ + { TARGET_PCS5G_BR + 12, 0x1012c000, 1 }, /* 0x61012c000 */ + { TARGET_PORT_CONF, 0x10130000, 1 }, /* 0x610130000 */ + { TARGET_DEV2G5 + 3, 0x10404000, 1 }, /* 0x610404000 */ + { TARGET_DEV5G + 3, 0x10408000, 1 }, /* 0x610408000 */ + { TARGET_PCS5G_BR + 3, 0x1040c000, 1 }, /* 0x61040c000 */ + { TARGET_DEV2G5 + 4, 0x10410000, 1 }, /* 0x610410000 */ + { TARGET_DEV5G + 4, 0x10414000, 1 }, /* 0x610414000 */ + { TARGET_PCS5G_BR + 4, 0x10418000, 1 }, /* 0x610418000 */ + { TARGET_DEV2G5 + 5, 0x1041c000, 1 }, /* 0x61041c000 */ + { TARGET_DEV5G + 5, 0x10420000, 1 }, /* 0x610420000 */ + { TARGET_PCS5G_BR + 5, 0x10424000, 1 }, /* 0x610424000 */ + { TARGET_DEV2G5 + 13, 0x10428000, 1 }, /* 0x610428000 */ + { TARGET_DEV10G + 1, 0x1042c000, 1 }, /* 0x61042c000 */ + { TARGET_PCS10G_BR + 1, 0x10430000, 1 }, /* 0x610430000 */ + { TARGET_DEV2G5 + 24, 0x10434000, 1 }, /* 0x610434000 */ + { TARGET_DEV2G5 + 25, 0x10438000, 1 }, /* 0x610438000 */ + { TARGET_DEV2G5 + 26, 0x1043c000, 1 }, /* 0x61043c000 */ + { TARGET_DEV2G5 + 27, 0x10440000, 1 }, /* 0x610440000 */ + { TARGET_DEV2G5 + 28, 0x10444000, 1 }, /* 0x610444000 */ + { TARGET_DEV2G5 + 29, 0x10448000, 1 }, /* 0x610448000 */ + { TARGET_DEV2G5 + 30, 0x1044c000, 1 }, /* 0x61044c000 */ + { TARGET_DEV2G5 + 31, 0x10450000, 1 }, /* 0x610450000 */ + { TARGET_DEV2G5 + 48, 0x10454000, 1 }, /* 0x610454000 */ + { TARGET_DEV10G + 4, 0x10458000, 1 }, /* 0x610458000 */ + { TARGET_PCS10G_BR + 4, 0x1045c000, 1 }, /* 0x61045c000 */ + { TARGET_DEV2G5 + 49, 0x10460000, 1 }, /* 0x610460000 */ + { TARGET_DEV10G + 5, 0x10464000, 1 }, /* 0x610464000 */ + { TARGET_PCS10G_BR + 5, 0x10468000, 1 }, /* 0x610468000 */ + { TARGET_DEV2G5 + 50, 0x1046c000, 1 }, /* 0x61046c000 */ + { TARGET_DEV10G + 6, 0x10470000, 1 }, /* 0x610470000 */ + { TARGET_PCS10G_BR + 6, 0x10474000, 1 }, /* 0x610474000 */ + { TARGET_DEV2G5 + 51, 0x10478000, 1 }, /* 0x610478000 */ + { TARGET_DEV10G + 7, 0x1047c000, 1 }, /* 0x61047c000 */ + { TARGET_PCS10G_BR + 7, 0x10480000, 1 }, /* 0x610480000 */ + { TARGET_DEV2G5 + 52, 0x10484000, 1 }, /* 0x610484000 */ + { TARGET_DEV10G + 8, 0x10488000, 1 }, /* 0x610488000 */ + { TARGET_PCS10G_BR + 8, 0x1048c000, 1 }, /* 0x61048c000 */ + { TARGET_DEV2G5 + 53, 0x10490000, 1 }, /* 0x610490000 */ + { TARGET_DEV10G + 9, 0x10494000, 1 }, /* 0x610494000 */ + { TARGET_PCS10G_BR + 9, 0x10498000, 1 }, /* 0x610498000 */ + { TARGET_DEV2G5 + 54, 0x1049c000, 1 }, /* 0x61049c000 */ + { TARGET_DEV10G + 10, 0x104a0000, 1 }, /* 0x6104a0000 */ + { TARGET_PCS10G_BR + 10, 0x104a4000, 1 }, /* 0x6104a4000 */ + { TARGET_DEV2G5 + 55, 0x104a8000, 1 }, /* 0x6104a8000 */ + { TARGET_DEV10G + 11, 0x104ac000, 1 }, /* 0x6104ac000 */ + { TARGET_PCS10G_BR + 11, 0x104b0000, 1 }, /* 0x6104b0000 */ + { TARGET_DEV2G5 + 56, 0x104b4000, 1 }, /* 0x6104b4000 */ + { TARGET_DEV25G, 0x104b8000, 1 }, /* 0x6104b8000 */ + { TARGET_PCS25G_BR, 0x104bc000, 1 }, /* 0x6104bc000 */ + { TARGET_DEV2G5 + 58, 0x104c4000, 1 }, /* 0x6104c4000 */ + { TARGET_DEV25G + 2, 0x104c8000, 1 }, /* 0x6104c8000 */ + { TARGET_PCS25G_BR + 2, 0x104cc000, 1 }, /* 0x6104cc000 */ + { TARGET_DEV2G5 + 61, 0x104d4000, 1 }, /* 0x6104d4000 */ + { TARGET_DEV25G + 5, 0x104d8000, 1 }, /* 0x6104d8000 */ + { TARGET_PCS25G_BR + 5, 0x104dc000, 1 }, /* 0x6104dc000 */ + { TARGET_DEV2G5 + 62, 0x104e4000, 1 }, /* 0x6104e4000 */ + { TARGET_DEV25G + 6, 0x104e8000, 1 }, /* 0x6104e8000 */ + { TARGET_PCS25G_BR + 6, 0x104ec000, 1 }, /* 0x6104ec000 */ + { TARGET_DEV2G5 + 63, 0x104f4000, 1 }, /* 0x6104f4000 */ + { TARGET_DEV25G + 7, 0x104f8000, 1 }, /* 0x6104f8000 */ + { TARGET_PCS25G_BR + 7, 0x104fc000, 1 }, /* 0x6104fc000 */ + { TARGET_DSM, 0x10504000, 1 }, /* 0x610504000 */ + { TARGET_ASM, 0x10600000, 1 }, /* 0x610600000 */ + { TARGET_GCB, 0x11010000, 2 }, /* 0x611010000 */ + { TARGET_QS, 0x11030000, 2 }, /* 0x611030000 */ + { TARGET_ANA_ACL, 0x11050000, 2 }, /* 0x611050000 */ + { TARGET_LRN, 0x11060000, 2 }, /* 0x611060000 */ + { TARGET_VCAP_SUPER, 0x11080000, 2 }, /* 0x611080000 */ + { TARGET_QSYS, 0x110a0000, 2 }, /* 0x6110a0000 */ + { TARGET_QFWD, 0x110b0000, 2 }, /* 0x6110b0000 */ + { TARGET_XQS, 0x110c0000, 2 }, /* 0x6110c0000 */ + { TARGET_CLKGEN, 0x11100000, 2 }, /* 0x611100000 */ + { TARGET_ANA_AC_POL, 0x11200000, 2 }, /* 0x611200000 */ + { TARGET_QRES, 0x11280000, 2 }, /* 0x611280000 */ + { TARGET_EACL, 0x112c0000, 2 }, /* 0x6112c0000 */ + { TARGET_ANA_CL, 0x11400000, 2 }, /* 0x611400000 */ + { TARGET_ANA_L3, 0x11480000, 2 }, /* 0x611480000 */ + { TARGET_HSCH, 0x11580000, 2 }, /* 0x611580000 */ + { TARGET_REW, 0x11600000, 2 }, /* 0x611600000 */ + { TARGET_ANA_L2, 0x11800000, 2 }, /* 0x611800000 */ + { TARGET_ANA_AC, 0x11900000, 2 }, /* 0x611900000 */ + { TARGET_VOP, 0x11a00000, 2 }, /* 0x611a00000 */ +}; + +static int sparx5_create_targets(struct sparx5 *sparx5) +{ + struct resource *iores[IO_RANGES]; + void __iomem *iomem[IO_RANGES]; + void __iomem *begin[IO_RANGES]; + int range_id[IO_RANGES]; + int idx, jdx; + + for (idx = 0, jdx = 0; jdx < ARRAY_SIZE(sparx5_main_iomap); jdx++) { + const struct sparx5_main_io_resource *iomap = &sparx5_main_iomap[jdx]; + + if (idx == iomap->range) { + range_id[idx] = jdx; + idx++; + } + } + for (idx = 0; idx < IO_RANGES; idx++) { + iores[idx] = platform_get_resource(sparx5->pdev, IORESOURCE_MEM, + idx); + if (!iores[idx]) { + dev_err(sparx5->dev, "Invalid resource\n"); + return -EINVAL; + } + iomem[idx] = devm_ioremap(sparx5->dev, + iores[idx]->start, + iores[idx]->end - iores[idx]->start + + 1); + if (!iomem[idx]) { + dev_err(sparx5->dev, "Unable to get switch registers: %s\n", + iores[idx]->name); + return -ENOMEM; + } + begin[idx] = iomem[idx] - sparx5_main_iomap[range_id[idx]].offset; + } + for (jdx = 0; jdx < ARRAY_SIZE(sparx5_main_iomap); jdx++) { + const struct sparx5_main_io_resource *iomap = &sparx5_main_iomap[jdx]; + + sparx5->regs[iomap->id] = begin[iomap->range] + iomap->offset; + } + return 0; +} + +static int sparx5_create_port(struct sparx5 *sparx5, + struct initial_port_config *config) +{ + struct sparx5_port *spx5_port; + struct net_device *ndev; + struct phylink *phylink; + int err; + + ndev = sparx5_create_netdev(sparx5, config->portno); + if (IS_ERR(ndev)) { + dev_err(sparx5->dev, "Could not create net device: %02u\n", + config->portno); + return PTR_ERR(ndev); + } + spx5_port = netdev_priv(ndev); + spx5_port->of_node = config->node; + spx5_port->serdes = config->serdes; + spx5_port->pvid = NULL_VID; + spx5_port->signd_internal = true; + spx5_port->signd_active_high = true; + spx5_port->signd_enable = true; + spx5_port->max_vlan_tags = SPX5_PORT_MAX_TAGS_NONE; + spx5_port->vlan_type = SPX5_VLAN_PORT_TYPE_UNAWARE; + spx5_port->custom_etype = 0x8880; /* Vitesse */ + spx5_port->phylink_pcs.poll = true; + spx5_port->phylink_pcs.ops = &sparx5_phylink_pcs_ops; + sparx5->ports[config->portno] = spx5_port; + + err = sparx5_port_init(sparx5, spx5_port, &config->conf); + if (err) { + dev_err(sparx5->dev, "port init failed\n"); + return err; + } + spx5_port->conf = config->conf; + + /* Setup VLAN */ + sparx5_vlan_port_setup(sparx5, spx5_port->portno); + + /* Create a phylink for PHY management. Also handles SFPs */ + spx5_port->phylink_config.dev = &spx5_port->ndev->dev; + spx5_port->phylink_config.type = PHYLINK_NETDEV; + spx5_port->phylink_config.pcs_poll = true; + + phylink = phylink_create(&spx5_port->phylink_config, + of_fwnode_handle(config->node), + config->conf.phy_mode, + &sparx5_phylink_mac_ops); + if (IS_ERR(phylink)) + return PTR_ERR(phylink); + + spx5_port->phylink = phylink; + phylink_set_pcs(phylink, &spx5_port->phylink_pcs); + + return 0; +} + +static int sparx5_init_ram(struct sparx5 *s5) +{ + const struct sparx5_ram_config spx5_ram_cfg[] = { + {spx5_reg_get(s5, ANA_AC_STAT_RESET), ANA_AC_STAT_RESET_RESET}, + {spx5_reg_get(s5, ASM_STAT_CFG), ASM_STAT_CFG_STAT_CNT_CLR_SHOT}, + {spx5_reg_get(s5, QSYS_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, REW_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, VOP_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, ANA_AC_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, ASM_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, EACL_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, VCAP_SUPER_RAM_INIT), QSYS_RAM_INIT_RAM_INIT}, + {spx5_reg_get(s5, DSM_RAM_INIT), QSYS_RAM_INIT_RAM_INIT} + }; + const struct sparx5_ram_config *cfg; + u32 value, pending, jdx, idx; + + for (jdx = 0; jdx < 10; jdx++) { + pending = ARRAY_SIZE(spx5_ram_cfg); + for (idx = 0; idx < ARRAY_SIZE(spx5_ram_cfg); idx++) { + cfg = &spx5_ram_cfg[idx]; + if (jdx == 0) { + writel(cfg->init_val, cfg->init_reg); + } else { + value = readl(cfg->init_reg); + if ((value & cfg->init_val) != cfg->init_val) + pending--; + } + } + if (!pending) + break; + usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); + } + + if (pending > 0) { + /* Still initializing, should be complete in + * less than 1ms + */ + dev_err(s5->dev, "Memory initialization error\n"); + return -EINVAL; + } + return 0; +} + +static int sparx5_init_switchcore(struct sparx5 *sparx5) +{ + u32 value; + int err = 0; + + spx5_rmw(EACL_POL_EACL_CFG_EACL_FORCE_INIT_SET(1), + EACL_POL_EACL_CFG_EACL_FORCE_INIT, + sparx5, + EACL_POL_EACL_CFG); + + spx5_rmw(EACL_POL_EACL_CFG_EACL_FORCE_INIT_SET(0), + EACL_POL_EACL_CFG_EACL_FORCE_INIT, + sparx5, + EACL_POL_EACL_CFG); + + /* Initialize memories, if not done already */ + value = spx5_rd(sparx5, HSCH_RESET_CFG); + if (!(value & HSCH_RESET_CFG_CORE_ENA)) { + err = sparx5_init_ram(sparx5); + if (err) + return err; + } + + /* Reset counters */ + spx5_wr(ANA_AC_STAT_RESET_RESET_SET(1), sparx5, ANA_AC_STAT_RESET); + spx5_wr(ASM_STAT_CFG_STAT_CNT_CLR_SHOT_SET(1), sparx5, ASM_STAT_CFG); + + /* Enable switch-core and queue system */ + spx5_wr(HSCH_RESET_CFG_CORE_ENA_SET(1), sparx5, HSCH_RESET_CFG); + + return 0; +} + +static int sparx5_init_coreclock(struct sparx5 *sparx5) +{ + enum sparx5_core_clockfreq freq = sparx5->coreclock; + u32 clk_div, clk_period, pol_upd_int, idx; + + /* Verify if core clock frequency is supported on target. + * If 'VTSS_CORE_CLOCK_DEFAULT' then the highest supported + * freq. is used + */ + switch (sparx5->target_ct) { + case SPX5_TARGET_CT_7546: + if (sparx5->coreclock == SPX5_CORE_CLOCK_DEFAULT) + freq = SPX5_CORE_CLOCK_250MHZ; + else if (sparx5->coreclock != SPX5_CORE_CLOCK_250MHZ) + freq = 0; /* Not supported */ + break; + case SPX5_TARGET_CT_7549: + case SPX5_TARGET_CT_7552: + case SPX5_TARGET_CT_7556: + if (sparx5->coreclock == SPX5_CORE_CLOCK_DEFAULT) + freq = SPX5_CORE_CLOCK_500MHZ; + else if (sparx5->coreclock != SPX5_CORE_CLOCK_500MHZ) + freq = 0; /* Not supported */ + break; + case SPX5_TARGET_CT_7558: + case SPX5_TARGET_CT_7558TSN: + if (sparx5->coreclock == SPX5_CORE_CLOCK_DEFAULT) + freq = SPX5_CORE_CLOCK_625MHZ; + else if (sparx5->coreclock != SPX5_CORE_CLOCK_625MHZ) + freq = 0; /* Not supported */ + break; + case SPX5_TARGET_CT_7546TSN: + if (sparx5->coreclock == SPX5_CORE_CLOCK_DEFAULT) + freq = SPX5_CORE_CLOCK_625MHZ; + break; + case SPX5_TARGET_CT_7549TSN: + case SPX5_TARGET_CT_7552TSN: + case SPX5_TARGET_CT_7556TSN: + if (sparx5->coreclock == SPX5_CORE_CLOCK_DEFAULT) + freq = SPX5_CORE_CLOCK_625MHZ; + else if (sparx5->coreclock == SPX5_CORE_CLOCK_250MHZ) + freq = 0; /* Not supported */ + break; + default: + dev_err(sparx5->dev, "Target (%#04x) not supported\n", + sparx5->target_ct); + return -ENODEV; + } + + switch (freq) { + case SPX5_CORE_CLOCK_250MHZ: + clk_div = 10; + pol_upd_int = 312; + break; + case SPX5_CORE_CLOCK_500MHZ: + clk_div = 5; + pol_upd_int = 624; + break; + case SPX5_CORE_CLOCK_625MHZ: + clk_div = 4; + pol_upd_int = 780; + break; + default: + dev_err(sparx5->dev, "%d coreclock not supported on (%#04x)\n", + sparx5->coreclock, sparx5->target_ct); + return -EINVAL; + } + + /* Update state with chosen frequency */ + sparx5->coreclock = freq; + + /* Configure the LCPLL */ + spx5_rmw(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV_SET(clk_div) | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV_SET(0) | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR_SET(0) | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL_SET(0) | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA_SET(0) | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA_SET(1), + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA | + CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA, + sparx5, + CLKGEN_LCPLL1_CORE_CLK_CFG); + + clk_period = sparx5_clk_period(freq); + + spx5_rmw(HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS_SET(clk_period / 100), + HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS, + sparx5, + HSCH_SYS_CLK_PER); + + spx5_rmw(ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS_SET(clk_period / 100), + ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS, + sparx5, + ANA_AC_POL_BDLB_DLB_CTRL); + + spx5_rmw(ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS_SET(clk_period / 100), + ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS, + sparx5, + ANA_AC_POL_SLB_DLB_CTRL); + + spx5_rmw(LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS_SET(clk_period / 100), + LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS, + sparx5, + LRN_AUTOAGE_CFG_1); + + for (idx = 0; idx < 3; idx++) + spx5_rmw(GCB_SIO_CLOCK_SYS_CLK_PERIOD_SET(clk_period / 100), + GCB_SIO_CLOCK_SYS_CLK_PERIOD, + sparx5, + GCB_SIO_CLOCK(idx)); + + spx5_rmw(HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY_SET + ((256 * 1000) / clk_period), + HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY, + sparx5, + HSCH_TAS_STATEMACHINE_CFG); + + spx5_rmw(ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT_SET(pol_upd_int), + ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT, + sparx5, + ANA_AC_POL_POL_UPD_INT_CFG); + + return 0; +} + +static int sparx5_qlim_set(struct sparx5 *sparx5) +{ + u32 res, dp, prio; + + for (res = 0; res < 2; res++) { + for (prio = 0; prio < 8; prio++) + spx5_wr(0xFFF, sparx5, + QRES_RES_CFG(prio + 630 + res * 1024)); + + for (dp = 0; dp < 4; dp++) + spx5_wr(0xFFF, sparx5, + QRES_RES_CFG(dp + 638 + res * 1024)); + } + + /* Set 80,90,95,100% of memory size for top watermarks */ + spx5_wr(QLIM_WM(80), sparx5, XQS_QLIMIT_SHR_QLIM_CFG(0)); + spx5_wr(QLIM_WM(90), sparx5, XQS_QLIMIT_SHR_CTOP_CFG(0)); + spx5_wr(QLIM_WM(95), sparx5, XQS_QLIMIT_SHR_ATOP_CFG(0)); + spx5_wr(QLIM_WM(100), sparx5, XQS_QLIMIT_SHR_TOP_CFG(0)); + + return 0; +} + +/* Some boards needs to map the SGPIO for signal detect explicitly to the + * port module + */ +static void sparx5_board_init(struct sparx5 *sparx5) +{ + int idx; + + if (!sparx5->sd_sgpio_remapping) + return; + + /* Enable SGPIO Signal Detect remapping */ + spx5_rmw(GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL, + GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL, + sparx5, + GCB_HW_SGPIO_SD_CFG); + + /* Refer to LOS SGPIO */ + for (idx = 0; idx < SPX5_PORTS; idx++) + if (sparx5->ports[idx]) + if (sparx5->ports[idx]->conf.sd_sgpio != ~0) + spx5_wr(sparx5->ports[idx]->conf.sd_sgpio, + sparx5, + GCB_HW_SGPIO_TO_SD_MAP_CFG(idx)); +} + +static int sparx5_start(struct sparx5 *sparx5) +{ + u8 broadcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + char queue_name[32]; + u32 idx; + int err; + + /* Setup own UPSIDs */ + for (idx = 0; idx < 3; idx++) { + spx5_wr(idx, sparx5, ANA_AC_OWN_UPSID(idx)); + spx5_wr(idx, sparx5, ANA_CL_OWN_UPSID(idx)); + spx5_wr(idx, sparx5, ANA_L2_OWN_UPSID(idx)); + spx5_wr(idx, sparx5, REW_OWN_UPSID(idx)); + } + + /* Enable CPU ports */ + for (idx = SPX5_PORTS; idx < SPX5_PORTS_ALL; idx++) + spx5_rmw(QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(1), + QFWD_SWITCH_PORT_MODE_PORT_ENA, + sparx5, + QFWD_SWITCH_PORT_MODE(idx)); + + /* Init masks */ + sparx5_update_fwd(sparx5); + + /* CPU copy CPU pgids */ + spx5_wr(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(1), + sparx5, ANA_AC_PGID_MISC_CFG(PGID_CPU)); + spx5_wr(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(1), + sparx5, ANA_AC_PGID_MISC_CFG(PGID_BCAST)); + + /* Recalc injected frame FCS */ + for (idx = SPX5_PORT_CPU_0; idx <= SPX5_PORT_CPU_1; idx++) + spx5_rmw(ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA_SET(1), + ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA, + sparx5, ANA_CL_FILTER_CTRL(idx)); + + /* Init MAC table, ageing */ + sparx5_mact_init(sparx5); + + /* Setup VLANs */ + sparx5_vlan_init(sparx5); + + /* Add host mode BC address (points only to CPU) */ + sparx5_mact_learn(sparx5, PGID_CPU, broadcast, NULL_VID); + + /* Enable queue limitation watermarks */ + sparx5_qlim_set(sparx5); + + err = sparx5_config_auto_calendar(sparx5); + if (err) + return err; + + err = sparx5_config_dsm_calendar(sparx5); + if (err) + return err; + + /* Init stats */ + err = sparx_stats_init(sparx5); + if (err) + return err; + + /* Init mact_sw struct */ + mutex_init(&sparx5->mact_lock); + INIT_LIST_HEAD(&sparx5->mact_entries); + snprintf(queue_name, sizeof(queue_name), "%s-mact", + dev_name(sparx5->dev)); + sparx5->mact_queue = create_singlethread_workqueue(queue_name); + INIT_DELAYED_WORK(&sparx5->mact_work, sparx5_mact_pull_work); + queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work, + SPX5_MACT_PULL_DELAY); + + err = sparx5_register_netdevs(sparx5); + if (err) + return err; + + sparx5_board_init(sparx5); + err = sparx5_register_notifier_blocks(sparx5); + + /* Start register based INJ/XTR */ + err = -ENXIO; + if (err && sparx5->xtr_irq >= 0) { + err = devm_request_irq(sparx5->dev, sparx5->xtr_irq, + sparx5_xtr_handler, IRQF_SHARED, + "sparx5-xtr", sparx5); + if (!err) + err = sparx5_manual_injection_mode(sparx5); + if (err) + sparx5->xtr_irq = -ENXIO; + } else { + sparx5->xtr_irq = -ENXIO; + } + return err; +} + +static void sparx5_cleanup_ports(struct sparx5 *sparx5) +{ + sparx5_unregister_netdevs(sparx5); + sparx5_destroy_netdevs(sparx5); +} + +static int mchp_sparx5_probe(struct platform_device *pdev) +{ + struct initial_port_config *configs, *config; + struct device_node *np = pdev->dev.of_node; + struct device_node *ports, *portnp; + struct reset_control *reset; + struct sparx5 *sparx5; + int idx = 0, err = 0; + + if (!np && !pdev->dev.platform_data) + return -ENODEV; + + sparx5 = devm_kzalloc(&pdev->dev, sizeof(*sparx5), GFP_KERNEL); + if (!sparx5) + return -ENOMEM; + + platform_set_drvdata(pdev, sparx5); + sparx5->pdev = pdev; + sparx5->dev = &pdev->dev; + + /* Do switch core reset if available */ + reset = devm_reset_control_get_optional_shared(&pdev->dev, "switch"); + if (IS_ERR(reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(reset), + "Failed to get switch reset controller.\n"); + reset_control_reset(reset); + + /* Default values, some from DT */ + sparx5->coreclock = SPX5_CORE_CLOCK_DEFAULT; + + ports = of_get_child_by_name(np, "ethernet-ports"); + if (!ports) { + dev_err(sparx5->dev, "no ethernet-ports child node found\n"); + return -ENODEV; + } + sparx5->port_count = of_get_child_count(ports); + + configs = kcalloc(sparx5->port_count, + sizeof(struct initial_port_config), GFP_KERNEL); + if (!configs) { + err = -ENOMEM; + goto cleanup_pnode; + } + + for_each_available_child_of_node(ports, portnp) { + struct sparx5_port_config *conf; + struct phy *serdes; + u32 portno; + + err = of_property_read_u32(portnp, "reg", &portno); + if (err) { + dev_err(sparx5->dev, "port reg property error\n"); + continue; + } + config = &configs[idx]; + conf = &config->conf; + conf->speed = SPEED_UNKNOWN; + conf->bandwidth = SPEED_UNKNOWN; + err = of_get_phy_mode(portnp, &conf->phy_mode); + if (err) { + dev_err(sparx5->dev, "port %u: missing phy-mode\n", + portno); + continue; + } + err = of_property_read_u32(portnp, "microchip,bandwidth", + &conf->bandwidth); + if (err) { + dev_err(sparx5->dev, "port %u: missing bandwidth\n", + portno); + continue; + } + err = of_property_read_u32(portnp, "microchip,sd-sgpio", &conf->sd_sgpio); + if (err) + conf->sd_sgpio = ~0; + else + sparx5->sd_sgpio_remapping = true; + serdes = devm_of_phy_get(sparx5->dev, portnp, NULL); + if (IS_ERR(serdes)) { + err = dev_err_probe(sparx5->dev, PTR_ERR(serdes), + "port %u: missing serdes\n", + portno); + goto cleanup_config; + } + config->portno = portno; + config->node = portnp; + config->serdes = serdes; + + conf->media = PHY_MEDIA_DAC; + conf->serdes_reset = true; + conf->portmode = conf->phy_mode; + conf->power_down = true; + idx++; + } + + err = sparx5_create_targets(sparx5); + if (err) + goto cleanup_config; + + if (!of_get_mac_address(np, sparx5->base_mac)) { + dev_info(sparx5->dev, "MAC addr was not set, use random MAC\n"); + eth_random_addr(sparx5->base_mac); + sparx5->base_mac[5] = 0; + } + + sparx5->xtr_irq = platform_get_irq_byname(sparx5->pdev, "xtr"); + + /* Read chip ID to check CPU interface */ + sparx5->chip_id = spx5_rd(sparx5, GCB_CHIP_ID); + + sparx5->target_ct = (enum spx5_target_chiptype) + GCB_CHIP_ID_PART_ID_GET(sparx5->chip_id); + + /* Initialize Switchcore and internal RAMs */ + err = sparx5_init_switchcore(sparx5); + if (err) { + dev_err(sparx5->dev, "Switchcore initialization error\n"); + goto cleanup_config; + } + + /* Initialize the LC-PLL (core clock) and set affected registers */ + err = sparx5_init_coreclock(sparx5); + if (err) { + dev_err(sparx5->dev, "LC-PLL initialization error\n"); + goto cleanup_config; + } + + for (idx = 0; idx < sparx5->port_count; ++idx) { + config = &configs[idx]; + if (!config->node) + continue; + + err = sparx5_create_port(sparx5, config); + if (err) { + dev_err(sparx5->dev, "port create error\n"); + goto cleanup_ports; + } + } + + err = sparx5_start(sparx5); + if (err) { + dev_err(sparx5->dev, "Start failed\n"); + goto cleanup_ports; + } + goto cleanup_config; + +cleanup_ports: + sparx5_cleanup_ports(sparx5); +cleanup_config: + kfree(configs); +cleanup_pnode: + of_node_put(ports); + return err; +} + +static int mchp_sparx5_remove(struct platform_device *pdev) +{ + struct sparx5 *sparx5 = platform_get_drvdata(pdev); + + if (sparx5->xtr_irq) { + disable_irq(sparx5->xtr_irq); + sparx5->xtr_irq = -ENXIO; + } + sparx5_cleanup_ports(sparx5); + /* Unregister netdevs */ + sparx5_unregister_notifier_blocks(sparx5); + + return 0; +} + +static const struct of_device_id mchp_sparx5_match[] = { + { .compatible = "microchip,sparx5-switch" }, + { } +}; +MODULE_DEVICE_TABLE(of, mchp_sparx5_match); + +static struct platform_driver mchp_sparx5_driver = { + .probe = mchp_sparx5_probe, + .remove = mchp_sparx5_remove, + .driver = { + .name = "sparx5-switch", + .of_match_table = mchp_sparx5_match, + }, +}; + +module_platform_driver(mchp_sparx5_driver); + +MODULE_DESCRIPTION("Microchip Sparx5 switch driver"); +MODULE_AUTHOR("Steen Hegelund <steen.hegelund@microchip.com>"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h new file mode 100644 index 000000000000..4d5f44c3a421 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h @@ -0,0 +1,375 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#ifndef __SPARX5_MAIN_H__ +#define __SPARX5_MAIN_H__ + +#include <linux/types.h> +#include <linux/phy/phy.h> +#include <linux/netdevice.h> +#include <linux/phy.h> +#include <linux/if_vlan.h> +#include <linux/bitmap.h> +#include <linux/phylink.h> +#include <linux/hrtimer.h> + +/* Target chip type */ +enum spx5_target_chiptype { + SPX5_TARGET_CT_7546 = 0x7546, /* SparX-5-64 Enterprise */ + SPX5_TARGET_CT_7549 = 0x7549, /* SparX-5-90 Enterprise */ + SPX5_TARGET_CT_7552 = 0x7552, /* SparX-5-128 Enterprise */ + SPX5_TARGET_CT_7556 = 0x7556, /* SparX-5-160 Enterprise */ + SPX5_TARGET_CT_7558 = 0x7558, /* SparX-5-200 Enterprise */ + SPX5_TARGET_CT_7546TSN = 0x47546, /* SparX-5-64i Industrial */ + SPX5_TARGET_CT_7549TSN = 0x47549, /* SparX-5-90i Industrial */ + SPX5_TARGET_CT_7552TSN = 0x47552, /* SparX-5-128i Industrial */ + SPX5_TARGET_CT_7556TSN = 0x47556, /* SparX-5-160i Industrial */ + SPX5_TARGET_CT_7558TSN = 0x47558, /* SparX-5-200i Industrial */ +}; + +enum sparx5_port_max_tags { + SPX5_PORT_MAX_TAGS_NONE, /* No extra tags allowed */ + SPX5_PORT_MAX_TAGS_ONE, /* Single tag allowed */ + SPX5_PORT_MAX_TAGS_TWO /* Single and double tag allowed */ +}; + +enum sparx5_vlan_port_type { + SPX5_VLAN_PORT_TYPE_UNAWARE, /* VLAN unaware port */ + SPX5_VLAN_PORT_TYPE_C, /* C-port */ + SPX5_VLAN_PORT_TYPE_S, /* S-port */ + SPX5_VLAN_PORT_TYPE_S_CUSTOM /* S-port using custom type */ +}; + +#define SPX5_PORTS 65 +#define SPX5_PORT_CPU (SPX5_PORTS) /* Next port is CPU port */ +#define SPX5_PORT_CPU_0 (SPX5_PORT_CPU + 0) /* CPU Port 65 */ +#define SPX5_PORT_CPU_1 (SPX5_PORT_CPU + 1) /* CPU Port 66 */ +#define SPX5_PORT_VD0 (SPX5_PORT_CPU + 2) /* VD0/Port 67 used for IPMC */ +#define SPX5_PORT_VD1 (SPX5_PORT_CPU + 3) /* VD1/Port 68 used for AFI/OAM */ +#define SPX5_PORT_VD2 (SPX5_PORT_CPU + 4) /* VD2/Port 69 used for IPinIP*/ +#define SPX5_PORTS_ALL (SPX5_PORT_CPU + 5) /* Total number of ports */ + +#define PGID_BASE SPX5_PORTS /* Starts after port PGIDs */ +#define PGID_UC_FLOOD (PGID_BASE + 0) +#define PGID_MC_FLOOD (PGID_BASE + 1) +#define PGID_IPV4_MC_DATA (PGID_BASE + 2) +#define PGID_IPV4_MC_CTRL (PGID_BASE + 3) +#define PGID_IPV6_MC_DATA (PGID_BASE + 4) +#define PGID_IPV6_MC_CTRL (PGID_BASE + 5) +#define PGID_BCAST (PGID_BASE + 6) +#define PGID_CPU (PGID_BASE + 7) + +#define IFH_LEN 9 /* 36 bytes */ +#define NULL_VID 0 +#define SPX5_MACT_PULL_DELAY (2 * HZ) +#define SPX5_STATS_CHECK_DELAY (1 * HZ) +#define SPX5_PRIOS 8 /* Number of priority queues */ +#define SPX5_BUFFER_CELL_SZ 184 /* Cell size */ +#define SPX5_BUFFER_MEMORY 4194280 /* 22795 words * 184 bytes */ + +#define XTR_QUEUE 0 +#define INJ_QUEUE 0 + +struct sparx5; + +struct sparx5_port_config { + phy_interface_t portmode; + u32 bandwidth; + int speed; + int duplex; + enum phy_media media; + bool inband; + bool power_down; + bool autoneg; + bool serdes_reset; + u32 pause; + u32 pause_adv; + phy_interface_t phy_mode; + u32 sd_sgpio; +}; + +struct sparx5_port { + struct net_device *ndev; + struct sparx5 *sparx5; + struct device_node *of_node; + struct phy *serdes; + struct sparx5_port_config conf; + struct phylink_config phylink_config; + struct phylink *phylink; + struct phylink_pcs phylink_pcs; + u16 portno; + /* Ingress default VLAN (pvid) */ + u16 pvid; + /* Egress default VLAN (vid) */ + u16 vid; + bool signd_internal; + bool signd_active_high; + bool signd_enable; + bool flow_control; + enum sparx5_port_max_tags max_vlan_tags; + enum sparx5_vlan_port_type vlan_type; + u32 custom_etype; + u32 ifh[IFH_LEN]; + bool vlan_aware; + struct hrtimer inj_timer; +}; + +enum sparx5_core_clockfreq { + SPX5_CORE_CLOCK_DEFAULT, /* Defaults to the highest supported frequency */ + SPX5_CORE_CLOCK_250MHZ, /* 250MHZ core clock frequency */ + SPX5_CORE_CLOCK_500MHZ, /* 500MHZ core clock frequency */ + SPX5_CORE_CLOCK_625MHZ, /* 625MHZ core clock frequency */ +}; + +struct sparx5 { + struct platform_device *pdev; + struct device *dev; + u32 chip_id; + enum spx5_target_chiptype target_ct; + void __iomem *regs[NUM_TARGETS]; + int port_count; + struct mutex lock; /* MAC reg lock */ + /* port structures are in net device */ + struct sparx5_port *ports[SPX5_PORTS]; + enum sparx5_core_clockfreq coreclock; + /* Statistics */ + u32 num_stats; + u32 num_ethtool_stats; + const char * const *stats_layout; + u64 *stats; + /* Workqueue for reading stats */ + struct mutex queue_stats_lock; + struct delayed_work stats_work; + struct workqueue_struct *stats_queue; + /* Notifiers */ + struct notifier_block netdevice_nb; + struct notifier_block switchdev_nb; + struct notifier_block switchdev_blocking_nb; + /* Switch state */ + u8 base_mac[ETH_ALEN]; + /* Associated bridge device (when bridged) */ + struct net_device *hw_bridge_dev; + /* Bridged interfaces */ + DECLARE_BITMAP(bridge_mask, SPX5_PORTS); + DECLARE_BITMAP(bridge_fwd_mask, SPX5_PORTS); + DECLARE_BITMAP(bridge_lrn_mask, SPX5_PORTS); + DECLARE_BITMAP(vlan_mask[VLAN_N_VID], SPX5_PORTS); + /* SW MAC table */ + struct list_head mact_entries; + /* mac table list (mact_entries) mutex */ + struct mutex mact_lock; + struct delayed_work mact_work; + struct workqueue_struct *mact_queue; + /* Board specifics */ + bool sd_sgpio_remapping; + /* Register based inj/xtr */ + int xtr_irq; +}; + +/* sparx5_switchdev.c */ +int sparx5_register_notifier_blocks(struct sparx5 *sparx5); +void sparx5_unregister_notifier_blocks(struct sparx5 *sparx5); + +/* sparx5_packet.c */ +irqreturn_t sparx5_xtr_handler(int irq, void *_priv); +int sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev); +int sparx5_manual_injection_mode(struct sparx5 *sparx5); +void sparx5_port_inj_timer_setup(struct sparx5_port *port); + +/* sparx5_mactable.c */ +void sparx5_mact_pull_work(struct work_struct *work); +int sparx5_mact_learn(struct sparx5 *sparx5, int port, + const unsigned char mac[ETH_ALEN], u16 vid); +bool sparx5_mact_getnext(struct sparx5 *sparx5, + unsigned char mac[ETH_ALEN], u16 *vid, u32 *pcfg2); +int sparx5_mact_forget(struct sparx5 *sparx5, + const unsigned char mac[ETH_ALEN], u16 vid); +int sparx5_add_mact_entry(struct sparx5 *sparx5, + struct sparx5_port *port, + const unsigned char *addr, u16 vid); +int sparx5_del_mact_entry(struct sparx5 *sparx5, + const unsigned char *addr, + u16 vid); +int sparx5_mc_sync(struct net_device *dev, const unsigned char *addr); +int sparx5_mc_unsync(struct net_device *dev, const unsigned char *addr); +void sparx5_set_ageing(struct sparx5 *sparx5, int msecs); +void sparx5_mact_init(struct sparx5 *sparx5); + +/* sparx5_vlan.c */ +void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable); +void sparx5_update_fwd(struct sparx5 *sparx5); +void sparx5_vlan_init(struct sparx5 *sparx5); +void sparx5_vlan_port_setup(struct sparx5 *sparx5, int portno); +int sparx5_vlan_vid_add(struct sparx5_port *port, u16 vid, bool pvid, + bool untagged); +int sparx5_vlan_vid_del(struct sparx5_port *port, u16 vid); +void sparx5_vlan_port_apply(struct sparx5 *sparx5, struct sparx5_port *port); + +/* sparx5_calendar.c */ +int sparx5_config_auto_calendar(struct sparx5 *sparx5); +int sparx5_config_dsm_calendar(struct sparx5 *sparx5); + +/* sparx5_ethtool.c */ +void sparx5_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *stats); +int sparx_stats_init(struct sparx5 *sparx5); + +/* sparx5_netdev.c */ +bool sparx5_netdevice_check(const struct net_device *dev); +struct net_device *sparx5_create_netdev(struct sparx5 *sparx5, u32 portno); +int sparx5_register_netdevs(struct sparx5 *sparx5); +void sparx5_destroy_netdevs(struct sparx5 *sparx5); +void sparx5_unregister_netdevs(struct sparx5 *sparx5); + +/* Clock period in picoseconds */ +static inline u32 sparx5_clk_period(enum sparx5_core_clockfreq cclock) +{ + switch (cclock) { + case SPX5_CORE_CLOCK_250MHZ: + return 4000; + case SPX5_CORE_CLOCK_500MHZ: + return 2000; + case SPX5_CORE_CLOCK_625MHZ: + default: + return 1600; + } +} + +static inline bool sparx5_is_baser(phy_interface_t interface) +{ + return interface == PHY_INTERFACE_MODE_5GBASER || + interface == PHY_INTERFACE_MODE_10GBASER || + interface == PHY_INTERFACE_MODE_25GBASER; +} + +extern const struct phylink_mac_ops sparx5_phylink_mac_ops; +extern const struct phylink_pcs_ops sparx5_phylink_pcs_ops; +extern const struct ethtool_ops sparx5_ethtool_ops; + +/* Calculate raw offset */ +static inline __pure int spx5_offset(int id, int tinst, int tcnt, + int gbase, int ginst, + int gcnt, int gwidth, + int raddr, int rinst, + int rcnt, int rwidth) +{ + WARN_ON((tinst) >= tcnt); + WARN_ON((ginst) >= gcnt); + WARN_ON((rinst) >= rcnt); + return gbase + ((ginst) * gwidth) + + raddr + ((rinst) * rwidth); +} + +/* Read, Write and modify registers content. + * The register definition macros start at the id + */ +static inline void __iomem *spx5_addr(void __iomem *base[], + int id, int tinst, int tcnt, + int gbase, int ginst, + int gcnt, int gwidth, + int raddr, int rinst, + int rcnt, int rwidth) +{ + WARN_ON((tinst) >= tcnt); + WARN_ON((ginst) >= gcnt); + WARN_ON((rinst) >= rcnt); + return base[id + (tinst)] + + gbase + ((ginst) * gwidth) + + raddr + ((rinst) * rwidth); +} + +static inline void __iomem *spx5_inst_addr(void __iomem *base, + int gbase, int ginst, + int gcnt, int gwidth, + int raddr, int rinst, + int rcnt, int rwidth) +{ + WARN_ON((ginst) >= gcnt); + WARN_ON((rinst) >= rcnt); + return base + + gbase + ((ginst) * gwidth) + + raddr + ((rinst) * rwidth); +} + +static inline u32 spx5_rd(struct sparx5 *sparx5, int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + return readl(spx5_addr(sparx5->regs, id, tinst, tcnt, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); +} + +static inline u32 spx5_inst_rd(void __iomem *iomem, int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + return readl(spx5_inst_addr(iomem, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); +} + +static inline void spx5_wr(u32 val, struct sparx5 *sparx5, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + writel(val, spx5_addr(sparx5->regs, id, tinst, tcnt, + gbase, ginst, gcnt, gwidth, + raddr, rinst, rcnt, rwidth)); +} + +static inline void spx5_inst_wr(u32 val, void __iomem *iomem, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + writel(val, spx5_inst_addr(iomem, + gbase, ginst, gcnt, gwidth, + raddr, rinst, rcnt, rwidth)); +} + +static inline void spx5_rmw(u32 val, u32 mask, struct sparx5 *sparx5, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + u32 nval; + + nval = readl(spx5_addr(sparx5->regs, id, tinst, tcnt, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); + nval = (nval & ~mask) | (val & mask); + writel(nval, spx5_addr(sparx5->regs, id, tinst, tcnt, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); +} + +static inline void spx5_inst_rmw(u32 val, u32 mask, void __iomem *iomem, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + u32 nval; + + nval = readl(spx5_inst_addr(iomem, gbase, ginst, gcnt, gwidth, raddr, + rinst, rcnt, rwidth)); + nval = (nval & ~mask) | (val & mask); + writel(nval, spx5_inst_addr(iomem, gbase, ginst, gcnt, gwidth, raddr, + rinst, rcnt, rwidth)); +} + +static inline void __iomem *spx5_inst_get(struct sparx5 *sparx5, int id, int tinst) +{ + return sparx5->regs[id + tinst]; +} + +static inline void __iomem *spx5_reg_get(struct sparx5 *sparx5, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + return spx5_addr(sparx5->regs, id, tinst, tcnt, + gbase, ginst, gcnt, gwidth, + raddr, rinst, rcnt, rwidth); +} + +#endif /* __SPARX5_MAIN_H__ */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h new file mode 100644 index 000000000000..5ab2373a7178 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h @@ -0,0 +1,4642 @@ +/* SPDX-License-Identifier: GPL-2.0+ + * Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. + */ + +/* This file is autogenerated by cml-utils 2021-05-06 13:06:37 +0200. + * Commit ID: 9ae4ec441e25e4b9003f4e514df5cb12a36b84d3 + */ + +#ifndef _SPARX5_MAIN_REGS_H_ +#define _SPARX5_MAIN_REGS_H_ + +#include <linux/bitfield.h> +#include <linux/types.h> +#include <linux/bug.h> + +enum sparx5_target { + TARGET_ANA_AC = 1, + TARGET_ANA_ACL = 2, + TARGET_ANA_AC_POL = 4, + TARGET_ANA_CL = 6, + TARGET_ANA_L2 = 7, + TARGET_ANA_L3 = 8, + TARGET_ASM = 9, + TARGET_CLKGEN = 11, + TARGET_CPU = 12, + TARGET_DEV10G = 17, + TARGET_DEV25G = 29, + TARGET_DEV2G5 = 37, + TARGET_DEV5G = 102, + TARGET_DSM = 115, + TARGET_EACL = 116, + TARGET_FDMA = 117, + TARGET_GCB = 118, + TARGET_HSCH = 119, + TARGET_LRN = 122, + TARGET_PCEP = 129, + TARGET_PCS10G_BR = 132, + TARGET_PCS25G_BR = 144, + TARGET_PCS5G_BR = 160, + TARGET_PORT_CONF = 173, + TARGET_QFWD = 175, + TARGET_QRES = 176, + TARGET_QS = 177, + TARGET_QSYS = 178, + TARGET_REW = 179, + TARGET_VCAP_SUPER = 326, + TARGET_VOP = 327, + TARGET_XQS = 331, + NUM_TARGETS = 332 +}; + +#define __REG(...) __VA_ARGS__ + +/* ANA_AC:RAM_CTRL:RAM_INIT */ +#define ANA_AC_RAM_INIT __REG(TARGET_ANA_AC, 0, 1, 839108, 0, 1, 4, 0, 0, 1, 4) + +#define ANA_AC_RAM_INIT_RAM_INIT BIT(1) +#define ANA_AC_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(ANA_AC_RAM_INIT_RAM_INIT, x) +#define ANA_AC_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(ANA_AC_RAM_INIT_RAM_INIT, x) + +#define ANA_AC_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define ANA_AC_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(ANA_AC_RAM_INIT_RAM_CFG_HOOK, x) +#define ANA_AC_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(ANA_AC_RAM_INIT_RAM_CFG_HOOK, x) + +/* ANA_AC:PS_COMMON:OWN_UPSID */ +#define ANA_AC_OWN_UPSID(r) __REG(TARGET_ANA_AC, 0, 1, 894472, 0, 1, 352, 52, r, 3, 4) + +#define ANA_AC_OWN_UPSID_OWN_UPSID GENMASK(4, 0) +#define ANA_AC_OWN_UPSID_OWN_UPSID_SET(x)\ + FIELD_PREP(ANA_AC_OWN_UPSID_OWN_UPSID, x) +#define ANA_AC_OWN_UPSID_OWN_UPSID_GET(x)\ + FIELD_GET(ANA_AC_OWN_UPSID_OWN_UPSID, x) + +/* ANA_AC:SRC:SRC_CFG */ +#define ANA_AC_SRC_CFG(g) __REG(TARGET_ANA_AC, 0, 1, 849920, g, 102, 16, 0, 0, 1, 4) + +/* ANA_AC:SRC:SRC_CFG1 */ +#define ANA_AC_SRC_CFG1(g) __REG(TARGET_ANA_AC, 0, 1, 849920, g, 102, 16, 4, 0, 1, 4) + +/* ANA_AC:SRC:SRC_CFG2 */ +#define ANA_AC_SRC_CFG2(g) __REG(TARGET_ANA_AC, 0, 1, 849920, g, 102, 16, 8, 0, 1, 4) + +#define ANA_AC_SRC_CFG2_PORT_MASK2 BIT(0) +#define ANA_AC_SRC_CFG2_PORT_MASK2_SET(x)\ + FIELD_PREP(ANA_AC_SRC_CFG2_PORT_MASK2, x) +#define ANA_AC_SRC_CFG2_PORT_MASK2_GET(x)\ + FIELD_GET(ANA_AC_SRC_CFG2_PORT_MASK2, x) + +/* ANA_AC:PGID:PGID_CFG */ +#define ANA_AC_PGID_CFG(g) __REG(TARGET_ANA_AC, 0, 1, 786432, g, 3290, 16, 0, 0, 1, 4) + +/* ANA_AC:PGID:PGID_CFG1 */ +#define ANA_AC_PGID_CFG1(g) __REG(TARGET_ANA_AC, 0, 1, 786432, g, 3290, 16, 4, 0, 1, 4) + +/* ANA_AC:PGID:PGID_CFG2 */ +#define ANA_AC_PGID_CFG2(g) __REG(TARGET_ANA_AC, 0, 1, 786432, g, 3290, 16, 8, 0, 1, 4) + +#define ANA_AC_PGID_CFG2_PORT_MASK2 BIT(0) +#define ANA_AC_PGID_CFG2_PORT_MASK2_SET(x)\ + FIELD_PREP(ANA_AC_PGID_CFG2_PORT_MASK2, x) +#define ANA_AC_PGID_CFG2_PORT_MASK2_GET(x)\ + FIELD_GET(ANA_AC_PGID_CFG2_PORT_MASK2, x) + +/* ANA_AC:PGID:PGID_MISC_CFG */ +#define ANA_AC_PGID_MISC_CFG(g) __REG(TARGET_ANA_AC, 0, 1, 786432, g, 3290, 16, 12, 0, 1, 4) + +#define ANA_AC_PGID_MISC_CFG_PGID_CPU_QU GENMASK(6, 4) +#define ANA_AC_PGID_MISC_CFG_PGID_CPU_QU_SET(x)\ + FIELD_PREP(ANA_AC_PGID_MISC_CFG_PGID_CPU_QU, x) +#define ANA_AC_PGID_MISC_CFG_PGID_CPU_QU_GET(x)\ + FIELD_GET(ANA_AC_PGID_MISC_CFG_PGID_CPU_QU, x) + +#define ANA_AC_PGID_MISC_CFG_STACK_TYPE_ENA BIT(1) +#define ANA_AC_PGID_MISC_CFG_STACK_TYPE_ENA_SET(x)\ + FIELD_PREP(ANA_AC_PGID_MISC_CFG_STACK_TYPE_ENA, x) +#define ANA_AC_PGID_MISC_CFG_STACK_TYPE_ENA_GET(x)\ + FIELD_GET(ANA_AC_PGID_MISC_CFG_STACK_TYPE_ENA, x) + +#define ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA BIT(0) +#define ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(x)\ + FIELD_PREP(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, x) +#define ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_GET(x)\ + FIELD_GET(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, x) + +/* ANA_AC:STAT_GLOBAL_CFG_PORT:STAT_GLOBAL_EVENT_MASK */ +#define ANA_AC_PORT_SGE_CFG(r) __REG(TARGET_ANA_AC, 0, 1, 851552, 0, 1, 20, 0, r, 4, 4) + +#define ANA_AC_PORT_SGE_CFG_MASK GENMASK(15, 0) +#define ANA_AC_PORT_SGE_CFG_MASK_SET(x)\ + FIELD_PREP(ANA_AC_PORT_SGE_CFG_MASK, x) +#define ANA_AC_PORT_SGE_CFG_MASK_GET(x)\ + FIELD_GET(ANA_AC_PORT_SGE_CFG_MASK, x) + +/* ANA_AC:STAT_GLOBAL_CFG_PORT:STAT_RESET */ +#define ANA_AC_STAT_RESET __REG(TARGET_ANA_AC, 0, 1, 851552, 0, 1, 20, 16, 0, 1, 4) + +#define ANA_AC_STAT_RESET_RESET BIT(0) +#define ANA_AC_STAT_RESET_RESET_SET(x)\ + FIELD_PREP(ANA_AC_STAT_RESET_RESET, x) +#define ANA_AC_STAT_RESET_RESET_GET(x)\ + FIELD_GET(ANA_AC_STAT_RESET_RESET, x) + +/* ANA_AC:STAT_CNT_CFG_PORT:STAT_CFG */ +#define ANA_AC_PORT_STAT_CFG(g, r) __REG(TARGET_ANA_AC, 0, 1, 843776, g, 70, 64, 4, r, 4, 4) + +#define ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK GENMASK(11, 4) +#define ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK_SET(x)\ + FIELD_PREP(ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK, x) +#define ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK_GET(x)\ + FIELD_GET(ANA_AC_PORT_STAT_CFG_CFG_PRIO_MASK, x) + +#define ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE GENMASK(3, 1) +#define ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE_SET(x)\ + FIELD_PREP(ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE, x) +#define ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE_GET(x)\ + FIELD_GET(ANA_AC_PORT_STAT_CFG_CFG_CNT_FRM_TYPE, x) + +#define ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE BIT(0) +#define ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE_SET(x)\ + FIELD_PREP(ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE, x) +#define ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE_GET(x)\ + FIELD_GET(ANA_AC_PORT_STAT_CFG_CFG_CNT_BYTE, x) + +/* ANA_AC:STAT_CNT_CFG_PORT:STAT_LSB_CNT */ +#define ANA_AC_PORT_STAT_LSB_CNT(g, r) __REG(TARGET_ANA_AC, 0, 1, 843776, g, 70, 64, 20, r, 4, 4) + +/* ANA_ACL:COMMON:OWN_UPSID */ +#define ANA_ACL_OWN_UPSID(r) __REG(TARGET_ANA_ACL, 0, 1, 32768, 0, 1, 592, 580, r, 3, 4) + +#define ANA_ACL_OWN_UPSID_OWN_UPSID GENMASK(4, 0) +#define ANA_ACL_OWN_UPSID_OWN_UPSID_SET(x)\ + FIELD_PREP(ANA_ACL_OWN_UPSID_OWN_UPSID, x) +#define ANA_ACL_OWN_UPSID_OWN_UPSID_GET(x)\ + FIELD_GET(ANA_ACL_OWN_UPSID_OWN_UPSID, x) + +/* ANA_AC_POL:POL_ALL_CFG:POL_UPD_INT_CFG */ +#define ANA_AC_POL_POL_UPD_INT_CFG __REG(TARGET_ANA_AC_POL, 0, 1, 75968, 0, 1, 1160, 1148, 0, 1, 4) + +#define ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT GENMASK(9, 0) +#define ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT_SET(x)\ + FIELD_PREP(ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT, x) +#define ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT_GET(x)\ + FIELD_GET(ANA_AC_POL_POL_UPD_INT_CFG_POL_UPD_INT, x) + +/* ANA_AC_POL:COMMON_BDLB:DLB_CTRL */ +#define ANA_AC_POL_BDLB_DLB_CTRL __REG(TARGET_ANA_AC_POL, 0, 1, 79048, 0, 1, 8, 0, 0, 1, 4) + +#define ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS GENMASK(26, 19) +#define ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS_SET(x)\ + FIELD_PREP(ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS, x) +#define ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS_GET(x)\ + FIELD_GET(ANA_AC_POL_BDLB_DLB_CTRL_CLK_PERIOD_01NS, x) + +#define ANA_AC_POL_BDLB_DLB_CTRL_BASE_TICK_CNT GENMASK(18, 4) +#define ANA_AC_POL_BDLB_DLB_CTRL_BASE_TICK_CNT_SET(x)\ + FIELD_PREP(ANA_AC_POL_BDLB_DLB_CTRL_BASE_TICK_CNT, x) +#define ANA_AC_POL_BDLB_DLB_CTRL_BASE_TICK_CNT_GET(x)\ + FIELD_GET(ANA_AC_POL_BDLB_DLB_CTRL_BASE_TICK_CNT, x) + +#define ANA_AC_POL_BDLB_DLB_CTRL_LEAK_ENA BIT(1) +#define ANA_AC_POL_BDLB_DLB_CTRL_LEAK_ENA_SET(x)\ + FIELD_PREP(ANA_AC_POL_BDLB_DLB_CTRL_LEAK_ENA, x) +#define ANA_AC_POL_BDLB_DLB_CTRL_LEAK_ENA_GET(x)\ + FIELD_GET(ANA_AC_POL_BDLB_DLB_CTRL_LEAK_ENA, x) + +#define ANA_AC_POL_BDLB_DLB_CTRL_DLB_ADD_ENA BIT(0) +#define ANA_AC_POL_BDLB_DLB_CTRL_DLB_ADD_ENA_SET(x)\ + FIELD_PREP(ANA_AC_POL_BDLB_DLB_CTRL_DLB_ADD_ENA, x) +#define ANA_AC_POL_BDLB_DLB_CTRL_DLB_ADD_ENA_GET(x)\ + FIELD_GET(ANA_AC_POL_BDLB_DLB_CTRL_DLB_ADD_ENA, x) + +/* ANA_AC_POL:COMMON_BUM_SLB:DLB_CTRL */ +#define ANA_AC_POL_SLB_DLB_CTRL __REG(TARGET_ANA_AC_POL, 0, 1, 79056, 0, 1, 20, 0, 0, 1, 4) + +#define ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS GENMASK(26, 19) +#define ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS_SET(x)\ + FIELD_PREP(ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS, x) +#define ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS_GET(x)\ + FIELD_GET(ANA_AC_POL_SLB_DLB_CTRL_CLK_PERIOD_01NS, x) + +#define ANA_AC_POL_SLB_DLB_CTRL_BASE_TICK_CNT GENMASK(18, 4) +#define ANA_AC_POL_SLB_DLB_CTRL_BASE_TICK_CNT_SET(x)\ + FIELD_PREP(ANA_AC_POL_SLB_DLB_CTRL_BASE_TICK_CNT, x) +#define ANA_AC_POL_SLB_DLB_CTRL_BASE_TICK_CNT_GET(x)\ + FIELD_GET(ANA_AC_POL_SLB_DLB_CTRL_BASE_TICK_CNT, x) + +#define ANA_AC_POL_SLB_DLB_CTRL_LEAK_ENA BIT(1) +#define ANA_AC_POL_SLB_DLB_CTRL_LEAK_ENA_SET(x)\ + FIELD_PREP(ANA_AC_POL_SLB_DLB_CTRL_LEAK_ENA, x) +#define ANA_AC_POL_SLB_DLB_CTRL_LEAK_ENA_GET(x)\ + FIELD_GET(ANA_AC_POL_SLB_DLB_CTRL_LEAK_ENA, x) + +#define ANA_AC_POL_SLB_DLB_CTRL_DLB_ADD_ENA BIT(0) +#define ANA_AC_POL_SLB_DLB_CTRL_DLB_ADD_ENA_SET(x)\ + FIELD_PREP(ANA_AC_POL_SLB_DLB_CTRL_DLB_ADD_ENA, x) +#define ANA_AC_POL_SLB_DLB_CTRL_DLB_ADD_ENA_GET(x)\ + FIELD_GET(ANA_AC_POL_SLB_DLB_CTRL_DLB_ADD_ENA, x) + +/* ANA_CL:PORT:FILTER_CTRL */ +#define ANA_CL_FILTER_CTRL(g) __REG(TARGET_ANA_CL, 0, 1, 131072, g, 70, 512, 4, 0, 1, 4) + +#define ANA_CL_FILTER_CTRL_FILTER_SMAC_MC_DIS BIT(2) +#define ANA_CL_FILTER_CTRL_FILTER_SMAC_MC_DIS_SET(x)\ + FIELD_PREP(ANA_CL_FILTER_CTRL_FILTER_SMAC_MC_DIS, x) +#define ANA_CL_FILTER_CTRL_FILTER_SMAC_MC_DIS_GET(x)\ + FIELD_GET(ANA_CL_FILTER_CTRL_FILTER_SMAC_MC_DIS, x) + +#define ANA_CL_FILTER_CTRL_FILTER_NULL_MAC_DIS BIT(1) +#define ANA_CL_FILTER_CTRL_FILTER_NULL_MAC_DIS_SET(x)\ + FIELD_PREP(ANA_CL_FILTER_CTRL_FILTER_NULL_MAC_DIS, x) +#define ANA_CL_FILTER_CTRL_FILTER_NULL_MAC_DIS_GET(x)\ + FIELD_GET(ANA_CL_FILTER_CTRL_FILTER_NULL_MAC_DIS, x) + +#define ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA BIT(0) +#define ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA_SET(x)\ + FIELD_PREP(ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA, x) +#define ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA_GET(x)\ + FIELD_GET(ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA, x) + +/* ANA_CL:PORT:VLAN_FILTER_CTRL */ +#define ANA_CL_VLAN_FILTER_CTRL(g, r) __REG(TARGET_ANA_CL, 0, 1, 131072, g, 70, 512, 8, r, 3, 4) + +#define ANA_CL_VLAN_FILTER_CTRL_TAG_REQUIRED_ENA BIT(10) +#define ANA_CL_VLAN_FILTER_CTRL_TAG_REQUIRED_ENA_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_TAG_REQUIRED_ENA, x) +#define ANA_CL_VLAN_FILTER_CTRL_TAG_REQUIRED_ENA_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_TAG_REQUIRED_ENA, x) + +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CTAG_DIS BIT(9) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CTAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_PRIO_CTAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CTAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_PRIO_CTAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_CTAG_DIS BIT(8) +#define ANA_CL_VLAN_FILTER_CTRL_CTAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_CTAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_CTAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_CTAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_STAG_DIS BIT(7) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_PRIO_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_PRIO_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST1_STAG_DIS BIT(6) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST1_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST1_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST1_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST1_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST2_STAG_DIS BIT(5) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST2_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST2_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST2_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST2_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST3_STAG_DIS BIT(4) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST3_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST3_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST3_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_PRIO_CUST3_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_STAG_DIS BIT(3) +#define ANA_CL_VLAN_FILTER_CTRL_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_CUST1_STAG_DIS BIT(2) +#define ANA_CL_VLAN_FILTER_CTRL_CUST1_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_CUST1_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_CUST1_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_CUST1_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_CUST2_STAG_DIS BIT(1) +#define ANA_CL_VLAN_FILTER_CTRL_CUST2_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_CUST2_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_CUST2_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_CUST2_STAG_DIS, x) + +#define ANA_CL_VLAN_FILTER_CTRL_CUST3_STAG_DIS BIT(0) +#define ANA_CL_VLAN_FILTER_CTRL_CUST3_STAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_FILTER_CTRL_CUST3_STAG_DIS, x) +#define ANA_CL_VLAN_FILTER_CTRL_CUST3_STAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_FILTER_CTRL_CUST3_STAG_DIS, x) + +/* ANA_CL:PORT:ETAG_FILTER_CTRL */ +#define ANA_CL_ETAG_FILTER_CTRL(g) __REG(TARGET_ANA_CL, 0, 1, 131072, g, 70, 512, 20, 0, 1, 4) + +#define ANA_CL_ETAG_FILTER_CTRL_ETAG_REQUIRED_ENA BIT(1) +#define ANA_CL_ETAG_FILTER_CTRL_ETAG_REQUIRED_ENA_SET(x)\ + FIELD_PREP(ANA_CL_ETAG_FILTER_CTRL_ETAG_REQUIRED_ENA, x) +#define ANA_CL_ETAG_FILTER_CTRL_ETAG_REQUIRED_ENA_GET(x)\ + FIELD_GET(ANA_CL_ETAG_FILTER_CTRL_ETAG_REQUIRED_ENA, x) + +#define ANA_CL_ETAG_FILTER_CTRL_ETAG_DIS BIT(0) +#define ANA_CL_ETAG_FILTER_CTRL_ETAG_DIS_SET(x)\ + FIELD_PREP(ANA_CL_ETAG_FILTER_CTRL_ETAG_DIS, x) +#define ANA_CL_ETAG_FILTER_CTRL_ETAG_DIS_GET(x)\ + FIELD_GET(ANA_CL_ETAG_FILTER_CTRL_ETAG_DIS, x) + +/* ANA_CL:PORT:VLAN_CTRL */ +#define ANA_CL_VLAN_CTRL(g) __REG(TARGET_ANA_CL, 0, 1, 131072, g, 70, 512, 32, 0, 1, 4) + +#define ANA_CL_VLAN_CTRL_PORT_VOE_TPID_AWARE_DIS GENMASK(30, 26) +#define ANA_CL_VLAN_CTRL_PORT_VOE_TPID_AWARE_DIS_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_VOE_TPID_AWARE_DIS, x) +#define ANA_CL_VLAN_CTRL_PORT_VOE_TPID_AWARE_DIS_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_VOE_TPID_AWARE_DIS, x) + +#define ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_PCP GENMASK(25, 23) +#define ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_PCP_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_PCP, x) +#define ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_PCP_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_PCP, x) + +#define ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_DEI BIT(22) +#define ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_DEI_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_DEI, x) +#define ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_DEI_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_VOE_DEFAULT_DEI, x) + +#define ANA_CL_VLAN_CTRL_VLAN_PCP_DEI_TRANS_ENA BIT(21) +#define ANA_CL_VLAN_CTRL_VLAN_PCP_DEI_TRANS_ENA_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_VLAN_PCP_DEI_TRANS_ENA, x) +#define ANA_CL_VLAN_CTRL_VLAN_PCP_DEI_TRANS_ENA_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_VLAN_PCP_DEI_TRANS_ENA, x) + +#define ANA_CL_VLAN_CTRL_VLAN_TAG_SEL BIT(20) +#define ANA_CL_VLAN_CTRL_VLAN_TAG_SEL_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_VLAN_TAG_SEL, x) +#define ANA_CL_VLAN_CTRL_VLAN_TAG_SEL_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_VLAN_TAG_SEL, x) + +#define ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA BIT(19) +#define ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA, x) +#define ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA, x) + +#define ANA_CL_VLAN_CTRL_VLAN_POP_CNT GENMASK(18, 17) +#define ANA_CL_VLAN_CTRL_VLAN_POP_CNT_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_VLAN_POP_CNT, x) +#define ANA_CL_VLAN_CTRL_VLAN_POP_CNT_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_VLAN_POP_CNT, x) + +#define ANA_CL_VLAN_CTRL_PORT_TAG_TYPE BIT(16) +#define ANA_CL_VLAN_CTRL_PORT_TAG_TYPE_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_TAG_TYPE, x) +#define ANA_CL_VLAN_CTRL_PORT_TAG_TYPE_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_TAG_TYPE, x) + +#define ANA_CL_VLAN_CTRL_PORT_PCP GENMASK(15, 13) +#define ANA_CL_VLAN_CTRL_PORT_PCP_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_PCP, x) +#define ANA_CL_VLAN_CTRL_PORT_PCP_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_PCP, x) + +#define ANA_CL_VLAN_CTRL_PORT_DEI BIT(12) +#define ANA_CL_VLAN_CTRL_PORT_DEI_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_DEI, x) +#define ANA_CL_VLAN_CTRL_PORT_DEI_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_DEI, x) + +#define ANA_CL_VLAN_CTRL_PORT_VID GENMASK(11, 0) +#define ANA_CL_VLAN_CTRL_PORT_VID_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_PORT_VID, x) +#define ANA_CL_VLAN_CTRL_PORT_VID_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_PORT_VID, x) + +/* ANA_CL:PORT:VLAN_CTRL_2 */ +#define ANA_CL_VLAN_CTRL_2(g) __REG(TARGET_ANA_CL, 0, 1, 131072, g, 70, 512, 36, 0, 1, 4) + +#define ANA_CL_VLAN_CTRL_2_VLAN_PUSH_CNT GENMASK(1, 0) +#define ANA_CL_VLAN_CTRL_2_VLAN_PUSH_CNT_SET(x)\ + FIELD_PREP(ANA_CL_VLAN_CTRL_2_VLAN_PUSH_CNT, x) +#define ANA_CL_VLAN_CTRL_2_VLAN_PUSH_CNT_GET(x)\ + FIELD_GET(ANA_CL_VLAN_CTRL_2_VLAN_PUSH_CNT, x) + +/* ANA_CL:PORT:CAPTURE_BPDU_CFG */ +#define ANA_CL_CAPTURE_BPDU_CFG(g) __REG(TARGET_ANA_CL, 0, 1, 131072, g, 70, 512, 196, 0, 1, 4) + +/* ANA_CL:COMMON:OWN_UPSID */ +#define ANA_CL_OWN_UPSID(r) __REG(TARGET_ANA_CL, 0, 1, 166912, 0, 1, 756, 0, r, 3, 4) + +#define ANA_CL_OWN_UPSID_OWN_UPSID GENMASK(4, 0) +#define ANA_CL_OWN_UPSID_OWN_UPSID_SET(x)\ + FIELD_PREP(ANA_CL_OWN_UPSID_OWN_UPSID, x) +#define ANA_CL_OWN_UPSID_OWN_UPSID_GET(x)\ + FIELD_GET(ANA_CL_OWN_UPSID_OWN_UPSID, x) + +/* ANA_L2:COMMON:AUTO_LRN_CFG */ +#define ANA_L2_AUTO_LRN_CFG __REG(TARGET_ANA_L2, 0, 1, 566024, 0, 1, 700, 24, 0, 1, 4) + +/* ANA_L2:COMMON:AUTO_LRN_CFG1 */ +#define ANA_L2_AUTO_LRN_CFG1 __REG(TARGET_ANA_L2, 0, 1, 566024, 0, 1, 700, 28, 0, 1, 4) + +/* ANA_L2:COMMON:AUTO_LRN_CFG2 */ +#define ANA_L2_AUTO_LRN_CFG2 __REG(TARGET_ANA_L2, 0, 1, 566024, 0, 1, 700, 32, 0, 1, 4) + +#define ANA_L2_AUTO_LRN_CFG2_AUTO_LRN_ENA2 BIT(0) +#define ANA_L2_AUTO_LRN_CFG2_AUTO_LRN_ENA2_SET(x)\ + FIELD_PREP(ANA_L2_AUTO_LRN_CFG2_AUTO_LRN_ENA2, x) +#define ANA_L2_AUTO_LRN_CFG2_AUTO_LRN_ENA2_GET(x)\ + FIELD_GET(ANA_L2_AUTO_LRN_CFG2_AUTO_LRN_ENA2, x) + +/* ANA_L2:COMMON:OWN_UPSID */ +#define ANA_L2_OWN_UPSID(r) __REG(TARGET_ANA_L2, 0, 1, 566024, 0, 1, 700, 672, r, 3, 4) + +#define ANA_L2_OWN_UPSID_OWN_UPSID GENMASK(4, 0) +#define ANA_L2_OWN_UPSID_OWN_UPSID_SET(x)\ + FIELD_PREP(ANA_L2_OWN_UPSID_OWN_UPSID, x) +#define ANA_L2_OWN_UPSID_OWN_UPSID_GET(x)\ + FIELD_GET(ANA_L2_OWN_UPSID_OWN_UPSID, x) + +/* ANA_L3:COMMON:VLAN_CTRL */ +#define ANA_L3_VLAN_CTRL __REG(TARGET_ANA_L3, 0, 1, 493632, 0, 1, 184, 4, 0, 1, 4) + +#define ANA_L3_VLAN_CTRL_VLAN_ENA BIT(0) +#define ANA_L3_VLAN_CTRL_VLAN_ENA_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CTRL_VLAN_ENA, x) +#define ANA_L3_VLAN_CTRL_VLAN_ENA_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CTRL_VLAN_ENA, x) + +/* ANA_L3:VLAN:VLAN_CFG */ +#define ANA_L3_VLAN_CFG(g) __REG(TARGET_ANA_L3, 0, 1, 0, g, 5120, 64, 8, 0, 1, 4) + +#define ANA_L3_VLAN_CFG_VLAN_MSTP_PTR GENMASK(30, 24) +#define ANA_L3_VLAN_CFG_VLAN_MSTP_PTR_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_MSTP_PTR, x) +#define ANA_L3_VLAN_CFG_VLAN_MSTP_PTR_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_MSTP_PTR, x) + +#define ANA_L3_VLAN_CFG_VLAN_FID GENMASK(20, 8) +#define ANA_L3_VLAN_CFG_VLAN_FID_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_FID, x) +#define ANA_L3_VLAN_CFG_VLAN_FID_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_FID, x) + +#define ANA_L3_VLAN_CFG_VLAN_IGR_FILTER_ENA BIT(6) +#define ANA_L3_VLAN_CFG_VLAN_IGR_FILTER_ENA_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_IGR_FILTER_ENA, x) +#define ANA_L3_VLAN_CFG_VLAN_IGR_FILTER_ENA_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_IGR_FILTER_ENA, x) + +#define ANA_L3_VLAN_CFG_VLAN_SEC_FWD_ENA BIT(5) +#define ANA_L3_VLAN_CFG_VLAN_SEC_FWD_ENA_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_SEC_FWD_ENA, x) +#define ANA_L3_VLAN_CFG_VLAN_SEC_FWD_ENA_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_SEC_FWD_ENA, x) + +#define ANA_L3_VLAN_CFG_VLAN_FLOOD_DIS BIT(4) +#define ANA_L3_VLAN_CFG_VLAN_FLOOD_DIS_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_FLOOD_DIS, x) +#define ANA_L3_VLAN_CFG_VLAN_FLOOD_DIS_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_FLOOD_DIS, x) + +#define ANA_L3_VLAN_CFG_VLAN_LRN_DIS BIT(3) +#define ANA_L3_VLAN_CFG_VLAN_LRN_DIS_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_LRN_DIS, x) +#define ANA_L3_VLAN_CFG_VLAN_LRN_DIS_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_LRN_DIS, x) + +#define ANA_L3_VLAN_CFG_VLAN_RLEG_ENA BIT(2) +#define ANA_L3_VLAN_CFG_VLAN_RLEG_ENA_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_RLEG_ENA, x) +#define ANA_L3_VLAN_CFG_VLAN_RLEG_ENA_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_RLEG_ENA, x) + +#define ANA_L3_VLAN_CFG_VLAN_PRIVATE_ENA BIT(1) +#define ANA_L3_VLAN_CFG_VLAN_PRIVATE_ENA_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_PRIVATE_ENA, x) +#define ANA_L3_VLAN_CFG_VLAN_PRIVATE_ENA_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_PRIVATE_ENA, x) + +#define ANA_L3_VLAN_CFG_VLAN_MIRROR_ENA BIT(0) +#define ANA_L3_VLAN_CFG_VLAN_MIRROR_ENA_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_CFG_VLAN_MIRROR_ENA, x) +#define ANA_L3_VLAN_CFG_VLAN_MIRROR_ENA_GET(x)\ + FIELD_GET(ANA_L3_VLAN_CFG_VLAN_MIRROR_ENA, x) + +/* ANA_L3:VLAN:VLAN_MASK_CFG */ +#define ANA_L3_VLAN_MASK_CFG(g) __REG(TARGET_ANA_L3, 0, 1, 0, g, 5120, 64, 16, 0, 1, 4) + +/* ANA_L3:VLAN:VLAN_MASK_CFG1 */ +#define ANA_L3_VLAN_MASK_CFG1(g) __REG(TARGET_ANA_L3, 0, 1, 0, g, 5120, 64, 20, 0, 1, 4) + +/* ANA_L3:VLAN:VLAN_MASK_CFG2 */ +#define ANA_L3_VLAN_MASK_CFG2(g) __REG(TARGET_ANA_L3, 0, 1, 0, g, 5120, 64, 24, 0, 1, 4) + +#define ANA_L3_VLAN_MASK_CFG2_VLAN_PORT_MASK2 BIT(0) +#define ANA_L3_VLAN_MASK_CFG2_VLAN_PORT_MASK2_SET(x)\ + FIELD_PREP(ANA_L3_VLAN_MASK_CFG2_VLAN_PORT_MASK2, x) +#define ANA_L3_VLAN_MASK_CFG2_VLAN_PORT_MASK2_GET(x)\ + FIELD_GET(ANA_L3_VLAN_MASK_CFG2_VLAN_PORT_MASK2, x) + +/* ASM:DEV_STATISTICS:RX_IN_BYTES_CNT */ +#define ASM_RX_IN_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 0, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SYMBOL_ERR_CNT */ +#define ASM_RX_SYMBOL_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 4, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_PAUSE_CNT */ +#define ASM_RX_PAUSE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 8, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_UNSUP_OPCODE_CNT */ +#define ASM_RX_UNSUP_OPCODE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 12, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_OK_BYTES_CNT */ +#define ASM_RX_OK_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 16, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_BAD_BYTES_CNT */ +#define ASM_RX_BAD_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 20, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_UC_CNT */ +#define ASM_RX_UC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 24, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_MC_CNT */ +#define ASM_RX_MC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 28, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_BC_CNT */ +#define ASM_RX_BC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 32, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_CRC_ERR_CNT */ +#define ASM_RX_CRC_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 36, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_UNDERSIZE_CNT */ +#define ASM_RX_UNDERSIZE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 40, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_FRAGMENTS_CNT */ +#define ASM_RX_FRAGMENTS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 44, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_IN_RANGE_LEN_ERR_CNT */ +#define ASM_RX_IN_RANGE_LEN_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 48, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_OUT_OF_RANGE_LEN_ERR_CNT */ +#define ASM_RX_OUT_OF_RANGE_LEN_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 52, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_OVERSIZE_CNT */ +#define ASM_RX_OVERSIZE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 56, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_JABBERS_CNT */ +#define ASM_RX_JABBERS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 60, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE64_CNT */ +#define ASM_RX_SIZE64_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 64, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE65TO127_CNT */ +#define ASM_RX_SIZE65TO127_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 68, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE128TO255_CNT */ +#define ASM_RX_SIZE128TO255_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 72, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE256TO511_CNT */ +#define ASM_RX_SIZE256TO511_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 76, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE512TO1023_CNT */ +#define ASM_RX_SIZE512TO1023_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 80, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE1024TO1518_CNT */ +#define ASM_RX_SIZE1024TO1518_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 84, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_SIZE1519TOMAX_CNT */ +#define ASM_RX_SIZE1519TOMAX_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 88, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_IPG_SHRINK_CNT */ +#define ASM_RX_IPG_SHRINK_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 92, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_OUT_BYTES_CNT */ +#define ASM_TX_OUT_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 96, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_PAUSE_CNT */ +#define ASM_TX_PAUSE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 100, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_OK_BYTES_CNT */ +#define ASM_TX_OK_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 104, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_UC_CNT */ +#define ASM_TX_UC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 108, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_MC_CNT */ +#define ASM_TX_MC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 112, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_BC_CNT */ +#define ASM_TX_BC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 116, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE64_CNT */ +#define ASM_TX_SIZE64_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 120, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE65TO127_CNT */ +#define ASM_TX_SIZE65TO127_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 124, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE128TO255_CNT */ +#define ASM_TX_SIZE128TO255_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 128, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE256TO511_CNT */ +#define ASM_TX_SIZE256TO511_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 132, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE512TO1023_CNT */ +#define ASM_TX_SIZE512TO1023_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 136, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE1024TO1518_CNT */ +#define ASM_TX_SIZE1024TO1518_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 140, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_SIZE1519TOMAX_CNT */ +#define ASM_TX_SIZE1519TOMAX_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 144, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_ALIGNMENT_LOST_CNT */ +#define ASM_RX_ALIGNMENT_LOST_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 148, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_TAGGED_FRMS_CNT */ +#define ASM_RX_TAGGED_FRMS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 152, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_UNTAGGED_FRMS_CNT */ +#define ASM_RX_UNTAGGED_FRMS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 156, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_TAGGED_FRMS_CNT */ +#define ASM_TX_TAGGED_FRMS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 160, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_UNTAGGED_FRMS_CNT */ +#define ASM_TX_UNTAGGED_FRMS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 164, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SYMBOL_ERR_CNT */ +#define ASM_PMAC_RX_SYMBOL_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 168, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_PAUSE_CNT */ +#define ASM_PMAC_RX_PAUSE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 172, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_UNSUP_OPCODE_CNT */ +#define ASM_PMAC_RX_UNSUP_OPCODE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 176, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_OK_BYTES_CNT */ +#define ASM_PMAC_RX_OK_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 180, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_BAD_BYTES_CNT */ +#define ASM_PMAC_RX_BAD_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 184, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_UC_CNT */ +#define ASM_PMAC_RX_UC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 188, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_MC_CNT */ +#define ASM_PMAC_RX_MC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 192, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_BC_CNT */ +#define ASM_PMAC_RX_BC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 196, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_CRC_ERR_CNT */ +#define ASM_PMAC_RX_CRC_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 200, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_UNDERSIZE_CNT */ +#define ASM_PMAC_RX_UNDERSIZE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 204, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_FRAGMENTS_CNT */ +#define ASM_PMAC_RX_FRAGMENTS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 208, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_IN_RANGE_LEN_ERR_CNT */ +#define ASM_PMAC_RX_IN_RANGE_LEN_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 212, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_OUT_OF_RANGE_LEN_ERR_CNT */ +#define ASM_PMAC_RX_OUT_OF_RANGE_LEN_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 216, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_OVERSIZE_CNT */ +#define ASM_PMAC_RX_OVERSIZE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 220, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_JABBERS_CNT */ +#define ASM_PMAC_RX_JABBERS_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 224, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE64_CNT */ +#define ASM_PMAC_RX_SIZE64_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 228, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE65TO127_CNT */ +#define ASM_PMAC_RX_SIZE65TO127_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 232, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE128TO255_CNT */ +#define ASM_PMAC_RX_SIZE128TO255_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 236, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE256TO511_CNT */ +#define ASM_PMAC_RX_SIZE256TO511_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 240, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE512TO1023_CNT */ +#define ASM_PMAC_RX_SIZE512TO1023_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 244, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE1024TO1518_CNT */ +#define ASM_PMAC_RX_SIZE1024TO1518_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 248, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_SIZE1519TOMAX_CNT */ +#define ASM_PMAC_RX_SIZE1519TOMAX_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 252, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_PAUSE_CNT */ +#define ASM_PMAC_TX_PAUSE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 256, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_OK_BYTES_CNT */ +#define ASM_PMAC_TX_OK_BYTES_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 260, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_UC_CNT */ +#define ASM_PMAC_TX_UC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 264, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_MC_CNT */ +#define ASM_PMAC_TX_MC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 268, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_BC_CNT */ +#define ASM_PMAC_TX_BC_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 272, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE64_CNT */ +#define ASM_PMAC_TX_SIZE64_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 276, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE65TO127_CNT */ +#define ASM_PMAC_TX_SIZE65TO127_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 280, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE128TO255_CNT */ +#define ASM_PMAC_TX_SIZE128TO255_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 284, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE256TO511_CNT */ +#define ASM_PMAC_TX_SIZE256TO511_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 288, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE512TO1023_CNT */ +#define ASM_PMAC_TX_SIZE512TO1023_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 292, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE1024TO1518_CNT */ +#define ASM_PMAC_TX_SIZE1024TO1518_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 296, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_TX_SIZE1519TOMAX_CNT */ +#define ASM_PMAC_TX_SIZE1519TOMAX_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 300, 0, 1, 4) + +/* ASM:DEV_STATISTICS:PMAC_RX_ALIGNMENT_LOST_CNT */ +#define ASM_PMAC_RX_ALIGNMENT_LOST_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 304, 0, 1, 4) + +/* ASM:DEV_STATISTICS:MM_RX_ASSEMBLY_ERR_CNT */ +#define ASM_MM_RX_ASSEMBLY_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 308, 0, 1, 4) + +/* ASM:DEV_STATISTICS:MM_RX_SMD_ERR_CNT */ +#define ASM_MM_RX_SMD_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 312, 0, 1, 4) + +/* ASM:DEV_STATISTICS:MM_RX_ASSEMBLY_OK_CNT */ +#define ASM_MM_RX_ASSEMBLY_OK_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 316, 0, 1, 4) + +/* ASM:DEV_STATISTICS:MM_RX_MERGE_FRAG_CNT */ +#define ASM_MM_RX_MERGE_FRAG_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 320, 0, 1, 4) + +/* ASM:DEV_STATISTICS:MM_TX_PFRAGMENT_CNT */ +#define ASM_MM_TX_PFRAGMENT_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 324, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_MULTI_COLL_CNT */ +#define ASM_TX_MULTI_COLL_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 328, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_LATE_COLL_CNT */ +#define ASM_TX_LATE_COLL_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 332, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_XCOLL_CNT */ +#define ASM_TX_XCOLL_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 336, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_DEFER_CNT */ +#define ASM_TX_DEFER_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 340, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_XDEFER_CNT */ +#define ASM_TX_XDEFER_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 344, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_BACKOFF1_CNT */ +#define ASM_TX_BACKOFF1_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 348, 0, 1, 4) + +/* ASM:DEV_STATISTICS:TX_CSENSE_CNT */ +#define ASM_TX_CSENSE_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 352, 0, 1, 4) + +/* ASM:DEV_STATISTICS:RX_IN_BYTES_MSB_CNT */ +#define ASM_RX_IN_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 356, 0, 1, 4) + +#define ASM_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT, x) +#define ASM_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:RX_OK_BYTES_MSB_CNT */ +#define ASM_RX_OK_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 360, 0, 1, 4) + +#define ASM_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT, x) +#define ASM_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:PMAC_RX_OK_BYTES_MSB_CNT */ +#define ASM_PMAC_RX_OK_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 364, 0, 1, 4) + +#define ASM_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT, x) +#define ASM_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:RX_BAD_BYTES_MSB_CNT */ +#define ASM_RX_BAD_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 368, 0, 1, 4) + +#define ASM_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT, x) +#define ASM_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:PMAC_RX_BAD_BYTES_MSB_CNT */ +#define ASM_PMAC_RX_BAD_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 372, 0, 1, 4) + +#define ASM_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT, x) +#define ASM_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:TX_OUT_BYTES_MSB_CNT */ +#define ASM_TX_OUT_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 376, 0, 1, 4) + +#define ASM_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT, x) +#define ASM_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:TX_OK_BYTES_MSB_CNT */ +#define ASM_TX_OK_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 380, 0, 1, 4) + +#define ASM_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT, x) +#define ASM_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:PMAC_TX_OK_BYTES_MSB_CNT */ +#define ASM_PMAC_TX_OK_BYTES_MSB_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 384, 0, 1, 4) + +#define ASM_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT GENMASK(3, 0) +#define ASM_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(ASM_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT, x) +#define ASM_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(ASM_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT, x) + +/* ASM:DEV_STATISTICS:RX_SYNC_LOST_ERR_CNT */ +#define ASM_RX_SYNC_LOST_ERR_CNT(g) __REG(TARGET_ASM, 0, 1, 0, g, 65, 512, 388, 0, 1, 4) + +/* ASM:CFG:STAT_CFG */ +#define ASM_STAT_CFG __REG(TARGET_ASM, 0, 1, 33280, 0, 1, 1088, 0, 0, 1, 4) + +#define ASM_STAT_CFG_STAT_CNT_CLR_SHOT BIT(0) +#define ASM_STAT_CFG_STAT_CNT_CLR_SHOT_SET(x)\ + FIELD_PREP(ASM_STAT_CFG_STAT_CNT_CLR_SHOT, x) +#define ASM_STAT_CFG_STAT_CNT_CLR_SHOT_GET(x)\ + FIELD_GET(ASM_STAT_CFG_STAT_CNT_CLR_SHOT, x) + +/* ASM:CFG:PORT_CFG */ +#define ASM_PORT_CFG(r) __REG(TARGET_ASM, 0, 1, 33280, 0, 1, 1088, 540, r, 67, 4) + +#define ASM_PORT_CFG_CSC_STAT_DIS BIT(12) +#define ASM_PORT_CFG_CSC_STAT_DIS_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_CSC_STAT_DIS, x) +#define ASM_PORT_CFG_CSC_STAT_DIS_GET(x)\ + FIELD_GET(ASM_PORT_CFG_CSC_STAT_DIS, x) + +#define ASM_PORT_CFG_HIH_AFTER_PREAMBLE_ENA BIT(11) +#define ASM_PORT_CFG_HIH_AFTER_PREAMBLE_ENA_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_HIH_AFTER_PREAMBLE_ENA, x) +#define ASM_PORT_CFG_HIH_AFTER_PREAMBLE_ENA_GET(x)\ + FIELD_GET(ASM_PORT_CFG_HIH_AFTER_PREAMBLE_ENA, x) + +#define ASM_PORT_CFG_IGN_TAXI_ABORT_ENA BIT(10) +#define ASM_PORT_CFG_IGN_TAXI_ABORT_ENA_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_IGN_TAXI_ABORT_ENA, x) +#define ASM_PORT_CFG_IGN_TAXI_ABORT_ENA_GET(x)\ + FIELD_GET(ASM_PORT_CFG_IGN_TAXI_ABORT_ENA, x) + +#define ASM_PORT_CFG_NO_PREAMBLE_ENA BIT(9) +#define ASM_PORT_CFG_NO_PREAMBLE_ENA_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_NO_PREAMBLE_ENA, x) +#define ASM_PORT_CFG_NO_PREAMBLE_ENA_GET(x)\ + FIELD_GET(ASM_PORT_CFG_NO_PREAMBLE_ENA, x) + +#define ASM_PORT_CFG_SKIP_PREAMBLE_ENA BIT(8) +#define ASM_PORT_CFG_SKIP_PREAMBLE_ENA_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_SKIP_PREAMBLE_ENA, x) +#define ASM_PORT_CFG_SKIP_PREAMBLE_ENA_GET(x)\ + FIELD_GET(ASM_PORT_CFG_SKIP_PREAMBLE_ENA, x) + +#define ASM_PORT_CFG_FRM_AGING_DIS BIT(7) +#define ASM_PORT_CFG_FRM_AGING_DIS_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_FRM_AGING_DIS, x) +#define ASM_PORT_CFG_FRM_AGING_DIS_GET(x)\ + FIELD_GET(ASM_PORT_CFG_FRM_AGING_DIS, x) + +#define ASM_PORT_CFG_PAD_ENA BIT(6) +#define ASM_PORT_CFG_PAD_ENA_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_PAD_ENA, x) +#define ASM_PORT_CFG_PAD_ENA_GET(x)\ + FIELD_GET(ASM_PORT_CFG_PAD_ENA, x) + +#define ASM_PORT_CFG_INJ_DISCARD_CFG GENMASK(5, 4) +#define ASM_PORT_CFG_INJ_DISCARD_CFG_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_INJ_DISCARD_CFG, x) +#define ASM_PORT_CFG_INJ_DISCARD_CFG_GET(x)\ + FIELD_GET(ASM_PORT_CFG_INJ_DISCARD_CFG, x) + +#define ASM_PORT_CFG_INJ_FORMAT_CFG GENMASK(3, 2) +#define ASM_PORT_CFG_INJ_FORMAT_CFG_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_INJ_FORMAT_CFG, x) +#define ASM_PORT_CFG_INJ_FORMAT_CFG_GET(x)\ + FIELD_GET(ASM_PORT_CFG_INJ_FORMAT_CFG, x) + +#define ASM_PORT_CFG_VSTAX2_AWR_ENA BIT(1) +#define ASM_PORT_CFG_VSTAX2_AWR_ENA_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_VSTAX2_AWR_ENA, x) +#define ASM_PORT_CFG_VSTAX2_AWR_ENA_GET(x)\ + FIELD_GET(ASM_PORT_CFG_VSTAX2_AWR_ENA, x) + +#define ASM_PORT_CFG_PFRM_FLUSH BIT(0) +#define ASM_PORT_CFG_PFRM_FLUSH_SET(x)\ + FIELD_PREP(ASM_PORT_CFG_PFRM_FLUSH, x) +#define ASM_PORT_CFG_PFRM_FLUSH_GET(x)\ + FIELD_GET(ASM_PORT_CFG_PFRM_FLUSH, x) + +/* ASM:RAM_CTRL:RAM_INIT */ +#define ASM_RAM_INIT __REG(TARGET_ASM, 0, 1, 34832, 0, 1, 4, 0, 0, 1, 4) + +#define ASM_RAM_INIT_RAM_INIT BIT(1) +#define ASM_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(ASM_RAM_INIT_RAM_INIT, x) +#define ASM_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(ASM_RAM_INIT_RAM_INIT, x) + +#define ASM_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define ASM_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(ASM_RAM_INIT_RAM_CFG_HOOK, x) +#define ASM_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(ASM_RAM_INIT_RAM_CFG_HOOK, x) + +/* CLKGEN:LCPLL1:LCPLL1_CORE_CLK_CFG */ +#define CLKGEN_LCPLL1_CORE_CLK_CFG __REG(TARGET_CLKGEN, 0, 1, 12, 0, 1, 36, 0, 0, 1, 4) + +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV GENMASK(7, 0) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV_SET(x)\ + FIELD_PREP(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV, x) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV_GET(x)\ + FIELD_GET(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_DIV, x) + +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV GENMASK(10, 8) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV_SET(x)\ + FIELD_PREP(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV, x) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV_GET(x)\ + FIELD_GET(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_PRE_DIV, x) + +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR BIT(11) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR_SET(x)\ + FIELD_PREP(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR, x) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR_GET(x)\ + FIELD_GET(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_DIR, x) + +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL GENMASK(13, 12) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL_SET(x)\ + FIELD_PREP(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL, x) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL_GET(x)\ + FIELD_GET(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_SEL, x) + +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA BIT(14) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA_SET(x)\ + FIELD_PREP(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA, x) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA_GET(x)\ + FIELD_GET(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_ROT_ENA, x) + +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA BIT(15) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA_SET(x)\ + FIELD_PREP(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA, x) +#define CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA_GET(x)\ + FIELD_GET(CLKGEN_LCPLL1_CORE_CLK_CFG_CORE_CLK_ENA, x) + +/* CPU:CPU_REGS:PROC_CTRL */ +#define CPU_PROC_CTRL __REG(TARGET_CPU, 0, 1, 0, 0, 1, 204, 176, 0, 1, 4) + +#define CPU_PROC_CTRL_AARCH64_MODE_ENA BIT(12) +#define CPU_PROC_CTRL_AARCH64_MODE_ENA_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_AARCH64_MODE_ENA, x) +#define CPU_PROC_CTRL_AARCH64_MODE_ENA_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_AARCH64_MODE_ENA, x) + +#define CPU_PROC_CTRL_L2_RST_INVALIDATE_DIS BIT(11) +#define CPU_PROC_CTRL_L2_RST_INVALIDATE_DIS_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_L2_RST_INVALIDATE_DIS, x) +#define CPU_PROC_CTRL_L2_RST_INVALIDATE_DIS_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_L2_RST_INVALIDATE_DIS, x) + +#define CPU_PROC_CTRL_L1_RST_INVALIDATE_DIS BIT(10) +#define CPU_PROC_CTRL_L1_RST_INVALIDATE_DIS_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_L1_RST_INVALIDATE_DIS, x) +#define CPU_PROC_CTRL_L1_RST_INVALIDATE_DIS_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_L1_RST_INVALIDATE_DIS, x) + +#define CPU_PROC_CTRL_BE_EXCEP_MODE BIT(9) +#define CPU_PROC_CTRL_BE_EXCEP_MODE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_BE_EXCEP_MODE, x) +#define CPU_PROC_CTRL_BE_EXCEP_MODE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_BE_EXCEP_MODE, x) + +#define CPU_PROC_CTRL_VINITHI BIT(8) +#define CPU_PROC_CTRL_VINITHI_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_VINITHI, x) +#define CPU_PROC_CTRL_VINITHI_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_VINITHI, x) + +#define CPU_PROC_CTRL_CFGTE BIT(7) +#define CPU_PROC_CTRL_CFGTE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_CFGTE, x) +#define CPU_PROC_CTRL_CFGTE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_CFGTE, x) + +#define CPU_PROC_CTRL_CP15S_DISABLE BIT(6) +#define CPU_PROC_CTRL_CP15S_DISABLE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_CP15S_DISABLE, x) +#define CPU_PROC_CTRL_CP15S_DISABLE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_CP15S_DISABLE, x) + +#define CPU_PROC_CTRL_PROC_CRYPTO_DISABLE BIT(5) +#define CPU_PROC_CTRL_PROC_CRYPTO_DISABLE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_PROC_CRYPTO_DISABLE, x) +#define CPU_PROC_CTRL_PROC_CRYPTO_DISABLE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_PROC_CRYPTO_DISABLE, x) + +#define CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA BIT(4) +#define CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA, x) +#define CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA, x) + +#define CPU_PROC_CTRL_ACP_AWCACHE BIT(3) +#define CPU_PROC_CTRL_ACP_AWCACHE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_ACP_AWCACHE, x) +#define CPU_PROC_CTRL_ACP_AWCACHE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_ACP_AWCACHE, x) + +#define CPU_PROC_CTRL_ACP_ARCACHE BIT(2) +#define CPU_PROC_CTRL_ACP_ARCACHE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_ACP_ARCACHE, x) +#define CPU_PROC_CTRL_ACP_ARCACHE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_ACP_ARCACHE, x) + +#define CPU_PROC_CTRL_L2_FLUSH_REQ BIT(1) +#define CPU_PROC_CTRL_L2_FLUSH_REQ_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_L2_FLUSH_REQ, x) +#define CPU_PROC_CTRL_L2_FLUSH_REQ_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_L2_FLUSH_REQ, x) + +#define CPU_PROC_CTRL_ACP_DISABLE BIT(0) +#define CPU_PROC_CTRL_ACP_DISABLE_SET(x)\ + FIELD_PREP(CPU_PROC_CTRL_ACP_DISABLE, x) +#define CPU_PROC_CTRL_ACP_DISABLE_GET(x)\ + FIELD_GET(CPU_PROC_CTRL_ACP_DISABLE, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_ENA_CFG */ +#define DEV10G_MAC_ENA_CFG(t) __REG(TARGET_DEV10G, t, 12, 0, 0, 1, 60, 0, 0, 1, 4) + +#define DEV10G_MAC_ENA_CFG_RX_ENA BIT(4) +#define DEV10G_MAC_ENA_CFG_RX_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ENA_CFG_RX_ENA, x) +#define DEV10G_MAC_ENA_CFG_RX_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ENA_CFG_RX_ENA, x) + +#define DEV10G_MAC_ENA_CFG_TX_ENA BIT(0) +#define DEV10G_MAC_ENA_CFG_TX_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ENA_CFG_TX_ENA, x) +#define DEV10G_MAC_ENA_CFG_TX_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ENA_CFG_TX_ENA, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_MAXLEN_CFG */ +#define DEV10G_MAC_MAXLEN_CFG(t) __REG(TARGET_DEV10G, t, 12, 0, 0, 1, 60, 8, 0, 1, 4) + +#define DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK BIT(16) +#define DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_SET(x)\ + FIELD_PREP(DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, x) +#define DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_GET(x)\ + FIELD_GET(DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, x) + +#define DEV10G_MAC_MAXLEN_CFG_MAX_LEN GENMASK(15, 0) +#define DEV10G_MAC_MAXLEN_CFG_MAX_LEN_SET(x)\ + FIELD_PREP(DEV10G_MAC_MAXLEN_CFG_MAX_LEN, x) +#define DEV10G_MAC_MAXLEN_CFG_MAX_LEN_GET(x)\ + FIELD_GET(DEV10G_MAC_MAXLEN_CFG_MAX_LEN, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_NUM_TAGS_CFG */ +#define DEV10G_MAC_NUM_TAGS_CFG(t) __REG(TARGET_DEV10G, t, 12, 0, 0, 1, 60, 12, 0, 1, 4) + +#define DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS GENMASK(1, 0) +#define DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS_SET(x)\ + FIELD_PREP(DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS, x) +#define DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS_GET(x)\ + FIELD_GET(DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_TAGS_CFG */ +#define DEV10G_MAC_TAGS_CFG(t, r) __REG(TARGET_DEV10G, t, 12, 0, 0, 1, 60, 16, r, 3, 4) + +#define DEV10G_MAC_TAGS_CFG_TAG_ID GENMASK(31, 16) +#define DEV10G_MAC_TAGS_CFG_TAG_ID_SET(x)\ + FIELD_PREP(DEV10G_MAC_TAGS_CFG_TAG_ID, x) +#define DEV10G_MAC_TAGS_CFG_TAG_ID_GET(x)\ + FIELD_GET(DEV10G_MAC_TAGS_CFG_TAG_ID, x) + +#define DEV10G_MAC_TAGS_CFG_TAG_ENA BIT(4) +#define DEV10G_MAC_TAGS_CFG_TAG_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_TAGS_CFG_TAG_ENA, x) +#define DEV10G_MAC_TAGS_CFG_TAG_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_TAGS_CFG_TAG_ENA, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_ADV_CHK_CFG */ +#define DEV10G_MAC_ADV_CHK_CFG(t) __REG(TARGET_DEV10G, t, 12, 0, 0, 1, 60, 28, 0, 1, 4) + +#define DEV10G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA BIT(24) +#define DEV10G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA, x) +#define DEV10G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA, x) + +#define DEV10G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA BIT(20) +#define DEV10G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA, x) +#define DEV10G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA, x) + +#define DEV10G_MAC_ADV_CHK_CFG_SFD_CHK_ENA BIT(16) +#define DEV10G_MAC_ADV_CHK_CFG_SFD_CHK_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_SFD_CHK_ENA, x) +#define DEV10G_MAC_ADV_CHK_CFG_SFD_CHK_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_SFD_CHK_ENA, x) + +#define DEV10G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS BIT(12) +#define DEV10G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS, x) +#define DEV10G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS, x) + +#define DEV10G_MAC_ADV_CHK_CFG_PRM_CHK_ENA BIT(8) +#define DEV10G_MAC_ADV_CHK_CFG_PRM_CHK_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_PRM_CHK_ENA, x) +#define DEV10G_MAC_ADV_CHK_CFG_PRM_CHK_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_PRM_CHK_ENA, x) + +#define DEV10G_MAC_ADV_CHK_CFG_OOR_ERR_ENA BIT(4) +#define DEV10G_MAC_ADV_CHK_CFG_OOR_ERR_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_OOR_ERR_ENA, x) +#define DEV10G_MAC_ADV_CHK_CFG_OOR_ERR_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_OOR_ERR_ENA, x) + +#define DEV10G_MAC_ADV_CHK_CFG_INR_ERR_ENA BIT(0) +#define DEV10G_MAC_ADV_CHK_CFG_INR_ERR_ENA_SET(x)\ + FIELD_PREP(DEV10G_MAC_ADV_CHK_CFG_INR_ERR_ENA, x) +#define DEV10G_MAC_ADV_CHK_CFG_INR_ERR_ENA_GET(x)\ + FIELD_GET(DEV10G_MAC_ADV_CHK_CFG_INR_ERR_ENA, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_TX_MONITOR_STICKY */ +#define DEV10G_MAC_TX_MONITOR_STICKY(t) __REG(TARGET_DEV10G, t, 12, 0, 0, 1, 60, 48, 0, 1, 4) + +#define DEV10G_MAC_TX_MONITOR_STICKY_LOCAL_ERR_STATE_STICKY BIT(4) +#define DEV10G_MAC_TX_MONITOR_STICKY_LOCAL_ERR_STATE_STICKY_SET(x)\ + FIELD_PREP(DEV10G_MAC_TX_MONITOR_STICKY_LOCAL_ERR_STATE_STICKY, x) +#define DEV10G_MAC_TX_MONITOR_STICKY_LOCAL_ERR_STATE_STICKY_GET(x)\ + FIELD_GET(DEV10G_MAC_TX_MONITOR_STICKY_LOCAL_ERR_STATE_STICKY, x) + +#define DEV10G_MAC_TX_MONITOR_STICKY_REMOTE_ERR_STATE_STICKY BIT(3) +#define DEV10G_MAC_TX_MONITOR_STICKY_REMOTE_ERR_STATE_STICKY_SET(x)\ + FIELD_PREP(DEV10G_MAC_TX_MONITOR_STICKY_REMOTE_ERR_STATE_STICKY, x) +#define DEV10G_MAC_TX_MONITOR_STICKY_REMOTE_ERR_STATE_STICKY_GET(x)\ + FIELD_GET(DEV10G_MAC_TX_MONITOR_STICKY_REMOTE_ERR_STATE_STICKY, x) + +#define DEV10G_MAC_TX_MONITOR_STICKY_LINK_INTERRUPTION_STATE_STICKY BIT(2) +#define DEV10G_MAC_TX_MONITOR_STICKY_LINK_INTERRUPTION_STATE_STICKY_SET(x)\ + FIELD_PREP(DEV10G_MAC_TX_MONITOR_STICKY_LINK_INTERRUPTION_STATE_STICKY, x) +#define DEV10G_MAC_TX_MONITOR_STICKY_LINK_INTERRUPTION_STATE_STICKY_GET(x)\ + FIELD_GET(DEV10G_MAC_TX_MONITOR_STICKY_LINK_INTERRUPTION_STATE_STICKY, x) + +#define DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY BIT(1) +#define DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY_SET(x)\ + FIELD_PREP(DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY, x) +#define DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY_GET(x)\ + FIELD_GET(DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY, x) + +#define DEV10G_MAC_TX_MONITOR_STICKY_DIS_STATE_STICKY BIT(0) +#define DEV10G_MAC_TX_MONITOR_STICKY_DIS_STATE_STICKY_SET(x)\ + FIELD_PREP(DEV10G_MAC_TX_MONITOR_STICKY_DIS_STATE_STICKY, x) +#define DEV10G_MAC_TX_MONITOR_STICKY_DIS_STATE_STICKY_GET(x)\ + FIELD_GET(DEV10G_MAC_TX_MONITOR_STICKY_DIS_STATE_STICKY, x) + +/* DEV10G:DEV_CFG_STATUS:DEV_RST_CTRL */ +#define DEV10G_DEV_RST_CTRL(t) __REG(TARGET_DEV10G, t, 12, 436, 0, 1, 52, 0, 0, 1, 4) + +#define DEV10G_DEV_RST_CTRL_PARDET_MODE_ENA BIT(28) +#define DEV10G_DEV_RST_CTRL_PARDET_MODE_ENA_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_PARDET_MODE_ENA, x) +#define DEV10G_DEV_RST_CTRL_PARDET_MODE_ENA_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_PARDET_MODE_ENA, x) + +#define DEV10G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS BIT(27) +#define DEV10G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) +#define DEV10G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) + +#define DEV10G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS GENMASK(26, 25) +#define DEV10G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS, x) +#define DEV10G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS, x) + +#define DEV10G_DEV_RST_CTRL_SERDES_SPEED_SEL GENMASK(24, 23) +#define DEV10G_DEV_RST_CTRL_SERDES_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_SERDES_SPEED_SEL, x) +#define DEV10G_DEV_RST_CTRL_SERDES_SPEED_SEL_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_SERDES_SPEED_SEL, x) + +#define DEV10G_DEV_RST_CTRL_SPEED_SEL GENMASK(22, 20) +#define DEV10G_DEV_RST_CTRL_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_SPEED_SEL, x) +#define DEV10G_DEV_RST_CTRL_SPEED_SEL_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_SPEED_SEL, x) + +#define DEV10G_DEV_RST_CTRL_PCS_TX_RST BIT(12) +#define DEV10G_DEV_RST_CTRL_PCS_TX_RST_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_PCS_TX_RST, x) +#define DEV10G_DEV_RST_CTRL_PCS_TX_RST_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_PCS_TX_RST, x) + +#define DEV10G_DEV_RST_CTRL_PCS_RX_RST BIT(8) +#define DEV10G_DEV_RST_CTRL_PCS_RX_RST_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_PCS_RX_RST, x) +#define DEV10G_DEV_RST_CTRL_PCS_RX_RST_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_PCS_RX_RST, x) + +#define DEV10G_DEV_RST_CTRL_MAC_TX_RST BIT(4) +#define DEV10G_DEV_RST_CTRL_MAC_TX_RST_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_MAC_TX_RST, x) +#define DEV10G_DEV_RST_CTRL_MAC_TX_RST_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_MAC_TX_RST, x) + +#define DEV10G_DEV_RST_CTRL_MAC_RX_RST BIT(0) +#define DEV10G_DEV_RST_CTRL_MAC_RX_RST_SET(x)\ + FIELD_PREP(DEV10G_DEV_RST_CTRL_MAC_RX_RST, x) +#define DEV10G_DEV_RST_CTRL_MAC_RX_RST_GET(x)\ + FIELD_GET(DEV10G_DEV_RST_CTRL_MAC_RX_RST, x) + +/* DEV10G:PCS25G_CFG_STATUS:PCS25G_CFG */ +#define DEV10G_PCS25G_CFG(t) __REG(TARGET_DEV10G, t, 12, 488, 0, 1, 32, 0, 0, 1, 4) + +#define DEV10G_PCS25G_CFG_PCS25G_ENA BIT(0) +#define DEV10G_PCS25G_CFG_PCS25G_ENA_SET(x)\ + FIELD_PREP(DEV10G_PCS25G_CFG_PCS25G_ENA, x) +#define DEV10G_PCS25G_CFG_PCS25G_ENA_GET(x)\ + FIELD_GET(DEV10G_PCS25G_CFG_PCS25G_ENA, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_ENA_CFG */ +#define DEV25G_MAC_ENA_CFG(t) __REG(TARGET_DEV25G, t, 8, 0, 0, 1, 60, 0, 0, 1, 4) + +#define DEV25G_MAC_ENA_CFG_RX_ENA BIT(4) +#define DEV25G_MAC_ENA_CFG_RX_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ENA_CFG_RX_ENA, x) +#define DEV25G_MAC_ENA_CFG_RX_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ENA_CFG_RX_ENA, x) + +#define DEV25G_MAC_ENA_CFG_TX_ENA BIT(0) +#define DEV25G_MAC_ENA_CFG_TX_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ENA_CFG_TX_ENA, x) +#define DEV25G_MAC_ENA_CFG_TX_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ENA_CFG_TX_ENA, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_MAXLEN_CFG */ +#define DEV25G_MAC_MAXLEN_CFG(t) __REG(TARGET_DEV25G, t, 8, 0, 0, 1, 60, 8, 0, 1, 4) + +#define DEV25G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK BIT(16) +#define DEV25G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_SET(x)\ + FIELD_PREP(DEV25G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, x) +#define DEV25G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_GET(x)\ + FIELD_GET(DEV25G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, x) + +#define DEV25G_MAC_MAXLEN_CFG_MAX_LEN GENMASK(15, 0) +#define DEV25G_MAC_MAXLEN_CFG_MAX_LEN_SET(x)\ + FIELD_PREP(DEV25G_MAC_MAXLEN_CFG_MAX_LEN, x) +#define DEV25G_MAC_MAXLEN_CFG_MAX_LEN_GET(x)\ + FIELD_GET(DEV25G_MAC_MAXLEN_CFG_MAX_LEN, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_ADV_CHK_CFG */ +#define DEV25G_MAC_ADV_CHK_CFG(t) __REG(TARGET_DEV25G, t, 8, 0, 0, 1, 60, 28, 0, 1, 4) + +#define DEV25G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA BIT(24) +#define DEV25G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA, x) +#define DEV25G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA, x) + +#define DEV25G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA BIT(20) +#define DEV25G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA, x) +#define DEV25G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA, x) + +#define DEV25G_MAC_ADV_CHK_CFG_SFD_CHK_ENA BIT(16) +#define DEV25G_MAC_ADV_CHK_CFG_SFD_CHK_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_SFD_CHK_ENA, x) +#define DEV25G_MAC_ADV_CHK_CFG_SFD_CHK_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_SFD_CHK_ENA, x) + +#define DEV25G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS BIT(12) +#define DEV25G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS, x) +#define DEV25G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS, x) + +#define DEV25G_MAC_ADV_CHK_CFG_PRM_CHK_ENA BIT(8) +#define DEV25G_MAC_ADV_CHK_CFG_PRM_CHK_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_PRM_CHK_ENA, x) +#define DEV25G_MAC_ADV_CHK_CFG_PRM_CHK_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_PRM_CHK_ENA, x) + +#define DEV25G_MAC_ADV_CHK_CFG_OOR_ERR_ENA BIT(4) +#define DEV25G_MAC_ADV_CHK_CFG_OOR_ERR_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_OOR_ERR_ENA, x) +#define DEV25G_MAC_ADV_CHK_CFG_OOR_ERR_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_OOR_ERR_ENA, x) + +#define DEV25G_MAC_ADV_CHK_CFG_INR_ERR_ENA BIT(0) +#define DEV25G_MAC_ADV_CHK_CFG_INR_ERR_ENA_SET(x)\ + FIELD_PREP(DEV25G_MAC_ADV_CHK_CFG_INR_ERR_ENA, x) +#define DEV25G_MAC_ADV_CHK_CFG_INR_ERR_ENA_GET(x)\ + FIELD_GET(DEV25G_MAC_ADV_CHK_CFG_INR_ERR_ENA, x) + +/* DEV10G:DEV_CFG_STATUS:DEV_RST_CTRL */ +#define DEV25G_DEV_RST_CTRL(t) __REG(TARGET_DEV25G, t, 8, 436, 0, 1, 52, 0, 0, 1, 4) + +#define DEV25G_DEV_RST_CTRL_PARDET_MODE_ENA BIT(28) +#define DEV25G_DEV_RST_CTRL_PARDET_MODE_ENA_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_PARDET_MODE_ENA, x) +#define DEV25G_DEV_RST_CTRL_PARDET_MODE_ENA_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_PARDET_MODE_ENA, x) + +#define DEV25G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS BIT(27) +#define DEV25G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) +#define DEV25G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) + +#define DEV25G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS GENMASK(26, 25) +#define DEV25G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS, x) +#define DEV25G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS, x) + +#define DEV25G_DEV_RST_CTRL_SERDES_SPEED_SEL GENMASK(24, 23) +#define DEV25G_DEV_RST_CTRL_SERDES_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_SERDES_SPEED_SEL, x) +#define DEV25G_DEV_RST_CTRL_SERDES_SPEED_SEL_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_SERDES_SPEED_SEL, x) + +#define DEV25G_DEV_RST_CTRL_SPEED_SEL GENMASK(22, 20) +#define DEV25G_DEV_RST_CTRL_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_SPEED_SEL, x) +#define DEV25G_DEV_RST_CTRL_SPEED_SEL_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_SPEED_SEL, x) + +#define DEV25G_DEV_RST_CTRL_PCS_TX_RST BIT(12) +#define DEV25G_DEV_RST_CTRL_PCS_TX_RST_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_PCS_TX_RST, x) +#define DEV25G_DEV_RST_CTRL_PCS_TX_RST_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_PCS_TX_RST, x) + +#define DEV25G_DEV_RST_CTRL_PCS_RX_RST BIT(8) +#define DEV25G_DEV_RST_CTRL_PCS_RX_RST_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_PCS_RX_RST, x) +#define DEV25G_DEV_RST_CTRL_PCS_RX_RST_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_PCS_RX_RST, x) + +#define DEV25G_DEV_RST_CTRL_MAC_TX_RST BIT(4) +#define DEV25G_DEV_RST_CTRL_MAC_TX_RST_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_MAC_TX_RST, x) +#define DEV25G_DEV_RST_CTRL_MAC_TX_RST_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_MAC_TX_RST, x) + +#define DEV25G_DEV_RST_CTRL_MAC_RX_RST BIT(0) +#define DEV25G_DEV_RST_CTRL_MAC_RX_RST_SET(x)\ + FIELD_PREP(DEV25G_DEV_RST_CTRL_MAC_RX_RST, x) +#define DEV25G_DEV_RST_CTRL_MAC_RX_RST_GET(x)\ + FIELD_GET(DEV25G_DEV_RST_CTRL_MAC_RX_RST, x) + +/* DEV10G:PCS25G_CFG_STATUS:PCS25G_CFG */ +#define DEV25G_PCS25G_CFG(t) __REG(TARGET_DEV25G, t, 8, 488, 0, 1, 32, 0, 0, 1, 4) + +#define DEV25G_PCS25G_CFG_PCS25G_ENA BIT(0) +#define DEV25G_PCS25G_CFG_PCS25G_ENA_SET(x)\ + FIELD_PREP(DEV25G_PCS25G_CFG_PCS25G_ENA, x) +#define DEV25G_PCS25G_CFG_PCS25G_ENA_GET(x)\ + FIELD_GET(DEV25G_PCS25G_CFG_PCS25G_ENA, x) + +/* DEV10G:PCS25G_CFG_STATUS:PCS25G_SD_CFG */ +#define DEV25G_PCS25G_SD_CFG(t) __REG(TARGET_DEV25G, t, 8, 488, 0, 1, 32, 4, 0, 1, 4) + +#define DEV25G_PCS25G_SD_CFG_SD_SEL BIT(8) +#define DEV25G_PCS25G_SD_CFG_SD_SEL_SET(x)\ + FIELD_PREP(DEV25G_PCS25G_SD_CFG_SD_SEL, x) +#define DEV25G_PCS25G_SD_CFG_SD_SEL_GET(x)\ + FIELD_GET(DEV25G_PCS25G_SD_CFG_SD_SEL, x) + +#define DEV25G_PCS25G_SD_CFG_SD_POL BIT(4) +#define DEV25G_PCS25G_SD_CFG_SD_POL_SET(x)\ + FIELD_PREP(DEV25G_PCS25G_SD_CFG_SD_POL, x) +#define DEV25G_PCS25G_SD_CFG_SD_POL_GET(x)\ + FIELD_GET(DEV25G_PCS25G_SD_CFG_SD_POL, x) + +#define DEV25G_PCS25G_SD_CFG_SD_ENA BIT(0) +#define DEV25G_PCS25G_SD_CFG_SD_ENA_SET(x)\ + FIELD_PREP(DEV25G_PCS25G_SD_CFG_SD_ENA, x) +#define DEV25G_PCS25G_SD_CFG_SD_ENA_GET(x)\ + FIELD_GET(DEV25G_PCS25G_SD_CFG_SD_ENA, x) + +/* DEV1G:DEV_CFG_STATUS:DEV_RST_CTRL */ +#define DEV2G5_DEV_RST_CTRL(t) __REG(TARGET_DEV2G5, t, 65, 0, 0, 1, 36, 0, 0, 1, 4) + +#define DEV2G5_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS BIT(23) +#define DEV2G5_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) +#define DEV2G5_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) + +#define DEV2G5_DEV_RST_CTRL_SPEED_SEL GENMASK(22, 20) +#define DEV2G5_DEV_RST_CTRL_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_SPEED_SEL, x) +#define DEV2G5_DEV_RST_CTRL_SPEED_SEL_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_SPEED_SEL, x) + +#define DEV2G5_DEV_RST_CTRL_USX_PCS_TX_RST BIT(17) +#define DEV2G5_DEV_RST_CTRL_USX_PCS_TX_RST_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_USX_PCS_TX_RST, x) +#define DEV2G5_DEV_RST_CTRL_USX_PCS_TX_RST_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_USX_PCS_TX_RST, x) + +#define DEV2G5_DEV_RST_CTRL_USX_PCS_RX_RST BIT(16) +#define DEV2G5_DEV_RST_CTRL_USX_PCS_RX_RST_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_USX_PCS_RX_RST, x) +#define DEV2G5_DEV_RST_CTRL_USX_PCS_RX_RST_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_USX_PCS_RX_RST, x) + +#define DEV2G5_DEV_RST_CTRL_PCS_TX_RST BIT(12) +#define DEV2G5_DEV_RST_CTRL_PCS_TX_RST_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_PCS_TX_RST, x) +#define DEV2G5_DEV_RST_CTRL_PCS_TX_RST_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_PCS_TX_RST, x) + +#define DEV2G5_DEV_RST_CTRL_PCS_RX_RST BIT(8) +#define DEV2G5_DEV_RST_CTRL_PCS_RX_RST_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_PCS_RX_RST, x) +#define DEV2G5_DEV_RST_CTRL_PCS_RX_RST_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_PCS_RX_RST, x) + +#define DEV2G5_DEV_RST_CTRL_MAC_TX_RST BIT(4) +#define DEV2G5_DEV_RST_CTRL_MAC_TX_RST_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_MAC_TX_RST, x) +#define DEV2G5_DEV_RST_CTRL_MAC_TX_RST_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_MAC_TX_RST, x) + +#define DEV2G5_DEV_RST_CTRL_MAC_RX_RST BIT(0) +#define DEV2G5_DEV_RST_CTRL_MAC_RX_RST_SET(x)\ + FIELD_PREP(DEV2G5_DEV_RST_CTRL_MAC_RX_RST, x) +#define DEV2G5_DEV_RST_CTRL_MAC_RX_RST_GET(x)\ + FIELD_GET(DEV2G5_DEV_RST_CTRL_MAC_RX_RST, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_ENA_CFG */ +#define DEV2G5_MAC_ENA_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 0, 0, 1, 4) + +#define DEV2G5_MAC_ENA_CFG_RX_ENA BIT(4) +#define DEV2G5_MAC_ENA_CFG_RX_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_ENA_CFG_RX_ENA, x) +#define DEV2G5_MAC_ENA_CFG_RX_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_ENA_CFG_RX_ENA, x) + +#define DEV2G5_MAC_ENA_CFG_TX_ENA BIT(0) +#define DEV2G5_MAC_ENA_CFG_TX_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_ENA_CFG_TX_ENA, x) +#define DEV2G5_MAC_ENA_CFG_TX_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_ENA_CFG_TX_ENA, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_MODE_CFG */ +#define DEV2G5_MAC_MODE_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 4, 0, 1, 4) + +#define DEV2G5_MAC_MODE_CFG_FC_WORD_SYNC_ENA BIT(8) +#define DEV2G5_MAC_MODE_CFG_FC_WORD_SYNC_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_MODE_CFG_FC_WORD_SYNC_ENA, x) +#define DEV2G5_MAC_MODE_CFG_FC_WORD_SYNC_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_MODE_CFG_FC_WORD_SYNC_ENA, x) + +#define DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA BIT(4) +#define DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA, x) +#define DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA, x) + +#define DEV2G5_MAC_MODE_CFG_FDX_ENA BIT(0) +#define DEV2G5_MAC_MODE_CFG_FDX_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_MODE_CFG_FDX_ENA, x) +#define DEV2G5_MAC_MODE_CFG_FDX_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_MODE_CFG_FDX_ENA, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_MAXLEN_CFG */ +#define DEV2G5_MAC_MAXLEN_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 8, 0, 1, 4) + +#define DEV2G5_MAC_MAXLEN_CFG_MAX_LEN GENMASK(15, 0) +#define DEV2G5_MAC_MAXLEN_CFG_MAX_LEN_SET(x)\ + FIELD_PREP(DEV2G5_MAC_MAXLEN_CFG_MAX_LEN, x) +#define DEV2G5_MAC_MAXLEN_CFG_MAX_LEN_GET(x)\ + FIELD_GET(DEV2G5_MAC_MAXLEN_CFG_MAX_LEN, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_TAGS_CFG */ +#define DEV2G5_MAC_TAGS_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 12, 0, 1, 4) + +#define DEV2G5_MAC_TAGS_CFG_TAG_ID GENMASK(31, 16) +#define DEV2G5_MAC_TAGS_CFG_TAG_ID_SET(x)\ + FIELD_PREP(DEV2G5_MAC_TAGS_CFG_TAG_ID, x) +#define DEV2G5_MAC_TAGS_CFG_TAG_ID_GET(x)\ + FIELD_GET(DEV2G5_MAC_TAGS_CFG_TAG_ID, x) + +#define DEV2G5_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA BIT(3) +#define DEV2G5_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA, x) +#define DEV2G5_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA, x) + +#define DEV2G5_MAC_TAGS_CFG_PB_ENA GENMASK(2, 1) +#define DEV2G5_MAC_TAGS_CFG_PB_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_TAGS_CFG_PB_ENA, x) +#define DEV2G5_MAC_TAGS_CFG_PB_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_TAGS_CFG_PB_ENA, x) + +#define DEV2G5_MAC_TAGS_CFG_VLAN_AWR_ENA BIT(0) +#define DEV2G5_MAC_TAGS_CFG_VLAN_AWR_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_TAGS_CFG_VLAN_AWR_ENA, x) +#define DEV2G5_MAC_TAGS_CFG_VLAN_AWR_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_TAGS_CFG_VLAN_AWR_ENA, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_TAGS_CFG2 */ +#define DEV2G5_MAC_TAGS_CFG2(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 16, 0, 1, 4) + +#define DEV2G5_MAC_TAGS_CFG2_TAG_ID3 GENMASK(31, 16) +#define DEV2G5_MAC_TAGS_CFG2_TAG_ID3_SET(x)\ + FIELD_PREP(DEV2G5_MAC_TAGS_CFG2_TAG_ID3, x) +#define DEV2G5_MAC_TAGS_CFG2_TAG_ID3_GET(x)\ + FIELD_GET(DEV2G5_MAC_TAGS_CFG2_TAG_ID3, x) + +#define DEV2G5_MAC_TAGS_CFG2_TAG_ID2 GENMASK(15, 0) +#define DEV2G5_MAC_TAGS_CFG2_TAG_ID2_SET(x)\ + FIELD_PREP(DEV2G5_MAC_TAGS_CFG2_TAG_ID2, x) +#define DEV2G5_MAC_TAGS_CFG2_TAG_ID2_GET(x)\ + FIELD_GET(DEV2G5_MAC_TAGS_CFG2_TAG_ID2, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_ADV_CHK_CFG */ +#define DEV2G5_MAC_ADV_CHK_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 20, 0, 1, 4) + +#define DEV2G5_MAC_ADV_CHK_CFG_LEN_DROP_ENA BIT(0) +#define DEV2G5_MAC_ADV_CHK_CFG_LEN_DROP_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_ADV_CHK_CFG_LEN_DROP_ENA, x) +#define DEV2G5_MAC_ADV_CHK_CFG_LEN_DROP_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_ADV_CHK_CFG_LEN_DROP_ENA, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_IFG_CFG */ +#define DEV2G5_MAC_IFG_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 24, 0, 1, 4) + +#define DEV2G5_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK BIT(17) +#define DEV2G5_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK_SET(x)\ + FIELD_PREP(DEV2G5_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK, x) +#define DEV2G5_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK_GET(x)\ + FIELD_GET(DEV2G5_MAC_IFG_CFG_RESTORE_OLD_IPG_CHECK, x) + +#define DEV2G5_MAC_IFG_CFG_TX_IFG GENMASK(12, 8) +#define DEV2G5_MAC_IFG_CFG_TX_IFG_SET(x)\ + FIELD_PREP(DEV2G5_MAC_IFG_CFG_TX_IFG, x) +#define DEV2G5_MAC_IFG_CFG_TX_IFG_GET(x)\ + FIELD_GET(DEV2G5_MAC_IFG_CFG_TX_IFG, x) + +#define DEV2G5_MAC_IFG_CFG_RX_IFG2 GENMASK(7, 4) +#define DEV2G5_MAC_IFG_CFG_RX_IFG2_SET(x)\ + FIELD_PREP(DEV2G5_MAC_IFG_CFG_RX_IFG2, x) +#define DEV2G5_MAC_IFG_CFG_RX_IFG2_GET(x)\ + FIELD_GET(DEV2G5_MAC_IFG_CFG_RX_IFG2, x) + +#define DEV2G5_MAC_IFG_CFG_RX_IFG1 GENMASK(3, 0) +#define DEV2G5_MAC_IFG_CFG_RX_IFG1_SET(x)\ + FIELD_PREP(DEV2G5_MAC_IFG_CFG_RX_IFG1, x) +#define DEV2G5_MAC_IFG_CFG_RX_IFG1_GET(x)\ + FIELD_GET(DEV2G5_MAC_IFG_CFG_RX_IFG1, x) + +/* DEV1G:MAC_CFG_STATUS:MAC_HDX_CFG */ +#define DEV2G5_MAC_HDX_CFG(t) __REG(TARGET_DEV2G5, t, 65, 52, 0, 1, 36, 28, 0, 1, 4) + +#define DEV2G5_MAC_HDX_CFG_BYPASS_COL_SYNC BIT(26) +#define DEV2G5_MAC_HDX_CFG_BYPASS_COL_SYNC_SET(x)\ + FIELD_PREP(DEV2G5_MAC_HDX_CFG_BYPASS_COL_SYNC, x) +#define DEV2G5_MAC_HDX_CFG_BYPASS_COL_SYNC_GET(x)\ + FIELD_GET(DEV2G5_MAC_HDX_CFG_BYPASS_COL_SYNC, x) + +#define DEV2G5_MAC_HDX_CFG_SEED GENMASK(23, 16) +#define DEV2G5_MAC_HDX_CFG_SEED_SET(x)\ + FIELD_PREP(DEV2G5_MAC_HDX_CFG_SEED, x) +#define DEV2G5_MAC_HDX_CFG_SEED_GET(x)\ + FIELD_GET(DEV2G5_MAC_HDX_CFG_SEED, x) + +#define DEV2G5_MAC_HDX_CFG_SEED_LOAD BIT(12) +#define DEV2G5_MAC_HDX_CFG_SEED_LOAD_SET(x)\ + FIELD_PREP(DEV2G5_MAC_HDX_CFG_SEED_LOAD, x) +#define DEV2G5_MAC_HDX_CFG_SEED_LOAD_GET(x)\ + FIELD_GET(DEV2G5_MAC_HDX_CFG_SEED_LOAD, x) + +#define DEV2G5_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA BIT(8) +#define DEV2G5_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA_SET(x)\ + FIELD_PREP(DEV2G5_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA, x) +#define DEV2G5_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA_GET(x)\ + FIELD_GET(DEV2G5_MAC_HDX_CFG_RETRY_AFTER_EXC_COL_ENA, x) + +#define DEV2G5_MAC_HDX_CFG_LATE_COL_POS GENMASK(6, 0) +#define DEV2G5_MAC_HDX_CFG_LATE_COL_POS_SET(x)\ + FIELD_PREP(DEV2G5_MAC_HDX_CFG_LATE_COL_POS, x) +#define DEV2G5_MAC_HDX_CFG_LATE_COL_POS_GET(x)\ + FIELD_GET(DEV2G5_MAC_HDX_CFG_LATE_COL_POS, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_CFG */ +#define DEV2G5_PCS1G_CFG(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 0, 0, 1, 4) + +#define DEV2G5_PCS1G_CFG_LINK_STATUS_TYPE BIT(4) +#define DEV2G5_PCS1G_CFG_LINK_STATUS_TYPE_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_CFG_LINK_STATUS_TYPE, x) +#define DEV2G5_PCS1G_CFG_LINK_STATUS_TYPE_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_CFG_LINK_STATUS_TYPE, x) + +#define DEV2G5_PCS1G_CFG_AN_LINK_CTRL_ENA BIT(1) +#define DEV2G5_PCS1G_CFG_AN_LINK_CTRL_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_CFG_AN_LINK_CTRL_ENA, x) +#define DEV2G5_PCS1G_CFG_AN_LINK_CTRL_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_CFG_AN_LINK_CTRL_ENA, x) + +#define DEV2G5_PCS1G_CFG_PCS_ENA BIT(0) +#define DEV2G5_PCS1G_CFG_PCS_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_CFG_PCS_ENA, x) +#define DEV2G5_PCS1G_CFG_PCS_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_CFG_PCS_ENA, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_MODE_CFG */ +#define DEV2G5_PCS1G_MODE_CFG(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 4, 0, 1, 4) + +#define DEV2G5_PCS1G_MODE_CFG_UNIDIR_MODE_ENA BIT(4) +#define DEV2G5_PCS1G_MODE_CFG_UNIDIR_MODE_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_MODE_CFG_UNIDIR_MODE_ENA, x) +#define DEV2G5_PCS1G_MODE_CFG_UNIDIR_MODE_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_MODE_CFG_UNIDIR_MODE_ENA, x) + +#define DEV2G5_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA BIT(1) +#define DEV2G5_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, x) +#define DEV2G5_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, x) + +#define DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA BIT(0) +#define DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA, x) +#define DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_SD_CFG */ +#define DEV2G5_PCS1G_SD_CFG(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 8, 0, 1, 4) + +#define DEV2G5_PCS1G_SD_CFG_SD_SEL BIT(8) +#define DEV2G5_PCS1G_SD_CFG_SD_SEL_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_SD_CFG_SD_SEL, x) +#define DEV2G5_PCS1G_SD_CFG_SD_SEL_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_SD_CFG_SD_SEL, x) + +#define DEV2G5_PCS1G_SD_CFG_SD_POL BIT(4) +#define DEV2G5_PCS1G_SD_CFG_SD_POL_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_SD_CFG_SD_POL, x) +#define DEV2G5_PCS1G_SD_CFG_SD_POL_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_SD_CFG_SD_POL, x) + +#define DEV2G5_PCS1G_SD_CFG_SD_ENA BIT(0) +#define DEV2G5_PCS1G_SD_CFG_SD_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_SD_CFG_SD_ENA, x) +#define DEV2G5_PCS1G_SD_CFG_SD_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_SD_CFG_SD_ENA, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_ANEG_CFG */ +#define DEV2G5_PCS1G_ANEG_CFG(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 12, 0, 1, 4) + +#define DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY GENMASK(31, 16) +#define DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY, x) +#define DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY, x) + +#define DEV2G5_PCS1G_ANEG_CFG_SW_RESOLVE_ENA BIT(8) +#define DEV2G5_PCS1G_ANEG_CFG_SW_RESOLVE_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_CFG_SW_RESOLVE_ENA, x) +#define DEV2G5_PCS1G_ANEG_CFG_SW_RESOLVE_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_CFG_SW_RESOLVE_ENA, x) + +#define DEV2G5_PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT BIT(1) +#define DEV2G5_PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT, x) +#define DEV2G5_PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT, x) + +#define DEV2G5_PCS1G_ANEG_CFG_ANEG_ENA BIT(0) +#define DEV2G5_PCS1G_ANEG_CFG_ANEG_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_CFG_ANEG_ENA, x) +#define DEV2G5_PCS1G_ANEG_CFG_ANEG_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_CFG_ANEG_ENA, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_LB_CFG */ +#define DEV2G5_PCS1G_LB_CFG(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 20, 0, 1, 4) + +#define DEV2G5_PCS1G_LB_CFG_RA_ENA BIT(4) +#define DEV2G5_PCS1G_LB_CFG_RA_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LB_CFG_RA_ENA, x) +#define DEV2G5_PCS1G_LB_CFG_RA_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LB_CFG_RA_ENA, x) + +#define DEV2G5_PCS1G_LB_CFG_GMII_PHY_LB_ENA BIT(1) +#define DEV2G5_PCS1G_LB_CFG_GMII_PHY_LB_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LB_CFG_GMII_PHY_LB_ENA, x) +#define DEV2G5_PCS1G_LB_CFG_GMII_PHY_LB_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LB_CFG_GMII_PHY_LB_ENA, x) + +#define DEV2G5_PCS1G_LB_CFG_TBI_HOST_LB_ENA BIT(0) +#define DEV2G5_PCS1G_LB_CFG_TBI_HOST_LB_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LB_CFG_TBI_HOST_LB_ENA, x) +#define DEV2G5_PCS1G_LB_CFG_TBI_HOST_LB_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LB_CFG_TBI_HOST_LB_ENA, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_ANEG_STATUS */ +#define DEV2G5_PCS1G_ANEG_STATUS(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 32, 0, 1, 4) + +#define DEV2G5_PCS1G_ANEG_STATUS_LP_ADV_ABILITY GENMASK(31, 16) +#define DEV2G5_PCS1G_ANEG_STATUS_LP_ADV_ABILITY_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_STATUS_LP_ADV_ABILITY, x) +#define DEV2G5_PCS1G_ANEG_STATUS_LP_ADV_ABILITY_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_STATUS_LP_ADV_ABILITY, x) + +#define DEV2G5_PCS1G_ANEG_STATUS_PR BIT(4) +#define DEV2G5_PCS1G_ANEG_STATUS_PR_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_STATUS_PR, x) +#define DEV2G5_PCS1G_ANEG_STATUS_PR_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_STATUS_PR, x) + +#define DEV2G5_PCS1G_ANEG_STATUS_PAGE_RX_STICKY BIT(3) +#define DEV2G5_PCS1G_ANEG_STATUS_PAGE_RX_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_STATUS_PAGE_RX_STICKY, x) +#define DEV2G5_PCS1G_ANEG_STATUS_PAGE_RX_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_STATUS_PAGE_RX_STICKY, x) + +#define DEV2G5_PCS1G_ANEG_STATUS_ANEG_COMPLETE BIT(0) +#define DEV2G5_PCS1G_ANEG_STATUS_ANEG_COMPLETE_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_ANEG_STATUS_ANEG_COMPLETE, x) +#define DEV2G5_PCS1G_ANEG_STATUS_ANEG_COMPLETE_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_ANEG_STATUS_ANEG_COMPLETE, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_LINK_STATUS */ +#define DEV2G5_PCS1G_LINK_STATUS(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 40, 0, 1, 4) + +#define DEV2G5_PCS1G_LINK_STATUS_DELAY_VAR GENMASK(15, 12) +#define DEV2G5_PCS1G_LINK_STATUS_DELAY_VAR_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LINK_STATUS_DELAY_VAR, x) +#define DEV2G5_PCS1G_LINK_STATUS_DELAY_VAR_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LINK_STATUS_DELAY_VAR, x) + +#define DEV2G5_PCS1G_LINK_STATUS_SIGNAL_DETECT BIT(8) +#define DEV2G5_PCS1G_LINK_STATUS_SIGNAL_DETECT_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LINK_STATUS_SIGNAL_DETECT, x) +#define DEV2G5_PCS1G_LINK_STATUS_SIGNAL_DETECT_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LINK_STATUS_SIGNAL_DETECT, x) + +#define DEV2G5_PCS1G_LINK_STATUS_LINK_STATUS BIT(4) +#define DEV2G5_PCS1G_LINK_STATUS_LINK_STATUS_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LINK_STATUS_LINK_STATUS, x) +#define DEV2G5_PCS1G_LINK_STATUS_LINK_STATUS_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LINK_STATUS_LINK_STATUS, x) + +#define DEV2G5_PCS1G_LINK_STATUS_SYNC_STATUS BIT(0) +#define DEV2G5_PCS1G_LINK_STATUS_SYNC_STATUS_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_LINK_STATUS_SYNC_STATUS, x) +#define DEV2G5_PCS1G_LINK_STATUS_SYNC_STATUS_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_LINK_STATUS_SYNC_STATUS, x) + +/* DEV1G:PCS1G_CFG_STATUS:PCS1G_STICKY */ +#define DEV2G5_PCS1G_STICKY(t) __REG(TARGET_DEV2G5, t, 65, 88, 0, 1, 68, 48, 0, 1, 4) + +#define DEV2G5_PCS1G_STICKY_LINK_DOWN_STICKY BIT(4) +#define DEV2G5_PCS1G_STICKY_LINK_DOWN_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_STICKY_LINK_DOWN_STICKY, x) +#define DEV2G5_PCS1G_STICKY_LINK_DOWN_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_STICKY_LINK_DOWN_STICKY, x) + +#define DEV2G5_PCS1G_STICKY_OUT_OF_SYNC_STICKY BIT(0) +#define DEV2G5_PCS1G_STICKY_OUT_OF_SYNC_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS1G_STICKY_OUT_OF_SYNC_STICKY, x) +#define DEV2G5_PCS1G_STICKY_OUT_OF_SYNC_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS1G_STICKY_OUT_OF_SYNC_STICKY, x) + +/* DEV1G:PCS_FX100_CONFIGURATION:PCS_FX100_CFG */ +#define DEV2G5_PCS_FX100_CFG(t) __REG(TARGET_DEV2G5, t, 65, 164, 0, 1, 4, 0, 0, 1, 4) + +#define DEV2G5_PCS_FX100_CFG_SD_SEL BIT(26) +#define DEV2G5_PCS_FX100_CFG_SD_SEL_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_SD_SEL, x) +#define DEV2G5_PCS_FX100_CFG_SD_SEL_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_SD_SEL, x) + +#define DEV2G5_PCS_FX100_CFG_SD_POL BIT(25) +#define DEV2G5_PCS_FX100_CFG_SD_POL_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_SD_POL, x) +#define DEV2G5_PCS_FX100_CFG_SD_POL_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_SD_POL, x) + +#define DEV2G5_PCS_FX100_CFG_SD_ENA BIT(24) +#define DEV2G5_PCS_FX100_CFG_SD_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_SD_ENA, x) +#define DEV2G5_PCS_FX100_CFG_SD_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_SD_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_LOOPBACK_ENA BIT(20) +#define DEV2G5_PCS_FX100_CFG_LOOPBACK_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_LOOPBACK_ENA, x) +#define DEV2G5_PCS_FX100_CFG_LOOPBACK_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_LOOPBACK_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_SWAP_MII_ENA BIT(16) +#define DEV2G5_PCS_FX100_CFG_SWAP_MII_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_SWAP_MII_ENA, x) +#define DEV2G5_PCS_FX100_CFG_SWAP_MII_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_SWAP_MII_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_RXBITSEL GENMASK(15, 12) +#define DEV2G5_PCS_FX100_CFG_RXBITSEL_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_RXBITSEL, x) +#define DEV2G5_PCS_FX100_CFG_RXBITSEL_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_RXBITSEL, x) + +#define DEV2G5_PCS_FX100_CFG_SIGDET_CFG GENMASK(10, 9) +#define DEV2G5_PCS_FX100_CFG_SIGDET_CFG_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_SIGDET_CFG, x) +#define DEV2G5_PCS_FX100_CFG_SIGDET_CFG_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_SIGDET_CFG, x) + +#define DEV2G5_PCS_FX100_CFG_LINKHYST_TM_ENA BIT(8) +#define DEV2G5_PCS_FX100_CFG_LINKHYST_TM_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_LINKHYST_TM_ENA, x) +#define DEV2G5_PCS_FX100_CFG_LINKHYST_TM_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_LINKHYST_TM_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_LINKHYSTTIMER GENMASK(7, 4) +#define DEV2G5_PCS_FX100_CFG_LINKHYSTTIMER_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_LINKHYSTTIMER, x) +#define DEV2G5_PCS_FX100_CFG_LINKHYSTTIMER_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_LINKHYSTTIMER, x) + +#define DEV2G5_PCS_FX100_CFG_UNIDIR_MODE_ENA BIT(3) +#define DEV2G5_PCS_FX100_CFG_UNIDIR_MODE_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_UNIDIR_MODE_ENA, x) +#define DEV2G5_PCS_FX100_CFG_UNIDIR_MODE_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_UNIDIR_MODE_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_FEFCHK_ENA BIT(2) +#define DEV2G5_PCS_FX100_CFG_FEFCHK_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_FEFCHK_ENA, x) +#define DEV2G5_PCS_FX100_CFG_FEFCHK_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_FEFCHK_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_FEFGEN_ENA BIT(1) +#define DEV2G5_PCS_FX100_CFG_FEFGEN_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_FEFGEN_ENA, x) +#define DEV2G5_PCS_FX100_CFG_FEFGEN_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_FEFGEN_ENA, x) + +#define DEV2G5_PCS_FX100_CFG_PCS_ENA BIT(0) +#define DEV2G5_PCS_FX100_CFG_PCS_ENA_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_CFG_PCS_ENA, x) +#define DEV2G5_PCS_FX100_CFG_PCS_ENA_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_CFG_PCS_ENA, x) + +/* DEV1G:PCS_FX100_STATUS:PCS_FX100_STATUS */ +#define DEV2G5_PCS_FX100_STATUS(t) __REG(TARGET_DEV2G5, t, 65, 168, 0, 1, 4, 0, 0, 1, 4) + +#define DEV2G5_PCS_FX100_STATUS_EDGE_POS_PTP GENMASK(11, 8) +#define DEV2G5_PCS_FX100_STATUS_EDGE_POS_PTP_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_EDGE_POS_PTP, x) +#define DEV2G5_PCS_FX100_STATUS_EDGE_POS_PTP_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_EDGE_POS_PTP, x) + +#define DEV2G5_PCS_FX100_STATUS_PCS_ERROR_STICKY BIT(7) +#define DEV2G5_PCS_FX100_STATUS_PCS_ERROR_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_PCS_ERROR_STICKY, x) +#define DEV2G5_PCS_FX100_STATUS_PCS_ERROR_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_PCS_ERROR_STICKY, x) + +#define DEV2G5_PCS_FX100_STATUS_FEF_FOUND_STICKY BIT(6) +#define DEV2G5_PCS_FX100_STATUS_FEF_FOUND_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_FEF_FOUND_STICKY, x) +#define DEV2G5_PCS_FX100_STATUS_FEF_FOUND_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_FEF_FOUND_STICKY, x) + +#define DEV2G5_PCS_FX100_STATUS_SSD_ERROR_STICKY BIT(5) +#define DEV2G5_PCS_FX100_STATUS_SSD_ERROR_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_SSD_ERROR_STICKY, x) +#define DEV2G5_PCS_FX100_STATUS_SSD_ERROR_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_SSD_ERROR_STICKY, x) + +#define DEV2G5_PCS_FX100_STATUS_SYNC_LOST_STICKY BIT(4) +#define DEV2G5_PCS_FX100_STATUS_SYNC_LOST_STICKY_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_SYNC_LOST_STICKY, x) +#define DEV2G5_PCS_FX100_STATUS_SYNC_LOST_STICKY_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_SYNC_LOST_STICKY, x) + +#define DEV2G5_PCS_FX100_STATUS_FEF_STATUS BIT(2) +#define DEV2G5_PCS_FX100_STATUS_FEF_STATUS_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_FEF_STATUS, x) +#define DEV2G5_PCS_FX100_STATUS_FEF_STATUS_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_FEF_STATUS, x) + +#define DEV2G5_PCS_FX100_STATUS_SIGNAL_DETECT BIT(1) +#define DEV2G5_PCS_FX100_STATUS_SIGNAL_DETECT_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_SIGNAL_DETECT, x) +#define DEV2G5_PCS_FX100_STATUS_SIGNAL_DETECT_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_SIGNAL_DETECT, x) + +#define DEV2G5_PCS_FX100_STATUS_SYNC_STATUS BIT(0) +#define DEV2G5_PCS_FX100_STATUS_SYNC_STATUS_SET(x)\ + FIELD_PREP(DEV2G5_PCS_FX100_STATUS_SYNC_STATUS, x) +#define DEV2G5_PCS_FX100_STATUS_SYNC_STATUS_GET(x)\ + FIELD_GET(DEV2G5_PCS_FX100_STATUS_SYNC_STATUS, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_ENA_CFG */ +#define DEV5G_MAC_ENA_CFG(t) __REG(TARGET_DEV5G, t, 13, 0, 0, 1, 60, 0, 0, 1, 4) + +#define DEV5G_MAC_ENA_CFG_RX_ENA BIT(4) +#define DEV5G_MAC_ENA_CFG_RX_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ENA_CFG_RX_ENA, x) +#define DEV5G_MAC_ENA_CFG_RX_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ENA_CFG_RX_ENA, x) + +#define DEV5G_MAC_ENA_CFG_TX_ENA BIT(0) +#define DEV5G_MAC_ENA_CFG_TX_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ENA_CFG_TX_ENA, x) +#define DEV5G_MAC_ENA_CFG_TX_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ENA_CFG_TX_ENA, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_MAXLEN_CFG */ +#define DEV5G_MAC_MAXLEN_CFG(t) __REG(TARGET_DEV5G, t, 13, 0, 0, 1, 60, 8, 0, 1, 4) + +#define DEV5G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK BIT(16) +#define DEV5G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_SET(x)\ + FIELD_PREP(DEV5G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, x) +#define DEV5G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_GET(x)\ + FIELD_GET(DEV5G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, x) + +#define DEV5G_MAC_MAXLEN_CFG_MAX_LEN GENMASK(15, 0) +#define DEV5G_MAC_MAXLEN_CFG_MAX_LEN_SET(x)\ + FIELD_PREP(DEV5G_MAC_MAXLEN_CFG_MAX_LEN, x) +#define DEV5G_MAC_MAXLEN_CFG_MAX_LEN_GET(x)\ + FIELD_GET(DEV5G_MAC_MAXLEN_CFG_MAX_LEN, x) + +/* DEV10G:MAC_CFG_STATUS:MAC_ADV_CHK_CFG */ +#define DEV5G_MAC_ADV_CHK_CFG(t) __REG(TARGET_DEV5G, t, 13, 0, 0, 1, 60, 28, 0, 1, 4) + +#define DEV5G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA BIT(24) +#define DEV5G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA, x) +#define DEV5G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_EXT_EOP_CHK_ENA, x) + +#define DEV5G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA BIT(20) +#define DEV5G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA, x) +#define DEV5G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_EXT_SOP_CHK_ENA, x) + +#define DEV5G_MAC_ADV_CHK_CFG_SFD_CHK_ENA BIT(16) +#define DEV5G_MAC_ADV_CHK_CFG_SFD_CHK_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_SFD_CHK_ENA, x) +#define DEV5G_MAC_ADV_CHK_CFG_SFD_CHK_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_SFD_CHK_ENA, x) + +#define DEV5G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS BIT(12) +#define DEV5G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS, x) +#define DEV5G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_PRM_SHK_CHK_DIS, x) + +#define DEV5G_MAC_ADV_CHK_CFG_PRM_CHK_ENA BIT(8) +#define DEV5G_MAC_ADV_CHK_CFG_PRM_CHK_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_PRM_CHK_ENA, x) +#define DEV5G_MAC_ADV_CHK_CFG_PRM_CHK_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_PRM_CHK_ENA, x) + +#define DEV5G_MAC_ADV_CHK_CFG_OOR_ERR_ENA BIT(4) +#define DEV5G_MAC_ADV_CHK_CFG_OOR_ERR_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_OOR_ERR_ENA, x) +#define DEV5G_MAC_ADV_CHK_CFG_OOR_ERR_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_OOR_ERR_ENA, x) + +#define DEV5G_MAC_ADV_CHK_CFG_INR_ERR_ENA BIT(0) +#define DEV5G_MAC_ADV_CHK_CFG_INR_ERR_ENA_SET(x)\ + FIELD_PREP(DEV5G_MAC_ADV_CHK_CFG_INR_ERR_ENA, x) +#define DEV5G_MAC_ADV_CHK_CFG_INR_ERR_ENA_GET(x)\ + FIELD_GET(DEV5G_MAC_ADV_CHK_CFG_INR_ERR_ENA, x) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SYMBOL_ERR_CNT */ +#define DEV5G_RX_SYMBOL_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 0, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_PAUSE_CNT */ +#define DEV5G_RX_PAUSE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 4, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_UNSUP_OPCODE_CNT */ +#define DEV5G_RX_UNSUP_OPCODE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 8, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_UC_CNT */ +#define DEV5G_RX_UC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 12, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_MC_CNT */ +#define DEV5G_RX_MC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 16, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_BC_CNT */ +#define DEV5G_RX_BC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 20, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_CRC_ERR_CNT */ +#define DEV5G_RX_CRC_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 24, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_UNDERSIZE_CNT */ +#define DEV5G_RX_UNDERSIZE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 28, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_FRAGMENTS_CNT */ +#define DEV5G_RX_FRAGMENTS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 32, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_IN_RANGE_LEN_ERR_CNT */ +#define DEV5G_RX_IN_RANGE_LEN_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 36, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_OUT_OF_RANGE_LEN_ERR_CNT */ +#define DEV5G_RX_OUT_OF_RANGE_LEN_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 40, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_OVERSIZE_CNT */ +#define DEV5G_RX_OVERSIZE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 44, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_JABBERS_CNT */ +#define DEV5G_RX_JABBERS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 48, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE64_CNT */ +#define DEV5G_RX_SIZE64_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 52, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE65TO127_CNT */ +#define DEV5G_RX_SIZE65TO127_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 56, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE128TO255_CNT */ +#define DEV5G_RX_SIZE128TO255_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 60, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE256TO511_CNT */ +#define DEV5G_RX_SIZE256TO511_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 64, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE512TO1023_CNT */ +#define DEV5G_RX_SIZE512TO1023_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 68, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE1024TO1518_CNT */ +#define DEV5G_RX_SIZE1024TO1518_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 72, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_SIZE1519TOMAX_CNT */ +#define DEV5G_RX_SIZE1519TOMAX_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 76, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_IPG_SHRINK_CNT */ +#define DEV5G_RX_IPG_SHRINK_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 80, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_PAUSE_CNT */ +#define DEV5G_TX_PAUSE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 84, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_UC_CNT */ +#define DEV5G_TX_UC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 88, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_MC_CNT */ +#define DEV5G_TX_MC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 92, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_BC_CNT */ +#define DEV5G_TX_BC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 96, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE64_CNT */ +#define DEV5G_TX_SIZE64_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 100, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE65TO127_CNT */ +#define DEV5G_TX_SIZE65TO127_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 104, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE128TO255_CNT */ +#define DEV5G_TX_SIZE128TO255_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 108, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE256TO511_CNT */ +#define DEV5G_TX_SIZE256TO511_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 112, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE512TO1023_CNT */ +#define DEV5G_TX_SIZE512TO1023_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 116, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE1024TO1518_CNT */ +#define DEV5G_TX_SIZE1024TO1518_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 120, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_SIZE1519TOMAX_CNT */ +#define DEV5G_TX_SIZE1519TOMAX_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 124, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_ALIGNMENT_LOST_CNT */ +#define DEV5G_RX_ALIGNMENT_LOST_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 128, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_TAGGED_FRMS_CNT */ +#define DEV5G_RX_TAGGED_FRMS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 132, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_UNTAGGED_FRMS_CNT */ +#define DEV5G_RX_UNTAGGED_FRMS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 136, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_TAGGED_FRMS_CNT */ +#define DEV5G_TX_TAGGED_FRMS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 140, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:TX_UNTAGGED_FRMS_CNT */ +#define DEV5G_TX_UNTAGGED_FRMS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 144, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SYMBOL_ERR_CNT */ +#define DEV5G_PMAC_RX_SYMBOL_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 148, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_PAUSE_CNT */ +#define DEV5G_PMAC_RX_PAUSE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 152, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_UNSUP_OPCODE_CNT */ +#define DEV5G_PMAC_RX_UNSUP_OPCODE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 156, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_UC_CNT */ +#define DEV5G_PMAC_RX_UC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 160, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_MC_CNT */ +#define DEV5G_PMAC_RX_MC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 164, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_BC_CNT */ +#define DEV5G_PMAC_RX_BC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 168, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_CRC_ERR_CNT */ +#define DEV5G_PMAC_RX_CRC_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 172, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_UNDERSIZE_CNT */ +#define DEV5G_PMAC_RX_UNDERSIZE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 176, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_FRAGMENTS_CNT */ +#define DEV5G_PMAC_RX_FRAGMENTS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 180, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_IN_RANGE_LEN_ERR_CNT */ +#define DEV5G_PMAC_RX_IN_RANGE_LEN_ERR_CNT(t) __REG(TARGET_DEV5G,\ + t, 13, 60, 0, 1, 312, 184, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_OUT_OF_RANGE_LEN_ERR_CNT */ +#define DEV5G_PMAC_RX_OUT_OF_RANGE_LEN_ERR_CNT(t) __REG(TARGET_DEV5G,\ + t, 13, 60, 0, 1, 312, 188, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_OVERSIZE_CNT */ +#define DEV5G_PMAC_RX_OVERSIZE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 192, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_JABBERS_CNT */ +#define DEV5G_PMAC_RX_JABBERS_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 196, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE64_CNT */ +#define DEV5G_PMAC_RX_SIZE64_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 200, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE65TO127_CNT */ +#define DEV5G_PMAC_RX_SIZE65TO127_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 204, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE128TO255_CNT */ +#define DEV5G_PMAC_RX_SIZE128TO255_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 208, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE256TO511_CNT */ +#define DEV5G_PMAC_RX_SIZE256TO511_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 212, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE512TO1023_CNT */ +#define DEV5G_PMAC_RX_SIZE512TO1023_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 216, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE1024TO1518_CNT */ +#define DEV5G_PMAC_RX_SIZE1024TO1518_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 220, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_SIZE1519TOMAX_CNT */ +#define DEV5G_PMAC_RX_SIZE1519TOMAX_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 224, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_PAUSE_CNT */ +#define DEV5G_PMAC_TX_PAUSE_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 228, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_UC_CNT */ +#define DEV5G_PMAC_TX_UC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 232, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_MC_CNT */ +#define DEV5G_PMAC_TX_MC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 236, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_BC_CNT */ +#define DEV5G_PMAC_TX_BC_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 240, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE64_CNT */ +#define DEV5G_PMAC_TX_SIZE64_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 244, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE65TO127_CNT */ +#define DEV5G_PMAC_TX_SIZE65TO127_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 248, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE128TO255_CNT */ +#define DEV5G_PMAC_TX_SIZE128TO255_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 252, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE256TO511_CNT */ +#define DEV5G_PMAC_TX_SIZE256TO511_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 256, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE512TO1023_CNT */ +#define DEV5G_PMAC_TX_SIZE512TO1023_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 260, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE1024TO1518_CNT */ +#define DEV5G_PMAC_TX_SIZE1024TO1518_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 264, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_TX_SIZE1519TOMAX_CNT */ +#define DEV5G_PMAC_TX_SIZE1519TOMAX_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 268, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_ALIGNMENT_LOST_CNT */ +#define DEV5G_PMAC_RX_ALIGNMENT_LOST_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 272, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:MM_RX_ASSEMBLY_ERR_CNT */ +#define DEV5G_MM_RX_ASSEMBLY_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 276, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:MM_RX_SMD_ERR_CNT */ +#define DEV5G_MM_RX_SMD_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 280, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:MM_RX_ASSEMBLY_OK_CNT */ +#define DEV5G_MM_RX_ASSEMBLY_OK_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 284, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:MM_RX_MERGE_FRAG_CNT */ +#define DEV5G_MM_RX_MERGE_FRAG_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 288, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:MM_TX_PFRAGMENT_CNT */ +#define DEV5G_MM_TX_PFRAGMENT_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 292, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_HIH_CKSM_ERR_CNT */ +#define DEV5G_RX_HIH_CKSM_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 296, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:RX_XGMII_PROT_ERR_CNT */ +#define DEV5G_RX_XGMII_PROT_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 300, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_HIH_CKSM_ERR_CNT */ +#define DEV5G_PMAC_RX_HIH_CKSM_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 304, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_32BIT:PMAC_RX_XGMII_PROT_ERR_CNT */ +#define DEV5G_PMAC_RX_XGMII_PROT_ERR_CNT(t) __REG(TARGET_DEV5G, t, 13, 60, 0, 1, 312, 308, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:RX_IN_BYTES_CNT */ +#define DEV5G_RX_IN_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 0, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:RX_IN_BYTES_MSB_CNT */ +#define DEV5G_RX_IN_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 4, 0, 1, 4) + +#define DEV5G_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT, x) +#define DEV5G_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_RX_IN_BYTES_MSB_CNT_RX_IN_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:RX_OK_BYTES_CNT */ +#define DEV5G_RX_OK_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 8, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:RX_OK_BYTES_MSB_CNT */ +#define DEV5G_RX_OK_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 12, 0, 1, 4) + +#define DEV5G_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT, x) +#define DEV5G_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_RX_OK_BYTES_MSB_CNT_RX_OK_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:RX_BAD_BYTES_CNT */ +#define DEV5G_RX_BAD_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 16, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:RX_BAD_BYTES_MSB_CNT */ +#define DEV5G_RX_BAD_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 20, 0, 1, 4) + +#define DEV5G_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT, x) +#define DEV5G_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_RX_BAD_BYTES_MSB_CNT_RX_BAD_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:TX_OUT_BYTES_CNT */ +#define DEV5G_TX_OUT_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 24, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:TX_OUT_BYTES_MSB_CNT */ +#define DEV5G_TX_OUT_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 28, 0, 1, 4) + +#define DEV5G_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT, x) +#define DEV5G_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_TX_OUT_BYTES_MSB_CNT_TX_OUT_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:TX_OK_BYTES_CNT */ +#define DEV5G_TX_OK_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 32, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:TX_OK_BYTES_MSB_CNT */ +#define DEV5G_TX_OK_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 36, 0, 1, 4) + +#define DEV5G_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT, x) +#define DEV5G_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_TX_OK_BYTES_MSB_CNT_TX_OK_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:PMAC_RX_OK_BYTES_CNT */ +#define DEV5G_PMAC_RX_OK_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 40, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:PMAC_RX_OK_BYTES_MSB_CNT */ +#define DEV5G_PMAC_RX_OK_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 44, 0, 1, 4) + +#define DEV5G_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT, x) +#define DEV5G_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_PMAC_RX_OK_BYTES_MSB_CNT_PMAC_RX_OK_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:PMAC_RX_BAD_BYTES_CNT */ +#define DEV5G_PMAC_RX_BAD_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 48, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:PMAC_RX_BAD_BYTES_MSB_CNT */ +#define DEV5G_PMAC_RX_BAD_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 52, 0, 1, 4) + +#define DEV5G_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT, x) +#define DEV5G_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_PMAC_RX_BAD_BYTES_MSB_CNT_PMAC_RX_BAD_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_STATISTICS_40BIT:PMAC_TX_OK_BYTES_CNT */ +#define DEV5G_PMAC_TX_OK_BYTES_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 56, 0, 1, 4) + +/* DEV10G:DEV_STATISTICS_40BIT:PMAC_TX_OK_BYTES_MSB_CNT */ +#define DEV5G_PMAC_TX_OK_BYTES_MSB_CNT(t) __REG(TARGET_DEV5G, t, 13, 372, 0, 1, 64, 60, 0, 1, 4) + +#define DEV5G_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT GENMASK(7, 0) +#define DEV5G_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT_SET(x)\ + FIELD_PREP(DEV5G_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT, x) +#define DEV5G_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT_GET(x)\ + FIELD_GET(DEV5G_PMAC_TX_OK_BYTES_MSB_CNT_PMAC_TX_OK_BYTES_MSB_CNT, x) + +/* DEV10G:DEV_CFG_STATUS:DEV_RST_CTRL */ +#define DEV5G_DEV_RST_CTRL(t) __REG(TARGET_DEV5G, t, 13, 436, 0, 1, 52, 0, 0, 1, 4) + +#define DEV5G_DEV_RST_CTRL_PARDET_MODE_ENA BIT(28) +#define DEV5G_DEV_RST_CTRL_PARDET_MODE_ENA_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_PARDET_MODE_ENA, x) +#define DEV5G_DEV_RST_CTRL_PARDET_MODE_ENA_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_PARDET_MODE_ENA, x) + +#define DEV5G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS BIT(27) +#define DEV5G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) +#define DEV5G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_USXGMII_OSET_FILTER_DIS, x) + +#define DEV5G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS GENMASK(26, 25) +#define DEV5G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS, x) +#define DEV5G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_MUXED_USXGMII_NETWORK_PORTS, x) + +#define DEV5G_DEV_RST_CTRL_SERDES_SPEED_SEL GENMASK(24, 23) +#define DEV5G_DEV_RST_CTRL_SERDES_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_SERDES_SPEED_SEL, x) +#define DEV5G_DEV_RST_CTRL_SERDES_SPEED_SEL_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_SERDES_SPEED_SEL, x) + +#define DEV5G_DEV_RST_CTRL_SPEED_SEL GENMASK(22, 20) +#define DEV5G_DEV_RST_CTRL_SPEED_SEL_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_SPEED_SEL, x) +#define DEV5G_DEV_RST_CTRL_SPEED_SEL_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_SPEED_SEL, x) + +#define DEV5G_DEV_RST_CTRL_PCS_TX_RST BIT(12) +#define DEV5G_DEV_RST_CTRL_PCS_TX_RST_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_PCS_TX_RST, x) +#define DEV5G_DEV_RST_CTRL_PCS_TX_RST_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_PCS_TX_RST, x) + +#define DEV5G_DEV_RST_CTRL_PCS_RX_RST BIT(8) +#define DEV5G_DEV_RST_CTRL_PCS_RX_RST_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_PCS_RX_RST, x) +#define DEV5G_DEV_RST_CTRL_PCS_RX_RST_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_PCS_RX_RST, x) + +#define DEV5G_DEV_RST_CTRL_MAC_TX_RST BIT(4) +#define DEV5G_DEV_RST_CTRL_MAC_TX_RST_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_MAC_TX_RST, x) +#define DEV5G_DEV_RST_CTRL_MAC_TX_RST_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_MAC_TX_RST, x) + +#define DEV5G_DEV_RST_CTRL_MAC_RX_RST BIT(0) +#define DEV5G_DEV_RST_CTRL_MAC_RX_RST_SET(x)\ + FIELD_PREP(DEV5G_DEV_RST_CTRL_MAC_RX_RST, x) +#define DEV5G_DEV_RST_CTRL_MAC_RX_RST_GET(x)\ + FIELD_GET(DEV5G_DEV_RST_CTRL_MAC_RX_RST, x) + +/* DSM:RAM_CTRL:RAM_INIT */ +#define DSM_RAM_INIT __REG(TARGET_DSM, 0, 1, 0, 0, 1, 4, 0, 0, 1, 4) + +#define DSM_RAM_INIT_RAM_INIT BIT(1) +#define DSM_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(DSM_RAM_INIT_RAM_INIT, x) +#define DSM_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(DSM_RAM_INIT_RAM_INIT, x) + +#define DSM_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define DSM_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(DSM_RAM_INIT_RAM_CFG_HOOK, x) +#define DSM_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(DSM_RAM_INIT_RAM_CFG_HOOK, x) + +/* DSM:CFG:BUF_CFG */ +#define DSM_BUF_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 0, r, 67, 4) + +#define DSM_BUF_CFG_CSC_STAT_DIS BIT(13) +#define DSM_BUF_CFG_CSC_STAT_DIS_SET(x)\ + FIELD_PREP(DSM_BUF_CFG_CSC_STAT_DIS, x) +#define DSM_BUF_CFG_CSC_STAT_DIS_GET(x)\ + FIELD_GET(DSM_BUF_CFG_CSC_STAT_DIS, x) + +#define DSM_BUF_CFG_AGING_ENA BIT(12) +#define DSM_BUF_CFG_AGING_ENA_SET(x)\ + FIELD_PREP(DSM_BUF_CFG_AGING_ENA, x) +#define DSM_BUF_CFG_AGING_ENA_GET(x)\ + FIELD_GET(DSM_BUF_CFG_AGING_ENA, x) + +#define DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS BIT(11) +#define DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS_SET(x)\ + FIELD_PREP(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS, x) +#define DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS_GET(x)\ + FIELD_GET(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS, x) + +#define DSM_BUF_CFG_UNDERFLOW_WATCHDOG_TIMEOUT GENMASK(10, 0) +#define DSM_BUF_CFG_UNDERFLOW_WATCHDOG_TIMEOUT_SET(x)\ + FIELD_PREP(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_TIMEOUT, x) +#define DSM_BUF_CFG_UNDERFLOW_WATCHDOG_TIMEOUT_GET(x)\ + FIELD_GET(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_TIMEOUT, x) + +/* DSM:CFG:DEV_TX_STOP_WM_CFG */ +#define DSM_DEV_TX_STOP_WM_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 1360, r, 67, 4) + +#define DSM_DEV_TX_STOP_WM_CFG_FAST_STARTUP_ENA BIT(9) +#define DSM_DEV_TX_STOP_WM_CFG_FAST_STARTUP_ENA_SET(x)\ + FIELD_PREP(DSM_DEV_TX_STOP_WM_CFG_FAST_STARTUP_ENA, x) +#define DSM_DEV_TX_STOP_WM_CFG_FAST_STARTUP_ENA_GET(x)\ + FIELD_GET(DSM_DEV_TX_STOP_WM_CFG_FAST_STARTUP_ENA, x) + +#define DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA BIT(8) +#define DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA_SET(x)\ + FIELD_PREP(DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA, x) +#define DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA_GET(x)\ + FIELD_GET(DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA, x) + +#define DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM GENMASK(7, 1) +#define DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_SET(x)\ + FIELD_PREP(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM, x) +#define DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_GET(x)\ + FIELD_GET(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM, x) + +#define DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR BIT(0) +#define DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_SET(x)\ + FIELD_PREP(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR, x) +#define DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_GET(x)\ + FIELD_GET(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR, x) + +/* DSM:CFG:RX_PAUSE_CFG */ +#define DSM_RX_PAUSE_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 1628, r, 67, 4) + +#define DSM_RX_PAUSE_CFG_RX_PAUSE_EN BIT(1) +#define DSM_RX_PAUSE_CFG_RX_PAUSE_EN_SET(x)\ + FIELD_PREP(DSM_RX_PAUSE_CFG_RX_PAUSE_EN, x) +#define DSM_RX_PAUSE_CFG_RX_PAUSE_EN_GET(x)\ + FIELD_GET(DSM_RX_PAUSE_CFG_RX_PAUSE_EN, x) + +#define DSM_RX_PAUSE_CFG_FC_OBEY_LOCAL BIT(0) +#define DSM_RX_PAUSE_CFG_FC_OBEY_LOCAL_SET(x)\ + FIELD_PREP(DSM_RX_PAUSE_CFG_FC_OBEY_LOCAL, x) +#define DSM_RX_PAUSE_CFG_FC_OBEY_LOCAL_GET(x)\ + FIELD_GET(DSM_RX_PAUSE_CFG_FC_OBEY_LOCAL, x) + +/* DSM:CFG:MAC_CFG */ +#define DSM_MAC_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 2432, r, 67, 4) + +#define DSM_MAC_CFG_TX_PAUSE_VAL GENMASK(31, 16) +#define DSM_MAC_CFG_TX_PAUSE_VAL_SET(x)\ + FIELD_PREP(DSM_MAC_CFG_TX_PAUSE_VAL, x) +#define DSM_MAC_CFG_TX_PAUSE_VAL_GET(x)\ + FIELD_GET(DSM_MAC_CFG_TX_PAUSE_VAL, x) + +#define DSM_MAC_CFG_HDX_BACKPREASSURE BIT(2) +#define DSM_MAC_CFG_HDX_BACKPREASSURE_SET(x)\ + FIELD_PREP(DSM_MAC_CFG_HDX_BACKPREASSURE, x) +#define DSM_MAC_CFG_HDX_BACKPREASSURE_GET(x)\ + FIELD_GET(DSM_MAC_CFG_HDX_BACKPREASSURE, x) + +#define DSM_MAC_CFG_SEND_PAUSE_FRM_TWICE BIT(1) +#define DSM_MAC_CFG_SEND_PAUSE_FRM_TWICE_SET(x)\ + FIELD_PREP(DSM_MAC_CFG_SEND_PAUSE_FRM_TWICE, x) +#define DSM_MAC_CFG_SEND_PAUSE_FRM_TWICE_GET(x)\ + FIELD_GET(DSM_MAC_CFG_SEND_PAUSE_FRM_TWICE, x) + +#define DSM_MAC_CFG_TX_PAUSE_XON_XOFF BIT(0) +#define DSM_MAC_CFG_TX_PAUSE_XON_XOFF_SET(x)\ + FIELD_PREP(DSM_MAC_CFG_TX_PAUSE_XON_XOFF, x) +#define DSM_MAC_CFG_TX_PAUSE_XON_XOFF_GET(x)\ + FIELD_GET(DSM_MAC_CFG_TX_PAUSE_XON_XOFF, x) + +/* DSM:CFG:MAC_ADDR_BASE_HIGH_CFG */ +#define DSM_MAC_ADDR_BASE_HIGH_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 2700, r, 65, 4) + +#define DSM_MAC_ADDR_BASE_HIGH_CFG_MAC_ADDR_HIGH GENMASK(23, 0) +#define DSM_MAC_ADDR_BASE_HIGH_CFG_MAC_ADDR_HIGH_SET(x)\ + FIELD_PREP(DSM_MAC_ADDR_BASE_HIGH_CFG_MAC_ADDR_HIGH, x) +#define DSM_MAC_ADDR_BASE_HIGH_CFG_MAC_ADDR_HIGH_GET(x)\ + FIELD_GET(DSM_MAC_ADDR_BASE_HIGH_CFG_MAC_ADDR_HIGH, x) + +/* DSM:CFG:MAC_ADDR_BASE_LOW_CFG */ +#define DSM_MAC_ADDR_BASE_LOW_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 2960, r, 65, 4) + +#define DSM_MAC_ADDR_BASE_LOW_CFG_MAC_ADDR_LOW GENMASK(23, 0) +#define DSM_MAC_ADDR_BASE_LOW_CFG_MAC_ADDR_LOW_SET(x)\ + FIELD_PREP(DSM_MAC_ADDR_BASE_LOW_CFG_MAC_ADDR_LOW, x) +#define DSM_MAC_ADDR_BASE_LOW_CFG_MAC_ADDR_LOW_GET(x)\ + FIELD_GET(DSM_MAC_ADDR_BASE_LOW_CFG_MAC_ADDR_LOW, x) + +/* DSM:CFG:TAXI_CAL_CFG */ +#define DSM_TAXI_CAL_CFG(r) __REG(TARGET_DSM, 0, 1, 20, 0, 1, 3528, 3224, r, 9, 4) + +#define DSM_TAXI_CAL_CFG_CAL_IDX GENMASK(20, 15) +#define DSM_TAXI_CAL_CFG_CAL_IDX_SET(x)\ + FIELD_PREP(DSM_TAXI_CAL_CFG_CAL_IDX, x) +#define DSM_TAXI_CAL_CFG_CAL_IDX_GET(x)\ + FIELD_GET(DSM_TAXI_CAL_CFG_CAL_IDX, x) + +#define DSM_TAXI_CAL_CFG_CAL_CUR_LEN GENMASK(14, 9) +#define DSM_TAXI_CAL_CFG_CAL_CUR_LEN_SET(x)\ + FIELD_PREP(DSM_TAXI_CAL_CFG_CAL_CUR_LEN, x) +#define DSM_TAXI_CAL_CFG_CAL_CUR_LEN_GET(x)\ + FIELD_GET(DSM_TAXI_CAL_CFG_CAL_CUR_LEN, x) + +#define DSM_TAXI_CAL_CFG_CAL_CUR_VAL GENMASK(8, 5) +#define DSM_TAXI_CAL_CFG_CAL_CUR_VAL_SET(x)\ + FIELD_PREP(DSM_TAXI_CAL_CFG_CAL_CUR_VAL, x) +#define DSM_TAXI_CAL_CFG_CAL_CUR_VAL_GET(x)\ + FIELD_GET(DSM_TAXI_CAL_CFG_CAL_CUR_VAL, x) + +#define DSM_TAXI_CAL_CFG_CAL_PGM_VAL GENMASK(4, 1) +#define DSM_TAXI_CAL_CFG_CAL_PGM_VAL_SET(x)\ + FIELD_PREP(DSM_TAXI_CAL_CFG_CAL_PGM_VAL, x) +#define DSM_TAXI_CAL_CFG_CAL_PGM_VAL_GET(x)\ + FIELD_GET(DSM_TAXI_CAL_CFG_CAL_PGM_VAL, x) + +#define DSM_TAXI_CAL_CFG_CAL_PGM_ENA BIT(0) +#define DSM_TAXI_CAL_CFG_CAL_PGM_ENA_SET(x)\ + FIELD_PREP(DSM_TAXI_CAL_CFG_CAL_PGM_ENA, x) +#define DSM_TAXI_CAL_CFG_CAL_PGM_ENA_GET(x)\ + FIELD_GET(DSM_TAXI_CAL_CFG_CAL_PGM_ENA, x) + +/* EACL:POL_CFG:POL_EACL_CFG */ +#define EACL_POL_EACL_CFG __REG(TARGET_EACL, 0, 1, 150608, 0, 1, 780, 768, 0, 1, 4) + +#define EACL_POL_EACL_CFG_EACL_CNT_MARKED_AS_DROPPED BIT(5) +#define EACL_POL_EACL_CFG_EACL_CNT_MARKED_AS_DROPPED_SET(x)\ + FIELD_PREP(EACL_POL_EACL_CFG_EACL_CNT_MARKED_AS_DROPPED, x) +#define EACL_POL_EACL_CFG_EACL_CNT_MARKED_AS_DROPPED_GET(x)\ + FIELD_GET(EACL_POL_EACL_CFG_EACL_CNT_MARKED_AS_DROPPED, x) + +#define EACL_POL_EACL_CFG_EACL_ALLOW_FP_COPY BIT(4) +#define EACL_POL_EACL_CFG_EACL_ALLOW_FP_COPY_SET(x)\ + FIELD_PREP(EACL_POL_EACL_CFG_EACL_ALLOW_FP_COPY, x) +#define EACL_POL_EACL_CFG_EACL_ALLOW_FP_COPY_GET(x)\ + FIELD_GET(EACL_POL_EACL_CFG_EACL_ALLOW_FP_COPY, x) + +#define EACL_POL_EACL_CFG_EACL_ALLOW_CPU_COPY BIT(3) +#define EACL_POL_EACL_CFG_EACL_ALLOW_CPU_COPY_SET(x)\ + FIELD_PREP(EACL_POL_EACL_CFG_EACL_ALLOW_CPU_COPY, x) +#define EACL_POL_EACL_CFG_EACL_ALLOW_CPU_COPY_GET(x)\ + FIELD_GET(EACL_POL_EACL_CFG_EACL_ALLOW_CPU_COPY, x) + +#define EACL_POL_EACL_CFG_EACL_FORCE_CLOSE BIT(2) +#define EACL_POL_EACL_CFG_EACL_FORCE_CLOSE_SET(x)\ + FIELD_PREP(EACL_POL_EACL_CFG_EACL_FORCE_CLOSE, x) +#define EACL_POL_EACL_CFG_EACL_FORCE_CLOSE_GET(x)\ + FIELD_GET(EACL_POL_EACL_CFG_EACL_FORCE_CLOSE, x) + +#define EACL_POL_EACL_CFG_EACL_FORCE_OPEN BIT(1) +#define EACL_POL_EACL_CFG_EACL_FORCE_OPEN_SET(x)\ + FIELD_PREP(EACL_POL_EACL_CFG_EACL_FORCE_OPEN, x) +#define EACL_POL_EACL_CFG_EACL_FORCE_OPEN_GET(x)\ + FIELD_GET(EACL_POL_EACL_CFG_EACL_FORCE_OPEN, x) + +#define EACL_POL_EACL_CFG_EACL_FORCE_INIT BIT(0) +#define EACL_POL_EACL_CFG_EACL_FORCE_INIT_SET(x)\ + FIELD_PREP(EACL_POL_EACL_CFG_EACL_FORCE_INIT, x) +#define EACL_POL_EACL_CFG_EACL_FORCE_INIT_GET(x)\ + FIELD_GET(EACL_POL_EACL_CFG_EACL_FORCE_INIT, x) + +/* EACL:RAM_CTRL:RAM_INIT */ +#define EACL_RAM_INIT __REG(TARGET_EACL, 0, 1, 118736, 0, 1, 4, 0, 0, 1, 4) + +#define EACL_RAM_INIT_RAM_INIT BIT(1) +#define EACL_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(EACL_RAM_INIT_RAM_INIT, x) +#define EACL_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(EACL_RAM_INIT_RAM_INIT, x) + +#define EACL_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define EACL_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(EACL_RAM_INIT_RAM_CFG_HOOK, x) +#define EACL_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(EACL_RAM_INIT_RAM_CFG_HOOK, x) + +/* FDMA:FDMA:FDMA_CH_ACTIVATE */ +#define FDMA_CH_ACTIVATE __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 0, 0, 1, 4) + +#define FDMA_CH_ACTIVATE_CH_ACTIVATE GENMASK(7, 0) +#define FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(x)\ + FIELD_PREP(FDMA_CH_ACTIVATE_CH_ACTIVATE, x) +#define FDMA_CH_ACTIVATE_CH_ACTIVATE_GET(x)\ + FIELD_GET(FDMA_CH_ACTIVATE_CH_ACTIVATE, x) + +/* FDMA:FDMA:FDMA_CH_RELOAD */ +#define FDMA_CH_RELOAD __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 4, 0, 1, 4) + +#define FDMA_CH_RELOAD_CH_RELOAD GENMASK(7, 0) +#define FDMA_CH_RELOAD_CH_RELOAD_SET(x)\ + FIELD_PREP(FDMA_CH_RELOAD_CH_RELOAD, x) +#define FDMA_CH_RELOAD_CH_RELOAD_GET(x)\ + FIELD_GET(FDMA_CH_RELOAD_CH_RELOAD, x) + +/* FDMA:FDMA:FDMA_CH_DISABLE */ +#define FDMA_CH_DISABLE __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 8, 0, 1, 4) + +#define FDMA_CH_DISABLE_CH_DISABLE GENMASK(7, 0) +#define FDMA_CH_DISABLE_CH_DISABLE_SET(x)\ + FIELD_PREP(FDMA_CH_DISABLE_CH_DISABLE, x) +#define FDMA_CH_DISABLE_CH_DISABLE_GET(x)\ + FIELD_GET(FDMA_CH_DISABLE_CH_DISABLE, x) + +/* FDMA:FDMA:FDMA_DCB_LLP */ +#define FDMA_DCB_LLP(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 52, r, 8, 4) + +/* FDMA:FDMA:FDMA_DCB_LLP1 */ +#define FDMA_DCB_LLP1(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 84, r, 8, 4) + +/* FDMA:FDMA:FDMA_DCB_LLP_PREV */ +#define FDMA_DCB_LLP_PREV(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 116, r, 8, 4) + +/* FDMA:FDMA:FDMA_DCB_LLP_PREV1 */ +#define FDMA_DCB_LLP_PREV1(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 148, r, 8, 4) + +/* FDMA:FDMA:FDMA_CH_CFG */ +#define FDMA_CH_CFG(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 224, r, 8, 4) + +#define FDMA_CH_CFG_CH_XTR_STATUS_MODE BIT(7) +#define FDMA_CH_CFG_CH_XTR_STATUS_MODE_SET(x)\ + FIELD_PREP(FDMA_CH_CFG_CH_XTR_STATUS_MODE, x) +#define FDMA_CH_CFG_CH_XTR_STATUS_MODE_GET(x)\ + FIELD_GET(FDMA_CH_CFG_CH_XTR_STATUS_MODE, x) + +#define FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY BIT(6) +#define FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(x)\ + FIELD_PREP(FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY, x) +#define FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_GET(x)\ + FIELD_GET(FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY, x) + +#define FDMA_CH_CFG_CH_INJ_PORT BIT(5) +#define FDMA_CH_CFG_CH_INJ_PORT_SET(x)\ + FIELD_PREP(FDMA_CH_CFG_CH_INJ_PORT, x) +#define FDMA_CH_CFG_CH_INJ_PORT_GET(x)\ + FIELD_GET(FDMA_CH_CFG_CH_INJ_PORT, x) + +#define FDMA_CH_CFG_CH_DCB_DB_CNT GENMASK(4, 1) +#define FDMA_CH_CFG_CH_DCB_DB_CNT_SET(x)\ + FIELD_PREP(FDMA_CH_CFG_CH_DCB_DB_CNT, x) +#define FDMA_CH_CFG_CH_DCB_DB_CNT_GET(x)\ + FIELD_GET(FDMA_CH_CFG_CH_DCB_DB_CNT, x) + +#define FDMA_CH_CFG_CH_MEM BIT(0) +#define FDMA_CH_CFG_CH_MEM_SET(x)\ + FIELD_PREP(FDMA_CH_CFG_CH_MEM, x) +#define FDMA_CH_CFG_CH_MEM_GET(x)\ + FIELD_GET(FDMA_CH_CFG_CH_MEM, x) + +/* FDMA:FDMA:FDMA_CH_TRANSLATE */ +#define FDMA_CH_TRANSLATE(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 256, r, 8, 4) + +#define FDMA_CH_TRANSLATE_OFFSET GENMASK(15, 0) +#define FDMA_CH_TRANSLATE_OFFSET_SET(x)\ + FIELD_PREP(FDMA_CH_TRANSLATE_OFFSET, x) +#define FDMA_CH_TRANSLATE_OFFSET_GET(x)\ + FIELD_GET(FDMA_CH_TRANSLATE_OFFSET, x) + +/* FDMA:FDMA:FDMA_XTR_CFG */ +#define FDMA_XTR_CFG __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 364, 0, 1, 4) + +#define FDMA_XTR_CFG_XTR_FIFO_WM GENMASK(15, 11) +#define FDMA_XTR_CFG_XTR_FIFO_WM_SET(x)\ + FIELD_PREP(FDMA_XTR_CFG_XTR_FIFO_WM, x) +#define FDMA_XTR_CFG_XTR_FIFO_WM_GET(x)\ + FIELD_GET(FDMA_XTR_CFG_XTR_FIFO_WM, x) + +#define FDMA_XTR_CFG_XTR_ARB_SAT GENMASK(10, 0) +#define FDMA_XTR_CFG_XTR_ARB_SAT_SET(x)\ + FIELD_PREP(FDMA_XTR_CFG_XTR_ARB_SAT, x) +#define FDMA_XTR_CFG_XTR_ARB_SAT_GET(x)\ + FIELD_GET(FDMA_XTR_CFG_XTR_ARB_SAT, x) + +/* FDMA:FDMA:FDMA_PORT_CTRL */ +#define FDMA_PORT_CTRL(r) __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 376, r, 2, 4) + +#define FDMA_PORT_CTRL_INJ_STOP BIT(4) +#define FDMA_PORT_CTRL_INJ_STOP_SET(x)\ + FIELD_PREP(FDMA_PORT_CTRL_INJ_STOP, x) +#define FDMA_PORT_CTRL_INJ_STOP_GET(x)\ + FIELD_GET(FDMA_PORT_CTRL_INJ_STOP, x) + +#define FDMA_PORT_CTRL_INJ_STOP_FORCE BIT(3) +#define FDMA_PORT_CTRL_INJ_STOP_FORCE_SET(x)\ + FIELD_PREP(FDMA_PORT_CTRL_INJ_STOP_FORCE, x) +#define FDMA_PORT_CTRL_INJ_STOP_FORCE_GET(x)\ + FIELD_GET(FDMA_PORT_CTRL_INJ_STOP_FORCE, x) + +#define FDMA_PORT_CTRL_XTR_STOP BIT(2) +#define FDMA_PORT_CTRL_XTR_STOP_SET(x)\ + FIELD_PREP(FDMA_PORT_CTRL_XTR_STOP, x) +#define FDMA_PORT_CTRL_XTR_STOP_GET(x)\ + FIELD_GET(FDMA_PORT_CTRL_XTR_STOP, x) + +#define FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY BIT(1) +#define FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY_SET(x)\ + FIELD_PREP(FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY, x) +#define FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY_GET(x)\ + FIELD_GET(FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY, x) + +#define FDMA_PORT_CTRL_XTR_BUF_RST BIT(0) +#define FDMA_PORT_CTRL_XTR_BUF_RST_SET(x)\ + FIELD_PREP(FDMA_PORT_CTRL_XTR_BUF_RST, x) +#define FDMA_PORT_CTRL_XTR_BUF_RST_GET(x)\ + FIELD_GET(FDMA_PORT_CTRL_XTR_BUF_RST, x) + +/* FDMA:FDMA:FDMA_INTR_DCB */ +#define FDMA_INTR_DCB __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 384, 0, 1, 4) + +#define FDMA_INTR_DCB_INTR_DCB GENMASK(7, 0) +#define FDMA_INTR_DCB_INTR_DCB_SET(x)\ + FIELD_PREP(FDMA_INTR_DCB_INTR_DCB, x) +#define FDMA_INTR_DCB_INTR_DCB_GET(x)\ + FIELD_GET(FDMA_INTR_DCB_INTR_DCB, x) + +/* FDMA:FDMA:FDMA_INTR_DCB_ENA */ +#define FDMA_INTR_DCB_ENA __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 388, 0, 1, 4) + +#define FDMA_INTR_DCB_ENA_INTR_DCB_ENA GENMASK(7, 0) +#define FDMA_INTR_DCB_ENA_INTR_DCB_ENA_SET(x)\ + FIELD_PREP(FDMA_INTR_DCB_ENA_INTR_DCB_ENA, x) +#define FDMA_INTR_DCB_ENA_INTR_DCB_ENA_GET(x)\ + FIELD_GET(FDMA_INTR_DCB_ENA_INTR_DCB_ENA, x) + +/* FDMA:FDMA:FDMA_INTR_DB */ +#define FDMA_INTR_DB __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 392, 0, 1, 4) + +#define FDMA_INTR_DB_INTR_DB GENMASK(7, 0) +#define FDMA_INTR_DB_INTR_DB_SET(x)\ + FIELD_PREP(FDMA_INTR_DB_INTR_DB, x) +#define FDMA_INTR_DB_INTR_DB_GET(x)\ + FIELD_GET(FDMA_INTR_DB_INTR_DB, x) + +/* FDMA:FDMA:FDMA_INTR_DB_ENA */ +#define FDMA_INTR_DB_ENA __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 396, 0, 1, 4) + +#define FDMA_INTR_DB_ENA_INTR_DB_ENA GENMASK(7, 0) +#define FDMA_INTR_DB_ENA_INTR_DB_ENA_SET(x)\ + FIELD_PREP(FDMA_INTR_DB_ENA_INTR_DB_ENA, x) +#define FDMA_INTR_DB_ENA_INTR_DB_ENA_GET(x)\ + FIELD_GET(FDMA_INTR_DB_ENA_INTR_DB_ENA, x) + +/* FDMA:FDMA:FDMA_INTR_ERR */ +#define FDMA_INTR_ERR __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 400, 0, 1, 4) + +#define FDMA_INTR_ERR_INTR_PORT_ERR GENMASK(9, 8) +#define FDMA_INTR_ERR_INTR_PORT_ERR_SET(x)\ + FIELD_PREP(FDMA_INTR_ERR_INTR_PORT_ERR, x) +#define FDMA_INTR_ERR_INTR_PORT_ERR_GET(x)\ + FIELD_GET(FDMA_INTR_ERR_INTR_PORT_ERR, x) + +#define FDMA_INTR_ERR_INTR_CH_ERR GENMASK(7, 0) +#define FDMA_INTR_ERR_INTR_CH_ERR_SET(x)\ + FIELD_PREP(FDMA_INTR_ERR_INTR_CH_ERR, x) +#define FDMA_INTR_ERR_INTR_CH_ERR_GET(x)\ + FIELD_GET(FDMA_INTR_ERR_INTR_CH_ERR, x) + +/* FDMA:FDMA:FDMA_ERRORS */ +#define FDMA_ERRORS __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 412, 0, 1, 4) + +#define FDMA_ERRORS_ERR_XTR_WR GENMASK(31, 30) +#define FDMA_ERRORS_ERR_XTR_WR_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_XTR_WR, x) +#define FDMA_ERRORS_ERR_XTR_WR_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_XTR_WR, x) + +#define FDMA_ERRORS_ERR_XTR_OVF GENMASK(29, 28) +#define FDMA_ERRORS_ERR_XTR_OVF_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_XTR_OVF, x) +#define FDMA_ERRORS_ERR_XTR_OVF_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_XTR_OVF, x) + +#define FDMA_ERRORS_ERR_XTR_TAXI32_OVF GENMASK(27, 26) +#define FDMA_ERRORS_ERR_XTR_TAXI32_OVF_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_XTR_TAXI32_OVF, x) +#define FDMA_ERRORS_ERR_XTR_TAXI32_OVF_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_XTR_TAXI32_OVF, x) + +#define FDMA_ERRORS_ERR_DCB_XTR_DATAL GENMASK(25, 24) +#define FDMA_ERRORS_ERR_DCB_XTR_DATAL_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_DCB_XTR_DATAL, x) +#define FDMA_ERRORS_ERR_DCB_XTR_DATAL_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_DCB_XTR_DATAL, x) + +#define FDMA_ERRORS_ERR_DCB_RD GENMASK(23, 16) +#define FDMA_ERRORS_ERR_DCB_RD_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_DCB_RD, x) +#define FDMA_ERRORS_ERR_DCB_RD_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_DCB_RD, x) + +#define FDMA_ERRORS_ERR_INJ_RD GENMASK(15, 10) +#define FDMA_ERRORS_ERR_INJ_RD_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_INJ_RD, x) +#define FDMA_ERRORS_ERR_INJ_RD_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_INJ_RD, x) + +#define FDMA_ERRORS_ERR_INJ_OUT_OF_SYNC GENMASK(9, 8) +#define FDMA_ERRORS_ERR_INJ_OUT_OF_SYNC_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_INJ_OUT_OF_SYNC, x) +#define FDMA_ERRORS_ERR_INJ_OUT_OF_SYNC_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_INJ_OUT_OF_SYNC, x) + +#define FDMA_ERRORS_ERR_CH_WR GENMASK(7, 0) +#define FDMA_ERRORS_ERR_CH_WR_SET(x)\ + FIELD_PREP(FDMA_ERRORS_ERR_CH_WR, x) +#define FDMA_ERRORS_ERR_CH_WR_GET(x)\ + FIELD_GET(FDMA_ERRORS_ERR_CH_WR, x) + +/* FDMA:FDMA:FDMA_ERRORS_2 */ +#define FDMA_ERRORS_2 __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 416, 0, 1, 4) + +#define FDMA_ERRORS_2_ERR_XTR_FRAG GENMASK(1, 0) +#define FDMA_ERRORS_2_ERR_XTR_FRAG_SET(x)\ + FIELD_PREP(FDMA_ERRORS_2_ERR_XTR_FRAG, x) +#define FDMA_ERRORS_2_ERR_XTR_FRAG_GET(x)\ + FIELD_GET(FDMA_ERRORS_2_ERR_XTR_FRAG, x) + +/* FDMA:FDMA:FDMA_CTRL */ +#define FDMA_CTRL __REG(TARGET_FDMA, 0, 1, 8, 0, 1, 428, 424, 0, 1, 4) + +#define FDMA_CTRL_NRESET BIT(0) +#define FDMA_CTRL_NRESET_SET(x)\ + FIELD_PREP(FDMA_CTRL_NRESET, x) +#define FDMA_CTRL_NRESET_GET(x)\ + FIELD_GET(FDMA_CTRL_NRESET, x) + +/* DEVCPU_GCB:CHIP_REGS:CHIP_ID */ +#define GCB_CHIP_ID __REG(TARGET_GCB, 0, 1, 0, 0, 1, 424, 0, 0, 1, 4) + +#define GCB_CHIP_ID_REV_ID GENMASK(31, 28) +#define GCB_CHIP_ID_REV_ID_SET(x)\ + FIELD_PREP(GCB_CHIP_ID_REV_ID, x) +#define GCB_CHIP_ID_REV_ID_GET(x)\ + FIELD_GET(GCB_CHIP_ID_REV_ID, x) + +#define GCB_CHIP_ID_PART_ID GENMASK(27, 12) +#define GCB_CHIP_ID_PART_ID_SET(x)\ + FIELD_PREP(GCB_CHIP_ID_PART_ID, x) +#define GCB_CHIP_ID_PART_ID_GET(x)\ + FIELD_GET(GCB_CHIP_ID_PART_ID, x) + +#define GCB_CHIP_ID_MFG_ID GENMASK(11, 1) +#define GCB_CHIP_ID_MFG_ID_SET(x)\ + FIELD_PREP(GCB_CHIP_ID_MFG_ID, x) +#define GCB_CHIP_ID_MFG_ID_GET(x)\ + FIELD_GET(GCB_CHIP_ID_MFG_ID, x) + +#define GCB_CHIP_ID_ONE BIT(0) +#define GCB_CHIP_ID_ONE_SET(x)\ + FIELD_PREP(GCB_CHIP_ID_ONE, x) +#define GCB_CHIP_ID_ONE_GET(x)\ + FIELD_GET(GCB_CHIP_ID_ONE, x) + +/* DEVCPU_GCB:CHIP_REGS:SOFT_RST */ +#define GCB_SOFT_RST __REG(TARGET_GCB, 0, 1, 0, 0, 1, 424, 8, 0, 1, 4) + +#define GCB_SOFT_RST_SOFT_NON_CFG_RST BIT(2) +#define GCB_SOFT_RST_SOFT_NON_CFG_RST_SET(x)\ + FIELD_PREP(GCB_SOFT_RST_SOFT_NON_CFG_RST, x) +#define GCB_SOFT_RST_SOFT_NON_CFG_RST_GET(x)\ + FIELD_GET(GCB_SOFT_RST_SOFT_NON_CFG_RST, x) + +#define GCB_SOFT_RST_SOFT_SWC_RST BIT(1) +#define GCB_SOFT_RST_SOFT_SWC_RST_SET(x)\ + FIELD_PREP(GCB_SOFT_RST_SOFT_SWC_RST, x) +#define GCB_SOFT_RST_SOFT_SWC_RST_GET(x)\ + FIELD_GET(GCB_SOFT_RST_SOFT_SWC_RST, x) + +#define GCB_SOFT_RST_SOFT_CHIP_RST BIT(0) +#define GCB_SOFT_RST_SOFT_CHIP_RST_SET(x)\ + FIELD_PREP(GCB_SOFT_RST_SOFT_CHIP_RST, x) +#define GCB_SOFT_RST_SOFT_CHIP_RST_GET(x)\ + FIELD_GET(GCB_SOFT_RST_SOFT_CHIP_RST, x) + +/* DEVCPU_GCB:CHIP_REGS:HW_SGPIO_SD_CFG */ +#define GCB_HW_SGPIO_SD_CFG __REG(TARGET_GCB, 0, 1, 0, 0, 1, 424, 20, 0, 1, 4) + +#define GCB_HW_SGPIO_SD_CFG_SD_HIGH_ENA BIT(1) +#define GCB_HW_SGPIO_SD_CFG_SD_HIGH_ENA_SET(x)\ + FIELD_PREP(GCB_HW_SGPIO_SD_CFG_SD_HIGH_ENA, x) +#define GCB_HW_SGPIO_SD_CFG_SD_HIGH_ENA_GET(x)\ + FIELD_GET(GCB_HW_SGPIO_SD_CFG_SD_HIGH_ENA, x) + +#define GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL BIT(0) +#define GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL_SET(x)\ + FIELD_PREP(GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL, x) +#define GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL_GET(x)\ + FIELD_GET(GCB_HW_SGPIO_SD_CFG_SD_MAP_SEL, x) + +/* DEVCPU_GCB:CHIP_REGS:HW_SGPIO_TO_SD_MAP_CFG */ +#define GCB_HW_SGPIO_TO_SD_MAP_CFG(r) __REG(TARGET_GCB, 0, 1, 0, 0, 1, 424, 24, r, 65, 4) + +#define GCB_HW_SGPIO_TO_SD_MAP_CFG_SGPIO_TO_SD_SEL GENMASK(8, 0) +#define GCB_HW_SGPIO_TO_SD_MAP_CFG_SGPIO_TO_SD_SEL_SET(x)\ + FIELD_PREP(GCB_HW_SGPIO_TO_SD_MAP_CFG_SGPIO_TO_SD_SEL, x) +#define GCB_HW_SGPIO_TO_SD_MAP_CFG_SGPIO_TO_SD_SEL_GET(x)\ + FIELD_GET(GCB_HW_SGPIO_TO_SD_MAP_CFG_SGPIO_TO_SD_SEL, x) + +/* DEVCPU_GCB:SIO_CTRL:SIO_CLOCK */ +#define GCB_SIO_CLOCK(g) __REG(TARGET_GCB, 0, 1, 876, g, 3, 280, 20, 0, 1, 4) + +#define GCB_SIO_CLOCK_SIO_CLK_FREQ GENMASK(19, 8) +#define GCB_SIO_CLOCK_SIO_CLK_FREQ_SET(x)\ + FIELD_PREP(GCB_SIO_CLOCK_SIO_CLK_FREQ, x) +#define GCB_SIO_CLOCK_SIO_CLK_FREQ_GET(x)\ + FIELD_GET(GCB_SIO_CLOCK_SIO_CLK_FREQ, x) + +#define GCB_SIO_CLOCK_SYS_CLK_PERIOD GENMASK(7, 0) +#define GCB_SIO_CLOCK_SYS_CLK_PERIOD_SET(x)\ + FIELD_PREP(GCB_SIO_CLOCK_SYS_CLK_PERIOD, x) +#define GCB_SIO_CLOCK_SYS_CLK_PERIOD_GET(x)\ + FIELD_GET(GCB_SIO_CLOCK_SYS_CLK_PERIOD, x) + +/* HSCH:HSCH_MISC:SYS_CLK_PER */ +#define HSCH_SYS_CLK_PER __REG(TARGET_HSCH, 0, 1, 163104, 0, 1, 648, 640, 0, 1, 4) + +#define HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS GENMASK(7, 0) +#define HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS_SET(x)\ + FIELD_PREP(HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS, x) +#define HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS_GET(x)\ + FIELD_GET(HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS, x) + +/* HSCH:SYSTEM:FLUSH_CTRL */ +#define HSCH_FLUSH_CTRL __REG(TARGET_HSCH, 0, 1, 184000, 0, 1, 312, 4, 0, 1, 4) + +#define HSCH_FLUSH_CTRL_FLUSH_ENA BIT(27) +#define HSCH_FLUSH_CTRL_FLUSH_ENA_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_ENA, x) +#define HSCH_FLUSH_CTRL_FLUSH_ENA_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_ENA, x) + +#define HSCH_FLUSH_CTRL_FLUSH_SRC BIT(26) +#define HSCH_FLUSH_CTRL_FLUSH_SRC_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_SRC, x) +#define HSCH_FLUSH_CTRL_FLUSH_SRC_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_SRC, x) + +#define HSCH_FLUSH_CTRL_FLUSH_DST BIT(25) +#define HSCH_FLUSH_CTRL_FLUSH_DST_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_DST, x) +#define HSCH_FLUSH_CTRL_FLUSH_DST_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_DST, x) + +#define HSCH_FLUSH_CTRL_FLUSH_PORT GENMASK(24, 18) +#define HSCH_FLUSH_CTRL_FLUSH_PORT_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_PORT, x) +#define HSCH_FLUSH_CTRL_FLUSH_PORT_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_PORT, x) + +#define HSCH_FLUSH_CTRL_FLUSH_QUEUE BIT(17) +#define HSCH_FLUSH_CTRL_FLUSH_QUEUE_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_QUEUE, x) +#define HSCH_FLUSH_CTRL_FLUSH_QUEUE_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_QUEUE, x) + +#define HSCH_FLUSH_CTRL_FLUSH_SE BIT(16) +#define HSCH_FLUSH_CTRL_FLUSH_SE_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_SE, x) +#define HSCH_FLUSH_CTRL_FLUSH_SE_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_SE, x) + +#define HSCH_FLUSH_CTRL_FLUSH_HIER GENMASK(15, 0) +#define HSCH_FLUSH_CTRL_FLUSH_HIER_SET(x)\ + FIELD_PREP(HSCH_FLUSH_CTRL_FLUSH_HIER, x) +#define HSCH_FLUSH_CTRL_FLUSH_HIER_GET(x)\ + FIELD_GET(HSCH_FLUSH_CTRL_FLUSH_HIER, x) + +/* HSCH:SYSTEM:PORT_MODE */ +#define HSCH_PORT_MODE(r) __REG(TARGET_HSCH, 0, 1, 184000, 0, 1, 312, 8, r, 70, 4) + +#define HSCH_PORT_MODE_DEQUEUE_DIS BIT(4) +#define HSCH_PORT_MODE_DEQUEUE_DIS_SET(x)\ + FIELD_PREP(HSCH_PORT_MODE_DEQUEUE_DIS, x) +#define HSCH_PORT_MODE_DEQUEUE_DIS_GET(x)\ + FIELD_GET(HSCH_PORT_MODE_DEQUEUE_DIS, x) + +#define HSCH_PORT_MODE_AGE_DIS BIT(3) +#define HSCH_PORT_MODE_AGE_DIS_SET(x)\ + FIELD_PREP(HSCH_PORT_MODE_AGE_DIS, x) +#define HSCH_PORT_MODE_AGE_DIS_GET(x)\ + FIELD_GET(HSCH_PORT_MODE_AGE_DIS, x) + +#define HSCH_PORT_MODE_TRUNC_ENA BIT(2) +#define HSCH_PORT_MODE_TRUNC_ENA_SET(x)\ + FIELD_PREP(HSCH_PORT_MODE_TRUNC_ENA, x) +#define HSCH_PORT_MODE_TRUNC_ENA_GET(x)\ + FIELD_GET(HSCH_PORT_MODE_TRUNC_ENA, x) + +#define HSCH_PORT_MODE_EIR_REMARK_ENA BIT(1) +#define HSCH_PORT_MODE_EIR_REMARK_ENA_SET(x)\ + FIELD_PREP(HSCH_PORT_MODE_EIR_REMARK_ENA, x) +#define HSCH_PORT_MODE_EIR_REMARK_ENA_GET(x)\ + FIELD_GET(HSCH_PORT_MODE_EIR_REMARK_ENA, x) + +#define HSCH_PORT_MODE_CPU_PRIO_MODE BIT(0) +#define HSCH_PORT_MODE_CPU_PRIO_MODE_SET(x)\ + FIELD_PREP(HSCH_PORT_MODE_CPU_PRIO_MODE, x) +#define HSCH_PORT_MODE_CPU_PRIO_MODE_GET(x)\ + FIELD_GET(HSCH_PORT_MODE_CPU_PRIO_MODE, x) + +/* HSCH:SYSTEM:OUTB_SHARE_ENA */ +#define HSCH_OUTB_SHARE_ENA(r) __REG(TARGET_HSCH, 0, 1, 184000, 0, 1, 312, 288, r, 5, 4) + +#define HSCH_OUTB_SHARE_ENA_OUTB_SHARE_ENA GENMASK(7, 0) +#define HSCH_OUTB_SHARE_ENA_OUTB_SHARE_ENA_SET(x)\ + FIELD_PREP(HSCH_OUTB_SHARE_ENA_OUTB_SHARE_ENA, x) +#define HSCH_OUTB_SHARE_ENA_OUTB_SHARE_ENA_GET(x)\ + FIELD_GET(HSCH_OUTB_SHARE_ENA_OUTB_SHARE_ENA, x) + +/* HSCH:MMGT:RESET_CFG */ +#define HSCH_RESET_CFG __REG(TARGET_HSCH, 0, 1, 162368, 0, 1, 16, 8, 0, 1, 4) + +#define HSCH_RESET_CFG_CORE_ENA BIT(0) +#define HSCH_RESET_CFG_CORE_ENA_SET(x)\ + FIELD_PREP(HSCH_RESET_CFG_CORE_ENA, x) +#define HSCH_RESET_CFG_CORE_ENA_GET(x)\ + FIELD_GET(HSCH_RESET_CFG_CORE_ENA, x) + +/* HSCH:TAS_CONFIG:TAS_STATEMACHINE_CFG */ +#define HSCH_TAS_STATEMACHINE_CFG __REG(TARGET_HSCH, 0, 1, 162384, 0, 1, 12, 8, 0, 1, 4) + +#define HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY GENMASK(7, 0) +#define HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY_SET(x)\ + FIELD_PREP(HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY, x) +#define HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY_GET(x)\ + FIELD_GET(HSCH_TAS_STATEMACHINE_CFG_REVISIT_DLY, x) + +/* LRN:COMMON:COMMON_ACCESS_CTRL */ +#define LRN_COMMON_ACCESS_CTRL __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 0, 0, 1, 4) + +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_COL GENMASK(21, 20) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_COL_SET(x)\ + FIELD_PREP(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_COL, x) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_COL_GET(x)\ + FIELD_GET(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_COL, x) + +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_TYPE BIT(19) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_TYPE_SET(x)\ + FIELD_PREP(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_TYPE, x) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_TYPE_GET(x)\ + FIELD_GET(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_TYPE, x) + +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_ROW GENMASK(18, 5) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_ROW_SET(x)\ + FIELD_PREP(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_ROW, x) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_ROW_GET(x)\ + FIELD_GET(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_DIRECT_ROW, x) + +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD GENMASK(4, 1) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_SET(x)\ + FIELD_PREP(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD, x) +#define LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD_GET(x)\ + FIELD_GET(LRN_COMMON_ACCESS_CTRL_CPU_ACCESS_CMD, x) + +#define LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT BIT(0) +#define LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_SET(x)\ + FIELD_PREP(LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT, x) +#define LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT_GET(x)\ + FIELD_GET(LRN_COMMON_ACCESS_CTRL_MAC_TABLE_ACCESS_SHOT, x) + +/* LRN:COMMON:MAC_ACCESS_CFG_0 */ +#define LRN_MAC_ACCESS_CFG_0 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 4, 0, 1, 4) + +#define LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_FID GENMASK(28, 16) +#define LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_FID_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_FID, x) +#define LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_FID_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_FID, x) + +#define LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_MAC_MSB GENMASK(15, 0) +#define LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_MAC_MSB_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_MAC_MSB, x) +#define LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_MAC_MSB_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_0_MAC_ENTRY_MAC_MSB, x) + +/* LRN:COMMON:MAC_ACCESS_CFG_1 */ +#define LRN_MAC_ACCESS_CFG_1 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 8, 0, 1, 4) + +/* LRN:COMMON:MAC_ACCESS_CFG_2 */ +#define LRN_MAC_ACCESS_CFG_2 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 12, 0, 1, 4) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_SRC_KILL_FWD BIT(28) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_SRC_KILL_FWD_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_SRC_KILL_FWD, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_SRC_KILL_FWD_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_SRC_KILL_FWD, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_NXT_LRN_ALL BIT(27) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_NXT_LRN_ALL_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_NXT_LRN_ALL, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_NXT_LRN_ALL_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_NXT_LRN_ALL, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_QU GENMASK(26, 24) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_QU_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_QU, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_QU_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_QU, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_COPY BIT(23) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_COPY_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_COPY, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_COPY_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_CPU_COPY, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLAN_IGNORE BIT(22) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLAN_IGNORE_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLAN_IGNORE, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLAN_IGNORE_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLAN_IGNORE, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_MIRROR BIT(21) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_MIRROR_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_MIRROR, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_MIRROR_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_MIRROR, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_FLAG GENMASK(20, 19) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_FLAG_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_FLAG, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_FLAG_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_FLAG, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_INTERVAL GENMASK(18, 17) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_INTERVAL_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_INTERVAL, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_INTERVAL_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_AGE_INTERVAL, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_LOCKED BIT(16) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_LOCKED_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_LOCKED, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_LOCKED_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_LOCKED, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD BIT(15) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_VLD, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE GENMASK(14, 12) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_TYPE, x) + +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR GENMASK(11, 0) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR, x) +#define LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR, x) + +/* LRN:COMMON:MAC_ACCESS_CFG_3 */ +#define LRN_MAC_ACCESS_CFG_3 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 16, 0, 1, 4) + +#define LRN_MAC_ACCESS_CFG_3_MAC_ENTRY_ISDX_LIMIT_IDX GENMASK(10, 0) +#define LRN_MAC_ACCESS_CFG_3_MAC_ENTRY_ISDX_LIMIT_IDX_SET(x)\ + FIELD_PREP(LRN_MAC_ACCESS_CFG_3_MAC_ENTRY_ISDX_LIMIT_IDX, x) +#define LRN_MAC_ACCESS_CFG_3_MAC_ENTRY_ISDX_LIMIT_IDX_GET(x)\ + FIELD_GET(LRN_MAC_ACCESS_CFG_3_MAC_ENTRY_ISDX_LIMIT_IDX, x) + +/* LRN:COMMON:SCAN_NEXT_CFG */ +#define LRN_SCAN_NEXT_CFG __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 20, 0, 1, 4) + +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_FLAG_UPDATE_SEL GENMASK(21, 19) +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_FLAG_UPDATE_SEL_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_AGE_FLAG_UPDATE_SEL, x) +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_FLAG_UPDATE_SEL_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_AGE_FLAG_UPDATE_SEL, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NXT_LRN_ALL_UPDATE_SEL GENMASK(18, 17) +#define LRN_SCAN_NEXT_CFG_SCAN_NXT_LRN_ALL_UPDATE_SEL_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NXT_LRN_ALL_UPDATE_SEL, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NXT_LRN_ALL_UPDATE_SEL_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NXT_LRN_ALL_UPDATE_SEL, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_FILTER_SEL GENMASK(16, 15) +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_FILTER_SEL_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_AGE_FILTER_SEL, x) +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_FILTER_SEL_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_AGE_FILTER_SEL, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_MOVE_FOUND_ENA BIT(14) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_MOVE_FOUND_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NEXT_MOVE_FOUND_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_MOVE_FOUND_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NEXT_MOVE_FOUND_ENA, x) + +#define LRN_SCAN_NEXT_CFG_NXT_LRN_ALL_FILTER_ENA BIT(13) +#define LRN_SCAN_NEXT_CFG_NXT_LRN_ALL_FILTER_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_NXT_LRN_ALL_FILTER_ENA, x) +#define LRN_SCAN_NEXT_CFG_NXT_LRN_ALL_FILTER_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_NXT_LRN_ALL_FILTER_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_USE_PORT_FILTER_ENA BIT(12) +#define LRN_SCAN_NEXT_CFG_SCAN_USE_PORT_FILTER_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_USE_PORT_FILTER_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_USE_PORT_FILTER_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_USE_PORT_FILTER_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_REMOVE_FOUND_ENA BIT(11) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_REMOVE_FOUND_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NEXT_REMOVE_FOUND_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_REMOVE_FOUND_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NEXT_REMOVE_FOUND_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA BIT(10) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NEXT_UNTIL_FOUND_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_INC_AGE_BITS_ENA BIT(9) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_INC_AGE_BITS_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NEXT_INC_AGE_BITS_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_INC_AGE_BITS_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NEXT_INC_AGE_BITS_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_AGED_ONLY_ENA BIT(8) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_AGED_ONLY_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NEXT_AGED_ONLY_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_AGED_ONLY_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NEXT_AGED_ONLY_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_IGNORE_LOCKED_ENA BIT(7) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_IGNORE_LOCKED_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_NEXT_IGNORE_LOCKED_ENA, x) +#define LRN_SCAN_NEXT_CFG_SCAN_NEXT_IGNORE_LOCKED_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_NEXT_IGNORE_LOCKED_ENA, x) + +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_INTERVAL_MASK GENMASK(6, 3) +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_INTERVAL_MASK_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_SCAN_AGE_INTERVAL_MASK, x) +#define LRN_SCAN_NEXT_CFG_SCAN_AGE_INTERVAL_MASK_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_SCAN_AGE_INTERVAL_MASK, x) + +#define LRN_SCAN_NEXT_CFG_ISDX_LIMIT_IDX_FILTER_ENA BIT(2) +#define LRN_SCAN_NEXT_CFG_ISDX_LIMIT_IDX_FILTER_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_ISDX_LIMIT_IDX_FILTER_ENA, x) +#define LRN_SCAN_NEXT_CFG_ISDX_LIMIT_IDX_FILTER_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_ISDX_LIMIT_IDX_FILTER_ENA, x) + +#define LRN_SCAN_NEXT_CFG_FID_FILTER_ENA BIT(1) +#define LRN_SCAN_NEXT_CFG_FID_FILTER_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_FID_FILTER_ENA, x) +#define LRN_SCAN_NEXT_CFG_FID_FILTER_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_FID_FILTER_ENA, x) + +#define LRN_SCAN_NEXT_CFG_ADDR_FILTER_ENA BIT(0) +#define LRN_SCAN_NEXT_CFG_ADDR_FILTER_ENA_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_ADDR_FILTER_ENA, x) +#define LRN_SCAN_NEXT_CFG_ADDR_FILTER_ENA_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_ADDR_FILTER_ENA, x) + +/* LRN:COMMON:SCAN_NEXT_CFG_1 */ +#define LRN_SCAN_NEXT_CFG_1 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 24, 0, 1, 4) + +#define LRN_SCAN_NEXT_CFG_1_PORT_MOVE_NEW_ADDR GENMASK(30, 16) +#define LRN_SCAN_NEXT_CFG_1_PORT_MOVE_NEW_ADDR_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_1_PORT_MOVE_NEW_ADDR, x) +#define LRN_SCAN_NEXT_CFG_1_PORT_MOVE_NEW_ADDR_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_1_PORT_MOVE_NEW_ADDR, x) + +#define LRN_SCAN_NEXT_CFG_1_SCAN_ENTRY_ADDR_MASK GENMASK(14, 0) +#define LRN_SCAN_NEXT_CFG_1_SCAN_ENTRY_ADDR_MASK_SET(x)\ + FIELD_PREP(LRN_SCAN_NEXT_CFG_1_SCAN_ENTRY_ADDR_MASK, x) +#define LRN_SCAN_NEXT_CFG_1_SCAN_ENTRY_ADDR_MASK_GET(x)\ + FIELD_GET(LRN_SCAN_NEXT_CFG_1_SCAN_ENTRY_ADDR_MASK, x) + +/* LRN:COMMON:AUTOAGE_CFG */ +#define LRN_AUTOAGE_CFG(r) __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 36, r, 4, 4) + +#define LRN_AUTOAGE_CFG_UNIT_SIZE GENMASK(29, 28) +#define LRN_AUTOAGE_CFG_UNIT_SIZE_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_UNIT_SIZE, x) +#define LRN_AUTOAGE_CFG_UNIT_SIZE_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_UNIT_SIZE, x) + +#define LRN_AUTOAGE_CFG_PERIOD_VAL GENMASK(27, 0) +#define LRN_AUTOAGE_CFG_PERIOD_VAL_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_PERIOD_VAL, x) +#define LRN_AUTOAGE_CFG_PERIOD_VAL_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_PERIOD_VAL, x) + +/* LRN:COMMON:AUTOAGE_CFG_1 */ +#define LRN_AUTOAGE_CFG_1 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 52, 0, 1, 4) + +#define LRN_AUTOAGE_CFG_1_PAUSE_AUTO_AGE_ENA BIT(25) +#define LRN_AUTOAGE_CFG_1_PAUSE_AUTO_AGE_ENA_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_PAUSE_AUTO_AGE_ENA, x) +#define LRN_AUTOAGE_CFG_1_PAUSE_AUTO_AGE_ENA_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_PAUSE_AUTO_AGE_ENA, x) + +#define LRN_AUTOAGE_CFG_1_CELLS_BETWEEN_ENTRY_SCAN GENMASK(24, 15) +#define LRN_AUTOAGE_CFG_1_CELLS_BETWEEN_ENTRY_SCAN_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_CELLS_BETWEEN_ENTRY_SCAN, x) +#define LRN_AUTOAGE_CFG_1_CELLS_BETWEEN_ENTRY_SCAN_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_CELLS_BETWEEN_ENTRY_SCAN, x) + +#define LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS GENMASK(14, 7) +#define LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS, x) +#define LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_CLK_PERIOD_01NS, x) + +#define LRN_AUTOAGE_CFG_1_USE_PORT_FILTER_ENA BIT(6) +#define LRN_AUTOAGE_CFG_1_USE_PORT_FILTER_ENA_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_USE_PORT_FILTER_ENA, x) +#define LRN_AUTOAGE_CFG_1_USE_PORT_FILTER_ENA_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_USE_PORT_FILTER_ENA, x) + +#define LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_SHOT GENMASK(5, 2) +#define LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_SHOT_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_SHOT, x) +#define LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_SHOT_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_SHOT, x) + +#define LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_STOP_SHOT BIT(1) +#define LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_STOP_SHOT_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_STOP_SHOT, x) +#define LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_STOP_SHOT_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_FORCE_HW_SCAN_STOP_SHOT, x) + +#define LRN_AUTOAGE_CFG_1_FORCE_IDLE_ENA BIT(0) +#define LRN_AUTOAGE_CFG_1_FORCE_IDLE_ENA_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_1_FORCE_IDLE_ENA, x) +#define LRN_AUTOAGE_CFG_1_FORCE_IDLE_ENA_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_1_FORCE_IDLE_ENA, x) + +/* LRN:COMMON:AUTOAGE_CFG_2 */ +#define LRN_AUTOAGE_CFG_2 __REG(TARGET_LRN, 0, 1, 0, 0, 1, 72, 56, 0, 1, 4) + +#define LRN_AUTOAGE_CFG_2_NEXT_ROW GENMASK(17, 4) +#define LRN_AUTOAGE_CFG_2_NEXT_ROW_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_2_NEXT_ROW, x) +#define LRN_AUTOAGE_CFG_2_NEXT_ROW_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_2_NEXT_ROW, x) + +#define LRN_AUTOAGE_CFG_2_SCAN_ONGOING_STATUS GENMASK(3, 0) +#define LRN_AUTOAGE_CFG_2_SCAN_ONGOING_STATUS_SET(x)\ + FIELD_PREP(LRN_AUTOAGE_CFG_2_SCAN_ONGOING_STATUS, x) +#define LRN_AUTOAGE_CFG_2_SCAN_ONGOING_STATUS_GET(x)\ + FIELD_GET(LRN_AUTOAGE_CFG_2_SCAN_ONGOING_STATUS, x) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_REGION_CTRL_2_OFF_OUTBOUND_0 */ +#define PCEP_RCTRL_2_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 4, 0, 1, 4) + +#define PCEP_RCTRL_2_OUT_0_MSG_CODE GENMASK(7, 0) +#define PCEP_RCTRL_2_OUT_0_MSG_CODE_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_MSG_CODE, x) +#define PCEP_RCTRL_2_OUT_0_MSG_CODE_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_MSG_CODE, x) + +#define PCEP_RCTRL_2_OUT_0_TAG GENMASK(15, 8) +#define PCEP_RCTRL_2_OUT_0_TAG_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_TAG, x) +#define PCEP_RCTRL_2_OUT_0_TAG_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_TAG, x) + +#define PCEP_RCTRL_2_OUT_0_TAG_SUBSTITUTE_EN BIT(16) +#define PCEP_RCTRL_2_OUT_0_TAG_SUBSTITUTE_EN_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_TAG_SUBSTITUTE_EN, x) +#define PCEP_RCTRL_2_OUT_0_TAG_SUBSTITUTE_EN_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_TAG_SUBSTITUTE_EN, x) + +#define PCEP_RCTRL_2_OUT_0_FUNC_BYPASS BIT(19) +#define PCEP_RCTRL_2_OUT_0_FUNC_BYPASS_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_FUNC_BYPASS, x) +#define PCEP_RCTRL_2_OUT_0_FUNC_BYPASS_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_FUNC_BYPASS, x) + +#define PCEP_RCTRL_2_OUT_0_SNP BIT(20) +#define PCEP_RCTRL_2_OUT_0_SNP_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_SNP, x) +#define PCEP_RCTRL_2_OUT_0_SNP_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_SNP, x) + +#define PCEP_RCTRL_2_OUT_0_INHIBIT_PAYLOAD BIT(22) +#define PCEP_RCTRL_2_OUT_0_INHIBIT_PAYLOAD_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_INHIBIT_PAYLOAD, x) +#define PCEP_RCTRL_2_OUT_0_INHIBIT_PAYLOAD_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_INHIBIT_PAYLOAD, x) + +#define PCEP_RCTRL_2_OUT_0_HEADER_SUBSTITUTE_EN BIT(23) +#define PCEP_RCTRL_2_OUT_0_HEADER_SUBSTITUTE_EN_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_HEADER_SUBSTITUTE_EN, x) +#define PCEP_RCTRL_2_OUT_0_HEADER_SUBSTITUTE_EN_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_HEADER_SUBSTITUTE_EN, x) + +#define PCEP_RCTRL_2_OUT_0_CFG_SHIFT_MODE BIT(28) +#define PCEP_RCTRL_2_OUT_0_CFG_SHIFT_MODE_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_CFG_SHIFT_MODE, x) +#define PCEP_RCTRL_2_OUT_0_CFG_SHIFT_MODE_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_CFG_SHIFT_MODE, x) + +#define PCEP_RCTRL_2_OUT_0_INVERT_MODE BIT(29) +#define PCEP_RCTRL_2_OUT_0_INVERT_MODE_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_INVERT_MODE, x) +#define PCEP_RCTRL_2_OUT_0_INVERT_MODE_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_INVERT_MODE, x) + +#define PCEP_RCTRL_2_OUT_0_REGION_EN BIT(31) +#define PCEP_RCTRL_2_OUT_0_REGION_EN_SET(x)\ + FIELD_PREP(PCEP_RCTRL_2_OUT_0_REGION_EN, x) +#define PCEP_RCTRL_2_OUT_0_REGION_EN_GET(x)\ + FIELD_GET(PCEP_RCTRL_2_OUT_0_REGION_EN, x) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_LWR_BASE_ADDR_OFF_OUTBOUND_0 */ +#define PCEP_ADDR_LWR_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 8, 0, 1, 4) + +#define PCEP_ADDR_LWR_OUT_0_LWR_BASE_HW GENMASK(15, 0) +#define PCEP_ADDR_LWR_OUT_0_LWR_BASE_HW_SET(x)\ + FIELD_PREP(PCEP_ADDR_LWR_OUT_0_LWR_BASE_HW, x) +#define PCEP_ADDR_LWR_OUT_0_LWR_BASE_HW_GET(x)\ + FIELD_GET(PCEP_ADDR_LWR_OUT_0_LWR_BASE_HW, x) + +#define PCEP_ADDR_LWR_OUT_0_LWR_BASE_RW GENMASK(31, 16) +#define PCEP_ADDR_LWR_OUT_0_LWR_BASE_RW_SET(x)\ + FIELD_PREP(PCEP_ADDR_LWR_OUT_0_LWR_BASE_RW, x) +#define PCEP_ADDR_LWR_OUT_0_LWR_BASE_RW_GET(x)\ + FIELD_GET(PCEP_ADDR_LWR_OUT_0_LWR_BASE_RW, x) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_UPPER_BASE_ADDR_OFF_OUTBOUND_0 */ +#define PCEP_ADDR_UPR_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 12, 0, 1, 4) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_LIMIT_ADDR_OFF_OUTBOUND_0 */ +#define PCEP_ADDR_LIM_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 16, 0, 1, 4) + +#define PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_HW GENMASK(15, 0) +#define PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_HW_SET(x)\ + FIELD_PREP(PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_HW, x) +#define PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_HW_GET(x)\ + FIELD_GET(PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_HW, x) + +#define PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_RW GENMASK(31, 16) +#define PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_RW_SET(x)\ + FIELD_PREP(PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_RW, x) +#define PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_RW_GET(x)\ + FIELD_GET(PCEP_ADDR_LIM_OUT_0_LIMIT_ADDR_RW, x) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_LWR_TARGET_ADDR_OFF_OUTBOUND_0 */ +#define PCEP_ADDR_LWR_TGT_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 20, 0, 1, 4) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_UPPER_TARGET_ADDR_OFF_OUTBOUND_0 */ +#define PCEP_ADDR_UPR_TGT_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 24, 0, 1, 4) + +/* PCIE_DM_EP:PF0_ATU_CAP:IATU_UPPR_LIMIT_ADDR_OFF_OUTBOUND_0 */ +#define PCEP_ADDR_UPR_LIM_OUT_0 __REG(TARGET_PCEP, 0, 1, 3145728, 0, 1, 130852, 32, 0, 1, 4) + +#define PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_RW GENMASK(1, 0) +#define PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_RW_SET(x)\ + FIELD_PREP(PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_RW, x) +#define PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_RW_GET(x)\ + FIELD_GET(PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_RW, x) + +#define PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_HW GENMASK(31, 2) +#define PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_HW_SET(x)\ + FIELD_PREP(PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_HW, x) +#define PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_HW_GET(x)\ + FIELD_GET(PCEP_ADDR_UPR_LIM_OUT_0_UPPR_LIMIT_ADDR_HW, x) + +/* PCS_10GBASE_R:PCS_10GBR_CFG:PCS_CFG */ +#define PCS10G_BR_PCS_CFG(t) __REG(TARGET_PCS10G_BR, t, 12, 0, 0, 1, 56, 0, 0, 1, 4) + +#define PCS10G_BR_PCS_CFG_PCS_ENA BIT(31) +#define PCS10G_BR_PCS_CFG_PCS_ENA_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_PCS_ENA, x) +#define PCS10G_BR_PCS_CFG_PCS_ENA_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_PCS_ENA, x) + +#define PCS10G_BR_PCS_CFG_PMA_LOOPBACK_ENA BIT(30) +#define PCS10G_BR_PCS_CFG_PMA_LOOPBACK_ENA_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_PMA_LOOPBACK_ENA, x) +#define PCS10G_BR_PCS_CFG_PMA_LOOPBACK_ENA_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_PMA_LOOPBACK_ENA, x) + +#define PCS10G_BR_PCS_CFG_SH_CNT_MAX GENMASK(29, 24) +#define PCS10G_BR_PCS_CFG_SH_CNT_MAX_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_SH_CNT_MAX, x) +#define PCS10G_BR_PCS_CFG_SH_CNT_MAX_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_SH_CNT_MAX, x) + +#define PCS10G_BR_PCS_CFG_RX_DATA_FLIP BIT(18) +#define PCS10G_BR_PCS_CFG_RX_DATA_FLIP_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_RX_DATA_FLIP, x) +#define PCS10G_BR_PCS_CFG_RX_DATA_FLIP_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_RX_DATA_FLIP, x) + +#define PCS10G_BR_PCS_CFG_RESYNC_ENA BIT(15) +#define PCS10G_BR_PCS_CFG_RESYNC_ENA_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_RESYNC_ENA, x) +#define PCS10G_BR_PCS_CFG_RESYNC_ENA_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_RESYNC_ENA, x) + +#define PCS10G_BR_PCS_CFG_LF_GEN_DIS BIT(14) +#define PCS10G_BR_PCS_CFG_LF_GEN_DIS_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_LF_GEN_DIS, x) +#define PCS10G_BR_PCS_CFG_LF_GEN_DIS_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_LF_GEN_DIS, x) + +#define PCS10G_BR_PCS_CFG_RX_TEST_MODE BIT(13) +#define PCS10G_BR_PCS_CFG_RX_TEST_MODE_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_RX_TEST_MODE, x) +#define PCS10G_BR_PCS_CFG_RX_TEST_MODE_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_RX_TEST_MODE, x) + +#define PCS10G_BR_PCS_CFG_RX_SCR_DISABLE BIT(12) +#define PCS10G_BR_PCS_CFG_RX_SCR_DISABLE_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_RX_SCR_DISABLE, x) +#define PCS10G_BR_PCS_CFG_RX_SCR_DISABLE_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_RX_SCR_DISABLE, x) + +#define PCS10G_BR_PCS_CFG_TX_DATA_FLIP BIT(7) +#define PCS10G_BR_PCS_CFG_TX_DATA_FLIP_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_TX_DATA_FLIP, x) +#define PCS10G_BR_PCS_CFG_TX_DATA_FLIP_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_TX_DATA_FLIP, x) + +#define PCS10G_BR_PCS_CFG_AN_LINK_CTRL_ENA BIT(6) +#define PCS10G_BR_PCS_CFG_AN_LINK_CTRL_ENA_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_AN_LINK_CTRL_ENA, x) +#define PCS10G_BR_PCS_CFG_AN_LINK_CTRL_ENA_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_AN_LINK_CTRL_ENA, x) + +#define PCS10G_BR_PCS_CFG_TX_TEST_MODE BIT(4) +#define PCS10G_BR_PCS_CFG_TX_TEST_MODE_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_TX_TEST_MODE, x) +#define PCS10G_BR_PCS_CFG_TX_TEST_MODE_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_TX_TEST_MODE, x) + +#define PCS10G_BR_PCS_CFG_TX_SCR_DISABLE BIT(3) +#define PCS10G_BR_PCS_CFG_TX_SCR_DISABLE_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_CFG_TX_SCR_DISABLE, x) +#define PCS10G_BR_PCS_CFG_TX_SCR_DISABLE_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_CFG_TX_SCR_DISABLE, x) + +/* PCS_10GBASE_R:PCS_10GBR_CFG:PCS_SD_CFG */ +#define PCS10G_BR_PCS_SD_CFG(t) __REG(TARGET_PCS10G_BR, t, 12, 0, 0, 1, 56, 4, 0, 1, 4) + +#define PCS10G_BR_PCS_SD_CFG_SD_SEL BIT(8) +#define PCS10G_BR_PCS_SD_CFG_SD_SEL_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_SD_CFG_SD_SEL, x) +#define PCS10G_BR_PCS_SD_CFG_SD_SEL_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_SD_CFG_SD_SEL, x) + +#define PCS10G_BR_PCS_SD_CFG_SD_POL BIT(4) +#define PCS10G_BR_PCS_SD_CFG_SD_POL_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_SD_CFG_SD_POL, x) +#define PCS10G_BR_PCS_SD_CFG_SD_POL_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_SD_CFG_SD_POL, x) + +#define PCS10G_BR_PCS_SD_CFG_SD_ENA BIT(0) +#define PCS10G_BR_PCS_SD_CFG_SD_ENA_SET(x)\ + FIELD_PREP(PCS10G_BR_PCS_SD_CFG_SD_ENA, x) +#define PCS10G_BR_PCS_SD_CFG_SD_ENA_GET(x)\ + FIELD_GET(PCS10G_BR_PCS_SD_CFG_SD_ENA, x) + +/* PCS_10GBASE_R:PCS_10GBR_CFG:PCS_CFG */ +#define PCS25G_BR_PCS_CFG(t) __REG(TARGET_PCS25G_BR, t, 8, 0, 0, 1, 56, 0, 0, 1, 4) + +#define PCS25G_BR_PCS_CFG_PCS_ENA BIT(31) +#define PCS25G_BR_PCS_CFG_PCS_ENA_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_PCS_ENA, x) +#define PCS25G_BR_PCS_CFG_PCS_ENA_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_PCS_ENA, x) + +#define PCS25G_BR_PCS_CFG_PMA_LOOPBACK_ENA BIT(30) +#define PCS25G_BR_PCS_CFG_PMA_LOOPBACK_ENA_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_PMA_LOOPBACK_ENA, x) +#define PCS25G_BR_PCS_CFG_PMA_LOOPBACK_ENA_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_PMA_LOOPBACK_ENA, x) + +#define PCS25G_BR_PCS_CFG_SH_CNT_MAX GENMASK(29, 24) +#define PCS25G_BR_PCS_CFG_SH_CNT_MAX_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_SH_CNT_MAX, x) +#define PCS25G_BR_PCS_CFG_SH_CNT_MAX_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_SH_CNT_MAX, x) + +#define PCS25G_BR_PCS_CFG_RX_DATA_FLIP BIT(18) +#define PCS25G_BR_PCS_CFG_RX_DATA_FLIP_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_RX_DATA_FLIP, x) +#define PCS25G_BR_PCS_CFG_RX_DATA_FLIP_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_RX_DATA_FLIP, x) + +#define PCS25G_BR_PCS_CFG_RESYNC_ENA BIT(15) +#define PCS25G_BR_PCS_CFG_RESYNC_ENA_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_RESYNC_ENA, x) +#define PCS25G_BR_PCS_CFG_RESYNC_ENA_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_RESYNC_ENA, x) + +#define PCS25G_BR_PCS_CFG_LF_GEN_DIS BIT(14) +#define PCS25G_BR_PCS_CFG_LF_GEN_DIS_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_LF_GEN_DIS, x) +#define PCS25G_BR_PCS_CFG_LF_GEN_DIS_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_LF_GEN_DIS, x) + +#define PCS25G_BR_PCS_CFG_RX_TEST_MODE BIT(13) +#define PCS25G_BR_PCS_CFG_RX_TEST_MODE_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_RX_TEST_MODE, x) +#define PCS25G_BR_PCS_CFG_RX_TEST_MODE_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_RX_TEST_MODE, x) + +#define PCS25G_BR_PCS_CFG_RX_SCR_DISABLE BIT(12) +#define PCS25G_BR_PCS_CFG_RX_SCR_DISABLE_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_RX_SCR_DISABLE, x) +#define PCS25G_BR_PCS_CFG_RX_SCR_DISABLE_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_RX_SCR_DISABLE, x) + +#define PCS25G_BR_PCS_CFG_TX_DATA_FLIP BIT(7) +#define PCS25G_BR_PCS_CFG_TX_DATA_FLIP_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_TX_DATA_FLIP, x) +#define PCS25G_BR_PCS_CFG_TX_DATA_FLIP_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_TX_DATA_FLIP, x) + +#define PCS25G_BR_PCS_CFG_AN_LINK_CTRL_ENA BIT(6) +#define PCS25G_BR_PCS_CFG_AN_LINK_CTRL_ENA_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_AN_LINK_CTRL_ENA, x) +#define PCS25G_BR_PCS_CFG_AN_LINK_CTRL_ENA_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_AN_LINK_CTRL_ENA, x) + +#define PCS25G_BR_PCS_CFG_TX_TEST_MODE BIT(4) +#define PCS25G_BR_PCS_CFG_TX_TEST_MODE_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_TX_TEST_MODE, x) +#define PCS25G_BR_PCS_CFG_TX_TEST_MODE_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_TX_TEST_MODE, x) + +#define PCS25G_BR_PCS_CFG_TX_SCR_DISABLE BIT(3) +#define PCS25G_BR_PCS_CFG_TX_SCR_DISABLE_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_CFG_TX_SCR_DISABLE, x) +#define PCS25G_BR_PCS_CFG_TX_SCR_DISABLE_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_CFG_TX_SCR_DISABLE, x) + +/* PCS_10GBASE_R:PCS_10GBR_CFG:PCS_SD_CFG */ +#define PCS25G_BR_PCS_SD_CFG(t) __REG(TARGET_PCS25G_BR, t, 8, 0, 0, 1, 56, 4, 0, 1, 4) + +#define PCS25G_BR_PCS_SD_CFG_SD_SEL BIT(8) +#define PCS25G_BR_PCS_SD_CFG_SD_SEL_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_SD_CFG_SD_SEL, x) +#define PCS25G_BR_PCS_SD_CFG_SD_SEL_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_SD_CFG_SD_SEL, x) + +#define PCS25G_BR_PCS_SD_CFG_SD_POL BIT(4) +#define PCS25G_BR_PCS_SD_CFG_SD_POL_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_SD_CFG_SD_POL, x) +#define PCS25G_BR_PCS_SD_CFG_SD_POL_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_SD_CFG_SD_POL, x) + +#define PCS25G_BR_PCS_SD_CFG_SD_ENA BIT(0) +#define PCS25G_BR_PCS_SD_CFG_SD_ENA_SET(x)\ + FIELD_PREP(PCS25G_BR_PCS_SD_CFG_SD_ENA, x) +#define PCS25G_BR_PCS_SD_CFG_SD_ENA_GET(x)\ + FIELD_GET(PCS25G_BR_PCS_SD_CFG_SD_ENA, x) + +/* PCS_10GBASE_R:PCS_10GBR_CFG:PCS_CFG */ +#define PCS5G_BR_PCS_CFG(t) __REG(TARGET_PCS5G_BR, t, 13, 0, 0, 1, 56, 0, 0, 1, 4) + +#define PCS5G_BR_PCS_CFG_PCS_ENA BIT(31) +#define PCS5G_BR_PCS_CFG_PCS_ENA_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_PCS_ENA, x) +#define PCS5G_BR_PCS_CFG_PCS_ENA_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_PCS_ENA, x) + +#define PCS5G_BR_PCS_CFG_PMA_LOOPBACK_ENA BIT(30) +#define PCS5G_BR_PCS_CFG_PMA_LOOPBACK_ENA_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_PMA_LOOPBACK_ENA, x) +#define PCS5G_BR_PCS_CFG_PMA_LOOPBACK_ENA_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_PMA_LOOPBACK_ENA, x) + +#define PCS5G_BR_PCS_CFG_SH_CNT_MAX GENMASK(29, 24) +#define PCS5G_BR_PCS_CFG_SH_CNT_MAX_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_SH_CNT_MAX, x) +#define PCS5G_BR_PCS_CFG_SH_CNT_MAX_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_SH_CNT_MAX, x) + +#define PCS5G_BR_PCS_CFG_RX_DATA_FLIP BIT(18) +#define PCS5G_BR_PCS_CFG_RX_DATA_FLIP_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_RX_DATA_FLIP, x) +#define PCS5G_BR_PCS_CFG_RX_DATA_FLIP_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_RX_DATA_FLIP, x) + +#define PCS5G_BR_PCS_CFG_RESYNC_ENA BIT(15) +#define PCS5G_BR_PCS_CFG_RESYNC_ENA_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_RESYNC_ENA, x) +#define PCS5G_BR_PCS_CFG_RESYNC_ENA_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_RESYNC_ENA, x) + +#define PCS5G_BR_PCS_CFG_LF_GEN_DIS BIT(14) +#define PCS5G_BR_PCS_CFG_LF_GEN_DIS_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_LF_GEN_DIS, x) +#define PCS5G_BR_PCS_CFG_LF_GEN_DIS_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_LF_GEN_DIS, x) + +#define PCS5G_BR_PCS_CFG_RX_TEST_MODE BIT(13) +#define PCS5G_BR_PCS_CFG_RX_TEST_MODE_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_RX_TEST_MODE, x) +#define PCS5G_BR_PCS_CFG_RX_TEST_MODE_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_RX_TEST_MODE, x) + +#define PCS5G_BR_PCS_CFG_RX_SCR_DISABLE BIT(12) +#define PCS5G_BR_PCS_CFG_RX_SCR_DISABLE_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_RX_SCR_DISABLE, x) +#define PCS5G_BR_PCS_CFG_RX_SCR_DISABLE_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_RX_SCR_DISABLE, x) + +#define PCS5G_BR_PCS_CFG_TX_DATA_FLIP BIT(7) +#define PCS5G_BR_PCS_CFG_TX_DATA_FLIP_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_TX_DATA_FLIP, x) +#define PCS5G_BR_PCS_CFG_TX_DATA_FLIP_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_TX_DATA_FLIP, x) + +#define PCS5G_BR_PCS_CFG_AN_LINK_CTRL_ENA BIT(6) +#define PCS5G_BR_PCS_CFG_AN_LINK_CTRL_ENA_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_AN_LINK_CTRL_ENA, x) +#define PCS5G_BR_PCS_CFG_AN_LINK_CTRL_ENA_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_AN_LINK_CTRL_ENA, x) + +#define PCS5G_BR_PCS_CFG_TX_TEST_MODE BIT(4) +#define PCS5G_BR_PCS_CFG_TX_TEST_MODE_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_TX_TEST_MODE, x) +#define PCS5G_BR_PCS_CFG_TX_TEST_MODE_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_TX_TEST_MODE, x) + +#define PCS5G_BR_PCS_CFG_TX_SCR_DISABLE BIT(3) +#define PCS5G_BR_PCS_CFG_TX_SCR_DISABLE_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_CFG_TX_SCR_DISABLE, x) +#define PCS5G_BR_PCS_CFG_TX_SCR_DISABLE_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_CFG_TX_SCR_DISABLE, x) + +/* PCS_10GBASE_R:PCS_10GBR_CFG:PCS_SD_CFG */ +#define PCS5G_BR_PCS_SD_CFG(t) __REG(TARGET_PCS5G_BR, t, 13, 0, 0, 1, 56, 4, 0, 1, 4) + +#define PCS5G_BR_PCS_SD_CFG_SD_SEL BIT(8) +#define PCS5G_BR_PCS_SD_CFG_SD_SEL_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_SD_CFG_SD_SEL, x) +#define PCS5G_BR_PCS_SD_CFG_SD_SEL_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_SD_CFG_SD_SEL, x) + +#define PCS5G_BR_PCS_SD_CFG_SD_POL BIT(4) +#define PCS5G_BR_PCS_SD_CFG_SD_POL_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_SD_CFG_SD_POL, x) +#define PCS5G_BR_PCS_SD_CFG_SD_POL_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_SD_CFG_SD_POL, x) + +#define PCS5G_BR_PCS_SD_CFG_SD_ENA BIT(0) +#define PCS5G_BR_PCS_SD_CFG_SD_ENA_SET(x)\ + FIELD_PREP(PCS5G_BR_PCS_SD_CFG_SD_ENA, x) +#define PCS5G_BR_PCS_SD_CFG_SD_ENA_GET(x)\ + FIELD_GET(PCS5G_BR_PCS_SD_CFG_SD_ENA, x) + +/* PORT_CONF:HW_CFG:DEV5G_MODES */ +#define PORT_CONF_DEV5G_MODES __REG(TARGET_PORT_CONF, 0, 1, 0, 0, 1, 24, 0, 0, 1, 4) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D0_MODE BIT(0) +#define PORT_CONF_DEV5G_MODES_DEV5G_D0_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D0_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D0_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D0_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D1_MODE BIT(1) +#define PORT_CONF_DEV5G_MODES_DEV5G_D1_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D1_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D1_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D1_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D2_MODE BIT(2) +#define PORT_CONF_DEV5G_MODES_DEV5G_D2_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D2_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D2_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D2_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D3_MODE BIT(3) +#define PORT_CONF_DEV5G_MODES_DEV5G_D3_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D3_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D3_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D3_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D4_MODE BIT(4) +#define PORT_CONF_DEV5G_MODES_DEV5G_D4_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D4_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D4_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D4_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D5_MODE BIT(5) +#define PORT_CONF_DEV5G_MODES_DEV5G_D5_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D5_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D5_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D5_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D6_MODE BIT(6) +#define PORT_CONF_DEV5G_MODES_DEV5G_D6_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D6_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D6_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D6_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D7_MODE BIT(7) +#define PORT_CONF_DEV5G_MODES_DEV5G_D7_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D7_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D7_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D7_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D8_MODE BIT(8) +#define PORT_CONF_DEV5G_MODES_DEV5G_D8_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D8_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D8_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D8_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D9_MODE BIT(9) +#define PORT_CONF_DEV5G_MODES_DEV5G_D9_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D9_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D9_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D9_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D10_MODE BIT(10) +#define PORT_CONF_DEV5G_MODES_DEV5G_D10_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D10_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D10_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D10_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D11_MODE BIT(11) +#define PORT_CONF_DEV5G_MODES_DEV5G_D11_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D11_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D11_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D11_MODE, x) + +#define PORT_CONF_DEV5G_MODES_DEV5G_D64_MODE BIT(12) +#define PORT_CONF_DEV5G_MODES_DEV5G_D64_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV5G_MODES_DEV5G_D64_MODE, x) +#define PORT_CONF_DEV5G_MODES_DEV5G_D64_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV5G_MODES_DEV5G_D64_MODE, x) + +/* PORT_CONF:HW_CFG:DEV10G_MODES */ +#define PORT_CONF_DEV10G_MODES __REG(TARGET_PORT_CONF, 0, 1, 0, 0, 1, 24, 4, 0, 1, 4) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D12_MODE BIT(0) +#define PORT_CONF_DEV10G_MODES_DEV10G_D12_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D12_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D12_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D12_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D13_MODE BIT(1) +#define PORT_CONF_DEV10G_MODES_DEV10G_D13_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D13_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D13_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D13_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D14_MODE BIT(2) +#define PORT_CONF_DEV10G_MODES_DEV10G_D14_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D14_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D14_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D14_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D15_MODE BIT(3) +#define PORT_CONF_DEV10G_MODES_DEV10G_D15_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D15_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D15_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D15_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D48_MODE BIT(4) +#define PORT_CONF_DEV10G_MODES_DEV10G_D48_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D48_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D48_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D48_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D49_MODE BIT(5) +#define PORT_CONF_DEV10G_MODES_DEV10G_D49_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D49_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D49_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D49_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D50_MODE BIT(6) +#define PORT_CONF_DEV10G_MODES_DEV10G_D50_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D50_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D50_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D50_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D51_MODE BIT(7) +#define PORT_CONF_DEV10G_MODES_DEV10G_D51_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D51_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D51_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D51_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D52_MODE BIT(8) +#define PORT_CONF_DEV10G_MODES_DEV10G_D52_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D52_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D52_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D52_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D53_MODE BIT(9) +#define PORT_CONF_DEV10G_MODES_DEV10G_D53_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D53_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D53_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D53_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D54_MODE BIT(10) +#define PORT_CONF_DEV10G_MODES_DEV10G_D54_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D54_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D54_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D54_MODE, x) + +#define PORT_CONF_DEV10G_MODES_DEV10G_D55_MODE BIT(11) +#define PORT_CONF_DEV10G_MODES_DEV10G_D55_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV10G_MODES_DEV10G_D55_MODE, x) +#define PORT_CONF_DEV10G_MODES_DEV10G_D55_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV10G_MODES_DEV10G_D55_MODE, x) + +/* PORT_CONF:HW_CFG:DEV25G_MODES */ +#define PORT_CONF_DEV25G_MODES __REG(TARGET_PORT_CONF, 0, 1, 0, 0, 1, 24, 8, 0, 1, 4) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D56_MODE BIT(0) +#define PORT_CONF_DEV25G_MODES_DEV25G_D56_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D56_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D56_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D56_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D57_MODE BIT(1) +#define PORT_CONF_DEV25G_MODES_DEV25G_D57_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D57_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D57_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D57_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D58_MODE BIT(2) +#define PORT_CONF_DEV25G_MODES_DEV25G_D58_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D58_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D58_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D58_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D59_MODE BIT(3) +#define PORT_CONF_DEV25G_MODES_DEV25G_D59_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D59_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D59_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D59_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D60_MODE BIT(4) +#define PORT_CONF_DEV25G_MODES_DEV25G_D60_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D60_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D60_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D60_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D61_MODE BIT(5) +#define PORT_CONF_DEV25G_MODES_DEV25G_D61_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D61_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D61_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D61_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D62_MODE BIT(6) +#define PORT_CONF_DEV25G_MODES_DEV25G_D62_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D62_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D62_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D62_MODE, x) + +#define PORT_CONF_DEV25G_MODES_DEV25G_D63_MODE BIT(7) +#define PORT_CONF_DEV25G_MODES_DEV25G_D63_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_DEV25G_MODES_DEV25G_D63_MODE, x) +#define PORT_CONF_DEV25G_MODES_DEV25G_D63_MODE_GET(x)\ + FIELD_GET(PORT_CONF_DEV25G_MODES_DEV25G_D63_MODE, x) + +/* PORT_CONF:HW_CFG:QSGMII_ENA */ +#define PORT_CONF_QSGMII_ENA __REG(TARGET_PORT_CONF, 0, 1, 0, 0, 1, 24, 12, 0, 1, 4) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_0 BIT(0) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_0_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_0, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_0_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_0, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_1 BIT(1) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_1_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_1, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_1_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_1, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_2 BIT(2) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_2_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_2, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_2_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_2, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_3 BIT(3) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_3_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_3, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_3_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_3, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_4 BIT(4) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_4_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_4, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_4_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_4, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_5 BIT(5) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_5_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_5, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_5_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_5, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_6 BIT(6) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_6_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_6, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_6_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_6, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_7 BIT(7) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_7_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_7, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_7_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_7, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_8 BIT(8) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_8_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_8, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_8_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_8, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_9 BIT(9) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_9_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_9, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_9_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_9, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_10 BIT(10) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_10_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_10, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_10_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_10, x) + +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_11 BIT(11) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_11_SET(x)\ + FIELD_PREP(PORT_CONF_QSGMII_ENA_QSGMII_ENA_11, x) +#define PORT_CONF_QSGMII_ENA_QSGMII_ENA_11_GET(x)\ + FIELD_GET(PORT_CONF_QSGMII_ENA_QSGMII_ENA_11, x) + +/* PORT_CONF:USGMII_CFG_STAT:USGMII_CFG */ +#define PORT_CONF_USGMII_CFG(g) __REG(TARGET_PORT_CONF, 0, 1, 72, g, 6, 8, 0, 0, 1, 4) + +#define PORT_CONF_USGMII_CFG_BYPASS_SCRAM BIT(9) +#define PORT_CONF_USGMII_CFG_BYPASS_SCRAM_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_BYPASS_SCRAM, x) +#define PORT_CONF_USGMII_CFG_BYPASS_SCRAM_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_BYPASS_SCRAM, x) + +#define PORT_CONF_USGMII_CFG_BYPASS_DESCRAM BIT(8) +#define PORT_CONF_USGMII_CFG_BYPASS_DESCRAM_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_BYPASS_DESCRAM, x) +#define PORT_CONF_USGMII_CFG_BYPASS_DESCRAM_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_BYPASS_DESCRAM, x) + +#define PORT_CONF_USGMII_CFG_FLIP_LANES BIT(7) +#define PORT_CONF_USGMII_CFG_FLIP_LANES_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_FLIP_LANES, x) +#define PORT_CONF_USGMII_CFG_FLIP_LANES_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_FLIP_LANES, x) + +#define PORT_CONF_USGMII_CFG_SHYST_DIS BIT(6) +#define PORT_CONF_USGMII_CFG_SHYST_DIS_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_SHYST_DIS, x) +#define PORT_CONF_USGMII_CFG_SHYST_DIS_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_SHYST_DIS, x) + +#define PORT_CONF_USGMII_CFG_E_DET_ENA BIT(5) +#define PORT_CONF_USGMII_CFG_E_DET_ENA_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_E_DET_ENA, x) +#define PORT_CONF_USGMII_CFG_E_DET_ENA_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_E_DET_ENA, x) + +#define PORT_CONF_USGMII_CFG_USE_I1_ENA BIT(4) +#define PORT_CONF_USGMII_CFG_USE_I1_ENA_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_USE_I1_ENA, x) +#define PORT_CONF_USGMII_CFG_USE_I1_ENA_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_USE_I1_ENA, x) + +#define PORT_CONF_USGMII_CFG_QUAD_MODE BIT(1) +#define PORT_CONF_USGMII_CFG_QUAD_MODE_SET(x)\ + FIELD_PREP(PORT_CONF_USGMII_CFG_QUAD_MODE, x) +#define PORT_CONF_USGMII_CFG_QUAD_MODE_GET(x)\ + FIELD_GET(PORT_CONF_USGMII_CFG_QUAD_MODE, x) + +/* QFWD:SYSTEM:SWITCH_PORT_MODE */ +#define QFWD_SWITCH_PORT_MODE(r) __REG(TARGET_QFWD, 0, 1, 0, 0, 1, 340, 0, r, 70, 4) + +#define QFWD_SWITCH_PORT_MODE_PORT_ENA BIT(19) +#define QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_PORT_ENA, x) +#define QFWD_SWITCH_PORT_MODE_PORT_ENA_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_PORT_ENA, x) + +#define QFWD_SWITCH_PORT_MODE_FWD_URGENCY GENMASK(18, 10) +#define QFWD_SWITCH_PORT_MODE_FWD_URGENCY_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_FWD_URGENCY, x) +#define QFWD_SWITCH_PORT_MODE_FWD_URGENCY_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_FWD_URGENCY, x) + +#define QFWD_SWITCH_PORT_MODE_YEL_RSRVD GENMASK(9, 6) +#define QFWD_SWITCH_PORT_MODE_YEL_RSRVD_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_YEL_RSRVD, x) +#define QFWD_SWITCH_PORT_MODE_YEL_RSRVD_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_YEL_RSRVD, x) + +#define QFWD_SWITCH_PORT_MODE_INGRESS_DROP_MODE BIT(5) +#define QFWD_SWITCH_PORT_MODE_INGRESS_DROP_MODE_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_INGRESS_DROP_MODE, x) +#define QFWD_SWITCH_PORT_MODE_INGRESS_DROP_MODE_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_INGRESS_DROP_MODE, x) + +#define QFWD_SWITCH_PORT_MODE_IGR_NO_SHARING BIT(4) +#define QFWD_SWITCH_PORT_MODE_IGR_NO_SHARING_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_IGR_NO_SHARING, x) +#define QFWD_SWITCH_PORT_MODE_IGR_NO_SHARING_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_IGR_NO_SHARING, x) + +#define QFWD_SWITCH_PORT_MODE_EGR_NO_SHARING BIT(3) +#define QFWD_SWITCH_PORT_MODE_EGR_NO_SHARING_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_EGR_NO_SHARING, x) +#define QFWD_SWITCH_PORT_MODE_EGR_NO_SHARING_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_EGR_NO_SHARING, x) + +#define QFWD_SWITCH_PORT_MODE_EGRESS_DROP_MODE BIT(2) +#define QFWD_SWITCH_PORT_MODE_EGRESS_DROP_MODE_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_EGRESS_DROP_MODE, x) +#define QFWD_SWITCH_PORT_MODE_EGRESS_DROP_MODE_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_EGRESS_DROP_MODE, x) + +#define QFWD_SWITCH_PORT_MODE_EGRESS_RSRV_DIS BIT(1) +#define QFWD_SWITCH_PORT_MODE_EGRESS_RSRV_DIS_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_EGRESS_RSRV_DIS, x) +#define QFWD_SWITCH_PORT_MODE_EGRESS_RSRV_DIS_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_EGRESS_RSRV_DIS, x) + +#define QFWD_SWITCH_PORT_MODE_LEARNALL_MORE BIT(0) +#define QFWD_SWITCH_PORT_MODE_LEARNALL_MORE_SET(x)\ + FIELD_PREP(QFWD_SWITCH_PORT_MODE_LEARNALL_MORE, x) +#define QFWD_SWITCH_PORT_MODE_LEARNALL_MORE_GET(x)\ + FIELD_GET(QFWD_SWITCH_PORT_MODE_LEARNALL_MORE, x) + +/* QRES:RES_CTRL:RES_CFG */ +#define QRES_RES_CFG(g) __REG(TARGET_QRES, 0, 1, 0, g, 5120, 16, 0, 0, 1, 4) + +#define QRES_RES_CFG_WM_HIGH GENMASK(11, 0) +#define QRES_RES_CFG_WM_HIGH_SET(x)\ + FIELD_PREP(QRES_RES_CFG_WM_HIGH, x) +#define QRES_RES_CFG_WM_HIGH_GET(x)\ + FIELD_GET(QRES_RES_CFG_WM_HIGH, x) + +/* QRES:RES_CTRL:RES_STAT */ +#define QRES_RES_STAT(g) __REG(TARGET_QRES, 0, 1, 0, g, 5120, 16, 4, 0, 1, 4) + +#define QRES_RES_STAT_MAXUSE GENMASK(20, 0) +#define QRES_RES_STAT_MAXUSE_SET(x)\ + FIELD_PREP(QRES_RES_STAT_MAXUSE, x) +#define QRES_RES_STAT_MAXUSE_GET(x)\ + FIELD_GET(QRES_RES_STAT_MAXUSE, x) + +/* QRES:RES_CTRL:RES_STAT_CUR */ +#define QRES_RES_STAT_CUR(g) __REG(TARGET_QRES, 0, 1, 0, g, 5120, 16, 8, 0, 1, 4) + +#define QRES_RES_STAT_CUR_INUSE GENMASK(20, 0) +#define QRES_RES_STAT_CUR_INUSE_SET(x)\ + FIELD_PREP(QRES_RES_STAT_CUR_INUSE, x) +#define QRES_RES_STAT_CUR_INUSE_GET(x)\ + FIELD_GET(QRES_RES_STAT_CUR_INUSE, x) + +/* DEVCPU_QS:XTR:XTR_GRP_CFG */ +#define QS_XTR_GRP_CFG(r) __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 0, r, 2, 4) + +#define QS_XTR_GRP_CFG_MODE GENMASK(3, 2) +#define QS_XTR_GRP_CFG_MODE_SET(x)\ + FIELD_PREP(QS_XTR_GRP_CFG_MODE, x) +#define QS_XTR_GRP_CFG_MODE_GET(x)\ + FIELD_GET(QS_XTR_GRP_CFG_MODE, x) + +#define QS_XTR_GRP_CFG_STATUS_WORD_POS BIT(1) +#define QS_XTR_GRP_CFG_STATUS_WORD_POS_SET(x)\ + FIELD_PREP(QS_XTR_GRP_CFG_STATUS_WORD_POS, x) +#define QS_XTR_GRP_CFG_STATUS_WORD_POS_GET(x)\ + FIELD_GET(QS_XTR_GRP_CFG_STATUS_WORD_POS, x) + +#define QS_XTR_GRP_CFG_BYTE_SWAP BIT(0) +#define QS_XTR_GRP_CFG_BYTE_SWAP_SET(x)\ + FIELD_PREP(QS_XTR_GRP_CFG_BYTE_SWAP, x) +#define QS_XTR_GRP_CFG_BYTE_SWAP_GET(x)\ + FIELD_GET(QS_XTR_GRP_CFG_BYTE_SWAP, x) + +/* DEVCPU_QS:XTR:XTR_RD */ +#define QS_XTR_RD(r) __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 8, r, 2, 4) + +/* DEVCPU_QS:XTR:XTR_FLUSH */ +#define QS_XTR_FLUSH __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 24, 0, 1, 4) + +#define QS_XTR_FLUSH_FLUSH GENMASK(1, 0) +#define QS_XTR_FLUSH_FLUSH_SET(x)\ + FIELD_PREP(QS_XTR_FLUSH_FLUSH, x) +#define QS_XTR_FLUSH_FLUSH_GET(x)\ + FIELD_GET(QS_XTR_FLUSH_FLUSH, x) + +/* DEVCPU_QS:XTR:XTR_DATA_PRESENT */ +#define QS_XTR_DATA_PRESENT __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 28, 0, 1, 4) + +#define QS_XTR_DATA_PRESENT_DATA_PRESENT GENMASK(1, 0) +#define QS_XTR_DATA_PRESENT_DATA_PRESENT_SET(x)\ + FIELD_PREP(QS_XTR_DATA_PRESENT_DATA_PRESENT, x) +#define QS_XTR_DATA_PRESENT_DATA_PRESENT_GET(x)\ + FIELD_GET(QS_XTR_DATA_PRESENT_DATA_PRESENT, x) + +/* DEVCPU_QS:INJ:INJ_GRP_CFG */ +#define QS_INJ_GRP_CFG(r) __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 0, r, 2, 4) + +#define QS_INJ_GRP_CFG_MODE GENMASK(3, 2) +#define QS_INJ_GRP_CFG_MODE_SET(x)\ + FIELD_PREP(QS_INJ_GRP_CFG_MODE, x) +#define QS_INJ_GRP_CFG_MODE_GET(x)\ + FIELD_GET(QS_INJ_GRP_CFG_MODE, x) + +#define QS_INJ_GRP_CFG_BYTE_SWAP BIT(0) +#define QS_INJ_GRP_CFG_BYTE_SWAP_SET(x)\ + FIELD_PREP(QS_INJ_GRP_CFG_BYTE_SWAP, x) +#define QS_INJ_GRP_CFG_BYTE_SWAP_GET(x)\ + FIELD_GET(QS_INJ_GRP_CFG_BYTE_SWAP, x) + +/* DEVCPU_QS:INJ:INJ_WR */ +#define QS_INJ_WR(r) __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 8, r, 2, 4) + +/* DEVCPU_QS:INJ:INJ_CTRL */ +#define QS_INJ_CTRL(r) __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 16, r, 2, 4) + +#define QS_INJ_CTRL_GAP_SIZE GENMASK(24, 21) +#define QS_INJ_CTRL_GAP_SIZE_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_GAP_SIZE, x) +#define QS_INJ_CTRL_GAP_SIZE_GET(x)\ + FIELD_GET(QS_INJ_CTRL_GAP_SIZE, x) + +#define QS_INJ_CTRL_ABORT BIT(20) +#define QS_INJ_CTRL_ABORT_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_ABORT, x) +#define QS_INJ_CTRL_ABORT_GET(x)\ + FIELD_GET(QS_INJ_CTRL_ABORT, x) + +#define QS_INJ_CTRL_EOF BIT(19) +#define QS_INJ_CTRL_EOF_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_EOF, x) +#define QS_INJ_CTRL_EOF_GET(x)\ + FIELD_GET(QS_INJ_CTRL_EOF, x) + +#define QS_INJ_CTRL_SOF BIT(18) +#define QS_INJ_CTRL_SOF_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_SOF, x) +#define QS_INJ_CTRL_SOF_GET(x)\ + FIELD_GET(QS_INJ_CTRL_SOF, x) + +#define QS_INJ_CTRL_VLD_BYTES GENMASK(17, 16) +#define QS_INJ_CTRL_VLD_BYTES_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_VLD_BYTES, x) +#define QS_INJ_CTRL_VLD_BYTES_GET(x)\ + FIELD_GET(QS_INJ_CTRL_VLD_BYTES, x) + +/* DEVCPU_QS:INJ:INJ_STATUS */ +#define QS_INJ_STATUS __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 24, 0, 1, 4) + +#define QS_INJ_STATUS_WMARK_REACHED GENMASK(5, 4) +#define QS_INJ_STATUS_WMARK_REACHED_SET(x)\ + FIELD_PREP(QS_INJ_STATUS_WMARK_REACHED, x) +#define QS_INJ_STATUS_WMARK_REACHED_GET(x)\ + FIELD_GET(QS_INJ_STATUS_WMARK_REACHED, x) + +#define QS_INJ_STATUS_FIFO_RDY GENMASK(3, 2) +#define QS_INJ_STATUS_FIFO_RDY_SET(x)\ + FIELD_PREP(QS_INJ_STATUS_FIFO_RDY, x) +#define QS_INJ_STATUS_FIFO_RDY_GET(x)\ + FIELD_GET(QS_INJ_STATUS_FIFO_RDY, x) + +#define QS_INJ_STATUS_INJ_IN_PROGRESS GENMASK(1, 0) +#define QS_INJ_STATUS_INJ_IN_PROGRESS_SET(x)\ + FIELD_PREP(QS_INJ_STATUS_INJ_IN_PROGRESS, x) +#define QS_INJ_STATUS_INJ_IN_PROGRESS_GET(x)\ + FIELD_GET(QS_INJ_STATUS_INJ_IN_PROGRESS, x) + +/* QSYS:PAUSE_CFG:PAUSE_CFG */ +#define QSYS_PAUSE_CFG(r) __REG(TARGET_QSYS, 0, 1, 544, 0, 1, 1128, 0, r, 70, 4) + +#define QSYS_PAUSE_CFG_PAUSE_START GENMASK(25, 14) +#define QSYS_PAUSE_CFG_PAUSE_START_SET(x)\ + FIELD_PREP(QSYS_PAUSE_CFG_PAUSE_START, x) +#define QSYS_PAUSE_CFG_PAUSE_START_GET(x)\ + FIELD_GET(QSYS_PAUSE_CFG_PAUSE_START, x) + +#define QSYS_PAUSE_CFG_PAUSE_STOP GENMASK(13, 2) +#define QSYS_PAUSE_CFG_PAUSE_STOP_SET(x)\ + FIELD_PREP(QSYS_PAUSE_CFG_PAUSE_STOP, x) +#define QSYS_PAUSE_CFG_PAUSE_STOP_GET(x)\ + FIELD_GET(QSYS_PAUSE_CFG_PAUSE_STOP, x) + +#define QSYS_PAUSE_CFG_PAUSE_ENA BIT(1) +#define QSYS_PAUSE_CFG_PAUSE_ENA_SET(x)\ + FIELD_PREP(QSYS_PAUSE_CFG_PAUSE_ENA, x) +#define QSYS_PAUSE_CFG_PAUSE_ENA_GET(x)\ + FIELD_GET(QSYS_PAUSE_CFG_PAUSE_ENA, x) + +#define QSYS_PAUSE_CFG_AGGRESSIVE_TAILDROP_ENA BIT(0) +#define QSYS_PAUSE_CFG_AGGRESSIVE_TAILDROP_ENA_SET(x)\ + FIELD_PREP(QSYS_PAUSE_CFG_AGGRESSIVE_TAILDROP_ENA, x) +#define QSYS_PAUSE_CFG_AGGRESSIVE_TAILDROP_ENA_GET(x)\ + FIELD_GET(QSYS_PAUSE_CFG_AGGRESSIVE_TAILDROP_ENA, x) + +/* QSYS:PAUSE_CFG:ATOP */ +#define QSYS_ATOP(r) __REG(TARGET_QSYS, 0, 1, 544, 0, 1, 1128, 284, r, 70, 4) + +#define QSYS_ATOP_ATOP GENMASK(11, 0) +#define QSYS_ATOP_ATOP_SET(x)\ + FIELD_PREP(QSYS_ATOP_ATOP, x) +#define QSYS_ATOP_ATOP_GET(x)\ + FIELD_GET(QSYS_ATOP_ATOP, x) + +/* QSYS:PAUSE_CFG:FWD_PRESSURE */ +#define QSYS_FWD_PRESSURE(r) __REG(TARGET_QSYS, 0, 1, 544, 0, 1, 1128, 564, r, 70, 4) + +#define QSYS_FWD_PRESSURE_FWD_PRESSURE GENMASK(11, 1) +#define QSYS_FWD_PRESSURE_FWD_PRESSURE_SET(x)\ + FIELD_PREP(QSYS_FWD_PRESSURE_FWD_PRESSURE, x) +#define QSYS_FWD_PRESSURE_FWD_PRESSURE_GET(x)\ + FIELD_GET(QSYS_FWD_PRESSURE_FWD_PRESSURE, x) + +#define QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS BIT(0) +#define QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS_SET(x)\ + FIELD_PREP(QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS, x) +#define QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS_GET(x)\ + FIELD_GET(QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS, x) + +/* QSYS:PAUSE_CFG:ATOP_TOT_CFG */ +#define QSYS_ATOP_TOT_CFG __REG(TARGET_QSYS, 0, 1, 544, 0, 1, 1128, 844, 0, 1, 4) + +#define QSYS_ATOP_TOT_CFG_ATOP_TOT GENMASK(11, 0) +#define QSYS_ATOP_TOT_CFG_ATOP_TOT_SET(x)\ + FIELD_PREP(QSYS_ATOP_TOT_CFG_ATOP_TOT, x) +#define QSYS_ATOP_TOT_CFG_ATOP_TOT_GET(x)\ + FIELD_GET(QSYS_ATOP_TOT_CFG_ATOP_TOT, x) + +/* QSYS:CALCFG:CAL_AUTO */ +#define QSYS_CAL_AUTO(r) __REG(TARGET_QSYS, 0, 1, 2304, 0, 1, 40, 0, r, 7, 4) + +#define QSYS_CAL_AUTO_CAL_AUTO GENMASK(29, 0) +#define QSYS_CAL_AUTO_CAL_AUTO_SET(x)\ + FIELD_PREP(QSYS_CAL_AUTO_CAL_AUTO, x) +#define QSYS_CAL_AUTO_CAL_AUTO_GET(x)\ + FIELD_GET(QSYS_CAL_AUTO_CAL_AUTO, x) + +/* QSYS:CALCFG:CAL_CTRL */ +#define QSYS_CAL_CTRL __REG(TARGET_QSYS, 0, 1, 2304, 0, 1, 40, 36, 0, 1, 4) + +#define QSYS_CAL_CTRL_CAL_MODE GENMASK(14, 11) +#define QSYS_CAL_CTRL_CAL_MODE_SET(x)\ + FIELD_PREP(QSYS_CAL_CTRL_CAL_MODE, x) +#define QSYS_CAL_CTRL_CAL_MODE_GET(x)\ + FIELD_GET(QSYS_CAL_CTRL_CAL_MODE, x) + +#define QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE GENMASK(10, 1) +#define QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE_SET(x)\ + FIELD_PREP(QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE, x) +#define QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE_GET(x)\ + FIELD_GET(QSYS_CAL_CTRL_CAL_AUTO_GRANT_RATE, x) + +#define QSYS_CAL_CTRL_CAL_AUTO_ERROR BIT(0) +#define QSYS_CAL_CTRL_CAL_AUTO_ERROR_SET(x)\ + FIELD_PREP(QSYS_CAL_CTRL_CAL_AUTO_ERROR, x) +#define QSYS_CAL_CTRL_CAL_AUTO_ERROR_GET(x)\ + FIELD_GET(QSYS_CAL_CTRL_CAL_AUTO_ERROR, x) + +/* QSYS:RAM_CTRL:RAM_INIT */ +#define QSYS_RAM_INIT __REG(TARGET_QSYS, 0, 1, 2344, 0, 1, 4, 0, 0, 1, 4) + +#define QSYS_RAM_INIT_RAM_INIT BIT(1) +#define QSYS_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(QSYS_RAM_INIT_RAM_INIT, x) +#define QSYS_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(QSYS_RAM_INIT_RAM_INIT, x) + +#define QSYS_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define QSYS_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(QSYS_RAM_INIT_RAM_CFG_HOOK, x) +#define QSYS_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(QSYS_RAM_INIT_RAM_CFG_HOOK, x) + +/* REW:COMMON:OWN_UPSID */ +#define REW_OWN_UPSID(r) __REG(TARGET_REW, 0, 1, 387264, 0, 1, 1232, 0, r, 3, 4) + +#define REW_OWN_UPSID_OWN_UPSID GENMASK(4, 0) +#define REW_OWN_UPSID_OWN_UPSID_SET(x)\ + FIELD_PREP(REW_OWN_UPSID_OWN_UPSID, x) +#define REW_OWN_UPSID_OWN_UPSID_GET(x)\ + FIELD_GET(REW_OWN_UPSID_OWN_UPSID, x) + +/* REW:PORT:PORT_VLAN_CFG */ +#define REW_PORT_VLAN_CFG(g) __REG(TARGET_REW, 0, 1, 360448, g, 70, 256, 0, 0, 1, 4) + +#define REW_PORT_VLAN_CFG_PORT_PCP GENMASK(15, 13) +#define REW_PORT_VLAN_CFG_PORT_PCP_SET(x)\ + FIELD_PREP(REW_PORT_VLAN_CFG_PORT_PCP, x) +#define REW_PORT_VLAN_CFG_PORT_PCP_GET(x)\ + FIELD_GET(REW_PORT_VLAN_CFG_PORT_PCP, x) + +#define REW_PORT_VLAN_CFG_PORT_DEI BIT(12) +#define REW_PORT_VLAN_CFG_PORT_DEI_SET(x)\ + FIELD_PREP(REW_PORT_VLAN_CFG_PORT_DEI, x) +#define REW_PORT_VLAN_CFG_PORT_DEI_GET(x)\ + FIELD_GET(REW_PORT_VLAN_CFG_PORT_DEI, x) + +#define REW_PORT_VLAN_CFG_PORT_VID GENMASK(11, 0) +#define REW_PORT_VLAN_CFG_PORT_VID_SET(x)\ + FIELD_PREP(REW_PORT_VLAN_CFG_PORT_VID, x) +#define REW_PORT_VLAN_CFG_PORT_VID_GET(x)\ + FIELD_GET(REW_PORT_VLAN_CFG_PORT_VID, x) + +/* REW:PORT:TAG_CTRL */ +#define REW_TAG_CTRL(g) __REG(TARGET_REW, 0, 1, 360448, g, 70, 256, 132, 0, 1, 4) + +#define REW_TAG_CTRL_TAG_CFG_OBEY_WAS_TAGGED BIT(13) +#define REW_TAG_CTRL_TAG_CFG_OBEY_WAS_TAGGED_SET(x)\ + FIELD_PREP(REW_TAG_CTRL_TAG_CFG_OBEY_WAS_TAGGED, x) +#define REW_TAG_CTRL_TAG_CFG_OBEY_WAS_TAGGED_GET(x)\ + FIELD_GET(REW_TAG_CTRL_TAG_CFG_OBEY_WAS_TAGGED, x) + +#define REW_TAG_CTRL_TAG_CFG GENMASK(12, 11) +#define REW_TAG_CTRL_TAG_CFG_SET(x)\ + FIELD_PREP(REW_TAG_CTRL_TAG_CFG, x) +#define REW_TAG_CTRL_TAG_CFG_GET(x)\ + FIELD_GET(REW_TAG_CTRL_TAG_CFG, x) + +#define REW_TAG_CTRL_TAG_TPID_CFG GENMASK(10, 8) +#define REW_TAG_CTRL_TAG_TPID_CFG_SET(x)\ + FIELD_PREP(REW_TAG_CTRL_TAG_TPID_CFG, x) +#define REW_TAG_CTRL_TAG_TPID_CFG_GET(x)\ + FIELD_GET(REW_TAG_CTRL_TAG_TPID_CFG, x) + +#define REW_TAG_CTRL_TAG_VID_CFG GENMASK(7, 6) +#define REW_TAG_CTRL_TAG_VID_CFG_SET(x)\ + FIELD_PREP(REW_TAG_CTRL_TAG_VID_CFG, x) +#define REW_TAG_CTRL_TAG_VID_CFG_GET(x)\ + FIELD_GET(REW_TAG_CTRL_TAG_VID_CFG, x) + +#define REW_TAG_CTRL_TAG_PCP_CFG GENMASK(5, 3) +#define REW_TAG_CTRL_TAG_PCP_CFG_SET(x)\ + FIELD_PREP(REW_TAG_CTRL_TAG_PCP_CFG, x) +#define REW_TAG_CTRL_TAG_PCP_CFG_GET(x)\ + FIELD_GET(REW_TAG_CTRL_TAG_PCP_CFG, x) + +#define REW_TAG_CTRL_TAG_DEI_CFG GENMASK(2, 0) +#define REW_TAG_CTRL_TAG_DEI_CFG_SET(x)\ + FIELD_PREP(REW_TAG_CTRL_TAG_DEI_CFG, x) +#define REW_TAG_CTRL_TAG_DEI_CFG_GET(x)\ + FIELD_GET(REW_TAG_CTRL_TAG_DEI_CFG, x) + +/* REW:RAM_CTRL:RAM_INIT */ +#define REW_RAM_INIT __REG(TARGET_REW, 0, 1, 378696, 0, 1, 4, 0, 0, 1, 4) + +#define REW_RAM_INIT_RAM_INIT BIT(1) +#define REW_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(REW_RAM_INIT_RAM_INIT, x) +#define REW_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(REW_RAM_INIT_RAM_INIT, x) + +#define REW_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define REW_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(REW_RAM_INIT_RAM_CFG_HOOK, x) +#define REW_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(REW_RAM_INIT_RAM_CFG_HOOK, x) + +/* VCAP_SUPER:RAM_CTRL:RAM_INIT */ +#define VCAP_SUPER_RAM_INIT __REG(TARGET_VCAP_SUPER, 0, 1, 1120, 0, 1, 4, 0, 0, 1, 4) + +#define VCAP_SUPER_RAM_INIT_RAM_INIT BIT(1) +#define VCAP_SUPER_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(VCAP_SUPER_RAM_INIT_RAM_INIT, x) +#define VCAP_SUPER_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(VCAP_SUPER_RAM_INIT_RAM_INIT, x) + +#define VCAP_SUPER_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define VCAP_SUPER_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(VCAP_SUPER_RAM_INIT_RAM_CFG_HOOK, x) +#define VCAP_SUPER_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(VCAP_SUPER_RAM_INIT_RAM_CFG_HOOK, x) + +/* VOP:RAM_CTRL:RAM_INIT */ +#define VOP_RAM_INIT __REG(TARGET_VOP, 0, 1, 279176, 0, 1, 4, 0, 0, 1, 4) + +#define VOP_RAM_INIT_RAM_INIT BIT(1) +#define VOP_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(VOP_RAM_INIT_RAM_INIT, x) +#define VOP_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(VOP_RAM_INIT_RAM_INIT, x) + +#define VOP_RAM_INIT_RAM_CFG_HOOK BIT(0) +#define VOP_RAM_INIT_RAM_CFG_HOOK_SET(x)\ + FIELD_PREP(VOP_RAM_INIT_RAM_CFG_HOOK, x) +#define VOP_RAM_INIT_RAM_CFG_HOOK_GET(x)\ + FIELD_GET(VOP_RAM_INIT_RAM_CFG_HOOK, x) + +/* XQS:SYSTEM:STAT_CFG */ +#define XQS_STAT_CFG __REG(TARGET_XQS, 0, 1, 6768, 0, 1, 872, 860, 0, 1, 4) + +#define XQS_STAT_CFG_STAT_CLEAR_SHOT GENMASK(21, 18) +#define XQS_STAT_CFG_STAT_CLEAR_SHOT_SET(x)\ + FIELD_PREP(XQS_STAT_CFG_STAT_CLEAR_SHOT, x) +#define XQS_STAT_CFG_STAT_CLEAR_SHOT_GET(x)\ + FIELD_GET(XQS_STAT_CFG_STAT_CLEAR_SHOT, x) + +#define XQS_STAT_CFG_STAT_VIEW GENMASK(17, 5) +#define XQS_STAT_CFG_STAT_VIEW_SET(x)\ + FIELD_PREP(XQS_STAT_CFG_STAT_VIEW, x) +#define XQS_STAT_CFG_STAT_VIEW_GET(x)\ + FIELD_GET(XQS_STAT_CFG_STAT_VIEW, x) + +#define XQS_STAT_CFG_STAT_SRV_PKT_ONLY BIT(4) +#define XQS_STAT_CFG_STAT_SRV_PKT_ONLY_SET(x)\ + FIELD_PREP(XQS_STAT_CFG_STAT_SRV_PKT_ONLY, x) +#define XQS_STAT_CFG_STAT_SRV_PKT_ONLY_GET(x)\ + FIELD_GET(XQS_STAT_CFG_STAT_SRV_PKT_ONLY, x) + +#define XQS_STAT_CFG_STAT_WRAP_DIS GENMASK(3, 0) +#define XQS_STAT_CFG_STAT_WRAP_DIS_SET(x)\ + FIELD_PREP(XQS_STAT_CFG_STAT_WRAP_DIS, x) +#define XQS_STAT_CFG_STAT_WRAP_DIS_GET(x)\ + FIELD_GET(XQS_STAT_CFG_STAT_WRAP_DIS, x) + +/* XQS:QLIMIT_SHR:QLIMIT_SHR_TOP_CFG */ +#define XQS_QLIMIT_SHR_TOP_CFG(g) __REG(TARGET_XQS, 0, 1, 7936, g, 4, 48, 0, 0, 1, 4) + +#define XQS_QLIMIT_SHR_TOP_CFG_QLIMIT_SHR_TOP GENMASK(14, 0) +#define XQS_QLIMIT_SHR_TOP_CFG_QLIMIT_SHR_TOP_SET(x)\ + FIELD_PREP(XQS_QLIMIT_SHR_TOP_CFG_QLIMIT_SHR_TOP, x) +#define XQS_QLIMIT_SHR_TOP_CFG_QLIMIT_SHR_TOP_GET(x)\ + FIELD_GET(XQS_QLIMIT_SHR_TOP_CFG_QLIMIT_SHR_TOP, x) + +/* XQS:QLIMIT_SHR:QLIMIT_SHR_ATOP_CFG */ +#define XQS_QLIMIT_SHR_ATOP_CFG(g) __REG(TARGET_XQS, 0, 1, 7936, g, 4, 48, 4, 0, 1, 4) + +#define XQS_QLIMIT_SHR_ATOP_CFG_QLIMIT_SHR_ATOP GENMASK(14, 0) +#define XQS_QLIMIT_SHR_ATOP_CFG_QLIMIT_SHR_ATOP_SET(x)\ + FIELD_PREP(XQS_QLIMIT_SHR_ATOP_CFG_QLIMIT_SHR_ATOP, x) +#define XQS_QLIMIT_SHR_ATOP_CFG_QLIMIT_SHR_ATOP_GET(x)\ + FIELD_GET(XQS_QLIMIT_SHR_ATOP_CFG_QLIMIT_SHR_ATOP, x) + +/* XQS:QLIMIT_SHR:QLIMIT_SHR_CTOP_CFG */ +#define XQS_QLIMIT_SHR_CTOP_CFG(g) __REG(TARGET_XQS, 0, 1, 7936, g, 4, 48, 8, 0, 1, 4) + +#define XQS_QLIMIT_SHR_CTOP_CFG_QLIMIT_SHR_CTOP GENMASK(14, 0) +#define XQS_QLIMIT_SHR_CTOP_CFG_QLIMIT_SHR_CTOP_SET(x)\ + FIELD_PREP(XQS_QLIMIT_SHR_CTOP_CFG_QLIMIT_SHR_CTOP, x) +#define XQS_QLIMIT_SHR_CTOP_CFG_QLIMIT_SHR_CTOP_GET(x)\ + FIELD_GET(XQS_QLIMIT_SHR_CTOP_CFG_QLIMIT_SHR_CTOP, x) + +/* XQS:QLIMIT_SHR:QLIMIT_SHR_QLIM_CFG */ +#define XQS_QLIMIT_SHR_QLIM_CFG(g) __REG(TARGET_XQS, 0, 1, 7936, g, 4, 48, 12, 0, 1, 4) + +#define XQS_QLIMIT_SHR_QLIM_CFG_QLIMIT_SHR_QLIM GENMASK(14, 0) +#define XQS_QLIMIT_SHR_QLIM_CFG_QLIMIT_SHR_QLIM_SET(x)\ + FIELD_PREP(XQS_QLIMIT_SHR_QLIM_CFG_QLIMIT_SHR_QLIM, x) +#define XQS_QLIMIT_SHR_QLIM_CFG_QLIMIT_SHR_QLIM_GET(x)\ + FIELD_GET(XQS_QLIMIT_SHR_QLIM_CFG_QLIMIT_SHR_QLIM, x) + +/* XQS:STAT:CNT */ +#define XQS_CNT(g) __REG(TARGET_XQS, 0, 1, 0, g, 1024, 4, 0, 0, 1, 4) + +#endif /* _SPARX5_MAIN_REGS_H_ */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c new file mode 100644 index 000000000000..9d485a9d1f1f --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" +#include "sparx5_port.h" + +/* The IFH bit position of the first VSTAX bit. This is because the + * VSTAX bit positions in Data sheet is starting from zero. + */ +#define VSTAX 73 + +static void ifh_encode_bitfield(void *ifh, u64 value, u32 pos, u32 width) +{ + u8 *ifh_hdr = ifh; + /* Calculate the Start IFH byte position of this IFH bit position */ + u32 byte = (35 - (pos / 8)); + /* Calculate the Start bit position in the Start IFH byte */ + u32 bit = (pos % 8); + u64 encode = GENMASK(bit + width - 1, bit) & (value << bit); + + /* Max width is 5 bytes - 40 bits. In worst case this will + * spread over 6 bytes - 48 bits + */ + compiletime_assert(width <= 40, "Unsupported width, must be <= 40"); + + /* The b0-b7 goes into the start IFH byte */ + if (encode & 0xFF) + ifh_hdr[byte] |= (u8)((encode & 0xFF)); + /* The b8-b15 goes into the next IFH byte */ + if (encode & 0xFF00) + ifh_hdr[byte - 1] |= (u8)((encode & 0xFF00) >> 8); + /* The b16-b23 goes into the next IFH byte */ + if (encode & 0xFF0000) + ifh_hdr[byte - 2] |= (u8)((encode & 0xFF0000) >> 16); + /* The b24-b31 goes into the next IFH byte */ + if (encode & 0xFF000000) + ifh_hdr[byte - 3] |= (u8)((encode & 0xFF000000) >> 24); + /* The b32-b39 goes into the next IFH byte */ + if (encode & 0xFF00000000) + ifh_hdr[byte - 4] |= (u8)((encode & 0xFF00000000) >> 32); + /* The b40-b47 goes into the next IFH byte */ + if (encode & 0xFF0000000000) + ifh_hdr[byte - 5] |= (u8)((encode & 0xFF0000000000) >> 40); +} + +static void sparx5_set_port_ifh(void *ifh_hdr, u16 portno) +{ + /* VSTAX.RSV = 1. MSBit must be 1 */ + ifh_encode_bitfield(ifh_hdr, 1, VSTAX + 79, 1); + /* VSTAX.INGR_DROP_MODE = Enable. Don't make head-of-line blocking */ + ifh_encode_bitfield(ifh_hdr, 1, VSTAX + 55, 1); + /* MISC.CPU_MASK/DPORT = Destination port */ + ifh_encode_bitfield(ifh_hdr, portno, 29, 8); + /* MISC.PIPELINE_PT */ + ifh_encode_bitfield(ifh_hdr, 16, 37, 5); + /* MISC.PIPELINE_ACT */ + ifh_encode_bitfield(ifh_hdr, 1, 42, 3); + /* FWD.SRC_PORT = CPU */ + ifh_encode_bitfield(ifh_hdr, SPX5_PORT_CPU, 46, 7); + /* FWD.SFLOW_ID (disable SFlow sampling) */ + ifh_encode_bitfield(ifh_hdr, 124, 57, 7); + /* FWD.UPDATE_FCS = Enable. Enforce update of FCS. */ + ifh_encode_bitfield(ifh_hdr, 1, 67, 1); +} + +static int sparx5_port_open(struct net_device *ndev) +{ + struct sparx5_port *port = netdev_priv(ndev); + int err = 0; + + sparx5_port_enable(port, true); + err = phylink_of_phy_connect(port->phylink, port->of_node, 0); + if (err) { + netdev_err(ndev, "Could not attach to PHY\n"); + return err; + } + + phylink_start(port->phylink); + + if (!ndev->phydev) { + /* power up serdes */ + port->conf.power_down = false; + if (port->conf.serdes_reset) + err = sparx5_serdes_set(port->sparx5, port, &port->conf); + else + err = phy_power_on(port->serdes); + if (err) + netdev_err(ndev, "%s failed\n", __func__); + } + + return err; +} + +static int sparx5_port_stop(struct net_device *ndev) +{ + struct sparx5_port *port = netdev_priv(ndev); + int err = 0; + + sparx5_port_enable(port, false); + phylink_stop(port->phylink); + phylink_disconnect_phy(port->phylink); + + if (!ndev->phydev) { + /* power down serdes */ + port->conf.power_down = true; + if (port->conf.serdes_reset) + err = sparx5_serdes_set(port->sparx5, port, &port->conf); + else + err = phy_power_off(port->serdes); + if (err) + netdev_err(ndev, "%s failed\n", __func__); + } + return 0; +} + +static void sparx5_set_rx_mode(struct net_device *dev) +{ + struct sparx5_port *port = netdev_priv(dev); + struct sparx5 *sparx5 = port->sparx5; + + if (!test_bit(port->portno, sparx5->bridge_mask)) + __dev_mc_sync(dev, sparx5_mc_sync, sparx5_mc_unsync); +} + +static int sparx5_port_get_phys_port_name(struct net_device *dev, + char *buf, size_t len) +{ + struct sparx5_port *port = netdev_priv(dev); + int ret; + + ret = snprintf(buf, len, "p%d", port->portno); + if (ret >= len) + return -EINVAL; + + return 0; +} + +static int sparx5_set_mac_address(struct net_device *dev, void *p) +{ + struct sparx5_port *port = netdev_priv(dev); + struct sparx5 *sparx5 = port->sparx5; + const struct sockaddr *addr = p; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + /* Remove current */ + sparx5_mact_forget(sparx5, dev->dev_addr, port->pvid); + + /* Add new */ + sparx5_mact_learn(sparx5, PGID_CPU, addr->sa_data, port->pvid); + + /* Record the address */ + ether_addr_copy(dev->dev_addr, addr->sa_data); + + return 0; +} + +static int sparx5_get_port_parent_id(struct net_device *dev, + struct netdev_phys_item_id *ppid) +{ + struct sparx5_port *sparx5_port = netdev_priv(dev); + struct sparx5 *sparx5 = sparx5_port->sparx5; + + ppid->id_len = sizeof(sparx5->base_mac); + memcpy(&ppid->id, &sparx5->base_mac, ppid->id_len); + + return 0; +} + +static const struct net_device_ops sparx5_port_netdev_ops = { + .ndo_open = sparx5_port_open, + .ndo_stop = sparx5_port_stop, + .ndo_start_xmit = sparx5_port_xmit_impl, + .ndo_set_rx_mode = sparx5_set_rx_mode, + .ndo_get_phys_port_name = sparx5_port_get_phys_port_name, + .ndo_set_mac_address = sparx5_set_mac_address, + .ndo_validate_addr = eth_validate_addr, + .ndo_get_stats64 = sparx5_get_stats64, + .ndo_get_port_parent_id = sparx5_get_port_parent_id, +}; + +bool sparx5_netdevice_check(const struct net_device *dev) +{ + return dev && (dev->netdev_ops == &sparx5_port_netdev_ops); +} + +struct net_device *sparx5_create_netdev(struct sparx5 *sparx5, u32 portno) +{ + struct sparx5_port *spx5_port; + struct net_device *ndev; + u64 val; + + ndev = devm_alloc_etherdev(sparx5->dev, sizeof(struct sparx5_port)); + if (!ndev) + return ERR_PTR(-ENOMEM); + + SET_NETDEV_DEV(ndev, sparx5->dev); + spx5_port = netdev_priv(ndev); + spx5_port->ndev = ndev; + spx5_port->sparx5 = sparx5; + spx5_port->portno = portno; + sparx5_set_port_ifh(spx5_port->ifh, portno); + + ndev->netdev_ops = &sparx5_port_netdev_ops; + ndev->ethtool_ops = &sparx5_ethtool_ops; + + val = ether_addr_to_u64(sparx5->base_mac) + portno + 1; + u64_to_ether_addr(val, ndev->dev_addr); + + return ndev; +} + +int sparx5_register_netdevs(struct sparx5 *sparx5) +{ + int portno; + int err; + + for (portno = 0; portno < SPX5_PORTS; portno++) + if (sparx5->ports[portno]) { + err = register_netdev(sparx5->ports[portno]->ndev); + if (err) { + dev_err(sparx5->dev, + "port: %02u: netdev registration failed\n", + portno); + return err; + } + sparx5_port_inj_timer_setup(sparx5->ports[portno]); + } + return 0; +} + +void sparx5_destroy_netdevs(struct sparx5 *sparx5) +{ + struct sparx5_port *port; + int portno; + + for (portno = 0; portno < SPX5_PORTS; portno++) { + port = sparx5->ports[portno]; + if (port && port->phylink) { + /* Disconnect the phy */ + rtnl_lock(); + sparx5_port_stop(port->ndev); + phylink_disconnect_phy(port->phylink); + rtnl_unlock(); + phylink_destroy(port->phylink); + port->phylink = NULL; + } + } +} + +void sparx5_unregister_netdevs(struct sparx5 *sparx5) +{ + int portno; + + for (portno = 0; portno < SPX5_PORTS; portno++) + if (sparx5->ports[portno]) + unregister_netdev(sparx5->ports[portno]->ndev); +} + diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c new file mode 100644 index 000000000000..09ca7a3bafdc --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" + +#define XTR_EOF_0 ntohl((__force __be32)0x80000000u) +#define XTR_EOF_1 ntohl((__force __be32)0x80000001u) +#define XTR_EOF_2 ntohl((__force __be32)0x80000002u) +#define XTR_EOF_3 ntohl((__force __be32)0x80000003u) +#define XTR_PRUNED ntohl((__force __be32)0x80000004u) +#define XTR_ABORT ntohl((__force __be32)0x80000005u) +#define XTR_ESCAPE ntohl((__force __be32)0x80000006u) +#define XTR_NOT_READY ntohl((__force __be32)0x80000007u) + +#define XTR_VALID_BYTES(x) (4 - ((x) & 3)) + +#define INJ_TIMEOUT_NS 50000 + +struct frame_info { + int src_port; +}; + +static void sparx5_xtr_flush(struct sparx5 *sparx5, u8 grp) +{ + /* Start flush */ + spx5_wr(QS_XTR_FLUSH_FLUSH_SET(BIT(grp)), sparx5, QS_XTR_FLUSH); + + /* Allow to drain */ + mdelay(1); + + /* All Queues normal */ + spx5_wr(0, sparx5, QS_XTR_FLUSH); +} + +static void sparx5_ifh_parse(u32 *ifh, struct frame_info *info) +{ + u8 *xtr_hdr = (u8 *)ifh; + + /* FWD is bit 45-72 (28 bits), but we only read the 27 LSB for now */ + u32 fwd = + ((u32)xtr_hdr[27] << 24) | + ((u32)xtr_hdr[28] << 16) | + ((u32)xtr_hdr[29] << 8) | + ((u32)xtr_hdr[30] << 0); + fwd = (fwd >> 5); + info->src_port = FIELD_GET(GENMASK(7, 1), fwd); +} + +static void sparx5_xtr_grp(struct sparx5 *sparx5, u8 grp, bool byte_swap) +{ + bool eof_flag = false, pruned_flag = false, abort_flag = false; + struct net_device *netdev; + struct sparx5_port *port; + struct frame_info fi; + int i, byte_cnt = 0; + struct sk_buff *skb; + u32 ifh[IFH_LEN]; + u32 *rxbuf; + + /* Get IFH */ + for (i = 0; i < IFH_LEN; i++) + ifh[i] = spx5_rd(sparx5, QS_XTR_RD(grp)); + + /* Decode IFH (whats needed) */ + sparx5_ifh_parse(ifh, &fi); + + /* Map to port netdev */ + port = fi.src_port < SPX5_PORTS ? + sparx5->ports[fi.src_port] : NULL; + if (!port || !port->ndev) { + dev_err(sparx5->dev, "Data on inactive port %d\n", fi.src_port); + sparx5_xtr_flush(sparx5, grp); + return; + } + + /* Have netdev, get skb */ + netdev = port->ndev; + skb = netdev_alloc_skb(netdev, netdev->mtu + ETH_HLEN); + if (!skb) { + sparx5_xtr_flush(sparx5, grp); + dev_err(sparx5->dev, "No skb allocated\n"); + netdev->stats.rx_dropped++; + return; + } + rxbuf = (u32 *)skb->data; + + /* Now, pull frame data */ + while (!eof_flag) { + u32 val = spx5_rd(sparx5, QS_XTR_RD(grp)); + u32 cmp = val; + + if (byte_swap) + cmp = ntohl((__force __be32)val); + + switch (cmp) { + case XTR_NOT_READY: + break; + case XTR_ABORT: + /* No accompanying data */ + abort_flag = true; + eof_flag = true; + break; + case XTR_EOF_0: + case XTR_EOF_1: + case XTR_EOF_2: + case XTR_EOF_3: + /* This assumes STATUS_WORD_POS == 1, Status + * just after last data + */ + byte_cnt -= (4 - XTR_VALID_BYTES(val)); + eof_flag = true; + break; + case XTR_PRUNED: + /* But get the last 4 bytes as well */ + eof_flag = true; + pruned_flag = true; + fallthrough; + case XTR_ESCAPE: + *rxbuf = spx5_rd(sparx5, QS_XTR_RD(grp)); + byte_cnt += 4; + rxbuf++; + break; + default: + *rxbuf = val; + byte_cnt += 4; + rxbuf++; + } + } + + if (abort_flag || pruned_flag || !eof_flag) { + netdev_err(netdev, "Discarded frame: abort:%d pruned:%d eof:%d\n", + abort_flag, pruned_flag, eof_flag); + kfree_skb(skb); + netdev->stats.rx_dropped++; + return; + } + + /* Everything we see on an interface that is in the HW bridge + * has already been forwarded + */ + if (test_bit(port->portno, sparx5->bridge_mask)) + skb->offload_fwd_mark = 1; + + /* Finish up skb */ + skb_put(skb, byte_cnt - ETH_FCS_LEN); + eth_skb_pad(skb); + skb->protocol = eth_type_trans(skb, netdev); + netif_rx(skb); + netdev->stats.rx_bytes += skb->len; + netdev->stats.rx_packets++; +} + +static int sparx5_inject(struct sparx5 *sparx5, + u32 *ifh, + struct sk_buff *skb, + struct net_device *ndev) +{ + int grp = INJ_QUEUE; + u32 val, w, count; + u8 *buf; + + val = spx5_rd(sparx5, QS_INJ_STATUS); + if (!(QS_INJ_STATUS_FIFO_RDY_GET(val) & BIT(grp))) { + pr_err_ratelimited("Injection: Queue not ready: 0x%lx\n", + QS_INJ_STATUS_FIFO_RDY_GET(val)); + return -EBUSY; + } + + /* Indicate SOF */ + spx5_wr(QS_INJ_CTRL_SOF_SET(1) | + QS_INJ_CTRL_GAP_SIZE_SET(1), + sparx5, QS_INJ_CTRL(grp)); + + /* Write the IFH to the chip. */ + for (w = 0; w < IFH_LEN; w++) + spx5_wr(ifh[w], sparx5, QS_INJ_WR(grp)); + + /* Write words, round up */ + count = DIV_ROUND_UP(skb->len, 4); + buf = skb->data; + for (w = 0; w < count; w++, buf += 4) { + val = get_unaligned((const u32 *)buf); + spx5_wr(val, sparx5, QS_INJ_WR(grp)); + } + + /* Add padding */ + while (w < (60 / 4)) { + spx5_wr(0, sparx5, QS_INJ_WR(grp)); + w++; + } + + /* Indicate EOF and valid bytes in last word */ + spx5_wr(QS_INJ_CTRL_GAP_SIZE_SET(1) | + QS_INJ_CTRL_VLD_BYTES_SET(skb->len < 60 ? 0 : skb->len % 4) | + QS_INJ_CTRL_EOF_SET(1), + sparx5, QS_INJ_CTRL(grp)); + + /* Add dummy CRC */ + spx5_wr(0, sparx5, QS_INJ_WR(grp)); + w++; + + val = spx5_rd(sparx5, QS_INJ_STATUS); + if (QS_INJ_STATUS_WMARK_REACHED_GET(val) & BIT(grp)) { + struct sparx5_port *port = netdev_priv(ndev); + + pr_err_ratelimited("Injection: Watermark reached: 0x%lx\n", + QS_INJ_STATUS_WMARK_REACHED_GET(val)); + netif_stop_queue(ndev); + hrtimer_start(&port->inj_timer, INJ_TIMEOUT_NS, + HRTIMER_MODE_REL); + } + + return NETDEV_TX_OK; +} + +int sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev) +{ + struct net_device_stats *stats = &dev->stats; + struct sparx5_port *port = netdev_priv(dev); + struct sparx5 *sparx5 = port->sparx5; + int ret; + + ret = sparx5_inject(sparx5, port->ifh, skb, dev); + + if (ret == NETDEV_TX_OK) { + stats->tx_bytes += skb->len; + stats->tx_packets++; + skb_tx_timestamp(skb); + dev_kfree_skb_any(skb); + } else { + stats->tx_dropped++; + } + return ret; +} + +static enum hrtimer_restart sparx5_injection_timeout(struct hrtimer *tmr) +{ + struct sparx5_port *port = container_of(tmr, struct sparx5_port, + inj_timer); + int grp = INJ_QUEUE; + u32 val; + + val = spx5_rd(port->sparx5, QS_INJ_STATUS); + if (QS_INJ_STATUS_WMARK_REACHED_GET(val) & BIT(grp)) { + pr_err_ratelimited("Injection: Reset watermark count\n"); + /* Reset Watermark count to restart */ + spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_SET(1), + DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR, + port->sparx5, + DSM_DEV_TX_STOP_WM_CFG(port->portno)); + } + netif_wake_queue(port->ndev); + return HRTIMER_NORESTART; +} + +int sparx5_manual_injection_mode(struct sparx5 *sparx5) +{ + const int byte_swap = 1; + int portno; + + /* Change mode to manual extraction and injection */ + spx5_wr(QS_XTR_GRP_CFG_MODE_SET(1) | + QS_XTR_GRP_CFG_STATUS_WORD_POS_SET(1) | + QS_XTR_GRP_CFG_BYTE_SWAP_SET(byte_swap), + sparx5, QS_XTR_GRP_CFG(XTR_QUEUE)); + spx5_wr(QS_INJ_GRP_CFG_MODE_SET(1) | + QS_INJ_GRP_CFG_BYTE_SWAP_SET(byte_swap), + sparx5, QS_INJ_GRP_CFG(INJ_QUEUE)); + + /* CPU ports capture setup */ + for (portno = SPX5_PORT_CPU_0; portno <= SPX5_PORT_CPU_1; portno++) { + /* ASM CPU port: No preamble, IFH, enable padding */ + spx5_wr(ASM_PORT_CFG_PAD_ENA_SET(1) | + ASM_PORT_CFG_NO_PREAMBLE_ENA_SET(1) | + ASM_PORT_CFG_INJ_FORMAT_CFG_SET(1), /* 1 = IFH */ + sparx5, ASM_PORT_CFG(portno)); + + /* Reset WM cnt to unclog queued frames */ + spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_SET(1), + DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR, + sparx5, + DSM_DEV_TX_STOP_WM_CFG(portno)); + + /* Set Disassembler Stop Watermark level */ + spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_SET(0), + DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM, + sparx5, + DSM_DEV_TX_STOP_WM_CFG(portno)); + + /* Enable Disassembler buffer underrun watchdog + */ + spx5_rmw(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS_SET(0), + DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS, + sparx5, + DSM_BUF_CFG(portno)); + } + return 0; +} + +irqreturn_t sparx5_xtr_handler(int irq, void *_sparx5) +{ + struct sparx5 *s5 = _sparx5; + int poll = 64; + + /* Check data in queue */ + while (spx5_rd(s5, QS_XTR_DATA_PRESENT) & BIT(XTR_QUEUE) && poll-- > 0) + sparx5_xtr_grp(s5, XTR_QUEUE, false); + + return IRQ_HANDLED; +} + +void sparx5_port_inj_timer_setup(struct sparx5_port *port) +{ + hrtimer_init(&port->inj_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + port->inj_timer.function = sparx5_injection_timeout; +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c b/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c new file mode 100644 index 000000000000..af70e2795125 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include <linux/module.h> +#include <linux/phylink.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/sfp.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" +#include "sparx5_port.h" + +static bool port_conf_has_changed(struct sparx5_port_config *a, struct sparx5_port_config *b) +{ + if (a->speed != b->speed || + a->portmode != b->portmode || + a->autoneg != b->autoneg || + a->pause_adv != b->pause_adv || + a->power_down != b->power_down || + a->media != b->media) + return true; + return false; +} + +static void sparx5_phylink_validate(struct phylink_config *config, + unsigned long *supported, + struct phylink_link_state *state) +{ + struct sparx5_port *port = netdev_priv(to_net_dev(config->dev)); + __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; + + phylink_set(mask, Autoneg); + phylink_set_port_modes(mask); + phylink_set(mask, Pause); + phylink_set(mask, Asym_Pause); + + switch (state->interface) { + case PHY_INTERFACE_MODE_5GBASER: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_25GBASER: + case PHY_INTERFACE_MODE_NA: + if (port->conf.bandwidth == SPEED_5000) + phylink_set(mask, 5000baseT_Full); + if (port->conf.bandwidth == SPEED_10000) { + phylink_set(mask, 5000baseT_Full); + phylink_set(mask, 10000baseT_Full); + phylink_set(mask, 10000baseCR_Full); + phylink_set(mask, 10000baseSR_Full); + phylink_set(mask, 10000baseLR_Full); + phylink_set(mask, 10000baseLRM_Full); + phylink_set(mask, 10000baseER_Full); + } + if (port->conf.bandwidth == SPEED_25000) { + phylink_set(mask, 5000baseT_Full); + phylink_set(mask, 10000baseT_Full); + phylink_set(mask, 10000baseCR_Full); + phylink_set(mask, 10000baseSR_Full); + phylink_set(mask, 10000baseLR_Full); + phylink_set(mask, 10000baseLRM_Full); + phylink_set(mask, 10000baseER_Full); + phylink_set(mask, 25000baseCR_Full); + phylink_set(mask, 25000baseSR_Full); + } + if (state->interface != PHY_INTERFACE_MODE_NA) + break; + fallthrough; + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_QSGMII: + phylink_set(mask, 10baseT_Half); + phylink_set(mask, 10baseT_Full); + phylink_set(mask, 100baseT_Half); + phylink_set(mask, 100baseT_Full); + phylink_set(mask, 1000baseT_Full); + phylink_set(mask, 1000baseX_Full); + if (state->interface != PHY_INTERFACE_MODE_NA) + break; + fallthrough; + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + if (state->interface != PHY_INTERFACE_MODE_2500BASEX) { + phylink_set(mask, 1000baseT_Full); + phylink_set(mask, 1000baseX_Full); + } + if (state->interface == PHY_INTERFACE_MODE_2500BASEX || + state->interface == PHY_INTERFACE_MODE_NA) { + phylink_set(mask, 2500baseT_Full); + phylink_set(mask, 2500baseX_Full); + } + break; + default: + bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); + return; + } + bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS); + bitmap_and(state->advertising, state->advertising, mask, + __ETHTOOL_LINK_MODE_MASK_NBITS); +} + +static void sparx5_phylink_mac_config(struct phylink_config *config, + unsigned int mode, + const struct phylink_link_state *state) +{ + /* Currently not used */ +} + +static void sparx5_phylink_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, + phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct sparx5_port *port = netdev_priv(to_net_dev(config->dev)); + struct sparx5_port_config conf; + int err; + + conf = port->conf; + conf.duplex = duplex; + conf.pause = 0; + conf.pause |= tx_pause ? MLO_PAUSE_TX : 0; + conf.pause |= rx_pause ? MLO_PAUSE_RX : 0; + conf.speed = speed; + /* Configure the port to speed/duplex/pause */ + err = sparx5_port_config(port->sparx5, port, &conf); + if (err) + netdev_err(port->ndev, "port config failed: %d\n", err); +} + +static void sparx5_phylink_mac_link_down(struct phylink_config *config, + unsigned int mode, + phy_interface_t interface) +{ + /* Currently not used */ +} + +static struct sparx5_port *sparx5_pcs_to_port(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct sparx5_port, phylink_pcs); +} + +static void sparx5_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct sparx5_port *port = sparx5_pcs_to_port(pcs); + struct sparx5_port_status status; + + sparx5_get_port_status(port->sparx5, port, &status); + state->link = status.link && !status.link_down; + state->an_complete = status.an_complete; + state->speed = status.speed; + state->duplex = status.duplex; + state->pause = status.pause; +} + +static int sparx5_pcs_config(struct phylink_pcs *pcs, + unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct sparx5_port *port = sparx5_pcs_to_port(pcs); + struct sparx5_port_config conf; + int ret = 0; + + conf = port->conf; + conf.power_down = false; + conf.portmode = interface; + conf.inband = phylink_autoneg_inband(mode); + conf.autoneg = phylink_test(advertising, Autoneg); + conf.pause_adv = 0; + if (phylink_test(advertising, Pause)) + conf.pause_adv |= ADVERTISE_1000XPAUSE; + if (phylink_test(advertising, Asym_Pause)) + conf.pause_adv |= ADVERTISE_1000XPSE_ASYM; + if (sparx5_is_baser(interface)) { + if (phylink_test(advertising, FIBRE)) + conf.media = PHY_MEDIA_SR; + else + conf.media = PHY_MEDIA_DAC; + } + if (!port_conf_has_changed(&port->conf, &conf)) + return ret; + /* Enable the PCS matching this interface type */ + ret = sparx5_port_pcs_set(port->sparx5, port, &conf); + if (ret) + netdev_err(port->ndev, "port PCS config failed: %d\n", ret); + return ret; +} + +static void sparx5_pcs_aneg_restart(struct phylink_pcs *pcs) +{ + /* Currently not used */ +} + +const struct phylink_pcs_ops sparx5_phylink_pcs_ops = { + .pcs_get_state = sparx5_pcs_get_state, + .pcs_config = sparx5_pcs_config, + .pcs_an_restart = sparx5_pcs_aneg_restart, +}; + +const struct phylink_mac_ops sparx5_phylink_mac_ops = { + .validate = sparx5_phylink_validate, + .mac_config = sparx5_phylink_mac_config, + .mac_link_down = sparx5_phylink_mac_link_down, + .mac_link_up = sparx5_phylink_mac_link_up, +}; diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_port.c b/drivers/net/ethernet/microchip/sparx5/sparx5_port.c new file mode 100644 index 000000000000..d2e3250928bf --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_port.c @@ -0,0 +1,1146 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include <linux/module.h> +#include <linux/phy/phy.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" +#include "sparx5_port.h" + +#define SPX5_ETYPE_TAG_C 0x8100 +#define SPX5_ETYPE_TAG_S 0x88a8 + +#define SPX5_WAIT_US 1000 +#define SPX5_WAIT_MAX_US 2000 + +enum port_error { + SPX5_PERR_SPEED, + SPX5_PERR_IFTYPE, +}; + +#define PAUSE_DISCARD 0xC +#define ETH_MAXLEN (ETH_DATA_LEN + ETH_HLEN + ETH_FCS_LEN) + +static void decode_sgmii_word(u16 lp_abil, struct sparx5_port_status *status) +{ + status->an_complete = true; + if (!(lp_abil & LPA_SGMII_LINK)) { + status->link = false; + return; + } + + switch (lp_abil & LPA_SGMII_SPD_MASK) { + case LPA_SGMII_10: + status->speed = SPEED_10; + break; + case LPA_SGMII_100: + status->speed = SPEED_100; + break; + case LPA_SGMII_1000: + status->speed = SPEED_1000; + break; + default: + status->link = false; + return; + } + if (lp_abil & LPA_SGMII_FULL_DUPLEX) + status->duplex = DUPLEX_FULL; + else + status->duplex = DUPLEX_HALF; +} + +static void decode_cl37_word(u16 lp_abil, uint16_t ld_abil, struct sparx5_port_status *status) +{ + status->link = !(lp_abil & ADVERTISE_RFAULT) && status->link; + status->an_complete = true; + status->duplex = (ADVERTISE_1000XFULL & lp_abil) ? + DUPLEX_FULL : DUPLEX_UNKNOWN; // 1G HDX not supported + + if ((ld_abil & ADVERTISE_1000XPAUSE) && + (lp_abil & ADVERTISE_1000XPAUSE)) { + status->pause = MLO_PAUSE_RX | MLO_PAUSE_TX; + } else if ((ld_abil & ADVERTISE_1000XPSE_ASYM) && + (lp_abil & ADVERTISE_1000XPSE_ASYM)) { + status->pause |= (lp_abil & ADVERTISE_1000XPAUSE) ? + MLO_PAUSE_TX : 0; + status->pause |= (ld_abil & ADVERTISE_1000XPAUSE) ? + MLO_PAUSE_RX : 0; + } else { + status->pause = MLO_PAUSE_NONE; + } +} + +static int sparx5_get_dev2g5_status(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_status *status) +{ + u32 portno = port->portno; + u16 lp_adv, ld_adv; + u32 value; + + /* Get PCS Link down sticky */ + value = spx5_rd(sparx5, DEV2G5_PCS1G_STICKY(portno)); + status->link_down = DEV2G5_PCS1G_STICKY_LINK_DOWN_STICKY_GET(value); + if (status->link_down) /* Clear the sticky */ + spx5_wr(value, sparx5, DEV2G5_PCS1G_STICKY(portno)); + + /* Get both current Link and Sync status */ + value = spx5_rd(sparx5, DEV2G5_PCS1G_LINK_STATUS(portno)); + status->link = DEV2G5_PCS1G_LINK_STATUS_LINK_STATUS_GET(value) && + DEV2G5_PCS1G_LINK_STATUS_SYNC_STATUS_GET(value); + + if (port->conf.portmode == PHY_INTERFACE_MODE_1000BASEX) + status->speed = SPEED_1000; + else if (port->conf.portmode == PHY_INTERFACE_MODE_2500BASEX) + status->speed = SPEED_2500; + + status->duplex = DUPLEX_FULL; + + /* Get PCS ANEG status register */ + value = spx5_rd(sparx5, DEV2G5_PCS1G_ANEG_STATUS(portno)); + + /* Aneg complete provides more information */ + if (DEV2G5_PCS1G_ANEG_STATUS_ANEG_COMPLETE_GET(value)) { + lp_adv = DEV2G5_PCS1G_ANEG_STATUS_LP_ADV_ABILITY_GET(value); + if (port->conf.portmode == PHY_INTERFACE_MODE_SGMII) { + decode_sgmii_word(lp_adv, status); + } else { + value = spx5_rd(sparx5, DEV2G5_PCS1G_ANEG_CFG(portno)); + ld_adv = DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY_GET(value); + decode_cl37_word(lp_adv, ld_adv, status); + } + } + return 0; +} + +static int sparx5_get_sfi_status(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_status *status) +{ + bool high_speed_dev = sparx5_is_baser(port->conf.portmode); + u32 portno = port->portno; + u32 value, dev, tinst; + void __iomem *inst; + + if (!high_speed_dev) { + netdev_err(port->ndev, "error: low speed and SFI mode\n"); + return -EINVAL; + } + + dev = sparx5_to_high_dev(portno); + tinst = sparx5_port_dev_index(portno); + inst = spx5_inst_get(sparx5, dev, tinst); + + value = spx5_inst_rd(inst, DEV10G_MAC_TX_MONITOR_STICKY(0)); + if (value != DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY) { + /* The link is or has been down. Clear the sticky bit */ + status->link_down = 1; + spx5_inst_wr(0xffffffff, inst, DEV10G_MAC_TX_MONITOR_STICKY(0)); + value = spx5_inst_rd(inst, DEV10G_MAC_TX_MONITOR_STICKY(0)); + } + status->link = (value == DEV10G_MAC_TX_MONITOR_STICKY_IDLE_STATE_STICKY); + status->duplex = DUPLEX_FULL; + if (port->conf.portmode == PHY_INTERFACE_MODE_5GBASER) + status->speed = SPEED_5000; + else if (port->conf.portmode == PHY_INTERFACE_MODE_10GBASER) + status->speed = SPEED_10000; + else + status->speed = SPEED_25000; + + return 0; +} + +/* Get link status of 1000Base-X/in-band and SFI ports. + */ +int sparx5_get_port_status(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_status *status) +{ + memset(status, 0, sizeof(*status)); + status->speed = port->conf.speed; + if (port->conf.power_down) { + status->link = false; + return 0; + } + switch (port->conf.portmode) { + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + return sparx5_get_dev2g5_status(sparx5, port, status); + case PHY_INTERFACE_MODE_5GBASER: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_25GBASER: + return sparx5_get_sfi_status(sparx5, port, status); + case PHY_INTERFACE_MODE_NA: + return 0; + default: + netdev_err(port->ndev, "Status not supported"); + return -ENODEV; + } + return 0; +} + +static int sparx5_port_error(struct sparx5_port *port, + struct sparx5_port_config *conf, + enum port_error errtype) +{ + switch (errtype) { + case SPX5_PERR_SPEED: + netdev_err(port->ndev, + "Interface does not support speed: %u: for %s\n", + conf->speed, phy_modes(conf->portmode)); + break; + case SPX5_PERR_IFTYPE: + netdev_err(port->ndev, + "Switch port does not support interface type: %s\n", + phy_modes(conf->portmode)); + break; + default: + netdev_err(port->ndev, + "Interface configuration error\n"); + } + + return -EINVAL; +} + +static int sparx5_port_verify_speed(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + if ((sparx5_port_is_2g5(port->portno) && + conf->speed > SPEED_2500) || + (sparx5_port_is_5g(port->portno) && + conf->speed > SPEED_5000) || + (sparx5_port_is_10g(port->portno) && + conf->speed > SPEED_10000)) + return sparx5_port_error(port, conf, SPX5_PERR_SPEED); + + switch (conf->portmode) { + case PHY_INTERFACE_MODE_NA: + return -EINVAL; + case PHY_INTERFACE_MODE_1000BASEX: + if (conf->speed != SPEED_1000 || + sparx5_port_is_2g5(port->portno)) + return sparx5_port_error(port, conf, SPX5_PERR_SPEED); + if (sparx5_port_is_2g5(port->portno)) + return sparx5_port_error(port, conf, SPX5_PERR_IFTYPE); + break; + case PHY_INTERFACE_MODE_2500BASEX: + if (conf->speed != SPEED_2500 || + sparx5_port_is_2g5(port->portno)) + return sparx5_port_error(port, conf, SPX5_PERR_SPEED); + break; + case PHY_INTERFACE_MODE_QSGMII: + if (port->portno > 47) + return sparx5_port_error(port, conf, SPX5_PERR_IFTYPE); + fallthrough; + case PHY_INTERFACE_MODE_SGMII: + if (conf->speed != SPEED_1000 && + conf->speed != SPEED_100 && + conf->speed != SPEED_10 && + conf->speed != SPEED_2500) + return sparx5_port_error(port, conf, SPX5_PERR_SPEED); + break; + case PHY_INTERFACE_MODE_5GBASER: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_25GBASER: + if ((conf->speed != SPEED_5000 && + conf->speed != SPEED_10000 && + conf->speed != SPEED_25000)) + return sparx5_port_error(port, conf, SPX5_PERR_SPEED); + break; + default: + return sparx5_port_error(port, conf, SPX5_PERR_IFTYPE); + } + return 0; +} + +static bool sparx5_dev_change(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + return sparx5_is_baser(port->conf.portmode) ^ + sparx5_is_baser(conf->portmode); +} + +static int sparx5_port_flush_poll(struct sparx5 *sparx5, u32 portno) +{ + u32 value, resource, prio, delay_cnt = 0; + bool poll_src = true; + char *mem = ""; + + /* Resource == 0: Memory tracked per source (SRC-MEM) + * Resource == 1: Frame references tracked per source (SRC-REF) + * Resource == 2: Memory tracked per destination (DST-MEM) + * Resource == 3: Frame references tracked per destination. (DST-REF) + */ + while (1) { + bool empty = true; + + for (resource = 0; resource < (poll_src ? 2 : 1); resource++) { + u32 base; + + base = (resource == 0 ? 2048 : 0) + SPX5_PRIOS * portno; + for (prio = 0; prio < SPX5_PRIOS; prio++) { + value = spx5_rd(sparx5, + QRES_RES_STAT(base + prio)); + if (value) { + mem = resource == 0 ? + "DST-MEM" : "SRC-MEM"; + empty = false; + } + } + } + + if (empty) + break; + + if (delay_cnt++ == 2000) { + dev_err(sparx5->dev, + "Flush timeout port %u. %s queue not empty\n", + portno, mem); + return -EINVAL; + } + + usleep_range(SPX5_WAIT_US, SPX5_WAIT_MAX_US); + } + return 0; +} + +static int sparx5_port_disable(struct sparx5 *sparx5, struct sparx5_port *port, bool high_spd_dev) +{ + u32 tinst = high_spd_dev ? + sparx5_port_dev_index(port->portno) : port->portno; + u32 dev = high_spd_dev ? + sparx5_to_high_dev(port->portno) : TARGET_DEV2G5; + void __iomem *devinst = spx5_inst_get(sparx5, dev, tinst); + u32 spd = port->conf.speed; + u32 spd_prm; + int err; + + if (high_spd_dev) { + /* 1: Reset the PCS Rx clock domain */ + spx5_inst_rmw(DEV10G_DEV_RST_CTRL_PCS_RX_RST, + DEV10G_DEV_RST_CTRL_PCS_RX_RST, + devinst, + DEV10G_DEV_RST_CTRL(0)); + + /* 2: Disable MAC frame reception */ + spx5_inst_rmw(0, + DEV10G_MAC_ENA_CFG_RX_ENA, + devinst, + DEV10G_MAC_ENA_CFG(0)); + } else { + /* 1: Reset the PCS Rx clock domain */ + spx5_inst_rmw(DEV2G5_DEV_RST_CTRL_PCS_RX_RST, + DEV2G5_DEV_RST_CTRL_PCS_RX_RST, + devinst, + DEV2G5_DEV_RST_CTRL(0)); + /* 2: Disable MAC frame reception */ + spx5_inst_rmw(0, + DEV2G5_MAC_ENA_CFG_RX_ENA, + devinst, + DEV2G5_MAC_ENA_CFG(0)); + } + /* 3: Disable traffic being sent to or from switch port->portno */ + spx5_rmw(0, + QFWD_SWITCH_PORT_MODE_PORT_ENA, + sparx5, + QFWD_SWITCH_PORT_MODE(port->portno)); + + /* 4: Disable dequeuing from the egress queues */ + spx5_rmw(HSCH_PORT_MODE_DEQUEUE_DIS, + HSCH_PORT_MODE_DEQUEUE_DIS, + sparx5, + HSCH_PORT_MODE(port->portno)); + + /* 5: Disable Flowcontrol */ + spx5_rmw(QSYS_PAUSE_CFG_PAUSE_STOP_SET(0xFFF - 1), + QSYS_PAUSE_CFG_PAUSE_STOP, + sparx5, + QSYS_PAUSE_CFG(port->portno)); + + spd_prm = spd == SPEED_10 ? 1000 : spd == SPEED_100 ? 100 : 10; + /* 6: Wait while the last frame is exiting the queues */ + usleep_range(8 * spd_prm, 10 * spd_prm); + + /* 7: Flush the queues accociated with the port->portno */ + spx5_rmw(HSCH_FLUSH_CTRL_FLUSH_PORT_SET(port->portno) | + HSCH_FLUSH_CTRL_FLUSH_DST_SET(1) | + HSCH_FLUSH_CTRL_FLUSH_SRC_SET(1) | + HSCH_FLUSH_CTRL_FLUSH_ENA_SET(1), + HSCH_FLUSH_CTRL_FLUSH_PORT | + HSCH_FLUSH_CTRL_FLUSH_DST | + HSCH_FLUSH_CTRL_FLUSH_SRC | + HSCH_FLUSH_CTRL_FLUSH_ENA, + sparx5, + HSCH_FLUSH_CTRL); + + /* 8: Enable dequeuing from the egress queues */ + spx5_rmw(0, + HSCH_PORT_MODE_DEQUEUE_DIS, + sparx5, + HSCH_PORT_MODE(port->portno)); + + /* 9: Wait until flushing is complete */ + err = sparx5_port_flush_poll(sparx5, port->portno); + if (err) + return err; + + /* 10: Reset the MAC clock domain */ + if (high_spd_dev) { + spx5_inst_rmw(DEV10G_DEV_RST_CTRL_PCS_TX_RST_SET(1) | + DEV10G_DEV_RST_CTRL_MAC_RX_RST_SET(1) | + DEV10G_DEV_RST_CTRL_MAC_TX_RST_SET(1), + DEV10G_DEV_RST_CTRL_PCS_TX_RST | + DEV10G_DEV_RST_CTRL_MAC_RX_RST | + DEV10G_DEV_RST_CTRL_MAC_TX_RST, + devinst, + DEV10G_DEV_RST_CTRL(0)); + + } else { + spx5_inst_rmw(DEV2G5_DEV_RST_CTRL_SPEED_SEL_SET(3) | + DEV2G5_DEV_RST_CTRL_PCS_TX_RST_SET(1) | + DEV2G5_DEV_RST_CTRL_PCS_RX_RST_SET(1) | + DEV2G5_DEV_RST_CTRL_MAC_TX_RST_SET(1) | + DEV2G5_DEV_RST_CTRL_MAC_RX_RST_SET(1), + DEV2G5_DEV_RST_CTRL_SPEED_SEL | + DEV2G5_DEV_RST_CTRL_PCS_TX_RST | + DEV2G5_DEV_RST_CTRL_PCS_RX_RST | + DEV2G5_DEV_RST_CTRL_MAC_TX_RST | + DEV2G5_DEV_RST_CTRL_MAC_RX_RST, + devinst, + DEV2G5_DEV_RST_CTRL(0)); + } + /* 11: Clear flushing */ + spx5_rmw(HSCH_FLUSH_CTRL_FLUSH_PORT_SET(port->portno) | + HSCH_FLUSH_CTRL_FLUSH_ENA_SET(0), + HSCH_FLUSH_CTRL_FLUSH_PORT | + HSCH_FLUSH_CTRL_FLUSH_ENA, + sparx5, + HSCH_FLUSH_CTRL); + + if (high_spd_dev) { + u32 pcs = sparx5_to_pcs_dev(port->portno); + void __iomem *pcsinst = spx5_inst_get(sparx5, pcs, tinst); + + /* 12: Disable 5G/10G/25 BaseR PCS */ + spx5_inst_rmw(PCS10G_BR_PCS_CFG_PCS_ENA_SET(0), + PCS10G_BR_PCS_CFG_PCS_ENA, + pcsinst, + PCS10G_BR_PCS_CFG(0)); + + if (sparx5_port_is_25g(port->portno)) + /* Disable 25G PCS */ + spx5_rmw(DEV25G_PCS25G_CFG_PCS25G_ENA_SET(0), + DEV25G_PCS25G_CFG_PCS25G_ENA, + sparx5, + DEV25G_PCS25G_CFG(tinst)); + } else { + /* 12: Disable 1G PCS */ + spx5_rmw(DEV2G5_PCS1G_CFG_PCS_ENA_SET(0), + DEV2G5_PCS1G_CFG_PCS_ENA, + sparx5, + DEV2G5_PCS1G_CFG(port->portno)); + } + + /* The port is now flushed and disabled */ + return 0; +} + +static int sparx5_port_fifo_sz(struct sparx5 *sparx5, + u32 portno, u32 speed) +{ + u32 sys_clk = sparx5_clk_period(sparx5->coreclock); + const u32 taxi_dist[SPX5_PORTS_ALL] = { + 6, 8, 10, 6, 8, 10, 6, 8, 10, 6, 8, 10, + 4, 4, 4, 4, + 11, 12, 13, 14, 15, 16, 17, 18, + 11, 12, 13, 14, 15, 16, 17, 18, + 11, 12, 13, 14, 15, 16, 17, 18, + 11, 12, 13, 14, 15, 16, 17, 18, + 4, 6, 8, 4, 6, 8, 6, 8, + 2, 2, 2, 2, 2, 2, 2, 4, 2 + }; + u32 mac_per = 6400, tmp1, tmp2, tmp3; + u32 fifo_width = 16; + u32 mac_width = 8; + u32 addition = 0; + + switch (speed) { + case SPEED_25000: + return 0; + case SPEED_10000: + mac_per = 6400; + mac_width = 8; + addition = 1; + break; + case SPEED_5000: + mac_per = 12800; + mac_width = 8; + addition = 0; + break; + case SPEED_2500: + mac_per = 3200; + mac_width = 1; + addition = 0; + break; + case SPEED_1000: + mac_per = 8000; + mac_width = 1; + addition = 0; + break; + case SPEED_100: + case SPEED_10: + return 1; + default: + break; + } + + tmp1 = 1000 * mac_width / fifo_width; + tmp2 = 3000 + ((12000 + 2 * taxi_dist[portno] * 1000) + * sys_clk / mac_per); + tmp3 = tmp1 * tmp2 / 1000; + return (tmp3 + 2000 + 999) / 1000 + addition; +} + +/* Configure port muxing: + * QSGMII: 4x2G5 devices + */ +static int sparx5_port_mux_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + u32 portno = port->portno; + u32 inst; + + if (port->conf.portmode == conf->portmode) + return 0; /* Nothing to do */ + + switch (conf->portmode) { + case PHY_INTERFACE_MODE_QSGMII: /* QSGMII: 4x2G5 devices. Mode Q' */ + inst = (portno - portno % 4) / 4; + spx5_rmw(BIT(inst), + BIT(inst), + sparx5, + PORT_CONF_QSGMII_ENA); + + if ((portno / 4 % 2) == 0) { + /* Affects d0-d3,d8-d11..d40-d43 */ + spx5_rmw(PORT_CONF_USGMII_CFG_BYPASS_SCRAM_SET(1) | + PORT_CONF_USGMII_CFG_BYPASS_DESCRAM_SET(1) | + PORT_CONF_USGMII_CFG_QUAD_MODE_SET(1), + PORT_CONF_USGMII_CFG_BYPASS_SCRAM | + PORT_CONF_USGMII_CFG_BYPASS_DESCRAM | + PORT_CONF_USGMII_CFG_QUAD_MODE, + sparx5, + PORT_CONF_USGMII_CFG((portno / 8))); + } + break; + default: + break; + } + return 0; +} + +static int sparx5_port_max_tags_set(struct sparx5 *sparx5, + struct sparx5_port *port) +{ + enum sparx5_port_max_tags max_tags = port->max_vlan_tags; + int tag_ct = max_tags == SPX5_PORT_MAX_TAGS_ONE ? 1 : + max_tags == SPX5_PORT_MAX_TAGS_TWO ? 2 : 0; + bool dtag = max_tags == SPX5_PORT_MAX_TAGS_TWO; + enum sparx5_vlan_port_type vlan_type = port->vlan_type; + bool dotag = max_tags != SPX5_PORT_MAX_TAGS_NONE; + u32 dev = sparx5_to_high_dev(port->portno); + u32 tinst = sparx5_port_dev_index(port->portno); + void __iomem *inst = spx5_inst_get(sparx5, dev, tinst); + u32 etype; + + etype = (vlan_type == SPX5_VLAN_PORT_TYPE_S_CUSTOM ? + port->custom_etype : + vlan_type == SPX5_VLAN_PORT_TYPE_C ? + SPX5_ETYPE_TAG_C : SPX5_ETYPE_TAG_S); + + spx5_wr(DEV2G5_MAC_TAGS_CFG_TAG_ID_SET(etype) | + DEV2G5_MAC_TAGS_CFG_PB_ENA_SET(dtag) | + DEV2G5_MAC_TAGS_CFG_VLAN_AWR_ENA_SET(dotag) | + DEV2G5_MAC_TAGS_CFG_VLAN_LEN_AWR_ENA_SET(dotag), + sparx5, + DEV2G5_MAC_TAGS_CFG(port->portno)); + + if (sparx5_port_is_2g5(port->portno)) + return 0; + + spx5_inst_rmw(DEV10G_MAC_TAGS_CFG_TAG_ID_SET(etype) | + DEV10G_MAC_TAGS_CFG_TAG_ENA_SET(dotag), + DEV10G_MAC_TAGS_CFG_TAG_ID | + DEV10G_MAC_TAGS_CFG_TAG_ENA, + inst, + DEV10G_MAC_TAGS_CFG(0, 0)); + + spx5_inst_rmw(DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS_SET(tag_ct), + DEV10G_MAC_NUM_TAGS_CFG_NUM_TAGS, + inst, + DEV10G_MAC_NUM_TAGS_CFG(0)); + + spx5_inst_rmw(DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK_SET(dotag), + DEV10G_MAC_MAXLEN_CFG_MAX_LEN_TAG_CHK, + inst, + DEV10G_MAC_MAXLEN_CFG(0)); + return 0; +} + +static int sparx5_port_fwd_urg(struct sparx5 *sparx5, u32 speed) +{ + u32 clk_period_ps = 1600; /* 625Mhz for now */ + u32 urg = 672000; + + switch (speed) { + case SPEED_10: + case SPEED_100: + case SPEED_1000: + urg = 672000; + break; + case SPEED_2500: + urg = 270000; + break; + case SPEED_5000: + urg = 135000; + break; + case SPEED_10000: + urg = 67200; + break; + case SPEED_25000: + urg = 27000; + break; + } + return urg / clk_period_ps - 1; +} + +static u16 sparx5_wm_enc(u16 value) +{ + if (value >= 2048) + return 2048 + value / 16; + + return value; +} + +static int sparx5_port_fc_setup(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + bool fc_obey = conf->pause & MLO_PAUSE_RX ? 1 : 0; + u32 pause_stop = 0xFFF - 1; /* FC gen disabled */ + + if (conf->pause & MLO_PAUSE_TX) + pause_stop = sparx5_wm_enc(4 * (ETH_MAXLEN / + SPX5_BUFFER_CELL_SZ)); + + /* Set HDX flowcontrol */ + spx5_rmw(DSM_MAC_CFG_HDX_BACKPREASSURE_SET(conf->duplex == DUPLEX_HALF), + DSM_MAC_CFG_HDX_BACKPREASSURE, + sparx5, + DSM_MAC_CFG(port->portno)); + + /* Obey flowcontrol */ + spx5_rmw(DSM_RX_PAUSE_CFG_RX_PAUSE_EN_SET(fc_obey), + DSM_RX_PAUSE_CFG_RX_PAUSE_EN, + sparx5, + DSM_RX_PAUSE_CFG(port->portno)); + + /* Disable forward pressure */ + spx5_rmw(QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS_SET(fc_obey), + QSYS_FWD_PRESSURE_FWD_PRESSURE_DIS, + sparx5, + QSYS_FWD_PRESSURE(port->portno)); + + /* Generate pause frames */ + spx5_rmw(QSYS_PAUSE_CFG_PAUSE_STOP_SET(pause_stop), + QSYS_PAUSE_CFG_PAUSE_STOP, + sparx5, + QSYS_PAUSE_CFG(port->portno)); + + return 0; +} + +static u16 sparx5_get_aneg_word(struct sparx5_port_config *conf) +{ + if (conf->portmode == PHY_INTERFACE_MODE_1000BASEX) /* cl-37 aneg */ + return (conf->pause_adv | ADVERTISE_LPACK | ADVERTISE_1000XFULL); + else + return 1; /* Enable SGMII Aneg */ +} + +int sparx5_serdes_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + int portmode, err, speed = conf->speed; + + if (conf->portmode == PHY_INTERFACE_MODE_QSGMII && + ((port->portno % 4) != 0)) { + return 0; + } + if (sparx5_is_baser(conf->portmode)) { + if (conf->portmode == PHY_INTERFACE_MODE_25GBASER) + speed = SPEED_25000; + else if (conf->portmode == PHY_INTERFACE_MODE_10GBASER) + speed = SPEED_10000; + else + speed = SPEED_5000; + } + + err = phy_set_media(port->serdes, conf->media); + if (err) + return err; + if (speed > 0) { + err = phy_set_speed(port->serdes, speed); + if (err) + return err; + } + if (conf->serdes_reset) { + err = phy_reset(port->serdes); + if (err) + return err; + } + + /* Configure SerDes with port parameters + * For BaseR, the serdes driver supports 10GGBASE-R and speed 5G/10G/25G + */ + portmode = conf->portmode; + if (sparx5_is_baser(conf->portmode)) + portmode = PHY_INTERFACE_MODE_10GBASER; + err = phy_set_mode_ext(port->serdes, PHY_MODE_ETHERNET, portmode); + if (err) + return err; + conf->serdes_reset = false; + return err; +} + +static int sparx5_port_pcs_low_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + bool sgmii = false, inband_aneg = false; + int err; + + if (port->conf.inband) { + if (conf->portmode == PHY_INTERFACE_MODE_SGMII || + conf->portmode == PHY_INTERFACE_MODE_QSGMII) + inband_aneg = true; /* Cisco-SGMII in-band-aneg */ + else if (conf->portmode == PHY_INTERFACE_MODE_1000BASEX && + conf->autoneg) + inband_aneg = true; /* Clause-37 in-band-aneg */ + + err = sparx5_serdes_set(sparx5, port, conf); + if (err) + return -EINVAL; + } else { + sgmii = true; /* Phy is connnected to the MAC */ + } + + /* Choose SGMII or 1000BaseX/2500BaseX PCS mode */ + spx5_rmw(DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(sgmii), + DEV2G5_PCS1G_MODE_CFG_SGMII_MODE_ENA, + sparx5, + DEV2G5_PCS1G_MODE_CFG(port->portno)); + + /* Enable PCS */ + spx5_wr(DEV2G5_PCS1G_CFG_PCS_ENA_SET(1), + sparx5, + DEV2G5_PCS1G_CFG(port->portno)); + + if (inband_aneg) { + u16 abil = sparx5_get_aneg_word(conf); + + /* Enable in-band aneg */ + spx5_wr(DEV2G5_PCS1G_ANEG_CFG_ADV_ABILITY_SET(abil) | + DEV2G5_PCS1G_ANEG_CFG_SW_RESOLVE_ENA_SET(1) | + DEV2G5_PCS1G_ANEG_CFG_ANEG_ENA_SET(1) | + DEV2G5_PCS1G_ANEG_CFG_ANEG_RESTART_ONE_SHOT_SET(1), + sparx5, + DEV2G5_PCS1G_ANEG_CFG(port->portno)); + } else { + spx5_wr(0, sparx5, DEV2G5_PCS1G_ANEG_CFG(port->portno)); + } + + /* Take PCS out of reset */ + spx5_rmw(DEV2G5_DEV_RST_CTRL_SPEED_SEL_SET(2) | + DEV2G5_DEV_RST_CTRL_PCS_TX_RST_SET(0) | + DEV2G5_DEV_RST_CTRL_PCS_RX_RST_SET(0), + DEV2G5_DEV_RST_CTRL_SPEED_SEL | + DEV2G5_DEV_RST_CTRL_PCS_TX_RST | + DEV2G5_DEV_RST_CTRL_PCS_RX_RST, + sparx5, + DEV2G5_DEV_RST_CTRL(port->portno)); + + return 0; +} + +static int sparx5_port_pcs_high_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + u32 clk_spd = conf->portmode == PHY_INTERFACE_MODE_5GBASER ? 1 : 0; + u32 pix = sparx5_port_dev_index(port->portno); + u32 dev = sparx5_to_high_dev(port->portno); + u32 pcs = sparx5_to_pcs_dev(port->portno); + void __iomem *devinst; + void __iomem *pcsinst; + int err; + + devinst = spx5_inst_get(sparx5, dev, pix); + pcsinst = spx5_inst_get(sparx5, pcs, pix); + + /* SFI : No in-band-aneg. Speeds 5G/10G/25G */ + err = sparx5_serdes_set(sparx5, port, conf); + if (err) + return -EINVAL; + if (conf->portmode == PHY_INTERFACE_MODE_25GBASER) { + /* Enable PCS for 25G device, speed 25G */ + spx5_rmw(DEV25G_PCS25G_CFG_PCS25G_ENA_SET(1), + DEV25G_PCS25G_CFG_PCS25G_ENA, + sparx5, + DEV25G_PCS25G_CFG(pix)); + } else { + /* Enable PCS for 5G/10G/25G devices, speed 5G/10G */ + spx5_inst_rmw(PCS10G_BR_PCS_CFG_PCS_ENA_SET(1), + PCS10G_BR_PCS_CFG_PCS_ENA, + pcsinst, + PCS10G_BR_PCS_CFG(0)); + } + + /* Enable 5G/10G/25G MAC module */ + spx5_inst_wr(DEV10G_MAC_ENA_CFG_RX_ENA_SET(1) | + DEV10G_MAC_ENA_CFG_TX_ENA_SET(1), + devinst, + DEV10G_MAC_ENA_CFG(0)); + + /* Take the device out of reset */ + spx5_inst_rmw(DEV10G_DEV_RST_CTRL_PCS_RX_RST_SET(0) | + DEV10G_DEV_RST_CTRL_PCS_TX_RST_SET(0) | + DEV10G_DEV_RST_CTRL_MAC_RX_RST_SET(0) | + DEV10G_DEV_RST_CTRL_MAC_TX_RST_SET(0) | + DEV10G_DEV_RST_CTRL_SPEED_SEL_SET(clk_spd), + DEV10G_DEV_RST_CTRL_PCS_RX_RST | + DEV10G_DEV_RST_CTRL_PCS_TX_RST | + DEV10G_DEV_RST_CTRL_MAC_RX_RST | + DEV10G_DEV_RST_CTRL_MAC_TX_RST | + DEV10G_DEV_RST_CTRL_SPEED_SEL, + devinst, + DEV10G_DEV_RST_CTRL(0)); + + return 0; +} + +/* Switch between 1G/2500 and 5G/10G/25G devices */ +static void sparx5_dev_switch(struct sparx5 *sparx5, int port, bool hsd) +{ + int bt_indx = BIT(sparx5_port_dev_index(port)); + + if (sparx5_port_is_5g(port)) { + spx5_rmw(hsd ? 0 : bt_indx, + bt_indx, + sparx5, + PORT_CONF_DEV5G_MODES); + } else if (sparx5_port_is_10g(port)) { + spx5_rmw(hsd ? 0 : bt_indx, + bt_indx, + sparx5, + PORT_CONF_DEV10G_MODES); + } else if (sparx5_port_is_25g(port)) { + spx5_rmw(hsd ? 0 : bt_indx, + bt_indx, + sparx5, + PORT_CONF_DEV25G_MODES); + } +} + +/* Configure speed/duplex dependent registers */ +static int sparx5_port_config_low_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + u32 clk_spd, gig_mode, tx_gap, hdx_gap_1, hdx_gap_2; + bool fdx = conf->duplex == DUPLEX_FULL; + int spd = conf->speed; + + clk_spd = spd == SPEED_10 ? 0 : spd == SPEED_100 ? 1 : 2; + gig_mode = spd == SPEED_1000 || spd == SPEED_2500; + tx_gap = spd == SPEED_1000 ? 4 : fdx ? 6 : 5; + hdx_gap_1 = spd == SPEED_1000 ? 0 : spd == SPEED_100 ? 1 : 2; + hdx_gap_2 = spd == SPEED_1000 ? 0 : spd == SPEED_100 ? 4 : 1; + + /* GIG/FDX mode */ + spx5_rmw(DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA_SET(gig_mode) | + DEV2G5_MAC_MODE_CFG_FDX_ENA_SET(fdx), + DEV2G5_MAC_MODE_CFG_GIGA_MODE_ENA | + DEV2G5_MAC_MODE_CFG_FDX_ENA, + sparx5, + DEV2G5_MAC_MODE_CFG(port->portno)); + + /* Set MAC IFG Gaps */ + spx5_wr(DEV2G5_MAC_IFG_CFG_TX_IFG_SET(tx_gap) | + DEV2G5_MAC_IFG_CFG_RX_IFG1_SET(hdx_gap_1) | + DEV2G5_MAC_IFG_CFG_RX_IFG2_SET(hdx_gap_2), + sparx5, + DEV2G5_MAC_IFG_CFG(port->portno)); + + /* Disabling frame aging when in HDX (due to HDX issue) */ + spx5_rmw(HSCH_PORT_MODE_AGE_DIS_SET(fdx == 0), + HSCH_PORT_MODE_AGE_DIS, + sparx5, + HSCH_PORT_MODE(port->portno)); + + /* Enable MAC module */ + spx5_wr(DEV2G5_MAC_ENA_CFG_RX_ENA | + DEV2G5_MAC_ENA_CFG_TX_ENA, + sparx5, + DEV2G5_MAC_ENA_CFG(port->portno)); + + /* Select speed and take MAC out of reset */ + spx5_rmw(DEV2G5_DEV_RST_CTRL_SPEED_SEL_SET(clk_spd) | + DEV2G5_DEV_RST_CTRL_MAC_TX_RST_SET(0) | + DEV2G5_DEV_RST_CTRL_MAC_RX_RST_SET(0), + DEV2G5_DEV_RST_CTRL_SPEED_SEL | + DEV2G5_DEV_RST_CTRL_MAC_TX_RST | + DEV2G5_DEV_RST_CTRL_MAC_RX_RST, + sparx5, + DEV2G5_DEV_RST_CTRL(port->portno)); + + return 0; +} + +int sparx5_port_pcs_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) + +{ + bool high_speed_dev = sparx5_is_baser(conf->portmode); + int err; + + if (sparx5_dev_change(sparx5, port, conf)) { + /* switch device */ + sparx5_dev_switch(sparx5, port->portno, high_speed_dev); + + /* Disable the not-in-use device */ + err = sparx5_port_disable(sparx5, port, !high_speed_dev); + if (err) + return err; + } + /* Disable the port before re-configuring */ + err = sparx5_port_disable(sparx5, port, high_speed_dev); + if (err) + return -EINVAL; + + if (high_speed_dev) + err = sparx5_port_pcs_high_set(sparx5, port, conf); + else + err = sparx5_port_pcs_low_set(sparx5, port, conf); + + if (err) + return -EINVAL; + + if (port->conf.inband) { + /* Enable/disable 1G counters in ASM */ + spx5_rmw(ASM_PORT_CFG_CSC_STAT_DIS_SET(high_speed_dev), + ASM_PORT_CFG_CSC_STAT_DIS, + sparx5, + ASM_PORT_CFG(port->portno)); + + /* Enable/disable 1G counters in DSM */ + spx5_rmw(DSM_BUF_CFG_CSC_STAT_DIS_SET(high_speed_dev), + DSM_BUF_CFG_CSC_STAT_DIS, + sparx5, + DSM_BUF_CFG(port->portno)); + } + + port->conf = *conf; + + return 0; +} + +int sparx5_port_config(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + bool high_speed_dev = sparx5_is_baser(conf->portmode); + int err, urgency, stop_wm; + + err = sparx5_port_verify_speed(sparx5, port, conf); + if (err) + return err; + + /* high speed device is already configured */ + if (!high_speed_dev) + sparx5_port_config_low_set(sparx5, port, conf); + + /* Configure flow control */ + err = sparx5_port_fc_setup(sparx5, port, conf); + if (err) + return err; + + /* Set the DSM stop watermark */ + stop_wm = sparx5_port_fifo_sz(sparx5, port->portno, conf->speed); + spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_SET(stop_wm), + DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM, + sparx5, + DSM_DEV_TX_STOP_WM_CFG(port->portno)); + + /* Enable port in queue system */ + urgency = sparx5_port_fwd_urg(sparx5, conf->speed); + spx5_rmw(QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(1) | + QFWD_SWITCH_PORT_MODE_FWD_URGENCY_SET(urgency), + QFWD_SWITCH_PORT_MODE_PORT_ENA | + QFWD_SWITCH_PORT_MODE_FWD_URGENCY, + sparx5, + QFWD_SWITCH_PORT_MODE(port->portno)); + + /* Save the new values */ + port->conf = *conf; + + return 0; +} + +/* Initialize port config to default */ +int sparx5_port_init(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf) +{ + u32 pause_start = sparx5_wm_enc(6 * (ETH_MAXLEN / SPX5_BUFFER_CELL_SZ)); + u32 atop = sparx5_wm_enc(20 * (ETH_MAXLEN / SPX5_BUFFER_CELL_SZ)); + u32 devhigh = sparx5_to_high_dev(port->portno); + u32 pix = sparx5_port_dev_index(port->portno); + u32 pcs = sparx5_to_pcs_dev(port->portno); + bool sd_pol = port->signd_active_high; + bool sd_sel = !port->signd_internal; + bool sd_ena = port->signd_enable; + u32 pause_stop = 0xFFF - 1; /* FC generate disabled */ + void __iomem *devinst; + void __iomem *pcsinst; + int err; + + devinst = spx5_inst_get(sparx5, devhigh, pix); + pcsinst = spx5_inst_get(sparx5, pcs, pix); + + /* Set the mux port mode */ + err = sparx5_port_mux_set(sparx5, port, conf); + if (err) + return err; + + /* Configure MAC vlan awareness */ + err = sparx5_port_max_tags_set(sparx5, port); + if (err) + return err; + + /* Set Max Length */ + spx5_rmw(DEV2G5_MAC_MAXLEN_CFG_MAX_LEN_SET(ETH_MAXLEN), + DEV2G5_MAC_MAXLEN_CFG_MAX_LEN, + sparx5, + DEV2G5_MAC_MAXLEN_CFG(port->portno)); + + /* 1G/2G5: Signal Detect configuration */ + spx5_wr(DEV2G5_PCS1G_SD_CFG_SD_POL_SET(sd_pol) | + DEV2G5_PCS1G_SD_CFG_SD_SEL_SET(sd_sel) | + DEV2G5_PCS1G_SD_CFG_SD_ENA_SET(sd_ena), + sparx5, + DEV2G5_PCS1G_SD_CFG(port->portno)); + + /* Set Pause WM hysteresis */ + spx5_rmw(QSYS_PAUSE_CFG_PAUSE_START_SET(pause_start) | + QSYS_PAUSE_CFG_PAUSE_STOP_SET(pause_stop) | + QSYS_PAUSE_CFG_PAUSE_ENA_SET(1), + QSYS_PAUSE_CFG_PAUSE_START | + QSYS_PAUSE_CFG_PAUSE_STOP | + QSYS_PAUSE_CFG_PAUSE_ENA, + sparx5, + QSYS_PAUSE_CFG(port->portno)); + + /* Port ATOP. Frames are tail dropped when this WM is hit */ + spx5_wr(QSYS_ATOP_ATOP_SET(atop), + sparx5, + QSYS_ATOP(port->portno)); + + /* Discard pause frame 01-80-C2-00-00-01 */ + spx5_wr(PAUSE_DISCARD, sparx5, ANA_CL_CAPTURE_BPDU_CFG(port->portno)); + + if (conf->portmode == PHY_INTERFACE_MODE_QSGMII || + conf->portmode == PHY_INTERFACE_MODE_SGMII) { + err = sparx5_serdes_set(sparx5, port, conf); + if (err) + return err; + + if (!sparx5_port_is_2g5(port->portno)) + /* Enable shadow device */ + spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA_SET(1), + DSM_DEV_TX_STOP_WM_CFG_DEV10G_SHADOW_ENA, + sparx5, + DSM_DEV_TX_STOP_WM_CFG(port->portno)); + + sparx5_dev_switch(sparx5, port->portno, false); + } + if (conf->portmode == PHY_INTERFACE_MODE_QSGMII) { + // All ports must be PCS enabled in QSGMII mode + spx5_rmw(DEV2G5_DEV_RST_CTRL_PCS_TX_RST_SET(0), + DEV2G5_DEV_RST_CTRL_PCS_TX_RST, + sparx5, + DEV2G5_DEV_RST_CTRL(port->portno)); + } + /* Default IFGs for 1G */ + spx5_wr(DEV2G5_MAC_IFG_CFG_TX_IFG_SET(6) | + DEV2G5_MAC_IFG_CFG_RX_IFG1_SET(0) | + DEV2G5_MAC_IFG_CFG_RX_IFG2_SET(0), + sparx5, + DEV2G5_MAC_IFG_CFG(port->portno)); + + if (sparx5_port_is_2g5(port->portno)) + return 0; /* Low speed device only - return */ + + /* Now setup the high speed device */ + if (conf->portmode == PHY_INTERFACE_MODE_NA) + conf->portmode = PHY_INTERFACE_MODE_10GBASER; + + if (sparx5_is_baser(conf->portmode)) + sparx5_dev_switch(sparx5, port->portno, true); + + /* Set Max Length */ + spx5_inst_rmw(DEV10G_MAC_MAXLEN_CFG_MAX_LEN_SET(ETH_MAXLEN), + DEV10G_MAC_MAXLEN_CFG_MAX_LEN, + devinst, + DEV10G_MAC_ENA_CFG(0)); + + /* Handle Signal Detect in 10G PCS */ + spx5_inst_wr(PCS10G_BR_PCS_SD_CFG_SD_POL_SET(sd_pol) | + PCS10G_BR_PCS_SD_CFG_SD_SEL_SET(sd_sel) | + PCS10G_BR_PCS_SD_CFG_SD_ENA_SET(sd_ena), + pcsinst, + PCS10G_BR_PCS_SD_CFG(0)); + + if (sparx5_port_is_25g(port->portno)) { + /* Handle Signal Detect in 25G PCS */ + spx5_wr(DEV25G_PCS25G_SD_CFG_SD_POL_SET(sd_pol) | + DEV25G_PCS25G_SD_CFG_SD_SEL_SET(sd_sel) | + DEV25G_PCS25G_SD_CFG_SD_ENA_SET(sd_ena), + sparx5, + DEV25G_PCS25G_SD_CFG(pix)); + } + + return 0; +} + +void sparx5_port_enable(struct sparx5_port *port, bool enable) +{ + struct sparx5 *sparx5 = port->sparx5; + + /* Enable port for frame transfer? */ + spx5_rmw(QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(enable), + QFWD_SWITCH_PORT_MODE_PORT_ENA, + sparx5, + QFWD_SWITCH_PORT_MODE(port->portno)); +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_port.h b/drivers/net/ethernet/microchip/sparx5/sparx5_port.h new file mode 100644 index 000000000000..fd05ab6436d1 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_port.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#ifndef __SPARX5_PORT_H__ +#define __SPARX5_PORT_H__ + +#include "sparx5_main.h" + +static inline bool sparx5_port_is_2g5(int portno) +{ + return portno >= 16 && portno <= 47; +} + +static inline bool sparx5_port_is_5g(int portno) +{ + return portno <= 11 || portno == 64; +} + +static inline bool sparx5_port_is_10g(int portno) +{ + return (portno >= 12 && portno <= 15) || (portno >= 48 && portno <= 55); +} + +static inline bool sparx5_port_is_25g(int portno) +{ + return portno >= 56 && portno <= 63; +} + +static inline u32 sparx5_to_high_dev(int port) +{ + if (sparx5_port_is_5g(port)) + return TARGET_DEV5G; + if (sparx5_port_is_10g(port)) + return TARGET_DEV10G; + return TARGET_DEV25G; +} + +static inline u32 sparx5_to_pcs_dev(int port) +{ + if (sparx5_port_is_5g(port)) + return TARGET_PCS5G_BR; + if (sparx5_port_is_10g(port)) + return TARGET_PCS10G_BR; + return TARGET_PCS25G_BR; +} + +static inline int sparx5_port_dev_index(int port) +{ + if (sparx5_port_is_2g5(port)) + return port; + if (sparx5_port_is_5g(port)) + return (port <= 11 ? port : 12); + if (sparx5_port_is_10g(port)) + return (port >= 12 && port <= 15) ? + port - 12 : port - 44; + return (port - 56); +} + +int sparx5_port_init(struct sparx5 *sparx5, + struct sparx5_port *spx5_port, + struct sparx5_port_config *conf); + +int sparx5_port_config(struct sparx5 *sparx5, + struct sparx5_port *spx5_port, + struct sparx5_port_config *conf); + +int sparx5_port_pcs_set(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_config *conf); + +int sparx5_serdes_set(struct sparx5 *sparx5, + struct sparx5_port *spx5_port, + struct sparx5_port_config *conf); + +struct sparx5_port_status { + bool link; + bool link_down; + int speed; + bool an_complete; + int duplex; + int pause; +}; + +int sparx5_get_port_status(struct sparx5 *sparx5, + struct sparx5_port *port, + struct sparx5_port_status *status); + +void sparx5_port_enable(struct sparx5_port *port, bool enable); + +#endif /* __SPARX5_PORT_H__ */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c new file mode 100644 index 000000000000..a72e3b3b596e --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include <linux/if_bridge.h> +#include <net/switchdev.h> + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" + +static struct workqueue_struct *sparx5_owq; + +struct sparx5_switchdev_event_work { + struct work_struct work; + struct switchdev_notifier_fdb_info fdb_info; + struct net_device *dev; + unsigned long event; +}; + +static void sparx5_port_attr_bridge_flags(struct sparx5_port *port, + struct switchdev_brport_flags flags) +{ + if (flags.mask & BR_MCAST_FLOOD) + sparx5_pgid_update_mask(port, PGID_MC_FLOOD, true); +} + +static void sparx5_attr_stp_state_set(struct sparx5_port *port, + u8 state) +{ + struct sparx5 *sparx5 = port->sparx5; + + if (!test_bit(port->portno, sparx5->bridge_mask)) { + netdev_err(port->ndev, + "Controlling non-bridged port %d?\n", port->portno); + return; + } + + switch (state) { + case BR_STATE_FORWARDING: + set_bit(port->portno, sparx5->bridge_fwd_mask); + fallthrough; + case BR_STATE_LEARNING: + set_bit(port->portno, sparx5->bridge_lrn_mask); + break; + + default: + /* All other states treated as blocking */ + clear_bit(port->portno, sparx5->bridge_fwd_mask); + clear_bit(port->portno, sparx5->bridge_lrn_mask); + break; + } + + /* apply the bridge_fwd_mask to all the ports */ + sparx5_update_fwd(sparx5); +} + +static void sparx5_port_attr_ageing_set(struct sparx5_port *port, + unsigned long ageing_clock_t) +{ + unsigned long ageing_jiffies = clock_t_to_jiffies(ageing_clock_t); + u32 ageing_time = jiffies_to_msecs(ageing_jiffies); + + sparx5_set_ageing(port->sparx5, ageing_time); +} + +static int sparx5_port_attr_set(struct net_device *dev, const void *ctx, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack) +{ + struct sparx5_port *port = netdev_priv(dev); + + switch (attr->id) { + case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS: + sparx5_port_attr_bridge_flags(port, attr->u.brport_flags); + break; + case SWITCHDEV_ATTR_ID_PORT_STP_STATE: + sparx5_attr_stp_state_set(port, attr->u.stp_state); + break; + case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME: + sparx5_port_attr_ageing_set(port, attr->u.ageing_time); + break; + case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING: + port->vlan_aware = attr->u.vlan_filtering; + sparx5_vlan_port_apply(port->sparx5, port); + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static int sparx5_port_bridge_join(struct sparx5_port *port, + struct net_device *bridge) +{ + struct sparx5 *sparx5 = port->sparx5; + + if (bitmap_empty(sparx5->bridge_mask, SPX5_PORTS)) + /* First bridged port */ + sparx5->hw_bridge_dev = bridge; + else + if (sparx5->hw_bridge_dev != bridge) + /* This is adding the port to a second bridge, this is + * unsupported + */ + return -ENODEV; + + set_bit(port->portno, sparx5->bridge_mask); + + /* Port enters in bridge mode therefor don't need to copy to CPU + * frames for multicast in case the bridge is not requesting them + */ + __dev_mc_unsync(port->ndev, sparx5_mc_unsync); + + return 0; +} + +static void sparx5_port_bridge_leave(struct sparx5_port *port, + struct net_device *bridge) +{ + struct sparx5 *sparx5 = port->sparx5; + + clear_bit(port->portno, sparx5->bridge_mask); + if (bitmap_empty(sparx5->bridge_mask, SPX5_PORTS)) + sparx5->hw_bridge_dev = NULL; + + /* Clear bridge vlan settings before updating the port settings */ + port->vlan_aware = 0; + port->pvid = NULL_VID; + port->vid = NULL_VID; + + /* Port enters in host more therefore restore mc list */ + __dev_mc_sync(port->ndev, sparx5_mc_sync, sparx5_mc_unsync); +} + +static int sparx5_port_changeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info) +{ + struct sparx5_port *port = netdev_priv(dev); + int err = 0; + + if (netif_is_bridge_master(info->upper_dev)) { + if (info->linking) + err = sparx5_port_bridge_join(port, info->upper_dev); + else + sparx5_port_bridge_leave(port, info->upper_dev); + + sparx5_vlan_port_apply(port->sparx5, port); + } + + return err; +} + +static int sparx5_port_add_addr(struct net_device *dev, bool up) +{ + struct sparx5_port *port = netdev_priv(dev); + struct sparx5 *sparx5 = port->sparx5; + u16 vid = port->pvid; + + if (up) + sparx5_mact_learn(sparx5, PGID_CPU, port->ndev->dev_addr, vid); + else + sparx5_mact_forget(sparx5, port->ndev->dev_addr, vid); + + return 0; +} + +static int sparx5_netdevice_port_event(struct net_device *dev, + struct notifier_block *nb, + unsigned long event, void *ptr) +{ + int err = 0; + + if (!sparx5_netdevice_check(dev)) + return 0; + + switch (event) { + case NETDEV_CHANGEUPPER: + err = sparx5_port_changeupper(dev, ptr); + break; + case NETDEV_PRE_UP: + err = sparx5_port_add_addr(dev, true); + break; + case NETDEV_DOWN: + err = sparx5_port_add_addr(dev, false); + break; + } + + return err; +} + +static int sparx5_netdevice_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + int ret = 0; + + ret = sparx5_netdevice_port_event(dev, nb, event, ptr); + + return notifier_from_errno(ret); +} + +static void sparx5_switchdev_bridge_fdb_event_work(struct work_struct *work) +{ + struct sparx5_switchdev_event_work *switchdev_work = + container_of(work, struct sparx5_switchdev_event_work, work); + struct net_device *dev = switchdev_work->dev; + struct switchdev_notifier_fdb_info *fdb_info; + struct sparx5_port *port; + struct sparx5 *sparx5; + + rtnl_lock(); + if (!sparx5_netdevice_check(dev)) + goto out; + + port = netdev_priv(dev); + sparx5 = port->sparx5; + + fdb_info = &switchdev_work->fdb_info; + + switch (switchdev_work->event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + if (!fdb_info->added_by_user) + break; + sparx5_add_mact_entry(sparx5, port, fdb_info->addr, + fdb_info->vid); + break; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + if (!fdb_info->added_by_user) + break; + sparx5_del_mact_entry(sparx5, fdb_info->addr, fdb_info->vid); + break; + } + +out: + rtnl_unlock(); + kfree(switchdev_work->fdb_info.addr); + kfree(switchdev_work); + dev_put(dev); +} + +static void sparx5_schedule_work(struct work_struct *work) +{ + queue_work(sparx5_owq, work); +} + +static int sparx5_switchdev_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + struct sparx5_switchdev_event_work *switchdev_work; + struct switchdev_notifier_fdb_info *fdb_info; + struct switchdev_notifier_info *info = ptr; + int err; + + switch (event) { + case SWITCHDEV_PORT_ATTR_SET: + err = switchdev_handle_port_attr_set(dev, ptr, + sparx5_netdevice_check, + sparx5_port_attr_set); + return notifier_from_errno(err); + case SWITCHDEV_FDB_ADD_TO_DEVICE: + fallthrough; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + switchdev_work = kzalloc(sizeof(*switchdev_work), GFP_ATOMIC); + if (!switchdev_work) + return NOTIFY_BAD; + + switchdev_work->dev = dev; + switchdev_work->event = event; + + fdb_info = container_of(info, + struct switchdev_notifier_fdb_info, + info); + INIT_WORK(&switchdev_work->work, + sparx5_switchdev_bridge_fdb_event_work); + memcpy(&switchdev_work->fdb_info, ptr, + sizeof(switchdev_work->fdb_info)); + switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC); + if (!switchdev_work->fdb_info.addr) + goto err_addr_alloc; + + ether_addr_copy((u8 *)switchdev_work->fdb_info.addr, + fdb_info->addr); + dev_hold(dev); + + sparx5_schedule_work(&switchdev_work->work); + break; + } + + return NOTIFY_DONE; +err_addr_alloc: + kfree(switchdev_work); + return NOTIFY_BAD; +} + +static void sparx5_sync_port_dev_addr(struct sparx5 *sparx5, + struct sparx5_port *port, + u16 vid, bool add) +{ + if (!port || + !test_bit(port->portno, sparx5->bridge_mask)) + return; /* Skip null/host interfaces */ + + /* Bridge connects to vid? */ + if (add) { + /* Add port MAC address from the VLAN */ + sparx5_mact_learn(sparx5, PGID_CPU, + port->ndev->dev_addr, vid); + } else { + /* Control port addr visibility depending on + * port VLAN connectivity. + */ + if (test_bit(port->portno, sparx5->vlan_mask[vid])) + sparx5_mact_learn(sparx5, PGID_CPU, + port->ndev->dev_addr, vid); + else + sparx5_mact_forget(sparx5, + port->ndev->dev_addr, vid); + } +} + +static void sparx5_sync_bridge_dev_addr(struct net_device *dev, + struct sparx5 *sparx5, + u16 vid, bool add) +{ + int i; + + /* First, handle bridge address'es */ + if (add) { + sparx5_mact_learn(sparx5, PGID_CPU, dev->dev_addr, + vid); + sparx5_mact_learn(sparx5, PGID_BCAST, dev->broadcast, + vid); + } else { + sparx5_mact_forget(sparx5, dev->dev_addr, vid); + sparx5_mact_forget(sparx5, dev->broadcast, vid); + } + + /* Now look at bridged ports */ + for (i = 0; i < SPX5_PORTS; i++) + sparx5_sync_port_dev_addr(sparx5, sparx5->ports[i], vid, add); +} + +static int sparx5_handle_port_vlan_add(struct net_device *dev, + struct notifier_block *nb, + const struct switchdev_obj_port_vlan *v) +{ + struct sparx5_port *port = netdev_priv(dev); + + if (netif_is_bridge_master(dev)) { + if (v->flags & BRIDGE_VLAN_INFO_BRENTRY) { + struct sparx5 *sparx5 = + container_of(nb, struct sparx5, + switchdev_blocking_nb); + + sparx5_sync_bridge_dev_addr(dev, sparx5, v->vid, true); + } + return 0; + } + + if (!sparx5_netdevice_check(dev)) + return -EOPNOTSUPP; + + return sparx5_vlan_vid_add(port, v->vid, + v->flags & BRIDGE_VLAN_INFO_PVID, + v->flags & BRIDGE_VLAN_INFO_UNTAGGED); +} + +static int sparx5_handle_port_obj_add(struct net_device *dev, + struct notifier_block *nb, + struct switchdev_notifier_port_obj_info *info) +{ + const struct switchdev_obj *obj = info->obj; + int err; + + switch (obj->id) { + case SWITCHDEV_OBJ_ID_PORT_VLAN: + err = sparx5_handle_port_vlan_add(dev, nb, + SWITCHDEV_OBJ_PORT_VLAN(obj)); + break; + default: + err = -EOPNOTSUPP; + break; + } + + info->handled = true; + return err; +} + +static int sparx5_handle_port_vlan_del(struct net_device *dev, + struct notifier_block *nb, + u16 vid) +{ + struct sparx5_port *port = netdev_priv(dev); + int ret; + + /* Master bridge? */ + if (netif_is_bridge_master(dev)) { + struct sparx5 *sparx5 = + container_of(nb, struct sparx5, + switchdev_blocking_nb); + + sparx5_sync_bridge_dev_addr(dev, sparx5, vid, false); + return 0; + } + + if (!sparx5_netdevice_check(dev)) + return -EOPNOTSUPP; + + ret = sparx5_vlan_vid_del(port, vid); + if (ret) + return ret; + + /* Delete the port MAC address with the matching VLAN information */ + sparx5_mact_forget(port->sparx5, port->ndev->dev_addr, vid); + + return 0; +} + +static int sparx5_handle_port_obj_del(struct net_device *dev, + struct notifier_block *nb, + struct switchdev_notifier_port_obj_info *info) +{ + const struct switchdev_obj *obj = info->obj; + int err; + + switch (obj->id) { + case SWITCHDEV_OBJ_ID_PORT_VLAN: + err = sparx5_handle_port_vlan_del(dev, nb, + SWITCHDEV_OBJ_PORT_VLAN(obj)->vid); + break; + default: + err = -EOPNOTSUPP; + break; + } + + info->handled = true; + return err; +} + +static int sparx5_switchdev_blocking_event(struct notifier_block *nb, + unsigned long event, + void *ptr) +{ + struct net_device *dev = switchdev_notifier_info_to_dev(ptr); + int err; + + switch (event) { + case SWITCHDEV_PORT_OBJ_ADD: + err = sparx5_handle_port_obj_add(dev, nb, ptr); + return notifier_from_errno(err); + case SWITCHDEV_PORT_OBJ_DEL: + err = sparx5_handle_port_obj_del(dev, nb, ptr); + return notifier_from_errno(err); + case SWITCHDEV_PORT_ATTR_SET: + err = switchdev_handle_port_attr_set(dev, ptr, + sparx5_netdevice_check, + sparx5_port_attr_set); + return notifier_from_errno(err); + } + + return NOTIFY_DONE; +} + +int sparx5_register_notifier_blocks(struct sparx5 *s5) +{ + int err; + + s5->netdevice_nb.notifier_call = sparx5_netdevice_event; + err = register_netdevice_notifier(&s5->netdevice_nb); + if (err) + return err; + + s5->switchdev_nb.notifier_call = sparx5_switchdev_event; + err = register_switchdev_notifier(&s5->switchdev_nb); + if (err) + goto err_switchdev_nb; + + s5->switchdev_blocking_nb.notifier_call = sparx5_switchdev_blocking_event; + err = register_switchdev_blocking_notifier(&s5->switchdev_blocking_nb); + if (err) + goto err_switchdev_blocking_nb; + + sparx5_owq = alloc_ordered_workqueue("sparx5_order", 0); + if (!sparx5_owq) { + err = -ENOMEM; + goto err_switchdev_blocking_nb; + } + + return 0; + +err_switchdev_blocking_nb: + unregister_switchdev_notifier(&s5->switchdev_nb); +err_switchdev_nb: + unregister_netdevice_notifier(&s5->netdevice_nb); + + return err; +} + +void sparx5_unregister_notifier_blocks(struct sparx5 *s5) +{ + destroy_workqueue(sparx5_owq); + + unregister_switchdev_blocking_notifier(&s5->switchdev_blocking_nb); + unregister_switchdev_notifier(&s5->switchdev_nb); + unregister_netdevice_notifier(&s5->netdevice_nb); +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c new file mode 100644 index 000000000000..4ce490a25f33 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries. + */ + +#include "sparx5_main_regs.h" +#include "sparx5_main.h" + +static int sparx5_vlant_set_mask(struct sparx5 *sparx5, u16 vid) +{ + u32 mask[3]; + + /* Divide up mask in 32 bit words */ + bitmap_to_arr32(mask, sparx5->vlan_mask[vid], SPX5_PORTS); + + /* Output mask to respective registers */ + spx5_wr(mask[0], sparx5, ANA_L3_VLAN_MASK_CFG(vid)); + spx5_wr(mask[1], sparx5, ANA_L3_VLAN_MASK_CFG1(vid)); + spx5_wr(mask[2], sparx5, ANA_L3_VLAN_MASK_CFG2(vid)); + + return 0; +} + +void sparx5_vlan_init(struct sparx5 *sparx5) +{ + u16 vid; + + spx5_rmw(ANA_L3_VLAN_CTRL_VLAN_ENA_SET(1), + ANA_L3_VLAN_CTRL_VLAN_ENA, + sparx5, + ANA_L3_VLAN_CTRL); + + /* Map VLAN = FID */ + for (vid = NULL_VID; vid < VLAN_N_VID; vid++) + spx5_rmw(ANA_L3_VLAN_CFG_VLAN_FID_SET(vid), + ANA_L3_VLAN_CFG_VLAN_FID, + sparx5, + ANA_L3_VLAN_CFG(vid)); +} + +void sparx5_vlan_port_setup(struct sparx5 *sparx5, int portno) +{ + struct sparx5_port *port = sparx5->ports[portno]; + + /* Configure PVID */ + spx5_rmw(ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA_SET(0) | + ANA_CL_VLAN_CTRL_PORT_VID_SET(port->pvid), + ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA | + ANA_CL_VLAN_CTRL_PORT_VID, + sparx5, + ANA_CL_VLAN_CTRL(port->portno)); +} + +int sparx5_vlan_vid_add(struct sparx5_port *port, u16 vid, bool pvid, + bool untagged) +{ + struct sparx5 *sparx5 = port->sparx5; + int ret; + + /* Make the port a member of the VLAN */ + set_bit(port->portno, sparx5->vlan_mask[vid]); + ret = sparx5_vlant_set_mask(sparx5, vid); + if (ret) + return ret; + + /* Default ingress vlan classification */ + if (pvid) + port->pvid = vid; + + /* Untagged egress vlan classification */ + if (untagged && port->vid != vid) { + if (port->vid) { + netdev_err(port->ndev, + "Port already has a native VLAN: %d\n", + port->vid); + return -EBUSY; + } + port->vid = vid; + } + + sparx5_vlan_port_apply(sparx5, port); + + return 0; +} + +int sparx5_vlan_vid_del(struct sparx5_port *port, u16 vid) +{ + struct sparx5 *sparx5 = port->sparx5; + int ret; + + /* 8021q removes VID 0 on module unload for all interfaces + * with VLAN filtering feature. We need to keep it to receive + * untagged traffic. + */ + if (vid == 0) + return 0; + + /* Stop the port from being a member of the vlan */ + clear_bit(port->portno, sparx5->vlan_mask[vid]); + ret = sparx5_vlant_set_mask(sparx5, vid); + if (ret) + return ret; + + /* Ingress */ + if (port->pvid == vid) + port->pvid = 0; + + /* Egress */ + if (port->vid == vid) + port->vid = 0; + + sparx5_vlan_port_apply(sparx5, port); + + return 0; +} + +void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable) +{ + struct sparx5 *sparx5 = port->sparx5; + u32 val, mask; + + /* mask is spread across 3 registers x 32 bit */ + if (port->portno < 32) { + mask = BIT(port->portno); + val = enable ? mask : 0; + spx5_rmw(val, mask, sparx5, ANA_AC_PGID_CFG(pgid)); + } else if (port->portno < 64) { + mask = BIT(port->portno - 32); + val = enable ? mask : 0; + spx5_rmw(val, mask, sparx5, ANA_AC_PGID_CFG1(pgid)); + } else if (port->portno < SPX5_PORTS) { + mask = BIT(port->portno - 64); + val = enable ? mask : 0; + spx5_rmw(val, mask, sparx5, ANA_AC_PGID_CFG2(pgid)); + } else { + netdev_err(port->ndev, "Invalid port no: %d\n", port->portno); + } +} + +void sparx5_update_fwd(struct sparx5 *sparx5) +{ + DECLARE_BITMAP(workmask, SPX5_PORTS); + u32 mask[3]; + int port; + + /* Divide up fwd mask in 32 bit words */ + bitmap_to_arr32(mask, sparx5->bridge_fwd_mask, SPX5_PORTS); + + /* Update flood masks */ + for (port = PGID_UC_FLOOD; port <= PGID_BCAST; port++) { + spx5_wr(mask[0], sparx5, ANA_AC_PGID_CFG(port)); + spx5_wr(mask[1], sparx5, ANA_AC_PGID_CFG1(port)); + spx5_wr(mask[2], sparx5, ANA_AC_PGID_CFG2(port)); + } + + /* Update SRC masks */ + for (port = 0; port < SPX5_PORTS; port++) { + if (test_bit(port, sparx5->bridge_fwd_mask)) { + /* Allow to send to all bridged but self */ + bitmap_copy(workmask, sparx5->bridge_fwd_mask, SPX5_PORTS); + clear_bit(port, workmask); + bitmap_to_arr32(mask, workmask, SPX5_PORTS); + spx5_wr(mask[0], sparx5, ANA_AC_SRC_CFG(port)); + spx5_wr(mask[1], sparx5, ANA_AC_SRC_CFG1(port)); + spx5_wr(mask[2], sparx5, ANA_AC_SRC_CFG2(port)); + } else { + spx5_wr(0, sparx5, ANA_AC_SRC_CFG(port)); + spx5_wr(0, sparx5, ANA_AC_SRC_CFG1(port)); + spx5_wr(0, sparx5, ANA_AC_SRC_CFG2(port)); + } + } + + /* Learning enabled only for bridged ports */ + bitmap_and(workmask, sparx5->bridge_fwd_mask, + sparx5->bridge_lrn_mask, SPX5_PORTS); + bitmap_to_arr32(mask, workmask, SPX5_PORTS); + + /* Apply learning mask */ + spx5_wr(mask[0], sparx5, ANA_L2_AUTO_LRN_CFG); + spx5_wr(mask[1], sparx5, ANA_L2_AUTO_LRN_CFG1); + spx5_wr(mask[2], sparx5, ANA_L2_AUTO_LRN_CFG2); +} + +void sparx5_vlan_port_apply(struct sparx5 *sparx5, + struct sparx5_port *port) + +{ + u32 val; + + /* Configure PVID, vlan aware */ + val = ANA_CL_VLAN_CTRL_VLAN_AWARE_ENA_SET(port->vlan_aware) | + ANA_CL_VLAN_CTRL_VLAN_POP_CNT_SET(port->vlan_aware) | + ANA_CL_VLAN_CTRL_PORT_VID_SET(port->pvid); + spx5_wr(val, sparx5, ANA_CL_VLAN_CTRL(port->portno)); + + val = 0; + if (port->vlan_aware && !port->pvid) + /* If port is vlan-aware and tagged, drop untagged and + * priority tagged frames. + */ + val = ANA_CL_VLAN_FILTER_CTRL_TAG_REQUIRED_ENA_SET(1) | + ANA_CL_VLAN_FILTER_CTRL_PRIO_CTAG_DIS_SET(1) | + ANA_CL_VLAN_FILTER_CTRL_PRIO_STAG_DIS_SET(1); + spx5_wr(val, sparx5, + ANA_CL_VLAN_FILTER_CTRL(port->portno, 0)); + + /* Egress configuration (REW_TAG_CFG): VLAN tag type to 8021Q */ + val = REW_TAG_CTRL_TAG_TPID_CFG_SET(0); + if (port->vlan_aware) { + if (port->vid) + /* Tag all frames except when VID == DEFAULT_VLAN */ + val |= REW_TAG_CTRL_TAG_CFG_SET(1); + else + val |= REW_TAG_CTRL_TAG_CFG_SET(3); + } + spx5_wr(val, sparx5, REW_TAG_CTRL(port->portno)); + + /* Egress VID */ + spx5_rmw(REW_PORT_VLAN_CFG_PORT_VID_SET(port->vid), + REW_PORT_VLAN_CFG_PORT_VID, + sparx5, + REW_PORT_VLAN_CFG(port->portno)); +} diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 04d067243457..fff78900fc8a 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1230,8 +1230,10 @@ static int mana_create_txq(struct mana_port_context *apc, cq->gdma_id = cq->gdma_cq->id; - if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) - return -EINVAL; + if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { + err = -EINVAL; + goto out; + } gc->cq_table[cq->gdma_id] = cq->gdma_cq; @@ -1387,8 +1389,7 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, gc = gd->gdma_context; - rxq = kzalloc(sizeof(*rxq) + - RX_BUFFERS_PER_QUEUE * sizeof(struct mana_recv_buf_oob), + rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE), GFP_KERNEL); if (!rxq) return NULL; diff --git a/drivers/net/ethernet/moxa/moxart_ether.c b/drivers/net/ethernet/moxa/moxart_ether.c index b85733942053..5249b64f4fc5 100644 --- a/drivers/net/ethernet/moxa/moxart_ether.c +++ b/drivers/net/ethernet/moxa/moxart_ether.c @@ -481,13 +481,12 @@ static int moxart_mac_probe(struct platform_device *pdev) priv->ndev = ndev; priv->pdev = pdev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ndev->base_addr = res->start; - priv->base = devm_ioremap_resource(p_dev, res); + priv->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(priv->base)) { ret = PTR_ERR(priv->base); goto init_fail; } + ndev->base_addr = res->start; spin_lock_init(&priv->txlock); diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 0c4283319d7f..adfb9781799e 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -379,6 +379,7 @@ static u32 ocelot_read_eq_avail(struct ocelot *ocelot, int port) int ocelot_port_flush(struct ocelot *ocelot, int port) { + unsigned int pause_ena; int err, val; /* Disable dequeuing from the egress queues */ @@ -387,6 +388,7 @@ int ocelot_port_flush(struct ocelot *ocelot, int port) QSYS_PORT_MODE, port); /* Disable flow control */ + ocelot_fields_read(ocelot, port, SYS_PAUSE_CFG_PAUSE_ENA, &pause_ena); ocelot_fields_write(ocelot, port, SYS_PAUSE_CFG_PAUSE_ENA, 0); /* Disable priority flow control */ @@ -422,6 +424,9 @@ int ocelot_port_flush(struct ocelot *ocelot, int port) /* Clear flushing again. */ ocelot_rmw_gix(ocelot, 0, REW_PORT_CFG_FLUSH_ENA, REW_PORT_CFG, port); + /* Re-enable flow control */ + ocelot_fields_write(ocelot, port, SYS_PAUSE_CFG_PAUSE_ENA, pause_ena); + return err; } EXPORT_SYMBOL(ocelot_port_flush); diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index aad33d22c33f..3e89e34f86d5 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -939,7 +939,7 @@ static void ocelot_port_attr_mc_set(struct ocelot *ocelot, int port, bool mc) ANA_PORT_CPU_FWD_CFG, port); } -static int ocelot_port_attr_set(struct net_device *dev, +static int ocelot_port_attr_set(struct net_device *dev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) { @@ -948,6 +948,9 @@ static int ocelot_port_attr_set(struct net_device *dev, int port = priv->chip_port; int err = 0; + if (ctx && ctx != priv) + return 0; + switch (attr->id) { case SWITCHDEV_ATTR_ID_PORT_STP_STATE: ocelot_port_attr_stp_state_set(ocelot, port, attr->u.stp_state); @@ -1058,12 +1061,16 @@ ocelot_port_obj_mrp_del_ring_role(struct net_device *dev, return ocelot_mrp_del_ring_role(ocelot, port, mrp); } -static int ocelot_port_obj_add(struct net_device *dev, +static int ocelot_port_obj_add(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) { + struct ocelot_port_private *priv = netdev_priv(dev); int ret = 0; + if (ctx && ctx != priv) + return 0; + switch (obj->id) { case SWITCHDEV_OBJ_ID_PORT_VLAN: ret = ocelot_port_obj_add_vlan(dev, @@ -1086,11 +1093,15 @@ static int ocelot_port_obj_add(struct net_device *dev, return ret; } -static int ocelot_port_obj_del(struct net_device *dev, +static int ocelot_port_obj_del(struct net_device *dev, const void *ctx, const struct switchdev_obj *obj) { + struct ocelot_port_private *priv = netdev_priv(dev); int ret = 0; + if (ctx && ctx != priv) + return 0; + switch (obj->id) { case SWITCHDEV_OBJ_ID_PORT_VLAN: ret = ocelot_vlan_vid_del(dev, @@ -1143,10 +1154,14 @@ static int ocelot_switchdev_sync(struct ocelot *ocelot, int port, struct net_device *bridge_dev, struct netlink_ext_ack *extack) { + struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct ocelot_port_private *priv; clock_t ageing_time; u8 stp_state; int err; + priv = container_of(ocelot_port, struct ocelot_port_private, port); + ocelot_inherit_brport_flags(ocelot, port, brport_dev); stp_state = br_port_get_stp_state(brport_dev); @@ -1160,16 +1175,12 @@ static int ocelot_switchdev_sync(struct ocelot *ocelot, int port, ageing_time = br_get_ageing_time(bridge_dev); ocelot_port_attr_ageing_set(ocelot, port, ageing_time); - err = br_mdb_replay(bridge_dev, brport_dev, + err = br_mdb_replay(bridge_dev, brport_dev, priv, true, &ocelot_switchdev_blocking_nb, extack); if (err && err != -EOPNOTSUPP) return err; - err = br_fdb_replay(bridge_dev, brport_dev, &ocelot_switchdev_nb); - if (err) - return err; - - err = br_vlan_replay(bridge_dev, brport_dev, + err = br_vlan_replay(bridge_dev, brport_dev, priv, true, &ocelot_switchdev_blocking_nb, extack); if (err && err != -EOPNOTSUPP) return err; diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index c84c8bf2bc20..fc99ad8e4a38 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -3815,6 +3815,7 @@ static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) dev_err(&pdev->dev, "invalid sram_size %dB or board span %ldB\n", mgp->sram_size, mgp->board_span); + status = -EINVAL; goto abort_with_ioremap; } memcpy_fromio(mgp->eeprom_strings, diff --git a/drivers/net/ethernet/natsemi/natsemi.c b/drivers/net/ethernet/natsemi/natsemi.c index b81e1487945c..51b4b25d15ad 100644 --- a/drivers/net/ethernet/natsemi/natsemi.c +++ b/drivers/net/ethernet/natsemi/natsemi.c @@ -969,7 +969,7 @@ static int natsemi_probe1(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; err_create_file: - unregister_netdev(dev); + unregister_netdev(dev); err_register_netdev: iounmap(ioaddr); @@ -3103,14 +3103,14 @@ static int netdev_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) case SIOCSMIIREG: /* Write MII PHY register. */ if (dev->if_port == PORT_TP) { if ((data->phy_id & 0x1f) == np->phy_addr_external) { - if ((data->reg_num & 0x1f) == MII_ADVERTISE) + if ((data->reg_num & 0x1f) == MII_ADVERTISE) np->advertising = data->val_in; mdio_write(dev, data->reg_num & 0x1f, data->val_in); } } else { if ((data->phy_id & 0x1f) == np->phy_addr_external) { - if ((data->reg_num & 0x1f) == MII_ADVERTISE) + if ((data->reg_num & 0x1f) == MII_ADVERTISE) np->advertising = data->val_in; } move_int_phy(dev, data->phy_id & 0x1f); diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c index 9cfcd5500462..0b017d4f5c08 100644 --- a/drivers/net/ethernet/neterion/s2io.c +++ b/drivers/net/ethernet/neterion/s2io.c @@ -1101,6 +1101,8 @@ static int s2io_print_pci_mode(struct s2io_nic *nic) * @nic: device private variable * @link: link status (UP/DOWN) used to enable/disable continuous * transmit interrupts + * @may_sleep: parameter indicates if sleeping when waiting for + * command complete * Description: The function configures transmit traffic interrupts * Return Value: SUCCESS on success and * '-1' on failure @@ -2743,7 +2745,7 @@ static int s2io_chk_rx_buffers(struct s2io_nic *nic, struct ring_info *ring) } /** - * s2io_poll - Rx interrupt handler for NAPI support + * s2io_poll_msix - Rx interrupt handler for NAPI support * @napi : pointer to the napi structure. * @budget : The number of packets that were budgeted to be processed * during one pass through the 'Poll" function. @@ -3323,6 +3325,8 @@ static void s2io_updt_xpak_counter(struct net_device *dev) * @addr: address * @busy_bit: bit to check for busy * @bit_state: state to check + * @may_sleep: parameter indicates if sleeping when waiting for + * command complete * Description: Function that waits for a command to Write into RMAC * ADDR DATA registers to be completed and returns either success or * error depending on whether the command was complete or not. @@ -4868,6 +4872,8 @@ static struct net_device_stats *s2io_get_stats(struct net_device *dev) /** * s2io_set_multicast - entry point for multicast address enable/disable. * @dev : pointer to the device structure + * @may_sleep: parameter indicates if sleeping when waiting for command + * complete * Description: * This function is a driver entry point which gets called by the kernel * whenever multicast addresses must be enabled/disabled. This also gets @@ -5288,7 +5294,7 @@ s2io_ethtool_set_link_ksettings(struct net_device *dev, } /** - * s2io_ethtol_get_link_ksettings - Return link specific information. + * s2io_ethtool_get_link_ksettings - Return link specific information. * @dev: pointer to netdev * @cmd : pointer to the structure with parameters given by ethtool * to return link information. diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.c b/drivers/net/ethernet/neterion/vxge/vxge-config.c index 5162b938a1ac..a3204a7ef750 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-config.c +++ b/drivers/net/ethernet/neterion/vxge/vxge-config.c @@ -3784,6 +3784,7 @@ vxge_hw_rts_rth_data0_data1_get(u32 j, u64 *data0, u64 *data1, VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_ENTRY_EN | VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_DATA( itable[j]); + return; default: return; } @@ -4884,7 +4885,7 @@ vpath_open_exit1: } /** - * vxge_hw_vpath_rx_doorbell_post - Close the handle got from previous vpath + * vxge_hw_vpath_rx_doorbell_init - Close the handle got from previous vpath * (vpath) open * @vp: Handle got from previous vpath open * diff --git a/drivers/net/ethernet/neterion/vxge/vxge-main.c b/drivers/net/ethernet/neterion/vxge/vxge-main.c index 87892bd992b1..82eef4c72f01 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-main.c +++ b/drivers/net/ethernet/neterion/vxge/vxge-main.c @@ -87,7 +87,7 @@ static unsigned int bw_percentage[VXGE_HW_MAX_VIRTUAL_PATHS] = module_param_array(bw_percentage, uint, NULL, 0); static struct vxge_drv_config *driver_config; -static enum vxge_hw_status vxge_reset_all_vpaths(struct vxgedev *vdev); +static void vxge_reset_all_vpaths(struct vxgedev *vdev); static inline int is_vxge_card_up(struct vxgedev *vdev) { @@ -1606,7 +1606,6 @@ static void vxge_config_ci_for_tti_rti(struct vxgedev *vdev) static int do_vxge_reset(struct vxgedev *vdev, int event) { - enum vxge_hw_status status; int ret = 0, vp_id, i; vxge_debug_entryexit(VXGE_TRACE, "%s:%d", __func__, __LINE__); @@ -1709,14 +1708,7 @@ static int do_vxge_reset(struct vxgedev *vdev, int event) netif_tx_stop_all_queues(vdev->ndev); if (event == VXGE_LL_FULL_RESET) { - status = vxge_reset_all_vpaths(vdev); - if (status != VXGE_HW_OK) { - vxge_debug_init(VXGE_ERR, - "fatal: %s: can not reset vpaths", - vdev->ndev->name); - ret = -EPERM; - goto out; - } + vxge_reset_all_vpaths(vdev); } if (event == VXGE_LL_COMPL_RESET) { @@ -1799,7 +1791,7 @@ static void vxge_reset(struct work_struct *work) } /** - * vxge_poll - Receive handler when Receive Polling is used. + * vxge_poll_msix - Receive handler when Receive Polling is used. * @napi: pointer to the napi structure. * @budget: Number of packets budgeted to be processed in this iteration. * @@ -1969,9 +1961,8 @@ static enum vxge_hw_status vxge_rth_configure(struct vxgedev *vdev) } /* reset vpaths */ -static enum vxge_hw_status vxge_reset_all_vpaths(struct vxgedev *vdev) +static void vxge_reset_all_vpaths(struct vxgedev *vdev) { - enum vxge_hw_status status = VXGE_HW_OK; struct vxge_vpath *vpath; int i; @@ -1986,18 +1977,16 @@ static enum vxge_hw_status vxge_reset_all_vpaths(struct vxgedev *vdev) "vxge_hw_vpath_recover_" "from_reset failed for vpath: " "%d", i); - return status; + return; } } else { vxge_debug_init(VXGE_ERR, "vxge_hw_vpath_reset failed for " "vpath:%d", i); - return status; + return; } } } - - return status; } /* close vpaths */ @@ -2676,11 +2665,7 @@ static int vxge_set_features(struct net_device *dev, netdev_features_t features) /* !netif_running() ensured by vxge_fix_features() */ vdev->devh->config.rth_en = !!(features & NETIF_F_RXHASH); - if (vxge_reset_all_vpaths(vdev) != VXGE_HW_OK) { - dev->features = features ^ NETIF_F_RXHASH; - vdev->devh->config.rth_en = !!(dev->features & NETIF_F_RXHASH); - return -EIO; - } + vxge_reset_all_vpaths(vdev); return 0; } @@ -3693,10 +3678,9 @@ static int vxge_config_vpaths(struct vxge_hw_device_config *device_config, driver_config->vpath_per_dev = 1; for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) - if (!vxge_bVALn(vpath_mask, i, 1)) - continue; - else + if (vxge_bVALn(vpath_mask, i, 1)) default_no_vpath++; + if (default_no_vpath < driver_config->vpath_per_dev) driver_config->vpath_per_dev = default_no_vpath; @@ -4752,7 +4736,7 @@ _exit0: } /** - * vxge_rem_nic - Free the PCI device + * vxge_remove - Free the PCI device * @pdev: structure containing the PCI related information of the device. * Description: This function is called by the Pci subsystem to release a * PCI device and free up all resource held up by the device. diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile index d31772ae511d..9cff3d48acbc 100644 --- a/drivers/net/ethernet/netronome/nfp/Makefile +++ b/drivers/net/ethernet/netronome/nfp/Makefile @@ -51,7 +51,8 @@ nfp-objs += \ flower/metadata.o \ flower/offload.o \ flower/tunnel_conf.o \ - flower/qos_conf.o + flower/qos_conf.o \ + flower/conntrack.o endif ifeq ($(CONFIG_BPF_SYSCALL),y) diff --git a/drivers/net/ethernet/netronome/nfp/ccm_mbox.c b/drivers/net/ethernet/netronome/nfp/ccm_mbox.c index f0783aa9e66e..4247bca09807 100644 --- a/drivers/net/ethernet/netronome/nfp/ccm_mbox.c +++ b/drivers/net/ethernet/netronome/nfp/ccm_mbox.c @@ -36,7 +36,7 @@ enum nfp_net_mbox_cmsg_state { }; /** - * struct nfp_ccm_mbox_skb_cb - CCM mailbox specific info + * struct nfp_ccm_mbox_cmsg_cb - CCM mailbox specific info * @state: processing state (/stage) of the message * @err: error encountered during processing if any * @max_len: max(request_len, reply_len) diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c new file mode 100644 index 000000000000..273d529d43c2 --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c @@ -0,0 +1,1180 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (C) 2021 Corigine, Inc. */ + +#include "conntrack.h" + +const struct rhashtable_params nfp_tc_ct_merge_params = { + .head_offset = offsetof(struct nfp_fl_ct_tc_merge, + hash_node), + .key_len = sizeof(unsigned long) * 2, + .key_offset = offsetof(struct nfp_fl_ct_tc_merge, cookie), + .automatic_shrinking = true, +}; + +const struct rhashtable_params nfp_nft_ct_merge_params = { + .head_offset = offsetof(struct nfp_fl_nft_tc_merge, + hash_node), + .key_len = sizeof(unsigned long) * 3, + .key_offset = offsetof(struct nfp_fl_nft_tc_merge, cookie), + .automatic_shrinking = true, +}; + +static struct flow_action_entry *get_flow_act(struct flow_rule *rule, + enum flow_action_id act_id); + +/** + * get_hashentry() - Wrapper around hashtable lookup. + * @ht: hashtable where entry could be found + * @key: key to lookup + * @params: hashtable params + * @size: size of entry to allocate if not in table + * + * Returns an entry from a hashtable. If entry does not exist + * yet allocate the memory for it and return the new entry. + */ +static void *get_hashentry(struct rhashtable *ht, void *key, + const struct rhashtable_params params, size_t size) +{ + void *result; + + result = rhashtable_lookup_fast(ht, key, params); + + if (result) + return result; + + result = kzalloc(size, GFP_KERNEL); + if (!result) + return ERR_PTR(-ENOMEM); + + return result; +} + +bool is_pre_ct_flow(struct flow_cls_offload *flow) +{ + struct flow_action_entry *act; + int i; + + flow_action_for_each(i, act, &flow->rule->action) { + if (act->id == FLOW_ACTION_CT && !act->ct.action) + return true; + } + return false; +} + +bool is_post_ct_flow(struct flow_cls_offload *flow) +{ + struct flow_rule *rule = flow_cls_offload_flow_rule(flow); + struct flow_dissector *dissector = rule->match.dissector; + struct flow_match_ct ct; + + if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { + flow_rule_match_ct(rule, &ct); + if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) + return true; + } + return false; +} + +static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, + struct nfp_fl_ct_flow_entry *entry2) +{ + unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys & + entry2->rule->match.dissector->used_keys; + bool out; + + /* check the overlapped fields one by one, the unmasked part + * should not conflict with each other. + */ + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) { + struct flow_match_control match1, match2; + + flow_rule_match_control(entry1->rule, &match1); + flow_rule_match_control(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) { + struct flow_match_basic match1, match2; + + flow_rule_match_basic(entry1->rule, &match1); + flow_rule_match_basic(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { + struct flow_match_ipv4_addrs match1, match2; + + flow_rule_match_ipv4_addrs(entry1->rule, &match1); + flow_rule_match_ipv4_addrs(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { + struct flow_match_ipv6_addrs match1, match2; + + flow_rule_match_ipv6_addrs(entry1->rule, &match1); + flow_rule_match_ipv6_addrs(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) { + struct flow_match_ports match1, match2; + + flow_rule_match_ports(entry1->rule, &match1); + flow_rule_match_ports(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs match1, match2; + + flow_rule_match_eth_addrs(entry1->rule, &match1); + flow_rule_match_eth_addrs(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan match1, match2; + + flow_rule_match_vlan(entry1->rule, &match1); + flow_rule_match_vlan(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) { + struct flow_match_mpls match1, match2; + + flow_rule_match_mpls(entry1->rule, &match1); + flow_rule_match_mpls(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) { + struct flow_match_tcp match1, match2; + + flow_rule_match_tcp(entry1->rule, &match1); + flow_rule_match_tcp(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) { + struct flow_match_ip match1, match2; + + flow_rule_match_ip(entry1->rule, &match1); + flow_rule_match_ip(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) { + struct flow_match_enc_keyid match1, match2; + + flow_rule_match_enc_keyid(entry1->rule, &match1); + flow_rule_match_enc_keyid(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { + struct flow_match_ipv4_addrs match1, match2; + + flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1); + flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { + struct flow_match_ipv6_addrs match1, match2; + + flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1); + flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) { + struct flow_match_control match1, match2; + + flow_rule_match_enc_control(entry1->rule, &match1); + flow_rule_match_enc_control(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) { + struct flow_match_ip match1, match2; + + flow_rule_match_enc_ip(entry1->rule, &match1); + flow_rule_match_enc_ip(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) { + struct flow_match_enc_opts match1, match2; + + flow_rule_match_enc_opts(entry1->rule, &match1); + flow_rule_match_enc_opts(entry2->rule, &match2); + COMPARE_UNMASKED_FIELDS(match1, match2, &out); + if (out) + goto check_failed; + } + + return 0; + +check_failed: + return -EINVAL; +} + +static int nfp_ct_check_mangle_merge(struct flow_action_entry *a_in, + struct flow_rule *rule) +{ + enum flow_action_mangle_base htype = a_in->mangle.htype; + u32 offset = a_in->mangle.offset; + + switch (htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_ETH: + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) + return -EOPNOTSUPP; + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { + struct flow_match_ip match; + + flow_rule_match_ip(rule, &match); + if (offset == offsetof(struct iphdr, ttl) && + match.mask->ttl) + return -EOPNOTSUPP; + if (offset == round_down(offsetof(struct iphdr, tos), 4) && + match.mask->tos) + return -EOPNOTSUPP; + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { + struct flow_match_ipv4_addrs match; + + flow_rule_match_ipv4_addrs(rule, &match); + if (offset == offsetof(struct iphdr, saddr) && + match.mask->src) + return -EOPNOTSUPP; + if (offset == offsetof(struct iphdr, daddr) && + match.mask->dst) + return -EOPNOTSUPP; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { + struct flow_match_ip match; + + flow_rule_match_ip(rule, &match); + if (offset == round_down(offsetof(struct ipv6hdr, hop_limit), 4) && + match.mask->ttl) + return -EOPNOTSUPP; + /* for ipv6, tos and flow_lbl are in the same word */ + if (offset == round_down(offsetof(struct ipv6hdr, flow_lbl), 4) && + match.mask->tos) + return -EOPNOTSUPP; + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { + struct flow_match_ipv6_addrs match; + + flow_rule_match_ipv6_addrs(rule, &match); + if (offset >= offsetof(struct ipv6hdr, saddr) && + offset < offsetof(struct ipv6hdr, daddr) && + memchr_inv(&match.mask->src, 0, sizeof(match.mask->src))) + return -EOPNOTSUPP; + if (offset >= offsetof(struct ipv6hdr, daddr) && + offset < sizeof(struct ipv6hdr) && + memchr_inv(&match.mask->dst, 0, sizeof(match.mask->dst))) + return -EOPNOTSUPP; + } + break; + case FLOW_ACT_MANGLE_HDR_TYPE_TCP: + case FLOW_ACT_MANGLE_HDR_TYPE_UDP: + /* currently only can modify ports */ + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) + return -EOPNOTSUPP; + break; + default: + break; + } + return 0; +} + +static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry, + struct nfp_fl_ct_flow_entry *post_ct_entry, + struct nfp_fl_ct_flow_entry *nft_entry) +{ + struct flow_action_entry *act; + int err, i; + + /* Check for pre_ct->action conflicts */ + flow_action_for_each(i, act, &pre_ct_entry->rule->action) { + switch (act->id) { + case FLOW_ACTION_MANGLE: + err = nfp_ct_check_mangle_merge(act, nft_entry->rule); + if (err) + return err; + err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule); + if (err) + return err; + break; + case FLOW_ACTION_VLAN_PUSH: + case FLOW_ACTION_VLAN_POP: + case FLOW_ACTION_VLAN_MANGLE: + case FLOW_ACTION_MPLS_PUSH: + case FLOW_ACTION_MPLS_POP: + case FLOW_ACTION_MPLS_MANGLE: + return -EOPNOTSUPP; + default: + break; + } + } + + /* Check for nft->action conflicts */ + flow_action_for_each(i, act, &nft_entry->rule->action) { + switch (act->id) { + case FLOW_ACTION_MANGLE: + err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule); + if (err) + return err; + break; + case FLOW_ACTION_VLAN_PUSH: + case FLOW_ACTION_VLAN_POP: + case FLOW_ACTION_VLAN_MANGLE: + case FLOW_ACTION_MPLS_PUSH: + case FLOW_ACTION_MPLS_POP: + case FLOW_ACTION_MPLS_MANGLE: + return -EOPNOTSUPP; + default: + break; + } + } + return 0; +} + +static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry, + struct nfp_fl_ct_flow_entry *nft_entry) +{ + struct flow_dissector *dissector = post_ct_entry->rule->match.dissector; + struct flow_action_entry *ct_met; + struct flow_match_ct ct; + int i; + + ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA); + if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) { + u32 *act_lbl; + + act_lbl = ct_met->ct_metadata.labels; + flow_rule_match_ct(post_ct_entry->rule, &ct); + for (i = 0; i < 4; i++) { + if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^ + (act_lbl[i] & ct.mask->ct_labels[i])) + return -EINVAL; + } + + if ((ct.key->ct_mark & ct.mask->ct_mark) ^ + (ct_met->ct_metadata.mark & ct.mask->ct_mark)) + return -EINVAL; + + return 0; + } + + return -EINVAL; +} + +static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry) +{ + return 0; +} + +static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie, + struct net_device *netdev) +{ + return 0; +} + +static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt, + struct nfp_fl_ct_flow_entry *nft_entry, + struct nfp_fl_ct_tc_merge *tc_m_entry) +{ + struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry; + struct nfp_fl_nft_tc_merge *nft_m_entry; + unsigned long new_cookie[3]; + int err; + + pre_ct_entry = tc_m_entry->pre_ct_parent; + post_ct_entry = tc_m_entry->post_ct_parent; + + err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry); + if (err) + return err; + + /* Check that the two tc flows are also compatible with + * the nft entry. No need to check the pre_ct and post_ct + * entries as that was already done during pre_merge. + * The nft entry does not have a netdev or chain populated, so + * skip this check. + */ + err = nfp_ct_merge_check(pre_ct_entry, nft_entry); + if (err) + return err; + err = nfp_ct_merge_check(post_ct_entry, nft_entry); + if (err) + return err; + err = nfp_ct_check_meta(post_ct_entry, nft_entry); + if (err) + return err; + + /* Combine tc_merge and nft cookies for this cookie. */ + new_cookie[0] = tc_m_entry->cookie[0]; + new_cookie[1] = tc_m_entry->cookie[1]; + new_cookie[2] = nft_entry->cookie; + nft_m_entry = get_hashentry(&zt->nft_merge_tb, + &new_cookie, + nfp_nft_ct_merge_params, + sizeof(*nft_m_entry)); + + if (IS_ERR(nft_m_entry)) + return PTR_ERR(nft_m_entry); + + /* nft_m_entry already present, not merging again */ + if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie))) + return 0; + + memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie)); + nft_m_entry->zt = zt; + nft_m_entry->tc_m_parent = tc_m_entry; + nft_m_entry->nft_parent = nft_entry; + nft_m_entry->tc_flower_cookie = 0; + /* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created + * it only combined them if the netdevs were the same, so can use any of them. + */ + nft_m_entry->netdev = pre_ct_entry->netdev; + + /* Add this entry to the tc_m_list and nft_flow lists */ + list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children); + list_add(&nft_m_entry->nft_flow_list, &nft_entry->children); + + /* Generate offload structure and send to nfp */ + err = nfp_fl_ct_add_offload(nft_m_entry); + if (err) + goto err_nft_ct_offload; + + err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node, + nfp_nft_ct_merge_params); + if (err) + goto err_nft_ct_merge_insert; + + zt->nft_merge_count++; + + return err; + +err_nft_ct_merge_insert: + nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie, + nft_m_entry->netdev); +err_nft_ct_offload: + list_del(&nft_m_entry->tc_merge_list); + list_del(&nft_m_entry->nft_flow_list); + kfree(nft_m_entry); + return err; +} + +static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt, + struct nfp_fl_ct_flow_entry *ct_entry1, + struct nfp_fl_ct_flow_entry *ct_entry2) +{ + struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry; + struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp; + struct nfp_fl_ct_tc_merge *m_entry; + unsigned long new_cookie[2]; + int err; + + if (ct_entry1->type == CT_TYPE_PRE_CT) { + pre_ct_entry = ct_entry1; + post_ct_entry = ct_entry2; + } else { + post_ct_entry = ct_entry1; + pre_ct_entry = ct_entry2; + } + + if (post_ct_entry->netdev != pre_ct_entry->netdev) + return -EINVAL; + /* Checks that the chain_index of the filter matches the + * chain_index of the GOTO action. + */ + if (post_ct_entry->chain_index != pre_ct_entry->chain_index) + return -EINVAL; + + err = nfp_ct_merge_check(post_ct_entry, pre_ct_entry); + if (err) + return err; + + new_cookie[0] = pre_ct_entry->cookie; + new_cookie[1] = post_ct_entry->cookie; + m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie, + nfp_tc_ct_merge_params, sizeof(*m_entry)); + if (IS_ERR(m_entry)) + return PTR_ERR(m_entry); + + /* m_entry already present, not merging again */ + if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie))) + return 0; + + memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie)); + m_entry->zt = zt; + m_entry->post_ct_parent = post_ct_entry; + m_entry->pre_ct_parent = pre_ct_entry; + + /* Add this entry to the pre_ct and post_ct lists */ + list_add(&m_entry->post_ct_list, &post_ct_entry->children); + list_add(&m_entry->pre_ct_list, &pre_ct_entry->children); + INIT_LIST_HEAD(&m_entry->children); + + err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node, + nfp_tc_ct_merge_params); + if (err) + goto err_ct_tc_merge_insert; + zt->tc_merge_count++; + + /* Merge with existing nft flows */ + list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list, + list_node) { + nfp_ct_do_nft_merge(zt, nft_entry, m_entry); + } + + return 0; + +err_ct_tc_merge_insert: + list_del(&m_entry->post_ct_list); + list_del(&m_entry->pre_ct_list); + kfree(m_entry); + return err; +} + +static struct +nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv, + u16 zone, bool wildcarded) +{ + struct nfp_fl_ct_zone_entry *zt; + int err; + + if (wildcarded && priv->ct_zone_wc) + return priv->ct_zone_wc; + + if (!wildcarded) { + zt = get_hashentry(&priv->ct_zone_table, &zone, + nfp_zone_table_params, sizeof(*zt)); + + /* If priv is set this is an existing entry, just return it */ + if (IS_ERR(zt) || zt->priv) + return zt; + } else { + zt = kzalloc(sizeof(*zt), GFP_KERNEL); + if (!zt) + return ERR_PTR(-ENOMEM); + } + + zt->zone = zone; + zt->priv = priv; + zt->nft = NULL; + + /* init the various hash tables and lists*/ + INIT_LIST_HEAD(&zt->pre_ct_list); + INIT_LIST_HEAD(&zt->post_ct_list); + INIT_LIST_HEAD(&zt->nft_flows_list); + + err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params); + if (err) + goto err_tc_merge_tb_init; + + err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params); + if (err) + goto err_nft_merge_tb_init; + + if (wildcarded) { + priv->ct_zone_wc = zt; + } else { + err = rhashtable_insert_fast(&priv->ct_zone_table, + &zt->hash_node, + nfp_zone_table_params); + if (err) + goto err_zone_insert; + } + + return zt; + +err_zone_insert: + rhashtable_destroy(&zt->nft_merge_tb); +err_nft_merge_tb_init: + rhashtable_destroy(&zt->tc_merge_tb); +err_tc_merge_tb_init: + kfree(zt); + return ERR_PTR(err); +} + +static struct +nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt, + struct net_device *netdev, + struct flow_cls_offload *flow, + bool is_nft, struct netlink_ext_ack *extack) +{ + struct nf_flow_match *nft_match = NULL; + struct nfp_fl_ct_flow_entry *entry; + struct nfp_fl_ct_map_entry *map; + struct flow_action_entry *act; + int err, i; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return ERR_PTR(-ENOMEM); + + entry->rule = flow_rule_alloc(flow->rule->action.num_entries); + if (!entry->rule) { + err = -ENOMEM; + goto err_pre_ct_rule; + } + + /* nft flows gets destroyed after callback return, so need + * to do a full copy instead of just a reference. + */ + if (is_nft) { + nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL); + if (!nft_match) { + err = -ENOMEM; + goto err_pre_ct_act; + } + memcpy(&nft_match->dissector, flow->rule->match.dissector, + sizeof(nft_match->dissector)); + memcpy(&nft_match->mask, flow->rule->match.mask, + sizeof(nft_match->mask)); + memcpy(&nft_match->key, flow->rule->match.key, + sizeof(nft_match->key)); + entry->rule->match.dissector = &nft_match->dissector; + entry->rule->match.mask = &nft_match->mask; + entry->rule->match.key = &nft_match->key; + } else { + entry->rule->match.dissector = flow->rule->match.dissector; + entry->rule->match.mask = flow->rule->match.mask; + entry->rule->match.key = flow->rule->match.key; + } + + entry->zt = zt; + entry->netdev = netdev; + entry->cookie = flow->cookie; + entry->chain_index = flow->common.chain_index; + entry->tun_offset = NFP_FL_CT_NO_TUN; + + /* Copy over action data. Unfortunately we do not get a handle to the + * original tcf_action data, and the flow objects gets destroyed, so we + * cannot just save a pointer to this either, so need to copy over the + * data unfortunately. + */ + entry->rule->action.num_entries = flow->rule->action.num_entries; + flow_action_for_each(i, act, &flow->rule->action) { + struct flow_action_entry *new_act; + + new_act = &entry->rule->action.entries[i]; + memcpy(new_act, act, sizeof(struct flow_action_entry)); + /* Entunnel is a special case, need to allocate and copy + * tunnel info. + */ + if (act->id == FLOW_ACTION_TUNNEL_ENCAP) { + struct ip_tunnel_info *tun = act->tunnel; + size_t tun_size = sizeof(*tun) + tun->options_len; + + new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC); + if (!new_act->tunnel) { + err = -ENOMEM; + goto err_pre_ct_tun_cp; + } + entry->tun_offset = i; + } + } + + INIT_LIST_HEAD(&entry->children); + + /* Now add a ct map entry to flower-priv */ + map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie, + nfp_ct_map_params, sizeof(*map)); + if (IS_ERR(map)) { + NL_SET_ERR_MSG_MOD(extack, + "offload error: ct map entry creation failed"); + err = -ENOMEM; + goto err_ct_flow_insert; + } + map->cookie = flow->cookie; + map->ct_entry = entry; + err = rhashtable_insert_fast(&zt->priv->ct_map_table, + &map->hash_node, + nfp_ct_map_params); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "offload error: ct map entry table add failed"); + goto err_map_insert; + } + + return entry; + +err_map_insert: + kfree(map); +err_ct_flow_insert: + if (entry->tun_offset != NFP_FL_CT_NO_TUN) + kfree(entry->rule->action.entries[entry->tun_offset].tunnel); +err_pre_ct_tun_cp: + kfree(nft_match); +err_pre_ct_act: + kfree(entry->rule); +err_pre_ct_rule: + kfree(entry); + return ERR_PTR(err); +} + +static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry) +{ + struct nfp_fl_ct_zone_entry *zt; + int err; + + zt = m_entry->zt; + + /* Flow is in HW, need to delete */ + if (m_entry->tc_flower_cookie) { + err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie, + m_entry->netdev); + if (err) + return; + } + + WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb, + &m_entry->hash_node, + nfp_nft_ct_merge_params)); + zt->nft_merge_count--; + list_del(&m_entry->tc_merge_list); + list_del(&m_entry->nft_flow_list); + + kfree(m_entry); +} + +static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow) +{ + struct nfp_fl_nft_tc_merge *m_entry, *tmp; + + /* These post entries are parts of two lists, one is a list of nft_entries + * and the other is of from a list of tc_merge structures. Iterate + * through the relevant list and cleanup the entries. + */ + + if (is_nft_flow) { + /* Need to iterate through list of nft_flow entries*/ + struct nfp_fl_ct_flow_entry *ct_entry = entry; + + list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, + nft_flow_list) { + cleanup_nft_merge_entry(m_entry); + } + } else { + /* Need to iterate through list of tc_merged_flow entries*/ + struct nfp_fl_ct_tc_merge *ct_entry = entry; + + list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, + tc_merge_list) { + cleanup_nft_merge_entry(m_entry); + } + } +} + +static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent) +{ + struct nfp_fl_ct_zone_entry *zt; + int err; + + zt = m_ent->zt; + err = rhashtable_remove_fast(&zt->tc_merge_tb, + &m_ent->hash_node, + nfp_tc_ct_merge_params); + if (err) + pr_warn("WARNING: could not remove merge_entry from hashtable\n"); + zt->tc_merge_count--; + list_del(&m_ent->post_ct_list); + list_del(&m_ent->pre_ct_list); + + if (!list_empty(&m_ent->children)) + nfp_free_nft_merge_children(m_ent, false); + kfree(m_ent); +} + +static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry) +{ + struct nfp_fl_ct_tc_merge *m_ent, *tmp; + + switch (entry->type) { + case CT_TYPE_PRE_CT: + list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) { + nfp_del_tc_merge_entry(m_ent); + } + break; + case CT_TYPE_POST_CT: + list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) { + nfp_del_tc_merge_entry(m_ent); + } + break; + default: + break; + } +} + +void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry) +{ + list_del(&entry->list_node); + + if (!list_empty(&entry->children)) { + if (entry->type == CT_TYPE_NFT) + nfp_free_nft_merge_children(entry, true); + else + nfp_free_tc_merge_children(entry); + } + + if (entry->tun_offset != NFP_FL_CT_NO_TUN) + kfree(entry->rule->action.entries[entry->tun_offset].tunnel); + + if (entry->type == CT_TYPE_NFT) { + struct nf_flow_match *nft_match; + + nft_match = container_of(entry->rule->match.dissector, + struct nf_flow_match, dissector); + kfree(nft_match); + } + + kfree(entry->rule); + kfree(entry); +} + +static struct flow_action_entry *get_flow_act(struct flow_rule *rule, + enum flow_action_id act_id) +{ + struct flow_action_entry *act = NULL; + int i; + + flow_action_for_each(i, act, &rule->action) { + if (act->id == act_id) + return act; + } + return NULL; +} + +static void +nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1, + struct nfp_fl_ct_zone_entry *zt_src, + struct nfp_fl_ct_zone_entry *zt_dst) +{ + struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp; + struct list_head *ct_list; + + if (ct_entry1->type == CT_TYPE_PRE_CT) + ct_list = &zt_src->post_ct_list; + else if (ct_entry1->type == CT_TYPE_POST_CT) + ct_list = &zt_src->pre_ct_list; + else + return; + + list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list, + list_node) { + nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1); + } +} + +static void +nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry, + struct nfp_fl_ct_zone_entry *zt) +{ + struct nfp_fl_ct_tc_merge *tc_merge_entry; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&zt->tc_merge_tb, &iter); + rhashtable_walk_start(&iter); + while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) { + if (IS_ERR(tc_merge_entry)) + continue; + rhashtable_walk_stop(&iter); + nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + +int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv, + struct net_device *netdev, + struct flow_cls_offload *flow, + struct netlink_ext_ack *extack) +{ + struct flow_action_entry *ct_act, *ct_goto; + struct nfp_fl_ct_flow_entry *ct_entry; + struct nfp_fl_ct_zone_entry *zt; + int err; + + ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT); + if (!ct_act) { + NL_SET_ERR_MSG_MOD(extack, + "unsupported offload: Conntrack action empty in conntrack offload"); + return -EOPNOTSUPP; + } + + ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO); + if (!ct_goto) { + NL_SET_ERR_MSG_MOD(extack, + "unsupported offload: Conntrack requires ACTION_GOTO"); + return -EOPNOTSUPP; + } + + zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false); + if (IS_ERR(zt)) { + NL_SET_ERR_MSG_MOD(extack, + "offload error: Could not create zone table entry"); + return PTR_ERR(zt); + } + + if (!zt->nft) { + zt->nft = ct_act->ct.flow_table; + err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "offload error: Could not register nft_callback"); + return err; + } + } + + /* Add entry to pre_ct_list */ + ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack); + if (IS_ERR(ct_entry)) + return PTR_ERR(ct_entry); + ct_entry->type = CT_TYPE_PRE_CT; + ct_entry->chain_index = ct_goto->chain_index; + list_add(&ct_entry->list_node, &zt->pre_ct_list); + zt->pre_ct_count++; + + nfp_ct_merge_tc_entries(ct_entry, zt, zt); + + /* Need to check and merge with tables in the wc_zone as well */ + if (priv->ct_zone_wc) + nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt); + + return 0; +} + +int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv, + struct net_device *netdev, + struct flow_cls_offload *flow, + struct netlink_ext_ack *extack) +{ + struct flow_rule *rule = flow_cls_offload_flow_rule(flow); + struct nfp_fl_ct_flow_entry *ct_entry; + struct nfp_fl_ct_zone_entry *zt; + bool wildcarded = false; + struct flow_match_ct ct; + + flow_rule_match_ct(rule, &ct); + if (!ct.mask->ct_zone) { + wildcarded = true; + } else if (ct.mask->ct_zone != U16_MAX) { + NL_SET_ERR_MSG_MOD(extack, + "unsupported offload: partially wildcarded ct_zone is not supported"); + return -EOPNOTSUPP; + } + + zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded); + if (IS_ERR(zt)) { + NL_SET_ERR_MSG_MOD(extack, + "offload error: Could not create zone table entry"); + return PTR_ERR(zt); + } + + /* Add entry to post_ct_list */ + ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack); + if (IS_ERR(ct_entry)) + return PTR_ERR(ct_entry); + + ct_entry->type = CT_TYPE_POST_CT; + ct_entry->chain_index = flow->common.chain_index; + list_add(&ct_entry->list_node, &zt->post_ct_list); + zt->post_ct_count++; + + if (wildcarded) { + /* Iterate through all zone tables if not empty, look for merges with + * pre_ct entries and merge them. + */ + struct rhashtable_iter iter; + struct nfp_fl_ct_zone_entry *zone_table; + + rhashtable_walk_enter(&priv->ct_zone_table, &iter); + rhashtable_walk_start(&iter); + while ((zone_table = rhashtable_walk_next(&iter)) != NULL) { + if (IS_ERR(zone_table)) + continue; + rhashtable_walk_stop(&iter); + nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); + } else { + nfp_ct_merge_tc_entries(ct_entry, zt, zt); + } + + return 0; +} + +static int +nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow) +{ + struct nfp_fl_ct_map_entry *ct_map_ent; + struct nfp_fl_ct_flow_entry *ct_entry; + struct netlink_ext_ack *extack = NULL; + + ASSERT_RTNL(); + + extack = flow->common.extack; + switch (flow->command) { + case FLOW_CLS_REPLACE: + /* Netfilter can request offload multiple times for the same + * flow - protect against adding duplicates. + */ + ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, + nfp_ct_map_params); + if (!ct_map_ent) { + ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack); + if (IS_ERR(ct_entry)) + return PTR_ERR(ct_entry); + ct_entry->type = CT_TYPE_NFT; + list_add(&ct_entry->list_node, &zt->nft_flows_list); + zt->nft_flows_count++; + nfp_ct_merge_nft_with_tc(ct_entry, zt); + } + return 0; + case FLOW_CLS_DESTROY: + ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, + nfp_ct_map_params); + return nfp_fl_ct_del_flow(ct_map_ent); + case FLOW_CLS_STATS: + return 0; + default: + break; + } + return -EINVAL; +} + +int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv) +{ + struct flow_cls_offload *flow = type_data; + struct nfp_fl_ct_zone_entry *zt = cb_priv; + int err = -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_CLSFLOWER: + rtnl_lock(); + err = nfp_fl_ct_offload_nft_flow(zt, flow); + rtnl_unlock(); + break; + default: + return -EOPNOTSUPP; + } + return err; +} + +static void +nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt) +{ + struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp; + struct nfp_fl_ct_map_entry *ct_map_ent; + + list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list, + list_node) { + ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, + &nft_entry->cookie, + nfp_ct_map_params); + nfp_fl_ct_del_flow(ct_map_ent); + } +} + +int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent) +{ + struct nfp_fl_ct_flow_entry *ct_entry; + struct nfp_fl_ct_zone_entry *zt; + struct rhashtable *m_table; + + if (!ct_map_ent) + return -ENOENT; + + zt = ct_map_ent->ct_entry->zt; + ct_entry = ct_map_ent->ct_entry; + m_table = &zt->priv->ct_map_table; + + switch (ct_entry->type) { + case CT_TYPE_PRE_CT: + zt->pre_ct_count--; + rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, + nfp_ct_map_params); + nfp_fl_ct_clean_flow_entry(ct_entry); + kfree(ct_map_ent); + + /* If this is the last pre_ct_rule it means that it is + * very likely that the nft table will be cleaned up next, + * as this happens on the removal of the last act_ct flow. + * However we cannot deregister the callback on the removal + * of the last nft flow as this runs into a deadlock situation. + * So deregister the callback on removal of the last pre_ct flow + * and remove any remaining nft flow entries. We also cannot + * save this state and delete the callback later since the + * nft table would already have been freed at that time. + */ + if (!zt->pre_ct_count) { + nf_flow_table_offload_del_cb(zt->nft, + nfp_fl_ct_handle_nft_flow, + zt); + zt->nft = NULL; + nfp_fl_ct_clean_nft_entries(zt); + } + break; + case CT_TYPE_POST_CT: + zt->post_ct_count--; + rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, + nfp_ct_map_params); + nfp_fl_ct_clean_flow_entry(ct_entry); + kfree(ct_map_ent); + break; + case CT_TYPE_NFT: + zt->nft_flows_count--; + rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, + nfp_ct_map_params); + nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry); + kfree(ct_map_ent); + default: + break; + } + + return 0; +} diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h new file mode 100644 index 000000000000..170b6cdb8cd0 --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* Copyright (C) 2021 Corigine, Inc. */ + +#ifndef __NFP_FLOWER_CONNTRACK_H__ +#define __NFP_FLOWER_CONNTRACK_H__ 1 + +#include <net/netfilter/nf_flow_table.h> +#include "main.h" + +#define NFP_FL_CT_NO_TUN 0xff + +#define COMPARE_UNMASKED_FIELDS(__match1, __match2, __out) \ + do { \ + typeof(__match1) _match1 = (__match1); \ + typeof(__match2) _match2 = (__match2); \ + bool *_out = (__out); \ + int i, size = sizeof(*(_match1).key); \ + char *k1, *m1, *k2, *m2; \ + *_out = false; \ + k1 = (char *)_match1.key; \ + m1 = (char *)_match1.mask; \ + k2 = (char *)_match2.key; \ + m2 = (char *)_match2.mask; \ + for (i = 0; i < size; i++) \ + if ((k1[i] & m1[i] & m2[i]) ^ \ + (k2[i] & m1[i] & m2[i])) { \ + *_out = true; \ + break; \ + } \ + } while (0) \ + +extern const struct rhashtable_params nfp_zone_table_params; +extern const struct rhashtable_params nfp_ct_map_params; +extern const struct rhashtable_params nfp_tc_ct_merge_params; +extern const struct rhashtable_params nfp_nft_ct_merge_params; + +/** + * struct nfp_fl_ct_zone_entry - Zone entry containing conntrack flow information + * @zone: The zone number, used as lookup key in hashtable + * @hash_node: Used by the hashtable + * @priv: Pointer to nfp_flower_priv data + * @nft: Pointer to nf_flowtable for this zone + * + * @pre_ct_list: The pre_ct_list of nfp_fl_ct_flow_entry entries + * @pre_ct_count: Keep count of the number of pre_ct entries + * + * @post_ct_list: The post_ct_list of nfp_fl_ct_flow_entry entries + * @post_ct_count: Keep count of the number of post_ct entries + * + * @tc_merge_tb: The table of merged tc flows + * @tc_merge_count: Keep count of the number of merged tc entries + * + * @nft_flows_list: The list of nft relatednfp_fl_ct_flow_entry entries + * @nft_flows_count: Keep count of the number of nft_flow entries + * + * @nft_merge_tb: The table of merged tc+nft flows + * @nft_merge_count: Keep count of the number of merged tc+nft entries + */ +struct nfp_fl_ct_zone_entry { + u16 zone; + struct rhash_head hash_node; + + struct nfp_flower_priv *priv; + struct nf_flowtable *nft; + + struct list_head pre_ct_list; + unsigned int pre_ct_count; + + struct list_head post_ct_list; + unsigned int post_ct_count; + + struct rhashtable tc_merge_tb; + unsigned int tc_merge_count; + + struct list_head nft_flows_list; + unsigned int nft_flows_count; + + struct rhashtable nft_merge_tb; + unsigned int nft_merge_count; +}; + +enum ct_entry_type { + CT_TYPE_PRE_CT, + CT_TYPE_NFT, + CT_TYPE_POST_CT, +}; + +/** + * struct nfp_fl_ct_flow_entry - Flow entry containing conntrack flow information + * @cookie: Flow cookie, same as original TC flow, used as key + * @list_node: Used by the list + * @chain_index: Chain index of the original flow + * @netdev: netdev structure. + * @type: Type of pre-entry from enum ct_entry_type + * @zt: Reference to the zone table this belongs to + * @children: List of tc_merge flows this flow forms part of + * @rule: Reference to the original TC flow rule + * @stats: Used to cache stats for updating + * @tun_offset: Used to indicate tunnel action offset in action list + */ +struct nfp_fl_ct_flow_entry { + unsigned long cookie; + struct list_head list_node; + u32 chain_index; + enum ct_entry_type type; + struct net_device *netdev; + struct nfp_fl_ct_zone_entry *zt; + struct list_head children; + struct flow_rule *rule; + struct flow_stats stats; + u8 tun_offset; // Set to NFP_FL_CT_NO_TUN if no tun +}; + +/** + * struct nfp_fl_ct_tc_merge - Merge of two flows from tc + * @cookie: Flow cookie, combination of pre and post ct cookies + * @hash_node: Used by the hashtable + * @pre_ct_list: This entry is part of a pre_ct_list + * @post_ct_list: This entry is part of a post_ct_list + * @zt: Reference to the zone table this belongs to + * @pre_ct_parent: The pre_ct_parent + * @post_ct_parent: The post_ct_parent + * @children: List of nft merged entries + */ +struct nfp_fl_ct_tc_merge { + unsigned long cookie[2]; + struct rhash_head hash_node; + struct list_head pre_ct_list; + struct list_head post_ct_list; + struct nfp_fl_ct_zone_entry *zt; + struct nfp_fl_ct_flow_entry *pre_ct_parent; + struct nfp_fl_ct_flow_entry *post_ct_parent; + struct list_head children; +}; + +/** + * struct nfp_fl_nft_tc_merge - Merge of tc_merge flows with nft flow + * @netdev: Ingress netdev name + * @cookie: Flow cookie, combination of tc_merge and nft cookies + * @hash_node: Used by the hashtable + * @zt: Reference to the zone table this belongs to + * @nft_flow_list: This entry is part of a nft_flows_list + * @tc_merge_list: This entry is part of a ct_merge_list + * @tc_m_parent: The tc_merge parent + * @nft_parent: The nft_entry parent + * @tc_flower_cookie: The cookie of the flow offloaded to the nfp + * @flow_pay: Reference to the offloaded flow struct + */ +struct nfp_fl_nft_tc_merge { + struct net_device *netdev; + unsigned long cookie[3]; + struct rhash_head hash_node; + struct nfp_fl_ct_zone_entry *zt; + struct list_head nft_flow_list; + struct list_head tc_merge_list; + struct nfp_fl_ct_tc_merge *tc_m_parent; + struct nfp_fl_ct_flow_entry *nft_parent; + unsigned long tc_flower_cookie; + struct nfp_fl_payload *flow_pay; +}; + +/** + * struct nfp_fl_ct_map_entry - Map between flow cookie and specific ct_flow + * @cookie: Flow cookie, same as original TC flow, used as key + * @hash_node: Used by the hashtable + * @ct_entry: Pointer to corresponding ct_entry + */ +struct nfp_fl_ct_map_entry { + unsigned long cookie; + struct rhash_head hash_node; + struct nfp_fl_ct_flow_entry *ct_entry; +}; + +bool is_pre_ct_flow(struct flow_cls_offload *flow); +bool is_post_ct_flow(struct flow_cls_offload *flow); + +/** + * nfp_fl_ct_handle_pre_ct() - Handles -trk conntrack rules + * @priv: Pointer to app priv + * @netdev: netdev structure. + * @flow: TC flower classifier offload structure. + * @extack: Extack pointer for errors + * + * Adds a new entry to the relevant zone table and tries to + * merge with other +trk+est entries and offload if possible. + * + * Return: negative value on error, 0 if configured successfully. + */ +int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv, + struct net_device *netdev, + struct flow_cls_offload *flow, + struct netlink_ext_ack *extack); +/** + * nfp_fl_ct_handle_post_ct() - Handles +trk+est conntrack rules + * @priv: Pointer to app priv + * @netdev: netdev structure. + * @flow: TC flower classifier offload structure. + * @extack: Extack pointer for errors + * + * Adds a new entry to the relevant zone table and tries to + * merge with other -trk entries and offload if possible. + * + * Return: negative value on error, 0 if configured successfully. + */ +int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv, + struct net_device *netdev, + struct flow_cls_offload *flow, + struct netlink_ext_ack *extack); + +/** + * nfp_fl_ct_clean_flow_entry() - Free a nfp_fl_ct_flow_entry + * @entry: Flow entry to cleanup + */ +void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry); + +/** + * nfp_fl_ct_del_flow() - Handle flow_del callbacks for conntrack + * @ct_map_ent: ct map entry for the flow that needs deleting + */ +int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent); + +/** + * nfp_fl_ct_handle_nft_flow() - Handle flower flow callbacks for nft table + * @type: Type provided by callback + * @type_data: Callback data + * @cb_priv: Pointer to data provided when registering the callback, in this + * case it's the zone table. + */ +int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, + void *cb_priv); +#endif diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h index 31377923ea3d..0fbd682ccf72 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.h +++ b/drivers/net/ethernet/netronome/nfp/flower/main.h @@ -193,6 +193,9 @@ struct nfp_fl_internal_ports { * @qos_stats_lock: Lock on qos stats updates * @pre_tun_rule_cnt: Number of pre-tunnel rules offloaded * @merge_table: Hash table to store merged flows + * @ct_zone_table: Hash table used to store the different zones + * @ct_zone_wc: Special zone entry for wildcarded zone matches + * @ct_map_table: Hash table used to referennce ct flows */ struct nfp_flower_priv { struct nfp_app *app; @@ -227,6 +230,9 @@ struct nfp_flower_priv { spinlock_t qos_stats_lock; /* Protect the qos stats */ int pre_tun_rule_cnt; struct rhashtable merge_table; + struct rhashtable ct_zone_table; + struct nfp_fl_ct_zone_entry *ct_zone_wc; + struct rhashtable ct_map_table; }; /** diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c index 327bb56b3ef5..621113650a9b 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c +++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c @@ -9,6 +9,7 @@ #include <net/pkt_cls.h> #include "cmsg.h" +#include "conntrack.h" #include "main.h" #include "../nfp_app.h" @@ -496,6 +497,20 @@ const struct rhashtable_params merge_table_params = { .key_len = sizeof(u64), }; +const struct rhashtable_params nfp_zone_table_params = { + .head_offset = offsetof(struct nfp_fl_ct_zone_entry, hash_node), + .key_len = sizeof(u16), + .key_offset = offsetof(struct nfp_fl_ct_zone_entry, zone), + .automatic_shrinking = false, +}; + +const struct rhashtable_params nfp_ct_map_params = { + .head_offset = offsetof(struct nfp_fl_ct_map_entry, hash_node), + .key_len = sizeof(unsigned long), + .key_offset = offsetof(struct nfp_fl_ct_map_entry, cookie), + .automatic_shrinking = true, +}; + int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, unsigned int host_num_mems) { @@ -516,6 +531,14 @@ int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, if (err) goto err_free_stats_ctx_table; + err = rhashtable_init(&priv->ct_zone_table, &nfp_zone_table_params); + if (err) + goto err_free_merge_table; + + err = rhashtable_init(&priv->ct_map_table, &nfp_ct_map_params); + if (err) + goto err_free_ct_zone_table; + get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); /* Init ring buffer and unallocated mask_ids. */ @@ -523,7 +546,7 @@ int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); if (!priv->mask_ids.mask_id_free_list.buf) - goto err_free_merge_table; + goto err_free_ct_map_table; priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; @@ -560,6 +583,10 @@ err_free_last_used: kfree(priv->mask_ids.last_used); err_free_mask_id: kfree(priv->mask_ids.mask_id_free_list.buf); +err_free_ct_map_table: + rhashtable_destroy(&priv->ct_map_table); +err_free_ct_zone_table: + rhashtable_destroy(&priv->ct_zone_table); err_free_merge_table: rhashtable_destroy(&priv->merge_table); err_free_stats_ctx_table: @@ -569,6 +596,100 @@ err_free_flow_table: return -ENOMEM; } +static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt) +{ + if (!zt) + return; + + if (!list_empty(&zt->pre_ct_list)) { + struct rhashtable *m_table = &zt->priv->ct_map_table; + struct nfp_fl_ct_flow_entry *entry, *tmp; + struct nfp_fl_ct_map_entry *map; + + WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n"); + list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list, + list_node) { + map = rhashtable_lookup_fast(m_table, + &entry->cookie, + nfp_ct_map_params); + WARN_ON_ONCE(rhashtable_remove_fast(m_table, + &map->hash_node, + nfp_ct_map_params)); + nfp_fl_ct_clean_flow_entry(entry); + kfree(map); + } + } + + if (!list_empty(&zt->post_ct_list)) { + struct rhashtable *m_table = &zt->priv->ct_map_table; + struct nfp_fl_ct_flow_entry *entry, *tmp; + struct nfp_fl_ct_map_entry *map; + + WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n"); + list_for_each_entry_safe(entry, tmp, &zt->post_ct_list, + list_node) { + map = rhashtable_lookup_fast(m_table, + &entry->cookie, + nfp_ct_map_params); + WARN_ON_ONCE(rhashtable_remove_fast(m_table, + &map->hash_node, + nfp_ct_map_params)); + nfp_fl_ct_clean_flow_entry(entry); + kfree(map); + } + } + + if (zt->nft) { + nf_flow_table_offload_del_cb(zt->nft, + nfp_fl_ct_handle_nft_flow, + zt); + zt->nft = NULL; + } + + if (!list_empty(&zt->nft_flows_list)) { + struct rhashtable *m_table = &zt->priv->ct_map_table; + struct nfp_fl_ct_flow_entry *entry, *tmp; + struct nfp_fl_ct_map_entry *map; + + WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n"); + list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list, + list_node) { + map = rhashtable_lookup_fast(m_table, + &entry->cookie, + nfp_ct_map_params); + WARN_ON_ONCE(rhashtable_remove_fast(m_table, + &map->hash_node, + nfp_ct_map_params)); + nfp_fl_ct_clean_flow_entry(entry); + kfree(map); + } + } + + rhashtable_free_and_destroy(&zt->tc_merge_tb, + nfp_check_rhashtable_empty, NULL); + rhashtable_free_and_destroy(&zt->nft_merge_tb, + nfp_check_rhashtable_empty, NULL); + + kfree(zt); +} + +static void nfp_free_zone_table_entry(void *ptr, void *arg) +{ + struct nfp_fl_ct_zone_entry *zt = ptr; + + nfp_zone_table_entry_destroy(zt); +} + +static void nfp_free_map_table_entry(void *ptr, void *arg) +{ + struct nfp_fl_ct_map_entry *map = ptr; + + if (!map) + return; + + kfree(map); +} + void nfp_flower_metadata_cleanup(struct nfp_app *app) { struct nfp_flower_priv *priv = app->priv; @@ -582,6 +703,12 @@ void nfp_flower_metadata_cleanup(struct nfp_app *app) nfp_check_rhashtable_empty, NULL); rhashtable_free_and_destroy(&priv->merge_table, nfp_check_rhashtable_empty, NULL); + rhashtable_free_and_destroy(&priv->ct_zone_table, + nfp_free_zone_table_entry, NULL); + nfp_zone_table_entry_destroy(priv->ct_zone_wc); + + rhashtable_free_and_destroy(&priv->ct_map_table, + nfp_free_map_table_entry, NULL); kvfree(priv->stats); kfree(priv->mask_ids.mask_id_free_list.buf); kfree(priv->mask_ids.last_used); diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index e95969c462e4..2406d33356ad 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -7,6 +7,7 @@ #include "cmsg.h" #include "main.h" +#include "conntrack.h" #include "../nfpcore/nfp_cpp.h" #include "../nfpcore/nfp_nsp.h" #include "../nfp_app.h" @@ -1276,6 +1277,20 @@ nfp_flower_validate_pre_tun_rule(struct nfp_app *app, return 0; } +static bool offload_pre_check(struct flow_cls_offload *flow) +{ + struct flow_rule *rule = flow_cls_offload_flow_rule(flow); + struct flow_dissector *dissector = rule->match.dissector; + + if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) + return false; + + if (flow->common.chain_index) + return false; + + return true; +} + /** * nfp_flower_add_offload() - Adds a new flow to hardware. * @app: Pointer to the APP handle @@ -1302,6 +1317,15 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, if (nfp_netdev_is_nfp_repr(netdev)) port = nfp_port_from_netdev(netdev); + if (is_pre_ct_flow(flow)) + return nfp_fl_ct_handle_pre_ct(priv, netdev, flow, extack); + + if (is_post_ct_flow(flow)) + return nfp_fl_ct_handle_post_ct(priv, netdev, flow, extack); + + if (!offload_pre_check(flow)) + return -EOPNOTSUPP; + key_layer = kmalloc(sizeof(*key_layer), GFP_KERNEL); if (!key_layer) return -ENOMEM; @@ -1481,6 +1505,7 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, struct flow_cls_offload *flow) { struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_ct_map_entry *ct_map_ent; struct netlink_ext_ack *extack = NULL; struct nfp_fl_payload *nfp_flow; struct nfp_port *port = NULL; @@ -1490,6 +1515,14 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, if (nfp_netdev_is_nfp_repr(netdev)) port = nfp_port_from_netdev(netdev); + /* Check ct_map_table */ + ct_map_ent = rhashtable_lookup_fast(&priv->ct_map_table, &flow->cookie, + nfp_ct_map_params); + if (ct_map_ent) { + err = nfp_fl_ct_del_flow(ct_map_ent); + return err; + } + nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev); if (!nfp_flow) { NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot remove flow that does not exist"); @@ -1646,9 +1679,10 @@ nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev, static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) { + struct flow_cls_common_offload *common = type_data; struct nfp_repr *repr = cb_priv; - if (!tc_cls_can_offload_and_chain0(repr->netdev, type_data)) + if (!tc_can_offload_extack(repr->netdev, common->extack)) return -EOPNOTSUPP; switch (type) { @@ -1746,10 +1780,6 @@ static int nfp_flower_setup_indr_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) { struct nfp_flower_indr_block_cb_priv *priv = cb_priv; - struct flow_cls_offload *flower = type_data; - - if (flower->common.chain_index) - return -EOPNOTSUPP; switch (type) { case TC_SETUP_CLSFLOWER: diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c index d19c02e99114..ab70179728f6 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c +++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c @@ -21,7 +21,7 @@ #define NFP_TUN_PRE_TUN_IPV6_BIT BIT(7) /** - * struct nfp_tun_pre_run_rule - rule matched before decap + * struct nfp_tun_pre_tun_rule - rule matched before decap * @flags: options for the rule offset * @port_idx: index of destination MAC address for the rule * @vlan_tci: VLAN info associated with MAC diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index eeb30680b4dc..5dfa4799c34f 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -1819,7 +1819,6 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) struct xdp_buff xdp; int idx; - rcu_read_lock(); xdp_prog = READ_ONCE(dp->xdp_prog); true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz; xdp_init_buff(&xdp, PAGE_SIZE - NFP_NET_RX_BUF_HEADROOM, @@ -2036,7 +2035,6 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) if (!nfp_net_xdp_complete(tx_ring)) pkts_polled = budget; } - rcu_read_unlock(); return pkts_polled; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index b3cabc274121..3b8e675087de 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -103,6 +103,7 @@ nfp_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) case NFP_PORT_PF_PORT: case NFP_PORT_VF_PORT: nfp_repr_vnic_get_stats64(repr->port, stats); + break; default: break; } diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c index 94994a939277..d7ac0307797f 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c @@ -905,8 +905,7 @@ area_cache_put(struct nfp_cpp *cpp, struct nfp_cpp_area_cache *cache) return; /* Move to front of LRU */ - list_del(&cache->entry); - list_add(&cache->entry, &cpp->area_cache_list); + list_move(&cache->entry, &cpp->area_cache_list); mutex_unlock(&cpp->area_cache_mutex); } diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.c index d4e02542e2e9..e2e5fd003ad6 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.c @@ -24,7 +24,7 @@ #define NFFW_FWID_ALL 255 -/** +/* * NFFW_INFO_VERSION history: * 0: This was never actually used (before versioning), but it refers to * the previous struct which had FWINFO_CNT = MEINFO_CNT = 120 that later diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c index a6861df9904f..2d097dcb7bda 100644 --- a/drivers/net/ethernet/ni/nixge.c +++ b/drivers/net/ethernet/ni/nixge.c @@ -1224,7 +1224,6 @@ static int nixge_of_get_resources(struct platform_device *pdev) const struct of_device_id *of_id; enum nixge_version version; struct resource *ctrlres; - struct resource *dmares; struct net_device *ndev; struct nixge_priv *priv; @@ -1236,12 +1235,9 @@ static int nixge_of_get_resources(struct platform_device *pdev) version = (enum nixge_version)of_id->data; if (version <= NIXGE_V2) - dmares = platform_get_resource(pdev, IORESOURCE_MEM, 0); + priv->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); else - dmares = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "dma"); - - priv->dma_regs = devm_ioremap_resource(&pdev->dev, dmares); + priv->dma_regs = devm_platform_ioremap_resource_byname(pdev, "dma"); if (IS_ERR(priv->dma_regs)) { netdev_err(ndev, "failed to map dma regs\n"); return PTR_ERR(priv->dma_regs); diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe.h b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe.h index a6823c4d355d..108f312bc542 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe.h +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe.h @@ -596,8 +596,6 @@ struct pch_gbe_adapter { #define pch_gbe_hw_to_adapter(hw) container_of(hw, struct pch_gbe_adapter, hw) -extern const char pch_driver_version[]; - /* pch_gbe_main.c */ int pch_gbe_up(struct pch_gbe_adapter *adapter); void pch_gbe_down(struct pch_gbe_adapter *adapter); diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c index a58f14aca10c..660b07cb5b92 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c @@ -8,6 +8,8 @@ #include "pch_gbe.h" #include "pch_gbe_phy.h" +static const char pch_driver_version[] = "1.01"; + /* * pch_gbe_stats - Stats item information */ diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c index 334af49e5add..e351f3d1608f 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c @@ -8,15 +8,16 @@ #include "pch_gbe.h" #include "pch_gbe_phy.h" + +#include <linux/gpio/consumer.h> +#include <linux/gpio/machine.h> +#include <linux/iopoll.h> #include <linux/module.h> #include <linux/net_tstamp.h> #include <linux/ptp_classify.h> #include <linux/ptp_pch.h> #include <linux/gpio.h> -#define DRV_VERSION "1.01" -const char pch_driver_version[] = DRV_VERSION; - #define PCH_GBE_MAR_ENTRIES 16 #define PCH_GBE_SHORT_PKT 64 #define DSC_INIT16 0xC000 @@ -97,8 +98,6 @@ const char pch_driver_version[] = DRV_VERSION; #define PTP_L4_MULTICAST_SA "01:00:5e:00:01:81" #define PTP_L2_MULTICAST_SA "01:1b:19:00:00:00" -#define MINNOW_PHY_RESET_GPIO 13 - static int pch_gbe_mdio_read(struct net_device *netdev, int addr, int reg); static void pch_gbe_mdio_write(struct net_device *netdev, int addr, int reg, int data); @@ -108,7 +107,7 @@ static int pch_ptp_match(struct sk_buff *skb, u16 uid_hi, u32 uid_lo, u16 seqid) { u8 *data = skb->data; unsigned int offset; - u16 *hi, *id; + u16 hi, id; u32 lo; if (ptp_classify_raw(skb) == PTP_CLASS_NONE) @@ -119,14 +118,11 @@ static int pch_ptp_match(struct sk_buff *skb, u16 uid_hi, u32 uid_lo, u16 seqid) if (skb->len < offset + OFF_PTP_SEQUENCE_ID + sizeof(seqid)) return 0; - hi = (u16 *)(data + offset + OFF_PTP_SOURCE_UUID); - id = (u16 *)(data + offset + OFF_PTP_SEQUENCE_ID); - - memcpy(&lo, &hi[1], sizeof(lo)); + hi = get_unaligned_be16(data + offset + OFF_PTP_SOURCE_UUID + 0); + lo = get_unaligned_be32(data + offset + OFF_PTP_SOURCE_UUID + 2); + id = get_unaligned_be16(data + offset + OFF_PTP_SEQUENCE_ID); - return (uid_hi == *hi && - uid_lo == lo && - seqid == *id); + return (uid_hi == hi && uid_lo == lo && seqid == id); } static void @@ -136,7 +132,6 @@ pch_rx_timestamp(struct pch_gbe_adapter *adapter, struct sk_buff *skb) struct pci_dev *pdev; u64 ns; u32 hi, lo, val; - u16 uid, seq; if (!adapter->hwts_rx_en) return; @@ -152,10 +147,7 @@ pch_rx_timestamp(struct pch_gbe_adapter *adapter, struct sk_buff *skb) lo = pch_src_uuid_lo_read(pdev); hi = pch_src_uuid_hi_read(pdev); - uid = hi & 0xffff; - seq = (hi >> 16) & 0xffff; - - if (!pch_ptp_match(skb, htons(uid), htonl(lo), htons(seq))) + if (!pch_ptp_match(skb, hi, lo, hi >> 16)) goto out; ns = pch_rx_snap_read(pdev); @@ -298,15 +290,12 @@ static s32 pch_gbe_mac_read_mac_addr(struct pch_gbe_hw *hw) * @reg: Pointer of register * @bit: Busy bit */ -static void pch_gbe_wait_clr_bit(void *reg, u32 bit) +static void pch_gbe_wait_clr_bit(void __iomem *reg, u32 bit) { u32 tmp; /* wait busy */ - tmp = 1000; - while ((ioread32(reg) & bit) && --tmp) - cpu_relax(); - if (!tmp) + if (readx_poll_timeout_atomic(ioread32, reg, tmp, !(tmp & bit), 0, 10)) pr_err("Error: busy bit is not cleared\n"); } @@ -490,18 +479,13 @@ u16 pch_gbe_mac_ctrl_miim(struct pch_gbe_hw *hw, u32 addr, u32 dir, u32 reg, u16 data) { struct pch_gbe_adapter *adapter = pch_gbe_hw_to_adapter(hw); - u32 data_out = 0; - unsigned int i; unsigned long flags; + u32 data_out; spin_lock_irqsave(&hw->miim_lock, flags); - for (i = 100; i; --i) { - if ((ioread32(&hw->reg->MIIM) & PCH_GBE_MIIM_OPER_READY)) - break; - udelay(20); - } - if (i == 0) { + if (readx_poll_timeout_atomic(ioread32, &hw->reg->MIIM, data_out, + data_out & PCH_GBE_MIIM_OPER_READY, 20, 2000)) { netdev_err(adapter->netdev, "pch-gbe.miim won't go Ready\n"); spin_unlock_irqrestore(&hw->miim_lock, flags); return 0; /* No way to indicate timeout error */ @@ -509,12 +493,8 @@ u16 pch_gbe_mac_ctrl_miim(struct pch_gbe_hw *hw, u32 addr, u32 dir, u32 reg, iowrite32(((reg << PCH_GBE_MIIM_REG_ADDR_SHIFT) | (addr << PCH_GBE_MIIM_PHY_ADDR_SHIFT) | dir | data), &hw->reg->MIIM); - for (i = 0; i < 100; i++) { - udelay(20); - data_out = ioread32(&hw->reg->MIIM); - if ((data_out & PCH_GBE_MIIM_OPER_READY)) - break; - } + readx_poll_timeout_atomic(ioread32, &hw->reg->MIIM, data_out, + data_out & PCH_GBE_MIIM_OPER_READY, 20, 2000); spin_unlock_irqrestore(&hw->miim_lock, flags); netdev_dbg(adapter->netdev, "PHY %s: reg=%d, data=0x%04X\n", @@ -2532,9 +2512,13 @@ static int pch_gbe_probe(struct pci_dev *pdev, adapter->pdev = pdev; adapter->hw.back = adapter; adapter->hw.reg = pcim_iomap_table(pdev)[PCH_GBE_PCI_BAR]; + adapter->pdata = (struct pch_gbe_privdata *)pci_id->driver_data; - if (adapter->pdata && adapter->pdata->platform_init) - adapter->pdata->platform_init(pdev); + if (adapter->pdata && adapter->pdata->platform_init) { + ret = adapter->pdata->platform_init(pdev); + if (ret) + goto err_free_netdev; + } adapter->ptp_pdev = pci_get_domain_bus_and_slot(pci_domain_nr(adapter->pdev->bus), @@ -2624,26 +2608,45 @@ err_free_netdev: return ret; } +static void pch_gbe_gpio_remove_table(void *table) +{ + gpiod_remove_lookup_table(table); +} + +static int pch_gbe_gpio_add_table(struct device *dev, void *table) +{ + gpiod_add_lookup_table(table); + return devm_add_action_or_reset(dev, pch_gbe_gpio_remove_table, table); +} + +static struct gpiod_lookup_table pch_gbe_minnow_gpio_table = { + .dev_id = "0000:02:00.1", + .table = { + GPIO_LOOKUP("sch_gpio.33158", 13, NULL, GPIO_ACTIVE_LOW), + {} + }, +}; + /* The AR803X PHY on the MinnowBoard requires a physical pin to be toggled to * ensure it is awake for probe and init. Request the line and reset the PHY. */ static int pch_gbe_minnow_platform_init(struct pci_dev *pdev) { - unsigned long flags = GPIOF_DIR_OUT | GPIOF_INIT_HIGH | GPIOF_EXPORT; - unsigned gpio = MINNOW_PHY_RESET_GPIO; + struct gpio_desc *gpiod; int ret; - ret = devm_gpio_request_one(&pdev->dev, gpio, flags, - "minnow_phy_reset"); - if (ret) { - dev_err(&pdev->dev, - "ERR: Can't request PHY reset GPIO line '%d'\n", gpio); + ret = pch_gbe_gpio_add_table(&pdev->dev, &pch_gbe_minnow_gpio_table); + if (ret) return ret; - } - gpio_set_value(gpio, 0); + gpiod = devm_gpiod_get(&pdev->dev, NULL, GPIOD_OUT_HIGH); + if (IS_ERR(gpiod)) + return dev_err_probe(&pdev->dev, PTR_ERR(gpiod), + "Can't request PHY reset GPIO line\n"); + + gpiod_set_value(gpiod, 1); usleep_range(1250, 1500); - gpio_set_value(gpio, 1); + gpiod_set_value(gpiod, 0); usleep_range(1250, 1500); return ret; @@ -2722,7 +2725,6 @@ module_pci_driver(pch_gbe_driver); MODULE_DESCRIPTION("EG20T PCH Gigabit ethernet Driver"); MODULE_AUTHOR("LAPIS SEMICONDUCTOR, <tshimizu818@gmail.com>"); MODULE_LICENSE("GPL"); -MODULE_VERSION(DRV_VERSION); MODULE_DEVICE_TABLE(pci, pch_gbe_pcidev_id); /* pch_gbe_main.c */ diff --git a/drivers/net/ethernet/pensando/Kconfig b/drivers/net/ethernet/pensando/Kconfig index 5f8b0bb3af6e..202973a82712 100644 --- a/drivers/net/ethernet/pensando/Kconfig +++ b/drivers/net/ethernet/pensando/Kconfig @@ -20,6 +20,7 @@ if NET_VENDOR_PENSANDO config IONIC tristate "Pensando Ethernet IONIC Support" depends on 64BIT && PCI + depends on PTP_1588_CLOCK || !PTP_1588_CLOCK select NET_DEVLINK select DIMLIB help diff --git a/drivers/net/ethernet/qlogic/Kconfig b/drivers/net/ethernet/qlogic/Kconfig index 6b5ddb07ee83..98f430905ffa 100644 --- a/drivers/net/ethernet/qlogic/Kconfig +++ b/drivers/net/ethernet/qlogic/Kconfig @@ -110,6 +110,9 @@ config QED_RDMA config QED_ISCSI bool +config QED_NVMETCP + bool + config QED_FCOE bool diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c index 08f9477d2ee8..35ec9aab3dc7 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c @@ -1685,6 +1685,7 @@ netxen_process_rcv_ring(struct nx_host_sds_ring *sds_ring, int max) break; case NETXEN_NIC_RESPONSE_DESC: netxen_handle_fw_message(desc_cnt, consumer, sds_ring); + goto skip; default: goto skip; } diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c index 7e6bac85495d..344ea1143454 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c @@ -1602,6 +1602,8 @@ err_out_free_netdev: free_netdev(netdev); err_out_free_res: + if (NX_IS_REVISION_P3(pdev->revision)) + pci_disable_pcie_error_reporting(pdev); pci_release_regions(pdev); err_out_disable_pdev: diff --git a/drivers/net/ethernet/qlogic/qed/Makefile b/drivers/net/ethernet/qlogic/qed/Makefile index 8251755ec18c..0d9c2fe0245d 100644 --- a/drivers/net/ethernet/qlogic/qed/Makefile +++ b/drivers/net/ethernet/qlogic/qed/Makefile @@ -28,6 +28,11 @@ qed-$(CONFIG_QED_ISCSI) += qed_iscsi.o qed-$(CONFIG_QED_LL2) += qed_ll2.o qed-$(CONFIG_QED_OOO) += qed_ooo.o +qed-$(CONFIG_QED_NVMETCP) += \ + qed_nvmetcp.o \ + qed_nvmetcp_fw_funcs.o \ + qed_nvmetcp_ip_services.o + qed-$(CONFIG_QED_RDMA) += \ qed_iwarp.o \ qed_rdma.o \ diff --git a/drivers/net/ethernet/qlogic/qed/qed.h b/drivers/net/ethernet/qlogic/qed/qed.h index a20cb8a0c377..b590c70539b5 100644 --- a/drivers/net/ethernet/qlogic/qed/qed.h +++ b/drivers/net/ethernet/qlogic/qed/qed.h @@ -49,6 +49,8 @@ extern const struct qed_common_ops qed_common_ops_pass; #define QED_MIN_WIDS (4) #define QED_PF_DEMS_SIZE (4) +#define QED_LLH_DONT_CARE 0 + /* cau states */ enum qed_coalescing_mode { QED_COAL_MODE_DISABLE, @@ -200,6 +202,7 @@ enum qed_pci_personality { QED_PCI_ETH, QED_PCI_FCOE, QED_PCI_ISCSI, + QED_PCI_NVMETCP, QED_PCI_ETH_ROCE, QED_PCI_ETH_IWARP, QED_PCI_ETH_RDMA, @@ -239,6 +242,7 @@ enum QED_FEATURE { QED_PF_L2_QUE, QED_VF, QED_RDMA_CNQ, + QED_NVMETCP_CQ, QED_ISCSI_CQ, QED_FCOE_CQ, QED_VF_L2_QUE, @@ -284,6 +288,8 @@ struct qed_hw_info { ((dev)->hw_info.personality == QED_PCI_FCOE) #define QED_IS_ISCSI_PERSONALITY(dev) \ ((dev)->hw_info.personality == QED_PCI_ISCSI) +#define QED_IS_NVMETCP_PERSONALITY(dev) \ + ((dev)->hw_info.personality == QED_PCI_NVMETCP) /* Resource Allocation scheme results */ u32 resc_start[QED_MAX_RESC]; @@ -592,6 +598,7 @@ struct qed_hwfn { struct qed_ooo_info *p_ooo_info; struct qed_rdma_info *p_rdma_info; struct qed_iscsi_info *p_iscsi_info; + struct qed_nvmetcp_info *p_nvmetcp_info; struct qed_fcoe_info *p_fcoe_info; struct qed_pf_params pf_params; @@ -828,6 +835,7 @@ struct qed_dev { struct qed_eth_cb_ops *eth; struct qed_fcoe_cb_ops *fcoe; struct qed_iscsi_cb_ops *iscsi; + struct qed_nvmetcp_cb_ops *nvmetcp; } protocol_ops; void *ops_cookie; @@ -999,4 +1007,10 @@ int qed_mfw_fill_tlv_data(struct qed_hwfn *hwfn, void qed_hw_info_set_offload_tc(struct qed_hw_info *p_info, u8 tc); void qed_periodic_db_rec_start(struct qed_hwfn *p_hwfn); + +int qed_llh_add_src_tcp_port_filter(struct qed_dev *cdev, u16 src_port); +int qed_llh_add_dst_tcp_port_filter(struct qed_dev *cdev, u16 dest_port); +void qed_llh_remove_src_tcp_port_filter(struct qed_dev *cdev, u16 src_port); +void qed_llh_remove_dst_tcp_port_filter(struct qed_dev *cdev, u16 src_port); +void qed_llh_clear_all_filters(struct qed_dev *cdev); #endif /* _QED_H */ diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c index 0a22f8ce9a2c..5a0a3cbcc1c1 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c +++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c @@ -94,14 +94,14 @@ struct src_ent { static bool src_proto(enum protocol_type type) { - return type == PROTOCOLID_ISCSI || + return type == PROTOCOLID_TCP_ULP || type == PROTOCOLID_FCOE || type == PROTOCOLID_IWARP; } static bool tm_cid_proto(enum protocol_type type) { - return type == PROTOCOLID_ISCSI || + return type == PROTOCOLID_TCP_ULP || type == PROTOCOLID_FCOE || type == PROTOCOLID_ROCE || type == PROTOCOLID_IWARP; @@ -2072,7 +2072,6 @@ int qed_cxt_set_pf_params(struct qed_hwfn *p_hwfn, u32 rdma_tasks) PROTOCOLID_FCOE, p_params->num_cons, 0); - qed_cxt_set_proto_tid_count(p_hwfn, PROTOCOLID_FCOE, QED_CXT_FCOE_TID_SEG, 0, p_params->num_tasks, true); @@ -2090,13 +2089,12 @@ int qed_cxt_set_pf_params(struct qed_hwfn *p_hwfn, u32 rdma_tasks) if (p_params->num_cons && p_params->num_tasks) { qed_cxt_set_proto_cid_count(p_hwfn, - PROTOCOLID_ISCSI, + PROTOCOLID_TCP_ULP, p_params->num_cons, 0); - qed_cxt_set_proto_tid_count(p_hwfn, - PROTOCOLID_ISCSI, - QED_CXT_ISCSI_TID_SEG, + PROTOCOLID_TCP_ULP, + QED_CXT_TCP_ULP_TID_SEG, 0, p_params->num_tasks, true); @@ -2106,6 +2104,29 @@ int qed_cxt_set_pf_params(struct qed_hwfn *p_hwfn, u32 rdma_tasks) } break; } + case QED_PCI_NVMETCP: + { + struct qed_nvmetcp_pf_params *p_params; + + p_params = &p_hwfn->pf_params.nvmetcp_pf_params; + + if (p_params->num_cons && p_params->num_tasks) { + qed_cxt_set_proto_cid_count(p_hwfn, + PROTOCOLID_TCP_ULP, + p_params->num_cons, + 0); + qed_cxt_set_proto_tid_count(p_hwfn, + PROTOCOLID_TCP_ULP, + QED_CXT_TCP_ULP_TID_SEG, + 0, + p_params->num_tasks, + true); + } else { + DP_INFO(p_hwfn->cdev, + "NvmeTCP personality used without setting params!\n"); + } + break; + } default: return -EINVAL; } @@ -2129,8 +2150,9 @@ int qed_cxt_get_tid_mem_info(struct qed_hwfn *p_hwfn, seg = QED_CXT_FCOE_TID_SEG; break; case QED_PCI_ISCSI: - proto = PROTOCOLID_ISCSI; - seg = QED_CXT_ISCSI_TID_SEG; + case QED_PCI_NVMETCP: + proto = PROTOCOLID_TCP_ULP; + seg = QED_CXT_TCP_ULP_TID_SEG; break; default: return -EINVAL; @@ -2455,8 +2477,9 @@ int qed_cxt_get_task_ctx(struct qed_hwfn *p_hwfn, seg = QED_CXT_FCOE_TID_SEG; break; case QED_PCI_ISCSI: - proto = PROTOCOLID_ISCSI; - seg = QED_CXT_ISCSI_TID_SEG; + case QED_PCI_NVMETCP: + proto = PROTOCOLID_TCP_ULP; + seg = QED_CXT_TCP_ULP_TID_SEG; break; default: return -EINVAL; diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.h b/drivers/net/ethernet/qlogic/qed/qed_cxt.h index 056e79620a0e..8adb7ed0c12d 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_cxt.h +++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.h @@ -50,7 +50,7 @@ int qed_cxt_get_cid_info(struct qed_hwfn *p_hwfn, int qed_cxt_get_tid_mem_info(struct qed_hwfn *p_hwfn, struct qed_tid_mem *p_info); -#define QED_CXT_ISCSI_TID_SEG PROTOCOLID_ISCSI +#define QED_CXT_TCP_ULP_TID_SEG PROTOCOLID_TCP_ULP #define QED_CXT_ROCE_TID_SEG PROTOCOLID_ROCE #define QED_CXT_FCOE_TID_SEG PROTOCOLID_FCOE enum qed_cxt_elem_type { diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c index 17d5b649eb36..e81dd34a3cac 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c +++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c @@ -1266,9 +1266,11 @@ int qed_dcbx_get_config_params(struct qed_hwfn *p_hwfn, p_hwfn->p_dcbx_info->set.ver_num |= DCBX_CONFIG_VERSION_STATIC; p_hwfn->p_dcbx_info->set.enabled = dcbx_info->operational.enabled; + BUILD_BUG_ON(sizeof(dcbx_info->operational.params) != + sizeof(p_hwfn->p_dcbx_info->set.config.params)); memcpy(&p_hwfn->p_dcbx_info->set.config.params, &dcbx_info->operational.params, - sizeof(struct qed_dcbx_admin_params)); + sizeof(p_hwfn->p_dcbx_info->set.config.params)); p_hwfn->p_dcbx_info->set.config.valid = true; memcpy(params, &p_hwfn->p_dcbx_info->set, sizeof(struct qed_dcbx_set)); diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c index d2f5855b2ea7..0410c3604abd 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_dev.c +++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c @@ -37,6 +37,7 @@ #include "qed_sriov.h" #include "qed_vf.h" #include "qed_rdma.h" +#include "qed_nvmetcp.h" static DEFINE_SPINLOCK(qm_lock); @@ -667,7 +668,8 @@ qed_llh_set_engine_affin(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) } /* Storage PF is bound to a single engine while L2 PF uses both */ - if (QED_IS_FCOE_PERSONALITY(p_hwfn) || QED_IS_ISCSI_PERSONALITY(p_hwfn)) + if (QED_IS_FCOE_PERSONALITY(p_hwfn) || QED_IS_ISCSI_PERSONALITY(p_hwfn) || + QED_IS_NVMETCP_PERSONALITY(p_hwfn)) eng = cdev->fir_affin ? QED_ENG1 : QED_ENG0; else /* L2_PERSONALITY */ eng = QED_BOTH_ENG; @@ -1164,6 +1166,9 @@ void qed_llh_remove_mac_filter(struct qed_dev *cdev, if (!test_bit(QED_MF_LLH_MAC_CLSS, &cdev->mf_bits)) goto out; + if (QED_IS_NVMETCP_PERSONALITY(p_hwfn)) + return; + ether_addr_copy(filter.mac.addr, mac_addr); rc = qed_llh_shadow_remove_filter(cdev, ppfid, &filter, &filter_idx, &ref_cnt); @@ -1381,6 +1386,11 @@ void qed_resc_free(struct qed_dev *cdev) qed_ooo_free(p_hwfn); } + if (p_hwfn->hw_info.personality == QED_PCI_NVMETCP) { + qed_nvmetcp_free(p_hwfn); + qed_ooo_free(p_hwfn); + } + if (QED_IS_RDMA_PERSONALITY(p_hwfn) && rdma_info) { qed_spq_unregister_async_cb(p_hwfn, rdma_info->proto); qed_rdma_info_free(p_hwfn); @@ -1423,6 +1433,7 @@ static u32 qed_get_pq_flags(struct qed_hwfn *p_hwfn) flags |= PQ_FLAGS_OFLD; break; case QED_PCI_ISCSI: + case QED_PCI_NVMETCP: flags |= PQ_FLAGS_ACK | PQ_FLAGS_OOO | PQ_FLAGS_OFLD; break; case QED_PCI_ETH_ROCE: @@ -2263,10 +2274,11 @@ int qed_resc_alloc(struct qed_dev *cdev) * at the same time */ n_eqes += num_cons + 2 * MAX_NUM_VFS_BB + n_srq; - } else if (p_hwfn->hw_info.personality == QED_PCI_ISCSI) { + } else if (p_hwfn->hw_info.personality == QED_PCI_ISCSI || + p_hwfn->hw_info.personality == QED_PCI_NVMETCP) { num_cons = qed_cxt_get_proto_cid_count(p_hwfn, - PROTOCOLID_ISCSI, + PROTOCOLID_TCP_ULP, NULL); n_eqes += 2 * num_cons; } @@ -2313,6 +2325,15 @@ int qed_resc_alloc(struct qed_dev *cdev) goto alloc_err; } + if (p_hwfn->hw_info.personality == QED_PCI_NVMETCP) { + rc = qed_nvmetcp_alloc(p_hwfn); + if (rc) + goto alloc_err; + rc = qed_ooo_alloc(p_hwfn); + if (rc) + goto alloc_err; + } + if (QED_IS_RDMA_PERSONALITY(p_hwfn)) { rc = qed_rdma_info_alloc(p_hwfn); if (rc) @@ -2393,6 +2414,11 @@ void qed_resc_setup(struct qed_dev *cdev) qed_iscsi_setup(p_hwfn); qed_ooo_setup(p_hwfn); } + + if (p_hwfn->hw_info.personality == QED_PCI_NVMETCP) { + qed_nvmetcp_setup(p_hwfn); + qed_ooo_setup(p_hwfn); + } } } @@ -2854,7 +2880,8 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn, /* Protocol Configuration */ STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_TCP_RT_OFFSET, - (p_hwfn->hw_info.personality == QED_PCI_ISCSI) ? 1 : 0); + ((p_hwfn->hw_info.personality == QED_PCI_ISCSI) || + (p_hwfn->hw_info.personality == QED_PCI_NVMETCP)) ? 1 : 0); STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_FCOE_RT_OFFSET, (p_hwfn->hw_info.personality == QED_PCI_FCOE) ? 1 : 0); STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_ROCE_RT_OFFSET, 0); @@ -3535,14 +3562,21 @@ static void qed_hw_set_feat(struct qed_hwfn *p_hwfn) feat_num[QED_ISCSI_CQ] = min_t(u32, sb_cnt.cnt, RESC_NUM(p_hwfn, QED_CMDQS_CQS)); + + if (QED_IS_NVMETCP_PERSONALITY(p_hwfn)) + feat_num[QED_NVMETCP_CQ] = min_t(u32, sb_cnt.cnt, + RESC_NUM(p_hwfn, + QED_CMDQS_CQS)); + DP_VERBOSE(p_hwfn, NETIF_MSG_PROBE, - "#PF_L2_QUEUES=%d VF_L2_QUEUES=%d #ROCE_CNQ=%d FCOE_CQ=%d ISCSI_CQ=%d #SBS=%d\n", + "#PF_L2_QUEUES=%d VF_L2_QUEUES=%d #ROCE_CNQ=%d FCOE_CQ=%d ISCSI_CQ=%d NVMETCP_CQ=%d #SBS=%d\n", (int)FEAT_NUM(p_hwfn, QED_PF_L2_QUE), (int)FEAT_NUM(p_hwfn, QED_VF_L2_QUE), (int)FEAT_NUM(p_hwfn, QED_RDMA_CNQ), (int)FEAT_NUM(p_hwfn, QED_FCOE_CQ), (int)FEAT_NUM(p_hwfn, QED_ISCSI_CQ), + (int)FEAT_NUM(p_hwfn, QED_NVMETCP_CQ), (int)sb_cnt.cnt); } @@ -3734,7 +3768,8 @@ int qed_hw_get_dflt_resc(struct qed_hwfn *p_hwfn, break; case QED_BDQ: if (p_hwfn->hw_info.personality != QED_PCI_ISCSI && - p_hwfn->hw_info.personality != QED_PCI_FCOE) + p_hwfn->hw_info.personality != QED_PCI_FCOE && + p_hwfn->hw_info.personality != QED_PCI_NVMETCP) *p_resc_num = 0; else *p_resc_num = 1; @@ -3755,7 +3790,8 @@ int qed_hw_get_dflt_resc(struct qed_hwfn *p_hwfn, *p_resc_start = 0; else if (p_hwfn->cdev->num_ports_in_engine == 4) *p_resc_start = p_hwfn->port_id; - else if (p_hwfn->hw_info.personality == QED_PCI_ISCSI) + else if (p_hwfn->hw_info.personality == QED_PCI_ISCSI || + p_hwfn->hw_info.personality == QED_PCI_NVMETCP) *p_resc_start = p_hwfn->port_id; else if (p_hwfn->hw_info.personality == QED_PCI_FCOE) *p_resc_start = p_hwfn->port_id + 2; @@ -5326,3 +5362,93 @@ void qed_set_fw_mac_addr(__le16 *fw_msb, ((u8 *)fw_lsb)[0] = mac[5]; ((u8 *)fw_lsb)[1] = mac[4]; } + +static int qed_llh_shadow_remove_all_filters(struct qed_dev *cdev, u8 ppfid) +{ + struct qed_llh_info *p_llh_info = cdev->p_llh_info; + struct qed_llh_filter_info *p_filters; + int rc; + + rc = qed_llh_shadow_sanity(cdev, ppfid, 0, "remove_all"); + if (rc) + return rc; + + p_filters = p_llh_info->pp_filters[ppfid]; + memset(p_filters, 0, NIG_REG_LLH_FUNC_FILTER_EN_SIZE * + sizeof(*p_filters)); + + return 0; +} + +static void qed_llh_clear_ppfid_filters(struct qed_dev *cdev, u8 ppfid) +{ + struct qed_hwfn *p_hwfn = QED_LEADING_HWFN(cdev); + struct qed_ptt *p_ptt = qed_ptt_acquire(p_hwfn); + u8 filter_idx, abs_ppfid; + int rc = 0; + + if (!p_ptt) + return; + + if (!test_bit(QED_MF_LLH_PROTO_CLSS, &cdev->mf_bits) && + !test_bit(QED_MF_LLH_MAC_CLSS, &cdev->mf_bits)) + goto out; + + rc = qed_llh_abs_ppfid(cdev, ppfid, &abs_ppfid); + if (rc) + goto out; + + rc = qed_llh_shadow_remove_all_filters(cdev, ppfid); + if (rc) + goto out; + + for (filter_idx = 0; filter_idx < NIG_REG_LLH_FUNC_FILTER_EN_SIZE; + filter_idx++) { + rc = qed_llh_remove_filter(p_hwfn, p_ptt, + abs_ppfid, filter_idx); + if (rc) + goto out; + } +out: + qed_ptt_release(p_hwfn, p_ptt); +} + +int qed_llh_add_src_tcp_port_filter(struct qed_dev *cdev, u16 src_port) +{ + return qed_llh_add_protocol_filter(cdev, 0, + QED_LLH_FILTER_TCP_SRC_PORT, + src_port, QED_LLH_DONT_CARE); +} + +void qed_llh_remove_src_tcp_port_filter(struct qed_dev *cdev, u16 src_port) +{ + qed_llh_remove_protocol_filter(cdev, 0, + QED_LLH_FILTER_TCP_SRC_PORT, + src_port, QED_LLH_DONT_CARE); +} + +int qed_llh_add_dst_tcp_port_filter(struct qed_dev *cdev, u16 dest_port) +{ + return qed_llh_add_protocol_filter(cdev, 0, + QED_LLH_FILTER_TCP_DEST_PORT, + QED_LLH_DONT_CARE, dest_port); +} + +void qed_llh_remove_dst_tcp_port_filter(struct qed_dev *cdev, u16 dest_port) +{ + qed_llh_remove_protocol_filter(cdev, 0, + QED_LLH_FILTER_TCP_DEST_PORT, + QED_LLH_DONT_CARE, dest_port); +} + +void qed_llh_clear_all_filters(struct qed_dev *cdev) +{ + u8 ppfid; + + if (!test_bit(QED_MF_LLH_PROTO_CLSS, &cdev->mf_bits) && + !test_bit(QED_MF_LLH_MAC_CLSS, &cdev->mf_bits)) + return; + + for (ppfid = 0; ppfid < cdev->p_llh_info->num_ppfid; ppfid++) + qed_llh_clear_ppfid_filters(cdev, ppfid); +} diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h index 559df9f4d656..fb1baa2da2d0 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h +++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h @@ -20,6 +20,7 @@ #include <linux/qed/fcoe_common.h> #include <linux/qed/eth_common.h> #include <linux/qed/iscsi_common.h> +#include <linux/qed/nvmetcp_common.h> #include <linux/qed/iwarp_common.h> #include <linux/qed/rdma_common.h> #include <linux/qed/roce_common.h> @@ -1118,7 +1119,7 @@ struct outer_tag_config_struct { /* personality per PF */ enum personality_type { BAD_PERSONALITY_TYP, - PERSONALITY_ISCSI, + PERSONALITY_TCP_ULP, PERSONALITY_FCOE, PERSONALITY_RDMA_AND_ETH, PERSONALITY_RDMA, @@ -12147,7 +12148,8 @@ struct public_func { #define FUNC_MF_CFG_PROTOCOL_ISCSI 0x00000010 #define FUNC_MF_CFG_PROTOCOL_FCOE 0x00000020 #define FUNC_MF_CFG_PROTOCOL_ROCE 0x00000030 -#define FUNC_MF_CFG_PROTOCOL_MAX 0x00000030 +#define FUNC_MF_CFG_PROTOCOL_NVMETCP 0x00000040 +#define FUNC_MF_CFG_PROTOCOL_MAX 0x00000040 #define FUNC_MF_CFG_MIN_BW_MASK 0x0000ff00 #define FUNC_MF_CFG_MIN_BW_SHIFT 8 diff --git a/drivers/net/ethernet/qlogic/qed/qed_iscsi.c b/drivers/net/ethernet/qlogic/qed/qed_iscsi.c index 4eae4ee3538f..db926d8b3033 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_iscsi.c +++ b/drivers/net/ethernet/qlogic/qed/qed_iscsi.c @@ -158,7 +158,7 @@ qed_sp_iscsi_func_start(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_INIT_FUNC, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; @@ -250,7 +250,7 @@ qed_sp_iscsi_func_start(struct qed_hwfn *p_hwfn, p_hwfn->p_iscsi_info->event_context = event_context; p_hwfn->p_iscsi_info->event_cb = async_event_cb; - qed_spq_register_async_cb(p_hwfn, PROTOCOLID_ISCSI, + qed_spq_register_async_cb(p_hwfn, PROTOCOLID_TCP_ULP, qed_iscsi_async_event); return qed_spq_post(p_hwfn, p_ent, NULL); @@ -286,7 +286,7 @@ static int qed_sp_iscsi_conn_offload(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_OFFLOAD_CONN, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; @@ -453,7 +453,7 @@ static int qed_sp_iscsi_conn_update(struct qed_hwfn *p_hwfn, struct iscsi_conn_update_ramrod_params *p_ramrod = NULL; struct qed_spq_entry *p_ent = NULL; struct qed_sp_init_data init_data; - int rc = -EINVAL; + int rc; u32 dval; /* Get SPQ entry */ @@ -465,7 +465,7 @@ static int qed_sp_iscsi_conn_update(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_UPDATE_CONN, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; @@ -506,7 +506,7 @@ qed_sp_iscsi_mac_update(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_MAC_UPDATE, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; @@ -548,7 +548,7 @@ static int qed_sp_iscsi_conn_terminate(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_TERMINATION_CONN, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; @@ -582,7 +582,7 @@ static int qed_sp_iscsi_conn_clear_sq(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_CLEAR_SQ, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; @@ -606,13 +606,13 @@ static int qed_sp_iscsi_func_stop(struct qed_hwfn *p_hwfn, rc = qed_sp_init_request(p_hwfn, &p_ent, ISCSI_RAMROD_CMD_ID_DESTROY_FUNC, - PROTOCOLID_ISCSI, &init_data); + PROTOCOLID_TCP_ULP, &init_data); if (rc) return rc; rc = qed_spq_post(p_hwfn, p_ent, NULL); - qed_spq_unregister_async_cb(p_hwfn, PROTOCOLID_ISCSI); + qed_spq_unregister_async_cb(p_hwfn, PROTOCOLID_TCP_ULP); return rc; } @@ -786,7 +786,7 @@ static int qed_iscsi_acquire_connection(struct qed_hwfn *p_hwfn, u32 icid; spin_lock_bh(&p_hwfn->p_iscsi_info->lock); - rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_ISCSI, &icid); + rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_TCP_ULP, &icid); spin_unlock_bh(&p_hwfn->p_iscsi_info->lock); if (rc) return rc; diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c index 49783f365079..02a4610d9330 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c +++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c @@ -960,7 +960,8 @@ static int qed_sp_ll2_rx_queue_start(struct qed_hwfn *p_hwfn, if (test_bit(QED_MF_LL2_NON_UNICAST, &p_hwfn->cdev->mf_bits) && p_ramrod->main_func_queue && conn_type != QED_LL2_TYPE_ROCE && - conn_type != QED_LL2_TYPE_IWARP) { + conn_type != QED_LL2_TYPE_IWARP && + (!QED_IS_NVMETCP_PERSONALITY(p_hwfn))) { p_ramrod->mf_si_bcast_accept_all = 1; p_ramrod->mf_si_mcast_accept_all = 1; } else { @@ -1037,8 +1038,8 @@ static int qed_sp_ll2_tx_queue_start(struct qed_hwfn *p_hwfn, case QED_LL2_TYPE_FCOE: p_ramrod->conn_type = PROTOCOLID_FCOE; break; - case QED_LL2_TYPE_ISCSI: - p_ramrod->conn_type = PROTOCOLID_ISCSI; + case QED_LL2_TYPE_TCP_ULP: + p_ramrod->conn_type = PROTOCOLID_TCP_ULP; break; case QED_LL2_TYPE_ROCE: p_ramrod->conn_type = PROTOCOLID_ROCE; @@ -1047,8 +1048,9 @@ static int qed_sp_ll2_tx_queue_start(struct qed_hwfn *p_hwfn, p_ramrod->conn_type = PROTOCOLID_IWARP; break; case QED_LL2_TYPE_OOO: - if (p_hwfn->hw_info.personality == QED_PCI_ISCSI) - p_ramrod->conn_type = PROTOCOLID_ISCSI; + if (p_hwfn->hw_info.personality == QED_PCI_ISCSI || + p_hwfn->hw_info.personality == QED_PCI_NVMETCP) + p_ramrod->conn_type = PROTOCOLID_TCP_ULP; else p_ramrod->conn_type = PROTOCOLID_IWARP; break; @@ -1634,7 +1636,8 @@ int qed_ll2_establish_connection(void *cxt, u8 connection_handle) if (rc) goto out; - if (!QED_IS_RDMA_PERSONALITY(p_hwfn)) + if (!QED_IS_RDMA_PERSONALITY(p_hwfn) && + !QED_IS_NVMETCP_PERSONALITY(p_hwfn)) qed_wr(p_hwfn, p_ptt, PRS_REG_USE_LIGHT_L2, 1); qed_ll2_establish_connection_ooo(p_hwfn, p_ll2_conn); @@ -2376,7 +2379,8 @@ out: static bool qed_ll2_is_storage_eng1(struct qed_dev *cdev) { return (QED_IS_FCOE_PERSONALITY(QED_LEADING_HWFN(cdev)) || - QED_IS_ISCSI_PERSONALITY(QED_LEADING_HWFN(cdev))) && + QED_IS_ISCSI_PERSONALITY(QED_LEADING_HWFN(cdev)) || + QED_IS_NVMETCP_PERSONALITY(QED_LEADING_HWFN(cdev))) && (QED_AFFIN_HWFN(cdev) != QED_LEADING_HWFN(cdev)); } @@ -2402,11 +2406,13 @@ static int qed_ll2_stop(struct qed_dev *cdev) if (cdev->ll2->handle == QED_LL2_UNUSED_HANDLE) return 0; + if (!QED_IS_NVMETCP_PERSONALITY(p_hwfn)) + qed_llh_remove_mac_filter(cdev, 0, cdev->ll2_mac_address); qed_llh_remove_mac_filter(cdev, 0, cdev->ll2_mac_address); eth_zero_addr(cdev->ll2_mac_address); - if (QED_IS_ISCSI_PERSONALITY(p_hwfn)) + if (QED_IS_ISCSI_PERSONALITY(p_hwfn) || QED_IS_NVMETCP_PERSONALITY(p_hwfn)) qed_ll2_stop_ooo(p_hwfn); /* In CMT mode, LL2 is always started on engine 0 for a storage PF */ @@ -2442,7 +2448,8 @@ static int __qed_ll2_start(struct qed_hwfn *p_hwfn, conn_type = QED_LL2_TYPE_FCOE; break; case QED_PCI_ISCSI: - conn_type = QED_LL2_TYPE_ISCSI; + case QED_PCI_NVMETCP: + conn_type = QED_LL2_TYPE_TCP_ULP; break; case QED_PCI_ETH_ROCE: conn_type = QED_LL2_TYPE_ROCE; @@ -2567,7 +2574,7 @@ static int qed_ll2_start(struct qed_dev *cdev, struct qed_ll2_params *params) } } - if (QED_IS_ISCSI_PERSONALITY(p_hwfn)) { + if (QED_IS_ISCSI_PERSONALITY(p_hwfn) || QED_IS_NVMETCP_PERSONALITY(p_hwfn)) { DP_VERBOSE(cdev, QED_MSG_STORAGE, "Starting OOO LL2 queue\n"); rc = qed_ll2_start_ooo(p_hwfn, params); if (rc) { @@ -2576,10 +2583,13 @@ static int qed_ll2_start(struct qed_dev *cdev, struct qed_ll2_params *params) } } - rc = qed_llh_add_mac_filter(cdev, 0, params->ll2_mac_address); - if (rc) { - DP_NOTICE(cdev, "Failed to add an LLH filter\n"); - goto err3; + if (!QED_IS_NVMETCP_PERSONALITY(p_hwfn)) { + rc = qed_llh_add_mac_filter(cdev, 0, params->ll2_mac_address); + if (rc) { + DP_NOTICE(cdev, "Failed to add an LLH filter\n"); + goto err3; + } + } ether_addr_copy(cdev->ll2_mac_address, params->ll2_mac_address); @@ -2587,7 +2597,7 @@ static int qed_ll2_start(struct qed_dev *cdev, struct qed_ll2_params *params) return 0; err3: - if (QED_IS_ISCSI_PERSONALITY(p_hwfn)) + if (QED_IS_ISCSI_PERSONALITY(p_hwfn) || QED_IS_NVMETCP_PERSONALITY(p_hwfn)) qed_ll2_stop_ooo(p_hwfn); err2: if (b_is_storage_eng1) diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c index cd882c453394..4387292c37e2 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c +++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c @@ -2446,6 +2446,9 @@ qed_mcp_get_shmem_proto(struct qed_hwfn *p_hwfn, case FUNC_MF_CFG_PROTOCOL_ISCSI: *p_proto = QED_PCI_ISCSI; break; + case FUNC_MF_CFG_PROTOCOL_NVMETCP: + *p_proto = QED_PCI_NVMETCP; + break; case FUNC_MF_CFG_PROTOCOL_FCOE: *p_proto = QED_PCI_FCOE; break; diff --git a/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c b/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c index 3e3192a3ad9b..6190adf965bc 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c +++ b/drivers/net/ethernet/qlogic/qed/qed_mng_tlv.c @@ -1306,7 +1306,8 @@ int qed_mfw_process_tlv_req(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) } if ((tlv_group & QED_MFW_TLV_ISCSI) && - p_hwfn->hw_info.personality != QED_PCI_ISCSI) { + p_hwfn->hw_info.personality != QED_PCI_ISCSI && + p_hwfn->hw_info.personality != QED_PCI_NVMETCP) { DP_VERBOSE(p_hwfn, QED_MSG_SP, "Skipping iSCSI TLVs for non-iSCSI function\n"); tlv_group &= ~QED_MFW_TLV_ISCSI; diff --git a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp.c b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp.c new file mode 100644 index 000000000000..f19128c8d9cc --- /dev/null +++ b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp.c @@ -0,0 +1,829 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +/* Copyright 2021 Marvell. All rights reserved. */ + +#include <linux/types.h> +#include <asm/byteorder.h> +#include <asm/param.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/etherdevice.h> +#include <linux/kernel.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/stddef.h> +#include <linux/string.h> +#include <linux/errno.h> +#include <linux/list.h> +#include <linux/qed/qed_nvmetcp_if.h> +#include "qed.h" +#include "qed_cxt.h" +#include "qed_dev_api.h" +#include "qed_hsi.h" +#include "qed_hw.h" +#include "qed_int.h" +#include "qed_nvmetcp.h" +#include "qed_ll2.h" +#include "qed_mcp.h" +#include "qed_sp.h" +#include "qed_reg_addr.h" +#include "qed_nvmetcp_fw_funcs.h" + +static int qed_nvmetcp_async_event(struct qed_hwfn *p_hwfn, u8 fw_event_code, + u16 echo, union event_ring_data *data, + u8 fw_return_code) +{ + if (p_hwfn->p_nvmetcp_info->event_cb) { + struct qed_nvmetcp_info *p_nvmetcp = p_hwfn->p_nvmetcp_info; + + return p_nvmetcp->event_cb(p_nvmetcp->event_context, + fw_event_code, data); + } else { + DP_NOTICE(p_hwfn, "nvmetcp async completion is not set\n"); + + return -EINVAL; + } +} + +static int qed_sp_nvmetcp_func_start(struct qed_hwfn *p_hwfn, + enum spq_mode comp_mode, + struct qed_spq_comp_cb *p_comp_addr, + void *event_context, + nvmetcp_event_cb_t async_event_cb) +{ + struct nvmetcp_init_ramrod_params *p_ramrod = NULL; + struct qed_nvmetcp_pf_params *p_params = NULL; + struct scsi_init_func_queues *p_queue = NULL; + struct nvmetcp_spe_func_init *p_init = NULL; + struct qed_sp_init_data init_data = {}; + struct qed_spq_entry *p_ent = NULL; + int rc = 0; + u16 val; + u8 i; + + /* Get SPQ entry */ + init_data.cid = qed_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = comp_mode; + init_data.p_comp_data = p_comp_addr; + rc = qed_sp_init_request(p_hwfn, &p_ent, + NVMETCP_RAMROD_CMD_ID_INIT_FUNC, + PROTOCOLID_TCP_ULP, &init_data); + if (rc) + return rc; + + p_ramrod = &p_ent->ramrod.nvmetcp_init; + p_init = &p_ramrod->nvmetcp_init_spe; + p_params = &p_hwfn->pf_params.nvmetcp_pf_params; + p_queue = &p_init->q_params; + p_init->num_sq_pages_in_ring = p_params->num_sq_pages_in_ring; + p_init->num_r2tq_pages_in_ring = p_params->num_r2tq_pages_in_ring; + p_init->num_uhq_pages_in_ring = p_params->num_uhq_pages_in_ring; + p_init->ll2_rx_queue_id = RESC_START(p_hwfn, QED_LL2_RAM_QUEUE) + + p_params->ll2_ooo_queue_id; + SET_FIELD(p_init->flags, NVMETCP_SPE_FUNC_INIT_NVMETCP_MODE, 1); + p_init->func_params.log_page_size = ilog2(PAGE_SIZE); + p_init->func_params.num_tasks = cpu_to_le16(p_params->num_tasks); + p_init->debug_flags = p_params->debug_mode; + DMA_REGPAIR_LE(p_queue->glbl_q_params_addr, + p_params->glbl_q_params_addr); + p_queue->cq_num_entries = cpu_to_le16(QED_NVMETCP_FW_CQ_SIZE); + p_queue->num_queues = p_params->num_queues; + val = RESC_START(p_hwfn, QED_CMDQS_CQS); + p_queue->queue_relative_offset = cpu_to_le16((u16)val); + p_queue->cq_sb_pi = p_params->gl_rq_pi; + + for (i = 0; i < p_params->num_queues; i++) { + val = qed_get_igu_sb_id(p_hwfn, i); + p_queue->cq_cmdq_sb_num_arr[i] = cpu_to_le16(val); + } + + SET_FIELD(p_queue->q_validity, + SCSI_INIT_FUNC_QUEUES_CMD_VALID, 0); + p_queue->cmdq_num_entries = 0; + p_queue->bdq_resource_id = (u8)RESC_START(p_hwfn, QED_BDQ); + p_ramrod->tcp_init.two_msl_timer = cpu_to_le32(QED_TCP_TWO_MSL_TIMER); + p_ramrod->tcp_init.tx_sws_timer = cpu_to_le16(QED_TCP_SWS_TIMER); + p_init->half_way_close_timeout = cpu_to_le16(QED_TCP_HALF_WAY_CLOSE_TIMEOUT); + p_ramrod->tcp_init.max_fin_rt = QED_TCP_MAX_FIN_RT; + SET_FIELD(p_ramrod->nvmetcp_init_spe.params, + NVMETCP_SPE_FUNC_INIT_MAX_SYN_RT, QED_TCP_MAX_FIN_RT); + p_hwfn->p_nvmetcp_info->event_context = event_context; + p_hwfn->p_nvmetcp_info->event_cb = async_event_cb; + qed_spq_register_async_cb(p_hwfn, PROTOCOLID_TCP_ULP, + qed_nvmetcp_async_event); + + return qed_spq_post(p_hwfn, p_ent, NULL); +} + +static int qed_sp_nvmetcp_func_stop(struct qed_hwfn *p_hwfn, + enum spq_mode comp_mode, + struct qed_spq_comp_cb *p_comp_addr) +{ + struct qed_spq_entry *p_ent = NULL; + struct qed_sp_init_data init_data; + int rc; + + /* Get SPQ entry */ + memset(&init_data, 0, sizeof(init_data)); + init_data.cid = qed_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = comp_mode; + init_data.p_comp_data = p_comp_addr; + rc = qed_sp_init_request(p_hwfn, &p_ent, + NVMETCP_RAMROD_CMD_ID_DESTROY_FUNC, + PROTOCOLID_TCP_ULP, &init_data); + if (rc) + return rc; + + rc = qed_spq_post(p_hwfn, p_ent, NULL); + qed_spq_unregister_async_cb(p_hwfn, PROTOCOLID_TCP_ULP); + + return rc; +} + +static int qed_fill_nvmetcp_dev_info(struct qed_dev *cdev, + struct qed_dev_nvmetcp_info *info) +{ + struct qed_hwfn *hwfn = QED_AFFIN_HWFN(cdev); + int rc; + + memset(info, 0, sizeof(*info)); + rc = qed_fill_dev_info(cdev, &info->common); + info->port_id = MFW_PORT(hwfn); + info->num_cqs = FEAT_NUM(hwfn, QED_NVMETCP_CQ); + + return rc; +} + +static void qed_register_nvmetcp_ops(struct qed_dev *cdev, + struct qed_nvmetcp_cb_ops *ops, + void *cookie) +{ + cdev->protocol_ops.nvmetcp = ops; + cdev->ops_cookie = cookie; +} + +static int qed_nvmetcp_stop(struct qed_dev *cdev) +{ + int rc; + + if (!(cdev->flags & QED_FLAG_STORAGE_STARTED)) { + DP_NOTICE(cdev, "nvmetcp already stopped\n"); + + return 0; + } + + if (!hash_empty(cdev->connections)) { + DP_NOTICE(cdev, + "Can't stop nvmetcp - not all connections were returned\n"); + + return -EINVAL; + } + + /* Stop the nvmetcp */ + rc = qed_sp_nvmetcp_func_stop(QED_AFFIN_HWFN(cdev), QED_SPQ_MODE_EBLOCK, + NULL); + cdev->flags &= ~QED_FLAG_STORAGE_STARTED; + + return rc; +} + +static int qed_nvmetcp_start(struct qed_dev *cdev, + struct qed_nvmetcp_tid *tasks, + void *event_context, + nvmetcp_event_cb_t async_event_cb) +{ + struct qed_tid_mem *tid_info; + int rc; + + if (cdev->flags & QED_FLAG_STORAGE_STARTED) { + DP_NOTICE(cdev, "nvmetcp already started;\n"); + + return 0; + } + + rc = qed_sp_nvmetcp_func_start(QED_AFFIN_HWFN(cdev), + QED_SPQ_MODE_EBLOCK, NULL, + event_context, async_event_cb); + if (rc) { + DP_NOTICE(cdev, "Failed to start nvmetcp\n"); + + return rc; + } + + cdev->flags |= QED_FLAG_STORAGE_STARTED; + hash_init(cdev->connections); + + if (!tasks) + return 0; + + tid_info = kzalloc(sizeof(*tid_info), GFP_KERNEL); + if (!tid_info) { + qed_nvmetcp_stop(cdev); + + return -ENOMEM; + } + + rc = qed_cxt_get_tid_mem_info(QED_AFFIN_HWFN(cdev), tid_info); + if (rc) { + DP_NOTICE(cdev, "Failed to gather task information\n"); + qed_nvmetcp_stop(cdev); + kfree(tid_info); + + return rc; + } + + /* Fill task information */ + tasks->size = tid_info->tid_size; + tasks->num_tids_per_block = tid_info->num_tids_per_block; + memcpy(tasks->blocks, tid_info->blocks, + MAX_TID_BLOCKS_NVMETCP * sizeof(u8 *)); + kfree(tid_info); + + return 0; +} + +static struct qed_hash_nvmetcp_con *qed_nvmetcp_get_hash(struct qed_dev *cdev, + u32 handle) +{ + struct qed_hash_nvmetcp_con *hash_con = NULL; + + if (!(cdev->flags & QED_FLAG_STORAGE_STARTED)) + return NULL; + + hash_for_each_possible(cdev->connections, hash_con, node, handle) { + if (hash_con->con->icid == handle) + break; + } + + if (!hash_con || hash_con->con->icid != handle) + return NULL; + + return hash_con; +} + +static int qed_sp_nvmetcp_conn_offload(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn *p_conn, + enum spq_mode comp_mode, + struct qed_spq_comp_cb *p_comp_addr) +{ + struct nvmetcp_spe_conn_offload *p_ramrod = NULL; + struct tcp_offload_params_opt2 *p_tcp = NULL; + struct qed_sp_init_data init_data = { 0 }; + struct qed_spq_entry *p_ent = NULL; + dma_addr_t r2tq_pbl_addr; + dma_addr_t xhq_pbl_addr; + dma_addr_t uhq_pbl_addr; + u16 physical_q; + int rc = 0; + u8 i; + + /* Get SPQ entry */ + init_data.cid = p_conn->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = comp_mode; + init_data.p_comp_data = p_comp_addr; + rc = qed_sp_init_request(p_hwfn, &p_ent, + NVMETCP_RAMROD_CMD_ID_OFFLOAD_CONN, + PROTOCOLID_TCP_ULP, &init_data); + if (rc) + return rc; + + p_ramrod = &p_ent->ramrod.nvmetcp_conn_offload; + + /* Transmission PQ is the first of the PF */ + physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); + p_conn->physical_q0 = cpu_to_le16(physical_q); + p_ramrod->nvmetcp.physical_q0 = cpu_to_le16(physical_q); + + /* nvmetcp Pure-ACK PQ */ + physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK); + p_conn->physical_q1 = cpu_to_le16(physical_q); + p_ramrod->nvmetcp.physical_q1 = cpu_to_le16(physical_q); + p_ramrod->conn_id = cpu_to_le16(p_conn->conn_id); + DMA_REGPAIR_LE(p_ramrod->nvmetcp.sq_pbl_addr, p_conn->sq_pbl_addr); + r2tq_pbl_addr = qed_chain_get_pbl_phys(&p_conn->r2tq); + DMA_REGPAIR_LE(p_ramrod->nvmetcp.r2tq_pbl_addr, r2tq_pbl_addr); + xhq_pbl_addr = qed_chain_get_pbl_phys(&p_conn->xhq); + DMA_REGPAIR_LE(p_ramrod->nvmetcp.xhq_pbl_addr, xhq_pbl_addr); + uhq_pbl_addr = qed_chain_get_pbl_phys(&p_conn->uhq); + DMA_REGPAIR_LE(p_ramrod->nvmetcp.uhq_pbl_addr, uhq_pbl_addr); + p_ramrod->nvmetcp.flags = p_conn->offl_flags; + p_ramrod->nvmetcp.default_cq = p_conn->default_cq; + p_ramrod->nvmetcp.initial_ack = 0; + DMA_REGPAIR_LE(p_ramrod->nvmetcp.nvmetcp.cccid_itid_table_addr, + p_conn->nvmetcp_cccid_itid_table_addr); + p_ramrod->nvmetcp.nvmetcp.cccid_max_range = + cpu_to_le16(p_conn->nvmetcp_cccid_max_range); + p_tcp = &p_ramrod->tcp; + qed_set_fw_mac_addr(&p_tcp->remote_mac_addr_hi, + &p_tcp->remote_mac_addr_mid, + &p_tcp->remote_mac_addr_lo, p_conn->remote_mac); + qed_set_fw_mac_addr(&p_tcp->local_mac_addr_hi, + &p_tcp->local_mac_addr_mid, + &p_tcp->local_mac_addr_lo, p_conn->local_mac); + p_tcp->vlan_id = cpu_to_le16(p_conn->vlan_id); + p_tcp->flags = cpu_to_le16(p_conn->tcp_flags); + p_tcp->ip_version = p_conn->ip_version; + if (p_tcp->ip_version == TCP_IPV6) { + for (i = 0; i < 4; i++) { + p_tcp->remote_ip[i] = cpu_to_le32(p_conn->remote_ip[i]); + p_tcp->local_ip[i] = cpu_to_le32(p_conn->local_ip[i]); + } + } else { + p_tcp->remote_ip[0] = cpu_to_le32(p_conn->remote_ip[0]); + p_tcp->local_ip[0] = cpu_to_le32(p_conn->local_ip[0]); + } + + p_tcp->flow_label = cpu_to_le32(p_conn->flow_label); + p_tcp->ttl = p_conn->ttl; + p_tcp->tos_or_tc = p_conn->tos_or_tc; + p_tcp->remote_port = cpu_to_le16(p_conn->remote_port); + p_tcp->local_port = cpu_to_le16(p_conn->local_port); + p_tcp->mss = cpu_to_le16(p_conn->mss); + p_tcp->rcv_wnd_scale = p_conn->rcv_wnd_scale; + p_tcp->connect_mode = p_conn->connect_mode; + p_tcp->cwnd = cpu_to_le32(p_conn->cwnd); + p_tcp->ka_max_probe_cnt = p_conn->ka_max_probe_cnt; + p_tcp->ka_timeout = cpu_to_le32(p_conn->ka_timeout); + p_tcp->max_rt_time = cpu_to_le32(p_conn->max_rt_time); + p_tcp->ka_interval = cpu_to_le32(p_conn->ka_interval); + + return qed_spq_post(p_hwfn, p_ent, NULL); +} + +static int qed_sp_nvmetcp_conn_update(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn *p_conn, + enum spq_mode comp_mode, + struct qed_spq_comp_cb *p_comp_addr) +{ + struct nvmetcp_conn_update_ramrod_params *p_ramrod = NULL; + struct qed_spq_entry *p_ent = NULL; + struct qed_sp_init_data init_data; + int rc = -EINVAL; + u32 dval; + + /* Get SPQ entry */ + memset(&init_data, 0, sizeof(init_data)); + init_data.cid = p_conn->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = comp_mode; + init_data.p_comp_data = p_comp_addr; + + rc = qed_sp_init_request(p_hwfn, &p_ent, + NVMETCP_RAMROD_CMD_ID_UPDATE_CONN, + PROTOCOLID_TCP_ULP, &init_data); + if (rc) + return rc; + + p_ramrod = &p_ent->ramrod.nvmetcp_conn_update; + p_ramrod->conn_id = cpu_to_le16(p_conn->conn_id); + p_ramrod->flags = p_conn->update_flag; + p_ramrod->max_seq_size = cpu_to_le32(p_conn->max_seq_size); + dval = p_conn->max_recv_pdu_length; + p_ramrod->max_recv_pdu_length = cpu_to_le32(dval); + dval = p_conn->max_send_pdu_length; + p_ramrod->max_send_pdu_length = cpu_to_le32(dval); + p_ramrod->first_seq_length = cpu_to_le32(p_conn->first_seq_length); + + return qed_spq_post(p_hwfn, p_ent, NULL); +} + +static int qed_sp_nvmetcp_conn_terminate(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn *p_conn, + enum spq_mode comp_mode, + struct qed_spq_comp_cb *p_comp_addr) +{ + struct nvmetcp_spe_conn_termination *p_ramrod = NULL; + struct qed_spq_entry *p_ent = NULL; + struct qed_sp_init_data init_data; + int rc = -EINVAL; + + /* Get SPQ entry */ + memset(&init_data, 0, sizeof(init_data)); + init_data.cid = p_conn->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = comp_mode; + init_data.p_comp_data = p_comp_addr; + rc = qed_sp_init_request(p_hwfn, &p_ent, + NVMETCP_RAMROD_CMD_ID_TERMINATION_CONN, + PROTOCOLID_TCP_ULP, &init_data); + if (rc) + return rc; + + p_ramrod = &p_ent->ramrod.nvmetcp_conn_terminate; + p_ramrod->conn_id = cpu_to_le16(p_conn->conn_id); + p_ramrod->abortive = p_conn->abortive_dsconnect; + + return qed_spq_post(p_hwfn, p_ent, NULL); +} + +static int qed_sp_nvmetcp_conn_clear_sq(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn *p_conn, + enum spq_mode comp_mode, + struct qed_spq_comp_cb *p_comp_addr) +{ + struct qed_spq_entry *p_ent = NULL; + struct qed_sp_init_data init_data; + int rc = -EINVAL; + + /* Get SPQ entry */ + memset(&init_data, 0, sizeof(init_data)); + init_data.cid = p_conn->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = comp_mode; + init_data.p_comp_data = p_comp_addr; + rc = qed_sp_init_request(p_hwfn, &p_ent, + NVMETCP_RAMROD_CMD_ID_CLEAR_SQ, + PROTOCOLID_TCP_ULP, &init_data); + if (rc) + return rc; + + return qed_spq_post(p_hwfn, p_ent, NULL); +} + +static void __iomem *qed_nvmetcp_get_db_addr(struct qed_hwfn *p_hwfn, u32 cid) +{ + return (u8 __iomem *)p_hwfn->doorbells + + qed_db_addr(cid, DQ_DEMS_LEGACY); +} + +static int qed_nvmetcp_allocate_connection(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn **p_out_conn) +{ + struct qed_chain_init_params params = { + .mode = QED_CHAIN_MODE_PBL, + .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE, + .cnt_type = QED_CHAIN_CNT_TYPE_U16, + }; + struct qed_nvmetcp_pf_params *p_params = NULL; + struct qed_nvmetcp_conn *p_conn = NULL; + int rc = 0; + + /* Try finding a free connection that can be used */ + spin_lock_bh(&p_hwfn->p_nvmetcp_info->lock); + if (!list_empty(&p_hwfn->p_nvmetcp_info->free_list)) + p_conn = list_first_entry(&p_hwfn->p_nvmetcp_info->free_list, + struct qed_nvmetcp_conn, list_entry); + if (p_conn) { + list_del(&p_conn->list_entry); + spin_unlock_bh(&p_hwfn->p_nvmetcp_info->lock); + *p_out_conn = p_conn; + + return 0; + } + spin_unlock_bh(&p_hwfn->p_nvmetcp_info->lock); + + /* Need to allocate a new connection */ + p_params = &p_hwfn->pf_params.nvmetcp_pf_params; + p_conn = kzalloc(sizeof(*p_conn), GFP_KERNEL); + if (!p_conn) + return -ENOMEM; + + params.num_elems = p_params->num_r2tq_pages_in_ring * + QED_CHAIN_PAGE_SIZE / sizeof(struct nvmetcp_wqe); + params.elem_size = sizeof(struct nvmetcp_wqe); + rc = qed_chain_alloc(p_hwfn->cdev, &p_conn->r2tq, ¶ms); + if (rc) + goto nomem_r2tq; + + params.num_elems = p_params->num_uhq_pages_in_ring * + QED_CHAIN_PAGE_SIZE / sizeof(struct iscsi_uhqe); + params.elem_size = sizeof(struct iscsi_uhqe); + rc = qed_chain_alloc(p_hwfn->cdev, &p_conn->uhq, ¶ms); + if (rc) + goto nomem_uhq; + + params.elem_size = sizeof(struct iscsi_xhqe); + rc = qed_chain_alloc(p_hwfn->cdev, &p_conn->xhq, ¶ms); + if (rc) + goto nomem; + + p_conn->free_on_delete = true; + *p_out_conn = p_conn; + + return 0; + +nomem: + qed_chain_free(p_hwfn->cdev, &p_conn->uhq); +nomem_uhq: + qed_chain_free(p_hwfn->cdev, &p_conn->r2tq); +nomem_r2tq: + kfree(p_conn); + + return -ENOMEM; +} + +static int qed_nvmetcp_acquire_connection(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn **p_out_conn) +{ + struct qed_nvmetcp_conn *p_conn = NULL; + int rc = 0; + u32 icid; + + spin_lock_bh(&p_hwfn->p_nvmetcp_info->lock); + rc = qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_TCP_ULP, &icid); + spin_unlock_bh(&p_hwfn->p_nvmetcp_info->lock); + + if (rc) + return rc; + + rc = qed_nvmetcp_allocate_connection(p_hwfn, &p_conn); + if (rc) { + spin_lock_bh(&p_hwfn->p_nvmetcp_info->lock); + qed_cxt_release_cid(p_hwfn, icid); + spin_unlock_bh(&p_hwfn->p_nvmetcp_info->lock); + + return rc; + } + + p_conn->icid = icid; + p_conn->conn_id = (u16)icid; + p_conn->fw_cid = (p_hwfn->hw_info.opaque_fid << 16) | icid; + *p_out_conn = p_conn; + + return rc; +} + +static void qed_nvmetcp_release_connection(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn *p_conn) +{ + spin_lock_bh(&p_hwfn->p_nvmetcp_info->lock); + list_add_tail(&p_conn->list_entry, &p_hwfn->p_nvmetcp_info->free_list); + qed_cxt_release_cid(p_hwfn, p_conn->icid); + spin_unlock_bh(&p_hwfn->p_nvmetcp_info->lock); +} + +static void qed_nvmetcp_free_connection(struct qed_hwfn *p_hwfn, + struct qed_nvmetcp_conn *p_conn) +{ + qed_chain_free(p_hwfn->cdev, &p_conn->xhq); + qed_chain_free(p_hwfn->cdev, &p_conn->uhq); + qed_chain_free(p_hwfn->cdev, &p_conn->r2tq); + kfree(p_conn); +} + +int qed_nvmetcp_alloc(struct qed_hwfn *p_hwfn) +{ + struct qed_nvmetcp_info *p_nvmetcp_info; + + p_nvmetcp_info = kzalloc(sizeof(*p_nvmetcp_info), GFP_KERNEL); + if (!p_nvmetcp_info) + return -ENOMEM; + + INIT_LIST_HEAD(&p_nvmetcp_info->free_list); + p_hwfn->p_nvmetcp_info = p_nvmetcp_info; + + return 0; +} + +void qed_nvmetcp_setup(struct qed_hwfn *p_hwfn) +{ + spin_lock_init(&p_hwfn->p_nvmetcp_info->lock); +} + +void qed_nvmetcp_free(struct qed_hwfn *p_hwfn) +{ + struct qed_nvmetcp_conn *p_conn = NULL; + + if (!p_hwfn->p_nvmetcp_info) + return; + + while (!list_empty(&p_hwfn->p_nvmetcp_info->free_list)) { + p_conn = list_first_entry(&p_hwfn->p_nvmetcp_info->free_list, + struct qed_nvmetcp_conn, list_entry); + if (p_conn) { + list_del(&p_conn->list_entry); + qed_nvmetcp_free_connection(p_hwfn, p_conn); + } + } + + kfree(p_hwfn->p_nvmetcp_info); + p_hwfn->p_nvmetcp_info = NULL; +} + +static int qed_nvmetcp_acquire_conn(struct qed_dev *cdev, + u32 *handle, + u32 *fw_cid, void __iomem **p_doorbell) +{ + struct qed_hash_nvmetcp_con *hash_con; + int rc; + + /* Allocate a hashed connection */ + hash_con = kzalloc(sizeof(*hash_con), GFP_ATOMIC); + if (!hash_con) + return -ENOMEM; + + /* Acquire the connection */ + rc = qed_nvmetcp_acquire_connection(QED_AFFIN_HWFN(cdev), + &hash_con->con); + if (rc) { + DP_NOTICE(cdev, "Failed to acquire Connection\n"); + kfree(hash_con); + + return rc; + } + + /* Added the connection to hash table */ + *handle = hash_con->con->icid; + *fw_cid = hash_con->con->fw_cid; + hash_add(cdev->connections, &hash_con->node, *handle); + if (p_doorbell) + *p_doorbell = qed_nvmetcp_get_db_addr(QED_AFFIN_HWFN(cdev), + *handle); + + return 0; +} + +static int qed_nvmetcp_release_conn(struct qed_dev *cdev, u32 handle) +{ + struct qed_hash_nvmetcp_con *hash_con; + + hash_con = qed_nvmetcp_get_hash(cdev, handle); + if (!hash_con) { + DP_NOTICE(cdev, "Failed to find connection for handle %d\n", + handle); + + return -EINVAL; + } + + hlist_del(&hash_con->node); + qed_nvmetcp_release_connection(QED_AFFIN_HWFN(cdev), hash_con->con); + kfree(hash_con); + + return 0; +} + +static int qed_nvmetcp_offload_conn(struct qed_dev *cdev, u32 handle, + struct qed_nvmetcp_params_offload *conn_info) +{ + struct qed_hash_nvmetcp_con *hash_con; + struct qed_nvmetcp_conn *con; + + hash_con = qed_nvmetcp_get_hash(cdev, handle); + if (!hash_con) { + DP_NOTICE(cdev, "Failed to find connection for handle %d\n", + handle); + + return -EINVAL; + } + + /* Update the connection with information from the params */ + con = hash_con->con; + + /* FW initializations */ + con->layer_code = NVMETCP_SLOW_PATH_LAYER_CODE; + con->sq_pbl_addr = conn_info->sq_pbl_addr; + con->nvmetcp_cccid_max_range = conn_info->nvmetcp_cccid_max_range; + con->nvmetcp_cccid_itid_table_addr = conn_info->nvmetcp_cccid_itid_table_addr; + con->default_cq = conn_info->default_cq; + SET_FIELD(con->offl_flags, NVMETCP_CONN_OFFLOAD_PARAMS_TARGET_MODE, 0); + SET_FIELD(con->offl_flags, NVMETCP_CONN_OFFLOAD_PARAMS_NVMETCP_MODE, 1); + SET_FIELD(con->offl_flags, NVMETCP_CONN_OFFLOAD_PARAMS_TCP_ON_CHIP_1B, 1); + + /* Networking and TCP stack initializations */ + ether_addr_copy(con->local_mac, conn_info->src.mac); + ether_addr_copy(con->remote_mac, conn_info->dst.mac); + memcpy(con->local_ip, conn_info->src.ip, sizeof(con->local_ip)); + memcpy(con->remote_ip, conn_info->dst.ip, sizeof(con->remote_ip)); + con->local_port = conn_info->src.port; + con->remote_port = conn_info->dst.port; + con->vlan_id = conn_info->vlan_id; + + if (conn_info->timestamp_en) + SET_FIELD(con->tcp_flags, TCP_OFFLOAD_PARAMS_OPT2_TS_EN, 1); + + if (conn_info->delayed_ack_en) + SET_FIELD(con->tcp_flags, TCP_OFFLOAD_PARAMS_OPT2_DA_EN, 1); + + if (conn_info->tcp_keep_alive_en) + SET_FIELD(con->tcp_flags, TCP_OFFLOAD_PARAMS_OPT2_KA_EN, 1); + + if (conn_info->ecn_en) + SET_FIELD(con->tcp_flags, TCP_OFFLOAD_PARAMS_OPT2_ECN_EN, 1); + + con->ip_version = conn_info->ip_version; + con->flow_label = QED_TCP_FLOW_LABEL; + con->ka_max_probe_cnt = conn_info->ka_max_probe_cnt; + con->ka_timeout = conn_info->ka_timeout; + con->ka_interval = conn_info->ka_interval; + con->max_rt_time = conn_info->max_rt_time; + con->ttl = conn_info->ttl; + con->tos_or_tc = conn_info->tos_or_tc; + con->mss = conn_info->mss; + con->cwnd = conn_info->cwnd; + con->rcv_wnd_scale = conn_info->rcv_wnd_scale; + con->connect_mode = 0; + + return qed_sp_nvmetcp_conn_offload(QED_AFFIN_HWFN(cdev), con, + QED_SPQ_MODE_EBLOCK, NULL); +} + +static int qed_nvmetcp_update_conn(struct qed_dev *cdev, + u32 handle, + struct qed_nvmetcp_params_update *conn_info) +{ + struct qed_hash_nvmetcp_con *hash_con; + struct qed_nvmetcp_conn *con; + + hash_con = qed_nvmetcp_get_hash(cdev, handle); + if (!hash_con) { + DP_NOTICE(cdev, "Failed to find connection for handle %d\n", + handle); + + return -EINVAL; + } + + /* Update the connection with information from the params */ + con = hash_con->con; + SET_FIELD(con->update_flag, + ISCSI_CONN_UPDATE_RAMROD_PARAMS_INITIAL_R2T, 0); + SET_FIELD(con->update_flag, + ISCSI_CONN_UPDATE_RAMROD_PARAMS_IMMEDIATE_DATA, 1); + if (conn_info->hdr_digest_en) + SET_FIELD(con->update_flag, ISCSI_CONN_UPDATE_RAMROD_PARAMS_HD_EN, 1); + + if (conn_info->data_digest_en) + SET_FIELD(con->update_flag, ISCSI_CONN_UPDATE_RAMROD_PARAMS_DD_EN, 1); + + /* Placeholder - initialize pfv, cpda, hpda */ + + con->max_seq_size = conn_info->max_io_size; + con->max_recv_pdu_length = conn_info->max_recv_pdu_length; + con->max_send_pdu_length = conn_info->max_send_pdu_length; + con->first_seq_length = conn_info->max_io_size; + + return qed_sp_nvmetcp_conn_update(QED_AFFIN_HWFN(cdev), con, + QED_SPQ_MODE_EBLOCK, NULL); +} + +static int qed_nvmetcp_clear_conn_sq(struct qed_dev *cdev, u32 handle) +{ + struct qed_hash_nvmetcp_con *hash_con; + + hash_con = qed_nvmetcp_get_hash(cdev, handle); + if (!hash_con) { + DP_NOTICE(cdev, "Failed to find connection for handle %d\n", + handle); + + return -EINVAL; + } + + return qed_sp_nvmetcp_conn_clear_sq(QED_AFFIN_HWFN(cdev), hash_con->con, + QED_SPQ_MODE_EBLOCK, NULL); +} + +static int qed_nvmetcp_destroy_conn(struct qed_dev *cdev, + u32 handle, u8 abrt_conn) +{ + struct qed_hash_nvmetcp_con *hash_con; + + hash_con = qed_nvmetcp_get_hash(cdev, handle); + if (!hash_con) { + DP_NOTICE(cdev, "Failed to find connection for handle %d\n", + handle); + + return -EINVAL; + } + + hash_con->con->abortive_dsconnect = abrt_conn; + + return qed_sp_nvmetcp_conn_terminate(QED_AFFIN_HWFN(cdev), hash_con->con, + QED_SPQ_MODE_EBLOCK, NULL); +} + +static const struct qed_nvmetcp_ops qed_nvmetcp_ops_pass = { + .common = &qed_common_ops_pass, + .ll2 = &qed_ll2_ops_pass, + .fill_dev_info = &qed_fill_nvmetcp_dev_info, + .register_ops = &qed_register_nvmetcp_ops, + .start = &qed_nvmetcp_start, + .stop = &qed_nvmetcp_stop, + .acquire_conn = &qed_nvmetcp_acquire_conn, + .release_conn = &qed_nvmetcp_release_conn, + .offload_conn = &qed_nvmetcp_offload_conn, + .update_conn = &qed_nvmetcp_update_conn, + .destroy_conn = &qed_nvmetcp_destroy_conn, + .clear_sq = &qed_nvmetcp_clear_conn_sq, + .add_src_tcp_port_filter = &qed_llh_add_src_tcp_port_filter, + .remove_src_tcp_port_filter = &qed_llh_remove_src_tcp_port_filter, + .add_dst_tcp_port_filter = &qed_llh_add_dst_tcp_port_filter, + .remove_dst_tcp_port_filter = &qed_llh_remove_dst_tcp_port_filter, + .clear_all_filters = &qed_llh_clear_all_filters, + .init_read_io = &init_nvmetcp_host_read_task, + .init_write_io = &init_nvmetcp_host_write_task, + .init_icreq_exchange = &init_nvmetcp_init_conn_req_task, + .init_task_cleanup = &init_cleanup_task_nvmetcp +}; + +const struct qed_nvmetcp_ops *qed_get_nvmetcp_ops(void) +{ + return &qed_nvmetcp_ops_pass; +} +EXPORT_SYMBOL(qed_get_nvmetcp_ops); + +void qed_put_nvmetcp_ops(void) +{ +} +EXPORT_SYMBOL(qed_put_nvmetcp_ops); diff --git a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp.h b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp.h new file mode 100644 index 000000000000..e5e9d075bf4f --- /dev/null +++ b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ +/* Copyright 2021 Marvell. All rights reserved. */ + +#ifndef _QED_NVMETCP_H +#define _QED_NVMETCP_H + +#include <linux/types.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/qed/tcp_common.h> +#include <linux/qed/qed_nvmetcp_if.h> +#include <linux/qed/qed_chain.h> +#include "qed.h" +#include "qed_hsi.h" +#include "qed_mcp.h" +#include "qed_sp.h" + +#define QED_NVMETCP_FW_CQ_SIZE (4 * 1024) + +/* tcp parameters */ +#define QED_TCP_FLOW_LABEL 0 +#define QED_TCP_TWO_MSL_TIMER 4000 +#define QED_TCP_HALF_WAY_CLOSE_TIMEOUT 10 +#define QED_TCP_MAX_FIN_RT 2 +#define QED_TCP_SWS_TIMER 5000 + +struct qed_nvmetcp_info { + spinlock_t lock; /* Connection resources. */ + struct list_head free_list; + u16 max_num_outstanding_tasks; + void *event_context; + nvmetcp_event_cb_t event_cb; +}; + +struct qed_hash_nvmetcp_con { + struct hlist_node node; + struct qed_nvmetcp_conn *con; +}; + +struct qed_nvmetcp_conn { + struct list_head list_entry; + bool free_on_delete; + u16 conn_id; + u32 icid; + u32 fw_cid; + u8 layer_code; + u8 offl_flags; + u8 connect_mode; + dma_addr_t sq_pbl_addr; + struct qed_chain r2tq; + struct qed_chain xhq; + struct qed_chain uhq; + u8 local_mac[6]; + u8 remote_mac[6]; + u8 ip_version; + u8 ka_max_probe_cnt; + u16 vlan_id; + u16 tcp_flags; + u32 remote_ip[4]; + u32 local_ip[4]; + u32 flow_label; + u32 ka_timeout; + u32 ka_interval; + u32 max_rt_time; + u8 ttl; + u8 tos_or_tc; + u16 remote_port; + u16 local_port; + u16 mss; + u8 rcv_wnd_scale; + u32 rcv_wnd; + u32 cwnd; + u8 update_flag; + u8 default_cq; + u8 abortive_dsconnect; + u32 max_seq_size; + u32 max_recv_pdu_length; + u32 max_send_pdu_length; + u32 first_seq_length; + u16 physical_q0; + u16 physical_q1; + u16 nvmetcp_cccid_max_range; + dma_addr_t nvmetcp_cccid_itid_table_addr; +}; + +#if IS_ENABLED(CONFIG_QED_NVMETCP) +int qed_nvmetcp_alloc(struct qed_hwfn *p_hwfn); +void qed_nvmetcp_setup(struct qed_hwfn *p_hwfn); +void qed_nvmetcp_free(struct qed_hwfn *p_hwfn); + +#else /* IS_ENABLED(CONFIG_QED_NVMETCP) */ +static inline int qed_nvmetcp_alloc(struct qed_hwfn *p_hwfn) +{ + return -EINVAL; +} + +static inline void qed_nvmetcp_setup(struct qed_hwfn *p_hwfn) {} +static inline void qed_nvmetcp_free(struct qed_hwfn *p_hwfn) {} + +#endif /* IS_ENABLED(CONFIG_QED_NVMETCP) */ + +#endif diff --git a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.c b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.c new file mode 100644 index 000000000000..c1dd71d19f3f --- /dev/null +++ b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +/* Copyright 2021 Marvell. All rights reserved. */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <asm/byteorder.h> +#include <linux/qed/common_hsi.h> +#include <linux/qed/storage_common.h> +#include <linux/qed/nvmetcp_common.h> +#include <linux/qed/qed_nvmetcp_if.h> +#include "qed_nvmetcp_fw_funcs.h" + +#define NVMETCP_NUM_SGES_IN_CACHE 0x4 + +bool nvmetcp_is_slow_sgl(u16 num_sges, bool small_mid_sge) +{ + return (num_sges > SCSI_NUM_SGES_SLOW_SGL_THR && small_mid_sge); +} + +void init_scsi_sgl_context(struct scsi_sgl_params *ctx_sgl_params, + struct scsi_cached_sges *ctx_data_desc, + struct storage_sgl_task_params *sgl_params) +{ + u8 num_sges_to_init = (u8)(sgl_params->num_sges > NVMETCP_NUM_SGES_IN_CACHE ? + NVMETCP_NUM_SGES_IN_CACHE : sgl_params->num_sges); + u8 sge_index; + + /* sgl params */ + ctx_sgl_params->sgl_addr.lo = cpu_to_le32(sgl_params->sgl_phys_addr.lo); + ctx_sgl_params->sgl_addr.hi = cpu_to_le32(sgl_params->sgl_phys_addr.hi); + ctx_sgl_params->sgl_total_length = cpu_to_le32(sgl_params->total_buffer_size); + ctx_sgl_params->sgl_num_sges = cpu_to_le16(sgl_params->num_sges); + + for (sge_index = 0; sge_index < num_sges_to_init; sge_index++) { + ctx_data_desc->sge[sge_index].sge_addr.lo = + cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.lo); + ctx_data_desc->sge[sge_index].sge_addr.hi = + cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.hi); + ctx_data_desc->sge[sge_index].sge_len = + cpu_to_le32(sgl_params->sgl[sge_index].sge_len); + } +} + +static inline u32 calc_rw_task_size(struct nvmetcp_task_params *task_params, + enum nvmetcp_task_type task_type) +{ + u32 io_size; + + if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE) + io_size = task_params->tx_io_size; + else + io_size = task_params->rx_io_size; + + if (unlikely(!io_size)) + return 0; + + return io_size; +} + +static inline void init_sqe(struct nvmetcp_task_params *task_params, + struct storage_sgl_task_params *sgl_task_params, + enum nvmetcp_task_type task_type) +{ + if (!task_params->sqe) + return; + + memset(task_params->sqe, 0, sizeof(*task_params->sqe)); + task_params->sqe->task_id = cpu_to_le16(task_params->itid); + + switch (task_type) { + case NVMETCP_TASK_TYPE_HOST_WRITE: { + u32 buf_size = 0; + u32 num_sges = 0; + + SET_FIELD(task_params->sqe->contlen_cdbsize, + NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1); + SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE, + NVMETCP_WQE_TYPE_NORMAL); + if (task_params->tx_io_size) { + if (task_params->send_write_incapsule) + buf_size = calc_rw_task_size(task_params, task_type); + + if (nvmetcp_is_slow_sgl(sgl_task_params->num_sges, + sgl_task_params->small_mid_sge)) + num_sges = NVMETCP_WQE_NUM_SGES_SLOWIO; + else + num_sges = min((u16)sgl_task_params->num_sges, + (u16)SCSI_NUM_SGES_SLOW_SGL_THR); + } + SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES, num_sges); + SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN, buf_size); + } break; + + case NVMETCP_TASK_TYPE_HOST_READ: { + SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE, + NVMETCP_WQE_TYPE_NORMAL); + SET_FIELD(task_params->sqe->contlen_cdbsize, + NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1); + } break; + + case NVMETCP_TASK_TYPE_INIT_CONN_REQUEST: { + SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE, + NVMETCP_WQE_TYPE_MIDDLE_PATH); + + if (task_params->tx_io_size) { + SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN, + task_params->tx_io_size); + SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES, + min((u16)sgl_task_params->num_sges, + (u16)SCSI_NUM_SGES_SLOW_SGL_THR)); + } + } break; + + case NVMETCP_TASK_TYPE_CLEANUP: + SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE, + NVMETCP_WQE_TYPE_TASK_CLEANUP); + + default: + break; + } +} + +/* The following function initializes of NVMeTCP task params */ +static inline void +init_nvmetcp_task_params(struct e5_nvmetcp_task_context *context, + struct nvmetcp_task_params *task_params, + enum nvmetcp_task_type task_type) +{ + context->ystorm_st_context.state.cccid = task_params->host_cccid; + SET_FIELD(context->ustorm_st_context.error_flags, USTORM_NVMETCP_TASK_ST_CTX_NVME_TCP, 1); + context->ustorm_st_context.nvme_tcp_opaque_lo = cpu_to_le32(task_params->opq.lo); + context->ustorm_st_context.nvme_tcp_opaque_hi = cpu_to_le32(task_params->opq.hi); +} + +/* The following function initializes default values to all tasks */ +static inline void +init_default_nvmetcp_task(struct nvmetcp_task_params *task_params, + void *pdu_header, void *nvme_cmd, + enum nvmetcp_task_type task_type) +{ + struct e5_nvmetcp_task_context *context = task_params->context; + const u8 val_byte = context->mstorm_ag_context.cdu_validation; + u8 dw_index; + + memset(context, 0, sizeof(*context)); + init_nvmetcp_task_params(context, task_params, + (enum nvmetcp_task_type)task_type); + + /* Swapping requirements used below, will be removed in future FW versions */ + if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE || + task_type == NVMETCP_TASK_TYPE_HOST_READ) { + for (dw_index = 0; + dw_index < QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32); + dw_index++) + context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] = + cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index])); + + for (dw_index = QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32); + dw_index < QED_NVMETCP_CMD_HDR_SIZE / sizeof(u32); + dw_index++) + context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] = + cpu_to_le32(__swab32(((u32 *)nvme_cmd)[dw_index - 2])); + } else { + for (dw_index = 0; + dw_index < QED_NVMETCP_NON_IO_HDR_SIZE / sizeof(u32); + dw_index++) + context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] = + cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index])); + } + + /* M-Storm Context: */ + context->mstorm_ag_context.cdu_validation = val_byte; + context->mstorm_st_context.task_type = (u8)(task_type); + context->mstorm_ag_context.task_cid = cpu_to_le16(task_params->conn_icid); + + /* Ustorm Context: */ + SET_FIELD(context->ustorm_ag_context.flags1, E5_USTORM_NVMETCP_TASK_AG_CTX_R2T2RECV, 1); + context->ustorm_st_context.task_type = (u8)(task_type); + context->ustorm_st_context.cq_rss_number = task_params->cq_rss_number; + context->ustorm_ag_context.icid = cpu_to_le16(task_params->conn_icid); +} + +/* The following function initializes the U-Storm Task Contexts */ +static inline void +init_ustorm_task_contexts(struct ustorm_nvmetcp_task_st_ctx *ustorm_st_context, + struct e5_ustorm_nvmetcp_task_ag_ctx *ustorm_ag_context, + u32 remaining_recv_len, + u32 expected_data_transfer_len, u8 num_sges, + bool tx_dif_conn_err_en) +{ + /* Remaining data to be received in bytes. Used in validations*/ + ustorm_st_context->rem_rcv_len = cpu_to_le32(remaining_recv_len); + ustorm_ag_context->exp_data_acked = cpu_to_le32(expected_data_transfer_len); + ustorm_st_context->exp_data_transfer_len = cpu_to_le32(expected_data_transfer_len); + SET_FIELD(ustorm_st_context->reg1_map, REG1_NUM_SGES, num_sges); + SET_FIELD(ustorm_ag_context->flags2, E5_USTORM_NVMETCP_TASK_AG_CTX_DIF_ERROR_CF_EN, + tx_dif_conn_err_en ? 1 : 0); +} + +/* The following function initializes Local Completion Contexts: */ +static inline void +set_local_completion_context(struct e5_nvmetcp_task_context *context) +{ + SET_FIELD(context->ystorm_st_context.state.flags, + YSTORM_NVMETCP_TASK_STATE_LOCAL_COMP, 1); + SET_FIELD(context->ustorm_st_context.flags, + USTORM_NVMETCP_TASK_ST_CTX_LOCAL_COMP, 1); +} + +/* Common Fastpath task init function: */ +static inline void +init_rw_nvmetcp_task(struct nvmetcp_task_params *task_params, + enum nvmetcp_task_type task_type, + void *pdu_header, void *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params) +{ + struct e5_nvmetcp_task_context *context = task_params->context; + u32 task_size = calc_rw_task_size(task_params, task_type); + bool slow_io = false; + u8 num_sges = 0; + + init_default_nvmetcp_task(task_params, pdu_header, nvme_cmd, task_type); + + /* Tx/Rx: */ + if (task_params->tx_io_size) { + /* if data to transmit: */ + init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params, + &context->ystorm_st_context.state.data_desc, + sgl_task_params); + slow_io = nvmetcp_is_slow_sgl(sgl_task_params->num_sges, + sgl_task_params->small_mid_sge); + num_sges = + (u8)(!slow_io ? min((u32)sgl_task_params->num_sges, + (u32)SCSI_NUM_SGES_SLOW_SGL_THR) : + NVMETCP_WQE_NUM_SGES_SLOWIO); + if (slow_io) { + SET_FIELD(context->ystorm_st_context.state.flags, + YSTORM_NVMETCP_TASK_STATE_SLOW_IO, 1); + } + } else if (task_params->rx_io_size) { + /* if data to receive: */ + init_scsi_sgl_context(&context->mstorm_st_context.sgl_params, + &context->mstorm_st_context.data_desc, + sgl_task_params); + num_sges = + (u8)(!nvmetcp_is_slow_sgl(sgl_task_params->num_sges, + sgl_task_params->small_mid_sge) ? + min((u32)sgl_task_params->num_sges, + (u32)SCSI_NUM_SGES_SLOW_SGL_THR) : + NVMETCP_WQE_NUM_SGES_SLOWIO); + context->mstorm_st_context.rem_task_size = cpu_to_le32(task_size); + } + + /* Ustorm context: */ + init_ustorm_task_contexts(&context->ustorm_st_context, + &context->ustorm_ag_context, + /* Remaining Receive length is the Task Size */ + task_size, + /* The size of the transmitted task */ + task_size, + /* num_sges */ + num_sges, + false); + + /* Set exp_data_acked */ + if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE) { + if (task_params->send_write_incapsule) + context->ustorm_ag_context.exp_data_acked = task_size; + else + context->ustorm_ag_context.exp_data_acked = 0; + } else if (task_type == NVMETCP_TASK_TYPE_HOST_READ) { + context->ustorm_ag_context.exp_data_acked = 0; + } + + context->ustorm_ag_context.exp_cont_len = 0; + init_sqe(task_params, sgl_task_params, task_type); +} + +static void +init_common_initiator_read_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_cmd_pdu *cmd_pdu_header, + struct nvme_command *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params) +{ + init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_READ, + cmd_pdu_header, nvme_cmd, sgl_task_params); +} + +void init_nvmetcp_host_read_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_cmd_pdu *cmd_pdu_header, + struct nvme_command *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params) +{ + init_common_initiator_read_task(task_params, (void *)cmd_pdu_header, + (void *)nvme_cmd, sgl_task_params); +} + +static void +init_common_initiator_write_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_cmd_pdu *cmd_pdu_header, + struct nvme_command *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params) +{ + init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_WRITE, + cmd_pdu_header, nvme_cmd, sgl_task_params); +} + +void init_nvmetcp_host_write_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_cmd_pdu *cmd_pdu_header, + struct nvme_command *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params) +{ + init_common_initiator_write_task(task_params, (void *)cmd_pdu_header, + (void *)nvme_cmd, sgl_task_params); +} + +static void +init_common_login_request_task(struct nvmetcp_task_params *task_params, + void *login_req_pdu_header, + struct storage_sgl_task_params *tx_sgl_task_params, + struct storage_sgl_task_params *rx_sgl_task_params) +{ + struct e5_nvmetcp_task_context *context = task_params->context; + + init_default_nvmetcp_task(task_params, (void *)login_req_pdu_header, NULL, + NVMETCP_TASK_TYPE_INIT_CONN_REQUEST); + + /* Ustorm Context: */ + init_ustorm_task_contexts(&context->ustorm_st_context, + &context->ustorm_ag_context, + + /* Remaining Receive length is the Task Size */ + task_params->rx_io_size ? + rx_sgl_task_params->total_buffer_size : 0, + + /* The size of the transmitted task */ + task_params->tx_io_size ? + tx_sgl_task_params->total_buffer_size : 0, + 0, /* num_sges */ + 0); /* tx_dif_conn_err_en */ + + /* SGL context: */ + if (task_params->tx_io_size) + init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params, + &context->ystorm_st_context.state.data_desc, + tx_sgl_task_params); + if (task_params->rx_io_size) + init_scsi_sgl_context(&context->mstorm_st_context.sgl_params, + &context->mstorm_st_context.data_desc, + rx_sgl_task_params); + + context->mstorm_st_context.rem_task_size = + cpu_to_le32(task_params->rx_io_size ? + rx_sgl_task_params->total_buffer_size : 0); + init_sqe(task_params, tx_sgl_task_params, NVMETCP_TASK_TYPE_INIT_CONN_REQUEST); +} + +/* The following function initializes Login task in Host mode: */ +void init_nvmetcp_init_conn_req_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_icreq_pdu *init_conn_req_pdu_hdr, + struct storage_sgl_task_params *tx_sgl_task_params, + struct storage_sgl_task_params *rx_sgl_task_params) +{ + init_common_login_request_task(task_params, init_conn_req_pdu_hdr, + tx_sgl_task_params, rx_sgl_task_params); +} + +void init_cleanup_task_nvmetcp(struct nvmetcp_task_params *task_params) +{ + init_sqe(task_params, NULL, NVMETCP_TASK_TYPE_CLEANUP); +} diff --git a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.h b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.h new file mode 100644 index 000000000000..1d5ddc217bdb --- /dev/null +++ b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ +/* Copyright 2021 Marvell. All rights reserved. */ + +#ifndef _QED_NVMETCP_FW_FUNCS_H +#define _QED_NVMETCP_FW_FUNCS_H + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/list.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <asm/byteorder.h> +#include <linux/qed/common_hsi.h> +#include <linux/qed/storage_common.h> +#include <linux/qed/nvmetcp_common.h> +#include <linux/qed/qed_nvmetcp_if.h> + +#if IS_ENABLED(CONFIG_QED_NVMETCP) + +void init_nvmetcp_host_read_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_cmd_pdu *cmd_pdu_header, + struct nvme_command *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params); +void init_nvmetcp_host_write_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_cmd_pdu *cmd_pdu_header, + struct nvme_command *nvme_cmd, + struct storage_sgl_task_params *sgl_task_params); +void init_nvmetcp_init_conn_req_task(struct nvmetcp_task_params *task_params, + struct nvme_tcp_icreq_pdu *init_conn_req_pdu_hdr, + struct storage_sgl_task_params *tx_sgl_task_params, + struct storage_sgl_task_params *rx_sgl_task_params); +void init_cleanup_task_nvmetcp(struct nvmetcp_task_params *task_params); + +#else /* IS_ENABLED(CONFIG_QED_NVMETCP) */ + +#endif /* IS_ENABLED(CONFIG_QED_NVMETCP) */ + +#endif /* _QED_NVMETCP_FW_FUNCS_H */ diff --git a/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c new file mode 100644 index 000000000000..96a2077fd315 --- /dev/null +++ b/drivers/net/ethernet/qlogic/qed/qed_nvmetcp_ip_services.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +/* + * Copyright 2021 Marvell. All rights reserved. + */ + +#include <linux/types.h> +#include <asm/byteorder.h> +#include <asm/param.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/etherdevice.h> +#include <linux/kernel.h> +#include <linux/stddef.h> +#include <linux/errno.h> + +#include <net/tcp.h> + +#include <linux/qed/qed_nvmetcp_ip_services_if.h> + +#define QED_IP_RESOL_TIMEOUT 4 + +int qed_route_ipv4(struct sockaddr_storage *local_addr, + struct sockaddr_storage *remote_addr, + struct sockaddr *hardware_address, + struct net_device **ndev) +{ + struct neighbour *neigh = NULL; + __be32 *loc_ip, *rem_ip; + struct rtable *rt; + int rc = -ENXIO; + int retry; + + loc_ip = &((struct sockaddr_in *)local_addr)->sin_addr.s_addr; + rem_ip = &((struct sockaddr_in *)remote_addr)->sin_addr.s_addr; + *ndev = NULL; + rt = ip_route_output(&init_net, *rem_ip, *loc_ip, 0/*tos*/, 0/*oif*/); + if (IS_ERR(rt)) { + pr_err("lookup route failed\n"); + rc = PTR_ERR(rt); + goto return_err; + } + + neigh = dst_neigh_lookup(&rt->dst, rem_ip); + if (!neigh) { + rc = -ENOMEM; + ip_rt_put(rt); + goto return_err; + } + + *ndev = rt->dst.dev; + ip_rt_put(rt); + + /* If not resolved, kick-off state machine towards resolution */ + if (!(neigh->nud_state & NUD_VALID)) + neigh_event_send(neigh, NULL); + + /* query neighbor until resolved or timeout */ + retry = QED_IP_RESOL_TIMEOUT; + while (!(neigh->nud_state & NUD_VALID) && retry > 0) { + msleep(1000); + retry--; + } + + if (neigh->nud_state & NUD_VALID) { + /* copy resolved MAC address */ + neigh_ha_snapshot(hardware_address->sa_data, neigh, *ndev); + hardware_address->sa_family = (*ndev)->type; + rc = 0; + } + + neigh_release(neigh); + if (!(*loc_ip)) { + *loc_ip = inet_select_addr(*ndev, *rem_ip, RT_SCOPE_UNIVERSE); + local_addr->ss_family = AF_INET; + } + +return_err: + + return rc; +} +EXPORT_SYMBOL(qed_route_ipv4); + +int qed_route_ipv6(struct sockaddr_storage *local_addr, + struct sockaddr_storage *remote_addr, + struct sockaddr *hardware_address, + struct net_device **ndev) +{ + struct neighbour *neigh = NULL; + struct dst_entry *dst; + struct flowi6 fl6; + int rc = -ENXIO; + int retry; + + memset(&fl6, 0, sizeof(fl6)); + fl6.saddr = ((struct sockaddr_in6 *)local_addr)->sin6_addr; + fl6.daddr = ((struct sockaddr_in6 *)remote_addr)->sin6_addr; + dst = ip6_route_output(&init_net, NULL, &fl6); + if (!dst || dst->error) { + if (dst) { + dst_release(dst); + pr_err("lookup route failed %d\n", dst->error); + } + + goto out; + } + + neigh = dst_neigh_lookup(dst, &fl6.daddr); + if (neigh) { + *ndev = ip6_dst_idev(dst)->dev; + + /* If not resolved, kick-off state machine towards resolution */ + if (!(neigh->nud_state & NUD_VALID)) + neigh_event_send(neigh, NULL); + + /* query neighbor until resolved or timeout */ + retry = QED_IP_RESOL_TIMEOUT; + while (!(neigh->nud_state & NUD_VALID) && retry > 0) { + msleep(1000); + retry--; + } + + if (neigh->nud_state & NUD_VALID) { + neigh_ha_snapshot((u8 *)hardware_address->sa_data, + neigh, *ndev); + hardware_address->sa_family = (*ndev)->type; + rc = 0; + } + + neigh_release(neigh); + + if (ipv6_addr_any(&fl6.saddr)) { + if (ipv6_dev_get_saddr(dev_net(*ndev), *ndev, + &fl6.daddr, 0, &fl6.saddr)) { + pr_err("Unable to find source IP address\n"); + goto out; + } + + local_addr->ss_family = AF_INET6; + ((struct sockaddr_in6 *)local_addr)->sin6_addr = + fl6.saddr; + } + } + + dst_release(dst); + +out: + + return rc; +} +EXPORT_SYMBOL(qed_route_ipv6); + +void qed_vlan_get_ndev(struct net_device **ndev, u16 *vlan_id) +{ + if (is_vlan_dev(*ndev)) { + *vlan_id = vlan_dev_vlan_id(*ndev); + *ndev = vlan_dev_real_dev(*ndev); + } +} +EXPORT_SYMBOL(qed_vlan_get_ndev); + +struct pci_dev *qed_validate_ndev(struct net_device *ndev) +{ + struct pci_dev *pdev = NULL; + struct net_device *upper; + + for_each_pci_dev(pdev) { + if (pdev && pdev->driver && + !strcmp(pdev->driver->name, "qede")) { + upper = pci_get_drvdata(pdev); + if (upper->ifindex == ndev->ifindex) + return pdev; + } + } + + return NULL; +} +EXPORT_SYMBOL(qed_validate_ndev); + +__be16 qed_get_in_port(struct sockaddr_storage *sa) +{ + return sa->ss_family == AF_INET + ? ((struct sockaddr_in *)sa)->sin_port + : ((struct sockaddr_in6 *)sa)->sin6_port; +} +EXPORT_SYMBOL(qed_get_in_port); + +int qed_fetch_tcp_port(struct sockaddr_storage local_ip_addr, + struct socket **sock, u16 *port) +{ + struct sockaddr_storage sa; + int rc = 0; + + rc = sock_create(local_ip_addr.ss_family, SOCK_STREAM, IPPROTO_TCP, + sock); + if (rc) { + pr_warn("failed to create socket: %d\n", rc); + goto err; + } + + (*sock)->sk->sk_allocation = GFP_KERNEL; + sk_set_memalloc((*sock)->sk); + + rc = kernel_bind(*sock, (struct sockaddr *)&local_ip_addr, + sizeof(local_ip_addr)); + + if (rc) { + pr_warn("failed to bind socket: %d\n", rc); + goto err_sock; + } + + rc = kernel_getsockname(*sock, (struct sockaddr *)&sa); + if (rc < 0) { + pr_warn("getsockname() failed: %d\n", rc); + goto err_sock; + } + + *port = ntohs(qed_get_in_port(&sa)); + + return 0; + +err_sock: + sock_release(*sock); + sock = NULL; +err: + + return rc; +} +EXPORT_SYMBOL(qed_fetch_tcp_port); + +void qed_return_tcp_port(struct socket *sock) +{ + if (sock && sock->sk) { + tcp_set_state(sock->sk, TCP_CLOSE); + sock_release(sock); + } +} +EXPORT_SYMBOL(qed_return_tcp_port); diff --git a/drivers/net/ethernet/qlogic/qed/qed_ooo.c b/drivers/net/ethernet/qlogic/qed/qed_ooo.c index 88353aa404dc..b8c5641b29a8 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ooo.c +++ b/drivers/net/ethernet/qlogic/qed/qed_ooo.c @@ -16,7 +16,7 @@ #include "qed_ll2.h" #include "qed_ooo.h" #include "qed_cxt.h" - +#include "qed_nvmetcp.h" static struct qed_ooo_archipelago *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn, struct qed_ooo_info @@ -83,7 +83,8 @@ int qed_ooo_alloc(struct qed_hwfn *p_hwfn) switch (p_hwfn->hw_info.personality) { case QED_PCI_ISCSI: - proto = PROTOCOLID_ISCSI; + case QED_PCI_NVMETCP: + proto = PROTOCOLID_TCP_ULP; break; case QED_PCI_ETH_RDMA: case QED_PCI_ETH_IWARP: diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp.h b/drivers/net/ethernet/qlogic/qed/qed_sp.h index 993f1357b6fc..60ff3222bf55 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_sp.h +++ b/drivers/net/ethernet/qlogic/qed/qed_sp.h @@ -100,6 +100,11 @@ union ramrod_data { struct iscsi_spe_conn_mac_update iscsi_conn_mac_update; struct iscsi_spe_conn_termination iscsi_conn_terminate; + struct nvmetcp_init_ramrod_params nvmetcp_init; + struct nvmetcp_spe_conn_offload nvmetcp_conn_offload; + struct nvmetcp_conn_update_ramrod_params nvmetcp_conn_update; + struct nvmetcp_spe_conn_termination nvmetcp_conn_terminate; + struct vf_start_ramrod_data vf_start; struct vf_stop_ramrod_data vf_stop; }; diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c index aa71adcf31ee..b4ed54ffef9b 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c +++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c @@ -385,7 +385,8 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn, p_ramrod->personality = PERSONALITY_FCOE; break; case QED_PCI_ISCSI: - p_ramrod->personality = PERSONALITY_ISCSI; + case QED_PCI_NVMETCP: + p_ramrod->personality = PERSONALITY_TCP_ULP; break; case QED_PCI_ETH_ROCE: case QED_PCI_ETH_IWARP: diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index 8e150dd4f899..065e9004598e 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -1089,13 +1089,7 @@ static bool qede_rx_xdp(struct qede_dev *edev, xdp_prepare_buff(&xdp, page_address(bd->data), *data_offset, *len, false); - /* Queues always have a full reset currently, so for the time - * being until there's atomic program replace just mark read - * side for map helpers. - */ - rcu_read_lock(); act = bpf_prog_run_xdp(prog, &xdp); - rcu_read_unlock(); /* Recalculate, as XDP might have changed the headers */ *data_offset = xdp.data - xdp.data_hard_start; diff --git a/drivers/net/ethernet/qlogic/qede/qede_rdma.c b/drivers/net/ethernet/qlogic/qede/qede_rdma.c index 2f6598086d9b..6304514a6f2c 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_rdma.c +++ b/drivers/net/ethernet/qlogic/qede/qede_rdma.c @@ -247,12 +247,10 @@ static struct qede_rdma_event_work * qede_rdma_get_free_event_node(struct qede_dev *edev) { struct qede_rdma_event_work *event_node = NULL; - struct list_head *list_node = NULL; bool found = false; - list_for_each(list_node, &edev->rdma_info.rdma_event_list) { - event_node = list_entry(list_node, struct qede_rdma_event_work, - list); + list_for_each_entry(event_node, &edev->rdma_info.rdma_event_list, + list) { if (!work_pending(&event_node->work)) { found = true; break; diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c index 214e347097a7..2376b2729633 100644 --- a/drivers/net/ethernet/qlogic/qla3xxx.c +++ b/drivers/net/ethernet/qlogic/qla3xxx.c @@ -114,7 +114,7 @@ static int ql_sem_spinlock(struct ql3_adapter *qdev, value = readl(&port_regs->CommonRegs.semaphoreReg); if ((value & (sem_mask >> 16)) == sem_bits) return 0; - ssleep(1); + mdelay(1000); } while (--seconds); return -1; } diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c index d2c190732d3e..0a2f34fc8b24 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_init.c @@ -746,7 +746,7 @@ static int qlcnic_83xx_idc_unknown_state(struct qlcnic_adapter *adapter) } /** - * qlcnic_83xx_idc_cold_state + * qlcnic_83xx_idc_cold_state_handler * * @adapter: adapter structure * diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_vnic.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_vnic.c index c4297aea7d15..711609503ba6 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_vnic.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_vnic.c @@ -180,7 +180,7 @@ static int qlcnic_83xx_init_non_privileged_vnic(struct qlcnic_adapter *adapter) } /** - * qlcnic_83xx_vnic_opmode + * qlcnic_83xx_config_vnic_opmode * * @adapter: adapter structure * Identify virtual NIC operational modes. diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c index d8a3ecaed3fc..d8f0863b3934 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c @@ -1048,7 +1048,7 @@ int qlcnic_do_lb_test(struct qlcnic_adapter *adapter, u8 mode) for (i = 0; i < QLCNIC_NUM_ILB_PKT; i++) { skb = netdev_alloc_skb(adapter->netdev, QLCNIC_ILB_PKT_SIZE); if (!skb) - break; + goto error; qlcnic_create_loopback_buff(skb->data, adapter->mac_addr); skb_put(skb, QLCNIC_ILB_PKT_SIZE); adapter->ahw->diag_cnt = 0; @@ -1072,6 +1072,7 @@ int qlcnic_do_lb_test(struct qlcnic_adapter *adapter, u8 mode) cnt++; } if (cnt != i) { +error: dev_err(&adapter->pdev->dev, "LB Test: failed, TX[%d], RX[%d]\n", i, cnt); if (mode != QLCNIC_ILB_MODE) diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c index e1b8490bed0a..4b8bc46f55c2 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c @@ -460,12 +460,10 @@ int qlcnic_82xx_sre_macaddr_change(struct qlcnic_adapter *adapter, u8 *addr, int qlcnic_nic_del_mac(struct qlcnic_adapter *adapter, const u8 *addr) { struct qlcnic_mac_vlan_list *cur; - struct list_head *head; int err = -EINVAL; /* Delete MAC from the existing list */ - list_for_each(head, &adapter->mac_list) { - cur = list_entry(head, struct qlcnic_mac_vlan_list, list); + list_for_each_entry(cur, &adapter->mac_list, list) { if (ether_addr_equal(addr, cur->mac_addr)) { err = qlcnic_sre_macaddr_change(adapter, cur->mac_addr, 0, QLCNIC_MAC_DEL); @@ -483,11 +481,9 @@ int qlcnic_nic_add_mac(struct qlcnic_adapter *adapter, const u8 *addr, u16 vlan, enum qlcnic_mac_type mac_type) { struct qlcnic_mac_vlan_list *cur; - struct list_head *head; /* look up if already exists */ - list_for_each(head, &adapter->mac_list) { - cur = list_entry(head, struct qlcnic_mac_vlan_list, list); + list_for_each_entry(cur, &adapter->mac_list, list) { if (ether_addr_equal(addr, cur->mac_addr) && cur->vlan_id == vlan) return 0; diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.h index 601d22495a88..95ecc84dddcd 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.h +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.h @@ -203,7 +203,6 @@ int qlcnic_82xx_set_nic_info(struct qlcnic_adapter *, struct qlcnic_info *); int qlcnic_82xx_get_pci_info(struct qlcnic_adapter *, struct qlcnic_pci_info*); int qlcnic_82xx_alloc_mbx_args(struct qlcnic_cmd_args *, struct qlcnic_adapter *, u32); -int qlcnic_82xx_hw_write_wx_2M(struct qlcnic_adapter *, ulong, u32); int qlcnic_82xx_get_board_info(struct qlcnic_adapter *); int qlcnic_82xx_config_led(struct qlcnic_adapter *, u32, u32); void qlcnic_82xx_get_func_no(struct qlcnic_adapter *); diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c index bdf15d2a6431..af4c516a9e7c 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c @@ -1390,6 +1390,7 @@ static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int ma break; case QLCNIC_RESPONSE_DESC: qlcnic_handle_fw_message(desc_cnt, consumer, sds_ring); + goto skip; default: goto skip; } diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c index 96b947fde646..a4fa507903ee 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c @@ -319,10 +319,8 @@ int qlcnic_read_mac_addr(struct qlcnic_adapter *adapter) static void qlcnic_delete_adapter_mac(struct qlcnic_adapter *adapter) { struct qlcnic_mac_vlan_list *cur; - struct list_head *head; - list_for_each(head, &adapter->mac_list) { - cur = list_entry(head, struct qlcnic_mac_vlan_list, list); + list_for_each_entry(cur, &adapter->mac_list, list) { if (ether_addr_equal_unaligned(adapter->mac_addr, cur->mac_addr)) { qlcnic_sre_macaddr_change(adapter, cur->mac_addr, 0, QLCNIC_MAC_DEL); @@ -2690,6 +2688,7 @@ err_out_free_hw_res: kfree(ahw); err_out_free_res: + pci_disable_pcie_error_reporting(pdev); pci_release_regions(pdev); err_out_disable_pdev: @@ -3343,9 +3342,6 @@ qlcnic_can_start_firmware(struct qlcnic_adapter *adapter) do { msleep(1000); prev_state = QLC_SHARED_REG_RD32(adapter, QLCNIC_CRB_DEV_STATE); - - if (prev_state == QLCNIC_DEV_QUISCENT) - continue; } while ((prev_state != QLCNIC_DEV_READY) && --dev_init_timeo); if (!dev_init_timeo) { @@ -3455,6 +3451,7 @@ wait_npar: adapter->fw_wait_cnt = 0; return; } + break; case QLCNIC_DEV_FAILED: break; default: diff --git a/drivers/net/ethernet/qualcomm/qca_debug.c b/drivers/net/ethernet/qualcomm/qca_debug.c index 702aa217a27a..d59fff2fbcc6 100644 --- a/drivers/net/ethernet/qualcomm/qca_debug.c +++ b/drivers/net/ethernet/qualcomm/qca_debug.c @@ -62,6 +62,7 @@ static const char qcaspi_gstrings_stats[][ETH_GSTRING_LEN] = { "SPI errors", "Write verify errors", "Buffer available errors", + "Bad signature", }; #ifdef CONFIG_DEBUG_FS diff --git a/drivers/net/ethernet/qualcomm/qca_spi.c b/drivers/net/ethernet/qualcomm/qca_spi.c index ab9b02574a15..b64c254e00ba 100644 --- a/drivers/net/ethernet/qualcomm/qca_spi.c +++ b/drivers/net/ethernet/qualcomm/qca_spi.c @@ -504,8 +504,12 @@ qcaspi_qca7k_sync(struct qcaspi *qca, int event) qcaspi_read_register(qca, SPI_REG_SIGNATURE, &signature); qcaspi_read_register(qca, SPI_REG_SIGNATURE, &signature); if (signature != QCASPI_GOOD_SIGNATURE) { + if (qca->sync == QCASPI_SYNC_READY) + qca->stats.bad_signature++; + qca->sync = QCASPI_SYNC_UNKNOWN; netdev_dbg(qca->net_dev, "sync: got CPU on, but signature was invalid, restart\n"); + return; } else { /* ensure that the WRBUF is empty */ qcaspi_read_register(qca, SPI_REG_WRBUF_SPC_AVA, @@ -523,10 +527,14 @@ qcaspi_qca7k_sync(struct qcaspi *qca, int event) switch (qca->sync) { case QCASPI_SYNC_READY: - /* Read signature, if not valid go to unknown state. */ + /* Check signature twice, if not valid go to unknown state. */ qcaspi_read_register(qca, SPI_REG_SIGNATURE, &signature); + if (signature != QCASPI_GOOD_SIGNATURE) + qcaspi_read_register(qca, SPI_REG_SIGNATURE, &signature); + if (signature != QCASPI_GOOD_SIGNATURE) { qca->sync = QCASPI_SYNC_UNKNOWN; + qca->stats.bad_signature++; netdev_dbg(qca->net_dev, "sync: bad signature, restart\n"); /* don't reset right away */ return; @@ -653,8 +661,7 @@ qcaspi_intr_handler(int irq, void *data) struct qcaspi *qca = data; qca->intr_req++; - if (qca->spi_thread && - qca->spi_thread->state != TASK_RUNNING) + if (qca->spi_thread) wake_up_process(qca->spi_thread); return IRQ_HANDLED; @@ -777,8 +784,7 @@ qcaspi_netdev_xmit(struct sk_buff *skb, struct net_device *dev) netif_trans_update(dev); - if (qca->spi_thread && - qca->spi_thread->state != TASK_RUNNING) + if (qca->spi_thread) wake_up_process(qca->spi_thread); return NETDEV_TX_OK; diff --git a/drivers/net/ethernet/qualcomm/qca_spi.h b/drivers/net/ethernet/qualcomm/qca_spi.h index d13a67e20d65..3067356106f0 100644 --- a/drivers/net/ethernet/qualcomm/qca_spi.h +++ b/drivers/net/ethernet/qualcomm/qca_spi.h @@ -75,6 +75,7 @@ struct qcaspi_stats { u64 spi_err; u64 write_verify_failed; u64 buf_avail_err; + u64 bad_signature; }; struct qcaspi { diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c index 8d51b0cb545c..27b1663c476e 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c @@ -163,7 +163,8 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev, struct ifla_rmnet_flags *flags; flags = nla_data(data[IFLA_RMNET_FLAGS]); - data_format = flags->flags & flags->mask; + data_format &= ~flags->mask; + data_format |= flags->flags & flags->mask; } netdev_dbg(dev, "data format [0x%08X]\n", data_format); @@ -336,7 +337,8 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[], old_data_format = port->data_format; flags = nla_data(data[IFLA_RMNET_FLAGS]); - port->data_format = flags->flags & flags->mask; + port->data_format &= ~flags->mask; + port->data_format |= flags->flags & flags->mask; if (rmnet_vnd_update_dev_mtu(port, real_dev)) { port->data_format = old_data_format; diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h index 8d8d4690a074..3d3cba56c516 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2013-2014, 2016-2018 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2014, 2016-2018, 2021 The Linux Foundation. + * All rights reserved. * * RMNET Data configuration engine */ @@ -48,6 +49,7 @@ struct rmnet_pcpu_stats { struct rmnet_priv_stats { u64 csum_ok; + u64 csum_ip4_header_bad; u64 csum_valid_unset; u64 csum_validation_failed; u64 csum_err_bad_buffer; @@ -56,6 +58,7 @@ struct rmnet_priv_stats { u64 csum_fragmented_pkt; u64 csum_skipped; u64 csum_sw; + u64 csum_hw; }; struct rmnet_priv { diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c index 0be5ac7ab261..bfbd7847f946 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2021, The Linux Foundation. All rights reserved. * * RMNET Data ingress/egress handler */ @@ -82,12 +82,18 @@ __rmnet_map_ingress_handler(struct sk_buff *skb, skb->dev = ep->egress_dev; - /* Subtract MAP header */ - skb_pull(skb, sizeof(struct rmnet_map_header)); - rmnet_set_skb_proto(skb); - - if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { - if (!rmnet_map_checksum_downlink_packet(skb, len + pad)) + if ((port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) && + (map_header->flags & MAP_NEXT_HEADER_FLAG)) { + if (rmnet_map_process_next_hdr_packet(skb, len)) + goto free_skb; + skb_pull(skb, sizeof(*map_header)); + rmnet_set_skb_proto(skb); + } else { + /* Subtract MAP header */ + skb_pull(skb, sizeof(*map_header)); + rmnet_set_skb_proto(skb); + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4 && + !rmnet_map_checksum_downlink_packet(skb, len + pad)) skb->ip_summed = CHECKSUM_UNNECESSARY; } @@ -128,7 +134,7 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, struct rmnet_port *port, u8 mux_id, struct net_device *orig_dev) { - int required_headroom, additional_header_len; + int required_headroom, additional_header_len, csum_type = 0; struct rmnet_map_header *map_header; additional_header_len = 0; @@ -136,18 +142,23 @@ static int rmnet_map_egress_handler(struct sk_buff *skb, if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) { additional_header_len = sizeof(struct rmnet_map_ul_csum_header); - required_headroom += additional_header_len; + csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV4; + } else if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) { + additional_header_len = sizeof(struct rmnet_map_v5_csum_header); + csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV5; } - if (skb_headroom(skb) < required_headroom) { - if (pskb_expand_head(skb, required_headroom, 0, GFP_ATOMIC)) - return -ENOMEM; - } + required_headroom += additional_header_len; + + if (skb_cow_head(skb, required_headroom) < 0) + return -ENOMEM; - if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) - rmnet_map_checksum_uplink_packet(skb, orig_dev); + if (csum_type) + rmnet_map_checksum_uplink_packet(skb, port, orig_dev, + csum_type); - map_header = rmnet_map_add_map_header(skb, additional_header_len, 0); + map_header = rmnet_map_add_map_header(skb, additional_header_len, + port, 0); if (!map_header) return -ENOMEM; diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h index 2aea153f4247..e5a0b38f7dbe 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2021, The Linux Foundation. All rights reserved. */ #ifndef _RMNET_MAP_H_ @@ -43,10 +43,15 @@ enum rmnet_map_commands { struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, struct rmnet_port *port); struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, - int hdrlen, int pad); + int hdrlen, + struct rmnet_port *port, + int pad); void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port); int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len); void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, - struct net_device *orig_dev); + struct rmnet_port *port, + struct net_device *orig_dev, + int csum_type); +int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, u16 len); #endif /* _RMNET_MAP_H_ */ diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 0ac2ff828320..3676976c875b 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2018, 2021, The Linux Foundation. All rights reserved. * * RMNET Data MAP protocol */ @@ -8,6 +8,7 @@ #include <linux/ip.h> #include <linux/ipv6.h> #include <net/ip6_checksum.h> +#include <linux/bitfield.h> #include "rmnet_config.h" #include "rmnet_map.h" #include "rmnet_private.h" @@ -18,23 +19,13 @@ static __sum16 *rmnet_map_get_csum_field(unsigned char protocol, const void *txporthdr) { - __sum16 *check = NULL; + if (protocol == IPPROTO_TCP) + return &((struct tcphdr *)txporthdr)->check; - switch (protocol) { - case IPPROTO_TCP: - check = &(((struct tcphdr *)txporthdr)->check); - break; - - case IPPROTO_UDP: - check = &(((struct udphdr *)txporthdr)->check); - break; + if (protocol == IPPROTO_UDP) + return &((struct udphdr *)txporthdr)->check; - default: - check = NULL; - break; - } - - return check; + return NULL; } static int @@ -42,71 +33,74 @@ rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb, struct rmnet_map_dl_csum_trailer *csum_trailer, struct rmnet_priv *priv) { - __sum16 *csum_field, csum_temp, pseudo_csum, hdr_csum, ip_payload_csum; - u16 csum_value, csum_value_final; - struct iphdr *ip4h; - void *txporthdr; - __be16 addend; - - ip4h = (struct iphdr *)(skb->data); - if ((ntohs(ip4h->frag_off) & IP_MF) || - ((ntohs(ip4h->frag_off) & IP_OFFSET) > 0)) { + struct iphdr *ip4h = (struct iphdr *)skb->data; + void *txporthdr = skb->data + ip4h->ihl * 4; + __sum16 *csum_field, pseudo_csum; + __sum16 ip_payload_csum; + + /* Computing the checksum over just the IPv4 header--including its + * checksum field--should yield 0. If it doesn't, the IP header + * is bad, so return an error and let the IP layer drop it. + */ + if (ip_fast_csum(ip4h, ip4h->ihl)) { + priv->stats.csum_ip4_header_bad++; + return -EINVAL; + } + + /* We don't support checksum offload on IPv4 fragments */ + if (ip_is_fragment(ip4h)) { priv->stats.csum_fragmented_pkt++; return -EOPNOTSUPP; } - txporthdr = skb->data + ip4h->ihl * 4; - + /* Checksum offload is only supported for UDP and TCP protocols */ csum_field = rmnet_map_get_csum_field(ip4h->protocol, txporthdr); - if (!csum_field) { priv->stats.csum_err_invalid_transport++; return -EPROTONOSUPPORT; } - /* RFC 768 - Skip IPv4 UDP packets where sender checksum field is 0 */ - if (*csum_field == 0 && ip4h->protocol == IPPROTO_UDP) { + /* RFC 768: UDP checksum is optional for IPv4, and is 0 if unused */ + if (!*csum_field && ip4h->protocol == IPPROTO_UDP) { priv->stats.csum_skipped++; return 0; } - csum_value = ~ntohs(csum_trailer->csum_value); - hdr_csum = ~ip_fast_csum(ip4h, (int)ip4h->ihl); - ip_payload_csum = csum16_sub((__force __sum16)csum_value, - (__force __be16)hdr_csum); - - pseudo_csum = ~csum_tcpudp_magic(ip4h->saddr, ip4h->daddr, - ntohs(ip4h->tot_len) - ip4h->ihl * 4, - ip4h->protocol, 0); - addend = (__force __be16)ntohs((__force __be16)pseudo_csum); - pseudo_csum = csum16_add(ip_payload_csum, addend); - - addend = (__force __be16)ntohs((__force __be16)*csum_field); - csum_temp = ~csum16_sub(pseudo_csum, addend); - csum_value_final = (__force u16)csum_temp; - - if (unlikely(csum_value_final == 0)) { - switch (ip4h->protocol) { - case IPPROTO_UDP: - /* RFC 768 - DL4 1's complement rule for UDP csum 0 */ - csum_value_final = ~csum_value_final; - break; - - case IPPROTO_TCP: - /* DL4 Non-RFC compliant TCP checksum found */ - if (*csum_field == (__force __sum16)0xFFFF) - csum_value_final = ~csum_value_final; - break; - } - } - - if (csum_value_final == ntohs((__force __be16)*csum_field)) { - priv->stats.csum_ok++; - return 0; - } else { + /* The checksum value in the trailer is computed over the entire + * IP packet, including the IP header and payload. To derive the + * transport checksum from this, we first subract the contribution + * of the IP header from the trailer checksum. We then add the + * checksum computed over the pseudo header. + * + * We verified above that the IP header contributes zero to the + * trailer checksum. Therefore the checksum in the trailer is + * just the checksum computed over the IP payload. + + * If the IP payload arrives intact, adding the pseudo header + * checksum to the IP payload checksum will yield 0xffff (negative + * zero). This means the trailer checksum and the pseudo checksum + * are additive inverses of each other. Put another way, the + * message passes the checksum test if the trailer checksum value + * is the negated pseudo header checksum. + * + * Knowing this, we don't even need to examine the transport + * header checksum value; it is already accounted for in the + * checksum value found in the trailer. + */ + ip_payload_csum = csum_trailer->csum_value; + + pseudo_csum = csum_tcpudp_magic(ip4h->saddr, ip4h->daddr, + ntohs(ip4h->tot_len) - ip4h->ihl * 4, + ip4h->protocol, 0); + + /* The cast is required to ensure only the low 16 bits are examined */ + if (ip_payload_csum != (__sum16)~pseudo_csum) { priv->stats.csum_validation_failed++; return -EINVAL; } + + priv->stats.csum_ok++; + return 0; } #if IS_ENABLED(CONFIG_IPV6) @@ -115,76 +109,66 @@ rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb, struct rmnet_map_dl_csum_trailer *csum_trailer, struct rmnet_priv *priv) { - __sum16 *csum_field, ip6_payload_csum, pseudo_csum, csum_temp; - u16 csum_value, csum_value_final; - __be16 ip6_hdr_csum, addend; - struct ipv6hdr *ip6h; - void *txporthdr; - u32 length; - - ip6h = (struct ipv6hdr *)(skb->data); - - txporthdr = skb->data + sizeof(struct ipv6hdr); + struct ipv6hdr *ip6h = (struct ipv6hdr *)skb->data; + void *txporthdr = skb->data + sizeof(*ip6h); + __sum16 *csum_field, pseudo_csum; + __sum16 ip6_payload_csum; + __be16 ip_header_csum; + + /* Checksum offload is only supported for UDP and TCP protocols; + * the packet cannot include any IPv6 extension headers + */ csum_field = rmnet_map_get_csum_field(ip6h->nexthdr, txporthdr); - if (!csum_field) { priv->stats.csum_err_invalid_transport++; return -EPROTONOSUPPORT; } - csum_value = ~ntohs(csum_trailer->csum_value); - ip6_hdr_csum = (__force __be16) - ~ntohs((__force __be16)ip_compute_csum(ip6h, - (int)(txporthdr - (void *)(skb->data)))); - ip6_payload_csum = csum16_sub((__force __sum16)csum_value, - ip6_hdr_csum); - - length = (ip6h->nexthdr == IPPROTO_UDP) ? - ntohs(((struct udphdr *)txporthdr)->len) : - ntohs(ip6h->payload_len); - pseudo_csum = ~(csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, - length, ip6h->nexthdr, 0)); - addend = (__force __be16)ntohs((__force __be16)pseudo_csum); - pseudo_csum = csum16_add(ip6_payload_csum, addend); - - addend = (__force __be16)ntohs((__force __be16)*csum_field); - csum_temp = ~csum16_sub(pseudo_csum, addend); - csum_value_final = (__force u16)csum_temp; - - if (unlikely(csum_value_final == 0)) { - switch (ip6h->nexthdr) { - case IPPROTO_UDP: - /* RFC 2460 section 8.1 - * DL6 One's complement rule for UDP checksum 0 - */ - csum_value_final = ~csum_value_final; - break; - - case IPPROTO_TCP: - /* DL6 Non-RFC compliant TCP checksum found */ - if (*csum_field == (__force __sum16)0xFFFF) - csum_value_final = ~csum_value_final; - break; - } - } - - if (csum_value_final == ntohs((__force __be16)*csum_field)) { - priv->stats.csum_ok++; - return 0; - } else { + /* The checksum value in the trailer is computed over the entire + * IP packet, including the IP header and payload. To derive the + * transport checksum from this, we first subract the contribution + * of the IP header from the trailer checksum. We then add the + * checksum computed over the pseudo header. + */ + ip_header_csum = (__force __be16)ip_fast_csum(ip6h, sizeof(*ip6h) / 4); + ip6_payload_csum = csum16_sub(csum_trailer->csum_value, ip_header_csum); + + pseudo_csum = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, + ntohs(ip6h->payload_len), + ip6h->nexthdr, 0); + + /* It's sufficient to compare the IP payload checksum with the + * negated pseudo checksum to determine whether the packet + * checksum was good. (See further explanation in comments + * in rmnet_map_ipv4_dl_csum_trailer()). + * + * The cast is required to ensure only the low 16 bits are + * examined. + */ + if (ip6_payload_csum != (__sum16)~pseudo_csum) { priv->stats.csum_validation_failed++; return -EINVAL; } + + priv->stats.csum_ok++; + return 0; +} +#else +static int +rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb, + struct rmnet_map_dl_csum_trailer *csum_trailer, + struct rmnet_priv *priv) +{ + return 0; } #endif -static void rmnet_map_complement_ipv4_txporthdr_csum_field(void *iphdr) +static void rmnet_map_complement_ipv4_txporthdr_csum_field(struct iphdr *ip4h) { - struct iphdr *ip4h = (struct iphdr *)iphdr; void *txphdr; u16 *csum; - txphdr = iphdr + ip4h->ihl * 4; + txphdr = (void *)ip4h + ip4h->ihl * 4; if (ip4h->protocol == IPPROTO_TCP || ip4h->protocol == IPPROTO_UDP) { csum = (u16 *)rmnet_map_get_csum_field(ip4h->protocol, txphdr); @@ -193,15 +177,14 @@ static void rmnet_map_complement_ipv4_txporthdr_csum_field(void *iphdr) } static void -rmnet_map_ipv4_ul_csum_header(void *iphdr, +rmnet_map_ipv4_ul_csum_header(struct iphdr *iphdr, struct rmnet_map_ul_csum_header *ul_header, struct sk_buff *skb) { - struct iphdr *ip4h = iphdr; u16 val; val = MAP_CSUM_UL_ENABLED_FLAG; - if (ip4h->protocol == IPPROTO_UDP) + if (iphdr->protocol == IPPROTO_UDP) val |= MAP_CSUM_UL_UDP_FLAG; val |= skb->csum_offset & MAP_CSUM_UL_OFFSET_MASK; @@ -214,13 +197,13 @@ rmnet_map_ipv4_ul_csum_header(void *iphdr, } #if IS_ENABLED(CONFIG_IPV6) -static void rmnet_map_complement_ipv6_txporthdr_csum_field(void *ip6hdr) +static void +rmnet_map_complement_ipv6_txporthdr_csum_field(struct ipv6hdr *ip6h) { - struct ipv6hdr *ip6h = (struct ipv6hdr *)ip6hdr; void *txphdr; u16 *csum; - txphdr = ip6hdr + sizeof(struct ipv6hdr); + txphdr = ip6h + 1; if (ip6h->nexthdr == IPPROTO_TCP || ip6h->nexthdr == IPPROTO_UDP) { csum = (u16 *)rmnet_map_get_csum_field(ip6h->nexthdr, txphdr); @@ -229,15 +212,14 @@ static void rmnet_map_complement_ipv6_txporthdr_csum_field(void *ip6hdr) } static void -rmnet_map_ipv6_ul_csum_header(void *ip6hdr, +rmnet_map_ipv6_ul_csum_header(struct ipv6hdr *ipv6hdr, struct rmnet_map_ul_csum_header *ul_header, struct sk_buff *skb) { - struct ipv6hdr *ip6h = ip6hdr; u16 val; val = MAP_CSUM_UL_ENABLED_FLAG; - if (ip6h->nexthdr == IPPROTO_UDP) + if (ipv6hdr->nexthdr == IPPROTO_UDP) val |= MAP_CSUM_UL_UDP_FLAG; val |= skb->csum_offset & MAP_CSUM_UL_OFFSET_MASK; @@ -246,16 +228,73 @@ rmnet_map_ipv6_ul_csum_header(void *ip6hdr, skb->ip_summed = CHECKSUM_NONE; - rmnet_map_complement_ipv6_txporthdr_csum_field(ip6hdr); + rmnet_map_complement_ipv6_txporthdr_csum_field(ipv6hdr); +} +#else +static void +rmnet_map_ipv6_ul_csum_header(void *ip6hdr, + struct rmnet_map_ul_csum_header *ul_header, + struct sk_buff *skb) +{ } #endif +static void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, + struct net_device *orig_dev) +{ + struct rmnet_priv *priv = netdev_priv(orig_dev); + struct rmnet_map_v5_csum_header *ul_header; + + ul_header = skb_push(skb, sizeof(*ul_header)); + memset(ul_header, 0, sizeof(*ul_header)); + ul_header->header_info = u8_encode_bits(RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD, + MAPV5_HDRINFO_HDR_TYPE_FMASK); + + if (skb->ip_summed == CHECKSUM_PARTIAL) { + void *iph = ip_hdr(skb); + __sum16 *check; + void *trans; + u8 proto; + + if (skb->protocol == htons(ETH_P_IP)) { + u16 ip_len = ((struct iphdr *)iph)->ihl * 4; + + proto = ((struct iphdr *)iph)->protocol; + trans = iph + ip_len; + } else if (IS_ENABLED(CONFIG_IPV6) && + skb->protocol == htons(ETH_P_IPV6)) { + u16 ip_len = sizeof(struct ipv6hdr); + + proto = ((struct ipv6hdr *)iph)->nexthdr; + trans = iph + ip_len; + } else { + priv->stats.csum_err_invalid_ip_version++; + goto sw_csum; + } + + check = rmnet_map_get_csum_field(proto, trans); + if (check) { + skb->ip_summed = CHECKSUM_NONE; + /* Ask for checksum offloading */ + ul_header->csum_info |= MAPV5_CSUMINFO_VALID_FLAG; + priv->stats.csum_hw++; + return; + } + } + +sw_csum: + priv->stats.csum_sw++; +} + /* Adds MAP header to front of skb->data * Padding is calculated and set appropriately in MAP header. Mux ID is * initialized to 0. */ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, - int hdrlen, int pad) + int hdrlen, + struct rmnet_port *port, + int pad) { struct rmnet_map_header *map_header; u32 padding, map_datalen; @@ -266,6 +305,10 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb, skb_push(skb, sizeof(struct rmnet_map_header)); memset(map_header, 0, sizeof(struct rmnet_map_header)); + /* Set next_hdr bit for csum offload packets */ + if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) + map_header->flags |= MAP_NEXT_HEADER_FLAG; + if (pad == RMNET_MAP_NO_PAD_BYTES) { map_header->pkt_len = htons(map_datalen); return map_header; @@ -300,8 +343,11 @@ done: struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, struct rmnet_port *port) { + struct rmnet_map_v5_csum_header *next_hdr = NULL; struct rmnet_map_header *maph; + void *data = skb->data; struct sk_buff *skbn; + u8 nexthdr_type; u32 packet_len; if (skb->len == 0) @@ -310,8 +356,18 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, maph = (struct rmnet_map_header *)skb->data; packet_len = ntohs(maph->pkt_len) + sizeof(*maph); - if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) + if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) { packet_len += sizeof(struct rmnet_map_dl_csum_trailer); + } else if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) { + if (!(maph->flags & MAP_CMD_FLAG)) { + packet_len += sizeof(*next_hdr); + if (maph->flags & MAP_NEXT_HEADER_FLAG) + next_hdr = data + sizeof(*maph); + else + /* Mapv5 data pkt without csum hdr is invalid */ + return NULL; + } + } if (((int)skb->len - (int)packet_len) < 0) return NULL; @@ -320,6 +376,13 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, if (!maph->pkt_len) return NULL; + if (next_hdr) { + nexthdr_type = u8_get_bits(next_hdr->header_info, + MAPV5_HDRINFO_HDR_TYPE_FMASK); + if (nexthdr_type != RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD) + return NULL; + } + skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, GFP_ATOMIC); if (!skbn) return NULL; @@ -355,28 +418,19 @@ int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len) return -EINVAL; } - if (skb->protocol == htons(ETH_P_IP)) { + if (skb->protocol == htons(ETH_P_IP)) return rmnet_map_ipv4_dl_csum_trailer(skb, csum_trailer, priv); - } else if (skb->protocol == htons(ETH_P_IPV6)) { -#if IS_ENABLED(CONFIG_IPV6) + + if (IS_ENABLED(CONFIG_IPV6) && skb->protocol == htons(ETH_P_IPV6)) return rmnet_map_ipv6_dl_csum_trailer(skb, csum_trailer, priv); -#else - priv->stats.csum_err_invalid_ip_version++; - return -EPROTONOSUPPORT; -#endif - } else { - priv->stats.csum_err_invalid_ip_version++; - return -EPROTONOSUPPORT; - } - return 0; + priv->stats.csum_err_invalid_ip_version++; + + return -EPROTONOSUPPORT; } -/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP - * packets that are supported for UL checksum offload. - */ -void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, - struct net_device *orig_dev) +static void rmnet_map_v4_checksum_uplink_packet(struct sk_buff *skb, + struct net_device *orig_dev) { struct rmnet_priv *priv = netdev_priv(orig_dev); struct rmnet_map_ul_csum_header *ul_header; @@ -389,28 +443,80 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))) goto sw_csum; - if (skb->ip_summed == CHECKSUM_PARTIAL) { - iphdr = (char *)ul_header + - sizeof(struct rmnet_map_ul_csum_header); + if (skb->ip_summed != CHECKSUM_PARTIAL) + goto sw_csum; - if (skb->protocol == htons(ETH_P_IP)) { - rmnet_map_ipv4_ul_csum_header(iphdr, ul_header, skb); - return; - } else if (skb->protocol == htons(ETH_P_IPV6)) { -#if IS_ENABLED(CONFIG_IPV6) - rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb); - return; -#else - priv->stats.csum_err_invalid_ip_version++; - goto sw_csum; -#endif - } else { - priv->stats.csum_err_invalid_ip_version++; - } + iphdr = (char *)ul_header + + sizeof(struct rmnet_map_ul_csum_header); + + if (skb->protocol == htons(ETH_P_IP)) { + rmnet_map_ipv4_ul_csum_header(iphdr, ul_header, skb); + priv->stats.csum_hw++; + return; + } + + if (IS_ENABLED(CONFIG_IPV6) && skb->protocol == htons(ETH_P_IPV6)) { + rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb); + priv->stats.csum_hw++; + return; } + priv->stats.csum_err_invalid_ip_version++; + sw_csum: memset(ul_header, 0, sizeof(*ul_header)); priv->stats.csum_sw++; } + +/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP + * packets that are supported for UL checksum offload. + */ +void rmnet_map_checksum_uplink_packet(struct sk_buff *skb, + struct rmnet_port *port, + struct net_device *orig_dev, + int csum_type) +{ + switch (csum_type) { + case RMNET_FLAGS_EGRESS_MAP_CKSUMV4: + rmnet_map_v4_checksum_uplink_packet(skb, orig_dev); + break; + case RMNET_FLAGS_EGRESS_MAP_CKSUMV5: + rmnet_map_v5_checksum_uplink_packet(skb, port, orig_dev); + break; + default: + break; + } +} + +/* Process a MAPv5 packet header */ +int rmnet_map_process_next_hdr_packet(struct sk_buff *skb, + u16 len) +{ + struct rmnet_priv *priv = netdev_priv(skb->dev); + struct rmnet_map_v5_csum_header *next_hdr; + u8 nexthdr_type; + + next_hdr = (struct rmnet_map_v5_csum_header *)(skb->data + + sizeof(struct rmnet_map_header)); + + nexthdr_type = u8_get_bits(next_hdr->header_info, + MAPV5_HDRINFO_HDR_TYPE_FMASK); + + if (nexthdr_type != RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD) + return -EINVAL; + + if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { + priv->stats.csum_sw++; + } else if (next_hdr->csum_info & MAPV5_CSUMINFO_VALID_FLAG) { + priv->stats.csum_ok++; + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + priv->stats.csum_valid_unset++; + } + + /* Pull csum v5 header */ + skb_pull(skb, sizeof(*next_hdr)); + + return 0; +} diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c index 41fbd2ceeede..13d8eb43a485 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c @@ -126,24 +126,24 @@ static void rmnet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *s) { struct rmnet_priv *priv = netdev_priv(dev); - struct rmnet_vnd_stats total_stats; + struct rmnet_vnd_stats total_stats = { }; struct rmnet_pcpu_stats *pcpu_ptr; + struct rmnet_vnd_stats snapshot; unsigned int cpu, start; - memset(&total_stats, 0, sizeof(struct rmnet_vnd_stats)); - for_each_possible_cpu(cpu) { pcpu_ptr = per_cpu_ptr(priv->pcpu_stats, cpu); do { start = u64_stats_fetch_begin_irq(&pcpu_ptr->syncp); - total_stats.rx_pkts += pcpu_ptr->stats.rx_pkts; - total_stats.rx_bytes += pcpu_ptr->stats.rx_bytes; - total_stats.tx_pkts += pcpu_ptr->stats.tx_pkts; - total_stats.tx_bytes += pcpu_ptr->stats.tx_bytes; + snapshot = pcpu_ptr->stats; /* struct assignment */ } while (u64_stats_fetch_retry_irq(&pcpu_ptr->syncp, start)); - total_stats.tx_drops += pcpu_ptr->stats.tx_drops; + total_stats.rx_pkts += snapshot.rx_pkts; + total_stats.rx_bytes += snapshot.rx_bytes; + total_stats.tx_pkts += snapshot.tx_pkts; + total_stats.tx_bytes += snapshot.tx_bytes; + total_stats.tx_drops += snapshot.tx_drops; } s->rx_packets = total_stats.rx_pkts; @@ -166,6 +166,7 @@ static const struct net_device_ops rmnet_vnd_ops = { static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = { "Checksum ok", + "Bad IPv4 header checksum", "Checksum valid bit not set", "Checksum validation failed", "Checksum error bad buffer", @@ -174,6 +175,7 @@ static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = { "Checksum skipped on ip fragment", "Checksum skipped", "Checksum computed in software", + "Checksum computed in hardware", }; static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf) @@ -354,4 +356,4 @@ int rmnet_vnd_update_dev_mtu(struct rmnet_port *port, } return 0; -}
\ No newline at end of file +} diff --git a/drivers/net/ethernet/rdc/r6040.c b/drivers/net/ethernet/rdc/r6040.c index 7c74318620b1..47e9998b62f0 100644 --- a/drivers/net/ethernet/rdc/r6040.c +++ b/drivers/net/ethernet/rdc/r6040.c @@ -200,7 +200,7 @@ static int r6040_phy_read(void __iomem *ioaddr, int phy_addr, int reg) int limit = MAC_DEF_TIMEOUT; u16 cmd; - iowrite16(MDIO_READ + reg + (phy_addr << 8), ioaddr + MMDIO); + iowrite16(MDIO_READ | reg | (phy_addr << 8), ioaddr + MMDIO); /* Wait for the read bit to be cleared */ while (limit--) { cmd = ioread16(ioaddr + MMDIO); @@ -224,7 +224,7 @@ static int r6040_phy_write(void __iomem *ioaddr, iowrite16(val, ioaddr + MMWD); /* Write the command to the MDIO bus */ - iowrite16(MDIO_WRITE + reg + (phy_addr << 8), ioaddr + MMDIO); + iowrite16(MDIO_WRITE | reg | (phy_addr << 8), ioaddr + MMDIO); /* Wait for the write bit to be cleared */ while (limit--) { cmd = ioread16(ioaddr + MMDIO); @@ -544,7 +544,7 @@ static int r6040_rx(struct net_device *dev, int limit) skb_ptr->dev = priv->dev; /* Do not count the CRC */ - skb_put(skb_ptr, descptr->len - 4); + skb_put(skb_ptr, descptr->len - ETH_FCS_LEN); dma_unmap_single(&priv->pdev->dev, le32_to_cpu(descptr->buf), MAX_BUF_SIZE, DMA_FROM_DEVICE); skb_ptr->protocol = eth_type_trans(skb_ptr, priv->dev); @@ -552,7 +552,7 @@ static int r6040_rx(struct net_device *dev, int limit) /* Send to upper layer */ netif_receive_skb(skb_ptr); dev->stats.rx_packets++; - dev->stats.rx_bytes += descptr->len - 4; + dev->stats.rx_bytes += descptr->len - ETH_FCS_LEN; /* put new skb into descriptor */ descptr->skb_ptr = new_skb; @@ -943,6 +943,7 @@ static const struct ethtool_ops netdev_ethtool_ops = { .get_ts_info = ethtool_op_get_ts_info, .get_link_ksettings = phy_ethtool_get_link_ksettings, .set_link_ksettings = phy_ethtool_set_link_ksettings, + .nway_reset = phy_ethtool_nway_reset, }; static const struct net_device_ops r6040_netdev_ops = { diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index 4e44313b7651..9677e257e9a1 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -6,7 +6,7 @@ Copyright (C) 2000, 2001 David S. Miller (davem@redhat.com) [sungem.c] Copyright 2001 Manfred Spraul [natsemi.c] Copyright 1999-2001 by Donald Becker. [natsemi.c] - Written 1997-2001 by Donald Becker. [8139too.c] + Written 1997-2001 by Donald Becker. [8139too.c] Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>. [acenic.c] This software may be used and distributed according to the terms of @@ -947,8 +947,8 @@ static struct net_device_stats *cp_get_stats(struct net_device *dev) /* The chip only need report frame silently dropped. */ spin_lock_irqsave(&cp->lock, flags); - if (netif_running(dev) && netif_device_present(dev)) - __cp_get_stats(cp); + if (netif_running(dev) && netif_device_present(dev)) + __cp_get_stats(cp); spin_unlock_irqrestore(&cp->lock, flags); return &dev->stats; diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c index 1e5a453dea14..f0608f050050 100644 --- a/drivers/net/ethernet/realtek/8139too.c +++ b/drivers/net/ethernet/realtek/8139too.c @@ -11,7 +11,7 @@ -----<snip>----- - Written 1997-2001 by Donald Becker. + Written 1997-2001 by Donald Becker. This software may be used and distributed according to the terms of the GNU General Public License (GPL), incorporated herein by reference. Drivers based on or derived from this @@ -548,8 +548,8 @@ static const struct { { "RTL-8100", HW_REVID(1, 1, 1, 1, 0, 1, 0), - HasLWake, - }, + HasLWake, + }, { "RTL-8100B/8139D", HW_REVID(1, 1, 1, 0, 1, 0, 1), diff --git a/drivers/net/ethernet/realtek/atp.c b/drivers/net/ethernet/realtek/atp.c index 9e3b35c97e63..b6c849b258a0 100644 --- a/drivers/net/ethernet/realtek/atp.c +++ b/drivers/net/ethernet/realtek/atp.c @@ -497,8 +497,8 @@ static void write_packet(long ioaddr, int length, unsigned char *packet, int pad { if (length & 1) { - length++; - pad_len++; + length++; + pad_len++; } outb(EOC+MAR, ioaddr + PAR_DATA); diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 2c89cde7da1e..f744557c33a3 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -34,8 +34,6 @@ #include "r8169.h" #include "r8169_firmware.h" -#define MODULENAME "r8169" - #define FIRMWARE_8168D_1 "rtl_nic/rtl8168d-1.fw" #define FIRMWARE_8168D_2 "rtl_nic/rtl8168d-2.fw" #define FIRMWARE_8168E_1 "rtl_nic/rtl8168e-1.fw" @@ -1454,7 +1452,7 @@ static void rtl8169_get_drvinfo(struct net_device *dev, struct rtl8169_private *tp = netdev_priv(dev); struct rtl_fw *rtl_fw = tp->rtl_fw; - strlcpy(info->driver, MODULENAME, sizeof(info->driver)); + strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version)); if (rtl_fw) @@ -1671,7 +1669,7 @@ static void rtl8169_get_strings(struct net_device *dev, u32 stringset, u8 *data) { switch(stringset) { case ETH_SS_STATS: - memcpy(data, *rtl8169_gstrings, sizeof(rtl8169_gstrings)); + memcpy(data, rtl8169_gstrings, sizeof(rtl8169_gstrings)); break; } } @@ -3510,7 +3508,6 @@ static void rtl_hw_start_8106(struct rtl8169_private *tp) rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000); rtl_pcie_state_l2l3_disable(tp); - rtl_hw_aspm_clkreq_enable(tp, true); } DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond) @@ -4117,6 +4114,7 @@ static unsigned int rtl_quirk_packet_padto(struct rtl8169_private *tp, case RTL_GIGA_MAC_VER_61: case RTL_GIGA_MAC_VER_63: padto = max_t(unsigned int, padto, ETH_ZLEN); + break; default: break; } @@ -5305,7 +5303,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENODEV; } - rc = pcim_iomap_regions(pdev, BIT(region), MODULENAME); + rc = pcim_iomap_regions(pdev, BIT(region), KBUILD_MODNAME); if (rc < 0) { dev_err(&pdev->dev, "cannot remap MMIO, aborting\n"); return rc; @@ -5440,7 +5438,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } static struct pci_driver rtl8169_pci_driver = { - .name = MODULENAME, + .name = KBUILD_MODNAME, .id_table = rtl8169_pci_tbl, .probe = rtl_init_one, .remove = rtl_remove_one, diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 4afff320dfd0..69c50f81e1cb 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -2047,13 +2047,6 @@ static int ravb_probe(struct platform_device *pdev) return -EINVAL; } - /* Get base address */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&pdev->dev, "invalid resource\n"); - return -EINVAL; - } - ndev = alloc_etherdev_mqs(sizeof(struct ravb_private), NUM_TX_QUEUE, NUM_RX_QUEUE); if (!ndev) @@ -2065,9 +2058,6 @@ static int ravb_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); pm_runtime_get_sync(&pdev->dev); - /* The Ether-specific entries in the device structure. */ - ndev->base_addr = res->start; - chip_id = (enum ravb_chip_id)of_device_get_match_data(&pdev->dev); if (chip_id == RCAR_GEN3) @@ -2089,12 +2079,15 @@ static int ravb_probe(struct platform_device *pdev) priv->num_rx_ring[RAVB_BE] = BE_RX_RING_SIZE; priv->num_tx_ring[RAVB_NC] = NC_TX_RING_SIZE; priv->num_rx_ring[RAVB_NC] = NC_RX_RING_SIZE; - priv->addr = devm_ioremap_resource(&pdev->dev, res); + priv->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(priv->addr)) { error = PTR_ERR(priv->addr); goto out_release; } + /* The Ether-specific entries in the device structure. */ + ndev->base_addr = res->start; + spin_lock_init(&priv->lock); INIT_WORK(&priv->work, ravb_tx_timeout_work); diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index c5b154868c1f..840478692a37 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -2287,7 +2287,7 @@ static void sh_eth_get_strings(struct net_device *ndev, u32 stringset, u8 *data) { switch (stringset) { case ETH_SS_STATS: - memcpy(data, *sh_eth_gstrings_stats, + memcpy(data, sh_eth_gstrings_stats, sizeof(sh_eth_gstrings_stats)); break; } @@ -3225,9 +3225,6 @@ static int sh_eth_drv_probe(struct platform_device *pdev) struct net_device *ndev; int ret; - /* get base addr */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ndev = alloc_etherdev(sizeof(struct sh_eth_private)); if (!ndev) return -ENOMEM; @@ -3245,7 +3242,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev) mdp = netdev_priv(ndev); mdp->num_tx_ring = TX_RING_SIZE; mdp->num_rx_ring = RX_RING_SIZE; - mdp->addr = devm_ioremap_resource(&pdev->dev, res); + mdp->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(mdp->addr)) { ret = PTR_ERR(mdp->addr); goto out_release; diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c index 971f1e54b652..090bcd2fb758 100644 --- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c +++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c @@ -789,7 +789,7 @@ static void sxgbe_tx_queue_clean(struct sxgbe_tx_queue *tqueue) } /** - * sxgbe_tx_clean: + * sxgbe_tx_all_clean: * @priv: driver private structure * Description: it reclaims resources after transmission completes. */ @@ -1015,7 +1015,7 @@ static void sxgbe_tx_timer(struct timer_list *t) } /** - * sxgbe_init_tx_coalesce: init tx mitigation options. + * sxgbe_tx_init_coalesce: init tx mitigation options. * @priv: driver private structure * Description: * This inits the transmit coalesce parameters: i.e. timer rate, diff --git a/drivers/net/ethernet/seeq/ether3.c b/drivers/net/ethernet/seeq/ether3.c index 65c98837ec45..16a4cbae9326 100644 --- a/drivers/net/ethernet/seeq/ether3.c +++ b/drivers/net/ethernet/seeq/ether3.c @@ -617,7 +617,7 @@ if (next_ptr < RX_START || next_ptr >= RX_END) { break; } /* - * ignore our own packets... + * ignore our own packets... */ if (!(*(unsigned long *)&dev->dev_addr[0] ^ *(unsigned long *)&addrs[2+6]) && !(*(unsigned short *)&dev->dev_addr[4] ^ *(unsigned short *)&addrs[2+10])) { @@ -672,7 +672,7 @@ done: */ if (!(ether3_inw(REG_STATUS) & STAT_RXON)) { dev->stats.rx_dropped++; - ether3_outw(next_ptr, REG_RECVPTR); + ether3_outw(next_ptr, REG_RECVPTR); ether3_outw(priv(dev)->regs.command | CMD_RXON, REG_COMMAND); } @@ -690,11 +690,11 @@ static void ether3_tx(struct net_device *dev) do { unsigned long status; - /* + /* * Read the packet header - */ + */ ether3_setbuffer(dev, buffer_read, tx_tail * 0x600); - status = ether3_readlong(dev); + status = ether3_readlong(dev); /* * Check to see if this packet has been transmitted diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index c3f35da1b82a..e7e2223aebbf 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -370,9 +370,9 @@ static int efx_ef10_get_mac_address_vf(struct efx_nic *efx, u8 *mac_address) return 0; } -static ssize_t efx_ef10_show_link_control_flag(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t link_control_flag_show(struct device *dev, + struct device_attribute *attr, + char *buf) { struct efx_nic *efx = dev_get_drvdata(dev); @@ -382,9 +382,9 @@ static ssize_t efx_ef10_show_link_control_flag(struct device *dev, ? 1 : 0); } -static ssize_t efx_ef10_show_primary_flag(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t primary_flag_show(struct device *dev, + struct device_attribute *attr, + char *buf) { struct efx_nic *efx = dev_get_drvdata(dev); @@ -519,9 +519,8 @@ static void efx_ef10_cleanup_vlans(struct efx_nic *efx) mutex_unlock(&nic_data->vlan_lock); } -static DEVICE_ATTR(link_control_flag, 0444, efx_ef10_show_link_control_flag, - NULL); -static DEVICE_ATTR(primary_flag, 0444, efx_ef10_show_primary_flag, NULL); +static DEVICE_ATTR_RO(link_control_flag); +static DEVICE_ATTR_RO(primary_flag); static int efx_ef10_probe(struct efx_nic *efx) { @@ -1070,7 +1069,8 @@ static int efx_ef10_probe_vf(struct efx_nic *efx) /* If the parent PF has no VF data structure, it doesn't know about this * VF so fail probe. The VF needs to be re-created. This can happen - * if the PF driver is unloaded while the VF is assigned to a guest. + * if the PF driver was unloaded while any VF was assigned to a guest + * (using Xen, only). */ pci_dev_pf = efx->pci_dev->physfn; if (pci_dev_pf) { diff --git a/drivers/net/ethernet/sfc/ef10_sriov.c b/drivers/net/ethernet/sfc/ef10_sriov.c index 21fa6c0e8873..752d6406f07e 100644 --- a/drivers/net/ethernet/sfc/ef10_sriov.c +++ b/drivers/net/ethernet/sfc/ef10_sriov.c @@ -122,8 +122,7 @@ static void efx_ef10_sriov_free_vf_vports(struct efx_nic *efx) struct ef10_vf *vf = nic_data->vf + i; /* If VF is assigned, do not free the vport */ - if (vf->pci_dev && - vf->pci_dev->dev_flags & PCI_DEV_FLAGS_ASSIGNED) + if (vf->pci_dev && pci_is_dev_assigned(vf->pci_dev)) continue; if (vf->vport_assigned) { @@ -207,9 +206,7 @@ static int efx_ef10_sriov_alloc_vf_vswitching(struct efx_nic *efx) return 0; fail: - efx_ef10_sriov_free_vf_vports(efx); - kfree(nic_data->vf); - nic_data->vf = NULL; + efx_ef10_sriov_free_vf_vswitching(efx); return rc; } @@ -402,12 +399,17 @@ fail1: return rc; } +/* Disable SRIOV and remove VFs + * If some VFs are attached to a guest (using Xen, only) nothing is + * done if force=false, and vports are freed if force=true (for the non + * attachedc ones, only) but SRIOV is not disabled and VFs are not + * removed in either case. + */ static int efx_ef10_pci_sriov_disable(struct efx_nic *efx, bool force) { struct pci_dev *dev = efx->pci_dev; - unsigned int vfs_assigned = 0; - - vfs_assigned = pci_vfs_assigned(dev); + unsigned int vfs_assigned = pci_vfs_assigned(dev); + int rc = 0; if (vfs_assigned && !force) { netif_info(efx, drv, efx->net_dev, "VFs are assigned to guests; " @@ -417,10 +419,12 @@ static int efx_ef10_pci_sriov_disable(struct efx_nic *efx, bool force) if (!vfs_assigned) pci_disable_sriov(dev); + else + rc = -EBUSY; efx_ef10_sriov_free_vf_vswitching(efx); efx->vf_count = 0; - return 0; + return rc; } int efx_ef10_sriov_configure(struct efx_nic *efx, int num_vfs) @@ -439,24 +443,18 @@ int efx_ef10_sriov_init(struct efx_nic *efx) void efx_ef10_sriov_fini(struct efx_nic *efx) { struct efx_ef10_nic_data *nic_data = efx->nic_data; - unsigned int i; int rc; if (!nic_data->vf) { - /* Remove any un-assigned orphaned VFs */ + /* Remove any un-assigned orphaned VFs. This can happen if the PF driver + * was unloaded while any VF was assigned to a guest (using Xen, only). + */ if (pci_num_vf(efx->pci_dev) && !pci_vfs_assigned(efx->pci_dev)) pci_disable_sriov(efx->pci_dev); return; } - /* Remove any VFs in the host */ - for (i = 0; i < efx->vf_count; ++i) { - struct efx_nic *vf_efx = nic_data->vf[i].efx; - - if (vf_efx) - vf_efx->pci_dev->driver->remove(vf_efx->pci_dev); - } - + /* Disable SRIOV and remove any VFs in the host */ rc = efx_ef10_pci_sriov_disable(efx, true); if (rc) netif_dbg(efx, drv, efx->net_dev, diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index c746ca7235f1..37fcf2eb0741 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -689,13 +689,13 @@ static struct notifier_block efx_netdev_notifier = { .notifier_call = efx_netdev_event, }; -static ssize_t -show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) +static ssize_t phy_type_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct efx_nic *efx = dev_get_drvdata(dev); return sprintf(buf, "%d\n", efx->phy_type); } -static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); +static DEVICE_ATTR_RO(phy_type); static int efx_register_netdev(struct efx_nic *efx) { @@ -722,8 +722,7 @@ static int efx_register_netdev(struct efx_nic *efx) efx->state = STATE_READY; smp_mb(); /* ensure we change state before checking reset_pending */ if (efx->reset_pending) { - netif_err(efx, probe, efx->net_dev, - "aborting probe due to scheduled reset\n"); + pci_err(efx->pci_dev, "aborting probe due to scheduled reset\n"); rc = -EIO; goto fail_locked; } @@ -990,8 +989,7 @@ static int efx_pci_probe_main(struct efx_nic *efx) rc = efx->type->init(efx); up_write(&efx->filter_sem); if (rc) { - netif_err(efx, probe, efx->net_dev, - "failed to initialise NIC\n"); + pci_err(efx->pci_dev, "failed to initialise NIC\n"); goto fail3; } @@ -1038,8 +1036,8 @@ static int efx_pci_probe_post_io(struct efx_nic *efx) if (efx->type->sriov_init) { rc = efx->type->sriov_init(efx); if (rc) - netif_err(efx, probe, efx->net_dev, - "SR-IOV can't be enabled rc %d\n", rc); + pci_err(efx->pci_dev, "SR-IOV can't be enabled rc %d\n", + rc); } /* Determine netdevice features */ @@ -1106,8 +1104,7 @@ static int efx_pci_probe(struct pci_dev *pci_dev, if (rc) goto fail1; - netif_info(efx, probe, efx->net_dev, - "Solarflare NIC detected\n"); + pci_info(pci_dev, "Solarflare NIC detected\n"); if (!efx->type->is_vf) efx_probe_vpd_strings(efx); diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index de797e1ac5a9..896b59253197 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -1160,8 +1160,9 @@ void efx_fini_io(struct efx_nic *efx) } #ifdef CONFIG_SFC_MCDI_LOGGING -static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, - char *buf) +static ssize_t mcdi_logging_show(struct device *dev, + struct device_attribute *attr, + char *buf) { struct efx_nic *efx = dev_get_drvdata(dev); struct efx_mcdi_iface *mcdi = efx_mcdi(efx); @@ -1169,8 +1170,9 @@ static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr, return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled); } -static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t mcdi_logging_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { struct efx_nic *efx = dev_get_drvdata(dev); struct efx_mcdi_iface *mcdi = efx_mcdi(efx); @@ -1180,7 +1182,7 @@ static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr, return count; } -static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log); +static DEVICE_ATTR_RW(mcdi_logging); void efx_init_mcdi_logging(struct efx_nic *efx) { diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c index 5e7a57b680ca..9ec752a43c75 100644 --- a/drivers/net/ethernet/sfc/falcon/efx.c +++ b/drivers/net/ethernet/sfc/falcon/efx.c @@ -2254,12 +2254,12 @@ static struct notifier_block ef4_netdev_notifier = { }; static ssize_t -show_phy_type(struct device *dev, struct device_attribute *attr, char *buf) +phy_type_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ef4_nic *efx = dev_get_drvdata(dev); return sprintf(buf, "%d\n", efx->phy_type); } -static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL); +static DEVICE_ATTR_RO(phy_type); static int ef4_register_netdev(struct ef4_nic *efx) { diff --git a/drivers/net/ethernet/sfc/falcon/falcon_boards.c b/drivers/net/ethernet/sfc/falcon/falcon_boards.c index 729a05c1b0cf..2d2d8099011e 100644 --- a/drivers/net/ethernet/sfc/falcon/falcon_boards.c +++ b/drivers/net/ethernet/sfc/falcon/falcon_boards.c @@ -354,16 +354,16 @@ fail_on: return rc; } -static ssize_t show_phy_flash_cfg(struct device *dev, +static ssize_t phy_flash_cfg_show(struct device *dev, struct device_attribute *attr, char *buf) { struct ef4_nic *efx = dev_get_drvdata(dev); return sprintf(buf, "%d\n", !!(efx->phy_mode & PHY_MODE_SPECIAL)); } -static ssize_t set_phy_flash_cfg(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t phy_flash_cfg_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { struct ef4_nic *efx = dev_get_drvdata(dev); enum ef4_phy_mode old_mode, new_mode; @@ -396,7 +396,7 @@ static ssize_t set_phy_flash_cfg(struct device *dev, return err ? err : count; } -static DEVICE_ATTR(phy_flash_cfg, 0644, show_phy_flash_cfg, set_phy_flash_cfg); +static DEVICE_ATTR_RW(phy_flash_cfg); static void sfe4001_fini(struct ef4_nic *efx) { diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c index 49df02ecee91..148dcd48b58d 100644 --- a/drivers/net/ethernet/sfc/farch.c +++ b/drivers/net/ethernet/sfc/farch.c @@ -1668,13 +1668,17 @@ void efx_farch_rx_pull_indir_table(struct efx_nic *efx) */ void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw) { - unsigned vi_count, buftbl_min, total_tx_channels; - + unsigned vi_count, total_tx_channels; #ifdef CONFIG_SFC_SRIOV - struct siena_nic_data *nic_data = efx->nic_data; + struct siena_nic_data *nic_data; + unsigned buftbl_min; #endif total_tx_channels = efx->n_tx_channels + efx->n_extra_tx_channels; + vi_count = max(efx->n_channels, total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL); + +#ifdef CONFIG_SFC_SRIOV + nic_data = efx->nic_data; /* Account for the buffer table entries backing the datapath channels * and the descriptor caches for those channels. */ @@ -1682,9 +1686,6 @@ void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw) total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL * EFX_MAX_DMAQ_SIZE + efx->n_channels * EFX_MAX_EVQ_SIZE) * sizeof(efx_qword_t) / EFX_BUF_SIZE); - vi_count = max(efx->n_channels, total_tx_channels * EFX_MAX_TXQ_PER_CHANNEL); - -#ifdef CONFIG_SFC_SRIOV if (efx->type->sriov_wanted) { if (efx->type->sriov_wanted(efx)) { unsigned vi_dc_entries, buftbl_free; diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c index d1e908846f5d..22fbb0ae77fb 100644 --- a/drivers/net/ethernet/sfc/nic.c +++ b/drivers/net/ethernet/sfc/nic.c @@ -90,6 +90,7 @@ int efx_nic_init_interrupt(struct efx_nic *efx) efx->pci_dev->irq); goto fail1; } + efx->irqs_hooked = true; return 0; } diff --git a/drivers/net/ethernet/sfc/rx.c b/drivers/net/ethernet/sfc/rx.c index 17b8119c48e5..606750938b89 100644 --- a/drivers/net/ethernet/sfc/rx.c +++ b/drivers/net/ethernet/sfc/rx.c @@ -260,18 +260,14 @@ static bool efx_do_xdp(struct efx_nic *efx, struct efx_channel *channel, s16 offset; int err; - rcu_read_lock(); - xdp_prog = rcu_dereference(efx->xdp_prog); - if (!xdp_prog) { - rcu_read_unlock(); + xdp_prog = rcu_dereference_bh(efx->xdp_prog); + if (!xdp_prog) return true; - } rx_queue = efx_channel_get_rx_queue(channel); if (unlikely(channel->rx_pkt_n_frags > 1)) { /* We can't do XDP on fragmented packets - drop. */ - rcu_read_unlock(); efx_free_rx_buffers(rx_queue, rx_buf, channel->rx_pkt_n_frags); if (net_ratelimit()) @@ -296,7 +292,6 @@ static bool efx_do_xdp(struct efx_nic *efx, struct efx_channel *channel, rx_buf->len, false); xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); - rcu_read_unlock(); offset = (u8 *)xdp.data - *ehp; diff --git a/drivers/net/ethernet/sgi/ioc3-eth.c b/drivers/net/ethernet/sgi/ioc3-eth.c index 6eef0f45b133..2b29fd4cbdf4 100644 --- a/drivers/net/ethernet/sgi/ioc3-eth.c +++ b/drivers/net/ethernet/sgi/ioc3-eth.c @@ -835,6 +835,10 @@ static int ioc3eth_probe(struct platform_device *pdev) int err; regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs) { + dev_err(&pdev->dev, "Invalid resource\n"); + return -EINVAL; + } /* get mac addr from one wire prom */ if (ioc3eth_get_mac_addr(regs, mac_addr)) return -EPROBE_DEFER; /* not available yet */ diff --git a/drivers/net/ethernet/sis/sis900.c b/drivers/net/ethernet/sis/sis900.c index 620c26f71be8..ca9c00b7f588 100644 --- a/drivers/net/ethernet/sis/sis900.c +++ b/drivers/net/ethernet/sis/sis900.c @@ -678,12 +678,12 @@ static int sis900_mii_probe(struct net_device *net_dev) /* Reset phy if default phy is internal sis900 */ if ((sis_priv->mii->phy_id0 == 0x001D) && ((sis_priv->mii->phy_id1&0xFFF0) == 0x8000)) - status = sis900_reset_phy(net_dev, sis_priv->cur_phy); + status = sis900_reset_phy(net_dev, sis_priv->cur_phy); /* workaround for ICS1893 PHY */ if ((sis_priv->mii->phy_id0 == 0x0015) && ((sis_priv->mii->phy_id1&0xFFF0) == 0xF440)) - mdio_write(net_dev, sis_priv->cur_phy, 0x0018, 0xD200); + mdio_write(net_dev, sis_priv->cur_phy, 0x0018, 0xD200); if(status & MII_STAT_LINK){ while (poll_bit) { @@ -727,7 +727,7 @@ static int sis900_mii_probe(struct net_device *net_dev) static u16 sis900_default_phy(struct net_device * net_dev) { struct sis900_private *sis_priv = netdev_priv(net_dev); - struct mii_phy *phy = NULL, *phy_home = NULL, + struct mii_phy *phy = NULL, *phy_home = NULL, *default_phy = NULL, *phy_lan = NULL; u16 status; @@ -1339,18 +1339,18 @@ static void sis900_timer(struct timer_list *t) } else { /* Link ON -> OFF */ if (!(status & MII_STAT_LINK)){ - netif_carrier_off(net_dev); + netif_carrier_off(net_dev); if(netif_msg_link(sis_priv)) - printk(KERN_INFO "%s: Media Link Off\n", net_dev->name); + printk(KERN_INFO "%s: Media Link Off\n", net_dev->name); - /* Change mode issue */ - if ((mii_phy->phy_id0 == 0x001D) && - ((mii_phy->phy_id1 & 0xFFF0) == 0x8000)) - sis900_reset_phy(net_dev, sis_priv->cur_phy); + /* Change mode issue */ + if ((mii_phy->phy_id0 == 0x001D) && + ((mii_phy->phy_id1 & 0xFFF0) == 0x8000)) + sis900_reset_phy(net_dev, sis_priv->cur_phy); sis630_set_eq(net_dev, sis_priv->chipset_rev); - goto LookForLink; + goto LookForLink; } } @@ -2331,7 +2331,7 @@ static int sis900_set_config(struct net_device *dev, struct ifmap *map) case IF_PORT_10BASE2: /* 10Base2 */ case IF_PORT_AUI: /* AUI */ case IF_PORT_100BASEFX: /* 100BaseFx */ - /* These Modes are not supported (are they?)*/ + /* These Modes are not supported (are they?)*/ return -EOPNOTSUPP; default: diff --git a/drivers/net/ethernet/smsc/smc9194.c b/drivers/net/ethernet/smsc/smc9194.c index 4b2330deed47..bf7c8c8b1350 100644 --- a/drivers/net/ethernet/smsc/smc9194.c +++ b/drivers/net/ethernet/smsc/smc9194.c @@ -182,8 +182,8 @@ struct smc_local { struct sk_buff * saved_skb; /* - . This keeps track of how many packets that I have - . sent out. When an TX_EMPTY interrupt comes, I know + . This keeps track of how many packets that I have + . sent out. When an TX_EMPTY interrupt comes, I know . that all of these have been sent. */ int packets_waiting; @@ -343,7 +343,7 @@ static void smc_reset( int ioaddr ) /* Note: It doesn't seem that waiting for the MMU busy is needed here, but this is a place where future chipsets _COULD_ break. Be wary - of issuing another MMU command right after this */ + of issuing another MMU command right after this */ outb( 0, ioaddr + INT_MASK ); } @@ -521,9 +521,9 @@ static netdev_tx_t smc_wait_to_send_packet(struct sk_buff *skb, SMC_SELECT_BANK( 2 ); outw( MC_ALLOC | numPages, ioaddr + MMU_CMD ); /* - . Performance Hack + . Performance Hack . - . wait a short amount of time.. if I can send a packet now, I send + . wait a short amount of time.. if I can send a packet now, I send . it now. Otherwise, I enable an interrupt and wait for one to be . available. . @@ -540,17 +540,17 @@ static netdev_tx_t smc_wait_to_send_packet(struct sk_buff *skb, if ( status & IM_ALLOC_INT ) { /* acknowledge the interrupt */ outb( IM_ALLOC_INT, ioaddr + INTERRUPT ); - break; + break; } - } while ( -- time_out ); + } while ( -- time_out ); - if ( !time_out ) { + if ( !time_out ) { /* oh well, wait until the chip finds memory later */ SMC_ENABLE_INT( IM_ALLOC_INT ); PRINTK2((CARDNAME": memory allocation deferred.\n")); /* it's deferred, but I'll handle it later */ return NETDEV_TX_OK; - } + } /* or YES! I can send the packet now.. */ smc_hardware_send_packet(dev); netif_wake_queue(dev); @@ -616,7 +616,7 @@ static void smc_hardware_send_packet( struct net_device * dev ) #endif /* send the packet length ( +6 for status, length and ctl byte ) - and the status word ( set to zeros ) */ + and the status word ( set to zeros ) */ #ifdef USE_32_BIT outl( (length +6 ) << 16 , ioaddr + DATA_1 ); #else @@ -629,8 +629,8 @@ static void smc_hardware_send_packet( struct net_device * dev ) /* send the actual data . I _think_ it's faster to send the longs first, and then . mop up by sending the last word. It depends heavily - . on alignment, at least on the 486. Maybe it would be - . a good idea to check which is optimal? But that could take + . on alignment, at least on the 486. Maybe it would be + . a good idea to check which is optimal? But that could take . almost as much time as is saved? */ #ifdef USE_32_BIT @@ -757,7 +757,7 @@ static int __init smc_findirq(int ioaddr) outb( IM_ALLOC_INT, ioaddr + INT_MASK ); /* - . Allocate 512 bytes of memory. Note that the chip was just + . Allocate 512 bytes of memory. Note that the chip was just . reset so all the memory is available */ outw( MC_ALLOC | 1, ioaddr + MMU_CMD ); @@ -871,7 +871,7 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) goto err_out; } /* The above MIGHT indicate a device, but I need to write to further - test this. */ + test this. */ outw( 0x0, ioaddr + BANK_SELECT ); bank = inw( ioaddr + BANK_SELECT ); if ( (bank & 0xFF00 ) != 0x3300 ) { @@ -879,7 +879,7 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) goto err_out; } /* well, we've already written once, so hopefully another time won't - hurt. This time, I need to switch the bank register to bank 1, + hurt. This time, I need to switch the bank register to bank 1, so I can access the base address register */ SMC_SELECT_BANK(1); base_address_register = inw( ioaddr + BASE ); @@ -917,7 +917,7 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) dev->base_addr = ioaddr; /* - . Get the MAC address ( bank 1, regs 4 - 9 ) + . Get the MAC address ( bank 1, regs 4 - 9 ) */ SMC_SELECT_BANK( 1 ); for ( i = 0; i < 6; i += 2 ) { @@ -938,8 +938,8 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) /* Now, I want to find out more about the chip. This is sort of - redundant, but it's cleaner to have it in both, rather than having - one VERY long probe procedure. + redundant, but it's cleaner to have it in both, rather than having + one VERY long probe procedure. */ SMC_SELECT_BANK(3); revision_register = inw( ioaddr + REVISION ); @@ -967,7 +967,7 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) /* . If dev->irq is 0, then the device has to be banged on to see . what the IRQ is. - . + . . This banging doesn't always detect the IRQ, for unknown reasons. . a workaround is to reset the chip and try again. . @@ -978,7 +978,7 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) . . Specifying an IRQ is done with the assumption that the user knows . what (s)he is doing. No checking is done!!!! - . + . */ if ( dev->irq < 2 ) { int trials; @@ -1070,7 +1070,7 @@ static int smc_open(struct net_device *dev) } /* - According to Becker, I have to set the hardware address + According to Becker, I have to set the hardware address at this point, because the (l)user can set it with an ioctl. Easily done... */ diff --git a/drivers/net/ethernet/smsc/smc91x.c b/drivers/net/ethernet/smsc/smc91x.c index cbde83f620a0..813ea941b91a 100644 --- a/drivers/net/ethernet/smsc/smc91x.c +++ b/drivers/net/ethernet/smsc/smc91x.c @@ -671,19 +671,19 @@ smc_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) status = SMC_GET_INT(lp); if (status & IM_ALLOC_INT) { SMC_ACK_INT(lp, IM_ALLOC_INT); - break; + break; } - } while (--poll_count); + } while (--poll_count); smc_special_unlock(&lp->lock, flags); lp->pending_tx_skb = skb; - if (!poll_count) { + if (!poll_count) { /* oh well, wait until the chip finds memory later */ netif_stop_queue(dev); DBG(2, dev, "TX memory allocation deferred.\n"); SMC_ENABLE_INT(lp, IM_ALLOC_INT); - } else { + } else { /* * Allocation succeeded: push packet to the chip's own memory * immediately. @@ -1790,7 +1790,7 @@ static int smc_findirq(struct smc_local *lp) SMC_SET_INT_MASK(lp, IM_ALLOC_INT); /* - * Allocate 512 bytes of memory. Note that the chip was just + * Allocate 512 bytes of memory. Note that the chip was just * reset so all the memory is available */ SMC_SET_MMU_CMD(lp, MC_ALLOC | 1); @@ -1998,8 +1998,8 @@ static int smc_probe(struct net_device *dev, void __iomem *ioaddr, /* Grab the IRQ */ retval = request_irq(dev->irq, smc_interrupt, irq_flags, dev->name, dev); - if (retval) - goto err_out; + if (retval) + goto err_out; #ifdef CONFIG_ARCH_PXA # ifdef SMC_USE_PXA_DMA diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index dfc85cc68173..20d148c019d8 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -958,7 +958,6 @@ static int netsec_process_rx(struct netsec_priv *priv, int budget) xdp_init_buff(&xdp, PAGE_SIZE, &dring->xdp_rxq); - rcu_read_lock(); xdp_prog = READ_ONCE(priv->xdp_prog); dma_dir = page_pool_get_dma_dir(dring->page_pool); @@ -1069,8 +1068,6 @@ next: } netsec_finalize_xdp_rx(priv, xdp_act, xdp_xmit); - rcu_read_unlock(); - return done; } diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c index fcbb4bb31408..5eb6bb4f7b6c 100644 --- a/drivers/net/ethernet/socionext/sni_ave.c +++ b/drivers/net/ethernet/socionext/sni_ave.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -/** +/* * sni_ave.c - Socionext UniPhier AVE ethernet driver * Copyright 2014 Panasonic Corporation * Copyright 2015-2017 Socionext Inc. diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig index 7737e4d0bb9e..ac3c248d4f9b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Kconfig +++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig @@ -66,6 +66,18 @@ config DWMAC_ANARION This selects the Anarion SoC glue layer support for the stmmac driver. +config DWMAC_INGENIC + tristate "Ingenic MAC support" + default MACH_INGENIC + depends on OF && HAS_IOMEM && (MACH_INGENIC || COMPILE_TEST) + select MFD_SYSCON + help + Support for ethernet controller on Ingenic SoCs. + + This selects Ingenic SoCs glue layer support for the stmmac + device driver. This driver is used on for the Ingenic SoCs + MAC ethernet controller. + config DWMAC_IPQ806X tristate "QCA IPQ806x DWMAC support" default ARCH_QCOM @@ -238,6 +250,15 @@ config DWMAC_INTEL This selects the Intel platform specific bus support for the stmmac driver. This driver is used for Intel Quark/EHL/TGL. +config DWMAC_LOONGSON + tristate "Loongson PCI DWMAC support" + default MACH_LOONGSON64 + depends on STMMAC_ETH && PCI + depends on COMMON_CLK + help + This selects the LOONGSON PCI bus support for the stmmac driver, + Support for ethernet controller on Loongson-2K1000 SoC and LS7A1000 bridge. + config STMMAC_PCI tristate "STMMAC PCI bus support" depends on STMMAC_ETH && PCI diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile index f2e478b884b0..d4e12e9ace4f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Makefile +++ b/drivers/net/ethernet/stmicro/stmmac/Makefile @@ -14,6 +14,7 @@ stmmac-$(CONFIG_STMMAC_SELFTESTS) += stmmac_selftests.o # Ordering matters. Generic driver must be last. obj-$(CONFIG_STMMAC_PLATFORM) += stmmac-platform.o obj-$(CONFIG_DWMAC_ANARION) += dwmac-anarion.o +obj-$(CONFIG_DWMAC_INGENIC) += dwmac-ingenic.o obj-$(CONFIG_DWMAC_IPQ806X) += dwmac-ipq806x.o obj-$(CONFIG_DWMAC_LPC18XX) += dwmac-lpc18xx.o obj-$(CONFIG_DWMAC_MEDIATEK) += dwmac-mediatek.o @@ -36,4 +37,5 @@ dwmac-altr-socfpga-objs := altr_tse_pcs.o dwmac-socfpga.o obj-$(CONFIG_STMMAC_PCI) += stmmac-pci.o obj-$(CONFIG_DWMAC_INTEL) += dwmac-intel.o +obj-$(CONFIG_DWMAC_LOONGSON) += dwmac-loongson.o stmmac-pci-objs:= stmmac_pci.o diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index 619e3c0760d6..5fecc83f175b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -503,8 +503,7 @@ struct mac_device_info { const struct stmmac_hwtimestamp *ptp; const struct stmmac_tc_ops *tc; const struct stmmac_mmc_ops *mmc; - const struct mdio_xpcs_ops *xpcs; - struct mdio_xpcs_args xpcs_args; + struct dw_xpcs *xpcs; struct mii_regs mii; /* MII register Addresses */ struct mac_link link; void __iomem *pcsr; /* vpointer to device CSRs */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c new file mode 100644 index 000000000000..9a6d819b84ae --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-ingenic.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dwmac-ingenic.c - Ingenic SoCs DWMAC specific glue layer + * + * Copyright (c) 2021 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_net.h> +#include <linux/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/stmmac.h> + +#include "stmmac_platform.h" + +#define MACPHYC_TXCLK_SEL_MASK GENMASK(31, 31) +#define MACPHYC_TXCLK_SEL_OUTPUT 0x1 +#define MACPHYC_TXCLK_SEL_INPUT 0x0 +#define MACPHYC_MODE_SEL_MASK GENMASK(31, 31) +#define MACPHYC_MODE_SEL_RMII 0x0 +#define MACPHYC_TX_SEL_MASK GENMASK(19, 19) +#define MACPHYC_TX_SEL_ORIGIN 0x0 +#define MACPHYC_TX_SEL_DELAY 0x1 +#define MACPHYC_TX_DELAY_MASK GENMASK(18, 12) +#define MACPHYC_RX_SEL_MASK GENMASK(11, 11) +#define MACPHYC_RX_SEL_ORIGIN 0x0 +#define MACPHYC_RX_SEL_DELAY 0x1 +#define MACPHYC_RX_DELAY_MASK GENMASK(10, 4) +#define MACPHYC_SOFT_RST_MASK GENMASK(3, 3) +#define MACPHYC_PHY_INFT_MASK GENMASK(2, 0) +#define MACPHYC_PHY_INFT_RMII 0x4 +#define MACPHYC_PHY_INFT_RGMII 0x1 +#define MACPHYC_PHY_INFT_GMII 0x0 +#define MACPHYC_PHY_INFT_MII 0x0 + +#define MACPHYC_TX_DELAY_PS_MAX 2496 +#define MACPHYC_TX_DELAY_PS_MIN 20 + +#define MACPHYC_RX_DELAY_PS_MAX 2496 +#define MACPHYC_RX_DELAY_PS_MIN 20 + +enum ingenic_mac_version { + ID_JZ4775, + ID_X1000, + ID_X1600, + ID_X1830, + ID_X2000, +}; + +struct ingenic_mac { + const struct ingenic_soc_info *soc_info; + struct device *dev; + struct regmap *regmap; + + int rx_delay; + int tx_delay; +}; + +struct ingenic_soc_info { + enum ingenic_mac_version version; + u32 mask; + + int (*set_mode)(struct plat_stmmacenet_data *plat_dat); +}; + +static int ingenic_mac_init(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + int ret; + + if (mac->soc_info->set_mode) { + ret = mac->soc_info->set_mode(plat_dat); + if (ret) + return ret; + } + + return 0; +} + +static int jz4775_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_MII: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_MII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_MII\n"); + break; + + case PHY_INTERFACE_MODE_GMII: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_GMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_GMII\n"); + break; + + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + val = FIELD_PREP(MACPHYC_TXCLK_SEL_MASK, MACPHYC_TXCLK_SEL_INPUT) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RGMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RGMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int x1000_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, 0); +} + +static int x1600_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int x1830_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_MODE_SEL_MASK, MACPHYC_MODE_SEL_RMII) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int x2000_mac_set_mode(struct plat_stmmacenet_data *plat_dat) +{ + struct ingenic_mac *mac = plat_dat->bsp_priv; + unsigned int val; + + switch (plat_dat->interface) { + case PHY_INTERFACE_MODE_RMII: + val = FIELD_PREP(MACPHYC_TX_SEL_MASK, MACPHYC_TX_SEL_ORIGIN) | + FIELD_PREP(MACPHYC_RX_SEL_MASK, MACPHYC_RX_SEL_ORIGIN) | + FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RMII); + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RMII\n"); + break; + + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + val = FIELD_PREP(MACPHYC_PHY_INFT_MASK, MACPHYC_PHY_INFT_RGMII); + + if (mac->tx_delay == 0) + val |= FIELD_PREP(MACPHYC_TX_SEL_MASK, MACPHYC_TX_SEL_ORIGIN); + else + val |= FIELD_PREP(MACPHYC_TX_SEL_MASK, MACPHYC_TX_SEL_DELAY) | + FIELD_PREP(MACPHYC_TX_DELAY_MASK, (mac->tx_delay + 9750) / 19500 - 1); + + if (mac->rx_delay == 0) + val |= FIELD_PREP(MACPHYC_RX_SEL_MASK, MACPHYC_RX_SEL_ORIGIN); + else + val |= FIELD_PREP(MACPHYC_RX_SEL_MASK, MACPHYC_RX_SEL_DELAY) | + FIELD_PREP(MACPHYC_RX_DELAY_MASK, (mac->rx_delay + 9750) / 19500 - 1); + + dev_dbg(mac->dev, "MAC PHY Control Register: PHY_INTERFACE_MODE_RGMII\n"); + break; + + default: + dev_err(mac->dev, "Unsupported interface %d", plat_dat->interface); + return -EINVAL; + } + + /* Update MAC PHY control register */ + return regmap_update_bits(mac->regmap, 0, mac->soc_info->mask, val); +} + +static int ingenic_mac_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + struct ingenic_mac *mac; + const struct ingenic_soc_info *data; + u32 tx_delay_ps, rx_delay_ps; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return ret; + + plat_dat = stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return PTR_ERR(plat_dat); + + mac = devm_kzalloc(&pdev->dev, sizeof(*mac), GFP_KERNEL); + if (!mac) { + ret = -ENOMEM; + goto err_remove_config_dt; + } + + data = of_device_get_match_data(&pdev->dev); + if (!data) { + dev_err(&pdev->dev, "No of match data provided\n"); + ret = -EINVAL; + goto err_remove_config_dt; + } + + /* Get MAC PHY control register */ + mac->regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "mode-reg"); + if (IS_ERR(mac->regmap)) { + dev_err(&pdev->dev, "%s: Failed to get syscon regmap\n", __func__); + ret = PTR_ERR(mac->regmap); + goto err_remove_config_dt; + } + + if (!of_property_read_u32(pdev->dev.of_node, "tx-clk-delay-ps", &tx_delay_ps)) { + if (tx_delay_ps >= MACPHYC_TX_DELAY_PS_MIN && + tx_delay_ps <= MACPHYC_TX_DELAY_PS_MAX) { + mac->tx_delay = tx_delay_ps * 1000; + } else { + dev_err(&pdev->dev, "Invalid TX clock delay: %dps\n", tx_delay_ps); + return -EINVAL; + } + } + + if (!of_property_read_u32(pdev->dev.of_node, "rx-clk-delay-ps", &rx_delay_ps)) { + if (rx_delay_ps >= MACPHYC_RX_DELAY_PS_MIN && + rx_delay_ps <= MACPHYC_RX_DELAY_PS_MAX) { + mac->rx_delay = rx_delay_ps * 1000; + } else { + dev_err(&pdev->dev, "Invalid RX clock delay: %dps\n", rx_delay_ps); + return -EINVAL; + } + } + + mac->soc_info = data; + mac->dev = &pdev->dev; + + plat_dat->bsp_priv = mac; + + ret = ingenic_mac_init(plat_dat); + if (ret) + goto err_remove_config_dt; + + ret = stmmac_dvr_probe(&pdev->dev, plat_dat, &stmmac_res); + if (ret) + goto err_remove_config_dt; + + return 0; + +err_remove_config_dt: + stmmac_remove_config_dt(pdev, plat_dat); + + return ret; +} + +#ifdef CONFIG_PM_SLEEP +static int ingenic_mac_suspend(struct device *dev) +{ + int ret; + + ret = stmmac_suspend(dev); + + return ret; +} + +static int ingenic_mac_resume(struct device *dev) +{ + struct net_device *ndev = dev_get_drvdata(dev); + struct stmmac_priv *priv = netdev_priv(ndev); + int ret; + + ret = ingenic_mac_init(priv->plat); + if (ret) + return ret; + + ret = stmmac_resume(dev); + + return ret; +} +#endif /* CONFIG_PM_SLEEP */ + +static SIMPLE_DEV_PM_OPS(ingenic_mac_pm_ops, ingenic_mac_suspend, ingenic_mac_resume); + +static struct ingenic_soc_info jz4775_soc_info = { + .version = ID_JZ4775, + .mask = MACPHYC_TXCLK_SEL_MASK | MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = jz4775_mac_set_mode, +}; + +static struct ingenic_soc_info x1000_soc_info = { + .version = ID_X1000, + .mask = MACPHYC_SOFT_RST_MASK, + + .set_mode = x1000_mac_set_mode, +}; + +static struct ingenic_soc_info x1600_soc_info = { + .version = ID_X1600, + .mask = MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = x1600_mac_set_mode, +}; + +static struct ingenic_soc_info x1830_soc_info = { + .version = ID_X1830, + .mask = MACPHYC_MODE_SEL_MASK | MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = x1830_mac_set_mode, +}; + +static struct ingenic_soc_info x2000_soc_info = { + .version = ID_X2000, + .mask = MACPHYC_TX_SEL_MASK | MACPHYC_TX_DELAY_MASK | MACPHYC_RX_SEL_MASK | + MACPHYC_RX_DELAY_MASK | MACPHYC_SOFT_RST_MASK | MACPHYC_PHY_INFT_MASK, + + .set_mode = x2000_mac_set_mode, +}; + +static const struct of_device_id ingenic_mac_of_matches[] = { + { .compatible = "ingenic,jz4775-mac", .data = &jz4775_soc_info }, + { .compatible = "ingenic,x1000-mac", .data = &x1000_soc_info }, + { .compatible = "ingenic,x1600-mac", .data = &x1600_soc_info }, + { .compatible = "ingenic,x1830-mac", .data = &x1830_soc_info }, + { .compatible = "ingenic,x2000-mac", .data = &x2000_soc_info }, + { } +}; +MODULE_DEVICE_TABLE(of, ingenic_mac_of_matches); + +static struct platform_driver ingenic_mac_driver = { + .probe = ingenic_mac_probe, + .remove = stmmac_pltfr_remove, + .driver = { + .name = "ingenic-mac", + .pm = pm_ptr(&ingenic_mac_pm_ops), + .of_match_table = ingenic_mac_of_matches, + }, +}; +module_platform_driver(ingenic_mac_driver); + +MODULE_AUTHOR("周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>"); +MODULE_DESCRIPTION("Ingenic SoCs DWMAC specific glue layer"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 80728a4c0e3f..8e8778cfbbad 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -10,22 +10,6 @@ #include "stmmac.h" #include "stmmac_ptp.h" -#define INTEL_MGBE_ADHOC_ADDR 0x15 -#define INTEL_MGBE_XPCS_ADDR 0x16 - -/* Selection for PTP Clock Freq belongs to PSE & PCH GbE */ -#define PSE_PTP_CLK_FREQ_MASK (GMAC_GPO0 | GMAC_GPO3) -#define PSE_PTP_CLK_FREQ_19_2MHZ (GMAC_GPO0) -#define PSE_PTP_CLK_FREQ_200MHZ (GMAC_GPO0 | GMAC_GPO3) -#define PSE_PTP_CLK_FREQ_256MHZ (0) -#define PCH_PTP_CLK_FREQ_MASK (GMAC_GPO0) -#define PCH_PTP_CLK_FREQ_19_2MHZ (GMAC_GPO0) -#define PCH_PTP_CLK_FREQ_200MHZ (0) - -/* Cross-timestamping defines */ -#define ART_CPUID_LEAF 0x15 -#define EHL_PSE_ART_MHZ 19200000 - struct intel_priv_data { int mdio_adhoc_addr; /* mdio address for serdes & etc */ unsigned long crossts_adj; @@ -102,6 +86,22 @@ static int intel_serdes_powerup(struct net_device *ndev, void *priv_data) serdes_phy_addr = intel_priv->mdio_adhoc_addr; + /* Set the serdes rate and the PCLK rate */ + data = mdiobus_read(priv->mii, serdes_phy_addr, + SERDES_GCR0); + + data &= ~SERDES_RATE_MASK; + data &= ~SERDES_PCLK_MASK; + + if (priv->plat->max_speed == 2500) + data |= SERDES_RATE_PCIE_GEN2 << SERDES_RATE_PCIE_SHIFT | + SERDES_PCLK_37p5MHZ << SERDES_PCLK_SHIFT; + else + data |= SERDES_RATE_PCIE_GEN1 << SERDES_RATE_PCIE_SHIFT | + SERDES_PCLK_70MHZ << SERDES_PCLK_SHIFT; + + mdiobus_write(priv->mii, serdes_phy_addr, SERDES_GCR0, data); + /* assert clk_req */ data = mdiobus_read(priv->mii, serdes_phy_addr, SERDES_GCR0); data |= SERDES_PLL_CLK; @@ -230,6 +230,32 @@ static void intel_serdes_powerdown(struct net_device *ndev, void *intel_data) } } +static void intel_speed_mode_2500(struct net_device *ndev, void *intel_data) +{ + struct intel_priv_data *intel_priv = intel_data; + struct stmmac_priv *priv = netdev_priv(ndev); + int serdes_phy_addr = 0; + u32 data = 0; + + serdes_phy_addr = intel_priv->mdio_adhoc_addr; + + /* Determine the link speed mode: 2.5Gbps/1Gbps */ + data = mdiobus_read(priv->mii, serdes_phy_addr, + SERDES_GCR); + + if (((data & SERDES_LINK_MODE_MASK) >> SERDES_LINK_MODE_SHIFT) == + SERDES_LINK_MODE_2G5) { + dev_info(priv->device, "Link Speed Mode: 2.5Gbps\n"); + priv->plat->max_speed = 2500; + priv->plat->phy_interface = PHY_INTERFACE_MODE_2500BASEX; + priv->plat->mdio_bus_data->xpcs_an_inband = false; + } else { + priv->plat->max_speed = 1000; + priv->plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + priv->plat->mdio_bus_data->xpcs_an_inband = true; + } +} + /* Program PTP Clock Frequency for different variant of * Intel mGBE that has slightly different GPO mapping */ @@ -429,6 +455,17 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->force_sf_dma_mode = 0; plat->tso_en = 1; + /* Multiplying factor to the clk_eee_i clock time + * period to make it closer to 100 ns. This value + * should be programmed such that the clk_eee_time_period * + * (MULT_FACT_100NS + 1) should be within 80 ns to 120 ns + * clk_eee frequency is 19.2Mhz + * clk_eee_time_period is 52ns + * 52ns * (1 + 1) = 104ns + * MULT_FACT_100NS = 1 + */ + plat->mult_fact_100ns = 1; + plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP; for (i = 0; i < plat->rx_queues_to_use; i++) { @@ -556,6 +593,17 @@ static int ehl_common_data(struct pci_dev *pdev, plat->rx_queues_to_use = 8; plat->tx_queues_to_use = 8; plat->clk_ptp_rate = 200000000; + plat->use_phy_wol = 1; + + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 1; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 0; + plat->safety_feat_cfg->edpp = 0; + plat->safety_feat_cfg->prtyen = 0; + plat->safety_feat_cfg->tmouten = 0; return intel_mgbe_common_data(pdev, plat); } @@ -565,7 +613,7 @@ static int ehl_sgmii_data(struct pci_dev *pdev, { plat->bus_id = 1; plat->phy_interface = PHY_INTERFACE_MODE_SGMII; - + plat->speed_mode_2500 = intel_speed_mode_2500; plat->serdes_powerup = intel_serdes_powerup; plat->serdes_powerdown = intel_serdes_powerdown; @@ -618,6 +666,7 @@ static int ehl_pse0_sgmii1g_data(struct pci_dev *pdev, struct plat_stmmacenet_data *plat) { plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; plat->serdes_powerup = intel_serdes_powerup; plat->serdes_powerdown = intel_serdes_powerdown; return ehl_pse0_common_data(pdev, plat); @@ -656,6 +705,7 @@ static int ehl_pse1_sgmii1g_data(struct pci_dev *pdev, struct plat_stmmacenet_data *plat) { plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; plat->serdes_powerup = intel_serdes_powerup; plat->serdes_powerdown = intel_serdes_powerdown; return ehl_pse1_common_data(pdev, plat); @@ -672,6 +722,16 @@ static int tgl_common_data(struct pci_dev *pdev, plat->tx_queues_to_use = 4; plat->clk_ptp_rate = 200000000; + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 0; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 0; + plat->safety_feat_cfg->edpp = 0; + plat->safety_feat_cfg->prtyen = 0; + plat->safety_feat_cfg->tmouten = 0; + return intel_mgbe_common_data(pdev, plat); } @@ -680,6 +740,7 @@ static int tgl_sgmii_phy0_data(struct pci_dev *pdev, { plat->bus_id = 1; plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; plat->serdes_powerup = intel_serdes_powerup; plat->serdes_powerdown = intel_serdes_powerdown; return tgl_common_data(pdev, plat); @@ -694,6 +755,7 @@ static int tgl_sgmii_phy1_data(struct pci_dev *pdev, { plat->bus_id = 2; plat->phy_interface = PHY_INTERFACE_MODE_SGMII; + plat->speed_mode_2500 = intel_speed_mode_2500; plat->serdes_powerup = intel_serdes_powerup; plat->serdes_powerdown = intel_serdes_powerdown; return tgl_common_data(pdev, plat); @@ -948,6 +1010,12 @@ static int intel_eth_pci_probe(struct pci_dev *pdev, if (!plat->dma_cfg) return -ENOMEM; + plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, + sizeof(*plat->safety_feat_cfg), + GFP_KERNEL); + if (!plat->safety_feat_cfg) + return -ENOMEM; + /* Enable pci device */ ret = pcim_enable_device(pdev); if (ret) { @@ -1020,7 +1088,7 @@ err_alloc_irq: /** * intel_eth_pci_remove * - * @pdev: platform device pointer + * @pdev: pci device pointer * Description: this function calls the main to free the net resources * and releases the PCI resources. */ @@ -1050,6 +1118,7 @@ static int __maybe_unused intel_eth_pci_suspend(struct device *dev) return ret; pci_wake_from_d3(pdev, true); + pci_set_power_state(pdev, PCI_D3hot); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h index 542acb8ce467..0a37987478c1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.h @@ -9,6 +9,7 @@ #define POLL_DELAY_US 8 /* SERDES Register */ +#define SERDES_GCR 0x0 /* Global Conguration */ #define SERDES_GSR0 0x5 /* Global Status Reg0 */ #define SERDES_GCR0 0xb /* Global Configuration Reg0 */ @@ -17,8 +18,36 @@ #define SERDES_PHY_RX_CLK BIT(1) /* PSE SGMII PHY rx clk */ #define SERDES_RST BIT(2) /* Serdes Reset */ #define SERDES_PWR_ST_MASK GENMASK(6, 4) /* Serdes Power state*/ +#define SERDES_RATE_MASK GENMASK(9, 8) +#define SERDES_PCLK_MASK GENMASK(14, 12) /* PCLK rate to PHY */ +#define SERDES_LINK_MODE_MASK GENMASK(2, 1) +#define SERDES_LINK_MODE_SHIFT 1 #define SERDES_PWR_ST_SHIFT 4 #define SERDES_PWR_ST_P0 0x0 #define SERDES_PWR_ST_P3 0x3 +#define SERDES_LINK_MODE_2G5 0x3 +#define SERSED_LINK_MODE_1G 0x2 +#define SERDES_PCLK_37p5MHZ 0x0 +#define SERDES_PCLK_70MHZ 0x1 +#define SERDES_RATE_PCIE_GEN1 0x0 +#define SERDES_RATE_PCIE_GEN2 0x1 +#define SERDES_RATE_PCIE_SHIFT 8 +#define SERDES_PCLK_SHIFT 12 + +#define INTEL_MGBE_ADHOC_ADDR 0x15 +#define INTEL_MGBE_XPCS_ADDR 0x16 + +/* Cross-timestamping defines */ +#define ART_CPUID_LEAF 0x15 +#define EHL_PSE_ART_MHZ 19200000 + +/* Selection for PTP Clock Freq belongs to PSE & PCH GbE */ +#define PSE_PTP_CLK_FREQ_MASK (GMAC_GPO0 | GMAC_GPO3) +#define PSE_PTP_CLK_FREQ_19_2MHZ (GMAC_GPO0) +#define PSE_PTP_CLK_FREQ_200MHZ (GMAC_GPO0 | GMAC_GPO3) +#define PSE_PTP_CLK_FREQ_256MHZ (0) +#define PCH_PTP_CLK_FREQ_MASK (GMAC_GPO0) +#define PCH_PTP_CLK_FREQ_19_2MHZ (GMAC_GPO0) +#define PCH_PTP_CLK_FREQ_200MHZ (0) #endif /* __DWMAC_INTEL_H__ */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c new file mode 100644 index 000000000000..e108b0d2bd28 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2020, Loongson Corporation + */ + +#include <linux/clk-provider.h> +#include <linux/pci.h> +#include <linux/dmi.h> +#include <linux/device.h> +#include <linux/of_irq.h> +#include "stmmac.h" + +static int loongson_default_data(struct plat_stmmacenet_data *plat) +{ + plat->clk_csr = 2; /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ + plat->has_gmac = 1; + plat->force_sf_dma_mode = 1; + + /* Set default value for multicast hash bins */ + plat->multicast_filter_bins = HASH_TABLE_SIZE; + + /* Set default value for unicast filter entries */ + plat->unicast_filter_entries = 1; + + /* Set the maxmtu to a default of JUMBO_LEN */ + plat->maxmtu = JUMBO_LEN; + + /* Set default number of RX and TX queues to use */ + plat->tx_queues_to_use = 1; + plat->rx_queues_to_use = 1; + + /* Disable Priority config by default */ + plat->tx_queues_cfg[0].use_prio = false; + plat->rx_queues_cfg[0].use_prio = false; + + /* Disable RX queues routing by default */ + plat->rx_queues_cfg[0].pkt_route = 0x0; + + /* Default to phy auto-detection */ + plat->phy_addr = -1; + + plat->dma_cfg->pbl = 32; + plat->dma_cfg->pblx8 = true; + + plat->multicast_filter_bins = 256; + return 0; +} + +static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct plat_stmmacenet_data *plat; + struct stmmac_resources res; + bool mdio = false; + int ret, i; + struct device_node *np; + + np = dev_of_node(&pdev->dev); + + if (!np) { + pr_info("dwmac_loongson_pci: No OF node\n"); + return -ENODEV; + } + + if (!of_device_is_compatible(np, "loongson, pci-gmac")) { + pr_info("dwmac_loongson_pci: Incompatible OF node\n"); + return -ENODEV; + } + + plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL); + if (!plat) + return -ENOMEM; + + if (plat->mdio_node) { + dev_err(&pdev->dev, "Found MDIO subnode\n"); + mdio = true; + } + + if (mdio) { + plat->mdio_bus_data = devm_kzalloc(&pdev->dev, + sizeof(*plat->mdio_bus_data), + GFP_KERNEL); + if (!plat->mdio_bus_data) + return -ENOMEM; + plat->mdio_bus_data->needs_reset = true; + } + + plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), GFP_KERNEL); + if (!plat->dma_cfg) + return -ENOMEM; + + /* Enable pci device */ + ret = pci_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", __func__); + return ret; + } + + /* Get the base address of device */ + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (pci_resource_len(pdev, i) == 0) + continue; + ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (ret) + return ret; + break; + } + + plat->bus_id = of_alias_get_id(np, "ethernet"); + if (plat->bus_id < 0) + plat->bus_id = pci_dev_id(pdev); + + plat->phy_interface = device_get_phy_mode(&pdev->dev); + if (plat->phy_interface < 0) + dev_err(&pdev->dev, "phy_mode not found\n"); + + plat->interface = PHY_INTERFACE_MODE_GMII; + + pci_set_master(pdev); + + loongson_default_data(plat); + pci_enable_msi(pdev); + memset(&res, 0, sizeof(res)); + res.addr = pcim_iomap_table(pdev)[0]; + + res.irq = of_irq_get_byname(np, "macirq"); + if (res.irq < 0) { + dev_err(&pdev->dev, "IRQ macirq not found\n"); + ret = -ENODEV; + } + + res.wol_irq = of_irq_get_byname(np, "eth_wake_irq"); + if (res.wol_irq < 0) { + dev_info(&pdev->dev, "IRQ eth_wake_irq not found, using macirq\n"); + res.wol_irq = res.irq; + } + + res.lpi_irq = of_irq_get_byname(np, "eth_lpi"); + if (res.lpi_irq < 0) { + dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); + ret = -ENODEV; + } + + return stmmac_dvr_probe(&pdev->dev, plat, &res); +} + +static void loongson_dwmac_remove(struct pci_dev *pdev) +{ + int i; + + stmmac_dvr_remove(&pdev->dev); + + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (pci_resource_len(pdev, i) == 0) + continue; + pcim_iounmap_regions(pdev, BIT(i)); + break; + } + + pci_disable_device(pdev); +} + +static int __maybe_unused loongson_dwmac_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = stmmac_suspend(dev); + if (ret) + return ret; + + ret = pci_save_state(pdev); + if (ret) + return ret; + + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + return 0; +} + +static int __maybe_unused loongson_dwmac_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pci_restore_state(pdev); + pci_set_power_state(pdev, PCI_D0); + + ret = pci_enable_device(pdev); + if (ret) + return ret; + + pci_set_master(pdev); + + return stmmac_resume(dev); +} + +static SIMPLE_DEV_PM_OPS(loongson_dwmac_pm_ops, loongson_dwmac_suspend, + loongson_dwmac_resume); + +static const struct pci_device_id loongson_dwmac_id_table[] = { + { PCI_VDEVICE(LOONGSON, 0x7a03) }, + {} +}; +MODULE_DEVICE_TABLE(pci, loongson_dwmac_id_table); + +struct pci_driver loongson_dwmac_driver = { + .name = "dwmac-loongson-pci", + .id_table = loongson_dwmac_id_table, + .probe = loongson_dwmac_probe, + .remove = loongson_dwmac_remove, + .driver = { + .pm = &loongson_dwmac_pm_ops, + }, +}; + +module_pci_driver(loongson_dwmac_driver); + +MODULE_DESCRIPTION("Loongson DWMAC PCI driver"); +MODULE_AUTHOR("Qing Zhang <zhangqing@loongson.cn>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index 84382fc5cc4d..5c74b6279d69 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -454,7 +454,6 @@ static int qcom_ethqos_probe(struct platform_device *pdev) struct stmmac_resources stmmac_res; const struct ethqos_emac_driver_data *data; struct qcom_ethqos *ethqos; - struct resource *res; int ret; ret = stmmac_get_platform_resources(pdev, &stmmac_res); @@ -474,8 +473,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) } ethqos->pdev = pdev; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rgmii"); - ethqos->rgmii_base = devm_ioremap_resource(&pdev->dev, res); + ethqos->rgmii_base = devm_platform_ioremap_resource_byname(pdev, "rgmii"); if (IS_ERR(ethqos->rgmii_base)) { ret = PTR_ERR(ethqos->rgmii_base); goto err_mem; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index 8d28a536e1bb..280ac0129572 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -33,11 +33,13 @@ struct rk_gmac_ops { void (*set_rgmii_speed)(struct rk_priv_data *bsp_priv, int speed); void (*set_rmii_speed)(struct rk_priv_data *bsp_priv, int speed); void (*integrated_phy_powerup)(struct rk_priv_data *bsp_priv); + u32 regs[]; }; struct rk_priv_data { struct platform_device *pdev; phy_interface_t phy_iface; + int id; struct regulator *regulator; bool suspended; const struct rk_gmac_ops *ops; @@ -482,6 +484,54 @@ static const struct rk_gmac_ops rk3288_ops = { .set_rmii_speed = rk3288_set_rmii_speed, }; +#define RK3308_GRF_MAC_CON0 0x04a0 + +/* RK3308_GRF_MAC_CON0 */ +#define RK3308_GMAC_PHY_INTF_SEL_RMII (GRF_CLR_BIT(2) | GRF_CLR_BIT(3) | \ + GRF_BIT(4)) +#define RK3308_GMAC_FLOW_CTRL GRF_BIT(3) +#define RK3308_GMAC_FLOW_CTRL_CLR GRF_CLR_BIT(3) +#define RK3308_GMAC_SPEED_10M GRF_CLR_BIT(0) +#define RK3308_GMAC_SPEED_100M GRF_BIT(0) + +static void rk3308_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + regmap_write(bsp_priv->grf, RK3308_GRF_MAC_CON0, + RK3308_GMAC_PHY_INTF_SEL_RMII); +} + +static void rk3308_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + if (speed == 10) { + regmap_write(bsp_priv->grf, RK3308_GRF_MAC_CON0, + RK3308_GMAC_SPEED_10M); + } else if (speed == 100) { + regmap_write(bsp_priv->grf, RK3308_GRF_MAC_CON0, + RK3308_GMAC_SPEED_100M); + } else { + dev_err(dev, "unknown speed value for RMII! speed=%d", speed); + } +} + +static const struct rk_gmac_ops rk3308_ops = { + .set_to_rmii = rk3308_set_to_rmii, + .set_rmii_speed = rk3308_set_rmii_speed, +}; + #define RK3328_GRF_MAC_CON0 0x0900 #define RK3328_GRF_MAC_CON1 0x0904 #define RK3328_GRF_MAC_CON2 0x0908 @@ -948,6 +998,107 @@ static const struct rk_gmac_ops rk3399_ops = { .set_rmii_speed = rk3399_set_rmii_speed, }; +#define RK3568_GRF_GMAC0_CON0 0x0380 +#define RK3568_GRF_GMAC0_CON1 0x0384 +#define RK3568_GRF_GMAC1_CON0 0x0388 +#define RK3568_GRF_GMAC1_CON1 0x038c + +/* RK3568_GRF_GMAC0_CON1 && RK3568_GRF_GMAC1_CON1 */ +#define RK3568_GMAC_PHY_INTF_SEL_RGMII \ + (GRF_BIT(4) | GRF_CLR_BIT(5) | GRF_CLR_BIT(6)) +#define RK3568_GMAC_PHY_INTF_SEL_RMII \ + (GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | GRF_BIT(6)) +#define RK3568_GMAC_FLOW_CTRL GRF_BIT(3) +#define RK3568_GMAC_FLOW_CTRL_CLR GRF_CLR_BIT(3) +#define RK3568_GMAC_RXCLK_DLY_ENABLE GRF_BIT(1) +#define RK3568_GMAC_RXCLK_DLY_DISABLE GRF_CLR_BIT(1) +#define RK3568_GMAC_TXCLK_DLY_ENABLE GRF_BIT(0) +#define RK3568_GMAC_TXCLK_DLY_DISABLE GRF_CLR_BIT(0) + +/* RK3568_GRF_GMAC0_CON0 && RK3568_GRF_GMAC1_CON0 */ +#define RK3568_GMAC_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 8) +#define RK3568_GMAC_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 0) + +static void rk3568_set_to_rgmii(struct rk_priv_data *bsp_priv, + int tx_delay, int rx_delay) +{ + struct device *dev = &bsp_priv->pdev->dev; + u32 con0, con1; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + con0 = (bsp_priv->id == 1) ? RK3568_GRF_GMAC1_CON0 : + RK3568_GRF_GMAC0_CON0; + con1 = (bsp_priv->id == 1) ? RK3568_GRF_GMAC1_CON1 : + RK3568_GRF_GMAC0_CON1; + + regmap_write(bsp_priv->grf, con0, + RK3568_GMAC_CLK_RX_DL_CFG(rx_delay) | + RK3568_GMAC_CLK_TX_DL_CFG(tx_delay)); + + regmap_write(bsp_priv->grf, con1, + RK3568_GMAC_PHY_INTF_SEL_RGMII | + RK3568_GMAC_RXCLK_DLY_ENABLE | + RK3568_GMAC_TXCLK_DLY_ENABLE); +} + +static void rk3568_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + u32 con1; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "%s: Missing rockchip,grf property\n", __func__); + return; + } + + con1 = (bsp_priv->id == 1) ? RK3568_GRF_GMAC1_CON1 : + RK3568_GRF_GMAC0_CON1; + regmap_write(bsp_priv->grf, con1, RK3568_GMAC_PHY_INTF_SEL_RMII); +} + +static void rk3568_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned long rate; + int ret; + + switch (speed) { + case 10: + rate = 2500000; + break; + case 100: + rate = 25000000; + break; + case 1000: + rate = 125000000; + break; + default: + dev_err(dev, "unknown speed value for GMAC speed=%d", speed); + return; + } + + ret = clk_set_rate(bsp_priv->clk_mac_speed, rate); + if (ret) + dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n", + __func__, rate, ret); +} + +static const struct rk_gmac_ops rk3568_ops = { + .set_to_rgmii = rk3568_set_to_rgmii, + .set_to_rmii = rk3568_set_to_rmii, + .set_rgmii_speed = rk3568_set_gmac_speed, + .set_rmii_speed = rk3568_set_gmac_speed, + .regs = { + 0xfe2a0000, /* gmac0 */ + 0xfe010000, /* gmac1 */ + 0x0, /* sentinel */ + }, +}; + #define RV1108_GRF_GMAC_CON0 0X0900 /* RV1108_GRF_GMAC_CON0 */ @@ -1216,6 +1367,7 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, { struct rk_priv_data *bsp_priv; struct device *dev = &pdev->dev; + struct resource *res; int ret; const char *strings = NULL; int value; @@ -1227,6 +1379,22 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, of_get_phy_mode(dev->of_node, &bsp_priv->phy_iface); bsp_priv->ops = ops; + /* Some SoCs have multiple MAC controllers, which need + * to be distinguished. + */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res) { + int i = 0; + + while (ops->regs[i]) { + if (ops->regs[i] == res->start) { + bsp_priv->id = i; + break; + } + i++; + } + } + bsp_priv->regulator = devm_regulator_get_optional(dev, "phy"); if (IS_ERR(bsp_priv->regulator)) { if (PTR_ERR(bsp_priv->regulator) == -EPROBE_DEFER) { @@ -1294,11 +1462,36 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, return bsp_priv; } +static int rk_gmac_check_ops(struct rk_priv_data *bsp_priv) +{ + switch (bsp_priv->phy_iface) { + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + if (!bsp_priv->ops->set_to_rgmii) + return -EINVAL; + break; + case PHY_INTERFACE_MODE_RMII: + if (!bsp_priv->ops->set_to_rmii) + return -EINVAL; + break; + default: + dev_err(&bsp_priv->pdev->dev, + "unsupported interface %d", bsp_priv->phy_iface); + } + return 0; +} + static int rk_gmac_powerup(struct rk_priv_data *bsp_priv) { int ret; struct device *dev = &bsp_priv->pdev->dev; + ret = rk_gmac_check_ops(bsp_priv); + if (ret) + return ret; + ret = gmac_clk_enable(bsp_priv, true); if (ret) return ret; @@ -1369,10 +1562,12 @@ static void rk_fix_speed(void *priv, unsigned int speed) case PHY_INTERFACE_MODE_RGMII_ID: case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: - bsp_priv->ops->set_rgmii_speed(bsp_priv, speed); + if (bsp_priv->ops->set_rgmii_speed) + bsp_priv->ops->set_rgmii_speed(bsp_priv, speed); break; case PHY_INTERFACE_MODE_RMII: - bsp_priv->ops->set_rmii_speed(bsp_priv, speed); + if (bsp_priv->ops->set_rmii_speed) + bsp_priv->ops->set_rmii_speed(bsp_priv, speed); break; default: dev_err(dev, "unsupported interface %d", bsp_priv->phy_iface); @@ -1400,7 +1595,11 @@ static int rk_gmac_probe(struct platform_device *pdev) if (IS_ERR(plat_dat)) return PTR_ERR(plat_dat); - plat_dat->has_gmac = true; + /* If the stmmac is not already selected as gmac4, + * then make sure we fallback to gmac. + */ + if (!plat_dat->has_gmac4) + plat_dat->has_gmac = true; plat_dat->fix_mac_speed = rk_fix_speed; plat_dat->bsp_priv = rk_gmac_setup(pdev, plat_dat, data); @@ -1477,10 +1676,12 @@ static const struct of_device_id rk_gmac_dwmac_match[] = { { .compatible = "rockchip,rk3128-gmac", .data = &rk3128_ops }, { .compatible = "rockchip,rk3228-gmac", .data = &rk3228_ops }, { .compatible = "rockchip,rk3288-gmac", .data = &rk3288_ops }, + { .compatible = "rockchip,rk3308-gmac", .data = &rk3308_ops }, { .compatible = "rockchip,rk3328-gmac", .data = &rk3328_ops }, { .compatible = "rockchip,rk3366-gmac", .data = &rk3366_ops }, { .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops }, { .compatible = "rockchip,rk3399-gmac", .data = &rk3399_ops }, + { .compatible = "rockchip,rk3568-gmac", .data = &rk3568_ops }, { .compatible = "rockchip,rv1108-gmac", .data = &rv1108_ops }, { } }; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c index 527077c98ebc..fc3b0acc8f99 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c @@ -30,7 +30,7 @@ struct sunxi_priv_data { static int sun7i_gmac_init(struct platform_device *pdev, void *priv) { struct sunxi_priv_data *gmac = priv; - int ret; + int ret = 0; if (gmac->regulator) { ret = regulator_enable(gmac->regulator); @@ -51,11 +51,11 @@ static int sun7i_gmac_init(struct platform_device *pdev, void *priv) } else { clk_set_rate(gmac->tx_clk, SUN7I_GMAC_MII_RATE); ret = clk_prepare(gmac->tx_clk); - if (ret) - return ret; + if (ret && gmac->regulator) + regulator_disable(gmac->regulator); } - return 0; + return ret; } static void sun7i_gmac_exit(struct platform_device *pdev, void *priv) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h index b70d44ac0990..3c73453725f9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h @@ -76,10 +76,10 @@ enum power_event { #define LPI_CTRL_STATUS_TLPIEN 0x00000001 /* Transmit LPI Entry */ /* GMAC HW ADDR regs */ -#define GMAC_ADDR_HIGH(reg) (((reg > 15) ? 0x00000800 : 0x00000040) + \ - (reg * 8)) -#define GMAC_ADDR_LOW(reg) (((reg > 15) ? 0x00000804 : 0x00000044) + \ - (reg * 8)) +#define GMAC_ADDR_HIGH(reg) ((reg > 15) ? 0x00000800 + (reg - 16) * 8 : \ + 0x00000040 + (reg * 8)) +#define GMAC_ADDR_LOW(reg) ((reg > 15) ? 0x00000804 + (reg - 16) * 8 : \ + 0x00000044 + (reg * 8)) #define GMAC_MAX_PERFECT_ADDRESSES 1 #define GMAC_PCS_BASE 0x000000c0 /* PCS register base */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index f35c03c9f91e..67ba083eb90c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -1358,6 +1358,7 @@ int dwmac4_setup(struct stmmac_priv *priv) mac->link.speed10 = GMAC_CONFIG_PS; mac->link.speed100 = GMAC_CONFIG_FES | GMAC_CONFIG_PS; mac->link.speed1000 = 0; + mac->link.speed2500 = GMAC_CONFIG_FES; mac->link.speed_mask = GMAC_CONFIG_FES | GMAC_CONFIG_PS; mac->mii.addr = GMAC_MDIO_ADDR; mac->mii.data = GMAC_MDIO_DATA; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c index d8c6ff725237..9c2d40f853ed 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c @@ -183,7 +183,8 @@ static void dwmac5_handle_dma_err(struct net_device *ndev, STAT_OFF(dma_errors), stats); } -int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) +int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_feat_cfg) { u32 value; @@ -193,11 +194,16 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) /* 1. Enable Safety Features */ value = readl(ioaddr + MTL_ECC_CONTROL); value |= MEEAO; /* MTL ECC Error Addr Status Override */ - value |= TSOEE; /* TSO ECC */ - value |= MRXPEE; /* MTL RX Parser ECC */ - value |= MESTEE; /* MTL EST ECC */ - value |= MRXEE; /* MTL RX FIFO ECC */ - value |= MTXEE; /* MTL TX FIFO ECC */ + if (safety_feat_cfg->tsoee) + value |= TSOEE; /* TSO ECC */ + if (safety_feat_cfg->mrxpee) + value |= MRXPEE; /* MTL RX Parser ECC */ + if (safety_feat_cfg->mestee) + value |= MESTEE; /* MTL EST ECC */ + if (safety_feat_cfg->mrxee) + value |= MRXEE; /* MTL RX FIFO ECC */ + if (safety_feat_cfg->mtxee) + value |= MTXEE; /* MTL TX FIFO ECC */ writel(value, ioaddr + MTL_ECC_CONTROL); /* 2. Enable MTL Safety Interrupts */ @@ -219,13 +225,16 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) /* 5. Enable Parity and Timeout for FSM */ value = readl(ioaddr + MAC_FSM_CONTROL); - value |= PRTYEN; /* FSM Parity Feature */ - value |= TMOUTEN; /* FSM Timeout Feature */ + if (safety_feat_cfg->prtyen) + value |= PRTYEN; /* FSM Parity Feature */ + if (safety_feat_cfg->tmouten) + value |= TMOUTEN; /* FSM Timeout Feature */ writel(value, ioaddr + MAC_FSM_CONTROL); /* 4. Enable Data Parity Protection */ value = readl(ioaddr + MTL_DPP_CONTROL); - value |= EDPP; + if (safety_feat_cfg->edpp) + value |= EDPP; writel(value, ioaddr + MTL_DPP_CONTROL); /* @@ -235,7 +244,8 @@ int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp) if (asp <= 0x2) return 0; - value |= EPSI; + if (safety_feat_cfg->epsi) + value |= EPSI; writel(value, ioaddr + MTL_DPP_CONTROL); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h index 6b2fd37b29ad..53c138d0ff48 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.h @@ -137,7 +137,8 @@ #define GMAC_INT_FPE_EN BIT(17) -int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp); +int dwmac5_safety_feat_config(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_cfg); int dwmac5_safety_feat_irq_status(struct net_device *ndev, void __iomem *ioaddr, unsigned int asp, struct stmmac_safety_stats *stats); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index ad4df9bddcf3..c4d78fa93663 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -801,7 +801,9 @@ static void dwxgmac3_handle_dma_err(struct net_device *ndev, dwxgmac3_dma_errors, STAT_OFF(dma_errors), stats); } -static int dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp) +static int +dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_cfg) { u32 value; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 6d5e0f2b03ce..6dc1c98ebec8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -348,7 +348,8 @@ struct stmmac_ops { void (*pcs_rane)(void __iomem *ioaddr, bool restart); void (*pcs_get_adv_lp)(void __iomem *ioaddr, struct rgmii_adv *adv); /* Safety Features */ - int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp); + int (*safety_feat_config)(void __iomem *ioaddr, unsigned int asp, + struct stmmac_safety_feature_cfg *safety_cfg); int (*safety_feat_irq_status)(struct net_device *ndev, void __iomem *ioaddr, unsigned int asp, struct stmmac_safety_stats *stats); @@ -612,18 +613,6 @@ struct stmmac_mmc_ops { #define stmmac_mmc_read(__priv, __args...) \ stmmac_do_void_callback(__priv, mmc, read, __args) -/* XPCS callbacks */ -#define stmmac_xpcs_validate(__priv, __args...) \ - stmmac_do_callback(__priv, xpcs, validate, __args) -#define stmmac_xpcs_config(__priv, __args...) \ - stmmac_do_callback(__priv, xpcs, config, __args) -#define stmmac_xpcs_get_state(__priv, __args...) \ - stmmac_do_callback(__priv, xpcs, get_state, __args) -#define stmmac_xpcs_link_up(__priv, __args...) \ - stmmac_do_callback(__priv, xpcs, link_up, __args) -#define stmmac_xpcs_probe(__priv, __args...) \ - stmmac_do_callback(__priv, xpcs, probe, __args) - struct stmmac_regs_off { u32 ptp_off; u32 mmc_off; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index b6cd43eda7ac..e735134e8487 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -75,7 +75,7 @@ struct stmmac_tx_queue { unsigned int cur_tx; unsigned int dirty_tx; dma_addr_t dma_tx_phy; - u32 tx_tail_addr; + dma_addr_t tx_tail_addr; u32 mss; }; @@ -311,6 +311,7 @@ enum stmmac_state { int stmmac_mdio_unregister(struct net_device *ndev); int stmmac_mdio_register(struct net_device *ndev); int stmmac_mdio_reset(struct mii_bus *mii); +int stmmac_xpcs_setup(struct mii_bus *mii); void stmmac_set_ethtool_ops(struct net_device *netdev); void stmmac_ptp_register(struct stmmac_priv *priv); @@ -338,9 +339,9 @@ static inline bool stmmac_xdp_is_enabled(struct stmmac_priv *priv) static inline unsigned int stmmac_rx_offset(struct stmmac_priv *priv) { if (stmmac_xdp_is_enabled(priv)) - return XDP_PACKET_HEADROOM; + return XDP_PACKET_HEADROOM + NET_IP_ALIGN; - return 0; + return NET_SKB_PAD + NET_IP_ALIGN; } void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index 61b11639ee0c..d0ce608b81c3 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -720,6 +720,14 @@ static int stmmac_ethtool_op_set_eee(struct net_device *dev, netdev_warn(priv->dev, "Setting EEE tx-lpi is not supported\n"); + if (priv->hw->xpcs) { + ret = xpcs_config_eee(priv->hw->xpcs, + priv->plat->mult_fact_100ns, + edata->eee_enabled); + if (ret) + return ret; + } + if (!edata->eee_enabled) stmmac_disable_eee_mode(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 345b4c6d1fd4..8d9d6ecf8c63 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -931,6 +931,11 @@ static void stmmac_validate(struct phylink_config *config, if ((max_speed > 0) && (max_speed < 1000)) { phylink_set(mask, 1000baseT_Full); phylink_set(mask, 1000baseX_Full); + } else if (priv->plat->has_gmac4) { + if (!max_speed || max_speed >= 2500) { + phylink_set(mac_supported, 2500baseT_Full); + phylink_set(mac_supported, 2500baseX_Full); + } } else if (priv->plat->has_xgmac) { if (!max_speed || (max_speed >= 2500)) { phylink_set(mac_supported, 2500baseT_Full); @@ -996,29 +1001,14 @@ static void stmmac_validate(struct phylink_config *config, linkmode_andnot(state->advertising, state->advertising, mask); /* If PCS is supported, check which modes it supports. */ - stmmac_xpcs_validate(priv, &priv->hw->xpcs_args, supported, state); -} - -static void stmmac_mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state) -{ - struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); - - state->link = 0; - stmmac_xpcs_get_state(priv, &priv->hw->xpcs_args, state); + if (priv->hw->xpcs) + xpcs_validate(priv->hw->xpcs, supported, state); } static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { - struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); - - stmmac_xpcs_config(priv, &priv->hw->xpcs_args, state); -} - -static void stmmac_mac_an_restart(struct phylink_config *config) -{ - /* Not Supported */ + /* Nothing to do, xpcs_config() handles everything */ } static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) @@ -1031,8 +1021,8 @@ static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up) if (is_up && *hs_enable) { stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY); } else { - *lo_state = FPE_EVENT_UNKNOWN; - *lp_state = FPE_EVENT_UNKNOWN; + *lo_state = FPE_STATE_OFF; + *lp_state = FPE_STATE_OFF; } } @@ -1060,8 +1050,6 @@ static void stmmac_mac_link_up(struct phylink_config *config, struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); u32 ctrl; - stmmac_xpcs_link_up(priv, &priv->hw->xpcs_args, speed, interface); - ctrl = readl(priv->ioaddr + MAC_CTRL_REG); ctrl &= ~priv->hw->link.speed_mask; @@ -1154,9 +1142,7 @@ static void stmmac_mac_link_up(struct phylink_config *config, static const struct phylink_mac_ops stmmac_phylink_mac_ops = { .validate = stmmac_validate, - .mac_pcs_get_state = stmmac_mac_pcs_get_state, .mac_config = stmmac_mac_config, - .mac_an_restart = stmmac_mac_an_restart, .mac_link_down = stmmac_mac_link_down, .mac_link_up = stmmac_mac_link_up, }; @@ -1196,7 +1182,6 @@ static void stmmac_check_pcs_mode(struct stmmac_priv *priv) */ static int stmmac_init_phy(struct net_device *dev) { - struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; struct stmmac_priv *priv = netdev_priv(dev); struct device_node *node; int ret; @@ -1222,14 +1207,19 @@ static int stmmac_init_phy(struct net_device *dev) ret = phylink_connect_phy(priv->phylink, phydev); } - phylink_ethtool_get_wol(priv->phylink, &wol); - device_set_wakeup_capable(priv->device, !!wol.supported); + if (!priv->plat->pmt) { + struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; + + phylink_ethtool_get_wol(priv->phylink, &wol); + device_set_wakeup_capable(priv->device, !!wol.supported); + } return ret; } static int stmmac_phy_setup(struct stmmac_priv *priv) { + struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data; struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node); int mode = priv->plat->phy_interface; struct phylink *phylink; @@ -1237,8 +1227,9 @@ static int stmmac_phy_setup(struct stmmac_priv *priv) priv->phylink_config.dev = &priv->dev->dev; priv->phylink_config.type = PHYLINK_NETDEV; priv->phylink_config.pcs_poll = true; - priv->phylink_config.ovr_an_inband = - priv->plat->mdio_bus_data->xpcs_an_inband; + if (priv->plat->mdio_bus_data) + priv->phylink_config.ovr_an_inband = + mdio_bus_data->xpcs_an_inband; if (!fwnode) fwnode = dev_fwnode(priv->device); @@ -1248,6 +1239,9 @@ static int stmmac_phy_setup(struct stmmac_priv *priv) if (IS_ERR(phylink)) return PTR_ERR(phylink); + if (priv->hw->xpcs) + phylink_set_pcs(phylink, &priv->hw->xpcs->pcs); + priv->phylink = phylink; return 0; } @@ -3169,7 +3163,8 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) { if (priv->dma_cap.asp) { netdev_info(priv->dev, "Enabling Safety Features\n"); - stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp); + stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, + priv->plat->safety_feat_cfg); } else { netdev_info(priv->dev, "No Safety Features support found\n"); } @@ -3411,8 +3406,8 @@ static void stmmac_free_irq(struct net_device *dev, static int stmmac_request_irq_multi_msi(struct net_device *dev) { - enum request_irq_err irq_err = REQ_IRQ_ERR_NO; struct stmmac_priv *priv = netdev_priv(dev); + enum request_irq_err irq_err; cpumask_t cpu_mask; int irq_idx = 0; char *int_name; @@ -3559,8 +3554,8 @@ irq_error: static int stmmac_request_irq_single(struct net_device *dev) { - enum request_irq_err irq_err = REQ_IRQ_ERR_NO; struct stmmac_priv *priv = netdev_priv(dev); + enum request_irq_err irq_err; int ret; ret = request_irq(dev->irq, stmmac_interrupt, @@ -3570,7 +3565,7 @@ static int stmmac_request_irq_single(struct net_device *dev) "%s: ERROR: allocating the IRQ %d (error: %d)\n", __func__, dev->irq, ret); irq_err = REQ_IRQ_ERR_MAC; - return ret; + goto irq_error; } /* Request the Wake IRQ in case of another line @@ -3584,7 +3579,7 @@ static int stmmac_request_irq_single(struct net_device *dev) "%s: ERROR: allocating the WoL IRQ %d (%d)\n", __func__, priv->wol_irq, ret); irq_err = REQ_IRQ_ERR_WOL; - return ret; + goto irq_error; } } @@ -3634,6 +3629,7 @@ static int stmmac_request_irq(struct net_device *dev) int stmmac_open(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); + int mode = priv->plat->phy_interface; int bfsize = 0; u32 chan; int ret; @@ -3646,7 +3642,8 @@ int stmmac_open(struct net_device *dev) if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI && - priv->hw->xpcs_args.an_mode != DW_AN_C73) { + (!priv->hw->xpcs || + xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) { ret = stmmac_init_phy(dev); if (ret) { netdev_err(priv->dev, @@ -4654,7 +4651,6 @@ static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, return res; } -/* This function assumes rcu_read_lock() is held by the caller. */ static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, struct bpf_prog *prog, struct xdp_buff *xdp) @@ -4696,17 +4692,14 @@ static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, struct bpf_prog *prog; int res; - rcu_read_lock(); - prog = READ_ONCE(priv->xdp_prog); if (!prog) { res = STMMAC_XDP_PASS; - goto unlock; + goto out; } res = __stmmac_xdp_run_prog(priv, prog, xdp); -unlock: - rcu_read_unlock(); +out: return ERR_PTR(-res); } @@ -4976,10 +4969,8 @@ read_again: buf->xdp->data_end = buf->xdp->data + buf1_len; xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool); - rcu_read_lock(); prog = READ_ONCE(priv->xdp_prog); res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); - rcu_read_unlock(); switch (res) { case STMMAC_XDP_PASS: @@ -5134,7 +5125,7 @@ read_again: /* Buffer is good. Go on. */ - prefetch(page_address(buf->page)); + prefetch(page_address(buf->page) + buf->page_offset); if (buf->sec_page) prefetch(page_address(buf->sec_page)); @@ -5167,12 +5158,9 @@ read_again: dma_sync_single_for_cpu(priv->device, buf->addr, buf1_len, dma_dir); - xdp.data = page_address(buf->page) + buf->page_offset; - xdp.data_end = xdp.data + buf1_len; - xdp.data_hard_start = page_address(buf->page); - xdp_set_data_meta_invalid(&xdp); - xdp.frame_sz = buf_sz; - xdp.rxq = &rx_q->xdp_rxq; + xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq); + xdp_prepare_buff(&xdp, page_address(buf->page), + buf->page_offset, buf1_len, false); pre_len = xdp.data_end - xdp.data_hard_start - buf->page_offset; @@ -5888,12 +5876,21 @@ static int stmmac_set_mac_address(struct net_device *ndev, void *addr) struct stmmac_priv *priv = netdev_priv(ndev); int ret = 0; + ret = pm_runtime_get_sync(priv->device); + if (ret < 0) { + pm_runtime_put_noidle(priv->device); + return ret; + } + ret = eth_mac_addr(ndev, addr); if (ret) - return ret; + goto set_mac_error; stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); +set_mac_error: + pm_runtime_put(priv->device); + return ret; } @@ -6188,12 +6185,6 @@ static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid bool is_double = false; int ret; - ret = pm_runtime_get_sync(priv->device); - if (ret < 0) { - pm_runtime_put_noidle(priv->device); - return ret; - } - if (be16_to_cpu(proto) == ETH_P_8021AD) is_double = true; @@ -6219,6 +6210,12 @@ static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vi bool is_double = false; int ret; + ret = pm_runtime_get_sync(priv->device); + if (ret < 0) { + pm_runtime_put_noidle(priv->device); + return ret; + } + if (be16_to_cpu(proto) == ETH_P_8021AD) is_double = true; @@ -6532,7 +6529,8 @@ static int stmmac_hw_init(struct stmmac_priv *priv) * register (if supported). */ priv->plat->enh_desc = priv->dma_cap.enh_desc; - priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up; + priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && + !priv->plat->use_phy_wol; priv->hw->pmt = priv->plat->pmt; if (priv->dma_cap.hash_tb_sz) { priv->hw->multicast_filter_bins = @@ -6841,6 +6839,11 @@ int stmmac_dvr_probe(struct device *device, reset_control_reset(priv->plat->stmmac_rst); } + ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); + if (ret == -ENOTSUPP) + dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", + ERR_PTR(ret)); + /* Init MAC and get the capabilities */ ret = stmmac_hw_init(priv); if (ret) @@ -6992,6 +6995,15 @@ int stmmac_dvr_probe(struct device *device, } } + if (priv->plat->speed_mode_2500) + priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv); + + if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) { + ret = stmmac_xpcs_setup(priv->mii); + if (ret) + goto error_xpcs_setup; + } + ret = stmmac_phy_setup(priv); if (ret) { netdev_err(ndev, "failed to setup phy (%d)\n", ret); @@ -7028,6 +7040,7 @@ error_serdes_powerup: unregister_netdev(ndev); error_netdev_register: phylink_destroy(priv->phylink); +error_xpcs_setup: error_phy_setup: if (priv->hw->pcs != STMMAC_PCS_TBI && priv->hw->pcs != STMMAC_PCS_RTBI) @@ -7036,7 +7049,6 @@ error_mdio_register: stmmac_napi_del(ndev); error_hw_init: destroy_workqueue(priv->wq); - stmmac_bus_clks_config(priv, false); bitmap_free(priv->af_xdp_zc_qps); return ret; @@ -7073,6 +7085,7 @@ int stmmac_dvr_remove(struct device *dev) phylink_destroy(priv->phylink); if (priv->plat->stmmac_rst) reset_control_assert(priv->plat->stmmac_rst); + reset_control_assert(priv->plat->stmmac_ahb_rst); pm_runtime_put(dev); pm_runtime_disable(dev); if (priv->hw->pcs != STMMAC_PCS_TBI && diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c index b750074f8f9c..a5d150c5f3d8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c @@ -397,6 +397,41 @@ int stmmac_mdio_reset(struct mii_bus *bus) return 0; } +int stmmac_xpcs_setup(struct mii_bus *bus) +{ + struct net_device *ndev = bus->priv; + struct mdio_device *mdiodev; + struct stmmac_priv *priv; + struct dw_xpcs *xpcs; + int mode, addr; + + priv = netdev_priv(ndev); + mode = priv->plat->phy_interface; + + /* Try to probe the XPCS by scanning all addresses. */ + for (addr = 0; addr < PHY_MAX_ADDR; addr++) { + mdiodev = mdio_device_create(bus, addr); + if (IS_ERR(mdiodev)) + continue; + + xpcs = xpcs_create(mdiodev, mode); + if (IS_ERR_OR_NULL(xpcs)) { + mdio_device_free(mdiodev); + continue; + } + + priv->hw->xpcs = xpcs; + break; + } + + if (!priv->hw->xpcs) { + dev_warn(priv->device, "No xPCS found\n"); + return -ENODEV; + } + + return 0; +} + /** * stmmac_mdio_register * @ndev: net device structure @@ -444,14 +479,6 @@ int stmmac_mdio_register(struct net_device *ndev) max_addr = PHY_MAX_ADDR; } - if (mdio_bus_data->has_xpcs) { - priv->hw->xpcs = mdio_xpcs_get_ops(); - if (!priv->hw->xpcs) { - err = -ENODEV; - goto bus_register_fail; - } - } - if (mdio_bus_data->needs_reset) new_bus->reset = &stmmac_mdio_reset; @@ -503,30 +530,10 @@ int stmmac_mdio_register(struct net_device *ndev) found = 1; } - /* Try to probe the XPCS by scanning all addresses. */ - if (priv->hw->xpcs) { - struct mdio_xpcs_args *xpcs = &priv->hw->xpcs_args; - int ret, mode = priv->plat->phy_interface; - max_addr = PHY_MAX_ADDR; - - xpcs->bus = new_bus; - - for (addr = 0; addr < max_addr; addr++) { - xpcs->addr = addr; - - ret = stmmac_xpcs_probe(priv, xpcs, mode); - if (!ret) { - found = 1; - break; - } - } - } - if (!found && !mdio_node) { dev_warn(dev, "No PHY found\n"); - mdiobus_unregister(new_bus); - mdiobus_free(new_bus); - return -ENODEV; + err = -ENODEV; + goto no_phy_found; } bus_register_done: @@ -534,6 +541,8 @@ bus_register_done: return 0; +no_phy_found: + mdiobus_unregister(new_bus); bus_register_fail: mdiobus_free(new_bus); return err; @@ -551,6 +560,11 @@ int stmmac_mdio_unregister(struct net_device *ndev) if (!priv->mii) return 0; + if (priv->hw->xpcs) { + mdio_device_free(priv->hw->xpcs->mdiodev); + xpcs_destroy(priv->hw->xpcs); + } + mdiobus_unregister(priv->mii); priv->mii->priv = NULL; mdiobus_free(priv->mii); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index 95e0e4d6f74d..fcf17d8a0494 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -174,6 +174,12 @@ static int stmmac_pci_probe(struct pci_dev *pdev, if (!plat->dma_cfg) return -ENOMEM; + plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, + sizeof(*plat->safety_feat_cfg), + GFP_KERNEL); + if (!plat->safety_feat_cfg) + return -ENOMEM; + /* Enable pci device */ ret = pci_enable_device(pdev); if (ret) { @@ -203,6 +209,16 @@ static int stmmac_pci_probe(struct pci_dev *pdev, res.wol_irq = pdev->irq; res.irq = pdev->irq; + plat->safety_feat_cfg->tsoee = 1; + plat->safety_feat_cfg->mrxpee = 1; + plat->safety_feat_cfg->mestee = 1; + plat->safety_feat_cfg->mrxee = 1; + plat->safety_feat_cfg->mtxee = 1; + plat->safety_feat_cfg->epsi = 1; + plat->safety_feat_cfg->edpp = 1; + plat->safety_feat_cfg->prtyen = 1; + plat->safety_feat_cfg->tmouten = 1; + return stmmac_dvr_probe(&pdev->dev, plat, &res); } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 1e17a23d9118..072eff8079d0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -230,8 +230,6 @@ static int stmmac_mtl_setup(struct platform_device *pdev, plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WFQ; else if (of_property_read_bool(tx_node, "snps,tx-sched-dwrr")) plat->tx_sched_algorithm = MTL_TX_ALGORITHM_DWRR; - else if (of_property_read_bool(tx_node, "snps,tx-sched-sp")) - plat->tx_sched_algorithm = MTL_TX_ALGORITHM_SP; else plat->tx_sched_algorithm = MTL_TX_ALGORITHM_SP; @@ -602,6 +600,13 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) goto error_hw_init; } + plat->stmmac_ahb_rst = devm_reset_control_get_optional_shared( + &pdev->dev, "ahb"); + if (IS_ERR(plat->stmmac_ahb_rst)) { + ret = plat->stmmac_ahb_rst; + goto error_hw_init; + } + return plat; error_hw_init: @@ -622,6 +627,8 @@ error_pclk_get: void stmmac_remove_config_dt(struct platform_device *pdev, struct plat_stmmacenet_data *plat) { + clk_disable_unprepare(plat->stmmac_clk); + clk_disable_unprepare(plat->pclk); of_node_put(plat->phy_node); of_node_put(plat->mdio_node); } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 4e70efc45458..92dab609d4f8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -573,10 +573,8 @@ static int tc_add_flow(struct stmmac_priv *priv, for (i = 0; i < ARRAY_SIZE(tc_flow_parsers); i++) { ret = tc_flow_parsers[i].fn(priv, cls, entry); - if (!ret) { + if (!ret) entry->in_use = true; - continue; - } } if (!entry->in_use) diff --git a/drivers/net/ethernet/sun/cassini.c b/drivers/net/ethernet/sun/cassini.c index 54f45d8c79a7..981685c88308 100644 --- a/drivers/net/ethernet/sun/cassini.c +++ b/drivers/net/ethernet/sun/cassini.c @@ -486,7 +486,7 @@ page_err: /* initialize spare pool of rx buffers, but allocate during the open */ static void cas_spare_init(struct cas *cp) { - spin_lock(&cp->rx_inuse_lock); + spin_lock(&cp->rx_inuse_lock); INIT_LIST_HEAD(&cp->rx_inuse_list); spin_unlock(&cp->rx_inuse_lock); diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c index 707ccdd03b19..74e748662ec0 100644 --- a/drivers/net/ethernet/sun/niu.c +++ b/drivers/net/ethernet/sun/niu.c @@ -8144,10 +8144,10 @@ static int niu_pci_vpd_scan_props(struct niu *np, u32 start, u32 end) "VPD_SCAN: Reading in property [%s] len[%d]\n", namebuf, prop_len); for (i = 0; i < prop_len; i++) { - err = niu_pci_eeprom_read(np, off + i); - if (err >= 0) - *prop_buf = err; - ++prop_buf; + err = niu_pci_eeprom_read(np, off + i); + if (err < 0) + return err; + *prop_buf++ = err; } } @@ -8158,14 +8158,14 @@ static int niu_pci_vpd_scan_props(struct niu *np, u32 start, u32 end) } /* ESPC_PIO_EN_ENABLE must be set */ -static void niu_pci_vpd_fetch(struct niu *np, u32 start) +static int niu_pci_vpd_fetch(struct niu *np, u32 start) { u32 offset; int err; err = niu_pci_eeprom_read16_swp(np, start + 1); if (err < 0) - return; + return err; offset = err + 3; @@ -8174,12 +8174,14 @@ static void niu_pci_vpd_fetch(struct niu *np, u32 start) u32 end; err = niu_pci_eeprom_read(np, here); + if (err < 0) + return err; if (err != 0x90) - return; + return -EINVAL; err = niu_pci_eeprom_read16_swp(np, here + 1); if (err < 0) - return; + return err; here = start + offset + 3; end = start + offset + err; @@ -8187,9 +8189,12 @@ static void niu_pci_vpd_fetch(struct niu *np, u32 start) offset += err; err = niu_pci_vpd_scan_props(np, here, end); - if (err < 0 || err == 1) - return; + if (err < 0) + return err; + if (err == 1) + return -EINVAL; } + return 0; } /* ESPC_PIO_EN_ENABLE must be set */ @@ -9280,8 +9285,11 @@ static int niu_get_invariants(struct niu *np) offset = niu_pci_vpd_offset(np); netif_printk(np, probe, KERN_DEBUG, np->dev, "%s() VPD offset [%08x]\n", __func__, offset); - if (offset) - niu_pci_vpd_fetch(np, offset); + if (offset) { + err = niu_pci_vpd_fetch(np, offset); + if (err < 0) + return err; + } nw64(ESPC_PIO_EN, 0); if (np->flags & NIU_FLAGS_VPD_VALID) { diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c index 9790656cf970..cfb9e21b18b7 100644 --- a/drivers/net/ethernet/sun/sungem.c +++ b/drivers/net/ethernet/sun/sungem.c @@ -1258,8 +1258,8 @@ static void gem_begin_auto_negotiation(struct gem *gp, &advertising, ep->link_modes.advertising); if (gp->phy_type != phy_mii_mdio0 && - gp->phy_type != phy_mii_mdio1) - goto non_mii; + gp->phy_type != phy_mii_mdio1) + goto non_mii; /* Setup advertise */ if (found_mii_phy(gp)) @@ -1410,7 +1410,7 @@ static int gem_set_link_modes(struct gem *gp) if (gp->phy_type == phy_serialink || gp->phy_type == phy_serdes) { - u32 pcs_lpa = readl(gp->regs + PCS_MIILP); + u32 pcs_lpa = readl(gp->regs + PCS_MIILP); if (pcs_lpa & (PCS_MIIADV_SP | PCS_MIIADV_AP)) pause = 1; @@ -1892,7 +1892,7 @@ static void gem_init_mac(struct gem *gp) static void gem_init_pause_thresholds(struct gem *gp) { - u32 cfg; + u32 cfg; /* Calculate pause thresholds. Setting the OFF threshold to the * full RX fifo size effectively disables PAUSE generation which @@ -1914,15 +1914,15 @@ static void gem_init_pause_thresholds(struct gem *gp) /* Configure the chip "burst" DMA mode & enable some * HW bug fixes on Apple version */ - cfg = 0; - if (gp->pdev->vendor == PCI_VENDOR_ID_APPLE) + cfg = 0; + if (gp->pdev->vendor == PCI_VENDOR_ID_APPLE) cfg |= GREG_CFG_RONPAULBIT | GREG_CFG_ENBUG2FIX; #if !defined(CONFIG_SPARC64) && !defined(CONFIG_ALPHA) - cfg |= GREG_CFG_IBURST; + cfg |= GREG_CFG_IBURST; #endif - cfg |= ((31 << 1) & GREG_CFG_TXDMALIM); - cfg |= ((31 << 6) & GREG_CFG_RXDMALIM); - writel(cfg, gp->regs + GREG_CFG); + cfg |= ((31 << 1) & GREG_CFG_TXDMALIM); + cfg |= ((31 << 6) & GREG_CFG_RXDMALIM); + writel(cfg, gp->regs + GREG_CFG); /* If Infinite Burst didn't stick, then use different * thresholds (and Apple bug fixes don't exist) diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index 54b53dbdb33c..a2c1a404c52d 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -2286,8 +2286,8 @@ static netdev_tx_t happy_meal_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct happy_meal *hp = netdev_priv(dev); - int entry; - u32 tx_flags; + int entry; + u32 tx_flags; tx_flags = TXFLAG_OWN; if (skb->ip_summed == CHECKSUM_PARTIAL) { @@ -2301,7 +2301,7 @@ static netdev_tx_t happy_meal_start_xmit(struct sk_buff *skb, spin_lock_irq(&hp->happy_lock); - if (TX_BUFFS_AVAIL(hp) <= (skb_shinfo(skb)->nr_frags + 1)) { + if (TX_BUFFS_AVAIL(hp) <= (skb_shinfo(skb)->nr_frags + 1)) { netif_stop_queue(dev); spin_unlock_irq(&hp->happy_lock); printk(KERN_ERR "%s: BUG! Tx Ring full when queue awake!\n", diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 6a67b026df0b..718539cdd2f2 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -1506,12 +1506,12 @@ static void am65_cpsw_nuss_free_tx_chns(void *data) for (i = 0; i < common->tx_ch_num; i++) { struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i]; - if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) - k3_udma_glue_release_tx_chn(tx_chn->tx_chn); - if (!IS_ERR_OR_NULL(tx_chn->desc_pool)) k3_cppi_desc_pool_destroy(tx_chn->desc_pool); + if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) + k3_udma_glue_release_tx_chn(tx_chn->tx_chn); + memset(tx_chn, 0, sizeof(*tx_chn)); } } @@ -1531,12 +1531,12 @@ void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common) netif_napi_del(&tx_chn->napi_tx); - if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) - k3_udma_glue_release_tx_chn(tx_chn->tx_chn); - if (!IS_ERR_OR_NULL(tx_chn->desc_pool)) k3_cppi_desc_pool_destroy(tx_chn->desc_pool); + if (!IS_ERR_OR_NULL(tx_chn->tx_chn)) + k3_udma_glue_release_tx_chn(tx_chn->tx_chn); + memset(tx_chn, 0, sizeof(*tx_chn)); } } @@ -1624,11 +1624,11 @@ static void am65_cpsw_nuss_free_rx_chns(void *data) rx_chn = &common->rx_chns; - if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) - k3_udma_glue_release_rx_chn(rx_chn->rx_chn); - if (!IS_ERR_OR_NULL(rx_chn->desc_pool)) k3_cppi_desc_pool_destroy(rx_chn->desc_pool); + + if (!IS_ERR_OR_NULL(rx_chn->rx_chn)) + k3_udma_glue_release_rx_chn(rx_chn->rx_chn); } static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common) diff --git a/drivers/net/ethernet/ti/am65-cpsw-switchdev.c b/drivers/net/ethernet/ti/am65-cpsw-switchdev.c index 23cfb91e9c4d..9c29b363e9ae 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-switchdev.c +++ b/drivers/net/ethernet/ti/am65-cpsw-switchdev.c @@ -84,7 +84,7 @@ static int am65_cpsw_port_attr_br_flags_pre_set(struct net_device *netdev, return 0; } -static int am65_cpsw_port_attr_set(struct net_device *ndev, +static int am65_cpsw_port_attr_set(struct net_device *ndev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) { @@ -302,7 +302,7 @@ static int am65_cpsw_port_mdb_del(struct am65_cpsw_port *port, return 0; } -static int am65_cpsw_port_obj_add(struct net_device *ndev, +static int am65_cpsw_port_obj_add(struct net_device *ndev, const void *ctx, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) { @@ -329,7 +329,7 @@ static int am65_cpsw_port_obj_add(struct net_device *ndev, return err; } -static int am65_cpsw_port_obj_del(struct net_device *ndev, +static int am65_cpsw_port_obj_del(struct net_device *ndev, const void *ctx, const struct switchdev_obj *obj) { struct switchdev_obj_port_vlan *vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); diff --git a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c index 9caaae79fc95..c30a6e510aa3 100644 --- a/drivers/net/ethernet/ti/am65-cpts.c +++ b/drivers/net/ethernet/ti/am65-cpts.c @@ -1037,11 +1037,9 @@ static int am65_cpts_probe(struct platform_device *pdev) struct device_node *node = pdev->dev.of_node; struct device *dev = &pdev->dev; struct am65_cpts *cpts; - struct resource *res; void __iomem *base; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpts"); - base = devm_ioremap_resource(dev, res); + base = devm_platform_ioremap_resource_byname(pdev, "cpts"); if (IS_ERR(base)) return PTR_ERR(base); diff --git a/drivers/net/ethernet/ti/cpsw-phy-sel.c b/drivers/net/ethernet/ti/cpsw-phy-sel.c index 6e72ecbe5cf7..e8f38e3f7706 100644 --- a/drivers/net/ethernet/ti/cpsw-phy-sel.c +++ b/drivers/net/ethernet/ti/cpsw-phy-sel.c @@ -206,7 +206,6 @@ static const struct of_device_id cpsw_phy_sel_id_table[] = { static int cpsw_phy_sel_probe(struct platform_device *pdev) { - struct resource *res; const struct of_device_id *of_id; struct cpsw_phy_sel_priv *priv; @@ -223,8 +222,7 @@ static int cpsw_phy_sel_probe(struct platform_device *pdev) priv->dev = &pdev->dev; priv->cpsw_phy_sel = of_id->data; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "gmii-sel"); - priv->gmii_sel = devm_ioremap_resource(&pdev->dev, res); + priv->gmii_sel = devm_platform_ioremap_resource_byname(pdev, "gmii-sel"); if (IS_ERR(priv->gmii_sel)) return PTR_ERR(priv->gmii_sel); diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index c0cd7de88316..cbbd0f665796 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -430,8 +430,8 @@ static void cpsw_rx_handler(void *token, int len, int status) cpts_rx_timestamp(cpsw->cpts, skb); skb->protocol = eth_type_trans(skb, ndev); - /* unmap page as no netstack skb page recycling */ - page_pool_release_page(pool, page); + /* mark skb for recycling */ + skb_mark_for_recycle(skb, page, pool); netif_receive_skb(skb); ndev->stats.rx_bytes += len; @@ -1532,8 +1532,7 @@ static int cpsw_probe(struct platform_device *pdev) } cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000; - ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ss_regs = devm_ioremap_resource(dev, ss_res); + ss_regs = devm_platform_get_and_ioremap_resource(pdev, 0, &ss_res); if (IS_ERR(ss_regs)) return PTR_ERR(ss_regs); cpsw->regs = ss_regs; diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index d828f856237a..0c75e0576ee1 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -70,7 +70,7 @@ enum { }; /** - * struct ale_dev_id - The ALE version/SoC specific configuration + * struct cpsw_ale_dev_id - The ALE version/SoC specific configuration * @dev_id: ALE version/SoC id * @features: features supported by ALE * @tbl_entries: number of ALE entries diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 69b7a4e0220a..57d279fdcc9f 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -373,8 +373,8 @@ static void cpsw_rx_handler(void *token, int len, int status) cpts_rx_timestamp(cpsw->cpts, skb); skb->protocol = eth_type_trans(skb, ndev); - /* unmap page as no netstack skb page recycling */ - page_pool_release_page(pool, page); + /* mark skb for recycling */ + skb_mark_for_recycle(skb, page, pool); netif_receive_skb(skb); ndev->stats.rx_bytes += len; @@ -1883,8 +1883,7 @@ static int cpsw_probe(struct platform_device *pdev) } cpsw->bus_freq_mhz = clk_get_rate(clk) / 1000000; - ss_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ss_regs = devm_ioremap_resource(dev, ss_res); + ss_regs = devm_platform_get_and_ioremap_resource(pdev, 0, &ss_res); if (IS_ERR(ss_regs)) { ret = PTR_ERR(ss_regs); return ret; diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index 5862f0a4a975..ecc2a6b7e28f 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -1328,13 +1328,9 @@ int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, struct bpf_prog *prog; u32 act; - rcu_read_lock(); - prog = READ_ONCE(priv->xdp_prog); - if (!prog) { - ret = CPSW_XDP_PASS; - goto out; - } + if (!prog) + return CPSW_XDP_PASS; act = bpf_prog_run_xdp(prog, xdp); /* XDP prog might have changed packet data and boundaries */ @@ -1378,10 +1374,8 @@ int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, ndev->stats.rx_bytes += *len; ndev->stats.rx_packets++; out: - rcu_read_unlock(); return ret; drop: - rcu_read_unlock(); page_pool_recycle_direct(cpsw->page_pool[ch], page); return ret; } diff --git a/drivers/net/ethernet/ti/cpsw_switchdev.c b/drivers/net/ethernet/ti/cpsw_switchdev.c index 05a64fb7a04f..f7fb6e17dadd 100644 --- a/drivers/net/ethernet/ti/cpsw_switchdev.c +++ b/drivers/net/ethernet/ti/cpsw_switchdev.c @@ -86,7 +86,7 @@ static int cpsw_port_attr_br_flags_pre_set(struct net_device *netdev, return 0; } -static int cpsw_port_attr_set(struct net_device *ndev, +static int cpsw_port_attr_set(struct net_device *ndev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) { @@ -310,7 +310,7 @@ static int cpsw_port_mdb_del(struct cpsw_priv *priv, return err; } -static int cpsw_port_obj_add(struct net_device *ndev, +static int cpsw_port_obj_add(struct net_device *ndev, const void *ctx, const struct switchdev_obj *obj, struct netlink_ext_ack *extack) { @@ -338,7 +338,7 @@ static int cpsw_port_obj_add(struct net_device *ndev, return err; } -static int cpsw_port_obj_del(struct net_device *ndev, +static int cpsw_port_obj_del(struct net_device *ndev, const void *ctx, const struct switchdev_obj *obj) { struct switchdev_obj_port_vlan *vlan = SWITCHDEV_OBJ_PORT_VLAN(obj); diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c index f9417b44cae8..c674e34b6839 100644 --- a/drivers/net/ethernet/ti/davinci_emac.c +++ b/drivers/net/ethernet/ti/davinci_emac.c @@ -1814,13 +1814,12 @@ static int davinci_emac_probe(struct platform_device *pdev) priv->bus_freq_mhz = (u32)(emac_bus_frequency / 1000000); /* Get EMAC platform data */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv->emac_base_phys = res->start + pdata->ctrl_reg_offset; - priv->remap_addr = devm_ioremap_resource(&pdev->dev, res); + priv->remap_addr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(priv->remap_addr)) { rc = PTR_ERR(priv->remap_addr); goto no_pdata; } + priv->emac_base_phys = res->start + pdata->ctrl_reg_offset; res_ctrl = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (res_ctrl) { diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index 9030e619e543..97942b0e3897 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -1350,8 +1350,8 @@ int netcp_txpipe_open(struct netcp_tx_pipe *tx_pipe) tx_pipe->dma_queue = knav_queue_open(name, tx_pipe->dma_queue_id, KNAV_QUEUE_SHARED); if (IS_ERR(tx_pipe->dma_queue)) { - dev_err(dev, "Could not open DMA queue for channel \"%s\": %d\n", - name, ret); + dev_err(dev, "Could not open DMA queue for channel \"%s\": %pe\n", + name, tx_pipe->dma_queue); ret = PTR_ERR(tx_pipe->dma_queue); goto err; } diff --git a/drivers/net/ethernet/via/via-velocity.c b/drivers/net/ethernet/via/via-velocity.c index fecc4d7b00b0..88426b5e410b 100644 --- a/drivers/net/ethernet/via/via-velocity.c +++ b/drivers/net/ethernet/via/via-velocity.c @@ -1897,7 +1897,7 @@ static void velocity_error(struct velocity_info *vptr, int status) } /** - * tx_srv - transmit interrupt service + * velocity_tx_srv - transmit interrupt service * @vptr: Velocity * * Scan the queues looking for transmitted packets that @@ -2453,7 +2453,7 @@ static int velocity_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) } /** - * velocity_get_status - statistics callback + * velocity_get_stats - statistics callback * @dev: network device * * Callback from the network layer to allow driver statistics @@ -3723,7 +3723,7 @@ static int __init velocity_init_module(void) } /** - * velocity_cleanup - module unload + * velocity_cleanup_module - module unload * * When the velocity hardware is unloaded this function is called. * It will clean up the notifiers and the unregister the PCI diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c index ec5db481c9cd..811815f8cd3b 100644 --- a/drivers/net/ethernet/wiznet/w5100.c +++ b/drivers/net/ethernet/wiznet/w5100.c @@ -263,19 +263,14 @@ static int w5100_writebulk_direct(struct net_device *ndev, u32 addr, static int w5100_mmio_init(struct net_device *ndev) { struct platform_device *pdev = to_platform_device(ndev->dev.parent); - struct w5100_priv *priv = netdev_priv(ndev); struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev); - struct resource *mem; spin_lock_init(&mmio_priv->reg_lock); - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mmio_priv->base = devm_ioremap_resource(&pdev->dev, mem); + mmio_priv->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); if (IS_ERR(mmio_priv->base)) return PTR_ERR(mmio_priv->base); - netdev_info(ndev, "at 0x%llx irq %d\n", (u64)mem->start, priv->irq); - return 0; } diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index a1f5f07f4ca9..60a4f79b8fa1 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -774,12 +774,15 @@ static void temac_start_xmit_done(struct net_device *ndev) stat = be32_to_cpu(cur_p->app0); while (stat & STS_CTRL_APP0_CMPLT) { + /* Make sure that the other fields are read after bd is + * released by dma + */ + rmb(); dma_unmap_single(ndev->dev.parent, be32_to_cpu(cur_p->phys), be32_to_cpu(cur_p->len), DMA_TO_DEVICE); skb = (struct sk_buff *)ptr_from_txbd(cur_p); if (skb) dev_consume_skb_irq(skb); - cur_p->app0 = 0; cur_p->app1 = 0; cur_p->app2 = 0; cur_p->app3 = 0; @@ -788,6 +791,12 @@ static void temac_start_xmit_done(struct net_device *ndev) ndev->stats.tx_packets++; ndev->stats.tx_bytes += be32_to_cpu(cur_p->len); + /* app0 must be visible last, as it is used to flag + * availability of the bd + */ + smp_mb(); + cur_p->app0 = 0; + lp->tx_bd_ci++; if (lp->tx_bd_ci >= lp->tx_bd_num) lp->tx_bd_ci = 0; @@ -814,6 +823,9 @@ static inline int temac_check_tx_bd_space(struct temac_local *lp, int num_frag) if (cur_p->app0) return NETDEV_TX_BUSY; + /* Make sure to read next bd app0 after this one */ + rmb(); + tail++; if (tail >= lp->tx_bd_num) tail = 0; @@ -849,7 +861,7 @@ temac_start_xmit(struct sk_buff *skb, struct net_device *ndev) smp_mb(); /* Space might have just been freed - check again */ - if (temac_check_tx_bd_space(lp, num_frag)) + if (temac_check_tx_bd_space(lp, num_frag + 1)) return NETDEV_TX_BUSY; netif_wake_queue(ndev); @@ -876,7 +888,6 @@ temac_start_xmit(struct sk_buff *skb, struct net_device *ndev) return NETDEV_TX_OK; } cur_p->phys = cpu_to_be32(skb_dma_addr); - ptr_to_txbd((void *)skb, cur_p); for (ii = 0; ii < num_frag; ii++) { if (++lp->tx_bd_tail >= lp->tx_bd_num) @@ -915,6 +926,11 @@ temac_start_xmit(struct sk_buff *skb, struct net_device *ndev) } cur_p->app0 |= cpu_to_be32(STS_CTRL_APP0_EOP); + /* Mark last fragment with skb address, so it can be consumed + * in temac_start_xmit_done() + */ + ptr_to_txbd((void *)skb, cur_p); + tail_p = lp->tx_bd_p + sizeof(*lp->tx_bd_v) * lp->tx_bd_tail; lp->tx_bd_tail++; if (lp->tx_bd_tail >= lp->tx_bd_num) @@ -926,6 +942,9 @@ temac_start_xmit(struct sk_buff *skb, struct net_device *ndev) wmb(); lp->dma_out(lp, TX_TAILDESC_PTR, tail_p); /* DMA start */ + if (temac_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) + netif_stop_queue(ndev); + return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index b508c9453f40..13cd799541aa 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -1543,6 +1543,7 @@ static void axienet_validate(struct phylink_config *config, case PHY_INTERFACE_MODE_MII: phylink_set(mask, 100baseT_Full); phylink_set(mask, 10baseT_Full); + fallthrough; default: break; } @@ -1893,8 +1894,7 @@ static int axienet_probe(struct platform_device *pdev) goto cleanup_clk; /* Map device registers */ - ethres = platform_get_resource(pdev, IORESOURCE_MEM, 0); - lp->regs = devm_ioremap_resource(&pdev->dev, ethres); + lp->regs = devm_platform_get_and_ioremap_resource(pdev, 0, ðres); if (IS_ERR(lp->regs)) { ret = PTR_ERR(lp->regs); goto cleanup_clk; @@ -2009,9 +2009,7 @@ static int axienet_probe(struct platform_device *pdev) lp->eth_irq = platform_get_irq_optional(pdev, 0); } else { /* Check for these resources directly on the Ethernet node. */ - struct resource *res = platform_get_resource(pdev, - IORESOURCE_MEM, 1); - lp->dma_regs = devm_ioremap_resource(&pdev->dev, res); + lp->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 1, NULL); lp->rx_irq = platform_get_irq(pdev, 1); lp->tx_irq = platform_get_irq(pdev, 0); lp->eth_irq = platform_get_irq_optional(pdev, 2); diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c index d9d58a7dabee..b06377fe7293 100644 --- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c +++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c @@ -1189,9 +1189,8 @@ static int xemaclite_of_probe(struct platform_device *ofdev) } dev_info(dev, - "Xilinx EmacLite at 0x%08lX mapped to 0x%08lX, irq=%d\n", - (unsigned long __force)ndev->mem_start, - (unsigned long __force)lp->base_addr, ndev->irq); + "Xilinx EmacLite at 0x%08lX mapped to 0x%p, irq=%d\n", + (unsigned long __force)ndev->mem_start, lp->base_addr, ndev->irq); return 0; error: diff --git a/drivers/net/ethernet/xircom/xirc2ps_cs.c b/drivers/net/ethernet/xircom/xirc2ps_cs.c index 2049d76a0e68..4f6db6f5c272 100644 --- a/drivers/net/ethernet/xircom/xirc2ps_cs.c +++ b/drivers/net/ethernet/xircom/xirc2ps_cs.c @@ -1232,7 +1232,7 @@ do_start_xmit(struct sk_buff *skb, struct net_device *dev) if (pktlen < ETH_ZLEN) { if (skb_padto(skb, ETH_ZLEN)) - return NETDEV_TX_OK; + return NETDEV_TX_OK; pktlen = ETH_ZLEN; } diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c index cb89323855d8..85c66af9e56d 100644 --- a/drivers/net/ethernet/xscale/ixp4xx_eth.c +++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c @@ -1425,7 +1425,6 @@ static int ixp4xx_eth_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; struct eth_plat_info *plat; struct net_device *ndev; - struct resource *res; struct port *port; int err; @@ -1482,10 +1481,7 @@ static int ixp4xx_eth_probe(struct platform_device *pdev) port->id = plat->npe; /* Get the port resource and remap */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - port->regs = devm_ioremap_resource(dev, res); + port->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); if (IS_ERR(port->regs)) return PTR_ERR(port->regs); @@ -1531,8 +1527,8 @@ static int ixp4xx_eth_probe(struct platform_device *pdev) phydev = of_phy_get_and_connect(ndev, np, ixp4xx_adjust_link); } else { phydev = mdiobus_get_phy(mdio_bus, plat->phy); - if (IS_ERR(phydev)) { - err = PTR_ERR(phydev); + if (!phydev) { + err = -ENODEV; dev_err(dev, "could not connect phydev (%d)\n", err); goto err_free_mem; } |