diff options
Diffstat (limited to 'drivers/net/ethernet/broadcom/bnxt')
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt.c | 134 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt.h | 21 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c | 6 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c | 9 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 8 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c | 18 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h | 4 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c | 4 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c | 29 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 144 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 7 |
12 files changed, 273 insertions, 113 deletions
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index f758b2e0591f..7134d2c3eb1c 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -54,6 +54,7 @@ #include <net/pkt_cls.h> #include <linux/hwmon.h> #include <linux/hwmon-sysfs.h> +#include <net/page_pool.h> #include "bnxt_hsi.h" #include "bnxt.h" @@ -668,19 +669,20 @@ next_tx_int: } static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, + struct bnxt_rx_ring_info *rxr, gfp_t gfp) { struct device *dev = &bp->pdev->dev; struct page *page; - page = alloc_page(gfp); + page = page_pool_dev_alloc_pages(rxr->page_pool); if (!page) return NULL; *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); if (dma_mapping_error(dev, *mapping)) { - __free_page(page); + page_pool_recycle_direct(rxr->page_pool, page); return NULL; } *mapping += bp->rx_dma_offset; @@ -716,7 +718,8 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, dma_addr_t mapping; if (BNXT_RX_PAGE_MODE(bp)) { - struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); + struct page *page = + __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); if (!page) return -ENOMEM; @@ -1989,6 +1992,9 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } } + if (event & BNXT_REDIRECT_EVENT) + xdp_do_flush_map(); + if (event & BNXT_TX_EVENT) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; u16 prod = txr->tx_prod; @@ -2130,12 +2136,12 @@ static int bnxt_poll(struct napi_struct *napi, int budget) } } if (bp->flags & BNXT_FLAG_DIM) { - struct net_dim_sample dim_sample; + struct dim_sample dim_sample; - net_dim_sample(cpr->event_ctr, - cpr->rx_packets, - cpr->rx_bytes, - &dim_sample); + dim_update_sample(cpr->event_ctr, + cpr->rx_packets, + cpr->rx_bytes, + &dim_sample); net_dim(&cpr->dim, dim_sample); } return work_done; @@ -2254,9 +2260,23 @@ static void bnxt_free_tx_skbs(struct bnxt *bp) for (j = 0; j < max_idx;) { struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; - struct sk_buff *skb = tx_buf->skb; + struct sk_buff *skb; int k, last; + if (i < bp->tx_nr_rings_xdp && + tx_buf->action == XDP_REDIRECT) { + dma_unmap_single(&pdev->dev, + dma_unmap_addr(tx_buf, mapping), + dma_unmap_len(tx_buf, len), + PCI_DMA_TODEVICE); + xdp_return_frame(tx_buf->xdpf); + tx_buf->action = 0; + tx_buf->xdpf = NULL; + j++; + continue; + } + + skb = tx_buf->skb; if (!skb) { j++; continue; @@ -2343,7 +2363,7 @@ static void bnxt_free_rx_skbs(struct bnxt *bp) dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); - __free_page(data); + page_pool_recycle_direct(rxr->page_pool, data); } else { dma_unmap_single_attrs(&pdev->dev, mapping, bp->rx_buf_use_size, @@ -2480,6 +2500,9 @@ static void bnxt_free_rx_rings(struct bnxt *bp) if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) xdp_rxq_info_unreg(&rxr->xdp_rxq); + page_pool_destroy(rxr->page_pool); + rxr->page_pool = NULL; + kfree(rxr->rx_tpa); rxr->rx_tpa = NULL; @@ -2494,6 +2517,26 @@ static void bnxt_free_rx_rings(struct bnxt *bp) } } +static int bnxt_alloc_rx_page_pool(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + struct page_pool_params pp = { 0 }; + + pp.pool_size = bp->rx_ring_size; + pp.nid = dev_to_node(&bp->pdev->dev); + pp.dev = &bp->pdev->dev; + pp.dma_dir = DMA_BIDIRECTIONAL; + + rxr->page_pool = page_pool_create(&pp); + if (IS_ERR(rxr->page_pool)) { + int err = PTR_ERR(rxr->page_pool); + + rxr->page_pool = NULL; + return err; + } + return 0; +} + static int bnxt_alloc_rx_rings(struct bnxt *bp) { int i, rc, agg_rings = 0, tpa_rings = 0; @@ -2513,10 +2556,22 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) ring = &rxr->rx_ring_struct; + rc = bnxt_alloc_rx_page_pool(bp, rxr); + if (rc) + return rc; + rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); if (rc < 0) return rc; + rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, + MEM_TYPE_PAGE_POOL, + rxr->page_pool); + if (rc) { + xdp_rxq_info_unreg(&rxr->xdp_rxq); + return rc; + } + rc = bnxt_alloc_ring(bp, &ring->ring_mem); if (rc) return rc; @@ -2622,8 +2677,6 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) mapping = txr->tx_push_mapping + sizeof(struct tx_push_bd); txr->data_mapping = cpu_to_le64(mapping); - - memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); } qidx = bp->tc_to_qidx[j]; ring->queue_id = bp->q_info[qidx].queue_id; @@ -3022,7 +3075,7 @@ static int bnxt_alloc_vnics(struct bnxt *bp) int num_vnics = 1; #ifdef CONFIG_RFS_ACCEL - if (bp->flags & BNXT_FLAG_RFS) + if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) num_vnics += bp->rx_nr_rings; #endif @@ -5508,7 +5561,16 @@ static int bnxt_cp_rings_in_use(struct bnxt *bp) static int bnxt_get_func_stat_ctxs(struct bnxt *bp) { - return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp); + int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); + int cp = bp->cp_nr_rings; + + if (!ulp_stat) + return cp; + + if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) + return bnxt_get_ulp_msix_base(bp) + ulp_stat; + + return cp + ulp_stat; } static bool bnxt_need_reserve_rings(struct bnxt *bp) @@ -7124,6 +7186,9 @@ static int bnxt_alloc_rfs_vnics(struct bnxt *bp) #ifdef CONFIG_RFS_ACCEL int i, rc = 0; + if (bp->flags & BNXT_FLAG_CHIP_P5) + return 0; + for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_vnic_info *vnic; u16 vnic_id = i + 1; @@ -7477,11 +7542,7 @@ unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) { - unsigned int stat; - - stat = bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_ulp_stat_ctxs(bp); - stat -= bp->cp_nr_rings; - return stat; + return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); } int bnxt_get_avail_msix(struct bnxt *bp, int num) @@ -7813,7 +7874,7 @@ static void bnxt_enable_napi(struct bnxt *bp) if (bp->bnapi[i]->rx_ring) { INIT_WORK(&cpr->dim.work, bnxt_dim_work); - cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; } napi_enable(&bp->bnapi[i]->napi); } @@ -9587,7 +9648,7 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, return -ENOMEM; vnics = 1; - if (bp->flags & BNXT_FLAG_RFS) + if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) vnics += rx_rings; if (bp->flags & BNXT_FLAG_AGG_RINGS) @@ -9847,32 +9908,19 @@ static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, } } -static int bnxt_setup_tc_block(struct net_device *dev, - struct tc_block_offload *f) -{ - struct bnxt *bp = netdev_priv(dev); - - if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) - return -EOPNOTSUPP; - - switch (f->command) { - case TC_BLOCK_BIND: - return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb, - bp, bp, f->extack); - case TC_BLOCK_UNBIND: - tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp); - return 0; - default: - return -EOPNOTSUPP; - } -} +static LIST_HEAD(bnxt_block_cb_list); static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { + struct bnxt *bp = netdev_priv(dev); + switch (type) { case TC_SETUP_BLOCK: - return bnxt_setup_tc_block(dev, type_data); + return flow_block_cb_setup_simple(type_data, + &bnxt_block_cb_list, + bnxt_setup_tc_block_cb, + bp, bp, true); case TC_SETUP_QDISC_MQPRIO: { struct tc_mqprio_qopt *mqprio = type_data; @@ -10233,6 +10281,7 @@ static const struct net_device_ops bnxt_netdev_ops = { .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, .ndo_bpf = bnxt_xdp, + .ndo_xdp_xmit = bnxt_xdp_xmit, .ndo_bridge_getlink = bnxt_bridge_getlink, .ndo_bridge_setlink = bnxt_bridge_setlink, .ndo_get_devlink_port = bnxt_get_devlink_port, @@ -10262,10 +10311,10 @@ static void bnxt_remove_one(struct pci_dev *pdev) bnxt_dcb_free(bp); kfree(bp->edev); bp->edev = NULL; + bnxt_cleanup_pci(bp); bnxt_free_ctx_mem(bp); kfree(bp->ctx); bp->ctx = NULL; - bnxt_cleanup_pci(bp); bnxt_free_port_stats(bp); free_netdev(dev); } @@ -10859,6 +10908,7 @@ static void bnxt_shutdown(struct pci_dev *pdev) if (system_state == SYSTEM_POWER_OFF) { bnxt_clear_int_mode(bp); + pci_disable_device(pdev); pci_wake_from_d3(pdev, bp->wol); pci_set_power_state(pdev, PCI_D3hot); } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index be438d82f939..16694b704d15 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -24,7 +24,9 @@ #include <net/devlink.h> #include <net/dst_metadata.h> #include <net/xdp.h> -#include <linux/net_dim.h> +#include <linux/dim.h> + +struct page_pool; struct tx_bd { __le32 tx_bd_len_flags_type; @@ -587,15 +589,21 @@ struct nqe_cn { #define BNXT_HWRM_CHNL_CHIMP 0 #define BNXT_HWRM_CHNL_KONG 1 -#define BNXT_RX_EVENT 1 -#define BNXT_AGG_EVENT 2 -#define BNXT_TX_EVENT 4 +#define BNXT_RX_EVENT 1 +#define BNXT_AGG_EVENT 2 +#define BNXT_TX_EVENT 4 +#define BNXT_REDIRECT_EVENT 8 struct bnxt_sw_tx_bd { - struct sk_buff *skb; + union { + struct sk_buff *skb; + struct xdp_frame *xdpf; + }; DEFINE_DMA_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_LEN(len); u8 is_gso; u8 is_push; + u8 action; union { unsigned short nr_frags; u16 rx_prod; @@ -793,6 +801,7 @@ struct bnxt_rx_ring_info { struct bnxt_ring_struct rx_ring_struct; struct bnxt_ring_struct rx_agg_ring_struct; struct xdp_rxq_info xdp_rxq; + struct page_pool *page_pool; }; struct bnxt_cp_ring_info { @@ -810,7 +819,7 @@ struct bnxt_cp_ring_info { u64 rx_bytes; u64 event_ctr; - struct net_dim dim; + struct dim dim; union { struct tx_cmp *cp_desc_ring[MAX_CP_PAGES]; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c index 70775158c8c4..07301cb87c03 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c @@ -396,7 +396,7 @@ static int bnxt_hwrm_queue_dscp_qcaps(struct bnxt *bp) bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_DSCP_QCAPS, -1, -1); mutex_lock(&bp->hwrm_cmd_lock); - rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); + rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); if (!rc) { bp->max_dscp_value = (1 << resp->num_dscp_bits) - 1; if (bp->max_dscp_value < 0x3f) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c index 94e208e9789f..61393f351a77 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c @@ -11,7 +11,7 @@ #include <linux/module.h> #include <linux/pci.h> #include "bnxt_hsi.h" -#include <linux/net_dim.h> +#include <linux/dim.h> #include "bnxt.h" #include "bnxt_debugfs.h" @@ -21,7 +21,7 @@ static ssize_t debugfs_dim_read(struct file *filep, char __user *buffer, size_t count, loff_t *ppos) { - struct net_dim *dim = filep->private_data; + struct dim *dim = filep->private_data; int len; char *buf; @@ -61,7 +61,7 @@ static const struct file_operations debugfs_dim_fops = { .read = debugfs_dim_read, }; -static struct dentry *debugfs_dim_ring_init(struct net_dim *dim, int ring_idx, +static struct dentry *debugfs_dim_ring_init(struct dim *dim, int ring_idx, struct dentry *dd) { static char qname[16]; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c index afa97c8bb081..6f6576dc417a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c @@ -7,26 +7,25 @@ * the Free Software Foundation. */ -#include <linux/net_dim.h> +#include <linux/dim.h> #include "bnxt_hsi.h" #include "bnxt.h" void bnxt_dim_work(struct work_struct *work) { - struct net_dim *dim = container_of(work, struct net_dim, - work); + struct dim *dim = container_of(work, struct dim, work); struct bnxt_cp_ring_info *cpr = container_of(dim, struct bnxt_cp_ring_info, dim); struct bnxt_napi *bnapi = container_of(cpr, struct bnxt_napi, cp_ring); - struct net_dim_cq_moder cur_moder = + struct dim_cq_moder cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); cpr->rx_ring_coal.coal_ticks = cur_moder.usec; cpr->rx_ring_coal.coal_bufs = cur_moder.pkts; bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi); - dim->state = NET_DIM_START_MEASURE; + dim->state = DIM_START_MEASURE; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index a6c7baf38036..c7ee63d69679 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -2799,7 +2799,7 @@ static int bnxt_run_loopback(struct bnxt *bp) dev_kfree_skb(skb); return -EIO; } - bnxt_xmit_xdp(bp, txr, map, pkt_size, 0); + bnxt_xmit_bd(bp, txr, map, pkt_size); /* Sync BD data before updating doorbell */ wmb(); @@ -2842,7 +2842,7 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest, bool offline = false; u8 test_results = 0; u8 test_mask = 0; - int rc, i; + int rc = 0, i; if (!bp->num_tests || !BNXT_SINGLE_PF(bp)) return; @@ -2913,9 +2913,9 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest, } bnxt_hwrm_phy_loopback(bp, false, false); bnxt_half_close_nic(bp); - bnxt_open_nic(bp, false, true); + rc = bnxt_open_nic(bp, false, true); } - if (bnxt_test_irq(bp)) { + if (rc || bnxt_test_irq(bp)) { buf[BNXT_IRQ_TEST_IDX] = 1; etest->flags |= ETH_TEST_FL_FAILED; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c index 44d6c5743fb9..6fe4a7174271 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c @@ -170,10 +170,10 @@ static int bnxt_tc_parse_actions(struct bnxt *bp, } static int bnxt_tc_parse_flow(struct bnxt *bp, - struct tc_cls_flower_offload *tc_flow_cmd, + struct flow_cls_offload *tc_flow_cmd, struct bnxt_tc_flow *flow) { - struct flow_rule *rule = tc_cls_flower_offload_flow_rule(tc_flow_cmd); + struct flow_rule *rule = flow_cls_offload_flow_rule(tc_flow_cmd); struct flow_dissector *dissector = rule->match.dissector; /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ @@ -1262,7 +1262,7 @@ static void bnxt_tc_set_src_fid(struct bnxt *bp, struct bnxt_tc_flow *flow, * The hash-tables are already protected by the rhashtable API. */ static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *tc_flow_cmd) + struct flow_cls_offload *tc_flow_cmd) { struct bnxt_tc_flow_node *new_node, *old_node; struct bnxt_tc_info *tc_info = bp->tc_info; @@ -1348,7 +1348,7 @@ done: } static int bnxt_tc_del_flow(struct bnxt *bp, - struct tc_cls_flower_offload *tc_flow_cmd) + struct flow_cls_offload *tc_flow_cmd) { struct bnxt_tc_info *tc_info = bp->tc_info; struct bnxt_tc_flow_node *flow_node; @@ -1363,7 +1363,7 @@ static int bnxt_tc_del_flow(struct bnxt *bp, } static int bnxt_tc_get_flow_stats(struct bnxt *bp, - struct tc_cls_flower_offload *tc_flow_cmd) + struct flow_cls_offload *tc_flow_cmd) { struct bnxt_tc_flow_stats stats, *curr_stats, *prev_stats; struct bnxt_tc_info *tc_info = bp->tc_info; @@ -1585,14 +1585,14 @@ void bnxt_tc_flow_stats_work(struct bnxt *bp) } int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *cls_flower) + struct flow_cls_offload *cls_flower) { switch (cls_flower->command) { - case TC_CLSFLOWER_REPLACE: + case FLOW_CLS_REPLACE: return bnxt_tc_add_flow(bp, src_fid, cls_flower); - case TC_CLSFLOWER_DESTROY: + case FLOW_CLS_DESTROY: return bnxt_tc_del_flow(bp, cls_flower); - case TC_CLSFLOWER_STATS: + case FLOW_CLS_STATS: return bnxt_tc_get_flow_stats(bp, cls_flower); default: return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h index 8a0968967bc5..ffec57d1a5ec 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h @@ -196,7 +196,7 @@ struct bnxt_tc_flow_node { }; int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *cls_flower); + struct flow_cls_offload *cls_flower); int bnxt_init_tc(struct bnxt *bp); void bnxt_shutdown_tc(struct bnxt *bp); void bnxt_tc_flow_stats_work(struct bnxt *bp); @@ -209,7 +209,7 @@ static inline bool bnxt_tc_flower_enabled(struct bnxt *bp) #else /* CONFIG_BNXT_FLOWER_OFFLOAD */ static inline int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *cls_flower) + struct flow_cls_offload *cls_flower) { return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c index bfa342a98d08..fc77caf0a076 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c @@ -157,8 +157,10 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, if (BNXT_NEW_RM(bp)) { struct bnxt_hw_resc *hw_resc = &bp->hw_resc; + int resv_msix; - avail_msix = hw_resc->resv_irqs - bp->cp_nr_rings; + resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings; + avail_msix = min_t(int, resv_msix, avail_msix); edev->ulp_tbl[ulp_id].msix_requested = avail_msix; } bnxt_fill_msix_vecs(bp, ent); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c index f760921389a3..f9bf7d7250ab 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c @@ -161,34 +161,19 @@ static int bnxt_vf_rep_setup_tc_block_cb(enum tc_setup_type type, } } -static int bnxt_vf_rep_setup_tc_block(struct net_device *dev, - struct tc_block_offload *f) -{ - struct bnxt_vf_rep *vf_rep = netdev_priv(dev); - - if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) - return -EOPNOTSUPP; - - switch (f->command) { - case TC_BLOCK_BIND: - return tcf_block_cb_register(f->block, - bnxt_vf_rep_setup_tc_block_cb, - vf_rep, vf_rep, f->extack); - case TC_BLOCK_UNBIND: - tcf_block_cb_unregister(f->block, - bnxt_vf_rep_setup_tc_block_cb, vf_rep); - return 0; - default: - return -EOPNOTSUPP; - } -} +static LIST_HEAD(bnxt_vf_block_cb_list); static int bnxt_vf_rep_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { + struct bnxt_vf_rep *vf_rep = netdev_priv(dev); + switch (type) { case TC_SETUP_BLOCK: - return bnxt_vf_rep_setup_tc_block(dev, type_data); + return flow_block_cb_setup_simple(type_data, + &bnxt_vf_block_cb_list, + bnxt_vf_rep_setup_tc_block_cb, + vf_rep, vf_rep, true); default: return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 0184ef6f05a7..c6f6f2033880 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -15,12 +15,14 @@ #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/filter.h> +#include <net/page_pool.h> #include "bnxt_hsi.h" #include "bnxt.h" #include "bnxt_xdp.h" -void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len, u16 rx_prod) +struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len) { struct bnxt_sw_tx_bd *tx_buf; struct tx_bd *txbd; @@ -29,7 +31,6 @@ void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, prod = txr->tx_prod; tx_buf = &txr->tx_buf_ring[prod]; - tx_buf->rx_prod = rx_prod; txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; flags = (len << TX_BD_LEN_SHIFT) | (1 << TX_BD_FLAGS_BD_CNT_SHIFT) | @@ -40,30 +41,67 @@ void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, prod = NEXT_TX(prod); txr->tx_prod = prod; + return tx_buf; +} + +static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len, u16 rx_prod) +{ + struct bnxt_sw_tx_bd *tx_buf; + + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len); + tx_buf->rx_prod = rx_prod; + tx_buf->action = XDP_TX; +} + +static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len, + struct xdp_frame *xdpf) +{ + struct bnxt_sw_tx_bd *tx_buf; + + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len); + tx_buf->action = XDP_REDIRECT; + tx_buf->xdpf = xdpf; + dma_unmap_addr_set(tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, 0); } void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; + bool rx_doorbell_needed = false; struct bnxt_sw_tx_bd *tx_buf; u16 tx_cons = txr->tx_cons; u16 last_tx_cons = tx_cons; - u16 rx_prod; int i; for (i = 0; i < nr_pkts; i++) { - last_tx_cons = tx_cons; + tx_buf = &txr->tx_buf_ring[tx_cons]; + + if (tx_buf->action == XDP_REDIRECT) { + struct pci_dev *pdev = bp->pdev; + + dma_unmap_single(&pdev->dev, + dma_unmap_addr(tx_buf, mapping), + dma_unmap_len(tx_buf, len), + PCI_DMA_TODEVICE); + xdp_return_frame(tx_buf->xdpf); + tx_buf->action = 0; + tx_buf->xdpf = NULL; + } else if (tx_buf->action == XDP_TX) { + rx_doorbell_needed = true; + last_tx_cons = tx_cons; + } tx_cons = NEXT_TX(tx_cons); } txr->tx_cons = tx_cons; - if (bnxt_tx_avail(bp, txr) == bp->tx_ring_size) { - rx_prod = rxr->rx_prod; - } else { + if (rx_doorbell_needed) { tx_buf = &txr->tx_buf_ring[last_tx_cons]; - rx_prod = tx_buf->rx_prod; + bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod); } - bnxt_db_write(bp, &rxr->rx_db, rx_prod); } /* returns the following: @@ -88,19 +126,19 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, return false; pdev = bp->pdev; - txr = rxr->bnapi->tx_ring; rx_buf = &rxr->rx_buf_ring[cons]; offset = bp->rx_offset; + mapping = rx_buf->mapping - bp->rx_dma_offset; + dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); + + txr = rxr->bnapi->tx_ring; xdp.data_hard_start = *data_ptr - offset; xdp.data = *data_ptr; xdp_set_data_meta_invalid(&xdp); xdp.data_end = *data_ptr + *len; xdp.rxq = &rxr->xdp_rxq; orig_data = xdp.data; - mapping = rx_buf->mapping - bp->rx_dma_offset; - - dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); rcu_read_lock(); act = bpf_prog_run_xdp(xdp_prog, &xdp); @@ -132,10 +170,34 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, *event = BNXT_TX_EVENT; dma_sync_single_for_device(&pdev->dev, mapping + offset, *len, bp->rx_dir); - bnxt_xmit_xdp(bp, txr, mapping + offset, *len, - NEXT_RX(rxr->rx_prod)); + __bnxt_xmit_xdp(bp, txr, mapping + offset, *len, + NEXT_RX(rxr->rx_prod)); bnxt_reuse_rx_data(rxr, cons, page); return true; + case XDP_REDIRECT: + /* if we are calling this here then we know that the + * redirect is coming from a frame received by the + * bnxt_en driver. + */ + dma_unmap_page_attrs(&pdev->dev, mapping, + PAGE_SIZE, bp->rx_dir, + DMA_ATTR_WEAK_ORDERING); + + /* if we are unable to allocate a new buffer, abort and reuse */ + if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) { + trace_xdp_exception(bp->dev, xdp_prog, act); + bnxt_reuse_rx_data(rxr, cons, page); + return true; + } + + if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) { + trace_xdp_exception(bp->dev, xdp_prog, act); + page_pool_recycle_direct(rxr->page_pool, page); + return true; + } + + *event |= BNXT_REDIRECT_EVENT; + break; default: bpf_warn_invalid_xdp_action(act); /* Fall thru */ @@ -149,6 +211,56 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, return true; } +int bnxt_xdp_xmit(struct net_device *dev, int num_frames, + struct xdp_frame **frames, u32 flags) +{ + struct bnxt *bp = netdev_priv(dev); + struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog); + struct pci_dev *pdev = bp->pdev; + struct bnxt_tx_ring_info *txr; + dma_addr_t mapping; + int drops = 0; + int ring; + int i; + + if (!test_bit(BNXT_STATE_OPEN, &bp->state) || + !bp->tx_nr_rings_xdp || + !xdp_prog) + return -EINVAL; + + ring = smp_processor_id() % bp->tx_nr_rings_xdp; + txr = &bp->tx_ring[ring]; + + for (i = 0; i < num_frames; i++) { + struct xdp_frame *xdp = frames[i]; + + if (!txr || !bnxt_tx_avail(bp, txr) || + !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) { + xdp_return_frame_rx_napi(xdp); + drops++; + continue; + } + + mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, + DMA_TO_DEVICE); + + if (dma_mapping_error(&pdev->dev, mapping)) { + xdp_return_frame_rx_napi(xdp); + drops++; + continue; + } + __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp); + } + + if (flags & XDP_XMIT_FLUSH) { + /* Sync BD data before updating doorbell */ + wmb(); + bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); + } + + return num_frames - drops; +} + /* Under rtnl_lock */ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index 414b748038ca..0df40c3beb05 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -10,12 +10,15 @@ #ifndef BNXT_XDP_H #define BNXT_XDP_H -void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len, u16 rx_prod); +struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len); void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts); bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, struct page *page, u8 **data_ptr, unsigned int *len, u8 *event); int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp); +int bnxt_xdp_xmit(struct net_device *dev, int num_frames, + struct xdp_frame **frames, u32 flags); #endif |