diff options
Diffstat (limited to '')
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt.c | 710 |
1 files changed, 519 insertions, 191 deletions
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 99d025b69079..b86f980fa7ea 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -245,6 +245,21 @@ static const u16 bnxt_async_events_arr[] = { ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, + ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER, +}; + +const u16 bnxt_bstore_to_trace[] = { + [BNXT_CTX_SRT] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT_TRACE, + [BNXT_CTX_SRT2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT2_TRACE, + [BNXT_CTX_CRT] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT_TRACE, + [BNXT_CTX_CRT2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT2_TRACE, + [BNXT_CTX_RIGP0] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP0_TRACE, + [BNXT_CTX_L2HWRM] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_L2_HWRM_TRACE, + [BNXT_CTX_REHWRM] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ROCE_HWRM_TRACE, + [BNXT_CTX_CA0] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA0_TRACE, + [BNXT_CTX_CA1] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA1_TRACE, + [BNXT_CTX_CA2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA2_TRACE, + [BNXT_CTX_RIGP1] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP1_TRACE, }; static struct workqueue_struct *bnxt_pf_wq; @@ -864,6 +879,11 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) bnapi->events &= ~BNXT_TX_CMP_EVENT; } +static bool bnxt_separate_head_pool(void) +{ + return PAGE_SIZE > BNXT_RX_PAGE_SIZE; +} + static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, struct bnxt_rx_ring_info *rxr, unsigned int *offset, @@ -886,27 +906,19 @@ static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, } static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, + struct bnxt_rx_ring_info *rxr, gfp_t gfp) { - u8 *data; - struct pci_dev *pdev = bp->pdev; + unsigned int offset; + struct page *page; - if (gfp == GFP_ATOMIC) - data = napi_alloc_frag(bp->rx_buf_size); - else - data = netdev_alloc_frag(bp->rx_buf_size); - if (!data) + page = page_pool_alloc_frag(rxr->head_pool, &offset, + bp->rx_buf_size, gfp); + if (!page) return NULL; - *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset, - bp->rx_buf_use_size, bp->rx_dir, - DMA_ATTR_WEAK_ORDERING); - - if (dma_mapping_error(&pdev->dev, *mapping)) { - skb_free_frag(data); - data = NULL; - } - return data; + *mapping = page_pool_get_dma_addr(page) + bp->rx_dma_offset + offset; + return page_address(page) + offset; } int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, @@ -928,7 +940,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, rx_buf->data = page; rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; } else { - u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp); + u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp); if (!data) return -ENOMEM; @@ -1179,13 +1191,14 @@ static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, } skb = napi_build_skb(data, bp->rx_buf_size); - dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, - bp->rx_dir, DMA_ATTR_WEAK_ORDERING); + dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, + bp->rx_dir); if (!skb) { - skb_free_frag(data); + page_pool_free_va(rxr->head_pool, data, true); return NULL; } + skb_mark_for_recycle(skb); skb_reserve(skb, bp->rx_offset); skb_put(skb, offset_and_len & 0xffff); return skb; @@ -1518,7 +1531,7 @@ static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, if (TPA_START_IS_IPV6(tpa_start1)) tpa_info->gso_type = SKB_GSO_TCPV6; /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ - else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP && + else if (!BNXT_CHIP_P4_PLUS(bp) && TPA_START_HASH_TYPE(tpa_start) == 3) tpa_info->gso_type = SKB_GSO_TCPV6; tpa_info->rss_hash = @@ -1840,7 +1853,8 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, u8 *new_data; dma_addr_t new_mapping; - new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC); + new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, rxr, + GFP_ATOMIC); if (!new_data) { bnxt_abort_tpa(cpr, idx, agg_bufs); cpr->sw_stats->rx.rx_oom_discards += 1; @@ -1852,16 +1866,16 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, tpa_info->mapping = new_mapping; skb = napi_build_skb(data, bp->rx_buf_size); - dma_unmap_single_attrs(&bp->pdev->dev, mapping, - bp->rx_buf_use_size, bp->rx_dir, - DMA_ATTR_WEAK_ORDERING); + dma_sync_single_for_cpu(&bp->pdev->dev, mapping, + bp->rx_buf_use_size, bp->rx_dir); if (!skb) { - skb_free_frag(data); + page_pool_free_va(rxr->head_pool, data, true); bnxt_abort_tpa(cpr, idx, agg_bufs); cpr->sw_stats->rx.rx_oom_discards += 1; return NULL; } + skb_mark_for_recycle(skb); skb_reserve(skb, bp->rx_offset); skb_put(skb, len); } @@ -2212,15 +2226,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) { type = bnxt_rss_ext_op(bp, rxcmp); } else { - u32 hash_type = RX_CMP_HASH_TYPE(rxcmp); + u32 itypes = RX_CMP_ITYPES(rxcmp); - /* RSS profiles 1 and 3 with extract code 0 for inner - * 4-tuple - */ - if (hash_type != 1 && hash_type != 3) - type = PKT_HASH_TYPE_L3; - else + if (itypes == RX_CMP_FLAGS_ITYPE_TCP || + itypes == RX_CMP_FLAGS_ITYPE_UDP) type = PKT_HASH_TYPE_L4; + else + type = PKT_HASH_TYPE_L3; } skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); } @@ -2254,11 +2266,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; - unsigned long flags; - spin_lock_irqsave(&ptp->ptp_lock, flags); - ns = timecounter_cyc2time(&ptp->tc, ts); - spin_unlock_irqrestore(&ptp->ptp_lock, flags); + ns = bnxt_timecounter_cyc2time(ptp, ts); memset(skb_hwtstamps(skb), 0, sizeof(*skb_hwtstamps(skb))); skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); @@ -2465,6 +2474,59 @@ static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info) return false; } +bool bnxt_bs_trace_avail(struct bnxt *bp, u16 type) +{ + u32 flags = bp->ctx->ctx_arr[type].flags; + + return (flags & BNXT_CTX_MEM_TYPE_VALID) && + ((flags & BNXT_CTX_MEM_FW_TRACE) || + (flags & BNXT_CTX_MEM_FW_BIN_TRACE)); +} + +static void bnxt_bs_trace_init(struct bnxt *bp, struct bnxt_ctx_mem_type *ctxm) +{ + u32 mem_size, pages, rem_bytes, magic_byte_offset; + u16 trace_type = bnxt_bstore_to_trace[ctxm->type]; + struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; + struct bnxt_ring_mem_info *rmem, *rmem_pg_tbl; + struct bnxt_bs_trace_info *bs_trace; + int last_pg; + + if (ctxm->instance_bmap && ctxm->instance_bmap > 1) + return; + + mem_size = ctxm->max_entries * ctxm->entry_size; + rem_bytes = mem_size % BNXT_PAGE_SIZE; + pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); + + last_pg = (pages - 1) & (MAX_CTX_PAGES - 1); + magic_byte_offset = (rem_bytes ? rem_bytes : BNXT_PAGE_SIZE) - 1; + + rmem = &ctx_pg[0].ring_mem; + bs_trace = &bp->bs_trace[trace_type]; + bs_trace->ctx_type = ctxm->type; + bs_trace->trace_type = trace_type; + if (pages > MAX_CTX_PAGES) { + int last_pg_dir = rmem->nr_pages - 1; + + rmem_pg_tbl = &ctx_pg[0].ctx_pg_tbl[last_pg_dir]->ring_mem; + bs_trace->magic_byte = rmem_pg_tbl->pg_arr[last_pg]; + } else { + bs_trace->magic_byte = rmem->pg_arr[last_pg]; + } + bs_trace->magic_byte += magic_byte_offset; + *bs_trace->magic_byte = BNXT_TRACE_BUF_MAGIC_BYTE; +} + +#define BNXT_EVENT_BUF_PRODUCER_TYPE(data1) \ + (((data1) & ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_MASK) >>\ + ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_SFT) + +#define BNXT_EVENT_BUF_PRODUCER_OFFSET(data2) \ + (((data2) & \ + ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_MASK) >>\ + ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_SFT) + #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \ ((data2) & \ ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK) @@ -2764,12 +2826,12 @@ static int bnxt_async_event_process(struct bnxt *bp, if (!ptp) goto async_event_process_exit; - spin_lock_irqsave(&ptp->ptp_lock, flags); bnxt_ptp_update_current_time(bp); ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) << BNXT_PHC_BITS) | ptp->current_time); + write_seqlock_irqsave(&ptp->ptp_lock, flags); bnxt_ptp_rtc_timecounter_init(ptp, ns); - spin_unlock_irqrestore(&ptp->ptp_lock, flags); + write_sequnlock_irqrestore(&ptp->ptp_lock, flags); } break; } @@ -2781,6 +2843,13 @@ static int bnxt_async_event_process(struct bnxt *bp, hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); goto async_event_process_exit; } + case ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER: { + u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1); + u32 offset = BNXT_EVENT_BUF_PRODUCER_OFFSET(data2); + + bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset); + goto async_event_process_exit; + } default: goto async_event_process_exit; } @@ -3102,7 +3171,7 @@ static int bnxt_poll(struct napi_struct *napi, int budget) cpr->rx_packets, cpr->rx_bytes, &dim_sample); - net_dim(&cpr->dim, dim_sample); + net_dim(&cpr->dim, &dim_sample); } return work_done; } @@ -3233,7 +3302,7 @@ poll_done: cpr_rx->rx_packets, cpr_rx->rx_bytes, &dim_sample); - net_dim(&cpr->dim, dim_sample); + net_dim(&cpr->dim, &dim_sample); } return work_done; } @@ -3311,28 +3380,22 @@ static void bnxt_free_tx_skbs(struct bnxt *bp) static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) { - struct pci_dev *pdev = bp->pdev; int i, max_idx; max_idx = bp->rx_nr_pages * RX_DESC_CNT; for (i = 0; i < max_idx; i++) { struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; - dma_addr_t mapping = rx_buf->mapping; void *data = rx_buf->data; if (!data) continue; rx_buf->data = NULL; - if (BNXT_RX_PAGE_MODE(bp)) { + if (BNXT_RX_PAGE_MODE(bp)) page_pool_recycle_direct(rxr->page_pool, data); - } else { - dma_unmap_single_attrs(&pdev->dev, mapping, - bp->rx_buf_use_size, bp->rx_dir, - DMA_ATTR_WEAK_ORDERING); - skb_free_frag(data); - } + else + page_pool_free_va(rxr->head_pool, data, true); } } @@ -3356,16 +3419,11 @@ static void bnxt_free_one_rx_agg_ring(struct bnxt *bp, struct bnxt_rx_ring_info } } -static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) +static void bnxt_free_one_tpa_info_data(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) { - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; - struct pci_dev *pdev = bp->pdev; - struct bnxt_tpa_idx_map *map; int i; - if (!rxr->rx_tpa) - goto skip_rx_tpa_free; - for (i = 0; i < bp->max_tpa; i++) { struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; u8 *data = tpa_info->data; @@ -3373,14 +3431,20 @@ static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr) if (!data) continue; - dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping, - bp->rx_buf_use_size, bp->rx_dir, - DMA_ATTR_WEAK_ORDERING); - tpa_info->data = NULL; - - skb_free_frag(data); + page_pool_free_va(rxr->head_pool, data, false); } +} + +static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + struct bnxt_tpa_idx_map *map; + + if (!rxr->rx_tpa) + goto skip_rx_tpa_free; + + bnxt_free_one_tpa_info_data(bp, rxr); skip_rx_tpa_free: if (!rxr->rx_buf_ring) @@ -3408,7 +3472,7 @@ static void bnxt_free_rx_skbs(struct bnxt *bp) return; for (i = 0; i < bp->rx_nr_rings; i++) - bnxt_free_one_rx_ring_skbs(bp, i); + bnxt_free_one_rx_ring_skbs(bp, &bp->rx_ring[i]); } static void bnxt_free_skbs(struct bnxt *bp) @@ -3434,6 +3498,35 @@ static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len) *(p2 + i + offset) = init_val; } +static size_t __bnxt_copy_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem, + void *buf, size_t offset, size_t head, + size_t tail) +{ + int i, head_page, start_idx, source_offset; + size_t len, rem_len, total_len, max_bytes; + + head_page = head / rmem->page_size; + source_offset = head % rmem->page_size; + total_len = (tail - head) & MAX_CTX_BYTES_MASK; + if (!total_len) + total_len = MAX_CTX_BYTES; + start_idx = head_page % MAX_CTX_PAGES; + max_bytes = (rmem->nr_pages - start_idx) * rmem->page_size - + source_offset; + total_len = min(total_len, max_bytes); + rem_len = total_len; + + for (i = start_idx; rem_len; i++, source_offset = 0) { + len = min((size_t)(rmem->page_size - source_offset), rem_len); + if (buf) + memcpy(buf + offset, rmem->pg_arr[i] + source_offset, + len); + offset += len; + rem_len -= len; + } + return total_len; +} + static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) { struct pci_dev *pdev = bp->pdev; @@ -3520,29 +3613,64 @@ static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) return 0; } +static void bnxt_free_one_tpa_info(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + int i; + + kfree(rxr->rx_tpa_idx_map); + rxr->rx_tpa_idx_map = NULL; + if (rxr->rx_tpa) { + for (i = 0; i < bp->max_tpa; i++) { + kfree(rxr->rx_tpa[i].agg_arr); + rxr->rx_tpa[i].agg_arr = NULL; + } + } + kfree(rxr->rx_tpa); + rxr->rx_tpa = NULL; +} + static void bnxt_free_tpa_info(struct bnxt *bp) { - int i, j; + int i; for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; - kfree(rxr->rx_tpa_idx_map); - rxr->rx_tpa_idx_map = NULL; - if (rxr->rx_tpa) { - for (j = 0; j < bp->max_tpa; j++) { - kfree(rxr->rx_tpa[j].agg_arr); - rxr->rx_tpa[j].agg_arr = NULL; - } - } - kfree(rxr->rx_tpa); - rxr->rx_tpa = NULL; + bnxt_free_one_tpa_info(bp, rxr); } } +static int bnxt_alloc_one_tpa_info(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + struct rx_agg_cmp *agg; + int i; + + rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), + GFP_KERNEL); + if (!rxr->rx_tpa) + return -ENOMEM; + + if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) + return 0; + for (i = 0; i < bp->max_tpa; i++) { + agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); + if (!agg) + return -ENOMEM; + rxr->rx_tpa[i].agg_arr = agg; + } + rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), + GFP_KERNEL); + if (!rxr->rx_tpa_idx_map) + return -ENOMEM; + + return 0; +} + static int bnxt_alloc_tpa_info(struct bnxt *bp) { - int i, j; + int i, rc; bp->max_tpa = MAX_TPA; if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { @@ -3553,25 +3681,10 @@ static int bnxt_alloc_tpa_info(struct bnxt *bp) for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; - struct rx_agg_cmp *agg; - rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), - GFP_KERNEL); - if (!rxr->rx_tpa) - return -ENOMEM; - - if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) - continue; - for (j = 0; j < bp->max_tpa; j++) { - agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); - if (!agg) - return -ENOMEM; - rxr->rx_tpa[j].agg_arr = agg; - } - rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), - GFP_KERNEL); - if (!rxr->rx_tpa_idx_map) - return -ENOMEM; + rc = bnxt_alloc_one_tpa_info(bp, rxr); + if (rc) + return rc; } return 0; } @@ -3595,7 +3708,9 @@ static void bnxt_free_rx_rings(struct bnxt *bp) xdp_rxq_info_unreg(&rxr->xdp_rxq); page_pool_destroy(rxr->page_pool); - rxr->page_pool = NULL; + if (bnxt_separate_head_pool()) + page_pool_destroy(rxr->head_pool); + rxr->page_pool = rxr->head_pool = NULL; kfree(rxr->rx_agg_bmap); rxr->rx_agg_bmap = NULL; @@ -3613,6 +3728,7 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, int numa_node) { struct page_pool_params pp = { 0 }; + struct page_pool *pool; pp.pool_size = bp->rx_agg_ring_size; if (BNXT_RX_PAGE_MODE(bp)) @@ -3625,13 +3741,37 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, pp.max_len = PAGE_SIZE; pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; - rxr->page_pool = page_pool_create(&pp); - if (IS_ERR(rxr->page_pool)) { - int err = PTR_ERR(rxr->page_pool); + pool = page_pool_create(&pp); + if (IS_ERR(pool)) + return PTR_ERR(pool); + rxr->page_pool = pool; - rxr->page_pool = NULL; - return err; + if (bnxt_separate_head_pool()) { + pp.pool_size = max(bp->rx_ring_size, 1024); + pool = page_pool_create(&pp); + if (IS_ERR(pool)) + goto err_destroy_pp; } + rxr->head_pool = pool; + + return 0; + +err_destroy_pp: + page_pool_destroy(rxr->page_pool); + rxr->page_pool = NULL; + return PTR_ERR(pool); +} + +static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) +{ + u16 mem_size; + + rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; + mem_size = rxr->rx_agg_bmap_size / 8; + rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); + if (!rxr->rx_agg_bmap) + return -ENOMEM; + return 0; } @@ -3679,19 +3819,15 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) ring->grp_idx = i; if (agg_rings) { - u16 mem_size; - ring = &rxr->rx_agg_ring_struct; rc = bnxt_alloc_ring(bp, &ring->ring_mem); if (rc) return rc; ring->grp_idx = i; - rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; - mem_size = rxr->rx_agg_bmap_size / 8; - rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); - if (!rxr->rx_agg_bmap) - return -ENOMEM; + rc = bnxt_alloc_rx_agg_bmap(bp, rxr); + if (rc) + return rc; } } if (bp->flags & BNXT_FLAG_TPA) @@ -4166,10 +4302,31 @@ static void bnxt_alloc_one_rx_ring_page(struct bnxt *bp, rxr->rx_agg_prod = prod; } +static int bnxt_alloc_one_tpa_info_data(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + dma_addr_t mapping; + u8 *data; + int i; + + for (i = 0; i < bp->max_tpa; i++) { + data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, + GFP_KERNEL); + if (!data) + return -ENOMEM; + + rxr->rx_tpa[i].data = data; + rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; + rxr->rx_tpa[i].mapping = mapping; + } + + return 0; +} + static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) { struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; - int i; + int rc; bnxt_alloc_one_rx_ring_skb(bp, rxr, ring_nr); @@ -4179,18 +4336,9 @@ static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) bnxt_alloc_one_rx_ring_page(bp, rxr, ring_nr); if (rxr->rx_tpa) { - dma_addr_t mapping; - u8 *data; - - for (i = 0; i < bp->max_tpa; i++) { - data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL); - if (!data) - return -ENOMEM; - - rxr->rx_tpa[i].data = data; - rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; - rxr->rx_tpa[i].mapping = mapping; - } + rc = bnxt_alloc_one_tpa_info_data(bp, rxr); + if (rc) + return rc; } return 0; } @@ -4558,7 +4706,7 @@ int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) struct net_device *dev = bp->dev; if (page_mode) { - bp->flags &= ~BNXT_FLAG_AGG_RINGS; + bp->flags &= ~(BNXT_FLAG_AGG_RINGS | BNXT_FLAG_NO_AGG_RINGS); bp->flags |= BNXT_FLAG_RX_PAGE_MODE; if (bp->xdp_prog->aux->xdp_has_frags) @@ -8153,6 +8301,9 @@ static int bnxt_hwrm_func_qcfg(struct bnxt *bp) if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; + if (flags & FUNC_QCFG_RESP_FLAGS_ENABLE_RDMA_SRIOV) + bp->fw_cap |= BNXT_FW_CAP_ENABLE_RDMA_SRIOV; + switch (resp->port_partition_type) { case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: @@ -8214,7 +8365,7 @@ static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max) struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; int n = 1; - if (!ctxm->max_entries) + if (!ctxm->max_entries || ctxm->pg_info) continue; if (ctxm->instance_bmap) @@ -8226,6 +8377,9 @@ static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max) return 0; } +static void bnxt_free_one_ctx_mem(struct bnxt *bp, + struct bnxt_ctx_mem_type *ctxm, bool force); + #define BNXT_CTX_INIT_VALID(flags) \ (!!((flags) & \ FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT)) @@ -8234,7 +8388,7 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) { struct hwrm_func_backing_store_qcaps_v2_output *resp; struct hwrm_func_backing_store_qcaps_v2_input *req; - struct bnxt_ctx_mem_info *ctx; + struct bnxt_ctx_mem_info *ctx = bp->ctx; u16 type; int rc; @@ -8242,16 +8396,20 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) if (rc) return rc; - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return -ENOMEM; - bp->ctx = ctx; + if (!ctx) { + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + bp->ctx = ctx; + } resp = hwrm_req_hold(bp, req); for (type = 0; type < BNXT_CTX_V2_MAX; ) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; u8 init_val, init_off, i; + u32 max_entries; + u16 entry_size; __le32 *p; u32 flags; @@ -8261,15 +8419,26 @@ static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) goto ctx_done; flags = le32_to_cpu(resp->flags); type = le16_to_cpu(resp->next_valid_type); - if (!(flags & FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID)) + if (!(flags & BNXT_CTX_MEM_TYPE_VALID)) { + bnxt_free_one_ctx_mem(bp, ctxm, true); continue; - + } + entry_size = le16_to_cpu(resp->entry_size); + max_entries = le32_to_cpu(resp->max_num_entries); + if (ctxm->mem_valid) { + if (!(flags & BNXT_CTX_MEM_PERSIST) || + ctxm->entry_size != entry_size || + ctxm->max_entries != max_entries) + bnxt_free_one_ctx_mem(bp, ctxm, true); + else + continue; + } ctxm->type = le16_to_cpu(resp->type); - ctxm->entry_size = le16_to_cpu(resp->entry_size); + ctxm->entry_size = entry_size; ctxm->flags = flags; ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); ctxm->entry_multiple = resp->entry_multiple; - ctxm->max_entries = le32_to_cpu(resp->max_num_entries); + ctxm->max_entries = max_entries; ctxm->min_entries = le32_to_cpu(resp->min_num_entries); init_val = resp->ctx_init_value; init_off = resp->ctx_init_offset; @@ -8294,7 +8463,8 @@ static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) struct hwrm_func_backing_store_qcaps_input *req; int rc; - if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx) + if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || + (bp->ctx && bp->ctx->flags & BNXT_CTX_FLAG_INITED)) return 0; if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) @@ -8635,6 +8805,36 @@ static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, return rc; } +static size_t bnxt_copy_ctx_pg_tbls(struct bnxt *bp, + struct bnxt_ctx_pg_info *ctx_pg, + void *buf, size_t offset, size_t head, + size_t tail) +{ + struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; + size_t nr_pages = ctx_pg->nr_pages; + int page_size = rmem->page_size; + size_t len = 0, total_len = 0; + u16 depth = rmem->depth; + + tail %= nr_pages * page_size; + do { + if (depth > 1) { + int i = head / (page_size * MAX_CTX_PAGES); + struct bnxt_ctx_pg_info *pg_tbl; + + pg_tbl = ctx_pg->ctx_pg_tbl[i]; + rmem = &pg_tbl->ring_mem; + } + len = __bnxt_copy_ring(bp, rmem, buf, offset, head, tail); + head += len; + offset += len; + total_len += len; + if (head >= nr_pages * page_size) + head = 0; + } while (head != tail); + return total_len; +} + static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, struct bnxt_ctx_pg_info *ctx_pg) { @@ -8685,6 +8885,8 @@ static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp, rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl, ctxm->init_value ? ctxm : NULL); } + if (!rc) + ctxm->mem_valid = 1; return rc; } @@ -8711,6 +8913,16 @@ static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp, hwrm_req_hold(bp, req); req->type = cpu_to_le16(ctxm->type); req->entry_size = cpu_to_le16(ctxm->entry_size); + if ((ctxm->flags & BNXT_CTX_MEM_PERSIST) && + bnxt_bs_trace_avail(bp, ctxm->type)) { + struct bnxt_bs_trace_info *bs_trace; + u32 enables; + + enables = FUNC_BACKING_STORE_CFG_V2_REQ_ENABLES_NEXT_BS_OFFSET; + req->enables = cpu_to_le32(enables); + bs_trace = &bp->bs_trace[bnxt_bstore_to_trace[ctxm->type]]; + req->next_bs_offset = cpu_to_le32(bs_trace->last_offset); + } req->subtype_valid_cnt = ctxm->split_entry_cnt; for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++) p[i] = cpu_to_le32(ctxm->split[i]); @@ -8740,21 +8952,42 @@ static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena) { struct bnxt_ctx_mem_info *ctx = bp->ctx; struct bnxt_ctx_mem_type *ctxm; - u16 last_type; + u16 last_type = BNXT_CTX_INV; int rc = 0; u16 type; - if (!ena) - return 0; - else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) - last_type = BNXT_CTX_MAX - 1; - else - last_type = BNXT_CTX_L2_MAX - 1; + for (type = BNXT_CTX_SRT; type <= BNXT_CTX_RIGP1; type++) { + ctxm = &ctx->ctx_arr[type]; + if (!bnxt_bs_trace_avail(bp, type)) + continue; + if (!ctxm->mem_valid) { + rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, + ctxm->max_entries, 1); + if (rc) { + netdev_warn(bp->dev, "Unable to setup ctx page for type:0x%x.\n", + type); + continue; + } + bnxt_bs_trace_init(bp, ctxm); + } + last_type = type; + } + + if (last_type == BNXT_CTX_INV) { + if (!ena) + return 0; + else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) + last_type = BNXT_CTX_MAX - 1; + else + last_type = BNXT_CTX_L2_MAX - 1; + } ctx->ctx_arr[last_type].last = 1; for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) { ctxm = &ctx->ctx_arr[type]; + if (!ctxm->mem_valid) + continue; rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last); if (rc) return rc; @@ -8762,21 +8995,63 @@ static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena) return 0; } -void bnxt_free_ctx_mem(struct bnxt *bp) +/** + * __bnxt_copy_ctx_mem - copy host context memory + * @bp: The driver context + * @ctxm: The pointer to the context memory type + * @buf: The destination buffer or NULL to just obtain the length + * @offset: The buffer offset to copy the data to + * @head: The head offset of context memory to copy from + * @tail: The tail offset (last byte + 1) of context memory to end the copy + * + * This function is called for debugging purposes to dump the host context + * used by the chip. + * + * Return: Length of memory copied + */ +static size_t __bnxt_copy_ctx_mem(struct bnxt *bp, + struct bnxt_ctx_mem_type *ctxm, void *buf, + size_t offset, size_t head, size_t tail) { - struct bnxt_ctx_mem_info *ctx = bp->ctx; - u16 type; + struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; + size_t len = 0, total_len = 0; + int i, n = 1; - if (!ctx) - return; + if (!ctx_pg) + return 0; - for (type = 0; type < BNXT_CTX_V2_MAX; type++) { - struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; - struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; - int i, n = 1; + if (ctxm->instance_bmap) + n = hweight32(ctxm->instance_bmap); + for (i = 0; i < n; i++) { + len = bnxt_copy_ctx_pg_tbls(bp, &ctx_pg[i], buf, offset, head, + tail); + offset += len; + total_len += len; + } + return total_len; +} - if (!ctx_pg) - continue; +size_t bnxt_copy_ctx_mem(struct bnxt *bp, struct bnxt_ctx_mem_type *ctxm, + void *buf, size_t offset) +{ + size_t tail = ctxm->max_entries * ctxm->entry_size; + + return __bnxt_copy_ctx_mem(bp, ctxm, buf, offset, 0, tail); +} + +static void bnxt_free_one_ctx_mem(struct bnxt *bp, + struct bnxt_ctx_mem_type *ctxm, bool force) +{ + struct bnxt_ctx_pg_info *ctx_pg; + int i, n = 1; + + ctxm->last = 0; + + if (ctxm->mem_valid && !force && (ctxm->flags & BNXT_CTX_MEM_PERSIST)) + return; + + ctx_pg = ctxm->pg_info; + if (ctx_pg) { if (ctxm->instance_bmap) n = hweight32(ctxm->instance_bmap); for (i = 0; i < n; i++) @@ -8784,11 +9059,27 @@ void bnxt_free_ctx_mem(struct bnxt *bp) kfree(ctx_pg); ctxm->pg_info = NULL; + ctxm->mem_valid = 0; } + memset(ctxm, 0, sizeof(*ctxm)); +} + +void bnxt_free_ctx_mem(struct bnxt *bp, bool force) +{ + struct bnxt_ctx_mem_info *ctx = bp->ctx; + u16 type; + + if (!ctx) + return; + + for (type = 0; type < BNXT_CTX_V2_MAX; type++) + bnxt_free_one_ctx_mem(bp, &ctx->ctx_arr[type], force); ctx->flags &= ~BNXT_CTX_FLAG_INITED; - kfree(ctx); - bp->ctx = NULL; + if (force) { + kfree(ctx); + bp->ctx = NULL; + } } static int bnxt_alloc_ctx_mem(struct bnxt *bp) @@ -9053,7 +9344,6 @@ static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) struct hwrm_port_mac_ptp_qcfg_output *resp; struct hwrm_port_mac_ptp_qcfg_input *req; struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; - bool phc_cfg; u8 flags; int rc; @@ -9100,8 +9390,9 @@ static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) rc = -ENODEV; goto exit; } - phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; - rc = bnxt_ptp_init(bp, phc_cfg); + ptp->rtc_configured = + (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; + rc = bnxt_ptp_init(bp); if (rc) netdev_warn(bp->dev, "PTP initialization failed.\n"); exit: @@ -9179,6 +9470,9 @@ static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) bp->flags |= BNXT_FLAG_UDP_GSO_CAP; if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_TX_PKT_TS_CMPL_SUPPORTED) bp->fw_cap |= BNXT_FW_CAP_TX_TS_CMP; + if (BNXT_PF(bp) && + (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_ROCE_VF_RESOURCE_MGMT_SUPPORTED)) + bp->fw_cap |= BNXT_FW_CAP_ROCE_VF_RESC_MGMT_SUPPORTED; bp->tx_push_thresh = 0; if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && @@ -10885,7 +11179,7 @@ static void bnxt_free_irq(struct bnxt *bp) irq = &bp->irq_tbl[map_idx]; if (irq->requested) { if (irq->have_cpumask) { - irq_set_affinity_hint(irq->vector, NULL); + irq_update_affinity_hint(irq->vector, NULL); free_cpumask_var(irq->cpu_mask); irq->have_cpumask = 0; } @@ -10940,10 +11234,10 @@ static int bnxt_request_irq(struct bnxt *bp) irq->have_cpumask = 1; cpumask_set_cpu(cpumask_local_spread(i, numa_node), irq->cpu_mask); - rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask); + rc = irq_update_affinity_hint(irq->vector, irq->cpu_mask); if (rc) { netdev_warn(bp->dev, - "Set affinity failed, IRQ = %d\n", + "Update affinity hint failed, IRQ = %d\n", irq->vector); break; } @@ -10988,7 +11282,8 @@ static void bnxt_init_napi(struct bnxt *bp) cp_nr_rings--; for (i = 0; i < cp_nr_rings; i++) { bnapi = bp->bnapi[i]; - netif_napi_add(bp->dev, &bnapi->napi, poll_fn); + netif_napi_add_config(bp->dev, &bnapi->napi, poll_fn, + bnapi->index); } if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { bnapi = bp->bnapi[cp_nr_rings]; @@ -11748,7 +12043,7 @@ static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) bnxt_ulp_irq_stop(bp); - bnxt_free_ctx_mem(bp); + bnxt_free_ctx_mem(bp, false); bnxt_dcb_free(bp); rc = bnxt_fw_init_one(bp); if (rc) { @@ -12882,7 +13177,7 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev, if (features & NETIF_F_GRO_HW) features &= ~NETIF_F_LRO; - /* Both CTAG and STAG VLAN accelaration on the RX side have to be + /* Both CTAG and STAG VLAN acceleration on the RX side have to be * turned on or off together. */ vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; @@ -13413,7 +13708,7 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) bnxt_reset_task(bp, true); break; } - bnxt_free_one_rx_ring_skbs(bp, i); + bnxt_free_one_rx_ring_skbs(bp, rxr); rxr->rx_prod = 0; rxr->rx_agg_prod = 0; rxr->rx_sw_agg_prod = 0; @@ -13461,7 +13756,7 @@ static void bnxt_fw_reset_close(struct bnxt *bp) bnxt_hwrm_func_drv_unrgtr(bp); if (pci_is_enabled(bp->pdev)) pci_disable_device(bp->pdev); - bnxt_free_ctx_mem(bp); + bnxt_free_ctx_mem(bp, false); } static bool is_bnxt_fw_ok(struct bnxt *bp) @@ -13495,12 +13790,13 @@ static void bnxt_force_fw_reset(struct bnxt *bp) test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) return; + /* we have to serialize with bnxt_refclk_read()*/ if (ptp) { unsigned long flags; - spin_lock_irqsave(&ptp->ptp_lock, flags); + write_seqlock_irqsave(&ptp->ptp_lock, flags); set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); - spin_unlock_irqrestore(&ptp->ptp_lock, flags); + write_sequnlock_irqrestore(&ptp->ptp_lock, flags); } else { set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); } @@ -13564,12 +13860,13 @@ void bnxt_fw_reset(struct bnxt *bp) struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; int n = 0, tmo; + /* we have to serialize with bnxt_refclk_read()*/ if (ptp) { unsigned long flags; - spin_lock_irqsave(&ptp->ptp_lock, flags); + write_seqlock_irqsave(&ptp->ptp_lock, flags); set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); - spin_unlock_irqrestore(&ptp->ptp_lock, flags); + write_sequnlock_irqrestore(&ptp->ptp_lock, flags); } else { set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); } @@ -14494,6 +14791,14 @@ static int bnxt_change_mtu(struct net_device *dev, int new_mtu) bnxt_close_nic(bp, true, false); WRITE_ONCE(dev->mtu, new_mtu); + + /* MTU change may change the AGG ring settings if an XDP multi-buffer + * program is attached. We need to set the AGG rings settings and + * rx_skb_func accordingly. + */ + if (READ_ONCE(bp->xdp_prog)) + bnxt_set_rx_skb_mode(bp, true); + bnxt_set_ring_params(bp); if (netif_running(dev)) @@ -15033,19 +15338,6 @@ static const struct netdev_stat_ops bnxt_stat_ops = { .get_base_stats = bnxt_get_base_stats, }; -static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) -{ - u16 mem_size; - - rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; - mem_size = rxr->rx_agg_bmap_size / 8; - rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); - if (!rxr->rx_agg_bmap) - return -ENOMEM; - - return 0; -} - static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) { struct bnxt_rx_ring_info *rxr, *clone; @@ -15094,15 +15386,25 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) goto err_free_rx_agg_ring; } + if (bp->flags & BNXT_FLAG_TPA) { + rc = bnxt_alloc_one_tpa_info(bp, clone); + if (rc) + goto err_free_tpa_info; + } + bnxt_init_one_rx_ring_rxbd(bp, clone); bnxt_init_one_rx_agg_ring_rxbd(bp, clone); bnxt_alloc_one_rx_ring_skb(bp, clone, idx); if (bp->flags & BNXT_FLAG_AGG_RINGS) bnxt_alloc_one_rx_ring_page(bp, clone, idx); + if (bp->flags & BNXT_FLAG_TPA) + bnxt_alloc_one_tpa_info_data(bp, clone); return 0; +err_free_tpa_info: + bnxt_free_one_tpa_info(bp, clone); err_free_rx_agg_ring: bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem); err_free_rx_ring: @@ -15110,9 +15412,11 @@ err_free_rx_ring: err_rxq_info_unreg: xdp_rxq_info_unreg(&clone->xdp_rxq); err_page_pool_destroy: - clone->page_pool->p.napi = NULL; page_pool_destroy(clone->page_pool); + if (bnxt_separate_head_pool()) + page_pool_destroy(clone->head_pool); clone->page_pool = NULL; + clone->head_pool = NULL; return rc; } @@ -15122,13 +15426,15 @@ static void bnxt_queue_mem_free(struct net_device *dev, void *qmem) struct bnxt *bp = netdev_priv(dev); struct bnxt_ring_struct *ring; - bnxt_free_one_rx_ring(bp, rxr); - bnxt_free_one_rx_agg_ring(bp, rxr); + bnxt_free_one_rx_ring_skbs(bp, rxr); xdp_rxq_info_unreg(&rxr->xdp_rxq); page_pool_destroy(rxr->page_pool); + if (bnxt_separate_head_pool()) + page_pool_destroy(rxr->head_pool); rxr->page_pool = NULL; + rxr->head_pool = NULL; ring = &rxr->rx_ring_struct; bnxt_free_ring(bp, &ring->ring_mem); @@ -15210,7 +15516,10 @@ static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) rxr->rx_agg_prod = clone->rx_agg_prod; rxr->rx_sw_agg_prod = clone->rx_sw_agg_prod; rxr->rx_next_cons = clone->rx_next_cons; + rxr->rx_tpa = clone->rx_tpa; + rxr->rx_tpa_idx_map = clone->rx_tpa_idx_map; rxr->page_pool = clone->page_pool; + rxr->head_pool = clone->head_pool; rxr->xdp_rxq = clone->xdp_rxq; bnxt_copy_rx_ring(bp, rxr, clone); @@ -15231,6 +15540,13 @@ static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) { vnic = &bp->vnic_info[i]; + + rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true); + if (rc) { + netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", + vnic->vnic_id, rc); + return rc; + } vnic->mru = bp->dev->mtu + ETH_HLEN + VLAN_HLEN; bnxt_hwrm_vnic_update(bp, vnic, VNIC_UPDATE_REQ_ENABLES_MRU_VALID); @@ -15262,6 +15578,8 @@ static int bnxt_queue_stop(struct net_device *dev, void *qmem, int idx) bnxt_hwrm_rx_agg_ring_free(bp, rxr, false); rxr->rx_next_cons = 0; page_pool_disable_direct_recycling(rxr->page_pool); + if (bnxt_separate_head_pool()) + page_pool_disable_direct_recycling(rxr->head_pool); memcpy(qmem, rxr, sizeof(*rxr)); bnxt_init_rx_ring_struct(bp, qmem); @@ -15316,7 +15634,7 @@ static void bnxt_remove_one(struct pci_dev *pdev) kfree(bp->fw_health); bp->fw_health = NULL; bnxt_cleanup_pci(bp); - bnxt_free_ctx_mem(bp); + bnxt_free_ctx_mem(bp, true); bnxt_free_crash_dump_mem(bp); kfree(bp->rss_indir_tbl); bp->rss_indir_tbl = NULL; @@ -15958,7 +16276,7 @@ init_err_pci_clean: kfree(bp->fw_health); bp->fw_health = NULL; bnxt_cleanup_pci(bp); - bnxt_free_ctx_mem(bp); + bnxt_free_ctx_mem(bp, true); bnxt_free_crash_dump_mem(bp); kfree(bp->rss_indir_tbl); bp->rss_indir_tbl = NULL; @@ -15984,6 +16302,7 @@ static void bnxt_shutdown(struct pci_dev *pdev) if (netif_running(dev)) dev_close(dev); + bnxt_ptp_clear(bp); bnxt_clear_int_mode(bp); pci_disable_device(pdev); @@ -16011,8 +16330,9 @@ static int bnxt_suspend(struct device *device) rc = bnxt_close(dev); } bnxt_hwrm_func_drv_unrgtr(bp); + bnxt_ptp_clear(bp); pci_disable_device(bp->pdev); - bnxt_free_ctx_mem(bp); + bnxt_free_ctx_mem(bp, false); rtnl_unlock(); return rc; } @@ -16054,6 +16374,10 @@ static int bnxt_resume(struct device *device) if (bp->fw_crash_mem) bnxt_hwrm_crash_dump_mem_cfg(bp); + if (bnxt_ptp_init(bp)) { + kfree(bp->ptp_cfg); + bp->ptp_cfg = NULL; + } bnxt_get_wol_settings(bp); if (netif_running(dev)) { rc = bnxt_open(dev); @@ -16124,7 +16448,7 @@ static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, if (pci_is_enabled(pdev)) pci_disable_device(pdev); - bnxt_free_ctx_mem(bp); + bnxt_free_ctx_mem(bp, false); rtnl_unlock(); /* Request a slot slot reset. */ @@ -16136,7 +16460,7 @@ static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, * @pdev: Pointer to PCI device * * Restart the card from scratch, as if from a cold-boot. - * At this point, the card has exprienced a hard reset, + * At this point, the card has experienced a hard reset, * followed by fixups by BIOS, and has its config space * set up identically to what it was at cold boot. */ @@ -16164,7 +16488,7 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) pci_set_master(pdev); /* Upon fatal error, our device internal logic that latches to * BAR value is getting reset and will restore only upon - * rewritting the BARs. + * rewriting the BARs. * * As pci_restore_state() does not re-write the BARs if the * value is same as saved value earlier, driver needs to @@ -16232,8 +16556,12 @@ static void bnxt_io_resume(struct pci_dev *pdev) rtnl_lock(); err = bnxt_hwrm_func_qcaps(bp); - if (!err && netif_running(netdev)) - err = bnxt_open(netdev); + if (!err) { + if (netif_running(netdev)) + err = bnxt_open(netdev); + else + err = bnxt_reserve_rings(bp, true); + } if (!err) netif_device_attach(netdev); |