diff options
Diffstat (limited to 'drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c')
-rw-r--r-- | drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c | 281 |
1 files changed, 166 insertions, 115 deletions
diff --git a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c index 26017fe9df93..0d8ed002adcb 100644 --- a/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c +++ b/drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c @@ -104,9 +104,11 @@ static void free_rx_fd(struct dpaa2_eth_priv *priv, /* We don't support any other format */ return; - /* For S/G frames, we first need to free all SG entries */ + /* For S/G frames, we first need to free all SG entries + * except the first one, which was taken care of already + */ sgt = vaddr + dpaa2_fd_get_offset(fd); - for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { + for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { addr = dpaa2_sg_get_addr(&sgt[i]); sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); dma_unmap_single(dev, addr, DPAA2_ETH_RX_BUF_SIZE, @@ -131,16 +133,15 @@ static struct sk_buff *build_linear_skb(struct dpaa2_eth_priv *priv, u16 fd_offset = dpaa2_fd_get_offset(fd); u32 fd_length = dpaa2_fd_get_len(fd); - skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_SIZE + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); + ch->buf_count--; + + skb = build_skb(fd_vaddr, DPAA2_ETH_SKB_SIZE); if (unlikely(!skb)) return NULL; skb_reserve(skb, fd_offset); skb_put(skb, fd_length); - ch->buf_count--; - return skb; } @@ -176,10 +177,21 @@ static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv, if (i == 0) { /* We build the skb around the first data buffer */ - skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_SIZE + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); - if (unlikely(!skb)) - return NULL; + skb = build_skb(sg_vaddr, DPAA2_ETH_SKB_SIZE); + if (unlikely(!skb)) { + /* Free the first SG entry now, since we already + * unmapped it and obtained the virtual address + */ + skb_free_frag(sg_vaddr); + + /* We still need to subtract the buffers used + * by this FD from our software counter + */ + while (!dpaa2_sg_is_final(&sgt[i]) && + i < DPAA2_ETH_MAX_SG_ENTRIES) + i++; + break; + } sg_offset = dpaa2_sg_get_offset(sge); skb_reserve(skb, sg_offset); @@ -206,6 +218,8 @@ static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv, break; } + WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); + /* Count all data buffers + SG table buffer */ ch->buf_count -= i + 2; @@ -557,10 +571,10 @@ static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) percpu_stats = this_cpu_ptr(priv->percpu_stats); percpu_extras = this_cpu_ptr(priv->percpu_extras); - if (unlikely(skb_headroom(skb) < DPAA2_ETH_NEEDED_HEADROOM(priv))) { + if (unlikely(skb_headroom(skb) < dpaa2_eth_needed_headroom(priv))) { struct sk_buff *ns; - ns = skb_realloc_headroom(skb, DPAA2_ETH_NEEDED_HEADROOM(priv)); + ns = skb_realloc_headroom(skb, dpaa2_eth_needed_headroom(priv)); if (unlikely(!ns)) { percpu_stats->tx_dropped++; goto err_alloc_headroom; @@ -718,6 +732,23 @@ static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) return 0; } +/* Free buffers acquired from the buffer pool or which were meant to + * be released in the pool + */ +static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count) +{ + struct device *dev = priv->net_dev->dev.parent; + void *vaddr; + int i; + + for (i = 0; i < count; i++) { + vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); + dma_unmap_single(dev, buf_array[i], DPAA2_ETH_RX_BUF_SIZE, + DMA_BIDIRECTIONAL); + skb_free_frag(vaddr); + } +} + /* Perform a single release command to add buffers * to the specified buffer pool */ @@ -727,17 +758,17 @@ static int add_bufs(struct dpaa2_eth_priv *priv, u16 bpid) u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; void *buf; dma_addr_t addr; - int i; + int i, err; for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { /* Allocate buffer visible to WRIOP + skb shared info + * alignment padding */ - buf = napi_alloc_frag(DPAA2_ETH_BUF_RAW_SIZE); + buf = napi_alloc_frag(dpaa2_eth_buf_raw_size(priv)); if (unlikely(!buf)) goto err_alloc; - buf = PTR_ALIGN(buf, DPAA2_ETH_RX_BUF_ALIGN); + buf = PTR_ALIGN(buf, priv->rx_buf_align); addr = dma_map_single(dev, buf, DPAA2_ETH_RX_BUF_SIZE, DMA_FROM_DEVICE); @@ -748,28 +779,33 @@ static int add_bufs(struct dpaa2_eth_priv *priv, u16 bpid) /* tracing point */ trace_dpaa2_eth_buf_seed(priv->net_dev, - buf, DPAA2_ETH_BUF_RAW_SIZE, + buf, dpaa2_eth_buf_raw_size(priv), addr, DPAA2_ETH_RX_BUF_SIZE, bpid); } release_bufs: - /* In case the portal is busy, retry until successful. - * The buffer release function would only fail if the QBMan portal - * was busy, which implies portal contention (i.e. more CPUs than - * portals, i.e. GPPs w/o affine DPIOs). For all practical purposes, - * there is little we can realistically do, short of giving up - - * in which case we'd risk depleting the buffer pool and never again - * receiving the Rx interrupt which would kick-start the refill logic. - * So just keep retrying, at the risk of being moved to ksoftirqd. - */ - while (dpaa2_io_service_release(NULL, bpid, buf_array, i)) + /* In case the portal is busy, retry until successful */ + while ((err = dpaa2_io_service_release(NULL, bpid, + buf_array, i)) == -EBUSY) cpu_relax(); + + /* If release command failed, clean up and bail out; + * not much else we can do about it + */ + if (err) { + free_bufs(priv, buf_array, i); + return 0; + } + return i; err_map: skb_free_frag(buf); err_alloc: + /* If we managed to allocate at least some buffers, + * release them to hardware + */ if (i) goto release_bufs; @@ -811,10 +847,8 @@ static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) */ static void drain_bufs(struct dpaa2_eth_priv *priv, int count) { - struct device *dev = priv->net_dev->dev.parent; u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; - void *vaddr; - int ret, i; + int ret; do { ret = dpaa2_io_service_acquire(NULL, priv->bpid, @@ -823,15 +857,7 @@ static void drain_bufs(struct dpaa2_eth_priv *priv, int count) netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); return; } - for (i = 0; i < ret; i++) { - /* Same logic as on regular Rx path */ - vaddr = dpaa2_iova_to_virt(priv->iommu_domain, - buf_array[i]); - dma_unmap_single(dev, buf_array[i], - DPAA2_ETH_RX_BUF_SIZE, - DMA_FROM_DEVICE); - skb_free_frag(vaddr); - } + free_bufs(priv, buf_array, ret); } while (ret); } @@ -927,13 +953,14 @@ static int dpaa2_eth_poll(struct napi_struct *napi, int budget) break; } - if (cleaned < budget) { - napi_complete_done(napi, cleaned); + if (cleaned < budget && napi_complete_done(napi, cleaned)) { /* Re-enable data available notifications */ do { err = dpaa2_io_service_rearm(NULL, &ch->nctx); cpu_relax(); } while (err == -EBUSY); + WARN_ONCE(err, "CDAN notifications rearm failed on core %d", + ch->nctx.desired_cpu); } ch->stats.frames += cleaned; @@ -1415,34 +1442,32 @@ static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv) err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); if (err) { dev_err(dev, "dpcon_open() failed\n"); - goto err_open; + goto free; } err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); if (err) { dev_err(dev, "dpcon_reset() failed\n"); - goto err_reset; + goto close; } err = dpcon_get_attributes(priv->mc_io, 0, dpcon->mc_handle, &attrs); if (err) { dev_err(dev, "dpcon_get_attributes() failed\n"); - goto err_get_attr; + goto close; } err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); if (err) { dev_err(dev, "dpcon_enable() failed\n"); - goto err_enable; + goto close; } return dpcon; -err_enable: -err_get_attr: -err_reset: +close: dpcon_close(priv->mc_io, 0, dpcon->mc_handle); -err_open: +free: fsl_mc_object_free(dpcon); return NULL; @@ -1749,66 +1774,32 @@ static void free_dpbp(struct dpaa2_eth_priv *priv) fsl_mc_object_free(priv->dpbp_dev); } -/* Configure the DPNI object this interface is associated with */ -static int setup_dpni(struct fsl_mc_device *ls_dev) +static int set_buffer_layout(struct dpaa2_eth_priv *priv) { - struct device *dev = &ls_dev->dev; - struct dpaa2_eth_priv *priv; - struct net_device *net_dev; + struct device *dev = priv->net_dev->dev.parent; struct dpni_buffer_layout buf_layout = {0}; int err; - net_dev = dev_get_drvdata(dev); - priv = netdev_priv(net_dev); - - /* get a handle for the DPNI object */ - err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); - if (err) { - dev_err(dev, "dpni_open() failed\n"); - goto err_open; - } - - ls_dev->mc_io = priv->mc_io; - ls_dev->mc_handle = priv->mc_token; - - err = dpni_reset(priv->mc_io, 0, priv->mc_token); - if (err) { - dev_err(dev, "dpni_reset() failed\n"); - goto err_reset; - } - - err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, - &priv->dpni_attrs); - if (err) { - dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); - goto err_get_attr; - } + /* We need to check for WRIOP version 1.0.0, but depending on the MC + * version, this number is not always provided correctly on rev1. + * We need to check for both alternatives in this situation. + */ + if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || + priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) + priv->rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; + else + priv->rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; - /* Configure buffer layouts */ - /* rx buffer */ - buf_layout.pass_parser_result = true; + /* tx buffer */ buf_layout.pass_frame_status = true; buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; - buf_layout.data_align = DPAA2_ETH_RX_BUF_ALIGN; - buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | - DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | - DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | - DPNI_BUF_LAYOUT_OPT_DATA_ALIGN; - err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, - DPNI_QUEUE_RX, &buf_layout); - if (err) { - dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); - goto err_buf_layout; - } - - /* tx buffer */ buf_layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE; err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, DPNI_QUEUE_TX, &buf_layout); if (err) { dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); - goto err_buf_layout; + return err; } /* tx-confirm buffer */ @@ -1817,7 +1808,7 @@ static int setup_dpni(struct fsl_mc_device *ls_dev) DPNI_QUEUE_TX_CONFIRM, &buf_layout); if (err) { dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); - goto err_buf_layout; + return err; } /* Now that we've set our tx buffer layout, retrieve the minimum @@ -1827,24 +1818,76 @@ static int setup_dpni(struct fsl_mc_device *ls_dev) &priv->tx_data_offset); if (err) { dev_err(dev, "dpni_get_tx_data_offset() failed\n"); - goto err_data_offset; + return err; } if ((priv->tx_data_offset % 64) != 0) dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", priv->tx_data_offset); - /* Accommodate software annotation space (SWA) */ - priv->tx_data_offset += DPAA2_ETH_SWA_SIZE; + /* rx buffer */ + buf_layout.pass_parser_result = true; + buf_layout.data_align = priv->rx_buf_align; + buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); + buf_layout.private_data_size = 0; + buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | + DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | + DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | + DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; + err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, + DPNI_QUEUE_RX, &buf_layout); + if (err) { + dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); + return err; + } return 0; +} -err_data_offset: -err_buf_layout: -err_get_attr: -err_reset: +/* Configure the DPNI object this interface is associated with */ +static int setup_dpni(struct fsl_mc_device *ls_dev) +{ + struct device *dev = &ls_dev->dev; + struct dpaa2_eth_priv *priv; + struct net_device *net_dev; + int err; + + net_dev = dev_get_drvdata(dev); + priv = netdev_priv(net_dev); + + /* get a handle for the DPNI object */ + err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); + if (err) { + dev_err(dev, "dpni_open() failed\n"); + return err; + } + + ls_dev->mc_io = priv->mc_io; + ls_dev->mc_handle = priv->mc_token; + + err = dpni_reset(priv->mc_io, 0, priv->mc_token); + if (err) { + dev_err(dev, "dpni_reset() failed\n"); + goto close; + } + + err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, + &priv->dpni_attrs); + if (err) { + dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); + goto close; + } + + err = set_buffer_layout(priv); + if (err) + goto close; + + + return 0; + +close: dpni_close(priv->mc_io, 0, priv->mc_token); -err_open: + return err; } @@ -2085,7 +2128,7 @@ static int bind_dpni(struct dpaa2_eth_priv *priv) */ err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_SUPPORTED); if (err) - netdev_err(net_dev, "Failed to configure hashing\n"); + dev_err(dev, "Failed to configure hashing\n"); /* Configure handling of error frames */ err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; @@ -2230,6 +2273,7 @@ static int netdev_init(struct net_device *net_dev) { struct device *dev = net_dev->dev.parent; struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + u16 rx_headroom, req_headroom; u8 bcast_addr[ETH_ALEN]; u8 num_queues; int err; @@ -2251,7 +2295,20 @@ static int netdev_init(struct net_device *net_dev) /* Reserve enough space to align buffer as per hardware requirement; * NOTE: priv->tx_data_offset MUST be initialized at this point. */ - net_dev->needed_headroom = DPAA2_ETH_NEEDED_HEADROOM(priv); + net_dev->needed_headroom = dpaa2_eth_needed_headroom(priv); + + /* If headroom guaranteed by hardware in the Rx frame buffer is + * smaller than the Tx headroom required by the stack, issue a + * one time warning. This will most likely mean skbs forwarded to + * another DPAA2 network interface will get reallocated, with a + * significant performance impact. + */ + req_headroom = LL_RESERVED_SPACE(net_dev) - ETH_HLEN; + rx_headroom = ALIGN(DPAA2_ETH_RX_HWA_SIZE + + dpaa2_eth_rx_head_room(priv), priv->rx_buf_align); + if (req_headroom > rx_headroom) + dev_info_once(dev, "Required headroom (%d) greater than available (%d)\n", + req_headroom, rx_headroom); /* Set MTU limits */ net_dev->min_mtu = 68; @@ -2303,7 +2360,7 @@ static irqreturn_t dpni_irq0_handler(int irq_num, void *arg) static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) { - u32 status = 0, clear = 0; + u32 status = ~0; struct device *dev = (struct device *)arg; struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); struct net_device *net_dev = dev_get_drvdata(dev); @@ -2313,18 +2370,12 @@ static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) DPNI_IRQ_INDEX, &status); if (unlikely(err)) { netdev_err(net_dev, "Can't get irq status (err %d)\n", err); - clear = 0xffffffff; - goto out; + return IRQ_HANDLED; } - if (status & DPNI_IRQ_EVENT_LINK_CHANGED) { - clear |= DPNI_IRQ_EVENT_LINK_CHANGED; + if (status & DPNI_IRQ_EVENT_LINK_CHANGED) link_state_update(netdev_priv(net_dev)); - } -out: - dpni_clear_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, - DPNI_IRQ_INDEX, clear); return IRQ_HANDLED; } |