diff options
Diffstat (limited to 'drivers/net/ethernet/cadence/macb_main.c')
| -rw-r--r-- | drivers/net/ethernet/cadence/macb_main.c | 610 |
1 files changed, 407 insertions, 203 deletions
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index ffce528aa00e..4f63f1ba3161 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -34,7 +34,11 @@ #include <linux/udp.h> #include <linux/tcp.h> #include <linux/iopoll.h> +#include <linux/phy/phy.h> #include <linux/pm_runtime.h> +#include <linux/ptp_classify.h> +#include <linux/reset.h> +#include <linux/firmware/xlnx-zynqmp.h> #include "macb.h" /* This structure is only used for MACB on SiFive FU540 devices */ @@ -335,11 +339,9 @@ static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum) struct macb *bp = bus->priv; int status; - status = pm_runtime_get_sync(&bp->pdev->dev); - if (status < 0) { - pm_runtime_put_noidle(&bp->pdev->dev); + status = pm_runtime_resume_and_get(&bp->pdev->dev); + if (status < 0) goto mdio_pm_exit; - } status = macb_mdio_wait_for_idle(bp); if (status < 0) @@ -389,11 +391,9 @@ static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum, struct macb *bp = bus->priv; int status; - status = pm_runtime_get_sync(&bp->pdev->dev); - if (status < 0) { - pm_runtime_put_noidle(&bp->pdev->dev); + status = pm_runtime_resume_and_get(&bp->pdev->dev); + if (status < 0) goto mdio_pm_exit; - } status = macb_mdio_wait_for_idle(bp); if (status < 0) @@ -506,79 +506,11 @@ static void macb_set_tx_clk(struct macb *bp, int speed) netdev_err(bp->dev, "adjusting tx_clk failed.\n"); } -static void macb_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct net_device *ndev = to_net_dev(config->dev); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - struct macb *bp = netdev_priv(ndev); - - /* We only support MII, RMII, GMII, RGMII & SGMII. */ - if (state->interface != PHY_INTERFACE_MODE_NA && - state->interface != PHY_INTERFACE_MODE_MII && - state->interface != PHY_INTERFACE_MODE_RMII && - state->interface != PHY_INTERFACE_MODE_GMII && - state->interface != PHY_INTERFACE_MODE_SGMII && - state->interface != PHY_INTERFACE_MODE_10GBASER && - !phy_interface_mode_is_rgmii(state->interface)) { - linkmode_zero(supported); - return; - } - - if (!macb_is_gem(bp) && - (state->interface == PHY_INTERFACE_MODE_GMII || - phy_interface_mode_is_rgmii(state->interface))) { - linkmode_zero(supported); - return; - } - - if (state->interface == PHY_INTERFACE_MODE_10GBASER && - !(bp->caps & MACB_CAPS_HIGH_SPEED && - bp->caps & MACB_CAPS_PCS)) { - linkmode_zero(supported); - return; - } - - phylink_set_port_modes(mask); - phylink_set(mask, Autoneg); - phylink_set(mask, Asym_Pause); - - if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && - (state->interface == PHY_INTERFACE_MODE_NA || - state->interface == PHY_INTERFACE_MODE_10GBASER)) { - phylink_set_10g_modes(mask); - phylink_set(mask, 10000baseKR_Full); - if (state->interface != PHY_INTERFACE_MODE_NA) - goto out; - } - - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - - if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && - (state->interface == PHY_INTERFACE_MODE_NA || - state->interface == PHY_INTERFACE_MODE_GMII || - state->interface == PHY_INTERFACE_MODE_SGMII || - phy_interface_mode_is_rgmii(state->interface))) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - - if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) - phylink_set(mask, 1000baseT_Half); - } -out: - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex) { - struct macb *bp = container_of(pcs, struct macb, phylink_pcs); + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); u32 config; config = gem_readl(bp, USX_CONTROL); @@ -592,7 +524,7 @@ static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state) { - struct macb *bp = container_of(pcs, struct macb, phylink_pcs); + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); u32 val; state->speed = SPEED_10000; @@ -612,7 +544,7 @@ static int macb_usx_pcs_config(struct phylink_pcs *pcs, const unsigned long *advertising, bool permit_pause_to_mac) { - struct macb *bp = container_of(pcs, struct macb, phylink_pcs); + struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | GEM_BIT(SIGNAL_OK)); @@ -795,28 +727,23 @@ static void macb_mac_link_up(struct phylink_config *config, netif_tx_wake_all_queues(ndev); } -static int macb_mac_prepare(struct phylink_config *config, unsigned int mode, - phy_interface_t interface) +static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) { struct net_device *ndev = to_net_dev(config->dev); struct macb *bp = netdev_priv(ndev); if (interface == PHY_INTERFACE_MODE_10GBASER) - bp->phylink_pcs.ops = &macb_phylink_usx_pcs_ops; + return &bp->phylink_usx_pcs; else if (interface == PHY_INTERFACE_MODE_SGMII) - bp->phylink_pcs.ops = &macb_phylink_pcs_ops; + return &bp->phylink_sgmii_pcs; else - bp->phylink_pcs.ops = NULL; - - if (bp->phylink_pcs.ops) - phylink_set_pcs(bp->phylink, &bp->phylink_pcs); - - return 0; + return NULL; } static const struct phylink_mac_ops macb_phylink_ops = { - .validate = macb_validate, - .mac_prepare = macb_mac_prepare, + .validate = phylink_generic_validate, + .mac_select_pcs = macb_mac_select_pcs, .mac_config = macb_mac_config, .mac_link_down = macb_mac_link_down, .mac_link_up = macb_mac_link_up, @@ -874,14 +801,47 @@ static int macb_mii_probe(struct net_device *dev) { struct macb *bp = netdev_priv(dev); + bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; + bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; + bp->phylink_config.dev = &dev->dev; bp->phylink_config.type = PHYLINK_NETDEV; + bp->phylink_config.mac_managed_pm = true; if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { bp->phylink_config.poll_fixed_state = true; bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; } + bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | + MAC_10 | MAC_100; + + __set_bit(PHY_INTERFACE_MODE_MII, + bp->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + bp->phylink_config.supported_interfaces); + + /* Determine what modes are supported */ + if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { + bp->phylink_config.mac_capabilities |= MAC_1000FD; + if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) + bp->phylink_config.mac_capabilities |= MAC_1000HD; + + __set_bit(PHY_INTERFACE_MODE_GMII, + bp->phylink_config.supported_interfaces); + phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_PCS) + __set_bit(PHY_INTERFACE_MODE_SGMII, + bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_HIGH_SPEED) { + __set_bit(PHY_INTERFACE_MODE_10GBASER, + bp->phylink_config.supported_interfaces); + bp->phylink_config.mac_capabilities |= MAC_10000FD; + } + } + bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, bp->phy_interface, &macb_phylink_ops); if (IS_ERR(bp->phylink)) { @@ -1002,7 +962,7 @@ static int macb_halt_tx(struct macb *bp) return -ETIMEDOUT; } -static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) +static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) { if (tx_skb->mapping) { if (tx_skb->mapped_as_page) @@ -1015,7 +975,7 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb) } if (tx_skb->skb) { - dev_kfree_skb_any(tx_skb->skb); + napi_consume_skb(tx_skb->skb, budget); tx_skb->skb = NULL; } } @@ -1068,12 +1028,13 @@ static void macb_tx_error_task(struct work_struct *work) (unsigned int)(queue - bp->queues), queue->tx_tail, queue->tx_head); - /* Prevent the queue IRQ handlers from running: each of them may call - * macb_tx_interrupt(), which in turn may call netif_wake_subqueue(). + /* Prevent the queue NAPI TX poll from running, as it calls + * macb_tx_complete(), which in turn may call netif_wake_subqueue(). * As explained below, we have to halt the transmission before updating * TBQP registers so we call netif_tx_stop_all_queues() to notify the * network engine about the macb/gem being halted. */ + napi_disable(&queue->napi_tx); spin_lock_irqsave(&bp->lock, flags); /* Make sure nobody is trying to queue up new packets */ @@ -1101,7 +1062,7 @@ static void macb_tx_error_task(struct work_struct *work) if (ctrl & MACB_BIT(TX_USED)) { /* skb is set for the last buffer of the frame */ while (!skb) { - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, 0); tail++; tx_skb = macb_tx_skb(queue, tail); skb = tx_skb->skb; @@ -1131,7 +1092,7 @@ static void macb_tx_error_task(struct work_struct *work) desc->ctrl = ctrl | MACB_BIT(TX_USED); } - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, 0); } /* Set end of TX queue */ @@ -1161,27 +1122,50 @@ static void macb_tx_error_task(struct work_struct *work) macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); spin_unlock_irqrestore(&bp->lock, flags); + napi_enable(&queue->napi_tx); } -static void macb_tx_interrupt(struct macb_queue *queue) +static bool ptp_one_step_sync(struct sk_buff *skb) { - unsigned int tail; - unsigned int head; - u32 status; - struct macb *bp = queue->bp; - u16 queue_index = queue - bp->queues; + struct ptp_header *hdr; + unsigned int ptp_class; + u8 msgtype; - status = macb_readl(bp, TSR); - macb_writel(bp, TSR, status); + /* No need to parse packet if PTP TS is not involved */ + if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) + goto not_oss; - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TCOMP)); + /* Identify and return whether PTP one step sync is being processed */ + ptp_class = ptp_classify_raw(skb); + if (ptp_class == PTP_CLASS_NONE) + goto not_oss; - netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n", - (unsigned long)status); + hdr = ptp_parse_header(skb, ptp_class); + if (!hdr) + goto not_oss; + if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) + goto not_oss; + + msgtype = ptp_get_msgtype(hdr, ptp_class); + if (msgtype == PTP_MSGTYPE_SYNC) + return true; + +not_oss: + return false; +} + +static int macb_tx_complete(struct macb_queue *queue, int budget) +{ + struct macb *bp = queue->bp; + u16 queue_index = queue - bp->queues; + unsigned int tail; + unsigned int head; + int packets = 0; + + spin_lock(&queue->tx_ptr_lock); head = queue->tx_head; - for (tail = queue->tx_tail; tail != head; tail++) { + for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { struct macb_tx_skb *tx_skb; struct sk_buff *skb; struct macb_dma_desc *desc; @@ -1207,8 +1191,8 @@ static void macb_tx_interrupt(struct macb_queue *queue) /* First, update TX stats if needed */ if (skb) { - if (unlikely(skb_shinfo(skb)->tx_flags & - SKBTX_HW_TSTAMP) && + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + !ptp_one_step_sync(skb) && gem_ptp_do_txstamp(queue, skb, desc) == 0) { /* skb now belongs to timestamp buffer * and will be removed later @@ -1222,10 +1206,11 @@ static void macb_tx_interrupt(struct macb_queue *queue) queue->stats.tx_packets++; bp->dev->stats.tx_bytes += skb->len; queue->stats.tx_bytes += skb->len; + packets++; } /* Now we can safely release resources */ - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, budget); /* skb is set only for the last buffer of the frame. * WARNING: at this point skb has been freed by @@ -1241,6 +1226,9 @@ static void macb_tx_interrupt(struct macb_queue *queue) CIRC_CNT(queue->tx_head, queue->tx_tail, bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) netif_wake_subqueue(bp->dev, queue_index); + spin_unlock(&queue->tx_ptr_lock); + + return packets; } static void gem_rx_refill(struct macb_queue *queue) @@ -1258,7 +1246,6 @@ static void gem_rx_refill(struct macb_queue *queue) /* Make hw descriptor updates visible to CPU */ rmb(); - queue->rx_prepared_head++; desc = macb_rx_desc(queue, entry); if (!queue->rx_skbuff[entry]) { @@ -1297,6 +1284,7 @@ static void gem_rx_refill(struct macb_queue *queue) dma_wmb(); desc->addr &= ~MACB_BIT(RX_USED); } + queue->rx_prepared_head++; } /* Make descriptor updates visible to hardware */ @@ -1597,31 +1585,51 @@ static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, return received; } -static int macb_poll(struct napi_struct *napi, int budget) +static bool macb_rx_pending(struct macb_queue *queue) { - struct macb_queue *queue = container_of(napi, struct macb_queue, napi); struct macb *bp = queue->bp; - int work_done; - u32 status; + unsigned int entry; + struct macb_dma_desc *desc; + + entry = macb_rx_ring_wrap(bp, queue->rx_tail); + desc = macb_rx_desc(queue, entry); - status = macb_readl(bp, RSR); - macb_writel(bp, RSR, status); + /* Make hw descriptor updates visible to CPU */ + rmb(); - netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n", - (unsigned long)status, budget); + return (desc->addr & MACB_BIT(RX_USED)) != 0; +} + +static int macb_rx_poll(struct napi_struct *napi, int budget) +{ + struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); + struct macb *bp = queue->bp; + int work_done; work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); - if (work_done < budget) { - napi_complete_done(napi, work_done); - /* Packets received while interrupts were disabled */ - status = macb_readl(bp, RSR); - if (status) { + netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", + (unsigned int)(queue - bp->queues), work_done, budget); + + if (work_done < budget && napi_complete_done(napi, work_done)) { + queue_writel(queue, IER, bp->rx_intr_mask); + + /* Packet completions only seem to propagate to raise + * interrupts when interrupts are enabled at the time, so if + * packets were received while interrupts were disabled, + * they will not cause another interrupt to be generated when + * interrupts are re-enabled. + * Check for this case here to avoid losing a wakeup. This can + * potentially race with the interrupt handler doing the same + * actions if an interrupt is raised just after enabling them, + * but this should be harmless. + */ + if (macb_rx_pending(queue)) { + queue_writel(queue, IDR, bp->rx_intr_mask); if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(queue, ISR, MACB_BIT(RCOMP)); - napi_reschedule(napi); - } else { - queue_writel(queue, IER, bp->rx_intr_mask); + netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); + napi_schedule(napi); } } @@ -1630,6 +1638,90 @@ static int macb_poll(struct napi_struct *napi, int budget) return work_done; } +static void macb_tx_restart(struct macb_queue *queue) +{ + struct macb *bp = queue->bp; + unsigned int head_idx, tbqp; + + spin_lock(&queue->tx_ptr_lock); + + if (queue->tx_head == queue->tx_tail) + goto out_tx_ptr_unlock; + + tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp); + tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); + head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); + + if (tbqp == head_idx) + goto out_tx_ptr_unlock; + + spin_lock_irq(&bp->lock); + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock_irq(&bp->lock); + +out_tx_ptr_unlock: + spin_unlock(&queue->tx_ptr_lock); +} + +static bool macb_tx_complete_pending(struct macb_queue *queue) +{ + bool retval = false; + + spin_lock(&queue->tx_ptr_lock); + if (queue->tx_head != queue->tx_tail) { + /* Make hw descriptor updates visible to CPU */ + rmb(); + + if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) + retval = true; + } + spin_unlock(&queue->tx_ptr_lock); + return retval; +} + +static int macb_tx_poll(struct napi_struct *napi, int budget) +{ + struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); + struct macb *bp = queue->bp; + int work_done; + + work_done = macb_tx_complete(queue, budget); + + rmb(); // ensure txubr_pending is up to date + if (queue->txubr_pending) { + queue->txubr_pending = false; + netdev_vdbg(bp->dev, "poll: tx restart\n"); + macb_tx_restart(queue); + } + + netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", + (unsigned int)(queue - bp->queues), work_done, budget); + + if (work_done < budget && napi_complete_done(napi, work_done)) { + queue_writel(queue, IER, MACB_BIT(TCOMP)); + + /* Packet completions only seem to propagate to raise + * interrupts when interrupts are enabled at the time, so if + * packets were sent while interrupts were disabled, + * they will not cause another interrupt to be generated when + * interrupts are re-enabled. + * Check for this case here to avoid losing a wakeup. This can + * potentially race with the interrupt handler doing the same + * actions if an interrupt is raised just after enabling them, + * but this should be harmless. + */ + if (macb_tx_complete_pending(queue)) { + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP)); + netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); + napi_schedule(napi); + } + } + + return work_done; +} + static void macb_hresp_error_task(struct tasklet_struct *t) { struct macb *bp = from_tasklet(bp, t, hresp_err_tasklet); @@ -1669,21 +1761,6 @@ static void macb_hresp_error_task(struct tasklet_struct *t) netif_tx_start_all_queues(dev); } -static void macb_tx_restart(struct macb_queue *queue) -{ - unsigned int head = queue->tx_head; - unsigned int tail = queue->tx_tail; - struct macb *bp = queue->bp; - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TXUBR)); - - if (head == tail) - return; - - macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); -} - static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) { struct macb_queue *queue = dev_id; @@ -1780,9 +1857,27 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(queue, ISR, MACB_BIT(RCOMP)); - if (napi_schedule_prep(&queue->napi)) { + if (napi_schedule_prep(&queue->napi_rx)) { netdev_vdbg(bp->dev, "scheduling RX softirq\n"); - __napi_schedule(&queue->napi); + __napi_schedule(&queue->napi_rx); + } + } + + if (status & (MACB_BIT(TCOMP) | + MACB_BIT(TXUBR))) { + queue_writel(queue, IDR, MACB_BIT(TCOMP)); + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, MACB_BIT(TCOMP) | + MACB_BIT(TXUBR)); + + if (status & MACB_BIT(TXUBR)) { + queue->txubr_pending = true; + wmb(); // ensure softirq can see update + } + + if (napi_schedule_prep(&queue->napi_tx)) { + netdev_vdbg(bp->dev, "scheduling TX softirq\n"); + __napi_schedule(&queue->napi_tx); } } @@ -1796,12 +1891,6 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) break; } - if (status & MACB_BIT(TCOMP)) - macb_tx_interrupt(queue); - - if (status & MACB_BIT(TXUBR)) - macb_tx_restart(queue); - /* Link change detection isn't possible with RMII, so we'll * add that if/when we get our hands on a full-blown MII PHY. */ @@ -2007,7 +2096,8 @@ static unsigned int macb_tx_map(struct macb *bp, ctrl |= MACB_BF(TX_LSO, lso_ctrl); ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); if ((bp->dev->features & NETIF_F_HW_CSUM) && - skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl) + skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && + !ptp_one_step_sync(skb)) ctrl |= MACB_BIT(TX_NOCRC); } else /* Only set MSS/MFS on payload descriptors @@ -2034,7 +2124,7 @@ dma_error: for (i = queue->tx_head; i != tx_head; i++) { tx_skb = macb_tx_skb(queue, i); - macb_tx_unmap(bp, tx_skb); + macb_tx_unmap(bp, tx_skb, 0); } return 0; @@ -2105,7 +2195,7 @@ static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) if (!(ndev->features & NETIF_F_HW_CSUM) || !((*skb)->ip_summed != CHECKSUM_PARTIAL) || - skb_shinfo(*skb)->gso_size) /* Not available for GSO */ + skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) return 0; if (padlen <= 0) { @@ -2156,7 +2246,6 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) u16 queue_index = skb_get_queue_mapping(skb); struct macb *bp = netdev_priv(dev); struct macb_queue *queue = &bp->queues[queue_index]; - unsigned long flags; unsigned int desc_cnt, nr_frags, frag_size, f; unsigned int hdrlen; bool is_lso; @@ -2180,7 +2269,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) /* only queue eth + ip headers separately for UDP */ hdrlen = skb_transport_offset(skb); else - hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb); + hdrlen = skb_tcp_all_headers(skb); if (skb_headlen(skb) < hdrlen) { netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); /* if this is required, would need to copy to single buffer */ @@ -2213,16 +2302,16 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); } - spin_lock_irqsave(&bp->lock, flags); + spin_lock_bh(&queue->tx_ptr_lock); /* This is a hard error, log it. */ if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < desc_cnt) { netif_stop_subqueue(dev, queue_index); - spin_unlock_irqrestore(&bp->lock, flags); netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", queue->tx_head, queue->tx_tail); - return NETDEV_TX_BUSY; + ret = NETDEV_TX_BUSY; + goto unlock; } /* Map socket buffer for DMA transfer */ @@ -2235,13 +2324,15 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) wmb(); skb_tx_timestamp(skb); + spin_lock_irq(&bp->lock); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock_irq(&bp->lock); if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) netif_stop_subqueue(dev, queue_index); unlock: - spin_unlock_irqrestore(&bp->lock, flags); + spin_unlock_bh(&queue->tx_ptr_lock); return ret; } @@ -2761,9 +2852,9 @@ static int macb_open(struct net_device *dev) netdev_dbg(bp->dev, "open\n"); - err = pm_runtime_get_sync(&bp->pdev->dev); + err = pm_runtime_resume_and_get(&bp->pdev->dev); if (err < 0) - goto pm_exit; + return err; /* RX buffers initialization */ macb_init_rx_buffer_size(bp, bufsz); @@ -2775,15 +2866,21 @@ static int macb_open(struct net_device *dev) goto pm_exit; } - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - napi_enable(&queue->napi); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_enable(&queue->napi_rx); + napi_enable(&queue->napi_tx); + } macb_init_hw(bp); - err = macb_phylink_connect(bp); + err = phy_power_on(bp->sgmii_phy); if (err) goto reset_hw; + err = macb_phylink_connect(bp); + if (err) + goto phy_off; + netif_tx_start_all_queues(dev); if (bp->ptp_info) @@ -2791,10 +2888,15 @@ static int macb_open(struct net_device *dev) return 0; +phy_off: + phy_power_off(bp->sgmii_phy); + reset_hw: macb_reset_hw(bp); - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - napi_disable(&queue->napi); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } macb_free_consistent(bp); pm_exit: pm_runtime_put_sync(&bp->pdev->dev); @@ -2810,12 +2912,16 @@ static int macb_close(struct net_device *dev) netif_tx_stop_all_queues(dev); - for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) - napi_disable(&queue->napi); + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } phylink_stop(bp->phylink); phylink_disconnect_phy(bp->phylink); + phy_power_off(bp->sgmii_phy); + spin_lock_irqsave(&bp->lock, flags); macb_reset_hw(bp); netif_carrier_off(dev); @@ -3101,7 +3207,9 @@ static int macb_set_link_ksettings(struct net_device *netdev, } static void macb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); @@ -3113,7 +3221,9 @@ static void macb_get_ringparam(struct net_device *netdev, } static int macb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); u32 new_rx_size, new_tx_size; @@ -3374,7 +3484,8 @@ static int gem_add_flow_filter(struct net_device *netdev, fs->flow_type, (int)fs->ring_cookie, fs->location, htonl(fs->h_u.tcp_ip4_spec.ip4src), htonl(fs->h_u.tcp_ip4_spec.ip4dst), - htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst)); + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); spin_lock_irqsave(&bp->rx_fs_lock, flags); @@ -3427,8 +3538,8 @@ static int gem_del_flow_filter(struct net_device *netdev, fs->flow_type, (int)fs->ring_cookie, fs->location, htonl(fs->h_u.tcp_ip4_spec.ip4src), htonl(fs->h_u.tcp_ip4_spec.ip4dst), - htons(fs->h_u.tcp_ip4_spec.psrc), - htons(fs->h_u.tcp_ip4_spec.pdst)); + be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), + be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); gem_writel_n(bp, SCRT2, fs->location, 0); @@ -3867,7 +3978,9 @@ static int macb_init(struct platform_device *pdev) queue = &bp->queues[q]; queue->bp = bp; - netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT); + spin_lock_init(&queue->tx_ptr_lock); + netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); + netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); if (hw_q) { queue->ISR = GEM_ISR(hw_q - 1); queue->IER = GEM_IER(hw_q - 1); @@ -4137,11 +4250,9 @@ static int at91ether_open(struct net_device *dev) u32 ctl; int ret; - ret = pm_runtime_get_sync(&lp->pdev->dev); - if (ret < 0) { - pm_runtime_put_noidle(&lp->pdev->dev); + ret = pm_runtime_resume_and_get(&lp->pdev->dev); + if (ret < 0) return ret; - } /* Clear internal statistics */ ctl = macb_readl(lp, NCR); @@ -4492,6 +4603,61 @@ static int fu540_c000_init(struct platform_device *pdev) return macb_init(pdev); } +static int init_reset_optional(struct platform_device *pdev) +{ + struct net_device *dev = platform_get_drvdata(pdev); + struct macb *bp = netdev_priv(dev); + int ret; + + if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { + /* Ensure PHY device used in SGMII mode is ready */ + bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL); + + if (IS_ERR(bp->sgmii_phy)) + return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy), + "failed to get SGMII PHY\n"); + + ret = phy_init(bp->sgmii_phy); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to init SGMII PHY\n"); + } + + ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); + if (!ret) { + u32 pm_info[2]; + + ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", + pm_info, ARRAY_SIZE(pm_info)); + if (ret) { + dev_err(&pdev->dev, "Failed to read power management information\n"); + goto err_out_phy_exit; + } + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); + if (ret) + goto err_out_phy_exit; + + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); + if (ret) + goto err_out_phy_exit; + } + + /* Fully reset controller at hardware level if mapped in device tree */ + ret = device_reset_optional(&pdev->dev); + if (ret) { + phy_exit(bp->sgmii_phy); + return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); + } + + ret = macb_init(pdev); + +err_out_phy_exit: + if (ret) + phy_exit(bp->sgmii_phy); + + return ret; +} + static const struct macb_usrio_config sama7g5_usrio = { .mii = 0, .rmii = 1, @@ -4518,8 +4684,8 @@ static const struct macb_config at91sam9260_config = { }; static const struct macb_config sama5d3macb_config = { - .caps = MACB_CAPS_SG_DISABLED - | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .caps = MACB_CAPS_SG_DISABLED | + MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, .clk_init = macb_clk_init, .init = macb_init, .usrio = &macb_default_usrio, @@ -4550,8 +4716,8 @@ static const struct macb_config sama5d29_config = { }; static const struct macb_config sama5d3_config = { - .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE - | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, + .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, @@ -4583,11 +4749,11 @@ static const struct macb_config np4_config = { static const struct macb_config zynqmp_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, .dma_burst_length = 16, .clk_init = macb_clk_init, - .init = macb_init, + .init = init_reset_optional, .jumbo_max_len = 10240, .usrio = &macb_default_usrio, }; @@ -4601,6 +4767,17 @@ static const struct macb_config zynq_config = { .usrio = &macb_default_usrio, }; +static const struct macb_config mpfs_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .usrio = &macb_default_usrio, + .jumbo_max_len = 10240, +}; + static const struct macb_config sama7g5_gem_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | MACB_CAPS_MIIONRGMII, @@ -4619,8 +4796,17 @@ static const struct macb_config sama7g5_emac_config = { .usrio = &sama7g5_usrio, }; +static const struct macb_config versal_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK, + .dma_burst_length = 16, + .clk_init = macb_clk_init, + .init = init_reset_optional, + .jumbo_max_len = 10240, + .usrio = &macb_default_usrio, +}; + static const struct of_device_id macb_dt_ids[] = { - { .compatible = "cdns,at32ap7000-macb" }, { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, { .compatible = "cdns,macb" }, { .compatible = "cdns,np4-macb", .data = &np4_config }, @@ -4634,11 +4820,15 @@ static const struct of_device_id macb_dt_ids[] = { { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, { .compatible = "cdns,emac", .data = &emac_config }, - { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, - { .compatible = "cdns,zynq-gem", .data = &zynq_config }, + { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ + { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, + { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, + { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, + { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, + { .compatible = "xlnx,versal-gem", .data = &versal_config}, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, macb_dt_ids); @@ -4646,8 +4836,8 @@ MODULE_DEVICE_TABLE(of, macb_dt_ids); static const struct macb_config default_gem_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP, .dma_burst_length = 16, .clk_init = macb_clk_init, .init = macb_init, @@ -4749,7 +4939,7 @@ static int macb_probe(struct platform_device *pdev) #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { - dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); bp->hw_dma_cap |= HW_DMA_CAP_64B; } #endif @@ -4763,8 +4953,8 @@ static int macb_probe(struct platform_device *pdev) /* MTU range: 68 - 1500 or 10240 */ dev->min_mtu = GEM_MTU_MIN_SIZE; - if (bp->caps & MACB_CAPS_JUMBO) - dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN; + if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) + dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; else dev->max_mtu = ETH_DATA_LEN; @@ -4804,7 +4994,7 @@ static int macb_probe(struct platform_device *pdev) err = macb_mii_init(bp); if (err) - goto err_out_free_netdev; + goto err_out_phy_exit; netif_carrier_off(dev); @@ -4829,6 +5019,9 @@ err_out_unregister_mdio: mdiobus_unregister(bp->mii_bus); mdiobus_free(bp->mii_bus); +err_out_phy_exit: + phy_exit(bp->sgmii_phy); + err_out_free_netdev: free_netdev(dev); @@ -4850,6 +5043,7 @@ static int macb_remove(struct platform_device *pdev) if (dev) { bp = netdev_priv(dev); + phy_exit(bp->sgmii_phy); mdiobus_unregister(bp->mii_bus); mdiobus_free(bp->mii_bus); @@ -4930,12 +5124,15 @@ static int __maybe_unused macb_suspend(struct device *dev) netif_device_detach(netdev); for (q = 0, queue = bp->queues; q < bp->num_queues; - ++q, ++queue) - napi_disable(&queue->napi); + ++q, ++queue) { + napi_disable(&queue->napi_rx); + napi_disable(&queue->napi_tx); + } if (!(bp->wol & MACB_WOL_ENABLED)) { rtnl_lock(); phylink_stop(bp->phylink); + phy_exit(bp->sgmii_phy); rtnl_unlock(); spin_lock_irqsave(&bp->lock, flags); macb_reset_hw(bp); @@ -5009,8 +5206,10 @@ static int __maybe_unused macb_resume(struct device *dev) } for (q = 0, queue = bp->queues; q < bp->num_queues; - ++q, ++queue) - napi_enable(&queue->napi); + ++q, ++queue) { + napi_enable(&queue->napi_rx); + napi_enable(&queue->napi_tx); + } if (netdev->hw_features & NETIF_F_NTUPLE) gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); @@ -5023,6 +5222,9 @@ static int __maybe_unused macb_resume(struct device *dev) macb_set_rx_mode(netdev); macb_restore_features(bp); rtnl_lock(); + if (!device_may_wakeup(&bp->dev->dev)) + phy_init(bp->sgmii_phy); + phylink_start(bp->phylink); rtnl_unlock(); @@ -5040,7 +5242,7 @@ static int __maybe_unused macb_runtime_suspend(struct device *dev) if (!(device_may_wakeup(dev))) macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); - else + else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); return 0; @@ -5056,8 +5258,10 @@ static int __maybe_unused macb_runtime_resume(struct device *dev) clk_prepare_enable(bp->hclk); clk_prepare_enable(bp->tx_clk); clk_prepare_enable(bp->rx_clk); + clk_prepare_enable(bp->tsu_clk); + } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { + clk_prepare_enable(bp->tsu_clk); } - clk_prepare_enable(bp->tsu_clk); return 0; } |